From f635e485fce303979599c0869d5afeeadc6cac0c Mon Sep 17 00:00:00 2001 From: Jacob Moore Date: Thu, 15 Feb 2024 15:40:37 -0600 Subject: [PATCH 01/64] STYLE: Uncrustify Parallel-Explicit solvers --- src/Parallel-Solvers/FEA_Module.cpp | 509 +- src/Parallel-Solvers/FEA_Module.h | 545 +- src/Parallel-Solvers/FEA_Module_Inertial.cpp | 3627 +++++------ src/Parallel-Solvers/FEA_Module_Inertial.h | 79 +- .../Parallel-Explicit/CMakeLists.txt | 2 +- .../FEA_Module_Dynamic_Elasticity.cpp | 2982 ++++----- .../FEA_Module_Dynamic_Elasticity.h | 1069 ++-- .../Dynamic_Elastic_Solver/boundary.cpp | 80 +- .../elastic_optimization.cpp | 2833 ++++----- .../Dynamic_Elastic_Solver/force_elastic.cpp | 2208 +++---- .../force_gradients_elastic.cpp | 1280 ++-- .../Dynamic_Elastic_Solver/geometry.cpp | 682 +-- .../Dynamic_Elastic_Solver/mesh.h | 761 ++- .../Dynamic_Elastic_Solver/momentum.cpp | 485 +- .../Dynamic_Elastic_Solver/properties.cpp | 193 +- .../Dynamic_Elastic_Solver/state.h | 104 +- .../time_integration.cpp | 236 +- .../Eulerian_Solver/FEA_Module_Eulerian.cpp | 1823 +++--- .../Eulerian_Solver/FEA_Module_Eulerian.h | 488 +- .../Simulation_Parameters_Eulerian.h | 162 +- .../Eulerian_Solver/eulerian_geometry.cpp | 42 +- .../eulerian_time_integration.cpp | 240 +- .../Parallel-Explicit/Explicit_Solver.cpp | 5312 +++++++++-------- .../Parallel-Explicit/Explicit_Solver.h | 179 +- .../SGH_Solver/CMakeLists.txt | 20 +- .../SGH_Solver/FEA_Module_SGH.cpp | 2615 -------- .../SGH_Solver/FEA_Module_SGH.h | 697 --- .../SGH_Solver/energy_sgh.cpp | 57 - .../SGH_Solver/force_gradients_sgh.cpp | 2061 ------- .../Parallel-Explicit/SGH_Solver/geometry.cpp | 1631 ----- .../SGH_Solver/include/FEA_Module_SGH.h | 700 +++ .../SGH_Solver/{ => include}/mesh.h | 763 ++- .../SGH_Solver/include/state.h | 80 + .../Parallel-Explicit/SGH_Solver/momentum.cpp | 417 -- .../SGH_Solver/power_gradients_sgh.cpp | 469 -- .../SGH_Solver/sgh_optimization.cpp | 1757 ------ .../SGH_Solver/src/FEA_Module_SGH.cpp | 1725 ++++++ .../SGH_Solver/{ => src}/boundary.cpp | 80 +- .../SGH_Solver/src/energy_sgh.cpp | 54 + .../SGH_Solver/src/force_gradients_sgh.cpp | 2063 +++++++ .../SGH_Solver/{ => src}/force_sgh.cpp | 888 ++- .../SGH_Solver/src/geometry.cpp | 1643 +++++ .../SGH_Solver/src/momentum.cpp | 391 ++ .../SGH_Solver/src/power_gradients_sgh.cpp | 495 ++ .../SGH_Solver/{ => src}/properties.cpp | 179 +- .../SGH_Solver/src/setup_sgh.cpp | 958 +++ .../SGH_Solver/src/sgh_optimization.cpp | 1862 ++++++ .../SGH_Solver/src/time_integration.cpp | 231 + .../Parallel-Explicit/SGH_Solver/state.h | 108 - .../SGH_Solver/time_integration.cpp | 237 - .../Kinetic_Energy_Minimize.h | 634 +- .../Parallel-Explicit/example_simple.yaml | 4 +- .../Parallel-Explicit/main.cpp | 80 +- .../Parallel-Explicit/outputs.cpp | 1279 ++-- src/Parallel-Solvers/Solver.cpp | 5180 ++++++++-------- src/Parallel-Solvers/Solver.h | 428 +- src/Parallel-Solvers/node_combination.cpp | 73 +- src/Parallel-Solvers/node_combination.h | 147 +- 58 files changed, 28728 insertions(+), 27199 deletions(-) delete mode 100644 src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/FEA_Module_SGH.cpp delete mode 100644 src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/FEA_Module_SGH.h delete mode 100644 src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/energy_sgh.cpp delete mode 100644 src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/force_gradients_sgh.cpp delete mode 100644 src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/geometry.cpp create mode 100644 src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/FEA_Module_SGH.h rename src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/{ => include}/mesh.h (60%) create mode 100644 src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/state.h delete mode 100644 src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/momentum.cpp delete mode 100644 src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/power_gradients_sgh.cpp delete mode 100644 src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/sgh_optimization.cpp create mode 100644 src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/FEA_Module_SGH.cpp rename src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/{ => src}/boundary.cpp (50%) create mode 100644 src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/energy_sgh.cpp create mode 100644 src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp rename src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/{ => src}/force_sgh.cpp (54%) create mode 100644 src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/geometry.cpp create mode 100644 src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/momentum.cpp create mode 100644 src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/power_gradients_sgh.cpp rename src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/{ => src}/properties.cpp (63%) create mode 100644 src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/setup_sgh.cpp create mode 100644 src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/sgh_optimization.cpp create mode 100644 src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/time_integration.cpp delete mode 100644 src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/state.h delete mode 100644 src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/time_integration.cpp diff --git a/src/Parallel-Solvers/FEA_Module.cpp b/src/Parallel-Solvers/FEA_Module.cpp index 6544cf279..3cf5165ca 100644 --- a/src/Parallel-Solvers/FEA_Module.cpp +++ b/src/Parallel-Solvers/FEA_Module.cpp @@ -11,14 +11,14 @@ This program is open source under the BSD-3 License. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - + 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - + 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - + 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. @@ -43,91 +43,93 @@ #define BC_EPSILON 1.0e-8 using namespace utils; -FEA_Module::FEA_Module(Solver *Solver_Pointer) { - - Solver_Pointer_ = Solver_Pointer; - simparam = &Solver_Pointer->simparam; - - num_dim = simparam->num_dims; - num_gauss_points = simparam->num_gauss_points; - - //obtain global and local node and element counts - num_nodes = Solver_Pointer->num_nodes; - num_elem = Solver_Pointer->num_elem; - nall_nodes = Solver_Pointer->nall_nodes; - rnum_elem = Solver_Pointer->rnum_elem; - nlocal_nodes = Solver_Pointer->nlocal_nodes; - nghost_nodes = Solver_Pointer->nghost_nodes; - max_nodes_per_element = Solver_Pointer->max_nodes_per_element; - - hessvec_count = update_count = 0; - linear_solve_time = hessvec_time = hessvec_linear_time = 0; - last_compute_step = -1; - - Matrix_alloc=0; - gradient_print_sync = 0; - //RCP pointer to *this (Parallel Nonlinear Solver Object) - //FEM_pass = Teuchos::rcp(this); - - //Trilinos output stream - std::ostream &out = std::cout; - fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - (*fos).setOutputToRootOnly(0); - - //MPI Data copy - myrank = Solver_Pointer->myrank; - nranks = Solver_Pointer->nranks; - world = Solver_Pointer->world; - importer = Solver_Pointer->importer; - ghost_importer = Solver_Pointer->ghost_importer; - node_sorting_importer = Solver_Pointer->node_sorting_importer; - element_sorting_importer = Solver_Pointer->element_sorting_importer; - dof_importer = Solver_Pointer->dof_importer; - - //obtain node and element maps - comm = Solver_Pointer->comm; - map = Solver_Pointer->map; //map of node indices - ghost_node_map = Solver_Pointer->ghost_node_map; //map of node indices with ghosts on each rank - all_node_map = Solver_Pointer->all_node_map; //map of node indices with ghosts on each rank - element_map = Solver_Pointer->element_map; //non overlapping map of elements owned by each rank used in reduction ops - all_element_map = Solver_Pointer->all_element_map; //overlapping map of elements connected to the local nodes in each rank - local_dof_map = Solver_Pointer->local_dof_map; //map of local dofs (typically num_node_local*num_dim) - all_dof_map = Solver_Pointer->all_dof_map; //map of local and ghost dofs (typically num_node_all*num_dim) - - //obtain mesh coordinates, densities, and element connectivity - global_nodes_in_elem_distributed = Solver_Pointer->global_nodes_in_elem_distributed; //element to node connectivity table - node_nconn_distributed = Solver_Pointer->node_nconn_distributed; //how many elements a node is connected to - node_coords_distributed = Solver_Pointer->node_coords_distributed; - all_node_coords_distributed = Solver_Pointer->all_node_coords_distributed; - initial_node_coords_distributed = Solver_Pointer->node_coords_distributed; - all_initial_node_coords_distributed = Solver_Pointer->all_node_coords_distributed; - design_node_densities_distributed = Solver_Pointer->design_node_densities_distributed; - filtered_node_densities_distributed = Solver_Pointer->filtered_node_densities_distributed; - test_node_densities_distributed = Solver_Pointer->test_node_densities_distributed; - all_node_densities_distributed = Solver_Pointer->all_node_densities_distributed; - all_filtered_node_densities_distributed = Solver_Pointer->all_filtered_node_densities_distributed; - Global_Element_Densities = Solver_Pointer->Global_Element_Densities; - Element_Types = Solver_Pointer->Element_Types; - - //element select data - element_select = Solver_Pointer->element_select; - element_select->choose_3Delem_type(Element_Types(0), elem); - - //obtain boundary condition and loading data - nboundary_patches = Solver_Pointer->nboundary_patches; - Boundary_Patches = Solver_Pointer->Boundary_Patches; - node_specified_bcs = false; - //initialize for default - num_boundary_conditions = 0; - - //flag init - body_term_flag = nonzero_bc_flag = false; - - //output data - noutput = 0; +FEA_Module::FEA_Module(Solver* Solver_Pointer) +{ + Solver_Pointer_ = Solver_Pointer; + simparam = &Solver_Pointer->simparam; + + num_dim = simparam->num_dims; + num_gauss_points = simparam->num_gauss_points; + + // obtain global and local node and element counts + num_nodes = Solver_Pointer->num_nodes; + num_elem = Solver_Pointer->num_elem; + nall_nodes = Solver_Pointer->nall_nodes; + rnum_elem = Solver_Pointer->rnum_elem; + nlocal_nodes = Solver_Pointer->nlocal_nodes; + nghost_nodes = Solver_Pointer->nghost_nodes; + max_nodes_per_element = Solver_Pointer->max_nodes_per_element; + + hessvec_count = update_count = 0; + linear_solve_time = hessvec_time = hessvec_linear_time = 0; + last_compute_step = -1; + + Matrix_alloc = 0; + gradient_print_sync = 0; + // RCP pointer to *this (Parallel Nonlinear Solver Object) + // FEM_pass = Teuchos::rcp(this); + + // Trilinos output stream + std::ostream& out = std::cout; + fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + (*fos).setOutputToRootOnly(0); + + // MPI Data copy + myrank = Solver_Pointer->myrank; + nranks = Solver_Pointer->nranks; + world = Solver_Pointer->world; + importer = Solver_Pointer->importer; + ghost_importer = Solver_Pointer->ghost_importer; + node_sorting_importer = Solver_Pointer->node_sorting_importer; + element_sorting_importer = Solver_Pointer->element_sorting_importer; + dof_importer = Solver_Pointer->dof_importer; + + // obtain node and element maps + comm = Solver_Pointer->comm; + map = Solver_Pointer->map; // map of node indices + ghost_node_map = Solver_Pointer->ghost_node_map; // map of node indices with ghosts on each rank + all_node_map = Solver_Pointer->all_node_map; // map of node indices with ghosts on each rank + element_map = Solver_Pointer->element_map; // non overlapping map of elements owned by each rank used in reduction ops + all_element_map = Solver_Pointer->all_element_map; // overlapping map of elements connected to the local nodes in each rank + local_dof_map = Solver_Pointer->local_dof_map; // map of local dofs (typically num_node_local*num_dim) + all_dof_map = Solver_Pointer->all_dof_map; // map of local and ghost dofs (typically num_node_all*num_dim) + + // obtain mesh coordinates, densities, and element connectivity + global_nodes_in_elem_distributed = Solver_Pointer->global_nodes_in_elem_distributed; // element to node connectivity table + node_nconn_distributed = Solver_Pointer->node_nconn_distributed; // how many elements a node is connected to + node_coords_distributed = Solver_Pointer->node_coords_distributed; + all_node_coords_distributed = Solver_Pointer->all_node_coords_distributed; + initial_node_coords_distributed = Solver_Pointer->node_coords_distributed; + all_initial_node_coords_distributed = Solver_Pointer->all_node_coords_distributed; + design_node_densities_distributed = Solver_Pointer->design_node_densities_distributed; + filtered_node_densities_distributed = Solver_Pointer->filtered_node_densities_distributed; + test_node_densities_distributed = Solver_Pointer->test_node_densities_distributed; + all_node_densities_distributed = Solver_Pointer->all_node_densities_distributed; + all_filtered_node_densities_distributed = Solver_Pointer->all_filtered_node_densities_distributed; + Global_Element_Densities = Solver_Pointer->Global_Element_Densities; + Element_Types = Solver_Pointer->Element_Types; + + // element select data + element_select = Solver_Pointer->element_select; + element_select->choose_3Delem_type(Element_Types(0), elem); + + // obtain boundary condition and loading data + nboundary_patches = Solver_Pointer->nboundary_patches; + Boundary_Patches = Solver_Pointer->Boundary_Patches; + node_specified_bcs = false; + // initialize for default + num_boundary_conditions = 0; + + // flag init + body_term_flag = nonzero_bc_flag = false; + + // output data + noutput = 0; } -FEA_Module::~FEA_Module() {} +FEA_Module::~FEA_Module() +{ +} /* ---------------------------------------------------------------------- find which boundary patches correspond to the given BC. @@ -135,98 +137,106 @@ FEA_Module::~FEA_Module() {} val = plane value, cylinder radius, shell radius ------------------------------------------------------------------------- */ -void FEA_Module::tag_boundaries(int bc_tag, real_t val, int bdy_set, real_t *patch_limits){ - - int is_on_set; - /* - if (bdy_set == num_bdy_sets_){ - std::cout << " ERROR: number of boundary sets must be increased by " - << bdy_set-num_bdy_sets_+1 << std::endl; - exit(0); - } - */ - - //test patch limits for feasibility - if(patch_limits != NULL){ - //test for upper bounds being greater than lower bounds - if(patch_limits[1] <= patch_limits[0]) std::cout << " Warning: patch limits for boundary condition are infeasible " << patch_limits[0] << " and " << patch_limits[1] << std::endl; - if(patch_limits[3] <= patch_limits[2]) std::cout << " Warning: patch limits for boundary condition are infeasible " << patch_limits[2] << " and " << patch_limits[3] << std::endl; - } - - // save the boundary vertices to this set that are on the plane - int counter = 0; - for (int iboundary_patch = 0; iboundary_patch < nboundary_patches; iboundary_patch++) { - - // check to see if this patch is on the specified plane - is_on_set = check_boundary(Boundary_Patches(iboundary_patch), bc_tag, val, patch_limits); // no=0, yes=1 - - if (is_on_set == 1){ - Boundary_Condition_Patches(bdy_set,counter) = iboundary_patch; - counter ++; +void FEA_Module::tag_boundaries(int bc_tag, real_t val, int bdy_set, real_t* patch_limits) +{ + int is_on_set; + /* + if (bdy_set == num_bdy_sets_){ + std::cout << " ERROR: number of boundary sets must be increased by " + << bdy_set-num_bdy_sets_+1 << std::endl; + exit(0); + } + */ + + // test patch limits for feasibility + if (patch_limits != NULL) + { + // test for upper bounds being greater than lower bounds + if (patch_limits[1] <= patch_limits[0]) + { + std::cout << " Warning: patch limits for boundary condition are infeasible " << patch_limits[0] << " and " << patch_limits[1] << std::endl; + } + if (patch_limits[3] <= patch_limits[2]) + { + std::cout << " Warning: patch limits for boundary condition are infeasible " << patch_limits[2] << " and " << patch_limits[3] << std::endl; + } } - } // end for bdy_patch - - // save the number of bdy patches in the set - NBoundary_Condition_Patches(bdy_set) = counter; - - *fos << " tagged boundary patches " << std::endl; + + // save the boundary vertices to this set that are on the plane + int counter = 0; + for (int iboundary_patch = 0; iboundary_patch < nboundary_patches; iboundary_patch++) + { + // check to see if this patch is on the specified plane + is_on_set = check_boundary(Boundary_Patches(iboundary_patch), bc_tag, val, patch_limits); // no=0, yes=1 + + if (is_on_set == 1) + { + Boundary_Condition_Patches(bdy_set, counter) = iboundary_patch; + counter++; + } + } // end for bdy_patch + + // save the number of bdy patches in the set + NBoundary_Condition_Patches(bdy_set) = counter; + + *fos << " tagged boundary patches " << std::endl; } /* ------------------------------------------------------------------------------------------- Communicate ghosts using the current optimization design data ---------------------------------------------------------------------------------------------- */ -void FEA_Module::comm_densities(Teuchos::RCP zp){ - - //set density vector to the current value chosen by the optimizer - test_node_densities_distributed = zp; - - //debug print of design vector - //std::ostream &out = std::cout; - //Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - //if(myrank==0) - //*fos << "Density data :" << std::endl; - //node_densities_distributed->describe(*fos,Teuchos::VERB_EXTREME); - //*fos << std::endl; - //std::fflush(stdout); - - //communicate design densities - //create import object using local node indices map and all indices map - Tpetra::Import importer(map, all_node_map); - - //comms to get ghosts - all_node_densities_distributed->doImport(*test_node_densities_distributed, importer, Tpetra::INSERT); - - //update_count++; - //if(update_count==1){ - //MPI_Barrier(world); - //MPI_Abort(world,4); - //} +void FEA_Module::comm_densities(Teuchos::RCP zp) +{ + // set density vector to the current value chosen by the optimizer + test_node_densities_distributed = zp; + + // debug print of design vector + // std::ostream &out = std::cout; + // Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + // if(myrank==0) + // *fos << "Density data :" << std::endl; + // node_densities_distributed->describe(*fos,Teuchos::VERB_EXTREME); + // *fos << std::endl; + // std::fflush(stdout); + + // communicate design densities + // create import object using local node indices map and all indices map + Tpetra::Import importer(map, all_node_map); + + // comms to get ghosts + all_node_densities_distributed->doImport(*test_node_densities_distributed, importer, Tpetra::INSERT); + + // update_count++; + // if(update_count==1){ + // MPI_Barrier(world); + // MPI_Abort(world,4); + // } } -void FEA_Module::comm_filtered_densities(){ - - //debug print of design vector - //std::ostream &out = std::cout; - //Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - //if(myrank==0) - //*fos << "Density data :" << std::endl; - //node_densities_distributed->describe(*fos,Teuchos::VERB_EXTREME); - //*fos << std::endl; - //std::fflush(stdout); - - //communicate design densities - //create import object using local node indices map and all indices map - Tpetra::Import importer(map, all_node_map); - - //comms to get ghosts - all_filtered_node_densities_distributed->doImport(*filtered_node_densities_distributed, importer, Tpetra::INSERT); - - //update_count++; - //if(update_count==1){ - //MPI_Barrier(world); - //MPI_Abort(world,4); - //} +void FEA_Module::comm_filtered_densities() +{ + // debug print of design vector + // std::ostream &out = std::cout; + // Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + // if(myrank==0) + // *fos << "Density data :" << std::endl; + // node_densities_distributed->describe(*fos,Teuchos::VERB_EXTREME); + // *fos << std::endl; + // std::fflush(stdout); + + // communicate design densities + // create import object using local node indices map and all indices map + Tpetra::Import importer(map, all_node_map); + + // comms to get ghosts + all_filtered_node_densities_distributed->doImport(*filtered_node_densities_distributed, importer, Tpetra::INSERT); + + // update_count++; + // if(update_count==1){ + // MPI_Barrier(world); + // MPI_Abort(world,4); + // } } /* ---------------------------------------------------------------------- @@ -235,89 +245,116 @@ void FEA_Module::comm_filtered_densities(){ val = plane value, radius, radius ------------------------------------------------------------------------- */ -int FEA_Module::check_boundary(Node_Combination &Patch_Nodes, int bc_tag, real_t val, real_t *patch_limits){ - - int is_on_set = 1; - const_host_vec_array all_node_coords = all_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - - //Nodes on the Patch - auto node_list = Patch_Nodes.node_set; - int num_dim = simparam->num_dims; - size_t nnodes = node_list.size(); - size_t node_rid; - real_t node_coord[num_dim]; - int dim_other1, dim_other2; - CArrayKokkos node_on_flags(nnodes, "node_on_flags"); - - //initialize - for(int inode = 0; inode < nnodes; inode++) node_on_flags(inode) = 0; - - if(bc_tag==0){ - dim_other1 = 1; - dim_other2 = 2; - } - else if(bc_tag==1){ - dim_other1 = 0; - dim_other2 = 2; - } - else if(bc_tag==2){ - dim_other1 = 0; - dim_other2 = 1; - } - - - //test for planes - if(bc_tag < 3) - for(int inode = 0; inode < nnodes; inode++){ - - node_rid = all_node_map->getLocalElement(node_list(inode)); - for(int init=0; init < num_dim; init++){ - node_coord[init] = all_node_coords(node_rid,init); +int FEA_Module::check_boundary(Node_Combination& Patch_Nodes, int bc_tag, real_t val, real_t* patch_limits) +{ + int is_on_set = 1; + const_host_vec_array all_node_coords = all_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + + // Nodes on the Patch + auto node_list = Patch_Nodes.node_set; + int num_dim = simparam->num_dims; + size_t nnodes = node_list.size(); + size_t node_rid; + real_t node_coord[num_dim]; + int dim_other1, dim_other2; + CArrayKokkos node_on_flags(nnodes, "node_on_flags"); + + // initialize + for (int inode = 0; inode < nnodes; inode++) + { + node_on_flags(inode) = 0; + } + + if (bc_tag == 0) + { + dim_other1 = 1; + dim_other2 = 2; } - if ( fabs(node_coord[bc_tag] - val) <= BC_EPSILON){ node_on_flags(inode) = 1; - - //test if within patch segment if user specified - if(patch_limits!=NULL){ - if (node_coord[dim_other1] - patch_limits[0] <= -BC_EPSILON) node_on_flags(inode) = 0; - if (node_coord[dim_other1] - patch_limits[1] >= BC_EPSILON) node_on_flags(inode) = 0; - if (node_coord[dim_other2] - patch_limits[2] <= -BC_EPSILON) node_on_flags(inode) = 0; - if (node_coord[dim_other2] - patch_limits[3] >= BC_EPSILON) node_on_flags(inode) = 0; - } + else if (bc_tag == 1) + { + dim_other1 = 0; + dim_other2 = 2; } - //debug print of node id and node coord - //std::cout << "node coords on task " << myrank << " for node " << node_rid << std::endl; - //std::cout << "coord " <getLocalElement(node_list(inode)); + for (int init = 0; init < num_dim; init++) + { + node_coord[init] = all_node_coords(node_rid, init); + } + if (fabs(node_coord[bc_tag] - val) <= BC_EPSILON) + { + node_on_flags(inode) = 1; + + // test if within patch segment if user specified + if (patch_limits != NULL) + { + if (node_coord[dim_other1] - patch_limits[0] <= -BC_EPSILON) + { + node_on_flags(inode) = 0; + } + if (node_coord[dim_other1] - patch_limits[1] >= BC_EPSILON) + { + node_on_flags(inode) = 0; + } + if (node_coord[dim_other2] - patch_limits[2] <= -BC_EPSILON) + { + node_on_flags(inode) = 0; + } + if (node_coord[dim_other2] - patch_limits[3] >= BC_EPSILON) + { + node_on_flags(inode) = 0; + } + } + } + // debug print of node id and node coord + // std::cout << "node coords on task " << myrank << " for node " << node_rid << std::endl; + // std::cout << "coord " < #include #include -//#include "Tpetra_Details_makeColMap.hpp" +// #include "Tpetra_Details_makeColMap.hpp" #include "Tpetra_Details_DefaultTypes.hpp" #include "utilities.h" #include "matar.h" #include "elements.h" #include "node_combination.h" -//#include "Simulation_Parameters/Simulation_Parameters.h" +// #include "Simulation_Parameters/Simulation_Parameters.h" using namespace mtr; -//forward declare +// forward declare class Solver; -//forward declarations -namespace ROL{ - template - class Problem; -} +// forward declarations +namespace ROL +{ +template +class Problem; +} // namespace ROL class Simulation_Parameters; enum class FEA_MODULE_TYPE; enum class BOUNDARY_TYPE; -class FEA_Module{ - +class FEA_Module +{ public: - FEA_Module(Solver *Solver_Pointer); - virtual ~FEA_Module(); + FEA_Module(Solver* Solver_Pointer); + virtual ~FEA_Module(); + + // Trilinos type definitions + typedef Tpetra::Map<>::local_ordinal_type LO; + typedef Tpetra::Map<>::global_ordinal_type GO; + + typedef Tpetra::CrsMatrix MAT; + typedef const Tpetra::CrsMatrix const_MAT; + typedef Tpetra::MultiVector MV; + typedef Tpetra::MultiVector MCONN; + + typedef Kokkos::ViewTraits::size_type SizeType; + typedef Tpetra::Details::DefaultTypes::node_type node_type; + using traits = Kokkos::ViewTraits; + + using array_layout = typename traits::array_layout; + using execution_space = typename traits::execution_space; + using device_type = typename traits::device_type; + using memory_traits = typename traits::memory_traits; + using global_size_t = Tpetra::global_size_t; + + typedef Kokkos::View values_array; + typedef Kokkos::View global_indices_array; + typedef Kokkos::View indices_array; + // typedef Kokkos::View row_pointers; + typedef MAT::local_graph_device_type::row_map_type::non_const_type row_pointers; + // typedef Kokkos::DualView::t_dev vec_array; + typedef MV::dual_view_type::t_dev vec_array; + typedef MV::dual_view_type::t_host host_vec_array; + typedef Kokkos::View const_host_vec_array; + typedef Kokkos::View const_vec_array; + typedef MV::dual_view_type dual_vec_array; + typedef MCONN::dual_view_type dual_elem_conn_array; + typedef MCONN::dual_view_type::t_host host_elem_conn_array; + typedef MCONN::dual_view_type::t_dev elem_conn_array; + typedef Kokkos::View const_host_elem_conn_array; + typedef Kokkos::View const_elem_conn_array; + typedef Kokkos::View const_host_int_array; + typedef Kokkos::View const_host_bool_array; + + // initializes memory for arrays used in the global stiffness matrix assembly + // initialize data for boundaries of the model and storage for boundary conditions and applied loads + virtual void init_boundaries() {} + + // initializes memory for arrays used in the global stiffness matrix assembly + virtual void init_boundary_sets(int num_boundary_sets) {} + + virtual void init_assembly() {} + + virtual void assemble_matrix() {} + + virtual void assemble_vector() {} + + // interfaces between user input and creating data structures for bcs + virtual void generate_bcs() {} + + // interfaces between user input and creating data structures for applied loads + virtual void generate_applied_loads() {} + + virtual void setup() {} + + virtual void module_cleanup() {} + + virtual int solve() { return 0; } + + virtual int eigensolve() { return 0; } + + virtual void read_conditions_ansys_dat(std::ifstream* in, std::streampos before_condition_header) {} + + virtual void linear_solver_parameters() {} + + virtual void comm_variables(Teuchos::RCP zp) {} + + virtual void comm_densities(Teuchos::RCP zp); + + virtual void comm_filtered_densities(); + + virtual void update_linear_solve(Teuchos::RCP zp, int compute_step) {} + + virtual void update_forward_solve(Teuchos::RCP zp) {} + + virtual void local_matrix(int ielem, CArrayKokkos& Local_Matrix) {} + + virtual void local_matrix_multiply(int ielem, CArrayKokkos& Local_Matrix) {} + + virtual void Element_Material_Properties(size_t ielem, real_t& Element_Modulus, real_t& Poisson_Ratio, real_t density) {} + + virtual void Gradient_Element_Material_Properties(size_t ielem, real_t& Element_Modulus, real_t& Poisson_Ratio, real_t density) {} + + virtual void Concavity_Element_Material_Properties(size_t ielem, real_t& Element_Modulus, real_t& Poisson_Ratio, real_t density) {} + + virtual void Body_Term(size_t ielem, real_t density, real_t* forces) {} + + virtual void Gradient_Body_Term(size_t ielem, real_t density, real_t* forces) {} + + virtual void tag_boundaries(int this_bc_tag, real_t val, int bdy_set, real_t* patch_limits = NULL); + + virtual int check_boundary(Node_Combination& Patch_Nodes, int this_bc_tag, real_t val, real_t* patch_limits); + + virtual void compute_output() {} + + virtual void output_control() {} + + virtual void write_data(std::map& point_data_scalars_double, + std::map& point_data_vectors_double, + std::map& cell_data_scalars_double, + std::map& cell_data_scalars_int, + std::map>& cell_data_fields_double) {} + + virtual void sort_output(Teuchos::RCP> sorted_map) {} // node data outputs - //Trilinos type definitions - typedef Tpetra::Map<>::local_ordinal_type LO; - typedef Tpetra::Map<>::global_ordinal_type GO; + virtual void sort_element_output(Teuchos::RCP> sorted_element_map) {} // element data outputs - typedef Tpetra::CrsMatrix MAT; - typedef const Tpetra::CrsMatrix const_MAT; - typedef Tpetra::MultiVector MV; - typedef Tpetra::MultiVector MCONN; + virtual void collect_output(Teuchos::RCP> global_reduce_map) {} - typedef Kokkos::ViewTraits::size_type SizeType; - typedef Tpetra::Details::DefaultTypes::node_type node_type; - using traits = Kokkos::ViewTraits; - - using array_layout = typename traits::array_layout; - using execution_space = typename traits::execution_space; - using device_type = typename traits::device_type; - using memory_traits = typename traits::memory_traits; - using global_size_t = Tpetra::global_size_t; - - typedef Kokkos::View values_array; - typedef Kokkos::View global_indices_array; - typedef Kokkos::View indices_array; - //typedef Kokkos::View row_pointers; - typedef MAT::local_graph_device_type::row_map_type::non_const_type row_pointers; - //typedef Kokkos::DualView::t_dev vec_array; - typedef MV::dual_view_type::t_dev vec_array; - typedef MV::dual_view_type::t_host host_vec_array; - typedef Kokkos::View const_host_vec_array; - typedef Kokkos::View const_vec_array; - typedef MV::dual_view_type dual_vec_array; - typedef MCONN::dual_view_type dual_elem_conn_array; - typedef MCONN::dual_view_type::t_host host_elem_conn_array; - typedef MCONN::dual_view_type::t_dev elem_conn_array; - typedef Kokkos::View const_host_elem_conn_array; - typedef Kokkos::View const_elem_conn_array; - typedef Kokkos::View const_host_int_array; - typedef Kokkos::View const_host_bool_array; - - //initializes memory for arrays used in the global stiffness matrix assembly - //initialize data for boundaries of the model and storage for boundary conditions and applied loads - virtual void init_boundaries() {} - - //initializes memory for arrays used in the global stiffness matrix assembly - virtual void init_boundary_sets(int num_boundary_sets) {} + virtual void node_density_constraints(host_vec_array node_densities_lower_bound) {} - virtual void init_assembly() {} + virtual void compute_topology_optimization_adjoint() {} // Force does not depend on node coords and velocity - virtual void assemble_matrix() {} + virtual void compute_topology_optimization_adjoint_full() {} // Force depends on node coords and velocity - virtual void assemble_vector() {} + virtual void compute_topology_optimization_gradient(const_vec_array design_densities, vec_array gradients) {} - //interfaces between user input and creating data structures for bcs - virtual void generate_bcs() {} - - //interfaces between user input and creating data structures for applied loads - virtual void generate_applied_loads() {} + virtual void compute_topology_optimization_gradient_full(Teuchos::RCP design_densities_distributed, Teuchos::RCP design_gradients_distributed) {} - virtual void setup() {} + // interfacing information + FEA_MODULE_TYPE Module_Type; + int last_compute_step; - virtual void module_cleanup() {} + // output stream + Teuchos::RCP fos; - virtual int solve() {return 0;} + std::shared_ptr element_select; + elements::Element3D* elem; + elements::Element2D* elem2D; - virtual int eigensolve() {return 0;} + Simulation_Parameters* simparam; + Solver* Solver_Pointer_; + int num_dim; + int num_gauss_points; + int my_fea_module_index_; - virtual void read_conditions_ansys_dat(std::ifstream *in, std::streampos before_condition_header) {} + // Local FEA data + size_t nlocal_nodes; + dual_vec_array dual_node_coords; // coordinates of the nodes + dual_vec_array dual_node_densities; // topology optimization design variable + dual_elem_conn_array dual_nodes_in_elem; // dual view of element connectivity to nodes + host_elem_conn_array nodes_in_elem; // host view of element connectivity to nodes + CArrayKokkos Element_Types; + CArrayKokkos Nodes_Per_Element_Type; + CArrayKokkos corner_value_storage; + CArrayKokkos corner_vector_storage; + CArrayKokkos corner_gradient_storage; - virtual void linear_solver_parameters() {} + // Ghost data on this MPI rank + size_t nghost_nodes; + CArrayKokkos ghost_nodes; + CArrayKokkos ghost_node_ranks; - virtual void comm_variables(Teuchos::RCP zp) {} + // Local FEA data including ghosts + size_t nall_nodes; + size_t rnum_elem; - virtual void comm_densities(Teuchos::RCP zp); - - virtual void comm_filtered_densities(); - - virtual void update_linear_solve(Teuchos::RCP zp, int compute_step) {} - - virtual void update_forward_solve(Teuchos::RCP zp) {} - - virtual void local_matrix(int ielem, CArrayKokkos &Local_Matrix) {} - - virtual void local_matrix_multiply(int ielem, CArrayKokkos &Local_Matrix) {} - - virtual void Element_Material_Properties(size_t ielem, real_t &Element_Modulus, real_t &Poisson_Ratio, real_t density) {} - - virtual void Gradient_Element_Material_Properties(size_t ielem, real_t &Element_Modulus, real_t &Poisson_Ratio, real_t density) {} - - virtual void Concavity_Element_Material_Properties(size_t ielem, real_t &Element_Modulus, real_t &Poisson_Ratio, real_t density) {} - - virtual void Body_Term(size_t ielem, real_t density, real_t *forces) {} - - virtual void Gradient_Body_Term(size_t ielem, real_t density, real_t *forces) {} - - virtual void tag_boundaries(int this_bc_tag, real_t val, int bdy_set, real_t *patch_limits = NULL); - - virtual int check_boundary(Node_Combination &Patch_Nodes, int this_bc_tag, real_t val, real_t *patch_limits); - - virtual void compute_output(){} - - virtual void output_control(){} - - virtual void write_data(std::map &point_data_scalars_double, - std::map &point_data_vectors_double, - std::map &cell_data_scalars_double, - std::map &cell_data_scalars_int, - std::map > &cell_data_fields_double){} - - virtual void sort_output(Teuchos::RCP> sorted_map){} //node data outputs - - virtual void sort_element_output(Teuchos::RCP> sorted_element_map){} //element data outputs - - virtual void collect_output(Teuchos::RCP> global_reduce_map){} - - virtual void node_density_constraints(host_vec_array node_densities_lower_bound){} - - virtual void compute_topology_optimization_adjoint() {} //Force does not depend on node coords and velocity - - virtual void compute_topology_optimization_adjoint_full() {} //Force depends on node coords and velocity - - virtual void compute_topology_optimization_gradient(const_vec_array design_densities, vec_array gradients) {} - - virtual void compute_topology_optimization_gradient_full(Teuchos::RCP design_densities_distributed, Teuchos::RCP design_gradients_distributed) {} - - //interfacing information - FEA_MODULE_TYPE Module_Type; - int last_compute_step; - - //output stream - Teuchos::RCP fos; - - std::shared_ptr element_select; - elements::Element3D *elem; - elements::Element2D *elem2D; - - - Simulation_Parameters *simparam; - Solver *Solver_Pointer_; - int num_dim; - int num_gauss_points; - int my_fea_module_index_; - - //Local FEA data - size_t nlocal_nodes; - dual_vec_array dual_node_coords; //coordinates of the nodes - dual_vec_array dual_node_densities; //topology optimization design variable - dual_elem_conn_array dual_nodes_in_elem; //dual view of element connectivity to nodes - host_elem_conn_array nodes_in_elem; //host view of element connectivity to nodes - CArrayKokkos Element_Types; - CArrayKokkos Nodes_Per_Element_Type; - CArrayKokkos corner_value_storage; - CArrayKokkos corner_vector_storage; - CArrayKokkos corner_gradient_storage; - - //Ghost data on this MPI rank - size_t nghost_nodes; - CArrayKokkos ghost_nodes; - CArrayKokkos ghost_node_ranks; - - //Local FEA data including ghosts - size_t nall_nodes; - size_t rnum_elem; - - //Global FEA data - long long int num_nodes, num_elem; - Teuchos::RCP > comm; - Teuchos::RCP > map; //map of node indices - Teuchos::RCP > ghost_node_map; //map of node indices with ghosts on each rank - Teuchos::RCP > all_node_map; //map of node indices with ghosts on each rank - Teuchos::RCP > element_map; //non overlapping map of elements owned by each rank used in reduction ops - Teuchos::RCP > all_element_map; //overlapping map of elements connected to the local nodes in each rank - Teuchos::RCP > local_dof_map; //map of local dofs (typically num_node_local*num_dim) - Teuchos::RCP > all_dof_map; //map of local and ghost dofs (typically num_node_all*num_dim) - Teuchos::RCP global_nodes_in_elem_distributed; //element to node connectivity table - Teuchos::RCP node_nconn_distributed; //how many elements a node is connected to - Teuchos::RCP node_coords_distributed; - Teuchos::RCP all_node_coords_distributed; - Teuchos::RCP initial_node_coords_distributed; - Teuchos::RCP all_initial_node_coords_distributed; - Teuchos::RCP design_node_densities_distributed; - Teuchos::RCP filtered_node_densities_distributed; - Teuchos::RCP test_node_densities_distributed; - Teuchos::RCP all_node_densities_distributed; - Teuchos::RCP all_filtered_node_densities_distributed; - Teuchos::RCP Global_Element_Densities; - - //Boundary Conditions Data - //CArray Patch_Nodes; - size_t nboundary_patches; - size_t num_boundary_conditions; - int current_bdy_id; - CArrayKokkos Boundary_Patches; - CArrayKokkos Boundary_Condition_Patches; //set of patches corresponding to each boundary condition - CArrayKokkos NBoundary_Condition_Patches; - CArrayKokkos Boundary_Condition_Patches_strides; - - //element selection parameters and data - size_t max_nodes_per_element; - - //determines if rhs gets a contribution from bcs - bool nonzero_bc_flag; - - //body force parameters - bool body_term_flag; - real_t *gravity_vector; - - //lists what kind of boundary condition the nodal DOF is subjected to if any - CArrayKokkos Node_DOF_Boundary_Condition_Type; - //lists what kind of boundary condition each boundary set is assigned to - CArrayKokkos Boundary_Condition_Type_List; - - //number of displacement boundary conditions acting on nodes; used to size the reduced global stiffness map - size_t Number_DOF_BCS; - - //MPI data - int myrank; //index of this mpi rank in the world communicator - int nranks; //number of mpi ranks in the world communicator - MPI_Comm world; //stores the default communicator object (MPI_COMM_WORLD) - Teuchos::RCP> importer; //all node comms - Teuchos::RCP> ghost_importer; //ghost node comms - Teuchos::RCP> node_sorting_importer; //sorted node comms - Teuchos::RCP> element_sorting_importer; //sorted element comms - Teuchos::RCP> dof_importer; //ghost dof comms - - //! mapping used to get local ghost index from the global ID. - //typedef ::Tpetra::Details::FixedHashTable - //global_to_local_table_host_type; - - //global_to_local_table_host_type global2local_map; - //CArrayKokkos active_ranks; - - //Pertains to local mesh information being stored as prescribed by the row map - global_size_t min_gid; - global_size_t max_gid; - global_size_t index_base; - - //allocation flags to avoid repeat MV and global matrix construction - int Matrix_alloc; - - //debug flags - int gradient_print_sync; - - //Topology Optimization parameter - int penalty_power; - bool nodal_density_flag; - - //runtime and counters for performance output - double linear_solve_time, hessvec_time, hessvec_linear_time; - int update_count, hessvec_count; - - //nodal DOF output data - enum vector_styles {NODAL, DOF}; //multivector can store as ndof by 1 or nnode by vector_size - int noutput; - int displacement_index; - std::vector> output_dof_names; - std::vector module_outputs; - std::vector vector_style; - std::vector output_vector_sizes; - - //Pointer to ROL Problem for optimization solves - Teuchos::RCP> problem; - - //Explicit BC data kept for now until bc data is refactored/consolidated - // node ids in bdy_patch set - RaggedRightArrayKokkos bdy_nodes_in_set; - DCArrayKokkos num_bdy_nodes_in_set; - bool node_specified_bcs; //currently happens with ansys import - - // patch ids in bdy set - size_t num_bdy_sets; - DynamicRaggedRightArrayKokkos bdy_patches_in_set; - + // Global FEA data + long long int num_nodes, num_elem; + Teuchos::RCP> comm; + Teuchos::RCP> map; // map of node indices + Teuchos::RCP> ghost_node_map; // map of node indices with ghosts on each rank + Teuchos::RCP> all_node_map; // map of node indices with ghosts on each rank + Teuchos::RCP> element_map; // non overlapping map of elements owned by each rank used in reduction ops + Teuchos::RCP> all_element_map; // overlapping map of elements connected to the local nodes in each rank + Teuchos::RCP> local_dof_map; // map of local dofs (typically num_node_local*num_dim) + Teuchos::RCP> all_dof_map; // map of local and ghost dofs (typically num_node_all*num_dim) + Teuchos::RCP global_nodes_in_elem_distributed; // element to node connectivity table + Teuchos::RCP node_nconn_distributed; // how many elements a node is connected to + Teuchos::RCP node_coords_distributed; + Teuchos::RCP all_node_coords_distributed; + Teuchos::RCP initial_node_coords_distributed; + Teuchos::RCP all_initial_node_coords_distributed; + Teuchos::RCP design_node_densities_distributed; + Teuchos::RCP filtered_node_densities_distributed; + Teuchos::RCP test_node_densities_distributed; + Teuchos::RCP all_node_densities_distributed; + Teuchos::RCP all_filtered_node_densities_distributed; + Teuchos::RCP Global_Element_Densities; + + // Boundary Conditions Data + // CArray Patch_Nodes; + size_t nboundary_patches; + size_t num_boundary_conditions; + int current_bdy_id; + CArrayKokkos Boundary_Patches; + CArrayKokkos Boundary_Condition_Patches; // set of patches corresponding to each boundary condition + CArrayKokkos NBoundary_Condition_Patches; + CArrayKokkos Boundary_Condition_Patches_strides; + + // element selection parameters and data + size_t max_nodes_per_element; + + // determines if rhs gets a contribution from bcs + bool nonzero_bc_flag; + + // body force parameters + bool body_term_flag; + real_t* gravity_vector; + + // lists what kind of boundary condition the nodal DOF is subjected to if any + CArrayKokkos Node_DOF_Boundary_Condition_Type; + // lists what kind of boundary condition each boundary set is assigned to + CArrayKokkos Boundary_Condition_Type_List; + + // number of displacement boundary conditions acting on nodes; used to size the reduced global stiffness map + size_t Number_DOF_BCS; + + // MPI data + int myrank; // index of this mpi rank in the world communicator + int nranks; // number of mpi ranks in the world communicator + MPI_Comm world; // stores the default communicator object (MPI_COMM_WORLD) + Teuchos::RCP> importer; // all node comms + Teuchos::RCP> ghost_importer; // ghost node comms + Teuchos::RCP> node_sorting_importer; // sorted node comms + Teuchos::RCP> element_sorting_importer; // sorted element comms + Teuchos::RCP> dof_importer; // ghost dof comms + + // ! mapping used to get local ghost index from the global ID. + // typedef ::Tpetra::Details::FixedHashTable + // global_to_local_table_host_type; + + // global_to_local_table_host_type global2local_map; + // CArrayKokkos active_ranks; + + // Pertains to local mesh information being stored as prescribed by the row map + global_size_t min_gid; + global_size_t max_gid; + global_size_t index_base; + + // allocation flags to avoid repeat MV and global matrix construction + int Matrix_alloc; + + // debug flags + int gradient_print_sync; + + // Topology Optimization parameter + int penalty_power; + bool nodal_density_flag; + + // runtime and counters for performance output + double linear_solve_time, hessvec_time, hessvec_linear_time; + int update_count, hessvec_count; + + // nodal DOF output data + enum vector_styles { NODAL, DOF }; // multivector can store as ndof by 1 or nnode by vector_size + int noutput; + int displacement_index; + std::vector> output_dof_names; + std::vector module_outputs; + std::vector vector_style; + std::vector output_vector_sizes; + + // Pointer to ROL Problem for optimization solves + Teuchos::RCP> problem; + + // Explicit BC data kept for now until bc data is refactored/consolidated + // node ids in bdy_patch set + RaggedRightArrayKokkos bdy_nodes_in_set; + DCArrayKokkos num_bdy_nodes_in_set; + bool node_specified_bcs; // currently happens with ansys import + + // patch ids in bdy set + size_t num_bdy_sets; + DynamicRaggedRightArrayKokkos bdy_patches_in_set; }; #endif // end HEADER_H diff --git a/src/Parallel-Solvers/FEA_Module_Inertial.cpp b/src/Parallel-Solvers/FEA_Module_Inertial.cpp index ee7e29549..e84945ea7 100644 --- a/src/Parallel-Solvers/FEA_Module_Inertial.cpp +++ b/src/Parallel-Solvers/FEA_Module_Inertial.cpp @@ -11,14 +11,14 @@ This program is open source under the BSD-3 License. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - + 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - + 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - + 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. @@ -39,7 +39,7 @@ #include #include #include -#include +#include #include // fmin, fmax, abs note: fminl is long #include #include @@ -74,1795 +74,2090 @@ using namespace utils; - FEA_Module_Inertial::FEA_Module_Inertial( - Inertial_Parameters& params, Solver *Solver_Pointer, - const int my_fea_module_index) - : FEA_Module(Solver_Pointer) { - - //assign interfacing index - my_fea_module_index_ = my_fea_module_index; - Module_Type = FEA_MODULE_TYPE::Inertial; - - //acquire base class data from existing simparam in solver (gets yaml options etc.) - module_params = ¶ms; - simparam = &(Solver_Pointer->simparam); - - //TO parameters - nodal_density_flag = simparam->nodal_density_flag; - - //property initialization flags - mass_init = false; - com_init[0] = com_init[1] = com_init[2] = false; - - //property update counters - mass_update = com_update[0] = com_update[1] = com_update[2] = -1; - - //RCP initialization - mass_gradients_distributed = Teuchos::null; - center_of_mass_gradients_distributed = Teuchos::null; - - //construct per element inertial property vectors - Global_Element_Masses = Teuchos::rcp(new MV(element_map, 1)); - Global_Element_Volumes = Teuchos::rcp(new MV(element_map, 1)); - Global_Element_Moments_x = Teuchos::rcp(new MV(element_map, 1)); - Global_Element_Moments_y = Teuchos::rcp(new MV(element_map, 1)); - Global_Element_Moments_z = Teuchos::rcp(new MV(element_map, 1)); - Global_Element_Moments_of_Inertia_xx = Teuchos::rcp(new MV(element_map, 1)); - Global_Element_Moments_of_Inertia_yy = Teuchos::rcp(new MV(element_map, 1)); - Global_Element_Moments_of_Inertia_zz = Teuchos::rcp(new MV(element_map, 1)); - Global_Element_Moments_of_Inertia_xy = Teuchos::rcp(new MV(element_map, 1)); - Global_Element_Moments_of_Inertia_xz = Teuchos::rcp(new MV(element_map, 1)); - Global_Element_Moments_of_Inertia_yz = Teuchos::rcp(new MV(element_map, 1)); + Inertial_Parameters& params, Solver* Solver_Pointer, + const int my_fea_module_index) + : FEA_Module(Solver_Pointer) +{ + // assign interfacing index + my_fea_module_index_ = my_fea_module_index; + Module_Type = FEA_MODULE_TYPE::Inertial; + + // acquire base class data from existing simparam in solver (gets yaml options etc.) + module_params = ¶ms; + simparam = &(Solver_Pointer->simparam); + + // TO parameters + nodal_density_flag = simparam->nodal_density_flag; + + // property initialization flags + mass_init = false; + com_init[0] = com_init[1] = com_init[2] = false; + + // property update counters + mass_update = com_update[0] = com_update[1] = com_update[2] = -1; + + // RCP initialization + mass_gradients_distributed = Teuchos::null; + center_of_mass_gradients_distributed = Teuchos::null; + + // construct per element inertial property vectors + Global_Element_Masses = Teuchos::rcp(new MV(element_map, 1)); + Global_Element_Volumes = Teuchos::rcp(new MV(element_map, 1)); + Global_Element_Moments_x = Teuchos::rcp(new MV(element_map, 1)); + Global_Element_Moments_y = Teuchos::rcp(new MV(element_map, 1)); + Global_Element_Moments_z = Teuchos::rcp(new MV(element_map, 1)); + Global_Element_Moments_of_Inertia_xx = Teuchos::rcp(new MV(element_map, 1)); + Global_Element_Moments_of_Inertia_yy = Teuchos::rcp(new MV(element_map, 1)); + Global_Element_Moments_of_Inertia_zz = Teuchos::rcp(new MV(element_map, 1)); + Global_Element_Moments_of_Inertia_xy = Teuchos::rcp(new MV(element_map, 1)); + Global_Element_Moments_of_Inertia_xz = Teuchos::rcp(new MV(element_map, 1)); + Global_Element_Moments_of_Inertia_yz = Teuchos::rcp(new MV(element_map, 1)); } -FEA_Module_Inertial::~FEA_Module_Inertial(){ } +FEA_Module_Inertial::~FEA_Module_Inertial() +{ +} /* ---------------------------------------------------------------------- Compute the mass of each element; estimated with quadrature ------------------------------------------------------------------------- */ -void FEA_Module_Inertial::compute_element_masses(const_host_vec_array design_densities, bool max_flag, bool use_initial_coords){ - //local number of uniquely assigned elements - size_t nonoverlap_nelements = element_map->getLocalNumElements(); - //initialize memory for volume storage - host_vec_array Element_Masses = Global_Element_Masses->getLocalView(Tpetra::Access::ReadWrite); - if(!nodal_density_flag) compute_element_volumes(); - const_host_vec_array Element_Volumes = Global_Element_Volumes->getLocalView(Tpetra::Access::ReadOnly); - //local variable for host view in the dual view - const_host_vec_array all_node_coords; - if(use_initial_coords){ - all_node_coords = all_initial_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - } - else{ - all_node_coords = all_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - } - const_host_vec_array all_design_densities; - if(nodal_density_flag) - all_design_densities = all_node_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_host_elem_conn_array nodes_in_elem = global_nodes_in_elem_distributed->getLocalView (Tpetra::Access::ReadOnly); - int nodes_per_elem = elem->num_basis(); - int z_quad,y_quad,x_quad, direct_product_count; - size_t local_node_id; - LO ielem; - GO global_element_index; - - real_t Jacobian, current_density, weight_multiply; - //CArrayKokkos legendre_nodes_1D(num_gauss_points); - //CArrayKokkos legendre_weights_1D(num_gauss_points); - CArray legendre_nodes_1D(num_gauss_points); - CArray legendre_weights_1D(num_gauss_points); - real_t pointer_quad_coordinate[num_dim]; - real_t pointer_quad_coordinate_weight[num_dim]; - real_t pointer_interpolated_point[num_dim]; - real_t pointer_JT_row1[num_dim]; - real_t pointer_JT_row2[num_dim]; - real_t pointer_JT_row3[num_dim]; - ViewCArray quad_coordinate(pointer_quad_coordinate,num_dim); - ViewCArray quad_coordinate_weight(pointer_quad_coordinate_weight,num_dim); - ViewCArray interpolated_point(pointer_interpolated_point,num_dim); - ViewCArray JT_row1(pointer_JT_row1,num_dim); - ViewCArray JT_row2(pointer_JT_row2,num_dim); - ViewCArray JT_row3(pointer_JT_row3,num_dim); - - real_t pointer_basis_values[elem->num_basis()]; - real_t pointer_basis_derivative_s1[elem->num_basis()]; - real_t pointer_basis_derivative_s2[elem->num_basis()]; - real_t pointer_basis_derivative_s3[elem->num_basis()]; - ViewCArray basis_values(pointer_basis_values,elem->num_basis()); - ViewCArray basis_derivative_s1(pointer_basis_derivative_s1,elem->num_basis()); - ViewCArray basis_derivative_s2(pointer_basis_derivative_s2,elem->num_basis()); - ViewCArray basis_derivative_s3(pointer_basis_derivative_s3,elem->num_basis()); - CArrayKokkos nodal_positions(elem->num_basis(),num_dim); - CArrayKokkos nodal_density(elem->num_basis()); - - //initialize weights - elements::legendre_nodes_1D(legendre_nodes_1D,num_gauss_points); - elements::legendre_weights_1D(legendre_weights_1D,num_gauss_points); - - Solver::node_ordering_convention active_node_ordering_convention = Solver_Pointer_->active_node_ordering_convention; - CArrayKokkos convert_node_order(max_nodes_per_element); - if((active_node_ordering_convention == Solver::ENSIGHT && num_dim==3)||(active_node_ordering_convention == Solver::IJK && num_dim==2)){ - convert_node_order(0) = 0; - convert_node_order(1) = 1; - convert_node_order(2) = 3; - convert_node_order(3) = 2; - if(num_dim == 3){ - convert_node_order(4) = 4; - convert_node_order(5) = 5; - convert_node_order(6) = 7; - convert_node_order(7) = 6; +void FEA_Module_Inertial::compute_element_masses(const_host_vec_array design_densities, bool max_flag, bool use_initial_coords) +{ + // local number of uniquely assigned elements + size_t nonoverlap_nelements = element_map->getLocalNumElements(); + // initialize memory for volume storage + host_vec_array Element_Masses = Global_Element_Masses->getLocalView(Tpetra::Access::ReadWrite); + if (!nodal_density_flag) + { + compute_element_volumes(); } - } - else{ - convert_node_order(0) = 0; - convert_node_order(1) = 1; - convert_node_order(2) = 2; - convert_node_order(3) = 3; - if(num_dim==3){ - convert_node_order(4) = 4; - convert_node_order(5) = 5; - convert_node_order(6) = 6; - convert_node_order(7) = 7; + const_host_vec_array Element_Volumes = Global_Element_Volumes->getLocalView(Tpetra::Access::ReadOnly); + // local variable for host view in the dual view + const_host_vec_array all_node_coords; + if (use_initial_coords) + { + all_node_coords = all_initial_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); } - } - - //loop over elements and use quadrature rule to compute volume from Jacobian determinant - for(int nonoverlapping_ielem = 0; nonoverlapping_ielem < nonoverlap_nelements; nonoverlapping_ielem++){ - global_element_index = element_map->getGlobalElement(nonoverlapping_ielem); - ielem = all_element_map->getLocalElement(global_element_index); - if(nodal_density_flag){ - //acquire set of nodes for this local element - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - local_node_id = all_node_map->getLocalElement(nodes_in_elem(ielem, convert_node_order(node_loop))); - nodal_positions(node_loop,0) = all_node_coords(local_node_id,0); - nodal_positions(node_loop,1) = all_node_coords(local_node_id,1); - nodal_positions(node_loop,2) = all_node_coords(local_node_id,2); - if(nodal_density_flag) nodal_density(node_loop) = all_design_densities(local_node_id,0); - /* - if(myrank==1&&nodal_positions(node_loop,2)>10000000){ - std::cout << " LOCAL MATRIX DEBUG ON TASK " << myrank << std::endl; - std::cout << node_loop+1 <<" " << local_node_id <<" "<< nodes_in_elem(ielem, node_loop) << " "<< nodal_positions(node_loop,2) << std::endl; - std::fflush(stdout); - } - */ - //std::cout << local_node_id << " " << nodes_in_elem(ielem, node_loop) << " " - //<< nodal_positions(node_loop,0) << " " << nodal_positions(node_loop,1) << " "<< nodal_positions(node_loop,2) << " " << nodal_density(node_loop) <getLocalView(Tpetra::Access::ReadOnly); } - - //debug print of index - //std::cout << "nonoverlap element id on TASK " << myrank << " is " << nonoverlapping_ielem << std::endl; - //std::fflush(stdout); - - //initialize element mass - Element_Masses(nonoverlapping_ielem,0) = 0; - - if(Element_Types(ielem)==elements::elem_types::Hex8){ - direct_product_count = std::pow(num_gauss_points,num_dim); + const_host_vec_array all_design_densities; + if (nodal_density_flag) + { + all_design_densities = all_node_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); } - - //loop over quadrature points - for(int iquad=0; iquad < direct_product_count; iquad++){ - - //set current quadrature point - if(num_dim==3) z_quad = iquad/(num_gauss_points*num_gauss_points); - y_quad = (iquad % (num_gauss_points*num_gauss_points))/num_gauss_points; - x_quad = iquad % num_gauss_points; - quad_coordinate(0) = legendre_nodes_1D(x_quad); - quad_coordinate(1) = legendre_nodes_1D(y_quad); - if(num_dim==3) - quad_coordinate(2) = legendre_nodes_1D(z_quad); - - //set current quadrature weight - quad_coordinate_weight(0) = legendre_weights_1D(x_quad); - quad_coordinate_weight(1) = legendre_weights_1D(y_quad); - if(num_dim==3) - quad_coordinate_weight(2) = legendre_weights_1D(z_quad); - else - quad_coordinate_weight(2) = 1; - weight_multiply = quad_coordinate_weight(0)*quad_coordinate_weight(1)*quad_coordinate_weight(2); - - //compute shape functions at this point for the element type - elem->basis(basis_values,quad_coordinate); - - //compute all the necessary coordinates and derivatives at this point - - //compute shape function derivatives - elem->partial_xi_basis(basis_derivative_s1,quad_coordinate); - elem->partial_eta_basis(basis_derivative_s2,quad_coordinate); - elem->partial_mu_basis(basis_derivative_s3,quad_coordinate); - - //compute derivatives of x,y,z w.r.t the s,t,w isoparametric space needed by JT (Transpose of the Jacobian) - //derivative of x,y,z w.r.t s - JT_row1(0) = 0; - JT_row1(1) = 0; - JT_row1(2) = 0; - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - JT_row1(0) += nodal_positions(node_loop,0)*basis_derivative_s1(node_loop); - JT_row1(1) += nodal_positions(node_loop,1)*basis_derivative_s1(node_loop); - JT_row1(2) += nodal_positions(node_loop,2)*basis_derivative_s1(node_loop); - } - - //derivative of x,y,z w.r.t t - JT_row2(0) = 0; - JT_row2(1) = 0; - JT_row2(2) = 0; - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - JT_row2(0) += nodal_positions(node_loop,0)*basis_derivative_s2(node_loop); - JT_row2(1) += nodal_positions(node_loop,1)*basis_derivative_s2(node_loop); - JT_row2(2) += nodal_positions(node_loop,2)*basis_derivative_s2(node_loop); - } - - //derivative of x,y,z w.r.t w - JT_row3(0) = 0; - JT_row3(1) = 0; - JT_row3(2) = 0; - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - JT_row3(0) += nodal_positions(node_loop,0)*basis_derivative_s3(node_loop); - JT_row3(1) += nodal_positions(node_loop,1)*basis_derivative_s3(node_loop); - JT_row3(2) += nodal_positions(node_loop,2)*basis_derivative_s3(node_loop); - //debug print - /*if(myrank==1&&nodal_positions(node_loop,2)*basis_derivative_s3(node_loop)<-10000000){ - std::cout << " ELEMENT VOLUME JACOBIAN DEBUG ON TASK " << myrank << std::endl; - std::cout << node_loop+1 << " " << JT_row3(2) << " "<< nodal_positions(node_loop,2) <<" "<< basis_derivative_s3(node_loop) << std::endl; - std::fflush(stdout); - }*/ - } - - - //compute the determinant of the Jacobian - Jacobian = JT_row1(0)*(JT_row2(1)*JT_row3(2)-JT_row3(1)*JT_row2(2))- - JT_row1(1)*(JT_row2(0)*JT_row3(2)-JT_row3(0)*JT_row2(2))+ - JT_row1(2)*(JT_row2(0)*JT_row3(1)-JT_row3(0)*JT_row2(1)); - if(Jacobian<0) Jacobian = -Jacobian; - - //compute density - current_density = 0; - if(max_flag){ - current_density = 1; - } - else{ - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - current_density += nodal_density(node_loop)*basis_values(node_loop); + const_host_elem_conn_array nodes_in_elem = global_nodes_in_elem_distributed->getLocalView(Tpetra::Access::ReadOnly); + int nodes_per_elem = elem->num_basis(); + int z_quad, y_quad, x_quad, direct_product_count; + size_t local_node_id; + LO ielem; + GO global_element_index; + + real_t Jacobian, current_density, weight_multiply; + // CArrayKokkos legendre_nodes_1D(num_gauss_points); + // CArrayKokkos legendre_weights_1D(num_gauss_points); + CArray legendre_nodes_1D(num_gauss_points); + CArray legendre_weights_1D(num_gauss_points); + real_t pointer_quad_coordinate[num_dim]; + real_t pointer_quad_coordinate_weight[num_dim]; + real_t pointer_interpolated_point[num_dim]; + real_t pointer_JT_row1[num_dim]; + real_t pointer_JT_row2[num_dim]; + real_t pointer_JT_row3[num_dim]; + ViewCArray quad_coordinate(pointer_quad_coordinate, num_dim); + ViewCArray quad_coordinate_weight(pointer_quad_coordinate_weight, num_dim); + ViewCArray interpolated_point(pointer_interpolated_point, num_dim); + ViewCArray JT_row1(pointer_JT_row1, num_dim); + ViewCArray JT_row2(pointer_JT_row2, num_dim); + ViewCArray JT_row3(pointer_JT_row3, num_dim); + + real_t pointer_basis_values[elem->num_basis()]; + real_t pointer_basis_derivative_s1[elem->num_basis()]; + real_t pointer_basis_derivative_s2[elem->num_basis()]; + real_t pointer_basis_derivative_s3[elem->num_basis()]; + ViewCArray basis_values(pointer_basis_values, elem->num_basis()); + ViewCArray basis_derivative_s1(pointer_basis_derivative_s1, elem->num_basis()); + ViewCArray basis_derivative_s2(pointer_basis_derivative_s2, elem->num_basis()); + ViewCArray basis_derivative_s3(pointer_basis_derivative_s3, elem->num_basis()); + CArrayKokkos nodal_positions(elem->num_basis(), num_dim); + CArrayKokkos nodal_density(elem->num_basis()); + + // initialize weights + elements::legendre_nodes_1D(legendre_nodes_1D, num_gauss_points); + elements::legendre_weights_1D(legendre_weights_1D, num_gauss_points); + + Solver::node_ordering_convention active_node_ordering_convention = Solver_Pointer_->active_node_ordering_convention; + CArrayKokkos convert_node_order(max_nodes_per_element); + if ((active_node_ordering_convention == Solver::ENSIGHT && num_dim == 3) || (active_node_ordering_convention == Solver::IJK && num_dim == 2)) + { + convert_node_order(0) = 0; + convert_node_order(1) = 1; + convert_node_order(2) = 3; + convert_node_order(3) = 2; + if (num_dim == 3) + { + convert_node_order(4) = 4; + convert_node_order(5) = 5; + convert_node_order(6) = 7; + convert_node_order(7) = 6; } - } - - Element_Masses(nonoverlapping_ielem,0) += current_density*weight_multiply*Jacobian; } + else + { + convert_node_order(0) = 0; + convert_node_order(1) = 1; + convert_node_order(2) = 2; + convert_node_order(3) = 3; + if (num_dim == 3) + { + convert_node_order(4) = 4; + convert_node_order(5) = 5; + convert_node_order(6) = 6; + convert_node_order(7) = 7; + } } - else{ - Element_Masses(nonoverlapping_ielem,0) = Element_Volumes(nonoverlapping_ielem,0)*design_densities(nonoverlapping_ielem,0); + + // loop over elements and use quadrature rule to compute volume from Jacobian determinant + for (int nonoverlapping_ielem = 0; nonoverlapping_ielem < nonoverlap_nelements; nonoverlapping_ielem++) + { + global_element_index = element_map->getGlobalElement(nonoverlapping_ielem); + ielem = all_element_map->getLocalElement(global_element_index); + if (nodal_density_flag) + { + // acquire set of nodes for this local element + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + local_node_id = all_node_map->getLocalElement(nodes_in_elem(ielem, convert_node_order(node_loop))); + nodal_positions(node_loop, 0) = all_node_coords(local_node_id, 0); + nodal_positions(node_loop, 1) = all_node_coords(local_node_id, 1); + nodal_positions(node_loop, 2) = all_node_coords(local_node_id, 2); + if (nodal_density_flag) + { + nodal_density(node_loop) = all_design_densities(local_node_id, 0); + } + /* + if(myrank==1&&nodal_positions(node_loop,2)>10000000){ + std::cout << " LOCAL MATRIX DEBUG ON TASK " << myrank << std::endl; + std::cout << node_loop+1 <<" " << local_node_id <<" "<< nodes_in_elem(ielem, node_loop) << " "<< nodal_positions(node_loop,2) << std::endl; + std::fflush(stdout); + } + */ + // std::cout << local_node_id << " " << nodes_in_elem(ielem, node_loop) << " " + // << nodal_positions(node_loop,0) << " " << nodal_positions(node_loop,1) << " "<< nodal_positions(node_loop,2) << " " << nodal_density(node_loop) <basis(basis_values, quad_coordinate); + + // compute all the necessary coordinates and derivatives at this point + + // compute shape function derivatives + elem->partial_xi_basis(basis_derivative_s1, quad_coordinate); + elem->partial_eta_basis(basis_derivative_s2, quad_coordinate); + elem->partial_mu_basis(basis_derivative_s3, quad_coordinate); + + // compute derivatives of x,y,z w.r.t the s,t,w isoparametric space needed by JT (Transpose of the Jacobian) + // derivative of x,y,z w.r.t s + JT_row1(0) = 0; + JT_row1(1) = 0; + JT_row1(2) = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row1(0) += nodal_positions(node_loop, 0) * basis_derivative_s1(node_loop); + JT_row1(1) += nodal_positions(node_loop, 1) * basis_derivative_s1(node_loop); + JT_row1(2) += nodal_positions(node_loop, 2) * basis_derivative_s1(node_loop); + } + + // derivative of x,y,z w.r.t t + JT_row2(0) = 0; + JT_row2(1) = 0; + JT_row2(2) = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row2(0) += nodal_positions(node_loop, 0) * basis_derivative_s2(node_loop); + JT_row2(1) += nodal_positions(node_loop, 1) * basis_derivative_s2(node_loop); + JT_row2(2) += nodal_positions(node_loop, 2) * basis_derivative_s2(node_loop); + } + + // derivative of x,y,z w.r.t w + JT_row3(0) = 0; + JT_row3(1) = 0; + JT_row3(2) = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row3(0) += nodal_positions(node_loop, 0) * basis_derivative_s3(node_loop); + JT_row3(1) += nodal_positions(node_loop, 1) * basis_derivative_s3(node_loop); + JT_row3(2) += nodal_positions(node_loop, 2) * basis_derivative_s3(node_loop); + // debug print + /*if(myrank==1&&nodal_positions(node_loop,2)*basis_derivative_s3(node_loop)<-10000000){ + std::cout << " ELEMENT VOLUME JACOBIAN DEBUG ON TASK " << myrank << std::endl; + std::cout << node_loop+1 << " " << JT_row3(2) << " "<< nodal_positions(node_loop,2) <<" "<< basis_derivative_s3(node_loop) << std::endl; + std::fflush(stdout); + }*/ + } + + // compute the determinant of the Jacobian + Jacobian = JT_row1(0) * (JT_row2(1) * JT_row3(2) - JT_row3(1) * JT_row2(2)) - + JT_row1(1) * (JT_row2(0) * JT_row3(2) - JT_row3(0) * JT_row2(2)) + + JT_row1(2) * (JT_row2(0) * JT_row3(1) - JT_row3(0) * JT_row2(1)); + if (Jacobian < 0) + { + Jacobian = -Jacobian; + } + + // compute density + current_density = 0; + if (max_flag) + { + current_density = 1; + } + else + { + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + current_density += nodal_density(node_loop) * basis_values(node_loop); + } + } + + Element_Masses(nonoverlapping_ielem, 0) += current_density * weight_multiply * Jacobian; + } + } + else + { + Element_Masses(nonoverlapping_ielem, 0) = Element_Volumes(nonoverlapping_ielem, 0) * design_densities(nonoverlapping_ielem, 0); + } } - } - - //std::ostream &out = std::cout; - //Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - //if(myrank==0) - //*fos << "Global Element Masses:" << std::endl; - //Global_Element_Masses->describe(*fos,Teuchos::VERB_EXTREME); - //*fos << std::endl; + + // std::ostream &out = std::cout; + // Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + // if(myrank==0) + // *fos << "Global Element Masses:" << std::endl; + // Global_Element_Masses->describe(*fos,Teuchos::VERB_EXTREME); + // *fos << std::endl; } /* ---------------------------------------------------------------------- Compute the gradients of mass function with respect to nodal densities ------------------------------------------------------------------------- */ -void FEA_Module_Inertial::compute_nodal_gradients(const_host_vec_array design_variables, host_vec_array design_gradients, bool use_initial_coords){ - //local number of uniquely assigned elements - size_t nonoverlap_nelements = element_map->getLocalNumElements(); - //local variable for host view in the dual view - const_host_vec_array all_node_coords; - if(use_initial_coords){ - all_node_coords = all_initial_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - } - else{ - all_node_coords = all_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - } - const_host_elem_conn_array nodes_in_elem = global_nodes_in_elem_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_host_vec_array all_node_densities; - if(nodal_density_flag) - all_node_densities = all_node_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); - int nodes_per_elem = elem->num_basis(); - int z_quad,y_quad,x_quad, direct_product_count; - size_t local_node_id; - LO ielem; - GO global_element_index; - - real_t Jacobian, weight_multiply; - //CArrayKokkos legendre_nodes_1D(num_gauss_points); - //CArrayKokkos legendre_weights_1D(num_gauss_points); - CArray legendre_nodes_1D(num_gauss_points); - CArray legendre_weights_1D(num_gauss_points); - real_t pointer_quad_coordinate[num_dim]; - real_t pointer_quad_coordinate_weight[num_dim]; - real_t pointer_interpolated_point[num_dim]; - real_t pointer_JT_row1[num_dim]; - real_t pointer_JT_row2[num_dim]; - real_t pointer_JT_row3[num_dim]; - ViewCArray quad_coordinate(pointer_quad_coordinate,num_dim); - ViewCArray quad_coordinate_weight(pointer_quad_coordinate_weight,num_dim); - ViewCArray interpolated_point(pointer_interpolated_point,num_dim); - ViewCArray JT_row1(pointer_JT_row1,num_dim); - ViewCArray JT_row2(pointer_JT_row2,num_dim); - ViewCArray JT_row3(pointer_JT_row3,num_dim); - - real_t pointer_basis_values[elem->num_basis()]; - real_t pointer_basis_derivative_s1[elem->num_basis()]; - real_t pointer_basis_derivative_s2[elem->num_basis()]; - real_t pointer_basis_derivative_s3[elem->num_basis()]; - ViewCArray basis_values(pointer_basis_values,elem->num_basis()); - ViewCArray basis_derivative_s1(pointer_basis_derivative_s1,elem->num_basis()); - ViewCArray basis_derivative_s2(pointer_basis_derivative_s2,elem->num_basis()); - ViewCArray basis_derivative_s3(pointer_basis_derivative_s3,elem->num_basis()); - CArrayKokkos nodal_positions(elem->num_basis(),num_dim); - CArrayKokkos nodal_density(elem->num_basis()); - - //initialize weights - elements::legendre_nodes_1D(legendre_nodes_1D,num_gauss_points); - elements::legendre_weights_1D(legendre_weights_1D,num_gauss_points); - - Solver::node_ordering_convention active_node_ordering_convention = Solver_Pointer_->active_node_ordering_convention; - CArrayKokkos convert_node_order(max_nodes_per_element); - if((active_node_ordering_convention == Solver::ENSIGHT && num_dim==3)||(active_node_ordering_convention == Solver::IJK && num_dim==2)){ - convert_node_order(0) = 0; - convert_node_order(1) = 1; - convert_node_order(2) = 3; - convert_node_order(3) = 2; - if(num_dim == 3){ - convert_node_order(4) = 4; - convert_node_order(5) = 5; - convert_node_order(6) = 7; - convert_node_order(7) = 6; +void FEA_Module_Inertial::compute_nodal_gradients(const_host_vec_array design_variables, host_vec_array design_gradients, bool use_initial_coords) +{ + // local number of uniquely assigned elements + size_t nonoverlap_nelements = element_map->getLocalNumElements(); + // local variable for host view in the dual view + const_host_vec_array all_node_coords; + if (use_initial_coords) + { + all_node_coords = all_initial_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); } - } - else{ - convert_node_order(0) = 0; - convert_node_order(1) = 1; - convert_node_order(2) = 2; - convert_node_order(3) = 3; - if(num_dim==3){ - convert_node_order(4) = 4; - convert_node_order(5) = 5; - convert_node_order(6) = 6; - convert_node_order(7) = 7; + else + { + all_node_coords = all_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); } - } - - //initialize design gradients to 0 - for(int init = 0; init < nlocal_nodes; init++) - design_gradients(init,0) = 0; - - //loop over elements and use quadrature rule to compute volume from Jacobian determinant - for(int ielem = 0; ielem < rnum_elem; ielem++){ - //acquire set of nodes for this local element - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - local_node_id = all_node_map->getLocalElement(nodes_in_elem(ielem, convert_node_order(node_loop))); - nodal_positions(node_loop,0) = all_node_coords(local_node_id,0); - nodal_positions(node_loop,1) = all_node_coords(local_node_id,1); - nodal_positions(node_loop,2) = all_node_coords(local_node_id,2); - if(nodal_density_flag) nodal_density(node_loop) = all_node_densities(local_node_id,0); - /* - if(myrank==1&&nodal_positions(node_loop,2)>10000000){ - std::cout << " LOCAL MATRIX DEBUG ON TASK " << myrank << std::endl; - std::cout << node_loop+1 <<" " << local_node_id <<" "<< nodes_in_elem(ielem, node_loop) << " "<< nodal_positions(node_loop,2) << std::endl; - std::fflush(stdout); - } - */ - //std::cout << local_node_id << " " << nodes_in_elem(ielem, node_loop) << " " << nodal_positions(node_loop,0) << " " << nodal_positions(node_loop,1) << " "<< nodal_positions(node_loop,2) <getLocalView(Tpetra::Access::ReadOnly); + const_host_vec_array all_node_densities; + if (nodal_density_flag) + { + all_node_densities = all_node_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); } - - if(Element_Types(ielem)==elements::elem_types::Hex8){ - direct_product_count = std::pow(num_gauss_points,num_dim); + int nodes_per_elem = elem->num_basis(); + int z_quad, y_quad, x_quad, direct_product_count; + size_t local_node_id; + LO ielem; + GO global_element_index; + + real_t Jacobian, weight_multiply; + // CArrayKokkos legendre_nodes_1D(num_gauss_points); + // CArrayKokkos legendre_weights_1D(num_gauss_points); + CArray legendre_nodes_1D(num_gauss_points); + CArray legendre_weights_1D(num_gauss_points); + real_t pointer_quad_coordinate[num_dim]; + real_t pointer_quad_coordinate_weight[num_dim]; + real_t pointer_interpolated_point[num_dim]; + real_t pointer_JT_row1[num_dim]; + real_t pointer_JT_row2[num_dim]; + real_t pointer_JT_row3[num_dim]; + ViewCArray quad_coordinate(pointer_quad_coordinate, num_dim); + ViewCArray quad_coordinate_weight(pointer_quad_coordinate_weight, num_dim); + ViewCArray interpolated_point(pointer_interpolated_point, num_dim); + ViewCArray JT_row1(pointer_JT_row1, num_dim); + ViewCArray JT_row2(pointer_JT_row2, num_dim); + ViewCArray JT_row3(pointer_JT_row3, num_dim); + + real_t pointer_basis_values[elem->num_basis()]; + real_t pointer_basis_derivative_s1[elem->num_basis()]; + real_t pointer_basis_derivative_s2[elem->num_basis()]; + real_t pointer_basis_derivative_s3[elem->num_basis()]; + ViewCArray basis_values(pointer_basis_values, elem->num_basis()); + ViewCArray basis_derivative_s1(pointer_basis_derivative_s1, elem->num_basis()); + ViewCArray basis_derivative_s2(pointer_basis_derivative_s2, elem->num_basis()); + ViewCArray basis_derivative_s3(pointer_basis_derivative_s3, elem->num_basis()); + CArrayKokkos nodal_positions(elem->num_basis(), num_dim); + CArrayKokkos nodal_density(elem->num_basis()); + + // initialize weights + elements::legendre_nodes_1D(legendre_nodes_1D, num_gauss_points); + elements::legendre_weights_1D(legendre_weights_1D, num_gauss_points); + + Solver::node_ordering_convention active_node_ordering_convention = Solver_Pointer_->active_node_ordering_convention; + CArrayKokkos convert_node_order(max_nodes_per_element); + if ((active_node_ordering_convention == Solver::ENSIGHT && num_dim == 3) || (active_node_ordering_convention == Solver::IJK && num_dim == 2)) + { + convert_node_order(0) = 0; + convert_node_order(1) = 1; + convert_node_order(2) = 3; + convert_node_order(3) = 2; + if (num_dim == 3) + { + convert_node_order(4) = 4; + convert_node_order(5) = 5; + convert_node_order(6) = 7; + convert_node_order(7) = 6; + } } - - //loop over quadrature points - for(int iquad=0; iquad < direct_product_count; iquad++){ - - //set current quadrature point - if(num_dim==3) z_quad = iquad/(num_gauss_points*num_gauss_points); - y_quad = (iquad % (num_gauss_points*num_gauss_points))/num_gauss_points; - x_quad = iquad % num_gauss_points; - quad_coordinate(0) = legendre_nodes_1D(x_quad); - quad_coordinate(1) = legendre_nodes_1D(y_quad); - if(num_dim==3) - quad_coordinate(2) = legendre_nodes_1D(z_quad); - - //set current quadrature weight - quad_coordinate_weight(0) = legendre_weights_1D(x_quad); - quad_coordinate_weight(1) = legendre_weights_1D(y_quad); - if(num_dim==3) - quad_coordinate_weight(2) = legendre_weights_1D(z_quad); - else - quad_coordinate_weight(2) = 1; - weight_multiply = quad_coordinate_weight(0)*quad_coordinate_weight(1)*quad_coordinate_weight(2); - - //compute shape functions at this point for the element type - elem->basis(basis_values,quad_coordinate); - - //compute all the necessary coordinates and derivatives at this point - - //compute shape function derivatives - elem->partial_xi_basis(basis_derivative_s1,quad_coordinate); - elem->partial_eta_basis(basis_derivative_s2,quad_coordinate); - elem->partial_mu_basis(basis_derivative_s3,quad_coordinate); - - //compute derivatives of x,y,z w.r.t the s,t,w isoparametric space needed by JT (Transpose of the Jacobian) - //derivative of x,y,z w.r.t s - JT_row1(0) = 0; - JT_row1(1) = 0; - JT_row1(2) = 0; - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - JT_row1(0) += nodal_positions(node_loop,0)*basis_derivative_s1(node_loop); - JT_row1(1) += nodal_positions(node_loop,1)*basis_derivative_s1(node_loop); - JT_row1(2) += nodal_positions(node_loop,2)*basis_derivative_s1(node_loop); - } - - //derivative of x,y,z w.r.t t - JT_row2(0) = 0; - JT_row2(1) = 0; - JT_row2(2) = 0; - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - JT_row2(0) += nodal_positions(node_loop,0)*basis_derivative_s2(node_loop); - JT_row2(1) += nodal_positions(node_loop,1)*basis_derivative_s2(node_loop); - JT_row2(2) += nodal_positions(node_loop,2)*basis_derivative_s2(node_loop); - } - - //derivative of x,y,z w.r.t w - JT_row3(0) = 0; - JT_row3(1) = 0; - JT_row3(2) = 0; - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - JT_row3(0) += nodal_positions(node_loop,0)*basis_derivative_s3(node_loop); - JT_row3(1) += nodal_positions(node_loop,1)*basis_derivative_s3(node_loop); - JT_row3(2) += nodal_positions(node_loop,2)*basis_derivative_s3(node_loop); - //debug print - /*if(myrank==1&&nodal_positions(node_loop,2)*basis_derivative_s3(node_loop)<-10000000){ - std::cout << " ELEMENT VOLUME JACOBIAN DEBUG ON TASK " << myrank << std::endl; - std::cout << node_loop+1 << " " << JT_row3(2) << " "<< nodal_positions(node_loop,2) <<" "<< basis_derivative_s3(node_loop) << std::endl; - std::fflush(stdout); - }*/ - } - - - //compute the determinant of the Jacobian - Jacobian = JT_row1(0)*(JT_row2(1)*JT_row3(2)-JT_row3(1)*JT_row2(2))- - JT_row1(1)*(JT_row2(0)*JT_row3(2)-JT_row3(0)*JT_row2(2))+ - JT_row1(2)*(JT_row2(0)*JT_row3(1)-JT_row3(0)*JT_row2(1)); - if(Jacobian<0) Jacobian = -Jacobian; - - //assign contribution to every local node this element has - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - if(map->isNodeGlobalElement(nodes_in_elem(ielem, node_loop))){ - local_node_id = map->getLocalElement(nodes_in_elem(ielem, node_loop)); - design_gradients(local_node_id,0)+=weight_multiply*basis_values(node_loop)*Jacobian; + else + { + convert_node_order(0) = 0; + convert_node_order(1) = 1; + convert_node_order(2) = 2; + convert_node_order(3) = 3; + if (num_dim == 3) + { + convert_node_order(4) = 4; + convert_node_order(5) = 5; + convert_node_order(6) = 6; + convert_node_order(7) = 7; } - } } - - } + // initialize design gradients to 0 + for (int init = 0; init < nlocal_nodes; init++) + { + design_gradients(init, 0) = 0; + } + + // loop over elements and use quadrature rule to compute volume from Jacobian determinant + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + // acquire set of nodes for this local element + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + local_node_id = all_node_map->getLocalElement(nodes_in_elem(ielem, convert_node_order(node_loop))); + nodal_positions(node_loop, 0) = all_node_coords(local_node_id, 0); + nodal_positions(node_loop, 1) = all_node_coords(local_node_id, 1); + nodal_positions(node_loop, 2) = all_node_coords(local_node_id, 2); + if (nodal_density_flag) + { + nodal_density(node_loop) = all_node_densities(local_node_id, 0); + } + /* + if(myrank==1&&nodal_positions(node_loop,2)>10000000){ + std::cout << " LOCAL MATRIX DEBUG ON TASK " << myrank << std::endl; + std::cout << node_loop+1 <<" " << local_node_id <<" "<< nodes_in_elem(ielem, node_loop) << " "<< nodal_positions(node_loop,2) << std::endl; + std::fflush(stdout); + } + */ + // std::cout << local_node_id << " " << nodes_in_elem(ielem, node_loop) << " " << nodal_positions(node_loop,0) << " " << nodal_positions(node_loop,1) << " "<< nodal_positions(node_loop,2) <basis(basis_values, quad_coordinate); + + // compute all the necessary coordinates and derivatives at this point + + // compute shape function derivatives + elem->partial_xi_basis(basis_derivative_s1, quad_coordinate); + elem->partial_eta_basis(basis_derivative_s2, quad_coordinate); + elem->partial_mu_basis(basis_derivative_s3, quad_coordinate); + + // compute derivatives of x,y,z w.r.t the s,t,w isoparametric space needed by JT (Transpose of the Jacobian) + // derivative of x,y,z w.r.t s + JT_row1(0) = 0; + JT_row1(1) = 0; + JT_row1(2) = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row1(0) += nodal_positions(node_loop, 0) * basis_derivative_s1(node_loop); + JT_row1(1) += nodal_positions(node_loop, 1) * basis_derivative_s1(node_loop); + JT_row1(2) += nodal_positions(node_loop, 2) * basis_derivative_s1(node_loop); + } + + // derivative of x,y,z w.r.t t + JT_row2(0) = 0; + JT_row2(1) = 0; + JT_row2(2) = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row2(0) += nodal_positions(node_loop, 0) * basis_derivative_s2(node_loop); + JT_row2(1) += nodal_positions(node_loop, 1) * basis_derivative_s2(node_loop); + JT_row2(2) += nodal_positions(node_loop, 2) * basis_derivative_s2(node_loop); + } + + // derivative of x,y,z w.r.t w + JT_row3(0) = 0; + JT_row3(1) = 0; + JT_row3(2) = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row3(0) += nodal_positions(node_loop, 0) * basis_derivative_s3(node_loop); + JT_row3(1) += nodal_positions(node_loop, 1) * basis_derivative_s3(node_loop); + JT_row3(2) += nodal_positions(node_loop, 2) * basis_derivative_s3(node_loop); + // debug print + /*if(myrank==1&&nodal_positions(node_loop,2)*basis_derivative_s3(node_loop)<-10000000){ + std::cout << " ELEMENT VOLUME JACOBIAN DEBUG ON TASK " << myrank << std::endl; + std::cout << node_loop+1 << " " << JT_row3(2) << " "<< nodal_positions(node_loop,2) <<" "<< basis_derivative_s3(node_loop) << std::endl; + std::fflush(stdout); + }*/ + } + + // compute the determinant of the Jacobian + Jacobian = JT_row1(0) * (JT_row2(1) * JT_row3(2) - JT_row3(1) * JT_row2(2)) - + JT_row1(1) * (JT_row2(0) * JT_row3(2) - JT_row3(0) * JT_row2(2)) + + JT_row1(2) * (JT_row2(0) * JT_row3(1) - JT_row3(0) * JT_row2(1)); + if (Jacobian < 0) + { + Jacobian = -Jacobian; + } + + // assign contribution to every local node this element has + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + if (map->isNodeGlobalElement(nodes_in_elem(ielem, node_loop))) + { + local_node_id = map->getLocalElement(nodes_in_elem(ielem, node_loop)); + design_gradients(local_node_id, 0) += weight_multiply * basis_values(node_loop) * Jacobian; + } + } + } + } } /* ------------------------------------------------------------------------------------------------------------------------ Compute the moment of each element for a specified component; estimated with quadrature --------------------------------------------------------------------------------------------------------------------------- */ -void FEA_Module_Inertial::compute_element_moments(const_host_vec_array design_densities, bool max_flag, int moment_component, bool use_initial_coords){ - //local number of uniquely assigned elements - size_t nonoverlap_nelements = element_map->getLocalNumElements(); - //initialize memory for volume storage - host_vec_array Element_Masses = Global_Element_Masses->getLocalView(Tpetra::Access::ReadWrite); - host_vec_array Element_Moments; - - if(moment_component==0) Element_Moments = Global_Element_Moments_x->getLocalView(Tpetra::Access::ReadWrite); - if(moment_component==1) Element_Moments = Global_Element_Moments_y->getLocalView(Tpetra::Access::ReadWrite); - if(moment_component==2) Element_Moments = Global_Element_Moments_z->getLocalView(Tpetra::Access::ReadWrite); - - const_host_vec_array Element_Volumes = Global_Element_Volumes->getLocalView(Tpetra::Access::ReadOnly); - //local variable for host view in the dual view - const_host_vec_array all_node_coords = all_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_host_vec_array all_initial_node_coords; - if(use_initial_coords) - all_initial_node_coords = all_initial_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_host_vec_array all_design_densities; - if(nodal_density_flag) - all_design_densities = all_node_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_host_elem_conn_array nodes_in_elem = global_nodes_in_elem_distributed->getLocalView (Tpetra::Access::ReadOnly); - int nodes_per_elem = elem->num_basis(); - int z_quad,y_quad,x_quad, direct_product_count; - size_t local_node_id; - LO ielem; - GO global_element_index; - - Solver::node_ordering_convention active_node_ordering_convention = Solver_Pointer_->active_node_ordering_convention; - CArrayKokkos convert_node_order(max_nodes_per_element); - if((active_node_ordering_convention == Solver::ENSIGHT && num_dim==3)||(active_node_ordering_convention == Solver::IJK && num_dim==2)){ - convert_node_order(0) = 0; - convert_node_order(1) = 1; - convert_node_order(2) = 3; - convert_node_order(3) = 2; - if(num_dim == 3){ - convert_node_order(4) = 4; - convert_node_order(5) = 5; - convert_node_order(6) = 7; - convert_node_order(7) = 6; +void FEA_Module_Inertial::compute_element_moments(const_host_vec_array design_densities, bool max_flag, int moment_component, bool use_initial_coords) +{ + // local number of uniquely assigned elements + size_t nonoverlap_nelements = element_map->getLocalNumElements(); + // initialize memory for volume storage + host_vec_array Element_Masses = Global_Element_Masses->getLocalView(Tpetra::Access::ReadWrite); + host_vec_array Element_Moments; + + if (moment_component == 0) + { + Element_Moments = Global_Element_Moments_x->getLocalView(Tpetra::Access::ReadWrite); } - } - else{ - convert_node_order(0) = 0; - convert_node_order(1) = 1; - convert_node_order(2) = 2; - convert_node_order(3) = 3; - if(num_dim==3){ - convert_node_order(4) = 4; - convert_node_order(5) = 5; - convert_node_order(6) = 6; - convert_node_order(7) = 7; + if (moment_component == 1) + { + Element_Moments = Global_Element_Moments_y->getLocalView(Tpetra::Access::ReadWrite); } - } - - real_t Jacobian, current_density, weight_multiply; - //CArrayKokkos legendre_nodes_1D(num_gauss_points); - //CArrayKokkos legendre_weights_1D(num_gauss_points); - CArray legendre_nodes_1D(num_gauss_points); - CArray legendre_weights_1D(num_gauss_points); - real_t pointer_quad_coordinate[num_dim]; - real_t pointer_quad_coordinate_weight[num_dim]; - real_t pointer_interpolated_point[num_dim]; - real_t pointer_JT_row1[num_dim]; - real_t pointer_JT_row2[num_dim]; - real_t pointer_JT_row3[num_dim]; - ViewCArray quad_coordinate(pointer_quad_coordinate,num_dim); - ViewCArray quad_coordinate_weight(pointer_quad_coordinate_weight,num_dim); - ViewCArray interpolated_point(pointer_interpolated_point,num_dim); - ViewCArray JT_row1(pointer_JT_row1,num_dim); - ViewCArray JT_row2(pointer_JT_row2,num_dim); - ViewCArray JT_row3(pointer_JT_row3,num_dim); - - real_t pointer_basis_values[elem->num_basis()]; - real_t pointer_basis_derivative_s1[elem->num_basis()]; - real_t pointer_basis_derivative_s2[elem->num_basis()]; - real_t pointer_basis_derivative_s3[elem->num_basis()]; - ViewCArray basis_values(pointer_basis_values,elem->num_basis()); - ViewCArray basis_derivative_s1(pointer_basis_derivative_s1,elem->num_basis()); - ViewCArray basis_derivative_s2(pointer_basis_derivative_s2,elem->num_basis()); - ViewCArray basis_derivative_s3(pointer_basis_derivative_s3,elem->num_basis()); - CArrayKokkos nodal_positions(elem->num_basis(),num_dim); - CArrayKokkos initial_nodal_positions(elem->num_basis(),num_dim); - CArrayKokkos nodal_density(elem->num_basis()); - CArrayKokkos current_position(num_dim); - - //initialize weights - elements::legendre_nodes_1D(legendre_nodes_1D,num_gauss_points); - elements::legendre_weights_1D(legendre_weights_1D,num_gauss_points); - - //loop over elements and use quadrature rule to compute volume from Jacobian determinant - for(int nonoverlapping_ielem = 0; nonoverlapping_ielem < nonoverlap_nelements; nonoverlapping_ielem++){ - global_element_index = element_map->getGlobalElement(nonoverlapping_ielem); - ielem = all_element_map->getLocalElement(global_element_index); - //acquire set of nodes for this local element - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - local_node_id = all_node_map->getLocalElement(nodes_in_elem(ielem, convert_node_order(node_loop))); - nodal_positions(node_loop,0) = all_node_coords(local_node_id,0); - nodal_positions(node_loop,1) = all_node_coords(local_node_id,1); - nodal_positions(node_loop,2) = all_node_coords(local_node_id,2); - if(use_initial_coords){ - initial_nodal_positions(node_loop,0) = all_initial_node_coords(local_node_id,0); - initial_nodal_positions(node_loop,1) = all_initial_node_coords(local_node_id,1); - initial_nodal_positions(node_loop,2) = all_initial_node_coords(local_node_id,2); - } - - - if(nodal_density_flag) nodal_density(node_loop) = all_design_densities(local_node_id,0); - /* - if(myrank==1&&nodal_positions(node_loop,2)>10000000){ - std::cout << " LOCAL MATRIX DEBUG ON TASK " << myrank << std::endl; - std::cout << node_loop+1 <<" " << local_node_id <<" "<< nodes_in_elem(ielem, node_loop) << " "<< nodal_positions(node_loop,2) << std::endl; - std::fflush(stdout); - } - */ - //std::cout << local_node_id << " " << nodes_in_elem(ielem, node_loop) << " " - //<< nodal_positions(node_loop,0) << " " << nodal_positions(node_loop,1) << " "<< nodal_positions(node_loop,2) << " " << nodal_density(node_loop) <getLocalView(Tpetra::Access::ReadWrite); } - - //debug print of index - //std::cout << "nonoverlap element id on TASK " << myrank << " is " << nonoverlapping_ielem << std::endl; - //std::fflush(stdout); - - //initialize element mass - Element_Moments(nonoverlapping_ielem,0) = 0; - - if(Element_Types(ielem)==elements::elem_types::Hex8){ - direct_product_count = std::pow(num_gauss_points,num_dim); + + const_host_vec_array Element_Volumes = Global_Element_Volumes->getLocalView(Tpetra::Access::ReadOnly); + // local variable for host view in the dual view + const_host_vec_array all_node_coords = all_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_host_vec_array all_initial_node_coords; + if (use_initial_coords) + { + all_initial_node_coords = all_initial_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + } + const_host_vec_array all_design_densities; + if (nodal_density_flag) + { + all_design_densities = all_node_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); } - - //loop over quadrature points - for(int iquad=0; iquad < direct_product_count; iquad++){ - - //set current quadrature point - if(num_dim==3) z_quad = iquad/(num_gauss_points*num_gauss_points); - y_quad = (iquad % (num_gauss_points*num_gauss_points))/num_gauss_points; - x_quad = iquad % num_gauss_points; - quad_coordinate(0) = legendre_nodes_1D(x_quad); - quad_coordinate(1) = legendre_nodes_1D(y_quad); - if(num_dim==3) - quad_coordinate(2) = legendre_nodes_1D(z_quad); - - //set current quadrature weight - quad_coordinate_weight(0) = legendre_weights_1D(x_quad); - quad_coordinate_weight(1) = legendre_weights_1D(y_quad); - if(num_dim==3) - quad_coordinate_weight(2) = legendre_weights_1D(z_quad); - else - quad_coordinate_weight(2) = 1; - weight_multiply = quad_coordinate_weight(0)*quad_coordinate_weight(1)*quad_coordinate_weight(2); - - //compute shape functions at this point for the element type - elem->basis(basis_values,quad_coordinate); - - //compute all the necessary coordinates and derivatives at this point - - //compute shape function derivatives - elem->partial_xi_basis(basis_derivative_s1,quad_coordinate); - elem->partial_eta_basis(basis_derivative_s2,quad_coordinate); - elem->partial_mu_basis(basis_derivative_s3,quad_coordinate); - - //compute derivatives of x,y,z w.r.t the s,t,w isoparametric space needed by JT (Transpose of the Jacobian) - //derivative of x,y,z w.r.t s - JT_row1(0) = 0; - JT_row1(1) = 0; - JT_row1(2) = 0; - if(use_initial_coords){ - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - JT_row1(0) += initial_nodal_positions(node_loop,0)*basis_derivative_s1(node_loop); - JT_row1(1) += initial_nodal_positions(node_loop,1)*basis_derivative_s1(node_loop); - JT_row1(2) += initial_nodal_positions(node_loop,2)*basis_derivative_s1(node_loop); + const_host_elem_conn_array nodes_in_elem = global_nodes_in_elem_distributed->getLocalView(Tpetra::Access::ReadOnly); + int nodes_per_elem = elem->num_basis(); + int z_quad, y_quad, x_quad, direct_product_count; + size_t local_node_id; + LO ielem; + GO global_element_index; + + Solver::node_ordering_convention active_node_ordering_convention = Solver_Pointer_->active_node_ordering_convention; + CArrayKokkos convert_node_order(max_nodes_per_element); + if ((active_node_ordering_convention == Solver::ENSIGHT && num_dim == 3) || (active_node_ordering_convention == Solver::IJK && num_dim == 2)) + { + convert_node_order(0) = 0; + convert_node_order(1) = 1; + convert_node_order(2) = 3; + convert_node_order(3) = 2; + if (num_dim == 3) + { + convert_node_order(4) = 4; + convert_node_order(5) = 5; + convert_node_order(6) = 7; + convert_node_order(7) = 6; } - - //derivative of x,y,z w.r.t t - JT_row2(0) = 0; - JT_row2(1) = 0; - JT_row2(2) = 0; - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - JT_row2(0) += initial_nodal_positions(node_loop,0)*basis_derivative_s2(node_loop); - JT_row2(1) += initial_nodal_positions(node_loop,1)*basis_derivative_s2(node_loop); - JT_row2(2) += initial_nodal_positions(node_loop,2)*basis_derivative_s2(node_loop); + } + else + { + convert_node_order(0) = 0; + convert_node_order(1) = 1; + convert_node_order(2) = 2; + convert_node_order(3) = 3; + if (num_dim == 3) + { + convert_node_order(4) = 4; + convert_node_order(5) = 5; + convert_node_order(6) = 6; + convert_node_order(7) = 7; } + } - //derivative of x,y,z w.r.t w - JT_row3(0) = 0; - JT_row3(1) = 0; - JT_row3(2) = 0; - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - JT_row3(0) += initial_nodal_positions(node_loop,0)*basis_derivative_s3(node_loop); - JT_row3(1) += initial_nodal_positions(node_loop,1)*basis_derivative_s3(node_loop); - JT_row3(2) += initial_nodal_positions(node_loop,2)*basis_derivative_s3(node_loop); - //debug print - /*if(myrank==1&&nodal_positions(node_loop,2)*basis_derivative_s3(node_loop)<-10000000){ - std::cout << " ELEMENT VOLUME JACOBIAN DEBUG ON TASK " << myrank << std::endl; - std::cout << node_loop+1 << " " << JT_row3(2) << " "<< nodal_positions(node_loop,2) <<" "<< basis_derivative_s3(node_loop) << std::endl; - std::fflush(stdout); - }*/ - } + real_t Jacobian, current_density, weight_multiply; + // CArrayKokkos legendre_nodes_1D(num_gauss_points); + // CArrayKokkos legendre_weights_1D(num_gauss_points); + CArray legendre_nodes_1D(num_gauss_points); + CArray legendre_weights_1D(num_gauss_points); + real_t pointer_quad_coordinate[num_dim]; + real_t pointer_quad_coordinate_weight[num_dim]; + real_t pointer_interpolated_point[num_dim]; + real_t pointer_JT_row1[num_dim]; + real_t pointer_JT_row2[num_dim]; + real_t pointer_JT_row3[num_dim]; + ViewCArray quad_coordinate(pointer_quad_coordinate, num_dim); + ViewCArray quad_coordinate_weight(pointer_quad_coordinate_weight, num_dim); + ViewCArray interpolated_point(pointer_interpolated_point, num_dim); + ViewCArray JT_row1(pointer_JT_row1, num_dim); + ViewCArray JT_row2(pointer_JT_row2, num_dim); + ViewCArray JT_row3(pointer_JT_row3, num_dim); + + real_t pointer_basis_values[elem->num_basis()]; + real_t pointer_basis_derivative_s1[elem->num_basis()]; + real_t pointer_basis_derivative_s2[elem->num_basis()]; + real_t pointer_basis_derivative_s3[elem->num_basis()]; + ViewCArray basis_values(pointer_basis_values, elem->num_basis()); + ViewCArray basis_derivative_s1(pointer_basis_derivative_s1, elem->num_basis()); + ViewCArray basis_derivative_s2(pointer_basis_derivative_s2, elem->num_basis()); + ViewCArray basis_derivative_s3(pointer_basis_derivative_s3, elem->num_basis()); + CArrayKokkos nodal_positions(elem->num_basis(), num_dim); + CArrayKokkos initial_nodal_positions(elem->num_basis(), num_dim); + CArrayKokkos nodal_density(elem->num_basis()); + CArrayKokkos current_position(num_dim); + + // initialize weights + elements::legendre_nodes_1D(legendre_nodes_1D, num_gauss_points); + elements::legendre_weights_1D(legendre_weights_1D, num_gauss_points); + + // loop over elements and use quadrature rule to compute volume from Jacobian determinant + for (int nonoverlapping_ielem = 0; nonoverlapping_ielem < nonoverlap_nelements; nonoverlapping_ielem++) + { + global_element_index = element_map->getGlobalElement(nonoverlapping_ielem); + ielem = all_element_map->getLocalElement(global_element_index); + // acquire set of nodes for this local element + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + local_node_id = all_node_map->getLocalElement(nodes_in_elem(ielem, convert_node_order(node_loop))); + nodal_positions(node_loop, 0) = all_node_coords(local_node_id, 0); + nodal_positions(node_loop, 1) = all_node_coords(local_node_id, 1); + nodal_positions(node_loop, 2) = all_node_coords(local_node_id, 2); + if (use_initial_coords) + { + initial_nodal_positions(node_loop, 0) = all_initial_node_coords(local_node_id, 0); + initial_nodal_positions(node_loop, 1) = all_initial_node_coords(local_node_id, 1); + initial_nodal_positions(node_loop, 2) = all_initial_node_coords(local_node_id, 2); + } - } - else{ - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - JT_row1(0) += nodal_positions(node_loop,0)*basis_derivative_s1(node_loop); - JT_row1(1) += nodal_positions(node_loop,1)*basis_derivative_s1(node_loop); - JT_row1(2) += nodal_positions(node_loop,2)*basis_derivative_s1(node_loop); + if (nodal_density_flag) + { + nodal_density(node_loop) = all_design_densities(local_node_id, 0); + } + /* + if(myrank==1&&nodal_positions(node_loop,2)>10000000){ + std::cout << " LOCAL MATRIX DEBUG ON TASK " << myrank << std::endl; + std::cout << node_loop+1 <<" " << local_node_id <<" "<< nodes_in_elem(ielem, node_loop) << " "<< nodal_positions(node_loop,2) << std::endl; + std::fflush(stdout); + } + */ + // std::cout << local_node_id << " " << nodes_in_elem(ielem, node_loop) << " " + // << nodal_positions(node_loop,0) << " " << nodal_positions(node_loop,1) << " "<< nodal_positions(node_loop,2) << " " << nodal_density(node_loop) <num_basis(); node_loop++){ - JT_row2(0) += nodal_positions(node_loop,0)*basis_derivative_s2(node_loop); - JT_row2(1) += nodal_positions(node_loop,1)*basis_derivative_s2(node_loop); - JT_row2(2) += nodal_positions(node_loop,2)*basis_derivative_s2(node_loop); - } + // debug print of index + // std::cout << "nonoverlap element id on TASK " << myrank << " is " << nonoverlapping_ielem << std::endl; + // std::fflush(stdout); - //derivative of x,y,z w.r.t w - JT_row3(0) = 0; - JT_row3(1) = 0; - JT_row3(2) = 0; - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - JT_row3(0) += nodal_positions(node_loop,0)*basis_derivative_s3(node_loop); - JT_row3(1) += nodal_positions(node_loop,1)*basis_derivative_s3(node_loop); - JT_row3(2) += nodal_positions(node_loop,2)*basis_derivative_s3(node_loop); - //debug print - /*if(myrank==1&&nodal_positions(node_loop,2)*basis_derivative_s3(node_loop)<-10000000){ - std::cout << " ELEMENT VOLUME JACOBIAN DEBUG ON TASK " << myrank << std::endl; - std::cout << node_loop+1 << " " << JT_row3(2) << " "<< nodal_positions(node_loop,2) <<" "<< basis_derivative_s3(node_loop) << std::endl; - std::fflush(stdout); - }*/ - } - } - - - //compute the determinant of the Jacobian - Jacobian = JT_row1(0)*(JT_row2(1)*JT_row3(2)-JT_row3(1)*JT_row2(2))- - JT_row1(1)*(JT_row2(0)*JT_row3(2)-JT_row3(0)*JT_row2(2))+ - JT_row1(2)*(JT_row2(0)*JT_row3(1)-JT_row3(0)*JT_row2(1)); - if(Jacobian<0) Jacobian = -Jacobian; - - //compute density - if(max_flag){ - current_density = 1; - } - else{ - if(nodal_density_flag){ - current_density = 0; - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - current_density += nodal_density(node_loop)*basis_values(node_loop); - } - }// if - else{ - current_density = design_densities(nonoverlapping_ielem,0); + // initialize element mass + Element_Moments(nonoverlapping_ielem, 0) = 0; + + if (Element_Types(ielem) == elements::elem_types::Hex8) + { + direct_product_count = std::pow(num_gauss_points, num_dim); } - } - //compute current position - current_position(0) = current_position(1) = current_position(2) = 0; - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - current_position(moment_component) += nodal_positions(node_loop,moment_component)*basis_values(node_loop); - } + // loop over quadrature points + for (int iquad = 0; iquad < direct_product_count; iquad++) + { + // set current quadrature point + if (num_dim == 3) + { + z_quad = iquad / (num_gauss_points * num_gauss_points); + } + y_quad = (iquad % (num_gauss_points * num_gauss_points)) / num_gauss_points; + x_quad = iquad % num_gauss_points; + quad_coordinate(0) = legendre_nodes_1D(x_quad); + quad_coordinate(1) = legendre_nodes_1D(y_quad); + if (num_dim == 3) + { + quad_coordinate(2) = legendre_nodes_1D(z_quad); + } - Element_Moments(nonoverlapping_ielem,0) += current_density*current_position(moment_component)*weight_multiply*Jacobian; + // set current quadrature weight + quad_coordinate_weight(0) = legendre_weights_1D(x_quad); + quad_coordinate_weight(1) = legendre_weights_1D(y_quad); + if (num_dim == 3) + { + quad_coordinate_weight(2) = legendre_weights_1D(z_quad); + } + else + { + quad_coordinate_weight(2) = 1; + } + weight_multiply = quad_coordinate_weight(0) * quad_coordinate_weight(1) * quad_coordinate_weight(2); + + // compute shape functions at this point for the element type + elem->basis(basis_values, quad_coordinate); + + // compute all the necessary coordinates and derivatives at this point + + // compute shape function derivatives + elem->partial_xi_basis(basis_derivative_s1, quad_coordinate); + elem->partial_eta_basis(basis_derivative_s2, quad_coordinate); + elem->partial_mu_basis(basis_derivative_s3, quad_coordinate); + + // compute derivatives of x,y,z w.r.t the s,t,w isoparametric space needed by JT (Transpose of the Jacobian) + // derivative of x,y,z w.r.t s + JT_row1(0) = 0; + JT_row1(1) = 0; + JT_row1(2) = 0; + if (use_initial_coords) + { + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row1(0) += initial_nodal_positions(node_loop, 0) * basis_derivative_s1(node_loop); + JT_row1(1) += initial_nodal_positions(node_loop, 1) * basis_derivative_s1(node_loop); + JT_row1(2) += initial_nodal_positions(node_loop, 2) * basis_derivative_s1(node_loop); + } + + // derivative of x,y,z w.r.t t + JT_row2(0) = 0; + JT_row2(1) = 0; + JT_row2(2) = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row2(0) += initial_nodal_positions(node_loop, 0) * basis_derivative_s2(node_loop); + JT_row2(1) += initial_nodal_positions(node_loop, 1) * basis_derivative_s2(node_loop); + JT_row2(2) += initial_nodal_positions(node_loop, 2) * basis_derivative_s2(node_loop); + } + + // derivative of x,y,z w.r.t w + JT_row3(0) = 0; + JT_row3(1) = 0; + JT_row3(2) = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row3(0) += initial_nodal_positions(node_loop, 0) * basis_derivative_s3(node_loop); + JT_row3(1) += initial_nodal_positions(node_loop, 1) * basis_derivative_s3(node_loop); + JT_row3(2) += initial_nodal_positions(node_loop, 2) * basis_derivative_s3(node_loop); + // debug print + /*if(myrank==1&&nodal_positions(node_loop,2)*basis_derivative_s3(node_loop)<-10000000){ + std::cout << " ELEMENT VOLUME JACOBIAN DEBUG ON TASK " << myrank << std::endl; + std::cout << node_loop+1 << " " << JT_row3(2) << " "<< nodal_positions(node_loop,2) <<" "<< basis_derivative_s3(node_loop) << std::endl; + std::fflush(stdout); + }*/ + } + } + else + { + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row1(0) += nodal_positions(node_loop, 0) * basis_derivative_s1(node_loop); + JT_row1(1) += nodal_positions(node_loop, 1) * basis_derivative_s1(node_loop); + JT_row1(2) += nodal_positions(node_loop, 2) * basis_derivative_s1(node_loop); + } + + // derivative of x,y,z w.r.t t + JT_row2(0) = 0; + JT_row2(1) = 0; + JT_row2(2) = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row2(0) += nodal_positions(node_loop, 0) * basis_derivative_s2(node_loop); + JT_row2(1) += nodal_positions(node_loop, 1) * basis_derivative_s2(node_loop); + JT_row2(2) += nodal_positions(node_loop, 2) * basis_derivative_s2(node_loop); + } + + // derivative of x,y,z w.r.t w + JT_row3(0) = 0; + JT_row3(1) = 0; + JT_row3(2) = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row3(0) += nodal_positions(node_loop, 0) * basis_derivative_s3(node_loop); + JT_row3(1) += nodal_positions(node_loop, 1) * basis_derivative_s3(node_loop); + JT_row3(2) += nodal_positions(node_loop, 2) * basis_derivative_s3(node_loop); + // debug print + /*if(myrank==1&&nodal_positions(node_loop,2)*basis_derivative_s3(node_loop)<-10000000){ + std::cout << " ELEMENT VOLUME JACOBIAN DEBUG ON TASK " << myrank << std::endl; + std::cout << node_loop+1 << " " << JT_row3(2) << " "<< nodal_positions(node_loop,2) <<" "<< basis_derivative_s3(node_loop) << std::endl; + std::fflush(stdout); + }*/ + } + } + + // compute the determinant of the Jacobian + Jacobian = JT_row1(0) * (JT_row2(1) * JT_row3(2) - JT_row3(1) * JT_row2(2)) - + JT_row1(1) * (JT_row2(0) * JT_row3(2) - JT_row3(0) * JT_row2(2)) + + JT_row1(2) * (JT_row2(0) * JT_row3(1) - JT_row3(0) * JT_row2(1)); + if (Jacobian < 0) + { + Jacobian = -Jacobian; + } + + // compute density + if (max_flag) + { + current_density = 1; + } + else + { + if (nodal_density_flag) + { + current_density = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + current_density += nodal_density(node_loop) * basis_values(node_loop); + } + } // if + else + { + current_density = design_densities(nonoverlapping_ielem, 0); + } + } + + // compute current position + current_position(0) = current_position(1) = current_position(2) = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + current_position(moment_component) += nodal_positions(node_loop, moment_component) * basis_values(node_loop); + } + + Element_Moments(nonoverlapping_ielem, 0) += current_density * current_position(moment_component) * weight_multiply * Jacobian; + } } - } - - //std::ostream &out = std::cout; - //Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - //if(myrank==0) - //*fos << "Global Element Masses:" << std::endl; - //Global_Element_Masses->describe(*fos,Teuchos::VERB_EXTREME); - //*fos << std::endl; + + // std::ostream &out = std::cout; + // Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + // if(myrank==0) + // *fos << "Global Element Masses:" << std::endl; + // Global_Element_Masses->describe(*fos,Teuchos::VERB_EXTREME); + // *fos << std::endl; } /* --------------------------------------------------------------------------------------------------- Compute the gradients of the specified moment component with respect to design densities ------------------------------------------------------------------------------------------------------ */ -void FEA_Module_Inertial::compute_moment_gradients(const_host_vec_array design_variables, host_vec_array design_gradients, int moment_component, bool use_initial_coords){ - //local number of uniquely assigned elements - size_t nonoverlap_nelements = element_map->getLocalNumElements(); - //local variable for host view in the dual view - const_host_vec_array all_node_coords = all_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_host_vec_array all_initial_node_coords; - if(use_initial_coords) - all_initial_node_coords = all_initial_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_host_elem_conn_array nodes_in_elem = global_nodes_in_elem_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_host_vec_array all_node_densities; - if(nodal_density_flag) - all_node_densities = all_node_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); - int nodes_per_elem = elem->num_basis(); - int z_quad,y_quad,x_quad, direct_product_count; - size_t local_node_id; - LO ielem; - GO global_element_index; - - Solver::node_ordering_convention active_node_ordering_convention = Solver_Pointer_->active_node_ordering_convention; - CArrayKokkos convert_node_order(max_nodes_per_element); - if((active_node_ordering_convention == Solver::ENSIGHT && num_dim==3)||(active_node_ordering_convention == Solver::IJK && num_dim==2)){ - convert_node_order(0) = 0; - convert_node_order(1) = 1; - convert_node_order(2) = 3; - convert_node_order(3) = 2; - if(num_dim == 3){ - convert_node_order(4) = 4; - convert_node_order(5) = 5; - convert_node_order(6) = 7; - convert_node_order(7) = 6; +void FEA_Module_Inertial::compute_moment_gradients(const_host_vec_array design_variables, host_vec_array design_gradients, int moment_component, bool use_initial_coords) +{ + // local number of uniquely assigned elements + size_t nonoverlap_nelements = element_map->getLocalNumElements(); + // local variable for host view in the dual view + const_host_vec_array all_node_coords = all_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_host_vec_array all_initial_node_coords; + if (use_initial_coords) + { + all_initial_node_coords = all_initial_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); } - } - else{ - convert_node_order(0) = 0; - convert_node_order(1) = 1; - convert_node_order(2) = 2; - convert_node_order(3) = 3; - if(num_dim==3){ - convert_node_order(4) = 4; - convert_node_order(5) = 5; - convert_node_order(6) = 6; - convert_node_order(7) = 7; + const_host_elem_conn_array nodes_in_elem = global_nodes_in_elem_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_host_vec_array all_node_densities; + if (nodal_density_flag) + { + all_node_densities = all_node_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); } - } - - - real_t Jacobian, weight_multiply; - //CArrayKokkos legendre_nodes_1D(num_gauss_points); - //CArrayKokkos legendre_weights_1D(num_gauss_points); - CArray legendre_nodes_1D(num_gauss_points); - CArray legendre_weights_1D(num_gauss_points); - real_t pointer_quad_coordinate[num_dim]; - real_t pointer_quad_coordinate_weight[num_dim]; - real_t pointer_interpolated_point[num_dim]; - real_t pointer_JT_row1[num_dim]; - real_t pointer_JT_row2[num_dim]; - real_t pointer_JT_row3[num_dim]; - ViewCArray quad_coordinate(pointer_quad_coordinate,num_dim); - ViewCArray quad_coordinate_weight(pointer_quad_coordinate_weight,num_dim); - ViewCArray interpolated_point(pointer_interpolated_point,num_dim); - ViewCArray JT_row1(pointer_JT_row1,num_dim); - ViewCArray JT_row2(pointer_JT_row2,num_dim); - ViewCArray JT_row3(pointer_JT_row3,num_dim); - - real_t pointer_basis_values[elem->num_basis()]; - real_t pointer_basis_derivative_s1[elem->num_basis()]; - real_t pointer_basis_derivative_s2[elem->num_basis()]; - real_t pointer_basis_derivative_s3[elem->num_basis()]; - ViewCArray basis_values(pointer_basis_values,elem->num_basis()); - ViewCArray basis_derivative_s1(pointer_basis_derivative_s1,elem->num_basis()); - ViewCArray basis_derivative_s2(pointer_basis_derivative_s2,elem->num_basis()); - ViewCArray basis_derivative_s3(pointer_basis_derivative_s3,elem->num_basis()); - CArrayKokkos nodal_positions(elem->num_basis(),num_dim); - CArrayKokkos initial_nodal_positions(elem->num_basis(),num_dim); - CArrayKokkos nodal_density(elem->num_basis()); - CArrayKokkos current_position(num_dim); - - //initialize weights - elements::legendre_nodes_1D(legendre_nodes_1D,num_gauss_points); - elements::legendre_weights_1D(legendre_weights_1D,num_gauss_points); - - //initialize design gradients to 0 - for(int init = 0; init < nlocal_nodes; init++) - design_gradients(init,0) = 0; - - //loop over elements and use quadrature rule to compute volume from Jacobian determinant - for(int ielem = 0; ielem < rnum_elem; ielem++){ - //acquire set of nodes for this local element - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - local_node_id = all_node_map->getLocalElement(nodes_in_elem(ielem, convert_node_order(node_loop))); - nodal_positions(node_loop,0) = all_node_coords(local_node_id,0); - nodal_positions(node_loop,1) = all_node_coords(local_node_id,1); - nodal_positions(node_loop,2) = all_node_coords(local_node_id,2); - if(use_initial_coords){ - initial_nodal_positions(node_loop,0) = all_initial_node_coords(local_node_id,0); - initial_nodal_positions(node_loop,1) = all_initial_node_coords(local_node_id,1); - initial_nodal_positions(node_loop,2) = all_initial_node_coords(local_node_id,2); - } - if(nodal_density_flag) nodal_density(node_loop) = all_node_densities(local_node_id,0); - /* - if(myrank==1&&nodal_positions(node_loop,2)>10000000){ - std::cout << " LOCAL MATRIX DEBUG ON TASK " << myrank << std::endl; - std::cout << node_loop+1 <<" " << local_node_id <<" "<< nodes_in_elem(ielem, node_loop) << " "<< nodal_positions(node_loop,2) << std::endl; - std::fflush(stdout); - } - */ - //std::cout << local_node_id << " " << nodes_in_elem(ielem, node_loop) << " " << nodal_positions(node_loop,0) << " " << nodal_positions(node_loop,1) << " "<< nodal_positions(node_loop,2) <num_basis(); + int z_quad, y_quad, x_quad, direct_product_count; + size_t local_node_id; + LO ielem; + GO global_element_index; + + Solver::node_ordering_convention active_node_ordering_convention = Solver_Pointer_->active_node_ordering_convention; + CArrayKokkos convert_node_order(max_nodes_per_element); + if ((active_node_ordering_convention == Solver::ENSIGHT && num_dim == 3) || (active_node_ordering_convention == Solver::IJK && num_dim == 2)) + { + convert_node_order(0) = 0; + convert_node_order(1) = 1; + convert_node_order(2) = 3; + convert_node_order(3) = 2; + if (num_dim == 3) + { + convert_node_order(4) = 4; + convert_node_order(5) = 5; + convert_node_order(6) = 7; + convert_node_order(7) = 6; + } } - - //loop over quadrature points - for(int iquad=0; iquad < direct_product_count; iquad++){ - - //set current quadrature point - if(num_dim==3) z_quad = iquad/(num_gauss_points*num_gauss_points); - y_quad = (iquad % (num_gauss_points*num_gauss_points))/num_gauss_points; - x_quad = iquad % num_gauss_points; - quad_coordinate(0) = legendre_nodes_1D(x_quad); - quad_coordinate(1) = legendre_nodes_1D(y_quad); - if(num_dim==3) - quad_coordinate(2) = legendre_nodes_1D(z_quad); - - //set current quadrature weight - quad_coordinate_weight(0) = legendre_weights_1D(x_quad); - quad_coordinate_weight(1) = legendre_weights_1D(y_quad); - if(num_dim==3) - quad_coordinate_weight(2) = legendre_weights_1D(z_quad); - else - quad_coordinate_weight(2) = 1; - weight_multiply = quad_coordinate_weight(0)*quad_coordinate_weight(1)*quad_coordinate_weight(2); - - //compute shape functions at this point for the element type - elem->basis(basis_values,quad_coordinate); - - //compute all the necessary coordinates and derivatives at this point - - //compute shape function derivatives - elem->partial_xi_basis(basis_derivative_s1,quad_coordinate); - elem->partial_eta_basis(basis_derivative_s2,quad_coordinate); - elem->partial_mu_basis(basis_derivative_s3,quad_coordinate); - - //compute derivatives of x,y,z w.r.t the s,t,w isoparametric space needed by JT (Transpose of the Jacobian) - //derivative of x,y,z w.r.t s - JT_row1(0) = 0; - JT_row1(1) = 0; - JT_row1(2) = 0; - if(use_initial_coords){ - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - JT_row1(0) += initial_nodal_positions(node_loop,0)*basis_derivative_s1(node_loop); - JT_row1(1) += initial_nodal_positions(node_loop,1)*basis_derivative_s1(node_loop); - JT_row1(2) += initial_nodal_positions(node_loop,2)*basis_derivative_s1(node_loop); + else + { + convert_node_order(0) = 0; + convert_node_order(1) = 1; + convert_node_order(2) = 2; + convert_node_order(3) = 3; + if (num_dim == 3) + { + convert_node_order(4) = 4; + convert_node_order(5) = 5; + convert_node_order(6) = 6; + convert_node_order(7) = 7; } + } - //derivative of x,y,z w.r.t t - JT_row2(0) = 0; - JT_row2(1) = 0; - JT_row2(2) = 0; - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - JT_row2(0) += initial_nodal_positions(node_loop,0)*basis_derivative_s2(node_loop); - JT_row2(1) += initial_nodal_positions(node_loop,1)*basis_derivative_s2(node_loop); - JT_row2(2) += initial_nodal_positions(node_loop,2)*basis_derivative_s2(node_loop); - } + real_t Jacobian, weight_multiply; + // CArrayKokkos legendre_nodes_1D(num_gauss_points); + // CArrayKokkos legendre_weights_1D(num_gauss_points); + CArray legendre_nodes_1D(num_gauss_points); + CArray legendre_weights_1D(num_gauss_points); + real_t pointer_quad_coordinate[num_dim]; + real_t pointer_quad_coordinate_weight[num_dim]; + real_t pointer_interpolated_point[num_dim]; + real_t pointer_JT_row1[num_dim]; + real_t pointer_JT_row2[num_dim]; + real_t pointer_JT_row3[num_dim]; + ViewCArray quad_coordinate(pointer_quad_coordinate, num_dim); + ViewCArray quad_coordinate_weight(pointer_quad_coordinate_weight, num_dim); + ViewCArray interpolated_point(pointer_interpolated_point, num_dim); + ViewCArray JT_row1(pointer_JT_row1, num_dim); + ViewCArray JT_row2(pointer_JT_row2, num_dim); + ViewCArray JT_row3(pointer_JT_row3, num_dim); + + real_t pointer_basis_values[elem->num_basis()]; + real_t pointer_basis_derivative_s1[elem->num_basis()]; + real_t pointer_basis_derivative_s2[elem->num_basis()]; + real_t pointer_basis_derivative_s3[elem->num_basis()]; + ViewCArray basis_values(pointer_basis_values, elem->num_basis()); + ViewCArray basis_derivative_s1(pointer_basis_derivative_s1, elem->num_basis()); + ViewCArray basis_derivative_s2(pointer_basis_derivative_s2, elem->num_basis()); + ViewCArray basis_derivative_s3(pointer_basis_derivative_s3, elem->num_basis()); + CArrayKokkos nodal_positions(elem->num_basis(), num_dim); + CArrayKokkos initial_nodal_positions(elem->num_basis(), num_dim); + CArrayKokkos nodal_density(elem->num_basis()); + CArrayKokkos current_position(num_dim); + + // initialize weights + elements::legendre_nodes_1D(legendre_nodes_1D, num_gauss_points); + elements::legendre_weights_1D(legendre_weights_1D, num_gauss_points); + + // initialize design gradients to 0 + for (int init = 0; init < nlocal_nodes; init++) + { + design_gradients(init, 0) = 0; + } - //derivative of x,y,z w.r.t w - JT_row3(0) = 0; - JT_row3(1) = 0; - JT_row3(2) = 0; - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - JT_row3(0) += initial_nodal_positions(node_loop,0)*basis_derivative_s3(node_loop); - JT_row3(1) += initial_nodal_positions(node_loop,1)*basis_derivative_s3(node_loop); - JT_row3(2) += initial_nodal_positions(node_loop,2)*basis_derivative_s3(node_loop); + // loop over elements and use quadrature rule to compute volume from Jacobian determinant + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + // acquire set of nodes for this local element + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + local_node_id = all_node_map->getLocalElement(nodes_in_elem(ielem, convert_node_order(node_loop))); + nodal_positions(node_loop, 0) = all_node_coords(local_node_id, 0); + nodal_positions(node_loop, 1) = all_node_coords(local_node_id, 1); + nodal_positions(node_loop, 2) = all_node_coords(local_node_id, 2); + if (use_initial_coords) + { + initial_nodal_positions(node_loop, 0) = all_initial_node_coords(local_node_id, 0); + initial_nodal_positions(node_loop, 1) = all_initial_node_coords(local_node_id, 1); + initial_nodal_positions(node_loop, 2) = all_initial_node_coords(local_node_id, 2); + } + if (nodal_density_flag) + { + nodal_density(node_loop) = all_node_densities(local_node_id, 0); + } + /* + if(myrank==1&&nodal_positions(node_loop,2)>10000000){ + std::cout << " LOCAL MATRIX DEBUG ON TASK " << myrank << std::endl; + std::cout << node_loop+1 <<" " << local_node_id <<" "<< nodes_in_elem(ielem, node_loop) << " "<< nodal_positions(node_loop,2) << std::endl; + std::fflush(stdout); + } + */ + // std::cout << local_node_id << " " << nodes_in_elem(ielem, node_loop) << " " << nodal_positions(node_loop,0) << " " << nodal_positions(node_loop,1) << " "<< nodal_positions(node_loop,2) <num_basis(); node_loop++){ - JT_row1(0) += nodal_positions(node_loop,0)*basis_derivative_s1(node_loop); - JT_row1(1) += nodal_positions(node_loop,1)*basis_derivative_s1(node_loop); - JT_row1(2) += nodal_positions(node_loop,2)*basis_derivative_s1(node_loop); + if (Element_Types(ielem) == elements::elem_types::Hex8) + { + direct_product_count = std::pow(num_gauss_points, num_dim); } - //derivative of x,y,z w.r.t t - JT_row2(0) = 0; - JT_row2(1) = 0; - JT_row2(2) = 0; - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - JT_row2(0) += nodal_positions(node_loop,0)*basis_derivative_s2(node_loop); - JT_row2(1) += nodal_positions(node_loop,1)*basis_derivative_s2(node_loop); - JT_row2(2) += nodal_positions(node_loop,2)*basis_derivative_s2(node_loop); - } + // loop over quadrature points + for (int iquad = 0; iquad < direct_product_count; iquad++) + { + // set current quadrature point + if (num_dim == 3) + { + z_quad = iquad / (num_gauss_points * num_gauss_points); + } + y_quad = (iquad % (num_gauss_points * num_gauss_points)) / num_gauss_points; + x_quad = iquad % num_gauss_points; + quad_coordinate(0) = legendre_nodes_1D(x_quad); + quad_coordinate(1) = legendre_nodes_1D(y_quad); + if (num_dim == 3) + { + quad_coordinate(2) = legendre_nodes_1D(z_quad); + } - //derivative of x,y,z w.r.t w - JT_row3(0) = 0; - JT_row3(1) = 0; - JT_row3(2) = 0; - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - JT_row3(0) += nodal_positions(node_loop,0)*basis_derivative_s3(node_loop); - JT_row3(1) += nodal_positions(node_loop,1)*basis_derivative_s3(node_loop); - JT_row3(2) += nodal_positions(node_loop,2)*basis_derivative_s3(node_loop); - } - } - - - //compute the determinant of the Jacobian - Jacobian = JT_row1(0)*(JT_row2(1)*JT_row3(2)-JT_row3(1)*JT_row2(2))- - JT_row1(1)*(JT_row2(0)*JT_row3(2)-JT_row3(0)*JT_row2(2))+ - JT_row1(2)*(JT_row2(0)*JT_row3(1)-JT_row3(0)*JT_row2(1)); - if(Jacobian<0) Jacobian = -Jacobian; - - //compute current position - current_position(0) = current_position(1) = current_position(2) = 0; - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - current_position(moment_component) += nodal_positions(node_loop,moment_component)*basis_values(node_loop); - } - - //assign contribution to every local node this element has - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - if(map->isNodeGlobalElement(nodes_in_elem(ielem, node_loop))){ - local_node_id = map->getLocalElement(nodes_in_elem(ielem, node_loop)); - design_gradients(local_node_id,0)+=weight_multiply*basis_values(node_loop)*current_position(moment_component)*Jacobian; + // set current quadrature weight + quad_coordinate_weight(0) = legendre_weights_1D(x_quad); + quad_coordinate_weight(1) = legendre_weights_1D(y_quad); + if (num_dim == 3) + { + quad_coordinate_weight(2) = legendre_weights_1D(z_quad); + } + else + { + quad_coordinate_weight(2) = 1; + } + weight_multiply = quad_coordinate_weight(0) * quad_coordinate_weight(1) * quad_coordinate_weight(2); + + // compute shape functions at this point for the element type + elem->basis(basis_values, quad_coordinate); + + // compute all the necessary coordinates and derivatives at this point + + // compute shape function derivatives + elem->partial_xi_basis(basis_derivative_s1, quad_coordinate); + elem->partial_eta_basis(basis_derivative_s2, quad_coordinate); + elem->partial_mu_basis(basis_derivative_s3, quad_coordinate); + + // compute derivatives of x,y,z w.r.t the s,t,w isoparametric space needed by JT (Transpose of the Jacobian) + // derivative of x,y,z w.r.t s + JT_row1(0) = 0; + JT_row1(1) = 0; + JT_row1(2) = 0; + if (use_initial_coords) + { + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row1(0) += initial_nodal_positions(node_loop, 0) * basis_derivative_s1(node_loop); + JT_row1(1) += initial_nodal_positions(node_loop, 1) * basis_derivative_s1(node_loop); + JT_row1(2) += initial_nodal_positions(node_loop, 2) * basis_derivative_s1(node_loop); + } + + // derivative of x,y,z w.r.t t + JT_row2(0) = 0; + JT_row2(1) = 0; + JT_row2(2) = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row2(0) += initial_nodal_positions(node_loop, 0) * basis_derivative_s2(node_loop); + JT_row2(1) += initial_nodal_positions(node_loop, 1) * basis_derivative_s2(node_loop); + JT_row2(2) += initial_nodal_positions(node_loop, 2) * basis_derivative_s2(node_loop); + } + + // derivative of x,y,z w.r.t w + JT_row3(0) = 0; + JT_row3(1) = 0; + JT_row3(2) = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row3(0) += initial_nodal_positions(node_loop, 0) * basis_derivative_s3(node_loop); + JT_row3(1) += initial_nodal_positions(node_loop, 1) * basis_derivative_s3(node_loop); + JT_row3(2) += initial_nodal_positions(node_loop, 2) * basis_derivative_s3(node_loop); + } + } + else + { + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row1(0) += nodal_positions(node_loop, 0) * basis_derivative_s1(node_loop); + JT_row1(1) += nodal_positions(node_loop, 1) * basis_derivative_s1(node_loop); + JT_row1(2) += nodal_positions(node_loop, 2) * basis_derivative_s1(node_loop); + } + + // derivative of x,y,z w.r.t t + JT_row2(0) = 0; + JT_row2(1) = 0; + JT_row2(2) = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row2(0) += nodal_positions(node_loop, 0) * basis_derivative_s2(node_loop); + JT_row2(1) += nodal_positions(node_loop, 1) * basis_derivative_s2(node_loop); + JT_row2(2) += nodal_positions(node_loop, 2) * basis_derivative_s2(node_loop); + } + + // derivative of x,y,z w.r.t w + JT_row3(0) = 0; + JT_row3(1) = 0; + JT_row3(2) = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row3(0) += nodal_positions(node_loop, 0) * basis_derivative_s3(node_loop); + JT_row3(1) += nodal_positions(node_loop, 1) * basis_derivative_s3(node_loop); + JT_row3(2) += nodal_positions(node_loop, 2) * basis_derivative_s3(node_loop); + } + } + + // compute the determinant of the Jacobian + Jacobian = JT_row1(0) * (JT_row2(1) * JT_row3(2) - JT_row3(1) * JT_row2(2)) - + JT_row1(1) * (JT_row2(0) * JT_row3(2) - JT_row3(0) * JT_row2(2)) + + JT_row1(2) * (JT_row2(0) * JT_row3(1) - JT_row3(0) * JT_row2(1)); + if (Jacobian < 0) + { + Jacobian = -Jacobian; + } + + // compute current position + current_position(0) = current_position(1) = current_position(2) = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + current_position(moment_component) += nodal_positions(node_loop, moment_component) * basis_values(node_loop); + } + + // assign contribution to every local node this element has + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + if (map->isNodeGlobalElement(nodes_in_elem(ielem, node_loop))) + { + local_node_id = map->getLocalElement(nodes_in_elem(ielem, node_loop)); + design_gradients(local_node_id, 0) += weight_multiply * basis_values(node_loop) * current_position(moment_component) * Jacobian; + } + } } - } } - - } - } - /* ------------------------------------------------------------------------------------------------------------------------ Compute the moment of inertia of each element for a specified component of the inertia tensor; estimated with quadrature --------------------------------------------------------------------------------------------------------------------------- */ -void FEA_Module_Inertial::compute_element_moments_of_inertia(const_host_vec_array design_densities, bool max_flag, int inertia_component, bool use_initial_coords){ - //local number of uniquely assigned elements - size_t nonoverlap_nelements = element_map->getLocalNumElements(); - //initialize memory for volume storage - host_vec_array Element_Masses = Global_Element_Masses->getLocalView(Tpetra::Access::ReadWrite); - host_vec_array Element_Moments_of_Inertia; - - if(inertia_component==0) Element_Moments_of_Inertia = Global_Element_Moments_of_Inertia_xx->getLocalView(Tpetra::Access::ReadWrite); - if(inertia_component==1) Element_Moments_of_Inertia = Global_Element_Moments_of_Inertia_yy->getLocalView(Tpetra::Access::ReadWrite); - if(inertia_component==2) Element_Moments_of_Inertia = Global_Element_Moments_of_Inertia_zz->getLocalView(Tpetra::Access::ReadWrite); - if(inertia_component==3) Element_Moments_of_Inertia = Global_Element_Moments_of_Inertia_xy->getLocalView(Tpetra::Access::ReadWrite); - if(inertia_component==4) Element_Moments_of_Inertia = Global_Element_Moments_of_Inertia_xz->getLocalView(Tpetra::Access::ReadWrite); - if(inertia_component==5) Element_Moments_of_Inertia = Global_Element_Moments_of_Inertia_yz->getLocalView(Tpetra::Access::ReadWrite); - - const_host_vec_array Element_Volumes = Global_Element_Volumes->getLocalView(Tpetra::Access::ReadOnly); - //local variable for host view in the dual view - const_host_vec_array all_node_coords = all_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_host_vec_array all_initial_node_coords; - if(use_initial_coords) - all_initial_node_coords = all_initial_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_host_vec_array all_design_densities; - if(nodal_density_flag) - all_design_densities = all_node_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_host_elem_conn_array nodes_in_elem = global_nodes_in_elem_distributed->getLocalView (Tpetra::Access::ReadOnly); - double inertia_center[3]; - if(module_params->enable_inertia_center[0]){ - inertia_center[0] = module_params->moment_of_inertia_center[0]; - } - else{ - inertia_center[0] = center_of_mass[0]; - } - if(module_params->enable_inertia_center[1]){ - inertia_center[1] = module_params->moment_of_inertia_center[1]; - } - else{ - inertia_center[1] = center_of_mass[1]; - } - if(num_dim==3){ - if(module_params->enable_inertia_center[2]){ - inertia_center[2] = module_params->moment_of_inertia_center[2]; +void FEA_Module_Inertial::compute_element_moments_of_inertia(const_host_vec_array design_densities, bool max_flag, int inertia_component, bool use_initial_coords) +{ + // local number of uniquely assigned elements + size_t nonoverlap_nelements = element_map->getLocalNumElements(); + // initialize memory for volume storage + host_vec_array Element_Masses = Global_Element_Masses->getLocalView(Tpetra::Access::ReadWrite); + host_vec_array Element_Moments_of_Inertia; + + if (inertia_component == 0) + { + Element_Moments_of_Inertia = Global_Element_Moments_of_Inertia_xx->getLocalView(Tpetra::Access::ReadWrite); } - else{ - inertia_center[2] = center_of_mass[2]; + if (inertia_component == 1) + { + Element_Moments_of_Inertia = Global_Element_Moments_of_Inertia_yy->getLocalView(Tpetra::Access::ReadWrite); } - } - int nodes_per_elem = elem->num_basis(); - int z_quad,y_quad,x_quad, direct_product_count; - size_t local_node_id; - LO ielem; - GO global_element_index; - real_t delx1, delx2; - - real_t Jacobian, current_density, weight_multiply; - //CArrayKokkos legendre_nodes_1D(num_gauss_points); - //CArrayKokkos legendre_weights_1D(num_gauss_points); - CArray legendre_nodes_1D(num_gauss_points); - CArray legendre_weights_1D(num_gauss_points); - real_t pointer_quad_coordinate[num_dim]; - real_t pointer_quad_coordinate_weight[num_dim]; - real_t pointer_interpolated_point[num_dim]; - real_t pointer_JT_row1[num_dim]; - real_t pointer_JT_row2[num_dim]; - real_t pointer_JT_row3[num_dim]; - ViewCArray quad_coordinate(pointer_quad_coordinate,num_dim); - ViewCArray quad_coordinate_weight(pointer_quad_coordinate_weight,num_dim); - ViewCArray interpolated_point(pointer_interpolated_point,num_dim); - ViewCArray JT_row1(pointer_JT_row1,num_dim); - ViewCArray JT_row2(pointer_JT_row2,num_dim); - ViewCArray JT_row3(pointer_JT_row3,num_dim); - - real_t pointer_basis_values[elem->num_basis()]; - real_t pointer_basis_derivative_s1[elem->num_basis()]; - real_t pointer_basis_derivative_s2[elem->num_basis()]; - real_t pointer_basis_derivative_s3[elem->num_basis()]; - ViewCArray basis_values(pointer_basis_values,elem->num_basis()); - ViewCArray basis_derivative_s1(pointer_basis_derivative_s1,elem->num_basis()); - ViewCArray basis_derivative_s2(pointer_basis_derivative_s2,elem->num_basis()); - ViewCArray basis_derivative_s3(pointer_basis_derivative_s3,elem->num_basis()); - CArrayKokkos nodal_positions(elem->num_basis(),num_dim); - CArrayKokkos initial_nodal_positions(elem->num_basis(),num_dim); - CArrayKokkos nodal_density(elem->num_basis()); - CArrayKokkos current_position(num_dim); - - //initialize weights - elements::legendre_nodes_1D(legendre_nodes_1D,num_gauss_points); - elements::legendre_weights_1D(legendre_weights_1D,num_gauss_points); - - Solver::node_ordering_convention active_node_ordering_convention = Solver_Pointer_->active_node_ordering_convention; - CArrayKokkos convert_node_order(max_nodes_per_element); - if((active_node_ordering_convention == Solver::ENSIGHT && num_dim==3)||(active_node_ordering_convention == Solver::IJK && num_dim==2)){ - convert_node_order(0) = 0; - convert_node_order(1) = 1; - convert_node_order(2) = 3; - convert_node_order(3) = 2; - if(num_dim == 3){ - convert_node_order(4) = 4; - convert_node_order(5) = 5; - convert_node_order(6) = 7; - convert_node_order(7) = 6; + if (inertia_component == 2) + { + Element_Moments_of_Inertia = Global_Element_Moments_of_Inertia_zz->getLocalView(Tpetra::Access::ReadWrite); } - } - else{ - convert_node_order(0) = 0; - convert_node_order(1) = 1; - convert_node_order(2) = 2; - convert_node_order(3) = 3; - if(num_dim==3){ - convert_node_order(4) = 4; - convert_node_order(5) = 5; - convert_node_order(6) = 6; - convert_node_order(7) = 7; + if (inertia_component == 3) + { + Element_Moments_of_Inertia = Global_Element_Moments_of_Inertia_xy->getLocalView(Tpetra::Access::ReadWrite); } - } - - //loop over elements and use quadrature rule to compute volume from Jacobian determinant - for(int nonoverlapping_ielem = 0; nonoverlapping_ielem < nonoverlap_nelements; nonoverlapping_ielem++){ - global_element_index = element_map->getGlobalElement(nonoverlapping_ielem); - ielem = all_element_map->getLocalElement(global_element_index); - //acquire set of nodes for this local element - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - local_node_id = all_node_map->getLocalElement(nodes_in_elem(ielem, convert_node_order(node_loop))); - nodal_positions(node_loop,0) = all_node_coords(local_node_id,0); - nodal_positions(node_loop,1) = all_node_coords(local_node_id,1); - nodal_positions(node_loop,2) = all_node_coords(local_node_id,2); - if(use_initial_coords){ - initial_nodal_positions(node_loop,0) = all_initial_node_coords(local_node_id,0); - initial_nodal_positions(node_loop,1) = all_initial_node_coords(local_node_id,1); - initial_nodal_positions(node_loop,2) = all_initial_node_coords(local_node_id,2); - } - if(nodal_density_flag) nodal_density(node_loop) = all_design_densities(local_node_id,0); - /* - if(myrank==1&&nodal_positions(node_loop,2)>10000000){ - std::cout << " LOCAL MATRIX DEBUG ON TASK " << myrank << std::endl; - std::cout << node_loop+1 <<" " << local_node_id <<" "<< nodes_in_elem(ielem, node_loop) << " "<< nodal_positions(node_loop,2) << std::endl; - std::fflush(stdout); - } - */ - //std::cout << local_node_id << " " << nodes_in_elem(ielem, node_loop) << " " - //<< nodal_positions(node_loop,0) << " " << nodal_positions(node_loop,1) << " "<< nodal_positions(node_loop,2) << " " << nodal_density(node_loop) <getLocalView(Tpetra::Access::ReadWrite); } - - //debug print of index - //std::cout << "nonoverlap element id on TASK " << myrank << " is " << nonoverlapping_ielem << std::endl; - //std::fflush(stdout); - - //initialize element mass - Element_Moments_of_Inertia(nonoverlapping_ielem,0) = 0; - - if(Element_Types(ielem)==elements::elem_types::Hex8){ - direct_product_count = std::pow(num_gauss_points,num_dim); + if (inertia_component == 5) + { + Element_Moments_of_Inertia = Global_Element_Moments_of_Inertia_yz->getLocalView(Tpetra::Access::ReadWrite); } - - //loop over quadrature points - for(int iquad=0; iquad < direct_product_count; iquad++){ - - //set current quadrature point - if(num_dim==3) z_quad = iquad/(num_gauss_points*num_gauss_points); - y_quad = (iquad % (num_gauss_points*num_gauss_points))/num_gauss_points; - x_quad = iquad % num_gauss_points; - quad_coordinate(0) = legendre_nodes_1D(x_quad); - quad_coordinate(1) = legendre_nodes_1D(y_quad); - if(num_dim==3) - quad_coordinate(2) = legendre_nodes_1D(z_quad); - - //set current quadrature weight - quad_coordinate_weight(0) = legendre_weights_1D(x_quad); - quad_coordinate_weight(1) = legendre_weights_1D(y_quad); - if(num_dim==3) - quad_coordinate_weight(2) = legendre_weights_1D(z_quad); - else - quad_coordinate_weight(2) = 1; - weight_multiply = quad_coordinate_weight(0)*quad_coordinate_weight(1)*quad_coordinate_weight(2); - - //compute shape functions at this point for the element type - elem->basis(basis_values,quad_coordinate); - - //compute all the necessary coordinates and derivatives at this point - - //compute shape function derivatives - elem->partial_xi_basis(basis_derivative_s1,quad_coordinate); - elem->partial_eta_basis(basis_derivative_s2,quad_coordinate); - elem->partial_mu_basis(basis_derivative_s3,quad_coordinate); - - //compute derivatives of x,y,z w.r.t the s,t,w isoparametric space needed by JT (Transpose of the Jacobian) - //derivative of x,y,z w.r.t s - JT_row1(0) = 0; - JT_row1(1) = 0; - JT_row1(2) = 0; - if(use_initial_coords){ - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - JT_row1(0) += initial_nodal_positions(node_loop,0)*basis_derivative_s1(node_loop); - JT_row1(1) += initial_nodal_positions(node_loop,1)*basis_derivative_s1(node_loop); - JT_row1(2) += initial_nodal_positions(node_loop,2)*basis_derivative_s1(node_loop); - } - //derivative of x,y,z w.r.t t - JT_row2(0) = 0; - JT_row2(1) = 0; - JT_row2(2) = 0; - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - JT_row2(0) += initial_nodal_positions(node_loop,0)*basis_derivative_s2(node_loop); - JT_row2(1) += initial_nodal_positions(node_loop,1)*basis_derivative_s2(node_loop); - JT_row2(2) += initial_nodal_positions(node_loop,2)*basis_derivative_s2(node_loop); + const_host_vec_array Element_Volumes = Global_Element_Volumes->getLocalView(Tpetra::Access::ReadOnly); + // local variable for host view in the dual view + const_host_vec_array all_node_coords = all_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_host_vec_array all_initial_node_coords; + if (use_initial_coords) + { + all_initial_node_coords = all_initial_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + } + const_host_vec_array all_design_densities; + if (nodal_density_flag) + { + all_design_densities = all_node_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); + } + const_host_elem_conn_array nodes_in_elem = global_nodes_in_elem_distributed->getLocalView(Tpetra::Access::ReadOnly); + double inertia_center[3]; + if (module_params->enable_inertia_center[0]) + { + inertia_center[0] = module_params->moment_of_inertia_center[0]; + } + else + { + inertia_center[0] = center_of_mass[0]; + } + if (module_params->enable_inertia_center[1]) + { + inertia_center[1] = module_params->moment_of_inertia_center[1]; + } + else + { + inertia_center[1] = center_of_mass[1]; + } + if (num_dim == 3) + { + if (module_params->enable_inertia_center[2]) + { + inertia_center[2] = module_params->moment_of_inertia_center[2]; } - - //derivative of x,y,z w.r.t w - JT_row3(0) = 0; - JT_row3(1) = 0; - JT_row3(2) = 0; - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - JT_row3(0) += initial_nodal_positions(node_loop,0)*basis_derivative_s3(node_loop); - JT_row3(1) += initial_nodal_positions(node_loop,1)*basis_derivative_s3(node_loop); - JT_row3(2) += initial_nodal_positions(node_loop,2)*basis_derivative_s3(node_loop); + else + { + inertia_center[2] = center_of_mass[2]; } - - } - else{ - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - JT_row1(0) += nodal_positions(node_loop,0)*basis_derivative_s1(node_loop); - JT_row1(1) += nodal_positions(node_loop,1)*basis_derivative_s1(node_loop); - JT_row1(2) += nodal_positions(node_loop,2)*basis_derivative_s1(node_loop); + } + int nodes_per_elem = elem->num_basis(); + int z_quad, y_quad, x_quad, direct_product_count; + size_t local_node_id; + LO ielem; + GO global_element_index; + real_t delx1, delx2; + + real_t Jacobian, current_density, weight_multiply; + // CArrayKokkos legendre_nodes_1D(num_gauss_points); + // CArrayKokkos legendre_weights_1D(num_gauss_points); + CArray legendre_nodes_1D(num_gauss_points); + CArray legendre_weights_1D(num_gauss_points); + real_t pointer_quad_coordinate[num_dim]; + real_t pointer_quad_coordinate_weight[num_dim]; + real_t pointer_interpolated_point[num_dim]; + real_t pointer_JT_row1[num_dim]; + real_t pointer_JT_row2[num_dim]; + real_t pointer_JT_row3[num_dim]; + ViewCArray quad_coordinate(pointer_quad_coordinate, num_dim); + ViewCArray quad_coordinate_weight(pointer_quad_coordinate_weight, num_dim); + ViewCArray interpolated_point(pointer_interpolated_point, num_dim); + ViewCArray JT_row1(pointer_JT_row1, num_dim); + ViewCArray JT_row2(pointer_JT_row2, num_dim); + ViewCArray JT_row3(pointer_JT_row3, num_dim); + + real_t pointer_basis_values[elem->num_basis()]; + real_t pointer_basis_derivative_s1[elem->num_basis()]; + real_t pointer_basis_derivative_s2[elem->num_basis()]; + real_t pointer_basis_derivative_s3[elem->num_basis()]; + ViewCArray basis_values(pointer_basis_values, elem->num_basis()); + ViewCArray basis_derivative_s1(pointer_basis_derivative_s1, elem->num_basis()); + ViewCArray basis_derivative_s2(pointer_basis_derivative_s2, elem->num_basis()); + ViewCArray basis_derivative_s3(pointer_basis_derivative_s3, elem->num_basis()); + CArrayKokkos nodal_positions(elem->num_basis(), num_dim); + CArrayKokkos initial_nodal_positions(elem->num_basis(), num_dim); + CArrayKokkos nodal_density(elem->num_basis()); + CArrayKokkos current_position(num_dim); + + // initialize weights + elements::legendre_nodes_1D(legendre_nodes_1D, num_gauss_points); + elements::legendre_weights_1D(legendre_weights_1D, num_gauss_points); + + Solver::node_ordering_convention active_node_ordering_convention = Solver_Pointer_->active_node_ordering_convention; + CArrayKokkos convert_node_order(max_nodes_per_element); + if ((active_node_ordering_convention == Solver::ENSIGHT && num_dim == 3) || (active_node_ordering_convention == Solver::IJK && num_dim == 2)) + { + convert_node_order(0) = 0; + convert_node_order(1) = 1; + convert_node_order(2) = 3; + convert_node_order(3) = 2; + if (num_dim == 3) + { + convert_node_order(4) = 4; + convert_node_order(5) = 5; + convert_node_order(6) = 7; + convert_node_order(7) = 6; + } + } + else + { + convert_node_order(0) = 0; + convert_node_order(1) = 1; + convert_node_order(2) = 2; + convert_node_order(3) = 3; + if (num_dim == 3) + { + convert_node_order(4) = 4; + convert_node_order(5) = 5; + convert_node_order(6) = 6; + convert_node_order(7) = 7; } + } - //derivative of x,y,z w.r.t t - JT_row2(0) = 0; - JT_row2(1) = 0; - JT_row2(2) = 0; - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - JT_row2(0) += nodal_positions(node_loop,0)*basis_derivative_s2(node_loop); - JT_row2(1) += nodal_positions(node_loop,1)*basis_derivative_s2(node_loop); - JT_row2(2) += nodal_positions(node_loop,2)*basis_derivative_s2(node_loop); + // loop over elements and use quadrature rule to compute volume from Jacobian determinant + for (int nonoverlapping_ielem = 0; nonoverlapping_ielem < nonoverlap_nelements; nonoverlapping_ielem++) + { + global_element_index = element_map->getGlobalElement(nonoverlapping_ielem); + ielem = all_element_map->getLocalElement(global_element_index); + // acquire set of nodes for this local element + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + local_node_id = all_node_map->getLocalElement(nodes_in_elem(ielem, convert_node_order(node_loop))); + nodal_positions(node_loop, 0) = all_node_coords(local_node_id, 0); + nodal_positions(node_loop, 1) = all_node_coords(local_node_id, 1); + nodal_positions(node_loop, 2) = all_node_coords(local_node_id, 2); + if (use_initial_coords) + { + initial_nodal_positions(node_loop, 0) = all_initial_node_coords(local_node_id, 0); + initial_nodal_positions(node_loop, 1) = all_initial_node_coords(local_node_id, 1); + initial_nodal_positions(node_loop, 2) = all_initial_node_coords(local_node_id, 2); + } + if (nodal_density_flag) + { + nodal_density(node_loop) = all_design_densities(local_node_id, 0); + } + /* + if(myrank==1&&nodal_positions(node_loop,2)>10000000){ + std::cout << " LOCAL MATRIX DEBUG ON TASK " << myrank << std::endl; + std::cout << node_loop+1 <<" " << local_node_id <<" "<< nodes_in_elem(ielem, node_loop) << " "<< nodal_positions(node_loop,2) << std::endl; + std::fflush(stdout); + } + */ + // std::cout << local_node_id << " " << nodes_in_elem(ielem, node_loop) << " " + // << nodal_positions(node_loop,0) << " " << nodal_positions(node_loop,1) << " "<< nodal_positions(node_loop,2) << " " << nodal_density(node_loop) <num_basis(); node_loop++){ - JT_row3(0) += nodal_positions(node_loop,0)*basis_derivative_s3(node_loop); - JT_row3(1) += nodal_positions(node_loop,1)*basis_derivative_s3(node_loop); - JT_row3(2) += nodal_positions(node_loop,2)*basis_derivative_s3(node_loop); + // debug print of index + // std::cout << "nonoverlap element id on TASK " << myrank << " is " << nonoverlapping_ielem << std::endl; + // std::fflush(stdout); + + // initialize element mass + Element_Moments_of_Inertia(nonoverlapping_ielem, 0) = 0; + + if (Element_Types(ielem) == elements::elem_types::Hex8) + { + direct_product_count = std::pow(num_gauss_points, num_dim); } - } - - - //compute the determinant of the Jacobian - Jacobian = JT_row1(0)*(JT_row2(1)*JT_row3(2)-JT_row3(1)*JT_row2(2))- - JT_row1(1)*(JT_row2(0)*JT_row3(2)-JT_row3(0)*JT_row2(2))+ - JT_row1(2)*(JT_row2(0)*JT_row3(1)-JT_row3(0)*JT_row2(1)); - if(Jacobian<0) Jacobian = -Jacobian; - - //compute density - if(max_flag){ - current_density = 1; - } - else{ - if(nodal_density_flag){ - current_density = 0; - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - current_density += nodal_density(node_loop)*basis_values(node_loop); - } - }// if - else{ - current_density = design_densities(nonoverlapping_ielem,0); + + // loop over quadrature points + for (int iquad = 0; iquad < direct_product_count; iquad++) + { + // set current quadrature point + if (num_dim == 3) + { + z_quad = iquad / (num_gauss_points * num_gauss_points); + } + y_quad = (iquad % (num_gauss_points * num_gauss_points)) / num_gauss_points; + x_quad = iquad % num_gauss_points; + quad_coordinate(0) = legendre_nodes_1D(x_quad); + quad_coordinate(1) = legendre_nodes_1D(y_quad); + if (num_dim == 3) + { + quad_coordinate(2) = legendre_nodes_1D(z_quad); + } + + // set current quadrature weight + quad_coordinate_weight(0) = legendre_weights_1D(x_quad); + quad_coordinate_weight(1) = legendre_weights_1D(y_quad); + if (num_dim == 3) + { + quad_coordinate_weight(2) = legendre_weights_1D(z_quad); + } + else + { + quad_coordinate_weight(2) = 1; + } + weight_multiply = quad_coordinate_weight(0) * quad_coordinate_weight(1) * quad_coordinate_weight(2); + + // compute shape functions at this point for the element type + elem->basis(basis_values, quad_coordinate); + + // compute all the necessary coordinates and derivatives at this point + + // compute shape function derivatives + elem->partial_xi_basis(basis_derivative_s1, quad_coordinate); + elem->partial_eta_basis(basis_derivative_s2, quad_coordinate); + elem->partial_mu_basis(basis_derivative_s3, quad_coordinate); + + // compute derivatives of x,y,z w.r.t the s,t,w isoparametric space needed by JT (Transpose of the Jacobian) + // derivative of x,y,z w.r.t s + JT_row1(0) = 0; + JT_row1(1) = 0; + JT_row1(2) = 0; + if (use_initial_coords) + { + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row1(0) += initial_nodal_positions(node_loop, 0) * basis_derivative_s1(node_loop); + JT_row1(1) += initial_nodal_positions(node_loop, 1) * basis_derivative_s1(node_loop); + JT_row1(2) += initial_nodal_positions(node_loop, 2) * basis_derivative_s1(node_loop); + } + + // derivative of x,y,z w.r.t t + JT_row2(0) = 0; + JT_row2(1) = 0; + JT_row2(2) = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row2(0) += initial_nodal_positions(node_loop, 0) * basis_derivative_s2(node_loop); + JT_row2(1) += initial_nodal_positions(node_loop, 1) * basis_derivative_s2(node_loop); + JT_row2(2) += initial_nodal_positions(node_loop, 2) * basis_derivative_s2(node_loop); + } + + // derivative of x,y,z w.r.t w + JT_row3(0) = 0; + JT_row3(1) = 0; + JT_row3(2) = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row3(0) += initial_nodal_positions(node_loop, 0) * basis_derivative_s3(node_loop); + JT_row3(1) += initial_nodal_positions(node_loop, 1) * basis_derivative_s3(node_loop); + JT_row3(2) += initial_nodal_positions(node_loop, 2) * basis_derivative_s3(node_loop); + } + } + else + { + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row1(0) += nodal_positions(node_loop, 0) * basis_derivative_s1(node_loop); + JT_row1(1) += nodal_positions(node_loop, 1) * basis_derivative_s1(node_loop); + JT_row1(2) += nodal_positions(node_loop, 2) * basis_derivative_s1(node_loop); + } + + // derivative of x,y,z w.r.t t + JT_row2(0) = 0; + JT_row2(1) = 0; + JT_row2(2) = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row2(0) += nodal_positions(node_loop, 0) * basis_derivative_s2(node_loop); + JT_row2(1) += nodal_positions(node_loop, 1) * basis_derivative_s2(node_loop); + JT_row2(2) += nodal_positions(node_loop, 2) * basis_derivative_s2(node_loop); + } + + // derivative of x,y,z w.r.t w + JT_row3(0) = 0; + JT_row3(1) = 0; + JT_row3(2) = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row3(0) += nodal_positions(node_loop, 0) * basis_derivative_s3(node_loop); + JT_row3(1) += nodal_positions(node_loop, 1) * basis_derivative_s3(node_loop); + JT_row3(2) += nodal_positions(node_loop, 2) * basis_derivative_s3(node_loop); + } + } + + // compute the determinant of the Jacobian + Jacobian = JT_row1(0) * (JT_row2(1) * JT_row3(2) - JT_row3(1) * JT_row2(2)) - + JT_row1(1) * (JT_row2(0) * JT_row3(2) - JT_row3(0) * JT_row2(2)) + + JT_row1(2) * (JT_row2(0) * JT_row3(1) - JT_row3(0) * JT_row2(1)); + if (Jacobian < 0) + { + Jacobian = -Jacobian; + } + + // compute density + if (max_flag) + { + current_density = 1; + } + else + { + if (nodal_density_flag) + { + current_density = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + current_density += nodal_density(node_loop) * basis_values(node_loop); + } + } // if + else + { + current_density = design_densities(nonoverlapping_ielem, 0); + } + } + + // compute current position + current_position(0) = current_position(1) = current_position(2) = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + current_position(0) += nodal_positions(node_loop, 0) * basis_values(node_loop); + current_position(1) += nodal_positions(node_loop, 1) * basis_values(node_loop); + current_position(2) += nodal_positions(node_loop, 2) * basis_values(node_loop); + } + + if (inertia_component == 0) + { + delx1 = current_position(1) - inertia_center[1]; + delx2 = current_position(2) - inertia_center[2]; + Element_Moments_of_Inertia(nonoverlapping_ielem, 0) += current_density * (delx1 * delx1 + delx2 * delx2) * weight_multiply * Jacobian; + } + if (inertia_component == 1) + { + delx1 = current_position(0) - inertia_center[0]; + delx2 = current_position(2) - inertia_center[2]; + Element_Moments_of_Inertia(nonoverlapping_ielem, 0) += current_density * (delx1 * delx1 + delx2 * delx2) * weight_multiply * Jacobian; + } + if (inertia_component == 2) + { + delx1 = current_position(0) - inertia_center[0]; + delx2 = current_position(1) - inertia_center[1]; + Element_Moments_of_Inertia(nonoverlapping_ielem, 0) += current_density * (delx1 * delx1 + delx2 * delx2) * weight_multiply * Jacobian; + } + if (inertia_component == 3) + { + delx1 = current_position(0) - inertia_center[0]; + delx2 = current_position(1) - inertia_center[1]; + Element_Moments_of_Inertia(nonoverlapping_ielem, 0) -= current_density * (delx1 * delx2) * weight_multiply * Jacobian; + } + if (inertia_component == 4) + { + delx1 = current_position(0) - inertia_center[0]; + delx2 = current_position(2) - inertia_center[2]; + Element_Moments_of_Inertia(nonoverlapping_ielem, 0) -= current_density * (delx1 * delx2) * weight_multiply * Jacobian; + } + if (inertia_component == 5) + { + delx1 = current_position(1) - inertia_center[1]; + delx2 = current_position(2) - inertia_center[2]; + Element_Moments_of_Inertia(nonoverlapping_ielem, 0) -= current_density * (delx1 * delx2) * weight_multiply * Jacobian; + } } - } - - //compute current position - current_position(0) = current_position(1) = current_position(2) = 0; - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - current_position(0) += nodal_positions(node_loop,0)*basis_values(node_loop); - current_position(1) += nodal_positions(node_loop,1)*basis_values(node_loop); - current_position(2) += nodal_positions(node_loop,2)*basis_values(node_loop); - } - - if(inertia_component==0){ - delx1 = current_position(1) - inertia_center[1]; - delx2 = current_position(2) - inertia_center[2]; - Element_Moments_of_Inertia(nonoverlapping_ielem,0) += current_density*(delx1*delx1 + delx2*delx2)*weight_multiply*Jacobian; - } - if(inertia_component==1){ - delx1 = current_position(0) - inertia_center[0]; - delx2 = current_position(2) - inertia_center[2]; - Element_Moments_of_Inertia(nonoverlapping_ielem,0) += current_density*(delx1*delx1 + delx2*delx2)*weight_multiply*Jacobian; - } - if(inertia_component==2){ - delx1 = current_position(0) - inertia_center[0]; - delx2 = current_position(1) - inertia_center[1]; - Element_Moments_of_Inertia(nonoverlapping_ielem,0) += current_density*(delx1*delx1 + delx2*delx2)*weight_multiply*Jacobian; - } - if(inertia_component==3){ - delx1 = current_position(0) - inertia_center[0]; - delx2 = current_position(1) - inertia_center[1]; - Element_Moments_of_Inertia(nonoverlapping_ielem,0) -= current_density*(delx1*delx2)*weight_multiply*Jacobian; - } - if(inertia_component==4){ - delx1 = current_position(0) - inertia_center[0]; - delx2 = current_position(2) - inertia_center[2]; - Element_Moments_of_Inertia(nonoverlapping_ielem,0) -= current_density*(delx1*delx2)*weight_multiply*Jacobian; - } - if(inertia_component==5){ - delx1 = current_position(1) - inertia_center[1]; - delx2 = current_position(2) - inertia_center[2]; - Element_Moments_of_Inertia(nonoverlapping_ielem,0) -= current_density*(delx1*delx2)*weight_multiply*Jacobian; - } } - } - - //std::ostream &out = std::cout; - //Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - //if(myrank==0) - //*fos << "Global Element Masses:" << std::endl; - //Global_Element_Masses->describe(*fos,Teuchos::VERB_EXTREME); - //*fos << std::endl; + + // std::ostream &out = std::cout; + // Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + // if(myrank==0) + // *fos << "Global Element Masses:" << std::endl; + // Global_Element_Masses->describe(*fos,Teuchos::VERB_EXTREME); + // *fos << std::endl; } /* --------------------------------------------------------------------------------------------------- Compute the gradients of the specified moment of inertia component with respect to design densities ------------------------------------------------------------------------------------------------------ */ -void FEA_Module_Inertial::compute_moment_of_inertia_gradients(const_host_vec_array design_variables, host_vec_array design_gradients, int inertia_component, bool use_initial_coords){ - //local number of uniquely assigned elements - size_t nonoverlap_nelements = element_map->getLocalNumElements(); - //local variable for host view in the dual view - const_host_vec_array all_node_coords = all_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_host_vec_array all_initial_node_coords; - if(use_initial_coords) - all_initial_node_coords = all_initial_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_host_elem_conn_array nodes_in_elem = global_nodes_in_elem_distributed->getLocalView (Tpetra::Access::ReadOnly); - double inertia_center[3]; - - Solver::node_ordering_convention active_node_ordering_convention = Solver_Pointer_->active_node_ordering_convention; - CArrayKokkos convert_node_order(max_nodes_per_element); - if((active_node_ordering_convention == Solver::ENSIGHT && num_dim==3)||(active_node_ordering_convention == Solver::IJK && num_dim==2)){ - convert_node_order(0) = 0; - convert_node_order(1) = 1; - convert_node_order(2) = 3; - convert_node_order(3) = 2; - if(num_dim == 3){ - convert_node_order(4) = 4; - convert_node_order(5) = 5; - convert_node_order(6) = 7; - convert_node_order(7) = 6; +void FEA_Module_Inertial::compute_moment_of_inertia_gradients(const_host_vec_array design_variables, host_vec_array design_gradients, int inertia_component, bool use_initial_coords) +{ + // local number of uniquely assigned elements + size_t nonoverlap_nelements = element_map->getLocalNumElements(); + // local variable for host view in the dual view + const_host_vec_array all_node_coords = all_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_host_vec_array all_initial_node_coords; + if (use_initial_coords) + { + all_initial_node_coords = all_initial_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); } - } - else{ - convert_node_order(0) = 0; - convert_node_order(1) = 1; - convert_node_order(2) = 2; - convert_node_order(3) = 3; - if(num_dim==3){ - convert_node_order(4) = 4; - convert_node_order(5) = 5; - convert_node_order(6) = 6; - convert_node_order(7) = 7; + const_host_elem_conn_array nodes_in_elem = global_nodes_in_elem_distributed->getLocalView(Tpetra::Access::ReadOnly); + double inertia_center[3]; + + Solver::node_ordering_convention active_node_ordering_convention = Solver_Pointer_->active_node_ordering_convention; + CArrayKokkos convert_node_order(max_nodes_per_element); + if ((active_node_ordering_convention == Solver::ENSIGHT && num_dim == 3) || (active_node_ordering_convention == Solver::IJK && num_dim == 2)) + { + convert_node_order(0) = 0; + convert_node_order(1) = 1; + convert_node_order(2) = 3; + convert_node_order(3) = 2; + if (num_dim == 3) + { + convert_node_order(4) = 4; + convert_node_order(5) = 5; + convert_node_order(6) = 7; + convert_node_order(7) = 6; + } } - } - - if(module_params->enable_inertia_center[0]){ - inertia_center[0] = module_params->moment_of_inertia_center[0]; - } - else{ - inertia_center[0] = center_of_mass[0]; - } - if(module_params->enable_inertia_center[1]){ - inertia_center[1] = module_params->moment_of_inertia_center[1]; - } - else{ - inertia_center[1] = center_of_mass[1]; - } - if(num_dim==3){ - if(module_params->enable_inertia_center[2]){ - inertia_center[2] = module_params->moment_of_inertia_center[2]; + else + { + convert_node_order(0) = 0; + convert_node_order(1) = 1; + convert_node_order(2) = 2; + convert_node_order(3) = 3; + if (num_dim == 3) + { + convert_node_order(4) = 4; + convert_node_order(5) = 5; + convert_node_order(6) = 6; + convert_node_order(7) = 7; + } } - else{ - inertia_center[2] = center_of_mass[2]; + + if (module_params->enable_inertia_center[0]) + { + inertia_center[0] = module_params->moment_of_inertia_center[0]; } - } - const_host_vec_array all_node_densities; - if(nodal_density_flag) - all_node_densities = all_node_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); - int nodes_per_elem = elem->num_basis(); - int z_quad,y_quad,x_quad, direct_product_count; - size_t local_node_id; - LO ielem; - GO global_element_index; - real_t delx1, delx2; - - real_t Jacobian, weight_multiply; - //CArrayKokkos legendre_nodes_1D(num_gauss_points); - //CArrayKokkos legendre_weights_1D(num_gauss_points); - CArray legendre_nodes_1D(num_gauss_points); - CArray legendre_weights_1D(num_gauss_points); - real_t pointer_quad_coordinate[num_dim]; - real_t pointer_quad_coordinate_weight[num_dim]; - real_t pointer_interpolated_point[num_dim]; - real_t pointer_JT_row1[num_dim]; - real_t pointer_JT_row2[num_dim]; - real_t pointer_JT_row3[num_dim]; - ViewCArray quad_coordinate(pointer_quad_coordinate,num_dim); - ViewCArray quad_coordinate_weight(pointer_quad_coordinate_weight,num_dim); - ViewCArray interpolated_point(pointer_interpolated_point,num_dim); - ViewCArray JT_row1(pointer_JT_row1,num_dim); - ViewCArray JT_row2(pointer_JT_row2,num_dim); - ViewCArray JT_row3(pointer_JT_row3,num_dim); - - real_t pointer_basis_values[elem->num_basis()]; - real_t pointer_basis_derivative_s1[elem->num_basis()]; - real_t pointer_basis_derivative_s2[elem->num_basis()]; - real_t pointer_basis_derivative_s3[elem->num_basis()]; - ViewCArray basis_values(pointer_basis_values,elem->num_basis()); - ViewCArray basis_derivative_s1(pointer_basis_derivative_s1,elem->num_basis()); - ViewCArray basis_derivative_s2(pointer_basis_derivative_s2,elem->num_basis()); - ViewCArray basis_derivative_s3(pointer_basis_derivative_s3,elem->num_basis()); - CArrayKokkos nodal_positions(elem->num_basis(),num_dim); - CArrayKokkos initial_nodal_positions(elem->num_basis(),num_dim); - CArrayKokkos nodal_density(elem->num_basis()); - CArrayKokkos current_position(num_dim); - - //initialize weights - elements::legendre_nodes_1D(legendre_nodes_1D,num_gauss_points); - elements::legendre_weights_1D(legendre_weights_1D,num_gauss_points); - - //initialize design gradients to 0 - for(int init = 0; init < nlocal_nodes; init++) - design_gradients(init,0) = 0; - - //loop over elements and use quadrature rule to compute volume from Jacobian determinant - for(int ielem = 0; ielem < rnum_elem; ielem++){ - //acquire set of nodes for this local element - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - local_node_id = all_node_map->getLocalElement(nodes_in_elem(ielem, convert_node_order(node_loop))); - nodal_positions(node_loop,0) = all_node_coords(local_node_id,0); - nodal_positions(node_loop,1) = all_node_coords(local_node_id,1); - nodal_positions(node_loop,2) = all_node_coords(local_node_id,2); - if(use_initial_coords){ - initial_nodal_positions(node_loop,0) = all_initial_node_coords(local_node_id,0); - initial_nodal_positions(node_loop,1) = all_initial_node_coords(local_node_id,1); - initial_nodal_positions(node_loop,2) = all_initial_node_coords(local_node_id,2); - } - if(nodal_density_flag) nodal_density(node_loop) = all_node_densities(local_node_id,0); - /* - if(myrank==1&&nodal_positions(node_loop,2)>10000000){ - std::cout << " LOCAL MATRIX DEBUG ON TASK " << myrank << std::endl; - std::cout << node_loop+1 <<" " << local_node_id <<" "<< nodes_in_elem(ielem, node_loop) << " "<< nodal_positions(node_loop,2) << std::endl; - std::fflush(stdout); - } - */ - //std::cout << local_node_id << " " << nodes_in_elem(ielem, node_loop) << " " << nodal_positions(node_loop,0) << " " << nodal_positions(node_loop,1) << " "<< nodal_positions(node_loop,2) <enable_inertia_center[1]) + { + inertia_center[1] = module_params->moment_of_inertia_center[1]; } - - //loop over quadrature points - for(int iquad=0; iquad < direct_product_count; iquad++){ - - //set current quadrature point - if(num_dim==3) z_quad = iquad/(num_gauss_points*num_gauss_points); - y_quad = (iquad % (num_gauss_points*num_gauss_points))/num_gauss_points; - x_quad = iquad % num_gauss_points; - quad_coordinate(0) = legendre_nodes_1D(x_quad); - quad_coordinate(1) = legendre_nodes_1D(y_quad); - if(num_dim==3) - quad_coordinate(2) = legendre_nodes_1D(z_quad); - - //set current quadrature weight - quad_coordinate_weight(0) = legendre_weights_1D(x_quad); - quad_coordinate_weight(1) = legendre_weights_1D(y_quad); - if(num_dim==3) - quad_coordinate_weight(2) = legendre_weights_1D(z_quad); - else - quad_coordinate_weight(2) = 1; - weight_multiply = quad_coordinate_weight(0)*quad_coordinate_weight(1)*quad_coordinate_weight(2); - - //compute shape functions at this point for the element type - elem->basis(basis_values,quad_coordinate); - - //compute all the necessary coordinates and derivatives at this point - - //compute shape function derivatives - elem->partial_xi_basis(basis_derivative_s1,quad_coordinate); - elem->partial_eta_basis(basis_derivative_s2,quad_coordinate); - elem->partial_mu_basis(basis_derivative_s3,quad_coordinate); - - //compute derivatives of x,y,z w.r.t the s,t,w isoparametric space needed by JT (Transpose of the Jacobian) - //derivative of x,y,z w.r.t s - JT_row1(0) = 0; - JT_row1(1) = 0; - JT_row1(2) = 0; - if(use_initial_coords){ - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - JT_row1(0) += initial_nodal_positions(node_loop,0)*basis_derivative_s1(node_loop); - JT_row1(1) += initial_nodal_positions(node_loop,1)*basis_derivative_s1(node_loop); - JT_row1(2) += initial_nodal_positions(node_loop,2)*basis_derivative_s1(node_loop); + else + { + inertia_center[1] = center_of_mass[1]; + } + if (num_dim == 3) + { + if (module_params->enable_inertia_center[2]) + { + inertia_center[2] = module_params->moment_of_inertia_center[2]; } - - //derivative of x,y,z w.r.t t - JT_row2(0) = 0; - JT_row2(1) = 0; - JT_row2(2) = 0; - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - JT_row2(0) += initial_nodal_positions(node_loop,0)*basis_derivative_s2(node_loop); - JT_row2(1) += initial_nodal_positions(node_loop,1)*basis_derivative_s2(node_loop); - JT_row2(2) += initial_nodal_positions(node_loop,2)*basis_derivative_s2(node_loop); + else + { + inertia_center[2] = center_of_mass[2]; } + } + const_host_vec_array all_node_densities; + if (nodal_density_flag) + { + all_node_densities = all_node_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); + } + int nodes_per_elem = elem->num_basis(); + int z_quad, y_quad, x_quad, direct_product_count; + size_t local_node_id; + LO ielem; + GO global_element_index; + real_t delx1, delx2; + + real_t Jacobian, weight_multiply; + // CArrayKokkos legendre_nodes_1D(num_gauss_points); + // CArrayKokkos legendre_weights_1D(num_gauss_points); + CArray legendre_nodes_1D(num_gauss_points); + CArray legendre_weights_1D(num_gauss_points); + real_t pointer_quad_coordinate[num_dim]; + real_t pointer_quad_coordinate_weight[num_dim]; + real_t pointer_interpolated_point[num_dim]; + real_t pointer_JT_row1[num_dim]; + real_t pointer_JT_row2[num_dim]; + real_t pointer_JT_row3[num_dim]; + ViewCArray quad_coordinate(pointer_quad_coordinate, num_dim); + ViewCArray quad_coordinate_weight(pointer_quad_coordinate_weight, num_dim); + ViewCArray interpolated_point(pointer_interpolated_point, num_dim); + ViewCArray JT_row1(pointer_JT_row1, num_dim); + ViewCArray JT_row2(pointer_JT_row2, num_dim); + ViewCArray JT_row3(pointer_JT_row3, num_dim); + + real_t pointer_basis_values[elem->num_basis()]; + real_t pointer_basis_derivative_s1[elem->num_basis()]; + real_t pointer_basis_derivative_s2[elem->num_basis()]; + real_t pointer_basis_derivative_s3[elem->num_basis()]; + ViewCArray basis_values(pointer_basis_values, elem->num_basis()); + ViewCArray basis_derivative_s1(pointer_basis_derivative_s1, elem->num_basis()); + ViewCArray basis_derivative_s2(pointer_basis_derivative_s2, elem->num_basis()); + ViewCArray basis_derivative_s3(pointer_basis_derivative_s3, elem->num_basis()); + CArrayKokkos nodal_positions(elem->num_basis(), num_dim); + CArrayKokkos initial_nodal_positions(elem->num_basis(), num_dim); + CArrayKokkos nodal_density(elem->num_basis()); + CArrayKokkos current_position(num_dim); + + // initialize weights + elements::legendre_nodes_1D(legendre_nodes_1D, num_gauss_points); + elements::legendre_weights_1D(legendre_weights_1D, num_gauss_points); + + // initialize design gradients to 0 + for (int init = 0; init < nlocal_nodes; init++) + { + design_gradients(init, 0) = 0; + } - //derivative of x,y,z w.r.t w - JT_row3(0) = 0; - JT_row3(1) = 0; - JT_row3(2) = 0; - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - JT_row3(0) += initial_nodal_positions(node_loop,0)*basis_derivative_s3(node_loop); - JT_row3(1) += initial_nodal_positions(node_loop,1)*basis_derivative_s3(node_loop); - JT_row3(2) += initial_nodal_positions(node_loop,2)*basis_derivative_s3(node_loop); + // loop over elements and use quadrature rule to compute volume from Jacobian determinant + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + // acquire set of nodes for this local element + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + local_node_id = all_node_map->getLocalElement(nodes_in_elem(ielem, convert_node_order(node_loop))); + nodal_positions(node_loop, 0) = all_node_coords(local_node_id, 0); + nodal_positions(node_loop, 1) = all_node_coords(local_node_id, 1); + nodal_positions(node_loop, 2) = all_node_coords(local_node_id, 2); + if (use_initial_coords) + { + initial_nodal_positions(node_loop, 0) = all_initial_node_coords(local_node_id, 0); + initial_nodal_positions(node_loop, 1) = all_initial_node_coords(local_node_id, 1); + initial_nodal_positions(node_loop, 2) = all_initial_node_coords(local_node_id, 2); + } + if (nodal_density_flag) + { + nodal_density(node_loop) = all_node_densities(local_node_id, 0); + } + /* + if(myrank==1&&nodal_positions(node_loop,2)>10000000){ + std::cout << " LOCAL MATRIX DEBUG ON TASK " << myrank << std::endl; + std::cout << node_loop+1 <<" " << local_node_id <<" "<< nodes_in_elem(ielem, node_loop) << " "<< nodal_positions(node_loop,2) << std::endl; + std::fflush(stdout); + } + */ + // std::cout << local_node_id << " " << nodes_in_elem(ielem, node_loop) << " " << nodal_positions(node_loop,0) << " " << nodal_positions(node_loop,1) << " "<< nodal_positions(node_loop,2) <num_basis(); node_loop++){ - JT_row1(0) += nodal_positions(node_loop,0)*basis_derivative_s1(node_loop); - JT_row1(1) += nodal_positions(node_loop,1)*basis_derivative_s1(node_loop); - JT_row1(2) += nodal_positions(node_loop,2)*basis_derivative_s1(node_loop); + if (Element_Types(ielem) == elements::elem_types::Hex8) + { + direct_product_count = std::pow(num_gauss_points, num_dim); } - //derivative of x,y,z w.r.t t - JT_row2(0) = 0; - JT_row2(1) = 0; - JT_row2(2) = 0; - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - JT_row2(0) += nodal_positions(node_loop,0)*basis_derivative_s2(node_loop); - JT_row2(1) += nodal_positions(node_loop,1)*basis_derivative_s2(node_loop); - JT_row2(2) += nodal_positions(node_loop,2)*basis_derivative_s2(node_loop); - } + // loop over quadrature points + for (int iquad = 0; iquad < direct_product_count; iquad++) + { + // set current quadrature point + if (num_dim == 3) + { + z_quad = iquad / (num_gauss_points * num_gauss_points); + } + y_quad = (iquad % (num_gauss_points * num_gauss_points)) / num_gauss_points; + x_quad = iquad % num_gauss_points; + quad_coordinate(0) = legendre_nodes_1D(x_quad); + quad_coordinate(1) = legendre_nodes_1D(y_quad); + if (num_dim == 3) + { + quad_coordinate(2) = legendre_nodes_1D(z_quad); + } - //derivative of x,y,z w.r.t w - JT_row3(0) = 0; - JT_row3(1) = 0; - JT_row3(2) = 0; - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - JT_row3(0) += nodal_positions(node_loop,0)*basis_derivative_s3(node_loop); - JT_row3(1) += nodal_positions(node_loop,1)*basis_derivative_s3(node_loop); - JT_row3(2) += nodal_positions(node_loop,2)*basis_derivative_s3(node_loop); - } - } - - - //compute the determinant of the Jacobian - Jacobian = JT_row1(0)*(JT_row2(1)*JT_row3(2)-JT_row3(1)*JT_row2(2))- - JT_row1(1)*(JT_row2(0)*JT_row3(2)-JT_row3(0)*JT_row2(2))+ - JT_row1(2)*(JT_row2(0)*JT_row3(1)-JT_row3(0)*JT_row2(1)); - if(Jacobian<0) Jacobian = -Jacobian; - - //compute current position - current_position(0) = current_position(1) = current_position(2) = 0; - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - current_position(0) += nodal_positions(node_loop,0)*basis_values(node_loop); - current_position(1) += nodal_positions(node_loop,1)*basis_values(node_loop); - current_position(2) += nodal_positions(node_loop,2)*basis_values(node_loop); - } - - //assign contribution to every local node this element has - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - if(map->isNodeGlobalElement(nodes_in_elem(ielem, node_loop))){ - local_node_id = map->getLocalElement(nodes_in_elem(ielem, node_loop)); - if(inertia_component==0){ - delx1 = current_position(1) - inertia_center[1]; - delx2 = current_position(2) - inertia_center[2]; - design_gradients(local_node_id,0)+=weight_multiply*basis_values(node_loop)*(delx1*delx1 + delx2*delx2)*Jacobian; - } - if(inertia_component==1){ - delx1 = current_position(0) - inertia_center[0]; - delx2 = current_position(2) - inertia_center[2]; - design_gradients(local_node_id,0)+=weight_multiply*basis_values(node_loop)*(delx1*delx1 + delx2*delx2)*Jacobian; - } - if(inertia_component==2){ - delx1 = current_position(0) - inertia_center[0]; - delx2 = current_position(1) - inertia_center[1]; - design_gradients(local_node_id,0)+=weight_multiply*basis_values(node_loop)*(delx1*delx1 + delx2*delx2)*Jacobian; - } - if(inertia_component==3){ - delx1 = current_position(0) - inertia_center[0]; - delx2 = current_position(1) - inertia_center[1]; - design_gradients(local_node_id,0)-=weight_multiply*basis_values(node_loop)*(delx1*delx2)*Jacobian; - } - if(inertia_component==4){ - delx1 = current_position(0) - inertia_center[0]; - delx2 = current_position(2) - inertia_center[2]; - design_gradients(local_node_id,0)-=weight_multiply*basis_values(node_loop)*(delx1*delx2)*Jacobian; - } - if(inertia_component==5){ - delx1 = current_position(1) - inertia_center[1]; - delx2 = current_position(2) - inertia_center[2]; - design_gradients(local_node_id,0)-=weight_multiply*basis_values(node_loop)*(delx1*delx2)*Jacobian; + // set current quadrature weight + quad_coordinate_weight(0) = legendre_weights_1D(x_quad); + quad_coordinate_weight(1) = legendre_weights_1D(y_quad); + if (num_dim == 3) + { + quad_coordinate_weight(2) = legendre_weights_1D(z_quad); + } + else + { + quad_coordinate_weight(2) = 1; + } + weight_multiply = quad_coordinate_weight(0) * quad_coordinate_weight(1) * quad_coordinate_weight(2); + + // compute shape functions at this point for the element type + elem->basis(basis_values, quad_coordinate); + + // compute all the necessary coordinates and derivatives at this point + + // compute shape function derivatives + elem->partial_xi_basis(basis_derivative_s1, quad_coordinate); + elem->partial_eta_basis(basis_derivative_s2, quad_coordinate); + elem->partial_mu_basis(basis_derivative_s3, quad_coordinate); + + // compute derivatives of x,y,z w.r.t the s,t,w isoparametric space needed by JT (Transpose of the Jacobian) + // derivative of x,y,z w.r.t s + JT_row1(0) = 0; + JT_row1(1) = 0; + JT_row1(2) = 0; + if (use_initial_coords) + { + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row1(0) += initial_nodal_positions(node_loop, 0) * basis_derivative_s1(node_loop); + JT_row1(1) += initial_nodal_positions(node_loop, 1) * basis_derivative_s1(node_loop); + JT_row1(2) += initial_nodal_positions(node_loop, 2) * basis_derivative_s1(node_loop); + } + + // derivative of x,y,z w.r.t t + JT_row2(0) = 0; + JT_row2(1) = 0; + JT_row2(2) = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row2(0) += initial_nodal_positions(node_loop, 0) * basis_derivative_s2(node_loop); + JT_row2(1) += initial_nodal_positions(node_loop, 1) * basis_derivative_s2(node_loop); + JT_row2(2) += initial_nodal_positions(node_loop, 2) * basis_derivative_s2(node_loop); + } + + // derivative of x,y,z w.r.t w + JT_row3(0) = 0; + JT_row3(1) = 0; + JT_row3(2) = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row3(0) += initial_nodal_positions(node_loop, 0) * basis_derivative_s3(node_loop); + JT_row3(1) += initial_nodal_positions(node_loop, 1) * basis_derivative_s3(node_loop); + JT_row3(2) += initial_nodal_positions(node_loop, 2) * basis_derivative_s3(node_loop); + } + } + else + { + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row1(0) += nodal_positions(node_loop, 0) * basis_derivative_s1(node_loop); + JT_row1(1) += nodal_positions(node_loop, 1) * basis_derivative_s1(node_loop); + JT_row1(2) += nodal_positions(node_loop, 2) * basis_derivative_s1(node_loop); + } + + // derivative of x,y,z w.r.t t + JT_row2(0) = 0; + JT_row2(1) = 0; + JT_row2(2) = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row2(0) += nodal_positions(node_loop, 0) * basis_derivative_s2(node_loop); + JT_row2(1) += nodal_positions(node_loop, 1) * basis_derivative_s2(node_loop); + JT_row2(2) += nodal_positions(node_loop, 2) * basis_derivative_s2(node_loop); + } + + // derivative of x,y,z w.r.t w + JT_row3(0) = 0; + JT_row3(1) = 0; + JT_row3(2) = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row3(0) += nodal_positions(node_loop, 0) * basis_derivative_s3(node_loop); + JT_row3(1) += nodal_positions(node_loop, 1) * basis_derivative_s3(node_loop); + JT_row3(2) += nodal_positions(node_loop, 2) * basis_derivative_s3(node_loop); + } + } + + // compute the determinant of the Jacobian + Jacobian = JT_row1(0) * (JT_row2(1) * JT_row3(2) - JT_row3(1) * JT_row2(2)) - + JT_row1(1) * (JT_row2(0) * JT_row3(2) - JT_row3(0) * JT_row2(2)) + + JT_row1(2) * (JT_row2(0) * JT_row3(1) - JT_row3(0) * JT_row2(1)); + if (Jacobian < 0) + { + Jacobian = -Jacobian; + } + + // compute current position + current_position(0) = current_position(1) = current_position(2) = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + current_position(0) += nodal_positions(node_loop, 0) * basis_values(node_loop); + current_position(1) += nodal_positions(node_loop, 1) * basis_values(node_loop); + current_position(2) += nodal_positions(node_loop, 2) * basis_values(node_loop); + } + + // assign contribution to every local node this element has + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + if (map->isNodeGlobalElement(nodes_in_elem(ielem, node_loop))) + { + local_node_id = map->getLocalElement(nodes_in_elem(ielem, node_loop)); + if (inertia_component == 0) + { + delx1 = current_position(1) - inertia_center[1]; + delx2 = current_position(2) - inertia_center[2]; + design_gradients(local_node_id, 0) += weight_multiply * basis_values(node_loop) * (delx1 * delx1 + delx2 * delx2) * Jacobian; + } + if (inertia_component == 1) + { + delx1 = current_position(0) - inertia_center[0]; + delx2 = current_position(2) - inertia_center[2]; + design_gradients(local_node_id, 0) += weight_multiply * basis_values(node_loop) * (delx1 * delx1 + delx2 * delx2) * Jacobian; + } + if (inertia_component == 2) + { + delx1 = current_position(0) - inertia_center[0]; + delx2 = current_position(1) - inertia_center[1]; + design_gradients(local_node_id, 0) += weight_multiply * basis_values(node_loop) * (delx1 * delx1 + delx2 * delx2) * Jacobian; + } + if (inertia_component == 3) + { + delx1 = current_position(0) - inertia_center[0]; + delx2 = current_position(1) - inertia_center[1]; + design_gradients(local_node_id, 0) -= weight_multiply * basis_values(node_loop) * (delx1 * delx2) * Jacobian; + } + if (inertia_component == 4) + { + delx1 = current_position(0) - inertia_center[0]; + delx2 = current_position(2) - inertia_center[2]; + design_gradients(local_node_id, 0) -= weight_multiply * basis_values(node_loop) * (delx1 * delx2) * Jacobian; + } + if (inertia_component == 5) + { + delx1 = current_position(1) - inertia_center[1]; + delx2 = current_position(2) - inertia_center[2]; + design_gradients(local_node_id, 0) -= weight_multiply * basis_values(node_loop) * (delx1 * delx2) * Jacobian; + } + } } } - } } - - } - } /* ---------------------------------------------------------------------- Compute the volume of each element; estimated with quadrature ------------------------------------------------------------------------- */ -void FEA_Module_Inertial::compute_element_volumes(){ - //local number of uniquely assigned elements - size_t nonoverlap_nelements = element_map->getLocalNumElements(); - //local variable for host view in the dual view - const_host_vec_array all_node_coords = all_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_host_elem_conn_array nodes_in_elem = global_nodes_in_elem_distributed->getLocalView (Tpetra::Access::ReadOnly); - host_vec_array Element_Volumes = Global_Element_Volumes->getLocalView(Tpetra::Access::ReadWrite); - int nodes_per_elem = elem->num_basis(); - int z_quad,y_quad,x_quad, direct_product_count; - size_t local_node_id; - LO ielem; - GO global_element_index; - - real_t Jacobian, weight_multiply; - //CArrayKokkos legendre_nodes_1D(num_gauss_points); - //CArrayKokkos legendre_weights_1D(num_gauss_points); - CArray legendre_nodes_1D(num_gauss_points); - CArray legendre_weights_1D(num_gauss_points); - real_t pointer_quad_coordinate[num_dim]; - real_t pointer_quad_coordinate_weight[num_dim]; - real_t pointer_interpolated_point[num_dim]; - real_t pointer_JT_row1[num_dim]; - real_t pointer_JT_row2[num_dim]; - real_t pointer_JT_row3[num_dim]; - ViewCArray quad_coordinate(pointer_quad_coordinate,num_dim); - ViewCArray quad_coordinate_weight(pointer_quad_coordinate_weight,num_dim); - ViewCArray interpolated_point(pointer_interpolated_point,num_dim); - ViewCArray JT_row1(pointer_JT_row1,num_dim); - ViewCArray JT_row2(pointer_JT_row2,num_dim); - ViewCArray JT_row3(pointer_JT_row3,num_dim); - - real_t pointer_basis_values[elem->num_basis()]; - real_t pointer_basis_derivative_s1[elem->num_basis()]; - real_t pointer_basis_derivative_s2[elem->num_basis()]; - real_t pointer_basis_derivative_s3[elem->num_basis()]; - ViewCArray basis_values(pointer_basis_values,elem->num_basis()); - ViewCArray basis_derivative_s1(pointer_basis_derivative_s1,elem->num_basis()); - ViewCArray basis_derivative_s2(pointer_basis_derivative_s2,elem->num_basis()); - ViewCArray basis_derivative_s3(pointer_basis_derivative_s3,elem->num_basis()); - CArrayKokkos nodal_positions(elem->num_basis(),num_dim); - - //initialize weights - elements::legendre_nodes_1D(legendre_nodes_1D,num_gauss_points); - elements::legendre_weights_1D(legendre_weights_1D,num_gauss_points); - - Solver::node_ordering_convention active_node_ordering_convention = Solver_Pointer_->active_node_ordering_convention; - CArrayKokkos convert_node_order(max_nodes_per_element); - if((active_node_ordering_convention == Solver::ENSIGHT && num_dim==3)||(active_node_ordering_convention == Solver::IJK && num_dim==2)){ - convert_node_order(0) = 0; - convert_node_order(1) = 1; - convert_node_order(2) = 3; - convert_node_order(3) = 2; - if(num_dim == 3){ - convert_node_order(4) = 4; - convert_node_order(5) = 5; - convert_node_order(6) = 7; - convert_node_order(7) = 6; - } - } - else{ - convert_node_order(0) = 0; - convert_node_order(1) = 1; - convert_node_order(2) = 2; - convert_node_order(3) = 3; - if(num_dim==3){ - convert_node_order(4) = 4; - convert_node_order(5) = 5; - convert_node_order(6) = 6; - convert_node_order(7) = 7; - } - } - - - //loop over elements and use quadrature rule to compute volume from Jacobian determinant - for(int nonoverlapping_ielem = 0; nonoverlapping_ielem < nonoverlap_nelements; nonoverlapping_ielem++){ - global_element_index = element_map->getGlobalElement(nonoverlapping_ielem); - ielem = all_element_map->getLocalElement(global_element_index); - //debug print - //std::cout << "ELEMENT INDEX IS: " << ielem << " " <num_basis(); node_loop++){ - local_node_id = all_node_map->getLocalElement(nodes_in_elem(ielem, convert_node_order(node_loop))); - nodal_positions(node_loop,0) = all_node_coords(local_node_id,0); - nodal_positions(node_loop,1) = all_node_coords(local_node_id,1); - nodal_positions(node_loop,2) = all_node_coords(local_node_id,2); - /* - if(myrank==1&&nodal_positions(node_loop,2)>10000000){ - std::cout << " LOCAL MATRIX DEBUG ON TASK " << myrank << std::endl; - std::cout << node_loop+1 <<" " << local_node_id <<" "<< host_elem_conn_(ielem, node_loop) << " "<< nodal_positions(node_loop,2) << std::endl; - std::fflush(stdout); - } - */ - //std::cout << local_node_id << " " << nodes_in_elem(ielem, node_loop) << " " << nodal_positions(node_loop,0) << " " << nodal_positions(node_loop,1) << " "<< nodal_positions(node_loop,2) <getLocalNumElements(); + // local variable for host view in the dual view + const_host_vec_array all_node_coords = all_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_host_elem_conn_array nodes_in_elem = global_nodes_in_elem_distributed->getLocalView(Tpetra::Access::ReadOnly); + host_vec_array Element_Volumes = Global_Element_Volumes->getLocalView(Tpetra::Access::ReadWrite); + int nodes_per_elem = elem->num_basis(); + int z_quad, y_quad, x_quad, direct_product_count; + size_t local_node_id; + LO ielem; + GO global_element_index; + + real_t Jacobian, weight_multiply; + // CArrayKokkos legendre_nodes_1D(num_gauss_points); + // CArrayKokkos legendre_weights_1D(num_gauss_points); + CArray legendre_nodes_1D(num_gauss_points); + CArray legendre_weights_1D(num_gauss_points); + real_t pointer_quad_coordinate[num_dim]; + real_t pointer_quad_coordinate_weight[num_dim]; + real_t pointer_interpolated_point[num_dim]; + real_t pointer_JT_row1[num_dim]; + real_t pointer_JT_row2[num_dim]; + real_t pointer_JT_row3[num_dim]; + ViewCArray quad_coordinate(pointer_quad_coordinate, num_dim); + ViewCArray quad_coordinate_weight(pointer_quad_coordinate_weight, num_dim); + ViewCArray interpolated_point(pointer_interpolated_point, num_dim); + ViewCArray JT_row1(pointer_JT_row1, num_dim); + ViewCArray JT_row2(pointer_JT_row2, num_dim); + ViewCArray JT_row3(pointer_JT_row3, num_dim); + + real_t pointer_basis_values[elem->num_basis()]; + real_t pointer_basis_derivative_s1[elem->num_basis()]; + real_t pointer_basis_derivative_s2[elem->num_basis()]; + real_t pointer_basis_derivative_s3[elem->num_basis()]; + ViewCArray basis_values(pointer_basis_values, elem->num_basis()); + ViewCArray basis_derivative_s1(pointer_basis_derivative_s1, elem->num_basis()); + ViewCArray basis_derivative_s2(pointer_basis_derivative_s2, elem->num_basis()); + ViewCArray basis_derivative_s3(pointer_basis_derivative_s3, elem->num_basis()); + CArrayKokkos nodal_positions(elem->num_basis(), num_dim); + + // initialize weights + elements::legendre_nodes_1D(legendre_nodes_1D, num_gauss_points); + elements::legendre_weights_1D(legendre_weights_1D, num_gauss_points); + + Solver::node_ordering_convention active_node_ordering_convention = Solver_Pointer_->active_node_ordering_convention; + CArrayKokkos convert_node_order(max_nodes_per_element); + if ((active_node_ordering_convention == Solver::ENSIGHT && num_dim == 3) || (active_node_ordering_convention == Solver::IJK && num_dim == 2)) + { + convert_node_order(0) = 0; + convert_node_order(1) = 1; + convert_node_order(2) = 3; + convert_node_order(3) = 2; + if (num_dim == 3) + { + convert_node_order(4) = 4; + convert_node_order(5) = 5; + convert_node_order(6) = 7; + convert_node_order(7) = 6; + } } - - //initialize element volume - Element_Volumes(nonoverlapping_ielem,0) = 0; - - if(Element_Types(ielem)==elements::elem_types::Hex8){ - direct_product_count = std::pow(num_gauss_points,num_dim); + else + { + convert_node_order(0) = 0; + convert_node_order(1) = 1; + convert_node_order(2) = 2; + convert_node_order(3) = 3; + if (num_dim == 3) + { + convert_node_order(4) = 4; + convert_node_order(5) = 5; + convert_node_order(6) = 6; + convert_node_order(7) = 7; + } } - - //loop over quadrature points - for(int iquad=0; iquad < direct_product_count; iquad++){ - - //set current quadrature point - if(num_dim==3) z_quad = iquad/(num_gauss_points*num_gauss_points); - y_quad = (iquad % (num_gauss_points*num_gauss_points))/num_gauss_points; - x_quad = iquad % num_gauss_points; - quad_coordinate(0) = legendre_nodes_1D(x_quad); - quad_coordinate(1) = legendre_nodes_1D(y_quad); - if(num_dim==3) - quad_coordinate(2) = legendre_nodes_1D(z_quad); - - //set current quadrature weight - quad_coordinate_weight(0) = legendre_weights_1D(x_quad); - quad_coordinate_weight(1) = legendre_weights_1D(y_quad); - if(num_dim==3) - quad_coordinate_weight(2) = legendre_weights_1D(z_quad); - else - quad_coordinate_weight(2) = 1; - weight_multiply = quad_coordinate_weight(0)*quad_coordinate_weight(1)*quad_coordinate_weight(2); - - //compute shape functions at this point for the element type - elem->basis(basis_values,quad_coordinate); - - //compute all the necessary coordinates and derivatives at this point - - //compute shape function derivatives - elem->partial_xi_basis(basis_derivative_s1,quad_coordinate); - elem->partial_eta_basis(basis_derivative_s2,quad_coordinate); - elem->partial_mu_basis(basis_derivative_s3,quad_coordinate); - - //compute derivatives of x,y,z w.r.t the s,t,w isoparametric space needed by JT (Transpose of the Jacobian) - //derivative of x,y,z w.r.t s - JT_row1(0) = 0; - JT_row1(1) = 0; - JT_row1(2) = 0; - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - JT_row1(0) += nodal_positions(node_loop,0)*basis_derivative_s1(node_loop); - JT_row1(1) += nodal_positions(node_loop,1)*basis_derivative_s1(node_loop); - JT_row1(2) += nodal_positions(node_loop,2)*basis_derivative_s1(node_loop); - } - - //derivative of x,y,z w.r.t t - JT_row2(0) = 0; - JT_row2(1) = 0; - JT_row2(2) = 0; - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - JT_row2(0) += nodal_positions(node_loop,0)*basis_derivative_s2(node_loop); - JT_row2(1) += nodal_positions(node_loop,1)*basis_derivative_s2(node_loop); - JT_row2(2) += nodal_positions(node_loop,2)*basis_derivative_s2(node_loop); - } - - //derivative of x,y,z w.r.t w - JT_row3(0) = 0; - JT_row3(1) = 0; - JT_row3(2) = 0; - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - JT_row3(0) += nodal_positions(node_loop,0)*basis_derivative_s3(node_loop); - JT_row3(1) += nodal_positions(node_loop,1)*basis_derivative_s3(node_loop); - JT_row3(2) += nodal_positions(node_loop,2)*basis_derivative_s3(node_loop); - //debug print - /*if(myrank==1&&nodal_positions(node_loop,2)*basis_derivative_s3(node_loop)<-10000000){ - std::cout << " ELEMENT VOLUME JACOBIAN DEBUG ON TASK " << myrank << std::endl; - std::cout << node_loop+1 << " " << JT_row3(2) << " "<< nodal_positions(node_loop,2) <<" "<< basis_derivative_s3(node_loop) << std::endl; - std::fflush(stdout); - }*/ - } - - - //compute the determinant of the Jacobian - Jacobian = JT_row1(0)*(JT_row2(1)*JT_row3(2)-JT_row3(1)*JT_row2(2))- - JT_row1(1)*(JT_row2(0)*JT_row3(2)-JT_row3(0)*JT_row2(2))+ - JT_row1(2)*(JT_row2(0)*JT_row3(1)-JT_row3(0)*JT_row2(1)); - if(Jacobian<0) Jacobian = -Jacobian; - - Element_Volumes(nonoverlapping_ielem,0) += weight_multiply*Jacobian; + + // loop over elements and use quadrature rule to compute volume from Jacobian determinant + for (int nonoverlapping_ielem = 0; nonoverlapping_ielem < nonoverlap_nelements; nonoverlapping_ielem++) + { + global_element_index = element_map->getGlobalElement(nonoverlapping_ielem); + ielem = all_element_map->getLocalElement(global_element_index); + // debug print + // std::cout << "ELEMENT INDEX IS: " << ielem << " " <num_basis(); node_loop++) + { + local_node_id = all_node_map->getLocalElement(nodes_in_elem(ielem, convert_node_order(node_loop))); + nodal_positions(node_loop, 0) = all_node_coords(local_node_id, 0); + nodal_positions(node_loop, 1) = all_node_coords(local_node_id, 1); + nodal_positions(node_loop, 2) = all_node_coords(local_node_id, 2); + /* + if(myrank==1&&nodal_positions(node_loop,2)>10000000){ + std::cout << " LOCAL MATRIX DEBUG ON TASK " << myrank << std::endl; + std::cout << node_loop+1 <<" " << local_node_id <<" "<< host_elem_conn_(ielem, node_loop) << " "<< nodal_positions(node_loop,2) << std::endl; + std::fflush(stdout); + } + */ + // std::cout << local_node_id << " " << nodes_in_elem(ielem, node_loop) << " " << nodal_positions(node_loop,0) << " " << nodal_positions(node_loop,1) << " "<< nodal_positions(node_loop,2) <basis(basis_values, quad_coordinate); + + // compute all the necessary coordinates and derivatives at this point + + // compute shape function derivatives + elem->partial_xi_basis(basis_derivative_s1, quad_coordinate); + elem->partial_eta_basis(basis_derivative_s2, quad_coordinate); + elem->partial_mu_basis(basis_derivative_s3, quad_coordinate); + + // compute derivatives of x,y,z w.r.t the s,t,w isoparametric space needed by JT (Transpose of the Jacobian) + // derivative of x,y,z w.r.t s + JT_row1(0) = 0; + JT_row1(1) = 0; + JT_row1(2) = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row1(0) += nodal_positions(node_loop, 0) * basis_derivative_s1(node_loop); + JT_row1(1) += nodal_positions(node_loop, 1) * basis_derivative_s1(node_loop); + JT_row1(2) += nodal_positions(node_loop, 2) * basis_derivative_s1(node_loop); + } + + // derivative of x,y,z w.r.t t + JT_row2(0) = 0; + JT_row2(1) = 0; + JT_row2(2) = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row2(0) += nodal_positions(node_loop, 0) * basis_derivative_s2(node_loop); + JT_row2(1) += nodal_positions(node_loop, 1) * basis_derivative_s2(node_loop); + JT_row2(2) += nodal_positions(node_loop, 2) * basis_derivative_s2(node_loop); + } + + // derivative of x,y,z w.r.t w + JT_row3(0) = 0; + JT_row3(1) = 0; + JT_row3(2) = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row3(0) += nodal_positions(node_loop, 0) * basis_derivative_s3(node_loop); + JT_row3(1) += nodal_positions(node_loop, 1) * basis_derivative_s3(node_loop); + JT_row3(2) += nodal_positions(node_loop, 2) * basis_derivative_s3(node_loop); + // debug print + /*if(myrank==1&&nodal_positions(node_loop,2)*basis_derivative_s3(node_loop)<-10000000){ + std::cout << " ELEMENT VOLUME JACOBIAN DEBUG ON TASK " << myrank << std::endl; + std::cout << node_loop+1 << " " << JT_row3(2) << " "<< nodal_positions(node_loop,2) <<" "<< basis_derivative_s3(node_loop) << std::endl; + std::fflush(stdout); + }*/ + } + + // compute the determinant of the Jacobian + Jacobian = JT_row1(0) * (JT_row2(1) * JT_row3(2) - JT_row3(1) * JT_row2(2)) - + JT_row1(1) * (JT_row2(0) * JT_row3(2) - JT_row3(0) * JT_row2(2)) + + JT_row1(2) * (JT_row2(0) * JT_row3(1) - JT_row3(0) * JT_row2(1)); + if (Jacobian < 0) + { + Jacobian = -Jacobian; + } + + Element_Volumes(nonoverlapping_ielem, 0) += weight_multiply * Jacobian; + } } - } - - std::ostream &out = std::cout; - Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - //if(myrank==0) - //*fos << "Global Element Volumes:" << std::endl; - //Global_Element_Volumes->describe(*fos,Teuchos::VERB_EXTREME); - //*fos << std::endl; + + std::ostream& out = std::cout; + Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + // if(myrank==0) + // *fos << "Global Element Volumes:" << std::endl; + // Global_Element_Volumes->describe(*fos,Teuchos::VERB_EXTREME); + // *fos << std::endl; } /* ------------------------------------------------------------------------------------------- Communicate ghosts using the current optimization design data ---------------------------------------------------------------------------------------------- */ -void FEA_Module_Inertial::comm_variables(Teuchos::RCP zp){ - comm_densities(zp); +void FEA_Module_Inertial::comm_variables(Teuchos::RCP zp) +{ + comm_densities(zp); } diff --git a/src/Parallel-Solvers/FEA_Module_Inertial.h b/src/Parallel-Solvers/FEA_Module_Inertial.h index 395ef7c33..265adf32b 100644 --- a/src/Parallel-Solvers/FEA_Module_Inertial.h +++ b/src/Parallel-Solvers/FEA_Module_Inertial.h @@ -11,14 +11,14 @@ This program is open source under the BSD-3 License. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - + 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - + 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - + 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. @@ -42,58 +42,57 @@ class Inertial_Parameters; -class FEA_Module_Inertial: public FEA_Module{ - +class FEA_Module_Inertial : public FEA_Module +{ public: - FEA_Module_Inertial(Inertial_Parameters& params, Solver *Solver_Pointer, const int my_fea_module_index = 0); - ~FEA_Module_Inertial(); + FEA_Module_Inertial(Inertial_Parameters& params, Solver* Solver_Pointer, const int my_fea_module_index = 0); + ~FEA_Module_Inertial(); - void comm_variables(Teuchos::RCP zp); + void comm_variables(Teuchos::RCP zp); - void compute_element_volumes(); + void compute_element_volumes(); - void compute_element_masses(const_host_vec_array design_densities, bool max_flag, bool use_initial_coords = false); + void compute_element_masses(const_host_vec_array design_densities, bool max_flag, bool use_initial_coords = false); - void compute_element_moments(const_host_vec_array design_densities, bool max_flag, int moment_component, bool use_initial_coords = false); + void compute_element_moments(const_host_vec_array design_densities, bool max_flag, int moment_component, bool use_initial_coords = false); - void compute_element_moments_of_inertia(const_host_vec_array design_densities, bool max_flag, int inertia_component, bool use_initial_coords = false); + void compute_element_moments_of_inertia(const_host_vec_array design_densities, bool max_flag, int inertia_component, bool use_initial_coords = false); - void compute_nodal_gradients(const_host_vec_array design_densities, host_vec_array gradients, bool use_initial_coords = false); + void compute_nodal_gradients(const_host_vec_array design_densities, host_vec_array gradients, bool use_initial_coords = false); - void compute_moment_gradients(const_host_vec_array design_densities, host_vec_array gradients, int moment_component, bool use_initial_coords = false); + void compute_moment_gradients(const_host_vec_array design_densities, host_vec_array gradients, int moment_component, bool use_initial_coords = false); - void compute_moment_of_inertia_gradients(const_host_vec_array design_densities, host_vec_array gradients, int intertia_component, bool use_initial_coords = false); - - //forward declare - Inertial_Parameters *module_params; + void compute_moment_of_inertia_gradients(const_host_vec_array design_densities, host_vec_array gradients, int intertia_component, bool use_initial_coords = false); - //Global FEA data - Teuchos::RCP mass_gradients_distributed; - Teuchos::RCP center_of_mass_gradients_distributed; - Teuchos::RCP Global_Element_Volumes; - Teuchos::RCP Global_Element_Masses; - Teuchos::RCP Global_Element_Moments_x; - Teuchos::RCP Global_Element_Moments_y; - Teuchos::RCP Global_Element_Moments_z; - Teuchos::RCP Global_Element_Moments_of_Inertia_xx; - Teuchos::RCP Global_Element_Moments_of_Inertia_yy; - Teuchos::RCP Global_Element_Moments_of_Inertia_zz; - Teuchos::RCP Global_Element_Moments_of_Inertia_xy; - Teuchos::RCP Global_Element_Moments_of_Inertia_xz; - Teuchos::RCP Global_Element_Moments_of_Inertia_yz; + // forward declare + Inertial_Parameters* module_params; - //inertial properties - real_t mass, center_of_mass[3], moments_of_inertia[6]; + // Global FEA data + Teuchos::RCP mass_gradients_distributed; + Teuchos::RCP center_of_mass_gradients_distributed; + Teuchos::RCP Global_Element_Volumes; + Teuchos::RCP Global_Element_Masses; + Teuchos::RCP Global_Element_Moments_x; + Teuchos::RCP Global_Element_Moments_y; + Teuchos::RCP Global_Element_Moments_z; + Teuchos::RCP Global_Element_Moments_of_Inertia_xx; + Teuchos::RCP Global_Element_Moments_of_Inertia_yy; + Teuchos::RCP Global_Element_Moments_of_Inertia_zz; + Teuchos::RCP Global_Element_Moments_of_Inertia_xy; + Teuchos::RCP Global_Element_Moments_of_Inertia_xz; + Teuchos::RCP Global_Element_Moments_of_Inertia_yz; - bool use_initial_density; //if density variable is from initial configuration then jacobian is not needed + // inertial properties + real_t mass, center_of_mass[3], moments_of_inertia[6]; - //runtime flags - bool mass_init, com_init[3]; + bool use_initial_density; // if density variable is from initial configuration then jacobian is not needed - //update counters (first attempt at reducing redundant calls through ROL for Moments of Inertia and Center of Mass) - int mass_update, com_update[3]; - int mass_gradient_update, com_gradient_update[3]; + // runtime flags + bool mass_init, com_init[3]; + // update counters (first attempt at reducing redundant calls through ROL for Moments of Inertia and Center of Mass) + int mass_update, com_update[3]; + int mass_gradient_update, com_gradient_update[3]; }; #endif // end HEADER_H diff --git a/src/Parallel-Solvers/Parallel-Explicit/CMakeLists.txt b/src/Parallel-Solvers/Parallel-Explicit/CMakeLists.txt index c0dff6bb9..cbc46a920 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/CMakeLists.txt +++ b/src/Parallel-Solvers/Parallel-Explicit/CMakeLists.txt @@ -4,7 +4,7 @@ set(CMAKE_CXX_FLAGS "${Trilinos_CXX_COMPILER_FLAGS} ${CMAKE_CXX_FLAGS}") set(CMAKE_C_FLAGS "${Trilinos_C_COMPILER_FLAGS} ${CMAKE_C_FLAGS}") set(CMAKE_Fortran_FLAGS "${Trilinos_Fortran_COMPILER_FLAGS} ${CMAKE_Fortran_FLAGS}") -include_directories(SGH_Solver) +include_directories(SGH_Solver/include) include_directories(Dynamic_Elastic_Solver) #include_directories(Eulerian_Solver) include_directories(Topology_Optimization) diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.cpp b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.cpp index e09c67741..652d7e797 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.cpp @@ -11,14 +11,14 @@ This program is open source under the BSD-3 License. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - + 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - + 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - + 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. @@ -40,7 +40,7 @@ #include #include #include -#include +#include #include // fmin, fmax, abs note: fminl is long #include #include @@ -67,7 +67,7 @@ #include "Simulation_Parameters/FEA_Module/Dynamic_Elasticity_Parameters.h" #include "Simulation_Parameters/Simulation_Parameters_Explicit.h" -//optimization +// optimization #include "ROL_Solver.hpp" #include "Kinetic_Energy_Minimize.h" @@ -79,141 +79,143 @@ using namespace utils; - FEA_Module_Dynamic_Elasticity::FEA_Module_Dynamic_Elasticity( - Dynamic_Elasticity_Parameters& params, Solver *Solver_Pointer, - std::shared_ptr mesh_in, const int my_fea_module_index) - : FEA_Module(Solver_Pointer) { - - //assign interfacing index - my_fea_module_index_ = my_fea_module_index; - Module_Type = FEA_MODULE_TYPE::Dynamic_Elasticity; - - //recast solver pointer for non-base class access - Explicit_Solver_Pointer_ = dynamic_cast(Solver_Pointer); - module_params = ¶ms; - simparam = &(Explicit_Solver_Pointer_->simparam); - - mesh = mesh_in; - - //boundary condition data - max_boundary_sets = 0; - Local_Index_Boundary_Patches = Explicit_Solver_Pointer_->Local_Index_Boundary_Patches; - - //set Tpetra vector pointers - initial_node_velocities_distributed = Explicit_Solver_Pointer_->initial_node_velocities_distributed; - initial_node_coords_distributed = Explicit_Solver_Pointer_->initial_node_coords_distributed; - all_initial_node_coords_distributed = Explicit_Solver_Pointer_->all_initial_node_coords_distributed; - node_coords_distributed = Explicit_Solver_Pointer_->node_coords_distributed; - node_velocities_distributed = Explicit_Solver_Pointer_->node_velocities_distributed; - all_node_velocities_distributed = Explicit_Solver_Pointer_->all_node_velocities_distributed; - - if(simparam->topology_optimization_on||simparam->shape_optimization_on){ - all_cached_node_velocities_distributed = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - force_gradient_velocity = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - force_gradient_position = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - force_gradient_design = Teuchos::rcp(new MV(all_node_map, 1)); - corner_value_storage = Solver_Pointer->corner_value_storage; - corner_vector_storage = Solver_Pointer->corner_vector_storage; - relative_element_densities = DCArrayKokkos(rnum_elem, "relative_element_densities"); - } - - if(simparam->topology_optimization_on||simparam->shape_optimization_on||simparam->num_dims==2){ - node_masses_distributed = Teuchos::rcp(new MV(map, 1)); - ghost_node_masses_distributed = Teuchos::rcp(new MV(ghost_node_map, 1)); - adjoint_vector_distributed = Teuchos::rcp(new MV(map, simparam->num_dims)); - phi_adjoint_vector_distributed = Teuchos::rcp(new MV(map, simparam->num_dims)); - } - - //setup output - noutput = 0; - init_output(); - - //optimization flags - kinetic_energy_objective = false; - - - //set parameters - Dynamic_Options dynamic_options = simparam->dynamic_options; - time_value = dynamic_options.time_value; - time_final = dynamic_options.time_final; - dt_max = dynamic_options.dt_max; - dt_min = dynamic_options.dt_min; - dt_cfl = dynamic_options.dt_cfl; - graphics_time = simparam->output_options.graphics_time; - graphics_dt_ival = simparam->output_options.graphics_dt_ival; - graphics_cyc_ival = simparam->output_options.graphics_cyc_ival; - cycle_stop = dynamic_options.cycle_stop; - rk_num_stages = dynamic_options.rk_num_stages; - dt = dynamic_options.dt; - fuzz = dynamic_options.fuzz; - tiny = dynamic_options.tiny; - small = dynamic_options.small; - graphics_times = simparam->output_options.graphics_times; - graphics_id = simparam->output_options.graphics_id; - rk_num_bins = simparam->dynamic_options.rk_num_bins; - - if(simparam->topology_optimization_on){ - max_time_steps = BUFFER_GROW; - forward_solve_velocity_data = Teuchos::rcp(new std::vector>(max_time_steps+1)); - time_data.resize(max_time_steps+1); - forward_solve_coordinate_data = Teuchos::rcp(new std::vector>(max_time_steps+1)); - adjoint_vector_data = Teuchos::rcp(new std::vector>(max_time_steps+1)); - phi_adjoint_vector_data = Teuchos::rcp(new std::vector>(max_time_steps+1)); - //assign a multivector of corresponding size to each new timestep in the buffer - for(int istep = 0; istep < max_time_steps+1; istep++){ - (*forward_solve_velocity_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - (*forward_solve_coordinate_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - (*adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - (*phi_adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + Dynamic_Elasticity_Parameters& params, Solver* Solver_Pointer, + std::shared_ptr mesh_in, const int my_fea_module_index) + : FEA_Module(Solver_Pointer) +{ + // assign interfacing index + my_fea_module_index_ = my_fea_module_index; + Module_Type = FEA_MODULE_TYPE::Dynamic_Elasticity; + + // recast solver pointer for non-base class access + Explicit_Solver_Pointer_ = dynamic_cast(Solver_Pointer); + module_params = ¶ms; + simparam = &(Explicit_Solver_Pointer_->simparam); + + mesh = mesh_in; + + // boundary condition data + max_boundary_sets = 0; + Local_Index_Boundary_Patches = Explicit_Solver_Pointer_->Local_Index_Boundary_Patches; + + // set Tpetra vector pointers + initial_node_velocities_distributed = Explicit_Solver_Pointer_->initial_node_velocities_distributed; + initial_node_coords_distributed = Explicit_Solver_Pointer_->initial_node_coords_distributed; + all_initial_node_coords_distributed = Explicit_Solver_Pointer_->all_initial_node_coords_distributed; + node_coords_distributed = Explicit_Solver_Pointer_->node_coords_distributed; + node_velocities_distributed = Explicit_Solver_Pointer_->node_velocities_distributed; + all_node_velocities_distributed = Explicit_Solver_Pointer_->all_node_velocities_distributed; + + if (simparam->topology_optimization_on || simparam->shape_optimization_on) + { + all_cached_node_velocities_distributed = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + force_gradient_velocity = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + force_gradient_position = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + force_gradient_design = Teuchos::rcp(new MV(all_node_map, 1)); + corner_value_storage = Solver_Pointer->corner_value_storage; + corner_vector_storage = Solver_Pointer->corner_vector_storage; + relative_element_densities = DCArrayKokkos(rnum_elem, "relative_element_densities"); + } + + if (simparam->topology_optimization_on || simparam->shape_optimization_on || simparam->num_dims == 2) + { + node_masses_distributed = Teuchos::rcp(new MV(map, 1)); + ghost_node_masses_distributed = Teuchos::rcp(new MV(ghost_node_map, 1)); + adjoint_vector_distributed = Teuchos::rcp(new MV(map, simparam->num_dims)); + phi_adjoint_vector_distributed = Teuchos::rcp(new MV(map, simparam->num_dims)); + } + + // setup output + noutput = 0; + init_output(); + + // optimization flags + kinetic_energy_objective = false; + + // set parameters + Dynamic_Options dynamic_options = simparam->dynamic_options; + time_value = dynamic_options.time_value; + time_final = dynamic_options.time_final; + dt_max = dynamic_options.dt_max; + dt_min = dynamic_options.dt_min; + dt_cfl = dynamic_options.dt_cfl; + graphics_time = simparam->output_options.graphics_time; + graphics_dt_ival = simparam->output_options.graphics_dt_ival; + graphics_cyc_ival = simparam->output_options.graphics_cyc_ival; + cycle_stop = dynamic_options.cycle_stop; + rk_num_stages = dynamic_options.rk_num_stages; + dt = dynamic_options.dt; + fuzz = dynamic_options.fuzz; + tiny = dynamic_options.tiny; + small = dynamic_options.small; + graphics_times = simparam->output_options.graphics_times; + graphics_id = simparam->output_options.graphics_id; + rk_num_bins = simparam->dynamic_options.rk_num_bins; + + if (simparam->topology_optimization_on) + { + max_time_steps = BUFFER_GROW; + forward_solve_velocity_data = Teuchos::rcp(new std::vector>(max_time_steps + 1)); + time_data.resize(max_time_steps + 1); + forward_solve_coordinate_data = Teuchos::rcp(new std::vector>(max_time_steps + 1)); + adjoint_vector_data = Teuchos::rcp(new std::vector>(max_time_steps + 1)); + phi_adjoint_vector_data = Teuchos::rcp(new std::vector>(max_time_steps + 1)); + // assign a multivector of corresponding size to each new timestep in the buffer + for (int istep = 0; istep < max_time_steps + 1; istep++) + { + (*forward_solve_velocity_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*forward_solve_coordinate_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*phi_adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + } } - - } } -FEA_Module_Dynamic_Elasticity::~FEA_Module_Dynamic_Elasticity(){ - //delete simparam; +FEA_Module_Dynamic_Elasticity::~FEA_Module_Dynamic_Elasticity() +{ + // delete simparam; } /* ---------------------------------------------------------------------- Read ANSYS dat format mesh file ------------------------------------------------------------------------- */ -void FEA_Module_Dynamic_Elasticity::read_conditions_ansys_dat(std::ifstream *in, std::streampos before_condition_header){ - - char ch; - int num_dim = simparam->num_dims; - int buffer_lines = 1000; - int max_word = 30; - auto input_options = simparam->input_options.value(); - int p_order = input_options.p_order; - real_t unit_scaling = input_options.unit_scaling; - int local_node_index, current_column_index; - size_t strain_count; - std::string skip_line, read_line, substring, token; - std::stringstream line_parse, line_parse2; - CArrayKokkos read_buffer; - CArrayKokkos read_buffer_indices; - int buffer_loop, buffer_iteration, buffer_iterations, scan_loop, nodes_per_element, words_per_line; - size_t read_index_start, node_rid, elem_gid; - LO local_dof_id; - GO node_gid; - real_t dof_value; - host_vec_array node_densities; - +void FEA_Module_Dynamic_Elasticity::read_conditions_ansys_dat(std::ifstream* in, std::streampos before_condition_header) +{ + char ch; + int num_dim = simparam->num_dims; + int buffer_lines = 1000; + int max_word = 30; + auto input_options = simparam->input_options.value(); + int p_order = input_options.p_order; + real_t unit_scaling = input_options.unit_scaling; + int local_node_index, current_column_index; + size_t strain_count; + std::string skip_line, read_line, substring, token; + std::stringstream line_parse, line_parse2; + CArrayKokkos read_buffer; + CArrayKokkos read_buffer_indices; + int buffer_loop, buffer_iteration, buffer_iterations, scan_loop, nodes_per_element, words_per_line; + size_t read_index_start, node_rid, elem_gid; + LO local_dof_id; + GO node_gid; + real_t dof_value; + host_vec_array node_densities; } // end read_conditions_ansys_dat // ----------------------------------------------------------------------------- // Interfaces read in data with the elastic solver data; currently a hack to streamline -//------------------------------------------------------------------------------ -void FEA_Module_Dynamic_Elasticity::elastic_interface_setup(node_t &node, - elem_t &elem, - corner_t &corner){ - - const size_t num_dim = simparam->num_dims; +// ------------------------------------------------------------------------------ +void FEA_Module_Dynamic_Elasticity::elastic_interface_setup(node_t& node, + elem_t& elem, + corner_t& corner) +{ + const size_t num_dim = simparam->num_dims; const size_t rk_num_bins = simparam->dynamic_options.rk_num_bins; num_nodes_in_elem = 1; - for (int dim=0; dimnall_nodes; int myrank = Explicit_Solver_Pointer_->myrank; int nranks = Explicit_Solver_Pointer_->nranks; - //printf("Num nodes assigned to MPI rank %lu is %lu\n" , myrank, nall_nodes); + // printf("Num nodes assigned to MPI rank %lu is %lu\n" , myrank, nall_nodes); // intialize node variables mesh->initialize_nodes(nall_nodes); mesh->initialize_local_nodes(Explicit_Solver_Pointer_->nlocal_nodes); node.initialize(rk_num_bins, nall_nodes, num_dim); - //std::cout << "Bin counts " << rk_num_bins << " Node counts " << nall_nodes << " Num dim " << num_dim << std::endl; + // std::cout << "Bin counts " << rk_num_bins << " Node counts " << nall_nodes << " Num dim " << num_dim << std::endl; - //view scope + // view scope { - host_vec_array interface_node_coords = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); - //save node data to node.coords - //std::cout << "NODE DATA ON RANK " << myrank << std::endl; - if(num_dim==2){ - for(int inode = 0; inode < nall_nodes; inode++){ - //std::cout << "Node index " << inode+1 << " "; - node.coords.host(0,inode,0) = interface_node_coords(inode,0); - //std::cout << host_node_coords_state(0,inode,0)+1<< " "; - node.coords.host(0,inode,1) = interface_node_coords(inode,1); - //std::cout << host_node_coords_state(0,inode,1)+1<< " "; + host_vec_array interface_node_coords = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + // save node data to node.coords + // std::cout << "NODE DATA ON RANK " << myrank << std::endl; + if (num_dim == 2) + { + for (int inode = 0; inode < nall_nodes; inode++) + { + // std::cout << "Node index " << inode+1 << " "; + node.coords.host(0, inode, 0) = interface_node_coords(inode, 0); + // std::cout << host_node_coords_state(0,inode,0)+1<< " "; + node.coords.host(0, inode, 1) = interface_node_coords(inode, 1); + // std::cout << host_node_coords_state(0,inode,1)+1<< " "; + } } - } - else if(num_dim==3){ - for(int inode = 0; inode < nall_nodes; inode++){ - //std::cout << "Node index " << inode+1 << " "; - node.coords.host(0,inode,0) = interface_node_coords(inode,0); - //std::cout << host_node_coords_state(0,inode,0)+1<< " "; - node.coords.host(0,inode,1) = interface_node_coords(inode,1); - //std::cout << host_node_coords_state(0,inode,1)+1<< " "; - - node.coords.host(0,inode,2) = interface_node_coords(inode,2); - //std::cout << host_node_coords_state(0,inode,2)+1<< std::endl; + else if (num_dim == 3) + { + for (int inode = 0; inode < nall_nodes; inode++) + { + // std::cout << "Node index " << inode+1 << " "; + node.coords.host(0, inode, 0) = interface_node_coords(inode, 0); + // std::cout << host_node_coords_state(0,inode,0)+1<< " "; + node.coords.host(0, inode, 1) = interface_node_coords(inode, 1); + // std::cout << host_node_coords_state(0,inode,1)+1<< " "; + + node.coords.host(0, inode, 2) = interface_node_coords(inode, 2); + // std::cout << host_node_coords_state(0,inode,2)+1<< std::endl; + } } - } - } //end view scope - // --- read in the elements in the mesh --- - + } // end view scope + // --- read in the elements in the mesh --- + rnum_elem = Explicit_Solver_Pointer_->rnum_elem; - //printf("Num elems assigned to MPI rank %lu is %lu\n" , myrank, rnum_elem); + // printf("Num elems assigned to MPI rank %lu is %lu\n" , myrank, rnum_elem); // intialize elem variables mesh->initialize_elems(rnum_elem, num_dim); elem.initialize(rk_num_bins, nall_nodes, 3); // always 3D here, even for 2D nodes_in_elem = mesh->nodes_in_elem; - //save data to nodes_in_elem.host - //CArrayKokkos host_mesh_nodes_in_elem(rnum_elem, num_nodes_in_elem); - //view scope + // save data to nodes_in_elem.host + // CArrayKokkos host_mesh_nodes_in_elem(rnum_elem, num_nodes_in_elem); + // view scope { - host_elem_conn_array interface_nodes_in_elem = Explicit_Solver_Pointer_->global_nodes_in_elem_distributed->getLocalView (Tpetra::Access::ReadWrite); - //save node data to node.coords - //std::cout << "ELEMENT CONNECTIVITY ON RANK " << myrank << std::endl; - for(int ielem = 0; ielem < rnum_elem; ielem++){ - //std::cout << "Element index " << ielem+1 << " "; - for(int inode = 0; inode < num_nodes_in_elem; inode++){ - nodes_in_elem.host(ielem,inode) = Explicit_Solver_Pointer_->all_node_map->getLocalElement(interface_nodes_in_elem(ielem,inode)); - //debug print - //std::cout << nodes_in_elem.get_kokkos_dual_view().h_view(ielem*num_nodes_in_elem + inode)+1<< " "; + host_elem_conn_array interface_nodes_in_elem = Explicit_Solver_Pointer_->global_nodes_in_elem_distributed->getLocalView(Tpetra::Access::ReadWrite); + // save node data to node.coords + // std::cout << "ELEMENT CONNECTIVITY ON RANK " << myrank << std::endl; + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + // std::cout << "Element index " << ielem+1 << " "; + for (int inode = 0; inode < num_nodes_in_elem; inode++) + { + nodes_in_elem.host(ielem, inode) = Explicit_Solver_Pointer_->all_node_map->getLocalElement(interface_nodes_in_elem(ielem, inode)); + // debug print + // std::cout << nodes_in_elem.get_kokkos_dual_view().h_view(ielem*num_nodes_in_elem + inode)+1<< " "; + } + // std::cout << std::endl; } - //std::cout << std::endl; - } } // update device side nodes_in_elem.update_device(); - //debug print - - //CArrayKokkos device_mesh_nodes_in_elem(rnum_elem, num_nodes_in_elem); - //device_mesh_nodes_in_elem.get_kokkos_view() = nodes_in_elem.get_kokkos_dual_view().d_view; - //host_mesh_nodes_in_elem.get_kokkos_view() = nodes_in_elem.get_kokkos_dual_view().view_host(); + // debug print + + // CArrayKokkos device_mesh_nodes_in_elem(rnum_elem, num_nodes_in_elem); + // device_mesh_nodes_in_elem.get_kokkos_view() = nodes_in_elem.get_kokkos_dual_view().d_view; + // host_mesh_nodes_in_elem.get_kokkos_view() = nodes_in_elem.get_kokkos_dual_view().view_host(); /* if(myrank==1){ std::cout << "ELEMENT CONNECTIVITY ON RANK 1 in LOCAL INDICES" << myrank << std::endl; @@ -335,7 +343,7 @@ void FEA_Module_Dynamic_Elasticity::elastic_interface_setup(node_t &node, //host_node_coords_state = CArrayKokkos(rk_num_bins, nall_nodes, num_dim); //host_all_node_coords_state.get_kokkos_view() = Kokkos::View("debug", rk_num_bins*nall_nodes*num_dim); //save node data to node.coords - + //std::cout << "ALL NODE DATA ON RANK " << myrank << std::endl; for(int inode = 0; inode < nall_nodes; inode++){ //std::cout << "Node index " << inode+1 << " "; @@ -349,38 +357,38 @@ void FEA_Module_Dynamic_Elasticity::elastic_interface_setup(node_t &node, */ // save the node coords to the current RK value - for (size_t node_gid=0; node_gidinitialize_corners(num_corners); corner.initialize(num_corners, num_dim); - + /* for(int inode = 0; inode < nall_nodes; inode++){ std::cout << "Node index " << inode+1 << " "; @@ -393,538 +401,578 @@ void FEA_Module_Dynamic_Elasticity::elastic_interface_setup(node_t &node, } */ // Close mesh input file - //fclose(in); + // fclose(in); return; - } /* ---------------------------------------------------------------------------- Initialize sets of element boundary surfaces and arrays for input conditions ------------------------------------------------------------------------------- */ -void FEA_Module_Dynamic_Elasticity::init_boundaries() { - max_boundary_sets = module_params->boundary_conditions.size(); - int num_dim = simparam->num_dims; - - // set the number of boundary sets - if(myrank == 0) - std::cout << "building boundary sets " << std::endl; - - //initialize to 1 since there must be at least 1 boundary set anyway; read in may occure later - if(max_boundary_sets==0) max_boundary_sets = 1; - //std::cout << "NUM BOUNDARY CONDITIONS ON RANK " << myrank << " FOR INIT " << num_boundary_conditions <(nall_nodes*num_dim, "Node_DOF_Boundary_Condition_Type"); - - //initialize - for(int init=0; init < nall_nodes*num_dim; init++) - Node_DOF_Boundary_Condition_Type(init) = NONE; - - Number_DOF_BCS = 0; +void FEA_Module_Dynamic_Elasticity::init_boundaries() +{ + max_boundary_sets = module_params->boundary_conditions.size(); + int num_dim = simparam->num_dims; + + // set the number of boundary sets + if (myrank == 0) + { + std::cout << "building boundary sets " << std::endl; + } + + // initialize to 1 since there must be at least 1 boundary set anyway; read in may occure later + if (max_boundary_sets == 0) + { + max_boundary_sets = 1; + } + // std::cout << "NUM BOUNDARY CONDITIONS ON RANK " << myrank << " FOR INIT " << num_boundary_conditions <(nall_nodes * num_dim, "Node_DOF_Boundary_Condition_Type"); + + // initialize + for (int init = 0; init < nall_nodes * num_dim; init++) + { + Node_DOF_Boundary_Condition_Type(init) = NONE; + } + + Number_DOF_BCS = 0; } /* ---------------------------------------------------------------------- initialize storage for element boundary surfaces corresponding to user BCs ------------------------------------------------------------------------- */ -void FEA_Module_Dynamic_Elasticity::init_boundary_sets (int num_sets){ +void FEA_Module_Dynamic_Elasticity::init_boundary_sets(int num_sets) +{ + if (num_sets == 0) + { + std::cout << " Warning: number of boundary conditions = 0"; + return; + } + // initialize maximum + max_boundary_sets = num_sets; + // std::cout << " DEBUG PRINT "<(num_sets, "Boundary_Condition_Type_List"); + NBoundary_Condition_Patches = CArrayKokkos(num_sets, "NBoundary_Condition_Patches"); + // std::cout << "NBOUNDARY PATCHES ON RANK " << myrank << " FOR INIT IS " << nboundary_patches <(num_sets, nboundary_patches, "Boundary_Condition_Patches"); + + // initialize data + for (int iset = 0; iset < num_sets; iset++) + { + NBoundary_Condition_Patches(iset) = 0; + } - if(num_sets == 0){ - std::cout << " Warning: number of boundary conditions = 0"; - return; - } - //initialize maximum - max_boundary_sets = num_sets; - //std::cout << " DEBUG PRINT "<(num_sets, "Boundary_Condition_Type_List"); - NBoundary_Condition_Patches = CArrayKokkos(num_sets, "NBoundary_Condition_Patches"); - //std::cout << "NBOUNDARY PATCHES ON RANK " << myrank << " FOR INIT IS " << nboundary_patches <(num_sets, nboundary_patches, "Boundary_Condition_Patches"); - - //initialize data - for(int iset = 0; iset < num_sets; iset++) NBoundary_Condition_Patches(iset) = 0; - - //initialize - for(int ibdy=0; ibdy < num_sets; ibdy++) Boundary_Condition_Type_List(ibdy) = NONE; + // initialize + for (int ibdy = 0; ibdy < num_sets; ibdy++) + { + Boundary_Condition_Type_List(ibdy) = NONE; + } } /* ---------------------------------------------------------------------------- Grow boundary conditions sets of element boundary surfaces ------------------------------------------------------------------------------- */ -void FEA_Module_Dynamic_Elasticity::grow_boundary_sets(int num_sets){ - int num_dim = simparam->num_dims; +void FEA_Module_Dynamic_Elasticity::grow_boundary_sets(int num_sets) +{ + int num_dim = simparam->num_dims; - if(num_sets == 0){ - std::cout << " Warning: number of boundary conditions being set to 0"; - return; - } - - //std::cout << " DEBUG PRINT "<max_boundary_sets){ - //temporary storage for previous data - CArrayKokkos Temp_Boundary_Condition_Type_List = Boundary_Condition_Type_List; - CArrayKokkos Temp_NBoundary_Condition_Patches = NBoundary_Condition_Patches; - CArrayKokkos Temp_Boundary_Condition_Patches = Boundary_Condition_Patches; - - max_boundary_sets = num_sets + 5; //5 is an arbitrary buffer - Boundary_Condition_Type_List = CArrayKokkos(max_boundary_sets, "Boundary_Condition_Type_List"); - NBoundary_Condition_Patches = CArrayKokkos(max_boundary_sets, "NBoundary_Condition_Patches"); - //std::cout << "NBOUNDARY PATCHES ON RANK " << myrank << " FOR GROW " << nboundary_patches <(max_boundary_sets, nboundary_patches, "Boundary_Condition_Patches"); - - //copy previous data back over - //std::cout << "NUM BOUNDARY CONDITIONS ON RANK " << myrank << " FOR COPY " << max_boundary_sets < max_boundary_sets) + { + // temporary storage for previous data + CArrayKokkos Temp_Boundary_Condition_Type_List = Boundary_Condition_Type_List; + CArrayKokkos Temp_NBoundary_Condition_Patches = NBoundary_Condition_Patches; + CArrayKokkos Temp_Boundary_Condition_Patches = Boundary_Condition_Patches; + + max_boundary_sets = num_sets + 5; // 5 is an arbitrary buffer + Boundary_Condition_Type_List = CArrayKokkos(max_boundary_sets, "Boundary_Condition_Type_List"); + NBoundary_Condition_Patches = CArrayKokkos(max_boundary_sets, "NBoundary_Condition_Patches"); + // std::cout << "NBOUNDARY PATCHES ON RANK " << myrank << " FOR GROW " << nboundary_patches <(max_boundary_sets, nboundary_patches, "Boundary_Condition_Patches"); + + // copy previous data back over + // std::cout << "NUM BOUNDARY CONDITIONS ON RANK " << myrank << " FOR COPY " << max_boundary_sets <output(FIELD::velocity); - bool output_strain_flag = simparam->output(FIELD::strain); - bool output_stress_flag = simparam->output(FIELD::stress); - int num_dim = simparam->num_dims; - int Brows; - if(num_dim==3) Brows = 6; - else Brows = 3; - - if(output_velocity_flag){ - //displacement_index is accessed by writers at the solver level for deformed output - output_velocity_index = noutput; - noutput += 1; - module_outputs.resize(noutput); - - vector_style.resize(noutput); - vector_style[noutput-1] = DOF; - - output_vector_sizes.resize(noutput); - output_vector_sizes[noutput-1] = num_dim; - - output_dof_names.resize(noutput); - output_dof_names[noutput-1].resize(num_dim); - output_dof_names[noutput-1][0] = "vx"; - output_dof_names[noutput-1][1] = "vy"; - if(num_dim==3) - output_dof_names[noutput-1][2] = "vz"; - } - if(output_strain_flag){ - output_strain_index = noutput; - noutput += 1; - module_outputs.resize(noutput); - - vector_style.resize(noutput); - vector_style[noutput-1] = NODAL; - - output_vector_sizes.resize(noutput); - output_vector_sizes[noutput-1] = Brows; - - output_dof_names.resize(noutput); - output_dof_names[noutput-1].resize(Brows); - if(num_dim==2){ - output_dof_names[noutput-1][0] = "strain_xx"; - output_dof_names[noutput-1][1] = "strain_yy"; - output_dof_names[noutput-1][2] = "strain_xy"; +void FEA_Module_Dynamic_Elasticity::init_output() +{ + // check user parameters for output + bool output_velocity_flag = simparam->output(FIELD::velocity); + bool output_strain_flag = simparam->output(FIELD::strain); + bool output_stress_flag = simparam->output(FIELD::stress); + int num_dim = simparam->num_dims; + int Brows; + if (num_dim == 3) + { + Brows = 6; + } + else + { + Brows = 3; } - if(num_dim==3){ - output_dof_names[noutput-1][0] = "strain_xx"; - output_dof_names[noutput-1][1] = "strain_yy"; - output_dof_names[noutput-1][2] = "strain_zz"; - output_dof_names[noutput-1][3] = "strain_xy"; - output_dof_names[noutput-1][4] = "strain_xz"; - output_dof_names[noutput-1][5] = "strain_yz"; + + if (output_velocity_flag) + { + // displacement_index is accessed by writers at the solver level for deformed output + output_velocity_index = noutput; + noutput += 1; + module_outputs.resize(noutput); + + vector_style.resize(noutput); + vector_style[noutput - 1] = DOF; + + output_vector_sizes.resize(noutput); + output_vector_sizes[noutput - 1] = num_dim; + + output_dof_names.resize(noutput); + output_dof_names[noutput - 1].resize(num_dim); + output_dof_names[noutput - 1][0] = "vx"; + output_dof_names[noutput - 1][1] = "vy"; + if (num_dim == 3) + { + output_dof_names[noutput - 1][2] = "vz"; + } } - } - if(output_stress_flag){ - output_stress_index = noutput; - noutput += 1; - module_outputs.resize(noutput); - - vector_style.resize(noutput); - vector_style[noutput-1] = NODAL; - - output_vector_sizes.resize(noutput); - output_vector_sizes[noutput-1] = Brows; - - output_dof_names.resize(noutput); - output_dof_names[noutput-1].resize(Brows); - if(num_dim==2){ - output_dof_names[noutput-1][0] = "stress_xx"; - output_dof_names[noutput-1][1] = "stress_yy"; - output_dof_names[noutput-1][3] = "stress_xy"; + if (output_strain_flag) + { + output_strain_index = noutput; + noutput += 1; + module_outputs.resize(noutput); + + vector_style.resize(noutput); + vector_style[noutput - 1] = NODAL; + + output_vector_sizes.resize(noutput); + output_vector_sizes[noutput - 1] = Brows; + + output_dof_names.resize(noutput); + output_dof_names[noutput - 1].resize(Brows); + if (num_dim == 2) + { + output_dof_names[noutput - 1][0] = "strain_xx"; + output_dof_names[noutput - 1][1] = "strain_yy"; + output_dof_names[noutput - 1][2] = "strain_xy"; + } + if (num_dim == 3) + { + output_dof_names[noutput - 1][0] = "strain_xx"; + output_dof_names[noutput - 1][1] = "strain_yy"; + output_dof_names[noutput - 1][2] = "strain_zz"; + output_dof_names[noutput - 1][3] = "strain_xy"; + output_dof_names[noutput - 1][4] = "strain_xz"; + output_dof_names[noutput - 1][5] = "strain_yz"; + } } - if(num_dim==3){ - output_dof_names[noutput-1][0] = "stress_xx"; - output_dof_names[noutput-1][1] = "stress_yy"; - output_dof_names[noutput-1][2] = "stress_zz"; - output_dof_names[noutput-1][3] = "stress_xy"; - output_dof_names[noutput-1][4] = "stress_xz"; - output_dof_names[noutput-1][5] = "stress_yz"; + if (output_stress_flag) + { + output_stress_index = noutput; + noutput += 1; + module_outputs.resize(noutput); + + vector_style.resize(noutput); + vector_style[noutput - 1] = NODAL; + + output_vector_sizes.resize(noutput); + output_vector_sizes[noutput - 1] = Brows; + + output_dof_names.resize(noutput); + output_dof_names[noutput - 1].resize(Brows); + if (num_dim == 2) + { + output_dof_names[noutput - 1][0] = "stress_xx"; + output_dof_names[noutput - 1][1] = "stress_yy"; + output_dof_names[noutput - 1][3] = "stress_xy"; + } + if (num_dim == 3) + { + output_dof_names[noutput - 1][0] = "stress_xx"; + output_dof_names[noutput - 1][1] = "stress_yy"; + output_dof_names[noutput - 1][2] = "stress_zz"; + output_dof_names[noutput - 1][3] = "stress_xy"; + output_dof_names[noutput - 1][4] = "stress_xz"; + output_dof_names[noutput - 1][5] = "stress_yz"; + } } - } } /* ------------------------------------------------------------------------------------------- Prompts sorting for elastic response output data. For now, nodal strains. ---------------------------------------------------------------------------------------------- */ -void FEA_Module_Dynamic_Elasticity::sort_output(Teuchos::RCP > sorted_map){ - - +void FEA_Module_Dynamic_Elasticity::sort_output(Teuchos::RCP> sorted_map) +{ } /* ------------------------------------------------------------------------------------------- populate requests this module makes for output data ---------------------------------------------------------------------------------------------- */ -void FEA_Module_Dynamic_Elasticity::write_data(std::map &point_data_scalars_double, - std::map &point_data_vectors_double, - std::map &cell_data_scalars_double, - std::map &cell_data_scalars_int, - std::map > &cell_data_fields_double){ - - - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; +void FEA_Module_Dynamic_Elasticity::write_data(std::map& point_data_scalars_double, + std::map& point_data_vectors_double, + std::map& cell_data_scalars_double, + std::map& cell_data_scalars_int, + std::map>& cell_data_fields_double) +{ + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - for (const FIELD& field_name : simparam->output_options.output_fields) { - switch (field_name) + for (const FIELD& field_name : simparam->output_options.output_fields) { - - case FIELD::velocity: - // node "velocity" - node_vel.update_host(); - point_data_vectors_double["velocity"] = &node_vel.host(rk_level,0,0); - break; - - case FIELD::element_density: - // element "density" - elem_den.update_host(); - cell_data_scalars_double["element_density"] = elem_den.host_pointer(); - break; - - case FIELD::pressure: - // element "pressure" - elem_pres.update_host(); - cell_data_scalars_double["pressure"] = elem_pres.host_pointer(); - break; - - case FIELD::volume: - // element "volume" - elem_vol.update_host(); - cell_data_scalars_double["volume"] = elem_vol.host_pointer(); - break; - - case FIELD::mass: - // element "mass" - elem_mass.update_host(); - cell_data_scalars_double["mass"] = elem_mass.host_pointer(); - break; - - case FIELD::material_id: - // element "material_id" - elem_mat_id.update_host(); - cell_data_scalars_int["material_id"] = reinterpret_cast(elem_mat_id.host_pointer()); - break; - - case FIELD::user_vars: - // element "user_vars" - elem_user_output_vars.update_host(); - cell_data_fields_double["user_vars"] = std::make_pair(elem_user_output_vars.host_pointer(), + switch (field_name) + { + case FIELD::velocity: + // node "velocity" + node_vel.update_host(); + point_data_vectors_double["velocity"] = &node_vel.host(rk_level, 0, 0); + break; + + case FIELD::element_density: + // element "density" + elem_den.update_host(); + cell_data_scalars_double["element_density"] = elem_den.host_pointer(); + break; + + case FIELD::pressure: + // element "pressure" + elem_pres.update_host(); + cell_data_scalars_double["pressure"] = elem_pres.host_pointer(); + break; + + case FIELD::volume: + // element "volume" + elem_vol.update_host(); + cell_data_scalars_double["volume"] = elem_vol.host_pointer(); + break; + + case FIELD::mass: + // element "mass" + elem_mass.update_host(); + cell_data_scalars_double["mass"] = elem_mass.host_pointer(); + break; + + case FIELD::material_id: + // element "material_id" + elem_mat_id.update_host(); + cell_data_scalars_int["material_id"] = reinterpret_cast(elem_mat_id.host_pointer()); + break; + + case FIELD::user_vars: + // element "user_vars" + elem_user_output_vars.update_host(); + cell_data_fields_double["user_vars"] = std::make_pair(elem_user_output_vars.host_pointer(), elem_user_output_vars.dims(1)); - case FIELD::stress: - // element "stress" - elem_stress.update_host(); - cell_data_fields_double["stress"] = std::make_pair(&elem_stress.host(rk_level,0,0,0), 9); - break; - - default: - break; - - } // end switch - } // end if - - // element "mat_id" //uncomment if needed (works fine) - //sgh_module->elem_mat_id.update_host(); - //cell_data_scalars_int["mat_id"] = reinterpret_cast(&sgh_module->elem_mat_id.host(0)); - - // element "user_output_vars" //uncomment if needed (works fine) - //sgh_module->elem_user_output_vars.update_host(); - //cell_data_fields_double["user_output_vars"] = std::make_pair(&sgh_module->elem_user_output_vars.host_pointer(), - // sgh_module->elem_user_output_vars.dims(1)); - - // element "stress" //uncomment if needed (works fine) - //sgh_module->elem_stress.update_host(); - //cell_data_fields_double["stress"] = std::make_pair(&sgh_module->elem_stress.host(rk_level,0,0,0), 9); + case FIELD::stress: + // element "stress" + elem_stress.update_host(); + cell_data_fields_double["stress"] = std::make_pair(&elem_stress.host(rk_level, 0, 0, 0), 9); + break; + + default: + break; + } // end switch + } // end if + + // element "mat_id" //uncomment if needed (works fine) + // sgh_module->elem_mat_id.update_host(); + // cell_data_scalars_int["mat_id"] = reinterpret_cast(&sgh_module->elem_mat_id.host(0)); + + // element "user_output_vars" //uncomment if needed (works fine) + // sgh_module->elem_user_output_vars.update_host(); + // cell_data_fields_double["user_output_vars"] = std::make_pair(&sgh_module->elem_user_output_vars.host_pointer(), + // sgh_module->elem_user_output_vars.dims(1)); + + // element "stress" //uncomment if needed (works fine) + // sgh_module->elem_stress.update_host(); + // cell_data_fields_double["stress"] = std::make_pair(&sgh_module->elem_stress.host(rk_level,0,0,0), 9); } /* ------------------------------------------------------------------------------------------- Prompts sorting for elastic response output data. For now, nodal strains. ---------------------------------------------------------------------------------------------- */ -void FEA_Module_Dynamic_Elasticity::sort_element_output(Teuchos::RCP > sorted_map){ - //interface element density data - { - host_vec_array Element_Densities = Global_Element_Densities->getLocalView (Tpetra::Access::ReadWrite); - elem_den.update_host(); - for(int ielem = 0; ielem < rnum_elem; ielem++){ - Element_Densities(ielem,0) = elem_den.host(ielem); - } - } - +void FEA_Module_Dynamic_Elasticity::sort_element_output(Teuchos::RCP> sorted_map) +{ + // interface element density data + { + host_vec_array Element_Densities = Global_Element_Densities->getLocalView(Tpetra::Access::ReadWrite); + elem_den.update_host(); + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + Element_Densities(ielem, 0) = elem_den.host(ielem); + } + } } /* ------------------------------------------------------------------------------------------- Prompts computation of elastic response output data. For now, nodal strains. ---------------------------------------------------------------------------------------------- */ -void FEA_Module_Dynamic_Elasticity::collect_output(Teuchos::RCP > global_reduce_map){ - +void FEA_Module_Dynamic_Elasticity::collect_output(Teuchos::RCP> global_reduce_map) +{ } /* ------------------------------------------------------------------------------------------- Prompts computation of elastic response output data. For now, nodal strains. ---------------------------------------------------------------------------------------------- */ -void FEA_Module_Dynamic_Elasticity::compute_output(){ - +void FEA_Module_Dynamic_Elasticity::compute_output() +{ } /* ---------------------------------------------------------------------- Communicate updated nodal velocities to ghost nodes ------------------------------------------------------------------------- */ -void FEA_Module_Dynamic_Elasticity::comm_node_masses(){ - - //debug print of design vector - //std::ostream &out = std::cout; - //Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - //if(myrank==0) - //*fos << "Density data :" << std::endl; - //node_densities_distributed->describe(*fos,Teuchos::VERB_EXTREME); - //*fos << std::endl; - //std::fflush(stdout); - - //communicate design densities - //create import object using local node indices map and all indices map - //Tpetra::Import importer(map, ghost_node_map); - - //comms to get ghosts - ghost_node_masses_distributed->doImport(*node_masses_distributed, *ghost_importer, Tpetra::INSERT); - //all_node_map->describe(*fos,Teuchos::VERB_EXTREME); - //all_node_velocities_distributed->describe(*fos,Teuchos::VERB_EXTREME); - - //update_count++; - //if(update_count==1){ - //MPI_Barrier(world); - //MPI_Abort(world,4); - //} +void FEA_Module_Dynamic_Elasticity::comm_node_masses() +{ + // debug print of design vector + // std::ostream &out = std::cout; + // Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + // if(myrank==0) + // *fos << "Density data :" << std::endl; + // node_densities_distributed->describe(*fos,Teuchos::VERB_EXTREME); + // *fos << std::endl; + // std::fflush(stdout); + + // communicate design densities + // create import object using local node indices map and all indices map + // Tpetra::Import importer(map, ghost_node_map); + + // comms to get ghosts + ghost_node_masses_distributed->doImport(*node_masses_distributed, *ghost_importer, Tpetra::INSERT); + // all_node_map->describe(*fos,Teuchos::VERB_EXTREME); + // all_node_velocities_distributed->describe(*fos,Teuchos::VERB_EXTREME); + + // update_count++; + // if(update_count==1){ + // MPI_Barrier(world); + // MPI_Abort(world,4); + // } } /* ------------------------------------------------------------------------------------------- Communicate ghosts using the current optimization design data ---------------------------------------------------------------------------------------------- */ -void FEA_Module_Dynamic_Elasticity::comm_variables(Teuchos::RCP zp){ - - if(simparam->topology_optimization_on){ - //set density vector to the current value chosen by the optimizer - test_node_densities_distributed = zp; - - //debug print of design vector - //std::ostream &out = std::cout; - //Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - //if(myrank==0) - //*fos << "Density data :" << std::endl; - //node_densities_distributed->describe(*fos,Teuchos::VERB_EXTREME); - //*fos << std::endl; - //std::fflush(stdout); - - //communicate design densities - //create import object using local node indices map and all indices map - //Tpetra::Import importer(map, all_node_map); - - //comms to get ghosts - all_node_densities_distributed->doImport(*test_node_densities_distributed, *importer, Tpetra::INSERT); - } - else if(simparam->shape_optimization_on){ - //clause to communicate boundary node data if the boundary nodes are ghosts on this rank - } +void FEA_Module_Dynamic_Elasticity::comm_variables(Teuchos::RCP zp) +{ + if (simparam->topology_optimization_on) + { + // set density vector to the current value chosen by the optimizer + test_node_densities_distributed = zp; + + // debug print of design vector + // std::ostream &out = std::cout; + // Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + // if(myrank==0) + // *fos << "Density data :" << std::endl; + // node_densities_distributed->describe(*fos,Teuchos::VERB_EXTREME); + // *fos << std::endl; + // std::fflush(stdout); + + // communicate design densities + // create import object using local node indices map and all indices map + // Tpetra::Import importer(map, all_node_map); + + // comms to get ghosts + all_node_densities_distributed->doImport(*test_node_densities_distributed, *importer, Tpetra::INSERT); + } + else if (simparam->shape_optimization_on) + { + // clause to communicate boundary node data if the boundary nodes are ghosts on this rank + } } - /* ------------------------------------------------------------------------------------------- enforce constraints on nodes due to BCS ---------------------------------------------------------------------------------------------- */ -void FEA_Module_Dynamic_Elasticity::node_density_constraints(host_vec_array &node_densities_lower_bound){ - - const size_t num_dim = mesh->num_dims; - const_vec_array all_initial_node_coords = all_initial_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - const size_t num_lcs = module_params->loading_conditions.size(); - - const DCArrayKokkos mat_fill = simparam->mat_fill; - const DCArrayKokkos loading = module_params->loading; - - //debug check - //std::cout << "NUMBER OF LOADING CONDITIONS: " << num_lcs << std::endl; - - // walk over the nodes to update the velocity - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - double current_node_coords[3]; - double radius; - for (size_t dim = 0; dim < num_dim; dim++){ - current_node_coords[dim] = all_initial_node_coords(node_gid, dim); - } // end for dim - radius = sqrt(current_node_coords[0]*current_node_coords[0]+current_node_coords[1]*current_node_coords[1]+current_node_coords[2]*current_node_coords[2]); - for(size_t ilc=0; ilc < num_lcs; ilc++){ - //debug check - //std::cout << "LOADING CONDITION VOLUME TYPE: " << to_string(loading(ilc).volume) << std::endl; - - bool fill_this = loading(ilc).volume.contains(current_node_coords); - if(fill_this){ - node_densities_lower_bound(node_gid,0) = 1; +void FEA_Module_Dynamic_Elasticity::node_density_constraints(host_vec_array& node_densities_lower_bound) +{ + const size_t num_dim = mesh->num_dims; + const_vec_array all_initial_node_coords = all_initial_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + const size_t num_lcs = module_params->loading_conditions.size(); + + const DCArrayKokkos mat_fill = simparam->mat_fill; + const DCArrayKokkos loading = module_params->loading; + + // debug check + // std::cout << "NUMBER OF LOADING CONDITIONS: " << num_lcs << std::endl; + + // walk over the nodes to update the velocity + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + double current_node_coords[3]; + double radius; + for (size_t dim = 0; dim < num_dim; dim++) + { + current_node_coords[dim] = all_initial_node_coords(node_gid, dim); + } // end for dim + radius = sqrt(current_node_coords[0] * current_node_coords[0] + current_node_coords[1] * current_node_coords[1] + current_node_coords[2] * current_node_coords[2]); + for (size_t ilc = 0; ilc < num_lcs; ilc++) + { + // debug check + // std::cout << "LOADING CONDITION VOLUME TYPE: " << to_string(loading(ilc).volume) << std::endl; + + bool fill_this = loading(ilc).volume.contains(current_node_coords); + if (fill_this) + { + node_densities_lower_bound(node_gid, 0) = 1; + } } - } }); // end for parallel for over nodes - } /* ---------------------------------------------------------------------------- Setup elastic solver data ------------------------------------------------------------------------------- */ -void FEA_Module_Dynamic_Elasticity::setup(){ +void FEA_Module_Dynamic_Elasticity::setup() +{ Dynamic_Options dynamic_options = simparam->dynamic_options; - const size_t rk_level = dynamic_options.rk_num_bins - 1; - const size_t num_fills = simparam->regions.size(); - const size_t rk_num_bins = dynamic_options.rk_num_bins; - const size_t num_bcs = module_params->boundary_conditions.size(); - const size_t num_materials = simparam->materials.size(); - const int num_dim = simparam->num_dims; + const size_t rk_level = dynamic_options.rk_num_bins - 1; + const size_t num_fills = simparam->regions.size(); + const size_t rk_num_bins = dynamic_options.rk_num_bins; + const size_t num_bcs = module_params->boundary_conditions.size(); + const size_t num_materials = simparam->materials.size(); + const int num_dim = simparam->num_dims; // --------------------------------------------------------------------- // obtain mesh data - // --------------------------------------------------------------------- + // --------------------------------------------------------------------- elastic_interface_setup(node_interface, elem_interface, corner_interface); mesh->build_corner_connectivity(); - //debug print of corner ids - /* - if(myrank==1){ - for(int i = 0; i < mesh.num_nodes; i++){ - - // loop over all corners around the node and calculate the nodal force - for (size_t corner_lid=0; corner_lidgetGlobalElement(i) << " " << i << " " << all_node_map->getLocalElement(all_node_map->getGlobalElement(i)) << " " << corner_gid << " " << std::endl; - - } // end for corner_lid - //std::cout << explicit_solver_pointer->all_node_map->getGlobalElement(i) << " " << node_force[0] << " " << node_force[1] << " " << node_force[2] << std::endl; - //std::cout << explicit_solver_pointer->all_node_map->getGlobalElement(i) << " " << node_mass(i) << std::endl; - } - } - */ - /* - if(myrank==1){ - for(int i = 0; i < mesh.num_elems; i++){ - - // loop over all corners around the node and calculate the nodal force - for (size_t corner_lid=0; corner_lidgetGlobalElement(mesh.nodes_in_elem(i, corner_lid)) <<" " << corner_gid << " " << std::endl; - - } // end for corner_lid - //std::cout << explicit_solver_pointer->all_node_map->getGlobalElement(i) << " " << node_force[0] << " " << node_force[1] << " " << node_force[2] << std::endl; - //std::cout << explicit_solver_pointer->all_node_map->getGlobalElement(i) << " " << node_mass(i) << std::endl; - } - } - */ + // debug print of corner ids + /* + if(myrank==1){ + for(int i = 0; i < mesh.num_nodes; i++){ + + // loop over all corners around the node and calculate the nodal force + for (size_t corner_lid=0; corner_lidgetGlobalElement(i) << " " << i << " " << all_node_map->getLocalElement(all_node_map->getGlobalElement(i)) << " " << corner_gid << " " << std::endl; + + } // end for corner_lid + //std::cout << explicit_solver_pointer->all_node_map->getGlobalElement(i) << " " << node_force[0] << " " << node_force[1] << " " << node_force[2] << std::endl; + //std::cout << explicit_solver_pointer->all_node_map->getGlobalElement(i) << " " << node_mass(i) << std::endl; + } + } + */ + /* + if(myrank==1){ + for(int i = 0; i < mesh.num_elems; i++){ + + // loop over all corners around the node and calculate the nodal force + for (size_t corner_lid=0; corner_lidgetGlobalElement(mesh.nodes_in_elem(i, corner_lid)) <<" " << corner_gid << " " << std::endl; + + } // end for corner_lid + //std::cout << explicit_solver_pointer->all_node_map->getGlobalElement(i) << " " << node_force[0] << " " << node_force[1] << " " << node_force[2] << std::endl; + //std::cout << explicit_solver_pointer->all_node_map->getGlobalElement(i) << " " << node_mass(i) << std::endl; + } + } + */ mesh->build_elem_elem_connectivity(); mesh->num_bdy_patches = nboundary_patches; - if(num_dim==2){ - mesh->build_patch_connectivity(); - mesh->build_node_node_connectivity(); + if (num_dim == 2) + { + mesh->build_patch_connectivity(); + mesh->build_node_node_connectivity(); } - - // --------------------------------------------------------------------- - // allocate memory - // --------------------------------------------------------------------- - - // shorthand names - const size_t num_nodes = mesh->num_nodes; - const size_t num_elems = mesh->num_elems; + + // --------------------------------------------------------------------- + // allocate memory + // --------------------------------------------------------------------- + + // shorthand names + const size_t num_nodes = mesh->num_nodes; + const size_t num_elems = mesh->num_elems; const size_t num_corners = mesh->num_corners; - - // --- make dual views of data on CPU and GPU --- - // Notes: - // Instead of using a struct of dual types like the mesh type, - // individual dual views will be made for all the state - // variables. The motivation is to reduce memory movement - // when passing state into a function. Passing a struct by - // reference will copy the meta data and pointers for the - // variables held inside the struct. Since all the mesh - // variables are typically used by most functions, a single - // mesh struct or passing the arrays will be roughly equivalent - // for memory movement. - - + // --- make dual views of data on CPU and GPU --- + // Notes: + // Instead of using a struct of dual types like the mesh type, + // individual dual views will be made for all the state + // variables. The motivation is to reduce memory movement + // when passing state into a function. Passing a struct by + // reference will copy the meta data and pointers for the + // variables held inside the struct. Since all the mesh + // variables are typically used by most functions, a single + // mesh struct or passing the arrays will be roughly equivalent + // for memory movement. + // create Dual Views of the individual node struct variables - node_coords = DViewCArrayKokkos(node_interface.coords.get_kokkos_dual_view().view_host().data(),rk_num_bins,num_nodes,num_dim); + node_coords = DViewCArrayKokkos(node_interface.coords.get_kokkos_dual_view().view_host().data(), rk_num_bins, num_nodes, num_dim); - node_vel = DViewCArrayKokkos(node_interface.vel.get_kokkos_dual_view().view_host().data(),rk_num_bins,num_nodes,num_dim); + node_vel = DViewCArrayKokkos(node_interface.vel.get_kokkos_dual_view().view_host().data(), rk_num_bins, num_nodes, num_dim); + + node_mass = DViewCArrayKokkos(node_interface.mass.get_kokkos_dual_view().view_host().data(), num_nodes); - node_mass = DViewCArrayKokkos(node_interface.mass.get_kokkos_dual_view().view_host().data(),num_nodes); - - // create Dual Views of the individual elem struct variables - elem_den= DViewCArrayKokkos(&elem_interface.den(0), + elem_den = DViewCArrayKokkos(&elem_interface.den(0), num_elems); elem_pres = DViewCArrayKokkos(&elem_interface.pres(0), num_elems); - elem_stress = DViewCArrayKokkos(&elem_interface.stress(0,0,0,0), + elem_stress = DViewCArrayKokkos(&elem_interface.stress(0, 0, 0, 0), rk_num_bins, num_elems, 3, @@ -933,38 +981,37 @@ void FEA_Module_Dynamic_Elasticity::setup(){ elem_sspd = DViewCArrayKokkos(&elem_interface.sspd(0), num_elems); - elem_sie = DViewCArrayKokkos(&elem_interface.sie(0,0), + elem_sie = DViewCArrayKokkos(&elem_interface.sie(0, 0), rk_num_bins, num_elems); elem_vol = DViewCArrayKokkos(&elem_interface.vol(0), num_elems); - + elem_div = DViewCArrayKokkos(&elem_interface.div(0), num_elems); - elem_mass = DViewCArrayKokkos(&elem_interface.mass(0), num_elems); elem_mat_id = DViewCArrayKokkos(&elem_interface.mat_id(0), num_elems); - + // create Dual Views of the corner struct variables - corner_force = DViewCArrayKokkos (&corner_interface.force(0,0), - num_corners, + corner_force = DViewCArrayKokkos(&corner_interface.force(0, 0), + num_corners, num_dim); - corner_mass = DViewCArrayKokkos (&corner_interface.mass(0), + corner_mass = DViewCArrayKokkos(&corner_interface.mass(0), num_corners); - + // allocate elem_vel_grad - elem_vel_grad = DCArrayKokkos (num_elems,3,3); + elem_vel_grad = DCArrayKokkos(num_elems, 3, 3); // allocate material models - elem_eos = DCArrayKokkos (num_elems); - elem_strength = DCArrayKokkos (num_elems); - + elem_eos = DCArrayKokkos(num_elems); + elem_strength = DCArrayKokkos(num_elems); + // --------------------------------------------------------------------- // calculate geometry // --------------------------------------------------------------------- @@ -973,76 +1020,79 @@ void FEA_Module_Dynamic_Elasticity::setup(){ get_vol(); - //FEA_Module bc variable + // FEA_Module bc variable num_boundary_conditions = num_bcs; - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos mat_fill = simparam->mat_fill; - const DCArrayKokkos material = simparam->material; + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos mat_fill = simparam->mat_fill; + const DCArrayKokkos material = simparam->material; global_vars = simparam->global_vars; - state_vars = DCArrayKokkos (rnum_elem, simparam->max_num_state_vars); - elem_user_output_vars = DCArrayKokkos (rnum_elem, simparam->output_options.max_num_user_output_vars); - - //--- calculate bdy sets ---// - mesh->num_nodes_in_patch = 2*(num_dim-1); // 2 (2D) or 4 (3D) - mesh->num_patches_in_elem = 2*num_dim; // 4 (2D) or 6 (3D) + state_vars = DCArrayKokkos(rnum_elem, simparam->max_num_state_vars); + elem_user_output_vars = DCArrayKokkos(rnum_elem, simparam->output_options.max_num_user_output_vars); + + // --- calculate bdy sets ---// + mesh->num_nodes_in_patch = 2 * (num_dim - 1); // 2 (2D) or 4 (3D) + mesh->num_patches_in_elem = 2 * num_dim; // 4 (2D) or 6 (3D) mesh->init_bdy_sets(num_bcs); num_bdy_sets = mesh->num_bdy_sets; printf("Num BC's = %lu\n", num_bcs); // patch ids in bdy set bdy_patches_in_set = mesh->bdy_patches_in_set; - if(num_dim==2) - bdy_nodes = mesh->bdy_nodes; + if (num_dim == 2) + { + bdy_nodes = mesh->bdy_nodes; + } // tag boundary patches in the set tag_bdys(boundary, *mesh, node_coords); build_boundry_node_sets(*mesh); - + // node ids in bdy_patch set - bdy_nodes_in_set = mesh->bdy_nodes_in_set; + bdy_nodes_in_set = mesh->bdy_nodes_in_set; num_bdy_nodes_in_set = mesh->num_bdy_nodes_in_set; - - //assign mesh views needed by the FEA module + + // assign mesh views needed by the FEA module // elem ids in elem - elems_in_elem = mesh->elems_in_elem; + elems_in_elem = mesh->elems_in_elem; num_elems_in_elem = mesh->num_elems_in_elem; - //corners + // corners num_corners_in_node = mesh->num_corners_in_node; - corners_in_node = mesh->corners_in_node; - corners_in_elem = mesh->corners_in_elem; + corners_in_node = mesh->corners_in_node; + corners_in_elem = mesh->corners_in_elem; - //elem-node conn & node-node conn + // elem-node conn & node-node conn elems_in_node = mesh->elems_in_node; - if(num_dim==2){ - nodes_in_node = mesh->nodes_in_node; - num_nodes_in_node = mesh->num_nodes_in_node; - //patch conn - - patches_in_elem = mesh->patches_in_elem; - nodes_in_patch = mesh->nodes_in_patch; - elems_in_patch = mesh->elems_in_patch; + if (num_dim == 2) + { + nodes_in_node = mesh->nodes_in_node; + num_nodes_in_node = mesh->num_nodes_in_node; + // patch conn + + patches_in_elem = mesh->patches_in_elem; + nodes_in_patch = mesh->nodes_in_patch; + elems_in_patch = mesh->elems_in_patch; } // loop over BCs - for (size_t this_bdy = 0; this_bdy < num_bcs; this_bdy++){ - + for (size_t this_bdy = 0; this_bdy < num_bcs; this_bdy++) + { RUN_CLASS({ printf("Boundary Condition number %lu \n", this_bdy); printf(" Num bdy patches in this set = %lu \n", bdy_patches_in_set.stride(this_bdy)); printf(" Num bdy nodes in this set = %lu \n", bdy_nodes_in_set.stride(this_bdy)); }); Kokkos::fence(); - - }// end for + } // end for // elem_mat_id needs to be initialized before initialization of material models - for (int f_id = 0; f_id < num_fills; f_id++){ - FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { - elem_mat_id(elem_gid) = mat_fill(f_id).material_id; + for (int f_id = 0; f_id < num_fills; f_id++) + { + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { + elem_mat_id(elem_gid) = mat_fill(f_id).material_id; }); } elem_mat_id.update_host(); @@ -1054,7 +1104,7 @@ void FEA_Module_Dynamic_Elasticity::setup(){ global_vars, elem_user_output_vars, rnum_elem); - + // initialize strength model init_strength_model(elem_strength, material, @@ -1072,23 +1122,24 @@ void FEA_Module_Dynamic_Elasticity::setup(){ global_vars, elem_user_output_vars, rnum_elem); - - //--- apply the fill instructions over each of the Elements---// - - //initialize if topology optimization is used - if(simparam->topology_optimization_on){ - for(int elem_id = 0; elem_id < rnum_elem; elem_id++){ - relative_element_densities.host(elem_id) = 1; - }//for - relative_element_densities.update_device(); + + // --- apply the fill instructions over each of the Elements---// + + // initialize if topology optimization is used + if (simparam->topology_optimization_on) + { + for (int elem_id = 0; elem_id < rnum_elem; elem_id++) + { + relative_element_densities.host(elem_id) = 1; + } // for + relative_element_densities.update_device(); } - + // loop over the fill instructures - for (int f_id = 0; f_id < num_fills; f_id++){ - + for (int f_id = 0; f_id < num_fills; f_id++) + { // parallel loop over elements in mesh FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { - // calculate the coordinates and radius of the element double elem_coords[3]; // note:initialization with a list won't work elem_coords[0] = 0.0; @@ -1096,47 +1147,52 @@ void FEA_Module_Dynamic_Elasticity::setup(){ elem_coords[2] = 0.0; // get the coordinates of the element center - for (int node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ + for (int node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { elem_coords[0] += node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), 0); elem_coords[1] += node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), 1); - if (num_dim == 3){ + if (num_dim == 3) + { elem_coords[2] += node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), 2); - } else + } + else { elem_coords[2] = 0.0; } } // end loop over nodes in element - elem_coords[0] = elem_coords[0]/num_nodes_in_elem; - elem_coords[1] = elem_coords[1]/num_nodes_in_elem; - elem_coords[2] = elem_coords[2]/num_nodes_in_elem; + elem_coords[0] = elem_coords[0] / num_nodes_in_elem; + elem_coords[1] = elem_coords[1] / num_nodes_in_elem; + elem_coords[2] = elem_coords[2] / num_nodes_in_elem; // default is not to fill the element bool fill_this = mat_fill(f_id).volume.contains(elem_coords); // paint the material state on the element - if (fill_this){ - + if (fill_this) + { // density elem_den(elem_gid) = mat_fill(f_id).den; - + // mass - elem_mass(elem_gid) = elem_den(elem_gid)*elem_vol(elem_gid); - + elem_mass(elem_gid) = elem_den(elem_gid) * elem_vol(elem_gid); + // specific internal energy elem_sie(rk_level, elem_gid) = mat_fill(f_id).sie; - + size_t mat_id = elem_mat_id(elem_gid); // short name - + // --- stress tensor --- // always 3D even for 2D-RZ - for (size_t i=0; i<3; i++){ - for (size_t j=0; j<3; j++){ - elem_stress(rk_level,elem_gid,i,j) = 0.0; - } + for (size_t i = 0; i < 3; i++) + { + for (size_t j = 0; j < 3; j++) + { + elem_stress(rk_level, elem_gid, i, j) = 0.0; + } } // end for - + // short form for clean code - EOSParent * eos_model = elem_eos(elem_gid).model; + EOSParent* eos_model = elem_eos(elem_gid).model; // --- Pressure --- eos_model->calc_pressure(elem_pres, @@ -1148,7 +1204,7 @@ void FEA_Module_Dynamic_Elasticity::setup(){ elem_user_output_vars, elem_sspd, elem_den(elem_gid), - elem_sie(rk_level,elem_gid)); + elem_sie(rk_level, elem_gid)); // --- Sound speed --- eos_model->calc_sound_speed(elem_pres, @@ -1160,226 +1216,224 @@ void FEA_Module_Dynamic_Elasticity::setup(){ elem_user_output_vars, elem_sspd, elem_den(elem_gid), - elem_sie(rk_level,elem_gid)); - - // loop over the nodes of this element and apply velocity - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ + elem_sie(rk_level, elem_gid)); + // loop over the nodes of this element and apply velocity + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { // get the mesh node index size_t node_gid = nodes_in_elem(elem_gid, node_lid); - // --- Velocity --- - switch(mat_fill(f_id).velocity) + switch (mat_fill(f_id).velocity) { case VELOCITY_TYPE::cartesian: - { - - node_vel(rk_level, node_gid, 0) = mat_fill(f_id).u; - node_vel(rk_level, node_gid, 1) = mat_fill(f_id).v; - if (num_dim == 3) node_vel(rk_level, node_gid, 2) = mat_fill(f_id).w; - - - break; - } - case VELOCITY_TYPE::radial: - { - // Setting up cylindrical - double dir[2]; - dir[0] = 0.0; - dir[1] = 0.0; - double radius_val = 0.0; - - for(int dim=0; dim<2; dim++){ - dir[dim] = node_coords(rk_level, node_gid, dim); - radius_val += node_coords(rk_level, node_gid, dim)*node_coords(rk_level, node_gid, dim); - } // end for - radius_val = sqrt(radius_val); - - for(int dim=0; dim<2; dim++){ - if (radius_val > 1.0e-14){ - dir[dim] /= (radius_val); + { + node_vel(rk_level, node_gid, 0) = mat_fill(f_id).u; + node_vel(rk_level, node_gid, 1) = mat_fill(f_id).v; + if (num_dim == 3) + { + node_vel(rk_level, node_gid, 2) = mat_fill(f_id).w; } - else{ - dir[dim] = 0.0; + + break; + } + case VELOCITY_TYPE::radial: + { + // Setting up cylindrical + double dir[2]; + dir[0] = 0.0; + dir[1] = 0.0; + double radius_val = 0.0; + + for (int dim = 0; dim < 2; dim++) + { + dir[dim] = node_coords(rk_level, node_gid, dim); + radius_val += node_coords(rk_level, node_gid, dim) * node_coords(rk_level, node_gid, dim); + } // end for + radius_val = sqrt(radius_val); + + for (int dim = 0; dim < 2; dim++) + { + if (radius_val > 1.0e-14) + { + dir[dim] /= (radius_val); + } + else + { + dir[dim] = 0.0; + } + } // end for + + node_vel(rk_level, node_gid, 0) = mat_fill(f_id).speed * dir[0]; + node_vel(rk_level, node_gid, 1) = mat_fill(f_id).speed * dir[1]; + if (num_dim == 3) + { + node_vel(rk_level, node_gid, 2) = 0.0; } - } // end for - - - node_vel(rk_level, node_gid, 0) = mat_fill(f_id).speed*dir[0]; - node_vel(rk_level, node_gid, 1) = mat_fill(f_id).speed*dir[1]; - if (num_dim == 3) node_vel(rk_level, node_gid, 2) = 0.0; - - break; - } + + break; + } case VELOCITY_TYPE::spherical: - { - - // Setting up spherical - double dir[3]; - dir[0] = 0.0; - dir[1] = 0.0; - dir[2] = 0.0; - double radius_val = 0.0; - - for(int dim=0; dim<3; dim++){ - dir[dim] = node_coords(rk_level, node_gid, dim); - radius_val += node_coords(rk_level, node_gid, dim)*node_coords(rk_level, node_gid, dim); - } // end for - radius_val = sqrt(radius_val); - - for(int dim=0; dim<3; dim++){ - if (radius_val > 1.0e-14){ - dir[dim] /= (radius_val); + { + // Setting up spherical + double dir[3]; + dir[0] = 0.0; + dir[1] = 0.0; + dir[2] = 0.0; + double radius_val = 0.0; + + for (int dim = 0; dim < 3; dim++) + { + dir[dim] = node_coords(rk_level, node_gid, dim); + radius_val += node_coords(rk_level, node_gid, dim) * node_coords(rk_level, node_gid, dim); + } // end for + radius_val = sqrt(radius_val); + + for (int dim = 0; dim < 3; dim++) + { + if (radius_val > 1.0e-14) + { + dir[dim] /= (radius_val); + } + else + { + dir[dim] = 0.0; + } + } // end for + + node_vel(rk_level, node_gid, 0) = mat_fill(f_id).speed * dir[0]; + node_vel(rk_level, node_gid, 1) = mat_fill(f_id).speed * dir[1]; + if (num_dim == 3) + { + node_vel(rk_level, node_gid, 2) = mat_fill(f_id).speed * dir[2]; } - else{ - dir[dim] = 0.0; - } - } // end for - - node_vel(rk_level, node_gid, 0) = mat_fill(f_id).speed*dir[0]; - node_vel(rk_level, node_gid, 1) = mat_fill(f_id).speed*dir[1]; - if (num_dim == 3) node_vel(rk_level, node_gid, 2) = mat_fill(f_id).speed*dir[2]; - - break; - } + + break; + } case VELOCITY_TYPE::radial_linear: - { - - break; - } + { + break; + } case VELOCITY_TYPE::spherical_linear: - { - - break; - } + { + break; + } case VELOCITY_TYPE::tg_vortex: - { - - node_vel(rk_level, node_gid, 0) = sin(PI * node_coords(rk_level,node_gid, 0)) * cos(PI * node_coords(rk_level,node_gid, 1)); - node_vel(rk_level, node_gid, 1) = -1.0*cos(PI * node_coords(rk_level,node_gid, 0)) * sin(PI * node_coords(rk_level,node_gid, 1)); - if (num_dim == 3) node_vel(rk_level, node_gid, 2) = 0.0; + { + node_vel(rk_level, node_gid, 0) = sin(PI * node_coords(rk_level, node_gid, 0)) * cos(PI * node_coords(rk_level, node_gid, 1)); + node_vel(rk_level, node_gid, 1) = -1.0 * cos(PI * node_coords(rk_level, node_gid, 0)) * sin(PI * node_coords(rk_level, node_gid, 1)); + if (num_dim == 3) + { + node_vel(rk_level, node_gid, 2) = 0.0; + } - break; - } + break; + } } // end of switch + } // end loop over nodes of element - }// end loop over nodes of element - - - if(mat_fill(f_id).velocity == VELOCITY_TYPE::tg_vortex) + if (mat_fill(f_id).velocity == VELOCITY_TYPE::tg_vortex) { - elem_pres(elem_gid) = 0.25*( cos(2.0*PI*elem_coords[0]) + cos(2.0*PI*elem_coords[1]) ) + 1.0; - + elem_pres(elem_gid) = 0.25 * (cos(2.0 * PI * elem_coords[0]) + cos(2.0 * PI * elem_coords[1]) ) + 1.0; + // p = rho*ie*(gamma - 1) size_t mat_id = f_id; - double gamma = global_vars(mat_id,0); // gamma value + double gamma = global_vars(mat_id, 0); // gamma value elem_sie(rk_level, elem_gid) = - elem_pres(elem_gid)/(mat_fill(f_id).den*(gamma - 1.0)); + elem_pres(elem_gid) / (mat_fill(f_id).den * (gamma - 1.0)); } // end if - } // end if fill - }); // end FOR_ALL_CLASS element loop Kokkos::fence(); - - } // end for loop over fills - // apply BC's to velocity FEA_Module_Dynamic_Elasticity::boundary_velocity(*mesh, boundary, node_vel); - - + // calculate the corner massess if 2D - if(num_dim==2){ - + if (num_dim == 2) + { FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { - // facial area of the corners double corner_areas_array[4]; - - ViewCArrayKokkos corner_areas(&corner_areas_array[0],4); - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 4); - + + ViewCArrayKokkos corner_areas(&corner_areas_array[0], 4); + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 4); + get_area_weights2D(corner_areas, elem_gid, node_coords, elem_node_gids, rk_level); - + // loop over the corners of the element and calculate the mass - for (size_t corner_lid=0; corner_lid<4; corner_lid++){ - - size_t corner_gid = corners_in_elem(elem_gid, corner_lid); - corner_mass(corner_gid) = corner_areas(corner_lid)*elem_den(elem_gid); // node radius is added later - + for (size_t corner_lid = 0; corner_lid < 4; corner_lid++) + { + size_t corner_gid = corners_in_elem(elem_gid, corner_lid); + corner_mass(corner_gid) = corner_areas(corner_lid) * elem_den(elem_gid); // node radius is added later } // end for over corners }); - } // end of - - + // calculate the nodal mass FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - node_mass(node_gid) = 0.0; - - if(num_dim==3){ - - for(size_t elem_lid=0; elem_lidtopology_optimization_on||simparam->shape_optimization_on||simparam->num_dims==2){ - { - vec_array node_mass_interface = node_masses_distributed->getLocalView (Tpetra::Access::ReadWrite); + // current interface has differing mass arrays; this equates them until we unify memory + // view scope + if (simparam->topology_optimization_on || simparam->shape_optimization_on || simparam->num_dims == 2) + { + { + vec_array node_mass_interface = node_masses_distributed->getLocalView(Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - node_mass_interface(node_gid,0) = node_mass(node_gid); + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + node_mass_interface(node_gid, 0) = node_mass(node_gid); }); // end parallel for - } //end view scope - Kokkos::fence(); - //communicate ghost densities - comm_node_masses(); - - //this is forcing a copy to the device - //view scope - { - vec_array ghost_node_mass_interface = ghost_node_masses_distributed->getLocalView (Tpetra::Access::ReadWrite); + } // end view scope + Kokkos::fence(); + // communicate ghost densities + comm_node_masses(); + // this is forcing a copy to the device + // view scope + { + vec_array ghost_node_mass_interface = ghost_node_masses_distributed->getLocalView(Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, nlocal_nodes, nall_nodes, { - node_mass(node_gid) = ghost_node_mass_interface(node_gid-nlocal_nodes,0); + FOR_ALL_CLASS(node_gid, nlocal_nodes, nall_nodes, { + node_mass(node_gid) = ghost_node_mass_interface(node_gid - nlocal_nodes, 0); }); // end parallel for - } //end view scope - Kokkos::fence(); - } //endif - - //initialize if topology optimization is used - if(simparam->topology_optimization_on || simparam->shape_optimization_on){ - init_assembly(); - assemble_matrix(); + } // end view scope + Kokkos::fence(); + } // endif + + // initialize if topology optimization is used + if (simparam->topology_optimization_on || simparam->shape_optimization_on) + { + init_assembly(); + assemble_matrix(); } // update host copies of arrays modified in this function @@ -1388,27 +1442,27 @@ void FEA_Module_Dynamic_Elasticity::setup(){ elem_sie.update_host(); elem_stress.update_host(); elem_pres.update_host(); - elem_sspd.update_host(); + elem_sspd.update_host(); return; - } // end of setup /* ---------------------------------------------------------------------------- solve function called by solver ------------------------------------------------------------------------------- */ -void FEA_Module_Dynamic_Elasticity::module_cleanup(){ - cleanup_material_models(); +void FEA_Module_Dynamic_Elasticity::module_cleanup() +{ + cleanup_material_models(); } /* ---------------------------------------------------------------------------- Deallocate memory used for material models ------------------------------------------------------------------------------- */ -void FEA_Module_Dynamic_Elasticity::cleanup_material_models() { - - const DCArrayKokkos material = simparam->material; +void FEA_Module_Dynamic_Elasticity::cleanup_material_models() +{ + const DCArrayKokkos material = simparam->material; // destroy strength model destroy_strength_model(elem_strength, @@ -1428,50 +1482,46 @@ void FEA_Module_Dynamic_Elasticity::cleanup_material_models() { elem_user_output_vars, rnum_elem); return; - } // end cleanup_user_strength_model; - /** * Determines which of the boundary patches are associated with which boundary. - * + * * Modifies: bdy_patches_in_set */ -void FEA_Module_Dynamic_Elasticity::tag_bdys(const DCArrayKokkos &boundary, - mesh_t &mesh, - const DViewCArrayKokkos &node_coords){ - +void FEA_Module_Dynamic_Elasticity::tag_bdys(const DCArrayKokkos& boundary, + mesh_t& mesh, + const DViewCArrayKokkos& node_coords) +{ const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - size_t num_dim = simparam->num_dims; - int nboundary_patches = Explicit_Solver_Pointer_->nboundary_patches; - int num_nodes_in_patch = mesh.num_nodes_in_patch; - - //if (bdy_set == mesh.num_bdy_sets){ + size_t num_dim = simparam->num_dims; + int nboundary_patches = Explicit_Solver_Pointer_->nboundary_patches; + int num_nodes_in_patch = mesh.num_nodes_in_patch; + + // if (bdy_set == mesh.num_bdy_sets){ // printf(" ERROR: number of boundary sets must be increased by %zu", // bdy_set-mesh.num_bdy_sets+1); // exit(0); - //} // end if - - //error and debug flag - //DCArrayKokkos print_flag(1, "print_flag"); - //print_flag.host(0) = false; - //print_flag.update_device(); - - + // } // end if + + // error and debug flag + // DCArrayKokkos print_flag(1, "print_flag"); + // print_flag.host(0) = false; + // print_flag.update_device(); + FOR_ALL_CLASS(bdy_set, 0, num_bdy_sets, { - //for (size_t bdy_set = 0; bdy_set < num_bdy_sets; bdy_set++) { - + // for (size_t bdy_set = 0; bdy_set < num_bdy_sets; bdy_set++) { + // tag boundaries BOUNDARY_TYPE bc_type = boundary(bdy_set).surface.type; double val = boundary(bdy_set).surface.plane_position; - + // save the boundary patches to this set that are on the plane, spheres, etc. - for (size_t bdy_patch_lid=0; bdy_patch_lid < nboundary_patches; bdy_patch_lid++){ - + for (size_t bdy_patch_lid = 0; bdy_patch_lid < nboundary_patches; bdy_patch_lid++) + { // save the patch index size_t bdy_patch_gid = bdy_patch_lid; - - + // check to see if this patch is on the specified plane bool is_on_bdy = check_bdy(bdy_patch_gid, num_dim, @@ -1480,8 +1530,8 @@ void FEA_Module_Dynamic_Elasticity::tag_bdys(const DCArrayKokkos &b val, node_coords, rk_level); // no=0, yes=1 - - //debug check + + // debug check /* for (size_t patch_node_lid=0; patch_node_lid &b } */ - if (is_on_bdy){ - + if (is_on_bdy) + { size_t index = bdy_patches_in_set.stride(bdy_set); - + // increment the number of boundary patches saved - bdy_patches_in_set.stride(bdy_set) ++; - - + bdy_patches_in_set.stride(bdy_set)++; + bdy_patches_in_set(bdy_set, index) = bdy_patch_gid; } // end if - - } // end for bdy_patch - //} + // } }); // end FOR_ALL_CLASS bdy_sets - - //debug check - //print_flag.update_host(); - //if(print_flag.host(0)) std::cout << "found boundary node with id 549412" << std::endl; + + // debug check + // print_flag.update_host(); + // if(print_flag.host(0)) std::cout << "found boundary node with id 549412" << std::endl; return; } // end tag - KOKKOS_INLINE_FUNCTION -bool FEA_Module_Dynamic_Elasticity::check_bdy(const size_t patch_gid, - const int num_dim, - const int num_nodes_in_patch, - const BOUNDARY_TYPE bc_type, - const double val, - const DViewCArrayKokkos &node_coords, - const size_t rk_level) const { - +bool FEA_Module_Dynamic_Elasticity::check_bdy(const size_t patch_gid, + const int num_dim, + const int num_nodes_in_patch, + const BOUNDARY_TYPE bc_type, + const double val, + const DViewCArrayKokkos& node_coords, + const size_t rk_level) const +{ // default bool is not on the boundary size_t is_on_bdy = 0; - + // the patch coordinates double these_patch_coords[3]; // Note: cannot allocated array with num_dim - + // loop over the nodes on the patch - for (size_t patch_node_lid=0; patch_node_lid < num_nodes_in_patch; patch_node_lid++) { + for (size_t patch_node_lid = 0; patch_node_lid < num_nodes_in_patch; patch_node_lid++) + { // get the nodal_gid for this node in the patch - //size_t node_gid = mesh.nodes_in_patch(patch_gid, patch_node_lid); + // size_t node_gid = mesh.nodes_in_patch(patch_gid, patch_node_lid); size_t node_gid = Local_Index_Boundary_Patches(patch_gid, patch_node_lid); - for (size_t dim = 0; dim < num_dim; dim++){ + for (size_t dim = 0; dim < num_dim; dim++) + { these_patch_coords[dim] = node_coords(rk_level, node_gid, dim); // (rk, node_gid, dim) } - - if (bc_type == BOUNDARY_TYPE::x_plane) { - if ( fabs(these_patch_coords[0] - val) <= 1.0e-7 ) is_on_bdy += 1; + + if (bc_type == BOUNDARY_TYPE::x_plane) + { + if (fabs(these_patch_coords[0] - val) <= 1.0e-7) + { + is_on_bdy += 1; + } } - else if (bc_type == BOUNDARY_TYPE::y_plane) { - if ( fabs(these_patch_coords[1] - val) <= 1.0e-7 ) is_on_bdy += 1; + else if (bc_type == BOUNDARY_TYPE::y_plane) + { + if (fabs(these_patch_coords[1] - val) <= 1.0e-7) + { + is_on_bdy += 1; + } } - else if (bc_type == BOUNDARY_TYPE::z_plane) { - if ( fabs(these_patch_coords[2] - val) <= 1.0e-7 ) is_on_bdy += 1; + else if (bc_type == BOUNDARY_TYPE::z_plane) + { + if (fabs(these_patch_coords[2] - val) <= 1.0e-7) + { + is_on_bdy += 1; + } } - else if (bc_type == BOUNDARY_TYPE::cylinder){ - real_t R = sqrt(these_patch_coords[0]*these_patch_coords[0] + - these_patch_coords[1]*these_patch_coords[1]); - - if ( fabs(R - val) <= 1.0e-7 ) is_on_bdy += 1; + else if (bc_type == BOUNDARY_TYPE::cylinder) + { + real_t R = sqrt(these_patch_coords[0] * these_patch_coords[0] + + these_patch_coords[1] * these_patch_coords[1]); + + if (fabs(R - val) <= 1.0e-7) + { + is_on_bdy += 1; + } } - else if (bc_type == BOUNDARY_TYPE::sphere) { - real_t R = sqrt(these_patch_coords[0]*these_patch_coords[0] + - these_patch_coords[1]*these_patch_coords[1] + - these_patch_coords[2]*these_patch_coords[2]); - - if ( fabs(R - val) <= 1.0e-7 ) is_on_bdy += 1; + else if (bc_type == BOUNDARY_TYPE::sphere) + { + real_t R = sqrt(these_patch_coords[0] * these_patch_coords[0] + + these_patch_coords[1] * these_patch_coords[1] + + these_patch_coords[2] * these_patch_coords[2]); + + if (fabs(R - val) <= 1.0e-7) + { + is_on_bdy += 1; + } } } - + // if all nodes in the patch are on the surface return is_on_bdy == num_nodes_in_patch; } // end method to check bdy @@ -1570,94 +1638,82 @@ bool FEA_Module_Dynamic_Elasticity::check_bdy(const size_t patch_gid, Build set of nodes assigned to each boundary condition ------------------------------------------------------------------------------- */ -void FEA_Module_Dynamic_Elasticity::build_boundry_node_sets(mesh_t &mesh){ - +void FEA_Module_Dynamic_Elasticity::build_boundry_node_sets(mesh_t& mesh) +{ // build boundary nodes in each boundary set - int nboundary_patches = Explicit_Solver_Pointer_->nboundary_patches; + int nboundary_patches = Explicit_Solver_Pointer_->nboundary_patches; int num_nodes_in_patch = mesh.num_nodes_in_patch; - num_bdy_nodes_in_set = mesh.num_bdy_nodes_in_set = DCArrayKokkos (num_bdy_sets, "num_bdy_nodes_in_set"); - CArrayKokkos temp_count_num_bdy_nodes_in_set(num_bdy_sets, nall_nodes, "temp_count_num_bdy_nodes_in_set"); - - DynamicRaggedRightArrayKokkos temp_nodes_in_set (mesh.num_bdy_sets, nboundary_patches*mesh.num_nodes_in_patch, "temp_nodes_in_set"); - + num_bdy_nodes_in_set = mesh.num_bdy_nodes_in_set = DCArrayKokkos(num_bdy_sets, "num_bdy_nodes_in_set"); + CArrayKokkos temp_count_num_bdy_nodes_in_set(num_bdy_sets, nall_nodes, "temp_count_num_bdy_nodes_in_set"); + + DynamicRaggedRightArrayKokkos temp_nodes_in_set(mesh.num_bdy_sets, nboundary_patches * mesh.num_nodes_in_patch, "temp_nodes_in_set"); + // Parallel loop over boundary sets on device FOR_ALL_CLASS(bdy_set, 0, num_bdy_sets, { - // finde the number of patches_in_set size_t num_bdy_patches_in_set = bdy_patches_in_set.stride(bdy_set); num_bdy_nodes_in_set(bdy_set) = 0; - + // Loop over boundary patches in boundary set - for (size_t bdy_patch_gid = 0; bdy_patch_gid (mesh.num_bdy_nodes_in_set, "bdy_nodes_in_set"); + bdy_nodes_in_set = mesh.bdy_nodes_in_set = RaggedRightArrayKokkos(mesh.num_bdy_nodes_in_set, "bdy_nodes_in_set"); - FOR_ALL_CLASS (bdy_set, 0, num_bdy_sets, { - + FOR_ALL_CLASS(bdy_set, 0, num_bdy_sets, { // Loop over boundary patches in boundary set - for (size_t bdy_node_lid=0; bdy_node_liddynamic_options; - - const size_t rk_level = dynamic_options.rk_num_bins - 1; - time_value = dynamic_options.time_initial; - time_final = dynamic_options.time_final; - dt_max = dynamic_options.dt_max; - dt_min = dynamic_options.dt_min; - dt_cfl = dynamic_options.dt_cfl; - graphics_time = simparam->output_options.graphics_step; + + const size_t rk_level = dynamic_options.rk_num_bins - 1; + time_value = dynamic_options.time_initial; + time_final = dynamic_options.time_final; + dt_max = dynamic_options.dt_max; + dt_min = dynamic_options.dt_min; + dt_cfl = dynamic_options.dt_cfl; + graphics_time = simparam->output_options.graphics_step; graphics_dt_ival = simparam->output_options.graphics_step; - cycle_stop = dynamic_options.cycle_stop; - rk_num_stages = dynamic_options.rk_num_stages; - dt = dynamic_options.dt; - fuzz = dynamic_options.fuzz; - tiny = dynamic_options.tiny; + cycle_stop = dynamic_options.cycle_stop; + rk_num_stages = dynamic_options.rk_num_stages; + dt = dynamic_options.dt; + fuzz = dynamic_options.fuzz; + tiny = dynamic_options.tiny; small = dynamic_options.small; graphics_times = simparam->output_options.graphics_times; - graphics_id = simparam->output_options.graphics_id; - size_t num_bdy_nodes = mesh->num_bdy_nodes; - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos material = simparam->material; - int nTO_modules; - int old_max_forward_buffer; - unsigned long cycle; - const int num_dim = simparam->num_dims; - real_t objective_accumulation, global_objective_accumulation; - std::vector> FEA_Module_My_TO_Modules = simparam->FEA_Module_My_TO_Modules; - problem = Explicit_Solver_Pointer_->problem; //Pointer to ROL optimization problem object + graphics_id = simparam->output_options.graphics_id; + size_t num_bdy_nodes = mesh->num_bdy_nodes; + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos material = simparam->material; + int nTO_modules; + int old_max_forward_buffer; + unsigned long cycle; + const int num_dim = simparam->num_dims; + real_t objective_accumulation, global_objective_accumulation; + std::vector> FEA_Module_My_TO_Modules = simparam->FEA_Module_My_TO_Modules; + problem = Explicit_Solver_Pointer_->problem; // Pointer to ROL optimization problem object ROL::Ptr> obj_pointer; - //reset time accumulating objective and constraints + // reset time accumulating objective and constraints /* for(int imodule = 0 ; imodule < FEA_Module_My_TO_Modules[my_fea_module_index_].size(); imodule++){ current_module_index = FEA_Module_My_TO_Modules[my_fea_module_index_][imodule]; //test if module needs reset if(){ - + } } */ - //simple setup to just request KE for now; above loop to be expanded and used later for scanning modules - if(simparam->topology_optimization_on){ - obj_pointer = problem->getObjective(); - KineticEnergyMinimize_TopOpt& kinetic_energy_minimize_function = dynamic_cast(*obj_pointer); - kinetic_energy_minimize_function.objective_accumulation = 0; - global_objective_accumulation = objective_accumulation = 0; - kinetic_energy_objective = true; - if(max_time_steps +1 > forward_solve_velocity_data->size()){ - old_max_forward_buffer = forward_solve_velocity_data->size(); - time_data.resize(max_time_steps+1); - forward_solve_velocity_data->resize(max_time_steps+1); - forward_solve_coordinate_data->resize(max_time_steps+1); - adjoint_vector_data->resize(max_time_steps+1); - phi_adjoint_vector_data->resize(max_time_steps+1); - //assign a multivector of corresponding size to each new timestep in the buffer - for(int istep = old_max_forward_buffer; istep < max_time_steps+1; istep++){ - (*forward_solve_velocity_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - (*forward_solve_coordinate_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - (*adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - (*phi_adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + // simple setup to just request KE for now; above loop to be expanded and used later for scanning modules + if (simparam->topology_optimization_on) + { + obj_pointer = problem->getObjective(); + KineticEnergyMinimize_TopOpt& kinetic_energy_minimize_function = dynamic_cast(*obj_pointer); + kinetic_energy_minimize_function.objective_accumulation = 0; + global_objective_accumulation = objective_accumulation = 0; + kinetic_energy_objective = true; + if (max_time_steps + 1 > forward_solve_velocity_data->size()) + { + old_max_forward_buffer = forward_solve_velocity_data->size(); + time_data.resize(max_time_steps + 1); + forward_solve_velocity_data->resize(max_time_steps + 1); + forward_solve_coordinate_data->resize(max_time_steps + 1); + adjoint_vector_data->resize(max_time_steps + 1); + phi_adjoint_vector_data->resize(max_time_steps + 1); + // assign a multivector of corresponding size to each new timestep in the buffer + for (int istep = old_max_forward_buffer; istep < max_time_steps + 1; istep++) + { + (*forward_solve_velocity_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*forward_solve_coordinate_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*phi_adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + } } - } } - if(simparam->topology_optimization_on) - nTO_modules = simparam->TO_Module_List.size(); + if (simparam->topology_optimization_on) + { + nTO_modules = simparam->TO_Module_List.size(); + } int myrank = Explicit_Solver_Pointer_->myrank; - if(simparam->output_options.output_file_format==OUTPUT_FORMAT::vtk&&simparam->output_options.write_initial) + if (simparam->output_options.output_file_format == OUTPUT_FORMAT::vtk && simparam->output_options.write_initial) { - if(myrank==0) - printf("Writing outputs to file at %f \n", time_value); + if (myrank == 0) + { + printf("Writing outputs to file at %f \n", time_value); + } - double comm_time1 = Explicit_Solver_Pointer_->CPU_Time(); - Explicit_Solver_Pointer_->write_outputs(); - double comm_time2 = Explicit_Solver_Pointer_->CPU_Time(); - Explicit_Solver_Pointer_->output_time += comm_time2 - comm_time1; + double comm_time1 = Explicit_Solver_Pointer_->CPU_Time(); + Explicit_Solver_Pointer_->write_outputs(); + double comm_time2 = Explicit_Solver_Pointer_->CPU_Time(); + Explicit_Solver_Pointer_->output_time += comm_time2 - comm_time1; } - - CArrayKokkos node_extensive_mass(nall_nodes, "node_extensive_mass"); - + + CArrayKokkos node_extensive_mass(nall_nodes, "node_extensive_mass"); + // extensive energy tallies over the mesh elements local to this MPI rank double IE_t0 = 0.0; double KE_t0 = 0.0; double TE_t0 = 0.0; - + double IE_sum = 0.0; double KE_sum = 0.0; - + double IE_loc_sum = 0.0; double KE_loc_sum = 0.0; @@ -1783,138 +1848,139 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve(){ double global_TE_tend = 0.0; int nlocal_elem_non_overlapping = Explicit_Solver_Pointer_->nlocal_elem_non_overlapping; - + // extensive IE REDUCE_SUM_CLASS(elem_gid, 0, nlocal_elem_non_overlapping, IE_loc_sum, { - IE_loc_sum += elem_mass(elem_gid)*elem_sie(rk_level,elem_gid); - + IE_loc_sum += elem_mass(elem_gid) * elem_sie(rk_level, elem_gid); }, IE_sum); IE_t0 = IE_sum; - MPI_Allreduce(&IE_t0,&global_IE_t0,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD); - + MPI_Allreduce(&IE_t0, &global_IE_t0, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); + // extensive KE REDUCE_SUM_CLASS(node_gid, 0, nlocal_nodes, KE_loc_sum, { - double ke = 0; - for (size_t dim=0; dimtopology_optimization_on||simparam->shape_optimization_on){ - time_data[0] = 0; - //assign current velocity data to multivector - //view scope + // save initial data + if (simparam->topology_optimization_on || simparam->shape_optimization_on) { - vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView (Tpetra::Access::ReadWrite); - vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - node_velocities_interface(node_gid,idim) = node_vel(rk_level,node_gid,idim); - node_coords_interface(node_gid,idim) = node_coords(rk_level,node_gid,idim); - } + time_data[0] = 0; + // assign current velocity data to multivector + // view scope + { + vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); + vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + node_velocities_interface(node_gid, idim) = node_vel(rk_level, node_gid, idim); + node_coords_interface(node_gid, idim) = node_coords(rk_level, node_gid, idim); + } }); - } //end view scope - Kokkos::fence(); - - //communicate ghosts - double comm_time1 = Explicit_Solver_Pointer_->CPU_Time(); - - //active view scope; triggers host comms from updated data on device - { - const_host_vec_array node_velocities_host = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_host_vec_array node_coords_host = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - } - double comm_time2 = Explicit_Solver_Pointer_->CPU_Time(); - Explicit_Solver_Pointer_->dev2host_time += comm_time2-comm_time1; + } // end view scope + Kokkos::fence(); - //communicate ghost velocities - Explicit_Solver_Pointer_->comm_velocities(); - Explicit_Solver_Pointer_->comm_coordinates(); - - - double comm_time3 = Explicit_Solver_Pointer_->CPU_Time(); + // communicate ghosts + double comm_time1 = Explicit_Solver_Pointer_->CPU_Time(); - //view scope - { - const_vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array ghost_node_velocities_interface = Explicit_Solver_Pointer_->ghost_node_velocities_distributed->getLocalView (Tpetra::Access::ReadOnly); - vec_array all_node_velocities_interface = Explicit_Solver_Pointer_->all_node_velocities_distributed->getLocalView (Tpetra::Access::ReadWrite); - const_vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array ghost_node_coords_interface = Explicit_Solver_Pointer_->ghost_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - all_node_velocities_interface(node_gid,idim) = node_velocities_interface(node_gid,idim); - all_node_coords_interface(node_gid,idim) = node_coords_interface(node_gid,idim); + // active view scope; triggers host comms from updated data on device + { + const_host_vec_array node_velocities_host = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_host_vec_array node_coords_host = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); } + double comm_time2 = Explicit_Solver_Pointer_->CPU_Time(); + Explicit_Solver_Pointer_->dev2host_time += comm_time2 - comm_time1; + + // communicate ghost velocities + Explicit_Solver_Pointer_->comm_velocities(); + Explicit_Solver_Pointer_->comm_coordinates(); + + double comm_time3 = Explicit_Solver_Pointer_->CPU_Time(); + + // view scope + { + const_vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array ghost_node_velocities_interface = Explicit_Solver_Pointer_->ghost_node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); + vec_array all_node_velocities_interface = Explicit_Solver_Pointer_->all_node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); + const_vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array ghost_node_coords_interface = Explicit_Solver_Pointer_->ghost_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + all_node_velocities_interface(node_gid, idim) = node_velocities_interface(node_gid, idim); + all_node_coords_interface(node_gid, idim) = node_coords_interface(node_gid, idim); + } }); // end parallel for - Kokkos::fence(); + Kokkos::fence(); - FOR_ALL_CLASS(node_gid, nlocal_nodes, nlocal_nodes+nghost_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - all_node_velocities_interface(node_gid,idim) = ghost_node_velocities_interface(node_gid-nlocal_nodes,idim); - all_node_coords_interface(node_gid,idim) = ghost_node_coords_interface(node_gid-nlocal_nodes,idim); - } + FOR_ALL_CLASS(node_gid, nlocal_nodes, nlocal_nodes + nghost_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + all_node_velocities_interface(node_gid, idim) = ghost_node_velocities_interface(node_gid - nlocal_nodes, idim); + all_node_coords_interface(node_gid, idim) = ghost_node_coords_interface(node_gid - nlocal_nodes, idim); + } }); // end parallel for - Kokkos::fence(); - } //end view scope - - - (*forward_solve_velocity_data)[0]->assign(*Explicit_Solver_Pointer_->all_node_velocities_distributed); - (*forward_solve_coordinate_data)[0]->assign(*Explicit_Solver_Pointer_->all_node_coords_distributed); - } - - // loop over the max number of time integration cycles - for (cycle = 0; cycle < cycle_stop; cycle++) { - - // get the step - if(num_dim==2){ + Kokkos::fence(); + } // end view scope + + (*forward_solve_velocity_data)[0]->assign(*Explicit_Solver_Pointer_->all_node_velocities_distributed); + (*forward_solve_coordinate_data)[0]->assign(*Explicit_Solver_Pointer_->all_node_coords_distributed); + } + + // loop over the max number of time integration cycles + for (cycle = 0; cycle < cycle_stop; cycle++) + { + // get the step + if (num_dim == 2) + { get_timestep2D(*mesh, node_coords, node_vel, elem_sspd, elem_vol); } - else { + else + { get_timestep(*mesh, node_coords, node_vel, @@ -1923,29 +1989,35 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve(){ } // end if 2D double global_dt; - MPI_Allreduce(&dt,&global_dt,1,MPI_DOUBLE,MPI_MIN,MPI_COMM_WORLD); + MPI_Allreduce(&dt, &global_dt, 1, MPI_DOUBLE, MPI_MIN, MPI_COMM_WORLD); dt = global_dt; - // stop calculation if flag - //if (stop_calc == 1) break; - - - if(simparam->dynamic_options.output_time_sequence_level>=TIME_OUTPUT_LEVEL::high){ - if (cycle==0){ - if(myrank==0) - printf("cycle = %lu, time = %12.5e, time step = %12.5e \n", cycle, time_value, dt); + // stop calculation if flag + // if (stop_calc == 1) break; + + if (simparam->dynamic_options.output_time_sequence_level >= TIME_OUTPUT_LEVEL::high) + { + if (cycle == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %12.5e, time step = %12.5e \n", cycle, time_value, dt); + } + } + // print time step every 10 cycles + else if (cycle % 20 == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %12.5e, time step = %12.5e \n", cycle, time_value, dt); + } + } // end if } - // print time step every 10 cycles - else if (cycle%20==0){ - if(myrank==0) - printf("cycle = %lu, time = %12.5e, time step = %12.5e \n", cycle, time_value, dt); - } // end if - } - + // --------------------------------------------------------------------- // integrate the solution forward to t(n+1) via Runge Kutta (RK) method // --------------------------------------------------------------------- - + // save the values at t_n rk_init(node_coords, node_vel, @@ -1953,18 +2025,15 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve(){ elem_stress, rnum_elem, nall_nodes); - - - - // integrate solution forward in time - for (size_t rk_stage = 0; rk_stage < rk_num_stages; rk_stage++){ - + // integrate solution forward in time + for (size_t rk_stage = 0; rk_stage < rk_num_stages; rk_stage++) + { // ---- RK coefficient ---- - double rk_alpha = 1.0/((double)rk_num_stages - (double)rk_stage); - + double rk_alpha = 1.0 / ((double)rk_num_stages - (double)rk_stage); + // ---- Calculate velocity diveregence for the element ---- - get_divergence(elem_div, + get_divergence(elem_div, *mesh, node_coords, node_vel, @@ -1979,10 +2048,10 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve(){ for (size_t dim = 0; dim < num_dim; dim++){ node_force[dim] = 0.0; } // end for dim - + // loop over all corners around the node and calculate the nodal force for (size_t corner_lid=0; corner_lidall_node_map->getGlobalElement(i) << " " << corner_gid << " " << corner_force(corner_gid, 0) << " " << corner_force(corner_gid, 1) << " " << corner_force(corner_gid, 2) << std::endl; @@ -1990,7 +2059,7 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve(){ for (size_t dim = 0; dim < num_dim; dim++){ node_force[dim] += corner_force(corner_gid, dim); } // end for dim - + } // end for corner_lid //std::cout << Explicit_Solver_Pointer_->all_node_map->getGlobalElement(i) << " " << node_force[0] << " " << node_force[1] << " " << node_force[2] << std::endl; //std::cout << Explicit_Solver_Pointer_->all_node_map->getGlobalElement(i) << " " << node_mass(i) << std::endl; @@ -2018,7 +2087,7 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve(){ corner_force, rk_alpha, cycle); - applied_forces(material, + applied_forces(material, *mesh, node_coords, node_vel, @@ -2030,73 +2099,69 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve(){ corner_force, rk_alpha, cycle); - - + // ---- apply force boundary conditions to the boundary patches---- boundary_velocity(*mesh, boundary, node_vel); - //current interface has differing velocity arrays; this equates them until we unify memory - //first comm time interval point + // current interface has differing velocity arrays; this equates them until we unify memory + // first comm time interval point double comm_time1 = Explicit_Solver_Pointer_->CPU_Time(); - //view scope + // view scope { - vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - node_velocities_interface(node_gid,idim) = node_vel(rk_level,node_gid,idim); - } + vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + node_velocities_interface(node_gid, idim) = node_vel(rk_level, node_gid, idim); + } }); // end parallel for - } //end view scope + } // end view scope Kokkos::fence(); - - //active view scope + + // active view scope { - const_host_vec_array node_velocities_host = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView (Tpetra::Access::ReadOnly); + const_host_vec_array node_velocities_host = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); } double comm_time2 = Explicit_Solver_Pointer_->CPU_Time(); - Explicit_Solver_Pointer_->dev2host_time += comm_time2-comm_time1; - //communicate ghost velocities + Explicit_Solver_Pointer_->dev2host_time += comm_time2 - comm_time1; + // communicate ghost velocities Explicit_Solver_Pointer_->comm_velocities(); - + double comm_time3 = Explicit_Solver_Pointer_->CPU_Time(); - //this is forcing a copy to the device - //view scope + // this is forcing a copy to the device + // view scope { - vec_array ghost_node_velocities_interface = Explicit_Solver_Pointer_->ghost_node_velocities_distributed->getLocalView (Tpetra::Access::ReadWrite); + vec_array ghost_node_velocities_interface = Explicit_Solver_Pointer_->ghost_node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, nlocal_nodes, nall_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - node_vel(rk_level,node_gid,idim) = ghost_node_velocities_interface(node_gid-nlocal_nodes,idim); - } - + FOR_ALL_CLASS(node_gid, nlocal_nodes, nall_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + node_vel(rk_level, node_gid, idim) = ghost_node_velocities_interface(node_gid - nlocal_nodes, idim); + } }); // end parallel for - } //end view scope + } // end view scope Kokkos::fence(); - + double comm_time4 = Explicit_Solver_Pointer_->CPU_Time(); - Explicit_Solver_Pointer_->host2dev_time += comm_time4-comm_time3; - Explicit_Solver_Pointer_->communication_time += comm_time4-comm_time1; - //debug print vector values on a rank + Explicit_Solver_Pointer_->host2dev_time += comm_time4 - comm_time3; + Explicit_Solver_Pointer_->communication_time += comm_time4 - comm_time1; + // debug print vector values on a rank /* if(myrank==0) for(int i = 0; i < nall_nodes; i++){ std::cout << Explicit_Solver_Pointer_->all_node_map->getGlobalElement(i) << " " << node_vel(rk_level,i,0) << " " << node_vel(rk_level,i,1) << " " << node_vel(rk_level,i,2) << std::endl; } - */ - - + */ + // ---- Update nodal positions ---- update_position_elastic(rk_alpha, nall_nodes, node_coords, node_vel); - - + // ---- Calculate cell volume for next time step ---- get_vol(); - - - + // ---- Calculate elem state (den, pres, sound speed, stress) for next time step ---- /* if(num_dim==2){ @@ -2138,48 +2203,45 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve(){ // 2) hypo-elastic strength models are called in get_force // 3) strength models must be added by the user in user_mat.cpp - // calculate the new corner masses if 2D - if(num_dim==2){ - + if (num_dim == 2) + { // calculate the nodal areal mass FOR_ALL_CLASS(node_gid, 0, nall_nodes, { - node_mass(node_gid) = 0.0; - - if (node_coords(rk_level,node_gid,1) > tiny){ - node_mass(node_gid) = node_extensive_mass(node_gid)/node_coords(rk_level,node_gid,1); - } - //if(cycle==0&&node_gid==1&&myrank==0) - //std::cout << "index " << node_gid << " on rank " << myrank << " node vel " << node_vel(rk_level,node_gid,0) << " " << node_mass(node_gid) << std::endl << std::flush; + if (node_coords(rk_level, node_gid, 1) > tiny) + { + node_mass(node_gid) = node_extensive_mass(node_gid) / node_coords(rk_level, node_gid, 1); + } + // if(cycle==0&&node_gid==1&&myrank==0) + // std::cout << "index " << node_gid << " on rank " << myrank << " node vel " << node_vel(rk_level,node_gid,0) << " " << node_mass(node_gid) << std::endl << std::flush; }); // end parallel for over node_gid Kokkos::fence(); - //current interface has differing density arrays; this equates them until we unify memory - //view scope + // current interface has differing density arrays; this equates them until we unify memory + // view scope { - vec_array node_mass_interface = node_masses_distributed->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - node_mass_interface(node_gid,0) = node_mass(node_gid); + vec_array node_mass_interface = node_masses_distributed->getLocalView(Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + node_mass_interface(node_gid, 0) = node_mass(node_gid); }); // end parallel for - } //end view scope + } // end view scope Kokkos::fence(); - //communicate ghost densities + // communicate ghost densities comm_node_masses(); - //this is forcing a copy to the device - //view scope + // this is forcing a copy to the device + // view scope { - vec_array ghost_node_mass_interface = ghost_node_masses_distributed->getLocalView (Tpetra::Access::ReadWrite); + vec_array ghost_node_mass_interface = ghost_node_masses_distributed->getLocalView(Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, nlocal_nodes, nall_nodes, { - node_mass(node_gid) = ghost_node_mass_interface(node_gid-nlocal_nodes,0); + FOR_ALL_CLASS(node_gid, nlocal_nodes, nall_nodes, { + node_mass(node_gid) = ghost_node_mass_interface(node_gid - nlocal_nodes, 0); }); // end parallel for - } //end view scope + } // end view scope Kokkos::fence(); - - + // ----------------------------------------------- // Calcualte the areal mass for nodes on the axis // ----------------------------------------------- @@ -2192,318 +2254,334 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve(){ // 0---1 /* FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { - + // loop over the corners of the element and calculate the mass for (size_t node_lid=0; node_lid<4; node_lid++){ - + size_t node_gid = nodes_in_elem(elem_gid, node_lid); size_t node_minus_gid; size_t node_plus_gid; - - + + if (node_coords(rk_level,node_gid,1) < tiny){ // node is on the axis - + // minus node if (node_lid==0){ node_minus_gid = nodes_in_elem(elem_gid, 3); } else { node_minus_gid = nodes_in_elem(elem_gid, node_lid-1); } - + // plus node if (node_lid==3){ node_plus_gid = nodes_in_elem(elem_gid, 0); } else { node_plus_gid = nodes_in_elem(elem_gid, node_lid+1); } - + node_mass(node_gid) = fmax(node_mass(node_plus_gid), node_mass(node_minus_gid))/2.0; - + } // end if - + } // end for over corners - + }); // end parallel for over elem_gid Kokkos::fence(); */ - + FOR_ALL_CLASS(node_bdy_gid, 0, num_bdy_nodes, { - //FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + // FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { size_t node_gid = bdy_nodes(node_bdy_gid); - - if (node_coords(rk_level,node_gid,1) < tiny){ + + if (node_coords(rk_level, node_gid, 1) < tiny) + { // node is on the axis - - for(size_t node_lid=0; node_lid < num_nodes_in_node(node_gid); node_lid++){ - + + for (size_t node_lid = 0; node_lid < num_nodes_in_node(node_gid); node_lid++) + { size_t node_neighbor_gid = nodes_in_node(node_gid, node_lid); - + // if the node is off the axis, use it's areal mass on the boundary - if (node_coords(rk_level,node_neighbor_gid,1) > tiny){ - node_mass(node_gid) = fmax(node_mass(node_gid), node_mass(node_neighbor_gid)/2.0); + if (node_coords(rk_level, node_neighbor_gid, 1) > tiny) + { + node_mass(node_gid) = fmax(node_mass(node_gid), node_mass(node_neighbor_gid) / 2.0); } - } // end for over neighboring nodes - } // end if - }); // end parallel for over elem_gid - } // end of if 2D-RZ + } // end of RK loop + // increment the time + Explicit_Solver_Pointer_->time_value = simparam->dynamic_options.time_value = time_value += dt; + + if (simparam->topology_optimization_on || simparam->shape_optimization_on) + { + if (cycle >= max_time_steps) + { + max_time_steps = cycle + 1; + } - } // end of RK loop + if (max_time_steps + 1 > forward_solve_velocity_data->size()) + { + old_max_forward_buffer = forward_solve_velocity_data->size(); + time_data.resize(max_time_steps + BUFFER_GROW + 1); + forward_solve_velocity_data->resize(max_time_steps + BUFFER_GROW + 1); + forward_solve_coordinate_data->resize(max_time_steps + BUFFER_GROW + 1); + adjoint_vector_data->resize(max_time_steps + BUFFER_GROW + 1); + phi_adjoint_vector_data->resize(max_time_steps + BUFFER_GROW + 1); + // assign a multivector of corresponding size to each new timestep in the buffer + for (int istep = old_max_forward_buffer; istep < max_time_steps + BUFFER_GROW + 1; istep++) + { + (*forward_solve_velocity_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*forward_solve_coordinate_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*phi_adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + } + } - // increment the time - Explicit_Solver_Pointer_->time_value = simparam->dynamic_options.time_value = time_value+=dt; + time_data[cycle + 1] = dt + time_data[cycle]; - if(simparam->topology_optimization_on||simparam->shape_optimization_on){ - if(cycle >= max_time_steps) - max_time_steps = cycle + 1; + // assign current velocity data to multivector + // view scope + { + vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); + vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + node_velocities_interface(node_gid, idim) = node_vel(rk_level, node_gid, idim); + node_coords_interface(node_gid, idim) = node_coords(rk_level, node_gid, idim); + } + }); + } // end view scope + Kokkos::fence(); - if(max_time_steps + 1 > forward_solve_velocity_data->size()){ - old_max_forward_buffer = forward_solve_velocity_data->size(); - time_data.resize(max_time_steps + BUFFER_GROW +1); - forward_solve_velocity_data->resize(max_time_steps + BUFFER_GROW +1); - forward_solve_coordinate_data->resize(max_time_steps + BUFFER_GROW +1); - adjoint_vector_data->resize(max_time_steps + BUFFER_GROW +1); - phi_adjoint_vector_data->resize(max_time_steps + BUFFER_GROW +1); - //assign a multivector of corresponding size to each new timestep in the buffer - for(int istep = old_max_forward_buffer; istep < max_time_steps + BUFFER_GROW +1; istep++){ - (*forward_solve_velocity_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - (*forward_solve_coordinate_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - (*adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - (*phi_adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - } - } + // communicate ghosts + double comm_time1 = Explicit_Solver_Pointer_->CPU_Time(); - - time_data[cycle+1] = dt + time_data[cycle]; - - - //assign current velocity data to multivector - //view scope - { - vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView (Tpetra::Access::ReadWrite); - vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - node_velocities_interface(node_gid,idim) = node_vel(rk_level,node_gid,idim); - node_coords_interface(node_gid,idim) = node_coords(rk_level,node_gid,idim); + // active view scope; triggers host comms from updated data on device + { + const_host_vec_array node_velocities_host = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_host_vec_array node_coords_host = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); } - }); - } //end view scope - Kokkos::fence(); + double comm_time2 = Explicit_Solver_Pointer_->CPU_Time(); + Explicit_Solver_Pointer_->dev2host_time += comm_time2 - comm_time1; - //communicate ghosts - double comm_time1 = Explicit_Solver_Pointer_->CPU_Time(); - - //active view scope; triggers host comms from updated data on device - { - const_host_vec_array node_velocities_host = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_host_vec_array node_coords_host = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - } - double comm_time2 = Explicit_Solver_Pointer_->CPU_Time(); - Explicit_Solver_Pointer_->dev2host_time += comm_time2-comm_time1; + // communicate ghost velocities + Explicit_Solver_Pointer_->comm_velocities(); + Explicit_Solver_Pointer_->comm_coordinates(); - //communicate ghost velocities - Explicit_Solver_Pointer_->comm_velocities(); - Explicit_Solver_Pointer_->comm_coordinates(); - - - double comm_time3 = Explicit_Solver_Pointer_->CPU_Time(); + double comm_time3 = Explicit_Solver_Pointer_->CPU_Time(); - //view scope - { - const_vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array ghost_node_velocities_interface = Explicit_Solver_Pointer_->ghost_node_velocities_distributed->getLocalView (Tpetra::Access::ReadOnly); - vec_array all_node_velocities_interface = Explicit_Solver_Pointer_->all_node_velocities_distributed->getLocalView (Tpetra::Access::ReadWrite); - const_vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array ghost_node_coords_interface = Explicit_Solver_Pointer_->ghost_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - all_node_velocities_interface(node_gid,idim) = node_velocities_interface(node_gid,idim); - all_node_coords_interface(node_gid,idim) = node_coords_interface(node_gid,idim); - } + // view scope + { + const_vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array ghost_node_velocities_interface = Explicit_Solver_Pointer_->ghost_node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); + vec_array all_node_velocities_interface = Explicit_Solver_Pointer_->all_node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); + const_vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array ghost_node_coords_interface = Explicit_Solver_Pointer_->ghost_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + all_node_velocities_interface(node_gid, idim) = node_velocities_interface(node_gid, idim); + all_node_coords_interface(node_gid, idim) = node_coords_interface(node_gid, idim); + } }); // end parallel for - Kokkos::fence(); + Kokkos::fence(); - FOR_ALL_CLASS(node_gid, nlocal_nodes, nlocal_nodes+nghost_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - all_node_velocities_interface(node_gid,idim) = ghost_node_velocities_interface(node_gid-nlocal_nodes,idim); - all_node_coords_interface(node_gid,idim) = ghost_node_coords_interface(node_gid-nlocal_nodes,idim); - } + FOR_ALL_CLASS(node_gid, nlocal_nodes, nlocal_nodes + nghost_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + all_node_velocities_interface(node_gid, idim) = ghost_node_velocities_interface(node_gid - nlocal_nodes, idim); + all_node_coords_interface(node_gid, idim) = ghost_node_coords_interface(node_gid - nlocal_nodes, idim); + } }); // end parallel for - Kokkos::fence(); - } //end view scope - - double comm_time4 = Explicit_Solver_Pointer_->CPU_Time(); - Explicit_Solver_Pointer_->host2dev_time += comm_time4-comm_time3; - Explicit_Solver_Pointer_->communication_time += comm_time4-comm_time1; - - (*forward_solve_velocity_data)[cycle+1]->assign(*Explicit_Solver_Pointer_->all_node_velocities_distributed); - (*forward_solve_coordinate_data)[cycle+1]->assign(*Explicit_Solver_Pointer_->all_node_coords_distributed); - - //kinetic energy accumulation - if(kinetic_energy_objective){ - const_vec_array node_velocities_interface = (*forward_solve_velocity_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array previous_node_velocities_interface = (*forward_solve_velocity_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - KE_loc_sum = 0.0; - KE_sum = 0.0; - // extensive KE - REDUCE_SUM_CLASS(node_gid, 0, nlocal_nodes, KE_loc_sum, { - - double ke = 0; - for (size_t dim=0; dimCPU_Time(); + Explicit_Solver_Pointer_->host2dev_time += comm_time4 - comm_time3; + Explicit_Solver_Pointer_->communication_time += comm_time4 - comm_time1; + + (*forward_solve_velocity_data)[cycle + 1]->assign(*Explicit_Solver_Pointer_->all_node_velocities_distributed); + (*forward_solve_coordinate_data)[cycle + 1]->assign(*Explicit_Solver_Pointer_->all_node_coords_distributed); + + // kinetic energy accumulation + if (kinetic_energy_objective) + { + const_vec_array node_velocities_interface = (*forward_solve_velocity_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array previous_node_velocities_interface = (*forward_solve_velocity_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + KE_loc_sum = 0.0; + KE_sum = 0.0; + // extensive KE + REDUCE_SUM_CLASS(node_gid, 0, nlocal_nodes, KE_loc_sum, { + double ke = 0; + for (size_t dim = 0; dim < num_dim; dim++) + { + // midpoint integration approximation + ke += (node_velocities_interface(node_gid, dim) + previous_node_velocities_interface(node_gid, dim)) * (node_velocities_interface(node_gid, + dim) + previous_node_velocities_interface(node_gid, dim)) / 4; // 1/2 at end + } // end for + + if (num_dim == 2) + { + KE_loc_sum += node_mass(node_gid) * node_coords(rk_level, node_gid, 1) * ke; + } + else + { + KE_loc_sum += node_mass(node_gid) * ke; + } }, KE_sum); - Kokkos::fence(); - KE_sum = 0.5*KE_sum; - objective_accumulation += KE_sum*dt; + Kokkos::fence(); + KE_sum = 0.5 * KE_sum; + objective_accumulation += KE_sum * dt; + } + } + + size_t write = 0; + if ((cycle + 1) % graphics_cyc_ival == 0 && cycle > 0) + { + write = 1; } - } - - size_t write = 0; - if ((cycle+1)%graphics_cyc_ival == 0 && cycle>0){ - write = 1; - } - else if (cycle == cycle_stop) { - write = 1; - } - else if (time_value >= time_final&&simparam->output_options.write_final){ - write = 1; - } - else if (time_value >= graphics_time){ - write = 1; - } - + else if (cycle == cycle_stop) + { + write = 1; + } + else if (time_value >= time_final && simparam->output_options.write_final) + { + write = 1; + } + else if (time_value >= graphics_time) + { + write = 1; + } + // write outputs - if (write == 1){ - //interface nodal coordinate data (note: this is not needed if using write_outputs()) - //view scope + if (write == 1) + { + // interface nodal coordinate data (note: this is not needed if using write_outputs()) + // view scope { - vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - node_coords_interface(node_gid,idim) = node_coords(rk_level,node_gid,idim); - } + vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + node_coords_interface(node_gid, idim) = node_coords(rk_level, node_gid, idim); + } }); // end parallel for - } //end view scope - if(simparam->output_options.output_file_format==OUTPUT_FORMAT::vtk){ - if(myrank==0){ - printf("Writing outputs to file at %f \n", graphics_time); - } + } // end view scope + if (simparam->output_options.output_file_format == OUTPUT_FORMAT::vtk) + { + if (myrank == 0) + { + printf("Writing outputs to file at %f \n", graphics_time); + } - double comm_time1 = Explicit_Solver_Pointer_->CPU_Time(); - Explicit_Solver_Pointer_->write_outputs(); + double comm_time1 = Explicit_Solver_Pointer_->CPU_Time(); + Explicit_Solver_Pointer_->write_outputs(); - double comm_time2 = Explicit_Solver_Pointer_->CPU_Time(); - Explicit_Solver_Pointer_->output_time += comm_time2 - comm_time1; + double comm_time2 = Explicit_Solver_Pointer_->CPU_Time(); + Explicit_Solver_Pointer_->output_time += comm_time2 - comm_time1; } graphics_time = time_value + graphics_dt_ival; - } // end if - - - // end of calculation - if (time_value>=time_final) break; + } // end if - + // end of calculation + if (time_value >= time_final) + { + break; + } } // end for cycle loop last_time_step = cycle; - //simple setup to just calculate KE minimize objective for now - if(simparam->topology_optimization_on){ - KineticEnergyMinimize_TopOpt& kinetic_energy_minimize_function = dynamic_cast(*obj_pointer); + // simple setup to just calculate KE minimize objective for now + if (simparam->topology_optimization_on) + { + KineticEnergyMinimize_TopOpt& kinetic_energy_minimize_function = dynamic_cast(*obj_pointer); - //collect local objective values - MPI_Allreduce(&objective_accumulation,&global_objective_accumulation,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD); - kinetic_energy_minimize_function.objective_accumulation = global_objective_accumulation; + // collect local objective values + MPI_Allreduce(&objective_accumulation, &global_objective_accumulation, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); + kinetic_energy_minimize_function.objective_accumulation = global_objective_accumulation; - if(myrank==0) - std::cout << "CURRENT TIME INTEGRAL OF KINETIC ENERGY " << global_objective_accumulation << std::endl; + if (myrank == 0) + { + std::cout << "CURRENT TIME INTEGRAL OF KINETIC ENERGY " << global_objective_accumulation << std::endl; + } } - - + auto time_2 = std::chrono::high_resolution_clock::now(); auto time_difference = time_2 - time_1; - //double calc_time = std::chrono::duration_cast(diff).count(); + // double calc_time = std::chrono::duration_cast(diff).count(); double calc_time = std::chrono::duration_cast(time_difference).count(); - if(myrank==0) - printf("\nCalculation time in seconds: %f \n", calc_time*1e-09); - + if (myrank == 0) + { + printf("\nCalculation time in seconds: %f \n", calc_time * 1e-09); + } + IE_loc_sum = 0.0; KE_loc_sum = 0.0; - IE_sum = 0.0; - KE_sum = 0.0; - + IE_sum = 0.0; + KE_sum = 0.0; + // extensive IE REDUCE_SUM_CLASS(elem_gid, 0, nlocal_elem_non_overlapping, IE_loc_sum, { - - IE_loc_sum += elem_mass(elem_gid)*elem_sie(rk_level,elem_gid); - + IE_loc_sum += elem_mass(elem_gid) * elem_sie(rk_level, elem_gid); }, IE_sum); IE_tend = IE_sum; - //reduce over MPI ranks - MPI_Allreduce(&IE_tend,&global_IE_tend,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD); + // reduce over MPI ranks + MPI_Allreduce(&IE_tend, &global_IE_tend, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); // extensive KE REDUCE_SUM_CLASS(node_gid, 0, nlocal_nodes, KE_loc_sum, { - double ke = 0; - for (size_t dim=0; dim mesh_in, const int my_fea_module_index = 0); - ~FEA_Module_Dynamic_Elasticity(); - - //initialize data for boundaries of the model and storage for boundary conditions and applied loads - void elastic_interface_setup(node_t &node, elem_t &elem, corner_t &corner); - - void setup(); - - void cleanup_material_models(); - - int solve(); - - void module_cleanup(); - - void elastic_solve(); - - void get_force_vgradient_elastic(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &elem_mat_id, - const double rk_alpha, - const size_t cycle); - - void get_force_ugradient_elastic(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &elem_mat_id, - const double rk_alpha, - const size_t cycle); - - void get_force_dgradient_elastic(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &elem_mat_id, - const double rk_alpha, - const size_t cycle); - - void force_design_gradient_term(const_vec_array design_variables, vec_array design_gradients); - - void update_position_elastic(double rk_alpha, - const size_t num_nodes, - DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel); - - void get_vol(); - - void init_assembly(); - - KOKKOS_INLINE_FUNCTION - void get_vol_hex(const DViewCArrayKokkos &elem_vol, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const; - - - KOKKOS_INLINE_FUNCTION - void get_vol_quad(const DViewCArrayKokkos &elem_vol, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const; - - - KOKKOS_FUNCTION - double get_area_quad(const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const; - - - KOKKOS_FUNCTION - void get_bmatrix(const ViewCArrayKokkos &B_matrix, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const; - - - KOKKOS_FUNCTION - void get_bmatrix2D(const ViewCArrayKokkos &B_matrix, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const; - - KOKKOS_FUNCTION - void get_area_weights2D(const ViewCArrayKokkos &corner_areas, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const; - - - KOKKOS_INLINE_FUNCTION - double heron(const double x1, - const double y1, - const double x2, - const double y2, - const double x3, - const double y3) const; - - double average_element_density(const int nodes_per_elem, const CArray current_element_densities) const; - - void get_divergence(DViewCArrayKokkos &elem_div, - const mesh_t mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_vol); - - - void get_divergence2D(DViewCArrayKokkos &elem_div, - const mesh_t mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_vol); - - - KOKKOS_FUNCTION - void get_velgrad(ViewCArrayKokkos &vel_grad, - const ViewCArrayKokkos &elem_node_gids, - const DViewCArrayKokkos &node_vel, - const ViewCArrayKokkos &b_matrix, - const double elem_vol, - const size_t elem_gid, - const size_t rk_level) const; - - - KOKKOS_FUNCTION - void get_velgrad2D(ViewCArrayKokkos &vel_grad, - const ViewCArrayKokkos &elem_node_gids, - const DViewCArrayKokkos &node_vel, - const ViewCArrayKokkos &b_matrix, - const double elem_vol, - const double elem_area, - const size_t elem_gid, - const size_t rk_level) const; - - KOKKOS_INLINE_FUNCTION - void decompose_vel_grad(ViewCArrayKokkos &D_tensor, - ViewCArrayKokkos &W_tensor, - const ViewCArrayKokkos &vel_grad, - const ViewCArrayKokkos &elem_node_gids, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const double vol) const; - - void boundary_velocity(const mesh_t &mesh, - const DCArrayKokkos &boundary, - DViewCArrayKokkos &node_vel); - - void boundary_adjoint(const mesh_t &mesh, - const DCArrayKokkos &boundary, - vec_array &node_adjoint, - vec_array &node_phi_adjoint); - - void tag_bdys(const DCArrayKokkos &boundary, - mesh_t &mesh, - const DViewCArrayKokkos &node_coords); - - KOKKOS_INLINE_FUNCTION - bool check_bdy(const size_t patch_gid, - const int num_dim, - const int num_nodes_in_patch, - const BOUNDARY_TYPE this_bc_tag, - const double val, - const DViewCArrayKokkos &node_coords, - const size_t rk_level) const; - - void rk_init(DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &elem_sie, - DViewCArrayKokkos &elem_stress, - const size_t num_elems, - const size_t num_nodes); - - - void get_timestep(mesh_t &mesh, - DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &elem_sspd, - DViewCArrayKokkos &elem_vol); - - - void get_timestep2D(mesh_t &mesh, - DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &elem_sspd, - DViewCArrayKokkos &elem_vol); - - void update_state(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &elem_den, - DViewCArrayKokkos &elem_pres, - DViewCArrayKokkos &elem_stress, - DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_mass, - const DViewCArrayKokkos &elem_mat_id, - const double rk_alpha, - const size_t cycle); - - - void update_state2D(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &elem_den, - DViewCArrayKokkos &elem_pres, - DViewCArrayKokkos &elem_stress, - DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_mass, - const DViewCArrayKokkos &elem_mat_id, - const double rk_alpha, - const size_t cycle); - - void build_boundry_node_sets(mesh_t &mesh); - - void init_boundaries(); - - //initializes memory for arrays used in the global stiffness matrix assembly - void init_boundary_sets(int num_boundary_sets); - - void grow_boundary_sets(int num_boundary_sets); - - virtual void update_forward_solve(Teuchos::RCP zp); - - void comm_node_masses(); - - void comm_adjoint_vectors(int cycle); - - void comm_variables(Teuchos::RCP zp); - - void read_conditions_ansys_dat(std::ifstream *in, std::streampos before_condition_header); - - //interfaces between user input and creating data structures for bcs - void generate_bcs(); - - void Displacement_Boundary_Conditions(); - - void init_output(); - - void compute_output(); - - void sort_output(Teuchos::RCP > sorted_map); - - void sort_element_output(Teuchos::RCP > sorted_map); - - void collect_output(Teuchos::RCP > global_reduce_map); - - void write_data(std::map &point_data_scalars_double, - std::map &point_data_vectors_double, - std::map &cell_data_scalars_double, - std::map &cell_data_scalars_int, - std::map > &cell_data_fields_double); - - void write_outputs (const mesh_t &mesh, - DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &node_mass, - DViewCArrayKokkos &elem_den, - DViewCArrayKokkos &elem_pres, - DViewCArrayKokkos &elem_stress, - DViewCArrayKokkos &elem_sspd, - DViewCArrayKokkos &elem_sie, - DViewCArrayKokkos &elem_vol, - DViewCArrayKokkos &elem_mass, - DViewCArrayKokkos &elem_mat_id); - - - void ensight(const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &node_mass, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_mass, - const DViewCArrayKokkos &elem_mat_id); - - - void state_file(const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &node_mass, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_mass, - const DViewCArrayKokkos &elem_mat_id); - - void node_density_constraints(host_vec_array &node_densities_lower_bound); - - void compute_topology_optimization_adjoint(); //Force does not depend on node coords and velocity - - void compute_topology_optimization_adjoint_full(); //Force depends on node coords and velocity - - void compute_topology_optimization_gradient(const_vec_array design_densities, vec_array gradients); - - void compute_topology_optimization_gradient_full(Teuchos::RCP design_densities_distributed, Teuchos::RCP design_gradients_distributed); - - //elastic TO stuff - void get_force_elastic(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &node_mass, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &elem_mat_id, - DViewCArrayKokkos &corner_force, - const double rk_alpha, - const size_t cycle); - - void applied_forces(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &node_mass, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &elem_mat_id, - DViewCArrayKokkos &corner_force, - const double rk_alpha, - const size_t cycle); - - void Element_Material_Properties(size_t ielem, real_t &Element_Modulus, real_t &Poisson_Ratio, real_t density); - - void compute_stiffness_gradients(const_host_vec_array &design_densities, host_vec_array &gradients); - - void Gradient_Element_Material_Properties(size_t ielem, real_t &Element_Modulus, real_t &Poisson_Ratio, real_t density); - - void local_matrix_multiply(int ielem, CArrayKokkos &Local_Matrix); - - void assemble_matrix(); - - bool nodal_density_flag; - real_t penalty_power; - Teuchos::RCP Global_Stiffness_Matrix; - RaggedRightArrayKokkos Stiffness_Matrix; - DCArrayKokkos Stiffness_Matrix_Strides; - DCArrayKokkos Global_Stiffness_Matrix_Assembly_Map; - //end elastic TO data - - Dynamic_Elasticity_Parameters *module_params; - Simulation_Parameters_Explicit *simparam; - Explicit_Solver *Explicit_Solver_Pointer_; - - elements::ref_element *ref_elem; - - std::shared_ptr mesh; - //shallow copies of mesh class views - size_t num_nodes_in_elem; - // corner ids in node - RaggedRightArrayKokkos corners_in_node; - CArrayKokkos num_corners_in_node; - - // elem ids in node - RaggedRightArrayKokkos elems_in_node; - - // node ids in node - RaggedRightArrayKokkos nodes_in_node; - CArrayKokkos num_nodes_in_node; - - // node ids in elem - DCArrayKokkos nodes_in_elem; - - // corner ids in elem - CArrayKokkos corners_in_elem; - - // elem ids in elem - RaggedRightArrayKokkos elems_in_elem; - CArrayKokkos num_elems_in_elem; - - // patch ids in elem - CArrayKokkos patches_in_elem; - - // node ids in a patch - CArrayKokkos nodes_in_patch; - - // element ids in a patch - CArrayKokkos elems_in_patch; - - // bdy nodes - CArrayKokkos bdy_nodes; - - //Topology optimization filter variable - DCArrayKokkos relative_element_densities; - - //Local FEA data - host_elem_conn_array interface_nodes_in_elem; //host view of element connectivity to nodes - - //Global FEA data - Teuchos::RCP node_velocities_distributed; - Teuchos::RCP initial_node_coords_distributed; - Teuchos::RCP all_initial_node_coords_distributed; - Teuchos::RCP initial_node_velocities_distributed; - Teuchos::RCP all_node_velocities_distributed; - Teuchos::RCP all_cached_node_velocities_distributed; - Teuchos::RCP node_masses_distributed; - Teuchos::RCP ghost_node_masses_distributed; - Teuchos::RCP adjoint_vector_distributed; - Teuchos::RCP phi_adjoint_vector_distributed; - Teuchos::RCP>> forward_solve_velocity_data; - Teuchos::RCP>> forward_solve_coordinate_data; - Teuchos::RCP>> adjoint_vector_data; - Teuchos::RCP>> phi_adjoint_vector_data; - Teuchos::RCP force_gradient_design; - Teuchos::RCP force_gradient_position; - Teuchos::RCP force_gradient_velocity; - - //Local FEA data - DCArrayKokkos Global_Gradient_Matrix_Assembly_Map; - RaggedRightArrayKokkos Graph_Matrix; //stores global indices - RaggedRightArrayKokkos DOF_Graph_Matrix; //stores global indices - RaggedRightArrayKokkos Force_Gradient_Positions; - RaggedRightArrayKokkos Force_Gradient_Velocities; - DCArrayKokkos Gradient_Matrix_Strides; - DCArrayKokkos Graph_Matrix_Strides; - RaggedRightArrayKokkos Original_Gradient_Entries; - RaggedRightArrayKokkos Original_Gradient_Entry_Indices; - DCArrayKokkos Original_Gradient_Entries_Strides; - - //distributed matrices - Teuchos::RCP distributed_force_gradient_positions; - Teuchos::RCP distributed_force_gradient_velocities; - - std::vector time_data; - unsigned long max_time_steps, last_time_step; - - // --------------------------------------------------------------------- - // state data type declarations (must stay in scope for output after run) - // --------------------------------------------------------------------- - node_t node_interface; - elem_t elem_interface; - corner_t corner_interface; - - //Dual View wrappers - // Dual Views of the individual node struct variables - DViewCArrayKokkos node_coords; - DViewCArrayKokkos node_vel; - DViewCArrayKokkos node_mass; - - // Dual Views of the individual elem struct variables - DViewCArrayKokkos elem_den; - DViewCArrayKokkos elem_pres; - DViewCArrayKokkos elem_stress; // always 3D even in 2D-RZ - DViewCArrayKokkos elem_sspd; - DViewCArrayKokkos elem_sie; - DViewCArrayKokkos elem_vol; - DViewCArrayKokkos elem_div; - DViewCArrayKokkos elem_mass; - DViewCArrayKokkos elem_mat_id; - - // Element velocity gradient - DCArrayKokkos elem_vel_grad; - - // for storing global variables used in user material model - DCArrayKokkos global_vars; - - // for storing state variables used in user material model - DCArrayKokkos state_vars; - - //elem_user_output_vars allow users to output variables of interest per element - DCArrayKokkos elem_user_output_vars; - - //material models - DCArrayKokkos elem_eos; - DCArrayKokkos elem_strength; - - // Dual Views of the corner struct variables - DViewCArrayKokkos corner_force; - DViewCArrayKokkos corner_mass; - - //Boundary Conditions Data - DCArrayKokkos Local_Index_Boundary_Patches; - //CArray Patch_Nodes; - enum bc_type {NONE, POINT_LOADING_CONDITION, LINE_LOADING_CONDITION, SURFACE_LOADING_CONDITION}; - - //Boundary Conditions Data - int max_boundary_sets; - - //output dof data - //Global arrays with collected data used to print - int output_velocity_index, output_strain_index, output_stress_index; - - //parameters - double time_value, time_final, dt, dt_max, dt_min, dt_cfl, graphics_time, graphics_dt_ival; - size_t graphics_cyc_ival, cycle_stop, rk_num_stages, graphics_id; - double fuzz, tiny, small; - CArray graphics_times; - int rk_num_bins; - - //optimization flags - bool kinetic_energy_objective; + + FEA_Module_Dynamic_Elasticity(Dynamic_Elasticity_Parameters& params, Solver* Solver_Pointer, std::shared_ptr mesh_in, const int my_fea_module_index = 0); + ~FEA_Module_Dynamic_Elasticity(); + + // initialize data for boundaries of the model and storage for boundary conditions and applied loads + void elastic_interface_setup(node_t& node, elem_t& elem, corner_t& corner); + + void setup(); + + void cleanup_material_models(); + + int solve(); + + void module_cleanup(); + + void elastic_solve(); + + void get_force_vgradient_elastic(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_pres, + const DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + const double rk_alpha, + const size_t cycle); + + void get_force_ugradient_elastic(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_pres, + const DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + const double rk_alpha, + const size_t cycle); + + void get_force_dgradient_elastic(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_pres, + const DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + const double rk_alpha, + const size_t cycle); + + void force_design_gradient_term(const_vec_array design_variables, vec_array design_gradients); + + void update_position_elastic(double rk_alpha, + const size_t num_nodes, + DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel); + + void get_vol(); + + void init_assembly(); + + KOKKOS_INLINE_FUNCTION + void get_vol_hex(const DViewCArrayKokkos& elem_vol, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const; + + KOKKOS_INLINE_FUNCTION + void get_vol_quad(const DViewCArrayKokkos& elem_vol, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const; + + KOKKOS_FUNCTION + double get_area_quad(const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const; + + KOKKOS_FUNCTION + void get_bmatrix(const ViewCArrayKokkos& B_matrix, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const; + + KOKKOS_FUNCTION + void get_bmatrix2D(const ViewCArrayKokkos& B_matrix, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const; + + KOKKOS_FUNCTION + void get_area_weights2D(const ViewCArrayKokkos& corner_areas, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const; + + KOKKOS_INLINE_FUNCTION + double heron(const double x1, + const double y1, + const double x2, + const double y2, + const double x3, + const double y3) const; + + double average_element_density(const int nodes_per_elem, const CArray current_element_densities) const; + + void get_divergence(DViewCArrayKokkos& elem_div, + const mesh_t mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_vol); + + void get_divergence2D(DViewCArrayKokkos& elem_div, + const mesh_t mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_vol); + + KOKKOS_FUNCTION + void get_velgrad(ViewCArrayKokkos& vel_grad, + const ViewCArrayKokkos& elem_node_gids, + const DViewCArrayKokkos& node_vel, + const ViewCArrayKokkos& b_matrix, + const double elem_vol, + const size_t elem_gid, + const size_t rk_level) const; + + KOKKOS_FUNCTION + void get_velgrad2D(ViewCArrayKokkos& vel_grad, + const ViewCArrayKokkos& elem_node_gids, + const DViewCArrayKokkos& node_vel, + const ViewCArrayKokkos& b_matrix, + const double elem_vol, + const double elem_area, + const size_t elem_gid, + const size_t rk_level) const; + + KOKKOS_INLINE_FUNCTION + void decompose_vel_grad(ViewCArrayKokkos& D_tensor, + ViewCArrayKokkos& W_tensor, + const ViewCArrayKokkos& vel_grad, + const ViewCArrayKokkos& elem_node_gids, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const double vol) const; + + void boundary_velocity(const mesh_t& mesh, + const DCArrayKokkos& boundary, + DViewCArrayKokkos& node_vel); + + void boundary_adjoint(const mesh_t& mesh, + const DCArrayKokkos& boundary, + vec_array& node_adjoint, + vec_array& node_phi_adjoint); + + void tag_bdys(const DCArrayKokkos& boundary, + mesh_t& mesh, + const DViewCArrayKokkos& node_coords); + + KOKKOS_INLINE_FUNCTION + bool check_bdy(const size_t patch_gid, + const int num_dim, + const int num_nodes_in_patch, + const BOUNDARY_TYPE this_bc_tag, + const double val, + const DViewCArrayKokkos& node_coords, + const size_t rk_level) const; + + void rk_init(DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_sie, + DViewCArrayKokkos& elem_stress, + const size_t num_elems, + const size_t num_nodes); + + void get_timestep(mesh_t& mesh, + DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_sspd, + DViewCArrayKokkos& elem_vol); + + void get_timestep2D(mesh_t& mesh, + DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_sspd, + DViewCArrayKokkos& elem_vol); + + void update_state(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_den, + DViewCArrayKokkos& elem_pres, + DViewCArrayKokkos& elem_stress, + DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& elem_mat_id, + const double rk_alpha, + const size_t cycle); + + void update_state2D(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_den, + DViewCArrayKokkos& elem_pres, + DViewCArrayKokkos& elem_stress, + DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& elem_mat_id, + const double rk_alpha, + const size_t cycle); + + void build_boundry_node_sets(mesh_t& mesh); + + void init_boundaries(); + + // initializes memory for arrays used in the global stiffness matrix assembly + void init_boundary_sets(int num_boundary_sets); + + void grow_boundary_sets(int num_boundary_sets); + + virtual void update_forward_solve(Teuchos::RCP zp); + + void comm_node_masses(); + + void comm_adjoint_vectors(int cycle); + + void comm_variables(Teuchos::RCP zp); + + void read_conditions_ansys_dat(std::ifstream* in, std::streampos before_condition_header); + + // interfaces between user input and creating data structures for bcs + void generate_bcs(); + + void Displacement_Boundary_Conditions(); + + void init_output(); + + void compute_output(); + + void sort_output(Teuchos::RCP> sorted_map); + + void sort_element_output(Teuchos::RCP> sorted_map); + + void collect_output(Teuchos::RCP> global_reduce_map); + + void write_data(std::map& point_data_scalars_double, + std::map& point_data_vectors_double, + std::map& cell_data_scalars_double, + std::map& cell_data_scalars_int, + std::map>& cell_data_fields_double); + + void write_outputs(const mesh_t& mesh, + DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& node_mass, + DViewCArrayKokkos& elem_den, + DViewCArrayKokkos& elem_pres, + DViewCArrayKokkos& elem_stress, + DViewCArrayKokkos& elem_sspd, + DViewCArrayKokkos& elem_sie, + DViewCArrayKokkos& elem_vol, + DViewCArrayKokkos& elem_mass, + DViewCArrayKokkos& elem_mat_id); + + void ensight(const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_mass, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_pres, + const DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& elem_mat_id); + + void state_file(const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_mass, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_pres, + const DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& elem_mat_id); + + void node_density_constraints(host_vec_array& node_densities_lower_bound); + + void compute_topology_optimization_adjoint(); // Force does not depend on node coords and velocity + + void compute_topology_optimization_adjoint_full(); // Force depends on node coords and velocity + + void compute_topology_optimization_gradient(const_vec_array design_densities, vec_array gradients); + + void compute_topology_optimization_gradient_full(Teuchos::RCP design_densities_distributed, Teuchos::RCP design_gradients_distributed); + + // elastic TO stuff + void get_force_elastic(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_mass, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + DViewCArrayKokkos& corner_force, + const double rk_alpha, + const size_t cycle); + + void applied_forces(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_mass, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + DViewCArrayKokkos& corner_force, + const double rk_alpha, + const size_t cycle); + + void Element_Material_Properties(size_t ielem, real_t& Element_Modulus, real_t& Poisson_Ratio, real_t density); + + void compute_stiffness_gradients(const_host_vec_array& design_densities, host_vec_array& gradients); + + void Gradient_Element_Material_Properties(size_t ielem, real_t& Element_Modulus, real_t& Poisson_Ratio, real_t density); + + void local_matrix_multiply(int ielem, CArrayKokkos& Local_Matrix); + + void assemble_matrix(); + + bool nodal_density_flag; + real_t penalty_power; + Teuchos::RCP Global_Stiffness_Matrix; + RaggedRightArrayKokkos Stiffness_Matrix; + DCArrayKokkos Stiffness_Matrix_Strides; + DCArrayKokkos Global_Stiffness_Matrix_Assembly_Map; + // end elastic TO data + + Dynamic_Elasticity_Parameters* module_params; + Simulation_Parameters_Explicit* simparam; + Explicit_Solver* Explicit_Solver_Pointer_; + + elements::ref_element* ref_elem; + + std::shared_ptr mesh; + // shallow copies of mesh class views + size_t num_nodes_in_elem; + // corner ids in node + RaggedRightArrayKokkos corners_in_node; + CArrayKokkos num_corners_in_node; + + // elem ids in node + RaggedRightArrayKokkos elems_in_node; + + // node ids in node + RaggedRightArrayKokkos nodes_in_node; + CArrayKokkos num_nodes_in_node; + + // node ids in elem + DCArrayKokkos nodes_in_elem; + + // corner ids in elem + CArrayKokkos corners_in_elem; + + // elem ids in elem + RaggedRightArrayKokkos elems_in_elem; + CArrayKokkos num_elems_in_elem; + + // patch ids in elem + CArrayKokkos patches_in_elem; + + // node ids in a patch + CArrayKokkos nodes_in_patch; + + // element ids in a patch + CArrayKokkos elems_in_patch; + + // bdy nodes + CArrayKokkos bdy_nodes; + + // Topology optimization filter variable + DCArrayKokkos relative_element_densities; + + // Local FEA data + host_elem_conn_array interface_nodes_in_elem; // host view of element connectivity to nodes + + // Global FEA data + Teuchos::RCP node_velocities_distributed; + Teuchos::RCP initial_node_coords_distributed; + Teuchos::RCP all_initial_node_coords_distributed; + Teuchos::RCP initial_node_velocities_distributed; + Teuchos::RCP all_node_velocities_distributed; + Teuchos::RCP all_cached_node_velocities_distributed; + Teuchos::RCP node_masses_distributed; + Teuchos::RCP ghost_node_masses_distributed; + Teuchos::RCP adjoint_vector_distributed; + Teuchos::RCP phi_adjoint_vector_distributed; + Teuchos::RCP>> forward_solve_velocity_data; + Teuchos::RCP>> forward_solve_coordinate_data; + Teuchos::RCP>> adjoint_vector_data; + Teuchos::RCP>> phi_adjoint_vector_data; + Teuchos::RCP force_gradient_design; + Teuchos::RCP force_gradient_position; + Teuchos::RCP force_gradient_velocity; + + // Local FEA data + DCArrayKokkos Global_Gradient_Matrix_Assembly_Map; + RaggedRightArrayKokkos Graph_Matrix; // stores global indices + RaggedRightArrayKokkos DOF_Graph_Matrix; // stores global indices + RaggedRightArrayKokkos Force_Gradient_Positions; + RaggedRightArrayKokkos Force_Gradient_Velocities; + DCArrayKokkos Gradient_Matrix_Strides; + DCArrayKokkos Graph_Matrix_Strides; + RaggedRightArrayKokkos Original_Gradient_Entries; + RaggedRightArrayKokkos Original_Gradient_Entry_Indices; + DCArrayKokkos Original_Gradient_Entries_Strides; + + // distributed matrices + Teuchos::RCP distributed_force_gradient_positions; + Teuchos::RCP distributed_force_gradient_velocities; + + std::vector time_data; + unsigned long max_time_steps, last_time_step; + + // --------------------------------------------------------------------- + // state data type declarations (must stay in scope for output after run) + // --------------------------------------------------------------------- + node_t node_interface; + elem_t elem_interface; + corner_t corner_interface; + + // Dual View wrappers + // Dual Views of the individual node struct variables + DViewCArrayKokkos node_coords; + DViewCArrayKokkos node_vel; + DViewCArrayKokkos node_mass; + + // Dual Views of the individual elem struct variables + DViewCArrayKokkos elem_den; + DViewCArrayKokkos elem_pres; + DViewCArrayKokkos elem_stress; // always 3D even in 2D-RZ + DViewCArrayKokkos elem_sspd; + DViewCArrayKokkos elem_sie; + DViewCArrayKokkos elem_vol; + DViewCArrayKokkos elem_div; + DViewCArrayKokkos elem_mass; + DViewCArrayKokkos elem_mat_id; + + // Element velocity gradient + DCArrayKokkos elem_vel_grad; + + // for storing global variables used in user material model + DCArrayKokkos global_vars; + + // for storing state variables used in user material model + DCArrayKokkos state_vars; + + // elem_user_output_vars allow users to output variables of interest per element + DCArrayKokkos elem_user_output_vars; + + // material models + DCArrayKokkos elem_eos; + DCArrayKokkos elem_strength; + + // Dual Views of the corner struct variables + DViewCArrayKokkos corner_force; + DViewCArrayKokkos corner_mass; + + // Boundary Conditions Data + DCArrayKokkos Local_Index_Boundary_Patches; + // CArray Patch_Nodes; + enum bc_type { NONE, POINT_LOADING_CONDITION, LINE_LOADING_CONDITION, SURFACE_LOADING_CONDITION }; + + // Boundary Conditions Data + int max_boundary_sets; + + // output dof data + // Global arrays with collected data used to print + int output_velocity_index, output_strain_index, output_stress_index; + + // parameters + double time_value, time_final, dt, dt_max, dt_min, dt_cfl, graphics_time, graphics_dt_ival; + size_t graphics_cyc_ival, cycle_stop, rk_num_stages, graphics_id; + double fuzz, tiny, small; + CArray graphics_times; + int rk_num_bins; + + // optimization flags + bool kinetic_energy_objective; }; #endif // end HEADER_H diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/boundary.cpp b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/boundary.cpp index 06a017b8a..9f3a5c7a0 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/boundary.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/boundary.cpp @@ -1,76 +1,76 @@ // ------------------------------------------------------- // This function applys the boundary condition // to points on a list of patches created at setup -//-------------------------------------------------------- +// -------------------------------------------------------- #include "mesh.h" #include "state.h" #include "FEA_Module_Dynamic_Elasticity.h" #include "Simulation_Parameters/FEA_Module/Boundary_Conditions.h" -void FEA_Module_Dynamic_Elasticity::boundary_velocity(const mesh_t &mesh, - const DCArrayKokkos &boundary, - DViewCArrayKokkos &node_vel){ +void FEA_Module_Dynamic_Elasticity::boundary_velocity(const mesh_t& mesh, + const DCArrayKokkos& boundary, + DViewCArrayKokkos& node_vel) +{ + // error and debug flag + // DCArrayKokkos print_flag(1, "print_flag"); + // print_flag.host(0) = false; + // print_flag.update_device(); - //error and debug flag - //DCArrayKokkos print_flag(1, "print_flag"); - //print_flag.host(0) = false; - //print_flag.update_device(); - - const size_t rk_level = rk_num_bins - 1; - int num_dims = num_dim; + const size_t rk_level = rk_num_bins - 1; + int num_dims = num_dim; // Loop over boundary sets - for (size_t bdy_set=0; bdy_set #include #include -#include +#include #include // fmin, fmax, abs note: fminl is long #include #include @@ -34,7 +34,7 @@ #include "FEA_Module_Dynamic_Elasticity.h" #include "Explicit_Solver.h" -//optimization +// optimization #include "ROL_Solver.hpp" #include "Kinetic_Energy_Minimize.h" @@ -42,213 +42,228 @@ Compute new system response due to the design variable update ------------------------------------------------------------------------- */ -void FEA_Module_Dynamic_Elasticity::update_forward_solve(Teuchos::RCP zp){ - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - //local variable for host view in the dual view - int num_dim = simparam->num_dims; - int nodes_per_elem = max_nodes_per_element; - int local_node_index, current_row, current_column; - int max_stride = 0; - int current_module_index; - size_t access_index, row_access_index, row_counter; - GO global_index, global_dof_index; - LO local_dof_index; - const size_t num_fills = simparam->regions.size(); - const size_t rk_num_bins = simparam->dynamic_options.rk_num_bins; - const size_t num_bcs = module_params->boundary_conditions.size(); - const size_t num_materials = simparam->materials.size(); - real_t objective_accumulation; - - // --- Read in the nodes in the mesh --- - int myrank = Explicit_Solver_Pointer_->myrank; - int nranks = Explicit_Solver_Pointer_->nranks; - - const DCArrayKokkos mat_fill = simparam->mat_fill; - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos material = simparam->material; - CArray current_element_nodal_densities = CArray(num_nodes_in_elem); - - std::vector> FEA_Module_My_TO_Modules = simparam->FEA_Module_My_TO_Modules; - problem = Explicit_Solver_Pointer_->problem; //Pointer to ROL optimization problem object - ROL::Ptr> obj_pointer; - - //compute element averaged density ratios corresponding to nodal density design variables - {//view scope - const_host_vec_array all_node_densities = all_node_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); - //debug print - //std::cout << "NODE DENSITY TEST " << all_node_densities(0,0) << std::endl; - for(int elem_id = 0; elem_id < rnum_elem; elem_id++){ - for(int inode = 0; inode < num_nodes_in_elem; inode++){ - current_element_nodal_densities(inode) = all_node_densities(nodes_in_elem(elem_id,inode),0); - } - relative_element_densities.host(elem_id) = average_element_density(num_nodes_in_elem, current_element_nodal_densities); - }//for - } //view scope - //debug print - //std::cout << "ELEMENT RELATIVE DENSITY TEST " << relative_element_densities.host(0) << std::endl; - relative_element_densities.update_device(); - - //set density vector to the current value chosen by the optimizer - test_node_densities_distributed = zp; - - //reset nodal coordinates to initial values - node_coords_distributed->assign(*initial_node_coords_distributed); - - //comms for ghosts - Explicit_Solver_Pointer_->comm_coordinates(); - - //view scope - { - const_vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array ghost_node_coords_interface = Explicit_Solver_Pointer_->ghost_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - all_node_coords_interface(node_gid,idim) = node_coords_interface(node_gid,idim); - } +void FEA_Module_Dynamic_Elasticity::update_forward_solve(Teuchos::RCP zp) +{ + // local variable for host view in the dual view + int num_dim = simparam->num_dims; + int nodes_per_elem = max_nodes_per_element; + int local_node_index, current_row, current_column; + int max_stride = 0; + int current_module_index; + size_t access_index, row_access_index, row_counter; + GO global_index, global_dof_index; + LO local_dof_index; + + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + const size_t num_fills = simparam->regions.size(); + const size_t rk_num_bins = simparam->dynamic_options.rk_num_bins; + const size_t num_bcs = module_params->boundary_conditions.size(); + const size_t num_materials = simparam->materials.size(); + real_t objective_accumulation; + + // --- Read in the nodes in the mesh --- + int myrank = Explicit_Solver_Pointer_->myrank; + int nranks = Explicit_Solver_Pointer_->nranks; + + const DCArrayKokkos mat_fill = simparam->mat_fill; + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos material = simparam->material; + CArray current_element_nodal_densities = CArray(num_nodes_in_elem); + + std::vector> FEA_Module_My_TO_Modules = simparam->FEA_Module_My_TO_Modules; + problem = Explicit_Solver_Pointer_->problem; // Pointer to ROL optimization problem object + ROL::Ptr> obj_pointer; + + // compute element averaged density ratios corresponding to nodal density design variables + { // view scope + const_host_vec_array all_node_densities = all_node_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); + // debug print + // std::cout << "NODE DENSITY TEST " << all_node_densities(0,0) << std::endl; + for (int elem_id = 0; elem_id < rnum_elem; elem_id++) + { + for (int inode = 0; inode < num_nodes_in_elem; inode++) + { + current_element_nodal_densities(inode) = all_node_densities(nodes_in_elem(elem_id, inode), 0); + } + relative_element_densities.host(elem_id) = average_element_density(num_nodes_in_elem, current_element_nodal_densities); + } // for + } // view scope + // debug print + // std::cout << "ELEMENT RELATIVE DENSITY TEST " << relative_element_densities.host(0) << std::endl; + relative_element_densities.update_device(); + + // set density vector to the current value chosen by the optimizer + test_node_densities_distributed = zp; + + // reset nodal coordinates to initial values + node_coords_distributed->assign(*initial_node_coords_distributed); + + // comms for ghosts + Explicit_Solver_Pointer_->comm_coordinates(); + + // view scope + { + const_vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array ghost_node_coords_interface = Explicit_Solver_Pointer_->ghost_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + all_node_coords_interface(node_gid, idim) = node_coords_interface(node_gid, idim); + } }); // end parallel for - Kokkos::fence(); + Kokkos::fence(); - FOR_ALL_CLASS(node_gid, nlocal_nodes, nlocal_nodes+nghost_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - all_node_coords_interface(node_gid,idim) = ghost_node_coords_interface(node_gid-nlocal_nodes,idim); - } + FOR_ALL_CLASS(node_gid, nlocal_nodes, nlocal_nodes + nghost_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + all_node_coords_interface(node_gid, idim) = ghost_node_coords_interface(node_gid - nlocal_nodes, idim); + } }); // end parallel for - Kokkos::fence(); - } //end view scope - - //reset velocities to initial conditions - node_velocities_distributed->assign(*initial_node_velocities_distributed); - - //reset time accumulating objective and constraints - /* - for(int imodule = 0 ; imodule < FEA_Module_My_TO_Modules[my_fea_module_index_].size(); imodule++){ - current_module_index = FEA_Module_My_TO_Modules[my_fea_module_index_][imodule]; - //test if module needs reset - if(){ - - } - } - */ - //simple setup to just request KE for now; above loop to be expanded and used later for scanning modules - obj_pointer = problem->getObjective(); - KineticEnergyMinimize_TopOpt& kinetic_energy_minimize_function = dynamic_cast(*obj_pointer); - kinetic_energy_minimize_function.objective_accumulation = 0; - - //interface trial density vector - - //interfacing of vectors(should be removed later once made compatible) - //view scope - { - host_vec_array interface_node_coords = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); - for(size_t ibin = 0; ibin < rk_num_bins; ibin++){ - //save node data to node.coords - //std::cout << "NODE DATA ON RANK " << myrank << std::endl; - if(num_dim==2){ - for(int inode = 0; inode < nall_nodes; inode++){ - //std::cout << "Node index " << inode+1 << " "; - node_coords.host(ibin,inode,0) = interface_node_coords(inode,0); - //std::cout << host_node_coords_state(0,inode,0)+1<< " "; - node_coords.host(ibin,inode,1) = interface_node_coords(inode,1); - //std::cout << host_node_coords_state(0,inode,1)+1<< " "; - } - } - else if(num_dim==3){ - for(int inode = 0; inode < nall_nodes; inode++){ - //std::cout << "Node index " << inode+1 << " "; - node_coords.host(ibin,inode,0) = interface_node_coords(inode,0); - //std::cout << host_node_coords_state(0,inode,0)+1<< " "; - node_coords.host(ibin,inode,1) = interface_node_coords(inode,1); - //std::cout << host_node_coords_state(0,inode,1)+1<< " "; - - node_coords.host(ibin,inode,2) = interface_node_coords(inode,2); - //std::cout << host_node_coords_state(0,inode,2)+1<< std::endl; - } + Kokkos::fence(); + } // end view scope + + // reset velocities to initial conditions + node_velocities_distributed->assign(*initial_node_velocities_distributed); + + // reset time accumulating objective and constraints + /* + for(int imodule = 0 ; imodule < FEA_Module_My_TO_Modules[my_fea_module_index_].size(); imodule++){ + current_module_index = FEA_Module_My_TO_Modules[my_fea_module_index_][imodule]; + //test if module needs reset + if(){ + } } - } //end view scope + */ + // simple setup to just request KE for now; above loop to be expanded and used later for scanning modules + obj_pointer = problem->getObjective(); + KineticEnergyMinimize_TopOpt& kinetic_energy_minimize_function = dynamic_cast(*obj_pointer); + kinetic_energy_minimize_function.objective_accumulation = 0; + + // interface trial density vector + + // interfacing of vectors(should be removed later once made compatible) + // view scope + { + host_vec_array interface_node_coords = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + for (size_t ibin = 0; ibin < rk_num_bins; ibin++) + { + // save node data to node.coords + // std::cout << "NODE DATA ON RANK " << myrank << std::endl; + if (num_dim == 2) + { + for (int inode = 0; inode < nall_nodes; inode++) + { + // std::cout << "Node index " << inode+1 << " "; + node_coords.host(ibin, inode, 0) = interface_node_coords(inode, 0); + // std::cout << host_node_coords_state(0,inode,0)+1<< " "; + node_coords.host(ibin, inode, 1) = interface_node_coords(inode, 1); + // std::cout << host_node_coords_state(0,inode,1)+1<< " "; + } + } + else if (num_dim == 3) + { + for (int inode = 0; inode < nall_nodes; inode++) + { + // std::cout << "Node index " << inode+1 << " "; + node_coords.host(ibin, inode, 0) = interface_node_coords(inode, 0); + // std::cout << host_node_coords_state(0,inode,0)+1<< " "; + node_coords.host(ibin, inode, 1) = interface_node_coords(inode, 1); + // std::cout << host_node_coords_state(0,inode,1)+1<< " "; + + node_coords.host(ibin, inode, 2) = interface_node_coords(inode, 2); + // std::cout << host_node_coords_state(0,inode,2)+1<< std::endl; + } + } + } + } // end view scope // save the node coords to the current RK value - for (size_t node_gid=0; node_gid < nall_nodes; node_gid++){ - - for(int rk=1; rkcalc_pressure(elem_pres, + elem_coords[0] += node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), 0); + elem_coords[1] += node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), 1); + if (num_dim == 3) + { + elem_coords[2] += node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), 2); + } + else + { + elem_coords[2] = 0.0; + } + } // end loop over nodes in element + elem_coords[0] = elem_coords[0] / num_nodes_in_elem; + elem_coords[1] = elem_coords[1] / num_nodes_in_elem; + elem_coords[2] = elem_coords[2] / num_nodes_in_elem; + + // default is not to fill the element + bool fill_this = mat_fill(f_id).volume.contains(elem_coords); + + // paint the material state on the element + if (fill_this) + { + // density + elem_den(elem_gid) = mat_fill(f_id).den; + + // compute element average density from initial nodal density variables used as TO design variables + elem_den(elem_gid) = elem_den(elem_gid) * relative_element_densities(elem_gid); + + // mass + elem_mass(elem_gid) = elem_den(elem_gid) * elem_vol(elem_gid); + + // specific internal energy + elem_sie(rk_level, elem_gid) = mat_fill(f_id).sie; + + elem_mat_id(elem_gid) = mat_fill(f_id).material_id; + size_t mat_id = elem_mat_id(elem_gid); // short name + + // --- stress tensor --- + // always 3D even for 2D-RZ + for (size_t i = 0; i < 3; i++) + { + for (size_t j = 0; j < 3; j++) + { + elem_stress(rk_level, elem_gid, i, j) = 0.0; + } + } // end for + + // short form for clean code + EOSParent* eos_model = elem_eos(elem_gid).model; + + // --- Pressure --- + eos_model->calc_pressure(elem_pres, elem_stress, elem_gid, elem_mat_id(elem_gid), @@ -257,10 +272,10 @@ void FEA_Module_Dynamic_Elasticity::update_forward_solve(Teuchos::RCP elem_user_output_vars, elem_sspd, elem_den(elem_gid), - elem_sie(rk_level,elem_gid)); + elem_sie(rk_level, elem_gid)); - // --- Sound speed --- - eos_model->calc_sound_speed(elem_pres, + // --- Sound speed --- + eos_model->calc_sound_speed(elem_pres, elem_stress, elem_gid, elem_mat_id(elem_gid), @@ -269,229 +284,224 @@ void FEA_Module_Dynamic_Elasticity::update_forward_solve(Teuchos::RCP elem_user_output_vars, elem_sspd, elem_den(elem_gid), - elem_sie(rk_level,elem_gid)); - - // loop over the nodes of this element and apply velocity - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - - // get the mesh node index - size_t node_gid = nodes_in_elem(elem_gid, node_lid); + elem_sie(rk_level, elem_gid)); - - // --- Velocity --- - switch(mat_fill(f_id).velocity) + // loop over the nodes of this element and apply velocity + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { - case VELOCITY_TYPE::cartesian: - { - - node_vel(rk_level, node_gid, 0) = mat_fill(f_id).u; - node_vel(rk_level, node_gid, 1) = mat_fill(f_id).v; - if (num_dim == 3) node_vel(rk_level, node_gid, 2) = mat_fill(f_id).w; - - - break; - } - case VELOCITY_TYPE::radial: + // get the mesh node index + size_t node_gid = nodes_in_elem(elem_gid, node_lid); + + // --- Velocity --- + switch (mat_fill(f_id).velocity) { - // Setting up cylindrical - double dir[2]; - dir[0] = 0.0; - dir[1] = 0.0; - double radius_val = 0.0; - - for(int dim=0; dim<2; dim++){ - dir[dim] = node_coords(rk_level, node_gid, dim); - radius_val += node_coords(rk_level, node_gid, dim)*node_coords(rk_level, node_gid, dim); - } // end for - radius_val = sqrt(radius_val); - - for(int dim=0; dim<2; dim++){ - if (radius_val > 1.0e-14){ - dir[dim] /= (radius_val); + case VELOCITY_TYPE::cartesian: + { + node_vel(rk_level, node_gid, 0) = mat_fill(f_id).u; + node_vel(rk_level, node_gid, 1) = mat_fill(f_id).v; + if (num_dim == 3) + { + node_vel(rk_level, node_gid, 2) = mat_fill(f_id).w; + } + + break; } - else{ - dir[dim] = 0.0; + case VELOCITY_TYPE::radial: + { + // Setting up cylindrical + double dir[2]; + dir[0] = 0.0; + dir[1] = 0.0; + double radius_val = 0.0; + + for (int dim = 0; dim < 2; dim++) + { + dir[dim] = node_coords(rk_level, node_gid, dim); + radius_val += node_coords(rk_level, node_gid, dim) * node_coords(rk_level, node_gid, dim); + } // end for + radius_val = sqrt(radius_val); + + for (int dim = 0; dim < 2; dim++) + { + if (radius_val > 1.0e-14) + { + dir[dim] /= (radius_val); + } + else + { + dir[dim] = 0.0; + } + } // end for + + node_vel(rk_level, node_gid, 0) = mat_fill(f_id).speed * dir[0]; + node_vel(rk_level, node_gid, 1) = mat_fill(f_id).speed * dir[1]; + if (num_dim == 3) + { + node_vel(rk_level, node_gid, 2) = 0.0; + } + + break; } - } // end for - - - node_vel(rk_level, node_gid, 0) = mat_fill(f_id).speed*dir[0]; - node_vel(rk_level, node_gid, 1) = mat_fill(f_id).speed*dir[1]; - if (num_dim == 3) node_vel(rk_level, node_gid, 2) = 0.0; - - break; - } - case VELOCITY_TYPE::spherical: - { - - // Setting up spherical - double dir[3]; - dir[0] = 0.0; - dir[1] = 0.0; - dir[2] = 0.0; - double radius_val = 0.0; - - for(int dim=0; dim<3; dim++){ - dir[dim] = node_coords(rk_level, node_gid, dim); - radius_val += node_coords(rk_level, node_gid, dim)*node_coords(rk_level, node_gid, dim); - } // end for - radius_val = sqrt(radius_val); - - for(int dim=0; dim<3; dim++){ - if (radius_val > 1.0e-14){ - dir[dim] /= (radius_val); + case VELOCITY_TYPE::spherical: + { + // Setting up spherical + double dir[3]; + dir[0] = 0.0; + dir[1] = 0.0; + dir[2] = 0.0; + double radius_val = 0.0; + + for (int dim = 0; dim < 3; dim++) + { + dir[dim] = node_coords(rk_level, node_gid, dim); + radius_val += node_coords(rk_level, node_gid, dim) * node_coords(rk_level, node_gid, dim); + } // end for + radius_val = sqrt(radius_val); + + for (int dim = 0; dim < 3; dim++) + { + if (radius_val > 1.0e-14) + { + dir[dim] /= (radius_val); + } + else + { + dir[dim] = 0.0; + } + } // end for + + node_vel(rk_level, node_gid, 0) = mat_fill(f_id).speed * dir[0]; + node_vel(rk_level, node_gid, 1) = mat_fill(f_id).speed * dir[1]; + if (num_dim == 3) + { + node_vel(rk_level, node_gid, 2) = mat_fill(f_id).speed * dir[2]; + } + + break; } - else{ - dir[dim] = 0.0; + case VELOCITY_TYPE::radial_linear: + { + break; } - } // end for - - node_vel(rk_level, node_gid, 0) = mat_fill(f_id).speed*dir[0]; - node_vel(rk_level, node_gid, 1) = mat_fill(f_id).speed*dir[1]; - if (num_dim == 3) node_vel(rk_level, node_gid, 2) = mat_fill(f_id).speed*dir[2]; - - break; - } - case VELOCITY_TYPE::radial_linear: - { - - break; - } - case VELOCITY_TYPE::spherical_linear: - { - - break; - } - case VELOCITY_TYPE::tg_vortex: - { - - node_vel(rk_level, node_gid, 0) = sin(PI * node_coords(rk_level,node_gid, 0)) * cos(PI * node_coords(rk_level,node_gid, 1)); - node_vel(rk_level, node_gid, 1) = -1.0*cos(PI * node_coords(rk_level,node_gid, 0)) * sin(PI * node_coords(rk_level,node_gid, 1)); - if (num_dim == 3) node_vel(rk_level, node_gid, 2) = 0.0; - - break; - } - } // end of switch - - }// end loop over nodes of element - - - if(mat_fill(f_id).velocity == VELOCITY_TYPE::tg_vortex) - { - elem_pres(elem_gid) = 0.25*( cos(2.0*PI*elem_coords[0]) + cos(2.0*PI*elem_coords[1]) ) + 1.0; - - // p = rho*ie*(gamma - 1) - size_t mat_id = f_id; - double gamma = global_vars(mat_id,0); // gamma value - elem_sie(rk_level, elem_gid) = - elem_pres(elem_gid)/(mat_fill(f_id).den*(gamma - 1.0)); - } // end if + case VELOCITY_TYPE::spherical_linear: + { + break; + } + case VELOCITY_TYPE::tg_vortex: + { + node_vel(rk_level, node_gid, 0) = sin(PI * node_coords(rk_level, node_gid, 0)) * cos(PI * node_coords(rk_level, node_gid, 1)); + node_vel(rk_level, node_gid, 1) = -1.0 * cos(PI * node_coords(rk_level, node_gid, 0)) * sin(PI * node_coords(rk_level, node_gid, 1)); + if (num_dim == 3) + { + node_vel(rk_level, node_gid, 2) = 0.0; + } + + break; + } + } // end of switch + } // end loop over nodes of element - } // end if fill - + if (mat_fill(f_id).velocity == VELOCITY_TYPE::tg_vortex) + { + elem_pres(elem_gid) = 0.25 * (cos(2.0 * PI * elem_coords[0]) + cos(2.0 * PI * elem_coords[1]) ) + 1.0; + + // p = rho*ie*(gamma - 1) + size_t mat_id = f_id; + double gamma = global_vars(mat_id, 0); // gamma value + elem_sie(rk_level, elem_gid) = + elem_pres(elem_gid) / (mat_fill(f_id).den * (gamma - 1.0)); + } // end if + } // end if fill }); // end FOR_ALL_CLASS element loop - Kokkos::fence(); - - - } // end for loop over fills - }//end view scope - - - + Kokkos::fence(); + } // end for loop over fills + } // end view scope + // apply BC's to velocity FEA_Module_Dynamic_Elasticity::boundary_velocity(*mesh, boundary, node_vel); - - + // calculate the corner massess if 2D - if(num_dim==2){ - + if (num_dim == 2) + { FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { - // facial area of the corners double corner_areas_array[4]; - - ViewCArrayKokkos corner_areas(&corner_areas_array[0],4); - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 4); - + + ViewCArrayKokkos corner_areas(&corner_areas_array[0], 4); + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 4); + get_area_weights2D(corner_areas, elem_gid, node_coords, elem_node_gids, rk_level); - + // loop over the corners of the element and calculate the mass - for (size_t corner_lid=0; corner_lid<4; corner_lid++){ - - size_t corner_gid = corners_in_elem(elem_gid, corner_lid); - corner_mass(corner_gid) = corner_areas(corner_lid)*elem_den(elem_gid); // node radius is added later - + for (size_t corner_lid = 0; corner_lid < 4; corner_lid++) + { + size_t corner_gid = corners_in_elem(elem_gid, corner_lid); + corner_mass(corner_gid) = corner_areas(corner_lid) * elem_den(elem_gid); // node radius is added later } // end for over corners }); - } // end of - - + // calculate the nodal mass FOR_ALL_CLASS(node_gid, 0, nall_nodes, { - node_mass(node_gid) = 0.0; - - if(num_dim==3){ - - for(size_t elem_lid=0; elem_lid < num_corners_in_node(node_gid); elem_lid++){ - size_t elem_gid = elems_in_node(node_gid,elem_lid); - node_mass(node_gid) += 1.0/8.0*elem_mass(elem_gid); + + if (num_dim == 3) + { + for (size_t elem_lid = 0; elem_lid < num_corners_in_node(node_gid); elem_lid++) + { + size_t elem_gid = elems_in_node(node_gid, elem_lid); + node_mass(node_gid) += 1.0 / 8.0 * elem_mass(elem_gid); } // end for elem_lid - - }// end if dims=3 - else { - + } // end if dims=3 + else + { // 2D-RZ - for(size_t corner_lid=0; corner_lid < num_corners_in_node(node_gid); corner_lid++){ - - size_t corner_gid = corners_in_node(node_gid, corner_lid); + for (size_t corner_lid = 0; corner_lid < num_corners_in_node(node_gid); corner_lid++) + { + size_t corner_gid = corners_in_node(node_gid, corner_lid); node_mass(node_gid) += corner_mass(corner_gid); // sans the radius so it is areal node mass - - corner_mass(corner_gid) *= node_coords(rk_level,node_gid,1); // true corner mass now + + corner_mass(corner_gid) *= node_coords(rk_level, node_gid, 1); // true corner mass now } // end for elem_lid - } // end else - }); // end FOR_ALL_CLASS Kokkos::fence(); - - //current interface has differing mass arrays; this equates them until we unify memory - //view scope + // current interface has differing mass arrays; this equates them until we unify memory + // view scope { - vec_array node_mass_interface = node_masses_distributed->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - node_mass_interface(node_gid,0) = node_mass(node_gid); + vec_array node_mass_interface = node_masses_distributed->getLocalView(Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + node_mass_interface(node_gid, 0) = node_mass(node_gid); }); // end parallel for - } //end view scope + } // end view scope Kokkos::fence(); - //communicate ghost densities + // communicate ghost densities comm_node_masses(); - //this is forcing a copy to the device - //view scope + // this is forcing a copy to the device + // view scope { - vec_array ghost_node_mass_interface = ghost_node_masses_distributed->getLocalView (Tpetra::Access::ReadWrite); + vec_array ghost_node_mass_interface = ghost_node_masses_distributed->getLocalView(Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, nlocal_nodes, nall_nodes, { - node_mass(node_gid) = ghost_node_mass_interface(node_gid-nlocal_nodes,0); + FOR_ALL_CLASS(node_gid, nlocal_nodes, nall_nodes, { + node_mass(node_gid) = ghost_node_mass_interface(node_gid - nlocal_nodes, 0); }); // end parallel for - } //end view scope + } // end view scope Kokkos::fence(); - //update stiffness matrix - if(simparam->topology_optimization_on || simparam->shape_optimization_on){ - assemble_matrix(); + // update stiffness matrix + if (simparam->topology_optimization_on || simparam->shape_optimization_on) + { + assemble_matrix(); } - - //execute solve - elastic_solve(); + // execute solve + elastic_solve(); } /* ------------------------------------------------------------------------------------------- @@ -500,141 +510,163 @@ void FEA_Module_Dynamic_Elasticity::update_forward_solve(Teuchos::RCP double FEA_Module_Dynamic_Elasticity::average_element_density(const int nodes_per_elem, const CArray current_element_densities) const { - double result = 0; - for(int i=0; i < nodes_per_elem; i++){ - result += current_element_densities(i)/nodes_per_elem; - } + double result = 0; + for (int i = 0; i < nodes_per_elem; i++) + { + result += current_element_densities(i) / nodes_per_elem; + } - return result; + return result; } /* --------------------------------------------------------------------------------------------------------------- - Simpler adjoint vector solve for the kinetic energy minimization problem + Simpler adjoint vector solve for the kinetic energy minimization problem when force does not depend on u and v. ------------------------------------------------------------------------------------------------------------------ */ -void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_adjoint(){ - - size_t num_bdy_nodes = mesh->num_bdy_nodes; - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos material = simparam->material; - const int num_dim = simparam->num_dims; - real_t global_dt; - size_t current_data_index, next_data_index; - Teuchos::RCP previous_adjoint_vector_distributed, current_adjoint_vector_distributed, previous_velocity_vector_distributed, current_velocity_vector_distributed; - //initialize first adjoint vector at last_time_step to 0 as the terminal value - (*adjoint_vector_data)[last_time_step+1]->putScalar(0); - - //solve terminal value problem, proceeds in time backward. For simplicity, we use the same timestep data from the forward solve. - //A linear interpolant is assumed between velocity data points; velocity midpoint is used to update the adjoint. - if(myrank==0) - std::cout << "Computing adjoint vector " << time_data.size() << std::endl; - - for (int cycle = last_time_step; cycle >= 0; cycle--) { - //compute timestep from time data - global_dt = time_data[cycle+1] - time_data[cycle]; - - //print - if(simparam->dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ - if (cycle==last_time_step){ - if(myrank==0) - printf("cycle = %d, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } - // print time step every 10 cycles - else if (cycle%20==0){ - if(myrank==0) - printf("cycle = %d, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } // end if +void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_adjoint() +{ + size_t num_bdy_nodes = mesh->num_bdy_nodes; + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos material = simparam->material; + const int num_dim = simparam->num_dims; + real_t global_dt; + size_t current_data_index, next_data_index; + Teuchos::RCP previous_adjoint_vector_distributed, current_adjoint_vector_distributed, previous_velocity_vector_distributed, current_velocity_vector_distributed; + // initialize first adjoint vector at last_time_step to 0 as the terminal value + (*adjoint_vector_data)[last_time_step + 1]->putScalar(0); + + // solve terminal value problem, proceeds in time backward. For simplicity, we use the same timestep data from the forward solve. + // A linear interpolant is assumed between velocity data points; velocity midpoint is used to update the adjoint. + if (myrank == 0) + { + std::cout << "Computing adjoint vector " << time_data.size() << std::endl; } - //else if (cycle==1){ - //if(myrank==0) - //printf("cycle = %lu, time = %f, time step = %f \n", cycle-1, time_data[cycle-1], global_dt); - //} // end if - //compute adjoint vector for this data point; use velocity midpoint - //view scope + for (int cycle = last_time_step; cycle >= 0; cycle--) { - const_vec_array previous_velocity_vector = (*forward_solve_velocity_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - - const_vec_array previous_adjoint_vector = (*adjoint_vector_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView (Tpetra::Access::ReadWrite); - - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes + nghost_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - //cancellation of half from midpoint and 2 from adjoint equation already done - current_adjoint_vector(node_gid,idim) = -0.5*(current_velocity_vector(node_gid,idim)+previous_velocity_vector(node_gid,idim))*global_dt + previous_adjoint_vector(node_gid,idim); - } + // compute timestep from time data + global_dt = time_data[cycle + 1] - time_data[cycle]; + + // print + if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) + { + if (cycle == last_time_step) + { + if (myrank == 0) + { + printf("cycle = %d, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } + // print time step every 10 cycles + else if (cycle % 20 == 0) + { + if (myrank == 0) + { + printf("cycle = %d, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } // end if + } + // else if (cycle==1){ + // if(myrank==0) + // printf("cycle = %lu, time = %f, time step = %f \n", cycle-1, time_data[cycle-1], global_dt); + // } // end if + + // compute adjoint vector for this data point; use velocity midpoint + // view scope + { + const_vec_array previous_velocity_vector = (*forward_solve_velocity_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + + const_vec_array previous_adjoint_vector = (*adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadWrite); + + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes + nghost_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + // cancellation of half from midpoint and 2 from adjoint equation already done + current_adjoint_vector(node_gid, idim) = -0.5 * (current_velocity_vector(node_gid, idim) + previous_velocity_vector(node_gid, idim)) * global_dt + previous_adjoint_vector(node_gid, + idim); + } }); // end parallel for - Kokkos::fence(); - } //end view scope - - } + Kokkos::fence(); + } // end view scope + } } - /* ------------------------------------------------------------------------------ Coupled adjoint problem for the kinetic energy minimization problem --------------------------------------------------------------------------------- */ -void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_adjoint_full(){ - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - size_t num_bdy_nodes = mesh->num_bdy_nodes; - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos material = simparam->material; - const int num_dim = simparam->num_dims; - const real_t damping_constant = module_params->damping_constant; - real_t global_dt; - size_t current_data_index, next_data_index; - Teuchos::RCP previous_adjoint_vector_distributed, current_adjoint_vector_distributed, previous_velocity_vector_distributed, current_velocity_vector_distributed; - Teuchos::RCP previous_phi_adjoint_vector_distributed, current_phi_adjoint_vector_distributed; - //initialize first adjoint vector at last_time_step to 0 as the terminal value - (*adjoint_vector_data)[last_time_step+1]->putScalar(0); - (*phi_adjoint_vector_data)[last_time_step+1]->putScalar(0); - - //solve terminal value problem, proceeds in time backward. For simplicity, we use the same timestep data from the forward solve. - //A linear interpolant is assumed between velocity data points; velocity midpoint is used to update the adjoint. - if(myrank==0) - std::cout << "Computing adjoint vector " << time_data.size() << std::endl; - - for (int cycle = last_time_step; cycle >= 0; cycle--) { - //compute timestep from time data - global_dt = time_data[cycle+1] - time_data[cycle]; - - //print - if(simparam->dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ - if (cycle==last_time_step){ - if(myrank==0) - printf("cycle = %d, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } - // print time step every 10 cycles - else if (cycle%20==0){ - if(myrank==0) - printf("cycle = %d, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } // end if +void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_adjoint_full() +{ + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + size_t num_bdy_nodes = mesh->num_bdy_nodes; + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos material = simparam->material; + const int num_dim = simparam->num_dims; + const real_t damping_constant = module_params->damping_constant; + real_t global_dt; + size_t current_data_index, next_data_index; + Teuchos::RCP previous_adjoint_vector_distributed, current_adjoint_vector_distributed, previous_velocity_vector_distributed, current_velocity_vector_distributed; + Teuchos::RCP previous_phi_adjoint_vector_distributed, current_phi_adjoint_vector_distributed; + // initialize first adjoint vector at last_time_step to 0 as the terminal value + (*adjoint_vector_data)[last_time_step + 1]->putScalar(0); + (*phi_adjoint_vector_data)[last_time_step + 1]->putScalar(0); + + // solve terminal value problem, proceeds in time backward. For simplicity, we use the same timestep data from the forward solve. + // A linear interpolant is assumed between velocity data points; velocity midpoint is used to update the adjoint. + if (myrank == 0) + { + std::cout << "Computing adjoint vector " << time_data.size() << std::endl; } - - //view scope + + for (int cycle = last_time_step; cycle >= 0; cycle--) { - - //set velocity and position for this timestep - const_vec_array previous_velocity_vector = (*forward_solve_velocity_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - - const_vec_array previous_coordinate_vector = (*forward_solve_coordinate_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_coordinate_vector = (*forward_solve_coordinate_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - - //interface of arrays for current implementation of force calculation - - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes+nghost_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - node_vel(rk_level,node_gid,idim) = previous_velocity_vector(node_gid,idim); - node_coords(rk_level,node_gid,idim) = previous_coordinate_vector(node_gid,idim); + // compute timestep from time data + global_dt = time_data[cycle + 1] - time_data[cycle]; + + // print + if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) + { + if (cycle == last_time_step) + { + if (myrank == 0) + { + printf("cycle = %d, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } + // print time step every 10 cycles + else if (cycle % 20 == 0) + { + if (myrank == 0) + { + printf("cycle = %d, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } // end if } + + // view scope + { + // set velocity and position for this timestep + const_vec_array previous_velocity_vector = (*forward_solve_velocity_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + + const_vec_array previous_coordinate_vector = (*forward_solve_coordinate_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array current_coordinate_vector = (*forward_solve_coordinate_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + + // interface of arrays for current implementation of force calculation + + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes + nghost_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + node_vel(rk_level, node_gid, idim) = previous_velocity_vector(node_gid, idim); + node_coords(rk_level, node_gid, idim) = previous_coordinate_vector(node_gid, idim); + } }); - Kokkos::fence(); + Kokkos::fence(); - get_force_vgradient_elastic(material, + get_force_vgradient_elastic(material, *mesh, node_coords, node_vel, @@ -649,965 +681,1050 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_adjoint_full() 1, cycle); - //force_gradient_velocity->describe(*fos,Teuchos::VERB_EXTREME); - const_vec_array previous_force_gradient_position = force_gradient_position->getLocalView (Tpetra::Access::ReadOnly); - //const_vec_array current_force_gradient_position = force_gradient_position->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array previous_force_gradient_velocity = force_gradient_velocity->getLocalView (Tpetra::Access::ReadOnly); - //const_vec_array current_force_gradient_velocity = force_gradient_velocity->getLocalView (Tpetra::Access::ReadOnly); - //compute gradient of force with respect to velocity - - const_vec_array previous_adjoint_vector = (*adjoint_vector_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array phi_previous_adjoint_vector = (*phi_adjoint_vector_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - vec_array midpoint_adjoint_vector = adjoint_vector_distributed->getLocalView (Tpetra::Access::ReadWrite); - vec_array phi_midpoint_adjoint_vector = phi_adjoint_vector_distributed->getLocalView (Tpetra::Access::ReadWrite); - - //half step update for RK2 scheme - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - real_t rate_of_change; - real_t matrix_contribution; - size_t dof_id; - for (int idim = 0; idim < num_dim; idim++){ - matrix_contribution = 0; - //compute resulting row of force velocity gradient matrix transpose right multiplied by adjoint vector - /* - for(int idof = 0; idof < Gradient_Matrix_Strides(node_gid*num_dim+idim); idof++){ - dof_id = DOF_Graph_Matrix(node_gid*num_dim+idim,idof); - matrix_contribution += previous_adjoint_vector(dof_id/num_dim,dof_id%num_dim)*Force_Gradient_Velocities(node_gid*num_dim+idim,idof); - } - */ - matrix_contribution = -damping_constant*previous_adjoint_vector(node_gid,idim); - rate_of_change = previous_velocity_vector(node_gid,idim)- - matrix_contribution/node_mass(node_gid)- - phi_previous_adjoint_vector(node_gid,idim)/node_mass(node_gid); - midpoint_adjoint_vector(node_gid,idim) = -rate_of_change*global_dt/2 + previous_adjoint_vector(node_gid,idim); - matrix_contribution = 0; - //compute resulting row of force displacement gradient matrix transpose right multiplied by adjoint vector - for(int idof = 0; idof < Gradient_Matrix_Strides(node_gid*num_dim+idim); idof++){ - dof_id = DOF_Graph_Matrix(node_gid*num_dim+idim,idof); - matrix_contribution += -previous_adjoint_vector(dof_id/num_dim,dof_id%num_dim)*Force_Gradient_Positions(node_gid*num_dim+idim,idof); - } - rate_of_change = -matrix_contribution; - //rate_of_change = -0.0000001*previous_adjoint_vector(node_gid,idim); - phi_midpoint_adjoint_vector(node_gid,idim) = -rate_of_change*global_dt/2 + phi_previous_adjoint_vector(node_gid,idim); - } + // force_gradient_velocity->describe(*fos,Teuchos::VERB_EXTREME); + const_vec_array previous_force_gradient_position = force_gradient_position->getLocalView(Tpetra::Access::ReadOnly); + // const_vec_array current_force_gradient_position = force_gradient_position->getLocalView (Tpetra::Access::ReadOnly); + const_vec_array previous_force_gradient_velocity = force_gradient_velocity->getLocalView(Tpetra::Access::ReadOnly); + // const_vec_array current_force_gradient_velocity = force_gradient_velocity->getLocalView (Tpetra::Access::ReadOnly); + // compute gradient of force with respect to velocity + + const_vec_array previous_adjoint_vector = (*adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array phi_previous_adjoint_vector = (*phi_adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + vec_array midpoint_adjoint_vector = adjoint_vector_distributed->getLocalView(Tpetra::Access::ReadWrite); + vec_array phi_midpoint_adjoint_vector = phi_adjoint_vector_distributed->getLocalView(Tpetra::Access::ReadWrite); + + // half step update for RK2 scheme + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + real_t rate_of_change; + real_t matrix_contribution; + size_t dof_id; + for (int idim = 0; idim < num_dim; idim++) + { + matrix_contribution = 0; + // compute resulting row of force velocity gradient matrix transpose right multiplied by adjoint vector + /* + for(int idof = 0; idof < Gradient_Matrix_Strides(node_gid*num_dim+idim); idof++){ + dof_id = DOF_Graph_Matrix(node_gid*num_dim+idim,idof); + matrix_contribution += previous_adjoint_vector(dof_id/num_dim,dof_id%num_dim)*Force_Gradient_Velocities(node_gid*num_dim+idim,idof); + } + */ + matrix_contribution = -damping_constant * previous_adjoint_vector(node_gid, idim); + rate_of_change = previous_velocity_vector(node_gid, idim) - + matrix_contribution / node_mass(node_gid) - + phi_previous_adjoint_vector(node_gid, idim) / node_mass(node_gid); + midpoint_adjoint_vector(node_gid, idim) = -rate_of_change * global_dt / 2 + previous_adjoint_vector(node_gid, idim); + matrix_contribution = 0; + // compute resulting row of force displacement gradient matrix transpose right multiplied by adjoint vector + for (int idof = 0; idof < Gradient_Matrix_Strides(node_gid * num_dim + idim); idof++) + { + dof_id = DOF_Graph_Matrix(node_gid * num_dim + idim, idof); + matrix_contribution += -previous_adjoint_vector(dof_id / num_dim, dof_id % num_dim) * Force_Gradient_Positions(node_gid * num_dim + idim, idof); + } + rate_of_change = -matrix_contribution; + // rate_of_change = -0.0000001*previous_adjoint_vector(node_gid,idim); + phi_midpoint_adjoint_vector(node_gid, idim) = -rate_of_change * global_dt / 2 + phi_previous_adjoint_vector(node_gid, idim); + } }); // end parallel for - Kokkos::fence(); - - boundary_adjoint(*mesh, boundary,midpoint_adjoint_vector, phi_midpoint_adjoint_vector); - comm_adjoint_vectors(cycle); - //swap names to get ghost nodes for the midpoint vectors - vec_array current_adjoint_vector = adjoint_vector_distributed->getLocalView (Tpetra::Access::ReadWrite); - vec_array phi_current_adjoint_vector = phi_adjoint_vector_distributed->getLocalView (Tpetra::Access::ReadWrite); - midpoint_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView (Tpetra::Access::ReadWrite); - phi_midpoint_adjoint_vector = (*phi_adjoint_vector_data)[cycle]->getLocalView (Tpetra::Access::ReadWrite); - - //full step update with midpoint gradient for RK2 scheme - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - real_t rate_of_change; - real_t matrix_contribution; - size_t dof_id; - for (int idim = 0; idim < num_dim; idim++){ - matrix_contribution = 0; - //compute resulting row of force velocity gradient matrix transpose right multiplied by adjoint vector - /* - for(int idof = 0; idof < Gradient_Matrix_Strides(node_gid*num_dim+idim); idof++){ - dof_id = DOF_Graph_Matrix(node_gid*num_dim+idim,idof); - matrix_contribution += midpoint_adjoint_vector(dof_id/num_dim,dof_id%num_dim)*Force_Gradient_Velocities(node_gid*num_dim+idim,idof); - } - */ - matrix_contribution = -damping_constant*midpoint_adjoint_vector(node_gid,idim); - rate_of_change = (previous_velocity_vector(node_gid,idim) + current_velocity_vector(node_gid,idim))/2- - matrix_contribution/node_mass(node_gid)- - phi_midpoint_adjoint_vector(node_gid,idim)/node_mass(node_gid); - current_adjoint_vector(node_gid,idim) = -rate_of_change*global_dt + previous_adjoint_vector(node_gid,idim); - matrix_contribution = 0; - //compute resulting row of force displacement gradient matrix transpose right multiplied by adjoint vector - for(int idof = 0; idof < Gradient_Matrix_Strides(node_gid*num_dim+idim); idof++){ - dof_id = DOF_Graph_Matrix(node_gid*num_dim+idim,idof); - matrix_contribution += -midpoint_adjoint_vector(dof_id/num_dim,dof_id%num_dim)*Force_Gradient_Positions(node_gid*num_dim+idim,idof); - } - rate_of_change = -matrix_contribution; - //rate_of_change = -0.0000001*midpoint_adjoint_vector(node_gid,idim); - phi_current_adjoint_vector(node_gid,idim) = -rate_of_change*global_dt + phi_previous_adjoint_vector(node_gid,idim); - } + Kokkos::fence(); + + boundary_adjoint(*mesh, boundary, midpoint_adjoint_vector, phi_midpoint_adjoint_vector); + comm_adjoint_vectors(cycle); + // swap names to get ghost nodes for the midpoint vectors + vec_array current_adjoint_vector = adjoint_vector_distributed->getLocalView(Tpetra::Access::ReadWrite); + vec_array phi_current_adjoint_vector = phi_adjoint_vector_distributed->getLocalView(Tpetra::Access::ReadWrite); + midpoint_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadWrite); + phi_midpoint_adjoint_vector = (*phi_adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadWrite); + + // full step update with midpoint gradient for RK2 scheme + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + real_t rate_of_change; + real_t matrix_contribution; + size_t dof_id; + for (int idim = 0; idim < num_dim; idim++) + { + matrix_contribution = 0; + // compute resulting row of force velocity gradient matrix transpose right multiplied by adjoint vector + /* + for(int idof = 0; idof < Gradient_Matrix_Strides(node_gid*num_dim+idim); idof++){ + dof_id = DOF_Graph_Matrix(node_gid*num_dim+idim,idof); + matrix_contribution += midpoint_adjoint_vector(dof_id/num_dim,dof_id%num_dim)*Force_Gradient_Velocities(node_gid*num_dim+idim,idof); + } + */ + matrix_contribution = -damping_constant * midpoint_adjoint_vector(node_gid, idim); + rate_of_change = (previous_velocity_vector(node_gid, idim) + current_velocity_vector(node_gid, idim)) / 2 - + matrix_contribution / node_mass(node_gid) - + phi_midpoint_adjoint_vector(node_gid, idim) / node_mass(node_gid); + current_adjoint_vector(node_gid, idim) = -rate_of_change * global_dt + previous_adjoint_vector(node_gid, idim); + matrix_contribution = 0; + // compute resulting row of force displacement gradient matrix transpose right multiplied by adjoint vector + for (int idof = 0; idof < Gradient_Matrix_Strides(node_gid * num_dim + idim); idof++) + { + dof_id = DOF_Graph_Matrix(node_gid * num_dim + idim, idof); + matrix_contribution += -midpoint_adjoint_vector(dof_id / num_dim, dof_id % num_dim) * Force_Gradient_Positions(node_gid * num_dim + idim, idof); + } + rate_of_change = -matrix_contribution; + // rate_of_change = -0.0000001*midpoint_adjoint_vector(node_gid,idim); + phi_current_adjoint_vector(node_gid, idim) = -rate_of_change * global_dt + phi_previous_adjoint_vector(node_gid, idim); + } }); // end parallel for - Kokkos::fence(); + Kokkos::fence(); - - boundary_adjoint(*mesh, boundary,current_adjoint_vector, phi_current_adjoint_vector); + boundary_adjoint(*mesh, boundary, current_adjoint_vector, phi_current_adjoint_vector); + } // end view scope - } //end view scope - - comm_adjoint_vectors(cycle); - //phi_adjoint_vector_distributed->describe(*fos,Teuchos::VERB_EXTREME); - } + comm_adjoint_vectors(cycle); + // phi_adjoint_vector_distributed->describe(*fos,Teuchos::VERB_EXTREME); + } } - /* ---------------------------------------------------------------------------- Gradient calculation for the kinetic energy minimization problem ------------------------------------------------------------------------------- */ -void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient(const_vec_array design_variables, vec_array design_gradients){ - - size_t num_bdy_nodes = mesh->num_bdy_nodes; - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos material = simparam->material; - const int num_dim = simparam->num_dims; - int num_corners = rnum_elem*num_nodes_in_elem; - real_t global_dt; - size_t current_data_index, next_data_index; - CArrayKokkos current_element_velocities = CArrayKokkos(num_nodes_in_elem,num_dim); - CArrayKokkos current_element_adjoint = CArrayKokkos(num_nodes_in_elem,num_dim); - - if(myrank==0) - std::cout << "Computing accumulated kinetic energy gradient" << std::endl; +void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient(const_vec_array design_variables, vec_array design_gradients) +{ + size_t num_bdy_nodes = mesh->num_bdy_nodes; + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos material = simparam->material; + const int num_dim = simparam->num_dims; + int num_corners = rnum_elem * num_nodes_in_elem; + real_t global_dt; + size_t current_data_index, next_data_index; + CArrayKokkos current_element_velocities = CArrayKokkos(num_nodes_in_elem, num_dim); + CArrayKokkos current_element_adjoint = CArrayKokkos(num_nodes_in_elem, num_dim); + + if (myrank == 0) + { + std::cout << "Computing accumulated kinetic energy gradient" << std::endl; + } - compute_topology_optimization_adjoint(); + compute_topology_optimization_adjoint(); - //compute design gradients - FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { - design_gradients(node_id,0) = 0; + // compute design gradients + FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { + design_gradients(node_id, 0) = 0; }); // end parallel for - Kokkos::fence(); - - //gradient contribution from kinetic energy vMv product. - for (unsigned long cycle = 0; cycle < last_time_step+1; cycle++) { - //compute timestep from time data - global_dt = time_data[cycle+1] - time_data[cycle]; - - //print - if(simparam->dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ - if (cycle==0){ - if(myrank==0) - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } - // print time step every 10 cycles - else if (cycle%20==0){ - if(myrank==0) - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } // end if - } + Kokkos::fence(); - //compute adjoint vector for this data point; use velocity midpoint - //view scope - { - const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array next_velocity_vector = (*forward_solve_velocity_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array next_adjoint_vector = (*adjoint_vector_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - - FOR_ALL_CLASS(elem_id, 0, rnum_elem, { - size_t node_id; - size_t corner_id; - real_t inner_product; - //std::cout << elem_mass(elem_id) <dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) + { + if (cycle == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } } - } - - for (int inode = 0; inode < num_nodes_in_elem; inode++){ - //compute gradient of local element contribution to v^t*M*v product - corner_id = elem_id*num_nodes_in_elem + inode; - corner_value_storage(corner_id) = inner_product*global_dt; - } - + // print time step every 10 cycles + else if (cycle % 20 == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } // end if + } + + // compute adjoint vector for this data point; use velocity midpoint + // view scope + { + const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_velocity_vector = (*forward_solve_velocity_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_adjoint_vector = (*adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + + FOR_ALL_CLASS(elem_id, 0, rnum_elem, { + size_t node_id; + size_t corner_id; + real_t inner_product; + // std::cout << elem_mass(elem_id) <dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ - if (cycle==0){ - if(myrank==0) - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } - // print time step every 10 cycles - else if (cycle%20==0){ - if(myrank==0) - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } // end if - } + Kokkos::fence(); - //compute adjoint vector for this data point; use velocity midpoint - //view scope - { - //const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array next_adjoint_vector = (*adjoint_vector_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - //const_vec_array current_coord_vector = forward_solve_coordinate_data[cycle]->getLocalView (Tpetra::Access::ReadOnly); - //const_vec_array final_coordinates = forward_solve_coordinate_data[last_time_step+1]->getLocalView (Tpetra::Access::ReadOnly); - - FOR_ALL_CLASS(elem_id, 0, rnum_elem, { - size_t node_id; - size_t corner_id; - real_t inner_product; - //std::cout << elem_mass(elem_id) <dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) + { + if (cycle == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } + // print time step every 10 cycles + else if (cycle % 20 == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } // end if + } -void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient_full(Teuchos::RCP design_densities_distributed, Teuchos::RCP design_gradients_distributed){ + // compute adjoint vector for this data point; use velocity midpoint + // view scope + { + // const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); + const_vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_adjoint_vector = (*adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + // const_vec_array current_coord_vector = forward_solve_coordinate_data[cycle]->getLocalView (Tpetra::Access::ReadOnly); + // const_vec_array final_coordinates = forward_solve_coordinate_data[last_time_step+1]->getLocalView (Tpetra::Access::ReadOnly); + + FOR_ALL_CLASS(elem_id, 0, rnum_elem, { + size_t node_id; + size_t corner_id; + real_t inner_product; + // std::cout << elem_mass(elem_id) <num_bdy_nodes; - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos material = simparam->material; - const int num_dim = simparam->num_dims; - int num_corners = rnum_elem*num_nodes_in_elem; - real_t global_dt; - bool element_constant_density = true; - size_t current_data_index, next_data_index; - CArrayKokkos current_element_velocities = CArrayKokkos(num_nodes_in_elem,num_dim); - CArrayKokkos current_element_adjoint = CArrayKokkos(num_nodes_in_elem,num_dim); + inner_product = 0; + for (int ifill = 0; ifill < num_nodes_in_elem; ifill++) + { + node_id = nodes_in_elem(elem_id, ifill); + for (int idim = 0; idim < num_dim; idim++) + { + inner_product += 0.00001 * current_element_adjoint(ifill, idim); + // inner_product += 0.0001; + } + } - if(myrank==0) - std::cout << "Computing accumulated kinetic energy gradient" << std::endl; + for (int inode = 0; inode < num_nodes_in_elem; inode++) + { + // compute gradient of local element contribution to v^t*M*v product + corner_id = elem_id * num_nodes_in_elem + inode; + corner_value_storage(corner_id) = -inner_product * global_dt / (double)num_nodes_in_elem; + } + }); // end parallel for + Kokkos::fence(); - compute_topology_optimization_adjoint_full(); + // accumulate node values from corner storage + // multiply + FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { + size_t corner_id; + for (int icorner = 0; icorner < num_corners_in_node(node_id); icorner++) + { + corner_id = corners_in_node(node_id, icorner); + design_gradients(node_id, 0) += corner_value_storage(corner_id); + } + }); // end parallel for + Kokkos::fence(); + } // end view scope + } +} - { //view scope - vec_array design_gradients = design_gradients_distributed->getLocalView (Tpetra::Access::ReadWrite); - const_vec_array design_densities = design_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); - //compute design gradients - FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { - design_gradients(node_id,0) = 0; - }); // end parallel for - Kokkos::fence(); +/* ---------------------------------------------------------------------------- + Gradient for the (unsimplified) kinetic energy minimization problem +------------------------------------------------------------------------------- */ - //gradient contribution from kinetic energy v(dM/drho)v product. - if(simparam->dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ - if(myrank==0){ - std::cout << "v*dM/drho*v term" << std::endl; - } +void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient_full(Teuchos::RCP design_densities_distributed, Teuchos::RCP design_gradients_distributed) +{ + size_t num_bdy_nodes = mesh->num_bdy_nodes; + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos material = simparam->material; + const int num_dim = simparam->num_dims; + int num_corners = rnum_elem * num_nodes_in_elem; + real_t global_dt; + bool element_constant_density = true; + size_t current_data_index, next_data_index; + CArrayKokkos current_element_velocities = CArrayKokkos(num_nodes_in_elem, num_dim); + CArrayKokkos current_element_adjoint = CArrayKokkos(num_nodes_in_elem, num_dim); + + if (myrank == 0) + { + std::cout << "Computing accumulated kinetic energy gradient" << std::endl; } - for (unsigned long cycle = 0; cycle < last_time_step+1; cycle++) { - //compute timestep from time data - global_dt = time_data[cycle+1] - time_data[cycle]; - - //print - if(simparam->dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ + compute_topology_optimization_adjoint_full(); - if (cycle==0){ - if(myrank==0) - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } - // print time step every 10 cycles - else if (cycle%20==0){ - if(myrank==0) - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } // end if - } + { // view scope + vec_array design_gradients = design_gradients_distributed->getLocalView(Tpetra::Access::ReadWrite); + const_vec_array design_densities = design_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); + // compute design gradients + FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { + design_gradients(node_id, 0) = 0; + }); // end parallel for + Kokkos::fence(); - //compute adjoint vector for this data point; use velocity midpoint - //view scope + // gradient contribution from kinetic energy v(dM/drho)v product. + if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) { - const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array next_velocity_vector = (*forward_solve_velocity_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array next_adjoint_vector = (*adjoint_vector_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - - FOR_ALL_CLASS(elem_id, 0, rnum_elem, { - size_t node_id; - size_t corner_id; - real_t inner_product; - //std::cout << elem_mass(elem_id) <dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) + { + if (cycle == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } + // print time step every 10 cycles + else if (cycle % 20 == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } // end if } - }); // end parallel for - Kokkos::fence(); - } //end view scope + // compute adjoint vector for this data point; use velocity midpoint + // view scope + { + const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_velocity_vector = (*forward_solve_velocity_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_adjoint_vector = (*adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + + FOR_ALL_CLASS(elem_id, 0, rnum_elem, { + size_t node_id; + size_t corner_id; + real_t inner_product; + // std::cout << elem_mass(elem_id) <dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ - if(myrank==0){ - std::cout << "gradient term involving adjoint derivative" << std::endl; + // accumulate node values from corner storage + // multiply + FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { + size_t corner_id; + for (int icorner = 0; icorner < num_corners_in_node(node_id); icorner++) + { + corner_id = corners_in_node(node_id, icorner); + design_gradients(node_id, 0) += corner_value_storage(corner_id); + } + }); // end parallel for + Kokkos::fence(); + } // end view scope } - } - for (unsigned long cycle = 0; cycle < last_time_step+1; cycle++) { - //compute timestep from time data - global_dt = time_data[cycle+1] - time_data[cycle]; - //print - if(simparam->dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ + // multiply by Hex8 constants (the diagonlization here only works for Hex8 anyway) + FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { + design_gradients(node_id, 0) *= 0.5 / (double)num_nodes_in_elem / (double)num_nodes_in_elem; + // design_gradients(node_id,0) =0.00001; + }); // end parallel for + Kokkos::fence(); - if (cycle==0){ - if(myrank==0) - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + // gradient contribution from time derivative of adjoint \dot{lambda}(dM/drho)v product. + if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) + { + if (myrank == 0) + { + std::cout << "gradient term involving adjoint derivative" << std::endl; + } } - // print time step every 10 cycles - else if (cycle%20==0){ - if(myrank==0) - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } // end if - } - //compute adjoint vector for this data point; use velocity midpoint - //view scope + for (unsigned long cycle = 0; cycle < last_time_step + 1; cycle++) { - const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_phi_adjoint_vector = (*phi_adjoint_vector_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array next_velocity_vector = (*forward_solve_velocity_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array next_adjoint_vector = (*adjoint_vector_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array next_phi_adjoint_vector = (*phi_adjoint_vector_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - - const real_t damping_constant = module_params->damping_constant; - FOR_ALL_CLASS(elem_id, 0, rnum_elem, { - real_t lambda_dot_current; - real_t lambda_dot_next; - size_t node_id; - size_t corner_id; - real_t inner_product; - //std::cout << elem_mass(elem_id) <dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) + { + if (cycle == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } + // print time step every 10 cycles + else if (cycle % 20 == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } // end if } - inner_product = 0; - for(int ifill=0; ifill < num_nodes_in_elem; ifill++){ - node_id = nodes_in_elem(elem_id, ifill); - for(int idim=0; idim < num_dim; idim++){ - //lambda_dot_current = lambda_dot_next = (next_adjoint_vector(node_id,idim)-current_adjoint_vector(node_id,idim))/global_dt; - lambda_dot_current = current_velocity_vector(node_id,idim) + damping_constant*current_adjoint_vector(node_id,idim)/node_mass(node_id) - current_phi_adjoint_vector(node_id,idim)/node_mass(node_id); - lambda_dot_next = next_velocity_vector(node_id,idim) + damping_constant*next_adjoint_vector(node_id,idim)/node_mass(node_id) - next_phi_adjoint_vector(node_id,idim)/node_mass(node_id); - inner_product += elem_mass(elem_id)*(lambda_dot_current+lambda_dot_next)*current_element_velocities(ifill,idim)/2; - } - } + // compute adjoint vector for this data point; use velocity midpoint + // view scope + { + const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array current_phi_adjoint_vector = (*phi_adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_velocity_vector = (*forward_solve_velocity_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_adjoint_vector = (*adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_phi_adjoint_vector = (*phi_adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + + const real_t damping_constant = module_params->damping_constant; + FOR_ALL_CLASS(elem_id, 0, rnum_elem, { + real_t lambda_dot_current; + real_t lambda_dot_next; + size_t node_id; + size_t corner_id; + real_t inner_product; + // std::cout << elem_mass(elem_id) <getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_adjoint_vector = (*adjoint_vector_data)[0]->getLocalView (Tpetra::Access::ReadOnly); - - FOR_ALL_CLASS(elem_id, 0, rnum_elem, { - real_t lambda_dot; - size_t node_id; - size_t corner_id; - real_t inner_product; - //std::cout << elem_mass(elem_id) <getLocalView(Tpetra::Access::ReadOnly); + const_vec_array current_adjoint_vector = (*adjoint_vector_data)[0]->getLocalView(Tpetra::Access::ReadOnly); + + FOR_ALL_CLASS(elem_id, 0, rnum_elem, { + real_t lambda_dot; + size_t node_id; + size_t corner_id; + real_t inner_product; + // std::cout << elem_mass(elem_id) <getLocalView (Tpetra::Access::ReadWrite); - const_host_vec_array host_design_variables = design_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); - compute_stiffness_gradients(host_design_variables, host_design_gradients); - }//end view scope + Kokkos::fence(); + } // end view scope + } // end view scope + // force_design_gradient_term(design_variables, design_gradients); + // view scope + { + host_vec_array host_design_gradients = design_gradients_distributed->getLocalView(Tpetra::Access::ReadWrite); + const_host_vec_array host_design_variables = design_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); + compute_stiffness_gradients(host_design_variables, host_design_gradients); + } // end view scope } /* ---------------------------------------------------------------------- Initialize global vectors and array maps needed for matrix assembly ------------------------------------------------------------------------- */ -void FEA_Module_Dynamic_Elasticity::init_assembly(){ - int num_dim = simparam->num_dims; - //const_host_elem_conn_array nodes_in_elem = global_nodes_in_elem_distributed->getLocalView (Tpetra::Access::ReadOnly); - Gradient_Matrix_Strides = DCArrayKokkos (nlocal_nodes*num_dim, "Gradient_Matrix_Strides"); - CArrayKokkos Graph_Fill(nall_nodes, "nall_nodes"); - CArrayKokkos current_row_nodes_scanned; - int local_node_index, current_column_index; - size_t max_stride = 0; - size_t nodes_per_element; - nodal_density_flag = simparam->nodal_density_flag; - penalty_power = simparam->optimization_options.simp_penalty_power; - - //allocate stride arrays - CArrayKokkos Graph_Matrix_Strides_initial(nlocal_nodes, "Graph_Matrix_Strides_initial"); - DCArrayKokkos Dual_Graph_Matrix_Strides_initial(nlocal_nodes, "Host_Graph_Matrix_Strides_initial"); - Graph_Matrix_Strides = DCArrayKokkos(nlocal_nodes, "Graph_Matrix_Strides"); - - //allocate storage for the sparse stiffness matrix map used in the assembly process - Global_Stiffness_Matrix_Assembly_Map = Global_Gradient_Matrix_Assembly_Map = DCArrayKokkos(rnum_elem, - max_nodes_per_element,max_nodes_per_element, "Global_Gradient_Matrix_Assembly_Map"); - - //allocate array used to determine global node repeats in the sparse graph later - DCArrayKokkos node_indices_used(nall_nodes, "node_indices_used"); - - /*allocate array that stores which column the node index occured on for the current row - when removing repeats*/ - DCArrayKokkos column_index(nall_nodes, "column_index"); - - //initialize nlocal arrays - FOR_ALL_CLASS(inode, 0, nlocal_nodes, { - Graph_Matrix_Strides_initial(inode) = 0; - Graph_Matrix_Strides(inode) = 0; - Graph_Fill(inode) = 0; +void FEA_Module_Dynamic_Elasticity::init_assembly() +{ + int num_dim = simparam->num_dims; + // const_host_elem_conn_array nodes_in_elem = global_nodes_in_elem_distributed->getLocalView (Tpetra::Access::ReadOnly); + Gradient_Matrix_Strides = DCArrayKokkos(nlocal_nodes * num_dim, "Gradient_Matrix_Strides"); + CArrayKokkos Graph_Fill(nall_nodes, "nall_nodes"); + CArrayKokkos current_row_nodes_scanned; + int local_node_index, current_column_index; + size_t max_stride = 0; + size_t nodes_per_element; + nodal_density_flag = simparam->nodal_density_flag; + penalty_power = simparam->optimization_options.simp_penalty_power; + + // allocate stride arrays + CArrayKokkos Graph_Matrix_Strides_initial(nlocal_nodes, "Graph_Matrix_Strides_initial"); + DCArrayKokkos Dual_Graph_Matrix_Strides_initial(nlocal_nodes, "Host_Graph_Matrix_Strides_initial"); + Graph_Matrix_Strides = DCArrayKokkos(nlocal_nodes, "Graph_Matrix_Strides"); + + // allocate storage for the sparse stiffness matrix map used in the assembly process + Global_Stiffness_Matrix_Assembly_Map = Global_Gradient_Matrix_Assembly_Map = DCArrayKokkos(rnum_elem, + max_nodes_per_element, max_nodes_per_element, "Global_Gradient_Matrix_Assembly_Map"); + + // allocate array used to determine global node repeats in the sparse graph later + DCArrayKokkos node_indices_used(nall_nodes, "node_indices_used"); + + /*allocate array that stores which column the node index occured on for the current row + when removing repeats*/ + DCArrayKokkos column_index(nall_nodes, "column_index"); + + // initialize nlocal arrays + FOR_ALL_CLASS(inode, 0, nlocal_nodes, { + Graph_Matrix_Strides_initial(inode) = 0; + Graph_Matrix_Strides(inode) = 0; + Graph_Fill(inode) = 0; }); // end parallel for - Kokkos::fence(); + Kokkos::fence(); - //initialize nall arrays - //initialize nlocal arrays - FOR_ALL_CLASS(inode, 0, nall_nodes, { - node_indices_used(inode) = 0; - column_index(inode) = 0; + // initialize nall arrays + // initialize nlocal arrays + FOR_ALL_CLASS(inode, 0, nall_nodes, { + node_indices_used(inode) = 0; + column_index(inode) = 0; }); // end parallel for - Kokkos::fence(); - - //count upper bound of strides for Sparse Pattern Graph by allowing repeats due to connectivity - if(num_dim == 2) - for (int ielem = 0; ielem < rnum_elem; ielem++){ - element_select->choose_2Delem_type(Element_Types(ielem), elem2D); - nodes_per_element = elem2D->num_nodes(); - for (int lnode = 0; lnode < nodes_per_element; lnode++){ - local_node_index = nodes_in_elem(ielem, lnode); - if(local_node_index < nlocal_nodes){ - Dual_Graph_Matrix_Strides_initial.host(local_node_index) += nodes_per_element; + Kokkos::fence(); + + // count upper bound of strides for Sparse Pattern Graph by allowing repeats due to connectivity + if (num_dim == 2) + { + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + element_select->choose_2Delem_type(Element_Types(ielem), elem2D); + nodes_per_element = elem2D->num_nodes(); + for (int lnode = 0; lnode < nodes_per_element; lnode++) + { + local_node_index = nodes_in_elem(ielem, lnode); + if (local_node_index < nlocal_nodes) + { + Dual_Graph_Matrix_Strides_initial.host(local_node_index) += nodes_per_element; + } + } } - } } - if(num_dim == 3) - for (int ielem = 0; ielem < rnum_elem; ielem++){ - element_select->choose_3Delem_type(Element_Types(ielem), elem); - nodes_per_element = elem->num_nodes(); - for (int lnode = 0; lnode < nodes_per_element; lnode++){ - local_node_index = nodes_in_elem(ielem, lnode); - if(local_node_index < nlocal_nodes){ - Dual_Graph_Matrix_Strides_initial.host(local_node_index) += nodes_per_element; + if (num_dim == 3) + { + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + element_select->choose_3Delem_type(Element_Types(ielem), elem); + nodes_per_element = elem->num_nodes(); + for (int lnode = 0; lnode < nodes_per_element; lnode++) + { + local_node_index = nodes_in_elem(ielem, lnode); + if (local_node_index < nlocal_nodes) + { + Dual_Graph_Matrix_Strides_initial.host(local_node_index) += nodes_per_element; + } + } } - } } - - Dual_Graph_Matrix_Strides_initial.update_device(); - //equate strides for later - FOR_ALL_CLASS(inode, 0, nlocal_nodes, { - Graph_Matrix_Strides(inode) = Graph_Matrix_Strides_initial(inode) = Dual_Graph_Matrix_Strides_initial(inode); + Dual_Graph_Matrix_Strides_initial.update_device(); + + // equate strides for later + FOR_ALL_CLASS(inode, 0, nlocal_nodes, { + Graph_Matrix_Strides(inode) = Graph_Matrix_Strides_initial(inode) = Dual_Graph_Matrix_Strides_initial(inode); }); // end parallel for - - //for (int inode = 0; inode < nlocal_nodes; inode++) - //std::cout << Graph_Matrix_Strides_initial(inode) << std::endl; - - //compute maximum stride - size_t update = 0; - REDUCE_MAX_CLASS(inode, 0, nlocal_nodes, update, { - if(update < Graph_Matrix_Strides_initial(inode)) - update = Graph_Matrix_Strides_initial(inode); + + // for (int inode = 0; inode < nlocal_nodes; inode++) + // std::cout << Graph_Matrix_Strides_initial(inode) << std::endl; + + // compute maximum stride + size_t update = 0; + REDUCE_MAX_CLASS(inode, 0, nlocal_nodes, update, { + if (update < Graph_Matrix_Strides_initial(inode)) + { + update = Graph_Matrix_Strides_initial(inode); + } }, max_stride); - - //std::cout << "THE MAX STRIDE" << max_stride << std::endl; - //allocate array used in the repeat removal process - current_row_nodes_scanned = CArrayKokkos(max_stride, "current_row_nodes_scanned"); - - //allocate sparse graph with node repeats - RaggedRightArrayKokkos Repeat_Graph_Matrix(Graph_Matrix_Strides_initial); - RaggedRightArrayofVectorsKokkos Element_local_indices(Graph_Matrix_Strides_initial,num_dim); - - //Fill the initial Graph with repeats - if(num_dim == 2){ - for (int ielem = 0; ielem < rnum_elem; ielem++){ - element_select->choose_2Delem_type(Element_Types(ielem), elem2D); - nodes_per_element = elem2D->num_nodes(); - for (int lnode = 0; lnode < nodes_per_element; lnode++){ - local_node_index = nodes_in_elem(ielem, lnode); - if(local_node_index < nlocal_nodes){ - for (int jnode = 0; jnode < nodes_per_element; jnode++){ - current_column_index = Graph_Fill(local_node_index)+jnode; - Repeat_Graph_Matrix(local_node_index, current_column_index) = nodes_in_elem(ielem,jnode); - - //fill inverse map - Element_local_indices(local_node_index,current_column_index,0) = ielem; - Element_local_indices(local_node_index,current_column_index,1) = lnode; - Element_local_indices(local_node_index,current_column_index,2) = jnode; - - //fill forward map - Global_Gradient_Matrix_Assembly_Map(ielem,lnode,jnode) = current_column_index; - } - Graph_Fill(local_node_index) += nodes_per_element; + + // std::cout << "THE MAX STRIDE" << max_stride << std::endl; + // allocate array used in the repeat removal process + current_row_nodes_scanned = CArrayKokkos(max_stride, "current_row_nodes_scanned"); + + // allocate sparse graph with node repeats + RaggedRightArrayKokkos Repeat_Graph_Matrix(Graph_Matrix_Strides_initial); + RaggedRightArrayofVectorsKokkos Element_local_indices(Graph_Matrix_Strides_initial, num_dim); + + // Fill the initial Graph with repeats + if (num_dim == 2) + { + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + element_select->choose_2Delem_type(Element_Types(ielem), elem2D); + nodes_per_element = elem2D->num_nodes(); + for (int lnode = 0; lnode < nodes_per_element; lnode++) + { + local_node_index = nodes_in_elem(ielem, lnode); + if (local_node_index < nlocal_nodes) + { + for (int jnode = 0; jnode < nodes_per_element; jnode++) + { + current_column_index = Graph_Fill(local_node_index) + jnode; + Repeat_Graph_Matrix(local_node_index, current_column_index) = nodes_in_elem(ielem, jnode); + + // fill inverse map + Element_local_indices(local_node_index, current_column_index, 0) = ielem; + Element_local_indices(local_node_index, current_column_index, 1) = lnode; + Element_local_indices(local_node_index, current_column_index, 2) = jnode; + + // fill forward map + Global_Gradient_Matrix_Assembly_Map(ielem, lnode, jnode) = current_column_index; + } + Graph_Fill(local_node_index) += nodes_per_element; + } + } } - } } - } - - if(num_dim == 3){ - for (int ielem = 0; ielem < rnum_elem; ielem++){ - element_select->choose_3Delem_type(Element_Types(ielem), elem); - nodes_per_element = elem->num_nodes(); - for (int lnode = 0; lnode < nodes_per_element; lnode++){ - local_node_index = nodes_in_elem(ielem, lnode); - if(local_node_index < nlocal_nodes){ - for (int jnode = 0; jnode < nodes_per_element; jnode++){ - current_column_index = Graph_Fill(local_node_index)+jnode; - Repeat_Graph_Matrix(local_node_index, current_column_index) = nodes_in_elem(ielem,jnode); - - //fill inverse map - Element_local_indices(local_node_index,current_column_index,0) = ielem; - Element_local_indices(local_node_index,current_column_index,1) = lnode; - Element_local_indices(local_node_index,current_column_index,2) = jnode; - - //fill forward map - Global_Gradient_Matrix_Assembly_Map(ielem,lnode,jnode) = current_column_index; - } - Graph_Fill(local_node_index) += nodes_per_element; + + if (num_dim == 3) + { + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + element_select->choose_3Delem_type(Element_Types(ielem), elem); + nodes_per_element = elem->num_nodes(); + for (int lnode = 0; lnode < nodes_per_element; lnode++) + { + local_node_index = nodes_in_elem(ielem, lnode); + if (local_node_index < nlocal_nodes) + { + for (int jnode = 0; jnode < nodes_per_element; jnode++) + { + current_column_index = Graph_Fill(local_node_index) + jnode; + Repeat_Graph_Matrix(local_node_index, current_column_index) = nodes_in_elem(ielem, jnode); + + // fill inverse map + Element_local_indices(local_node_index, current_column_index, 0) = ielem; + Element_local_indices(local_node_index, current_column_index, 1) = lnode; + Element_local_indices(local_node_index, current_column_index, 2) = jnode; + + // fill forward map + Global_Gradient_Matrix_Assembly_Map(ielem, lnode, jnode) = current_column_index; + } + Graph_Fill(local_node_index) += nodes_per_element; + } + } } - } } - } - - //debug statement - //std::cout << "started run" << std::endl; - //std::cout << "Graph Matrix Strides Repeat on task " << myrank << std::endl; - //for (int inode = 0; inode < nlocal_nodes; inode++) - //std::cout << Graph_Matrix_Strides(inode) << std::endl; - RUN_CLASS({ - //remove repeats from the inital graph setup - int current_node; - //remove repeats from the inital graph setup - int current_element_index; - int element_row_index; - int element_column_index; - int current_stride; - int current_row_n_nodes_scanned; - for (int inode = 0; inode < nlocal_nodes; inode++){ - current_row_n_nodes_scanned = 0; - for (int istride = 0; istride < Graph_Matrix_Strides(inode); istride++){ - //convert global index in graph to its local index for the flagging array - current_node = Repeat_Graph_Matrix(inode,istride); - //debug - //if(current_node==-1) - //std::cout << "Graph Matrix node access on task " << myrank << std::endl; - //std::cout << Repeat_Graph_Matrix(inode,istride) << std::endl; - if(node_indices_used(current_node)){ - //set global assembly map index to the location in the graph matrix where this global node was first found - current_element_index = Element_local_indices(inode,istride,0); - element_row_index = Element_local_indices(inode,istride,1); - element_column_index = Element_local_indices(inode,istride,2); - Global_Gradient_Matrix_Assembly_Map(current_element_index,element_row_index, element_column_index) - = column_index(current_node); - - - //swap current node with the end of the current row and shorten the stride of the row - //first swap information about the inverse and forward maps - - current_stride = Graph_Matrix_Strides(inode); - if(istride!=current_stride-1){ - Element_local_indices(inode,istride,0) = Element_local_indices(inode,current_stride-1,0); - Element_local_indices(inode,istride,1) = Element_local_indices(inode,current_stride-1,1); - Element_local_indices(inode,istride,2) = Element_local_indices(inode,current_stride-1,2); - current_element_index = Element_local_indices(inode,istride,0); - element_row_index = Element_local_indices(inode,istride,1); - element_column_index = Element_local_indices(inode,istride,2); - - Global_Gradient_Matrix_Assembly_Map(current_element_index,element_row_index, element_column_index) - = istride; - - //now that the element map information has been copied, copy the global node index and delete the last index - - Repeat_Graph_Matrix(inode,istride) = Repeat_Graph_Matrix(inode,current_stride-1); - } - istride--; - Graph_Matrix_Strides(inode)--; + + // debug statement + // std::cout << "started run" << std::endl; + // std::cout << "Graph Matrix Strides Repeat on task " << myrank << std::endl; + // for (int inode = 0; inode < nlocal_nodes; inode++) + // std::cout << Graph_Matrix_Strides(inode) << std::endl; + RUN_CLASS({ + // remove repeats from the inital graph setup + int current_node; + // remove repeats from the inital graph setup + int current_element_index; + int element_row_index; + int element_column_index; + int current_stride; + int current_row_n_nodes_scanned; + for (int inode = 0; inode < nlocal_nodes; inode++) + { + current_row_n_nodes_scanned = 0; + for (int istride = 0; istride < Graph_Matrix_Strides(inode); istride++) + { + // convert global index in graph to its local index for the flagging array + current_node = Repeat_Graph_Matrix(inode, istride); + // debug + // if(current_node==-1) + // std::cout << "Graph Matrix node access on task " << myrank << std::endl; + // std::cout << Repeat_Graph_Matrix(inode,istride) << std::endl; + if (node_indices_used(current_node)) + { + // set global assembly map index to the location in the graph matrix where this global node was first found + current_element_index = Element_local_indices(inode, istride, 0); + element_row_index = Element_local_indices(inode, istride, 1); + element_column_index = Element_local_indices(inode, istride, 2); + Global_Gradient_Matrix_Assembly_Map(current_element_index, element_row_index, element_column_index) + = column_index(current_node); + + // swap current node with the end of the current row and shorten the stride of the row + // first swap information about the inverse and forward maps + + current_stride = Graph_Matrix_Strides(inode); + if (istride != current_stride - 1) + { + Element_local_indices(inode, istride, 0) = Element_local_indices(inode, current_stride - 1, 0); + Element_local_indices(inode, istride, 1) = Element_local_indices(inode, current_stride - 1, 1); + Element_local_indices(inode, istride, 2) = Element_local_indices(inode, current_stride - 1, 2); + current_element_index = Element_local_indices(inode, istride, 0); + element_row_index = Element_local_indices(inode, istride, 1); + element_column_index = Element_local_indices(inode, istride, 2); + + Global_Gradient_Matrix_Assembly_Map(current_element_index, element_row_index, element_column_index) + = istride; + + // now that the element map information has been copied, copy the global node index and delete the last index + + Repeat_Graph_Matrix(inode, istride) = Repeat_Graph_Matrix(inode, current_stride - 1); + } + istride--; + Graph_Matrix_Strides(inode)--; + } + else + { + /*this node hasn't shown up in the row before; add it to the list of nodes + that have been scanned uniquely. Use this list to reset the flag array + afterwards without having to loop over all the nodes in the system*/ + node_indices_used(current_node) = 1; + column_index(current_node) = istride; + current_row_nodes_scanned(current_row_n_nodes_scanned) = current_node; + current_row_n_nodes_scanned++; + } + } + // reset nodes used list for the next row of the sparse list + for (int node_reset = 0; node_reset < current_row_n_nodes_scanned; node_reset++) + { + node_indices_used(current_row_nodes_scanned(node_reset)) = 0; + } } - else{ - /*this node hasn't shown up in the row before; add it to the list of nodes - that have been scanned uniquely. Use this list to reset the flag array - afterwards without having to loop over all the nodes in the system*/ - node_indices_used(current_node) = 1; - column_index(current_node) = istride; - current_row_nodes_scanned(current_row_n_nodes_scanned) = current_node; - current_row_n_nodes_scanned++; + }); + Kokkos::fence(); + + Graph_Matrix_Strides.update_host(); + // copy reduced content to non_repeat storage + Graph_Matrix = RaggedRightArrayKokkos(Graph_Matrix_Strides); + + FOR_ALL_CLASS(inode, 0, nlocal_nodes, { + for (int istride = 0; istride < Graph_Matrix_Strides(inode); istride++) + { + Graph_Matrix(inode, istride) = Repeat_Graph_Matrix(inode, istride); } - } - //reset nodes used list for the next row of the sparse list - for(int node_reset = 0; node_reset < current_row_n_nodes_scanned; node_reset++) - node_indices_used(current_row_nodes_scanned(node_reset)) = 0; + }); // end parallel for - } - }); - Kokkos::fence(); - - Graph_Matrix_Strides.update_host(); - //copy reduced content to non_repeat storage - Graph_Matrix = RaggedRightArrayKokkos(Graph_Matrix_Strides); - - FOR_ALL_CLASS(inode, 0, nlocal_nodes, { - for(int istride = 0; istride < Graph_Matrix_Strides(inode); istride++){ - Graph_Matrix(inode,istride) = Repeat_Graph_Matrix(inode,istride); - } + // deallocate repeat matrix + + /*At this stage the sparse graph should have unique global indices on each row. + The constructed Assembly map (to the global sparse matrix) + is used to loop over each element's local stiffness matrix in the assembly process.*/ + + // expand strides for stiffness matrix by multipling by dim + FOR_ALL_CLASS(idof, 0, num_dim * nlocal_nodes, { + Gradient_Matrix_Strides(idof) = num_dim * Graph_Matrix_Strides(idof / num_dim); }); // end parallel for - //deallocate repeat matrix - - /*At this stage the sparse graph should have unique global indices on each row. - The constructed Assembly map (to the global sparse matrix) - is used to loop over each element's local stiffness matrix in the assembly process.*/ - - //expand strides for stiffness matrix by multipling by dim - FOR_ALL_CLASS(idof, 0, num_dim*nlocal_nodes, { - Gradient_Matrix_Strides(idof) = num_dim*Graph_Matrix_Strides(idof/num_dim); + Stiffness_Matrix = Force_Gradient_Positions = RaggedRightArrayKokkos(Gradient_Matrix_Strides); + Force_Gradient_Velocities = RaggedRightArrayKokkos(Gradient_Matrix_Strides); + DOF_Graph_Matrix = RaggedRightArrayKokkos(Gradient_Matrix_Strides); + + // set stiffness Matrix Graph + // debug print + // std::cout << "DOF GRAPH MATRIX ENTRIES ON TASK " << myrank << std::endl; + FOR_ALL_CLASS(idof, 0, num_dim * nlocal_nodes, { + for (int istride = 0; istride < Gradient_Matrix_Strides(idof); istride++) + { + DOF_Graph_Matrix(idof, istride) = Graph_Matrix(idof / num_dim, istride / num_dim) * num_dim + istride % num_dim; + } }); // end parallel for - Stiffness_Matrix = Force_Gradient_Positions = RaggedRightArrayKokkos(Gradient_Matrix_Strides); - Force_Gradient_Velocities = RaggedRightArrayKokkos(Gradient_Matrix_Strides); - DOF_Graph_Matrix = RaggedRightArrayKokkos (Gradient_Matrix_Strides); + Stiffness_Matrix_Strides = Gradient_Matrix_Strides; + + /* + //construct distributed gradient matrix from local kokkos data + //build column map for the global gradient matrix + Teuchos::RCP > colmap; + const Teuchos::RCP > dommap = local_dof_map; + + Tpetra::Details::makeColMap(colmap,dommap,DOF_Graph_Matrix.get_kokkos_view(), nullptr); + + size_t nnz = DOF_Graph_Matrix.size(); + + //debug print + //std::cout << "DOF GRAPH SIZE ON RANK " << myrank << " IS " << nnz << std::endl; - //set stiffness Matrix Graph - //debug print - //std::cout << "DOF GRAPH MATRIX ENTRIES ON TASK " << myrank << std::endl; - FOR_ALL_CLASS(idof, 0, num_dim*nlocal_nodes, { - for (int istride = 0; istride < Gradient_Matrix_Strides(idof); istride++){ - DOF_Graph_Matrix(idof,istride) = Graph_Matrix(idof/num_dim,istride/num_dim)*num_dim + istride%num_dim; + //local indices in the graph using the constructed column map + CArrayKokkos gradient_local_indices(nnz, "gradient_local_indices"); + + //row offsets with compatible template arguments + Kokkos::View row_offsets = DOF_Graph_Matrix.start_index_; + row_pointers row_offsets_pass("row_offsets", nlocal_nodes*num_dim+1); + for(int ipass = 0; ipass < nlocal_nodes*num_dim + 1; ipass++){ + row_offsets_pass(ipass) = row_offsets(ipass); } - }); // end parallel for - Stiffness_Matrix_Strides = Gradient_Matrix_Strides; - - /* - //construct distributed gradient matrix from local kokkos data - //build column map for the global gradient matrix - Teuchos::RCP > colmap; - const Teuchos::RCP > dommap = local_dof_map; - - Tpetra::Details::makeColMap(colmap,dommap,DOF_Graph_Matrix.get_kokkos_view(), nullptr); - - size_t nnz = DOF_Graph_Matrix.size(); - - //debug print - //std::cout << "DOF GRAPH SIZE ON RANK " << myrank << " IS " << nnz << std::endl; - - //local indices in the graph using the constructed column map - CArrayKokkos gradient_local_indices(nnz, "gradient_local_indices"); - - //row offsets with compatible template arguments - Kokkos::View row_offsets = DOF_Graph_Matrix.start_index_; - row_pointers row_offsets_pass("row_offsets", nlocal_nodes*num_dim+1); - for(int ipass = 0; ipass < nlocal_nodes*num_dim + 1; ipass++){ - row_offsets_pass(ipass) = row_offsets(ipass); - } - - size_t entrycount = 0; - for(int irow = 0; irow < nlocal_nodes*num_dim; irow++){ - for(int istride = 0; istride < Gradient_Matrix_Strides(irow); istride++){ - gradient_local_indices(entrycount) = colmap->getLocalElement(DOF_Graph_Matrix(irow,istride)); - entrycount++; + size_t entrycount = 0; + for(int irow = 0; irow < nlocal_nodes*num_dim; irow++){ + for(int istride = 0; istride < Gradient_Matrix_Strides(irow); istride++){ + gradient_local_indices(entrycount) = colmap->getLocalElement(DOF_Graph_Matrix(irow,istride)); + entrycount++; + } } - } - - - //sort values and indices - Tpetra::Import_Util::sortCrsEntries(row_offsets_pass, gradient_local_indices.get_kokkos_view(), Force_Gradient_Positions.get_kokkos_view()); - Tpetra::Import_Util::sortCrsEntries(row_offsets_pass, gradient_local_indices.get_kokkos_view(), Force_Gradient_Velocities.get_kokkos_view()); - - //Teuchos::RCP crs_matrix_params = Teuchos::rcp(new Teuchos::ParameterList("crsmatrix")); - //crs_matrix_params->set("sorted", false); - distributed_force_gradient_positions = Teuchos::rcp(new MAT(local_dof_map, colmap, row_offsets_pass, gradient_local_indices.get_kokkos_view(), Force_Gradient_Positions.get_kokkos_view())); - distributed_force_gradient_positions->fillComplete(); - distributed_force_gradient_velocities = Teuchos::rcp(new MAT(local_dof_map, colmap, row_offsets_pass, gradient_local_indices.get_kokkos_view(), Force_Gradient_Velocities.get_kokkos_view())); - distributed_force_gradient_velocities->fillComplete(); - */ - //distributed_force_gradient_positions->describe(*fos,Teuchos::VERB_EXTREME); - //distributed_force_gradient_velocities->describe(*fos,Teuchos::VERB_EXTREME); + + + //sort values and indices + Tpetra::Import_Util::sortCrsEntries(row_offsets_pass, gradient_local_indices.get_kokkos_view(), Force_Gradient_Positions.get_kokkos_view()); + Tpetra::Import_Util::sortCrsEntries(row_offsets_pass, gradient_local_indices.get_kokkos_view(), Force_Gradient_Velocities.get_kokkos_view()); + + //Teuchos::RCP crs_matrix_params = Teuchos::rcp(new Teuchos::ParameterList("crsmatrix")); + //crs_matrix_params->set("sorted", false); + distributed_force_gradient_positions = Teuchos::rcp(new MAT(local_dof_map, colmap, row_offsets_pass, gradient_local_indices.get_kokkos_view(), Force_Gradient_Positions.get_kokkos_view())); + distributed_force_gradient_positions->fillComplete(); + distributed_force_gradient_velocities = Teuchos::rcp(new MAT(local_dof_map, colmap, row_offsets_pass, gradient_local_indices.get_kokkos_view(), Force_Gradient_Velocities.get_kokkos_view())); + distributed_force_gradient_velocities->fillComplete(); + */ + // distributed_force_gradient_positions->describe(*fos,Teuchos::VERB_EXTREME); + // distributed_force_gradient_velocities->describe(*fos,Teuchos::VERB_EXTREME); } /* ---------------------------------------------------------------------- Enforce boundary conditions on the adjoint vectors ------------------------------------------------------------------------- */ -void FEA_Module_Dynamic_Elasticity::boundary_adjoint(const mesh_t &mesh, - const DCArrayKokkos &boundary, - vec_array &node_adjoint, - vec_array &node_phi_adjoint){ - - //error and debug flag - //DCArrayKokkos print_flag(1, "print_flag"); - //print_flag.host(0) = false; - //print_flag.update_device(); - - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - int num_dims = simparam->num_dims; +void FEA_Module_Dynamic_Elasticity::boundary_adjoint(const mesh_t& mesh, + const DCArrayKokkos& boundary, + vec_array& node_adjoint, + vec_array& node_phi_adjoint) +{ + // error and debug flag + // DCArrayKokkos print_flag(1, "print_flag"); + // print_flag.host(0) = false; + // print_flag.update_device(); + + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + int num_dims = simparam->num_dims; // Loop over boundary sets - for (size_t bdy_set=0; bdy_set fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - //if(myrank==0) - //*fos << "Density data :" << std::endl; - //node_densities_distributed->describe(*fos,Teuchos::VERB_EXTREME); - //*fos << std::endl; - //std::fflush(stdout); - - //communicate design densities - //create import object using local node indices map and all indices map - //Tpetra::Import importer(map, all_node_map); - - //comms to get ghosts - (*adjoint_vector_data)[cycle]->doImport(*adjoint_vector_distributed, *importer, Tpetra::INSERT); - (*phi_adjoint_vector_data)[cycle]->doImport(*phi_adjoint_vector_distributed, *importer, Tpetra::INSERT); - //all_node_map->describe(*fos,Teuchos::VERB_EXTREME); - //all_node_velocities_distributed->describe(*fos,Teuchos::VERB_EXTREME); - - //update_count++; - //if(update_count==1){ - //MPI_Barrier(world); - //MPI_Abort(world,4); - //} +void FEA_Module_Dynamic_Elasticity::comm_adjoint_vectors(int cycle) +{ + // debug print of design vector + // std::ostream &out = std::cout; + // Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + // if(myrank==0) + // *fos << "Density data :" << std::endl; + // node_densities_distributed->describe(*fos,Teuchos::VERB_EXTREME); + // *fos << std::endl; + // std::fflush(stdout); + + // communicate design densities + // create import object using local node indices map and all indices map + // Tpetra::Import importer(map, all_node_map); + + // comms to get ghosts + (*adjoint_vector_data)[cycle]->doImport(*adjoint_vector_distributed, *importer, Tpetra::INSERT); + (*phi_adjoint_vector_data)[cycle]->doImport(*phi_adjoint_vector_distributed, *importer, Tpetra::INSERT); + // all_node_map->describe(*fos,Teuchos::VERB_EXTREME); + // all_node_velocities_distributed->describe(*fos,Teuchos::VERB_EXTREME); + + // update_count++; + // if(update_count==1){ + // MPI_Barrier(world); + // MPI_Abort(world,4); + // } } diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/force_elastic.cpp b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/force_elastic.cpp index 20012dd28..4ef144028 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/force_elastic.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/force_elastic.cpp @@ -9,690 +9,796 @@ // ----------------------------------------------------------------------------- // This function calculates the corner forces and the evolves stress (hypo) -//------------------------------------------------------------------------------ -void FEA_Module_Dynamic_Elasticity::get_force_elastic(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &node_mass, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &elem_mat_id, - DViewCArrayKokkos &corner_force, - const double rk_alpha, - const size_t cycle - ){ - - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - const size_t num_dim = mesh.num_dims; - const real_t damping_constant = module_params->damping_constant; - const_vec_array all_initial_node_coords = all_initial_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); +// ------------------------------------------------------------------------------ +void FEA_Module_Dynamic_Elasticity::get_force_elastic(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_mass, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + DViewCArrayKokkos& corner_force, + const double rk_alpha, + const size_t cycle + ) +{ + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + const size_t num_dim = mesh.num_dims; + const real_t damping_constant = module_params->damping_constant; + const_vec_array all_initial_node_coords = all_initial_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); // walk over the nodes to update the velocity FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - size_t dof_id; double node_force[3]; - for (size_t dim = 0; dim < num_dim; dim++){ + for (size_t dim = 0; dim < num_dim; dim++) + { node_force[dim] = 0.0; } // end for dim - + // loop over all corners around the node and calculate the nodal force - for (size_t corner_lid=0; corner_lid &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &node_mass, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &elem_mat_id, - DViewCArrayKokkos &corner_force, - const double rk_alpha, - const size_t cycle - ){ - - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - const size_t num_dim = mesh.num_dims; - const_vec_array all_initial_node_coords = all_initial_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - const size_t num_lcs = module_params->loading.size(); - - const DCArrayKokkos mat_fill = simparam->mat_fill; - const DCArrayKokkos loading = module_params->loading; - - //debug check - //std::cout << "NUMBER OF LOADING CONDITIONS: " << num_lcs << std::endl; +// ------------------------------------------------------------------------------ +void FEA_Module_Dynamic_Elasticity::applied_forces(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_mass, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + DViewCArrayKokkos& corner_force, + const double rk_alpha, + const size_t cycle + ) +{ + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + const size_t num_dim = mesh.num_dims; + const_vec_array all_initial_node_coords = all_initial_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + const size_t num_lcs = module_params->loading.size(); + + const DCArrayKokkos mat_fill = simparam->mat_fill; + const DCArrayKokkos loading = module_params->loading; + + // debug check + // std::cout << "NUMBER OF LOADING CONDITIONS: " << num_lcs << std::endl; // walk over the nodes to update the velocity - //FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (size_t node_gid = 0; node_gid <= nlocal_nodes; node_gid++) { + // FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + for (size_t node_gid = 0; node_gid <= nlocal_nodes; node_gid++) + { double current_node_coords[3]; size_t dof_id; double node_force[3]; double applied_force[3]; double radius; - for (size_t dim = 0; dim < num_dim; dim++){ + for (size_t dim = 0; dim < num_dim; dim++) + { node_force[dim] = 0.0; current_node_coords[dim] = all_initial_node_coords(node_gid, dim); } // end for dim - radius = sqrt(current_node_coords[0]*current_node_coords[0]+current_node_coords[1]*current_node_coords[1]+current_node_coords[2]*current_node_coords[2]); - for(size_t ilc=0; ilc < num_lcs; ilc++){ - //debug check - //std::cout << "LOADING CONDITION VOLUME TYPE: " << to_string(loading(ilc).volume) << std::endl; - - bool fill_this = loading(ilc).contains(current_node_coords); - if(fill_this){ - // loop over all corners around the node and calculate the nodal force - for (size_t corner_lid=0; corner_lidnum_dims; - int nodes_per_elem; - int current_row_n_nodes_scanned; - int local_dof_index, local_node_index, current_row, current_column; - int max_stride = 0; - - CArrayKokkos Local_Stiffness_Matrix(num_dim*max_nodes_per_element,num_dim*max_nodes_per_element); - - //initialize stiffness Matrix entries to 0 - //debug print - //std::cout << "DOF GRAPH MATRIX ENTRIES ON TASK " << myrank << std::endl; - for (int idof = 0; idof < num_dim*nlocal_nodes; idof++){ - for (int istride = 0; istride < Stiffness_Matrix_Strides(idof); istride++){ - Stiffness_Matrix(idof,istride) = 0; - //debug print - //std::cout << "{" <num_dims; + int nodes_per_elem; + int current_row_n_nodes_scanned; + int local_dof_index, local_node_index, current_row, current_column; + int max_stride = 0; + + CArrayKokkos Local_Stiffness_Matrix(num_dim * max_nodes_per_element, num_dim * max_nodes_per_element); + + // initialize stiffness Matrix entries to 0 + // debug print + // std::cout << "DOF GRAPH MATRIX ENTRIES ON TASK " << myrank << std::endl; + for (int idof = 0; idof < num_dim * nlocal_nodes; idof++) + { + for (int istride = 0; istride < Stiffness_Matrix_Strides(idof); istride++) + { + Stiffness_Matrix(idof, istride) = 0; + // debug print + // std::cout << "{" <choose_2Delem_type(Element_Types(ielem), elem2D); - nodes_per_elem = elem2D->num_nodes(); - //construct local stiffness matrix for this element - local_matrix_multiply(ielem, Local_Stiffness_Matrix); - //assign entries of this local matrix to the sparse global matrix storage; - for (int inode = 0; inode < nodes_per_elem; inode++){ - //see if this node is local - local_node_index = nodes_in_elem.host(ielem,inode); - if(!map->isNodeLocalElement(local_node_index)) continue; - //set dof row start index - current_row = num_dim*local_node_index; - for(int jnode = 0; jnode < nodes_per_elem; jnode++){ - - current_column = num_dim*Global_Stiffness_Matrix_Assembly_Map(ielem,inode,jnode); - for (int idim = 0; idim < num_dim; idim++){ - for (int jdim = 0; jdim < num_dim; jdim++){ - - //debug print - //if(current_row + idim==15&¤t_column + jdim==4) - //std::cout << " Local stiffness matrix contribution for row " << current_row + idim +1 << " and column " << current_column + jdim + 1 << " : " << - //Local_Stiffness_Matrix(num_dim*inode + idim,num_dim*jnode + jdim) << " from " << ielem +1 << " i: " << num_dim*inode+idim+1 << " j: " << num_dim*jnode + jdim +1 << std::endl << std::endl; - //end debug - - Stiffness_Matrix(current_row + idim, current_column + jdim) += Local_Stiffness_Matrix(num_dim*inode + idim,num_dim*jnode + jdim); - } + + // reset unsorted DOF Graph corresponding to assembly mapped values + // debug print + for (int idof = 0; idof < num_dim * nlocal_nodes; idof++) + { + for (int istride = 0; istride < Stiffness_Matrix_Strides(idof); istride++) + { + DOF_Graph_Matrix(idof, istride) = Graph_Matrix(idof / num_dim, istride / num_dim) * num_dim + istride % num_dim; + // debug print + // std::cout << "{" <choose_3Delem_type(Element_Types(ielem), elem); - nodes_per_elem = elem->num_nodes(); - //construct local stiffness matrix for this element - local_matrix_multiply(ielem, Local_Stiffness_Matrix); - //assign entries of this local matrix to the sparse global matrix storage; - for (int inode = 0; inode < nodes_per_elem; inode++){ - //see if this node is local - local_node_index = nodes_in_elem.host(ielem,inode); - if(!map->isNodeLocalElement(local_node_index)) continue; - //set dof row start index - current_row = num_dim*local_node_index; - for(int jnode = 0; jnode < nodes_per_elem; jnode++){ - - current_column = num_dim*Global_Stiffness_Matrix_Assembly_Map(ielem,inode,jnode); - for (int idim = 0; idim < num_dim; idim++){ - for (int jdim = 0; jdim < num_dim; jdim++){ - - //debug print - //if(current_row + idim==15&¤t_column + jdim==4) - //std::cout << " Local stiffness matrix contribution for row " << current_row + idim +1 << " and column " << current_column + jdim + 1 << " : " << - //Local_Stiffness_Matrix(num_dim*inode + idim,num_dim*jnode + jdim) << " from " << ielem +1 << " i: " << num_dim*inode+idim+1 << " j: " << num_dim*jnode + jdim +1 << std::endl << std::endl; - //end debug - - Stiffness_Matrix(current_row + idim, current_column + jdim) += Local_Stiffness_Matrix(num_dim*inode + idim,num_dim*jnode + jdim); - } + + // assemble the global stiffness matrix + if (num_dim == 2) + { + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + element_select->choose_2Delem_type(Element_Types(ielem), elem2D); + nodes_per_elem = elem2D->num_nodes(); + // construct local stiffness matrix for this element + local_matrix_multiply(ielem, Local_Stiffness_Matrix); + // assign entries of this local matrix to the sparse global matrix storage; + for (int inode = 0; inode < nodes_per_elem; inode++) + { + // see if this node is local + local_node_index = nodes_in_elem.host(ielem, inode); + if (!map->isNodeLocalElement(local_node_index)) + { + continue; + } + // set dof row start index + current_row = num_dim * local_node_index; + for (int jnode = 0; jnode < nodes_per_elem; jnode++) + { + current_column = num_dim * Global_Stiffness_Matrix_Assembly_Map(ielem, inode, jnode); + for (int idim = 0; idim < num_dim; idim++) + { + for (int jdim = 0; jdim < num_dim; jdim++) + { + // debug print + // if(current_row + idim==15&¤t_column + jdim==4) + // std::cout << " Local stiffness matrix contribution for row " << current_row + idim +1 << " and column " << current_column + jdim + 1 << " : " << + // Local_Stiffness_Matrix(num_dim*inode + idim,num_dim*jnode + jdim) << " from " << ielem +1 << " i: " << num_dim*inode+idim+1 << " j: " << num_dim*jnode + jdim +1 << std::endl << std::endl; + // end debug + + Stiffness_Matrix(current_row + idim, current_column + jdim) += Local_Stiffness_Matrix(num_dim * inode + idim, num_dim * jnode + jdim); + } + } + } + } } - } } - } + if (num_dim == 3) + { + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + element_select->choose_3Delem_type(Element_Types(ielem), elem); + nodes_per_elem = elem->num_nodes(); + // construct local stiffness matrix for this element + local_matrix_multiply(ielem, Local_Stiffness_Matrix); + // assign entries of this local matrix to the sparse global matrix storage; + for (int inode = 0; inode < nodes_per_elem; inode++) + { + // see if this node is local + local_node_index = nodes_in_elem.host(ielem, inode); + if (!map->isNodeLocalElement(local_node_index)) + { + continue; + } + // set dof row start index + current_row = num_dim * local_node_index; + for (int jnode = 0; jnode < nodes_per_elem; jnode++) + { + current_column = num_dim * Global_Stiffness_Matrix_Assembly_Map(ielem, inode, jnode); + for (int idim = 0; idim < num_dim; idim++) + { + for (int jdim = 0; jdim < num_dim; jdim++) + { + // debug print + // if(current_row + idim==15&¤t_column + jdim==4) + // std::cout << " Local stiffness matrix contribution for row " << current_row + idim +1 << " and column " << current_column + jdim + 1 << " : " << + // Local_Stiffness_Matrix(num_dim*inode + idim,num_dim*jnode + jdim) << " from " << ielem +1 << " i: " << num_dim*inode+idim+1 << " j: " << num_dim*jnode + jdim +1 << std::endl << std::endl; + // end debug + + Stiffness_Matrix(current_row + idim, current_column + jdim) += Local_Stiffness_Matrix(num_dim * inode + idim, num_dim * jnode + jdim); + } + } + } + } + } + } - //debug print of A matrix - //*fos << "Global Stiffness Matrix :" << std::endl; - //Global_Stiffness_Matrix->describe(*fos,Teuchos::VERB_EXTREME); - //*fos << std::endl; + // debug print of A matrix + // *fos << "Global Stiffness Matrix :" << std::endl; + // Global_Stiffness_Matrix->describe(*fos,Teuchos::VERB_EXTREME); + // *fos << std::endl; - //filter small negative numbers (that should have been 0 from cancellation) from floating point error - /* - for (int idof = 0; idof < num_dim*nlocal_nodes; idof++){ - for (int istride = 0; istride < Stiffness_Matrix_Strides(idof); istride++){ - if(Stiffness_Matrix(idof,istride)<0.000000001*simparam->Elastic_Modulus*density_epsilon||Stiffness_Matrix(idof,istride)>-0.000000001*simparam->Elastic_Modulus*density_epsilon) - Stiffness_Matrix(idof,istride) = 0; + // filter small negative numbers (that should have been 0 from cancellation) from floating point error + /* + for (int idof = 0; idof < num_dim*nlocal_nodes; idof++){ + for (int istride = 0; istride < Stiffness_Matrix_Strides(idof); istride++){ + if(Stiffness_Matrix(idof,istride)<0.000000001*simparam->Elastic_Modulus*density_epsilon||Stiffness_Matrix(idof,istride)>-0.000000001*simparam->Elastic_Modulus*density_epsilon) + Stiffness_Matrix(idof,istride) = 0; + //debug print + //std::cout << "{" <get_unit_scaling(); - real_t penalty_product = 1; - real_t density_epsilon = simparam->optimization_options.density_epsilon; - if(density < 0) density = 0; - for(int i = 0; i < penalty_power; i++) - penalty_product *= density; - //relationship between density and stiffness - Element_Modulus = (density_epsilon + (1 - density_epsilon)*penalty_product)*module_params->material.elastic_modulus/unit_scaling/unit_scaling; - //Element_Modulus = density*simparam->Elastic_Modulus/unit_scaling/unit_scaling; - Poisson_Ratio = module_params->material.poisson_ratio; +void FEA_Module_Dynamic_Elasticity::Element_Material_Properties(size_t ielem, real_t& Element_Modulus, real_t& Poisson_Ratio, real_t density) +{ + real_t unit_scaling = simparam->get_unit_scaling(); + real_t penalty_product = 1; + real_t density_epsilon = simparam->optimization_options.density_epsilon; + if (density < 0) + { + density = 0; + } + for (int i = 0; i < penalty_power; i++) + { + penalty_product *= density; + } + // relationship between density and stiffness + Element_Modulus = (density_epsilon + (1 - density_epsilon) * penalty_product) * module_params->material.elastic_modulus / unit_scaling / unit_scaling; + // Element_Modulus = density*simparam->Elastic_Modulus/unit_scaling/unit_scaling; + Poisson_Ratio = module_params->material.poisson_ratio; } /* ---------------------------------------------------------------------- Retrieve derivative of material properties with respect to local density ------------------------------------------------------------------------- */ -void FEA_Module_Dynamic_Elasticity::Gradient_Element_Material_Properties(size_t ielem, real_t &Element_Modulus_Derivative, real_t &Poisson_Ratio, real_t density){ - real_t unit_scaling = simparam->get_unit_scaling(); - real_t penalty_product = 1; - real_t density_epsilon = simparam->optimization_options.density_epsilon; - Element_Modulus_Derivative = 0; - if(density < 0) density = 0; - for(int i = 0; i < penalty_power - 1; i++) - penalty_product *= density; - //relationship between density and stiffness - Element_Modulus_Derivative = penalty_power*(1 - density_epsilon)*penalty_product*module_params->material.elastic_modulus/unit_scaling/unit_scaling; - //Element_Modulus_Derivative = simparam->Elastic_Modulus/unit_scaling/unit_scaling; - Poisson_Ratio = module_params->material.poisson_ratio; +void FEA_Module_Dynamic_Elasticity::Gradient_Element_Material_Properties(size_t ielem, real_t& Element_Modulus_Derivative, real_t& Poisson_Ratio, real_t density) +{ + real_t unit_scaling = simparam->get_unit_scaling(); + real_t penalty_product = 1; + real_t density_epsilon = simparam->optimization_options.density_epsilon; + Element_Modulus_Derivative = 0; + if (density < 0) + { + density = 0; + } + for (int i = 0; i < penalty_power - 1; i++) + { + penalty_product *= density; + } + // relationship between density and stiffness + Element_Modulus_Derivative = penalty_power * (1 - density_epsilon) * penalty_product * module_params->material.elastic_modulus / unit_scaling / unit_scaling; + // Element_Modulus_Derivative = simparam->Elastic_Modulus/unit_scaling/unit_scaling; + Poisson_Ratio = module_params->material.poisson_ratio; } /* ---------------------------------------------------------------------- Construct the local stiffness matrix ------------------------------------------------------------------------- */ -void FEA_Module_Dynamic_Elasticity::local_matrix_multiply(int ielem, CArrayKokkos &Local_Matrix){ - //local variable for host view in the dual view - const_host_vec_array all_initial_node_coords = all_initial_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_host_vec_array Element_Densities; - //local variable for host view of densities from the dual view - - const_host_vec_array all_node_densities; - if(nodal_density_flag){ - if(simparam->optimization_options.density_filter == DENSITY_FILTER::helmholtz_filter) - all_node_densities = all_filtered_node_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); - else - all_node_densities = all_node_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); - } - else{ - Element_Densities = Global_Element_Densities->getLocalView(Tpetra::Access::ReadOnly); - } - int num_dim = simparam->num_dims; - int nodes_per_elem = elem->num_basis(); - int num_gauss_points = simparam->num_gauss_points; - int z_quad,y_quad,x_quad, direct_product_count; - size_t local_node_id; - - direct_product_count = std::pow(num_gauss_points,num_dim); - real_t Elastic_Constant, Shear_Term, Pressure_Term, matrix_term; - real_t matrix_subterm1, matrix_subterm2, matrix_subterm3, invJacobian, Jacobian, weight_multiply; - real_t Element_Modulus, Poisson_Ratio; - //CArrayKokkos legendre_nodes_1D(num_gauss_points); - //CArrayKokkos legendre_weights_1D(num_gauss_points); - CArray legendre_nodes_1D(num_gauss_points); - CArray legendre_weights_1D(num_gauss_points); - real_t pointer_quad_coordinate[num_dim]; - real_t pointer_quad_coordinate_weight[num_dim]; - real_t pointer_interpolated_point[num_dim]; - real_t pointer_JT_row1[num_dim]; - real_t pointer_JT_row2[num_dim]; - real_t pointer_JT_row3[num_dim]; - ViewCArray quad_coordinate(pointer_quad_coordinate,num_dim); - ViewCArray quad_coordinate_weight(pointer_quad_coordinate_weight,num_dim); - ViewCArray interpolated_point(pointer_interpolated_point,num_dim); - ViewCArray JT_row1(pointer_JT_row1,num_dim); - ViewCArray JT_row2(pointer_JT_row2,num_dim); - ViewCArray JT_row3(pointer_JT_row3,num_dim); - - real_t pointer_basis_values[elem->num_basis()]; - real_t pointer_basis_derivative_s1[elem->num_basis()]; - real_t pointer_basis_derivative_s2[elem->num_basis()]; - real_t pointer_basis_derivative_s3[elem->num_basis()]; - ViewCArray basis_values(pointer_basis_values,elem->num_basis()); - ViewCArray basis_derivative_s1(pointer_basis_derivative_s1,elem->num_basis()); - ViewCArray basis_derivative_s2(pointer_basis_derivative_s2,elem->num_basis()); - ViewCArray basis_derivative_s3(pointer_basis_derivative_s3,elem->num_basis()); - CArrayKokkos nodal_positions(elem->num_basis(),num_dim); - CArrayKokkos nodal_density(elem->num_basis()); - - size_t Brows; - if(num_dim==2) Brows = 3; - if(num_dim==3) Brows = 6; - FArrayKokkos B_matrix_contribution(Brows,num_dim*elem->num_basis()); - CArrayKokkos B_matrix(Brows,num_dim*elem->num_basis()); - FArrayKokkos CB_matrix_contribution(Brows,num_dim*elem->num_basis()); - CArrayKokkos CB_matrix(Brows,num_dim*elem->num_basis()); - CArrayKokkos C_matrix(Brows,Brows); - - //initialize weights - elements::legendre_nodes_1D(legendre_nodes_1D,num_gauss_points); - elements::legendre_weights_1D(legendre_weights_1D,num_gauss_points); - Solver::node_ordering_convention active_node_ordering_convention = Explicit_Solver_Pointer_->active_node_ordering_convention; - - real_t current_density = 1; - CArrayKokkos convert_node_order(max_nodes_per_element); - if((active_node_ordering_convention == Solver::ENSIGHT && num_dim==3)||(active_node_ordering_convention == Solver::IJK && num_dim==2)){ - convert_node_order(0) = 0; - convert_node_order(1) = 1; - convert_node_order(2) = 3; - convert_node_order(3) = 2; - if(num_dim == 3){ - convert_node_order(4) = 4; - convert_node_order(5) = 5; - convert_node_order(6) = 7; - convert_node_order(7) = 6; - } - } - else if((active_node_ordering_convention == Solver::IJK && num_dim==3)||(active_node_ordering_convention == Solver::ENSIGHT && num_dim==2)){ - convert_node_order(0) = 0; - convert_node_order(1) = 1; - convert_node_order(2) = 2; - convert_node_order(3) = 3; - if(num_dim==3){ - convert_node_order(4) = 4; - convert_node_order(5) = 5; - convert_node_order(6) = 6; - convert_node_order(7) = 7; - } - } - - //acquire set of nodes for this local element - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - local_node_id = nodes_in_elem.host(ielem, convert_node_order(node_loop)); - nodal_positions(node_loop,0) = all_initial_node_coords(local_node_id,0); - nodal_positions(node_loop,1) = all_initial_node_coords(local_node_id,1); - nodal_positions(node_loop,2) = all_initial_node_coords(local_node_id,2); - if(nodal_density_flag) nodal_density(node_loop) = all_node_densities(local_node_id,0); - /* - if(myrank==1&&nodal_positions(node_loop,2)>10000000){ - std::cout << " LOCAL MATRIX DEBUG ON TASK " << myrank << std::endl; - std::cout << node_loop+1 <<" " << local_node_id <<" "<< nodes_in_elem(ielem, node_loop) << " "<< nodal_positions(node_loop,2) << std::endl; - std::fflush(stdout); - } - */ - //std::cout << local_node_id << " " << nodes_in_elem(ielem, node_loop) << " " << nodal_positions(node_loop,0) << " " << nodal_positions(node_loop,1) << " "<< nodal_positions(node_loop,2) <& Local_Matrix) +{ + // local variable for host view in the dual view + const_host_vec_array all_initial_node_coords = all_initial_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_host_vec_array Element_Densities; + // local variable for host view of densities from the dual view - //loop over quadrature points - for(int iquad=0; iquad < direct_product_count; iquad++){ - - //set current quadrature point - if(num_dim==3) z_quad = iquad/(num_gauss_points*num_gauss_points); - y_quad = (iquad % (num_gauss_points*num_gauss_points))/num_gauss_points; - x_quad = iquad % num_gauss_points; - quad_coordinate(0) = legendre_nodes_1D(x_quad); - quad_coordinate(1) = legendre_nodes_1D(y_quad); - if(num_dim==3) - quad_coordinate(2) = legendre_nodes_1D(z_quad); - - //set current quadrature weight - quad_coordinate_weight(0) = legendre_weights_1D(x_quad); - quad_coordinate_weight(1) = legendre_weights_1D(y_quad); - if(num_dim==3) - quad_coordinate_weight(2) = legendre_weights_1D(z_quad); + const_host_vec_array all_node_densities; + if (nodal_density_flag) + { + if (simparam->optimization_options.density_filter == DENSITY_FILTER::helmholtz_filter) + { + all_node_densities = all_filtered_node_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); + } + else + { + all_node_densities = all_node_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); + } + } else - quad_coordinate_weight(2) = 1; - weight_multiply = quad_coordinate_weight(0)*quad_coordinate_weight(1)*quad_coordinate_weight(2); - - //compute shape functions at this point for the element type - elem->basis(basis_values,quad_coordinate); - - //compute density - current_density = relative_element_densities.host(ielem); - - //debug print - //std::cout << "Current Density " << current_density << std::endl; - - //look up element material properties at this point as a function of density - Element_Material_Properties((size_t) ielem,Element_Modulus,Poisson_Ratio, current_density); - Elastic_Constant = Element_Modulus/((1 + Poisson_Ratio)*(1 - 2*Poisson_Ratio)); - Shear_Term = 0.5-Poisson_Ratio; - Pressure_Term = 1 - Poisson_Ratio; - - //debug print - //std::cout << "Element Material Params " << Elastic_Constant << std::endl; - - //compute Elastic (C) matrix - if(num_dim==2){ - C_matrix(0,0) = Pressure_Term; - C_matrix(1,1) = Pressure_Term; - C_matrix(0,1) = Poisson_Ratio; - C_matrix(1,0) = Poisson_Ratio; - C_matrix(2,2) = Shear_Term; + { + Element_Densities = Global_Element_Densities->getLocalView(Tpetra::Access::ReadOnly); } - if(num_dim==3){ - C_matrix(0,0) = Pressure_Term; - C_matrix(1,1) = Pressure_Term; - C_matrix(2,2) = Pressure_Term; - C_matrix(0,1) = Poisson_Ratio; - C_matrix(0,2) = Poisson_Ratio; - C_matrix(1,0) = Poisson_Ratio; - C_matrix(1,2) = Poisson_Ratio; - C_matrix(2,0) = Poisson_Ratio; - C_matrix(2,1) = Poisson_Ratio; - C_matrix(3,3) = Shear_Term; - C_matrix(4,4) = Shear_Term; - C_matrix(5,5) = Shear_Term; + int num_dim = simparam->num_dims; + int nodes_per_elem = elem->num_basis(); + int num_gauss_points = simparam->num_gauss_points; + int z_quad, y_quad, x_quad, direct_product_count; + size_t local_node_id; + + direct_product_count = std::pow(num_gauss_points, num_dim); + real_t Elastic_Constant, Shear_Term, Pressure_Term, matrix_term; + real_t matrix_subterm1, matrix_subterm2, matrix_subterm3, invJacobian, Jacobian, weight_multiply; + real_t Element_Modulus, Poisson_Ratio; + // CArrayKokkos legendre_nodes_1D(num_gauss_points); + // CArrayKokkos legendre_weights_1D(num_gauss_points); + CArray legendre_nodes_1D(num_gauss_points); + CArray legendre_weights_1D(num_gauss_points); + real_t pointer_quad_coordinate[num_dim]; + real_t pointer_quad_coordinate_weight[num_dim]; + real_t pointer_interpolated_point[num_dim]; + real_t pointer_JT_row1[num_dim]; + real_t pointer_JT_row2[num_dim]; + real_t pointer_JT_row3[num_dim]; + ViewCArray quad_coordinate(pointer_quad_coordinate, num_dim); + ViewCArray quad_coordinate_weight(pointer_quad_coordinate_weight, num_dim); + ViewCArray interpolated_point(pointer_interpolated_point, num_dim); + ViewCArray JT_row1(pointer_JT_row1, num_dim); + ViewCArray JT_row2(pointer_JT_row2, num_dim); + ViewCArray JT_row3(pointer_JT_row3, num_dim); + + real_t pointer_basis_values[elem->num_basis()]; + real_t pointer_basis_derivative_s1[elem->num_basis()]; + real_t pointer_basis_derivative_s2[elem->num_basis()]; + real_t pointer_basis_derivative_s3[elem->num_basis()]; + ViewCArray basis_values(pointer_basis_values, elem->num_basis()); + ViewCArray basis_derivative_s1(pointer_basis_derivative_s1, elem->num_basis()); + ViewCArray basis_derivative_s2(pointer_basis_derivative_s2, elem->num_basis()); + ViewCArray basis_derivative_s3(pointer_basis_derivative_s3, elem->num_basis()); + CArrayKokkos nodal_positions(elem->num_basis(), num_dim); + CArrayKokkos nodal_density(elem->num_basis()); + + size_t Brows; + if (num_dim == 2) + { + Brows = 3; } - - /* - //debug print of elasticity matrix - std::cout << " ------------ELASTICITY MATRIX "<< ielem + 1 <<"--------------"<partial_xi_basis(basis_derivative_s1,quad_coordinate); - elem->partial_eta_basis(basis_derivative_s2,quad_coordinate); - elem->partial_mu_basis(basis_derivative_s3,quad_coordinate); - - //compute derivatives of x,y,z w.r.t the s,t,w isoparametric space needed by JT (Transpose of the Jacobian) - //derivative of x,y,z w.r.t s - JT_row1(0) = 0; - JT_row1(1) = 0; - JT_row1(2) = 0; - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - JT_row1(0) += nodal_positions(node_loop,0)*basis_derivative_s1(node_loop); - JT_row1(1) += nodal_positions(node_loop,1)*basis_derivative_s1(node_loop); - JT_row1(2) += nodal_positions(node_loop,2)*basis_derivative_s1(node_loop); + FArrayKokkos B_matrix_contribution(Brows, num_dim * elem->num_basis()); + CArrayKokkos B_matrix(Brows, num_dim * elem->num_basis()); + FArrayKokkos CB_matrix_contribution(Brows, num_dim * elem->num_basis()); + CArrayKokkos CB_matrix(Brows, num_dim * elem->num_basis()); + CArrayKokkos C_matrix(Brows, Brows); + + // initialize weights + elements::legendre_nodes_1D(legendre_nodes_1D, num_gauss_points); + elements::legendre_weights_1D(legendre_weights_1D, num_gauss_points); + Solver::node_ordering_convention active_node_ordering_convention = Explicit_Solver_Pointer_->active_node_ordering_convention; + + real_t current_density = 1; + CArrayKokkos convert_node_order(max_nodes_per_element); + if ((active_node_ordering_convention == Solver::ENSIGHT && num_dim == 3) || (active_node_ordering_convention == Solver::IJK && num_dim == 2)) + { + convert_node_order(0) = 0; + convert_node_order(1) = 1; + convert_node_order(2) = 3; + convert_node_order(3) = 2; + if (num_dim == 3) + { + convert_node_order(4) = 4; + convert_node_order(5) = 5; + convert_node_order(6) = 7; + convert_node_order(7) = 6; + } } - - //derivative of x,y,z w.r.t t - JT_row2(0) = 0; - JT_row2(1) = 0; - JT_row2(2) = 0; - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - JT_row2(0) += nodal_positions(node_loop,0)*basis_derivative_s2(node_loop); - JT_row2(1) += nodal_positions(node_loop,1)*basis_derivative_s2(node_loop); - JT_row2(2) += nodal_positions(node_loop,2)*basis_derivative_s2(node_loop); + else if ((active_node_ordering_convention == Solver::IJK && num_dim == 3) || (active_node_ordering_convention == Solver::ENSIGHT && num_dim == 2)) + { + convert_node_order(0) = 0; + convert_node_order(1) = 1; + convert_node_order(2) = 2; + convert_node_order(3) = 3; + if (num_dim == 3) + { + convert_node_order(4) = 4; + convert_node_order(5) = 5; + convert_node_order(6) = 6; + convert_node_order(7) = 7; + } } - //derivative of x,y,z w.r.t w - JT_row3(0) = 0; - JT_row3(1) = 0; - JT_row3(2) = 0; - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - JT_row3(0) += nodal_positions(node_loop,0)*basis_derivative_s3(node_loop); - JT_row3(1) += nodal_positions(node_loop,1)*basis_derivative_s3(node_loop); - JT_row3(2) += nodal_positions(node_loop,2)*basis_derivative_s3(node_loop); - //debug print - /*if(myrank==1&&nodal_positions(node_loop,2)*basis_derivative_s3(node_loop)<-10000000){ - std::cout << " LOCAL MATRIX DEBUG ON TASK " << myrank << std::endl; - std::cout << node_loop+1 << " " << JT_row3(2) << " "<< nodal_positions(node_loop,2) <<" "<< basis_derivative_s3(node_loop) << std::endl; - std::fflush(stdout); - }*/ - } - - - //compute the determinant of the Jacobian - Jacobian = JT_row1(0)*(JT_row2(1)*JT_row3(2)-JT_row3(1)*JT_row2(2))- - JT_row1(1)*(JT_row2(0)*JT_row3(2)-JT_row3(0)*JT_row2(2))+ - JT_row1(2)*(JT_row2(0)*JT_row3(1)-JT_row3(0)*JT_row2(1)); - if(Jacobian<0) Jacobian = -Jacobian; - invJacobian = 1/Jacobian; - //compute the contributions of this quadrature point to the B matrix - if(num_dim==2) - for(int ishape=0; ishape < nodes_per_elem; ishape++){ - B_matrix_contribution(0,ishape*num_dim) = (basis_derivative_s1(ishape)*(JT_row2(1)*JT_row3(2)-JT_row3(1)*JT_row2(2))- - basis_derivative_s2(ishape)*(JT_row1(1)*JT_row3(2)-JT_row3(1)*JT_row1(2))+ - basis_derivative_s3(ishape)*(JT_row1(1)*JT_row2(2)-JT_row2(1)*JT_row1(2))); - B_matrix_contribution(1,ishape*num_dim) = 0; - B_matrix_contribution(2,ishape*num_dim) = 0; - B_matrix_contribution(3,ishape*num_dim) = (-basis_derivative_s1(ishape)*(JT_row2(0)*JT_row3(2)-JT_row3(0)*JT_row2(2))+ - basis_derivative_s2(ishape)*(JT_row1(0)*JT_row3(2)-JT_row3(0)*JT_row1(2))- - basis_derivative_s3(ishape)*(JT_row1(0)*JT_row2(2)-JT_row2(0)*JT_row1(2))); - B_matrix_contribution(4,ishape*num_dim) = (basis_derivative_s1(ishape)*(JT_row2(0)*JT_row3(1)-JT_row3(0)*JT_row2(1))- - basis_derivative_s2(ishape)*(JT_row1(0)*JT_row3(1)-JT_row3(0)*JT_row1(1))+ - basis_derivative_s3(ishape)*(JT_row1(0)*JT_row2(1)-JT_row2(0)*JT_row1(1))); - B_matrix_contribution(5,ishape*num_dim) = 0; - B_matrix_contribution(0,ishape*num_dim+1) = 0; - B_matrix_contribution(1,ishape*num_dim+1) = (-basis_derivative_s1(ishape)*(JT_row2(0)*JT_row3(2)-JT_row3(0)*JT_row2(2))+ - basis_derivative_s2(ishape)*(JT_row1(0)*JT_row3(2)-JT_row3(0)*JT_row1(2))- - basis_derivative_s3(ishape)*(JT_row1(0)*JT_row2(2)-JT_row2(0)*JT_row1(2))); - B_matrix_contribution(2,ishape*num_dim+1) = 0; - B_matrix_contribution(3,ishape*num_dim+1) = (basis_derivative_s1(ishape)*(JT_row2(1)*JT_row3(2)-JT_row3(1)*JT_row2(2))- - basis_derivative_s2(ishape)*(JT_row1(1)*JT_row3(2)-JT_row3(1)*JT_row1(2))+ - basis_derivative_s3(ishape)*(JT_row1(1)*JT_row2(2)-JT_row2(1)*JT_row1(2))); - B_matrix_contribution(4,ishape*num_dim+1) = 0; - B_matrix_contribution(5,ishape*num_dim+1) = (basis_derivative_s1(ishape)*(JT_row2(0)*JT_row3(1)-JT_row3(0)*JT_row2(1))- - basis_derivative_s2(ishape)*(JT_row1(0)*JT_row3(1)-JT_row3(0)*JT_row1(1))+ - basis_derivative_s3(ishape)*(JT_row1(0)*JT_row2(1)-JT_row2(0)*JT_row1(1))); - B_matrix_contribution(0,ishape*num_dim+2) = 0; - B_matrix_contribution(1,ishape*num_dim+2) = 0; - B_matrix_contribution(2,ishape*num_dim+2) = (basis_derivative_s1(ishape)*(JT_row2(0)*JT_row3(1)-JT_row3(0)*JT_row2(1))- - basis_derivative_s2(ishape)*(JT_row1(0)*JT_row3(1)-JT_row3(0)*JT_row1(1))+ - basis_derivative_s3(ishape)*(JT_row1(0)*JT_row2(1)-JT_row2(0)*JT_row1(1))); - B_matrix_contribution(3,ishape*num_dim+2) = 0; - B_matrix_contribution(4,ishape*num_dim+2) = (basis_derivative_s1(ishape)*(JT_row2(1)*JT_row3(2)-JT_row3(1)*JT_row2(2))- - basis_derivative_s2(ishape)*(JT_row1(1)*JT_row3(2)-JT_row3(1)*JT_row1(2))+ - basis_derivative_s3(ishape)*(JT_row1(1)*JT_row2(2)-JT_row2(1)*JT_row1(2))); - B_matrix_contribution(5,ishape*num_dim+2) = (-basis_derivative_s1(ishape)*(JT_row2(0)*JT_row3(2)-JT_row3(0)*JT_row2(2))+ - basis_derivative_s2(ishape)*(JT_row1(0)*JT_row3(2)-JT_row3(0)*JT_row1(2))- - basis_derivative_s3(ishape)*(JT_row1(0)*JT_row2(2)-JT_row2(0)*JT_row1(2))); + // acquire set of nodes for this local element + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + local_node_id = nodes_in_elem.host(ielem, convert_node_order(node_loop)); + nodal_positions(node_loop, 0) = all_initial_node_coords(local_node_id, 0); + nodal_positions(node_loop, 1) = all_initial_node_coords(local_node_id, 1); + nodal_positions(node_loop, 2) = all_initial_node_coords(local_node_id, 2); + if (nodal_density_flag) + { + nodal_density(node_loop) = all_node_densities(local_node_id, 0); + } + /* + if(myrank==1&&nodal_positions(node_loop,2)>10000000){ + std::cout << " LOCAL MATRIX DEBUG ON TASK " << myrank << std::endl; + std::cout << node_loop+1 <<" " << local_node_id <<" "<< nodes_in_elem(ielem, node_loop) << " "<< nodal_positions(node_loop,2) << std::endl; + std::fflush(stdout); + } + */ + // std::cout << local_node_id << " " << nodes_in_elem(ielem, node_loop) << " " << nodal_positions(node_loop,0) << " " << nodal_positions(node_loop,1) << " "<< nodal_positions(node_loop,2) <basis(basis_values, quad_coordinate); + + // compute density + current_density = relative_element_densities.host(ielem); + + // debug print + // std::cout << "Current Density " << current_density << std::endl; + + // look up element material properties at this point as a function of density + Element_Material_Properties((size_t) ielem, Element_Modulus, Poisson_Ratio, current_density); + Elastic_Constant = Element_Modulus / ((1 + Poisson_Ratio) * (1 - 2 * Poisson_Ratio)); + Shear_Term = 0.5 - Poisson_Ratio; + Pressure_Term = 1 - Poisson_Ratio; + + // debug print + // std::cout << "Element Material Params " << Elastic_Constant << std::endl; + + // compute Elastic (C) matrix + if (num_dim == 2) + { + C_matrix(0, 0) = Pressure_Term; + C_matrix(1, 1) = Pressure_Term; + C_matrix(0, 1) = Poisson_Ratio; + C_matrix(1, 0) = Poisson_Ratio; + C_matrix(2, 2) = Shear_Term; + } + if (num_dim == 3) + { + C_matrix(0, 0) = Pressure_Term; + C_matrix(1, 1) = Pressure_Term; + C_matrix(2, 2) = Pressure_Term; + C_matrix(0, 1) = Poisson_Ratio; + C_matrix(0, 2) = Poisson_Ratio; + C_matrix(1, 0) = Poisson_Ratio; + C_matrix(1, 2) = Poisson_Ratio; + C_matrix(2, 0) = Poisson_Ratio; + C_matrix(2, 1) = Poisson_Ratio; + C_matrix(3, 3) = Shear_Term; + C_matrix(4, 4) = Shear_Term; + C_matrix(5, 5) = Shear_Term; + } + + /* + //debug print of elasticity matrix + std::cout << " ------------ELASTICITY MATRIX "<< ielem + 1 <<"--------------"<partial_xi_basis(basis_derivative_s1, quad_coordinate); + elem->partial_eta_basis(basis_derivative_s2, quad_coordinate); + elem->partial_mu_basis(basis_derivative_s3, quad_coordinate); + + // compute derivatives of x,y,z w.r.t the s,t,w isoparametric space needed by JT (Transpose of the Jacobian) + // derivative of x,y,z w.r.t s + JT_row1(0) = 0; + JT_row1(1) = 0; + JT_row1(2) = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row1(0) += nodal_positions(node_loop, 0) * basis_derivative_s1(node_loop); + JT_row1(1) += nodal_positions(node_loop, 1) * basis_derivative_s1(node_loop); + JT_row1(2) += nodal_positions(node_loop, 2) * basis_derivative_s1(node_loop); + } + + // derivative of x,y,z w.r.t t + JT_row2(0) = 0; + JT_row2(1) = 0; + JT_row2(2) = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row2(0) += nodal_positions(node_loop, 0) * basis_derivative_s2(node_loop); + JT_row2(1) += nodal_positions(node_loop, 1) * basis_derivative_s2(node_loop); + JT_row2(2) += nodal_positions(node_loop, 2) * basis_derivative_s2(node_loop); + } + + // derivative of x,y,z w.r.t w + JT_row3(0) = 0; + JT_row3(1) = 0; + JT_row3(2) = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row3(0) += nodal_positions(node_loop, 0) * basis_derivative_s3(node_loop); + JT_row3(1) += nodal_positions(node_loop, 1) * basis_derivative_s3(node_loop); + JT_row3(2) += nodal_positions(node_loop, 2) * basis_derivative_s3(node_loop); + // debug print + /*if(myrank==1&&nodal_positions(node_loop,2)*basis_derivative_s3(node_loop)<-10000000){ + std::cout << " LOCAL MATRIX DEBUG ON TASK " << myrank << std::endl; + std::cout << node_loop+1 << " " << JT_row3(2) << " "<< nodal_positions(node_loop,2) <<" "<< basis_derivative_s3(node_loop) << std::endl; + std::fflush(stdout); + }*/ + } + + // compute the determinant of the Jacobian + Jacobian = JT_row1(0) * (JT_row2(1) * JT_row3(2) - JT_row3(1) * JT_row2(2)) - + JT_row1(1) * (JT_row2(0) * JT_row3(2) - JT_row3(0) * JT_row2(2)) + + JT_row1(2) * (JT_row2(0) * JT_row3(1) - JT_row3(0) * JT_row2(1)); + if (Jacobian < 0) + { + Jacobian = -Jacobian; + } + invJacobian = 1 / Jacobian; + // compute the contributions of this quadrature point to the B matrix + if (num_dim == 2) + { + for (int ishape = 0; ishape < nodes_per_elem; ishape++) + { + B_matrix_contribution(0, ishape * num_dim) = (basis_derivative_s1(ishape) * (JT_row2(1) * JT_row3(2) - JT_row3(1) * JT_row2(2)) - + basis_derivative_s2(ishape) * (JT_row1(1) * JT_row3(2) - JT_row3(1) * JT_row1(2)) + + basis_derivative_s3(ishape) * (JT_row1(1) * JT_row2(2) - JT_row2(1) * JT_row1(2))); + B_matrix_contribution(1, ishape * num_dim) = 0; + B_matrix_contribution(2, ishape * num_dim) = 0; + B_matrix_contribution(3, ishape * num_dim) = (-basis_derivative_s1(ishape) * (JT_row2(0) * JT_row3(2) - JT_row3(0) * JT_row2(2)) + + basis_derivative_s2(ishape) * (JT_row1(0) * JT_row3(2) - JT_row3(0) * JT_row1(2)) - + basis_derivative_s3(ishape) * (JT_row1(0) * JT_row2(2) - JT_row2(0) * JT_row1(2))); + B_matrix_contribution(4, ishape * num_dim) = (basis_derivative_s1(ishape) * (JT_row2(0) * JT_row3(1) - JT_row3(0) * JT_row2(1)) - + basis_derivative_s2(ishape) * (JT_row1(0) * JT_row3(1) - JT_row3(0) * JT_row1(1)) + + basis_derivative_s3(ishape) * (JT_row1(0) * JT_row2(1) - JT_row2(0) * JT_row1(1))); + B_matrix_contribution(5, ishape * num_dim) = 0; + B_matrix_contribution(0, ishape * num_dim + 1) = 0; + B_matrix_contribution(1, ishape * num_dim + 1) = (-basis_derivative_s1(ishape) * (JT_row2(0) * JT_row3(2) - JT_row3(0) * JT_row2(2)) + + basis_derivative_s2(ishape) * (JT_row1(0) * JT_row3(2) - JT_row3(0) * JT_row1(2)) - + basis_derivative_s3(ishape) * (JT_row1(0) * JT_row2(2) - JT_row2(0) * JT_row1(2))); + B_matrix_contribution(2, ishape * num_dim + 1) = 0; + B_matrix_contribution(3, ishape * num_dim + 1) = (basis_derivative_s1(ishape) * (JT_row2(1) * JT_row3(2) - JT_row3(1) * JT_row2(2)) - + basis_derivative_s2(ishape) * (JT_row1(1) * JT_row3(2) - JT_row3(1) * JT_row1(2)) + + basis_derivative_s3(ishape) * (JT_row1(1) * JT_row2(2) - JT_row2(1) * JT_row1(2))); + B_matrix_contribution(4, ishape * num_dim + 1) = 0; + B_matrix_contribution(5, ishape * num_dim + 1) = (basis_derivative_s1(ishape) * (JT_row2(0) * JT_row3(1) - JT_row3(0) * JT_row2(1)) - + basis_derivative_s2(ishape) * (JT_row1(0) * JT_row3(1) - JT_row3(0) * JT_row1(1)) + + basis_derivative_s3(ishape) * (JT_row1(0) * JT_row2(1) - JT_row2(0) * JT_row1(1))); + B_matrix_contribution(0, ishape * num_dim + 2) = 0; + B_matrix_contribution(1, ishape * num_dim + 2) = 0; + B_matrix_contribution(2, ishape * num_dim + 2) = (basis_derivative_s1(ishape) * (JT_row2(0) * JT_row3(1) - JT_row3(0) * JT_row2(1)) - + basis_derivative_s2(ishape) * (JT_row1(0) * JT_row3(1) - JT_row3(0) * JT_row1(1)) + + basis_derivative_s3(ishape) * (JT_row1(0) * JT_row2(1) - JT_row2(0) * JT_row1(1))); + B_matrix_contribution(3, ishape * num_dim + 2) = 0; + B_matrix_contribution(4, ishape * num_dim + 2) = (basis_derivative_s1(ishape) * (JT_row2(1) * JT_row3(2) - JT_row3(1) * JT_row2(2)) - + basis_derivative_s2(ishape) * (JT_row1(1) * JT_row3(2) - JT_row3(1) * JT_row1(2)) + + basis_derivative_s3(ishape) * (JT_row1(1) * JT_row2(2) - JT_row2(1) * JT_row1(2))); + B_matrix_contribution(5, ishape * num_dim + 2) = (-basis_derivative_s1(ishape) * (JT_row2(0) * JT_row3(2) - JT_row3(0) * JT_row2(2)) + + basis_derivative_s2(ishape) * (JT_row1(0) * JT_row3(2) - JT_row3(0) * JT_row1(2)) - + basis_derivative_s3(ishape) * (JT_row1(0) * JT_row2(2) - JT_row2(0) * JT_row1(2))); + } + } + if (num_dim == 3) + { + for (int ishape = 0; ishape < nodes_per_elem; ishape++) + { + B_matrix_contribution(0, ishape * num_dim) = (basis_derivative_s1(ishape) * (JT_row2(1) * JT_row3(2) - JT_row3(1) * JT_row2(2)) - + basis_derivative_s2(ishape) * (JT_row1(1) * JT_row3(2) - JT_row3(1) * JT_row1(2)) + + basis_derivative_s3(ishape) * (JT_row1(1) * JT_row2(2) - JT_row2(1) * JT_row1(2))); + B_matrix_contribution(1, ishape * num_dim) = 0; + B_matrix_contribution(2, ishape * num_dim) = 0; + B_matrix_contribution(3, ishape * num_dim) = (-basis_derivative_s1(ishape) * (JT_row2(0) * JT_row3(2) - JT_row3(0) * JT_row2(2)) + + basis_derivative_s2(ishape) * (JT_row1(0) * JT_row3(2) - JT_row3(0) * JT_row1(2)) - + basis_derivative_s3(ishape) * (JT_row1(0) * JT_row2(2) - JT_row2(0) * JT_row1(2))); + B_matrix_contribution(4, ishape * num_dim) = (basis_derivative_s1(ishape) * (JT_row2(0) * JT_row3(1) - JT_row3(0) * JT_row2(1)) - + basis_derivative_s2(ishape) * (JT_row1(0) * JT_row3(1) - JT_row3(0) * JT_row1(1)) + + basis_derivative_s3(ishape) * (JT_row1(0) * JT_row2(1) - JT_row2(0) * JT_row1(1))); + B_matrix_contribution(5, ishape * num_dim) = 0; + B_matrix_contribution(0, ishape * num_dim + 1) = 0; + B_matrix_contribution(1, ishape * num_dim + 1) = (-basis_derivative_s1(ishape) * (JT_row2(0) * JT_row3(2) - JT_row3(0) * JT_row2(2)) + + basis_derivative_s2(ishape) * (JT_row1(0) * JT_row3(2) - JT_row3(0) * JT_row1(2)) - + basis_derivative_s3(ishape) * (JT_row1(0) * JT_row2(2) - JT_row2(0) * JT_row1(2))); + B_matrix_contribution(2, ishape * num_dim + 1) = 0; + B_matrix_contribution(3, ishape * num_dim + 1) = (basis_derivative_s1(ishape) * (JT_row2(1) * JT_row3(2) - JT_row3(1) * JT_row2(2)) - + basis_derivative_s2(ishape) * (JT_row1(1) * JT_row3(2) - JT_row3(1) * JT_row1(2)) + + basis_derivative_s3(ishape) * (JT_row1(1) * JT_row2(2) - JT_row2(1) * JT_row1(2))); + B_matrix_contribution(4, ishape * num_dim + 1) = 0; + B_matrix_contribution(5, ishape * num_dim + 1) = (basis_derivative_s1(ishape) * (JT_row2(0) * JT_row3(1) - JT_row3(0) * JT_row2(1)) - + basis_derivative_s2(ishape) * (JT_row1(0) * JT_row3(1) - JT_row3(0) * JT_row1(1)) + + basis_derivative_s3(ishape) * (JT_row1(0) * JT_row2(1) - JT_row2(0) * JT_row1(1))); + B_matrix_contribution(0, ishape * num_dim + 2) = 0; + B_matrix_contribution(1, ishape * num_dim + 2) = 0; + B_matrix_contribution(2, ishape * num_dim + 2) = (basis_derivative_s1(ishape) * (JT_row2(0) * JT_row3(1) - JT_row3(0) * JT_row2(1)) - + basis_derivative_s2(ishape) * (JT_row1(0) * JT_row3(1) - JT_row3(0) * JT_row1(1)) + + basis_derivative_s3(ishape) * (JT_row1(0) * JT_row2(1) - JT_row2(0) * JT_row1(1))); + B_matrix_contribution(3, ishape * num_dim + 2) = 0; + B_matrix_contribution(4, ishape * num_dim + 2) = (basis_derivative_s1(ishape) * (JT_row2(1) * JT_row3(2) - JT_row3(1) * JT_row2(2)) - + basis_derivative_s2(ishape) * (JT_row1(1) * JT_row3(2) - JT_row3(1) * JT_row1(2)) + + basis_derivative_s3(ishape) * (JT_row1(1) * JT_row2(2) - JT_row2(1) * JT_row1(2))); + B_matrix_contribution(5, ishape * num_dim + 2) = (-basis_derivative_s1(ishape) * (JT_row2(0) * JT_row3(2) - JT_row3(0) * JT_row2(2)) + + basis_derivative_s2(ishape) * (JT_row1(0) * JT_row3(2) - JT_row3(0) * JT_row1(2)) - + basis_derivative_s3(ishape) * (JT_row1(0) * JT_row2(2) - JT_row2(0) * JT_row1(2))); + } + } + /* + //debug print of B matrix per quadrature point + std::cout << " ------------B MATRIX QUADRATURE CONTRIBUTION"<< ielem + 1 <<"--------------"<getLocalView (Tpetra::Access::ReadOnly); - - const_host_vec_array all_initial_node_coords = all_initial_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_host_vec_array Element_Densities; - //local variable for host view of densities from the dual view - const_host_vec_array all_node_densities; - if(nodal_density_flag) - all_node_densities = all_node_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); - else - Element_Densities = Global_Element_Densities->getLocalView(Tpetra::Access::ReadOnly); - int num_dim = simparam->num_dims; - int nodes_per_elem = elem->num_basis(); - int num_gauss_points = simparam->num_gauss_points; - int z_quad,y_quad,x_quad, direct_product_count; - size_t local_node_id, local_dof_idx, local_dof_idy, local_dof_idz; - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - GO current_global_index; - - real_t global_dt; - size_t current_data_index, next_data_index; - CArrayKokkos current_element_adjoint = CArrayKokkos(num_nodes_in_elem*num_dim); - - direct_product_count = std::pow(num_gauss_points,num_dim); - real_t Element_Modulus_Gradient, Poisson_Ratio, gradient_force_density[3]; - real_t Elastic_Constant, Shear_Term, Pressure_Term; - real_t inner_product, matrix_term, Jacobian, invJacobian, weight_multiply; - //CArrayKokkos legendre_nodes_1D(num_gauss_points); - //CArrayKokkos legendre_weights_1D(num_gauss_points); - CArray legendre_nodes_1D(num_gauss_points); - CArray legendre_weights_1D(num_gauss_points); - real_t pointer_quad_coordinate[num_dim]; - real_t pointer_quad_coordinate_weight[num_dim]; - real_t pointer_interpolated_point[num_dim]; - real_t pointer_JT_row1[num_dim]; - real_t pointer_JT_row2[num_dim]; - real_t pointer_JT_row3[num_dim]; - ViewCArray quad_coordinate(pointer_quad_coordinate,num_dim); - ViewCArray quad_coordinate_weight(pointer_quad_coordinate_weight,num_dim); - ViewCArray interpolated_point(pointer_interpolated_point,num_dim); - ViewCArray JT_row1(pointer_JT_row1,num_dim); - ViewCArray JT_row2(pointer_JT_row2,num_dim); - ViewCArray JT_row3(pointer_JT_row3,num_dim); - - real_t pointer_basis_values[elem->num_basis()]; - real_t pointer_basis_derivative_s1[elem->num_basis()]; - real_t pointer_basis_derivative_s2[elem->num_basis()]; - real_t pointer_basis_derivative_s3[elem->num_basis()]; - ViewCArray basis_values(pointer_basis_values,elem->num_basis()); - ViewCArray basis_derivative_s1(pointer_basis_derivative_s1,elem->num_basis()); - ViewCArray basis_derivative_s2(pointer_basis_derivative_s2,elem->num_basis()); - ViewCArray basis_derivative_s3(pointer_basis_derivative_s3,elem->num_basis()); - CArrayKokkos nodal_positions(elem->num_basis(),num_dim); - CArrayKokkos current_nodal_displacements(elem->num_basis()*num_dim); - CArrayKokkos nodal_density(elem->num_basis()); - - size_t Brows; - if(num_dim==2) Brows = 3; - if(num_dim==3) Brows = 6; - FArrayKokkos B_matrix_contribution(Brows,num_dim*elem->num_basis()); - CArrayKokkos B_matrix(Brows,num_dim*elem->num_basis()); - FArrayKokkos CB_matrix_contribution(Brows,num_dim*elem->num_basis()); - CArrayKokkos CB_matrix(Brows,num_dim*elem->num_basis()); - CArrayKokkos C_matrix(Brows,Brows); - CArrayKokkos Local_Matrix_Contribution(num_dim*nodes_per_elem,num_dim*nodes_per_elem); - - //initialize weights - elements::legendre_nodes_1D(legendre_nodes_1D,num_gauss_points); - elements::legendre_weights_1D(legendre_weights_1D,num_gauss_points); - Solver::node_ordering_convention active_node_ordering_convention = Explicit_Solver_Pointer_->active_node_ordering_convention; - - real_t current_density = 1; - CArrayKokkos convert_node_order(max_nodes_per_element); - if((active_node_ordering_convention == Solver::ENSIGHT && num_dim==3)||(active_node_ordering_convention == Solver::IJK && num_dim==2)){ - convert_node_order(0) = 0; - convert_node_order(1) = 1; - convert_node_order(2) = 3; - convert_node_order(3) = 2; - if(num_dim == 3){ - convert_node_order(4) = 4; - convert_node_order(5) = 5; - convert_node_order(6) = 7; - convert_node_order(7) = 6; +void FEA_Module_Dynamic_Elasticity::compute_stiffness_gradients(const_host_vec_array& design_variables, host_vec_array& design_gradients) +{ + // local variable for host view in the dual view + const_host_vec_array all_node_coords = all_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + + const_host_vec_array all_initial_node_coords = all_initial_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_host_vec_array Element_Densities; + // local variable for host view of densities from the dual view + const_host_vec_array all_node_densities; + if (nodal_density_flag) + { + all_node_densities = all_node_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); } - } - else if((active_node_ordering_convention == Solver::IJK && num_dim==3)||(active_node_ordering_convention == Solver::ENSIGHT && num_dim==2)){ - convert_node_order(0) = 0; - convert_node_order(1) = 1; - convert_node_order(2) = 2; - convert_node_order(3) = 3; - if(num_dim==3){ - convert_node_order(4) = 4; - convert_node_order(5) = 5; - convert_node_order(6) = 6; - convert_node_order(7) = 7; + else + { + Element_Densities = Global_Element_Densities->getLocalView(Tpetra::Access::ReadOnly); + } + int num_dim = simparam->num_dims; + int nodes_per_elem = elem->num_basis(); + int num_gauss_points = simparam->num_gauss_points; + int z_quad, y_quad, x_quad, direct_product_count; + size_t local_node_id, local_dof_idx, local_dof_idy, local_dof_idz; + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + GO current_global_index; + + real_t global_dt; + size_t current_data_index, next_data_index; + CArrayKokkos current_element_adjoint = CArrayKokkos(num_nodes_in_elem * num_dim); + + direct_product_count = std::pow(num_gauss_points, num_dim); + real_t Element_Modulus_Gradient, Poisson_Ratio, gradient_force_density[3]; + real_t Elastic_Constant, Shear_Term, Pressure_Term; + real_t inner_product, matrix_term, Jacobian, invJacobian, weight_multiply; + // CArrayKokkos legendre_nodes_1D(num_gauss_points); + // CArrayKokkos legendre_weights_1D(num_gauss_points); + CArray legendre_nodes_1D(num_gauss_points); + CArray legendre_weights_1D(num_gauss_points); + real_t pointer_quad_coordinate[num_dim]; + real_t pointer_quad_coordinate_weight[num_dim]; + real_t pointer_interpolated_point[num_dim]; + real_t pointer_JT_row1[num_dim]; + real_t pointer_JT_row2[num_dim]; + real_t pointer_JT_row3[num_dim]; + ViewCArray quad_coordinate(pointer_quad_coordinate, num_dim); + ViewCArray quad_coordinate_weight(pointer_quad_coordinate_weight, num_dim); + ViewCArray interpolated_point(pointer_interpolated_point, num_dim); + ViewCArray JT_row1(pointer_JT_row1, num_dim); + ViewCArray JT_row2(pointer_JT_row2, num_dim); + ViewCArray JT_row3(pointer_JT_row3, num_dim); + + real_t pointer_basis_values[elem->num_basis()]; + real_t pointer_basis_derivative_s1[elem->num_basis()]; + real_t pointer_basis_derivative_s2[elem->num_basis()]; + real_t pointer_basis_derivative_s3[elem->num_basis()]; + ViewCArray basis_values(pointer_basis_values, elem->num_basis()); + ViewCArray basis_derivative_s1(pointer_basis_derivative_s1, elem->num_basis()); + ViewCArray basis_derivative_s2(pointer_basis_derivative_s2, elem->num_basis()); + ViewCArray basis_derivative_s3(pointer_basis_derivative_s3, elem->num_basis()); + CArrayKokkos nodal_positions(elem->num_basis(), num_dim); + CArrayKokkos current_nodal_displacements(elem->num_basis() * num_dim); + CArrayKokkos nodal_density(elem->num_basis()); + + size_t Brows; + if (num_dim == 2) + { + Brows = 3; + } + if (num_dim == 3) + { + Brows = 6; } - } - - //loop through each element and assign the contribution to compliance gradient for each of its local nodes - if(simparam->dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ - if(myrank==0){ - std::cout << "gradient term derivative of force" << std::endl; + FArrayKokkos B_matrix_contribution(Brows, num_dim * elem->num_basis()); + CArrayKokkos B_matrix(Brows, num_dim * elem->num_basis()); + FArrayKokkos CB_matrix_contribution(Brows, num_dim * elem->num_basis()); + CArrayKokkos CB_matrix(Brows, num_dim * elem->num_basis()); + CArrayKokkos C_matrix(Brows, Brows); + CArrayKokkos Local_Matrix_Contribution(num_dim * nodes_per_elem, num_dim * nodes_per_elem); + + // initialize weights + elements::legendre_nodes_1D(legendre_nodes_1D, num_gauss_points); + elements::legendre_weights_1D(legendre_weights_1D, num_gauss_points); + Solver::node_ordering_convention active_node_ordering_convention = Explicit_Solver_Pointer_->active_node_ordering_convention; + + real_t current_density = 1; + CArrayKokkos convert_node_order(max_nodes_per_element); + if ((active_node_ordering_convention == Solver::ENSIGHT && num_dim == 3) || (active_node_ordering_convention == Solver::IJK && num_dim == 2)) + { + convert_node_order(0) = 0; + convert_node_order(1) = 1; + convert_node_order(2) = 3; + convert_node_order(3) = 2; + if (num_dim == 3) + { + convert_node_order(4) = 4; + convert_node_order(5) = 5; + convert_node_order(6) = 7; + convert_node_order(7) = 6; } - } - - for (unsigned long cycle = 0; cycle < last_time_step+1; cycle++) { - //compute timestep from time data - global_dt = time_data[cycle+1] - time_data[cycle]; - - //print - if(simparam->dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ - if (cycle==0){ - if(myrank==0) - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } - // print time step every 10 cycles - else if (cycle%20==0){ - if(myrank==0) - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } // end if } - - //view scope + else if ((active_node_ordering_convention == Solver::IJK && num_dim == 3) || (active_node_ordering_convention == Solver::ENSIGHT && num_dim == 2)) { - //const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_host_vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_host_vec_array next_adjoint_vector = (*adjoint_vector_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - - const_host_vec_array next_coordinate_vector = (*forward_solve_coordinate_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_host_vec_array current_coordinate_vector = (*forward_solve_coordinate_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - - //interface of arrays for current implementation of force calculation - for(size_t ielem = 0; ielem < rnum_elem; ielem++){ - nodes_per_elem = elem->num_basis(); - - //initialize C matrix - for(int irow = 0; irow < Brows; irow++) - for(int icol = 0; icol < Brows; icol++) - C_matrix(irow,icol) = 0; - - //B matrix initialization - for(int irow=0; irow < Brows; irow++) - for(int icol=0; icol < num_dim*nodes_per_elem; icol++){ - CB_matrix(irow,icol) = 0; - } - - //acquire set of nodes and nodal displacements for this local element - for(int node_loop=0; node_loop < nodes_per_elem; node_loop++){ - local_node_id = nodes_in_elem.host(ielem, convert_node_order(node_loop)); - local_dof_idx = nodes_in_elem.host(ielem, convert_node_order(node_loop))*num_dim; - local_dof_idy = local_dof_idx + 1; - local_dof_idz = local_dof_idx + 2; - - nodal_positions(node_loop,0) = all_initial_node_coords(local_node_id,0); - nodal_positions(node_loop,1) = all_initial_node_coords(local_node_id,1); - current_nodal_displacements(node_loop*num_dim) = 0.5*(next_coordinate_vector(local_node_id, 0)-all_initial_node_coords(local_node_id,0)+current_coordinate_vector(local_node_id, 0)-all_initial_node_coords(local_node_id,0)); - current_nodal_displacements(node_loop*num_dim+1) = 0.5*(next_coordinate_vector(local_node_id, 1)-all_initial_node_coords(local_node_id,1)+current_coordinate_vector(local_node_id, 1)-all_initial_node_coords(local_node_id,1)); - current_element_adjoint(node_loop*num_dim) = 0.5*(current_adjoint_vector(local_node_id,0)+next_adjoint_vector(local_node_id,0)); - current_element_adjoint(node_loop*num_dim+1) = 0.5*(current_adjoint_vector(local_node_id,1)+next_adjoint_vector(local_node_id,1)); - - if(num_dim==3){ - nodal_positions(node_loop,2) = all_initial_node_coords(local_node_id,2); - current_nodal_displacements(node_loop*num_dim+2) = 0.5*(next_coordinate_vector(local_node_id, 2)-all_initial_node_coords(local_node_id,2)+current_coordinate_vector(local_node_id, 2)-all_initial_node_coords(local_node_id,2)); - current_element_adjoint(node_loop*num_dim+2) = 0.5*(current_adjoint_vector(local_node_id,2)+next_adjoint_vector(local_node_id,2)); - } - - if(nodal_density_flag) nodal_density(node_loop) = all_node_densities(local_node_id,0); - //debug print - /* - std::cout << "node index access x "<< local_node_id << std::endl; - std::cout << "local index access x "<< local_dof_idx << " displacement x " << current_nodal_displacements(node_loop*num_dim) <dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) + { + if (myrank == 0) + { + std::cout << "gradient term derivative of force" << std::endl; } - std::cout << " }"<< std::endl; - */ - - //loop over quadrature points - for(int iquad=0; iquad < direct_product_count; iquad++){ - - //set current quadrature point - if(num_dim==3) z_quad = iquad/(num_gauss_points*num_gauss_points); - y_quad = (iquad % (num_gauss_points*num_gauss_points))/num_gauss_points; - x_quad = iquad % num_gauss_points; - quad_coordinate(0) = legendre_nodes_1D(x_quad); - quad_coordinate(1) = legendre_nodes_1D(y_quad); - if(num_dim==3) - quad_coordinate(2) = legendre_nodes_1D(z_quad); - - //set current quadrature weight - quad_coordinate_weight(0) = legendre_weights_1D(x_quad); - quad_coordinate_weight(1) = legendre_weights_1D(y_quad); - if(num_dim==3) - quad_coordinate_weight(2) = legendre_weights_1D(z_quad); - else - quad_coordinate_weight(2) = 1; - weight_multiply = quad_coordinate_weight(0)*quad_coordinate_weight(1)*quad_coordinate_weight(2); - - //compute shape functions at this point for the element type - elem->basis(basis_values,quad_coordinate); - - //compute all the necessary coordinates and derivatives at this point - //compute shape function derivatives - elem->partial_xi_basis(basis_derivative_s1,quad_coordinate); - elem->partial_eta_basis(basis_derivative_s2,quad_coordinate); - elem->partial_mu_basis(basis_derivative_s3,quad_coordinate); - - //compute derivatives of x,y,z w.r.t the s,t,w isoparametric space needed by JT (Transpose of the Jacobian) - //derivative of x,y,z w.r.t s - JT_row1(0) = 0; - JT_row1(1) = 0; - JT_row1(2) = 0; - for(int node_loop=0; node_loop < nodes_per_elem; node_loop++){ - JT_row1(0) += nodal_positions(node_loop,0)*basis_derivative_s1(node_loop); - JT_row1(1) += nodal_positions(node_loop,1)*basis_derivative_s1(node_loop); - JT_row1(2) += nodal_positions(node_loop,2)*basis_derivative_s1(node_loop); - } - - //derivative of x,y,z w.r.t t - JT_row2(0) = 0; - JT_row2(1) = 0; - JT_row2(2) = 0; - for(int node_loop=0; node_loop < nodes_per_elem; node_loop++){ - JT_row2(0) += nodal_positions(node_loop,0)*basis_derivative_s2(node_loop); - JT_row2(1) += nodal_positions(node_loop,1)*basis_derivative_s2(node_loop); - JT_row2(2) += nodal_positions(node_loop,2)*basis_derivative_s2(node_loop); - } - - //derivative of x,y,z w.r.t w - JT_row3(0) = 0; - JT_row3(1) = 0; - JT_row3(2) = 0; - for(int node_loop=0; node_loop < nodes_per_elem; node_loop++){ - JT_row3(0) += nodal_positions(node_loop,0)*basis_derivative_s3(node_loop); - JT_row3(1) += nodal_positions(node_loop,1)*basis_derivative_s3(node_loop); - JT_row3(2) += nodal_positions(node_loop,2)*basis_derivative_s3(node_loop); - } - - //compute the determinant of the Jacobian - Jacobian = JT_row1(0)*(JT_row2(1)*JT_row3(2)-JT_row3(1)*JT_row2(2))- - JT_row1(1)*(JT_row2(0)*JT_row3(2)-JT_row3(0)*JT_row2(2))+ - JT_row1(2)*(JT_row2(0)*JT_row3(1)-JT_row3(0)*JT_row2(1)); - if(Jacobian<0) Jacobian = -Jacobian; - invJacobian = 1/Jacobian; - - //compute density - current_density = relative_element_densities.host(ielem); - - //debug print - //std::cout << "Current Density " << current_density << std::endl; - - //compute the contributions of this quadrature point to the B matrix - if(num_dim==2) - for(int ishape=0; ishape < nodes_per_elem; ishape++){ - B_matrix_contribution(0,ishape*num_dim) = (basis_derivative_s1(ishape)*(JT_row2(1)*JT_row3(2)-JT_row3(1)*JT_row2(2))- - basis_derivative_s2(ishape)*(JT_row1(1)*JT_row3(2)-JT_row3(1)*JT_row1(2))+ - basis_derivative_s3(ishape)*(JT_row1(1)*JT_row2(2)-JT_row2(1)*JT_row1(2))); - B_matrix_contribution(1,ishape*num_dim) = 0; - B_matrix_contribution(2,ishape*num_dim) = 0; - B_matrix_contribution(3,ishape*num_dim) = (-basis_derivative_s1(ishape)*(JT_row2(0)*JT_row3(2)-JT_row3(0)*JT_row2(2))+ - basis_derivative_s2(ishape)*(JT_row1(0)*JT_row3(2)-JT_row3(0)*JT_row1(2))- - basis_derivative_s3(ishape)*(JT_row1(0)*JT_row2(2)-JT_row2(0)*JT_row1(2))); - B_matrix_contribution(4,ishape*num_dim) = (basis_derivative_s1(ishape)*(JT_row2(0)*JT_row3(1)-JT_row3(0)*JT_row2(1))- - basis_derivative_s2(ishape)*(JT_row1(0)*JT_row3(1)-JT_row3(0)*JT_row1(1))+ - basis_derivative_s3(ishape)*(JT_row1(0)*JT_row2(1)-JT_row2(0)*JT_row1(1))); - B_matrix_contribution(5,ishape*num_dim) = 0; - B_matrix_contribution(0,ishape*num_dim+1) = 0; - B_matrix_contribution(1,ishape*num_dim+1) = (-basis_derivative_s1(ishape)*(JT_row2(0)*JT_row3(2)-JT_row3(0)*JT_row2(2))+ - basis_derivative_s2(ishape)*(JT_row1(0)*JT_row3(2)-JT_row3(0)*JT_row1(2))- - basis_derivative_s3(ishape)*(JT_row1(0)*JT_row2(2)-JT_row2(0)*JT_row1(2))); - B_matrix_contribution(2,ishape*num_dim+1) = 0; - B_matrix_contribution(3,ishape*num_dim+1) = (basis_derivative_s1(ishape)*(JT_row2(1)*JT_row3(2)-JT_row3(1)*JT_row2(2))- - basis_derivative_s2(ishape)*(JT_row1(1)*JT_row3(2)-JT_row3(1)*JT_row1(2))+ - basis_derivative_s3(ishape)*(JT_row1(1)*JT_row2(2)-JT_row2(1)*JT_row1(2))); - B_matrix_contribution(4,ishape*num_dim+1) = 0; - B_matrix_contribution(5,ishape*num_dim+1) = (basis_derivative_s1(ishape)*(JT_row2(0)*JT_row3(1)-JT_row3(0)*JT_row2(1))- - basis_derivative_s2(ishape)*(JT_row1(0)*JT_row3(1)-JT_row3(0)*JT_row1(1))+ - basis_derivative_s3(ishape)*(JT_row1(0)*JT_row2(1)-JT_row2(0)*JT_row1(1))); - B_matrix_contribution(0,ishape*num_dim+2) = 0; - B_matrix_contribution(1,ishape*num_dim+2) = 0; - B_matrix_contribution(2,ishape*num_dim+2) = (basis_derivative_s1(ishape)*(JT_row2(0)*JT_row3(1)-JT_row3(0)*JT_row2(1))- - basis_derivative_s2(ishape)*(JT_row1(0)*JT_row3(1)-JT_row3(0)*JT_row1(1))+ - basis_derivative_s3(ishape)*(JT_row1(0)*JT_row2(1)-JT_row2(0)*JT_row1(1))); - B_matrix_contribution(3,ishape*num_dim+2) = 0; - B_matrix_contribution(4,ishape*num_dim+2) = (basis_derivative_s1(ishape)*(JT_row2(1)*JT_row3(2)-JT_row3(1)*JT_row2(2))- - basis_derivative_s2(ishape)*(JT_row1(1)*JT_row3(2)-JT_row3(1)*JT_row1(2))+ - basis_derivative_s3(ishape)*(JT_row1(1)*JT_row2(2)-JT_row2(1)*JT_row1(2))); - B_matrix_contribution(5,ishape*num_dim+2) = (-basis_derivative_s1(ishape)*(JT_row2(0)*JT_row3(2)-JT_row3(0)*JT_row2(2))+ - basis_derivative_s2(ishape)*(JT_row1(0)*JT_row3(2)-JT_row3(0)*JT_row1(2))- - basis_derivative_s3(ishape)*(JT_row1(0)*JT_row2(2)-JT_row2(0)*JT_row1(2))); - } - if(num_dim==3) - for(int ishape=0; ishape < nodes_per_elem; ishape++){ - B_matrix_contribution(0,ishape*num_dim) = (basis_derivative_s1(ishape)*(JT_row2(1)*JT_row3(2)-JT_row3(1)*JT_row2(2))- - basis_derivative_s2(ishape)*(JT_row1(1)*JT_row3(2)-JT_row3(1)*JT_row1(2))+ - basis_derivative_s3(ishape)*(JT_row1(1)*JT_row2(2)-JT_row2(1)*JT_row1(2))); - B_matrix_contribution(1,ishape*num_dim) = 0; - B_matrix_contribution(2,ishape*num_dim) = 0; - B_matrix_contribution(3,ishape*num_dim) = (-basis_derivative_s1(ishape)*(JT_row2(0)*JT_row3(2)-JT_row3(0)*JT_row2(2))+ - basis_derivative_s2(ishape)*(JT_row1(0)*JT_row3(2)-JT_row3(0)*JT_row1(2))- - basis_derivative_s3(ishape)*(JT_row1(0)*JT_row2(2)-JT_row2(0)*JT_row1(2))); - B_matrix_contribution(4,ishape*num_dim) = (basis_derivative_s1(ishape)*(JT_row2(0)*JT_row3(1)-JT_row3(0)*JT_row2(1))- - basis_derivative_s2(ishape)*(JT_row1(0)*JT_row3(1)-JT_row3(0)*JT_row1(1))+ - basis_derivative_s3(ishape)*(JT_row1(0)*JT_row2(1)-JT_row2(0)*JT_row1(1))); - B_matrix_contribution(5,ishape*num_dim) = 0; - B_matrix_contribution(0,ishape*num_dim+1) = 0; - B_matrix_contribution(1,ishape*num_dim+1) = (-basis_derivative_s1(ishape)*(JT_row2(0)*JT_row3(2)-JT_row3(0)*JT_row2(2))+ - basis_derivative_s2(ishape)*(JT_row1(0)*JT_row3(2)-JT_row3(0)*JT_row1(2))- - basis_derivative_s3(ishape)*(JT_row1(0)*JT_row2(2)-JT_row2(0)*JT_row1(2))); - B_matrix_contribution(2,ishape*num_dim+1) = 0; - B_matrix_contribution(3,ishape*num_dim+1) = (basis_derivative_s1(ishape)*(JT_row2(1)*JT_row3(2)-JT_row3(1)*JT_row2(2))- - basis_derivative_s2(ishape)*(JT_row1(1)*JT_row3(2)-JT_row3(1)*JT_row1(2))+ - basis_derivative_s3(ishape)*(JT_row1(1)*JT_row2(2)-JT_row2(1)*JT_row1(2))); - B_matrix_contribution(4,ishape*num_dim+1) = 0; - B_matrix_contribution(5,ishape*num_dim+1) = (basis_derivative_s1(ishape)*(JT_row2(0)*JT_row3(1)-JT_row3(0)*JT_row2(1))- - basis_derivative_s2(ishape)*(JT_row1(0)*JT_row3(1)-JT_row3(0)*JT_row1(1))+ - basis_derivative_s3(ishape)*(JT_row1(0)*JT_row2(1)-JT_row2(0)*JT_row1(1))); - B_matrix_contribution(0,ishape*num_dim+2) = 0; - B_matrix_contribution(1,ishape*num_dim+2) = 0; - B_matrix_contribution(2,ishape*num_dim+2) = (basis_derivative_s1(ishape)*(JT_row2(0)*JT_row3(1)-JT_row3(0)*JT_row2(1))- - basis_derivative_s2(ishape)*(JT_row1(0)*JT_row3(1)-JT_row3(0)*JT_row1(1))+ - basis_derivative_s3(ishape)*(JT_row1(0)*JT_row2(1)-JT_row2(0)*JT_row1(1))); - B_matrix_contribution(3,ishape*num_dim+2) = 0; - B_matrix_contribution(4,ishape*num_dim+2) = (basis_derivative_s1(ishape)*(JT_row2(1)*JT_row3(2)-JT_row3(1)*JT_row2(2))- - basis_derivative_s2(ishape)*(JT_row1(1)*JT_row3(2)-JT_row3(1)*JT_row1(2))+ - basis_derivative_s3(ishape)*(JT_row1(1)*JT_row2(2)-JT_row2(1)*JT_row1(2))); - B_matrix_contribution(5,ishape*num_dim+2) = (-basis_derivative_s1(ishape)*(JT_row2(0)*JT_row3(2)-JT_row3(0)*JT_row2(2))+ - basis_derivative_s2(ishape)*(JT_row1(0)*JT_row3(2)-JT_row3(0)*JT_row1(2))- - basis_derivative_s3(ishape)*(JT_row1(0)*JT_row2(2)-JT_row2(0)*JT_row1(2))); - } - - //look up element material properties at this point as a function of density - Gradient_Element_Material_Properties(ielem, Element_Modulus_Gradient, Poisson_Ratio, current_density); - Elastic_Constant = Element_Modulus_Gradient/((1 + Poisson_Ratio)*(1 - 2*Poisson_Ratio)); - Shear_Term = 0.5 - Poisson_Ratio; - Pressure_Term = 1 - Poisson_Ratio; - - //debug print - //std::cout << "Element Material Params " << Elastic_Constant << std::endl; - - //compute Elastic (C) matrix - if(num_dim==2){ - C_matrix(0,0) = Pressure_Term; - C_matrix(1,1) = Pressure_Term; - C_matrix(0,1) = Poisson_Ratio; - C_matrix(1,0) = Poisson_Ratio; - C_matrix(2,2) = Shear_Term; - } - if(num_dim==3){ - C_matrix(0,0) = Pressure_Term; - C_matrix(1,1) = Pressure_Term; - C_matrix(2,2) = Pressure_Term; - C_matrix(0,1) = Poisson_Ratio; - C_matrix(0,2) = Poisson_Ratio; - C_matrix(1,0) = Poisson_Ratio; - C_matrix(1,2) = Poisson_Ratio; - C_matrix(2,0) = Poisson_Ratio; - C_matrix(2,1) = Poisson_Ratio; - C_matrix(3,3) = Shear_Term; - C_matrix(4,4) = Shear_Term; - C_matrix(5,5) = Shear_Term; - } + } - //compute the previous multiplied by the Elastic (C) Matrix - for(int irow=0; irow < Brows; irow++){ - for(int icol=0; icol < num_dim*nodes_per_elem; icol++){ - CB_matrix_contribution(irow,icol) = 0; - for(int span=0; span < Brows; span++){ - CB_matrix_contribution(irow,icol) += C_matrix(irow,span)*B_matrix_contribution(span,icol); - } - } - } - - //compute the contributions of this quadrature point to all the local stiffness matrix elements - for(int ifill=0; ifill < num_dim*nodes_per_elem; ifill++){ - for(int jfill=ifill; jfill < num_dim*nodes_per_elem; jfill++){ - matrix_term = 0; - for(int span = 0; span < Brows; span++){ - matrix_term += B_matrix_contribution(span,ifill)*CB_matrix_contribution(span,jfill); - } - Local_Matrix_Contribution(ifill,jfill) = matrix_term; - if(ifill!=jfill) - Local_Matrix_Contribution(jfill,ifill) = Local_Matrix_Contribution(ifill,jfill); + for (unsigned long cycle = 0; cycle < last_time_step + 1; cycle++) + { + // compute timestep from time data + global_dt = time_data[cycle + 1] - time_data[cycle]; + + // print + if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) + { + if (cycle == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } } - } + // print time step every 10 cycles + else if (cycle % 20 == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } // end if + } - //compute inner product for this quadrature point contribution - inner_product = 0; - for(int ifill=0; ifill < num_dim*nodes_per_elem; ifill++){ - for(int jfill=0; jfill < num_dim*nodes_per_elem; jfill++){ - inner_product += Local_Matrix_Contribution(ifill, jfill)*current_element_adjoint(ifill)*current_nodal_displacements(jfill); - //debug - //if(Local_Matrix_Contribution(ifill, jfill)<0) Local_Matrix_Contribution(ifill, jfill) = - Local_Matrix_Contribution(ifill, jfill); - //inner_product += Local_Matrix_Contribution(ifill, jfill); + // view scope + { + // const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); + const_host_vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_host_vec_array next_adjoint_vector = (*adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + + const_host_vec_array next_coordinate_vector = (*forward_solve_coordinate_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_host_vec_array current_coordinate_vector = (*forward_solve_coordinate_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + + // interface of arrays for current implementation of force calculation + for (size_t ielem = 0; ielem < rnum_elem; ielem++) + { + nodes_per_elem = elem->num_basis(); + + // initialize C matrix + for (int irow = 0; irow < Brows; irow++) + { + for (int icol = 0; icol < Brows; icol++) + { + C_matrix(irow, icol) = 0; + } + } + + // B matrix initialization + for (int irow = 0; irow < Brows; irow++) + { + for (int icol = 0; icol < num_dim * nodes_per_elem; icol++) + { + CB_matrix(irow, icol) = 0; + } + } + + // acquire set of nodes and nodal displacements for this local element + for (int node_loop = 0; node_loop < nodes_per_elem; node_loop++) + { + local_node_id = nodes_in_elem.host(ielem, convert_node_order(node_loop)); + local_dof_idx = nodes_in_elem.host(ielem, convert_node_order(node_loop)) * num_dim; + local_dof_idy = local_dof_idx + 1; + local_dof_idz = local_dof_idx + 2; + + nodal_positions(node_loop, 0) = all_initial_node_coords(local_node_id, 0); + nodal_positions(node_loop, 1) = all_initial_node_coords(local_node_id, 1); + current_nodal_displacements(node_loop * num_dim) = 0.5 * (next_coordinate_vector(local_node_id, 0) - all_initial_node_coords(local_node_id, + 0) + current_coordinate_vector(local_node_id, 0) - all_initial_node_coords(local_node_id, 0)); + current_nodal_displacements(node_loop * num_dim + 1) = 0.5 * (next_coordinate_vector(local_node_id, 1) - all_initial_node_coords(local_node_id, + 1) + current_coordinate_vector(local_node_id, 1) - all_initial_node_coords(local_node_id, 1)); + current_element_adjoint(node_loop * num_dim) = 0.5 * (current_adjoint_vector(local_node_id, 0) + next_adjoint_vector(local_node_id, 0)); + current_element_adjoint(node_loop * num_dim + 1) = 0.5 * (current_adjoint_vector(local_node_id, 1) + next_adjoint_vector(local_node_id, 1)); + + if (num_dim == 3) + { + nodal_positions(node_loop, 2) = all_initial_node_coords(local_node_id, 2); + current_nodal_displacements(node_loop * num_dim + 2) = 0.5 * (next_coordinate_vector(local_node_id, 2) - all_initial_node_coords(local_node_id, + 2) + current_coordinate_vector(local_node_id, 2) - all_initial_node_coords(local_node_id, 2)); + current_element_adjoint(node_loop * num_dim + 2) = 0.5 * (current_adjoint_vector(local_node_id, 2) + next_adjoint_vector(local_node_id, 2)); + } + + if (nodal_density_flag) + { + nodal_density(node_loop) = all_node_densities(local_node_id, 0); + } + // debug print + /* + std::cout << "node index access x "<< local_node_id << std::endl; + std::cout << "local index access x "<< local_dof_idx << " displacement x " << current_nodal_displacements(node_loop*num_dim) <basis(basis_values, quad_coordinate); + + // compute all the necessary coordinates and derivatives at this point + // compute shape function derivatives + elem->partial_xi_basis(basis_derivative_s1, quad_coordinate); + elem->partial_eta_basis(basis_derivative_s2, quad_coordinate); + elem->partial_mu_basis(basis_derivative_s3, quad_coordinate); + + // compute derivatives of x,y,z w.r.t the s,t,w isoparametric space needed by JT (Transpose of the Jacobian) + // derivative of x,y,z w.r.t s + JT_row1(0) = 0; + JT_row1(1) = 0; + JT_row1(2) = 0; + for (int node_loop = 0; node_loop < nodes_per_elem; node_loop++) + { + JT_row1(0) += nodal_positions(node_loop, 0) * basis_derivative_s1(node_loop); + JT_row1(1) += nodal_positions(node_loop, 1) * basis_derivative_s1(node_loop); + JT_row1(2) += nodal_positions(node_loop, 2) * basis_derivative_s1(node_loop); + } + + // derivative of x,y,z w.r.t t + JT_row2(0) = 0; + JT_row2(1) = 0; + JT_row2(2) = 0; + for (int node_loop = 0; node_loop < nodes_per_elem; node_loop++) + { + JT_row2(0) += nodal_positions(node_loop, 0) * basis_derivative_s2(node_loop); + JT_row2(1) += nodal_positions(node_loop, 1) * basis_derivative_s2(node_loop); + JT_row2(2) += nodal_positions(node_loop, 2) * basis_derivative_s2(node_loop); + } + + // derivative of x,y,z w.r.t w + JT_row3(0) = 0; + JT_row3(1) = 0; + JT_row3(2) = 0; + for (int node_loop = 0; node_loop < nodes_per_elem; node_loop++) + { + JT_row3(0) += nodal_positions(node_loop, 0) * basis_derivative_s3(node_loop); + JT_row3(1) += nodal_positions(node_loop, 1) * basis_derivative_s3(node_loop); + JT_row3(2) += nodal_positions(node_loop, 2) * basis_derivative_s3(node_loop); + } + + // compute the determinant of the Jacobian + Jacobian = JT_row1(0) * (JT_row2(1) * JT_row3(2) - JT_row3(1) * JT_row2(2)) - + JT_row1(1) * (JT_row2(0) * JT_row3(2) - JT_row3(0) * JT_row2(2)) + + JT_row1(2) * (JT_row2(0) * JT_row3(1) - JT_row3(0) * JT_row2(1)); + if (Jacobian < 0) + { + Jacobian = -Jacobian; + } + invJacobian = 1 / Jacobian; + + // compute density + current_density = relative_element_densities.host(ielem); + + // debug print + // std::cout << "Current Density " << current_density << std::endl; + + // compute the contributions of this quadrature point to the B matrix + if (num_dim == 2) + { + for (int ishape = 0; ishape < nodes_per_elem; ishape++) + { + B_matrix_contribution(0, ishape * num_dim) = (basis_derivative_s1(ishape) * (JT_row2(1) * JT_row3(2) - JT_row3(1) * JT_row2(2)) - + basis_derivative_s2(ishape) * (JT_row1(1) * JT_row3(2) - JT_row3(1) * JT_row1(2)) + + basis_derivative_s3(ishape) * (JT_row1(1) * JT_row2(2) - JT_row2(1) * JT_row1(2))); + B_matrix_contribution(1, ishape * num_dim) = 0; + B_matrix_contribution(2, ishape * num_dim) = 0; + B_matrix_contribution(3, ishape * num_dim) = (-basis_derivative_s1(ishape) * (JT_row2(0) * JT_row3(2) - JT_row3(0) * JT_row2(2)) + + basis_derivative_s2(ishape) * (JT_row1(0) * JT_row3(2) - JT_row3(0) * JT_row1(2)) - + basis_derivative_s3(ishape) * (JT_row1(0) * JT_row2(2) - JT_row2(0) * JT_row1(2))); + B_matrix_contribution(4, ishape * num_dim) = (basis_derivative_s1(ishape) * (JT_row2(0) * JT_row3(1) - JT_row3(0) * JT_row2(1)) - + basis_derivative_s2(ishape) * (JT_row1(0) * JT_row3(1) - JT_row3(0) * JT_row1(1)) + + basis_derivative_s3(ishape) * (JT_row1(0) * JT_row2(1) - JT_row2(0) * JT_row1(1))); + B_matrix_contribution(5, ishape * num_dim) = 0; + B_matrix_contribution(0, ishape * num_dim + 1) = 0; + B_matrix_contribution(1, ishape * num_dim + 1) = (-basis_derivative_s1(ishape) * (JT_row2(0) * JT_row3(2) - JT_row3(0) * JT_row2(2)) + + basis_derivative_s2(ishape) * (JT_row1(0) * JT_row3(2) - JT_row3(0) * JT_row1(2)) - + basis_derivative_s3(ishape) * (JT_row1(0) * JT_row2(2) - JT_row2(0) * JT_row1(2))); + B_matrix_contribution(2, ishape * num_dim + 1) = 0; + B_matrix_contribution(3, ishape * num_dim + 1) = (basis_derivative_s1(ishape) * (JT_row2(1) * JT_row3(2) - JT_row3(1) * JT_row2(2)) - + basis_derivative_s2(ishape) * (JT_row1(1) * JT_row3(2) - JT_row3(1) * JT_row1(2)) + + basis_derivative_s3(ishape) * (JT_row1(1) * JT_row2(2) - JT_row2(1) * JT_row1(2))); + B_matrix_contribution(4, ishape * num_dim + 1) = 0; + B_matrix_contribution(5, ishape * num_dim + 1) = (basis_derivative_s1(ishape) * (JT_row2(0) * JT_row3(1) - JT_row3(0) * JT_row2(1)) - + basis_derivative_s2(ishape) * (JT_row1(0) * JT_row3(1) - JT_row3(0) * JT_row1(1)) + + basis_derivative_s3(ishape) * (JT_row1(0) * JT_row2(1) - JT_row2(0) * JT_row1(1))); + B_matrix_contribution(0, ishape * num_dim + 2) = 0; + B_matrix_contribution(1, ishape * num_dim + 2) = 0; + B_matrix_contribution(2, ishape * num_dim + 2) = (basis_derivative_s1(ishape) * (JT_row2(0) * JT_row3(1) - JT_row3(0) * JT_row2(1)) - + basis_derivative_s2(ishape) * (JT_row1(0) * JT_row3(1) - JT_row3(0) * JT_row1(1)) + + basis_derivative_s3(ishape) * (JT_row1(0) * JT_row2(1) - JT_row2(0) * JT_row1(1))); + B_matrix_contribution(3, ishape * num_dim + 2) = 0; + B_matrix_contribution(4, ishape * num_dim + 2) = (basis_derivative_s1(ishape) * (JT_row2(1) * JT_row3(2) - JT_row3(1) * JT_row2(2)) - + basis_derivative_s2(ishape) * (JT_row1(1) * JT_row3(2) - JT_row3(1) * JT_row1(2)) + + basis_derivative_s3(ishape) * (JT_row1(1) * JT_row2(2) - JT_row2(1) * JT_row1(2))); + B_matrix_contribution(5, ishape * num_dim + 2) = (-basis_derivative_s1(ishape) * (JT_row2(0) * JT_row3(2) - JT_row3(0) * JT_row2(2)) + + basis_derivative_s2(ishape) * (JT_row1(0) * JT_row3(2) - JT_row3(0) * JT_row1(2)) - + basis_derivative_s3(ishape) * (JT_row1(0) * JT_row2(2) - JT_row2(0) * JT_row1(2))); + } + } + if (num_dim == 3) + { + for (int ishape = 0; ishape < nodes_per_elem; ishape++) + { + B_matrix_contribution(0, ishape * num_dim) = (basis_derivative_s1(ishape) * (JT_row2(1) * JT_row3(2) - JT_row3(1) * JT_row2(2)) - + basis_derivative_s2(ishape) * (JT_row1(1) * JT_row3(2) - JT_row3(1) * JT_row1(2)) + + basis_derivative_s3(ishape) * (JT_row1(1) * JT_row2(2) - JT_row2(1) * JT_row1(2))); + B_matrix_contribution(1, ishape * num_dim) = 0; + B_matrix_contribution(2, ishape * num_dim) = 0; + B_matrix_contribution(3, ishape * num_dim) = (-basis_derivative_s1(ishape) * (JT_row2(0) * JT_row3(2) - JT_row3(0) * JT_row2(2)) + + basis_derivative_s2(ishape) * (JT_row1(0) * JT_row3(2) - JT_row3(0) * JT_row1(2)) - + basis_derivative_s3(ishape) * (JT_row1(0) * JT_row2(2) - JT_row2(0) * JT_row1(2))); + B_matrix_contribution(4, ishape * num_dim) = (basis_derivative_s1(ishape) * (JT_row2(0) * JT_row3(1) - JT_row3(0) * JT_row2(1)) - + basis_derivative_s2(ishape) * (JT_row1(0) * JT_row3(1) - JT_row3(0) * JT_row1(1)) + + basis_derivative_s3(ishape) * (JT_row1(0) * JT_row2(1) - JT_row2(0) * JT_row1(1))); + B_matrix_contribution(5, ishape * num_dim) = 0; + B_matrix_contribution(0, ishape * num_dim + 1) = 0; + B_matrix_contribution(1, ishape * num_dim + 1) = (-basis_derivative_s1(ishape) * (JT_row2(0) * JT_row3(2) - JT_row3(0) * JT_row2(2)) + + basis_derivative_s2(ishape) * (JT_row1(0) * JT_row3(2) - JT_row3(0) * JT_row1(2)) - + basis_derivative_s3(ishape) * (JT_row1(0) * JT_row2(2) - JT_row2(0) * JT_row1(2))); + B_matrix_contribution(2, ishape * num_dim + 1) = 0; + B_matrix_contribution(3, ishape * num_dim + 1) = (basis_derivative_s1(ishape) * (JT_row2(1) * JT_row3(2) - JT_row3(1) * JT_row2(2)) - + basis_derivative_s2(ishape) * (JT_row1(1) * JT_row3(2) - JT_row3(1) * JT_row1(2)) + + basis_derivative_s3(ishape) * (JT_row1(1) * JT_row2(2) - JT_row2(1) * JT_row1(2))); + B_matrix_contribution(4, ishape * num_dim + 1) = 0; + B_matrix_contribution(5, ishape * num_dim + 1) = (basis_derivative_s1(ishape) * (JT_row2(0) * JT_row3(1) - JT_row3(0) * JT_row2(1)) - + basis_derivative_s2(ishape) * (JT_row1(0) * JT_row3(1) - JT_row3(0) * JT_row1(1)) + + basis_derivative_s3(ishape) * (JT_row1(0) * JT_row2(1) - JT_row2(0) * JT_row1(1))); + B_matrix_contribution(0, ishape * num_dim + 2) = 0; + B_matrix_contribution(1, ishape * num_dim + 2) = 0; + B_matrix_contribution(2, ishape * num_dim + 2) = (basis_derivative_s1(ishape) * (JT_row2(0) * JT_row3(1) - JT_row3(0) * JT_row2(1)) - + basis_derivative_s2(ishape) * (JT_row1(0) * JT_row3(1) - JT_row3(0) * JT_row1(1)) + + basis_derivative_s3(ishape) * (JT_row1(0) * JT_row2(1) - JT_row2(0) * JT_row1(1))); + B_matrix_contribution(3, ishape * num_dim + 2) = 0; + B_matrix_contribution(4, ishape * num_dim + 2) = (basis_derivative_s1(ishape) * (JT_row2(1) * JT_row3(2) - JT_row3(1) * JT_row2(2)) - + basis_derivative_s2(ishape) * (JT_row1(1) * JT_row3(2) - JT_row3(1) * JT_row1(2)) + + basis_derivative_s3(ishape) * (JT_row1(1) * JT_row2(2) - JT_row2(1) * JT_row1(2))); + B_matrix_contribution(5, ishape * num_dim + 2) = (-basis_derivative_s1(ishape) * (JT_row2(0) * JT_row3(2) - JT_row3(0) * JT_row2(2)) + + basis_derivative_s2(ishape) * (JT_row1(0) * JT_row3(2) - JT_row3(0) * JT_row1(2)) - + basis_derivative_s3(ishape) * (JT_row1(0) * JT_row2(2) - JT_row2(0) * JT_row1(2))); + } + } + + // look up element material properties at this point as a function of density + Gradient_Element_Material_Properties(ielem, Element_Modulus_Gradient, Poisson_Ratio, current_density); + Elastic_Constant = Element_Modulus_Gradient / ((1 + Poisson_Ratio) * (1 - 2 * Poisson_Ratio)); + Shear_Term = 0.5 - Poisson_Ratio; + Pressure_Term = 1 - Poisson_Ratio; + + // debug print + // std::cout << "Element Material Params " << Elastic_Constant << std::endl; + + // compute Elastic (C) matrix + if (num_dim == 2) + { + C_matrix(0, 0) = Pressure_Term; + C_matrix(1, 1) = Pressure_Term; + C_matrix(0, 1) = Poisson_Ratio; + C_matrix(1, 0) = Poisson_Ratio; + C_matrix(2, 2) = Shear_Term; + } + if (num_dim == 3) + { + C_matrix(0, 0) = Pressure_Term; + C_matrix(1, 1) = Pressure_Term; + C_matrix(2, 2) = Pressure_Term; + C_matrix(0, 1) = Poisson_Ratio; + C_matrix(0, 2) = Poisson_Ratio; + C_matrix(1, 0) = Poisson_Ratio; + C_matrix(1, 2) = Poisson_Ratio; + C_matrix(2, 0) = Poisson_Ratio; + C_matrix(2, 1) = Poisson_Ratio; + C_matrix(3, 3) = Shear_Term; + C_matrix(4, 4) = Shear_Term; + C_matrix(5, 5) = Shear_Term; + } + + // compute the previous multiplied by the Elastic (C) Matrix + for (int irow = 0; irow < Brows; irow++) + { + for (int icol = 0; icol < num_dim * nodes_per_elem; icol++) + { + CB_matrix_contribution(irow, icol) = 0; + for (int span = 0; span < Brows; span++) + { + CB_matrix_contribution(irow, icol) += C_matrix(irow, span) * B_matrix_contribution(span, icol); + } + } + } + + // compute the contributions of this quadrature point to all the local stiffness matrix elements + for (int ifill = 0; ifill < num_dim * nodes_per_elem; ifill++) + { + for (int jfill = ifill; jfill < num_dim * nodes_per_elem; jfill++) + { + matrix_term = 0; + for (int span = 0; span < Brows; span++) + { + matrix_term += B_matrix_contribution(span, ifill) * CB_matrix_contribution(span, jfill); + } + Local_Matrix_Contribution(ifill, jfill) = matrix_term; + if (ifill != jfill) + { + Local_Matrix_Contribution(jfill, ifill) = Local_Matrix_Contribution(ifill, jfill); + } + } + } + + // compute inner product for this quadrature point contribution + inner_product = 0; + for (int ifill = 0; ifill < num_dim * nodes_per_elem; ifill++) + { + for (int jfill = 0; jfill < num_dim * nodes_per_elem; jfill++) + { + inner_product += Local_Matrix_Contribution(ifill, jfill) * current_element_adjoint(ifill) * current_nodal_displacements(jfill); + // debug + // if(Local_Matrix_Contribution(ifill, jfill)<0) Local_Matrix_Contribution(ifill, jfill) = - Local_Matrix_Contribution(ifill, jfill); + // inner_product += Local_Matrix_Contribution(ifill, jfill); + } + } + + // evaluate local stiffness matrix gradient with respect to igradient + for (int igradient = 0; igradient < nodes_per_elem; igradient++) + { + if (!map->isNodeLocalElement(nodes_in_elem.host(ielem, igradient))) + { + continue; + } + local_node_id = nodes_in_elem.host(ielem, igradient); + + // debug print + // std::cout << "contribution for " << igradient + 1 << " is " << inner_product << std::endl; + design_gradients(local_node_id, 0) -= -inner_product * Elastic_Constant * weight_multiply * invJacobian * global_dt / ((double)num_nodes_in_elem); + } + } } - } - - //evaluate local stiffness matrix gradient with respect to igradient - for(int igradient=0; igradient < nodes_per_elem; igradient++){ - if(!map->isNodeLocalElement(nodes_in_elem.host(ielem, igradient))) continue; - local_node_id = nodes_in_elem.host(ielem, igradient); - - //debug print - //std::cout << "contribution for " << igradient + 1 << " is " << inner_product << std::endl; - design_gradients(local_node_id,0) -= -inner_product*Elastic_Constant*weight_multiply*invJacobian*global_dt/((double)num_nodes_in_elem); - } } - } + // end view scope } - //end view scope - } } diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/force_gradients_elastic.cpp b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/force_gradients_elastic.cpp index e22e888ae..5e8999635 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/force_gradients_elastic.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/force_gradients_elastic.cpp @@ -5,7 +5,7 @@ #include #include #include -#include +#include #include // fmin, fmax, abs note: fminl is long #include #include @@ -22,88 +22,80 @@ #include "Simulation_Parameters/Simulation_Parameters_Explicit.h" #include "FEA_Module_Dynamic_Elasticity.h" - // ----------------------------------------------------------------------------- // This function calculates the corner forces and the evolves stress (hypo) -//------------------------------------------------------------------------------ -void FEA_Module_Dynamic_Elasticity::get_force_vgradient_elastic(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &elem_mat_id, - const double rk_alpha, - const size_t cycle - ){ - +// ------------------------------------------------------------------------------ +void FEA_Module_Dynamic_Elasticity::get_force_vgradient_elastic(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_pres, + const DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + const double rk_alpha, + const size_t cycle + ) +{ const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; const size_t num_dims = simparam->num_dims; // --- calculate the forces acting on the nodes from the element --- - FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { - - + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { const size_t num_nodes_in_elem = 8; - + // total Cauchy stress double tau_array[9]; - + // corner area normals double area_normal_array[24]; - + // estimate of shock direction double shock_dir_array[3]; - + // the sums in the Riemann solver double sum_array[4]; - + // corner shock impeadance x |corner area normal dot shock_dir| double muc_array[8]; - + // Riemann velocity double vel_star_array[3]; - + // velocity gradient double vel_grad_array[9]; - + // --- Create views of arrays to aid the force calculation --- - - ViewCArrayKokkos tau(tau_array, num_dims, num_dims); - ViewCArrayKokkos area_normal(area_normal_array, num_nodes_in_elem, num_dims); - ViewCArrayKokkos shock_dir(shock_dir_array, num_dims); - ViewCArrayKokkos sum(sum_array, 4); - ViewCArrayKokkos muc(muc_array, num_nodes_in_elem); - ViewCArrayKokkos vel_star(vel_star_array, num_dims); - ViewCArrayKokkos vel_grad(vel_grad_array, num_dims, num_dims); - - + + ViewCArrayKokkos tau(tau_array, num_dims, num_dims); + ViewCArrayKokkos area_normal(area_normal_array, num_nodes_in_elem, num_dims); + ViewCArrayKokkos shock_dir(shock_dir_array, num_dims); + ViewCArrayKokkos sum(sum_array, 4); + ViewCArrayKokkos muc(muc_array, num_nodes_in_elem); + ViewCArrayKokkos vel_star(vel_star_array, num_dims); + ViewCArrayKokkos vel_grad(vel_grad_array, num_dims, num_dims); + // --- abviatations of variables --- - + // element volume double vol = elem_vol(elem_gid); - + // create a view of the stress_matrix - ViewCArrayKokkos stress(&elem_stress(rk_level, elem_gid, 0,0), 3, 3); - - + ViewCArrayKokkos stress(&elem_stress(rk_level, elem_gid, 0, 0), 3, 3); + // cut out the node_gids for this element - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 8); - - - + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 8); + // get the B matrix which are the OUTWARD corner area normals get_bmatrix(area_normal, elem_gid, node_coords, elem_node_gids, rk_level); - - + // --- Calculate the velocity gradient --- get_velgrad(vel_grad, elem_node_gids, @@ -112,76 +104,74 @@ void FEA_Module_Dynamic_Elasticity::get_force_vgradient_elastic(const DCArrayKok vol, elem_gid, rk_level); - - + // the -1 is for the inward surface area normal, - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - for (size_t dim = 0; dim < num_dims; dim++){ - area_normal(node_lid, dim) = (-1.0)*area_normal(node_lid,dim); + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + for (size_t dim = 0; dim < num_dims; dim++) + { + area_normal(node_lid, dim) = (-1.0) * area_normal(node_lid, dim); } // end for } // end for - - - + double div = elem_div(elem_gid); - - + // vel = [u,v,w] // [du/dx, du/dy, du/dz] // vel_grad = [dv/dx, dv/dy, dv/dz] // [dw/dx, dw/dy, dw/dz] double curl[3]; - curl[0] = vel_grad(2,1) - vel_grad(1,2); // dw/dy - dv/dz - curl[1] = vel_grad(0,2) - vel_grad(2,0); // du/dz - dw/dx - curl[2] = vel_grad(1,0) - vel_grad(0,1); // dv/dx - du/dy - - double mag_curl = sqrt(curl[0]*curl[0] + curl[1]*curl[1] + curl[2]*curl[2]); - - + curl[0] = vel_grad(2, 1) - vel_grad(1, 2); // dw/dy - dv/dz + curl[1] = vel_grad(0, 2) - vel_grad(2, 0); // du/dz - dw/dx + curl[2] = vel_grad(1, 0) - vel_grad(0, 1); // dv/dx - du/dy + + double mag_curl = sqrt(curl[0] * curl[0] + curl[1] * curl[1] + curl[2] * curl[2]); + // --- Calculate the Cauchy stress --- - for (size_t i = 0; i < 3; i++){ - for (size_t j = 0; j < 3; j++){ - tau(i, j) = stress(i,j); + for (size_t i = 0; i < 3; i++) + { + for (size_t j = 0; j < 3; j++) + { + tau(i, j) = stress(i, j); // artificial viscosity can be added here to tau } // end for - } //end for + } // end for // add the pressure - for (int i = 0; i < num_dims; i++){ + for (int i = 0; i < num_dims; i++) + { tau(i, i) -= elem_pres(elem_gid); } // end for - - - // ---- Multidirectional Approximate Riemann solver (MARS) ---- // find the average velocity of the elem, it is an // estimate of the Riemann velocity - + // initialize to Riemann velocity to zero - for (size_t dim = 0; dim < num_dims; dim++){ + for (size_t dim = 0; dim < num_dims; dim++) + { vel_star(dim) = 0.0; } // loop over nodes and calculate an average velocity, which is // an estimate of Riemann velocity - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { // Get node gloabl index and create view of nodal velocity int node_gid = nodes_in_elem(elem_gid, node_lid); - - ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); - - vel_star(0) += 0.125*vel(0); - vel_star(1) += 0.125*vel(1); - vel_star(2) += 0.125*vel(2); - + + ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); + + vel_star(0) += 0.125 * vel(0); + vel_star(1) += 0.125 * vel(1); + vel_star(2) += 0.125 * vel(2); } // end for loop over nodes // find shock direction and shock impedance associated with each node - + // initialize sum term in MARS to zero - for (int i = 0; i < 4; i++){ + for (int i = 0; i < 4; i++) + { sum(i) = 0.0; } @@ -189,103 +179,102 @@ void FEA_Module_Dynamic_Elasticity::get_force_vgradient_elastic(const DCArrayKok double mag_vel; // magnitude of velocity // loop over the nodes of the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { - + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { // Get global node id size_t node_gid = nodes_in_elem(elem_gid, node_lid); // Create view of nodal velocity - ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); + ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); // Get an estimate of the shock direction. - mag_vel = sqrt( (vel(0) - vel_star(0) )*(vel(0) - vel_star(0) ) - + (vel(1) - vel_star(1) )*(vel(1) - vel_star(1) ) - + (vel(2) - vel_star(2) )*(vel(2) - vel_star(2) ) ); + mag_vel = sqrt( (vel(0) - vel_star(0) ) * (vel(0) - vel_star(0) ) + + (vel(1) - vel_star(1) ) * (vel(1) - vel_star(1) ) + + (vel(2) - vel_star(2) ) * (vel(2) - vel_star(2) ) ); - - if (mag_vel > small) { - + if (mag_vel > small) + { // estimate of the shock direction, a unit normal - for (int dim = 0; dim < num_dims; dim++){ + for (int dim = 0; dim < num_dims; dim++) + { shock_dir(dim) = (vel(dim) - vel_star(dim)) / mag_vel; } } - - else { - + else + { // if there is no velocity change, then use the surface area // normal as the shock direction - mag = sqrt( area_normal(node_lid, 0)*area_normal(node_lid, 0) - + area_normal(node_lid, 1)*area_normal(node_lid, 1) - + area_normal(node_lid, 2)*area_normal(node_lid, 2) ); - + mag = sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) + + area_normal(node_lid, 1) * area_normal(node_lid, 1) + + area_normal(node_lid, 2) * area_normal(node_lid, 2) ); + // estimate of the shock direction - for (int dim = 0; dim < num_dims; dim++){ - shock_dir(dim) = area_normal(node_lid, dim)/mag; + for (int dim = 0; dim < num_dims; dim++) + { + shock_dir(dim) = area_normal(node_lid, dim) / mag; } - } // end if mag_vel - // cell divergence indicates compression or expansions size_t mat_id = elem_mat_id(elem_gid); - if (div < 0){ // element in compression + if (div < 0) // element in compression + { muc(node_lid) = elem_den(elem_gid) * - (material(mat_id).q1*elem_sspd(elem_gid) + material(mat_id).q2*mag_vel); + (material(mat_id).q1 * elem_sspd(elem_gid) + material(mat_id).q2 * mag_vel); } - else { // element in expansion + else // element in expansion + { muc(node_lid) = elem_den(elem_gid) * - (material(mat_id).q1ex*elem_sspd(elem_gid) + material(mat_id).q2ex*mag_vel); + (material(mat_id).q1ex * elem_sspd(elem_gid) + material(mat_id).q2ex * mag_vel); } // end if on divergence sign - size_t use_shock_dir = 0; double mu_term; - + // Coding to use shock direction - if (use_shock_dir == 1){ + if (use_shock_dir == 1) + { // this is denominator of the Riamann solver and the multiplier // on velocity in the numerator. It filters on the shock // direction - mu_term = muc(node_lid)* - fabs( shock_dir(0)*area_normal(node_lid,0) - + shock_dir(1)*area_normal(node_lid,1) - + shock_dir(2)*area_normal(node_lid,2) ); + mu_term = muc(node_lid) * + fabs(shock_dir(0) * area_normal(node_lid, 0) + + shock_dir(1) * area_normal(node_lid, 1) + + shock_dir(2) * area_normal(node_lid, 2) ); } - else { - // Using a full tensoral Riemann jump relation - mu_term = muc(node_lid) - * sqrt( area_normal(node_lid, 0)*area_normal(node_lid, 0) - + area_normal(node_lid, 1)*area_normal(node_lid, 1) - + area_normal(node_lid, 2)*area_normal(node_lid, 2) ); + else + { + // Using a full tensoral Riemann jump relation + mu_term = muc(node_lid) + * sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) + + area_normal(node_lid, 1) * area_normal(node_lid, 1) + + area_normal(node_lid, 2) * area_normal(node_lid, 2) ); } - - sum(0) += mu_term*vel(0); - sum(1) += mu_term*vel(1); - sum(2) += mu_term*vel(2); + + sum(0) += mu_term * vel(0); + sum(1) += mu_term * vel(1); + sum(2) += mu_term * vel(2); sum(3) += mu_term; muc(node_lid) = mu_term; // the impeadance time surface area is stored here - } // end for node_lid loop over nodes of the elem - - - // The Riemann velocity, called vel_star - if (sum(3) > fuzz) { - for (size_t i = 0; i < num_dims; i++) { - vel_star(i) = sum(i)/sum(3); + if (sum(3) > fuzz) + { + for (size_t i = 0; i < num_dims; i++) + { + vel_star(i) = sum(i) / sum(3); } } - else { - for (int i = 0; i < num_dims; i++){ + else + { + for (int i = 0; i < num_dims; i++) + { vel_star(i) = 0.0; } } // end if - - // ---- Calculate the shock detector for the Riemann-solver ---- // // The dissipation from the Riemann problem is limited by phi @@ -298,7 +287,7 @@ void FEA_Module_Dynamic_Elasticity::get_force_vgradient_elastic(const DCArrayKok // phi = 0 highest-order solution // phi = 1 first order solution // - + double phi = 0.0; // the shock detector double r_face = 1.0; // the ratio on the face double r_min = 1.0; // the min ratio for the cell @@ -307,92 +296,83 @@ void FEA_Module_Dynamic_Elasticity::get_force_vgradient_elastic(const DCArrayKok double n_coef = 1.0; // the power on the limiting coefficient // (1=nominal, and n_coeff > 1 oscillatory) - // loop over the nieghboring cells - for (size_t elem_lid = 0; elem_lid < num_elems_in_elem(elem_gid); elem_lid++){ - + for (size_t elem_lid = 0; elem_lid < num_elems_in_elem(elem_gid); elem_lid++) + { // Get global index for neighboring cell size_t neighbor_gid = elems_in_elem(elem_gid, elem_lid); - + // calculate the velocity divergence in neighbor double div_neighbor = elem_div(neighbor_gid); - r_face = r_coef*(div_neighbor + small)/(div + small); + r_face = r_coef * (div_neighbor + small) / (div + small); // store the smallest face ratio r_min = fmin(r_face, r_min); - } // end for elem_lid - // calculate standard shock detector phi = 1.0 - fmax(0.0, r_min); phi = pow(phi, n_coef); // Mach number shock detector - double omega = 20.0;//20.0; // weighting factor on Mach number - double third = 1.0/3.0; + double omega = 20.0; // 20.0; // weighting factor on Mach number + double third = 1.0 / 3.0; double c_length = pow(vol, third); // characteristic length - double alpha = fmin(1.0, omega * (c_length * fabs(div))/(elem_sspd(elem_gid) + fuzz) ); - + double alpha = fmin(1.0, omega * (c_length * fabs(div)) / (elem_sspd(elem_gid) + fuzz) ); + // use Mach based detector with standard shock detector // turn off dissipation in expansion - //alpha = fmax(-fabs(div0)/div0 * alpha, 0.0); // this should be if(div0<0) alpha=alpha else alpha=0 - - phi = alpha*phi; - + // alpha = fmax(-fabs(div0)/div0 * alpha, 0.0); // this should be if(div0<0) alpha=alpha else alpha=0 + + phi = alpha * phi; + // curl limiter on Q - double phi_curl = fmin(1.0, 1.0*fabs(div)/(mag_curl + fuzz)); // disable Q when vorticity is high - //phi = phi_curl*phi; + double phi_curl = fmin(1.0, 1.0 * fabs(div) / (mag_curl + fuzz)); // disable Q when vorticity is high + // phi = phi_curl*phi; phi = 0; // ---- Calculate the Riemann force on each node ---- // loop over the each node in the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { - + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { size_t corner_lid = node_lid; // Get corner gid size_t corner_gid = corners_in_elem(elem_gid, corner_lid); - + // Get node gid size_t node_gid = nodes_in_elem(elem_gid, node_lid); - - // loop over dimension - for (int dim = 0; dim < num_dims; dim++){ + // loop over dimension + for (int dim = 0; dim < num_dims; dim++) + { corner_vector_storage(corner_gid, dim) = -0; - //corner_vector_storage(corner_gid, dim) = 0; - + // corner_vector_storage(corner_gid, dim) = 0; } // end loop over dimension - } // end for loop over nodes in elem - - - + // --- Update Stress --- // calculate the new stress at the next rk level, if it is a hypo model - - size_t mat_id = elem_mat_id(elem_gid); - - + size_t mat_id = elem_mat_id(elem_gid); }); // end parallel for loop over elements - //accumulate node values from corner storage - //force_gradient_velocity->putScalar(0); - //force_gradient_position->putScalar(0); - //set back to zero - FOR_ALL_CLASS(idof, 0, nlocal_nodes*num_dims, { - for(int jdof = 0; jdof < Gradient_Matrix_Strides(idof); jdof++){ - Force_Gradient_Velocities(idof,jdof) = 0; + // accumulate node values from corner storage + // force_gradient_velocity->putScalar(0); + // force_gradient_position->putScalar(0); + // set back to zero + FOR_ALL_CLASS(idof, 0, nlocal_nodes * num_dims, { + for (int jdof = 0; jdof < Gradient_Matrix_Strides(idof); jdof++) + { + Force_Gradient_Velocities(idof, jdof) = 0; } }); // end parallel for Kokkos::fence(); - - //vec_array force_gradient_velocity_view = force_gradient_velocity->getLocalView (Tpetra::Access::ReadWrite); + + // vec_array force_gradient_velocity_view = force_gradient_velocity->getLocalView (Tpetra::Access::ReadWrite); /* FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { size_t corner_id; @@ -405,101 +385,92 @@ void FEA_Module_Dynamic_Elasticity::get_force_vgradient_elastic(const DCArrayKok }); // end parallel for Kokkos::fence(); */ - + const real_t damping_constant = module_params->damping_constant; FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { - - Force_Gradient_Velocities(node_id*num_dims,0) = -damping_constant; - Force_Gradient_Velocities(node_id*num_dims+1,0) = -damping_constant; - Force_Gradient_Velocities(node_id*num_dims+2,0) = -damping_constant; + Force_Gradient_Velocities(node_id * num_dims, 0) = -damping_constant; + Force_Gradient_Velocities(node_id * num_dims + 1, 0) = -damping_constant; + Force_Gradient_Velocities(node_id * num_dims + 2, 0) = -damping_constant; }); // end parallel for Kokkos::fence(); - + return; - } // end of routine // ----------------------------------------------------------------------------- // This function calculates the corner forces and the evolves stress (hypo) -//------------------------------------------------------------------------------ -void FEA_Module_Dynamic_Elasticity::get_force_ugradient_elastic(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &elem_mat_id, - const double rk_alpha, - const size_t cycle - ){ - +// ------------------------------------------------------------------------------ +void FEA_Module_Dynamic_Elasticity::get_force_ugradient_elastic(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_pres, + const DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + const double rk_alpha, + const size_t cycle + ) +{ const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; const size_t num_dims = simparam->num_dims; // --- calculate the forces acting on the nodes from the element --- - FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { - - + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { const size_t num_nodes_in_elem = 8; - + // total Cauchy stress double tau_array[9]; - + // corner area normals double area_normal_array[24]; - + // estimate of shock direction double shock_dir_array[3]; - + // the sums in the Riemann solver double sum_array[4]; - + // corner shock impeadance x |corner area normal dot shock_dir| double muc_array[8]; - + // Riemann velocity double vel_star_array[3]; - + // velocity gradient double vel_grad_array[9]; - + // --- Create views of arrays to aid the force calculation --- - - ViewCArrayKokkos tau(tau_array, num_dims, num_dims); - ViewCArrayKokkos area_normal(area_normal_array, num_nodes_in_elem, num_dims); - ViewCArrayKokkos shock_dir(shock_dir_array, num_dims); - ViewCArrayKokkos sum(sum_array, 4); - ViewCArrayKokkos muc(muc_array, num_nodes_in_elem); - ViewCArrayKokkos vel_star(vel_star_array, num_dims); - ViewCArrayKokkos vel_grad(vel_grad_array, num_dims, num_dims); - - + + ViewCArrayKokkos tau(tau_array, num_dims, num_dims); + ViewCArrayKokkos area_normal(area_normal_array, num_nodes_in_elem, num_dims); + ViewCArrayKokkos shock_dir(shock_dir_array, num_dims); + ViewCArrayKokkos sum(sum_array, 4); + ViewCArrayKokkos muc(muc_array, num_nodes_in_elem); + ViewCArrayKokkos vel_star(vel_star_array, num_dims); + ViewCArrayKokkos vel_grad(vel_grad_array, num_dims, num_dims); + // --- abviatations of variables --- - + // element volume double vol = elem_vol(elem_gid); - + // create a view of the stress_matrix - ViewCArrayKokkos stress(&elem_stress(rk_level, elem_gid, 0,0), 3, 3); - - + ViewCArrayKokkos stress(&elem_stress(rk_level, elem_gid, 0, 0), 3, 3); + // cut out the node_gids for this element - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 8); - - - + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 8); + // get the B matrix which are the OUTWARD corner area normals get_bmatrix(area_normal, elem_gid, node_coords, elem_node_gids, rk_level); - - + // --- Calculate the velocity gradient --- get_velgrad(vel_grad, elem_node_gids, @@ -508,77 +479,74 @@ void FEA_Module_Dynamic_Elasticity::get_force_ugradient_elastic(const DCArrayKok vol, elem_gid, rk_level); - - + // the -1 is for the inward surface area normal, - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - for (size_t dim = 0; dim < num_dims; dim++){ - area_normal(node_lid, dim) = (-1.0)*area_normal(node_lid,dim); + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + for (size_t dim = 0; dim < num_dims; dim++) + { + area_normal(node_lid, dim) = (-1.0) * area_normal(node_lid, dim); } // end for } // end for - - - + double div = elem_div(elem_gid); - - + // vel = [u,v,w] // [du/dx, du/dy, du/dz] // vel_grad = [dv/dx, dv/dy, dv/dz] // [dw/dx, dw/dy, dw/dz] double curl[3]; - curl[0] = vel_grad(2,1) - vel_grad(1,2); // dw/dy - dv/dz - curl[1] = vel_grad(0,2) - vel_grad(2,0); // du/dz - dw/dx - curl[2] = vel_grad(1,0) - vel_grad(0,1); // dv/dx - du/dy - - double mag_curl = sqrt(curl[0]*curl[0] + curl[1]*curl[1] + curl[2]*curl[2]); - - + curl[0] = vel_grad(2, 1) - vel_grad(1, 2); // dw/dy - dv/dz + curl[1] = vel_grad(0, 2) - vel_grad(2, 0); // du/dz - dw/dx + curl[2] = vel_grad(1, 0) - vel_grad(0, 1); // dv/dx - du/dy + + double mag_curl = sqrt(curl[0] * curl[0] + curl[1] * curl[1] + curl[2] * curl[2]); + // --- Calculate the Cauchy stress --- - for (size_t i = 0; i < 3; i++){ - for (size_t j = 0; j < 3; j++){ - tau(i, j) = stress(i,j); + for (size_t i = 0; i < 3; i++) + { + for (size_t j = 0; j < 3; j++) + { + tau(i, j) = stress(i, j); // artificial viscosity can be added here to tau } // end for - } //end for + } // end for // add the pressure - for (int i = 0; i < num_dims; i++){ + for (int i = 0; i < num_dims; i++) + { tau(i, i) -= elem_pres(elem_gid); } // end for - - - // ---- Multidirectional Approximate Riemann solver (MARS) ---- // find the average velocity of the elem, it is an // estimate of the Riemann velocity - + // initialize to Riemann velocity to zero - for (size_t dim = 0; dim < num_dims; dim++){ + for (size_t dim = 0; dim < num_dims; dim++) + { vel_star(dim) = 0.0; } // loop over nodes and calculate an average velocity, which is // an estimate of Riemann velocity - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { // Get node gloabl index and create view of nodal velocity int node_gid = nodes_in_elem(elem_gid, node_lid); - - ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); - - vel_star(0) += 0.125*vel(0); - vel_star(1) += 0.125*vel(1); - vel_star(2) += 0.125*vel(2); - + ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); + + vel_star(0) += 0.125 * vel(0); + vel_star(1) += 0.125 * vel(1); + vel_star(2) += 0.125 * vel(2); } // end for loop over nodes // find shock direction and shock impedance associated with each node - + // initialize sum term in MARS to zero - for (int i = 0; i < 4; i++){ + for (int i = 0; i < 4; i++) + { sum(i) = 0.0; } @@ -586,103 +554,102 @@ void FEA_Module_Dynamic_Elasticity::get_force_ugradient_elastic(const DCArrayKok double mag_vel; // magnitude of velocity // loop over the nodes of the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { - + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { // Get global node id size_t node_gid = nodes_in_elem(elem_gid, node_lid); // Create view of nodal velocity - ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); + ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); // Get an estimate of the shock direction. - mag_vel = sqrt( (vel(0) - vel_star(0) )*(vel(0) - vel_star(0) ) - + (vel(1) - vel_star(1) )*(vel(1) - vel_star(1) ) - + (vel(2) - vel_star(2) )*(vel(2) - vel_star(2) ) ); + mag_vel = sqrt( (vel(0) - vel_star(0) ) * (vel(0) - vel_star(0) ) + + (vel(1) - vel_star(1) ) * (vel(1) - vel_star(1) ) + + (vel(2) - vel_star(2) ) * (vel(2) - vel_star(2) ) ); - - if (mag_vel > small) { - + if (mag_vel > small) + { // estimate of the shock direction, a unit normal - for (int dim = 0; dim < num_dims; dim++){ + for (int dim = 0; dim < num_dims; dim++) + { shock_dir(dim) = (vel(dim) - vel_star(dim)) / mag_vel; } } - - else { - + else + { // if there is no velocity change, then use the surface area // normal as the shock direction - mag = sqrt( area_normal(node_lid, 0)*area_normal(node_lid, 0) - + area_normal(node_lid, 1)*area_normal(node_lid, 1) - + area_normal(node_lid, 2)*area_normal(node_lid, 2) ); - + mag = sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) + + area_normal(node_lid, 1) * area_normal(node_lid, 1) + + area_normal(node_lid, 2) * area_normal(node_lid, 2) ); + // estimate of the shock direction - for (int dim = 0; dim < num_dims; dim++){ - shock_dir(dim) = area_normal(node_lid, dim)/mag; + for (int dim = 0; dim < num_dims; dim++) + { + shock_dir(dim) = area_normal(node_lid, dim) / mag; } - } // end if mag_vel - // cell divergence indicates compression or expansions size_t mat_id = elem_mat_id(elem_gid); - if (div < 0){ // element in compression + if (div < 0) // element in compression + { muc(node_lid) = elem_den(elem_gid) * - (material(mat_id).q1*elem_sspd(elem_gid) + material(mat_id).q2*mag_vel); + (material(mat_id).q1 * elem_sspd(elem_gid) + material(mat_id).q2 * mag_vel); } - else { // element in expansion + else // element in expansion + { muc(node_lid) = elem_den(elem_gid) * - (material(mat_id).q1ex*elem_sspd(elem_gid) + material(mat_id).q2ex*mag_vel); + (material(mat_id).q1ex * elem_sspd(elem_gid) + material(mat_id).q2ex * mag_vel); } // end if on divergence sign - size_t use_shock_dir = 0; double mu_term; - + // Coding to use shock direction - if (use_shock_dir == 1){ + if (use_shock_dir == 1) + { // this is denominator of the Riamann solver and the multiplier // on velocity in the numerator. It filters on the shock // direction - mu_term = muc(node_lid)* - fabs( shock_dir(0)*area_normal(node_lid,0) - + shock_dir(1)*area_normal(node_lid,1) - + shock_dir(2)*area_normal(node_lid,2) ); + mu_term = muc(node_lid) * + fabs(shock_dir(0) * area_normal(node_lid, 0) + + shock_dir(1) * area_normal(node_lid, 1) + + shock_dir(2) * area_normal(node_lid, 2) ); } - else { - // Using a full tensoral Riemann jump relation - mu_term = muc(node_lid) - * sqrt( area_normal(node_lid, 0)*area_normal(node_lid, 0) - + area_normal(node_lid, 1)*area_normal(node_lid, 1) - + area_normal(node_lid, 2)*area_normal(node_lid, 2) ); + else + { + // Using a full tensoral Riemann jump relation + mu_term = muc(node_lid) + * sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) + + area_normal(node_lid, 1) * area_normal(node_lid, 1) + + area_normal(node_lid, 2) * area_normal(node_lid, 2) ); } - - sum(0) += mu_term*vel(0); - sum(1) += mu_term*vel(1); - sum(2) += mu_term*vel(2); + + sum(0) += mu_term * vel(0); + sum(1) += mu_term * vel(1); + sum(2) += mu_term * vel(2); sum(3) += mu_term; muc(node_lid) = mu_term; // the impeadance time surface area is stored here - } // end for node_lid loop over nodes of the elem - - - // The Riemann velocity, called vel_star - if (sum(3) > fuzz) { - for (size_t i = 0; i < num_dims; i++) { - vel_star(i) = sum(i)/sum(3); + if (sum(3) > fuzz) + { + for (size_t i = 0; i < num_dims; i++) + { + vel_star(i) = sum(i) / sum(3); } } - else { - for (int i = 0; i < num_dims; i++){ + else + { + for (int i = 0; i < num_dims; i++) + { vel_star(i) = 0.0; } } // end if - - // ---- Calculate the shock detector for the Riemann-solver ---- // // The dissipation from the Riemann problem is limited by phi @@ -695,7 +662,7 @@ void FEA_Module_Dynamic_Elasticity::get_force_ugradient_elastic(const DCArrayKok // phi = 0 highest-order solution // phi = 1 first order solution // - + double phi = 0.0; // the shock detector double r_face = 1.0; // the ratio on the face double r_min = 1.0; // the min ratio for the cell @@ -704,299 +671,299 @@ void FEA_Module_Dynamic_Elasticity::get_force_ugradient_elastic(const DCArrayKok double n_coef = 1.0; // the power on the limiting coefficient // (1=nominal, and n_coeff > 1 oscillatory) - // loop over the nieghboring cells - for (size_t elem_lid = 0; elem_lid < num_elems_in_elem(elem_gid); elem_lid++){ - + for (size_t elem_lid = 0; elem_lid < num_elems_in_elem(elem_gid); elem_lid++) + { // Get global index for neighboring cell size_t neighbor_gid = elems_in_elem(elem_gid, elem_lid); - + // calculate the velocity divergence in neighbor double div_neighbor = elem_div(neighbor_gid); - r_face = r_coef*(div_neighbor + small)/(div + small); + r_face = r_coef * (div_neighbor + small) / (div + small); // store the smallest face ratio r_min = fmin(r_face, r_min); - } // end for elem_lid - // calculate standard shock detector phi = 1.0 - fmax(0.0, r_min); phi = pow(phi, n_coef); // Mach number shock detector - double omega = 20.0;//20.0; // weighting factor on Mach number - double third = 1.0/3.0; + double omega = 20.0; // 20.0; // weighting factor on Mach number + double third = 1.0 / 3.0; double c_length = pow(vol, third); // characteristic length - double alpha = fmin(1.0, omega * (c_length * fabs(div))/(elem_sspd(elem_gid) + fuzz) ); - + double alpha = fmin(1.0, omega * (c_length * fabs(div)) / (elem_sspd(elem_gid) + fuzz) ); + // use Mach based detector with standard shock detector // turn off dissipation in expansion - //alpha = fmax(-fabs(div0)/div0 * alpha, 0.0); // this should be if(div0<0) alpha=alpha else alpha=0 - - phi = alpha*phi; - + // alpha = fmax(-fabs(div0)/div0 * alpha, 0.0); // this should be if(div0<0) alpha=alpha else alpha=0 + + phi = alpha * phi; + // curl limiter on Q - double phi_curl = fmin(1.0, 1.0*fabs(div)/(mag_curl + fuzz)); // disable Q when vorticity is high - //phi = phi_curl*phi; + double phi_curl = fmin(1.0, 1.0 * fabs(div) / (mag_curl + fuzz)); // disable Q when vorticity is high + // phi = phi_curl*phi; phi = 0; // ---- Calculate the Riemann force on each node ---- // loop over the each node in the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { - + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { size_t corner_lid = node_lid; // Get corner gid size_t corner_gid = corners_in_elem(elem_gid, corner_lid); - + // Get node gid size_t node_gid = nodes_in_elem(elem_gid, node_lid); - - // loop over dimension - for (int dim = 0; dim < num_dims; dim++){ + // loop over dimension + for (int dim = 0; dim < num_dims; dim++) + { corner_vector_storage(corner_gid, dim) = -0.0001; - //corner_vector_storage(corner_gid, dim) = 0; - + // corner_vector_storage(corner_gid, dim) = 0; } // end loop over dimension - } // end for loop over nodes in elem - - - + // --- Update Stress --- // calculate the new stress at the next rk level, if it is a hypo model - - size_t mat_id = elem_mat_id(elem_gid); - - + size_t mat_id = elem_mat_id(elem_gid); }); // end parallel for loop over elements - //accumulate node values from corner storage - //force_gradient_position->putScalar(0); - //set back to zero - FOR_ALL_CLASS(idof, 0, nlocal_nodes*num_dims, { - for(int jdof = 0; jdof < Gradient_Matrix_Strides(idof); jdof++){ - Force_Gradient_Positions(idof,jdof) = 0; + // accumulate node values from corner storage + // force_gradient_position->putScalar(0); + // set back to zero + FOR_ALL_CLASS(idof, 0, nlocal_nodes * num_dims, { + for (int jdof = 0; jdof < Gradient_Matrix_Strides(idof); jdof++) + { + Force_Gradient_Positions(idof, jdof) = 0; } }); // end parallel for Kokkos::fence(); - - //vec_array force_gradient_position_view = force_gradient_position->getLocalView (Tpetra::Access::ReadWrite); + + // vec_array force_gradient_position_view = force_gradient_position->getLocalView (Tpetra::Access::ReadWrite); FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { size_t corner_id; - for(int icorner=0; icorner < num_corners_in_node(node_id); icorner++){ - corner_id = corners_in_node(node_id,icorner); - Force_Gradient_Positions(node_id*num_dims,0) += -0.001; - Force_Gradient_Positions(node_id*num_dims+1,0) += -0.001; - Force_Gradient_Positions(node_id*num_dims+2,0) += -0.001; + for (int icorner = 0; icorner < num_corners_in_node(node_id); icorner++) + { + corner_id = corners_in_node(node_id, icorner); + Force_Gradient_Positions(node_id * num_dims, 0) += -0.001; + Force_Gradient_Positions(node_id * num_dims + 1, 0) += -0.001; + Force_Gradient_Positions(node_id * num_dims + 2, 0) += -0.001; } }); // end parallel for Kokkos::fence(); - return; - } // end of routine // -------------------------------------------------------------------------------------- // Computes corner contribution of gradient of force with respect to the design variable -//--------------------------------------------------------------------------------------- - -void FEA_Module_Dynamic_Elasticity::force_design_gradient_term(const_vec_array design_variables, vec_array design_gradients){ - - size_t num_bdy_nodes = mesh->num_bdy_nodes; - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos material = simparam->material; - const int num_dim = simparam->num_dims; - int num_corners = rnum_elem*num_nodes_in_elem; - real_t global_dt; - bool element_constant_density = true; - size_t current_data_index, next_data_index; - CArrayKokkos current_element_adjoint = CArrayKokkos(num_nodes_in_elem,num_dim); - - //gradient contribution from gradient of Force vector with respect to design variable. - for (unsigned long cycle = 0; cycle < last_time_step+1; cycle++) { - //compute timestep from time data - global_dt = time_data[cycle+1] - time_data[cycle]; - - //print - if (cycle==0){ - if(myrank==0) - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } +// --------------------------------------------------------------------------------------- + +void FEA_Module_Dynamic_Elasticity::force_design_gradient_term(const_vec_array design_variables, vec_array design_gradients) +{ + size_t num_bdy_nodes = mesh->num_bdy_nodes; + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos material = simparam->material; + const int num_dim = simparam->num_dims; + int num_corners = rnum_elem * num_nodes_in_elem; + real_t global_dt; + bool element_constant_density = true; + size_t current_data_index, next_data_index; + CArrayKokkos current_element_adjoint = CArrayKokkos(num_nodes_in_elem, num_dim); + + // gradient contribution from gradient of Force vector with respect to design variable. + for (unsigned long cycle = 0; cycle < last_time_step + 1; cycle++) + { + // compute timestep from time data + global_dt = time_data[cycle + 1] - time_data[cycle]; + + // print + if (cycle == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } // print time step every 10 cycles - else if (cycle%20==0){ - if(myrank==0) - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } // end if - - //compute adjoint vector for this data point; use velocity midpoint - //view scope - { - //const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array next_adjoint_vector = (*adjoint_vector_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - //const_vec_array current_coord_vector = forward_solve_coordinate_data[cycle]->getLocalView (Tpetra::Access::ReadOnly); - //const_vec_array final_coordinates = forward_solve_coordinate_data[last_time_step+1]->getLocalView (Tpetra::Access::ReadOnly); - CArrayKokkos inner_products(num_nodes_in_elem); - FOR_ALL_CLASS(elem_id, 0, rnum_elem, { - size_t node_id; - size_t corner_id; - real_t inner_product; - //std::cout << elem_mass(elem_id) <getLocalView (Tpetra::Access::ReadOnly); + const_vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_adjoint_vector = (*adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + // const_vec_array current_coord_vector = forward_solve_coordinate_data[cycle]->getLocalView (Tpetra::Access::ReadOnly); + // const_vec_array final_coordinates = forward_solve_coordinate_data[last_time_step+1]->getLocalView (Tpetra::Access::ReadOnly); + CArrayKokkos inner_products(num_nodes_in_elem); + FOR_ALL_CLASS(elem_id, 0, rnum_elem, { + size_t node_id; + size_t corner_id; + real_t inner_product; + // std::cout << elem_mass(elem_id) < &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &elem_mat_id, - const double rk_alpha, - const size_t cycle - ) { - +// ------------------------------------------------------------------------------ +void FEA_Module_Dynamic_Elasticity::get_force_dgradient_elastic(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_pres, + const DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + const double rk_alpha, + const size_t cycle + ) +{ const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; const size_t num_dims = simparam->num_dims; // --- calculate the forces acting on the nodes from the element --- - FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { - + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { const size_t num_nodes_in_elem = 8; - + // total Cauchy stress double tau_array[9]; - + // corner area normals double area_normal_array[24]; - + // estimate of shock direction double shock_dir_array[3]; - + // the sums in the Riemann solver double sum_array[4]; - + // corner shock impeadance x |corner area normal dot shock_dir| double muc_array[8]; - + // Riemann velocity double vel_star_array[3]; - + // velocity gradient double vel_grad_array[9]; - + // --- Create views of arrays to aid the force calculation --- - - ViewCArrayKokkos tau(tau_array, num_dims, num_dims); - ViewCArrayKokkos area_normal(area_normal_array, num_nodes_in_elem, num_dims); - ViewCArrayKokkos shock_dir(shock_dir_array, num_dims); - ViewCArrayKokkos sum(sum_array, 4); - ViewCArrayKokkos muc(muc_array, num_nodes_in_elem); - ViewCArrayKokkos vel_star(vel_star_array, num_dims); - ViewCArrayKokkos vel_grad(vel_grad_array, num_dims, num_dims); - - + + ViewCArrayKokkos tau(tau_array, num_dims, num_dims); + ViewCArrayKokkos area_normal(area_normal_array, num_nodes_in_elem, num_dims); + ViewCArrayKokkos shock_dir(shock_dir_array, num_dims); + ViewCArrayKokkos sum(sum_array, 4); + ViewCArrayKokkos muc(muc_array, num_nodes_in_elem); + ViewCArrayKokkos vel_star(vel_star_array, num_dims); + ViewCArrayKokkos vel_grad(vel_grad_array, num_dims, num_dims); + // --- abviatations of variables --- - + // element volume double vol = elem_vol(elem_gid); - + // create a view of the stress_matrix - ViewCArrayKokkos stress(&elem_stress(rk_level, elem_gid, 0,0), 3, 3); - - + ViewCArrayKokkos stress(&elem_stress(rk_level, elem_gid, 0, 0), 3, 3); + // cut out the node_gids for this element - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 8); - - - + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 8); + // get the B matrix which are the OUTWARD corner area normals get_bmatrix(area_normal, elem_gid, node_coords, elem_node_gids, rk_level); - - + // --- Calculate the velocity gradient --- get_velgrad(vel_grad, elem_node_gids, @@ -1005,77 +972,74 @@ void FEA_Module_Dynamic_Elasticity::get_force_dgradient_elastic(const DCArrayKok vol, elem_gid, rk_level); - - + // the -1 is for the inward surface area normal, - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - for (size_t dim = 0; dim < num_dims; dim++){ - area_normal(node_lid, dim) = (-1.0)*area_normal(node_lid,dim); + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + for (size_t dim = 0; dim < num_dims; dim++) + { + area_normal(node_lid, dim) = (-1.0) * area_normal(node_lid, dim); } // end for } // end for - - - + double div = elem_div(elem_gid); - - + // vel = [u,v,w] // [du/dx, du/dy, du/dz] // vel_grad = [dv/dx, dv/dy, dv/dz] // [dw/dx, dw/dy, dw/dz] double curl[3]; - curl[0] = vel_grad(2,1) - vel_grad(1,2); // dw/dy - dv/dz - curl[1] = vel_grad(0,2) - vel_grad(2,0); // du/dz - dw/dx - curl[2] = vel_grad(1,0) - vel_grad(0,1); // dv/dx - du/dy - - double mag_curl = sqrt(curl[0]*curl[0] + curl[1]*curl[1] + curl[2]*curl[2]); - - + curl[0] = vel_grad(2, 1) - vel_grad(1, 2); // dw/dy - dv/dz + curl[1] = vel_grad(0, 2) - vel_grad(2, 0); // du/dz - dw/dx + curl[2] = vel_grad(1, 0) - vel_grad(0, 1); // dv/dx - du/dy + + double mag_curl = sqrt(curl[0] * curl[0] + curl[1] * curl[1] + curl[2] * curl[2]); + // --- Calculate the Cauchy stress --- - for (size_t i = 0; i < 3; i++){ - for (size_t j = 0; j < 3; j++){ - tau(i, j) = stress(i,j); + for (size_t i = 0; i < 3; i++) + { + for (size_t j = 0; j < 3; j++) + { + tau(i, j) = stress(i, j); // artificial viscosity can be added here to tau } // end for - } //end for + } // end for // add the pressure - for (int i = 0; i < num_dims; i++){ + for (int i = 0; i < num_dims; i++) + { tau(i, i) -= elem_pres(elem_gid); } // end for - - - // ---- Multidirectional Approximate Riemann solver (MARS) ---- // find the average velocity of the elem, it is an // estimate of the Riemann velocity - + // initialize to Riemann velocity to zero - for (size_t dim = 0; dim < num_dims; dim++){ + for (size_t dim = 0; dim < num_dims; dim++) + { vel_star(dim) = 0.0; } // loop over nodes and calculate an average velocity, which is // an estimate of Riemann velocity - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { // Get node gloabl index and create view of nodal velocity int node_gid = nodes_in_elem(elem_gid, node_lid); - - ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); - - vel_star(0) += 0.125*vel(0); - vel_star(1) += 0.125*vel(1); - vel_star(2) += 0.125*vel(2); - + ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); + + vel_star(0) += 0.125 * vel(0); + vel_star(1) += 0.125 * vel(1); + vel_star(2) += 0.125 * vel(2); } // end for loop over nodes // find shock direction and shock impedance associated with each node - + // initialize sum term in MARS to zero - for (int i = 0; i < 4; i++){ + for (int i = 0; i < 4; i++) + { sum(i) = 0.0; } @@ -1083,103 +1047,102 @@ void FEA_Module_Dynamic_Elasticity::get_force_dgradient_elastic(const DCArrayKok double mag_vel; // magnitude of velocity // loop over the nodes of the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { - + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { // Get global node id size_t node_gid = nodes_in_elem(elem_gid, node_lid); // Create view of nodal velocity - ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); + ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); // Get an estimate of the shock direction. - mag_vel = sqrt( (vel(0) - vel_star(0) )*(vel(0) - vel_star(0) ) - + (vel(1) - vel_star(1) )*(vel(1) - vel_star(1) ) - + (vel(2) - vel_star(2) )*(vel(2) - vel_star(2) ) ); + mag_vel = sqrt( (vel(0) - vel_star(0) ) * (vel(0) - vel_star(0) ) + + (vel(1) - vel_star(1) ) * (vel(1) - vel_star(1) ) + + (vel(2) - vel_star(2) ) * (vel(2) - vel_star(2) ) ); - - if (mag_vel > small) { - + if (mag_vel > small) + { // estimate of the shock direction, a unit normal - for (int dim = 0; dim < num_dims; dim++){ + for (int dim = 0; dim < num_dims; dim++) + { shock_dir(dim) = (vel(dim) - vel_star(dim)) / mag_vel; } } - - else { - + else + { // if there is no velocity change, then use the surface area // normal as the shock direction - mag = sqrt( area_normal(node_lid, 0)*area_normal(node_lid, 0) - + area_normal(node_lid, 1)*area_normal(node_lid, 1) - + area_normal(node_lid, 2)*area_normal(node_lid, 2) ); - + mag = sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) + + area_normal(node_lid, 1) * area_normal(node_lid, 1) + + area_normal(node_lid, 2) * area_normal(node_lid, 2) ); + // estimate of the shock direction - for (int dim = 0; dim < num_dims; dim++){ - shock_dir(dim) = area_normal(node_lid, dim)/mag; + for (int dim = 0; dim < num_dims; dim++) + { + shock_dir(dim) = area_normal(node_lid, dim) / mag; } - } // end if mag_vel - // cell divergence indicates compression or expansions size_t mat_id = elem_mat_id(elem_gid); - if (div < 0){ // element in compression + if (div < 0) // element in compression + { muc(node_lid) = elem_den(elem_gid) * - (material(mat_id).q1*elem_sspd(elem_gid) + material(mat_id).q2*mag_vel); + (material(mat_id).q1 * elem_sspd(elem_gid) + material(mat_id).q2 * mag_vel); } - else { // element in expansion + else // element in expansion + { muc(node_lid) = elem_den(elem_gid) * - (material(mat_id).q1ex*elem_sspd(elem_gid) + material(mat_id).q2ex*mag_vel); + (material(mat_id).q1ex * elem_sspd(elem_gid) + material(mat_id).q2ex * mag_vel); } // end if on divergence sign - size_t use_shock_dir = 0; double mu_term; - + // Coding to use shock direction - if (use_shock_dir == 1){ + if (use_shock_dir == 1) + { // this is denominator of the Riamann solver and the multiplier // on velocity in the numerator. It filters on the shock // direction - mu_term = muc(node_lid)* - fabs( shock_dir(0)*area_normal(node_lid,0) - + shock_dir(1)*area_normal(node_lid,1) - + shock_dir(2)*area_normal(node_lid,2) ); + mu_term = muc(node_lid) * + fabs(shock_dir(0) * area_normal(node_lid, 0) + + shock_dir(1) * area_normal(node_lid, 1) + + shock_dir(2) * area_normal(node_lid, 2) ); } - else { - // Using a full tensoral Riemann jump relation - mu_term = muc(node_lid) - * sqrt( area_normal(node_lid, 0)*area_normal(node_lid, 0) - + area_normal(node_lid, 1)*area_normal(node_lid, 1) - + area_normal(node_lid, 2)*area_normal(node_lid, 2) ); + else + { + // Using a full tensoral Riemann jump relation + mu_term = muc(node_lid) + * sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) + + area_normal(node_lid, 1) * area_normal(node_lid, 1) + + area_normal(node_lid, 2) * area_normal(node_lid, 2) ); } - - sum(0) += mu_term*vel(0); - sum(1) += mu_term*vel(1); - sum(2) += mu_term*vel(2); + + sum(0) += mu_term * vel(0); + sum(1) += mu_term * vel(1); + sum(2) += mu_term * vel(2); sum(3) += mu_term; muc(node_lid) = mu_term; // the impeadance time surface area is stored here - } // end for node_lid loop over nodes of the elem - - - // The Riemann velocity, called vel_star - if (sum(3) > fuzz) { - for (size_t i = 0; i < num_dims; i++) { - vel_star(i) = sum(i)/sum(3); + if (sum(3) > fuzz) + { + for (size_t i = 0; i < num_dims; i++) + { + vel_star(i) = sum(i) / sum(3); } } - else { - for (int i = 0; i < num_dims; i++){ + else + { + for (int i = 0; i < num_dims; i++) + { vel_star(i) = 0.0; } } // end if - - // ---- Calculate the shock detector for the Riemann-solver ---- // // The dissipation from the Riemann problem is limited by phi @@ -1192,7 +1155,7 @@ void FEA_Module_Dynamic_Elasticity::get_force_dgradient_elastic(const DCArrayKok // phi = 0 highest-order solution // phi = 1 first order solution // - + double phi = 0.0; // the shock detector double r_face = 1.0; // the ratio on the face double r_min = 1.0; // the min ratio for the cell @@ -1201,95 +1164,84 @@ void FEA_Module_Dynamic_Elasticity::get_force_dgradient_elastic(const DCArrayKok double n_coef = 1.0; // the power on the limiting coefficient // (1=nominal, and n_coeff > 1 oscillatory) - // loop over the nieghboring cells - for (size_t elem_lid = 0; elem_lid < num_elems_in_elem(elem_gid); elem_lid++){ - + for (size_t elem_lid = 0; elem_lid < num_elems_in_elem(elem_gid); elem_lid++) + { // Get global index for neighboring cell size_t neighbor_gid = elems_in_elem(elem_gid, elem_lid); - + // calculate the velocity divergence in neighbor double div_neighbor = elem_div(neighbor_gid); - r_face = r_coef*(div_neighbor + small)/(div + small); + r_face = r_coef * (div_neighbor + small) / (div + small); // store the smallest face ratio r_min = fmin(r_face, r_min); - } // end for elem_lid - // calculate standard shock detector phi = 1.0 - fmax(0.0, r_min); phi = pow(phi, n_coef); // Mach number shock detector - double omega = 20.0;//20.0; // weighting factor on Mach number - double third = 1.0/3.0; + double omega = 20.0; // 20.0; // weighting factor on Mach number + double third = 1.0 / 3.0; double c_length = pow(vol, third); // characteristic length - double alpha = fmin(1.0, omega * (c_length * fabs(div))/(elem_sspd(elem_gid) + fuzz) ); - + double alpha = fmin(1.0, omega * (c_length * fabs(div)) / (elem_sspd(elem_gid) + fuzz) ); + // use Mach based detector with standard shock detector // turn off dissipation in expansion - //alpha = fmax(-fabs(div0)/div0 * alpha, 0.0); // this should be if(div0<0) alpha=alpha else alpha=0 - - phi = alpha*phi; - + // alpha = fmax(-fabs(div0)/div0 * alpha, 0.0); // this should be if(div0<0) alpha=alpha else alpha=0 + + phi = alpha * phi; + // curl limiter on Q - double phi_curl = fmin(1.0, 1.0*fabs(div)/(mag_curl + fuzz)); // disable Q when vorticity is high - //phi = phi_curl*phi; + double phi_curl = fmin(1.0, 1.0 * fabs(div) / (mag_curl + fuzz)); // disable Q when vorticity is high + // phi = phi_curl*phi; phi = 0; // ---- Calculate the Riemann force on each node ---- // loop over the each node in the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { - + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { size_t corner_lid = node_lid; // Get corner gid size_t corner_gid = corners_in_elem(elem_gid, corner_lid); - + // Get node gid size_t node_gid = nodes_in_elem(elem_gid, node_lid); - - // loop over dimension - for (int dim = 0; dim < num_dims; dim++){ + // loop over dimension + for (int dim = 0; dim < num_dims; dim++) + { corner_vector_storage(corner_gid, dim) = 0.0001; - } // end loop over dimension - } // end for loop over nodes in elem - - - + // --- Update Stress --- // calculate the new stress at the next rk level, if it is a hypo model - - size_t mat_id = elem_mat_id(elem_gid); - - + size_t mat_id = elem_mat_id(elem_gid); }); // end parallel for loop over elements - //accumulate node values from corner storage + // accumulate node values from corner storage force_gradient_design->putScalar(0); - - vec_array force_gradient_design_view = force_gradient_design->getLocalView (Tpetra::Access::ReadWrite); + + vec_array force_gradient_design_view = force_gradient_design->getLocalView(Tpetra::Access::ReadWrite); FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { size_t corner_id; - for(int icorner=0; icorner < num_corners_in_node(node_id); icorner++){ - corner_id = corners_in_node(node_id,icorner); - force_gradient_design_view(node_id,0) += corner_vector_storage(corner_id, 0); - force_gradient_design_view(node_id,1) += corner_vector_storage(corner_id, 1); - force_gradient_design_view(node_id,2) += corner_vector_storage(corner_id, 2); + for (int icorner = 0; icorner < num_corners_in_node(node_id); icorner++) + { + corner_id = corners_in_node(node_id, icorner); + force_gradient_design_view(node_id, 0) += corner_vector_storage(corner_id, 0); + force_gradient_design_view(node_id, 1) += corner_vector_storage(corner_id, 1); + force_gradient_design_view(node_id, 2) += corner_vector_storage(corner_id, 2); } }); // end parallel for Kokkos::fence(); - return; - } // end of routine diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/geometry.cpp b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/geometry.cpp index 51aabc00e..7e4034a8a 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/geometry.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/geometry.cpp @@ -1,31 +1,28 @@ // ----------------------------------------------------------------------------- // This code handles the geometric information for the mesh for the SHG solver -//------------------------------------------------------------------------------ +// ------------------------------------------------------------------------------ #include "matar.h" #include "state.h" #include "FEA_Module_Dynamic_Elasticity.h" -void FEA_Module_Dynamic_Elasticity::update_position_elastic(double rk_alpha, - const size_t num_nodes, - DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel){ - +void FEA_Module_Dynamic_Elasticity::update_position_elastic(double rk_alpha, + const size_t num_nodes, + DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel) +{ const size_t rk_level = rk_num_bins - 1; - int num_dims = num_dims; + int num_dims = num_dims; // loop over all the nodes in the mesh FOR_ALL_CLASS(node_gid, 0, num_nodes, { - - for (int dim = 0; dim < num_dims; dim++){ - double half_vel = (node_vel(rk_level, node_gid, dim) + node_vel(0, node_gid, dim))*0.5; - node_coords(rk_level, node_gid, dim) = node_coords(0, node_gid, dim) + rk_alpha*dt*half_vel; + for (int dim = 0; dim < num_dims; dim++) + { + double half_vel = (node_vel(rk_level, node_gid, dim) + node_vel(0, node_gid, dim)) * 0.5; + node_coords(rk_level, node_gid, dim) = node_coords(0, node_gid, dim) + rk_alpha * dt * half_vel; } - }); // end parallel for over nodes - } // end subroutine - // ----------------------------------------------------------------------------- // This function claculates // B_p = J^{-T} \cdot (\nabla_{xi} \phi_p w @@ -36,303 +33,299 @@ void FEA_Module_Dynamic_Elasticity::update_position_elastic(double rk_alpha, // \nabla_{xi} is the gradient opperator in the reference coordinates // // B_p is the OUTWARD corner area normal at node p -//------------------------------------------------------------------------------ +// ------------------------------------------------------------------------------ KOKKOS_FUNCTION -void FEA_Module_Dynamic_Elasticity::get_bmatrix(const ViewCArrayKokkos &B_matrix, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const { - +void FEA_Module_Dynamic_Elasticity::get_bmatrix(const ViewCArrayKokkos& B_matrix, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const +{ const size_t num_nodes = 8; double x_array[8]; double y_array[8]; double z_array[8]; - + // x, y, z coordinates of elem vertices - auto x = ViewCArrayKokkos (x_array, num_nodes); - auto y = ViewCArrayKokkos (y_array, num_nodes); - auto z = ViewCArrayKokkos (z_array, num_nodes); + auto x = ViewCArrayKokkos(x_array, num_nodes); + auto y = ViewCArrayKokkos(y_array, num_nodes); + auto z = ViewCArrayKokkos(z_array, num_nodes); // get the coordinates of the nodes(rk,elem,node) in this element - for (int node_lid = 0; node_lid < num_nodes; node_lid++){ + for (int node_lid = 0; node_lid < num_nodes; node_lid++) + { x(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 0); y(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 1); z(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 2); } // end for - double twelth = 1./12.; - - B_matrix(0,0) = ( +y(1)*( -z(2) -z(3) +z(4) +z(5) ) - +y(2)*( +z(1) -z(3) ) - +y(3)*( +z(1) +z(2) -z(4) -z(7) ) - +y(4)*( -z(1) +z(3) -z(5) +z(7) ) - +y(5)*( -z(1) +z(4) ) - +y(7)*( +z(3) -z(4) ) )*twelth; - - B_matrix(1,0) = ( +y(0)*( +z(2) +z(3) -z(4) -z(5) ) - +y(2)*( -z(0) -z(3) +z(5) +z(6) ) - +y(3)*( -z(0) +z(2) ) - +y(4)*( +z(0) -z(5) ) - +y(5)*( +z(0) -z(2) +z(4) -z(6) ) - +y(6)*( -z(2) +z(5) ) )*twelth; - - B_matrix(2,0) = ( +y(0)*( -z(1) +z(3) ) - +y(1)*( +z(0) +z(3) -z(5) -z(6) ) - +y(3)*( -z(0) -z(1) +z(6) +z(7) ) - +y(5)*( +z(1) -z(6) ) - +y(6)*( +z(1) -z(3) +z(5) -z(7) ) - +y(7)*( -z(3) +z(6) ) )*twelth; - - B_matrix(3,0) = ( +y(0)*( -z(1) -z(2) +z(4) +z(7) ) - +y(1)*( +z(0) -z(2) ) - +y(2)*( +z(0) +z(1) -z(6) -z(7) ) - +y(4)*( -z(0) +z(7) ) - +y(6)*( +z(2) -z(7) ) - +y(7)*( -z(0) +z(2) -z(4) +z(6) ) )*twelth; - - B_matrix(4,0) = ( +y(0)*( +z(1) -z(3) +z(5) -z(7) ) - +y(1)*( -z(0) +z(5) ) - +y(3)*( +z(0) -z(7) ) - +y(5)*( -z(0) -z(1) +z(6) +z(7) ) - +y(6)*( -z(5) +z(7) ) - +y(7)*( +z(0) +z(3) -z(5) -z(6) ) )*twelth; - - B_matrix(5,0) = ( +y(0)*( +z(1) -z(4) ) - +y(1)*( -z(0) +z(2) -z(4) +z(6) ) - +y(2)*( -z(1) +z(6) ) - +y(4)*( +z(0) +z(1) -z(6) -z(7) ) - +y(6)*( -z(1) -z(2) +z(4) +z(7) ) - +y(7)*( +z(4) -z(6) ) )*twelth; - - B_matrix(6,0) = ( +y(1)*( +z(2) -z(5) ) - +y(2)*( -z(1) +z(3) -z(5) +z(7) ) - +y(3)*( -z(2) +z(7) ) - +y(4)*( +z(5) -z(7) ) - +y(5)*( +z(1) +z(2) -z(4) -z(7) ) - +y(7)*( -z(2) -z(3) +z(4) +z(5) ) )*twelth; - - B_matrix(7,0) = ( +y(0)*( -z(3) +z(4) ) - +y(2)*( +z(3) -z(6) ) - +y(3)*( +z(0) -z(2) +z(4) -z(6) ) - +y(4)*( -z(0) -z(3) +z(5) +z(6) ) - +y(5)*( -z(4) +z(6) ) - +y(6)*( +z(2) +z(3) -z(4) -z(5) ) )*twelth; - - B_matrix(0,1) = ( +z(1)*( -x(2) -x(3) +x(4) +x(5) ) - +z(2)*( +x(1) -x(3) ) - +z(3)*( +x(1) +x(2) -x(4) -x(7) ) - +z(4)*( -x(1) +x(3) -x(5) +x(7) ) - +z(5)*( -x(1) +x(4) ) - +z(7)*( +x(3) -x(4) ) )*twelth; - - B_matrix(1,1) = ( +z(0)*( +x(2) +x(3) -x(4) -x(5) ) - +z(2)*( -x(0) -x(3) +x(5) +x(6) ) - +z(3)*( -x(0) +x(2) ) - +z(4)*( +x(0) -x(5) ) - +z(5)*( +x(0) -x(2) +x(4) -x(6) ) - +z(6)*( -x(2) +x(5) ) )*twelth; - - B_matrix(2,1) = ( +z(0)*( -x(1) +x(3) ) - +z(1)*( +x(0) +x(3) -x(5) -x(6) ) - +z(3)*( -x(0) -x(1) +x(6) +x(7) ) - +z(5)*( +x(1) -x(6) ) - +z(6)*( +x(1) -x(3) +x(5) -x(7) ) - +z(7)*( -x(3) +x(6) ) )*twelth; - - B_matrix(3,1) = ( +z(0)*( -x(1) -x(2) +x(4) +x(7) ) - +z(1)*( +x(0) -x(2) ) - +z(2)*( +x(0) +x(1) -x(6) -x(7) ) - +z(4)*( -x(0) +x(7) ) - +z(6)*( +x(2) -x(7) ) - +z(7)*( -x(0) +x(2) -x(4) +x(6) ) )*twelth; - - B_matrix(4,1) = ( +z(0)*( +x(1) -x(3) +x(5) -x(7) ) - +z(1)*( -x(0) +x(5) ) - +z(3)*( +x(0) -x(7) ) - +z(5)*( -x(0) -x(1) +x(6) +x(7) ) - +z(6)*( -x(5) +x(7) ) - +z(7)*( +x(0) +x(3) -x(5) -x(6) ) )*twelth; - - B_matrix(5,1) = ( +z(0)*( +x(1) -x(4) ) - +z(1)*( -x(0) +x(2) -x(4) +x(6) ) - +z(2)*( -x(1) +x(6) ) - +z(4)*( +x(0) +x(1) -x(6) -x(7) ) - +z(6)*( -x(1) -x(2) +x(4) +x(7) ) - +z(7)*( +x(4) -x(6) ) )*twelth; - - B_matrix(6,1) = ( +z(1)*( +x(2) -x(5) ) - +z(2)*( -x(1) +x(3) -x(5) +x(7) ) - +z(3)*( -x(2) +x(7) ) - +z(4)*( +x(5) -x(7) ) - +z(5)*( +x(1) +x(2) -x(4) -x(7) ) - +z(7)*( -x(2) -x(3) +x(4) +x(5) ) )*twelth; - - B_matrix(7,1) = ( +z(0)*( -x(3) +x(4) ) - +z(2)*( +x(3) -x(6) ) - +z(3)*( +x(0) -x(2) +x(4) -x(6) ) - +z(4)*( -x(0) -x(3) +x(5) +x(6) ) - +z(5)*( -x(4) +x(6) ) - +z(6)*( +x(2) +x(3) -x(4) -x(5) ) )*twelth; - - B_matrix(0,2) = ( +x(1)*( -y(2) -y(3) +y(4) +y(5) ) - +x(2)*( +y(1) -y(3) ) - +x(3)*( +y(1) +y(2) -y(4) -y(7) ) - +x(4)*( -y(1) +y(3) -y(5) +y(7) ) - +x(5)*( -y(1) +y(4) ) - +x(7)*( +y(3) -y(4) ) )*twelth; - - B_matrix(1,2) = ( +x(0)*( +y(2) +y(3) -y(4) -y(5) ) - +x(2)*( -y(0) -y(3) +y(5) +y(6) ) - +x(3)*( -y(0) +y(2) ) - +x(4)*( +y(0) -y(5) ) - +x(5)*( +y(0) -y(2) +y(4) -y(6) ) - +x(6)*( -y(2) +y(5) ) )*twelth; - - B_matrix(2,2) = ( +x(0)*( -y(1) +y(3) ) - +x(1)*( +y(0) +y(3) -y(5) -y(6) ) - +x(3)*( -y(0) -y(1) +y(6) +y(7) ) - +x(5)*( +y(1) -y(6) ) - +x(6)*( +y(1) -y(3) +y(5) -y(7) ) - +x(7)*( -y(3) +y(6) ) )*twelth; - - B_matrix(3,2) = ( +x(0)*( -y(1) -y(2) +y(4) +y(7) ) - +x(1)*( +y(0) -y(2) ) - +x(2)*( +y(0) +y(1) -y(6) -y(7) ) - +x(4)*( -y(0) +y(7) ) - +x(6)*( +y(2) -y(7) ) - +x(7)*( -y(0) +y(2) -y(4) +y(6) ) )*twelth; - - B_matrix(4,2) = ( +x(0)*( +y(1) -y(3) +y(5) -y(7) ) - +x(1)*( -y(0) +y(5) ) - +x(3)*( +y(0) -y(7) ) - +x(5)*( -y(0) -y(1) +y(6) +y(7) ) - +x(6)*( -y(5) +y(7) ) - +x(7)*( +y(0) +y(3) -y(5) -y(6) ) )*twelth; - - B_matrix(5,2) = ( +x(0)*( +y(1) -y(4) ) - +x(1)*( -y(0) +y(2) -y(4) +y(6) ) - +x(2)*( -y(1) +y(6) ) - +x(4)*( +y(0) +y(1) -y(6) -y(7) ) - +x(6)*( -y(1) -y(2) +y(4) +y(7) ) - +x(7)*( +y(4) -y(6) ) )*twelth; - - B_matrix(6,2) = ( +x(1)*( +y(2) -y(5) ) - +x(2)*( -y(1) +y(3) -y(5) +y(7) ) - +x(3)*( -y(2) +y(7) ) - +x(4)*( +y(5) -y(7) ) - +x(5)*( +y(1) +y(2) -y(4) -y(7) ) - +x(7)*( -y(2) -y(3) +y(4) +y(5) ) )*twelth; - - B_matrix(7,2) = ( +x(0)*( -y(3) +y(4) ) - +x(2)*( +y(3) -y(6) ) - +x(3)*( +y(0) -y(2) +y(4) -y(6) ) - +x(4)*( -y(0) -y(3) +y(5) +y(6) ) - +x(5)*( -y(4) +y(6) ) - +x(6)*( +y(2) +y(3) -y(4) -y(5) ) )*twelth; - + double twelth = 1. / 12.; + + B_matrix(0, 0) = (+y(1) * (-z(2) - z(3) + z(4) + z(5) ) + + y(2) * (+z(1) - z(3) ) + + y(3) * (+z(1) + z(2) - z(4) - z(7) ) + + y(4) * (-z(1) + z(3) - z(5) + z(7) ) + + y(5) * (-z(1) + z(4) ) + + y(7) * (+z(3) - z(4) ) ) * twelth; + + B_matrix(1, 0) = (+y(0) * (+z(2) + z(3) - z(4) - z(5) ) + + y(2) * (-z(0) - z(3) + z(5) + z(6) ) + + y(3) * (-z(0) + z(2) ) + + y(4) * (+z(0) - z(5) ) + + y(5) * (+z(0) - z(2) + z(4) - z(6) ) + + y(6) * (-z(2) + z(5) ) ) * twelth; + + B_matrix(2, 0) = (+y(0) * (-z(1) + z(3) ) + + y(1) * (+z(0) + z(3) - z(5) - z(6) ) + + y(3) * (-z(0) - z(1) + z(6) + z(7) ) + + y(5) * (+z(1) - z(6) ) + + y(6) * (+z(1) - z(3) + z(5) - z(7) ) + + y(7) * (-z(3) + z(6) ) ) * twelth; + + B_matrix(3, 0) = (+y(0) * (-z(1) - z(2) + z(4) + z(7) ) + + y(1) * (+z(0) - z(2) ) + + y(2) * (+z(0) + z(1) - z(6) - z(7) ) + + y(4) * (-z(0) + z(7) ) + + y(6) * (+z(2) - z(7) ) + + y(7) * (-z(0) + z(2) - z(4) + z(6) ) ) * twelth; + + B_matrix(4, 0) = (+y(0) * (+z(1) - z(3) + z(5) - z(7) ) + + y(1) * (-z(0) + z(5) ) + + y(3) * (+z(0) - z(7) ) + + y(5) * (-z(0) - z(1) + z(6) + z(7) ) + + y(6) * (-z(5) + z(7) ) + + y(7) * (+z(0) + z(3) - z(5) - z(6) ) ) * twelth; + + B_matrix(5, 0) = (+y(0) * (+z(1) - z(4) ) + + y(1) * (-z(0) + z(2) - z(4) + z(6) ) + + y(2) * (-z(1) + z(6) ) + + y(4) * (+z(0) + z(1) - z(6) - z(7) ) + + y(6) * (-z(1) - z(2) + z(4) + z(7) ) + + y(7) * (+z(4) - z(6) ) ) * twelth; + + B_matrix(6, 0) = (+y(1) * (+z(2) - z(5) ) + + y(2) * (-z(1) + z(3) - z(5) + z(7) ) + + y(3) * (-z(2) + z(7) ) + + y(4) * (+z(5) - z(7) ) + + y(5) * (+z(1) + z(2) - z(4) - z(7) ) + + y(7) * (-z(2) - z(3) + z(4) + z(5) ) ) * twelth; + + B_matrix(7, 0) = (+y(0) * (-z(3) + z(4) ) + + y(2) * (+z(3) - z(6) ) + + y(3) * (+z(0) - z(2) + z(4) - z(6) ) + + y(4) * (-z(0) - z(3) + z(5) + z(6) ) + + y(5) * (-z(4) + z(6) ) + + y(6) * (+z(2) + z(3) - z(4) - z(5) ) ) * twelth; + + B_matrix(0, 1) = (+z(1) * (-x(2) - x(3) + x(4) + x(5) ) + + z(2) * (+x(1) - x(3) ) + + z(3) * (+x(1) + x(2) - x(4) - x(7) ) + + z(4) * (-x(1) + x(3) - x(5) + x(7) ) + + z(5) * (-x(1) + x(4) ) + + z(7) * (+x(3) - x(4) ) ) * twelth; + + B_matrix(1, 1) = (+z(0) * (+x(2) + x(3) - x(4) - x(5) ) + + z(2) * (-x(0) - x(3) + x(5) + x(6) ) + + z(3) * (-x(0) + x(2) ) + + z(4) * (+x(0) - x(5) ) + + z(5) * (+x(0) - x(2) + x(4) - x(6) ) + + z(6) * (-x(2) + x(5) ) ) * twelth; + + B_matrix(2, 1) = (+z(0) * (-x(1) + x(3) ) + + z(1) * (+x(0) + x(3) - x(5) - x(6) ) + + z(3) * (-x(0) - x(1) + x(6) + x(7) ) + + z(5) * (+x(1) - x(6) ) + + z(6) * (+x(1) - x(3) + x(5) - x(7) ) + + z(7) * (-x(3) + x(6) ) ) * twelth; + + B_matrix(3, 1) = (+z(0) * (-x(1) - x(2) + x(4) + x(7) ) + + z(1) * (+x(0) - x(2) ) + + z(2) * (+x(0) + x(1) - x(6) - x(7) ) + + z(4) * (-x(0) + x(7) ) + + z(6) * (+x(2) - x(7) ) + + z(7) * (-x(0) + x(2) - x(4) + x(6) ) ) * twelth; + + B_matrix(4, 1) = (+z(0) * (+x(1) - x(3) + x(5) - x(7) ) + + z(1) * (-x(0) + x(5) ) + + z(3) * (+x(0) - x(7) ) + + z(5) * (-x(0) - x(1) + x(6) + x(7) ) + + z(6) * (-x(5) + x(7) ) + + z(7) * (+x(0) + x(3) - x(5) - x(6) ) ) * twelth; + + B_matrix(5, 1) = (+z(0) * (+x(1) - x(4) ) + + z(1) * (-x(0) + x(2) - x(4) + x(6) ) + + z(2) * (-x(1) + x(6) ) + + z(4) * (+x(0) + x(1) - x(6) - x(7) ) + + z(6) * (-x(1) - x(2) + x(4) + x(7) ) + + z(7) * (+x(4) - x(6) ) ) * twelth; + + B_matrix(6, 1) = (+z(1) * (+x(2) - x(5) ) + + z(2) * (-x(1) + x(3) - x(5) + x(7) ) + + z(3) * (-x(2) + x(7) ) + + z(4) * (+x(5) - x(7) ) + + z(5) * (+x(1) + x(2) - x(4) - x(7) ) + + z(7) * (-x(2) - x(3) + x(4) + x(5) ) ) * twelth; + + B_matrix(7, 1) = (+z(0) * (-x(3) + x(4) ) + + z(2) * (+x(3) - x(6) ) + + z(3) * (+x(0) - x(2) + x(4) - x(6) ) + + z(4) * (-x(0) - x(3) + x(5) + x(6) ) + + z(5) * (-x(4) + x(6) ) + + z(6) * (+x(2) + x(3) - x(4) - x(5) ) ) * twelth; + + B_matrix(0, 2) = (+x(1) * (-y(2) - y(3) + y(4) + y(5) ) + + x(2) * (+y(1) - y(3) ) + + x(3) * (+y(1) + y(2) - y(4) - y(7) ) + + x(4) * (-y(1) + y(3) - y(5) + y(7) ) + + x(5) * (-y(1) + y(4) ) + + x(7) * (+y(3) - y(4) ) ) * twelth; + + B_matrix(1, 2) = (+x(0) * (+y(2) + y(3) - y(4) - y(5) ) + + x(2) * (-y(0) - y(3) + y(5) + y(6) ) + + x(3) * (-y(0) + y(2) ) + + x(4) * (+y(0) - y(5) ) + + x(5) * (+y(0) - y(2) + y(4) - y(6) ) + + x(6) * (-y(2) + y(5) ) ) * twelth; + + B_matrix(2, 2) = (+x(0) * (-y(1) + y(3) ) + + x(1) * (+y(0) + y(3) - y(5) - y(6) ) + + x(3) * (-y(0) - y(1) + y(6) + y(7) ) + + x(5) * (+y(1) - y(6) ) + + x(6) * (+y(1) - y(3) + y(5) - y(7) ) + + x(7) * (-y(3) + y(6) ) ) * twelth; + + B_matrix(3, 2) = (+x(0) * (-y(1) - y(2) + y(4) + y(7) ) + + x(1) * (+y(0) - y(2) ) + + x(2) * (+y(0) + y(1) - y(6) - y(7) ) + + x(4) * (-y(0) + y(7) ) + + x(6) * (+y(2) - y(7) ) + + x(7) * (-y(0) + y(2) - y(4) + y(6) ) ) * twelth; + + B_matrix(4, 2) = (+x(0) * (+y(1) - y(3) + y(5) - y(7) ) + + x(1) * (-y(0) + y(5) ) + + x(3) * (+y(0) - y(7) ) + + x(5) * (-y(0) - y(1) + y(6) + y(7) ) + + x(6) * (-y(5) + y(7) ) + + x(7) * (+y(0) + y(3) - y(5) - y(6) ) ) * twelth; + + B_matrix(5, 2) = (+x(0) * (+y(1) - y(4) ) + + x(1) * (-y(0) + y(2) - y(4) + y(6) ) + + x(2) * (-y(1) + y(6) ) + + x(4) * (+y(0) + y(1) - y(6) - y(7) ) + + x(6) * (-y(1) - y(2) + y(4) + y(7) ) + + x(7) * (+y(4) - y(6) ) ) * twelth; + + B_matrix(6, 2) = (+x(1) * (+y(2) - y(5) ) + + x(2) * (-y(1) + y(3) - y(5) + y(7) ) + + x(3) * (-y(2) + y(7) ) + + x(4) * (+y(5) - y(7) ) + + x(5) * (+y(1) + y(2) - y(4) - y(7) ) + + x(7) * (-y(2) - y(3) + y(4) + y(5) ) ) * twelth; + + B_matrix(7, 2) = (+x(0) * (-y(3) + y(4) ) + + x(2) * (+y(3) - y(6) ) + + x(3) * (+y(0) - y(2) + y(4) - y(6) ) + + x(4) * (-y(0) - y(3) + y(5) + y(6) ) + + x(5) * (-y(4) + y(6) ) + + x(6) * (+y(2) + y(3) - y(4) - y(5) ) ) * twelth; } // end subroutine - /* ---------------------------------------------------------------------------- Compute Volume of each finite element ------------------------------------------------------------------------------- */ -void FEA_Module_Dynamic_Elasticity::get_vol(){ - +void FEA_Module_Dynamic_Elasticity::get_vol() +{ const size_t rk_level = rk_num_bins - 1; const size_t num_dims = num_dim; - if (num_dims == 2){ + if (num_dims == 2) + { FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { - // cut out the node_gids for this element - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 4); + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 4); get_vol_quad(elem_vol, elem_gid, node_coords, elem_node_gids, rk_level); - }); Kokkos::fence(); } - else { + else + { FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { - // cut out the node_gids for this element - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 8); + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 8); get_vol_hex(elem_vol, elem_gid, node_coords, elem_node_gids, rk_level); - }); Kokkos::fence(); } // end if - + return; - } // end subroutine - // Exact volume for a hex element KOKKOS_INLINE_FUNCTION -void FEA_Module_Dynamic_Elasticity::get_vol_hex(const DViewCArrayKokkos &elem_vol, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const { - +void FEA_Module_Dynamic_Elasticity::get_vol_hex(const DViewCArrayKokkos& elem_vol, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const +{ const size_t num_nodes = 8; double x_array[8]; double y_array[8]; double z_array[8]; - + // x, y, z coordinates of elem vertices - auto x = ViewCArrayKokkos (x_array, num_nodes); - auto y = ViewCArrayKokkos (y_array, num_nodes); - auto z = ViewCArrayKokkos (z_array, num_nodes); - + auto x = ViewCArrayKokkos(x_array, num_nodes); + auto y = ViewCArrayKokkos(y_array, num_nodes); + auto z = ViewCArrayKokkos(z_array, num_nodes); + // get the coordinates of the nodes(rk,elem,node) in this element - for (int node_lid = 0; node_lid < num_nodes; node_lid++){ + for (int node_lid = 0; node_lid < num_nodes; node_lid++) + { x(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 0); y(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 1); z(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 2); } // end for - double twelth = 1./12.; - + double twelth = 1. / 12.; + // element volume elem_vol(elem_gid) = - (x(1)*(y(3)*(-z(0) + z(2)) + y(4)*( z(0) - z(5)) + y(0)*( z(2) + z(3) - z(4) - z(5)) + y(6)*(-z(2) + z(5)) + y(5)*(z(0) - z(2) + z(4) - z(6)) + y(2)*(-z(0) - z(3) + z(5) + z(6))) + - x(7)*(y(0)*(-z(3) + z(4)) + y(6)*( z(2) + z(3) - z(4) - z(5)) + y(2)*(z(3) - z(6)) + y(3)*(z(0) - z(2) + z(4) - z(6)) + y(5)*(-z(4) + z(6)) + y(4)*(-z(0) - z(3) + z(5) + z(6))) + - x(3)*(y(1)*( z(0) - z(2)) + y(7)*(-z(0) + z(2) - z(4) + z(6)) + y(6)*(z(2) - z(7)) + y(2)*(z(0) + z(1) - z(6) - z(7)) + y(4)*(-z(0) + z(7)) + y(0)*(-z(1) - z(2) + z(4) + z(7))) + - x(5)*(y(0)*( z(1) - z(4)) + y(7)*( z(4) - z(6)) + y(2)*(-z(1) + z(6)) + y(1)*(-z(0) + z(2) - z(4) + z(6)) + y(4)*(z(0) + z(1) - z(6) - z(7)) + y(6)*(-z(1) - z(2) + z(4) + z(7))) + - x(6)*(y(1)*( z(2) - z(5)) + y(7)*(-z(2) - z(3) + z(4) + z(5)) + y(5)*(z(1) + z(2) - z(4) - z(7)) + y(4)*(z(5) - z(7)) + y(3)*(-z(2) + z(7)) + y(2)*(-z(1) + z(3) - z(5) + z(7))) + - x(0)*(y(2)*( z(1) - z(3)) + y(7)*( z(3) - z(4)) + y(5)*(-z(1) + z(4)) + y(1)*(-z(2) - z(3) + z(4) + z(5)) + y(3)*(z(1) + z(2) - z(4) - z(7)) + y(4)*(-z(1) + z(3) - z(5) + z(7))) + - x(2)*(y(0)*(-z(1) + z(3)) + y(5)*( z(1) - z(6)) + y(1)*( z(0) + z(3) - z(5) - z(6)) + y(7)*(-z(3) + z(6)) + y(6)*(z(1) - z(3) + z(5) - z(7)) + y(3)*(-z(0) - z(1) + z(6) + z(7))) + - x(4)*(y(1)*(-z(0) + z(5)) + y(7)*( z(0) + z(3) - z(5) - z(6)) + y(3)*(z(0) - z(7)) + y(0)*(z(1) - z(3) + z(5) - z(7)) + y(6)*(-z(5) + z(7)) + y(5)*(-z(0) - z(1) + z(6) + z(7))))*twelth; + (x(1) * (y(3) * (-z(0) + z(2)) + y(4) * (z(0) - z(5)) + y(0) * (z(2) + z(3) - z(4) - z(5)) + y(6) * (-z(2) + z(5)) + y(5) * (z(0) - z(2) + z(4) - z(6)) + y(2) * (-z(0) - z(3) + z(5) + z(6))) + + x(7) * (y(0) * (-z(3) + z(4)) + y(6) * (z(2) + z(3) - z(4) - z(5)) + y(2) * (z(3) - z(6)) + y(3) * (z(0) - z(2) + z(4) - z(6)) + y(5) * (-z(4) + z(6)) + y(4) * (-z(0) - z(3) + z(5) + z(6))) + + x(3) * (y(1) * (z(0) - z(2)) + y(7) * (-z(0) + z(2) - z(4) + z(6)) + y(6) * (z(2) - z(7)) + y(2) * (z(0) + z(1) - z(6) - z(7)) + y(4) * (-z(0) + z(7)) + y(0) * (-z(1) - z(2) + z(4) + z(7))) + + x(5) * (y(0) * (z(1) - z(4)) + y(7) * (z(4) - z(6)) + y(2) * (-z(1) + z(6)) + y(1) * (-z(0) + z(2) - z(4) + z(6)) + y(4) * (z(0) + z(1) - z(6) - z(7)) + y(6) * (-z(1) - z(2) + z(4) + z(7))) + + x(6) * (y(1) * (z(2) - z(5)) + y(7) * (-z(2) - z(3) + z(4) + z(5)) + y(5) * (z(1) + z(2) - z(4) - z(7)) + y(4) * (z(5) - z(7)) + y(3) * (-z(2) + z(7)) + y(2) * (-z(1) + z(3) - z(5) + z(7))) + + x(0) * (y(2) * (z(1) - z(3)) + y(7) * (z(3) - z(4)) + y(5) * (-z(1) + z(4)) + y(1) * (-z(2) - z(3) + z(4) + z(5)) + y(3) * (z(1) + z(2) - z(4) - z(7)) + y(4) * (-z(1) + z(3) - z(5) + z(7))) + + x(2) * (y(0) * (-z(1) + z(3)) + y(5) * (z(1) - z(6)) + y(1) * (z(0) + z(3) - z(5) - z(6)) + y(7) * (-z(3) + z(6)) + y(6) * (z(1) - z(3) + z(5) - z(7)) + y(3) * (-z(0) - z(1) + z(6) + z(7))) + + x(4) * + (y(1) * (-z(0) + z(5)) + y(7) * (z(0) + z(3) - z(5) - z(6)) + y(3) * (z(0) - z(7)) + y(0) * (z(1) - z(3) + z(5) - z(7)) + y(6) * (-z(5) + z(7)) + y(5) * (-z(0) - z(1) + z(6) + z(7)))) * + twelth; return; - } // end subroutine - - KOKKOS_FUNCTION -void FEA_Module_Dynamic_Elasticity::get_bmatrix2D(const ViewCArrayKokkos &B_matrix, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const { - +void FEA_Module_Dynamic_Elasticity::get_bmatrix2D(const ViewCArrayKokkos& B_matrix, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const +{ const size_t num_nodes = 4; double x_array[4]; double y_array[4]; - + // x, y coordinates of elem vertices - auto x = ViewCArrayKokkos (x_array, num_nodes); - auto y = ViewCArrayKokkos (y_array, num_nodes); + auto x = ViewCArrayKokkos(x_array, num_nodes); + auto y = ViewCArrayKokkos(y_array, num_nodes); // get the coordinates of the nodes(rk,elem,node) in this element - for (int node_lid = 0; node_lid < num_nodes; node_lid++){ + for (int node_lid = 0; node_lid < num_nodes; node_lid++) + { x(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 0); y(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 1); } // end for @@ -340,26 +333,23 @@ void FEA_Module_Dynamic_Elasticity::get_bmatrix2D(const ViewCArrayKokkos &elem_vol, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const { - +void FEA_Module_Dynamic_Elasticity::get_vol_quad(const DViewCArrayKokkos& elem_vol, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const +{ // --- testing here --- /* double test_vol = 0.0; // getting the corner facial area double corner_areas_array[4]; ViewCArrayKokkos corner_areas(&corner_areas_array[0],4); - + get_area_weights2D(corner_areas, elem_gid, node_coords, elem_node_gids, rk_level); - + for(size_t node_lid=0; node_lid<4; node_lid++){ double y = node_coords(rk_level, elem_node_gids(node_lid), 1); // node radius test_vol += corner_areas(node_lid)*y; } // end for - + test_vol matches the Barlow volume formula */ // ------------------- - - + elem_vol(elem_gid) = 0.0; - + const size_t num_nodes = 4; double x_array[4]; double y_array[4]; - + // x, y coordinates of elem vertices - auto x = ViewCArrayKokkos (x_array, num_nodes); - auto y = ViewCArrayKokkos (y_array, num_nodes); - + auto x = ViewCArrayKokkos(x_array, num_nodes); + auto y = ViewCArrayKokkos(y_array, num_nodes); + // get the coordinates of the nodes(rk,elem,node) in this element - for (int node_lid = 0; node_lid < num_nodes; node_lid++){ + for (int node_lid = 0; node_lid < num_nodes; node_lid++) + { x(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 0); y(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 1); } // end for @@ -436,34 +424,33 @@ void FEA_Module_Dynamic_Elasticity::get_vol_quad(const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const { +double FEA_Module_Dynamic_Elasticity::get_area_quad(const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const +{ + double elem_area = 0.0; - double elem_area=0.0; - const size_t num_nodes = 4; double x_array[4]; double y_array[4]; - + // x, y coordinates of elem vertices - auto x = ViewCArrayKokkos (x_array, num_nodes); - auto y = ViewCArrayKokkos (y_array, num_nodes); - + auto x = ViewCArrayKokkos(x_array, num_nodes); + auto y = ViewCArrayKokkos(y_array, num_nodes); + // get the coordinates of the nodes(rk,elem,node) in this element - for (int node_lid = 0; node_lid < num_nodes; node_lid++){ + for (int node_lid = 0; node_lid < num_nodes; node_lid++) + { x(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 0); y(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 1); } // end for @@ -471,86 +458,79 @@ double FEA_Module_Dynamic_Elasticity::get_area_quad(const size_t elem_gid, /* ensight node order 0 1 2 3 Flanaghan node order 3 4 1 2 */ - + // element facial area - elem_area = 0.5*((x(0)-x(2))*(y(1)-y(3))+(x(3)-x(1))*(y(0)-y(2))); + elem_area = 0.5 * ((x(0) - x(2)) * (y(1) - y(3)) + (x(3) - x(1)) * (y(0) - y(2))); return elem_area; - } // end subroutine - KOKKOS_INLINE_FUNCTION double FEA_Module_Dynamic_Elasticity::heron(const double x1, - const double y1, - const double x2, - const double y2, - const double x3, - const double y3) const + const double y1, + const double x2, + const double y2, + const double x3, + const double y3) const { - double S,a,b,c,area; + double S, a, b, c, area; - S=0.0; - a=sqrt((x2-x1)*(x2-x1)+(y2-y1)*(y2-y1)); - S+=a; - b=sqrt((x3-x2)*(x3-x2)+(y3-y2)*(y3-y2)); - S+=b; - c=sqrt((x3-x1)*(x3-x1)+(y3-y1)*(y3-y1)); - S+=c; + S = 0.0; + a = sqrt((x2 - x1) * (x2 - x1) + (y2 - y1) * (y2 - y1)); + S += a; + b = sqrt((x3 - x2) * (x3 - x2) + (y3 - y2) * (y3 - y2)); + S += b; + c = sqrt((x3 - x1) * (x3 - x1) + (y3 - y1) * (y3 - y1)); + S += c; - S*=0.5; - area=sqrt(S*(S-a)*(S-b)*(S-c)); + S *= 0.5; + area = sqrt(S * (S - a) * (S - b) * (S - c)); - return area; + return area; } - - KOKKOS_FUNCTION -void FEA_Module_Dynamic_Elasticity::get_area_weights2D(const ViewCArrayKokkos &corner_areas, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const { - +void FEA_Module_Dynamic_Elasticity::get_area_weights2D(const ViewCArrayKokkos& corner_areas, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const +{ const size_t num_nodes = 4; double x_array[4]; double y_array[4]; - double rc,zc; - double A12,A23,A34,A41; - + double rc, zc; + double A12, A23, A34, A41; + // x, y coordinates of elem vertices - ViewCArrayKokkos x(x_array, num_nodes); - ViewCArrayKokkos y(y_array, num_nodes); + ViewCArrayKokkos x(x_array, num_nodes); + ViewCArrayKokkos y(y_array, num_nodes); // get the coordinates of the nodes(rk,elem,node) in this element - rc=zc=0.0; - for (int node_lid = 0; node_lid < num_nodes; node_lid++){ + rc = zc = 0.0; + for (int node_lid = 0; node_lid < num_nodes; node_lid++) + { x(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 0); y(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 1); - rc+=0.25*y(node_lid); - zc+=0.25*x(node_lid); + rc += 0.25 * y(node_lid); + zc += 0.25 * x(node_lid); } // end for /* ensight node order 0 1 2 3 Barlow node order 1 2 3 4 */ - - A12 = heron(x(0),y(0),zc,rc,x(1),y(1)); - A23 = heron(x(1),y(1),zc,rc,x(2),y(2)); - A34 = heron(x(2),y(2),zc,rc,x(3),y(3)); - A41 = heron(x(3),y(3),zc,rc,x(0),y(0)); - - corner_areas(0)=(5.*A41+5.*A12+A23+A34)/12.; - corner_areas(1)=(A41+5.*A12+5.*A23+A34)/12.; - corner_areas(2)=(A41+A12+5.*A23+5.*A34)/12.; - corner_areas(3)=(5.*A41+A12+A23+5.*A34)/12.; + A12 = heron(x(0), y(0), zc, rc, x(1), y(1)); + A23 = heron(x(1), y(1), zc, rc, x(2), y(2)); + A34 = heron(x(2), y(2), zc, rc, x(3), y(3)); + A41 = heron(x(3), y(3), zc, rc, x(0), y(0)); + corner_areas(0) = (5. * A41 + 5. * A12 + A23 + A34) / 12.; + corner_areas(1) = (A41 + 5. * A12 + 5. * A23 + A34) / 12.; + corner_areas(2) = (A41 + A12 + 5. * A23 + 5. * A34) / 12.; + corner_areas(3) = (5. * A41 + A12 + A23 + 5. * A34) / 12.; return; - } // end subroutine - diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/mesh.h b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/mesh.h index 25111f2ed..2a4d852c8 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/mesh.h +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/mesh.h @@ -1,7 +1,6 @@ #ifndef MESH_H #define MESH_H - #include "matar.h" #include "state.h" @@ -48,195 +47,181 @@ patch 6: [4,5,6,7] zeta-plus dir // sort in ascending order using bubble sort KOKKOS_INLINE_FUNCTION -void bubble_sort(size_t arr[], const size_t num){ - - for (size_t i=0; i<(num-1); i++){ - for (size_t j=0; j<(num-i-1); j++){ - - if (arr[j]>arr[j+1]){ +void bubble_sort(size_t arr[], const size_t num) +{ + for (size_t i = 0; i < (num - 1); i++) + { + for (size_t j = 0; j < (num - i - 1); j++) + { + if (arr[j] > arr[j + 1]) + { size_t temp = arr[j]; - arr[j] = arr[j+1]; - arr[j+1] = temp; + arr[j] = arr[j + 1]; + arr[j + 1] = temp; } // end if - } // end for j } // end for i } // end function - // mesh sizes and connectivity data structures -struct mesh_t { - +struct mesh_t +{ size_t num_dims; - + size_t num_nodes, num_local_nodes; - + size_t num_elems; size_t num_nodes_in_elem; size_t num_patches_in_elem; size_t num_corners; - + size_t num_patches; - + size_t num_bdy_patches; size_t num_bdy_nodes; size_t num_bdy_sets; size_t num_nodes_in_patch; - // ---- nodes ---- - + // corner ids in node - RaggedRightArrayKokkos corners_in_node; - CArrayKokkos num_corners_in_node; - + RaggedRightArrayKokkos corners_in_node; + CArrayKokkos num_corners_in_node; + // elem ids in node - RaggedRightArrayKokkos elems_in_node; - + RaggedRightArrayKokkos elems_in_node; + // node ids in node - RaggedRightArrayKokkos nodes_in_node; - CArrayKokkos num_nodes_in_node; - - + RaggedRightArrayKokkos nodes_in_node; + CArrayKokkos num_nodes_in_node; + // ---- elems ---- - + // node ids in elem - DCArrayKokkos nodes_in_elem; - + DCArrayKokkos nodes_in_elem; + // corner ids in elem - CArrayKokkos corners_in_elem; - + CArrayKokkos corners_in_elem; + // elem ids in elem - RaggedRightArrayKokkos elems_in_elem; - CArrayKokkos num_elems_in_elem; - + RaggedRightArrayKokkos elems_in_elem; + CArrayKokkos num_elems_in_elem; + // patch ids in elem - CArrayKokkos patches_in_elem; - - + CArrayKokkos patches_in_elem; + // ---- patches ---- - + // node ids in a patch - CArrayKokkos nodes_in_patch; - + CArrayKokkos nodes_in_patch; + // element ids in a patch - CArrayKokkos elems_in_patch; - - + CArrayKokkos elems_in_patch; + // ---- bdy ---- - + // bdy_patches - CArrayKokkos bdy_patches; - + CArrayKokkos bdy_patches; + // bdy nodes - CArrayKokkos bdy_nodes; - + CArrayKokkos bdy_nodes; + // patch ids in bdy set - DynamicRaggedRightArrayKokkos bdy_patches_in_set; - + DynamicRaggedRightArrayKokkos bdy_patches_in_set; + // node ids in bdy_patch set - RaggedRightArrayKokkos bdy_nodes_in_set; - DCArrayKokkos num_bdy_nodes_in_set; - + RaggedRightArrayKokkos bdy_nodes_in_set; + DCArrayKokkos num_bdy_nodes_in_set; + // initialization methods void initialize_nodes(const size_t num_nodes_inp) { num_nodes = num_nodes_inp; - + return; - }; // end method // initialization methods void initialize_local_nodes(const size_t num_nodes_inp) { num_local_nodes = num_nodes_inp; - + return; - }; // end method - - + // initialization methods void initialize_elems(const size_t num_elems_inp, const size_t num_dims_inp) { num_dims = num_dims_inp; num_nodes_in_elem = 1; - for (int dim=0; dim (num_elems, num_nodes_in_elem, "nodes_in_elem"); - corners_in_elem = CArrayKokkos (num_elems, num_nodes_in_elem, "corners_in_elem"); - + num_elems = num_elems_inp; + nodes_in_elem = DCArrayKokkos(num_elems, num_nodes_in_elem, "nodes_in_elem"); + corners_in_elem = CArrayKokkos(num_elems, num_nodes_in_elem, "corners_in_elem"); + return; - }; // end method - - + // initialization methods void initialize_corners(const size_t num_corners_inp) { num_corners = num_corners_inp; - + return; - }; // end method - - + // build the corner mesh connectivity arrays - void build_corner_connectivity(){ - - num_corners_in_node = CArrayKokkos (num_nodes); // stride sizes - + void build_corner_connectivity() + { + num_corners_in_node = CArrayKokkos(num_nodes); // stride sizes + // initializing the number of corners (node-cell pair) to be zero FOR_ALL_CLASS(node_gid, 0, num_nodes, { num_corners_in_node(node_gid) = 0; }); - - - for (size_t elem_gid = 0; elem_gid < num_elems; elem_gid++){ - FOR_ALL_CLASS(node_lid, 0, num_nodes_in_elem,{ - + + for (size_t elem_gid = 0; elem_gid < num_elems; elem_gid++) + { + FOR_ALL_CLASS(node_lid, 0, num_nodes_in_elem, { // get the global_id of the node size_t node_gid = nodes_in_elem(elem_gid, node_lid); // increment the number of corners attached to this point num_corners_in_node(node_gid) = num_corners_in_node(node_gid) + 1; - }); // end FOR_ALL over nodes in element } // end for elem_gid - - + // the stride sizes are the num_corners_in_node at the node - corners_in_node = RaggedRightArrayKokkos (num_corners_in_node, "corners_in_node"); + corners_in_node = RaggedRightArrayKokkos(num_corners_in_node, "corners_in_node"); - CArrayKokkos count_saved_corners_in_node(num_nodes, "count_saved_corners_in_node"); + CArrayKokkos count_saved_corners_in_node(num_nodes, "count_saved_corners_in_node"); // reset num_corners to zero FOR_ALL_CLASS(node_gid, 0, num_nodes, { count_saved_corners_in_node(node_gid) = 0; }); - - + // he elems_in_elem data type - elems_in_node = RaggedRightArrayKokkos (num_corners_in_node, "elems_in_node"); - + elems_in_node = RaggedRightArrayKokkos(num_corners_in_node, "elems_in_node"); + // populate the elems connected to a node list and corners in a node - for (size_t elem_gid = 0; elem_gid < num_elems; elem_gid++){ + for (size_t elem_gid = 0; elem_gid < num_elems; elem_gid++) + { FOR_ALL_CLASS(node_lid, 0, num_nodes_in_elem, { - // get the global_id of the node size_t node_gid = nodes_in_elem(elem_gid, node_lid); - + // the column index is the num corners saved size_t j = count_saved_corners_in_node(node_gid); // Save corner index to this node_gid - size_t corner_gid = node_lid + elem_gid*num_nodes_in_elem; + size_t corner_gid = node_lid + elem_gid * num_nodes_in_elem; corners_in_node(node_gid, j) = corner_gid; - + elems_in_node(node_gid, j) = elem_gid; // save the elem_gid // Save corner index to element @@ -245,133 +230,127 @@ struct mesh_t { // increment the number of corners saved to this node_gid count_saved_corners_in_node(node_gid) = count_saved_corners_in_node(node_gid) + 1; - }); // end FOR_ALL over nodes in element } // end for elem_gid - + return; - } // end of build_corner_connectivity - - + // build elem connectivity arrays - void build_elem_elem_connectivity(){ - + void build_elem_elem_connectivity() + { // find the max number of elems around a node size_t max_num_elems_in_node; size_t max_num_lcl; REDUCE_MAX_CLASS(node_gid, 0, num_nodes, max_num_lcl, { - // num_corners_in_node = num_elems_in_node size_t max_num = num_corners_in_node(node_gid); - - if (max_num > max_num_lcl) max_num_lcl = max_num; - + + if (max_num > max_num_lcl) + { + max_num_lcl = max_num; + } }, max_num_elems_in_node); // end parallel reduction on max Kokkos::fence(); - + // a temporary ragged array to save the elems around an elem - DynamicRaggedRightArrayKokkos temp_elems_in_elem(num_nodes, num_nodes_in_elem*max_num_elems_in_node, "temp_elems_in_elem"); - - num_elems_in_elem = CArrayKokkos (num_elems, "num_elems_in_elem"); + DynamicRaggedRightArrayKokkos temp_elems_in_elem(num_nodes, num_nodes_in_elem * max_num_elems_in_node, "temp_elems_in_elem"); + + num_elems_in_elem = CArrayKokkos(num_elems, "num_elems_in_elem"); FOR_ALL_CLASS(elem_gid, 0, num_elems, { num_elems_in_elem(elem_gid) = 0; }); Kokkos::fence(); - + // find and save neighboring elem_gids of an elem FOR_ALL_CLASS(elem_gid, 0, num_elems, { - for (int node_lid=0; node_lid (num_elems_in_elem, "elems_in_elem"); - + elems_in_elem = RaggedRightArrayKokkos(num_elems_in_elem, "elems_in_elem"); + FOR_ALL_CLASS(elem_gid, 0, num_elems, { - for (size_t i=0; i node_ordering_in_elem; // node lids in a patch - - num_nodes_in_patch = 2*(num_dims-1); // 2 (2D) or 4 (3D) - num_patches_in_elem = 2*num_dims; // 4 (2D) or 6 (3D) - + DViewCArrayKokkos node_ordering_in_elem; // node lids in a patch + + num_nodes_in_patch = 2 * (num_dims - 1); // 2 (2D) or 4 (3D) + num_patches_in_elem = 2 * num_dims; // 4 (2D) or 6 (3D) + size_t node_lids_in_patch_in_elem[24]; - - if(num_dims == 3) { - size_t temp_node_lids[24] = {0,4,7,3, - 1,2,6,5, - 0,1,5,4, - 2,3,7,6, - 0,3,2,1, - 4,5,6,7}; - - for (size_t i=0; i<24; i++){ + + if (num_dims == 3) + { + size_t temp_node_lids[24] = { 0, 4, 7, 3, + 1, 2, 6, 5, + 0, 1, 5, 4, + 2, 3, 7, 6, + 0, 3, 2, 1, + 4, 5, 6, 7 }; + + for (size_t i = 0; i < 24; i++) + { node_lids_in_patch_in_elem[i] = temp_node_lids[i]; } // end for i - } - else { + else + { // J // | // 3---2 @@ -379,447 +358,409 @@ struct mesh_t { // 0---1 // size_t temp_node_lids[8] = - {0,3, - 1,2, - 0,1, - 3,2}; - - for (size_t i=0; i<8; i++){ + { 0, 3, + 1, 2, + 0, 1, + 3, 2 }; + + for (size_t i = 0; i < 8; i++) + { node_lids_in_patch_in_elem[i] = temp_node_lids[i]; } // end for i - } // end if on dims - - node_ordering_in_elem = DViewCArrayKokkos (&node_lids_in_patch_in_elem[0],num_patches_in_elem,num_nodes_in_patch); - - + + node_ordering_in_elem = DViewCArrayKokkos(&node_lids_in_patch_in_elem[0], num_patches_in_elem, num_nodes_in_patch); + // for saviong the hash keys of the patches and then the nighboring elem_gid - CArrayKokkos hash_keys_in_elem (num_elems, num_patches_in_elem, "hash_keys_in_elem"); - + CArrayKokkos hash_keys_in_elem(num_elems, num_patches_in_elem, "hash_keys_in_elem"); + // for saving the adjacient patch_lid, which is the slide_lid - //CArrayKokkos neighboring_side_lids (num_elems, num_patches_in_elem); - + // CArrayKokkos neighboring_side_lids (num_elems, num_patches_in_elem); + // allocate memory for the patches in the elem - patches_in_elem = CArrayKokkos (num_elems, num_patches_in_elem, "patches_in_elem"); - + patches_in_elem = CArrayKokkos(num_elems, num_patches_in_elem, "patches_in_elem"); + // a temporary storaage for the patch_gids that are on the mesh boundary - CArrayKokkos temp_bdy_patches(num_elems*num_patches_in_elem, "temp_bdy_patches"); - + CArrayKokkos temp_bdy_patches(num_elems * num_patches_in_elem, "temp_bdy_patches"); + // step 1) calculate the hash values for each patch in the element FOR_ALL_CLASS(elem_gid, 0, num_elems, { - - for (size_t patch_lid = 0; patch_lid num_values(2, "num_values"); - + + DCArrayKokkos num_values(2, "num_values"); + // step 2: walk around the elements and save the elem pairs that have the same hash_key RUN_CLASS({ // serial execution on GPU - - size_t patch_gid = 0; + + size_t patch_gid = 0; size_t bdy_patch_gid = 0; - for (size_t elem_gid = 0; elem_gid (num_patches, 2, "elems_in_patch"); - nodes_in_patch = CArrayKokkos (num_patches, num_nodes_in_patch, "nodes_in_patch"); - + + elems_in_patch = CArrayKokkos(num_patches, 2, "elems_in_patch"); + nodes_in_patch = CArrayKokkos(num_patches, num_nodes_in_patch, "nodes_in_patch"); + // a temporary variable to help populate patch structures - CArrayKokkos num_elems_in_patch_saved (num_patches, "num_elems_in_patch_saved"); - + CArrayKokkos num_elems_in_patch_saved(num_patches, "num_elems_in_patch_saved"); + // initialize the number of elems in a patch saved to zero FOR_ALL_CLASS(patch_gid, 0, num_patches, { num_elems_in_patch_saved(patch_gid) = 0; }); - - for(size_t elem_gid=0; elem_gid (num_bdy_patches, "bdy_patches"); - + bdy_patches = CArrayKokkos(num_bdy_patches, "bdy_patches"); + FOR_ALL_CLASS(bdy_patch_gid, 0, num_bdy_patches, { bdy_patches(bdy_patch_gid) = temp_bdy_patches(bdy_patch_gid); }); // end FOR_ALL bdy_patch_gid - - - - + // find and store the boundary nodes - CArrayKokkos temp_bdy_nodes(num_nodes, "temp_bdy_nodes"); - CArrayKokkos hash_bdy_nodes(num_nodes, "hash_bdy_nodes"); - - FOR_ALL_CLASS (node_gid, 0, num_nodes, { + CArrayKokkos temp_bdy_nodes(num_nodes, "temp_bdy_nodes"); + CArrayKokkos hash_bdy_nodes(num_nodes, "hash_bdy_nodes"); + + FOR_ALL_CLASS(node_gid, 0, num_nodes, { hash_bdy_nodes(node_gid) = -1; }); // end for node_gid - + // Parallel loop over boundary patches - DCArrayKokkos num_bdy_nodes_saved(1); - + DCArrayKokkos num_bdy_nodes_saved(1); + RUN_CLASS({ num_bdy_nodes_saved(0) = 0; - for (size_t bdy_patch_gid=0; bdy_patch_gid (num_bdy_nodes, "bdy_nodes"); - - FOR_ALL_CLASS (node_gid, 0, num_bdy_nodes, { + + bdy_nodes = CArrayKokkos(num_bdy_nodes, "bdy_nodes"); + + FOR_ALL_CLASS(node_gid, 0, num_bdy_nodes, { bdy_nodes(node_gid) = temp_bdy_nodes(node_gid); }); // end for boundary node_gid - + printf("Num boundary nodes = %lu \n", num_bdy_nodes); - + return; - } // end patch connectivity method - - - + // build the patches - void build_node_node_connectivity(){ - - + void build_node_node_connectivity() + { // find the max number of elems around a node size_t max_num_elems_in_node; size_t max_num_lcl; REDUCE_MAX_CLASS(node_gid, 0, num_nodes, max_num_lcl, { - // num_corners_in_node = num_elems_in_node size_t max_num = num_corners_in_node(node_gid); - - if (max_num > max_num_lcl) max_num_lcl = max_num; - + + if (max_num > max_num_lcl) + { + max_num_lcl = max_num; + } }, max_num_elems_in_node); // end parallel reduction on max Kokkos::fence(); - + // each elem corner will contribute 3 edges to the node. Those edges will likely be the same // ones from an adjacent element so it is a safe estimate to multiply by 3 - DynamicRaggedRightArrayKokkos temp_nodes_in_nodes(num_nodes, max_num_elems_in_node*3, "temp_nodes_in_nodes"); - - num_nodes_in_node = CArrayKokkos (num_nodes, "num_nodes_in_node"); - + DynamicRaggedRightArrayKokkos temp_nodes_in_nodes(num_nodes, max_num_elems_in_node * 3, "temp_nodes_in_nodes"); + + num_nodes_in_node = CArrayKokkos(num_nodes, "num_nodes_in_node"); + // walk over the patches and save the node node connectivity RUN_CLASS({ - if (num_dims==3){ - - for (size_t patch_gid=0; patch_gid (num_nodes_in_node, "nodes_in_node"); - + + nodes_in_node = RaggedRightArrayKokkos(num_nodes_in_node, "nodes_in_node"); + // save the connectivity FOR_ALL_CLASS(node_gid, 0, num_nodes, { - size_t num_saved = 0; - for (size_t node_lid=0; node_lid (num_bcs, num_bdy_patches, "bdy_patches_in_set"); - + num_bdy_sets = num_bcs; + bdy_patches_in_set = DynamicRaggedRightArrayKokkos(num_bcs, num_bdy_patches, "bdy_patches_in_set"); + return; - } // end of init_bdy_sets method - - - }; // end mesh_t -#endif \ No newline at end of file +#endif \ No newline at end of file diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/momentum.cpp b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/momentum.cpp index 9fc15a8b4..98e83b455 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/momentum.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/momentum.cpp @@ -5,7 +5,7 @@ // ----------------------------------------------------------------------------- // This function evolves the velocity at the nodes of the mesh -//------------------------------------------------------------------------------ +// ------------------------------------------------------------------------------ /* void FEA_Module_Dynamic_Elasticity::update_velocity_elastic(double rk_alpha, const mesh_t &mesh, @@ -14,9 +14,9 @@ void FEA_Module_Dynamic_Elasticity::update_velocity_elastic(double rk_alpha, const DViewCArrayKokkos &corner_force ){ - const size_t rk_level = simparam->rk_num_bins - 1; + const size_t rk_level = simparam->rk_num_bins - 1; const size_t num_dims = mesh.num_dims; - + // walk over the nodes to update the velocity FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { @@ -24,399 +24,374 @@ void FEA_Module_Dynamic_Elasticity::update_velocity_elastic(double rk_alpha, for (size_t dim = 0; dim < num_dims; dim++){ node_force[dim] = 0.0; } // end for dim - + // loop over all corners around the node and calculate the nodal force for (size_t corner_lid=0; corner_lid &vel_grad, - const ViewCArrayKokkos &elem_node_gids, - const DViewCArrayKokkos &node_vel, - const ViewCArrayKokkos &b_matrix, - const double elem_vol, - const size_t elem_gid, - const size_t rk_level - ) const { - +void FEA_Module_Dynamic_Elasticity::get_velgrad(ViewCArrayKokkos& vel_grad, + const ViewCArrayKokkos& elem_node_gids, + const DViewCArrayKokkos& node_vel, + const ViewCArrayKokkos& b_matrix, + const double elem_vol, + const size_t elem_gid, + const size_t rk_level + ) const +{ const size_t num_nodes_in_elem = 8; - - double u_array[num_nodes_in_elem]; - double v_array[num_nodes_in_elem]; - double w_array[num_nodes_in_elem]; - ViewCArrayKokkos u(u_array, num_nodes_in_elem); // x-dir vel component - ViewCArrayKokkos v(v_array, num_nodes_in_elem); // y-dir vel component - ViewCArrayKokkos w(w_array, num_nodes_in_elem); // z-dir vel component - + + double u_array[num_nodes_in_elem]; + double v_array[num_nodes_in_elem]; + double w_array[num_nodes_in_elem]; + ViewCArrayKokkos u(u_array, num_nodes_in_elem); // x-dir vel component + ViewCArrayKokkos v(v_array, num_nodes_in_elem); // y-dir vel component + ViewCArrayKokkos w(w_array, num_nodes_in_elem); // z-dir vel component + // get the vertex velocities for the cell - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { // Get node gid size_t node_gid = elem_node_gids(node_lid); u(node_lid) = node_vel(rk_level, node_gid, 0); v(node_lid) = node_vel(rk_level, node_gid, 1); w(node_lid) = node_vel(rk_level, node_gid, 2); - } // end for - // --- calculate the velocity gradient terms --- - double inverse_vol = 1.0/elem_vol; - + double inverse_vol = 1.0 / elem_vol; + // x-dir - vel_grad(0,0) = (u(0)*b_matrix(0,0) + u(1)*b_matrix(1,0) - + u(2)*b_matrix(2,0) + u(3)*b_matrix(3,0) - + u(4)*b_matrix(4,0) + u(5)*b_matrix(5,0) - + u(6)*b_matrix(6,0) + u(7)*b_matrix(7,0))*inverse_vol; - - vel_grad(0,1) = (u(0)*b_matrix(0,1) + u(1)*b_matrix(1,1) - + u(2)*b_matrix(2,1) + u(3)*b_matrix(3,1) - + u(4)*b_matrix(4,1) + u(5)*b_matrix(5,1) - + u(6)*b_matrix(6,1) + u(7)*b_matrix(7,1))*inverse_vol; - - vel_grad(0,2) = (u(0)*b_matrix(0,2) + u(1)*b_matrix(1,2) - + u(2)*b_matrix(2,2) + u(3)*b_matrix(3,2) - + u(4)*b_matrix(4,2) + u(5)*b_matrix(5,2) - + u(6)*b_matrix(6,2) + u(7)*b_matrix(7,2))*inverse_vol; - + vel_grad(0, 0) = (u(0) * b_matrix(0, 0) + u(1) * b_matrix(1, 0) + + u(2) * b_matrix(2, 0) + u(3) * b_matrix(3, 0) + + u(4) * b_matrix(4, 0) + u(5) * b_matrix(5, 0) + + u(6) * b_matrix(6, 0) + u(7) * b_matrix(7, 0)) * inverse_vol; + + vel_grad(0, 1) = (u(0) * b_matrix(0, 1) + u(1) * b_matrix(1, 1) + + u(2) * b_matrix(2, 1) + u(3) * b_matrix(3, 1) + + u(4) * b_matrix(4, 1) + u(5) * b_matrix(5, 1) + + u(6) * b_matrix(6, 1) + u(7) * b_matrix(7, 1)) * inverse_vol; + + vel_grad(0, 2) = (u(0) * b_matrix(0, 2) + u(1) * b_matrix(1, 2) + + u(2) * b_matrix(2, 2) + u(3) * b_matrix(3, 2) + + u(4) * b_matrix(4, 2) + u(5) * b_matrix(5, 2) + + u(6) * b_matrix(6, 2) + u(7) * b_matrix(7, 2)) * inverse_vol; + // y-dir - vel_grad(1,0) = (v(0)*b_matrix(0,0) + v(1)*b_matrix(1,0) - + v(2)*b_matrix(2,0) + v(3)*b_matrix(3,0) - + v(4)*b_matrix(4,0) + v(5)*b_matrix(5,0) - + v(6)*b_matrix(6,0) + v(7)*b_matrix(7,0))*inverse_vol; - - vel_grad(1,1) = (v(0)*b_matrix(0,1) + v(1)*b_matrix(1,1) - + v(2)*b_matrix(2,1) + v(3)*b_matrix(3,1) - + v(4)*b_matrix(4,1) + v(5)*b_matrix(5,1) - + v(6)*b_matrix(6,1) + v(7)*b_matrix(7,1))*inverse_vol; - vel_grad(1,2) = (v(0)*b_matrix(0,2) + v(1)*b_matrix(1,2) - + v(2)*b_matrix(2,2) + v(3)*b_matrix(3,2) - + v(4)*b_matrix(4,2) + v(5)*b_matrix(5,2) - + v(6)*b_matrix(6,2) + v(7)*b_matrix(7,2))*inverse_vol; - + vel_grad(1, 0) = (v(0) * b_matrix(0, 0) + v(1) * b_matrix(1, 0) + + v(2) * b_matrix(2, 0) + v(3) * b_matrix(3, 0) + + v(4) * b_matrix(4, 0) + v(5) * b_matrix(5, 0) + + v(6) * b_matrix(6, 0) + v(7) * b_matrix(7, 0)) * inverse_vol; + + vel_grad(1, 1) = (v(0) * b_matrix(0, 1) + v(1) * b_matrix(1, 1) + + v(2) * b_matrix(2, 1) + v(3) * b_matrix(3, 1) + + v(4) * b_matrix(4, 1) + v(5) * b_matrix(5, 1) + + v(6) * b_matrix(6, 1) + v(7) * b_matrix(7, 1)) * inverse_vol; + vel_grad(1, 2) = (v(0) * b_matrix(0, 2) + v(1) * b_matrix(1, 2) + + v(2) * b_matrix(2, 2) + v(3) * b_matrix(3, 2) + + v(4) * b_matrix(4, 2) + v(5) * b_matrix(5, 2) + + v(6) * b_matrix(6, 2) + v(7) * b_matrix(7, 2)) * inverse_vol; + // z-dir - vel_grad(2,0) = (w(0)*b_matrix(0,0) + w(1)*b_matrix(1,0) - + w(2)*b_matrix(2,0) + w(3)*b_matrix(3,0) - + w(4)*b_matrix(4,0) + w(5)*b_matrix(5,0) - + w(6)*b_matrix(6,0) + w(7)*b_matrix(7,0))*inverse_vol; - - vel_grad(2,1) = (w(0)*b_matrix(0,1) + w(1)*b_matrix(1,1) - + w(2)*b_matrix(2,1) + w(3)*b_matrix(3,1) - + w(4)*b_matrix(4,1) + w(5)*b_matrix(5,1) - + w(6)*b_matrix(6,1) + w(7)*b_matrix(7,1))*inverse_vol; - - vel_grad(2,2) = (w(0)*b_matrix(0,2) + w(1)*b_matrix(1,2) - + w(2)*b_matrix(2,2) + w(3)*b_matrix(3,2) - + w(4)*b_matrix(4,2) + w(5)*b_matrix(5,2) - + w(6)*b_matrix(6,2) + w(7)*b_matrix(7,2))*inverse_vol; - - + vel_grad(2, 0) = (w(0) * b_matrix(0, 0) + w(1) * b_matrix(1, 0) + + w(2) * b_matrix(2, 0) + w(3) * b_matrix(3, 0) + + w(4) * b_matrix(4, 0) + w(5) * b_matrix(5, 0) + + w(6) * b_matrix(6, 0) + w(7) * b_matrix(7, 0)) * inverse_vol; + + vel_grad(2, 1) = (w(0) * b_matrix(0, 1) + w(1) * b_matrix(1, 1) + + w(2) * b_matrix(2, 1) + w(3) * b_matrix(3, 1) + + w(4) * b_matrix(4, 1) + w(5) * b_matrix(5, 1) + + w(6) * b_matrix(6, 1) + w(7) * b_matrix(7, 1)) * inverse_vol; + + vel_grad(2, 2) = (w(0) * b_matrix(0, 2) + w(1) * b_matrix(1, 2) + + w(2) * b_matrix(2, 2) + w(3) * b_matrix(3, 2) + + w(4) * b_matrix(4, 2) + w(5) * b_matrix(5, 2) + + w(6) * b_matrix(6, 2) + w(7) * b_matrix(7, 2)) * inverse_vol; + return; - } // end function - // ----------------------------------------------------------------------------- // This function calculates the velocity gradient -//------------------------------------------------------------------------------ +// ------------------------------------------------------------------------------ KOKKOS_FUNCTION -void FEA_Module_Dynamic_Elasticity::get_velgrad2D(ViewCArrayKokkos &vel_grad, - const ViewCArrayKokkos &elem_node_gids, - const DViewCArrayKokkos &node_vel, - const ViewCArrayKokkos &b_matrix, - const double elem_vol, - const double elem_area, - const size_t elem_gid, - const size_t rk_level - ) const { - +void FEA_Module_Dynamic_Elasticity::get_velgrad2D(ViewCArrayKokkos& vel_grad, + const ViewCArrayKokkos& elem_node_gids, + const DViewCArrayKokkos& node_vel, + const ViewCArrayKokkos& b_matrix, + const double elem_vol, + const double elem_area, + const size_t elem_gid, + const size_t rk_level + ) const +{ const size_t num_nodes_in_elem = 4; - - double u_array[num_nodes_in_elem]; - double v_array[num_nodes_in_elem]; - ViewCArrayKokkos u(u_array, num_nodes_in_elem); // x-dir vel component - ViewCArrayKokkos v(v_array, num_nodes_in_elem); // y-dir vel component - + + double u_array[num_nodes_in_elem]; + double v_array[num_nodes_in_elem]; + ViewCArrayKokkos u(u_array, num_nodes_in_elem); // x-dir vel component + ViewCArrayKokkos v(v_array, num_nodes_in_elem); // y-dir vel component + // get the vertex velocities for the cell - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { // Get node gid size_t node_gid = elem_node_gids(node_lid); u(node_lid) = node_vel(rk_level, node_gid, 0); // x-comp v(node_lid) = node_vel(rk_level, node_gid, 1); // y-comp - } // end for - + // initialize to zero - for (size_t i=0; i<3; i++){ - for (size_t j=0; j<3; j++){ - vel_grad(i,j) = 0.0; + for (size_t i = 0; i < 3; i++) + { + for (size_t j = 0; j < 3; j++) + { + vel_grad(i, j) = 0.0; } } - - double mean_radius = elem_vol/elem_area; - double elem_vel_r = 0.25*(v(0) + v(1) + v(2) + v(3)); - - + double mean_radius = elem_vol / elem_area; + double elem_vel_r = 0.25 * (v(0) + v(1) + v(2) + v(3)); + // --- calculate the velocity gradient terms --- - double inverse_area = 1.0/elem_area; - + double inverse_area = 1.0 / elem_area; + // x-dir - vel_grad(0,0) = (u(0)*b_matrix(0,0) + u(1)*b_matrix(1,0) - + u(2)*b_matrix(2,0) + u(3)*b_matrix(3,0))*inverse_area; - - vel_grad(0,1) = (u(0)*b_matrix(0,1) + u(1)*b_matrix(1,1) - + u(2)*b_matrix(2,1) + u(3)*b_matrix(3,1))*inverse_area; - - + vel_grad(0, 0) = (u(0) * b_matrix(0, 0) + u(1) * b_matrix(1, 0) + + u(2) * b_matrix(2, 0) + u(3) * b_matrix(3, 0)) * inverse_area; + + vel_grad(0, 1) = (u(0) * b_matrix(0, 1) + u(1) * b_matrix(1, 1) + + u(2) * b_matrix(2, 1) + u(3) * b_matrix(3, 1)) * inverse_area; + // y-dir - vel_grad(1,0) = (v(0)*b_matrix(0,0) + v(1)*b_matrix(1,0) - + v(2)*b_matrix(2,0) + v(3)*b_matrix(3,0))*inverse_area; - - vel_grad(1,1) = (v(0)*b_matrix(0,1) + v(1)*b_matrix(1,1) - + v(2)*b_matrix(2,1) + v(3)*b_matrix(3,1))*inverse_area; - - vel_grad(2,2) = elem_vel_r/mean_radius; // + avg(vel_R)/R - - return; - -} // end function + vel_grad(1, 0) = (v(0) * b_matrix(0, 0) + v(1) * b_matrix(1, 0) + + v(2) * b_matrix(2, 0) + v(3) * b_matrix(3, 0)) * inverse_area; + vel_grad(1, 1) = (v(0) * b_matrix(0, 1) + v(1) * b_matrix(1, 1) + + v(2) * b_matrix(2, 1) + v(3) * b_matrix(3, 1)) * inverse_area; + vel_grad(2, 2) = elem_vel_r / mean_radius; // + avg(vel_R)/R + return; +} // end function // ----------------------------------------------------------------------------- // This subroutine to calculate the velocity divergence in all elements -//------------------------------------------------------------------------------ -void FEA_Module_Dynamic_Elasticity::get_divergence(DViewCArrayKokkos &elem_div, - const mesh_t mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_vol - ){ - +// ------------------------------------------------------------------------------ +void FEA_Module_Dynamic_Elasticity::get_divergence(DViewCArrayKokkos& elem_div, + const mesh_t mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_vol + ) +{ const size_t rk_level = rk_num_bins - 1; // --- calculate the forces acting on the nodes from the element --- - FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { - + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { const size_t num_nodes_in_elem = 8; const size_t num_dims = 3; - + double u_array[num_nodes_in_elem]; double v_array[num_nodes_in_elem]; double w_array[num_nodes_in_elem]; - ViewCArrayKokkos u(u_array, num_nodes_in_elem); // x-dir vel component - ViewCArrayKokkos v(v_array, num_nodes_in_elem); // y-dir vel component - ViewCArrayKokkos w(w_array, num_nodes_in_elem); // z-dir vel component - + ViewCArrayKokkos u(u_array, num_nodes_in_elem); // x-dir vel component + ViewCArrayKokkos v(v_array, num_nodes_in_elem); // y-dir vel component + ViewCArrayKokkos w(w_array, num_nodes_in_elem); // z-dir vel component + // cut out the node_gids for this element - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 8); - + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 8); + // The b_matrix are the outward corner area normals double b_matrix_array[24]; - ViewCArrayKokkos b_matrix(b_matrix_array, num_nodes_in_elem, num_dims); + ViewCArrayKokkos b_matrix(b_matrix_array, num_nodes_in_elem, num_dims); get_bmatrix(b_matrix, elem_gid, node_coords, elem_node_gids, rk_level); - + // get the vertex velocities for the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { // Get node gid size_t node_gid = elem_node_gids(node_lid); - + u(node_lid) = node_vel(rk_level, node_gid, 0); v(node_lid) = node_vel(rk_level, node_gid, 1); w(node_lid) = node_vel(rk_level, node_gid, 2); - } // end for - - + // --- calculate the velocity divergence terms --- - double inverse_vol = 1.0/elem_vol(elem_gid); - + double inverse_vol = 1.0 / elem_vol(elem_gid); + elem_div(elem_gid) = 0.0; - + // x-dir - elem_div(elem_gid) += (u(0)*b_matrix(0,0) + u(1)*b_matrix(1,0) - + u(2)*b_matrix(2,0) + u(3)*b_matrix(3,0) - + u(4)*b_matrix(4,0) + u(5)*b_matrix(5,0) - + u(6)*b_matrix(6,0) + u(7)*b_matrix(7,0))*inverse_vol; - + elem_div(elem_gid) += (u(0) * b_matrix(0, 0) + u(1) * b_matrix(1, 0) + + u(2) * b_matrix(2, 0) + u(3) * b_matrix(3, 0) + + u(4) * b_matrix(4, 0) + u(5) * b_matrix(5, 0) + + u(6) * b_matrix(6, 0) + u(7) * b_matrix(7, 0)) * inverse_vol; + // y-dir - elem_div(elem_gid) += (v(0)*b_matrix(0,1) + v(1)*b_matrix(1,1) - + v(2)*b_matrix(2,1) + v(3)*b_matrix(3,1) - + v(4)*b_matrix(4,1) + v(5)*b_matrix(5,1) - + v(6)*b_matrix(6,1) + v(7)*b_matrix(7,1))*inverse_vol; - + elem_div(elem_gid) += (v(0) * b_matrix(0, 1) + v(1) * b_matrix(1, 1) + + v(2) * b_matrix(2, 1) + v(3) * b_matrix(3, 1) + + v(4) * b_matrix(4, 1) + v(5) * b_matrix(5, 1) + + v(6) * b_matrix(6, 1) + v(7) * b_matrix(7, 1)) * inverse_vol; + // z-dir - elem_div(elem_gid) += (w(0)*b_matrix(0,2) + w(1)*b_matrix(1,2) - + w(2)*b_matrix(2,2) + w(3)*b_matrix(3,2) - + w(4)*b_matrix(4,2) + w(5)*b_matrix(5,2) - + w(6)*b_matrix(6,2) + w(7)*b_matrix(7,2))*inverse_vol; - + elem_div(elem_gid) += (w(0) * b_matrix(0, 2) + w(1) * b_matrix(1, 2) + + w(2) * b_matrix(2, 2) + w(3) * b_matrix(3, 2) + + w(4) * b_matrix(4, 2) + w(5) * b_matrix(5, 2) + + w(6) * b_matrix(6, 2) + w(7) * b_matrix(7, 2)) * inverse_vol; }); // end parallel for over elem_gid - + return; - } // end subroutine - // ----------------------------------------------------------------------------- // This subroutine to calculate the velocity divergence in all elements -//------------------------------------------------------------------------------ -void FEA_Module_Dynamic_Elasticity::get_divergence2D(DViewCArrayKokkos &elem_div, - const mesh_t mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_vol - ){ - +// ------------------------------------------------------------------------------ +void FEA_Module_Dynamic_Elasticity::get_divergence2D(DViewCArrayKokkos& elem_div, + const mesh_t mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_vol + ) +{ const size_t rk_level = rk_num_bins - 1; // --- calculate the forces acting on the nodes from the element --- - FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { - + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { const size_t num_nodes_in_elem = 4; const size_t num_dims = 2; - + double u_array[num_nodes_in_elem]; double v_array[num_nodes_in_elem]; - ViewCArrayKokkos u(u_array, num_nodes_in_elem); // x-dir vel component - ViewCArrayKokkos v(v_array, num_nodes_in_elem); // y-dir vel component - - + ViewCArrayKokkos u(u_array, num_nodes_in_elem); // x-dir vel component + ViewCArrayKokkos v(v_array, num_nodes_in_elem); // y-dir vel component + // true volume RZ - //double r_array[num_nodes_in_elem]; - //ViewCArrayKokkos r(r_array, num_nodes_in_elem); // r-dir coordinate - - + // double r_array[num_nodes_in_elem]; + // ViewCArrayKokkos r(r_array, num_nodes_in_elem); // r-dir coordinate + // cut out the node_gids for this element - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 4); - + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 4); + // The b_matrix are the outward corner area normals double b_matrix_array[24]; - ViewCArrayKokkos b_matrix(b_matrix_array, num_nodes_in_elem, num_dims); + ViewCArrayKokkos b_matrix(b_matrix_array, num_nodes_in_elem, num_dims); get_bmatrix2D(b_matrix, elem_gid, node_coords, elem_node_gids, rk_level); - + // calculate the area of the quad double elem_area = get_area_quad(elem_gid, node_coords, elem_node_gids, rk_level); // true volume uses the elem_vol - - + // get the vertex velocities and node coordinate for the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { // Get node gid size_t node_gid = elem_node_gids(node_lid); - + u(node_lid) = node_vel(rk_level, node_gid, 0); v(node_lid) = node_vel(rk_level, node_gid, 1); - - //r(node_lid) = node_coords(rk_level, node_gid, 1); // true volume RZ - + + // r(node_lid) = node_coords(rk_level, node_gid, 1); // true volume RZ } // end for - - + // --- calculate the velocity divergence terms --- - double inverse_area = 1.0/elem_area; - - double mean_radius = elem_vol(elem_gid)/elem_area; - double elem_vel_r = 0.25*(v(0) + v(1) + v(2) + v(3)); - - + double inverse_area = 1.0 / elem_area; + + double mean_radius = elem_vol(elem_gid) / elem_area; + double elem_vel_r = 0.25 * (v(0) + v(1) + v(2) + v(3)); + elem_div(elem_gid) = 0.0; - + // x-dir - elem_div(elem_gid) += (u(0)*b_matrix(0,0) - + u(1)*b_matrix(1,0) - + u(2)*b_matrix(2,0) - + u(3)*b_matrix(3,0))*inverse_area; - - // y-dir (i.e., r direction) - elem_div(elem_gid) += (v(0)*b_matrix(0,1) - + v(1)*b_matrix(1,1) - + v(2)*b_matrix(2,1) - + v(3)*b_matrix(3,1))*inverse_area - + elem_vel_r/mean_radius; // + avg(u_R)/R + elem_div(elem_gid) += (u(0) * b_matrix(0, 0) + + u(1) * b_matrix(1, 0) + + u(2) * b_matrix(2, 0) + + u(3) * b_matrix(3, 0)) * inverse_area; + // y-dir (i.e., r direction) + elem_div(elem_gid) += (v(0) * b_matrix(0, 1) + + v(1) * b_matrix(1, 1) + + v(2) * b_matrix(2, 1) + + v(3) * b_matrix(3, 1)) * inverse_area + + elem_vel_r / mean_radius; // + avg(u_R)/R }); // end parallel for over elem_gid - + return; - } // end subroutine - // The velocity gradient can be decomposed into symmetric and antisymmetric tensors // L = vel_grad // D = sym(L) // W = antisym(L) KOKKOS_INLINE_FUNCTION -void FEA_Module_Dynamic_Elasticity::decompose_vel_grad(ViewCArrayKokkos &D_tensor, - ViewCArrayKokkos &W_tensor, - const ViewCArrayKokkos &vel_grad, - const ViewCArrayKokkos &elem_node_gids, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const double vol - ) const { - - +void FEA_Module_Dynamic_Elasticity::decompose_vel_grad(ViewCArrayKokkos& D_tensor, + ViewCArrayKokkos& W_tensor, + const ViewCArrayKokkos& vel_grad, + const ViewCArrayKokkos& elem_node_gids, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const double vol + ) const +{ // --- Calculate the velocity gradient --- - + const size_t num_dims = 3; - + // initialize to zero - for(size_t i=0; i &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &elem_den, - DViewCArrayKokkos &elem_pres, - DViewCArrayKokkos &elem_stress, - DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_mass, - const DViewCArrayKokkos &elem_mat_id, - const double rk_alpha, - const size_t cycle - ){ - +void FEA_Module_Dynamic_Elasticity::update_state(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_den, + DViewCArrayKokkos& elem_pres, + DViewCArrayKokkos& elem_stress, + DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& elem_mat_id, + const double rk_alpha, + const size_t cycle + ) +{ const size_t rk_level = rk_num_bins - 1; - int num_dims = num_dim; - + int num_dims = num_dim; + // loop over all the elements in the mesh - FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { - + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { const size_t num_dims = num_dims; const size_t num_nodes_in_elem = num_nodes_in_elem; // cut out the node_gids for this element - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), num_nodes_in_elem); - + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), num_nodes_in_elem); + // --- Density --- - elem_den(elem_gid) = elem_mass(elem_gid)/elem_vol(elem_gid); - + elem_den(elem_gid) = elem_mass(elem_gid) / elem_vol(elem_gid); + size_t mat_id = elem_mat_id(elem_gid); - //initialize elem pressure + // initialize elem pressure elem_pres(elem_gid) = 0; - + // --- Stress --- // hyper elastic plastic model - if(material(mat_id).strength_type == STRENGTH_TYPE::hyper){ - + if (material(mat_id).strength_type == STRENGTH_TYPE::hyper) + { // cut out the node_gids for this element - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), num_nodes_in_elem); - + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), num_nodes_in_elem); + // --- Density --- - elem_den(elem_gid) = elem_mass(elem_gid)/elem_vol(elem_gid); - + elem_den(elem_gid) = elem_mass(elem_gid) / elem_vol(elem_gid); + // corner area normals double area_array[24]; - ViewCArrayKokkos area(area_array, num_nodes_in_elem, num_dims); - + ViewCArrayKokkos area(area_array, num_nodes_in_elem, num_dims); + // velocity gradient double vel_grad_array[9]; - ViewCArrayKokkos vel_grad(vel_grad_array, num_dims, num_dims); - + ViewCArrayKokkos vel_grad(vel_grad_array, num_dims, num_dims); + // get the B matrix which are the OUTWARD corner area normals get_bmatrix(area, elem_gid, node_coords, elem_node_gids, rk_level); - - + // --- Calculate the velocity gradient --- get_velgrad(vel_grad, elem_node_gids, @@ -76,16 +74,18 @@ void FEA_Module_Dynamic_Elasticity::update_state(const DCArrayKokkos calc_stress(elem_pres, elem_stress, @@ -106,7 +106,6 @@ void FEA_Module_Dynamic_Elasticity::update_state(const DCArrayKokkos calc_sound_speed(elem_pres, @@ -133,79 +132,73 @@ void FEA_Module_Dynamic_Elasticity::update_state(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &elem_den, - DViewCArrayKokkos &elem_pres, - DViewCArrayKokkos &elem_stress, - DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_mass, - const DViewCArrayKokkos &elem_mat_id, - const double rk_alpha, - const size_t cycle - ){ - +void FEA_Module_Dynamic_Elasticity::update_state2D(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_den, + DViewCArrayKokkos& elem_pres, + DViewCArrayKokkos& elem_stress, + DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& elem_mat_id, + const double rk_alpha, + const size_t cycle + ) +{ const size_t rk_level = rk_num_bins - 1; - int num_dims = num_dim; - + int num_dims = num_dim; + // loop over all the elements in the mesh - FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { - + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { const size_t num_dims = num_dims; const size_t num_nodes_in_elem = num_nodes_in_elem; // cut out the node_gids for this element - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), num_nodes_in_elem); - + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), num_nodes_in_elem); + // --- Density --- - elem_den(elem_gid) = elem_mass(elem_gid)/elem_vol(elem_gid); - + elem_den(elem_gid) = elem_mass(elem_gid) / elem_vol(elem_gid); + size_t mat_id = elem_mat_id(elem_gid); - //initialize elem pressure + // initialize elem pressure elem_pres(elem_gid) = 0; - + // --- Stress --- // hyper elastic plastic model - if(material(mat_id).strength_type == STRENGTH_TYPE::hyper){ - + if (material(mat_id).strength_type == STRENGTH_TYPE::hyper) + { // cut out the node_gids for this element - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), num_nodes_in_elem); - + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), num_nodes_in_elem); + // --- Density --- - elem_den(elem_gid) = elem_mass(elem_gid)/elem_vol(elem_gid); - + elem_den(elem_gid) = elem_mass(elem_gid) / elem_vol(elem_gid); + // corner area normals double area_array[8]; - ViewCArrayKokkos area(area_array, num_nodes_in_elem, num_dims); - + ViewCArrayKokkos area(area_array, num_nodes_in_elem, num_dims); + // velocity gradient double vel_grad_array[4]; - ViewCArrayKokkos vel_grad(vel_grad_array, num_dims, num_dims); - + ViewCArrayKokkos vel_grad(vel_grad_array, num_dims, num_dims); + // get the B matrix which are the OUTWARD corner area normals get_bmatrix(area, elem_gid, node_coords, elem_node_gids, rk_level); - - + // --- Calculate the velocity gradient --- get_velgrad(vel_grad, elem_node_gids, @@ -214,7 +207,7 @@ void FEA_Module_Dynamic_Elasticity::update_state2D(const DCArrayKokkos calc_sound_speed(elem_pres, @@ -264,14 +256,9 @@ void FEA_Module_Dynamic_Elasticity::update_state2D(const DCArrayKokkos coords; + DCArrayKokkos coords; // velocity - DCArrayKokkos vel; + DCArrayKokkos vel; // mass at nodes - DCArrayKokkos mass; + DCArrayKokkos mass; // Includes Ghost Positions - //DCArrayKokkos all_coords; + // DCArrayKokkos all_coords; // Includes Ghost velocities - //DCArrayKokkos all_vel; + // DCArrayKokkos all_vel; // Includes Ghost masses - //DCArrayKokkos all_mass; + // DCArrayKokkos all_mass; - // initialization method (num_rk_storage_bins, num_nodes, num_dims) void initialize(size_t num_rk, size_t num_nodes, size_t num_dims) { - this->coords = DCArrayKokkos (num_rk, num_nodes, num_dims); - this->vel = DCArrayKokkos (num_rk, num_nodes, num_dims); - this->mass = DCArrayKokkos (num_nodes); + this->coords = DCArrayKokkos(num_rk, num_nodes, num_dims); + this->vel = DCArrayKokkos(num_rk, num_nodes, num_dims); + this->mass = DCArrayKokkos(num_nodes); }; // end method - }; // end node_t - // elem_state -struct elem_t { - +struct elem_t +{ // den - CArray den; - + CArray den; + // pres - CArray pres; - + CArray pres; + // stress - CArray stress; - + CArray stress; + // sspd - CArray sspd; - + CArray sspd; + // sie - CArray sie; + CArray sie; // vol - CArray vol; - + CArray vol; + // divergence of velocity - CArray div; - + CArray div; + // mass of elem - CArray mass; - + CArray mass; + // mat ids - CArray mat_id; - + CArray mat_id; + // initialization method (num_rk_storage_bins, num_cells, num_dims) void initialize(size_t num_rk, size_t num_elems, size_t num_dims) { - this->den = CArray (num_elems); - this->pres = CArray (num_elems); - this->stress = CArray (num_rk, num_elems, num_dims, num_dims); - this->sspd = CArray (num_elems); - this->sie = CArray (num_rk, num_elems); - this->vol = CArray (num_elems); - this->div = CArray (num_elems); - this->mass = CArray (num_elems); - this->mat_id = CArray (num_elems); + this->den = CArray(num_elems); + this->pres = CArray(num_elems); + this->stress = CArray(num_rk, num_elems, num_dims, num_dims); + this->sspd = CArray(num_elems); + this->sie = CArray(num_rk, num_elems); + this->vol = CArray(num_elems); + this->div = CArray(num_elems); + this->mass = CArray(num_elems); + this->mat_id = CArray(num_elems); }; // end method - }; // end elem_t - // corner_state -struct corner_t { - +struct corner_t +{ // force - CArray force; - + CArray force; + // mass of corner - CArray mass; + CArray mass; - // initialization method (num_corners, num_dims) void initialize(size_t num_corners, size_t num_dims) { - this->force = CArray (num_corners, num_dims); - this->mass = CArray (num_corners); + this->force = CArray(num_corners, num_dims); + this->mass = CArray(num_corners); }; // end method - }; // end corner_t -#endif +#endif diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/time_integration.cpp b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/time_integration.cpp index 5f79d7b6b..76d9eda73 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/time_integration.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/time_integration.cpp @@ -5,83 +5,75 @@ // ----------------------------------------------------------------------------- // This function saves the variables at rk_stage = 0, which is t_n -//------------------------------------------------------------------------------ -void FEA_Module_Dynamic_Elasticity::rk_init(DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &elem_sie, - DViewCArrayKokkos &elem_stress, - const size_t num_elems, - const size_t num_nodes){ - +// ------------------------------------------------------------------------------ +void FEA_Module_Dynamic_Elasticity::rk_init(DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_sie, + DViewCArrayKokkos& elem_stress, + const size_t num_elems, + const size_t num_nodes) +{ const size_t rk_level = rk_num_bins - 1; - int num_dims = num_dim; + int num_dims = num_dim; // save elem quantities FOR_ALL_CLASS(elem_gid, 0, num_elems, { - // stress is always 3D even with 2D-RZ - for(size_t i=0; i<3; i++){ - for(size_t j=0; j<3; j++){ - elem_stress(0,elem_gid,i,j) = elem_stress(rk_level,elem_gid,i,j); + for (size_t i = 0; i < 3; i++) + { + for (size_t j = 0; j < 3; j++) + { + elem_stress(0, elem_gid, i, j) = elem_stress(rk_level, elem_gid, i, j); } } // end for - elem_sie(0,elem_gid) = elem_sie(rk_level,elem_gid); - + elem_sie(0, elem_gid) = elem_sie(rk_level, elem_gid); }); // end parallel for - - + // save nodal quantities FOR_ALL_CLASS(node_gid, 0, num_nodes, { - - for(size_t i=0; i &node_coords, - DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &elem_sspd, - DViewCArrayKokkos &elem_vol){ - - +void FEA_Module_Dynamic_Elasticity::get_timestep(mesh_t& mesh, + DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_sspd, + DViewCArrayKokkos& elem_vol) +{ const size_t rk_level = rk_num_bins - 1; // increase dt by 10%, that is the largest dt value - dt = dt*1.1; - int num_dims = num_dim; + dt = dt * 1.1; + int num_dims = num_dim; double dt_lcl; double min_dt_calc; REDUCE_MIN_CLASS(elem_gid, 0, rnum_elem, dt_lcl, { - double coords0[24]; // element coords - ViewCArrayKokkos coords(coords0, 8, 3); + ViewCArrayKokkos coords(coords0, 8, 3); - double distance0[28]; // array for holding distances between each node - ViewCArrayKokkos dist(distance0, 28); - - // Getting the coordinates of the element - for(size_t node_lid = 0; node_lid < 8; node_lid++){ + ViewCArrayKokkos dist(distance0, 28); - for (size_t dim = 0; dim < num_dims; dim++){ - coords(node_lid, dim) = node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), dim); + // Getting the coordinates of the element + for (size_t node_lid = 0; node_lid < 8; node_lid++) + { + for (size_t dim = 0; dim < num_dims; dim++) + { + coords(node_lid, dim) = node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), dim); } // end for dim - } // end for loop over node_lid // loop conditions needed for distance calculation @@ -90,148 +82,150 @@ void FEA_Module_Dynamic_Elasticity::get_timestep(mesh_t &mesh, size_t a; size_t b; size_t loop = 0; - + // Only works for 3D // Solving for the magnitude of distance between each node - for (size_t i = 0; i < 28; i++){ - + for (size_t i = 0; i < 28; i++) + { a = countA; b = countB; - + // returns magnitude of distance between each node, 28 total options - dist(i) = fabs(sqrt(( pow((coords(b, 0) - coords(a, 0)), 2.0) - + pow((coords(b, 1) - coords(a, 1)), 2.0) - + pow((coords(b, 2) - coords(a, 2)), 2.0)))); + dist(i) = fabs(sqrt((pow((coords(b, 0) - coords(a, 0)), 2.0) + + pow((coords(b, 1) - coords(a, 1)), 2.0) + + pow((coords(b, 2) - coords(a, 2)), 2.0)))); countB++; countA++; - + // tricky indexing - if (countB > 7) { + if (countB > 7) + { loop++; countB = 1 + loop; countA = 0; } - } // endo for i - double dist_min = dist(0); - - for(int i = 0; i < 28; ++i){ + + for (int i = 0; i < 28; ++i) + { dist_min = fmin(dist(i), dist_min); } - + // local dt calc based on CFL - double dt_lcl_ = dt_cfl*dist_min/(elem_sspd(elem_gid) + fuzz); - + double dt_lcl_ = dt_cfl * dist_min / (elem_sspd(elem_gid) + fuzz); + // make dt be in bounds dt_lcl_ = fmin(dt_lcl_, dt_max); // make dt small than dt_max dt_lcl_ = fmax(dt_lcl_, dt_min); // make dt larger than dt_min - - - if (dt_lcl_ < dt_lcl) dt_lcl = dt_lcl_; - + + if (dt_lcl_ < dt_lcl) + { + dt_lcl = dt_lcl_; + } }, min_dt_calc); // end parallel reduction Kokkos::fence(); - + // save the min dt - if(min_dt_calc < dt) dt = min_dt_calc; - + if (min_dt_calc < dt) + { + dt = min_dt_calc; + } + // ensure time step hits the graphics time intervals - dt = fmin(dt, (graphics_time - time_value)+fuzz); - + dt = fmin(dt, (graphics_time - time_value) + fuzz); + // make dt be exact for final time - dt = fmin(dt, time_final-time_value); - + dt = fmin(dt, time_final - time_value); + return; - } // end get_timestep // ----------------------------------------------------------------------------- // This function calculates the time step by finding the shortest distance // between any two nodes in the mesh -//------------------------------------------------------------------------------ +// ------------------------------------------------------------------------------ // WARNING WARNING : Only works for 3D, 8 node elements -void FEA_Module_Dynamic_Elasticity::get_timestep2D(mesh_t &mesh, - DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &elem_sspd, - DViewCArrayKokkos &elem_vol){ - +void FEA_Module_Dynamic_Elasticity::get_timestep2D(mesh_t& mesh, + DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_sspd, + DViewCArrayKokkos& elem_vol) +{ const size_t rk_level = rk_num_bins - 1; // increase dt by 10%, that is the largest dt value - dt = dt*1.1; - int num_dims = num_dim; + dt = dt * 1.1; + int num_dims = num_dim; double dt_lcl; double min_dt_calc; REDUCE_MIN_CLASS(elem_gid, 0, rnum_elem, dt_lcl, { - double coords0[8]; // element coords - ViewCArrayKokkos coords(coords0, 4, 2); + ViewCArrayKokkos coords(coords0, 4, 2); - double distance0[6]; // array for holding distances between each node - ViewCArrayKokkos dist(distance0, 6); - - // Getting the coordinates of the nodes of the element - for(size_t node_lid = 0; node_lid < 4; node_lid++){ + ViewCArrayKokkos dist(distance0, 6); - for (size_t dim = 0; dim < num_dims; dim++){ - coords(node_lid, dim) = node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), dim); + // Getting the coordinates of the nodes of the element + for (size_t node_lid = 0; node_lid < 4; node_lid++) + { + for (size_t dim = 0; dim < num_dims; dim++) + { + coords(node_lid, dim) = node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), dim); } // end for dim - } // end for loop over node_lid - // Only works for 2D // Solving for the magnitude of distance between each node size_t count = 0; - for (size_t i = 0; i < 3; i++){ - for (size_t j=i+1; j<=3; j++){ - + for (size_t i = 0; i < 3; i++) + { + for (size_t j = i + 1; j <= 3; j++) + { // returns magnitude of distance between each node, 6 total options dist(count) = fabs( - sqrt( pow((coords(i, 0) - coords(j, 0)), 2.0) - + pow((coords(i, 1) - coords(j, 1)), 2.0) ) - ); - count ++; + sqrt(pow((coords(i, 0) - coords(j, 0)), 2.0) + + pow((coords(i, 1) - coords(j, 1)), 2.0) ) + ); + count++; } // end for j } // end for i - double dist_min = dist(0); - - for(int i = 0; i < 6; ++i){ + + for (int i = 0; i < 6; ++i) + { dist_min = fmin(dist(i), dist_min); } - + // local dt calc based on CFL - double dt_lcl_ = dt_cfl*dist_min/(elem_sspd(elem_gid) + fuzz); - + double dt_lcl_ = dt_cfl * dist_min / (elem_sspd(elem_gid) + fuzz); + // make dt be in bounds dt_lcl_ = fmin(dt_lcl_, dt_max); // make dt small than dt_max dt_lcl_ = fmax(dt_lcl_, dt_min); // make dt larger than dt_min - - - if (dt_lcl_ < dt_lcl) dt_lcl = dt_lcl_; - + + if (dt_lcl_ < dt_lcl) + { + dt_lcl = dt_lcl_; + } }, min_dt_calc); // end parallel reduction Kokkos::fence(); - + // save the min dt - if(min_dt_calc < dt) dt = min_dt_calc; - + if (min_dt_calc < dt) + { + dt = min_dt_calc; + } + // ensure time step hits the graphics time intervals - dt = fmin(dt, (graphics_time - time_value)+fuzz); - + dt = fmin(dt, (graphics_time - time_value) + fuzz); + // make dt be exact for final time - dt = fmin(dt, time_final-time_value); - + dt = fmin(dt, time_final - time_value); + return; - } // end get_timestep2D - - diff --git a/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/FEA_Module_Eulerian.cpp b/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/FEA_Module_Eulerian.cpp index 6602f6c77..d6319aa08 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/FEA_Module_Eulerian.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/FEA_Module_Eulerian.cpp @@ -11,14 +11,14 @@ This program is open source under the BSD-3 License. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - + 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - + 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - + 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. @@ -40,7 +40,7 @@ #include #include #include -#include +#include #include // fmin, fmax, abs note: fminl is long #include #include @@ -76,7 +76,7 @@ #include "FEA_Module_Eulerian.h" #include "Explicit_Solver_Eulerian.h" -//optimization +// optimization #include "ROL_Algorithm.hpp" #include "ROL_Solver.hpp" #include "ROL_LineSearchStep.hpp" @@ -100,617 +100,668 @@ using namespace utils; +FEA_Module_Eulerian::FEA_Module_Eulerian(Solver* Solver_Pointer, mesh_t& mesh, const int my_fea_module_index) : FEA_Module(Solver_Pointer), mesh(mesh) +{ + // assign interfacing index + my_fea_module_index_ = my_fea_module_index; + + // recast solver pointer for non-base class access + Explicit_Solver_Pointer_ = dynamic_cast(Solver_Pointer); + + // create parameter object + simparam = &Explicit_Solver_Pointer_->simparam; + // ---- Read input file, define state and boundary conditions ---- // + // simparam->input(); + + // TO parameters + simparam_dynamic_opt = Explicit_Solver_Pointer_->simparam_dynamic_opt; + + // create ref element object + // ref_elem = new elements::ref_element(); + // create mesh objects + // init_mesh = new swage::mesh_t(simparam); + // mesh = new swage::mesh_t(simparam); + + // boundary condition data + max_boundary_sets = 0; + Local_Index_Boundary_Patches = Explicit_Solver_Pointer_->Local_Index_Boundary_Patches; + + // set Tpetra vector pointers + initial_node_velocities_distributed = Explicit_Solver_Pointer_->initial_node_velocities_distributed; + initial_node_coords_distributed = Explicit_Solver_Pointer_->initial_node_coords_distributed; + all_initial_node_coords_distributed = Explicit_Solver_Pointer_->all_initial_node_coords_distributed; + node_coords_distributed = Explicit_Solver_Pointer_->node_coords_distributed; + node_velocities_distributed = Explicit_Solver_Pointer_->node_velocities_distributed; + all_node_velocities_distributed = Explicit_Solver_Pointer_->all_node_velocities_distributed; + if (simparam_dynamic_opt.topology_optimization_on || simparam_dynamic_opt.shape_optimization_on) + { + all_cached_node_velocities_distributed = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + force_gradient_velocity = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + force_gradient_position = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + force_gradient_design = Teuchos::rcp(new MV(all_node_map, 1)); + corner_value_storage = Solver_Pointer->corner_value_storage; + corner_vector_storage = Solver_Pointer->corner_vector_storage; + relative_element_densities = DCArrayKokkos(rnum_elem, "relative_element_densities"); + } + + if (simparam_dynamic_opt.topology_optimization_on || simparam_dynamic_opt.shape_optimization_on || simparam->num_dims == 2) + { + node_masses_distributed = Teuchos::rcp(new MV(map, 1)); + ghost_node_masses_distributed = Teuchos::rcp(new MV(ghost_node_map, 1)); + adjoint_vector_distributed = Teuchos::rcp(new MV(map, simparam->num_dims)); + phi_adjoint_vector_distributed = Teuchos::rcp(new MV(map, simparam->num_dims)); + } + + // setup output + noutput = 0; + init_output(); -FEA_Module_Eulerian::FEA_Module_Eulerian(Solver *Solver_Pointer, mesh_t& mesh, const int my_fea_module_index) :FEA_Module(Solver_Pointer), mesh(mesh){ - - //assign interfacing index - my_fea_module_index_ = my_fea_module_index; - - //recast solver pointer for non-base class access - Explicit_Solver_Pointer_ = dynamic_cast(Solver_Pointer); - - //create parameter object - simparam = &Explicit_Solver_Pointer_->simparam; - // ---- Read input file, define state and boundary conditions ---- // - //simparam->input(); - - //TO parameters - simparam_dynamic_opt = Explicit_Solver_Pointer_->simparam_dynamic_opt; - - //create ref element object - //ref_elem = new elements::ref_element(); - //create mesh objects - //init_mesh = new swage::mesh_t(simparam); - //mesh = new swage::mesh_t(simparam); - - //boundary condition data - max_boundary_sets = 0; - Local_Index_Boundary_Patches = Explicit_Solver_Pointer_->Local_Index_Boundary_Patches; - - //set Tpetra vector pointers - initial_node_velocities_distributed = Explicit_Solver_Pointer_->initial_node_velocities_distributed; - initial_node_coords_distributed = Explicit_Solver_Pointer_->initial_node_coords_distributed; - all_initial_node_coords_distributed = Explicit_Solver_Pointer_->all_initial_node_coords_distributed; - node_coords_distributed = Explicit_Solver_Pointer_->node_coords_distributed; - node_velocities_distributed = Explicit_Solver_Pointer_->node_velocities_distributed; - all_node_velocities_distributed = Explicit_Solver_Pointer_->all_node_velocities_distributed; - if(simparam_dynamic_opt.topology_optimization_on||simparam_dynamic_opt.shape_optimization_on){ - all_cached_node_velocities_distributed = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - force_gradient_velocity = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - force_gradient_position = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - force_gradient_design = Teuchos::rcp(new MV(all_node_map, 1)); - corner_value_storage = Solver_Pointer->corner_value_storage; - corner_vector_storage = Solver_Pointer->corner_vector_storage; - relative_element_densities = DCArrayKokkos(rnum_elem, "relative_element_densities"); - } - - if(simparam_dynamic_opt.topology_optimization_on||simparam_dynamic_opt.shape_optimization_on||simparam->num_dims==2){ - node_masses_distributed = Teuchos::rcp(new MV(map, 1)); - ghost_node_masses_distributed = Teuchos::rcp(new MV(ghost_node_map, 1)); - adjoint_vector_distributed = Teuchos::rcp(new MV(map, simparam->num_dims)); - phi_adjoint_vector_distributed = Teuchos::rcp(new MV(map, simparam->num_dims)); - } - - //setup output - noutput = 0; - init_output(); - - //optimization flags - kinetic_energy_objective = false; - - - //set parameters - Time_Variables tv = simparam->time_variables; - time_value = simparam->time_value; - time_final = tv.time_final; - dt_max = tv.dt_max; - dt_min = tv.dt_min; - dt_cfl = tv.dt_cfl; - graphics_time = simparam->graphics_options.graphics_time; - graphics_cyc_ival = simparam->graphics_options.graphics_cyc_ival; - graphics_dt_ival = simparam->graphics_options.graphics_dt_ival; - cycle_stop = tv.cycle_stop; - rk_num_stages = simparam->rk_num_stages; - dt = tv.dt; - fuzz = tv.fuzz; - tiny = tv.tiny; - small = tv.small; - graphics_times = simparam->graphics_options.graphics_times; - graphics_id = simparam->graphics_options.graphics_id; - - if(simparam_dynamic_opt.topology_optimization_on){ - max_time_steps = BUFFER_GROW; - forward_solve_velocity_data = Teuchos::rcp(new std::vector>(max_time_steps+1)); - time_data.resize(max_time_steps+1); - forward_solve_coordinate_data = Teuchos::rcp(new std::vector>(max_time_steps+1)); - adjoint_vector_data = Teuchos::rcp(new std::vector>(max_time_steps+1)); - phi_adjoint_vector_data = Teuchos::rcp(new std::vector>(max_time_steps+1)); - //assign a multivector of corresponding size to each new timestep in the buffer - for(int istep = 0; istep < max_time_steps+1; istep++){ - (*forward_solve_velocity_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - (*forward_solve_coordinate_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - (*adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - (*phi_adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + // optimization flags + kinetic_energy_objective = false; + + // set parameters + Time_Variables tv = simparam->time_variables; + time_value = simparam->time_value; + time_final = tv.time_final; + dt_max = tv.dt_max; + dt_min = tv.dt_min; + dt_cfl = tv.dt_cfl; + graphics_time = simparam->graphics_options.graphics_time; + graphics_cyc_ival = simparam->graphics_options.graphics_cyc_ival; + graphics_dt_ival = simparam->graphics_options.graphics_dt_ival; + cycle_stop = tv.cycle_stop; + rk_num_stages = simparam->rk_num_stages; + dt = tv.dt; + fuzz = tv.fuzz; + tiny = tv.tiny; + small = tv.small; + graphics_times = simparam->graphics_options.graphics_times; + graphics_id = simparam->graphics_options.graphics_id; + + if (simparam_dynamic_opt.topology_optimization_on) + { + max_time_steps = BUFFER_GROW; + forward_solve_velocity_data = Teuchos::rcp(new std::vector>(max_time_steps + 1)); + time_data.resize(max_time_steps + 1); + forward_solve_coordinate_data = Teuchos::rcp(new std::vector>(max_time_steps + 1)); + adjoint_vector_data = Teuchos::rcp(new std::vector>(max_time_steps + 1)); + phi_adjoint_vector_data = Teuchos::rcp(new std::vector>(max_time_steps + 1)); + // assign a multivector of corresponding size to each new timestep in the buffer + for (int istep = 0; istep < max_time_steps + 1; istep++) + { + (*forward_solve_velocity_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*forward_solve_coordinate_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*phi_adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + } } - - } } -FEA_Module_Eulerian::~FEA_Module_Eulerian(){ - //delete simparam; +FEA_Module_Eulerian::~FEA_Module_Eulerian() +{ + // delete simparam; } /* ---------------------------------------------------------------------- Read ANSYS dat format mesh file ------------------------------------------------------------------------- */ -void FEA_Module_Eulerian::read_conditions_ansys_dat(std::ifstream *in, std::streampos before_condition_header){ - - char ch; - int num_dim = simparam->num_dims; - int buffer_lines = 1000; - int max_word = 30; - int p_order = simparam->p_order; - real_t unit_scaling = simparam->get_unit_scaling(); - int local_node_index, current_column_index; - size_t strain_count; - std::string skip_line, read_line, substring, token; - std::stringstream line_parse, line_parse2; - CArrayKokkos read_buffer; - CArrayKokkos read_buffer_indices; - int buffer_loop, buffer_iteration, buffer_iterations, scan_loop, nodes_per_element, words_per_line; - size_t read_index_start, node_rid, elem_gid; - LO local_dof_id; - GO node_gid; - real_t dof_value; - host_vec_array node_densities; - +void FEA_Module_Eulerian::read_conditions_ansys_dat(std::ifstream* in, std::streampos before_condition_header) +{ + char ch; + int num_dim = simparam->num_dims; + int buffer_lines = 1000; + int max_word = 30; + int p_order = simparam->p_order; + real_t unit_scaling = simparam->get_unit_scaling(); + int local_node_index, current_column_index; + size_t strain_count; + std::string skip_line, read_line, substring, token; + std::stringstream line_parse, line_parse2; + CArrayKokkos read_buffer; + CArrayKokkos read_buffer_indices; + int buffer_loop, buffer_iteration, buffer_iterations, scan_loop, nodes_per_element, words_per_line; + size_t read_index_start, node_rid, elem_gid; + LO local_dof_id; + GO node_gid; + real_t dof_value; + host_vec_array node_densities; } // end read_conditions_ansys_dat - /* ---------------------------------------------------------------------------- Initialize sets of element boundary surfaces and arrays for input conditions ------------------------------------------------------------------------------- */ -void FEA_Module_Eulerian::init_boundaries(){ - max_boundary_sets = simparam->NB; - int num_dim = simparam->num_dims; - - // set the number of boundary sets - if(myrank == 0) - std::cout << "building boundary sets " << std::endl; - - //initialize to 1 since there must be at least 1 boundary set anyway; read in may occure later - if(max_boundary_sets==0) max_boundary_sets = 1; - //std::cout << "NUM BOUNDARY CONDITIONS ON RANK " << myrank << " FOR INIT " << num_boundary_conditions <(nall_nodes*num_dim, "Node_DOF_Boundary_Condition_Type"); - - //initialize - for(int init=0; init < nall_nodes*num_dim; init++) - Node_DOF_Boundary_Condition_Type(init) = NONE; - - Number_DOF_BCS = 0; +void FEA_Module_Eulerian::init_boundaries() +{ + max_boundary_sets = simparam->NB; + int num_dim = simparam->num_dims; + + // set the number of boundary sets + if (myrank == 0) + { + std::cout << "building boundary sets " << std::endl; + } + + // initialize to 1 since there must be at least 1 boundary set anyway; read in may occure later + if (max_boundary_sets == 0) + { + max_boundary_sets = 1; + } + // std::cout << "NUM BOUNDARY CONDITIONS ON RANK " << myrank << " FOR INIT " << num_boundary_conditions <(nall_nodes * num_dim, "Node_DOF_Boundary_Condition_Type"); + + // initialize + for (int init = 0; init < nall_nodes * num_dim; init++) + { + Node_DOF_Boundary_Condition_Type(init) = NONE; + } + + Number_DOF_BCS = 0; } /* ---------------------------------------------------------------------- initialize storage for element boundary surfaces corresponding to user BCs ------------------------------------------------------------------------- */ -void FEA_Module_Eulerian::init_boundary_sets (int num_sets){ +void FEA_Module_Eulerian::init_boundary_sets(int num_sets) +{ + if (num_sets == 0) + { + std::cout << " Warning: number of boundary conditions = 0"; + return; + } + // initialize maximum + max_boundary_sets = num_sets; + // std::cout << " DEBUG PRINT "<(num_sets, "Boundary_Condition_Type_List"); + NBoundary_Condition_Patches = CArrayKokkos(num_sets, "NBoundary_Condition_Patches"); + // std::cout << "NBOUNDARY PATCHES ON RANK " << myrank << " FOR INIT IS " << nboundary_patches <(num_sets, nboundary_patches, "Boundary_Condition_Patches"); + + // initialize data + for (int iset = 0; iset < num_sets; iset++) + { + NBoundary_Condition_Patches(iset) = 0; + } - if(num_sets == 0){ - std::cout << " Warning: number of boundary conditions = 0"; - return; - } - //initialize maximum - max_boundary_sets = num_sets; - //std::cout << " DEBUG PRINT "<(num_sets, "Boundary_Condition_Type_List"); - NBoundary_Condition_Patches = CArrayKokkos(num_sets, "NBoundary_Condition_Patches"); - //std::cout << "NBOUNDARY PATCHES ON RANK " << myrank << " FOR INIT IS " << nboundary_patches <(num_sets, nboundary_patches, "Boundary_Condition_Patches"); - - //initialize data - for(int iset = 0; iset < num_sets; iset++) NBoundary_Condition_Patches(iset) = 0; - - //initialize - for(int ibdy=0; ibdy < num_sets; ibdy++) Boundary_Condition_Type_List(ibdy) = NONE; + // initialize + for (int ibdy = 0; ibdy < num_sets; ibdy++) + { + Boundary_Condition_Type_List(ibdy) = NONE; + } } /* ---------------------------------------------------------------------------- Grow boundary conditions sets of element boundary surfaces ------------------------------------------------------------------------------- */ -void FEA_Module_Eulerian::grow_boundary_sets(int num_sets){ - int num_dim = simparam->num_dims; +void FEA_Module_Eulerian::grow_boundary_sets(int num_sets) +{ + int num_dim = simparam->num_dims; - if(num_sets == 0){ - std::cout << " Warning: number of boundary conditions being set to 0"; - return; - } - - //std::cout << " DEBUG PRINT "<max_boundary_sets){ - //temporary storage for previous data - CArrayKokkos Temp_Boundary_Condition_Type_List = Boundary_Condition_Type_List; - CArrayKokkos Temp_NBoundary_Condition_Patches = NBoundary_Condition_Patches; - CArrayKokkos Temp_Boundary_Condition_Patches = Boundary_Condition_Patches; - - max_boundary_sets = num_sets + 5; //5 is an arbitrary buffer - Boundary_Condition_Type_List = CArrayKokkos(max_boundary_sets, "Boundary_Condition_Type_List"); - NBoundary_Condition_Patches = CArrayKokkos(max_boundary_sets, "NBoundary_Condition_Patches"); - //std::cout << "NBOUNDARY PATCHES ON RANK " << myrank << " FOR GROW " << nboundary_patches <(max_boundary_sets, nboundary_patches, "Boundary_Condition_Patches"); - - //copy previous data back over - //std::cout << "NUM BOUNDARY CONDITIONS ON RANK " << myrank << " FOR COPY " << max_boundary_sets < max_boundary_sets) + { + // temporary storage for previous data + CArrayKokkos Temp_Boundary_Condition_Type_List = Boundary_Condition_Type_List; + CArrayKokkos Temp_NBoundary_Condition_Patches = NBoundary_Condition_Patches; + CArrayKokkos Temp_Boundary_Condition_Patches = Boundary_Condition_Patches; + + max_boundary_sets = num_sets + 5; // 5 is an arbitrary buffer + Boundary_Condition_Type_List = CArrayKokkos(max_boundary_sets, "Boundary_Condition_Type_List"); + NBoundary_Condition_Patches = CArrayKokkos(max_boundary_sets, "NBoundary_Condition_Patches"); + // std::cout << "NBOUNDARY PATCHES ON RANK " << myrank << " FOR GROW " << nboundary_patches <(max_boundary_sets, nboundary_patches, "Boundary_Condition_Patches"); + + // copy previous data back over + // std::cout << "NUM BOUNDARY CONDITIONS ON RANK " << myrank << " FOR COPY " << max_boundary_sets <output_options.output_velocity; - bool output_strain_flag = simparam->output_options.output_strain; - bool output_stress_flag = simparam->output_options.output_stress; - int num_dim = simparam->num_dims; - int Brows; - if(num_dim==3) Brows = 6; - else Brows = 3; - - if(output_velocity_flag){ - //displacement_index is accessed by writers at the solver level for deformed output - output_velocity_index = noutput; - noutput += 1; - module_outputs.resize(noutput); - - vector_style.resize(noutput); - vector_style[noutput-1] = DOF; - - output_vector_sizes.resize(noutput); - output_vector_sizes[noutput-1] = num_dim; - - output_dof_names.resize(noutput); - output_dof_names[noutput-1].resize(num_dim); - output_dof_names[noutput-1][0] = "vx"; - output_dof_names[noutput-1][1] = "vy"; - if(num_dim==3) - output_dof_names[noutput-1][2] = "vz"; - } - if(output_strain_flag){ - output_strain_index = noutput; - noutput += 1; - module_outputs.resize(noutput); - - vector_style.resize(noutput); - vector_style[noutput-1] = NODAL; - - output_vector_sizes.resize(noutput); - output_vector_sizes[noutput-1] = Brows; - - output_dof_names.resize(noutput); - output_dof_names[noutput-1].resize(Brows); - if(num_dim==2){ - output_dof_names[noutput-1][0] = "strain_xx"; - output_dof_names[noutput-1][1] = "strain_yy"; - output_dof_names[noutput-1][2] = "strain_xy"; +void FEA_Module_Eulerian::init_output() +{ + // check user parameters for output + bool output_velocity_flag = simparam->output_options.output_velocity; + bool output_strain_flag = simparam->output_options.output_strain; + bool output_stress_flag = simparam->output_options.output_stress; + int num_dim = simparam->num_dims; + int Brows; + if (num_dim == 3) + { + Brows = 6; } - if(num_dim==3){ - output_dof_names[noutput-1][0] = "strain_xx"; - output_dof_names[noutput-1][1] = "strain_yy"; - output_dof_names[noutput-1][2] = "strain_zz"; - output_dof_names[noutput-1][3] = "strain_xy"; - output_dof_names[noutput-1][4] = "strain_xz"; - output_dof_names[noutput-1][5] = "strain_yz"; + else + { + Brows = 3; } - } - if(output_stress_flag){ - output_stress_index = noutput; - noutput += 1; - module_outputs.resize(noutput); - - vector_style.resize(noutput); - vector_style[noutput-1] = NODAL; - - output_vector_sizes.resize(noutput); - output_vector_sizes[noutput-1] = Brows; - - output_dof_names.resize(noutput); - output_dof_names[noutput-1].resize(Brows); - if(num_dim==2){ - output_dof_names[noutput-1][0] = "stress_xx"; - output_dof_names[noutput-1][1] = "stress_yy"; - output_dof_names[noutput-1][3] = "stress_xy"; + + if (output_velocity_flag) + { + // displacement_index is accessed by writers at the solver level for deformed output + output_velocity_index = noutput; + noutput += 1; + module_outputs.resize(noutput); + + vector_style.resize(noutput); + vector_style[noutput - 1] = DOF; + + output_vector_sizes.resize(noutput); + output_vector_sizes[noutput - 1] = num_dim; + + output_dof_names.resize(noutput); + output_dof_names[noutput - 1].resize(num_dim); + output_dof_names[noutput - 1][0] = "vx"; + output_dof_names[noutput - 1][1] = "vy"; + if (num_dim == 3) + { + output_dof_names[noutput - 1][2] = "vz"; + } } - if(num_dim==3){ - output_dof_names[noutput-1][0] = "stress_xx"; - output_dof_names[noutput-1][1] = "stress_yy"; - output_dof_names[noutput-1][2] = "stress_zz"; - output_dof_names[noutput-1][3] = "stress_xy"; - output_dof_names[noutput-1][4] = "stress_xz"; - output_dof_names[noutput-1][5] = "stress_yz"; + if (output_strain_flag) + { + output_strain_index = noutput; + noutput += 1; + module_outputs.resize(noutput); + + vector_style.resize(noutput); + vector_style[noutput - 1] = NODAL; + + output_vector_sizes.resize(noutput); + output_vector_sizes[noutput - 1] = Brows; + + output_dof_names.resize(noutput); + output_dof_names[noutput - 1].resize(Brows); + if (num_dim == 2) + { + output_dof_names[noutput - 1][0] = "strain_xx"; + output_dof_names[noutput - 1][1] = "strain_yy"; + output_dof_names[noutput - 1][2] = "strain_xy"; + } + if (num_dim == 3) + { + output_dof_names[noutput - 1][0] = "strain_xx"; + output_dof_names[noutput - 1][1] = "strain_yy"; + output_dof_names[noutput - 1][2] = "strain_zz"; + output_dof_names[noutput - 1][3] = "strain_xy"; + output_dof_names[noutput - 1][4] = "strain_xz"; + output_dof_names[noutput - 1][5] = "strain_yz"; + } + } + if (output_stress_flag) + { + output_stress_index = noutput; + noutput += 1; + module_outputs.resize(noutput); + + vector_style.resize(noutput); + vector_style[noutput - 1] = NODAL; + + output_vector_sizes.resize(noutput); + output_vector_sizes[noutput - 1] = Brows; + + output_dof_names.resize(noutput); + output_dof_names[noutput - 1].resize(Brows); + if (num_dim == 2) + { + output_dof_names[noutput - 1][0] = "stress_xx"; + output_dof_names[noutput - 1][1] = "stress_yy"; + output_dof_names[noutput - 1][3] = "stress_xy"; + } + if (num_dim == 3) + { + output_dof_names[noutput - 1][0] = "stress_xx"; + output_dof_names[noutput - 1][1] = "stress_yy"; + output_dof_names[noutput - 1][2] = "stress_zz"; + output_dof_names[noutput - 1][3] = "stress_xy"; + output_dof_names[noutput - 1][4] = "stress_xz"; + output_dof_names[noutput - 1][5] = "stress_yz"; + } } - } } /* ------------------------------------------------------------------------------------------- Prompts sorting for elastic response output data. For now, nodal strains. ---------------------------------------------------------------------------------------------- */ -void FEA_Module_Eulerian::sort_output(Teuchos::RCP > sorted_map){ - - +void FEA_Module_Eulerian::sort_output(Teuchos::RCP> sorted_map) +{ } /* ------------------------------------------------------------------------------------------- Prompts computation of elastic response output data. For now, nodal strains. ---------------------------------------------------------------------------------------------- */ -void FEA_Module_Eulerian::collect_output(Teuchos::RCP > global_reduce_map){ - +void FEA_Module_Eulerian::collect_output(Teuchos::RCP> global_reduce_map) +{ } /* ------------------------------------------------------------------------------------------- Prompts computation of elastic response output data. For now, nodal strains. ---------------------------------------------------------------------------------------------- */ -void FEA_Module_Eulerian::compute_output(){ - +void FEA_Module_Eulerian::compute_output() +{ } /* ---------------------------------------------------------------------- Communicate updated nodal velocities to ghost nodes ------------------------------------------------------------------------- */ -void FEA_Module_Eulerian::comm_node_masses(){ - - //debug print of design vector - //std::ostream &out = std::cout; - //Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - //if(myrank==0) - //*fos << "Density data :" << std::endl; - //node_densities_distributed->describe(*fos,Teuchos::VERB_EXTREME); - //*fos << std::endl; - //std::fflush(stdout); - - //communicate design densities - //create import object using local node indices map and all indices map - //Tpetra::Import importer(map, ghost_node_map); - - //comms to get ghosts - ghost_node_masses_distributed->doImport(*node_masses_distributed, *ghost_importer, Tpetra::INSERT); - //all_node_map->describe(*fos,Teuchos::VERB_EXTREME); - //all_node_velocities_distributed->describe(*fos,Teuchos::VERB_EXTREME); - - //update_count++; - //if(update_count==1){ - //MPI_Barrier(world); - //MPI_Abort(world,4); - //} +void FEA_Module_Eulerian::comm_node_masses() +{ + // debug print of design vector + // std::ostream &out = std::cout; + // Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + // if(myrank==0) + // *fos << "Density data :" << std::endl; + // node_densities_distributed->describe(*fos,Teuchos::VERB_EXTREME); + // *fos << std::endl; + // std::fflush(stdout); + + // communicate design densities + // create import object using local node indices map and all indices map + // Tpetra::Import importer(map, ghost_node_map); + + // comms to get ghosts + ghost_node_masses_distributed->doImport(*node_masses_distributed, *ghost_importer, Tpetra::INSERT); + // all_node_map->describe(*fos,Teuchos::VERB_EXTREME); + // all_node_velocities_distributed->describe(*fos,Teuchos::VERB_EXTREME); + + // update_count++; + // if(update_count==1){ + // MPI_Barrier(world); + // MPI_Abort(world,4); + // } } /* ---------------------------------------------------------------------- Communicate updated nodal adjoint vectors to ghost nodes ------------------------------------------------------------------------- */ -void FEA_Module_Eulerian::comm_adjoint_vectors(int cycle){ - - //debug print of design vector - //std::ostream &out = std::cout; - //Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - //if(myrank==0) - //*fos << "Density data :" << std::endl; - //node_densities_distributed->describe(*fos,Teuchos::VERB_EXTREME); - //*fos << std::endl; - //std::fflush(stdout); - - //communicate design densities - //create import object using local node indices map and all indices map - //Tpetra::Import importer(map, all_node_map); - - //comms to get ghosts - (*adjoint_vector_data)[cycle]->doImport(*adjoint_vector_distributed, *importer, Tpetra::INSERT); - (*phi_adjoint_vector_data)[cycle]->doImport(*phi_adjoint_vector_distributed, *importer, Tpetra::INSERT); - //all_node_map->describe(*fos,Teuchos::VERB_EXTREME); - //all_node_velocities_distributed->describe(*fos,Teuchos::VERB_EXTREME); - - //update_count++; - //if(update_count==1){ - //MPI_Barrier(world); - //MPI_Abort(world,4); - //} +void FEA_Module_Eulerian::comm_adjoint_vectors(int cycle) +{ + // debug print of design vector + // std::ostream &out = std::cout; + // Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + // if(myrank==0) + // *fos << "Density data :" << std::endl; + // node_densities_distributed->describe(*fos,Teuchos::VERB_EXTREME); + // *fos << std::endl; + // std::fflush(stdout); + + // communicate design densities + // create import object using local node indices map and all indices map + // Tpetra::Import importer(map, all_node_map); + + // comms to get ghosts + (*adjoint_vector_data)[cycle]->doImport(*adjoint_vector_distributed, *importer, Tpetra::INSERT); + (*phi_adjoint_vector_data)[cycle]->doImport(*phi_adjoint_vector_distributed, *importer, Tpetra::INSERT); + // all_node_map->describe(*fos,Teuchos::VERB_EXTREME); + // all_node_velocities_distributed->describe(*fos,Teuchos::VERB_EXTREME); + + // update_count++; + // if(update_count==1){ + // MPI_Barrier(world); + // MPI_Abort(world,4); + // } } /* ------------------------------------------------------------------------------------------- Communicate ghosts using the current optimization design data ---------------------------------------------------------------------------------------------- */ -void FEA_Module_Eulerian::comm_variables(Teuchos::RCP zp){ - - if(simparam_dynamic_opt.topology_optimization_on){ - //set density vector to the current value chosen by the optimizer - test_node_densities_distributed = zp; - - //debug print of design vector - //std::ostream &out = std::cout; - //Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - //if(myrank==0) - //*fos << "Density data :" << std::endl; - //node_densities_distributed->describe(*fos,Teuchos::VERB_EXTREME); - //*fos << std::endl; - //std::fflush(stdout); - - //communicate design densities - //create import object using local node indices map and all indices map - //Tpetra::Import importer(map, all_node_map); - - //comms to get ghosts - all_node_densities_distributed->doImport(*test_node_densities_distributed, *importer, Tpetra::INSERT); - } - else if(simparam_dynamic_opt.shape_optimization_on){ - //clause to communicate boundary node data if the boundary nodes are ghosts on this rank - } +void FEA_Module_Eulerian::comm_variables(Teuchos::RCP zp) +{ + if (simparam_dynamic_opt.topology_optimization_on) + { + // set density vector to the current value chosen by the optimizer + test_node_densities_distributed = zp; + + // debug print of design vector + // std::ostream &out = std::cout; + // Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + // if(myrank==0) + // *fos << "Density data :" << std::endl; + // node_densities_distributed->describe(*fos,Teuchos::VERB_EXTREME); + // *fos << std::endl; + // std::fflush(stdout); + + // communicate design densities + // create import object using local node indices map and all indices map + // Tpetra::Import importer(map, all_node_map); + + // comms to get ghosts + all_node_densities_distributed->doImport(*test_node_densities_distributed, *importer, Tpetra::INSERT); + } + else if (simparam_dynamic_opt.shape_optimization_on) + { + // clause to communicate boundary node data if the boundary nodes are ghosts on this rank + } } - /* ------------------------------------------------------------------------------------------- enforce constraints on nodes due to BCS ---------------------------------------------------------------------------------------------- */ -void FEA_Module_Eulerian::node_density_constraints(host_vec_array node_densities_lower_bound){ - +void FEA_Module_Eulerian::node_density_constraints(host_vec_array node_densities_lower_bound) +{ } /* ---------------------------------------------------------------------------- Setup Eulerian solver data ------------------------------------------------------------------------------- */ -void FEA_Module_Eulerian::setup(){ - - const size_t rk_level = simparam->rk_num_bins - 1; - const size_t num_fills = simparam->region_options.size(); - const size_t rk_num_bins = simparam->rk_num_bins; - const size_t num_bcs = simparam->boundary_conditions.size(); +void FEA_Module_Eulerian::setup() +{ + const size_t rk_level = simparam->rk_num_bins - 1; + const size_t num_fills = simparam->region_options.size(); + const size_t rk_num_bins = simparam->rk_num_bins; + const size_t num_bcs = simparam->boundary_conditions.size(); const size_t num_materials = simparam->material_options.size(); - const int num_dim = simparam->num_dims; + const int num_dim = simparam->num_dims; - const DCArrayKokkos mat_fill = simparam->mat_fill; - const DCArrayKokkos boundary = simparam->boundary; - const DCArrayKokkos material = simparam->material; + const DCArrayKokkos mat_fill = simparam->mat_fill; + const DCArrayKokkos boundary = simparam->boundary; + const DCArrayKokkos material = simparam->material; global_vars = simparam->global_vars; - state_vars = DCArrayKokkos (rnum_elem, simparam->max_num_state_vars); - - //--- calculate bdy sets ---// - mesh.num_nodes_in_patch = 2*(num_dim-1); // 2 (2D) or 4 (3D) - mesh.num_patches_in_elem = 2*num_dim; // 4 (2D) or 6 (3D) + state_vars = DCArrayKokkos(rnum_elem, simparam->max_num_state_vars); + + // --- calculate bdy sets ---// + mesh.num_nodes_in_patch = 2 * (num_dim - 1); // 2 (2D) or 4 (3D) + mesh.num_patches_in_elem = 2 * num_dim; // 4 (2D) or 6 (3D) mesh.init_bdy_sets(num_bcs); num_bdy_sets = mesh.num_bdy_sets; printf("Num BC's = %lu\n", num_bcs); // patch ids in bdy set bdy_patches_in_set = mesh.bdy_patches_in_set; - if(num_dim==2) - bdy_nodes = mesh.bdy_nodes; - + if (num_dim == 2) + { + bdy_nodes = mesh.bdy_nodes; + } + // node ids in bdy_patch set - bdy_nodes_in_set = mesh.bdy_nodes_in_set; + bdy_nodes_in_set = mesh.bdy_nodes_in_set; num_bdy_nodes_in_set = mesh.num_bdy_nodes_in_set; - - //assign mesh views needed by the FEA module + + // assign mesh views needed by the FEA module // elem ids in elem - elems_in_elem = mesh.elems_in_elem; + elems_in_elem = mesh.elems_in_elem; num_elems_in_elem = mesh.num_elems_in_elem; - //corners + // corners num_corners_in_node = mesh.num_corners_in_node; - corners_in_node = mesh.corners_in_node; - corners_in_elem = mesh.corners_in_elem; + corners_in_node = mesh.corners_in_node; + corners_in_elem = mesh.corners_in_elem; - //elem-node conn & node-node conn + // elem-node conn & node-node conn elems_in_node = mesh.elems_in_node; - if(num_dim==2){ - nodes_in_node = mesh.nodes_in_node; - num_nodes_in_node = mesh.num_nodes_in_node; - //patch conn - - patches_in_elem = mesh.patches_in_elem; - nodes_in_patch = mesh.nodes_in_patch; - elems_in_patch = mesh.elems_in_patch; + if (num_dim == 2) + { + nodes_in_node = mesh.nodes_in_node; + num_nodes_in_node = mesh.num_nodes_in_node; + // patch conn + + patches_in_elem = mesh.patches_in_elem; + nodes_in_patch = mesh.nodes_in_patch; + elems_in_patch = mesh.elems_in_patch; } - - //initialize if topology optimization is used - if(simparam_dynamic_opt.topology_optimization_on||simparam_dynamic_opt.shape_optimization_on){ - //create parameter object - simparam_elasticity = Simulation_Parameters_Elasticity(); - init_assembly(); - assemble_matrix(); + + // initialize if topology optimization is used + if (simparam_dynamic_opt.topology_optimization_on || simparam_dynamic_opt.shape_optimization_on) + { + // create parameter object + simparam_elasticity = Simulation_Parameters_Elasticity(); + init_assembly(); + assemble_matrix(); } return; - } // end of setup /* ---------------------------------------------------------------------------- Eulerian solver loop ------------------------------------------------------------------------------- */ -void FEA_Module_Eulerian::euler_solve(){ - +void FEA_Module_Eulerian::euler_solve() +{ Time_Variables tv = simparam->time_variables; - - const size_t rk_level = simparam->rk_num_bins - 1; - time_value = simparam->time_value; - time_final = tv.time_final; - dt_max = tv.dt_max; - dt_min = tv.dt_min; - dt_cfl = tv.dt_cfl; - graphics_time = simparam->graphics_options.graphics_time; + + const size_t rk_level = simparam->rk_num_bins - 1; + time_value = simparam->time_value; + time_final = tv.time_final; + dt_max = tv.dt_max; + dt_min = tv.dt_min; + dt_cfl = tv.dt_cfl; + graphics_time = simparam->graphics_options.graphics_time; graphics_cyc_ival = simparam->graphics_options.graphics_cyc_ival; - graphics_dt_ival = simparam->graphics_options.graphics_dt_ival; - cycle_stop = tv.cycle_stop; - rk_num_stages = simparam->rk_num_stages; - dt = tv.dt; - fuzz = tv.fuzz; - tiny = tv.tiny; + graphics_dt_ival = simparam->graphics_options.graphics_dt_ival; + cycle_stop = tv.cycle_stop; + rk_num_stages = simparam->rk_num_stages; + dt = tv.dt; + fuzz = tv.fuzz; + tiny = tv.tiny; small = tv.small; graphics_times = simparam->graphics_options.graphics_times; - graphics_id = simparam->graphics_options.graphics_id; - size_t num_bdy_nodes = mesh.num_bdy_nodes; - const DCArrayKokkos boundary = simparam->boundary; - const DCArrayKokkos material = simparam->material; - int nTO_modules; - int old_max_forward_buffer; - size_t cycle; - const int num_dim = simparam->num_dims; - real_t objective_accumulation, global_objective_accumulation; - std::vector> FEA_Module_My_TO_Modules = simparam_dynamic_opt.FEA_Module_My_TO_Modules; - problem = Explicit_Solver_Pointer_->problem; //Pointer to ROL optimization problem object + graphics_id = simparam->graphics_options.graphics_id; + size_t num_bdy_nodes = mesh.num_bdy_nodes; + const DCArrayKokkos boundary = simparam->boundary; + const DCArrayKokkos material = simparam->material; + int nTO_modules; + int old_max_forward_buffer; + size_t cycle; + const int num_dim = simparam->num_dims; + real_t objective_accumulation, global_objective_accumulation; + std::vector> FEA_Module_My_TO_Modules = simparam_dynamic_opt.FEA_Module_My_TO_Modules; + problem = Explicit_Solver_Pointer_->problem; // Pointer to ROL optimization problem object ROL::Ptr> obj_pointer; - //reset time accumulating objective and constraints + // reset time accumulating objective and constraints /* for(int imodule = 0 ; imodule < FEA_Module_My_TO_Modules[my_fea_module_index_].size(); imodule++){ current_module_index = FEA_Module_My_TO_Modules[my_fea_module_index_][imodule]; //test if module needs reset if(){ - + } } */ - //simple setup to just request KE for now; above loop to be expanded and used later for scanning modules - if(simparam_dynamic_opt.topology_optimization_on){ - obj_pointer = problem->getObjective(); - KineticEnergyMinimize_TopOpt& kinetic_energy_minimize_function = dynamic_cast(*obj_pointer); - kinetic_energy_minimize_function.objective_accumulation = 0; - global_objective_accumulation = objective_accumulation = 0; - kinetic_energy_objective = true; - if(max_time_steps +1 > forward_solve_velocity_data->size()){ - old_max_forward_buffer = forward_solve_velocity_data->size(); - time_data.resize(max_time_steps+1); - forward_solve_velocity_data->resize(max_time_steps+1); - forward_solve_coordinate_data->resize(max_time_steps+1); - adjoint_vector_data->resize(max_time_steps+1); - phi_adjoint_vector_data->resize(max_time_steps+1); - //assign a multivector of corresponding size to each new timestep in the buffer - for(int istep = old_max_forward_buffer; istep < max_time_steps+1; istep++){ - (*forward_solve_velocity_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - (*forward_solve_coordinate_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - (*adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - (*phi_adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + // simple setup to just request KE for now; above loop to be expanded and used later for scanning modules + if (simparam_dynamic_opt.topology_optimization_on) + { + obj_pointer = problem->getObjective(); + KineticEnergyMinimize_TopOpt& kinetic_energy_minimize_function = dynamic_cast(*obj_pointer); + kinetic_energy_minimize_function.objective_accumulation = 0; + global_objective_accumulation = objective_accumulation = 0; + kinetic_energy_objective = true; + if (max_time_steps + 1 > forward_solve_velocity_data->size()) + { + old_max_forward_buffer = forward_solve_velocity_data->size(); + time_data.resize(max_time_steps + 1); + forward_solve_velocity_data->resize(max_time_steps + 1); + forward_solve_coordinate_data->resize(max_time_steps + 1); + adjoint_vector_data->resize(max_time_steps + 1); + phi_adjoint_vector_data->resize(max_time_steps + 1); + // assign a multivector of corresponding size to each new timestep in the buffer + for (int istep = old_max_forward_buffer; istep < max_time_steps + 1; istep++) + { + (*forward_solve_velocity_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*forward_solve_coordinate_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*phi_adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + } } - } } - if(simparam_dynamic_opt.topology_optimization_on) - nTO_modules = simparam_dynamic_opt.TO_Module_List.size(); + if (simparam_dynamic_opt.topology_optimization_on) + { + nTO_modules = simparam_dynamic_opt.TO_Module_List.size(); + } int myrank = Explicit_Solver_Pointer_->myrank; - if(simparam->output_options.output_file_format==OUTPUT_FORMAT::vtk) + if (simparam->output_options.output_file_format == OUTPUT_FORMAT::vtk) { - if(myrank==0) - printf("Writing outputs to file at %f \n", time_value); + if (myrank == 0) + { + printf("Writing outputs to file at %f \n", time_value); + } - double comm_time1 = Explicit_Solver_Pointer_->CPU_Time(); - //Explicit_Solver_Pointer_->write_outputs_new(); - double comm_time2 = Explicit_Solver_Pointer_->CPU_Time(); - Explicit_Solver_Pointer_->output_time += comm_time2 - comm_time1; + double comm_time1 = Explicit_Solver_Pointer_->CPU_Time(); + // Explicit_Solver_Pointer_->write_outputs_new(); + double comm_time2 = Explicit_Solver_Pointer_->CPU_Time(); + Explicit_Solver_Pointer_->output_time += comm_time2 - comm_time1; } /* write_outputs(mesh, @@ -730,18 +781,17 @@ void FEA_Module_Eulerian::euler_solve(){ graphics_id, time_value); */ - - - CArrayKokkos node_extensive_mass(nall_nodes, "node_extensive_mass"); - + + CArrayKokkos node_extensive_mass(nall_nodes, "node_extensive_mass"); + // extensive energy tallies over the mesh elements local to this MPI rank double IE_t0 = 0.0; double KE_t0 = 0.0; double TE_t0 = 0.0; - + double IE_sum = 0.0; double KE_sum = 0.0; - + double IE_loc_sum = 0.0; double KE_loc_sum = 0.0; @@ -760,138 +810,139 @@ void FEA_Module_Eulerian::euler_solve(){ double global_TE_tend = 0.0; int nlocal_elem_non_overlapping = Explicit_Solver_Pointer_->nlocal_elem_non_overlapping; - + // extensive IE REDUCE_SUM_CLASS(elem_gid, 0, nlocal_elem_non_overlapping, IE_loc_sum, { - IE_loc_sum += elem_mass(elem_gid)*elem_sie(rk_level,elem_gid); - + IE_loc_sum += elem_mass(elem_gid) * elem_sie(rk_level, elem_gid); }, IE_sum); IE_t0 = IE_sum; - MPI_Allreduce(&IE_t0,&global_IE_t0,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD); - + MPI_Allreduce(&IE_t0, &global_IE_t0, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); + // extensive KE REDUCE_SUM_CLASS(node_gid, 0, nlocal_nodes, KE_loc_sum, { - double ke = 0; - for (size_t dim=0; dimnode_velocities_distributed->getLocalView (Tpetra::Access::ReadWrite); - vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - node_velocities_interface(node_gid,idim) = node_vel(rk_level,node_gid,idim); - node_coords_interface(node_gid,idim) = node_coords(rk_level,node_gid,idim); - } + time_data[0] = 0; + // assign current velocity data to multivector + // view scope + { + vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); + vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + node_velocities_interface(node_gid, idim) = node_vel(rk_level, node_gid, idim); + node_coords_interface(node_gid, idim) = node_coords(rk_level, node_gid, idim); + } }); - } //end view scope - Kokkos::fence(); - - //communicate ghosts - double comm_time1 = Explicit_Solver_Pointer_->CPU_Time(); - - //active view scope; triggers host comms from updated data on device - { - const_host_vec_array node_velocities_host = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_host_vec_array node_coords_host = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - } - double comm_time2 = Explicit_Solver_Pointer_->CPU_Time(); - Explicit_Solver_Pointer_->dev2host_time += comm_time2-comm_time1; + } // end view scope + Kokkos::fence(); - //communicate ghost velocities - Explicit_Solver_Pointer_->comm_velocities(); - Explicit_Solver_Pointer_->comm_coordinates(); - - - double comm_time3 = Explicit_Solver_Pointer_->CPU_Time(); + // communicate ghosts + double comm_time1 = Explicit_Solver_Pointer_->CPU_Time(); - //view scope - { - const_vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array ghost_node_velocities_interface = Explicit_Solver_Pointer_->ghost_node_velocities_distributed->getLocalView (Tpetra::Access::ReadOnly); - vec_array all_node_velocities_interface = Explicit_Solver_Pointer_->all_node_velocities_distributed->getLocalView (Tpetra::Access::ReadWrite); - const_vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array ghost_node_coords_interface = Explicit_Solver_Pointer_->ghost_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - all_node_velocities_interface(node_gid,idim) = node_velocities_interface(node_gid,idim); - all_node_coords_interface(node_gid,idim) = node_coords_interface(node_gid,idim); + // active view scope; triggers host comms from updated data on device + { + const_host_vec_array node_velocities_host = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_host_vec_array node_coords_host = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); } + double comm_time2 = Explicit_Solver_Pointer_->CPU_Time(); + Explicit_Solver_Pointer_->dev2host_time += comm_time2 - comm_time1; + + // communicate ghost velocities + Explicit_Solver_Pointer_->comm_velocities(); + Explicit_Solver_Pointer_->comm_coordinates(); + + double comm_time3 = Explicit_Solver_Pointer_->CPU_Time(); + + // view scope + { + const_vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array ghost_node_velocities_interface = Explicit_Solver_Pointer_->ghost_node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); + vec_array all_node_velocities_interface = Explicit_Solver_Pointer_->all_node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); + const_vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array ghost_node_coords_interface = Explicit_Solver_Pointer_->ghost_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + all_node_velocities_interface(node_gid, idim) = node_velocities_interface(node_gid, idim); + all_node_coords_interface(node_gid, idim) = node_coords_interface(node_gid, idim); + } }); // end parallel for - Kokkos::fence(); + Kokkos::fence(); - FOR_ALL_CLASS(node_gid, nlocal_nodes, nlocal_nodes+nghost_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - all_node_velocities_interface(node_gid,idim) = ghost_node_velocities_interface(node_gid-nlocal_nodes,idim); - all_node_coords_interface(node_gid,idim) = ghost_node_coords_interface(node_gid-nlocal_nodes,idim); - } + FOR_ALL_CLASS(node_gid, nlocal_nodes, nlocal_nodes + nghost_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + all_node_velocities_interface(node_gid, idim) = ghost_node_velocities_interface(node_gid - nlocal_nodes, idim); + all_node_coords_interface(node_gid, idim) = ghost_node_coords_interface(node_gid - nlocal_nodes, idim); + } }); // end parallel for - Kokkos::fence(); - } //end view scope - - - (*forward_solve_velocity_data)[0]->assign(*Explicit_Solver_Pointer_->all_node_velocities_distributed); - (*forward_solve_coordinate_data)[0]->assign(*Explicit_Solver_Pointer_->all_node_coords_distributed); - } - - // loop over the max number of time integration cycles - for (cycle = 0; cycle < cycle_stop; cycle++) { - - // get the step - if(num_dim==2){ + Kokkos::fence(); + } // end view scope + + (*forward_solve_velocity_data)[0]->assign(*Explicit_Solver_Pointer_->all_node_velocities_distributed); + (*forward_solve_coordinate_data)[0]->assign(*Explicit_Solver_Pointer_->all_node_coords_distributed); + } + + // loop over the max number of time integration cycles + for (cycle = 0; cycle < cycle_stop; cycle++) + { + // get the step + if (num_dim == 2) + { get_timestep2D(mesh, node_coords, node_vel, elem_sspd, elem_vol); } - else { + else + { get_timestep(mesh, node_coords, node_vel, @@ -900,29 +951,32 @@ void FEA_Module_Eulerian::euler_solve(){ } // end if 2D double global_dt; - MPI_Allreduce(&dt,&global_dt,1,MPI_DOUBLE,MPI_MIN,MPI_COMM_WORLD); + MPI_Allreduce(&dt, &global_dt, 1, MPI_DOUBLE, MPI_MIN, MPI_COMM_WORLD); dt = global_dt; - // stop calculation if flag - //if (stop_calc == 1) break; - - + // stop calculation if flag + // if (stop_calc == 1) break; - if (cycle==0){ - if(myrank==0) - printf("cycle = %lu, time = %12.5e, time step = %12.5e \n", cycle, time_value, dt); + if (cycle == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %12.5e, time step = %12.5e \n", cycle, time_value, dt); + } } // print time step every 10 cycles - else if (cycle%20==0){ - if(myrank==0) - printf("cycle = %lu, time = %12.5e, time step = %12.5e \n", cycle, time_value, dt); + else if (cycle % 20 == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %12.5e, time step = %12.5e \n", cycle, time_value, dt); + } } // end if - - + // --------------------------------------------------------------------- // integrate the solution forward to t(n+1) via Runge Kutta (RK) method // --------------------------------------------------------------------- - + // save the values at t_n rk_init(node_coords, node_vel, @@ -930,107 +984,102 @@ void FEA_Module_Eulerian::euler_solve(){ elem_stress, rnum_elem, nall_nodes); - - - - // integrate solution forward in time - for (size_t rk_stage = 0; rk_stage < rk_num_stages; rk_stage++){ - + // integrate solution forward in time + for (size_t rk_stage = 0; rk_stage < rk_num_stages; rk_stage++) + { // ---- RK coefficient ---- - double rk_alpha = 1.0/((double)rk_num_stages - (double)rk_stage); + double rk_alpha = 1.0 / ((double)rk_num_stages - (double)rk_stage); - //current interface has differing velocity arrays; this equates them until we unify memory - //first comm time interval point + // current interface has differing velocity arrays; this equates them until we unify memory + // first comm time interval point double comm_time1 = Explicit_Solver_Pointer_->CPU_Time(); - //view scope + // view scope { - vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - node_velocities_interface(node_gid,idim) = node_vel(rk_level,node_gid,idim); - } + vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + node_velocities_interface(node_gid, idim) = node_vel(rk_level, node_gid, idim); + } }); // end parallel for - } //end view scope + } // end view scope Kokkos::fence(); - - //active view scope + + // active view scope { - const_host_vec_array node_velocities_host = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView (Tpetra::Access::ReadOnly); + const_host_vec_array node_velocities_host = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); } double comm_time2 = Explicit_Solver_Pointer_->CPU_Time(); - Explicit_Solver_Pointer_->dev2host_time += comm_time2-comm_time1; - //communicate ghost velocities + Explicit_Solver_Pointer_->dev2host_time += comm_time2 - comm_time1; + // communicate ghost velocities Explicit_Solver_Pointer_->comm_velocities(); - + double comm_time3 = Explicit_Solver_Pointer_->CPU_Time(); - //this is forcing a copy to the device - //view scope + // this is forcing a copy to the device + // view scope { - vec_array ghost_node_velocities_interface = Explicit_Solver_Pointer_->ghost_node_velocities_distributed->getLocalView (Tpetra::Access::ReadWrite); + vec_array ghost_node_velocities_interface = Explicit_Solver_Pointer_->ghost_node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, nlocal_nodes, nall_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - node_vel(rk_level,node_gid,idim) = ghost_node_velocities_interface(node_gid-nlocal_nodes,idim); - } - + FOR_ALL_CLASS(node_gid, nlocal_nodes, nall_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + node_vel(rk_level, node_gid, idim) = ghost_node_velocities_interface(node_gid - nlocal_nodes, idim); + } }); // end parallel for - } //end view scope + } // end view scope Kokkos::fence(); - + double comm_time4 = Explicit_Solver_Pointer_->CPU_Time(); - Explicit_Solver_Pointer_->host2dev_time += comm_time4-comm_time3; - Explicit_Solver_Pointer_->communication_time += comm_time4-comm_time1; - //debug print vector values on a rank + Explicit_Solver_Pointer_->host2dev_time += comm_time4 - comm_time3; + Explicit_Solver_Pointer_->communication_time += comm_time4 - comm_time1; + // debug print vector values on a rank /* if(myrank==0) for(int i = 0; i < nall_nodes; i++){ std::cout << Explicit_Solver_Pointer_->all_node_map->getGlobalElement(i) << " " << node_vel(rk_level,i,0) << " " << node_vel(rk_level,i,1) << " " << node_vel(rk_level,i,2) << std::endl; } - */ + */ - // calculate the new corner masses if 2D - if(num_dim==2){ - + if (num_dim == 2) + { // calculate the nodal areal mass FOR_ALL_CLASS(node_gid, 0, nall_nodes, { - node_mass(node_gid) = 0.0; - - if (node_coords(rk_level,node_gid,1) > tiny){ - node_mass(node_gid) = node_extensive_mass(node_gid)/node_coords(rk_level,node_gid,1); - } - //if(cycle==0&&node_gid==1&&myrank==0) - //std::cout << "index " << node_gid << " on rank " << myrank << " node vel " << node_vel(rk_level,node_gid,0) << " " << node_mass(node_gid) << std::endl << std::flush; + if (node_coords(rk_level, node_gid, 1) > tiny) + { + node_mass(node_gid) = node_extensive_mass(node_gid) / node_coords(rk_level, node_gid, 1); + } + // if(cycle==0&&node_gid==1&&myrank==0) + // std::cout << "index " << node_gid << " on rank " << myrank << " node vel " << node_vel(rk_level,node_gid,0) << " " << node_mass(node_gid) << std::endl << std::flush; }); // end parallel for over node_gid Kokkos::fence(); - //current interface has differing density arrays; this equates them until we unify memory - //view scope + // current interface has differing density arrays; this equates them until we unify memory + // view scope { - vec_array node_mass_interface = node_masses_distributed->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - node_mass_interface(node_gid,0) = node_mass(node_gid); + vec_array node_mass_interface = node_masses_distributed->getLocalView(Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + node_mass_interface(node_gid, 0) = node_mass(node_gid); }); // end parallel for - } //end view scope + } // end view scope Kokkos::fence(); - //communicate ghost densities + // communicate ghost densities comm_node_masses(); - //this is forcing a copy to the device - //view scope + // this is forcing a copy to the device + // view scope { - vec_array ghost_node_mass_interface = ghost_node_masses_distributed->getLocalView (Tpetra::Access::ReadWrite); + vec_array ghost_node_mass_interface = ghost_node_masses_distributed->getLocalView(Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, nlocal_nodes, nall_nodes, { - node_mass(node_gid) = ghost_node_mass_interface(node_gid-nlocal_nodes,0); + FOR_ALL_CLASS(node_gid, nlocal_nodes, nall_nodes, { + node_mass(node_gid) = ghost_node_mass_interface(node_gid - nlocal_nodes, 0); }); // end parallel for - } //end view scope + } // end view scope Kokkos::fence(); - - + // ----------------------------------------------- // Calcualte the areal mass for nodes on the axis // ----------------------------------------------- @@ -1043,339 +1092,355 @@ void FEA_Module_Eulerian::euler_solve(){ // 0---1 /* FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { - + // loop over the corners of the element and calculate the mass for (size_t node_lid=0; node_lid<4; node_lid++){ - + size_t node_gid = nodes_in_elem(elem_gid, node_lid); size_t node_minus_gid; size_t node_plus_gid; - - + + if (node_coords(rk_level,node_gid,1) < tiny){ // node is on the axis - + // minus node if (node_lid==0){ node_minus_gid = nodes_in_elem(elem_gid, 3); } else { node_minus_gid = nodes_in_elem(elem_gid, node_lid-1); } - + // plus node if (node_lid==3){ node_plus_gid = nodes_in_elem(elem_gid, 0); } else { node_plus_gid = nodes_in_elem(elem_gid, node_lid+1); } - + node_mass(node_gid) = fmax(node_mass(node_plus_gid), node_mass(node_minus_gid))/2.0; - + } // end if - + } // end for over corners - + }); // end parallel for over elem_gid Kokkos::fence(); */ - + FOR_ALL_CLASS(node_bdy_gid, 0, num_bdy_nodes, { - //FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + // FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { size_t node_gid = bdy_nodes(node_bdy_gid); - - if (node_coords(rk_level,node_gid,1) < tiny){ + + if (node_coords(rk_level, node_gid, 1) < tiny) + { // node is on the axis - - for(size_t node_lid=0; node_lid < num_nodes_in_node(node_gid); node_lid++){ - + + for (size_t node_lid = 0; node_lid < num_nodes_in_node(node_gid); node_lid++) + { size_t node_neighbor_gid = nodes_in_node(node_gid, node_lid); - + // if the node is off the axis, use it's areal mass on the boundary - if (node_coords(rk_level,node_neighbor_gid,1) > tiny){ - node_mass(node_gid) = fmax(node_mass(node_gid), node_mass(node_neighbor_gid)/2.0); + if (node_coords(rk_level, node_neighbor_gid, 1) > tiny) + { + node_mass(node_gid) = fmax(node_mass(node_gid), node_mass(node_neighbor_gid) / 2.0); } - } // end for over neighboring nodes - } // end if - }); // end parallel for over elem_gid - } // end of if 2D-RZ + } // end of RK loop + // increment the time + time_value += dt; + simparam->time_value = time_value; - } // end of RK loop + if (simparam_dynamic_opt.topology_optimization_on || simparam_dynamic_opt.shape_optimization_on) + { + if (cycle >= max_time_steps) + { + max_time_steps = cycle + 1; + } - // increment the time - time_value+=dt; - simparam->time_value = time_value; + if (max_time_steps + 1 > forward_solve_velocity_data->size()) + { + old_max_forward_buffer = forward_solve_velocity_data->size(); + time_data.resize(max_time_steps + BUFFER_GROW + 1); + forward_solve_velocity_data->resize(max_time_steps + BUFFER_GROW + 1); + forward_solve_coordinate_data->resize(max_time_steps + BUFFER_GROW + 1); + adjoint_vector_data->resize(max_time_steps + BUFFER_GROW + 1); + phi_adjoint_vector_data->resize(max_time_steps + BUFFER_GROW + 1); + // assign a multivector of corresponding size to each new timestep in the buffer + for (int istep = old_max_forward_buffer; istep < max_time_steps + BUFFER_GROW + 1; istep++) + { + (*forward_solve_velocity_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*forward_solve_coordinate_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*phi_adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + } + } - if(simparam_dynamic_opt.topology_optimization_on||simparam_dynamic_opt.shape_optimization_on){ - if(cycle >= max_time_steps) - max_time_steps = cycle + 1; + time_data[cycle + 1] = dt + time_data[cycle]; - if(max_time_steps + 1 > forward_solve_velocity_data->size()){ - old_max_forward_buffer = forward_solve_velocity_data->size(); - time_data.resize(max_time_steps + BUFFER_GROW +1); - forward_solve_velocity_data->resize(max_time_steps + BUFFER_GROW +1); - forward_solve_coordinate_data->resize(max_time_steps + BUFFER_GROW +1); - adjoint_vector_data->resize(max_time_steps + BUFFER_GROW +1); - phi_adjoint_vector_data->resize(max_time_steps + BUFFER_GROW +1); - //assign a multivector of corresponding size to each new timestep in the buffer - for(int istep = old_max_forward_buffer; istep < max_time_steps + BUFFER_GROW +1; istep++){ - (*forward_solve_velocity_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - (*forward_solve_coordinate_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - (*adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - (*phi_adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - } - } + // assign current velocity data to multivector + // view scope + { + vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); + vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + node_velocities_interface(node_gid, idim) = node_vel(rk_level, node_gid, idim); + node_coords_interface(node_gid, idim) = node_coords(rk_level, node_gid, idim); + } + }); + } // end view scope + Kokkos::fence(); - - time_data[cycle+1] = dt + time_data[cycle]; - - - //assign current velocity data to multivector - //view scope - { - vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView (Tpetra::Access::ReadWrite); - vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - node_velocities_interface(node_gid,idim) = node_vel(rk_level,node_gid,idim); - node_coords_interface(node_gid,idim) = node_coords(rk_level,node_gid,idim); + // communicate ghosts + double comm_time1 = Explicit_Solver_Pointer_->CPU_Time(); + + // active view scope; triggers host comms from updated data on device + { + const_host_vec_array node_velocities_host = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_host_vec_array node_coords_host = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); } - }); - } //end view scope - Kokkos::fence(); + double comm_time2 = Explicit_Solver_Pointer_->CPU_Time(); + Explicit_Solver_Pointer_->dev2host_time += comm_time2 - comm_time1; - //communicate ghosts - double comm_time1 = Explicit_Solver_Pointer_->CPU_Time(); - - //active view scope; triggers host comms from updated data on device - { - const_host_vec_array node_velocities_host = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_host_vec_array node_coords_host = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - } - double comm_time2 = Explicit_Solver_Pointer_->CPU_Time(); - Explicit_Solver_Pointer_->dev2host_time += comm_time2-comm_time1; + // communicate ghost velocities + Explicit_Solver_Pointer_->comm_velocities(); + Explicit_Solver_Pointer_->comm_coordinates(); - //communicate ghost velocities - Explicit_Solver_Pointer_->comm_velocities(); - Explicit_Solver_Pointer_->comm_coordinates(); - - - double comm_time3 = Explicit_Solver_Pointer_->CPU_Time(); + double comm_time3 = Explicit_Solver_Pointer_->CPU_Time(); - //view scope - { - const_vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array ghost_node_velocities_interface = Explicit_Solver_Pointer_->ghost_node_velocities_distributed->getLocalView (Tpetra::Access::ReadOnly); - vec_array all_node_velocities_interface = Explicit_Solver_Pointer_->all_node_velocities_distributed->getLocalView (Tpetra::Access::ReadWrite); - const_vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array ghost_node_coords_interface = Explicit_Solver_Pointer_->ghost_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - all_node_velocities_interface(node_gid,idim) = node_velocities_interface(node_gid,idim); - all_node_coords_interface(node_gid,idim) = node_coords_interface(node_gid,idim); - } + // view scope + { + const_vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array ghost_node_velocities_interface = Explicit_Solver_Pointer_->ghost_node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); + vec_array all_node_velocities_interface = Explicit_Solver_Pointer_->all_node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); + const_vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array ghost_node_coords_interface = Explicit_Solver_Pointer_->ghost_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + all_node_velocities_interface(node_gid, idim) = node_velocities_interface(node_gid, idim); + all_node_coords_interface(node_gid, idim) = node_coords_interface(node_gid, idim); + } }); // end parallel for - Kokkos::fence(); + Kokkos::fence(); - FOR_ALL_CLASS(node_gid, nlocal_nodes, nlocal_nodes+nghost_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - all_node_velocities_interface(node_gid,idim) = ghost_node_velocities_interface(node_gid-nlocal_nodes,idim); - all_node_coords_interface(node_gid,idim) = ghost_node_coords_interface(node_gid-nlocal_nodes,idim); - } + FOR_ALL_CLASS(node_gid, nlocal_nodes, nlocal_nodes + nghost_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + all_node_velocities_interface(node_gid, idim) = ghost_node_velocities_interface(node_gid - nlocal_nodes, idim); + all_node_coords_interface(node_gid, idim) = ghost_node_coords_interface(node_gid - nlocal_nodes, idim); + } }); // end parallel for - Kokkos::fence(); - } //end view scope - - double comm_time4 = Explicit_Solver_Pointer_->CPU_Time(); - Explicit_Solver_Pointer_->host2dev_time += comm_time4-comm_time3; - Explicit_Solver_Pointer_->communication_time += comm_time4-comm_time1; - - (*forward_solve_velocity_data)[cycle+1]->assign(*Explicit_Solver_Pointer_->all_node_velocities_distributed); - (*forward_solve_coordinate_data)[cycle+1]->assign(*Explicit_Solver_Pointer_->all_node_coords_distributed); - - //kinetic energy accumulation - if(kinetic_energy_objective){ - const_vec_array node_velocities_interface = (*forward_solve_velocity_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array previous_node_velocities_interface = (*forward_solve_velocity_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - KE_loc_sum = 0.0; - KE_sum = 0.0; - // extensive KE - REDUCE_SUM_CLASS(node_gid, 0, nlocal_nodes, KE_loc_sum, { - - double ke = 0; - for (size_t dim=0; dimCPU_Time(); + Explicit_Solver_Pointer_->host2dev_time += comm_time4 - comm_time3; + Explicit_Solver_Pointer_->communication_time += comm_time4 - comm_time1; + + (*forward_solve_velocity_data)[cycle + 1]->assign(*Explicit_Solver_Pointer_->all_node_velocities_distributed); + (*forward_solve_coordinate_data)[cycle + 1]->assign(*Explicit_Solver_Pointer_->all_node_coords_distributed); + + // kinetic energy accumulation + if (kinetic_energy_objective) + { + const_vec_array node_velocities_interface = (*forward_solve_velocity_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array previous_node_velocities_interface = (*forward_solve_velocity_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + KE_loc_sum = 0.0; + KE_sum = 0.0; + // extensive KE + REDUCE_SUM_CLASS(node_gid, 0, nlocal_nodes, KE_loc_sum, { + double ke = 0; + for (size_t dim = 0; dim < num_dim; dim++) + { + // midpoint integration approximation + ke += (node_velocities_interface(node_gid, dim) + node_velocities_interface(node_gid, dim)) * (node_velocities_interface(node_gid, dim) + node_velocities_interface(node_gid, + dim)) / 4; // 1/2 at end + } // end for + + if (num_dim == 2) + { + KE_loc_sum += node_mass(node_gid) * node_coords(rk_level, node_gid, 1) * ke; + } + else + { + KE_loc_sum += node_mass(node_gid) * ke; + } }, KE_sum); - Kokkos::fence(); - KE_sum = 0.5*KE_sum; - objective_accumulation += KE_sum*dt; + Kokkos::fence(); + KE_sum = 0.5 * KE_sum; + objective_accumulation += KE_sum * dt; + } + } + + size_t write = 0; + if ((cycle + 1) % graphics_cyc_ival == 0 && cycle > 0) + { + write = 1; + } + else if (cycle == cycle_stop) + { + write = 1; + } + else if (time_value >= time_final) + { + write = 1; + } + else if (time_value >= graphics_time) + { + write = 1; } - } - - size_t write = 0; - if ((cycle+1)%graphics_cyc_ival == 0 && cycle>0){ - write = 1; - } - else if (cycle == cycle_stop) { - write = 1; - } - else if (time_value >= time_final){ - write = 1; - } - else if (time_value >= graphics_time){ - write = 1; - } - + // write outputs - if (write == 1){ - //interface nodal coordinate data (note: this is not needed if using write_outputs_new()) - //view scope + if (write == 1) + { + // interface nodal coordinate data (note: this is not needed if using write_outputs_new()) + // view scope { - vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - node_coords_interface(node_gid,idim) = node_coords(rk_level,node_gid,idim); - } + vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + node_coords_interface(node_gid, idim) = node_coords(rk_level, node_gid, idim); + } }); // end parallel for - } //end view scope - if(simparam->output_options.output_file_format==OUTPUT_FORMAT::vtk){ - if(myrank==0){ - printf("Writing outputs to file at %f \n", graphics_time); - } + } // end view scope + if (simparam->output_options.output_file_format == OUTPUT_FORMAT::vtk) + { + if (myrank == 0) + { + printf("Writing outputs to file at %f \n", graphics_time); + } - double comm_time1 = Explicit_Solver_Pointer_->CPU_Time(); - //Explicit_Solver_Pointer_->write_outputs_new(); + double comm_time1 = Explicit_Solver_Pointer_->CPU_Time(); + // Explicit_Solver_Pointer_->write_outputs_new(); - double comm_time2 = Explicit_Solver_Pointer_->CPU_Time(); - Explicit_Solver_Pointer_->output_time += comm_time2 - comm_time1; + double comm_time2 = Explicit_Solver_Pointer_->CPU_Time(); + Explicit_Solver_Pointer_->output_time += comm_time2 - comm_time1; } - //Explicit_Solver_Pointer_->parallel_vtk_writer(); - //Explicit_Solver_Pointer_->parallel_vtk_writer_new(); - //Explicit_Solver_Pointer_->parallel_tecplot_writer(); - /* - write_outputs(mesh, - Explicit_Solver_Pointer_, - node_coords, - node_vel, - node_mass, - elem_den, - elem_pres, - elem_stress, - elem_sspd, - elem_sie, - elem_vol, - elem_mass, - elem_mat_id, - graphics_times, - graphics_id, - time_value); - */ + // Explicit_Solver_Pointer_->parallel_vtk_writer(); + // Explicit_Solver_Pointer_->parallel_vtk_writer_new(); + // Explicit_Solver_Pointer_->parallel_tecplot_writer(); + /* + write_outputs(mesh, + Explicit_Solver_Pointer_, + node_coords, + node_vel, + node_mass, + elem_den, + elem_pres, + elem_stress, + elem_sspd, + elem_sie, + elem_vol, + elem_mass, + elem_mat_id, + graphics_times, + graphics_id, + time_value); + */ graphics_time = time_value + graphics_dt_ival; - } // end if - - - // end of calculation - if (time_value>=time_final) break; + } // end if - + // end of calculation + if (time_value >= time_final) + { + break; + } } // end for cycle loop last_time_step = cycle; - //simple setup to just calculate KE minimize objective for now - if(simparam_dynamic_opt.topology_optimization_on){ - KineticEnergyMinimize_TopOpt& kinetic_energy_minimize_function = dynamic_cast(*obj_pointer); + // simple setup to just calculate KE minimize objective for now + if (simparam_dynamic_opt.topology_optimization_on) + { + KineticEnergyMinimize_TopOpt& kinetic_energy_minimize_function = dynamic_cast(*obj_pointer); - //collect local objective values - MPI_Allreduce(&objective_accumulation,&global_objective_accumulation,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD); - kinetic_energy_minimize_function.objective_accumulation = global_objective_accumulation; + // collect local objective values + MPI_Allreduce(&objective_accumulation, &global_objective_accumulation, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); + kinetic_energy_minimize_function.objective_accumulation = global_objective_accumulation; - if(myrank==0) - std::cout << "CURRENT TIME INTEGRAL OF KINETIC ENERGY " << global_objective_accumulation << std::endl; + if (myrank == 0) + { + std::cout << "CURRENT TIME INTEGRAL OF KINETIC ENERGY " << global_objective_accumulation << std::endl; + } } - - + auto time_2 = std::chrono::high_resolution_clock::now(); auto time_difference = time_2 - time_1; - //double calc_time = std::chrono::duration_cast(diff).count(); + // double calc_time = std::chrono::duration_cast(diff).count(); double calc_time = std::chrono::duration_cast(time_difference).count(); - if(myrank==0) - printf("\nCalculation time in seconds: %f \n", calc_time*1e-09); - + if (myrank == 0) + { + printf("\nCalculation time in seconds: %f \n", calc_time * 1e-09); + } + IE_loc_sum = 0.0; KE_loc_sum = 0.0; - IE_sum = 0.0; - KE_sum = 0.0; - + IE_sum = 0.0; + KE_sum = 0.0; + // extensive IE REDUCE_SUM_CLASS(elem_gid, 0, nlocal_elem_non_overlapping, IE_loc_sum, { - - IE_loc_sum += elem_mass(elem_gid)*elem_sie(rk_level,elem_gid); - + IE_loc_sum += elem_mass(elem_gid) * elem_sie(rk_level, elem_gid); }, IE_sum); IE_tend = IE_sum; - //reduce over MPI ranks - MPI_Allreduce(&IE_tend,&global_IE_tend,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD); + // reduce over MPI ranks + MPI_Allreduce(&IE_tend, &global_IE_tend, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); // extensive KE REDUCE_SUM_CLASS(node_gid, 0, nlocal_nodes, KE_loc_sum, { - double ke = 0; - for (size_t dim=0; dim zp); - - void read_conditions_ansys_dat(std::ifstream *in, std::streampos before_condition_header); - - //interfaces between user input and creating data structures for bcs - void generate_bcs(); - - void Displacement_Boundary_Conditions(); - - void init_output(); - - void compute_output(); - - void sort_output(Teuchos::RCP > sorted_map); - - void collect_output(Teuchos::RCP > global_reduce_map); - - void init_boundaries(); - - //initializes memory for arrays used in the global stiffness matrix assembly - void init_boundary_sets(int num_boundary_sets); - - void grow_boundary_sets(int num_boundary_sets); - - void rk_init(DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &elem_sie, - DViewCArrayKokkos &elem_stress, - const size_t num_elems, - const size_t num_nodes); - - void get_timestep(mesh_t &mesh, - DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &elem_sspd, - DViewCArrayKokkos &elem_vol); - - - void get_timestep2D(mesh_t &mesh, - DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &elem_sspd, - DViewCArrayKokkos &elem_vol); - - void write_outputs (const mesh_t &mesh, - DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &node_mass, - DViewCArrayKokkos &elem_den, - DViewCArrayKokkos &elem_pres, - DViewCArrayKokkos &elem_stress, - DViewCArrayKokkos &elem_sspd, - DViewCArrayKokkos &elem_sie, - DViewCArrayKokkos &elem_vol, - DViewCArrayKokkos &elem_mass, - DViewCArrayKokkos &elem_mat_id); - - void node_density_constraints(host_vec_array node_densities_lower_bound); - - void example_function(double rk_alpha, - const size_t num_nodes, - DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel); - - KOKKOS_FUNCTION - void example_device_function(const ViewCArrayKokkos &B_matrix, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const; - - - bool nodal_density_flag; - real_t penalty_power; - Teuchos::RCP Global_Stiffness_Matrix; - RaggedRightArrayKokkos Stiffness_Matrix; - DCArrayKokkos Stiffness_Matrix_Strides; - DCArrayKokkos Global_Stiffness_Matrix_Assembly_Map; - //end elastic TO data - - Simulation_Parameters_Eulerian *simparam; - Simulation_Parameters_Elasticity simparam_elasticity; - Simulation_Parameters_Dynamic_Optimization simparam_dynamic_opt; - Explicit_Solver_Eulerian *Explicit_Solver_Pointer_; - - elements::ref_element *ref_elem; - - mesh_t& mesh; - //shallow copies of mesh class views - size_t num_nodes_in_elem; - // corner ids in node - RaggedRightArrayKokkos corners_in_node; - CArrayKokkos num_corners_in_node; - - // elem ids in node - RaggedRightArrayKokkos elems_in_node; - - // node ids in node - RaggedRightArrayKokkos nodes_in_node; - CArrayKokkos num_nodes_in_node; - - // node ids in elem - DCArrayKokkos nodes_in_elem; - - // corner ids in elem - CArrayKokkos corners_in_elem; - - // elem ids in elem - RaggedRightArrayKokkos elems_in_elem; - CArrayKokkos num_elems_in_elem; - - // patch ids in elem - CArrayKokkos patches_in_elem; - - // node ids in a patch - CArrayKokkos nodes_in_patch; - - // element ids in a patch - CArrayKokkos elems_in_patch; - - // patch ids in bdy set - size_t num_bdy_sets; - DynamicRaggedRightArrayKokkos bdy_patches_in_set; - - // bdy nodes - CArrayKokkos bdy_nodes; - - // node ids in bdy_patch set - RaggedRightArrayKokkos bdy_nodes_in_set; - DCArrayKokkos num_bdy_nodes_in_set; - - //Topology optimization filter variable - DCArrayKokkos relative_element_densities; - - //Local FEA data - host_elem_conn_array interface_nodes_in_elem; //host view of element connectivity to nodes - - //Global FEA data - Teuchos::RCP node_velocities_distributed; - Teuchos::RCP initial_node_coords_distributed; - Teuchos::RCP all_initial_node_coords_distributed; - Teuchos::RCP initial_node_velocities_distributed; - Teuchos::RCP all_node_velocities_distributed; - Teuchos::RCP all_cached_node_velocities_distributed; - Teuchos::RCP node_masses_distributed; - Teuchos::RCP ghost_node_masses_distributed; - Teuchos::RCP adjoint_vector_distributed; - Teuchos::RCP phi_adjoint_vector_distributed; - Teuchos::RCP>> forward_solve_velocity_data; - Teuchos::RCP>> forward_solve_coordinate_data; - Teuchos::RCP>> adjoint_vector_data; - Teuchos::RCP>> phi_adjoint_vector_data; - Teuchos::RCP force_gradient_design; - Teuchos::RCP force_gradient_position; - Teuchos::RCP force_gradient_velocity; - - //Local FEA data - DCArrayKokkos Global_Gradient_Matrix_Assembly_Map; - RaggedRightArrayKokkos Graph_Matrix; //stores global indices - RaggedRightArrayKokkos DOF_Graph_Matrix; //stores global indices - RaggedRightArrayKokkos Force_Gradient_Positions; - RaggedRightArrayKokkos Force_Gradient_Velocities; - DCArrayKokkos Gradient_Matrix_Strides; - DCArrayKokkos Graph_Matrix_Strides; - RaggedRightArrayKokkos Original_Gradient_Entries; - RaggedRightArrayKokkos Original_Gradient_Entry_Indices; - DCArrayKokkos Original_Gradient_Entries_Strides; - - //distributed matrices - Teuchos::RCP distributed_force_gradient_positions; - Teuchos::RCP distributed_force_gradient_velocities; - - std::vector time_data; - int max_time_steps, last_time_step; - - //Dual View wrappers - // Dual Views of the individual node struct variables - DViewCArrayKokkos node_coords; - DViewCArrayKokkos node_vel; - DViewCArrayKokkos node_mass; - - // Dual Views of the individual elem struct variables - DViewCArrayKokkos elem_den; - DViewCArrayKokkos elem_pres; - DViewCArrayKokkos elem_stress; // always 3D even in 2D-RZ - DViewCArrayKokkos elem_sspd; - DViewCArrayKokkos elem_sie; - DViewCArrayKokkos elem_vol; - DViewCArrayKokkos elem_div; - DViewCArrayKokkos elem_mass; - DViewCArrayKokkos elem_mat_id; - - // Element velocity gradient - DCArrayKokkos elem_vel_grad; - - // for storing global variables used in user material model - DCArrayKokkos global_vars; - - // for storing global variables used in user material model - DCArrayKokkos state_vars; - - // Dual Views of the corner struct variables - DViewCArrayKokkos corner_force; - DViewCArrayKokkos corner_mass; - - //Boundary Conditions Data - DCArrayKokkos Local_Index_Boundary_Patches; - //CArray Patch_Nodes; - enum bc_type {NONE, POINT_LOADING_CONDITION, LINE_LOADING_CONDITION, SURFACE_LOADING_CONDITION}; - - //Boundary Conditions Data - int max_boundary_sets; - - //output dof data - //Global arrays with collected data used to print - int output_velocity_index, output_strain_index, output_stress_index; - - //file parameters - DCArrayKokkos read_from_file; - - //parameters - double time_value, time_final, dt, dt_max, dt_min, dt_cfl, graphics_time, graphics_dt_ival; - size_t graphics_cyc_ival, cycle_stop, rk_num_stages, graphics_id; - double fuzz, tiny, small; - CArray graphics_times; - - //optimization flags - bool kinetic_energy_objective; + + FEA_Module_Eulerian(Solver* Solver_Pointer, mesh_t& mesh, const int my_fea_module_index = 0); + ~FEA_Module_Eulerian(); + + void setup(); + + void euler_solve(); + + void comm_node_masses(); + + void comm_adjoint_vectors(int cycle); + + void comm_variables(Teuchos::RCP zp); + + void read_conditions_ansys_dat(std::ifstream* in, std::streampos before_condition_header); + + // interfaces between user input and creating data structures for bcs + void generate_bcs(); + + void Displacement_Boundary_Conditions(); + + void init_output(); + + void compute_output(); + + void sort_output(Teuchos::RCP> sorted_map); + + void collect_output(Teuchos::RCP> global_reduce_map); + + void init_boundaries(); + + // initializes memory for arrays used in the global stiffness matrix assembly + void init_boundary_sets(int num_boundary_sets); + + void grow_boundary_sets(int num_boundary_sets); + + void rk_init(DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_sie, + DViewCArrayKokkos& elem_stress, + const size_t num_elems, + const size_t num_nodes); + + void get_timestep(mesh_t& mesh, + DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_sspd, + DViewCArrayKokkos& elem_vol); + + void get_timestep2D(mesh_t& mesh, + DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_sspd, + DViewCArrayKokkos& elem_vol); + + void write_outputs(const mesh_t& mesh, + DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& node_mass, + DViewCArrayKokkos& elem_den, + DViewCArrayKokkos& elem_pres, + DViewCArrayKokkos& elem_stress, + DViewCArrayKokkos& elem_sspd, + DViewCArrayKokkos& elem_sie, + DViewCArrayKokkos& elem_vol, + DViewCArrayKokkos& elem_mass, + DViewCArrayKokkos& elem_mat_id); + + void node_density_constraints(host_vec_array node_densities_lower_bound); + + void example_function(double rk_alpha, + const size_t num_nodes, + DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel); + + KOKKOS_FUNCTION + void example_device_function(const ViewCArrayKokkos& B_matrix, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const; + + bool nodal_density_flag; + real_t penalty_power; + Teuchos::RCP Global_Stiffness_Matrix; + RaggedRightArrayKokkos Stiffness_Matrix; + DCArrayKokkos Stiffness_Matrix_Strides; + DCArrayKokkos Global_Stiffness_Matrix_Assembly_Map; + // end elastic TO data + + Simulation_Parameters_Eulerian* simparam; + Simulation_Parameters_Elasticity simparam_elasticity; + Simulation_Parameters_Dynamic_Optimization simparam_dynamic_opt; + Explicit_Solver_Eulerian* Explicit_Solver_Pointer_; + + elements::ref_element* ref_elem; + + mesh_t& mesh; + // shallow copies of mesh class views + size_t num_nodes_in_elem; + // corner ids in node + RaggedRightArrayKokkos corners_in_node; + CArrayKokkos num_corners_in_node; + + // elem ids in node + RaggedRightArrayKokkos elems_in_node; + + // node ids in node + RaggedRightArrayKokkos nodes_in_node; + CArrayKokkos num_nodes_in_node; + + // node ids in elem + DCArrayKokkos nodes_in_elem; + + // corner ids in elem + CArrayKokkos corners_in_elem; + + // elem ids in elem + RaggedRightArrayKokkos elems_in_elem; + CArrayKokkos num_elems_in_elem; + + // patch ids in elem + CArrayKokkos patches_in_elem; + + // node ids in a patch + CArrayKokkos nodes_in_patch; + + // element ids in a patch + CArrayKokkos elems_in_patch; + + // patch ids in bdy set + size_t num_bdy_sets; + DynamicRaggedRightArrayKokkos bdy_patches_in_set; + + // bdy nodes + CArrayKokkos bdy_nodes; + + // node ids in bdy_patch set + RaggedRightArrayKokkos bdy_nodes_in_set; + DCArrayKokkos num_bdy_nodes_in_set; + + // Topology optimization filter variable + DCArrayKokkos relative_element_densities; + + // Local FEA data + host_elem_conn_array interface_nodes_in_elem; // host view of element connectivity to nodes + + // Global FEA data + Teuchos::RCP node_velocities_distributed; + Teuchos::RCP initial_node_coords_distributed; + Teuchos::RCP all_initial_node_coords_distributed; + Teuchos::RCP initial_node_velocities_distributed; + Teuchos::RCP all_node_velocities_distributed; + Teuchos::RCP all_cached_node_velocities_distributed; + Teuchos::RCP node_masses_distributed; + Teuchos::RCP ghost_node_masses_distributed; + Teuchos::RCP adjoint_vector_distributed; + Teuchos::RCP phi_adjoint_vector_distributed; + Teuchos::RCP>> forward_solve_velocity_data; + Teuchos::RCP>> forward_solve_coordinate_data; + Teuchos::RCP>> adjoint_vector_data; + Teuchos::RCP>> phi_adjoint_vector_data; + Teuchos::RCP force_gradient_design; + Teuchos::RCP force_gradient_position; + Teuchos::RCP force_gradient_velocity; + + // Local FEA data + DCArrayKokkos Global_Gradient_Matrix_Assembly_Map; + RaggedRightArrayKokkos Graph_Matrix; // stores global indices + RaggedRightArrayKokkos DOF_Graph_Matrix; // stores global indices + RaggedRightArrayKokkos Force_Gradient_Positions; + RaggedRightArrayKokkos Force_Gradient_Velocities; + DCArrayKokkos Gradient_Matrix_Strides; + DCArrayKokkos Graph_Matrix_Strides; + RaggedRightArrayKokkos Original_Gradient_Entries; + RaggedRightArrayKokkos Original_Gradient_Entry_Indices; + DCArrayKokkos Original_Gradient_Entries_Strides; + + // distributed matrices + Teuchos::RCP distributed_force_gradient_positions; + Teuchos::RCP distributed_force_gradient_velocities; + + std::vector time_data; + int max_time_steps, last_time_step; + + // Dual View wrappers + // Dual Views of the individual node struct variables + DViewCArrayKokkos node_coords; + DViewCArrayKokkos node_vel; + DViewCArrayKokkos node_mass; + + // Dual Views of the individual elem struct variables + DViewCArrayKokkos elem_den; + DViewCArrayKokkos elem_pres; + DViewCArrayKokkos elem_stress; // always 3D even in 2D-RZ + DViewCArrayKokkos elem_sspd; + DViewCArrayKokkos elem_sie; + DViewCArrayKokkos elem_vol; + DViewCArrayKokkos elem_div; + DViewCArrayKokkos elem_mass; + DViewCArrayKokkos elem_mat_id; + + // Element velocity gradient + DCArrayKokkos elem_vel_grad; + + // for storing global variables used in user material model + DCArrayKokkos global_vars; + + // for storing global variables used in user material model + DCArrayKokkos state_vars; + + // Dual Views of the corner struct variables + DViewCArrayKokkos corner_force; + DViewCArrayKokkos corner_mass; + + // Boundary Conditions Data + DCArrayKokkos Local_Index_Boundary_Patches; + // CArray Patch_Nodes; + enum bc_type { NONE, POINT_LOADING_CONDITION, LINE_LOADING_CONDITION, SURFACE_LOADING_CONDITION }; + + // Boundary Conditions Data + int max_boundary_sets; + + // output dof data + // Global arrays with collected data used to print + int output_velocity_index, output_strain_index, output_stress_index; + + // file parameters + DCArrayKokkos read_from_file; + + // parameters + double time_value, time_final, dt, dt_max, dt_min, dt_cfl, graphics_time, graphics_dt_ival; + size_t graphics_cyc_ival, cycle_stop, rk_num_stages, graphics_id; + double fuzz, tiny, small; + CArray graphics_times; + + // optimization flags + bool kinetic_energy_objective; }; #endif // end HEADER_H diff --git a/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/Simulation_Parameters_Eulerian.h b/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/Simulation_Parameters_Eulerian.h index ace620341..87017b8fa 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/Simulation_Parameters_Eulerian.h +++ b/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/Simulation_Parameters_Eulerian.h @@ -11,14 +11,14 @@ This program is open source under the BSD-3 License. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - + 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - + 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - + 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. @@ -52,80 +52,94 @@ using namespace mtr; -struct Simulation_Parameters_Eulerian : Simulation_Parameters { - Time_Variables time_variables; - std::vector region_options; - std::vector material_options; - std::vector boundary_conditions; - Graphics_Options graphics_options; - - bool gravity_flag = false; - bool report_runtime = true; - - size_t rk_num_stages = 2; - int NB = 6; // number of boundaries - int NBSF = 4; //number of surface density force conditions - int NBV = 2; //number of surface sets used to specify a fixed displacement on nodes belonging to respective surfaces - - //Non-serialized fields - int num_gauss_points = 2; - size_t max_num_global_vars; - size_t rk_num_bins; - double time_value = 0.0; - DCArrayKokkos global_vars; - - DCArrayKokkos mat_fill; - DCArrayKokkos material; - DCArrayKokkos boundary; - std::vector gravity_vector {9.81, 0., 0.}; - - void init_material_variable_arrays(size_t nglobal_vars) { - global_vars = DCArrayKokkos (material_options.size(), nglobal_vars); - - for (size_t i = 0; i < material_options.size(); i++) { - auto mat = material_options[i]; - - for (size_t j = 0; j < mat.global_vars.size(); j++) - global_vars.host(i, j) = mat.global_vars[j]; +struct Simulation_Parameters_Eulerian : Simulation_Parameters +{ + Time_Variables time_variables; + std::vector region_options; + std::vector material_options; + std::vector boundary_conditions; + Graphics_Options graphics_options; + + bool gravity_flag = false; + bool report_runtime = true; + + size_t rk_num_stages = 2; + int NB = 6; // number of boundaries + int NBSF = 4; // number of surface density force conditions + int NBV = 2; // number of surface sets used to specify a fixed displacement on nodes belonging to respective surfaces + + // Non-serialized fields + int num_gauss_points = 2; + size_t max_num_global_vars; + size_t rk_num_bins; + double time_value = 0.0; + DCArrayKokkos global_vars; + + DCArrayKokkos mat_fill; + DCArrayKokkos material; + DCArrayKokkos boundary; + std::vector gravity_vector { 9.81, 0., 0. }; + + void init_material_variable_arrays(size_t nglobal_vars) + { + global_vars = DCArrayKokkos(material_options.size(), nglobal_vars); + + for (size_t i = 0; i < material_options.size(); i++) + { + auto mat = material_options[i]; + + for (size_t j = 0; j < mat.global_vars.size(); j++) + { + global_vars.host(i, j) = mat.global_vars[j]; + } + } + } + + template void from_vector(DCArrayKokkos& array, const std::vector& vec) + { + array = DCArrayKokkos(vec.size()); + for (size_t i = 0; i < vec.size(); i++) + { + array.host(i) = *(T*)&vec[i]; + } + } + + void derive_kokkos_arrays() + { + max_num_global_vars = 0; + for (auto mo : material_options) + { + max_num_global_vars = std::max(max_num_global_vars, mo.global_vars.size()); + } + + init_material_variable_arrays(max_num_global_vars); + + from_vector(mat_fill, region_options); + from_vector(material, material_options); + from_vector(boundary, boundary_conditions); + + // Send to device. + mat_fill.update_device(); + boundary.update_device(); + material.update_device(); + global_vars.update_device(); + } + + void derive() + { + derive_kokkos_arrays(); + rk_num_bins = rk_num_stages; + } + + void validate() + { + validate_module_is_specified(FEA_MODULE_TYPE::Eulerian); } - } - - template void from_vector(DCArrayKokkos& array, const std::vector& vec) { - array = DCArrayKokkos(vec.size()); - for (size_t i = 0; i < vec.size(); i++) - array.host(i) = *(T*)&vec[i]; - } - void derive_kokkos_arrays() { - max_num_global_vars = 0; - for (auto mo : material_options) - max_num_global_vars = std::max(max_num_global_vars, mo.global_vars.size()); - - init_material_variable_arrays(max_num_global_vars); - - from_vector(mat_fill, region_options); - from_vector(material, material_options); - from_vector(boundary, boundary_conditions); - - // Send to device. - mat_fill.update_device(); - boundary.update_device(); - material.update_device(); - global_vars.update_device(); - - } - - void derive() { - derive_kokkos_arrays(); - rk_num_bins = rk_num_stages; - } - void validate() { - validate_module_is_specified(FEA_MODULE_TYPE::Eulerian); - } }; -IMPL_YAML_SERIALIZABLE_WITH_BASE(Simulation_Parameters_Eulerian, Simulation_Parameters, - time_variables, material_options, region_options, +IMPL_YAML_SERIALIZABLE_WITH_BASE(Simulation_Parameters_Eulerian, Simulation_Parameters, + time_variables, material_options, region_options, boundary_conditions, gravity_flag, report_runtime, rk_num_stages, NB, NBSF, NBV, graphics_options -) + ) #endif // end HEADER_H diff --git a/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/eulerian_geometry.cpp b/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/eulerian_geometry.cpp index d07b69392..174ac6b14 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/eulerian_geometry.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/eulerian_geometry.cpp @@ -1,32 +1,29 @@ // ----------------------------------------------------------------------------- // This code handles the geometric information for the mesh for the SHG solver -//------------------------------------------------------------------------------ +// ------------------------------------------------------------------------------ #include "matar.h" #include "mesh.h" #include "state.h" #include "FEA_Module_Eulerian.h" -void FEA_Module_Eulerian::example_function(double rk_alpha, - const size_t num_nodes, - DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel){ - +void FEA_Module_Eulerian::example_function(double rk_alpha, + const size_t num_nodes, + DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel) +{ const size_t rk_level = rk_num_bins - 1; - int num_dims = num_dim; + int num_dims = num_dim; // example device loop FOR_ALL_CLASS(node_gid, 0, num_nodes, { - - for (int dim = 0; dim < num_dims; dim++){ - double half_vel = (node_vel(rk_level, node_gid, dim) + node_vel(0, node_gid, dim))*0.5; - node_coords(rk_level, node_gid, dim) = node_coords(0, node_gid, dim) + rk_alpha*dt*half_vel; + for (int dim = 0; dim < num_dims; dim++) + { + double half_vel = (node_vel(rk_level, node_gid, dim) + node_vel(0, node_gid, dim)) * 0.5; + node_coords(rk_level, node_gid, dim) = node_coords(0, node_gid, dim) + rk_alpha * dt * half_vel; } - }); // end parallel for over nodes - } // end subroutine - // ----------------------------------------------------------------------------- // This function claculates // B_p = J^{-T} \cdot (\nabla_{xi} \phi_p w @@ -37,15 +34,12 @@ void FEA_Module_Eulerian::example_function(double rk_alpha, // \nabla_{xi} is the gradient opperator in the reference coordinates // // B_p is the OUTWARD corner area normal at node p -//------------------------------------------------------------------------------ +// ------------------------------------------------------------------------------ KOKKOS_FUNCTION -void FEA_Module_Eulerian::example_device_function(const ViewCArrayKokkos &B_matrix, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const { - - +void FEA_Module_Eulerian::example_device_function(const ViewCArrayKokkos& B_matrix, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const +{ } // end subroutine - - diff --git a/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/eulerian_time_integration.cpp b/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/eulerian_time_integration.cpp index fd95ab16e..2e99a4718 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/eulerian_time_integration.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/eulerian_time_integration.cpp @@ -5,83 +5,75 @@ // ----------------------------------------------------------------------------- // This function saves the variables at rk_stage = 0, which is t_n -//------------------------------------------------------------------------------ -void FEA_Module_Eulerian::rk_init(DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &elem_sie, - DViewCArrayKokkos &elem_stress, - const size_t num_elems, - const size_t num_nodes){ - +// ------------------------------------------------------------------------------ +void FEA_Module_Eulerian::rk_init(DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_sie, + DViewCArrayKokkos& elem_stress, + const size_t num_elems, + const size_t num_nodes) +{ const size_t rk_level = rk_num_bins - 1; - int num_dims = num_dim; + int num_dims = num_dim; // save elem quantities FOR_ALL_CLASS(elem_gid, 0, num_elems, { - // stress is always 3D even with 2D-RZ - for(size_t i=0; i<3; i++){ - for(size_t j=0; j<3; j++){ - elem_stress(0,elem_gid,i,j) = elem_stress(rk_level,elem_gid,i,j); + for (size_t i = 0; i < 3; i++) + { + for (size_t j = 0; j < 3; j++) + { + elem_stress(0, elem_gid, i, j) = elem_stress(rk_level, elem_gid, i, j); } } // end for - elem_sie(0,elem_gid) = elem_sie(rk_level,elem_gid); - + elem_sie(0, elem_gid) = elem_sie(rk_level, elem_gid); }); // end parallel for - - + // save nodal quantities FOR_ALL_CLASS(node_gid, 0, num_nodes, { - - for(size_t i=0; i &node_coords, - DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &elem_sspd, - DViewCArrayKokkos &elem_vol){ - - --> const size_t rk_level = rk_num_bins - 1; +void FEA_Module_Eulerian::get_timestep(mesh_t& mesh, + DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_sspd, + DViewCArrayKokkos& elem_vol) +{ + ->const size_t rk_level = rk_num_bins - 1; // increase dt by 10%, that is the largest dt value - dt = dt*1.1; - int num_dims = num_dim; + dt = dt * 1.1; + int num_dims = num_dim; double dt_lcl; double min_dt_calc; REDUCE_MIN_CLASS(elem_gid, 0, rnum_elem, dt_lcl, { - double coords0[24]; // element coords - ViewCArrayKokkos coords(coords0, 8, 3); + ViewCArrayKokkos coords(coords0, 8, 3); - double distance0[28]; // array for holding distances between each node - ViewCArrayKokkos dist(distance0, 28); - - // Getting the coordinates of the element - for(size_t node_lid = 0; node_lid < 8; node_lid++){ + ViewCArrayKokkos dist(distance0, 28); - for (size_t dim = 0; dim < num_dims; dim++){ - coords(node_lid, dim) = node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), dim); + // Getting the coordinates of the element + for (size_t node_lid = 0; node_lid < 8; node_lid++) + { + for (size_t dim = 0; dim < num_dims; dim++) + { + coords(node_lid, dim) = node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), dim); } // end for dim - } // end for loop over node_lid // loop conditions needed for distance calculation @@ -90,148 +82,150 @@ void FEA_Module_Eulerian::get_timestep(mesh_t &mesh, size_t a; size_t b; size_t loop = 0; - + // Only works for 3D // Solving for the magnitude of distance between each node - for (size_t i = 0; i < 28; i++){ - + for (size_t i = 0; i < 28; i++) + { a = countA; b = countB; - + // returns magnitude of distance between each node, 28 total options - dist(i) = fabs(sqrt(( pow((coords(b, 0) - coords(a, 0)), 2.0) - + pow((coords(b, 1) - coords(a, 1)), 2.0) - + pow((coords(b, 2) - coords(a, 2)), 2.0)))); + dist(i) = fabs(sqrt((pow((coords(b, 0) - coords(a, 0)), 2.0) + + pow((coords(b, 1) - coords(a, 1)), 2.0) + + pow((coords(b, 2) - coords(a, 2)), 2.0)))); countB++; countA++; - + // tricky indexing - if (countB > 7) { + if (countB > 7) + { loop++; countB = 1 + loop; countA = 0; } - } // endo for i - double dist_min = dist(0); - - for(int i = 0; i < 28; ++i){ + + for (int i = 0; i < 28; ++i) + { dist_min = fmin(dist(i), dist_min); } - + // local dt calc based on CFL - double dt_lcl_ = dt_cfl*dist_min/(elem_sspd(elem_gid) + fuzz); - + double dt_lcl_ = dt_cfl * dist_min / (elem_sspd(elem_gid) + fuzz); + // make dt be in bounds dt_lcl_ = fmin(dt_lcl_, dt_max); // make dt small than dt_max dt_lcl_ = fmax(dt_lcl_, dt_min); // make dt larger than dt_min - - - if (dt_lcl_ < dt_lcl) dt_lcl = dt_lcl_; - + + if (dt_lcl_ < dt_lcl) + { + dt_lcl = dt_lcl_; + } }, min_dt_calc); // end parallel reduction Kokkos::fence(); - + // save the min dt - if(min_dt_calc < dt) dt = min_dt_calc; - + if (min_dt_calc < dt) + { + dt = min_dt_calc; + } + // ensure time step hits the graphics time intervals - dt = fmin(dt, (graphics_time - time_value)+fuzz); - + dt = fmin(dt, (graphics_time - time_value) + fuzz); + // make dt be exact for final time - dt = fmin(dt, time_final-time_value); - + dt = fmin(dt, time_final - time_value); + return; - } // end get_timestep // ----------------------------------------------------------------------------- // This function calculates the time step by finding the shortest distance // between any two nodes in the mesh -//------------------------------------------------------------------------------ +// ------------------------------------------------------------------------------ // WARNING WARNING : Only works for 3D, 8 node elements -void FEA_Module_Eulerian::get_timestep2D(mesh_t &mesh, - DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &elem_sspd, - DViewCArrayKokkos &elem_vol){ - --> const size_t rk_level = rk_num_bins - 1; +void FEA_Module_Eulerian::get_timestep2D(mesh_t& mesh, + DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_sspd, + DViewCArrayKokkos& elem_vol) +{ + ->const size_t rk_level = rk_num_bins - 1; // increase dt by 10%, that is the largest dt value - dt = dt*1.1; - int num_dims = num_dim; + dt = dt * 1.1; + int num_dims = num_dim; double dt_lcl; double min_dt_calc; REDUCE_MIN_CLASS(elem_gid, 0, rnum_elem, dt_lcl, { - double coords0[8]; // element coords - ViewCArrayKokkos coords(coords0, 4, 2); + ViewCArrayKokkos coords(coords0, 4, 2); - double distance0[6]; // array for holding distances between each node - ViewCArrayKokkos dist(distance0, 6); - - // Getting the coordinates of the nodes of the element - for(size_t node_lid = 0; node_lid < 4; node_lid++){ + ViewCArrayKokkos dist(distance0, 6); - for (size_t dim = 0; dim < num_dims; dim++){ - coords(node_lid, dim) = node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), dim); + // Getting the coordinates of the nodes of the element + for (size_t node_lid = 0; node_lid < 4; node_lid++) + { + for (size_t dim = 0; dim < num_dims; dim++) + { + coords(node_lid, dim) = node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), dim); } // end for dim - } // end for loop over node_lid - // Only works for 2D // Solving for the magnitude of distance between each node size_t count = 0; - for (size_t i = 0; i < 3; i++){ - for (size_t j=i+1; j<=3; j++){ - + for (size_t i = 0; i < 3; i++) + { + for (size_t j = i + 1; j <= 3; j++) + { // returns magnitude of distance between each node, 6 total options dist(count) = fabs( - sqrt( pow((coords(i, 0) - coords(j, 0)), 2.0) - + pow((coords(i, 1) - coords(j, 1)), 2.0) ) - ); - count ++; + sqrt(pow((coords(i, 0) - coords(j, 0)), 2.0) + + pow((coords(i, 1) - coords(j, 1)), 2.0) ) + ); + count++; } // end for j } // end for i - double dist_min = dist(0); - - for(int i = 0; i < 6; ++i){ + + for (int i = 0; i < 6; ++i) + { dist_min = fmin(dist(i), dist_min); } - + // local dt calc based on CFL - double dt_lcl_ = dt_cfl*dist_min/(elem_sspd(elem_gid) + fuzz); - + double dt_lcl_ = dt_cfl * dist_min / (elem_sspd(elem_gid) + fuzz); + // make dt be in bounds dt_lcl_ = fmin(dt_lcl_, dt_max); // make dt small than dt_max dt_lcl_ = fmax(dt_lcl_, dt_min); // make dt larger than dt_min - - - if (dt_lcl_ < dt_lcl) dt_lcl = dt_lcl_; - + + if (dt_lcl_ < dt_lcl) + { + dt_lcl = dt_lcl_; + } }, min_dt_calc); // end parallel reduction Kokkos::fence(); - + // save the min dt - if(min_dt_calc < dt) dt = min_dt_calc; - + if (min_dt_calc < dt) + { + dt = min_dt_calc; + } + // ensure time step hits the graphics time intervals - dt = fmin(dt, (graphics_time - time_value)+fuzz); - + dt = fmin(dt, (graphics_time - time_value) + fuzz); + // make dt be exact for final time - dt = fmin(dt, time_final-time_value); - + dt = fmin(dt, time_final - time_value); + return; - } // end get_timestep2D - - diff --git a/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp b/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp index 3cee69731..37e6d91bb 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp @@ -11,14 +11,14 @@ This program is open source under the BSD-3 License. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - + 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - + 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - + 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. @@ -33,14 +33,14 @@ WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - **********************************************************************************************/ - +**********************************************************************************************/ + #include #include #include #include #include -#include +#include #include // fmin, fmax, abs note: fminl is long #include #include @@ -63,14 +63,14 @@ #include "Simulation_Parameters/Simulation_Parameters_Explicit.h" #include "Simulation_Parameters/FEA_Module/FEA_Module_Headers.h" #include "FEA_Module_SGH.h" -//#include "FEA_Module_Eulerian.h" +// #include "FEA_Module_Eulerian.h" #include "FEA_Module_Dynamic_Elasticity.h" #include "FEA_Module_Inertial.h" #include "Explicit_Solver.h" #include "mesh.h" #include "state.h" -//Optimization Package +// Optimization Package #include "ROL_Algorithm.hpp" #include "ROL_Solver.hpp" #include "ROL_LineSearchStep.hpp" @@ -85,8 +85,8 @@ #include "ROL_ParameterList.hpp" #include -//Objective Functions and Constraint Functions -//#include "Topology_Optimization_Function_Headers.h" +// Objective Functions and Constraint Functions +// #include "Topology_Optimization_Function_Headers.h" #include "Mass_Constraint.h" #include "Moment_of_Inertia_Constraint.h" #include "Kinetic_Energy_Minimize.h" @@ -103,1411 +103,1561 @@ using namespace mtr; /* -Swage is a reference to a swage block used in blacksmithing. -Its a large metal block that has multiple shaps carved into -each surface to use for hammering metal into to form it. +Swage is a reference to a swage block used in blacksmithing. +Its a large metal block that has multiple shaps carved into +each surface to use for hammering metal into to form it. */ -Explicit_Solver::Explicit_Solver(Simulation_Parameters_Explicit params) : Solver(params) { - //create parameter objects - simparam = params; - //create ref element object - ref_elem = std::make_shared(); - //create mesh objects - //init_mesh = new swage::mesh_t(simparaA); - mesh = std::make_shared(); - - element_select = std::make_shared(); - num_nodes = 0; - - //boundary condition data - current_bdy_id = 0; - - //Trilinos output stream - std::ostream &out = std::cout; - fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - (*fos).setOutputToRootOnly(0); - - //file readin parameter - active_node_ordering_convention = ENSIGHT; - //default simulation parameters - time_value = 0; +Explicit_Solver::Explicit_Solver(Simulation_Parameters_Explicit params) : Solver(params) +{ + // create parameter objects + simparam = params; + // create ref element object + ref_elem = std::make_shared(); + // create mesh objects + // init_mesh = new swage::mesh_t(simparaA); + mesh = std::make_shared(); + + element_select = std::make_shared(); + num_nodes = 0; + + // boundary condition data + current_bdy_id = 0; + + // Trilinos output stream + std::ostream& out = std::cout; + fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + (*fos).setOutputToRootOnly(0); + + // file readin parameter + active_node_ordering_convention = ENSIGHT; + // default simulation parameters + time_value = 0; } -Explicit_Solver::~Explicit_Solver(){ - if (myrank == 0 && in != NULL) - delete in; +Explicit_Solver::~Explicit_Solver() +{ + if (myrank == 0 && in != NULL) + { + delete in; + } } -//============================================================================== +// ============================================================================== // Primary simulation runtime routine -//============================================================================== - - -void Explicit_Solver::run() { - - //MPI info - world = MPI_COMM_WORLD; //used for convenience to represent all the ranks in the job - MPI_Comm_rank(world,&myrank); - MPI_Comm_size(world,&nranks); - - if(myrank == 0){ - std::cout << "Starting Lagrangian SGH code" << std::endl; - } - - //initialize Trilinos communicator class - comm = Tpetra::getDefaultComm(); - - if (simparam.input_options.has_value()) { - const Input_Options& input_options = simparam.input_options.value(); - const char* mesh_file_name = input_options.mesh_file_name.c_str(); - switch (input_options.mesh_file_format) { - case MESH_FORMAT::tecplot: - read_mesh_tecplot(mesh_file_name); - break; - case MESH_FORMAT::vtk: - read_mesh_vtk(mesh_file_name); - break; - case MESH_FORMAT::ansys_dat: - read_mesh_ansys_dat(mesh_file_name); - break; - case MESH_FORMAT::ensight: - read_mesh_ensight(mesh_file_name); - break; - } - } else { - generate_mesh(simparam.mesh_generation_options.value()); - } - - //debug - //return; - init_maps(); - - init_state_vectors(); - - //set initial saved coordinates - //initial_node_coords_distributed->assign(*node_coords_distributed); - all_initial_node_coords_distributed->assign(*all_node_coords_distributed); - initial_node_coords_distributed = Teuchos::rcp(new MV(*all_initial_node_coords_distributed, map)); - - std::cout << "Num elements on process " << myrank << " = " << rnum_elem << std::endl; - - //initialize timing - init_clock(); - - //initialize runtime counters and timers - int hessvec_count = 0; - int update_count = 0; - file_index = 0; - real_t linear_solve_time = 0; - real_t hessvec_time = 0; - real_t hessvec_linear_time = 0; - - // ---- Find Boundaries on mesh ---- // - init_boundaries(); - - //set boundary conditions - //generate_tcs(); - //initialize TO design variable storage - if(simparam.topology_optimization_on || simparam.shape_optimization_on) - init_design(); - //process process list of requested FEA modules to construct list of objects - FEA_module_setup(); - - //Have modules read in boundary/loading conditions if file format provides it - /* - for(int imodule = 0; imodule < nfea_modules; imodule++){ - if(fea_module_must_read[imodule]){ - fea_modules[imodule]->read_conditions_ansys_dat(in, before_condition_header); +// ============================================================================== + +void Explicit_Solver::run() +{ + // MPI info + world = MPI_COMM_WORLD; // used for convenience to represent all the ranks in the job + MPI_Comm_rank(world, &myrank); + MPI_Comm_size(world, &nranks); + + if (myrank == 0) + { + std::cout << "Starting Lagrangian SGH code" << std::endl; } - else{ - fea_modules[imodule]->init_boundaries(); - - //set boundary conditions for FEA modules - fea_modules[imodule]->generate_bcs(); - - //set applied loading conditions for FEA modules - fea_modules[imodule]->generate_applied_loads(); - } - } - - //std::cout << "FEA MODULES " << nfea_modules << " " << simparam->nfea_modules << std::endl; - //call boundary routines on fea modules - - if(myrank == 0) - std::cout << "Starting init assembly" << std::endl <init_assembly(); - - //assemble the global solution (stiffness matrix etc. and nodal forces) - for(int imodule = 0; imodule < nfea_modules; imodule++) - fea_modules[imodule]->assemble_matrix(); - - if(myrank == 0) - std::cout << "Finished matrix assembly" << std::endl <assemble_vector(); - - for(int imodule = 0; imodule < nfea_modules; imodule++) - fea_modules[imodule]->linear_solver_parameters(); - - if(myrank == 0) - std::cout << "Starting First Solve" << std::endl <solve(); - if(solver_exit != EXIT_SUCCESS){ - std::cout << "Linear Explicit_Solver Error" << std::endl < design_gradients_distributed = Teuchos::rcp(new MV(map, 1)); - const_host_vec_array node_densities = design_node_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); - host_vec_array design_gradients = design_gradients_distributed->getLocalView (Tpetra::Access::ReadWrite); - compute_adjoint_gradients(node_densities, design_gradients); - std::ostream &out = std::cout; - Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - if(myrank==0) - *fos << "Strain Energy gradients :" << std::endl; - design_gradients_distributed->describe(*fos,Teuchos::VERB_EXTREME); - *fos << std::endl; - std::fflush(stdout); - */ - //return; - - //solver_exit = solve(); - //if(solver_exit == EXIT_SUCCESS){ - //std::cout << "Linear Explicit_Solver Error" << std::endl <setup(); - - for(int imodule = 0; imodule < nfea_modules; imodule++){ - if(myrank == 0) - std::cout << "Starting setup for FEA module " << imodule <setup(); - } - - //set initial saved velocities - initial_node_velocities_distributed->assign(*node_velocities_distributed); - - if(simparam.topology_optimization_on || simparam.shape_optimization_on){ - //design_node_densities_distributed->randomize(1,1); - setup_optimization_problem(); - //problem = ROL::makePtr>(obj,x); - } - else{ - // --------------------------------------------------------------------- - // Calculate the SGH solution - // --------------------------------------------------------------------- - //sgh_module->sgh_solve(); - for(int imodule = 0; imodule < nfea_modules; imodule++){ - if(myrank == 0) - std::cout << "Starting solve for FEA module " << imodule <solve(); + + // initialize Trilinos communicator class + comm = Tpetra::getDefaultComm(); + + if (simparam.input_options.has_value()) + { + const Input_Options& input_options = simparam.input_options.value(); + const char* mesh_file_name = input_options.mesh_file_name.c_str(); + switch (input_options.mesh_file_format) + { + case MESH_FORMAT::tecplot: + read_mesh_tecplot(mesh_file_name); + break; + case MESH_FORMAT::vtk: + read_mesh_vtk(mesh_file_name); + break; + case MESH_FORMAT::ansys_dat: + read_mesh_ansys_dat(mesh_file_name); + break; + case MESH_FORMAT::ensight: + read_mesh_ensight(mesh_file_name); + break; + } + } + else + { + generate_mesh(simparam.mesh_generation_options.value()); + } + + // debug + // return; + init_maps(); + + init_state_vectors(); + + // set initial saved coordinates + // initial_node_coords_distributed->assign(*node_coords_distributed); + all_initial_node_coords_distributed->assign(*all_node_coords_distributed); + initial_node_coords_distributed = Teuchos::rcp(new MV(*all_initial_node_coords_distributed, map)); + + std::cout << "Num elements on process " << myrank << " = " << rnum_elem << std::endl; + + // initialize timing + init_clock(); + + // initialize runtime counters and timers + int hessvec_count = 0; + int update_count = 0; + file_index = 0; + real_t linear_solve_time = 0; + real_t hessvec_time = 0; + real_t hessvec_linear_time = 0; + + // ---- Find Boundaries on mesh ---- // + init_boundaries(); + + // set boundary conditions + // generate_tcs(); + // initialize TO design variable storage + if (simparam.topology_optimization_on || simparam.shape_optimization_on) + { + init_design(); + } + // process process list of requested FEA modules to construct list of objects + FEA_module_setup(); + + // Have modules read in boundary/loading conditions if file format provides it + /* + for(int imodule = 0; imodule < nfea_modules; imodule++){ + if(fea_module_must_read[imodule]){ + fea_modules[imodule]->read_conditions_ansys_dat(in, before_condition_header); + } + else{ + fea_modules[imodule]->init_boundaries(); + + //set boundary conditions for FEA modules + fea_modules[imodule]->generate_bcs(); + + //set applied loading conditions for FEA modules + fea_modules[imodule]->generate_applied_loads(); } - } + } + + //std::cout << "FEA MODULES " << nfea_modules << " " << simparam->nfea_modules << std::endl; + //call boundary routines on fea modules - // clean up all material models - //sgh_module->cleanup_material_models(); - for(int imodule = 0; imodule < nfea_modules; imodule++){ if(myrank == 0) - std::cout << "Starting solve for FEA module " << imodule <module_cleanup(); - } - - //printf("Finished\n"); - - //benchmark simulation run time end - double current_cpu = CPU_Time(); - /* - for(int imodule = 0; imodule < nfea_modules; imodule++){ - linear_solve_time += fea_modules[imodule]->linear_solve_time; - hessvec_linear_time += fea_modules[imodule]->hessvec_linear_time; - } - */ - - std::cout << " RUNTIME OF CODE ON TASK " << myrank << " is "<< current_cpu-initial_CPU_time << " comms time " - << communication_time << " host to dev time " << host2dev_time << " dev to host time " << dev2host_time << std::endl; - - if(simparam.output_options.timer_output_level == TIMER_VERBOSITY::thorough){ - std::cout << " OUTPUT TIME OF CODE ON TASK " << myrank << " is "<< output_time << std::endl; - } - - //parallel_vtk_writer(); - - //test forward solve call - int ntests = 0; - if(simparam.topology_optimization_on){ - for(int itest = 0; itest < ntests; itest++){ - design_node_densities_distributed->randomize(1,1); - //test_node_densities_distributed = Teuchos::rcp(new MV(*design_node_densities_distributed)); - Teuchos::RCP test_gradients_distributed = Teuchos::rcp(new MV(map, 1)); - const_vec_array test_node_densities = design_node_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); - vec_array test_gradients = test_gradients_distributed->getLocalView (Tpetra::Access::ReadWrite); - - fea_modules[0]->comm_variables(design_node_densities_distributed); - fea_modules[0]->update_forward_solve(design_node_densities_distributed); - fea_modules[0]->compute_topology_optimization_adjoint(); - fea_modules[0]->compute_topology_optimization_gradient(test_node_densities, test_gradients); - // Data writers - //parallel_vtk_writer(); - } - } - - // vtk_writer(); - /* - if(myrank==0){ - std::cout << "Total number of solves and assembly " << fea_modules[0]->update_count <hessvec_count <init_assembly(); + + //assemble the global solution (stiffness matrix etc. and nodal forces) + for(int imodule = 0; imodule < nfea_modules; imodule++) + fea_modules[imodule]->assemble_matrix(); + + if(myrank == 0) + std::cout << "Finished matrix assembly" << std::endl <assemble_vector(); + + for(int imodule = 0; imodule < nfea_modules; imodule++) + fea_modules[imodule]->linear_solver_parameters(); + + if(myrank == 0) + std::cout << "Starting First Solve" << std::endl <solve(); + if(solver_exit != EXIT_SUCCESS){ + std::cout << "Linear Explicit_Solver Error" << std::endl < design_gradients_distributed = Teuchos::rcp(new MV(map, 1)); + const_host_vec_array node_densities = design_node_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); + host_vec_array design_gradients = design_gradients_distributed->getLocalView (Tpetra::Access::ReadWrite); + compute_adjoint_gradients(node_densities, design_gradients); + std::ostream &out = std::cout; + Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + if(myrank==0) + *fos << "Strain Energy gradients :" << std::endl; + design_gradients_distributed->describe(*fos,Teuchos::VERB_EXTREME); + *fos << std::endl; + std::fflush(stdout); + */ + // return; + + // solver_exit = solve(); + // if(solver_exit == EXIT_SUCCESS){ + // std::cout << "Linear Explicit_Solver Error" << std::endl <setup(); + + for (int imodule = 0; imodule < nfea_modules; imodule++) + { + if (myrank == 0) + { + std::cout << "Starting setup for FEA module " << imodule << std::endl << std::flush; + } + // allocate and fill sparse structures needed for global solution in each FEA module + fea_modules[imodule]->setup(); + } + + // set initial saved velocities + initial_node_velocities_distributed->assign(*node_velocities_distributed); + + if (simparam.topology_optimization_on || simparam.shape_optimization_on) + { + // design_node_densities_distributed->randomize(1,1); + setup_optimization_problem(); + // problem = ROL::makePtr>(obj,x); + } + else + { + // --------------------------------------------------------------------- + // Calculate the SGH solution + // --------------------------------------------------------------------- + // sgh_module->sgh_solve(); + for (int imodule = 0; imodule < nfea_modules; imodule++) + { + if (myrank == 0) + { + std::cout << "Starting solve for FEA module " << imodule << std::endl << std::flush; + } + // allocate and fill sparse structures needed for global solution in each FEA module + fea_modules[imodule]->solve(); + } + } + + // clean up all material models + // sgh_module->cleanup_material_models(); + for (int imodule = 0; imodule < nfea_modules; imodule++) + { + if (myrank == 0) + { + std::cout << "Starting solve for FEA module " << imodule << std::endl << std::flush; + } + // allocate and fill sparse structures needed for global solution in each FEA module + fea_modules[imodule]->module_cleanup(); + } + + // printf("Finished\n"); + + // benchmark simulation run time end + double current_cpu = CPU_Time(); + /* + for(int imodule = 0; imodule < nfea_modules; imodule++){ + linear_solve_time += fea_modules[imodule]->linear_solve_time; + hessvec_linear_time += fea_modules[imodule]->hessvec_linear_time; + } + */ + + std::cout << " RUNTIME OF CODE ON TASK " << myrank << " is " << current_cpu - initial_CPU_time << " comms time " + << communication_time << " host to dev time " << host2dev_time << " dev to host time " << dev2host_time << std::endl; + + if (simparam.output_options.timer_output_level == TIMER_VERBOSITY::thorough) + { + std::cout << " OUTPUT TIME OF CODE ON TASK " << myrank << " is " << output_time << std::endl; + } + + // parallel_vtk_writer(); + + // test forward solve call + int ntests = 0; + if (simparam.topology_optimization_on) + { + for (int itest = 0; itest < ntests; itest++) + { + design_node_densities_distributed->randomize(1, 1); + // test_node_densities_distributed = Teuchos::rcp(new MV(*design_node_densities_distributed)); + Teuchos::RCP test_gradients_distributed = Teuchos::rcp(new MV(map, 1)); + const_vec_array test_node_densities = design_node_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); + vec_array test_gradients = test_gradients_distributed->getLocalView(Tpetra::Access::ReadWrite); + + fea_modules[0]->comm_variables(design_node_densities_distributed); + fea_modules[0]->update_forward_solve(design_node_densities_distributed); + fea_modules[0]->compute_topology_optimization_adjoint(); + fea_modules[0]->compute_topology_optimization_gradient(test_node_densities, test_gradients); + // Data writers + // parallel_vtk_writer(); + } + } + + // vtk_writer(); + /* + if(myrank==0){ + std::cout << "Total number of solves and assembly " << fea_modules[0]->update_count <hessvec_count < read_buffer; - int buffer_loop, buffer_iteration, buffer_iterations, dof_limit, scan_loop, nodes_per_element; - size_t read_index_start, node_rid, elem_gid; - GO node_gid; - real_t dof_value; - host_vec_array node_densities; - bool zero_index_base = input_options.zero_index_base; - int negative_index_found = 0; - int global_negative_index_found = 0; - //Nodes_Per_Element_Type = elements::elem_types::Nodes_Per_Element_Type; - - //read the mesh - //PLACEHOLDER: ensight_format(MESH); - // abaqus_format(MESH); - // vtk_format(MESH) - - //task 0 reads file - if(myrank==0){ - in = new std::ifstream(); - in->open(MESH); - if (!(*in)) throw std::runtime_error(std::string("Can't open ") + MESH); - } - - //ANSYS dat file doesn't specify total number of nodes, which is needed for the node map. - //First pass reads in node section to determine the maximum number of nodes, second pass distributes node data - //The elements section header does specify element count - num_nodes = 0; - if(myrank==0){ - bool searching_for_nodes = true; - //skip lines at the top with nonessential info; stop skipping when "Nodes for the whole assembly" string is reached - while (searching_for_nodes&&in->good()) { - getline(*in, skip_line); - //std::cout << skip_line << std::endl; - line_parse.clear(); - line_parse.str(skip_line); - //stop when the NODES= string is reached - while (!line_parse.eof()){ - line_parse >> substring; - //std::cout << substring << std::endl; - if(!substring.compare("Nodes")){ - searching_for_nodes = false; - break; - } - } //while - - } - if(searching_for_nodes){ - std::cout << "FILE FORMAT ERROR" << std::endl; - } - //skip 2 lines - for (int j = 0; j < 2; j++) { - getline(*in, skip_line); - std::cout << skip_line << std::endl; - } //for - - //tally node count (bug in dat files seems to print wrong node count so read is done in two passes) - //stop when apparent "-1" zone delimiter is reacher - searching_for_nodes = true; - int node_tally = 0; - while (searching_for_nodes) { - getline(*in, read_line); - //std::cout << read_line << std::endl; - line_parse.clear(); - line_parse.str(read_line); - line_parse >> substring; - - //std::cout << substring << std::endl; - if(substring == "-1"){ - searching_for_nodes = false; - break; - } - else{ - node_tally++; +void Explicit_Solver::read_mesh_ansys_dat(const char* MESH) +{ + char ch; + int num_dim = simparam.num_dims; + Input_Options input_options = simparam.input_options.value(); + int p_order = input_options.p_order; + real_t unit_scaling = input_options.unit_scaling; + bool restart_file = simparam.restart_file; + int local_node_index, current_column_index; + size_t strain_count; + std::string skip_line, read_line, substring, token; + std::stringstream line_parse; + CArrayKokkos read_buffer; + int buffer_loop, buffer_iteration, buffer_iterations, dof_limit, scan_loop, nodes_per_element; + size_t read_index_start, node_rid, elem_gid; + GO node_gid; + real_t dof_value; + host_vec_array node_densities; + bool zero_index_base = input_options.zero_index_base; + int negative_index_found = 0; + int global_negative_index_found = 0; + // Nodes_Per_Element_Type = elements::elem_types::Nodes_Per_Element_Type; + + // read the mesh + // PLACEHOLDER: ensight_format(MESH); + // abaqus_format(MESH); + // vtk_format(MESH) + + // task 0 reads file + if (myrank == 0) + { + in = new std::ifstream(); + in->open(MESH); + if (!(*in)) + { + throw std::runtime_error(std::string("Can't open ") + MESH); + } + } + + // ANSYS dat file doesn't specify total number of nodes, which is needed for the node map. + // First pass reads in node section to determine the maximum number of nodes, second pass distributes node data + // The elements section header does specify element count + num_nodes = 0; + if (myrank == 0) + { + bool searching_for_nodes = true; + // skip lines at the top with nonessential info; stop skipping when "Nodes for the whole assembly" string is reached + while (searching_for_nodes && in->good()) { + getline(*in, skip_line); + // std::cout << skip_line << std::endl; + line_parse.clear(); + line_parse.str(skip_line); + // stop when the NODES= string is reached + while (!line_parse.eof()) { + line_parse >> substring; + // std::cout << substring << std::endl; + if (!substring.compare("Nodes")) + { + searching_for_nodes = false; + break; + } + } // while + } + if (searching_for_nodes) + { + std::cout << "FILE FORMAT ERROR" << std::endl; + } + // skip 2 lines + for (int j = 0; j < 2; j++) + { + getline(*in, skip_line); + std::cout << skip_line << std::endl; + } // for + + // tally node count (bug in dat files seems to print wrong node count so read is done in two passes) + // stop when apparent "-1" zone delimiter is reacher + searching_for_nodes = true; + int node_tally = 0; + while (searching_for_nodes) { + getline(*in, read_line); + // std::cout << read_line << std::endl; + line_parse.clear(); + line_parse.str(read_line); + line_parse >> substring; + + // std::cout << substring << std::endl; + if (substring == "-1") + { + searching_for_nodes = false; + break; + } + else + { + node_tally++; + } + } + num_nodes = node_tally; + std::cout << "declared node count: " << num_nodes << std::endl; + } + + // broadcast number of nodes + MPI_Bcast(&num_nodes, 1, MPI_LONG_LONG_INT, 0, world); + + // construct contiguous parallel row map now that we know the number of nodes + map = Teuchos::rcp(new Tpetra::Map(num_nodes, 0, comm)); + + // close and reopen file for second pass now that global node count is known + if (myrank == 0) + { + in->close(); + // in = new std::ifstream(); + in->open(MESH); + } + + // set the vertices in the mesh read in + nlocal_nodes = map->getLocalNumElements(); + // populate local row offset data from global data + global_size_t min_gid = map->getMinGlobalIndex(); + global_size_t max_gid = map->getMaxGlobalIndex(); + global_size_t index_base = map->getIndexBase(); + // debug print + // std::cout << "local node count on task: " << " " << nlocal_nodes << std::endl; + + // allocate node storage with dual view + // dual_node_coords = dual_vec_array("dual_node_coords", nlocal_nodes,num_dim); + // if(restart_file) + // dual_node_densities = dual_vec_array("dual_node_densities", nlocal_nodes,1); + + // local variable for host view in the dual view + node_coords_distributed = Teuchos::rcp(new MV(map, num_dim)); + // active view scope + { + host_vec_array node_coords = node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + // host_vec_array node_coords = dual_node_coords.view_host(); + if (restart_file) + { + design_node_densities_distributed = Teuchos::rcp(new MV(map, 1)); + node_densities = design_node_densities_distributed->getLocalView(Tpetra::Access::ReadWrite); + } + // notify that the host view is going to be modified in the file readin + // dual_node_coords.modify_host(); + // if(restart_file) + // dual_node_densities.modify_host(); + + // old swage method + // mesh->init_nodes(local_nrows); // add 1 for index starting at 1 + + std::cout << "Num nodes assigned to task " << myrank << " = " << nlocal_nodes << std::endl; + + // read the initial mesh coordinates + // x-coords + /*only task 0 reads in nodes and elements from the input file + stores node data in a buffer and communicates once the buffer cap is reached + or the data ends*/ + + words_per_line = input_options.words_per_line; + // if(restart_file) words_per_line++; + elem_words_per_line = input_options.elem_words_per_line; + + // allocate read buffer + read_buffer = CArrayKokkos(BUFFER_LINES, words_per_line, MAX_WORD); + + dof_limit = num_nodes; + buffer_iterations = dof_limit / BUFFER_LINES; + if (dof_limit % BUFFER_LINES != 0) + { + buffer_iterations++; + } + + // second pass to now read node coords with global node map defines + if (myrank == 0) + { + bool searching_for_nodes = true; + // skip lines at the top with nonessential info; stop skipping when "Nodes for the whole assembly" string is reached + while (searching_for_nodes && in->good()) { + getline(*in, skip_line); + // std::cout << skip_line << std::endl; + line_parse.clear(); + line_parse.str(skip_line); + // stop when the NODES= string is reached + while (!line_parse.eof()) { + line_parse >> substring; + // std::cout << substring << std::endl; + if (!substring.compare("Nodes")) + { + searching_for_nodes = false; + break; + } + } // while + } + if (searching_for_nodes) + { + std::cout << "FILE FORMAT ERROR" << std::endl; + } + // skip 2 lines + for (int j = 0; j < 2; j++) + { + getline(*in, skip_line); + std::cout << skip_line << std::endl; + } // for + } + + // read coords, also density if restarting + read_index_start = 0; + for (buffer_iteration = 0; buffer_iteration < buffer_iterations; buffer_iteration++) + { + // pack buffer on rank 0 + if (myrank == 0 && buffer_iteration < buffer_iterations - 1) + { + for (buffer_loop = 0; buffer_loop < BUFFER_LINES; buffer_loop++) + { + getline(*in, read_line); + line_parse.clear(); + line_parse.str(read_line); + + for (int iword = 0; iword < words_per_line; iword++) + { + // read portions of the line into the substring variable + line_parse >> substring; + // debug print + // std::cout<<" "<< substring <> substring; + // assign the substring variable as a word of the read buffer + strcpy(&read_buffer(buffer_loop, iword, 0), substring.c_str()); + } + buffer_loop++; + } + } + + // broadcast buffer to all ranks; each rank will determine which nodes in the buffer belong + MPI_Bcast(read_buffer.pointer(), BUFFER_LINES * words_per_line * MAX_WORD, MPI_CHAR, 0, world); + // broadcast how many nodes were read into this buffer iteration + MPI_Bcast(&buffer_loop, 1, MPI_INT, 0, world); + + // debug_print + // std::cout << "NODE BUFFER LOOP IS: " << buffer_loop << std::endl; + // for(int iprint=0; iprint < buffer_loop; iprint++) + // std::cout<<"buffer packing: " << std::string(&read_buffer(iprint,0,0)) << std::endl; + // return; + + // determine which data to store in the swage mesh members (the local node data) + // loop through read buffer + for (scan_loop = 0; scan_loop < buffer_loop; scan_loop++) + { + // set global node id (ensight specific order) + node_gid = read_index_start + scan_loop; + // let map decide if this node id belongs locally; if yes store data + if (map->isNodeGlobalElement(node_gid)) + { + // set local node index in this mpi rank + node_rid = map->getLocalElement(node_gid); + // extract nodal position from the read buffer + // for tecplot format this is the three coords in the same line + dof_value = atof(&read_buffer(scan_loop, 1, 0)); + node_coords(node_rid, 0) = dof_value * unit_scaling; + dof_value = atof(&read_buffer(scan_loop, 2, 0)); + node_coords(node_rid, 1) = dof_value * unit_scaling; + if (num_dim == 3) + { + dof_value = atof(&read_buffer(scan_loop, 3, 0)); + node_coords(node_rid, 2) = dof_value * unit_scaling; + } + // extract density if restarting + } + } + read_index_start += BUFFER_LINES; + } + } // end view scope + // repartition node distribution + repartition_nodes(); + + // synchronize device data + // dual_node_coords.sync_device(); + // dual_node_coords.modify_device(); + // if(restart_file){ + // dual_node_densities.sync_device(); + // dual_node_densities.modify_device(); + // } + + // debug print of nodal data + + // debug print nodal positions and indices + /* + std::cout << " ------------NODAL POSITIONS ON TASK " << myrank << " --------------"<getGlobalElement(inode) + 1 << " { "; + for (int istride = 0; istride < num_dim; istride++){ + std::cout << node_coords(inode,istride) << " , "; } - - } - num_nodes = node_tally; - std::cout << "declared node count: " << num_nodes << std::endl; - } - - //broadcast number of nodes - MPI_Bcast(&num_nodes,1,MPI_LONG_LONG_INT,0,world); - - //construct contiguous parallel row map now that we know the number of nodes - map = Teuchos::rcp( new Tpetra::Map(num_nodes,0,comm)); - - //close and reopen file for second pass now that global node count is known - if(myrank==0){ - in->close(); - //in = new std::ifstream(); - in->open(MESH); - } - - // set the vertices in the mesh read in - nlocal_nodes = map->getLocalNumElements(); - //populate local row offset data from global data - global_size_t min_gid = map->getMinGlobalIndex(); - global_size_t max_gid = map->getMaxGlobalIndex(); - global_size_t index_base = map->getIndexBase(); - //debug print - //std::cout << "local node count on task: " << " " << nlocal_nodes << std::endl; - - //allocate node storage with dual view - //dual_node_coords = dual_vec_array("dual_node_coords", nlocal_nodes,num_dim); - //if(restart_file) - //dual_node_densities = dual_vec_array("dual_node_densities", nlocal_nodes,1); - - //local variable for host view in the dual view - node_coords_distributed = Teuchos::rcp(new MV(map, num_dim)); - //active view scope - { - host_vec_array node_coords = node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); - //host_vec_array node_coords = dual_node_coords.view_host(); - if(restart_file){ - design_node_densities_distributed = Teuchos::rcp(new MV(map, 1)); - node_densities = design_node_densities_distributed->getLocalView (Tpetra::Access::ReadWrite); - } - //notify that the host view is going to be modified in the file readin - //dual_node_coords.modify_host(); - //if(restart_file) - //dual_node_densities.modify_host(); - - //old swage method - //mesh->init_nodes(local_nrows); // add 1 for index starting at 1 - - std::cout << "Num nodes assigned to task " << myrank << " = " << nlocal_nodes << std::endl; - - // read the initial mesh coordinates - // x-coords - /*only task 0 reads in nodes and elements from the input file - stores node data in a buffer and communicates once the buffer cap is reached - or the data ends*/ - - words_per_line = input_options.words_per_line; - //if(restart_file) words_per_line++; - elem_words_per_line = input_options.elem_words_per_line; - - //allocate read buffer - read_buffer = CArrayKokkos(BUFFER_LINES,words_per_line,MAX_WORD); - - dof_limit = num_nodes; - buffer_iterations = dof_limit/BUFFER_LINES; - if(dof_limit%BUFFER_LINES!=0) buffer_iterations++; - - //second pass to now read node coords with global node map defines - if(myrank==0){ - bool searching_for_nodes = true; - //skip lines at the top with nonessential info; stop skipping when "Nodes for the whole assembly" string is reached - while (searching_for_nodes&&in->good()) { - getline(*in, skip_line); - //std::cout << skip_line << std::endl; - line_parse.clear(); - line_parse.str(skip_line); - //stop when the NODES= string is reached - while (!line_parse.eof()){ - line_parse >> substring; - //std::cout << substring << std::endl; - if(!substring.compare("Nodes")){ - searching_for_nodes = false; - break; - } - } //while - - } - if(searching_for_nodes){ - std::cout << "FILE FORMAT ERROR" << std::endl; - } - //skip 2 lines - for (int j = 0; j < 2; j++) { - getline(*in, skip_line); - std::cout << skip_line << std::endl; - } //for - } - - //read coords, also density if restarting - read_index_start = 0; - for(buffer_iteration = 0; buffer_iteration < buffer_iterations; buffer_iteration++){ - //pack buffer on rank 0 - if(myrank==0&&buffer_iterationgood()) { + getline(*in, skip_line); + // std::cout << skip_line << std::endl; + line_parse.clear(); + line_parse.str(skip_line); + // stop when the NODES= string is reached + while (!line_parse.eof()) { + line_parse >> substring; + // std::cout << substring << std::endl; + if (!substring.compare("Elements")) + { + searching_for_elements = false; + break; + } + } // while + } + if (searching_for_elements) + { + std::cout << "FILE FORMAT ERROR" << std::endl; + } + + // read in element type from following line + getline(*in, read_line); + std::cout << read_line << std::endl; line_parse.clear(); line_parse.str(read_line); - - for(int iword = 0; iword < words_per_line; iword++){ - //read portions of the line into the substring variable line_parse >> substring; - //debug print - //std::cout<<" "<< substring <> substring; - //assign the substring variable as a word of the read buffer - strcpy(&read_buffer(buffer_loop,iword,0),substring.c_str()); + // parse element line out of jumble of comma delimited entries + line_parse.clear(); + line_parse.str(substring); + while (line_parse.good()) { + getline(line_parse, token, ','); } - buffer_loop++; - } - - } - - //broadcast buffer to all ranks; each rank will determine which nodes in the buffer belong - MPI_Bcast(read_buffer.pointer(),BUFFER_LINES*words_per_line*MAX_WORD,MPI_CHAR,0,world); - //broadcast how many nodes were read into this buffer iteration - MPI_Bcast(&buffer_loop,1,MPI_INT,0,world); - - //debug_print - //std::cout << "NODE BUFFER LOOP IS: " << buffer_loop << std::endl; - //for(int iprint=0; iprint < buffer_loop; iprint++) - //std::cout<<"buffer packing: " << std::string(&read_buffer(iprint,0,0)) << std::endl; - //return; - - //determine which data to store in the swage mesh members (the local node data) - //loop through read buffer - for(scan_loop = 0; scan_loop < buffer_loop; scan_loop++){ - //set global node id (ensight specific order) - node_gid = read_index_start + scan_loop; - //let map decide if this node id belongs locally; if yes store data - if(map->isNodeGlobalElement(node_gid)){ - //set local node index in this mpi rank - node_rid = map->getLocalElement(node_gid); - //extract nodal position from the read buffer - //for tecplot format this is the three coords in the same line - dof_value = atof(&read_buffer(scan_loop,1,0)); - node_coords(node_rid, 0) = dof_value * unit_scaling; - dof_value = atof(&read_buffer(scan_loop,2,0)); - node_coords(node_rid, 1) = dof_value * unit_scaling; - if(num_dim==3){ - dof_value = atof(&read_buffer(scan_loop,3,0)); - node_coords(node_rid, 2) = dof_value * unit_scaling; - } - //extract density if restarting - } - } - read_index_start+=BUFFER_LINES; - } - } //end view scope - //repartition node distribution - repartition_nodes(); - - //synchronize device data - //dual_node_coords.sync_device(); - //dual_node_coords.modify_device(); - //if(restart_file){ - //dual_node_densities.sync_device(); - //dual_node_densities.modify_device(); - //} - - //debug print of nodal data - - //debug print nodal positions and indices - /* - std::cout << " ------------NODAL POSITIONS ON TASK " << myrank << " --------------"<getGlobalElement(inode) + 1 << " { "; - for (int istride = 0; istride < num_dim; istride++){ - std::cout << node_coords(inode,istride) << " , "; - } - //std::cout << node_densities(inode,0); - std::cout << " }"<< std::endl; - } - */ - - //check that local assignments match global total - - - //read in element info - //seek element connectivity zone - int etype_index = 0; - if(myrank==0){ - bool searching_for_elements = true; - //skip lines at the top with nonessential info; stop skipping when "Nodes for the whole assembly" string is reached - while (searching_for_elements&&in->good()) { - getline(*in, skip_line); - //std::cout << skip_line << std::endl; - line_parse.clear(); - line_parse.str(skip_line); - //stop when the NODES= string is reached - while (!line_parse.eof()){ - line_parse >> substring; - //std::cout << substring << std::endl; - if(!substring.compare("Elements")){ - searching_for_elements = false; - break; + // element count should be the last token read in + num_elem = std::stoi(token); + + // skip line + for (int j = 0; j < 1; j++) + { + getline(*in, skip_line); + std::cout << skip_line << std::endl; } - } //while - } - if(searching_for_elements){ - std::cout << "FILE FORMAT ERROR" << std::endl; + + // broadcast element type + MPI_Bcast(&etype_index, 1, MPI_INT, 0, world); + + elements::elem_types::elem_type mesh_element_type; + int elem_words_per_line_no_nodes = elem_words_per_line; + + if (etype_index == 1) + { + mesh_element_type = elements::elem_types::Hex8; + nodes_per_element = 8; + elem_words_per_line += 8; + } + else if (etype_index == 2) + { + mesh_element_type = elements::elem_types::Hex20; + nodes_per_element = 20; + elem_words_per_line += 20; } + else if (etype_index == 3) + { + mesh_element_type = elements::elem_types::Hex32; + nodes_per_element = 32; + elem_words_per_line += 32; + } + else + { + *fos << "ERROR: ANSYS ELEMENT TYPE NOT FOUND OR RECOGNIZED" << std::endl; + exit_solver(0); + } + + // broadcast number of elements + MPI_Bcast(&num_elem, 1, MPI_LONG_LONG_INT, 0, world); + + *fos << "declared element count: " << num_elem << std::endl; + // std::cout<<"before initial mesh initialization"<> substring; - //std::cout << substring << std::endl; - if(!substring.compare("et,1,185")){ - //Hex8 type - etype_index = 1; + // read in element connectivity + // we're gonna reallocate for the words per line expected for the element connectivity + read_buffer = CArrayKokkos(BUFFER_LINES, elem_words_per_line, MAX_WORD); + CArrayKokkos node_store(nodes_per_element); + // calculate buffer iterations to read number of lines + buffer_iterations = num_elem / BUFFER_LINES; + int assign_flag; + + // dynamic buffer used to store elements before we know how many this rank needs + std::vector element_temp(BUFFER_LINES * elem_words_per_line); + std::vector global_indices_temp(BUFFER_LINES); + size_t buffer_max = BUFFER_LINES * elem_words_per_line; + size_t indices_buffer_max = BUFFER_LINES; + + if (num_elem % BUFFER_LINES != 0) + { + buffer_iterations++; } - else if(!substring.compare("et,1,186")){ - //Hex20 type - etype_index = 2; + read_index_start = 0; + // std::cout << "ELEMENT BUFFER ITERATIONS: " << buffer_iterations << std::endl; + rnum_elem = 0; + // std::cout << "BUFFER ITERATIONS IS: " << buffer_iterations << std::endl; + for (buffer_iteration = 0; buffer_iteration < buffer_iterations; buffer_iteration++) + { + // pack buffer on rank 0 + if (myrank == 0 && buffer_iteration < buffer_iterations - 1) + { + for (buffer_loop = 0; buffer_loop < BUFFER_LINES; buffer_loop++) + { + getline(*in, read_line); + line_parse.clear(); + line_parse.str(read_line); + for (int iword = 0; iword < elem_words_per_line; iword++) + { + // read portions of the line into the substring variable + line_parse >> substring; + // assign the substring variable as a word of the read buffer + strcpy(&read_buffer(buffer_loop, iword, 0), substring.c_str()); + } + } + } + else if (myrank == 0) + { + buffer_loop = 0; + while (buffer_iteration * BUFFER_LINES + buffer_loop < num_elem) { + getline(*in, read_line); + line_parse.clear(); + line_parse.str(read_line); + for (int iword = 0; iword < elem_words_per_line; iword++) + { + // read portions of the line into the substring variable + line_parse >> substring; + // assign the substring variable as a word of the read buffer + strcpy(&read_buffer(buffer_loop, iword, 0), substring.c_str()); + } + buffer_loop++; + // std::cout<<" "<< node_coords(node_gid, 0)<isNodeGlobalElement(node_gid) && !assign_flag) + { + assign_flag = 1; + rnum_elem++; + } + } + else + { + if (map->isNodeGlobalElement(node_gid - 1) && !assign_flag) + { + assign_flag = 1; + rnum_elem++; + } + } + } + + if (assign_flag) + { + for (int inode = 0; inode < nodes_per_element; inode++) + { + if ((rnum_elem - 1) * nodes_per_element + inode >= buffer_max) + { + element_temp.resize((rnum_elem - 1) * nodes_per_element + inode + BUFFER_LINES * nodes_per_element); + buffer_max = (rnum_elem - 1) * nodes_per_element + inode + BUFFER_LINES * nodes_per_element; + } + element_temp[(rnum_elem - 1) * nodes_per_element + inode] = node_store(inode); + // std::cout << "VECTOR STORAGE FOR ELEM " << rnum_elem << " ON TASK " << myrank << " NODE " << inode+1 << " IS " << node_store(inode) + 1 << std::endl; + } + // assign global element id to temporary list + if (rnum_elem - 1 >= indices_buffer_max) + { + global_indices_temp.resize(rnum_elem - 1 + BUFFER_LINES); + indices_buffer_max = rnum_elem - 1 + BUFFER_LINES; + } + global_indices_temp[rnum_elem - 1] = elem_gid; + } + } + read_index_start += BUFFER_LINES; } - else{ - etype_index = 0; - } - //for - - //seek element count line - //read in element count from the following line - getline(*in, read_line); - std::cout << read_line << std::endl; - line_parse.clear(); - line_parse.str(read_line); - line_parse >> substring; - //parse element line out of jumble of comma delimited entries - line_parse.clear(); - line_parse.str(substring); - while(line_parse.good()){ - getline(line_parse, token, ','); - } - //element count should be the last token read in - num_elem = std::stoi(token); - - //skip line - for (int j = 0; j < 1; j++) { - getline(*in, skip_line); - std::cout << skip_line << std::endl; - } - } - - //broadcast element type - MPI_Bcast(&etype_index,1,MPI_INT,0,world); - - elements::elem_types::elem_type mesh_element_type; - int elem_words_per_line_no_nodes = elem_words_per_line; - - if(etype_index==1){ - mesh_element_type = elements::elem_types::Hex8; - nodes_per_element = 8; - elem_words_per_line += 8; - } - else if(etype_index==2){ - mesh_element_type = elements::elem_types::Hex20; - nodes_per_element = 20; - elem_words_per_line += 20; - } - else if(etype_index==3){ - mesh_element_type = elements::elem_types::Hex32; - nodes_per_element = 32; - elem_words_per_line += 32; - } - else{ - *fos << "ERROR: ANSYS ELEMENT TYPE NOT FOUND OR RECOGNIZED" << std::endl; - exit_solver(0); - } - - //broadcast number of elements - MPI_Bcast(&num_elem,1,MPI_LONG_LONG_INT,0,world); - - *fos << "declared element count: " << num_elem << std::endl; - //std::cout<<"before initial mesh initialization"<(BUFFER_LINES,elem_words_per_line,MAX_WORD); - CArrayKokkos node_store(nodes_per_element); - - //calculate buffer iterations to read number of lines - buffer_iterations = num_elem/BUFFER_LINES; - int assign_flag; - - //dynamic buffer used to store elements before we know how many this rank needs - std::vector element_temp(BUFFER_LINES*elem_words_per_line); - std::vector global_indices_temp(BUFFER_LINES); - size_t buffer_max = BUFFER_LINES*elem_words_per_line; - size_t indices_buffer_max = BUFFER_LINES; - - if(num_elem%BUFFER_LINES!=0) buffer_iterations++; - read_index_start = 0; - //std::cout << "ELEMENT BUFFER ITERATIONS: " << buffer_iterations << std::endl; - rnum_elem = 0; - //std::cout << "BUFFER ITERATIONS IS: " << buffer_iterations << std::endl; - for(buffer_iteration = 0; buffer_iteration < buffer_iterations; buffer_iteration++){ - //pack buffer on rank 0 - if(myrank==0&&buffer_iteration> substring; - //assign the substring variable as a word of the read buffer - strcpy(&read_buffer(buffer_loop,iword,0),substring.c_str()); + + // check if ANSYS file has boundary and loading condition zones + bool No_Conditions = true; + if (myrank == 0) + { + if (in->good()) + { + before_condition_header = in->tellg(); } - } + bool searching_for_conditions = true; + // skip lines at the top with nonessential info; stop skipping when "Fixed Supports or Pressure" string is reached + while (searching_for_conditions && in->good()) { + getline(*in, skip_line); + // std::cout << skip_line << std::endl; + line_parse.clear(); + line_parse.str(skip_line); + // stop when the NODES= string is reached + while (!line_parse.eof()) { + line_parse >> substring; + // std::cout << substring << std::endl; + if (!substring.compare("Supports") || !substring.compare("Pressure")) + { + No_Conditions = searching_for_conditions = false; + break; + } + } // while + } // while } - else if(myrank==0){ - buffer_loop=0; - while(buffer_iteration*BUFFER_LINES+buffer_loop < num_elem) { - getline(*in,read_line); - line_parse.clear(); - line_parse.str(read_line); - for(int iword = 0; iword < elem_words_per_line; iword++){ - //read portions of the line into the substring variable - line_parse >> substring; - //assign the substring variable as a word of the read buffer - strcpy(&read_buffer(buffer_loop,iword,0),substring.c_str()); + + // broadcast search condition + MPI_Bcast(&No_Conditions, 1, MPI_CXX_BOOL, 0, world); + + // flag elasticity fea module for boundary/loading conditions readin that remains + if (!No_Conditions) + { + // check that the input file has configured some kind of acceptable module + simparam.validate_module_is_specified(FEA_MODULE_TYPE::Elasticity); + simparam.fea_module_must_read.insert(FEA_MODULE_TYPE::Elasticity); + } + + // Close mesh input file if no further readin is done by FEA modules for conditions + if (myrank == 0 && No_Conditions) + { + in->close(); + } + + std::cout << "RNUM ELEMENTS IS: " << rnum_elem << std::endl; + // copy temporary element storage to multivector storage + Element_Types = CArrayKokkos(rnum_elem); + + // set element object pointer + if (simparam.num_dims == 2) + { + element_select->choose_2Delem_type(mesh_element_type, elem2D); + max_nodes_per_element = elem2D->num_nodes(); + } + else if (simparam.num_dims == 3) + { + element_select->choose_3Delem_type(mesh_element_type, elem); + max_nodes_per_element = elem->num_nodes(); + } + + // 1 type per mesh for now + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + Element_Types(ielem) = mesh_element_type; + } + + dual_nodes_in_elem = dual_elem_conn_array("dual_nodes_in_elem", rnum_elem, max_nodes_per_element); + host_elem_conn_array nodes_in_elem = dual_nodes_in_elem.view_host(); + dual_nodes_in_elem.modify_host(); + + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + for (int inode = 0; inode < nodes_per_element; inode++) + { + nodes_in_elem(ielem, inode) = element_temp[ielem * nodes_per_element + inode]; } - buffer_loop++; - //std::cout<<" "<< node_coords(node_gid, 0)<isNodeGlobalElement(node_gid)&&!assign_flag){ - assign_flag = 1; - rnum_elem++; - } + // view storage for all local elements connected to local nodes on this rank + Kokkos::DualView All_Element_Global_Indices("All_Element_Global_Indices", rnum_elem); + // copy temporary global indices storage to view storage + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + All_Element_Global_Indices.h_view(ielem) = global_indices_temp[ielem]; + if (global_indices_temp[ielem] < 0) + { + negative_index_found = 1; } - else{ - if(map->isNodeGlobalElement(node_gid-1)&&!assign_flag){ - assign_flag = 1; - rnum_elem++; - } + } + + MPI_Allreduce(&negative_index_found, &global_negative_index_found, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD); + if (global_negative_index_found) + { + if (myrank == 0) + { + std::cout << "Node index less than or equal to zero detected; set \"zero_index_base: true\" under \"input_options\" in your yaml file if indices start at 0" << std::endl; } - } + exit_solver(0); + } - if(assign_flag){ - for(int inode = 0; inode < nodes_per_element; inode++){ - if((rnum_elem-1)*nodes_per_element + inode>=buffer_max){ - element_temp.resize((rnum_elem-1)*nodes_per_element + inode + BUFFER_LINES*nodes_per_element); - buffer_max = (rnum_elem-1)*nodes_per_element + inode + BUFFER_LINES*nodes_per_element; - } - element_temp[(rnum_elem-1)*nodes_per_element + inode] = node_store(inode); - //std::cout << "VECTOR STORAGE FOR ELEM " << rnum_elem << " ON TASK " << myrank << " NODE " << inode+1 << " IS " << node_store(inode) + 1 << std::endl; + // delete temporary element connectivity and index storage + std::vector().swap(element_temp); + std::vector().swap(global_indices_temp); + + // construct overlapping element map (since different ranks can own the same elements due to the local node map) + All_Element_Global_Indices.modify_host(); + All_Element_Global_Indices.sync_device(); + + all_element_map = Teuchos::rcp(new Tpetra::Map(Teuchos::OrdinalTraits::invalid(), All_Element_Global_Indices.d_view, 0, comm)); + + // element type selection (subject to change) + // ---- Set Element Type ---- // + // allocate element type memory + // elements::elem_type_t* elem_choice; + + int NE = 1; // number of element types in problem + + // Convert ijk index system to the finite element numbering convention + // for vertices in cell + CArrayKokkos convert_ensight_to_ijk(max_nodes_per_element); + CArrayKokkos tmp_ijk_indx(max_nodes_per_element); + convert_ensight_to_ijk(0) = 0; + convert_ensight_to_ijk(1) = 1; + convert_ensight_to_ijk(2) = 3; + convert_ensight_to_ijk(3) = 2; + convert_ensight_to_ijk(4) = 4; + convert_ensight_to_ijk(5) = 5; + convert_ensight_to_ijk(6) = 7; + convert_ensight_to_ijk(7) = 6; + + if (num_dim == 2) + { + for (int cell_rid = 0; cell_rid < rnum_elem; cell_rid++) + { + // set nodes per element + element_select->choose_2Delem_type(Element_Types(cell_rid), elem2D); + nodes_per_element = elem2D->num_nodes(); + for (int node_lid = 0; node_lid < nodes_per_element; node_lid++) + { + tmp_ijk_indx(node_lid) = nodes_in_elem(cell_rid, convert_ensight_to_ijk(node_lid)); + } + + for (int node_lid = 0; node_lid < nodes_per_element; node_lid++) + { + nodes_in_elem(cell_rid, node_lid) = tmp_ijk_indx(node_lid); + } } - //assign global element id to temporary list - if(rnum_elem-1>=indices_buffer_max){ - global_indices_temp.resize(rnum_elem-1 + BUFFER_LINES); - indices_buffer_max = rnum_elem-1 + BUFFER_LINES; + } + + if (num_dim == 3) + { + for (int cell_rid = 0; cell_rid < rnum_elem; cell_rid++) + { + // set nodes per element + element_select->choose_3Delem_type(Element_Types(cell_rid), elem); + nodes_per_element = elem->num_nodes(); + for (int node_lid = 0; node_lid < nodes_per_element; node_lid++) + { + tmp_ijk_indx(node_lid) = nodes_in_elem(cell_rid, convert_ensight_to_ijk(node_lid)); + } + + for (int node_lid = 0; node_lid < nodes_per_element; node_lid++) + { + nodes_in_elem(cell_rid, node_lid) = tmp_ijk_indx(node_lid); + } } - global_indices_temp[rnum_elem-1] = elem_gid; - } } - read_index_start+=BUFFER_LINES; - } - - //check if ANSYS file has boundary and loading condition zones - bool No_Conditions = true; - if(myrank==0){ - if(in->good()) - before_condition_header = in->tellg(); - bool searching_for_conditions = true; - //skip lines at the top with nonessential info; stop skipping when "Fixed Supports or Pressure" string is reached - while (searching_for_conditions&&in->good()) { - getline(*in, skip_line); - //std::cout << skip_line << std::endl; - line_parse.clear(); - line_parse.str(skip_line); - //stop when the NODES= string is reached - while (!line_parse.eof()){ - line_parse >> substring; - //std::cout << substring << std::endl; - if(!substring.compare("Supports")||!substring.compare("Pressure")){ - No_Conditions = searching_for_conditions = false; - break; - } - } //while - - } //while - } - - //broadcast search condition - MPI_Bcast(&No_Conditions,1,MPI_CXX_BOOL,0,world); - - //flag elasticity fea module for boundary/loading conditions readin that remains - if(!No_Conditions){ - // check that the input file has configured some kind of acceptable module - simparam.validate_module_is_specified(FEA_MODULE_TYPE::Elasticity); - simparam.fea_module_must_read.insert(FEA_MODULE_TYPE::Elasticity); - } - - // Close mesh input file if no further readin is done by FEA modules for conditions - if(myrank==0&&No_Conditions){ - in->close(); - } - - std::cout << "RNUM ELEMENTS IS: " << rnum_elem << std::endl; - //copy temporary element storage to multivector storage - Element_Types = CArrayKokkos(rnum_elem); - - //set element object pointer - if(simparam.num_dims==2){ - element_select->choose_2Delem_type(mesh_element_type, elem2D); - max_nodes_per_element = elem2D->num_nodes(); - } - else if(simparam.num_dims==3){ - element_select->choose_3Delem_type(mesh_element_type, elem); - max_nodes_per_element = elem->num_nodes(); - } - - //1 type per mesh for now - for(int ielem = 0; ielem < rnum_elem; ielem++) - Element_Types(ielem) = mesh_element_type; - - dual_nodes_in_elem = dual_elem_conn_array("dual_nodes_in_elem", rnum_elem, max_nodes_per_element); - host_elem_conn_array nodes_in_elem = dual_nodes_in_elem.view_host(); - dual_nodes_in_elem.modify_host(); - - for(int ielem = 0; ielem < rnum_elem; ielem++) - for(int inode = 0; inode < nodes_per_element; inode++){ - nodes_in_elem(ielem, inode) = element_temp[ielem*nodes_per_element + inode]; - } - - //view storage for all local elements connected to local nodes on this rank - Kokkos::DualView All_Element_Global_Indices("All_Element_Global_Indices",rnum_elem); - //copy temporary global indices storage to view storage - for(int ielem = 0; ielem < rnum_elem; ielem++){ - All_Element_Global_Indices.h_view(ielem) = global_indices_temp[ielem]; - if(global_indices_temp[ielem]<0){ - negative_index_found = 1; - } - } - - MPI_Allreduce(&negative_index_found,&global_negative_index_found,1,MPI_INT,MPI_MAX,MPI_COMM_WORLD); - if(global_negative_index_found){ - if(myrank==0){ - std::cout << "Node index less than or equal to zero detected; set \"zero_index_base: true\" under \"input_options\" in your yaml file if indices start at 0" << std::endl; - } - exit_solver(0); - } - - //delete temporary element connectivity and index storage - std::vector().swap(element_temp); - std::vector().swap(global_indices_temp); - - //construct overlapping element map (since different ranks can own the same elements due to the local node map) - All_Element_Global_Indices.modify_host(); - All_Element_Global_Indices.sync_device(); - - all_element_map = Teuchos::rcp( new Tpetra::Map(Teuchos::OrdinalTraits::invalid(),All_Element_Global_Indices.d_view,0,comm)); - - - //element type selection (subject to change) - // ---- Set Element Type ---- // - // allocate element type memory - //elements::elem_type_t* elem_choice; - - int NE = 1; // number of element types in problem - - // Convert ijk index system to the finite element numbering convention - // for vertices in cell - CArrayKokkos convert_ensight_to_ijk(max_nodes_per_element); - CArrayKokkos tmp_ijk_indx(max_nodes_per_element); - convert_ensight_to_ijk(0) = 0; - convert_ensight_to_ijk(1) = 1; - convert_ensight_to_ijk(2) = 3; - convert_ensight_to_ijk(3) = 2; - convert_ensight_to_ijk(4) = 4; - convert_ensight_to_ijk(5) = 5; - convert_ensight_to_ijk(6) = 7; - convert_ensight_to_ijk(7) = 6; - - if(num_dim==2) - for (int cell_rid = 0; cell_rid < rnum_elem; cell_rid++) { - //set nodes per element - element_select->choose_2Delem_type(Element_Types(cell_rid), elem2D); - nodes_per_element = elem2D->num_nodes(); - for (int node_lid = 0; node_lid < nodes_per_element; node_lid++){ - tmp_ijk_indx(node_lid) = nodes_in_elem(cell_rid, convert_ensight_to_ijk(node_lid)); - } - - for (int node_lid = 0; node_lid < nodes_per_element; node_lid++){ - nodes_in_elem(cell_rid, node_lid) = tmp_ijk_indx(node_lid); - } - } - - if(num_dim==3) - for (int cell_rid = 0; cell_rid < rnum_elem; cell_rid++) { - //set nodes per element - element_select->choose_3Delem_type(Element_Types(cell_rid), elem); - nodes_per_element = elem->num_nodes(); - for (int node_lid = 0; node_lid < nodes_per_element; node_lid++){ - tmp_ijk_indx(node_lid) = nodes_in_elem(cell_rid, convert_ensight_to_ijk(node_lid)); - } - - for (int node_lid = 0; node_lid < nodes_per_element; node_lid++){ - nodes_in_elem(cell_rid, node_lid) = tmp_ijk_indx(node_lid); - } - } - } // end read_mesh /* ---------------------------------------------------------------------- Initialize Distributed State Vectors Pertinent to this Solver ------------------------------------------------------------------------- */ -void Explicit_Solver::init_state_vectors(){ - int num_dim = simparam.num_dims; - //allocate node_velocities - //node_velocities_distributed = Teuchos::rcp(new MV(map, num_dim)); - initial_node_coords_distributed = Teuchos::rcp(new MV(map, num_dim)); - all_initial_node_coords_distributed = Teuchos::rcp(new MV(all_node_map, num_dim)); - initial_node_velocities_distributed = Teuchos::rcp(new MV(map, num_dim)); - all_node_velocities_distributed = Teuchos::rcp(new MV(all_node_map, num_dim)); - node_velocities_distributed = Teuchos::rcp(new MV(*all_node_velocities_distributed, map)); - //ghost_node_velocities_distributed = Teuchos::rcp(new MV(ghost_node_map, num_dim)); - ghost_node_velocities_distributed = Teuchos::rcp(new MV(*all_node_velocities_distributed, ghost_node_map, nlocal_nodes)); - if(simparam.topology_optimization_on){ - test_node_densities_distributed = Teuchos::rcp(new MV(map, 1)); - } - if(simparam.topology_optimization_on || simparam.shape_optimization_on){ - corner_value_storage = CArrayKokkos(rnum_elem*max_nodes_per_element); - corner_vector_storage = CArrayKokkos(rnum_elem*max_nodes_per_element,num_dim); - corner_gradient_storage = CArrayKokkos(rnum_elem*max_nodes_per_element, num_dim, max_nodes_per_element, num_dim); - } - all_node_densities_distributed = Teuchos::rcp(new MV(all_node_map, 1)); - Global_Element_Densities = Teuchos::rcp(new MV(all_element_map, 1)); +void Explicit_Solver::init_state_vectors() +{ + int num_dim = simparam.num_dims; + // allocate node_velocities + // node_velocities_distributed = Teuchos::rcp(new MV(map, num_dim)); + initial_node_coords_distributed = Teuchos::rcp(new MV(map, num_dim)); + all_initial_node_coords_distributed = Teuchos::rcp(new MV(all_node_map, num_dim)); + initial_node_velocities_distributed = Teuchos::rcp(new MV(map, num_dim)); + all_node_velocities_distributed = Teuchos::rcp(new MV(all_node_map, num_dim)); + node_velocities_distributed = Teuchos::rcp(new MV(*all_node_velocities_distributed, map)); + // ghost_node_velocities_distributed = Teuchos::rcp(new MV(ghost_node_map, num_dim)); + ghost_node_velocities_distributed = Teuchos::rcp(new MV(*all_node_velocities_distributed, ghost_node_map, nlocal_nodes)); + if (simparam.topology_optimization_on) + { + test_node_densities_distributed = Teuchos::rcp(new MV(map, 1)); + } + if (simparam.topology_optimization_on || simparam.shape_optimization_on) + { + corner_value_storage = CArrayKokkos(rnum_elem * max_nodes_per_element); + corner_vector_storage = CArrayKokkos(rnum_elem * max_nodes_per_element, num_dim); + corner_gradient_storage = CArrayKokkos(rnum_elem * max_nodes_per_element, num_dim, max_nodes_per_element, num_dim); + } + all_node_densities_distributed = Teuchos::rcp(new MV(all_node_map, 1)); + Global_Element_Densities = Teuchos::rcp(new MV(all_element_map, 1)); } /* ---------------------------------------------------------------------- - Construct list of objects for FEA modules + Construct list of objects for FEA modules ------------------------------------------------------------------------- */ -void Explicit_Solver::FEA_module_setup(){ - nfea_modules = simparam.fea_module_parameters.size(); - fea_module_must_read = simparam.fea_module_must_read; - - //allocate lists to size - fea_module_types = std::vector(); - fea_modules = std::vector(); - - for (auto& param : simparam.fea_module_parameters) { - fea_module_types.push_back(param->type); - param->apply( - [&](SGH_Parameters& param) { - sgh_module = new FEA_Module_SGH(param, this, mesh); - fea_modules.push_back(sgh_module); +void Explicit_Solver::FEA_module_setup() +{ + nfea_modules = simparam.fea_module_parameters.size(); + fea_module_must_read = simparam.fea_module_must_read; + + // allocate lists to size + fea_module_types = std::vector(); + fea_modules = std::vector(); + + for (auto& param : simparam.fea_module_parameters) + { + fea_module_types.push_back(param->type); + param->apply( + [&](SGH_Parameters& param) { + sgh_module = new FEA_Module_SGH(param, this, mesh); + fea_modules.push_back(sgh_module); }, - [&](Dynamic_Elasticity_Parameters& param) { - fea_modules.push_back(new FEA_Module_Dynamic_Elasticity(param, this, mesh)); + [&](Dynamic_Elasticity_Parameters& param) { + fea_modules.push_back(new FEA_Module_Dynamic_Elasticity(param, this, mesh)); }, - [&](Inertial_Parameters& param) { - fea_modules.push_back(new FEA_Module_Inertial(param, this)); + [&](Inertial_Parameters& param) { + fea_modules.push_back(new FEA_Module_Inertial(param, this)); }, - [&](const FEA_Module_Parameters& param) { - *fos << "PROGRAM IS ENDING DUE TO ERROR; UNDEFINED FEA MODULE REQUESTED WITH NAME \"" - << param.type <<"\"" << std::endl; - exit_solver(0); + [&](const FEA_Module_Parameters& param) { + *fos << "PROGRAM IS ENDING DUE TO ERROR; UNDEFINED FEA MODULE REQUESTED WITH NAME \"" + << param.type << "\"" << std::endl; + exit_solver(0); } - ); + ); - *fos << " " << fea_module_types.back() << " MODULE ALLOCATED AS " << fea_module_types.size() - 1 << std::endl; - } + *fos << " " << fea_module_types.back() << " MODULE ALLOCATED AS " << fea_module_types.size() - 1 << std::endl; + } } /* ---------------------------------------------------------------------- Setup Optimization Problem Object, Relevant Objective, and Constraints ------------------------------------------------------------------------- */ -void Explicit_Solver::setup_optimization_problem(){ - int num_dim = simparam.num_dims; - bool nodal_density_flag = simparam.nodal_density_flag; - int nTO_modules = simparam.TO_Module_List.size(); - //int nmulti_objective_modules = simparam->nmulti_objective_modules; - std::vector TO_Module_List = simparam.TO_Module_List; - std::vector TO_Module_My_FEA_Module = simparam.TO_Module_My_FEA_Module; - //std::vector Multi_Objective_Modules = simparam->Multi_Objective_Modules; - //std::vector Multi_Objective_Weights = simparam->Multi_Objective_Weights; - std::vector> Function_Arguments = simparam.Function_Arguments; - std::vector TO_Function_Type = simparam.TO_Function_Type; - std::vector>> Multi_Objective_Terms; - - std::string constraint_base, constraint_name; - std::stringstream number_union; - CArray Surface_Nodes; - GO current_node_index, current_element_index; - LO local_node_index, local_element_id; - int num_bdy_patches_in_set; - size_t node_id, patch_id, module_id; - int num_boundary_sets; - int local_surface_id; - const_host_vec_array design_densities; - typedef ROL::TpetraMultiVector ROL_MV; - const_host_elem_conn_array nodes_in_elem = global_nodes_in_elem_distributed->getLocalView (Tpetra::Access::ReadOnly); - - // fill parameter list with desired algorithmic options or leave as default - // Read optimization input parameter list. - std::string filename = "optimization_parameters.xml"; - auto parlist = ROL::getParametersFromXmlFile( filename ); - //ROL::ParameterList parlist; - - //Design variables to optimize - ROL::Ptr> x; - if(nodal_density_flag){ - x = ROL::makePtr>(design_node_densities_distributed); - } - else - x = ROL::makePtr>(Global_Element_Densities); - - //Instantiate (the one) objective function for the problem - ROL::Ptr> obj; - bool objective_declared = false; - for(int imodule = 0; imodule < nTO_modules; imodule++){ - if(TO_Function_Type[imodule] == FUNCTION_TYPE::OBJECTIVE){ - //check if previous module already defined an objective, there must be one objective module - if(objective_declared){ - // TODO: Put this validation earlier. - *fos << "PROGRAM IS ENDING DUE TO ERROR; ANOTHER OBJECTIVE FUNCTION WITH NAME \"" - << TO_Module_List[imodule] <<"\" ATTEMPTED TO REPLACE A PREVIOUS OBJECTIVE; THERE MUST BE ONE OBJECTIVE." << std::endl; - exit_solver(0); - } - if(TO_Module_List[imodule] == TO_MODULE_TYPE::Kinetic_Energy_Minimize){ - //debug print - *fos << " KINETIC ENERGY OBJECTIVE EXPECTS FEA MODULE INDEX " <(this, nodal_density_flag); - } - /* - else if(TO_Module_List[imodule] == "Heat_Capacity_Potential_Minimize"){ - //debug print - *fos << " HEAT CAPACITY POTENTIAL OBJECTIVE EXPECTS FEA MODULE INDEX " <(fea_modules[TO_Module_My_FEA_Module[imodule]], nodal_density_flag); - } - */ - //Multi-Objective case - /* - else if(TO_Module_List[imodule] == "Multi_Objective"){ - //allocate vector of Objective Functions to pass - Multi_Objective_Terms = std::vector>>(nmulti_objective_modules); - for(int imulti = 0; imulti < nmulti_objective_modules; imulti++){ - //get module index for objective term - module_id = Multi_Objective_Modules[imulti]; - if(TO_Module_List[module_id] == "Strain_Energy_Minimize"){ - //debug print - *fos << " STRAIN ENERGY OBJECTIVE EXPECTS FEA MODULE INDEX " <(fea_modules[TO_Module_My_FEA_Module[module_id]], nodal_density_flag); - } - else if(TO_Module_List[module_id] == "Heat_Capacity_Potential_Minimize"){ - //debug print - *fos << " HEAT CAPACITY POTENTIAL OBJECTIVE EXPECTS FEA MODULE INDEX " <(fea_modules[TO_Module_My_FEA_Module[module_id]], nodal_density_flag); - } - } - //allocate multi objective function - obj = ROL::makePtr(Multi_Objective_Terms, Multi_Objective_Weights); - } - */ - else{ - // TODO: Put validation earlier - *fos << "PROGRAM IS ENDING DUE TO ERROR; UNDEFINED OBJECTIVE FUNCTION REQUESTED WITH NAME \"" - << TO_Module_List[imodule] << "\"" << std::endl; - exit_solver(0); - } - objective_declared = true; - } - } - - //optimization problem interface that can have constraints added to it before passing to solver object - problem = ROL::makePtr>(obj,x); - - //ROL::Ptr> lin_icon = ROL::makePtr>(); - //ROL::Ptr> lin_imul = ROL::makePtr>(); - //ROL::Ptr> lin_ibnd = ROL::makePtr>(); - //problem.addLinearConstraint("Linear Inequality Constraint",lin_icon,lin_imul,lin_ibnd); - - // TypeG (generally constrained) specification - //ROL::Ptr> econ = ROL::makePtr>(); - //ROL::Ptr> emul = ROL::makePtr>(); - //problem.addConstraint("Equality Constraint",econ,emul); - - //ROL::Ptr> ineq_constraint = ROL::makePtr(this, nodal_density_flag); - //ROL::Ptr> lin_econ = ROL::makePtr>(); - //ROL::Ptr lin_emul = ROL::makePtr>(); - //problem.addLinearConstraint("Linear Equality Constraint",lin_econ,lin_mul); - - //ROL::Ptr> ineq_constraint = ROL::makePtr(fea_elasticity, nodal_density_flag); - //problem->addConstraint("Inequality Constraint",ineq_constraint,constraint_mul,constraint_bnd); - //problem->addConstraint("equality Constraint 2",eq_constraint2,constraint_mul2); - //problem->addConstraint("equality Constraint 3",eq_constraint3,constraint_mul3); - //problem->addLinearConstraint("Equality Constraint",eq_constraint,constraint_mul); - - for(int imodule = 0; imodule < nTO_modules; imodule++){ - number_union.str(constraint_base); - number_union << imodule + 1; - constraint_name = number_union.str(); - ROL::Ptr > li_ptr = ROL::makePtr>(1,0.0); - ROL::Ptr > constraint_mul = ROL::makePtr>(li_ptr); - if(TO_Function_Type[imodule] == FUNCTION_TYPE::EQUALITY_CONSTRAINT){ - //pointers are reference counting - ROL::Ptr> eq_constraint; - if(TO_Module_List[imodule]==TO_MODULE_TYPE::Mass_Constraint){ - - *fos << " MASS CONSTRAINT EXPECTS FEA MODULE INDEX " <(fea_modules[TO_Module_My_FEA_Module[imodule]], nodal_density_flag, Function_Arguments[imodule][0], false, true); - } - else if(TO_Module_List[imodule]==TO_MODULE_TYPE::Moment_of_Inertia_Constraint){ - *fos << " MOMENT OF INERTIA CONSTRAINT EXPECTS FEA MODULE INDEX " <(fea_modules[TO_Module_My_FEA_Module[imodule]], nodal_density_flag, Function_Arguments[imodule][1], Function_Arguments[imodule][0], false, true); - } - else{ - // TODO: Put validation earlier - *fos << "PROGRAM IS ENDING DUE TO ERROR; UNDEFINED EQUALITY CONSTRAINT FUNCTION REQUESTED WITH NAME \"" - << TO_Module_List[imodule] <<"\"" << std::endl; - exit_solver(0); - } - *fos << " ADDING CONSTRAINT " << constraint_name << std::endl; - problem->addConstraint(constraint_name, eq_constraint, constraint_mul); - } - - if(TO_Function_Type[imodule] == FUNCTION_TYPE::INEQUALITY_CONSTRAINT){ - //pointers are reference counting - ROL::Ptr> ineq_constraint; - ROL::Ptr > ll_ptr = ROL::makePtr>(1,Function_Arguments[imodule][0]); - ROL::Ptr > lu_ptr = ROL::makePtr>(1,Function_Arguments[imodule][1]); - ROL::Ptr > ll = ROL::makePtr>(ll_ptr); - ROL::Ptr > lu = ROL::makePtr>(lu_ptr); - ROL::Ptr> constraint_bnd = ROL::makePtr>(ll,lu); - if(TO_Module_List[imodule]==TO_MODULE_TYPE::Mass_Constraint){ - *fos << " MASS CONSTRAINT EXPECTS FEA MODULE INDEX " <(fea_modules[TO_Module_My_FEA_Module[imodule]], nodal_density_flag, true, true); - } - else if(TO_Module_List[imodule]==TO_MODULE_TYPE::Moment_of_Inertia_Constraint){ - *fos << " MOMENT OF INERTIA CONSTRAINT EXPECTS FEA MODULE INDEX " <(fea_modules[TO_Module_My_FEA_Module[imodule]], nodal_density_flag, Function_Arguments[imodule][1], Function_Arguments[imodule][0], true, true); - } - else{ - // TODO: Put this validation earlier - *fos << "PROGRAM IS ENDING DUE TO ERROR; UNDEFINED INEQUALITY CONSTRAINT FUNCTION REQUESTED WITH NAME \"" - << TO_Module_List[imodule] << "\"" << std::endl; - exit_solver(0); - } - *fos << " ADDING CONSTRAINT " << constraint_name << std::endl; - problem->addConstraint(constraint_name, ineq_constraint, constraint_mul, constraint_bnd); +void Explicit_Solver::setup_optimization_problem() +{ + int num_dim = simparam.num_dims; + bool nodal_density_flag = simparam.nodal_density_flag; + int nTO_modules = simparam.TO_Module_List.size(); + // int nmulti_objective_modules = simparam->nmulti_objective_modules; + std::vector TO_Module_List = simparam.TO_Module_List; + std::vector TO_Module_My_FEA_Module = simparam.TO_Module_My_FEA_Module; + // std::vector Multi_Objective_Modules = simparam->Multi_Objective_Modules; + // std::vector Multi_Objective_Weights = simparam->Multi_Objective_Weights; + std::vector> Function_Arguments = simparam.Function_Arguments; + std::vector TO_Function_Type = simparam.TO_Function_Type; + std::vector>> Multi_Objective_Terms; + + std::string constraint_base, constraint_name; + std::stringstream number_union; + CArray Surface_Nodes; + GO current_node_index, current_element_index; + LO local_node_index, local_element_id; + int num_bdy_patches_in_set; + size_t node_id, patch_id, module_id; + int num_boundary_sets; + int local_surface_id; + const_host_vec_array design_densities; + typedef ROL::TpetraMultiVector ROL_MV; + const_host_elem_conn_array nodes_in_elem = global_nodes_in_elem_distributed->getLocalView(Tpetra::Access::ReadOnly); + + // fill parameter list with desired algorithmic options or leave as default + // Read optimization input parameter list. + std::string filename = "optimization_parameters.xml"; + auto parlist = ROL::getParametersFromXmlFile(filename); + // ROL::ParameterList parlist; + + // Design variables to optimize + ROL::Ptr> x; + if (nodal_density_flag) + { + x = ROL::makePtr>(design_node_densities_distributed); + } + else + { + x = ROL::makePtr>(Global_Element_Densities); } - number_union.clear(); - } - //set bounds on design variables - if(nodal_density_flag){ - dual_vec_array dual_node_densities_upper_bound = dual_vec_array("dual_node_densities_upper_bound", nlocal_nodes, 1); - dual_vec_array dual_node_densities_lower_bound = dual_vec_array("dual_node_densities_lower_bound", nlocal_nodes, 1); - host_vec_array node_densities_upper_bound = dual_node_densities_upper_bound.view_host(); - host_vec_array node_densities_lower_bound = dual_node_densities_lower_bound.view_host(); - //notify that the host view is going to be modified in the file readin - dual_node_densities_upper_bound.modify_host(); - dual_node_densities_lower_bound.modify_host(); + // Instantiate (the one) objective function for the problem + ROL::Ptr> obj; + bool objective_declared = false; + for (int imodule = 0; imodule < nTO_modules; imodule++) + { + if (TO_Function_Type[imodule] == FUNCTION_TYPE::OBJECTIVE) + { + // check if previous module already defined an objective, there must be one objective module + if (objective_declared) + { + // TODO: Put this validation earlier. + *fos << "PROGRAM IS ENDING DUE TO ERROR; ANOTHER OBJECTIVE FUNCTION WITH NAME \"" + << TO_Module_List[imodule] << "\" ATTEMPTED TO REPLACE A PREVIOUS OBJECTIVE; THERE MUST BE ONE OBJECTIVE." << std::endl; + exit_solver(0); + } + if (TO_Module_List[imodule] == TO_MODULE_TYPE::Kinetic_Energy_Minimize) + { + // debug print + *fos << " KINETIC ENERGY OBJECTIVE EXPECTS FEA MODULE INDEX " << TO_Module_My_FEA_Module[imodule] << std::endl; + obj = ROL::makePtr(this, nodal_density_flag); + } + /* + else if(TO_Module_List[imodule] == "Heat_Capacity_Potential_Minimize"){ + //debug print + *fos << " HEAT CAPACITY POTENTIAL OBJECTIVE EXPECTS FEA MODULE INDEX " <(fea_modules[TO_Module_My_FEA_Module[imodule]], nodal_density_flag); + } + */ + // Multi-Objective case + /* + else if(TO_Module_List[imodule] == "Multi_Objective"){ + //allocate vector of Objective Functions to pass + Multi_Objective_Terms = std::vector>>(nmulti_objective_modules); + for(int imulti = 0; imulti < nmulti_objective_modules; imulti++){ + //get module index for objective term + module_id = Multi_Objective_Modules[imulti]; + if(TO_Module_List[module_id] == "Strain_Energy_Minimize"){ + //debug print + *fos << " STRAIN ENERGY OBJECTIVE EXPECTS FEA MODULE INDEX " <(fea_modules[TO_Module_My_FEA_Module[module_id]], nodal_density_flag); + } + else if(TO_Module_List[module_id] == "Heat_Capacity_Potential_Minimize"){ + //debug print + *fos << " HEAT CAPACITY POTENTIAL OBJECTIVE EXPECTS FEA MODULE INDEX " <(fea_modules[TO_Module_My_FEA_Module[module_id]], nodal_density_flag); + } + } + //allocate multi objective function + obj = ROL::makePtr(Multi_Objective_Terms, Multi_Objective_Weights); + } + */ + else + { + // TODO: Put validation earlier + *fos << "PROGRAM IS ENDING DUE TO ERROR; UNDEFINED OBJECTIVE FUNCTION REQUESTED WITH NAME \"" + << TO_Module_List[imodule] << "\"" << std::endl; + exit_solver(0); + } + objective_declared = true; + } + } - //initialize densities to 1 for now; in the future there might be an option to read in an initial condition for each node - for(int inode = 0; inode < nlocal_nodes; inode++){ - node_densities_upper_bound(inode,0) = 1; - node_densities_lower_bound(inode,0) = simparam.optimization_options.density_epsilon; + // optimization problem interface that can have constraints added to it before passing to solver object + problem = ROL::makePtr>(obj, x); + + // ROL::Ptr> lin_icon = ROL::makePtr>(); + // ROL::Ptr> lin_imul = ROL::makePtr>(); + // ROL::Ptr> lin_ibnd = ROL::makePtr>(); + // problem.addLinearConstraint("Linear Inequality Constraint",lin_icon,lin_imul,lin_ibnd); + + // TypeG (generally constrained) specification + // ROL::Ptr> econ = ROL::makePtr>(); + // ROL::Ptr> emul = ROL::makePtr>(); + // problem.addConstraint("Equality Constraint",econ,emul); + + // ROL::Ptr> ineq_constraint = ROL::makePtr(this, nodal_density_flag); + // ROL::Ptr> lin_econ = ROL::makePtr>(); + // ROL::Ptr lin_emul = ROL::makePtr>(); + // problem.addLinearConstraint("Linear Equality Constraint",lin_econ,lin_mul); + + // ROL::Ptr> ineq_constraint = ROL::makePtr(fea_elasticity, nodal_density_flag); + // problem->addConstraint("Inequality Constraint",ineq_constraint,constraint_mul,constraint_bnd); + // problem->addConstraint("equality Constraint 2",eq_constraint2,constraint_mul2); + // problem->addConstraint("equality Constraint 3",eq_constraint3,constraint_mul3); + // problem->addLinearConstraint("Equality Constraint",eq_constraint,constraint_mul); + + for (int imodule = 0; imodule < nTO_modules; imodule++) + { + number_union.str(constraint_base); + number_union << imodule + 1; + constraint_name = number_union.str(); + ROL::Ptr> li_ptr = ROL::makePtr>(1, 0.0); + ROL::Ptr> constraint_mul = ROL::makePtr>(li_ptr); + if (TO_Function_Type[imodule] == FUNCTION_TYPE::EQUALITY_CONSTRAINT) + { + // pointers are reference counting + ROL::Ptr> eq_constraint; + if (TO_Module_List[imodule] == TO_MODULE_TYPE::Mass_Constraint) + { + *fos << " MASS CONSTRAINT EXPECTS FEA MODULE INDEX " << TO_Module_My_FEA_Module[imodule] << std::endl; + eq_constraint = ROL::makePtr(fea_modules[TO_Module_My_FEA_Module[imodule]], nodal_density_flag, Function_Arguments[imodule][0], false, true); + } + else if (TO_Module_List[imodule] == TO_MODULE_TYPE::Moment_of_Inertia_Constraint) + { + *fos << " MOMENT OF INERTIA CONSTRAINT EXPECTS FEA MODULE INDEX " << TO_Module_My_FEA_Module[imodule] << std::endl; + eq_constraint = ROL::makePtr(fea_modules[TO_Module_My_FEA_Module[imodule]], nodal_density_flag, Function_Arguments[imodule][1], + Function_Arguments[imodule][0], false, true); + } + else + { + // TODO: Put validation earlier + *fos << "PROGRAM IS ENDING DUE TO ERROR; UNDEFINED EQUALITY CONSTRAINT FUNCTION REQUESTED WITH NAME \"" + << TO_Module_List[imodule] << "\"" << std::endl; + exit_solver(0); + } + *fos << " ADDING CONSTRAINT " << constraint_name << std::endl; + problem->addConstraint(constraint_name, eq_constraint, constraint_mul); + } + if (TO_Function_Type[imodule] == FUNCTION_TYPE::INEQUALITY_CONSTRAINT) + { + // pointers are reference counting + ROL::Ptr> ineq_constraint; + ROL::Ptr> ll_ptr = ROL::makePtr>(1, Function_Arguments[imodule][0]); + ROL::Ptr> lu_ptr = ROL::makePtr>(1, Function_Arguments[imodule][1]); + ROL::Ptr> ll = ROL::makePtr>(ll_ptr); + ROL::Ptr> lu = ROL::makePtr>(lu_ptr); + ROL::Ptr> constraint_bnd = ROL::makePtr>(ll, lu); + if (TO_Module_List[imodule] == TO_MODULE_TYPE::Mass_Constraint) + { + *fos << " MASS CONSTRAINT EXPECTS FEA MODULE INDEX " << TO_Module_My_FEA_Module[imodule] << std::endl; + ineq_constraint = ROL::makePtr(fea_modules[TO_Module_My_FEA_Module[imodule]], nodal_density_flag, true, true); + } + else if (TO_Module_List[imodule] == TO_MODULE_TYPE::Moment_of_Inertia_Constraint) + { + *fos << " MOMENT OF INERTIA CONSTRAINT EXPECTS FEA MODULE INDEX " << TO_Module_My_FEA_Module[imodule] << std::endl; + ineq_constraint = ROL::makePtr(fea_modules[TO_Module_My_FEA_Module[imodule]], nodal_density_flag, Function_Arguments[imodule][1], + Function_Arguments[imodule][0], true, true); + } + else + { + // TODO: Put this validation earlier + *fos << "PROGRAM IS ENDING DUE TO ERROR; UNDEFINED INEQUALITY CONSTRAINT FUNCTION REQUESTED WITH NAME \"" + << TO_Module_List[imodule] << "\"" << std::endl; + exit_solver(0); + } + *fos << " ADDING CONSTRAINT " << constraint_name << std::endl; + problem->addConstraint(constraint_name, ineq_constraint, constraint_mul, constraint_bnd); + } + number_union.clear(); } - //set lower bounds for nodes on surfaces with boundary and loading conditions - for(int imodule = 0; imodule < nfea_modules; imodule++){ - num_boundary_sets = fea_modules[imodule]->num_boundary_conditions; - for(int iboundary = 0; iboundary < num_boundary_sets; iboundary++){ - - num_bdy_patches_in_set = fea_modules[imodule]->bdy_patches_in_set.stride(iboundary); - - //loop over boundary patches for this boundary set - if(simparam.optimization_options.thick_condition_boundary){ - for (int bdy_patch_gid = 0; bdy_patch_gid < num_bdy_patches_in_set; bdy_patch_gid++){ - - // get the global id for this boundary patch - patch_id = fea_modules[imodule]->bdy_patches_in_set(iboundary, bdy_patch_gid); - Surface_Nodes = Boundary_Patches(patch_id).node_set; - current_element_index = Boundary_Patches(patch_id).element_id; - //debug print of local surface ids - //std::cout << " LOCAL SURFACE IDS " << std::endl; - //std::cout << local_surface_id << std::endl; - //acquire set of nodes for this face - for(int node_loop=0; node_loop < max_nodes_per_element; node_loop++){ - current_node_index = nodes_in_elem(current_element_index,node_loop); - if(map->isNodeGlobalElement(current_node_index)){ - local_node_index = map->getLocalElement(current_node_index); - node_densities_lower_bound(local_node_index,0) = 1; + + // set bounds on design variables + if (nodal_density_flag) + { + dual_vec_array dual_node_densities_upper_bound = dual_vec_array("dual_node_densities_upper_bound", nlocal_nodes, 1); + dual_vec_array dual_node_densities_lower_bound = dual_vec_array("dual_node_densities_lower_bound", nlocal_nodes, 1); + host_vec_array node_densities_upper_bound = dual_node_densities_upper_bound.view_host(); + host_vec_array node_densities_lower_bound = dual_node_densities_lower_bound.view_host(); + // notify that the host view is going to be modified in the file readin + dual_node_densities_upper_bound.modify_host(); + dual_node_densities_lower_bound.modify_host(); + + // initialize densities to 1 for now; in the future there might be an option to read in an initial condition for each node + for (int inode = 0; inode < nlocal_nodes; inode++) + { + node_densities_upper_bound(inode, 0) = 1; + node_densities_lower_bound(inode, 0) = simparam.optimization_options.density_epsilon; + } + // set lower bounds for nodes on surfaces with boundary and loading conditions + for (int imodule = 0; imodule < nfea_modules; imodule++) + { + num_boundary_sets = fea_modules[imodule]->num_boundary_conditions; + for (int iboundary = 0; iboundary < num_boundary_sets; iboundary++) + { + num_bdy_patches_in_set = fea_modules[imodule]->bdy_patches_in_set.stride(iboundary); + + // loop over boundary patches for this boundary set + if (simparam.optimization_options.thick_condition_boundary) + { + for (int bdy_patch_gid = 0; bdy_patch_gid < num_bdy_patches_in_set; bdy_patch_gid++) + { + // get the global id for this boundary patch + patch_id = fea_modules[imodule]->bdy_patches_in_set(iboundary, bdy_patch_gid); + Surface_Nodes = Boundary_Patches(patch_id).node_set; + current_element_index = Boundary_Patches(patch_id).element_id; + // debug print of local surface ids + // std::cout << " LOCAL SURFACE IDS " << std::endl; + // std::cout << local_surface_id << std::endl; + // acquire set of nodes for this face + for (int node_loop = 0; node_loop < max_nodes_per_element; node_loop++) + { + current_node_index = nodes_in_elem(current_element_index, node_loop); + if (map->isNodeGlobalElement(current_node_index)) + { + local_node_index = map->getLocalElement(current_node_index); + node_densities_lower_bound(local_node_index, 0) = 1; + } + } // node loop for + + // if + /* + else{ + Surface_Nodes = Boundary_Patches(patch_id).node_set; + local_surface_id = Boundary_Patches(patch_id).local_patch_id; + //debug print of local surface ids + //std::cout << " LOCAL SURFACE IDS " << std::endl; + //std::cout << local_surface_id << std::endl; + //acquire set of nodes for this face + for(int node_loop=0; node_loop < Surface_Nodes.size(); node_loop++){ + current_node_index = Surface_Nodes(node_loop); + if(map->isNodeGlobalElement(current_node_index)){ + local_node_index = map->getLocalElement(current_node_index); + node_densities_lower_bound(local_node_index,0) = 1; + } + }// node loop for + }//if + */ + } // boundary patch for } - }// node loop for - - //if - /* - else{ - Surface_Nodes = Boundary_Patches(patch_id).node_set; - local_surface_id = Boundary_Patches(patch_id).local_patch_id; - //debug print of local surface ids - //std::cout << " LOCAL SURFACE IDS " << std::endl; - //std::cout << local_surface_id << std::endl; - //acquire set of nodes for this face - for(int node_loop=0; node_loop < Surface_Nodes.size(); node_loop++){ - current_node_index = Surface_Nodes(node_loop); - if(map->isNodeGlobalElement(current_node_index)){ - local_node_index = map->getLocalElement(current_node_index); - node_densities_lower_bound(local_node_index,0) = 1; + else + { + for (int node_loop = 0; node_loop < fea_modules[imodule]->num_bdy_nodes_in_set(iboundary); node_loop++) + { + local_node_index = fea_modules[imodule]->bdy_nodes_in_set(iboundary, node_loop); + node_densities_lower_bound(local_node_index, 0) = 1; + } } - }// node loop for - }//if - */ - }//boundary patch for - } - else{ - for(int node_loop=0; node_loop < fea_modules[imodule]->num_bdy_nodes_in_set(iboundary); node_loop++){ - local_node_index = fea_modules[imodule]->bdy_nodes_in_set(iboundary, node_loop); - node_densities_lower_bound(local_node_index,0) = 1; - } + } // boundary set for + + // set node conditions due to point BCS that might not show up in boundary sets + // possible to have overlap in which nodes are set with the previous loop + fea_modules[imodule]->node_density_constraints(node_densities_lower_bound); + } // module for + + // sync device view + dual_node_densities_upper_bound.sync_device(); + dual_node_densities_lower_bound.sync_device(); + + // allocate global vector information + upper_bound_node_densities_distributed = Teuchos::rcp(new MV(map, dual_node_densities_upper_bound)); + lower_bound_node_densities_distributed = Teuchos::rcp(new MV(map, dual_node_densities_lower_bound)); + } + else + { + // initialize memory for volume storage + vec_array Element_Densities_Upper_Bound("Element Densities_Upper_Bound", rnum_elem, 1); + vec_array Element_Densities_Lower_Bound("Element Densities_Lower_Bound", rnum_elem, 1); + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + Element_Densities_Upper_Bound(ielem, 0) = 1; + Element_Densities_Lower_Bound(ielem, 0) = simparam.optimization_options.density_epsilon; } - }//boundary set for - - //set node conditions due to point BCS that might not show up in boundary sets - //possible to have overlap in which nodes are set with the previous loop - fea_modules[imodule]->node_density_constraints(node_densities_lower_bound); - }//module for - - //sync device view - dual_node_densities_upper_bound.sync_device(); - dual_node_densities_lower_bound.sync_device(); - - //allocate global vector information - upper_bound_node_densities_distributed = Teuchos::rcp(new MV(map, dual_node_densities_upper_bound)); - lower_bound_node_densities_distributed = Teuchos::rcp(new MV(map, dual_node_densities_lower_bound)); - - } - else{ - //initialize memory for volume storage - vec_array Element_Densities_Upper_Bound("Element Densities_Upper_Bound", rnum_elem, 1); - vec_array Element_Densities_Lower_Bound("Element Densities_Lower_Bound", rnum_elem, 1); - for(int ielem = 0; ielem < rnum_elem; ielem++){ - Element_Densities_Upper_Bound(ielem,0) = 1; - Element_Densities_Lower_Bound(ielem,0) = simparam.optimization_options.density_epsilon; - } - - //create global vector - Global_Element_Densities_Upper_Bound = Teuchos::rcp(new MV(element_map, Element_Densities_Upper_Bound)); - Global_Element_Densities_Lower_Bound = Teuchos::rcp(new MV(element_map, Element_Densities_Lower_Bound)); - } - - // Bound constraint defining the possible range of design density variables - ROL::Ptr > lower_bounds; - ROL::Ptr > upper_bounds; - if(nodal_density_flag){ - lower_bounds = ROL::makePtr>(lower_bound_node_densities_distributed); - upper_bounds = ROL::makePtr>(upper_bound_node_densities_distributed); - } - else{ - lower_bounds = ROL::makePtr>(Global_Element_Densities_Lower_Bound); - upper_bounds = ROL::makePtr>(Global_Element_Densities_Upper_Bound); - } - ROL::Ptr > bnd = ROL::makePtr>(lower_bounds, upper_bounds); - problem->addBoundConstraint(bnd); - - //compute initial constraint satisfaction - //ROL::Ptr ROL_Element_Masses = ROL::makePtr(fea_elasticity->Global_Element_Masses); - ROL::Elementwise::ReductionSum sumreduc; - if(nodal_density_flag) - design_densities = design_node_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); - else - design_densities = Global_Element_Densities->getLocalView (Tpetra::Access::ReadOnly); - //fea_elasticity->compute_element_masses(design_densities,true); - - //real_t initial_mass = ROL_Element_Masses->reduce(sumreduc); - - problem->setProjectionAlgorithm(*parlist); - //finalize problem - problem->finalize(false,true,*fos); - //problem->check(true,std::cout); - - //debug checks - ROL::Ptr> rol_x = - ROL::makePtr>(design_node_densities_distributed); - //construct direction vector for check - Teuchos::RCP directions_distributed = Teuchos::rcp(new MV(map, 1)); - directions_distributed->putScalar(-0.1); - directions_distributed->randomize(-0.8,1); - Kokkos::View direction_norm("gradient norm",1); - directions_distributed->norm2(direction_norm); - directions_distributed->scale(1/direction_norm(0)); - //set all but first component to 0 for debug - host_vec_array directions = directions_distributed->getLocalView (Tpetra::Access::ReadWrite); - //for(int init = 1; init < nlocal_nodes; init++) - //directions(4,0) = -0.3; - ROL::Ptr> rol_d = - ROL::makePtr>(directions_distributed); - obj->checkGradient(*rol_x, *rol_d); - //obj->checkHessVec(*rol_x, *rol_d); - //directions_distributed->putScalar(-0.000001); - //obj->checkGradient(*rol_x, *rol_d); - //directions_distributed->putScalar(-0.0000001); - //obj->checkGradient(*rol_x, *rol_d); - - - // Instantiate Solver. - ROL::Solver solver(problem,*parlist); - - // Solve optimization problem. - //std::ostream outStream; - //solver.solve(*fos); - - //print final constraint satisfaction - //fea_elasticity->compute_element_masses(design_densities,false); - //real_t final_mass = ROL_Element_Masses->reduce(sumreduc); - //if(myrank==0) - //std::cout << "Final Mass Constraint is " << final_mass/initial_mass << std::endl; -} + // create global vector + Global_Element_Densities_Upper_Bound = Teuchos::rcp(new MV(element_map, Element_Densities_Upper_Bound)); + Global_Element_Densities_Lower_Bound = Teuchos::rcp(new MV(element_map, Element_Densities_Lower_Bound)); + } + + // Bound constraint defining the possible range of design density variables + ROL::Ptr> lower_bounds; + ROL::Ptr> upper_bounds; + if (nodal_density_flag) + { + lower_bounds = ROL::makePtr>(lower_bound_node_densities_distributed); + upper_bounds = ROL::makePtr>(upper_bound_node_densities_distributed); + } + else + { + lower_bounds = ROL::makePtr>(Global_Element_Densities_Lower_Bound); + upper_bounds = ROL::makePtr>(Global_Element_Densities_Upper_Bound); + } + ROL::Ptr> bnd = ROL::makePtr>(lower_bounds, upper_bounds); + problem->addBoundConstraint(bnd); + + // compute initial constraint satisfaction + // ROL::Ptr ROL_Element_Masses = ROL::makePtr(fea_elasticity->Global_Element_Masses); + ROL::Elementwise::ReductionSum sumreduc; + if (nodal_density_flag) + { + design_densities = design_node_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); + } + else + { + design_densities = Global_Element_Densities->getLocalView(Tpetra::Access::ReadOnly); + } + // fea_elasticity->compute_element_masses(design_densities,true); + + // real_t initial_mass = ROL_Element_Masses->reduce(sumreduc); + + problem->setProjectionAlgorithm(*parlist); + // finalize problem + problem->finalize(false, true, *fos); + // problem->check(true,std::cout); + + // debug checks + ROL::Ptr> rol_x = + ROL::makePtr>(design_node_densities_distributed); + // construct direction vector for check + Teuchos::RCP directions_distributed = Teuchos::rcp(new MV(map, 1)); + directions_distributed->putScalar(-0.1); + directions_distributed->randomize(-0.8, 1); + Kokkos::View direction_norm("gradient norm", 1); + directions_distributed->norm2(direction_norm); + directions_distributed->scale(1 / direction_norm(0)); + // set all but first component to 0 for debug + host_vec_array directions = directions_distributed->getLocalView(Tpetra::Access::ReadWrite); + // for(int init = 1; init < nlocal_nodes; init++) + // directions(4,0) = -0.3; + ROL::Ptr> rol_d = + ROL::makePtr>(directions_distributed); + obj->checkGradient(*rol_x, *rol_d); + // obj->checkHessVec(*rol_x, *rol_d); + // directions_distributed->putScalar(-0.000001); + // obj->checkGradient(*rol_x, *rol_d); + // directions_distributed->putScalar(-0.0000001); + // obj->checkGradient(*rol_x, *rol_d); + + // Instantiate Solver. + ROL::Solver solver(problem, *parlist); + + // Solve optimization problem. + // std::ostream outStream; + // solver.solve(*fos); + + // print final constraint satisfaction + // fea_elasticity->compute_element_masses(design_densities,false); + // real_t final_mass = ROL_Element_Masses->reduce(sumreduc); + // if(myrank==0) + // std::cout << "Final Mass Constraint is " << final_mass/initial_mass << std::endl; +} /* ---------------------------------------------------------------------------- Initialize sets of element boundary surfaces and arrays for input conditions ------------------------------------------------------------------------------- */ -void Explicit_Solver::init_boundaries(){ - //int num_boundary_sets = simparam.NB; - int num_dim = simparam.num_dims; - size_t num_nodes_in_patch; - // build boundary mesh patches - if(myrank == 0) - std::cout << "Starting boundary patch setup" << std::endl <(nboundary_patches, num_nodes_in_patch, "Local_Index_Boundary_Patches"); - for(int iboundary = 0; iboundary < nboundary_patches; iboundary++){ - num_nodes_in_patch = Boundary_Patches(iboundary).node_set.size(); - for(int inode = 0; inode < num_nodes_in_patch; inode++){ - Local_Index_Boundary_Patches.host(iboundary,inode) = all_node_map->getLocalElement(Boundary_Patches(iboundary).node_set(inode)); - } - } - Local_Index_Boundary_Patches.update_device(); - - //std::cout << "Done with boundary patch setup" << std::endl <(nboundary_patches, num_nodes_in_patch, "Local_Index_Boundary_Patches"); + for (int iboundary = 0; iboundary < nboundary_patches; iboundary++) + { + num_nodes_in_patch = Boundary_Patches(iboundary).node_set.size(); + for (int inode = 0; inode < num_nodes_in_patch; inode++) + { + Local_Index_Boundary_Patches.host(iboundary, inode) = all_node_map->getLocalElement(Boundary_Patches(iboundary).node_set(inode)); + } + } + Local_Index_Boundary_Patches.update_device(); + + // std::cout << "Done with boundary patch setup" << std::endl <(num_sets, "NBoundary_Condition_Patches"); - Topology_Condition_Patches = CArrayKokkos(num_sets, nboundary_patches, "Boundary_Condition_Patches"); - - //initialize data - for(int iset = 0; iset < num_sets; iset++) NTopology_Condition_Patches(iset) = 0; +void Explicit_Solver::init_topology_conditions(int num_sets) +{ + // surface conditions + num_boundary_conditions = num_sets; + if (num_sets == 0) + { + std::cout << " Warning: number of boundary conditions = 0"; + return; + } + NTopology_Condition_Patches = CArrayKokkos(num_sets, "NBoundary_Condition_Patches"); + Topology_Condition_Patches = CArrayKokkos(num_sets, nboundary_patches, "Boundary_Condition_Patches"); + + // initialize data + for (int iset = 0; iset < num_sets; iset++) + { + NTopology_Condition_Patches(iset) = 0; + } } /* ---------------------------------------------------------------------- @@ -1516,43 +1666,53 @@ void Explicit_Solver::init_topology_conditions (int num_sets){ val = plane value, cylinder radius, shell radius ------------------------------------------------------------------------- */ -void Explicit_Solver::tag_boundaries(int bc_tag, real_t val, int bdy_set, real_t *patch_limits){ - - int num_dim = simparam.num_dims; - int is_on_set; - /* - if (bdy_set == num_bdy_sets_){ - std::cout << " ERROR: number of boundary sets must be increased by " - << bdy_set-num_bdy_sets_+1 << std::endl; - exit(0); - } - */ - - //test patch limits for feasibility - if(patch_limits != NULL){ - //test for upper bounds being greater than lower bounds - if(patch_limits[1] <= patch_limits[0]) std::cout << " Warning: patch limits for boundary condition are infeasible"; - if(num_dim==3) - if(patch_limits[2] <= patch_limits[3]) std::cout << " Warning: patch limits for boundary condition are infeasible"; - } - - // save the boundary vertices to this set that are on the plane - int counter = 0; - for (int iboundary_patch = 0; iboundary_patch < nboundary_patches; iboundary_patch++) { - - // check to see if this patch is on the specified plane - is_on_set = check_boundary(Boundary_Patches(iboundary_patch), bc_tag, val, patch_limits); // no=0, yes=1 - - if (is_on_set == 1){ - Topology_Condition_Patches(bdy_set,counter) = iboundary_patch; - counter ++; - } - } // end for bdy_patch - - // save the number of bdy patches in the set - NTopology_Condition_Patches(bdy_set) = counter; - - *fos << " tagged boundary patches " << std::endl; +void Explicit_Solver::tag_boundaries(int bc_tag, real_t val, int bdy_set, real_t* patch_limits) +{ + int num_dim = simparam.num_dims; + int is_on_set; + /* + if (bdy_set == num_bdy_sets_){ + std::cout << " ERROR: number of boundary sets must be increased by " + << bdy_set-num_bdy_sets_+1 << std::endl; + exit(0); + } + */ + + // test patch limits for feasibility + if (patch_limits != NULL) + { + // test for upper bounds being greater than lower bounds + if (patch_limits[1] <= patch_limits[0]) + { + std::cout << " Warning: patch limits for boundary condition are infeasible"; + } + if (num_dim == 3) + { + if (patch_limits[2] <= patch_limits[3]) + { + std::cout << " Warning: patch limits for boundary condition are infeasible"; + } + } + } + + // save the boundary vertices to this set that are on the plane + int counter = 0; + for (int iboundary_patch = 0; iboundary_patch < nboundary_patches; iboundary_patch++) + { + // check to see if this patch is on the specified plane + is_on_set = check_boundary(Boundary_Patches(iboundary_patch), bc_tag, val, patch_limits); // no=0, yes=1 + + if (is_on_set == 1) + { + Topology_Condition_Patches(bdy_set, counter) = iboundary_patch; + counter++; + } + } // end for bdy_patch + + // save the number of bdy patches in the set + NTopology_Condition_Patches(bdy_set) = counter; + + *fos << " tagged boundary patches " << std::endl; } /* ---------------------------------------------------------------------- @@ -1561,94 +1721,121 @@ void Explicit_Solver::tag_boundaries(int bc_tag, real_t val, int bdy_set, real_t val = plane value, radius, radius ------------------------------------------------------------------------- */ +int Explicit_Solver::check_boundary(Node_Combination& Patch_Nodes, int bc_tag, real_t val, real_t* patch_limits) +{ + int is_on_set = 1; + const_host_vec_array all_node_coords = all_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + + // Nodes on the Patch + auto node_list = Patch_Nodes.node_set; + int num_dim = simparam.num_dims; + size_t nnodes = node_list.size(); + size_t node_rid; + real_t node_coord[num_dim]; + int dim_other1, dim_other2; + CArrayKokkos node_on_flags(nnodes, "node_on_flags"); + + // initialize + for (int inode = 0; inode < nnodes; inode++) + { + node_on_flags(inode) = 0; + } -int Explicit_Solver::check_boundary(Node_Combination &Patch_Nodes, int bc_tag, real_t val, real_t *patch_limits){ - - int is_on_set = 1; - const_host_vec_array all_node_coords = all_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - - //Nodes on the Patch - auto node_list = Patch_Nodes.node_set; - int num_dim = simparam.num_dims; - size_t nnodes = node_list.size(); - size_t node_rid; - real_t node_coord[num_dim]; - int dim_other1, dim_other2; - CArrayKokkos node_on_flags(nnodes, "node_on_flags"); - - //initialize - for(int inode = 0; inode < nnodes; inode++) node_on_flags(inode) = 0; - - if(bc_tag==0){ - dim_other1 = 1; - dim_other2 = 2; - } - else if(bc_tag==1){ - dim_other1 = 0; - dim_other2 = 2; - } - else if(bc_tag==2){ - dim_other1 = 0; - dim_other2 = 1; - } - - - //test for planes - if(bc_tag < 3) - for(int inode = 0; inode < nnodes; inode++){ - - node_rid = all_node_map->getLocalElement(node_list(inode)); - for(int init=0; init < num_dim; init++){ - node_coord[init] = all_node_coords(node_rid,init); - } - if ( fabs(node_coord[bc_tag] - val) <= BC_EPSILON){ node_on_flags(inode) = 1; - - //test if within patch segment if user specified - if(patch_limits!=NULL){ - if (node_coord[dim_other1] - patch_limits[0] <= -BC_EPSILON) node_on_flags(inode) = 0; - if (node_coord[dim_other1] - patch_limits[1] >= BC_EPSILON) node_on_flags(inode) = 0; - if(num_dim==3){ - if (node_coord[dim_other2] - patch_limits[2] <= -BC_EPSILON) node_on_flags(inode) = 0; - if (node_coord[dim_other2] - patch_limits[3] >= BC_EPSILON) node_on_flags(inode) = 0; + if (bc_tag == 0) + { + dim_other1 = 1; + dim_other2 = 2; + } + else if (bc_tag == 1) + { + dim_other1 = 0; + dim_other2 = 2; + } + else if (bc_tag == 2) + { + dim_other1 = 0; + dim_other2 = 1; + } + + // test for planes + if (bc_tag < 3) + { + for (int inode = 0; inode < nnodes; inode++) + { + node_rid = all_node_map->getLocalElement(node_list(inode)); + for (int init = 0; init < num_dim; init++) + { + node_coord[init] = all_node_coords(node_rid, init); + } + if (fabs(node_coord[bc_tag] - val) <= BC_EPSILON) + { + node_on_flags(inode) = 1; + + // test if within patch segment if user specified + if (patch_limits != NULL) + { + if (node_coord[dim_other1] - patch_limits[0] <= -BC_EPSILON) + { + node_on_flags(inode) = 0; + } + if (node_coord[dim_other1] - patch_limits[1] >= BC_EPSILON) + { + node_on_flags(inode) = 0; + } + if (num_dim == 3) + { + if (node_coord[dim_other2] - patch_limits[2] <= -BC_EPSILON) + { + node_on_flags(inode) = 0; + } + if (node_coord[dim_other2] - patch_limits[3] >= BC_EPSILON) + { + node_on_flags(inode) = 0; + } + } + } + } + // debug print of node id and node coord + // std::cout << "node coords on task " << myrank << " for node " << node_rid << std::endl; + // std::cout << "coord " <(num_nodes,0,comm)); - - //importer from local node distribution to sorted distribution - //Tpetra::Import node_sorting_importer(map, sorted_map); - - sorted_node_coords_distributed = Teuchos::rcp(new MV(sorted_map, num_dim)); - sorted_node_velocities_distributed = Teuchos::rcp(new MV(sorted_map, num_dim)); - - //comms to sort - sorted_node_coords_distributed->doImport(*node_coords_distributed, *node_sorting_importer, Tpetra::INSERT); - sorted_node_velocities_distributed->doImport(*node_velocities_distributed, *node_sorting_importer, Tpetra::INSERT); - - //comms to sort FEA module related vector data - /* - for (int imodule = 0; imodule < nfea_modules; imodule++){ - fea_modules[imodule]->collect_output(global_reduce_map); - //collected_node_displacements_distributed->doImport(*(fea_elasticity->node_displacements_distributed), dof_collection_importer, Tpetra::INSERT); - } - */ - - //sorted nodal density information - if(simparam.topology_optimization_on){ - sorted_node_densities_distributed = Teuchos::rcp(new MV(sorted_map, 1)); - sorted_node_densities_distributed->doImport(*design_node_densities_distributed, *node_sorting_importer, Tpetra::INSERT); - } - //comms to sort - //collected_node_densities_distributed->doImport(*design_node_densities_distributed, node_collection_importer, Tpetra::INSERT); - - //comms to collect FEA module related vector data - for (int imodule = 0; imodule < nfea_modules; imodule++){ - fea_modules[imodule]->sort_output(sorted_map); - //collected_node_displacements_distributed->doImport(*(fea_elasticity->node_displacements_distributed), dof_collection_importer, Tpetra::INSERT); - } - - //Global_Element_Densities->describe(*fos,Teuchos::VERB_EXTREME); - - sorted_element_densities_distributed = Teuchos::rcp(new MV(sorted_element_map, 1)); - - for (int imodule = 0; imodule < nfea_modules; imodule++){ - fea_modules[imodule]->sort_element_output(sorted_element_map); - //collected_node_displacements_distributed->doImport(*(fea_elasticity->node_displacements_distributed), dof_collection_importer, Tpetra::INSERT); - } - - sorted_nodes_in_elem_distributed = Teuchos::rcp(new MCONN(sorted_element_map, max_nodes_per_element)); - - //comms - sorted_nodes_in_elem_distributed->doImport(*global_nodes_in_elem_distributed, *element_sorting_importer, Tpetra::INSERT); - sorted_element_densities_distributed->doImport(*Global_Element_Densities, *element_sorting_importer, Tpetra::INSERT); - +void Explicit_Solver::sort_information() +{ + int num_dim = simparam.num_dims; + // sorted_map = Teuchos::rcp( new Tpetra::Map(num_nodes,0,comm)); + + // importer from local node distribution to sorted distribution + // Tpetra::Import node_sorting_importer(map, sorted_map); + + sorted_node_coords_distributed = Teuchos::rcp(new MV(sorted_map, num_dim)); + sorted_node_velocities_distributed = Teuchos::rcp(new MV(sorted_map, num_dim)); + + // comms to sort + sorted_node_coords_distributed->doImport(*node_coords_distributed, *node_sorting_importer, Tpetra::INSERT); + sorted_node_velocities_distributed->doImport(*node_velocities_distributed, *node_sorting_importer, Tpetra::INSERT); + + // comms to sort FEA module related vector data + /* + for (int imodule = 0; imodule < nfea_modules; imodule++){ + fea_modules[imodule]->collect_output(global_reduce_map); + //collected_node_displacements_distributed->doImport(*(fea_elasticity->node_displacements_distributed), dof_collection_importer, Tpetra::INSERT); + } + */ + + // sorted nodal density information + if (simparam.topology_optimization_on) + { + sorted_node_densities_distributed = Teuchos::rcp(new MV(sorted_map, 1)); + sorted_node_densities_distributed->doImport(*design_node_densities_distributed, *node_sorting_importer, Tpetra::INSERT); + } + // comms to sort + // collected_node_densities_distributed->doImport(*design_node_densities_distributed, node_collection_importer, Tpetra::INSERT); + + // comms to collect FEA module related vector data + for (int imodule = 0; imodule < nfea_modules; imodule++) + { + fea_modules[imodule]->sort_output(sorted_map); + // collected_node_displacements_distributed->doImport(*(fea_elasticity->node_displacements_distributed), dof_collection_importer, Tpetra::INSERT); + } + + // Global_Element_Densities->describe(*fos,Teuchos::VERB_EXTREME); + + sorted_element_densities_distributed = Teuchos::rcp(new MV(sorted_element_map, 1)); + + for (int imodule = 0; imodule < nfea_modules; imodule++) + { + fea_modules[imodule]->sort_element_output(sorted_element_map); + // collected_node_displacements_distributed->doImport(*(fea_elasticity->node_displacements_distributed), dof_collection_importer, Tpetra::INSERT); + } + + sorted_nodes_in_elem_distributed = Teuchos::rcp(new MCONN(sorted_element_map, max_nodes_per_element)); + + // comms + sorted_nodes_in_elem_distributed->doImport(*global_nodes_in_elem_distributed, *element_sorting_importer, Tpetra::INSERT); + sorted_element_densities_distributed->doImport(*Global_Element_Densities, *element_sorting_importer, Tpetra::INSERT); } /* ---------------------------------------------------------------------- Collect Nodal Information on Rank 0 ------------------------------------------------------------------------- */ -void Explicit_Solver::collect_information(){ - GO nreduce_nodes = 0; - GO nreduce_elem = 0; - int num_dim = simparam.num_dims; +void Explicit_Solver::collect_information() +{ + GO nreduce_nodes = 0; + GO nreduce_elem = 0; + int num_dim = simparam.num_dims; + + // collect nodal coordinate information + if (myrank == 0) + { + nreduce_nodes = num_nodes; + } + Teuchos::RCP> global_reduce_map = + Teuchos::rcp(new Tpetra::Map(Teuchos::OrdinalTraits::invalid(), nreduce_nodes, 0, comm)); - //collect nodal coordinate information - if(myrank==0) nreduce_nodes = num_nodes; - Teuchos::RCP > global_reduce_map = - Teuchos::rcp(new Tpetra::Map(Teuchos::OrdinalTraits::invalid(),nreduce_nodes,0,comm)); - - //importer from local node distribution to collected distribution - Tpetra::Import node_collection_importer(map, global_reduce_map); + // importer from local node distribution to collected distribution + Tpetra::Import node_collection_importer(map, global_reduce_map); - collected_node_coords_distributed = Teuchos::rcp(new MV(global_reduce_map, num_dim)); - collected_node_velocities_distributed = Teuchos::rcp(new MV(global_reduce_map, num_dim)); + collected_node_coords_distributed = Teuchos::rcp(new MV(global_reduce_map, num_dim)); + collected_node_velocities_distributed = Teuchos::rcp(new MV(global_reduce_map, num_dim)); - //comms to collect - collected_node_coords_distributed->doImport(*node_coords_distributed, node_collection_importer, Tpetra::INSERT); - collected_node_velocities_distributed->doImport(*node_velocities_distributed, node_collection_importer, Tpetra::INSERT); + // comms to collect + collected_node_coords_distributed->doImport(*node_coords_distributed, node_collection_importer, Tpetra::INSERT); + collected_node_velocities_distributed->doImport(*node_velocities_distributed, node_collection_importer, Tpetra::INSERT); - //comms to collect FEA module related vector data - /* - for (int imodule = 0; imodule < nfea_modules; imodule++){ - fea_modules[imodule]->collect_output(global_reduce_map); - //collected_node_displacements_distributed->doImport(*(fea_elasticity->node_displacements_distributed), dof_collection_importer, Tpetra::INSERT); - } - */ + // comms to collect FEA module related vector data + /* + for (int imodule = 0; imodule < nfea_modules; imodule++){ + fea_modules[imodule]->collect_output(global_reduce_map); + //collected_node_displacements_distributed->doImport(*(fea_elasticity->node_displacements_distributed), dof_collection_importer, Tpetra::INSERT); + } + */ - //collected nodal density information - collected_node_densities_distributed = Teuchos::rcp(new MV(global_reduce_map, 1)); + // collected nodal density information + collected_node_densities_distributed = Teuchos::rcp(new MV(global_reduce_map, 1)); - //comms to collect - //collected_node_densities_distributed->doImport(*design_node_densities_distributed, node_collection_importer, Tpetra::INSERT); + // comms to collect + // collected_node_densities_distributed->doImport(*design_node_densities_distributed, node_collection_importer, Tpetra::INSERT); - //collect element connectivity data - if(myrank==0) nreduce_elem = num_elem; - Teuchos::RCP > global_reduce_element_map = - Teuchos::rcp(new Tpetra::Map(Teuchos::OrdinalTraits::invalid(),nreduce_elem,0,comm)); + // collect element connectivity data + if (myrank == 0) + { + nreduce_elem = num_elem; + } + Teuchos::RCP> global_reduce_element_map = + Teuchos::rcp(new Tpetra::Map(Teuchos::OrdinalTraits::invalid(), nreduce_elem, 0, comm)); - //importer from all element map to collected distribution - Tpetra::Import element_collection_importer(all_element_map, global_reduce_element_map); - - collected_nodes_in_elem_distributed = Teuchos::rcp(new MCONN(global_reduce_element_map, max_nodes_per_element)); + // importer from all element map to collected distribution + Tpetra::Import element_collection_importer(all_element_map, global_reduce_element_map); - //comms to collect - collected_nodes_in_elem_distributed->doImport(*global_nodes_in_elem_distributed, element_collection_importer, Tpetra::INSERT); + collected_nodes_in_elem_distributed = Teuchos::rcp(new MCONN(global_reduce_element_map, max_nodes_per_element)); - //collect element type data + // comms to collect + collected_nodes_in_elem_distributed->doImport(*global_nodes_in_elem_distributed, element_collection_importer, Tpetra::INSERT); + // collect element type data } /* ---------------------------------------------------------------------- Communicate updated nodal velocities to ghost nodes ------------------------------------------------------------------------- */ -void Explicit_Solver::comm_velocities(){ - - //debug print of design vector - //std::ostream &out = std::cout; - //Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - //if(myrank==0) - //*fos << "Density data :" << std::endl; - //node_densities_distributed->describe(*fos,Teuchos::VERB_EXTREME); - //*fos << std::endl; - //std::fflush(stdout); - - //communicate design densities - //create import object using local node indices map and all indices map - //Tpetra::Import importer(map, ghost_node_map); - - //comms to get ghosts - ghost_node_velocities_distributed->doImport(*node_velocities_distributed, *ghost_importer, Tpetra::INSERT); - //all_node_map->describe(*fos,Teuchos::VERB_EXTREME); - //all_node_velocities_distributed->describe(*fos,Teuchos::VERB_EXTREME); - - //update_count++; - //if(update_count==1){ - //MPI_Barrier(world); - //MPI_Abort(world,4); - //} +void Explicit_Solver::comm_velocities() +{ + // debug print of design vector + // std::ostream &out = std::cout; + // Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + // if(myrank==0) + // *fos << "Density data :" << std::endl; + // node_densities_distributed->describe(*fos,Teuchos::VERB_EXTREME); + // *fos << std::endl; + // std::fflush(stdout); + + // communicate design densities + // create import object using local node indices map and all indices map + // Tpetra::Import importer(map, ghost_node_map); + + // comms to get ghosts + ghost_node_velocities_distributed->doImport(*node_velocities_distributed, *ghost_importer, Tpetra::INSERT); + // all_node_map->describe(*fos,Teuchos::VERB_EXTREME); + // all_node_velocities_distributed->describe(*fos,Teuchos::VERB_EXTREME); + + // update_count++; + // if(update_count==1){ + // MPI_Barrier(world); + // MPI_Abort(world,4); + // } } /* ---------------------------------------------------------------------- Communicate nodal masses/densities to ghost nodes ------------------------------------------------------------------------- */ -void Explicit_Solver::comm_densities(){ - - //debug print of design vector - //std::ostream &out = std::cout; - //Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - //if(myrank==0) - //*fos << "Density data :" << std::endl; - //node_densities_distributed->describe(*fos,Teuchos::VERB_EXTREME); - //*fos << std::endl; - //std::fflush(stdout); - - //communicate design densities - //create import object using local node indices map and all indices map - //Tpetra::Import importer(map, all_node_map); - - //active view scope - { - const_host_vec_array node_densities_host = design_node_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); - } - //comms to get ghosts - all_node_densities_distributed->doImport(*design_node_densities_distributed, *importer, Tpetra::INSERT); - //all_node_map->describe(*fos,Teuchos::VERB_EXTREME); - //all_node_velocities_distributed->describe(*fos,Teuchos::VERB_EXTREME); - - //update_count++; - //if(update_count==1){ - //MPI_Barrier(world); - //MPI_Abort(world,4); - //} +void Explicit_Solver::comm_densities() +{ + // debug print of design vector + // std::ostream &out = std::cout; + // Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + // if(myrank==0) + // *fos << "Density data :" << std::endl; + // node_densities_distributed->describe(*fos,Teuchos::VERB_EXTREME); + // *fos << std::endl; + // std::fflush(stdout); + + // communicate design densities + // create import object using local node indices map and all indices map + // Tpetra::Import importer(map, all_node_map); + + // active view scope + { + const_host_vec_array node_densities_host = design_node_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); + } + // comms to get ghosts + all_node_densities_distributed->doImport(*design_node_densities_distributed, *importer, Tpetra::INSERT); + // all_node_map->describe(*fos,Teuchos::VERB_EXTREME); + // all_node_velocities_distributed->describe(*fos,Teuchos::VERB_EXTREME); + + // update_count++; + // if(update_count==1){ + // MPI_Barrier(world); + // MPI_Abort(world,4); + // } } /* ---------------------------------------------------------------------- Output Model Information in tecplot format ------------------------------------------------------------------------- */ -void Explicit_Solver::parallel_tecplot_writer(){ - int num_dim = simparam.num_dims; - std::string current_file_name; - std::string base_file_name= "TecplotTO"; - std::string base_file_name_undeformed= "TecplotTO_undeformed"; - std::stringstream current_line_stream; - std::string current_line; - std::string file_extension= ".dat"; - std::string file_count; - std::stringstream count_temp; - int time_step = 0; - int temp_convert; - int noutput, nvector; - bool displace_geometry = false; - // Convert ijk index system to the finite element numbering convention - // for vertices in cell - - //comm to vectors with contigously sorted global indices from unsorted zoltan2 repartition map - sort_information(); - //set host views of the communicated data to print out from - const_host_vec_array sorted_node_coords = sorted_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_host_vec_array sorted_node_velocities = sorted_node_velocities_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_host_vec_array sorted_node_densities; - if(simparam.topology_optimization_on){ - sorted_node_densities = sorted_node_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); - } - const_host_elem_conn_array sorted_nodes_in_elem = sorted_nodes_in_elem_distributed->getLocalView (Tpetra::Access::ReadOnly); - - CArrayKokkos convert_ijk_to_ensight(max_nodes_per_element); - CArrayKokkos tmp_ijk_indx(max_nodes_per_element); - convert_ijk_to_ensight(0) = 0; - convert_ijk_to_ensight(1) = 1; - convert_ijk_to_ensight(2) = 3; - convert_ijk_to_ensight(3) = 2; - if(num_dim==3){ - convert_ijk_to_ensight(4) = 4; - convert_ijk_to_ensight(5) = 5; - convert_ijk_to_ensight(6) = 7; - convert_ijk_to_ensight(7) = 6; - } - - MPI_File myfile_parallel; - MPI_Offset header_stream_offset = 0; - //initial undeformed geometry - count_temp.str(""); - count_temp << file_index; - file_index++; - file_count = count_temp.str(); - if(displace_geometry&&displacement_module>=0) - current_file_name = base_file_name_undeformed + file_count + file_extension; - else - current_file_name = base_file_name + file_count + file_extension; - MPI_File_open(MPI_COMM_WORLD, current_file_name.c_str(), - MPI_MODE_CREATE|MPI_MODE_WRONLY, +void Explicit_Solver::parallel_tecplot_writer() +{ + int num_dim = simparam.num_dims; + std::string current_file_name; + std::string base_file_name = "TecplotTO"; + std::string base_file_name_undeformed = "TecplotTO_undeformed"; + std::stringstream current_line_stream; + std::string current_line; + std::string file_extension = ".dat"; + std::string file_count; + std::stringstream count_temp; + int time_step = 0; + int temp_convert; + int noutput, nvector; + bool displace_geometry = false; + // Convert ijk index system to the finite element numbering convention + // for vertices in cell + + // comm to vectors with contigously sorted global indices from unsorted zoltan2 repartition map + sort_information(); + // set host views of the communicated data to print out from + const_host_vec_array sorted_node_coords = sorted_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_host_vec_array sorted_node_velocities = sorted_node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_host_vec_array sorted_node_densities; + if (simparam.topology_optimization_on) + { + sorted_node_densities = sorted_node_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); + } + const_host_elem_conn_array sorted_nodes_in_elem = sorted_nodes_in_elem_distributed->getLocalView(Tpetra::Access::ReadOnly); + + CArrayKokkos convert_ijk_to_ensight(max_nodes_per_element); + CArrayKokkos tmp_ijk_indx(max_nodes_per_element); + convert_ijk_to_ensight(0) = 0; + convert_ijk_to_ensight(1) = 1; + convert_ijk_to_ensight(2) = 3; + convert_ijk_to_ensight(3) = 2; + if (num_dim == 3) + { + convert_ijk_to_ensight(4) = 4; + convert_ijk_to_ensight(5) = 5; + convert_ijk_to_ensight(6) = 7; + convert_ijk_to_ensight(7) = 6; + } + + MPI_File myfile_parallel; + MPI_Offset header_stream_offset = 0; + // initial undeformed geometry + count_temp.str(""); + count_temp << file_index; + file_index++; + file_count = count_temp.str(); + if (displace_geometry && displacement_module >= 0) + { + current_file_name = base_file_name_undeformed + file_count + file_extension; + } + else + { + current_file_name = base_file_name + file_count + file_extension; + } + MPI_File_open(MPI_COMM_WORLD, current_file_name.c_str(), + MPI_MODE_CREATE | MPI_MODE_WRONLY, MPI_INFO_NULL, &myfile_parallel); - - int err = MPI_File_open(MPI_COMM_WORLD, current_file_name.c_str(), MPI_MODE_CREATE|MPI_MODE_EXCL|MPI_MODE_WRONLY, MPI_INFO_NULL, &myfile_parallel); - //allows overwriting the file if it already existed in the directory - if (err != MPI_SUCCESS) { - if (myrank == 0){ - MPI_File_delete(current_file_name.c_str(),MPI_INFO_NULL); - } - MPI_File_open(MPI_COMM_WORLD, current_file_name.c_str(), MPI_MODE_CREATE|MPI_MODE_EXCL|MPI_MODE_WRONLY, MPI_INFO_NULL, &myfile_parallel); - } - - //output header of the tecplot file - - //std::cout << current_file_name << std::endl; - current_line_stream << "TITLE=\"results for FEA simulation\"" "\n"; - current_line = current_line_stream.str(); - if(myrank == 0) - MPI_File_write(myfile_parallel,current_line.c_str(),current_line.length(), MPI_CHAR, MPI_STATUS_IGNORE); - header_stream_offset += current_line.length(); - //myfile << "VARIABLES = \"x\", \"y\", \"z\", \"density\", \"sigmaxx\", \"sigmayy\", \"sigmazz\", \"sigmaxy\", \"sigmaxz\", \"sigmayz\"" "\n"; - //else - current_line_stream.str(""); - if(num_dim==3) - current_line_stream << "VARIABLES = \"x\", \"y\", \"z\", \"vx\", \"vy\", \"vz\""; - else if(num_dim==2) - current_line_stream << "VARIABLES = \"x\", \"y\", \"vx\", \"vy\""; - current_line = current_line_stream.str(); - if(myrank == 0) - MPI_File_write(myfile_parallel,current_line.c_str(),current_line.length(), MPI_CHAR, MPI_STATUS_IGNORE); - header_stream_offset += current_line.length(); + + int err = MPI_File_open(MPI_COMM_WORLD, current_file_name.c_str(), MPI_MODE_CREATE | MPI_MODE_EXCL | MPI_MODE_WRONLY, MPI_INFO_NULL, &myfile_parallel); + // allows overwriting the file if it already existed in the directory + if (err != MPI_SUCCESS) + { + if (myrank == 0) + { + MPI_File_delete(current_file_name.c_str(), MPI_INFO_NULL); + } + MPI_File_open(MPI_COMM_WORLD, current_file_name.c_str(), MPI_MODE_CREATE | MPI_MODE_EXCL | MPI_MODE_WRONLY, MPI_INFO_NULL, &myfile_parallel); + } + + // output header of the tecplot file + + // std::cout << current_file_name << std::endl; + current_line_stream << "TITLE=\"results for FEA simulation\"" "\n"; + current_line = current_line_stream.str(); + if (myrank == 0) + { + MPI_File_write(myfile_parallel, current_line.c_str(), current_line.length(), MPI_CHAR, MPI_STATUS_IGNORE); + } + header_stream_offset += current_line.length(); + // myfile << "VARIABLES = \"x\", \"y\", \"z\", \"density\", \"sigmaxx\", \"sigmayy\", \"sigmazz\", \"sigmaxy\", \"sigmaxz\", \"sigmayz\"" "\n"; + // else + current_line_stream.str(""); + if (num_dim == 3) + { + current_line_stream << "VARIABLES = \"x\", \"y\", \"z\", \"vx\", \"vy\", \"vz\""; + } + else if (num_dim == 2) + { + current_line_stream << "VARIABLES = \"x\", \"y\", \"vx\", \"vy\""; + } + current_line = current_line_stream.str(); + if (myrank == 0) + { + MPI_File_write(myfile_parallel, current_line.c_str(), current_line.length(), MPI_CHAR, MPI_STATUS_IGNORE); + } + header_stream_offset += current_line.length(); /* for (int imodule = 0; imodule < nfea_modules; imodule++){ for(int ioutput = 0; ioutput < fea_modules[imodule]->noutput; ioutput++){ @@ -1929,53 +2142,64 @@ void Explicit_Solver::parallel_tecplot_writer(){ } } */ - current_line_stream.str(""); - current_line_stream << "\n"; - current_line = current_line_stream.str(); - if(myrank == 0) - MPI_File_write(myfile_parallel,current_line.c_str(),current_line.length(), MPI_CHAR, MPI_STATUS_IGNORE); - header_stream_offset += current_line.length(); - - current_line_stream.str(""); - if(num_dim==2){ - current_line_stream << "ZONE T=\"load step " << time_step << "\", NODES= " << num_nodes - << ", ELEMENTS= " << num_elem << ", DATAPACKING=POINT, ZONETYPE=FEQUADRILATERAL" "\n"; - } - else if(num_dim==3){ - current_line_stream << "ZONE T=\"load step " << time_step << "\", NODES= " << num_nodes - << ", ELEMENTS= " << num_elem << ", DATAPACKING=POINT, ZONETYPE=FEBRICK" "\n"; - } - current_line = current_line_stream.str(); - if(myrank == 0) - MPI_File_write(myfile_parallel,current_line.c_str(),current_line.length(), MPI_CHAR, MPI_STATUS_IGNORE); - header_stream_offset += current_line.length(); - - //output nodal data - //compute buffer output size and file stream offset for this MPI rank - int default_vector_count = 2; - int buffer_size_per_node_line = 26*default_vector_count*num_dim + 1; //25 width per number + 1 space times 6 entries plus line terminator - int nlocal_sorted_nodes = sorted_map->getLocalNumElements(); - GO first_node_global_id = sorted_map->getGlobalElement(0); - CArrayKokkos print_buffer(buffer_size_per_node_line*nlocal_sorted_nodes); - MPI_Offset file_stream_offset = buffer_size_per_node_line*first_node_global_id; - - //populate buffer - long long current_buffer_position = 0; - current_line_stream << std::fixed << std::setprecision(8); - for (int nodeline = 0; nodeline < nlocal_sorted_nodes; nodeline++) { current_line_stream.str(""); - current_line_stream << std::setw(25) << sorted_node_coords(nodeline,0) << " "; - current_line_stream << std::setw(25) << sorted_node_coords(nodeline,1) << " "; - if(num_dim==3) - current_line_stream << std::setw(25) << sorted_node_coords(nodeline,2) << " "; - - //velocity print - current_line_stream << std::setw(25) << sorted_node_velocities(nodeline,0) << " "; - current_line_stream << std::setw(25) << sorted_node_velocities(nodeline,1) << " "; - if(num_dim==3) - current_line_stream << std::setw(25) << sorted_node_velocities(nodeline,2) << " "; - - //myfile << std::setw(25) << collected_node_densities(nodeline,0) << " "; + current_line_stream << "\n"; + current_line = current_line_stream.str(); + if (myrank == 0) + { + MPI_File_write(myfile_parallel, current_line.c_str(), current_line.length(), MPI_CHAR, MPI_STATUS_IGNORE); + } + header_stream_offset += current_line.length(); + + current_line_stream.str(""); + if (num_dim == 2) + { + current_line_stream << "ZONE T=\"load step " << time_step << "\", NODES= " << num_nodes + << ", ELEMENTS= " << num_elem << ", DATAPACKING=POINT, ZONETYPE=FEQUADRILATERAL" "\n"; + } + else if (num_dim == 3) + { + current_line_stream << "ZONE T=\"load step " << time_step << "\", NODES= " << num_nodes + << ", ELEMENTS= " << num_elem << ", DATAPACKING=POINT, ZONETYPE=FEBRICK" "\n"; + } + current_line = current_line_stream.str(); + if (myrank == 0) + { + MPI_File_write(myfile_parallel, current_line.c_str(), current_line.length(), MPI_CHAR, MPI_STATUS_IGNORE); + } + header_stream_offset += current_line.length(); + + // output nodal data + // compute buffer output size and file stream offset for this MPI rank + int default_vector_count = 2; + int buffer_size_per_node_line = 26 * default_vector_count * num_dim + 1; // 25 width per number + 1 space times 6 entries plus line terminator + int nlocal_sorted_nodes = sorted_map->getLocalNumElements(); + GO first_node_global_id = sorted_map->getGlobalElement(0); + CArrayKokkos print_buffer(buffer_size_per_node_line * nlocal_sorted_nodes); + MPI_Offset file_stream_offset = buffer_size_per_node_line * first_node_global_id; + + // populate buffer + long long current_buffer_position = 0; + current_line_stream << std::fixed << std::setprecision(8); + for (int nodeline = 0; nodeline < nlocal_sorted_nodes; nodeline++) + { + current_line_stream.str(""); + current_line_stream << std::setw(25) << sorted_node_coords(nodeline, 0) << " "; + current_line_stream << std::setw(25) << sorted_node_coords(nodeline, 1) << " "; + if (num_dim == 3) + { + current_line_stream << std::setw(25) << sorted_node_coords(nodeline, 2) << " "; + } + + // velocity print + current_line_stream << std::setw(25) << sorted_node_velocities(nodeline, 0) << " "; + current_line_stream << std::setw(25) << sorted_node_velocities(nodeline, 1) << " "; + if (num_dim == 3) + { + current_line_stream << std::setw(25) << sorted_node_velocities(nodeline, 2) << " "; + } + + // myfile << std::setw(25) << collected_node_densities(nodeline,0) << " "; /* for (int imodule = 0; imodule < nfea_modules; imodule++){ noutput = fea_modules[imodule]->noutput; @@ -1987,174 +2211,195 @@ void Explicit_Solver::parallel_tecplot_writer(){ myfile << std::setw(25) << current_collected_output(nodeline*nvector + ivector,0) << " "; } } - if(fea_modules[imodule]->vector_style[ioutput] == FEA_Module::NODAL){ - nvector = fea_modules[imodule]->output_vector_sizes[ioutput]; - for(int ivector = 0; ivector < nvector; ivector++){ - myfile << std::setw(25) << current_collected_output(nodeline,ivector) << " "; - } + if(fea_modules[imodule]->vector_style[ioutput] == FEA_Module::NODAL){ + nvector = fea_modules[imodule]->output_vector_sizes[ioutput]; + for(int ivector = 0; ivector < nvector; ivector++){ + myfile << std::setw(25) << current_collected_output(nodeline,ivector) << " "; + } + } + } + } + */ + current_line_stream << std::endl; + + current_line = current_line_stream.str(); + + // copy current line over to C style string buffer (wrapped by matar) + strcpy(&print_buffer(current_buffer_position), current_line.c_str()); + + current_buffer_position += current_line.length(); + } + + // print buffers at offsets with collective MPI write + // MPI_Offset current_stream_position = MPI_File_get_position(myfile_parallel,0); + MPI_Barrier(world); + MPI_File_write_at_all(myfile_parallel, file_stream_offset + header_stream_offset, print_buffer.get_kokkos_view().data(), buffer_size_per_node_line * nlocal_sorted_nodes, MPI_CHAR, + MPI_STATUS_IGNORE); + // MPI_File_close(&myfile_parallel); + + // write element connectivity; reopen to reset offset baseline. + // err = MPI_File_open(MPI_COMM_WORLD, current_file_name.c_str(), MPI_MODE_APPEND|MPI_MODE_WRONLY, MPI_INFO_NULL, &myfile_parallel); + + MPI_Offset current_stream_position; + MPI_Barrier(world); + MPI_File_sync(myfile_parallel); + MPI_File_seek_shared(myfile_parallel, 0, MPI_SEEK_END); + MPI_File_sync(myfile_parallel); + MPI_File_get_position_shared(myfile_parallel, ¤t_stream_position); + + // debug check + // std::cout << "offset on rank " << myrank << " is " << file_stream_offset + header_stream_offset + current_buffer_position << std::endl; + // std::cout << "get position on rank " << myrank << " is " << current_stream_position << std::endl; + + // expand print buffer if needed + int buffer_size_per_element_line = 11 * max_nodes_per_element + 1; // 25 width per number plus 6 spaces plus line terminator + int nlocal_elements = sorted_element_map->getLocalNumElements(); + GO first_element_global_id = sorted_element_map->getGlobalElement(0); + if (buffer_size_per_element_line * nlocal_elements > print_buffer.size()) + { + print_buffer = CArrayKokkos(buffer_size_per_element_line * nlocal_elements); + } + file_stream_offset = buffer_size_per_element_line * first_element_global_id + current_stream_position; + + current_buffer_position = 0; + for (int elementline = 0; elementline < nlocal_elements; elementline++) + { + current_line_stream.str(""); + // convert node ordering + for (int ii = 0; ii < max_nodes_per_element; ii++) + { + if (active_node_ordering_convention == IJK) + { + temp_convert = convert_ijk_to_ensight(ii); + } + else + { + temp_convert = ii; } - } + current_line_stream << std::setw(10) << sorted_nodes_in_elem(elementline, temp_convert) + 1 << " "; } - */ - current_line_stream << std::endl; - - current_line = current_line_stream.str(); + current_line_stream << std::endl; + current_line = current_line_stream.str(); - //copy current line over to C style string buffer (wrapped by matar) - strcpy(&print_buffer(current_buffer_position),current_line.c_str()); - - current_buffer_position += current_line.length(); - } - - //print buffers at offsets with collective MPI write - //MPI_Offset current_stream_position = MPI_File_get_position(myfile_parallel,0); - MPI_Barrier(world); - MPI_File_write_at_all(myfile_parallel, file_stream_offset + header_stream_offset, print_buffer.get_kokkos_view().data(), buffer_size_per_node_line*nlocal_sorted_nodes, MPI_CHAR, MPI_STATUS_IGNORE); - //MPI_File_close(&myfile_parallel); - - //write element connectivity; reopen to reset offset baseline. - //err = MPI_File_open(MPI_COMM_WORLD, current_file_name.c_str(), MPI_MODE_APPEND|MPI_MODE_WRONLY, MPI_INFO_NULL, &myfile_parallel); - - MPI_Offset current_stream_position; - MPI_Barrier(world); - MPI_File_sync(myfile_parallel); - MPI_File_seek_shared(myfile_parallel, 0, MPI_SEEK_END); - MPI_File_sync(myfile_parallel); - MPI_File_get_position_shared(myfile_parallel, ¤t_stream_position); - - //debug check - //std::cout << "offset on rank " << myrank << " is " << file_stream_offset + header_stream_offset + current_buffer_position << std::endl; - //std::cout << "get position on rank " << myrank << " is " << current_stream_position << std::endl; - - //expand print buffer if needed - int buffer_size_per_element_line = 11*max_nodes_per_element + 1; //25 width per number plus 6 spaces plus line terminator - int nlocal_elements = sorted_element_map->getLocalNumElements(); - GO first_element_global_id = sorted_element_map->getGlobalElement(0); - if(buffer_size_per_element_line*nlocal_elements > print_buffer.size()) - print_buffer = CArrayKokkos(buffer_size_per_element_line*nlocal_elements); - file_stream_offset = buffer_size_per_element_line*first_element_global_id + current_stream_position; - - current_buffer_position = 0; - for (int elementline = 0; elementline < nlocal_elements; elementline++) { - current_line_stream.str(""); - //convert node ordering - for (int ii = 0; ii < max_nodes_per_element; ii++) { - if(active_node_ordering_convention == IJK) - temp_convert = convert_ijk_to_ensight(ii); - else - temp_convert = ii; - current_line_stream << std::setw(10) << sorted_nodes_in_elem(elementline, temp_convert) + 1 << " "; - } - current_line_stream << std::endl; - current_line = current_line_stream.str(); + // copy current line over to C style string buffer (wrapped by matar) + strcpy(&print_buffer(current_buffer_position), current_line.c_str()); - //copy current line over to C style string buffer (wrapped by matar) - strcpy(&print_buffer(current_buffer_position),current_line.c_str()); + current_buffer_position += current_line.length(); + } - current_buffer_position += current_line.length(); - } - - MPI_Barrier(world); - MPI_File_write_at_all(myfile_parallel, file_stream_offset, print_buffer.get_kokkos_view().data(), buffer_size_per_element_line*nlocal_elements, MPI_CHAR, MPI_STATUS_IGNORE); - - MPI_File_close(&myfile_parallel); + MPI_Barrier(world); + MPI_File_write_at_all(myfile_parallel, file_stream_offset, print_buffer.get_kokkos_view().data(), buffer_size_per_element_line * nlocal_elements, MPI_CHAR, MPI_STATUS_IGNORE); + MPI_File_close(&myfile_parallel); } /* ---------------------------------------------------------------------- Output Model Information in tecplot format ------------------------------------------------------------------------- */ -void Explicit_Solver::parallel_vtk_writer(){ - int num_dim = simparam.num_dims; - std::string current_file_name; - std::string base_file_name= "VTK"; - std::string base_file_name_undeformed= "VTK_undeformed"; - std::stringstream current_line_stream; - std::string current_line; - std::string file_extension= ".vtk"; - std::string file_count; - std::stringstream count_temp; - int time_step = 0; - int temp_convert; - int noutput, nvector; - bool displace_geometry = false; - MPI_Offset current_stream_position, header_stream_offset, file_stream_offset; - MPI_File myfile_parallel; - int buffer_size_per_element_line, nlocal_elements; - GO first_element_global_id; - int default_vector_count; - int buffer_size_per_node_line; - int nlocal_sorted_nodes; - GO first_node_global_id; - // Convert ijk index system to the finite element numbering convention - // for vertices in cell - - //comm to vectors with contigously sorted global indices from unsorted zoltan2 repartition map - sort_information(); - //set host views of the communicated data to print out from - const_host_vec_array sorted_node_coords = sorted_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_host_vec_array sorted_node_velocities = sorted_node_velocities_distributed->getLocalView (Tpetra::Access::ReadOnly); - //const_host_vec_array sorted_node_densities = sorted_node_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_host_elem_conn_array sorted_nodes_in_elem = sorted_nodes_in_elem_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_host_vec_array sorted_element_densities = sorted_element_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_host_vec_array sorted_node_densities; - if(simparam.topology_optimization_on){ - sorted_node_densities = sorted_node_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); - } - - CArrayKokkos convert_ijk_to_ensight(max_nodes_per_element); - CArrayKokkos tmp_ijk_indx(max_nodes_per_element); - convert_ijk_to_ensight(0) = 0; - convert_ijk_to_ensight(1) = 1; - convert_ijk_to_ensight(2) = 3; - convert_ijk_to_ensight(3) = 2; - if(num_dim==3){ - convert_ijk_to_ensight(4) = 4; - convert_ijk_to_ensight(5) = 5; - convert_ijk_to_ensight(6) = 7; - convert_ijk_to_ensight(7) = 6; - } - - header_stream_offset = 0; - //initial undeformed geometry - count_temp.str(""); - count_temp << file_index; - file_index++; - file_count = count_temp.str(); - if(displace_geometry&&displacement_module>=0) - current_file_name = base_file_name_undeformed + file_count + file_extension; - else - current_file_name = base_file_name + file_count + file_extension; - MPI_File_open(MPI_COMM_WORLD, current_file_name.c_str(), - MPI_MODE_CREATE|MPI_MODE_WRONLY, +void Explicit_Solver::parallel_vtk_writer() +{ + int num_dim = simparam.num_dims; + std::string current_file_name; + std::string base_file_name = "VTK"; + std::string base_file_name_undeformed = "VTK_undeformed"; + std::stringstream current_line_stream; + std::string current_line; + std::string file_extension = ".vtk"; + std::string file_count; + std::stringstream count_temp; + int time_step = 0; + int temp_convert; + int noutput, nvector; + bool displace_geometry = false; + MPI_Offset current_stream_position, header_stream_offset, file_stream_offset; + MPI_File myfile_parallel; + int buffer_size_per_element_line, nlocal_elements; + GO first_element_global_id; + int default_vector_count; + int buffer_size_per_node_line; + int nlocal_sorted_nodes; + GO first_node_global_id; + // Convert ijk index system to the finite element numbering convention + // for vertices in cell + + // comm to vectors with contigously sorted global indices from unsorted zoltan2 repartition map + sort_information(); + // set host views of the communicated data to print out from + const_host_vec_array sorted_node_coords = sorted_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_host_vec_array sorted_node_velocities = sorted_node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); + // const_host_vec_array sorted_node_densities = sorted_node_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); + const_host_elem_conn_array sorted_nodes_in_elem = sorted_nodes_in_elem_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_host_vec_array sorted_element_densities = sorted_element_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_host_vec_array sorted_node_densities; + if (simparam.topology_optimization_on) + { + sorted_node_densities = sorted_node_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); + } + + CArrayKokkos convert_ijk_to_ensight(max_nodes_per_element); + CArrayKokkos tmp_ijk_indx(max_nodes_per_element); + convert_ijk_to_ensight(0) = 0; + convert_ijk_to_ensight(1) = 1; + convert_ijk_to_ensight(2) = 3; + convert_ijk_to_ensight(3) = 2; + if (num_dim == 3) + { + convert_ijk_to_ensight(4) = 4; + convert_ijk_to_ensight(5) = 5; + convert_ijk_to_ensight(6) = 7; + convert_ijk_to_ensight(7) = 6; + } + + header_stream_offset = 0; + // initial undeformed geometry + count_temp.str(""); + count_temp << file_index; + file_index++; + file_count = count_temp.str(); + if (displace_geometry && displacement_module >= 0) + { + current_file_name = base_file_name_undeformed + file_count + file_extension; + } + else + { + current_file_name = base_file_name + file_count + file_extension; + } + MPI_File_open(MPI_COMM_WORLD, current_file_name.c_str(), + MPI_MODE_CREATE | MPI_MODE_WRONLY, MPI_INFO_NULL, &myfile_parallel); - - int err = MPI_File_open(MPI_COMM_WORLD, current_file_name.c_str(), MPI_MODE_CREATE|MPI_MODE_EXCL|MPI_MODE_WRONLY, MPI_INFO_NULL, &myfile_parallel); - //allows overwriting the file if it already existed in the directory - if (err != MPI_SUCCESS) { - if (myrank == 0){ - MPI_File_delete(current_file_name.c_str(),MPI_INFO_NULL); - } - MPI_File_open(MPI_COMM_WORLD, current_file_name.c_str(), MPI_MODE_CREATE|MPI_MODE_EXCL|MPI_MODE_WRONLY, MPI_INFO_NULL, &myfile_parallel); - } - - //output header of the tecplot file - - //std::cout << current_file_name << std::endl; - current_line_stream << "# vtk DataFile Version 2.0\n"; - current_line = current_line_stream.str(); - if(myrank == 0) - MPI_File_write(myfile_parallel,current_line.c_str(),current_line.length(), MPI_CHAR, MPI_STATUS_IGNORE); - header_stream_offset += current_line.length(); - //myfile << "VARIABLES = \"x\", \"y\", \"z\", \"density\", \"sigmaxx\", \"sigmayy\", \"sigmazz\", \"sigmaxy\", \"sigmaxz\", \"sigmayz\"" "\n"; - //else - current_line_stream.str(""); - current_line_stream << "Mesh for Fierro\n"; - current_line = current_line_stream.str(); - if(myrank == 0) - MPI_File_write(myfile_parallel,current_line.c_str(),current_line.length(), MPI_CHAR, MPI_STATUS_IGNORE); - header_stream_offset += current_line.length(); + + int err = MPI_File_open(MPI_COMM_WORLD, current_file_name.c_str(), MPI_MODE_CREATE | MPI_MODE_EXCL | MPI_MODE_WRONLY, MPI_INFO_NULL, &myfile_parallel); + // allows overwriting the file if it already existed in the directory + if (err != MPI_SUCCESS) + { + if (myrank == 0) + { + MPI_File_delete(current_file_name.c_str(), MPI_INFO_NULL); + } + MPI_File_open(MPI_COMM_WORLD, current_file_name.c_str(), MPI_MODE_CREATE | MPI_MODE_EXCL | MPI_MODE_WRONLY, MPI_INFO_NULL, &myfile_parallel); + } + + // output header of the tecplot file + + // std::cout << current_file_name << std::endl; + current_line_stream << "# vtk DataFile Version 2.0\n"; + current_line = current_line_stream.str(); + if (myrank == 0) + { + MPI_File_write(myfile_parallel, current_line.c_str(), current_line.length(), MPI_CHAR, MPI_STATUS_IGNORE); + } + header_stream_offset += current_line.length(); + // myfile << "VARIABLES = \"x\", \"y\", \"z\", \"density\", \"sigmaxx\", \"sigmayy\", \"sigmazz\", \"sigmaxy\", \"sigmaxz\", \"sigmayz\"" "\n"; + // else + current_line_stream.str(""); + current_line_stream << "Mesh for Fierro\n"; + current_line = current_line_stream.str(); + if (myrank == 0) + { + MPI_File_write(myfile_parallel, current_line.c_str(), current_line.length(), MPI_CHAR, MPI_STATUS_IGNORE); + } + header_stream_offset += current_line.length(); /* for (int imodule = 0; imodule < nfea_modules; imodule++){ for(int ioutput = 0; ioutput < fea_modules[imodule]->noutput; ioutput++){ @@ -2166,48 +2411,57 @@ void Explicit_Solver::parallel_vtk_writer(){ } */ - current_line_stream.str(""); - current_line_stream << "ASCII\n"; - current_line = current_line_stream.str(); - if(myrank == 0) - MPI_File_write(myfile_parallel,current_line.c_str(),current_line.length(), MPI_CHAR, MPI_STATUS_IGNORE); - header_stream_offset += current_line.length(); - - current_line_stream.str(""); - current_line_stream << "DATASET UNSTRUCTURED_GRID\n\n"; - current_line = current_line_stream.str(); - if(myrank == 0) - MPI_File_write(myfile_parallel,current_line.c_str(),current_line.length(), MPI_CHAR, MPI_STATUS_IGNORE); - header_stream_offset += current_line.length(); - - current_line_stream.str(""); - current_line_stream << "POINTS " << num_nodes - << " float\n"; - current_line = current_line_stream.str(); - if(myrank == 0) - MPI_File_write(myfile_parallel,current_line.c_str(),current_line.length(), MPI_CHAR, MPI_STATUS_IGNORE); - header_stream_offset += current_line.length(); - - //output nodal data - //compute buffer output size and file stream offset for this MPI rank - default_vector_count = 1; - buffer_size_per_node_line = 26*default_vector_count*num_dim + 1; //25 width per number + 1 space times 6 entries plus line terminator - nlocal_sorted_nodes = sorted_map->getLocalNumElements(); - first_node_global_id = sorted_map->getGlobalElement(0); - CArrayKokkos print_buffer(buffer_size_per_node_line*nlocal_sorted_nodes); - file_stream_offset = buffer_size_per_node_line*first_node_global_id; - - //populate buffer - long long current_buffer_position = 0; - current_line_stream << std::fixed << std::setprecision(8); - for (int nodeline = 0; nodeline < nlocal_sorted_nodes; nodeline++) { current_line_stream.str(""); - current_line_stream << std::left << std::setw(25) << sorted_node_coords(nodeline,0) << " "; - current_line_stream << std::left << std::setw(25) << sorted_node_coords(nodeline,1) << " "; - if(num_dim==3) - current_line_stream << std::left << std::setw(25) << sorted_node_coords(nodeline,2) << " "; - - //myfile << std::setw(25) << collected_node_densities(nodeline,0) << " "; + current_line_stream << "ASCII\n"; + current_line = current_line_stream.str(); + if (myrank == 0) + { + MPI_File_write(myfile_parallel, current_line.c_str(), current_line.length(), MPI_CHAR, MPI_STATUS_IGNORE); + } + header_stream_offset += current_line.length(); + + current_line_stream.str(""); + current_line_stream << "DATASET UNSTRUCTURED_GRID\n\n"; + current_line = current_line_stream.str(); + if (myrank == 0) + { + MPI_File_write(myfile_parallel, current_line.c_str(), current_line.length(), MPI_CHAR, MPI_STATUS_IGNORE); + } + header_stream_offset += current_line.length(); + + current_line_stream.str(""); + current_line_stream << "POINTS " << num_nodes + << " float\n"; + current_line = current_line_stream.str(); + if (myrank == 0) + { + MPI_File_write(myfile_parallel, current_line.c_str(), current_line.length(), MPI_CHAR, MPI_STATUS_IGNORE); + } + header_stream_offset += current_line.length(); + + // output nodal data + // compute buffer output size and file stream offset for this MPI rank + default_vector_count = 1; + buffer_size_per_node_line = 26 * default_vector_count * num_dim + 1; // 25 width per number + 1 space times 6 entries plus line terminator + nlocal_sorted_nodes = sorted_map->getLocalNumElements(); + first_node_global_id = sorted_map->getGlobalElement(0); + CArrayKokkos print_buffer(buffer_size_per_node_line * nlocal_sorted_nodes); + file_stream_offset = buffer_size_per_node_line * first_node_global_id; + + // populate buffer + long long current_buffer_position = 0; + current_line_stream << std::fixed << std::setprecision(8); + for (int nodeline = 0; nodeline < nlocal_sorted_nodes; nodeline++) + { + current_line_stream.str(""); + current_line_stream << std::left << std::setw(25) << sorted_node_coords(nodeline, 0) << " "; + current_line_stream << std::left << std::setw(25) << sorted_node_coords(nodeline, 1) << " "; + if (num_dim == 3) + { + current_line_stream << std::left << std::setw(25) << sorted_node_coords(nodeline, 2) << " "; + } + + // myfile << std::setw(25) << collected_node_densities(nodeline,0) << " "; /* for (int imodule = 0; imodule < nfea_modules; imodule++){ noutput = fea_modules[imodule]->noutput; @@ -2228,496 +2482,570 @@ void Explicit_Solver::parallel_vtk_writer(){ } } */ - current_line_stream << std::endl; - - current_line = current_line_stream.str(); + current_line_stream << std::endl; + + current_line = current_line_stream.str(); + + // copy current line over to C style string buffer (wrapped by matar) + strcpy(&print_buffer(current_buffer_position), current_line.c_str()); + + current_buffer_position += current_line.length(); + } + + // print buffers at offsets with collective MPI write + // MPI_Offset current_stream_position = MPI_File_get_position(myfile_parallel,0); + MPI_Barrier(world); + MPI_File_write_at_all(myfile_parallel, file_stream_offset + header_stream_offset, print_buffer.get_kokkos_view().data(), buffer_size_per_node_line * nlocal_sorted_nodes, MPI_CHAR, + MPI_STATUS_IGNORE); + // MPI_File_close(&myfile_parallel); + + // write element connectivity; reopen to reset offset baseline. + // err = MPI_File_open(MPI_COMM_WORLD, current_file_name.c_str(), MPI_MODE_APPEND|MPI_MODE_WRONLY, MPI_INFO_NULL, &myfile_parallel); + + header_stream_offset = 0; + MPI_Barrier(world); + MPI_File_sync(myfile_parallel); + MPI_File_seek_shared(myfile_parallel, 0, MPI_SEEK_END); + MPI_File_sync(myfile_parallel); + MPI_File_get_position_shared(myfile_parallel, ¤t_stream_position); + + // debug check + // std::cout << "offset on rank " << myrank << " is " << file_stream_offset + header_stream_offset + current_buffer_position << std::endl; + // std::cout << "get position on rank " << myrank << " is " << current_stream_position << std::endl; - //copy current line over to C style string buffer (wrapped by matar) - strcpy(&print_buffer(current_buffer_position),current_line.c_str()); - - current_buffer_position += current_line.length(); - } - - //print buffers at offsets with collective MPI write - //MPI_Offset current_stream_position = MPI_File_get_position(myfile_parallel,0); - MPI_Barrier(world); - MPI_File_write_at_all(myfile_parallel, file_stream_offset + header_stream_offset, print_buffer.get_kokkos_view().data(), buffer_size_per_node_line*nlocal_sorted_nodes, MPI_CHAR, MPI_STATUS_IGNORE); - //MPI_File_close(&myfile_parallel); - - //write element connectivity; reopen to reset offset baseline. - //err = MPI_File_open(MPI_COMM_WORLD, current_file_name.c_str(), MPI_MODE_APPEND|MPI_MODE_WRONLY, MPI_INFO_NULL, &myfile_parallel); - - header_stream_offset = 0; - MPI_Barrier(world); - MPI_File_sync(myfile_parallel); - MPI_File_seek_shared(myfile_parallel, 0, MPI_SEEK_END); - MPI_File_sync(myfile_parallel); - MPI_File_get_position_shared(myfile_parallel, ¤t_stream_position); - - //debug check - //std::cout << "offset on rank " << myrank << " is " << file_stream_offset + header_stream_offset + current_buffer_position << std::endl; - //std::cout << "get position on rank " << myrank << " is " << current_stream_position << std::endl; - - current_line_stream.str(""); - current_line_stream << std::endl << "CELLS " << num_elem << " " << num_elem*(max_nodes_per_element+1) << std::endl; - current_line = current_line_stream.str(); - //std::cout << current_line; - file_stream_offset = current_stream_position; - if(myrank == 0) - MPI_File_write_at(myfile_parallel, file_stream_offset, current_line.c_str(),current_line.length(), MPI_CHAR, MPI_STATUS_IGNORE); - header_stream_offset += current_line.length(); - - //expand print buffer if needed - buffer_size_per_element_line = 11*(max_nodes_per_element+1) + 1; //25 width per number plus 6 spaces plus line terminator - nlocal_elements = sorted_element_map->getLocalNumElements(); - first_element_global_id = sorted_element_map->getGlobalElement(0); - if(buffer_size_per_element_line*nlocal_elements > print_buffer.size()) - print_buffer = CArrayKokkos(buffer_size_per_element_line*nlocal_elements); - file_stream_offset = buffer_size_per_element_line*first_element_global_id + current_stream_position + header_stream_offset; - - current_buffer_position = 0; - for (int elementline = 0; elementline < nlocal_elements; elementline++) { current_line_stream.str(""); - //convert node ordering - current_line_stream << std::left << std::setw(10) << max_nodes_per_element << " "; - for (int ii = 0; ii < max_nodes_per_element; ii++) { - if(active_node_ordering_convention == IJK) - temp_convert = convert_ijk_to_ensight(ii); - else - temp_convert = ii; - current_line_stream << std::left << std::setw(10) << sorted_nodes_in_elem(elementline, temp_convert)<< " "; - } - current_line_stream << std::endl; + current_line_stream << std::endl << "CELLS " << num_elem << " " << num_elem * (max_nodes_per_element + 1) << std::endl; current_line = current_line_stream.str(); + // std::cout << current_line; + file_stream_offset = current_stream_position; + if (myrank == 0) + { + MPI_File_write_at(myfile_parallel, file_stream_offset, current_line.c_str(), current_line.length(), MPI_CHAR, MPI_STATUS_IGNORE); + } + header_stream_offset += current_line.length(); + + // expand print buffer if needed + buffer_size_per_element_line = 11 * (max_nodes_per_element + 1) + 1; // 25 width per number plus 6 spaces plus line terminator + nlocal_elements = sorted_element_map->getLocalNumElements(); + first_element_global_id = sorted_element_map->getGlobalElement(0); + if (buffer_size_per_element_line * nlocal_elements > print_buffer.size()) + { + print_buffer = CArrayKokkos(buffer_size_per_element_line * nlocal_elements); + } + file_stream_offset = buffer_size_per_element_line * first_element_global_id + current_stream_position + header_stream_offset; + + current_buffer_position = 0; + for (int elementline = 0; elementline < nlocal_elements; elementline++) + { + current_line_stream.str(""); + // convert node ordering + current_line_stream << std::left << std::setw(10) << max_nodes_per_element << " "; + for (int ii = 0; ii < max_nodes_per_element; ii++) + { + if (active_node_ordering_convention == IJK) + { + temp_convert = convert_ijk_to_ensight(ii); + } + else + { + temp_convert = ii; + } + current_line_stream << std::left << std::setw(10) << sorted_nodes_in_elem(elementline, temp_convert) << " "; + } + current_line_stream << std::endl; + current_line = current_line_stream.str(); + + // copy current line over to C style string buffer (wrapped by matar) + strcpy(&print_buffer(current_buffer_position), current_line.c_str()); + + current_buffer_position += current_line.length(); + } + + MPI_Barrier(world); + MPI_File_write_at_all(myfile_parallel, file_stream_offset, print_buffer.get_kokkos_view().data(), buffer_size_per_element_line * nlocal_elements, MPI_CHAR, MPI_STATUS_IGNORE); + + // print Element Types + + header_stream_offset = 0; + MPI_Barrier(world); + MPI_File_sync(myfile_parallel); + MPI_File_seek_shared(myfile_parallel, 0, MPI_SEEK_END); + MPI_File_sync(myfile_parallel); + MPI_File_get_position_shared(myfile_parallel, ¤t_stream_position); + + // debug check + // std::cout << "offset on rank " << myrank << " is " << file_stream_offset + header_stream_offset + current_buffer_position << std::endl; + // std::cout << "get position on rank " << myrank << " is " << current_stream_position << std::endl; - //copy current line over to C style string buffer (wrapped by matar) - strcpy(&print_buffer(current_buffer_position),current_line.c_str()); - - current_buffer_position += current_line.length(); - } - - MPI_Barrier(world); - MPI_File_write_at_all(myfile_parallel, file_stream_offset, print_buffer.get_kokkos_view().data(), buffer_size_per_element_line*nlocal_elements, MPI_CHAR, MPI_STATUS_IGNORE); - - //print Element Types - - header_stream_offset = 0; - MPI_Barrier(world); - MPI_File_sync(myfile_parallel); - MPI_File_seek_shared(myfile_parallel, 0, MPI_SEEK_END); - MPI_File_sync(myfile_parallel); - MPI_File_get_position_shared(myfile_parallel, ¤t_stream_position); - - //debug check - //std::cout << "offset on rank " << myrank << " is " << file_stream_offset + header_stream_offset + current_buffer_position << std::endl; - //std::cout << "get position on rank " << myrank << " is " << current_stream_position << std::endl; - - current_line_stream.str(""); - current_line_stream << std::endl << "CELL_TYPES " << num_elem << std::endl; - current_line = current_line_stream.str(); - //std::cout << current_line; - file_stream_offset = current_stream_position; - if(myrank == 0) - MPI_File_write_at(myfile_parallel, file_stream_offset, current_line.c_str(),current_line.length(), MPI_CHAR, MPI_STATUS_IGNORE); - header_stream_offset += current_line.length(); - - //expand print buffer if needed - buffer_size_per_element_line = 11; //10 width per number plus line terminator - nlocal_elements = sorted_element_map->getLocalNumElements(); - first_element_global_id = sorted_element_map->getGlobalElement(0); - if(buffer_size_per_element_line*nlocal_elements > print_buffer.size()) - print_buffer = CArrayKokkos(buffer_size_per_element_line*nlocal_elements); - file_stream_offset = buffer_size_per_element_line*first_element_global_id + current_stream_position + header_stream_offset; - - current_buffer_position = 0; - for (int elementline = 0; elementline < nlocal_elements; elementline++) { current_line_stream.str(""); - //convert node ordering - current_line_stream << std::left << std::setw(10) << 12 << std::endl; + current_line_stream << std::endl << "CELL_TYPES " << num_elem << std::endl; current_line = current_line_stream.str(); + // std::cout << current_line; + file_stream_offset = current_stream_position; + if (myrank == 0) + { + MPI_File_write_at(myfile_parallel, file_stream_offset, current_line.c_str(), current_line.length(), MPI_CHAR, MPI_STATUS_IGNORE); + } + header_stream_offset += current_line.length(); + + // expand print buffer if needed + buffer_size_per_element_line = 11; // 10 width per number plus line terminator + nlocal_elements = sorted_element_map->getLocalNumElements(); + first_element_global_id = sorted_element_map->getGlobalElement(0); + if (buffer_size_per_element_line * nlocal_elements > print_buffer.size()) + { + print_buffer = CArrayKokkos(buffer_size_per_element_line * nlocal_elements); + } + file_stream_offset = buffer_size_per_element_line * first_element_global_id + current_stream_position + header_stream_offset; + + current_buffer_position = 0; + for (int elementline = 0; elementline < nlocal_elements; elementline++) + { + current_line_stream.str(""); + // convert node ordering + current_line_stream << std::left << std::setw(10) << 12 << std::endl; + current_line = current_line_stream.str(); + + // copy current line over to C style string buffer (wrapped by matar) + strcpy(&print_buffer(current_buffer_position), current_line.c_str()); + + current_buffer_position += current_line.length(); + } + + MPI_Barrier(world); + MPI_File_write_at_all(myfile_parallel, file_stream_offset, print_buffer.get_kokkos_view().data(), buffer_size_per_element_line * nlocal_elements, MPI_CHAR, MPI_STATUS_IGNORE); + + // Print Node scalars + + header_stream_offset = 0; + MPI_Barrier(world); + MPI_File_sync(myfile_parallel); + MPI_File_seek_shared(myfile_parallel, 0, MPI_SEEK_END); + MPI_File_sync(myfile_parallel); + MPI_File_get_position_shared(myfile_parallel, ¤t_stream_position); + + // debug check + // std::cout << "offset on rank " << myrank << " is " << file_stream_offset + header_stream_offset + current_buffer_position << std::endl; + // std::cout << "get position on rank " << myrank << " is " << current_stream_position << std::endl; - //copy current line over to C style string buffer (wrapped by matar) - strcpy(&print_buffer(current_buffer_position),current_line.c_str()); - - current_buffer_position += current_line.length(); - } - - MPI_Barrier(world); - MPI_File_write_at_all(myfile_parallel, file_stream_offset, print_buffer.get_kokkos_view().data(), buffer_size_per_element_line*nlocal_elements, MPI_CHAR, MPI_STATUS_IGNORE); - - //Print Node scalars - - header_stream_offset = 0; - MPI_Barrier(world); - MPI_File_sync(myfile_parallel); - MPI_File_seek_shared(myfile_parallel, 0, MPI_SEEK_END); - MPI_File_sync(myfile_parallel); - MPI_File_get_position_shared(myfile_parallel, ¤t_stream_position); - - //debug check - //std::cout << "offset on rank " << myrank << " is " << file_stream_offset + header_stream_offset + current_buffer_position << std::endl; - //std::cout << "get position on rank " << myrank << " is " << current_stream_position << std::endl; - - current_line_stream.str(""); - current_line_stream << std::endl << "POINT_DATA " << num_nodes << std::endl; - current_line_stream << "SCALARS design_density float 1" << std::endl; - current_line_stream << "LOOKUP_TABLE default" << std::endl; - current_line = current_line_stream.str(); - //std::cout << current_line; - file_stream_offset = current_stream_position; - if(myrank == 0) - MPI_File_write_at(myfile_parallel, file_stream_offset, current_line.c_str(),current_line.length(), MPI_CHAR, MPI_STATUS_IGNORE); - header_stream_offset += current_line.length(); - - //expand print buffer if needed - default_vector_count = 1; - buffer_size_per_node_line = 26; //25 width per number + 1 space times 6 entries plus line terminator - nlocal_sorted_nodes = sorted_map->getLocalNumElements(); - first_node_global_id = sorted_map->getGlobalElement(0); - if(buffer_size_per_node_line*nlocal_sorted_nodes > print_buffer.size()) - print_buffer = CArrayKokkos(buffer_size_per_node_line*nlocal_sorted_nodes); - file_stream_offset = buffer_size_per_node_line*first_node_global_id + current_stream_position + header_stream_offset; - - current_buffer_position = 0; - current_line_stream << std::fixed << std::setprecision(8); - for (int nodeline = 0; nodeline < nlocal_sorted_nodes; nodeline++) { current_line_stream.str(""); - //convert node ordering - if(simparam.topology_optimization_on) - current_line_stream << std::left << std::setw(25) << sorted_node_densities(nodeline,0) << std::endl; - else - current_line_stream << std::left << std::setw(25) << 1 << std::endl; + current_line_stream << std::endl << "POINT_DATA " << num_nodes << std::endl; + current_line_stream << "SCALARS design_density float 1" << std::endl; + current_line_stream << "LOOKUP_TABLE default" << std::endl; current_line = current_line_stream.str(); + // std::cout << current_line; + file_stream_offset = current_stream_position; + if (myrank == 0) + { + MPI_File_write_at(myfile_parallel, file_stream_offset, current_line.c_str(), current_line.length(), MPI_CHAR, MPI_STATUS_IGNORE); + } + header_stream_offset += current_line.length(); + + // expand print buffer if needed + default_vector_count = 1; + buffer_size_per_node_line = 26; // 25 width per number + 1 space times 6 entries plus line terminator + nlocal_sorted_nodes = sorted_map->getLocalNumElements(); + first_node_global_id = sorted_map->getGlobalElement(0); + if (buffer_size_per_node_line * nlocal_sorted_nodes > print_buffer.size()) + { + print_buffer = CArrayKokkos(buffer_size_per_node_line * nlocal_sorted_nodes); + } + file_stream_offset = buffer_size_per_node_line * first_node_global_id + current_stream_position + header_stream_offset; + + current_buffer_position = 0; + current_line_stream << std::fixed << std::setprecision(8); + for (int nodeline = 0; nodeline < nlocal_sorted_nodes; nodeline++) + { + current_line_stream.str(""); + // convert node ordering + if (simparam.topology_optimization_on) + { + current_line_stream << std::left << std::setw(25) << sorted_node_densities(nodeline, 0) << std::endl; + } + else + { + current_line_stream << std::left << std::setw(25) << 1 << std::endl; + } + current_line = current_line_stream.str(); + + // copy current line over to C style string buffer (wrapped by matar) + strcpy(&print_buffer(current_buffer_position), current_line.c_str()); + + current_buffer_position += current_line.length(); + } + + MPI_Barrier(world); + MPI_File_write_at_all(myfile_parallel, file_stream_offset, print_buffer.get_kokkos_view().data(), buffer_size_per_node_line * nlocal_sorted_nodes, MPI_CHAR, MPI_STATUS_IGNORE); + + // Print Node vectors + + header_stream_offset = 0; + MPI_Barrier(world); + MPI_File_sync(myfile_parallel); + MPI_File_seek_shared(myfile_parallel, 0, MPI_SEEK_END); + MPI_File_sync(myfile_parallel); + MPI_File_get_position_shared(myfile_parallel, ¤t_stream_position); + + // debug check + // std::cout << "offset on rank " << myrank << " is " << file_stream_offset + header_stream_offset + current_buffer_position << std::endl; + // std::cout << "get position on rank " << myrank << " is " << current_stream_position << std::endl; - //copy current line over to C style string buffer (wrapped by matar) - strcpy(&print_buffer(current_buffer_position),current_line.c_str()); - - current_buffer_position += current_line.length(); - } - - MPI_Barrier(world); - MPI_File_write_at_all(myfile_parallel, file_stream_offset, print_buffer.get_kokkos_view().data(), buffer_size_per_node_line*nlocal_sorted_nodes, MPI_CHAR, MPI_STATUS_IGNORE); - - //Print Node vectors - - header_stream_offset = 0; - MPI_Barrier(world); - MPI_File_sync(myfile_parallel); - MPI_File_seek_shared(myfile_parallel, 0, MPI_SEEK_END); - MPI_File_sync(myfile_parallel); - MPI_File_get_position_shared(myfile_parallel, ¤t_stream_position); - - //debug check - //std::cout << "offset on rank " << myrank << " is " << file_stream_offset + header_stream_offset + current_buffer_position << std::endl; - //std::cout << "get position on rank " << myrank << " is " << current_stream_position << std::endl; - - current_line_stream.str(""); - current_line_stream << std::endl << "VECTORS velocity float" << std::endl; - current_line = current_line_stream.str(); - //std::cout << current_line; - file_stream_offset = current_stream_position; - if(myrank == 0) - MPI_File_write_at(myfile_parallel, file_stream_offset, current_line.c_str(),current_line.length(), MPI_CHAR, MPI_STATUS_IGNORE); - header_stream_offset += current_line.length(); - - //expand print buffer if needed - default_vector_count = 1; - buffer_size_per_node_line = num_dim*26 + 1; //25 width per number + 1 space times 6 entries plus line terminator - nlocal_sorted_nodes = sorted_map->getLocalNumElements(); - first_node_global_id = sorted_map->getGlobalElement(0); - if(buffer_size_per_node_line*nlocal_sorted_nodes > print_buffer.size()) - print_buffer = CArrayKokkos(buffer_size_per_node_line*nlocal_sorted_nodes); - file_stream_offset = buffer_size_per_node_line*first_node_global_id + current_stream_position + header_stream_offset; - - current_buffer_position = 0; - current_line_stream << std::fixed << std::setprecision(8); - for (int nodeline = 0; nodeline < nlocal_sorted_nodes; nodeline++) { current_line_stream.str(""); - //convert node ordering - //velocity print - current_line_stream << std::left << std::setw(25) << sorted_node_velocities(nodeline,0) << " "; - current_line_stream << std::left << std::setw(25) << sorted_node_velocities(nodeline,1) << " "; - if(num_dim==3) - current_line_stream << std::left << std::setw(25) << sorted_node_velocities(nodeline,2) << " "; - - current_line_stream << std::endl; + current_line_stream << std::endl << "VECTORS velocity float" << std::endl; current_line = current_line_stream.str(); + // std::cout << current_line; + file_stream_offset = current_stream_position; + if (myrank == 0) + { + MPI_File_write_at(myfile_parallel, file_stream_offset, current_line.c_str(), current_line.length(), MPI_CHAR, MPI_STATUS_IGNORE); + } + header_stream_offset += current_line.length(); + + // expand print buffer if needed + default_vector_count = 1; + buffer_size_per_node_line = num_dim * 26 + 1; // 25 width per number + 1 space times 6 entries plus line terminator + nlocal_sorted_nodes = sorted_map->getLocalNumElements(); + first_node_global_id = sorted_map->getGlobalElement(0); + if (buffer_size_per_node_line * nlocal_sorted_nodes > print_buffer.size()) + { + print_buffer = CArrayKokkos(buffer_size_per_node_line * nlocal_sorted_nodes); + } + file_stream_offset = buffer_size_per_node_line * first_node_global_id + current_stream_position + header_stream_offset; + + current_buffer_position = 0; + current_line_stream << std::fixed << std::setprecision(8); + for (int nodeline = 0; nodeline < nlocal_sorted_nodes; nodeline++) + { + current_line_stream.str(""); + // convert node ordering + // velocity print + current_line_stream << std::left << std::setw(25) << sorted_node_velocities(nodeline, 0) << " "; + current_line_stream << std::left << std::setw(25) << sorted_node_velocities(nodeline, 1) << " "; + if (num_dim == 3) + { + current_line_stream << std::left << std::setw(25) << sorted_node_velocities(nodeline, 2) << " "; + } + + current_line_stream << std::endl; + current_line = current_line_stream.str(); + + // copy current line over to C style string buffer (wrapped by matar) + strcpy(&print_buffer(current_buffer_position), current_line.c_str()); + + current_buffer_position += current_line.length(); + } + + MPI_Barrier(world); + MPI_File_write_at_all(myfile_parallel, file_stream_offset, print_buffer.get_kokkos_view().data(), buffer_size_per_node_line * nlocal_sorted_nodes, MPI_CHAR, MPI_STATUS_IGNORE); + + // print Element Fields + + header_stream_offset = 0; + MPI_Barrier(world); + MPI_File_sync(myfile_parallel); + MPI_File_seek_shared(myfile_parallel, 0, MPI_SEEK_END); + MPI_File_sync(myfile_parallel); + MPI_File_get_position_shared(myfile_parallel, ¤t_stream_position); + + // debug check + // std::cout << "offset on rank " << myrank << " is " << file_stream_offset + header_stream_offset + current_buffer_position << std::endl; + // std::cout << "get position on rank " << myrank << " is " << current_stream_position << std::endl; - //copy current line over to C style string buffer (wrapped by matar) - strcpy(&print_buffer(current_buffer_position),current_line.c_str()); - - current_buffer_position += current_line.length(); - } - - MPI_Barrier(world); - MPI_File_write_at_all(myfile_parallel, file_stream_offset, print_buffer.get_kokkos_view().data(), buffer_size_per_node_line*nlocal_sorted_nodes, MPI_CHAR, MPI_STATUS_IGNORE); - - //print Element Fields - - header_stream_offset = 0; - MPI_Barrier(world); - MPI_File_sync(myfile_parallel); - MPI_File_seek_shared(myfile_parallel, 0, MPI_SEEK_END); - MPI_File_sync(myfile_parallel); - MPI_File_get_position_shared(myfile_parallel, ¤t_stream_position); - - //debug check - //std::cout << "offset on rank " << myrank << " is " << file_stream_offset + header_stream_offset + current_buffer_position << std::endl; - //std::cout << "get position on rank " << myrank << " is " << current_stream_position << std::endl; - - current_line_stream.str(""); - current_line_stream << std::endl << "CELL_DATA " << num_elem << std::endl; - current_line_stream << "SCALARS element_density float 1" << std::endl; - current_line_stream << "LOOKUP_TABLE default" << std::endl; - current_line = current_line_stream.str(); - //std::cout << current_line; - file_stream_offset = current_stream_position; - if(myrank == 0) - MPI_File_write_at(myfile_parallel, file_stream_offset, current_line.c_str(),current_line.length(), MPI_CHAR, MPI_STATUS_IGNORE); - header_stream_offset += current_line.length(); - - //expand print buffer if needed - buffer_size_per_element_line = 26; //25 width per number plus line terminator - nlocal_elements = sorted_element_map->getLocalNumElements(); - first_element_global_id = sorted_element_map->getGlobalElement(0); - if(buffer_size_per_element_line*nlocal_elements > print_buffer.size()) - print_buffer = CArrayKokkos(buffer_size_per_element_line*nlocal_elements); - file_stream_offset = buffer_size_per_element_line*first_element_global_id + current_stream_position + header_stream_offset; - - current_buffer_position = 0; - current_line_stream << std::fixed << std::setprecision(8); - for (int elementline = 0; elementline < nlocal_elements; elementline++) { current_line_stream.str(""); - //convert node ordering - current_line_stream << std::left << std::setw(25) << sorted_element_densities(elementline,0) << std::endl; + current_line_stream << std::endl << "CELL_DATA " << num_elem << std::endl; + current_line_stream << "SCALARS element_density float 1" << std::endl; + current_line_stream << "LOOKUP_TABLE default" << std::endl; current_line = current_line_stream.str(); + // std::cout << current_line; + file_stream_offset = current_stream_position; + if (myrank == 0) + { + MPI_File_write_at(myfile_parallel, file_stream_offset, current_line.c_str(), current_line.length(), MPI_CHAR, MPI_STATUS_IGNORE); + } + header_stream_offset += current_line.length(); + + // expand print buffer if needed + buffer_size_per_element_line = 26; // 25 width per number plus line terminator + nlocal_elements = sorted_element_map->getLocalNumElements(); + first_element_global_id = sorted_element_map->getGlobalElement(0); + if (buffer_size_per_element_line * nlocal_elements > print_buffer.size()) + { + print_buffer = CArrayKokkos(buffer_size_per_element_line * nlocal_elements); + } + file_stream_offset = buffer_size_per_element_line * first_element_global_id + current_stream_position + header_stream_offset; + + current_buffer_position = 0; + current_line_stream << std::fixed << std::setprecision(8); + for (int elementline = 0; elementline < nlocal_elements; elementline++) + { + current_line_stream.str(""); + // convert node ordering + current_line_stream << std::left << std::setw(25) << sorted_element_densities(elementline, 0) << std::endl; + current_line = current_line_stream.str(); - //copy current line over to C style string buffer (wrapped by matar) - strcpy(&print_buffer(current_buffer_position),current_line.c_str()); + // copy current line over to C style string buffer (wrapped by matar) + strcpy(&print_buffer(current_buffer_position), current_line.c_str()); + + current_buffer_position += current_line.length(); + } - current_buffer_position += current_line.length(); - } - - MPI_Barrier(world); - MPI_File_write_at_all(myfile_parallel, file_stream_offset, print_buffer.get_kokkos_view().data(), buffer_size_per_element_line*nlocal_elements, MPI_CHAR, MPI_STATUS_IGNORE); - - MPI_Barrier(world); - MPI_File_sync(myfile_parallel); - MPI_File_close(&myfile_parallel); + MPI_Barrier(world); + MPI_File_write_at_all(myfile_parallel, file_stream_offset, print_buffer.get_kokkos_view().data(), buffer_size_per_element_line * nlocal_elements, MPI_CHAR, MPI_STATUS_IGNORE); + MPI_Barrier(world); + MPI_File_sync(myfile_parallel); + MPI_File_close(&myfile_parallel); } /* ---------------------------------------------------------------------- Output Model Information in tecplot format ------------------------------------------------------------------------- */ -void Explicit_Solver::tecplot_writer(){ - - int num_dim = simparam.num_dims; - std::string current_file_name; - std::string base_file_name= "TecplotTO"; - std::string base_file_name_undeformed= "TecplotTO_undeformed"; - std::stringstream ss; - std::string file_extension= ".dat"; - std::string file_count; - std::stringstream count_temp; - int time_step = 0; - int temp_convert; - int noutput, nvector; - bool displace_geometry = false; - const_host_vec_array current_collected_output; - int displacement_index; - collect_information(); - //set host views of the collected data to print out from - const_host_vec_array collected_node_coords = collected_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_host_vec_array collected_node_velocities = collected_node_velocities_distributed->getLocalView (Tpetra::Access::ReadOnly); - //const_host_vec_array collected_node_densities = collected_node_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_host_elem_conn_array collected_nodes_in_elem = collected_nodes_in_elem_distributed->getLocalView (Tpetra::Access::ReadOnly); - - // Convert ijk index system to the finite element numbering convention - // for vertices in cell - CArrayKokkos convert_ijk_to_ensight(max_nodes_per_element); - CArrayKokkos tmp_ijk_indx(max_nodes_per_element); - convert_ijk_to_ensight(0) = 0; - convert_ijk_to_ensight(1) = 1; - convert_ijk_to_ensight(2) = 3; - convert_ijk_to_ensight(3) = 2; - if(num_dim==3){ - convert_ijk_to_ensight(4) = 4; - convert_ijk_to_ensight(5) = 5; - convert_ijk_to_ensight(6) = 7; - convert_ijk_to_ensight(7) = 6; - } - - //compared to primitive unit cell, assumes orthogonal primitive unit cell - if(myrank==0){ - //initial undeformed geometry - count_temp.str(""); - count_temp << file_index; - file_index++; - file_count = count_temp.str(); - if(displace_geometry&&displacement_module>=0) - current_file_name = base_file_name_undeformed + file_count + file_extension; - else - current_file_name = base_file_name + file_count + file_extension; - std::ofstream myfile (current_file_name.c_str()); //output filestream object for file output - //read in position data - myfile << std::fixed << std::setprecision(8); - - //output header of the tecplot file - - myfile << "TITLE=\"results for TO code\"" "\n"; - //myfile << "VARIABLES = \"x\", \"y\", \"z\", \"density\", \"sigmaxx\", \"sigmayy\", \"sigmazz\", \"sigmaxy\", \"sigmaxz\", \"sigmayz\"" "\n"; - //else - myfile << "VARIABLES = \"x\", \"y\", \"z\", \"vx\", \"vy\", \"vz\""; - /* - for (int imodule = 0; imodule < nfea_modules; imodule++){ - for(int ioutput = 0; ioutput < fea_modules[imodule]->noutput; ioutput++){ - nvector = fea_modules[imodule]->output_vector_sizes[ioutput]; - for(int ivector = 0; ivector < nvector; ivector++){ - myfile << ", \"" << fea_modules[imodule]->output_dof_names[ioutput][ivector] << "\""; - } - } - } - */ - myfile << "\n"; - - if(num_dim==2){ - myfile << "ZONE T=\"load step " << time_step << "\", NODES= " << num_nodes - << ", ELEMENTS= " << num_elem << ", DATAPACKING=POINT, ZONETYPE=FEQUADRILATERAL" "\n"; - } - else if(num_dim==3){ - myfile << "ZONE T=\"load step " << time_step << "\", NODES= " << num_nodes - << ", ELEMENTS= " << num_elem << ", DATAPACKING=POINT, ZONETYPE=FEBRICK" "\n"; - } +void Explicit_Solver::tecplot_writer() +{ + int num_dim = simparam.num_dims; + std::string current_file_name; + std::string base_file_name = "TecplotTO"; + std::string base_file_name_undeformed = "TecplotTO_undeformed"; + std::stringstream ss; + std::string file_extension = ".dat"; + std::string file_count; + std::stringstream count_temp; + int time_step = 0; + int temp_convert; + int noutput, nvector; + bool displace_geometry = false; + const_host_vec_array current_collected_output; + int displacement_index; + collect_information(); + // set host views of the collected data to print out from + const_host_vec_array collected_node_coords = collected_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_host_vec_array collected_node_velocities = collected_node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); + // const_host_vec_array collected_node_densities = collected_node_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); + const_host_elem_conn_array collected_nodes_in_elem = collected_nodes_in_elem_distributed->getLocalView(Tpetra::Access::ReadOnly); + + // Convert ijk index system to the finite element numbering convention + // for vertices in cell + CArrayKokkos convert_ijk_to_ensight(max_nodes_per_element); + CArrayKokkos tmp_ijk_indx(max_nodes_per_element); + convert_ijk_to_ensight(0) = 0; + convert_ijk_to_ensight(1) = 1; + convert_ijk_to_ensight(2) = 3; + convert_ijk_to_ensight(3) = 2; + if (num_dim == 3) + { + convert_ijk_to_ensight(4) = 4; + convert_ijk_to_ensight(5) = 5; + convert_ijk_to_ensight(6) = 7; + convert_ijk_to_ensight(7) = 6; + } - for (int nodeline = 0; nodeline < num_nodes; nodeline++) { - myfile << std::setw(25) << collected_node_coords(nodeline,0) << " "; - myfile << std::setw(25) << collected_node_coords(nodeline,1) << " "; - if(num_dim==3) - myfile << std::setw(25) << collected_node_coords(nodeline,2) << " "; + // compared to primitive unit cell, assumes orthogonal primitive unit cell + if (myrank == 0) + { + // initial undeformed geometry + count_temp.str(""); + count_temp << file_index; + file_index++; + file_count = count_temp.str(); + if (displace_geometry && displacement_module >= 0) + { + current_file_name = base_file_name_undeformed + file_count + file_extension; + } + else + { + current_file_name = base_file_name + file_count + file_extension; + } + std::ofstream myfile(current_file_name.c_str()); // output filestream object for file output + // read in position data + myfile << std::fixed << std::setprecision(8); - //velocity print - myfile << std::setw(25) << collected_node_velocities(nodeline,0) << " "; - myfile << std::setw(25) << collected_node_velocities(nodeline,1) << " "; - if(num_dim==3) - myfile << std::setw(25) << collected_node_velocities(nodeline,2) << " "; + // output header of the tecplot file - //myfile << std::setw(25) << collected_node_densities(nodeline,0) << " "; + myfile << "TITLE=\"results for TO code\"" "\n"; + // myfile << "VARIABLES = \"x\", \"y\", \"z\", \"density\", \"sigmaxx\", \"sigmayy\", \"sigmazz\", \"sigmaxy\", \"sigmaxz\", \"sigmayz\"" "\n"; + // else + myfile << "VARIABLES = \"x\", \"y\", \"z\", \"vx\", \"vy\", \"vz\""; /* for (int imodule = 0; imodule < nfea_modules; imodule++){ - noutput = fea_modules[imodule]->noutput; - for(int ioutput = 0; ioutput < noutput; ioutput++){ - current_collected_output = fea_modules[imodule]->module_outputs[ioutput]; - if(fea_modules[imodule]->vector_style[ioutput] == FEA_Module::DOF){ - nvector = fea_modules[imodule]->output_vector_sizes[ioutput]; - for(int ivector = 0; ivector < nvector; ivector++){ - myfile << std::setw(25) << current_collected_output(nodeline*nvector + ivector,0) << " "; - } - } - if(fea_modules[imodule]->vector_style[ioutput] == FEA_Module::NODAL){ - nvector = fea_modules[imodule]->output_vector_sizes[ioutput]; - for(int ivector = 0; ivector < nvector; ivector++){ - myfile << std::setw(25) << current_collected_output(nodeline,ivector) << " "; - } + for(int ioutput = 0; ioutput < fea_modules[imodule]->noutput; ioutput++){ + nvector = fea_modules[imodule]->output_vector_sizes[ioutput]; + for(int ivector = 0; ivector < nvector; ivector++){ + myfile << ", \"" << fea_modules[imodule]->output_dof_names[ioutput][ivector] << "\""; } } } */ - myfile << std::endl; - } - for (int elementline = 0; elementline < num_elem; elementline++) { - //convert node ordering - for (int ii = 0; ii < max_nodes_per_element; ii++) { - if(active_node_ordering_convention == IJK) - temp_convert = convert_ijk_to_ensight(ii); - else - temp_convert = ii; - myfile << std::setw(10) << collected_nodes_in_elem(elementline, temp_convert) + 1 << " "; - } - myfile << " \n"; - } - myfile.close(); - } - if(myrank==0&&(displacement_module>=0&&displace_geometry)){ - //deformed geometry - count_temp.str(""); - count_temp << file_index; - file_index++; - file_count = count_temp.str(); - - current_file_name = base_file_name + file_count + file_extension; - std::ofstream myfile (current_file_name.c_str()); //output filestream object for file output - //read in position data - myfile << std::fixed << std::setprecision(8); - - //output header of the tecplot file - - myfile << "TITLE=\"results for TO code\" \n"; - myfile << "VARIABLES = \"x\", \"y\", \"z\", \"vx\", \"vy\", \"vz\""; - /* - for (int imodule = 0; imodule < nfea_modules; imodule++){ - for(int ioutput = 0; ioutput < fea_modules[imodule]->noutput; ioutput++){ - nvector = fea_modules[imodule]->output_vector_sizes[ioutput]; - for(int ivector = 0; ivector < nvector; ivector++){ - myfile << ", \"" << fea_modules[imodule]->output_dof_names[ioutput][ivector] << "\""; - } + myfile << "\n"; + + if (num_dim == 2) + { + myfile << "ZONE T=\"load step " << time_step << "\", NODES= " << num_nodes + << ", ELEMENTS= " << num_elem << ", DATAPACKING=POINT, ZONETYPE=FEQUADRILATERAL" "\n"; } - } - */ - myfile << "\n"; - - myfile << "ZONE T=\"load step " << time_step + 1 << "\", NODES= " << num_nodes - << ", ELEMENTS= " << num_elem << ", DATAPACKING=POINT, ZONETYPE=FEBRICK" "\n"; - - for (int nodeline = 0; nodeline < num_nodes; nodeline++) { - current_collected_output = fea_modules[displacement_module]->module_outputs[displacement_index]; - myfile << std::setw(25) << collected_node_coords(nodeline,0) + current_collected_output(nodeline*num_dim,0) << " "; - myfile << std::setw(25) << collected_node_coords(nodeline,1) + current_collected_output(nodeline*num_dim + 1,0) << " "; - if(num_dim==3) - myfile << std::setw(25) << collected_node_coords(nodeline,2) + current_collected_output(nodeline*num_dim + 2,0) << " "; - - //velocity print - myfile << std::setw(25) << collected_node_velocities(nodeline,0) << " "; - myfile << std::setw(25) << collected_node_velocities(nodeline,1) << " "; - if(num_dim==3) - myfile << std::setw(25) << collected_node_velocities(nodeline,2) << " "; - - //myfile << std::setw(25) << collected_node_densities(nodeline,0) << " "; - for (int imodule = 0; imodule < nfea_modules; imodule++){ - noutput = fea_modules[imodule]->noutput; - for(int ioutput = 0; ioutput < noutput; ioutput++){ - current_collected_output = fea_modules[imodule]->module_outputs[ioutput]; - if(fea_modules[imodule]->vector_style[ioutput] == FEA_Module::DOF){ - nvector = fea_modules[imodule]->output_vector_sizes[ioutput]; - for(int ivector = 0; ivector < nvector; ivector++){ - myfile << std::setw(25) << current_collected_output(nodeline*nvector + ivector,0) << " "; - } + else if (num_dim == 3) + { + myfile << "ZONE T=\"load step " << time_step << "\", NODES= " << num_nodes + << ", ELEMENTS= " << num_elem << ", DATAPACKING=POINT, ZONETYPE=FEBRICK" "\n"; + } + + for (int nodeline = 0; nodeline < num_nodes; nodeline++) + { + myfile << std::setw(25) << collected_node_coords(nodeline, 0) << " "; + myfile << std::setw(25) << collected_node_coords(nodeline, 1) << " "; + if (num_dim == 3) + { + myfile << std::setw(25) << collected_node_coords(nodeline, 2) << " "; } - if(fea_modules[imodule]->vector_style[ioutput] == FEA_Module::NODAL){ - nvector = fea_modules[imodule]->output_vector_sizes[ioutput]; - for(int ivector = 0; ivector < nvector; ivector++){ - myfile << std::setw(25) << current_collected_output(nodeline,ivector) << " "; + + // velocity print + myfile << std::setw(25) << collected_node_velocities(nodeline, 0) << " "; + myfile << std::setw(25) << collected_node_velocities(nodeline, 1) << " "; + if (num_dim == 3) + { + myfile << std::setw(25) << collected_node_velocities(nodeline, 2) << " "; + } + + // myfile << std::setw(25) << collected_node_densities(nodeline,0) << " "; + /* + for (int imodule = 0; imodule < nfea_modules; imodule++){ + noutput = fea_modules[imodule]->noutput; + for(int ioutput = 0; ioutput < noutput; ioutput++){ + current_collected_output = fea_modules[imodule]->module_outputs[ioutput]; + if(fea_modules[imodule]->vector_style[ioutput] == FEA_Module::DOF){ + nvector = fea_modules[imodule]->output_vector_sizes[ioutput]; + for(int ivector = 0; ivector < nvector; ivector++){ + myfile << std::setw(25) << current_collected_output(nodeline*nvector + ivector,0) << " "; + } + } + if(fea_modules[imodule]->vector_style[ioutput] == FEA_Module::NODAL){ + nvector = fea_modules[imodule]->output_vector_sizes[ioutput]; + for(int ivector = 0; ivector < nvector; ivector++){ + myfile << std::setw(25) << current_collected_output(nodeline,ivector) << " "; + } + } } } + */ + myfile << std::endl; + } + for (int elementline = 0; elementline < num_elem; elementline++) + { + // convert node ordering + for (int ii = 0; ii < max_nodes_per_element; ii++) + { + if (active_node_ordering_convention == IJK) + { + temp_convert = convert_ijk_to_ensight(ii); + } + else + { + temp_convert = ii; + } + myfile << std::setw(10) << collected_nodes_in_elem(elementline, temp_convert) + 1 << " "; + } + myfile << " \n"; + } + myfile.close(); + } + if (myrank == 0 && (displacement_module >= 0 && displace_geometry)) + { + // deformed geometry + count_temp.str(""); + count_temp << file_index; + file_index++; + file_count = count_temp.str(); + + current_file_name = base_file_name + file_count + file_extension; + std::ofstream myfile(current_file_name.c_str()); // output filestream object for file output + // read in position data + myfile << std::fixed << std::setprecision(8); + + // output header of the tecplot file + + myfile << "TITLE=\"results for TO code\" \n"; + myfile << "VARIABLES = \"x\", \"y\", \"z\", \"vx\", \"vy\", \"vz\""; + /* + for (int imodule = 0; imodule < nfea_modules; imodule++){ + for(int ioutput = 0; ioutput < fea_modules[imodule]->noutput; ioutput++){ + nvector = fea_modules[imodule]->output_vector_sizes[ioutput]; + for(int ivector = 0; ivector < nvector; ivector++){ + myfile << ", \"" << fea_modules[imodule]->output_dof_names[ioutput][ivector] << "\""; + } } } - myfile << std::endl; - } - for (int elementline = 0; elementline < num_elem; elementline++) { - //convert node ordering - for (int ii = 0; ii < max_nodes_per_element; ii++) { - if(active_node_ordering_convention == IJK) - temp_convert = convert_ijk_to_ensight(ii); - else - temp_convert = ii; - myfile << std::setw(10) << collected_nodes_in_elem(elementline, temp_convert) + 1 << " "; - } - myfile << " \n"; - } - myfile.close(); + */ + myfile << "\n"; + + myfile << "ZONE T=\"load step " << time_step + 1 << "\", NODES= " << num_nodes + << ", ELEMENTS= " << num_elem << ", DATAPACKING=POINT, ZONETYPE=FEBRICK" "\n"; + + for (int nodeline = 0; nodeline < num_nodes; nodeline++) + { + current_collected_output = fea_modules[displacement_module]->module_outputs[displacement_index]; + myfile << std::setw(25) << collected_node_coords(nodeline, 0) + current_collected_output(nodeline * num_dim, 0) << " "; + myfile << std::setw(25) << collected_node_coords(nodeline, 1) + current_collected_output(nodeline * num_dim + 1, 0) << " "; + if (num_dim == 3) + { + myfile << std::setw(25) << collected_node_coords(nodeline, 2) + current_collected_output(nodeline * num_dim + 2, 0) << " "; + } + + // velocity print + myfile << std::setw(25) << collected_node_velocities(nodeline, 0) << " "; + myfile << std::setw(25) << collected_node_velocities(nodeline, 1) << " "; + if (num_dim == 3) + { + myfile << std::setw(25) << collected_node_velocities(nodeline, 2) << " "; + } + + // myfile << std::setw(25) << collected_node_densities(nodeline,0) << " "; + for (int imodule = 0; imodule < nfea_modules; imodule++) + { + noutput = fea_modules[imodule]->noutput; + for (int ioutput = 0; ioutput < noutput; ioutput++) + { + current_collected_output = fea_modules[imodule]->module_outputs[ioutput]; + if (fea_modules[imodule]->vector_style[ioutput] == FEA_Module::DOF) + { + nvector = fea_modules[imodule]->output_vector_sizes[ioutput]; + for (int ivector = 0; ivector < nvector; ivector++) + { + myfile << std::setw(25) << current_collected_output(nodeline * nvector + ivector, 0) << " "; + } + } + if (fea_modules[imodule]->vector_style[ioutput] == FEA_Module::NODAL) + { + nvector = fea_modules[imodule]->output_vector_sizes[ioutput]; + for (int ivector = 0; ivector < nvector; ivector++) + { + myfile << std::setw(25) << current_collected_output(nodeline, ivector) << " "; + } + } + } + } + myfile << std::endl; + } + for (int elementline = 0; elementline < num_elem; elementline++) + { + // convert node ordering + for (int ii = 0; ii < max_nodes_per_element; ii++) + { + if (active_node_ordering_convention == IJK) + { + temp_convert = convert_ijk_to_ensight(ii); + } + else + { + temp_convert = ii; + } + myfile << std::setw(10) << collected_nodes_in_elem(elementline, temp_convert) + 1 << " "; + } + myfile << " \n"; + } + myfile.close(); } } + /* ---------------------------------------------------------------------- Output Model Information in vtk format ------------------------------------------------------------------------- */ @@ -2748,15 +3076,15 @@ void Explicit_Solver::vtk_writer(){ // Setup of file and directoring for exporting // --------------------------------------------------------------------------- - + FILE *out[20]; // the output files that are written to char filename[128]; - + struct stat st; - + if(stat("vtk",&st) != 0) system("mkdir vtk"); - + if(stat("vtk/data",&st) != 0) system("mkdir vtk/data"); @@ -2764,14 +3092,14 @@ void Explicit_Solver::vtk_writer(){ // --------------------------------------------------------------------------- // Write the Geometry file // --------------------------------------------------------------------------- - - + + sprintf(filename, "vtk/data/%s_%05d_%i.vtu", name, graphics_id, 0); // filename has the full string - + out[0] = fopen(filename, "w"); - + int num_nodes = mesh->num_nodes(); int num_cells = mesh->num_cells(); @@ -2781,7 +3109,7 @@ void Explicit_Solver::vtk_writer(){ fprintf(out[0],"\n"); fprintf(out[0],"\n", num_nodes, num_cells); - + // --------------------------------------------------------------------------- // Write point data @@ -2803,11 +3131,11 @@ void Explicit_Solver::vtk_writer(){ fprintf(out[0]," \n"); for(int cell_var = 0; cell_var < num_cell_vars; cell_var++){ - + fprintf(out[0],"\n", cell_var_names[cell_var]); - + for (int cell_rid = 0; cell_rid < num_cells; cell_rid++){ - + fprintf(out[0],"%f\n",(float) cell_rid); } // end for k over cells @@ -2825,16 +3153,16 @@ void Explicit_Solver::vtk_writer(){ real_t max_coord = 2.0; fprintf(out[0]," \n"); - + fprintf(out[0],"\n", num_dim); - + for (int node_gid = 0; node_gid < num_nodes; node_gid++){ - + fprintf(out[0],"%f %f %f \n",node_coords(node_gid, 0), node_coords(node_gid, 1), node_coords(node_gid, 2)); - } + } fprintf(out[0]," \n"); fprintf(out[0]," \n"); @@ -2855,7 +3183,7 @@ void Explicit_Solver::vtk_writer(){ // write nodes in a cell for (int cell_rid = 0; cell_rid < num_cells; cell_rid++){ - + for(int node = 0; node < 8; node++){ fprintf(out[0],"%i ", nodes_in_elem(cell_rid, node)); } @@ -2884,7 +3212,7 @@ void Explicit_Solver::vtk_writer(){ fprintf(out[0],"\n"); - + for (int cell_rid = 0; cell_rid < num_cells; cell_rid++){ fprintf(out[0],"%i \n", 6); @@ -2909,7 +3237,7 @@ void Explicit_Solver::vtk_writer(){ for (int cell_rid = 0; cell_rid < num_cells; cell_rid++){ fprintf(out[0],"%i \n", faceoffsets*(cell_rid+1)); } // end for k over cells - + fprintf(out[0]," \n"); fprintf(out[0],"\n"); @@ -2976,7 +3304,7 @@ void Explicit_Solver::ensight_writer(){ for (int cell_rid=0; cell_ridfield(cell_rid); + cell_fields(cell_rid, 0) = mat_pt->field(cell_rid); } // end for k over cells int num_elem = rnum_elem; @@ -2986,7 +3314,7 @@ void Explicit_Solver::ensight_writer(){ if(mesh->elem_order() == 0){ num_sub_1d = 1; } - + else{ num_sub_1d = mesh->elem_order()*2; } @@ -3029,7 +3357,7 @@ void Explicit_Solver::ensight_writer(){ // Use average temp from each node as cell temp for (int cell_rid = 0; cell_rid < num_cells; cell_rid++){ - cell_fields(cell_rid, 3) = mat_pt->field(cell_rid); + cell_fields(cell_rid, 3) = mat_pt->field(cell_rid); } // end for k over cells @@ -3038,8 +3366,8 @@ void Explicit_Solver::ensight_writer(){ auto vec_fields = CArray (num_nodes, num_vec_vars, 3); for (int node_gid = 0; node_gid < num_nodes; node_gid++){ - - vec_fields(node_gid, 0, 0) = node_coords(node_gid, 0); + + vec_fields(node_gid, 0, 0) = node_coords(node_gid, 0); vec_fields(node_gid, 0, 1) = node_coords(node_gid, 1); vec_fields(node_gid, 0, 2) = node_coords(node_gid, 2); @@ -3049,15 +3377,15 @@ void Explicit_Solver::ensight_writer(){ // Setup of file and directoring for exporting // --------------------------------------------------------------------------- - + FILE *out[20]; // the output files that are written to char filename[128]; - + struct stat st; - + if(stat("ensight",&st) != 0) system("mkdir ensight"); - + if(stat("ensight/data",&st) != 0) system("mkdir ensight/data"); @@ -3065,47 +3393,47 @@ void Explicit_Solver::ensight_writer(){ // --------------------------------------------------------------------------- // Write the Geometry file // --------------------------------------------------------------------------- - - + + sprintf(filename, "ensight/data/%s.%05d.geo", name, graphics_id); // filename has the full string - + out[0] = fopen(filename, "w"); - - + + fprintf(out[0],"A graphics dump by Cercion \n"); fprintf(out[0],"%s","EnSight Gold geometry\n"); fprintf(out[0],"%s","node id assign\n"); fprintf(out[0],"%s","element id assign\n"); - + fprintf(out[0],"part\n"); fprintf(out[0],"%10d\n",1); fprintf(out[0],"Mesh\n"); - - + + // --- vertices --- fprintf(out[0],"coordinates\n"); fprintf(out[0],"%10d\n",mesh->num_nodes()); - + // write all components of the point coordinates for (int node_gid=0; node_gidgetLocalView (Tpetra::Access::ReadWrite); - //notify that the host view is going to be modified in the file readin - //dual_node_densities.modify_host(); - - //debug Tecplot file readin of initial densities (only works on runs with 1 MPI rank this way) - std::string skip_line, read_line, substring; - std::stringstream line_parse; - real_t read_density; - //in = new std::ifstream(); - //in->open("TecplotDensity.dat"); - //skip 3 lines - //for (int j = 1; j <= 3; j++) { - //getline(*in, skip_line); - // std::cout << skip_line << std::endl; - //} - - - for(int inode = 0; inode < nlocal_nodes; inode++){ - //getline(*in,read_line); - //line_parse.clear(); - //line_parse.str(read_line); - //for(int iword = 0; iword < 10; iword++){ - //read portions of the line into the substring variable - - //if(iword==3){ line_parse >> read_density;} - //else {line_parse >> substring;} - //} - //initialize densities to 1 for now; in the future there might be an option to read in an initial condition for each node - //if(read_density < 0.3) read_density = 0.1; - //node_densities(inode,0) = read_density; - - node_densities(inode,0) = 1; - } - - //sync device view - //dual_node_densities.sync_device(); - } - //allocate global vector information - all_node_densities_distributed = Teuchos::rcp(new MV(all_node_map, 1)); - - //communicate ghost information to the all vector - //create import object using local node indices map and all indices map - Tpetra::Import importer(map, all_node_map); +void Explicit_Solver::init_design() +{ + int num_dim = simparam.num_dims; + bool nodal_density_flag = simparam.nodal_density_flag; + + // set densities + if (nodal_density_flag) + { + if (!simparam.restart_file) + { + design_node_densities_distributed = Teuchos::rcp(new MV(map, 1)); + host_vec_array node_densities = design_node_densities_distributed->getLocalView(Tpetra::Access::ReadWrite); + // notify that the host view is going to be modified in the file readin + // dual_node_densities.modify_host(); + + // debug Tecplot file readin of initial densities (only works on runs with 1 MPI rank this way) + std::string skip_line, read_line, substring; + std::stringstream line_parse; + real_t read_density; + // in = new std::ifstream(); + // in->open("TecplotDensity.dat"); + // skip 3 lines + // for (int j = 1; j <= 3; j++) { + // getline(*in, skip_line); + // std::cout << skip_line << std::endl; + // } + + for (int inode = 0; inode < nlocal_nodes; inode++) + { + // getline(*in,read_line); + // line_parse.clear(); + // line_parse.str(read_line); + // for(int iword = 0; iword < 10; iword++){ + // read portions of the line into the substring variable + + // if(iword==3){ line_parse >> read_density;} + // else {line_parse >> substring;} + // } + // initialize densities to 1 for now; in the future there might be an option to read in an initial condition for each node + // if(read_density < 0.3) read_density = 0.1; + // node_densities(inode,0) = read_density; + + node_densities(inode, 0) = 1; + } - //comms to get ghosts - all_node_densities_distributed->doImport(*design_node_densities_distributed, importer, Tpetra::INSERT); + // sync device view + // dual_node_densities.sync_device(); + } + // allocate global vector information + all_node_densities_distributed = Teuchos::rcp(new MV(all_node_map, 1)); + + // communicate ghost information to the all vector + // create import object using local node indices map and all indices map + Tpetra::Import importer(map, all_node_map); + + // comms to get ghosts + all_node_densities_distributed->doImport(*design_node_densities_distributed, importer, Tpetra::INSERT); + + // debug print + // std::ostream &out = std::cout; + // Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + // if(myrank==0) + // *fos << "Node Densities with Ghosts :" << std::endl; + // all_node_densities_distributed->describe(*fos,Teuchos::VERB_EXTREME); + // *fos << std::endl; + // std::fflush(stdout); + } + else + { + // initialize memory for volume storage + vec_array Element_Densities("Element Densities", rnum_elem, 1); + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + Element_Densities(ielem, 0) = 1; + } - //debug print - //std::ostream &out = std::cout; - //Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - //if(myrank==0) - //*fos << "Node Densities with Ghosts :" << std::endl; - //all_node_densities_distributed->describe(*fos,Teuchos::VERB_EXTREME); - //*fos << std::endl; - //std::fflush(stdout); - - } - else{ - //initialize memory for volume storage - vec_array Element_Densities("Element Densities", rnum_elem, 1); - for(int ielem = 0; ielem < rnum_elem; ielem++) - Element_Densities(ielem,0) = 1; - - //create global vector - Global_Element_Densities = Teuchos::rcp(new MV(all_element_map, Element_Densities)); - - //if(myrank==0) - //*fos << "Global Element Densities:" << std::endl; - //Global_Element_Densities->describe(*fos,Teuchos::VERB_EXTREME); - //*fos << std::endl; - } + // create global vector + Global_Element_Densities = Teuchos::rcp(new MV(all_element_map, Element_Densities)); + // if(myrank==0) + // *fos << "Global Element Densities:" << std::endl; + // Global_Element_Densities->describe(*fos,Teuchos::VERB_EXTREME); + // *fos << std::endl; + } } diff --git a/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.h b/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.h index 6a78e7a2e..8f89fe46a 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.h +++ b/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.h @@ -11,14 +11,14 @@ This program is open source under the BSD-3 License. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - + 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - + 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - + 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. @@ -34,10 +34,9 @@ OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. **********************************************************************************************/ - -#ifndef ELEMENTS_EXPLICIT_SOLVER_H -#define ELEMENTS_EXPLICIT_SOLVER_H +#ifndef ELEMENTS_EXPLICIT_SOLVER_H +#define ELEMENTS_EXPLICIT_SOLVER_H #include "Solver.h" #include "utilities.h" @@ -60,124 +59,124 @@ using namespace mtr; -class Explicit_Solver: public Solver { - +class Explicit_Solver : public Solver +{ public: - Explicit_Solver(Simulation_Parameters_Explicit); - ~Explicit_Solver(); + Explicit_Solver(Simulation_Parameters_Explicit); + ~Explicit_Solver(); + + virtual void run(); + + // void read_mesh_ensight(char *MESH); + + void read_mesh_ansys_dat(const char* MESH); + + void init_state_vectors(); + + // void repartition_nodes(); + + void comm_velocities(); + + void comm_densities(); - virtual void run(); + void init_design(); - //void read_mesh_ensight(char *MESH); + void collect_information(); - void read_mesh_ansys_dat(const char *MESH); + void sort_information(); - void init_state_vectors(); + // process input to decide TO problem and FEA modules + void FEA_module_setup(); - //void repartition_nodes(); + void setup_optimization_problem(); - void comm_velocities(); + // initialize data for boundaries of the model and storage for boundary conditions and applied loads + void init_boundaries(); - void comm_densities(); + // interfaces between user input and creating data structures for bcs + void topology_conditions(); - void init_design(); + // void vtk_writer(); - void collect_information(); + // void ensight_writer(); - void sort_information(); + // interfaces between user input and creating data structures for topology conditions + void generate_tcs(); - //process input to decide TO problem and FEA modules - void FEA_module_setup(); + void init_topology_conditions(int num_sets); - void setup_optimization_problem(); - - //initialize data for boundaries of the model and storage for boundary conditions and applied loads - void init_boundaries(); - - //interfaces between user input and creating data structures for bcs - void topology_conditions(); + void tecplot_writer(); - //void vtk_writer(); + void parallel_tecplot_writer(); - //void ensight_writer(); + void parallel_vtk_writer(); - //interfaces between user input and creating data structures for topology conditions - void generate_tcs(); + void write_outputs(); + void parallel_vtk_writer_new(); + void parallel_vtu_writer_new(); // not yet added + // maps for variable_name:pointer + std::map point_data_scalars_double; + std::map point_data_vectors_double; + std::map cell_data_scalars_double; + std::map cell_data_scalars_int; + std::map> cell_data_fields_double; - void init_topology_conditions (int num_sets); + // void init_boundary_sets(int num_boundary_sets); - void tecplot_writer(); + void tag_boundaries(int this_bc_tag, real_t val, int bdy_set, real_t* patch_limits = NULL); - void parallel_tecplot_writer(); + int check_boundary(Node_Combination& Patch_Nodes, int this_bc_tag, real_t val, real_t* patch_limits); - void parallel_vtk_writer(); + mesh_t* init_mesh; + std::shared_ptr mesh; - void write_outputs(); - void parallel_vtk_writer_new(); - void parallel_vtu_writer_new(); // not yet added - // maps for variable_name:pointer - std::map point_data_scalars_double; - std::map point_data_vectors_double; - std::map cell_data_scalars_double; - std::map cell_data_scalars_int; - std::map > cell_data_fields_double; - - //void init_boundary_sets(int num_boundary_sets); + Simulation_Parameters_Explicit simparam; - void tag_boundaries(int this_bc_tag, real_t val, int bdy_set, real_t *patch_limits = NULL); + // FEA simulations + class FEA_Module_SGH* sgh_module; - int check_boundary(Node_Combination &Patch_Nodes, int this_bc_tag, real_t val, real_t *patch_limits); - - mesh_t *init_mesh; - std::shared_ptr mesh; - - Simulation_Parameters_Explicit simparam; + // Global FEA data + Teuchos::RCP node_velocities_distributed; + Teuchos::RCP initial_node_velocities_distributed; + Teuchos::RCP all_node_velocities_distributed; + Teuchos::RCP ghost_node_velocities_distributed; + Teuchos::RCP all_cached_node_velocities_distributed; - //FEA simulations - class FEA_Module_SGH *sgh_module; + // Distributions of data used to print + Teuchos::RCP collected_node_velocities_distributed; + Teuchos::RCP sorted_node_velocities_distributed; - //Global FEA data - Teuchos::RCP node_velocities_distributed; - Teuchos::RCP initial_node_velocities_distributed; - Teuchos::RCP all_node_velocities_distributed; - Teuchos::RCP ghost_node_velocities_distributed; - Teuchos::RCP all_cached_node_velocities_distributed; + // Boundary Conditions Data + DCArrayKokkos Local_Index_Boundary_Patches; + CArrayKokkos Topology_Condition_Patches; // set of patches corresponding to each boundary condition + CArrayKokkos NTopology_Condition_Patches; - //Distributions of data used to print - Teuchos::RCP collected_node_velocities_distributed; - Teuchos::RCP sorted_node_velocities_distributed; - - //Boundary Conditions Data - DCArrayKokkos Local_Index_Boundary_Patches; - CArrayKokkos Topology_Condition_Patches; //set of patches corresponding to each boundary condition - CArrayKokkos NTopology_Condition_Patches; + // types of boundary conditions + enum tc_type { NONE, TO_SURFACE_CONSTRAINT, TO_BODY_CONSTRAINT }; - //types of boundary conditions - enum tc_type {NONE, TO_SURFACE_CONSTRAINT, TO_BODY_CONSTRAINT}; + // lists what kind of boundary condition each boundary set is assigned to + CArrayKokkos Boundary_Condition_Type_List; - //lists what kind of boundary condition each boundary set is assigned to - CArrayKokkos Boundary_Condition_Type_List; + // time data + real_t time_value; - //time data - real_t time_value; - - //number of displacement boundary conditions acting on nodes; used to size the reduced global stiffness map - size_t Number_DOF_BCS; + // number of displacement boundary conditions acting on nodes; used to size the reduced global stiffness map + size_t Number_DOF_BCS; - //! mapping used to get local ghost index from the global ID. - //typedef ::Tpetra::Details::FixedHashTable - //global_to_local_table_host_type; + // ! mapping used to get local ghost index from the global ID. + // typedef ::Tpetra::Details::FixedHashTable + // global_to_local_table_host_type; - //global_to_local_table_host_type global2local_map; - //CArrayKokkos active_ranks; + // global_to_local_table_host_type global2local_map; + // CArrayKokkos active_ranks; - //allocation flags to avoid repeat MV and global matrix construction - int Matrix_alloc; + // allocation flags to avoid repeat MV and global matrix construction + int Matrix_alloc; - //debug flags - int gradient_print_sync; + // debug flags + int gradient_print_sync; - Teuchos::RCP initial_node_coords_distributed; + Teuchos::RCP initial_node_coords_distributed; }; #endif // end Header Guard diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/CMakeLists.txt b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/CMakeLists.txt index 9f2cdc62b..23092b53c 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/CMakeLists.txt +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/CMakeLists.txt @@ -3,9 +3,23 @@ cmake_minimum_required(VERSION 3.1.3) set(CMAKE_CXX_FLAGS "${Trilinos_CXX_COMPILER_FLAGS} ${CMAKE_CXX_FLAGS}") set(CMAKE_C_FLAGS "${Trilinos_C_COMPILER_FLAGS} ${CMAKE_C_FLAGS}") set(CMAKE_Fortran_FLAGS "${Trilinos_Fortran_COMPILER_FLAGS} ${CMAKE_Fortran_FLAGS}") -set(SRC_Files geometry.cpp boundary.cpp time_integration.cpp momentum.cpp force_sgh.cpp -sgh_optimization.cpp force_gradients_sgh.cpp power_gradients_sgh.cpp energy_sgh.cpp properties.cpp) -set(FEA_Module_SRC FEA_Module_SGH.cpp ) + +include_directories(include) + +set(SRC_Files +src/geometry.cpp +src/boundary.cpp +src/time_integration.cpp +src/momentum.cpp +src/force_sgh.cpp +src/sgh_optimization.cpp +src/force_gradients_sgh.cpp +src/power_gradients_sgh.cpp +src/energy_sgh.cpp +src/properties.cpp +src/setup_sgh.cpp) + +set(FEA_Module_SRC src/FEA_Module_SGH.cpp ) INCLUDE_DIRECTORIES (${Trilinos_INCLUDE_DIRS} ${Trilinos_TPL_INCLUDE_DIRS}) LINK_DIRECTORIES (${Trilinos_LIBRARY_DIRS} ${Trilinos_TPL_LIBRARY_DIRS}) diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/FEA_Module_SGH.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/FEA_Module_SGH.cpp deleted file mode 100644 index f36f7c0f3..000000000 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/FEA_Module_SGH.cpp +++ /dev/null @@ -1,2615 +0,0 @@ -/********************************************************************************************** - © 2020. Triad National Security, LLC. All rights reserved. - This program was produced under U.S. Government contract 89233218CNA000001 for Los Alamos - National Laboratory (LANL), which is operated by Triad National Security, LLC for the U.S. - Department of Energy/National Nuclear Security Administration. All rights in the program are - reserved by Triad National Security, LLC, and the U.S. Department of Energy/National Nuclear - Security Administration. The Government is granted for itself and others acting on its behalf a - nonexclusive, paid-up, irrevocable worldwide license in this material to reproduce, prepare - derivative works, distribute copies to the public, perform publicly and display publicly, and - to permit others to do so. - This program is open source under the BSD-3 License. - Redistribution and use in source and binary forms, with or without modification, are permitted - provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright notice, this list of - conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright notice, this list of - conditions and the following disclaimer in the documentation and/or other materials - provided with the distribution. - - 3. Neither the name of the copyright holder nor the names of its contributors may be used - to endorse or promote products derived from this software without specific prior - written permission. - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS - IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR - CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR - OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - **********************************************************************************************/ - -#include -#include -#include -#include -#include -#include -#include // fmin, fmax, abs note: fminl is long -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include "Tpetra_Import.hpp" -#include "Tpetra_Import_Util2.hpp" - -#include "elements.h" -#include "swage.h" -#include "matar.h" -#include "utilities.h" -#include "node_combination.h" -#include "Simulation_Parameters/Simulation_Parameters_Explicit.h" -#include "Simulation_Parameters/FEA_Module/SGH_Parameters.h" -#include "FEA_Module_SGH.h" -#include "Explicit_Solver.h" - -//optimization -#include "ROL_Solver.hpp" -#include "Kinetic_Energy_Minimize.h" - -#define MAX_ELEM_NODES 8 -#define STRAIN_EPSILON 0.000000001 -#define DENSITY_EPSILON 0.0001 -#define BC_EPSILON 1.0e-6 -#define BUFFER_GROW 100 - -using namespace utils; - - -FEA_Module_SGH::FEA_Module_SGH( - SGH_Parameters& params, Solver *Solver_Pointer, - std::shared_ptr mesh_in, const int my_fea_module_index) - : FEA_Module(Solver_Pointer) { - //assign interfacing index - my_fea_module_index_ = my_fea_module_index; - Module_Type = FEA_MODULE_TYPE::SGH; - - //recast solver pointer for non-base class access - Explicit_Solver_Pointer_ = dynamic_cast(Solver_Pointer); - simparam = &(Explicit_Solver_Pointer_->simparam); - module_params = ¶ms; - - //create ref element object - //ref_elem = new elements::ref_element(); - //create mesh objects - //init_mesh = new swage::mesh_t(simparam); - //mesh = new swage::mesh_t(simparam); - - mesh = mesh_in; - - //boundary condition data - max_boundary_sets = 0; - Local_Index_Boundary_Patches = Explicit_Solver_Pointer_->Local_Index_Boundary_Patches; - - //set Tpetra vector pointers - initial_node_velocities_distributed = Explicit_Solver_Pointer_->initial_node_velocities_distributed; - initial_node_coords_distributed = Explicit_Solver_Pointer_->initial_node_coords_distributed; - all_initial_node_coords_distributed = Explicit_Solver_Pointer_->all_initial_node_coords_distributed; - node_coords_distributed = Explicit_Solver_Pointer_->node_coords_distributed; - node_velocities_distributed = Explicit_Solver_Pointer_->node_velocities_distributed; - all_node_velocities_distributed = Explicit_Solver_Pointer_->all_node_velocities_distributed; - if(simparam->topology_optimization_on||simparam->shape_optimization_on){ - all_cached_node_velocities_distributed = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - force_gradient_velocity = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - force_gradient_position = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - force_gradient_design = Teuchos::rcp(new MV(all_node_map, 1)); - corner_value_storage = Solver_Pointer->corner_value_storage; - corner_vector_storage = Solver_Pointer->corner_vector_storage; - corner_gradient_storage = Solver_Pointer->corner_gradient_storage; - relative_element_densities = DCArrayKokkos(rnum_elem, "relative_element_densities"); - adjoint_vector_distributed = Teuchos::rcp(new MV(map, simparam->num_dims)); - phi_adjoint_vector_distributed = Teuchos::rcp(new MV(map, simparam->num_dims)); - psi_adjoint_vector_distributed = Teuchos::rcp(new MV(all_element_map, 1)); - } - - if(simparam->topology_optimization_on||simparam->shape_optimization_on||simparam->num_dims==2){ - node_masses_distributed = Teuchos::rcp(new MV(map, 1)); - ghost_node_masses_distributed = Teuchos::rcp(new MV(ghost_node_map, 1)); - } - - //setup output - noutput = 0; - init_output(); - - //optimization flags - kinetic_energy_objective = false; - - - //set parameters - Dynamic_Options dynamic_options = simparam->dynamic_options; - time_value = dynamic_options.time_value; - time_final = dynamic_options.time_final; - dt_max = dynamic_options.dt_max; - dt_min = dynamic_options.dt_min; - dt_cfl = dynamic_options.dt_cfl; - graphics_time = simparam->output_options.graphics_time; - graphics_dt_ival = simparam->output_options.graphics_dt_ival; - graphics_cyc_ival = simparam->output_options.graphics_cyc_ival; - cycle_stop = dynamic_options.cycle_stop; - rk_num_stages = dynamic_options.rk_num_stages; - dt = dynamic_options.dt; - fuzz = dynamic_options.fuzz; - tiny = dynamic_options.tiny; - small = dynamic_options.small; - graphics_times = simparam->output_options.graphics_times; - graphics_id = simparam->output_options.graphics_id; - rk_num_bins = simparam->dynamic_options.rk_num_bins; - - if(simparam->topology_optimization_on){ - max_time_steps = BUFFER_GROW; - time_data.resize(max_time_steps+1); - element_internal_energy_distributed = Teuchos::rcp(new MV(all_element_map, 1)); - forward_solve_velocity_data = Teuchos::rcp(new std::vector>(max_time_steps+1)); - forward_solve_coordinate_data = Teuchos::rcp(new std::vector>(max_time_steps+1)); - forward_solve_internal_energy_data = Teuchos::rcp(new std::vector>(max_time_steps+1)); - adjoint_vector_data = Teuchos::rcp(new std::vector>(max_time_steps+1)); - phi_adjoint_vector_data = Teuchos::rcp(new std::vector>(max_time_steps+1)); - psi_adjoint_vector_data = Teuchos::rcp(new std::vector>(max_time_steps+1)); - //assign a multivector of corresponding size to each new timestep in the buffer - for(int istep = 0; istep < max_time_steps+1; istep++){ - (*forward_solve_velocity_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - (*forward_solve_coordinate_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - (*forward_solve_internal_energy_data)[istep] = Teuchos::rcp(new MV(all_element_map, 1)); - (*adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - (*phi_adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - (*psi_adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_element_map, 1)); - } - - } - - have_loading_conditions = false; - -} - -FEA_Module_SGH::~FEA_Module_SGH(){ - //delete simparam; -} - -/* ---------------------------------------------------------------------- - Read ANSYS dat format mesh file -------------------------------------------------------------------------- */ -void FEA_Module_SGH::read_conditions_ansys_dat(std::ifstream *in, std::streampos before_condition_header){ - - char ch; - int num_dim = simparam->num_dims; - int buffer_lines = 1000; - int max_word = 30; - auto input_options = simparam->input_options.value(); - int p_order = input_options.p_order; - real_t unit_scaling = input_options.unit_scaling; - int local_node_index, current_column_index; - size_t strain_count; - std::string skip_line, read_line, substring, token; - std::stringstream line_parse, line_parse2; - CArrayKokkos read_buffer; - CArrayKokkos read_buffer_indices; - int buffer_loop, buffer_iteration, buffer_iterations, scan_loop, nodes_per_element, words_per_line; - size_t read_index_start, node_rid, elem_gid; - LO local_dof_id; - GO node_gid; - real_t dof_value; - host_vec_array node_densities; - -} // end read_conditions_ansys_dat - -// ----------------------------------------------------------------------------- -// Interfaces read in data with the SGH solver data; currently a hack to streamline -//------------------------------------------------------------------------------ -void FEA_Module_SGH::sgh_interface_setup(node_t &node, - elem_t &elem, - corner_t &corner){ - - const size_t num_dim = simparam->num_dims; - const size_t rk_num_bins = simparam->dynamic_options.rk_num_bins; - - num_nodes_in_elem = 1; - for (int dim=0; dimnall_nodes; - int myrank = Explicit_Solver_Pointer_->myrank; - int nranks = Explicit_Solver_Pointer_->nranks; - //printf("Num nodes assigned to MPI rank %lu is %lu\n" , myrank, nall_nodes); - - // intialize node variables - mesh->initialize_nodes(nall_nodes); - mesh->initialize_local_nodes(Explicit_Solver_Pointer_->nlocal_nodes); - node.initialize(rk_num_bins, nall_nodes, num_dim); - //std::cout << "Bin counts " << rk_num_bins << " Node counts " << nall_nodes << " Num dim " << num_dim << std::endl; - - //view scope - { - host_vec_array interface_node_coords = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); - //save node data to node.coords - //std::cout << "NODE DATA ON RANK " << myrank << std::endl; - if(num_dim==2){ - for(int inode = 0; inode < nall_nodes; inode++){ - //std::cout << "Node index " << inode+1 << " "; - node.coords.host(0,inode,0) = interface_node_coords(inode,0); - //std::cout << host_node_coords_state(0,inode,0)+1<< " "; - node.coords.host(0,inode,1) = interface_node_coords(inode,1); - //std::cout << host_node_coords_state(0,inode,1)+1<< " "; - } - } - else if(num_dim==3){ - for(int inode = 0; inode < nall_nodes; inode++){ - //std::cout << "Node index " << inode+1 << " "; - node.coords.host(0,inode,0) = interface_node_coords(inode,0); - //std::cout << host_node_coords_state(0,inode,0)+1<< " "; - node.coords.host(0,inode,1) = interface_node_coords(inode,1); - //std::cout << host_node_coords_state(0,inode,1)+1<< " "; - - node.coords.host(0,inode,2) = interface_node_coords(inode,2); - //std::cout << host_node_coords_state(0,inode,2)+1<< std::endl; - } - } - } //end view scope - // --- read in the elements in the mesh --- - - rnum_elem = Explicit_Solver_Pointer_->rnum_elem; - //printf("Num elems assigned to MPI rank %lu is %lu\n" , myrank, rnum_elem); - - // intialize elem variables - mesh->initialize_elems(rnum_elem, num_dim); - elem.initialize(rk_num_bins, nall_nodes, 3); // always 3D here, even for 2D - nodes_in_elem = mesh->nodes_in_elem; - //save data to nodes_in_elem.host - //CArrayKokkos host_mesh_nodes_in_elem(rnum_elem, num_nodes_in_elem); - //view scope - { - host_elem_conn_array interface_nodes_in_elem = Explicit_Solver_Pointer_->global_nodes_in_elem_distributed->getLocalView (Tpetra::Access::ReadWrite); - //save node data to node.coords - //std::cout << "ELEMENT CONNECTIVITY ON RANK " << myrank << std::endl; - for(int ielem = 0; ielem < rnum_elem; ielem++){ - //std::cout << "Element index " << ielem+1 << " "; - for(int inode = 0; inode < num_nodes_in_elem; inode++){ - nodes_in_elem.host(ielem,inode) = Explicit_Solver_Pointer_->all_node_map->getLocalElement(interface_nodes_in_elem(ielem,inode)); - //debug print - //std::cout << nodes_in_elem.get_kokkos_dual_view().h_view(ielem*num_nodes_in_elem + inode)+1<< " "; - } - //std::cout << std::endl; - } - } - // update device side - nodes_in_elem.update_device(); - - //debug print - - //CArrayKokkos device_mesh_nodes_in_elem(rnum_elem, num_nodes_in_elem); - //device_mesh_nodes_in_elem.get_kokkos_view() = nodes_in_elem.get_kokkos_dual_view().d_view; - //host_mesh_nodes_in_elem.get_kokkos_view() = nodes_in_elem.get_kokkos_dual_view().view_host(); - /* - if(myrank==1){ - std::cout << "ELEMENT CONNECTIVITY ON RANK 1 in LOCAL INDICES" << myrank << std::endl; - for(int ielem = 0; ielem < rnum_elem; ielem++){ - std::cout << "Element index " << ielem+1 << " "; - for(int inode = 0; inode < num_nodes_in_elem; inode++){ - //debug print - //device_mesh_nodes_in_elem(ielem,inode) = Explicit_Solver_Pointer_->all_node_map->getLocalElement(interface_nodes_in_elem(ielem,inode)); - std::cout << nodes_in_elem(ielem, inode)+1<< " "; - } - std::cout << std::endl; - } - } - */ - /* - std::cout.flush(); - if(myrank==1){ - std::cout << "ELEMENT CONNECTIVITY ON RANK 1 in GLOBAL INDICES" << myrank << std::endl; - std::cout << "local node index of global index 275 on rank 1 " << Explicit_Solver_Pointer_->all_node_map->getLocalElement(275) << std::endl; - for(int ielem = 0; ielem < rnum_elem; ielem++){ - std::cout << ielem << " "; - for(int inode = 0; inode < num_nodes_in_elem; inode++){ - //debug print - //device_mesh_nodes_in_elem(ielem,inode) = Explicit_Solver_Pointer_->all_node_map->getLocalElement(interface_nodes_in_elem(ielem,inode)); - std::cout << Explicit_Solver_Pointer_->all_node_map->getGlobalElement(nodes_in_elem(ielem, inode))<< " "; - } - std::cout << std::endl; - } - } - std::cout.flush(); - */ - /* - size_t nall_nodes = Explicit_Solver_Pointer_->nall_nodes; - node.all_coords = DCArrayKokkos (rk_num_bins, nall_nodes, num_dim); - node.all_vel = DCArrayKokkos (rk_num_bins, nall_nodes, num_dim); - node.all_mass = DCArrayKokkos (nall_nodes); - - //save all data (nlocal +nghost) - CArrayKokkos host_all_node_coords_state(rk_num_bins, nall_nodes, num_dim); - host_vec_array interface_all_node_coords = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); - host_all_node_coords_state.get_kokkos_view() = node.all_coords.get_kokkos_dual_view().view_host(); - //host_node_coords_state = CArrayKokkos(rk_num_bins, nall_nodes, num_dim); - //host_all_node_coords_state.get_kokkos_view() = Kokkos::View("debug", rk_num_bins*nall_nodes*num_dim); - //save node data to node.coords - - //std::cout << "ALL NODE DATA ON RANK " << myrank << std::endl; - for(int inode = 0; inode < nall_nodes; inode++){ - //std::cout << "Node index " << inode+1 << " "; - node.all_coords.host(0,inode,0) = interface_all_node_coords(inode,0); - //std::cout << host_all_node_coords_state(0,inode,0)+1<< " "; - node.all_coords.host(0,inode,1) = interface_all_node_coords(inode,1); - //std::cout << host_all_node_coords_state(0,inode,1)+1<< " "; - node.all_coords.host(0,inode,2) = interface_all_node_coords(inode,2); - //std::cout << host_all_node_coords_state(0,inode,2)+1<< std::endl; - } - */ - - // save the node coords to the current RK value - for (size_t node_gid=0; node_gidinitialize_corners(num_corners); - corner.initialize(num_corners, num_dim); - - /* - for(int inode = 0; inode < nall_nodes; inode++){ - std::cout << "Node index " << inode+1 << " "; - for(int rk=0; rkboundary_conditions.size(); - int num_dim = simparam->num_dims; - - // set the number of boundary sets - if(myrank == 0) - std::cout << "building boundary sets " << std::endl; - - //initialize to 1 since there must be at least 1 boundary set anyway; read in may occure later - if(max_boundary_sets==0) max_boundary_sets = 1; - //std::cout << "NUM BOUNDARY CONDITIONS ON RANK " << myrank << " FOR INIT " << num_boundary_conditions <(nall_nodes*num_dim, "Node_DOF_Boundary_Condition_Type"); - - //initialize - for(int init=0; init < nall_nodes*num_dim; init++) - Node_DOF_Boundary_Condition_Type(init) = NONE; - - Number_DOF_BCS = 0; -} - -/* ---------------------------------------------------------------------- - initialize storage for element boundary surfaces corresponding to user BCs -------------------------------------------------------------------------- */ - -void FEA_Module_SGH::init_boundary_sets (int num_sets){ - - if(num_sets == 0){ - std::cout << " Warning: number of boundary conditions = 0"; - return; - } - //initialize maximum - max_boundary_sets = num_sets; - //std::cout << " DEBUG PRINT "<(num_sets, "Boundary_Condition_Type_List"); - NBoundary_Condition_Patches = CArrayKokkos(num_sets, "NBoundary_Condition_Patches"); - //std::cout << "NBOUNDARY PATCHES ON RANK " << myrank << " FOR INIT IS " << nboundary_patches <(num_sets, nboundary_patches, "Boundary_Condition_Patches"); - - //initialize data - for(int iset = 0; iset < num_sets; iset++) NBoundary_Condition_Patches(iset) = 0; - - //initialize - for(int ibdy=0; ibdy < num_sets; ibdy++) Boundary_Condition_Type_List(ibdy) = NONE; -} - -/* ---------------------------------------------------------------------------- - Grow boundary conditions sets of element boundary surfaces -------------------------------------------------------------------------------- */ - -void FEA_Module_SGH::grow_boundary_sets(int num_sets){ - int num_dim = simparam->num_dims; - - if(num_sets == 0){ - std::cout << " Warning: number of boundary conditions being set to 0"; - return; - } - - //std::cout << " DEBUG PRINT "<max_boundary_sets){ - //temporary storage for previous data - CArrayKokkos Temp_Boundary_Condition_Type_List = Boundary_Condition_Type_List; - CArrayKokkos Temp_NBoundary_Condition_Patches = NBoundary_Condition_Patches; - CArrayKokkos Temp_Boundary_Condition_Patches = Boundary_Condition_Patches; - - max_boundary_sets = num_sets + 5; //5 is an arbitrary buffer - Boundary_Condition_Type_List = CArrayKokkos(max_boundary_sets, "Boundary_Condition_Type_List"); - NBoundary_Condition_Patches = CArrayKokkos(max_boundary_sets, "NBoundary_Condition_Patches"); - //std::cout << "NBOUNDARY PATCHES ON RANK " << myrank << " FOR GROW " << nboundary_patches <(max_boundary_sets, nboundary_patches, "Boundary_Condition_Patches"); - - //copy previous data back over - //std::cout << "NUM BOUNDARY CONDITIONS ON RANK " << myrank << " FOR COPY " << max_boundary_sets <output(FIELD::velocity); - bool output_strain_flag = simparam->output(FIELD::strain); - bool output_stress_flag = simparam->output(FIELD::stress); - int num_dim = simparam->num_dims; - int Brows; - if(num_dim==3) Brows = 6; - else Brows = 3; - - //Implicit compliant code - if(output_velocity_flag){ - //displacement_index is accessed by writers at the solver level for deformed output - output_velocity_index = noutput; - noutput += 1; - module_outputs.resize(noutput); - - vector_style.resize(noutput); - vector_style[noutput-1] = DOF; - - output_vector_sizes.resize(noutput); - output_vector_sizes[noutput-1] = num_dim; - - output_dof_names.resize(noutput); - output_dof_names[noutput-1].resize(num_dim); - output_dof_names[noutput-1][0] = "vx"; - output_dof_names[noutput-1][1] = "vy"; - if(num_dim==3) - output_dof_names[noutput-1][2] = "vz"; - } - if(output_strain_flag){ - output_strain_index = noutput; - noutput += 1; - module_outputs.resize(noutput); - - vector_style.resize(noutput); - vector_style[noutput-1] = NODAL; - - output_vector_sizes.resize(noutput); - output_vector_sizes[noutput-1] = Brows; - - output_dof_names.resize(noutput); - output_dof_names[noutput-1].resize(Brows); - if(num_dim==2){ - output_dof_names[noutput-1][0] = "strain_xx"; - output_dof_names[noutput-1][1] = "strain_yy"; - output_dof_names[noutput-1][2] = "strain_xy"; - } - if(num_dim==3){ - output_dof_names[noutput-1][0] = "strain_xx"; - output_dof_names[noutput-1][1] = "strain_yy"; - output_dof_names[noutput-1][2] = "strain_zz"; - output_dof_names[noutput-1][3] = "strain_xy"; - output_dof_names[noutput-1][4] = "strain_xz"; - output_dof_names[noutput-1][5] = "strain_yz"; - } - } - if(output_stress_flag){ - output_stress_index = noutput; - noutput += 1; - module_outputs.resize(noutput); - - vector_style.resize(noutput); - vector_style[noutput-1] = NODAL; - - output_vector_sizes.resize(noutput); - output_vector_sizes[noutput-1] = Brows; - - output_dof_names.resize(noutput); - output_dof_names[noutput-1].resize(Brows); - if(num_dim==2){ - output_dof_names[noutput-1][0] = "stress_xx"; - output_dof_names[noutput-1][1] = "stress_yy"; - output_dof_names[noutput-1][3] = "stress_xy"; - } - if(num_dim==3){ - output_dof_names[noutput-1][0] = "stress_xx"; - output_dof_names[noutput-1][1] = "stress_yy"; - output_dof_names[noutput-1][2] = "stress_zz"; - output_dof_names[noutput-1][3] = "stress_xy"; - output_dof_names[noutput-1][4] = "stress_xz"; - output_dof_names[noutput-1][5] = "stress_yz"; - } - } - -} - -/* ------------------------------------------------------------------------------------------- - Prompts sorting for elastic response output data. For now, nodal strains. ----------------------------------------------------------------------------------------------- */ - -void FEA_Module_SGH::sort_output(Teuchos::RCP > sorted_map){ - - -} - -/* ------------------------------------------------------------------------------------------- - populate requests this module makes for output data ----------------------------------------------------------------------------------------------- */ - -void FEA_Module_SGH::write_data(std::map &point_data_scalars_double, - std::map &point_data_vectors_double, - std::map &cell_data_scalars_double, - std::map &cell_data_scalars_int, - std::map > &cell_data_fields_double){ - - - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - - for (const auto& field_name : simparam->output_options.output_fields) { - switch (field_name) - { - - case FIELD::velocity: - // node "velocity" - node_vel.update_host(); - point_data_vectors_double["velocity"] = &node_vel.host(rk_level,0,0); - break; - - case FIELD::element_density: - // element "density" - elem_den.update_host(); - cell_data_scalars_double["element_density"] = elem_den.host_pointer(); - break; - - case FIELD::pressure: - // element "pressure" - elem_pres.update_host(); - cell_data_scalars_double["pressure"] = elem_pres.host_pointer(); - break; - - case FIELD::SIE: - // element "SIE" - elem_sie.update_host(); - cell_data_scalars_double["SIE"] = &elem_sie.host(rk_level,0); - break; - - case FIELD::volume: - // element "volume" - elem_vol.update_host(); - cell_data_scalars_double["volume"] = elem_vol.host_pointer(); - break; - - case FIELD::mass: - // element "mass" - elem_mass.update_host(); - cell_data_scalars_double["mass"] = elem_mass.host_pointer(); - break; - - case FIELD::sound_speed: - // element "sspd" - elem_sspd.update_host(); - cell_data_scalars_double["sound_speed"] = elem_sspd.host_pointer(); - break; - - case FIELD::material_id: - // element "material_id" - elem_mat_id.update_host(); - cell_data_scalars_int["material_id"] = reinterpret_cast(elem_mat_id.host_pointer()); - break; - - case FIELD::user_vars: - // element "user_vars" - elem_user_output_vars.update_host(); - cell_data_fields_double["user_vars"] = std::make_pair(elem_user_output_vars.host_pointer(), - elem_user_output_vars.dims(1)); - case FIELD::stress: - // element "stress" - elem_stress.update_host(); - cell_data_fields_double["stress"] = std::make_pair(&elem_stress.host(rk_level,0,0,0), 9); - break; - - default: - break; - - } // end switch - } // end if - - // element "mat_id" //uncomment if needed (works fine) - //sgh_module->elem_mat_id.update_host(); - //cell_data_scalars_int["mat_id"] = reinterpret_cast(&sgh_module->elem_mat_id.host(0)); - - // element "user_output_vars" //uncomment if needed (works fine) - //sgh_module->elem_user_output_vars.update_host(); - //cell_data_fields_double["user_output_vars"] = std::make_pair(&sgh_module->elem_user_output_vars.host_pointer(), - // sgh_module->elem_user_output_vars.dims(1)); - - // element "stress" //uncomment if needed (works fine) - //sgh_module->elem_stress.update_host(); - //cell_data_fields_double["stress"] = std::make_pair(&sgh_module->elem_stress.host(rk_level,0,0,0), 9); -} - -/* ------------------------------------------------------------------------------------------- - Prompts sorting for elastic response output data. For now, nodal strains. ----------------------------------------------------------------------------------------------- */ - -void FEA_Module_SGH::sort_element_output(Teuchos::RCP > sorted_map){ - //interface element density data - { - host_vec_array Element_Densities = Global_Element_Densities->getLocalView (Tpetra::Access::ReadWrite); - elem_den.update_host(); - for(int ielem = 0; ielem < rnum_elem; ielem++){ - Element_Densities(ielem,0) = elem_den.host(ielem); - } - } - -} - -/* ------------------------------------------------------------------------------------------- - Prompts computation of elastic response output data. For now, nodal strains. ----------------------------------------------------------------------------------------------- */ - -void FEA_Module_SGH::collect_output(Teuchos::RCP > global_reduce_map){ - -} - -/* ------------------------------------------------------------------------------------------- - Prompts computation of elastic response output data. For now, nodal strains. ----------------------------------------------------------------------------------------------- */ - -void FEA_Module_SGH::compute_output(){ - -} - -/* ---------------------------------------------------------------------- - Communicate updated nodal velocities to ghost nodes -------------------------------------------------------------------------- */ - -void FEA_Module_SGH::comm_node_masses(){ - - //debug print of design vector - //std::ostream &out = std::cout; - //Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - //if(myrank==0) - //*fos << "Density data :" << std::endl; - //node_densities_distributed->describe(*fos,Teuchos::VERB_EXTREME); - //*fos << std::endl; - //std::fflush(stdout); - - //communicate design densities - //create import object using local node indices map and all indices map - //Tpetra::Import importer(map, ghost_node_map); - - //comms to get ghosts - ghost_node_masses_distributed->doImport(*node_masses_distributed, *ghost_importer, Tpetra::INSERT); - //all_node_map->describe(*fos,Teuchos::VERB_EXTREME); - //all_node_velocities_distributed->describe(*fos,Teuchos::VERB_EXTREME); - - //update_count++; - //if(update_count==1){ - //MPI_Barrier(world); - //MPI_Abort(world,4); - //} -} - -/* ------------------------------------------------------------------------------------------- - Communicate ghosts using the current optimization design data ----------------------------------------------------------------------------------------------- */ - -void FEA_Module_SGH::comm_variables(Teuchos::RCP zp){ - - if(simparam->topology_optimization_on){ - //set density vector to the current value chosen by the optimizer - test_node_densities_distributed = zp; - - //debug print of design vector - //std::ostream &out = std::cout; - //Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - //if(myrank==0) - //*fos << "Density data :" << std::endl; - //node_densities_distributed->describe(*fos,Teuchos::VERB_EXTREME); - //*fos << std::endl; - //std::fflush(stdout); - - //communicate design densities - //create import object using local node indices map and all indices map - //Tpetra::Import importer(map, all_node_map); - - //comms to get ghosts - all_node_densities_distributed->doImport(*test_node_densities_distributed, *importer, Tpetra::INSERT); - } - else if(simparam->shape_optimization_on){ - //clause to communicate boundary node data if the boundary nodes are ghosts on this rank - } -} - - -/* ------------------------------------------------------------------------------------------- - enforce constraints on nodes due to BCS ----------------------------------------------------------------------------------------------- */ - -void FEA_Module_SGH::node_density_constraints(host_vec_array node_densities_lower_bound){ - - const size_t num_dim = mesh->num_dims; - const_vec_array all_initial_node_coords = all_initial_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - const size_t num_lcs = module_params->loading_conditions.size(); - - const DCArrayKokkos mat_fill = simparam->mat_fill; - const DCArrayKokkos loading = module_params->loading; - - //debug check - //std::cout << "NUMBER OF LOADING CONDITIONS: " << num_lcs << std::endl; - - // walk over the nodes to update the velocity - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - double current_node_coords[3]; - double radius; - for (size_t dim = 0; dim < num_dim; dim++){ - current_node_coords[dim] = all_initial_node_coords(node_gid, dim); - } // end for dim - radius = sqrt(current_node_coords[0]*current_node_coords[0]+current_node_coords[1]*current_node_coords[1]+current_node_coords[2]*current_node_coords[2]); - for(size_t ilc=0; ilc < num_lcs; ilc++){ - //debug check - //std::cout << "LOADING CONDITION VOLUME TYPE: " << to_string(loading(ilc).volume) << std::endl; - - bool fill_this = loading(ilc).volume.contains(current_node_coords); - if(fill_this){ - node_densities_lower_bound(node_gid,0) = 1; - } - } - }); // end for parallel for over nodes - -} - -/* ---------------------------------------------------------------------------- - Setup SGH solver data -------------------------------------------------------------------------------- */ - -void FEA_Module_SGH::setup(){ - - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - const size_t num_fills = simparam->regions.size(); - const size_t rk_num_bins = simparam->dynamic_options.rk_num_bins; - const size_t num_bcs = module_params->boundary_conditions.size(); - const size_t num_materials = simparam->materials.size(); - const int num_dim = simparam->num_dims; - const size_t num_lcs = module_params->loading.size(); - if(num_lcs) - have_loading_conditions = true; - - - // --------------------------------------------------------------------- - // obtain mesh data - // --------------------------------------------------------------------- - sgh_interface_setup(node_interface, elem_interface, corner_interface); - mesh->build_corner_connectivity(); - //debug print of corner ids - /* - if(myrank==1){ - for(int i = 0; i < mesh.num_nodes; i++){ - - // loop over all corners around the node and calculate the nodal force - for (size_t corner_lid=0; corner_lidgetGlobalElement(i) << " " << i << " " << all_node_map->getLocalElement(all_node_map->getGlobalElement(i)) << " " << corner_gid << " " << std::endl; - - } // end for corner_lid - //std::cout << explicit_solver_pointer->all_node_map->getGlobalElement(i) << " " << node_force[0] << " " << node_force[1] << " " << node_force[2] << std::endl; - //std::cout << explicit_solver_pointer->all_node_map->getGlobalElement(i) << " " << node_mass(i) << std::endl; - } - } - */ - /* - if(myrank==1){ - for(int i = 0; i < mesh.num_elems; i++){ - - // loop over all corners around the node and calculate the nodal force - for (size_t corner_lid=0; corner_lidgetGlobalElement(mesh.nodes_in_elem(i, corner_lid)) <<" " << corner_gid << " " << std::endl; - - } // end for corner_lid - //std::cout << explicit_solver_pointer->all_node_map->getGlobalElement(i) << " " << node_force[0] << " " << node_force[1] << " " << node_force[2] << std::endl; - //std::cout << explicit_solver_pointer->all_node_map->getGlobalElement(i) << " " << node_mass(i) << std::endl; - } - } - */ - mesh->build_elem_elem_connectivity(); - mesh->num_bdy_patches = nboundary_patches; - if(num_dim==2){ - mesh->build_patch_connectivity(); - mesh->build_node_node_connectivity(); - } - - // --------------------------------------------------------------------- - // allocate memory - // --------------------------------------------------------------------- - - // shorthand names - const size_t num_nodes = mesh->num_nodes; - const size_t num_elems = mesh->num_elems; - const size_t num_corners = mesh->num_corners; - - - // --- make dual views of data on CPU and GPU --- - // Notes: - // Instead of using a struct of dual types like the mesh type, - // individual dual views will be made for all the state - // variables. The motivation is to reduce memory movement - // when passing state into a function. Passing a struct by - // reference will copy the meta data and pointers for the - // variables held inside the struct. Since all the mesh - // variables are typically used by most functions, a single - // mesh struct or passing the arrays will be roughly equivalent - // for memory movement. - - - // create Dual Views of the individual node struct variables - node_coords = DViewCArrayKokkos(node_interface.coords.get_kokkos_dual_view().view_host().data(),rk_num_bins,num_nodes,num_dim); - - node_vel = DViewCArrayKokkos(node_interface.vel.get_kokkos_dual_view().view_host().data(),rk_num_bins,num_nodes,num_dim); - - node_mass = DViewCArrayKokkos(node_interface.mass.get_kokkos_dual_view().view_host().data(),num_nodes); - - - // create Dual Views of the individual elem struct variables - elem_den= DViewCArrayKokkos(&elem_interface.den(0), - num_elems); - - elem_pres = DViewCArrayKokkos(&elem_interface.pres(0), - num_elems); - - elem_stress = DViewCArrayKokkos(&elem_interface.stress(0,0,0,0), - rk_num_bins, - num_elems, - 3, - 3); // always 3D even in 2D-RZ - - elem_sspd = DViewCArrayKokkos(&elem_interface.sspd(0), - num_elems); - - elem_sie = DViewCArrayKokkos(&elem_interface.sie(0,0), - rk_num_bins, - num_elems); - - elem_vol = DViewCArrayKokkos(&elem_interface.vol(0), - num_elems); - - elem_div = DViewCArrayKokkos(&elem_interface.div(0), - num_elems); - - - elem_mass = DViewCArrayKokkos(&elem_interface.mass(0), - num_elems); - - elem_mat_id = DViewCArrayKokkos(&elem_interface.mat_id(0), - num_elems); - - // create Dual Views of the corner struct variables - corner_force = DViewCArrayKokkos (&corner_interface.force(0,0), - num_corners, - num_dim); - - corner_mass = DViewCArrayKokkos (&corner_interface.mass(0), - num_corners); - - // allocate elem_vel_grad - elem_vel_grad = DCArrayKokkos (num_elems,3,3); - - // allocate material models - elem_eos = DCArrayKokkos (num_elems); - elem_strength = DCArrayKokkos (num_elems); - - // --------------------------------------------------------------------- - // calculate geometry - // --------------------------------------------------------------------- - node_coords.update_device(); - Kokkos::fence(); - - get_vol(); - - //FEA_Module bc variable - num_boundary_conditions = num_bcs; - - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos mat_fill = simparam->mat_fill; - const DCArrayKokkos material = simparam->material; - global_vars = simparam->global_vars; - state_vars = DCArrayKokkos (rnum_elem, simparam->max_num_state_vars); - elem_user_output_vars = DCArrayKokkos (rnum_elem, simparam->output_options.max_num_user_output_vars); - - //--- calculate bdy sets ---// - mesh->num_nodes_in_patch = 2*(num_dim-1); // 2 (2D) or 4 (3D) - mesh->num_patches_in_elem = 2*num_dim; // 4 (2D) or 6 (3D) - mesh->init_bdy_sets(num_bcs); - num_bdy_sets = mesh->num_bdy_sets; - printf("Num BC's = %lu\n", num_bcs); - - // patch ids in bdy set - bdy_patches_in_set = mesh->bdy_patches_in_set; - if(num_dim==2) - bdy_nodes = mesh->bdy_nodes; - - // tag boundary patches in the set - tag_bdys(boundary, *mesh, node_coords); - - build_boundry_node_sets(*mesh); - - // node ids in bdy_patch set - bdy_nodes_in_set = mesh->bdy_nodes_in_set; - num_bdy_nodes_in_set = mesh->num_bdy_nodes_in_set; - - //assign mesh views needed by the FEA module - - // elem ids in elem - elems_in_elem = mesh->elems_in_elem; - num_elems_in_elem = mesh->num_elems_in_elem; - - //corners - num_corners_in_node = mesh->num_corners_in_node; - corners_in_node = mesh->corners_in_node; - corners_in_elem = mesh->corners_in_elem; - - //elem-node conn & node-node conn - elems_in_node = mesh->elems_in_node; - if(num_dim==2){ - nodes_in_node = mesh->nodes_in_node; - num_nodes_in_node = mesh->num_nodes_in_node; - //patch conn - - patches_in_elem = mesh->patches_in_elem; - nodes_in_patch = mesh->nodes_in_patch; - elems_in_patch = mesh->elems_in_patch; - } - - // loop over BCs - for (size_t this_bdy = 0; this_bdy < num_bcs; this_bdy++){ - - RUN_CLASS({ - printf("Boundary Condition number %lu \n", this_bdy); - printf(" Num bdy patches in this set = %lu \n", bdy_patches_in_set.stride(this_bdy)); - printf(" Num bdy nodes in this set = %lu \n", bdy_nodes_in_set.stride(this_bdy)); - }); - Kokkos::fence(); - - }// end for - - // elem_mat_id needs to be initialized before initialization of material models - for (int f_id = 0; f_id < num_fills; f_id++){ - FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { - elem_mat_id(elem_gid) = mat_fill(f_id).material_id; - }); - } - elem_mat_id.update_host(); - - // function for initializing state_vars - init_state_vars(material, - elem_mat_id, - state_vars, - global_vars, - elem_user_output_vars, - rnum_elem); - - // initialize strength model - init_strength_model(elem_strength, - material, - elem_mat_id, - state_vars, - global_vars, - elem_user_output_vars, - rnum_elem); - - // initialize eos model - init_eos_model(elem_eos, - material, - elem_mat_id, - state_vars, - global_vars, - elem_user_output_vars, - rnum_elem); - - //--- apply the fill instructions over each of the Elements---// - - //initialize if topology optimization is used - if(simparam->topology_optimization_on){ - for(int elem_id = 0; elem_id < rnum_elem; elem_id++){ - relative_element_densities.host(elem_id) = 1; - }//for - relative_element_densities.update_device(); - } - - // loop over the fill instructures - for (int f_id = 0; f_id < num_fills; f_id++){ - - // parallel loop over elements in mesh - //for (size_t elem_gid = 0; elem_gid <= rnum_elem; elem_gid++) { - FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { - - // calculate the coordinates and radius of the element - double elem_coords[3]; // note:initialization with a list won't work - elem_coords[0] = 0.0; - elem_coords[1] = 0.0; - elem_coords[2] = 0.0; - - // get the coordinates of the element center - for (int node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - elem_coords[0] += node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), 0); - elem_coords[1] += node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), 1); - if (num_dim == 3){ - elem_coords[2] += node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), 2); - } else - { - elem_coords[2] = 0.0; - } - } // end loop over nodes in element - elem_coords[0] = elem_coords[0]/num_nodes_in_elem; - elem_coords[1] = elem_coords[1]/num_nodes_in_elem; - elem_coords[2] = elem_coords[2]/num_nodes_in_elem; - - // default is not to fill the element - bool fill_this = mat_fill(f_id).volume.contains(elem_coords); - - // paint the material state on the element - if (fill_this){ - - // density - elem_den(elem_gid) = mat_fill(f_id).den; - - // mass - elem_mass(elem_gid) = elem_den(elem_gid)*elem_vol(elem_gid); - - // specific internal energy - elem_sie(rk_level, elem_gid) = mat_fill(f_id).sie; - - size_t mat_id = elem_mat_id(elem_gid); // short name - - // --- stress tensor --- - // always 3D even for 2D-RZ - for (size_t i=0; i<3; i++){ - for (size_t j=0; j<3; j++){ - elem_stress(rk_level,elem_gid,i,j) = 0.0; - } - } // end for - - // short form for clean code - EOSParent * eos_model = elem_eos(elem_gid).model; - // --- Pressure --- - eos_model->calc_pressure(elem_pres, - elem_stress, - elem_gid, - elem_mat_id(elem_gid), - state_vars, - global_vars, - elem_user_output_vars, - elem_sspd, - elem_den(elem_gid), - elem_sie(rk_level,elem_gid)); - - // --- Sound speed --- - eos_model->calc_sound_speed(elem_pres, - elem_stress, - elem_gid, - elem_mat_id(elem_gid), - state_vars, - global_vars, - elem_user_output_vars, - elem_sspd, - elem_den(elem_gid), - elem_sie(rk_level,elem_gid)); - - // loop over the nodes of this element and apply velocity - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - - // get the mesh node index - size_t node_gid = nodes_in_elem(elem_gid, node_lid); - - - // --- Velocity --- - switch(mat_fill(f_id).velocity) - { - case VELOCITY_TYPE::cartesian: - { - - node_vel(rk_level, node_gid, 0) = mat_fill(f_id).u; - node_vel(rk_level, node_gid, 1) = mat_fill(f_id).v; - if (num_dim == 3) node_vel(rk_level, node_gid, 2) = mat_fill(f_id).w; - - - break; - } - case VELOCITY_TYPE::radial: - { - // Setting up cylindrical - double dir[2]; - dir[0] = 0.0; - dir[1] = 0.0; - double radius_val = 0.0; - - for(int dim=0; dim<2; dim++){ - dir[dim] = node_coords(rk_level, node_gid, dim); - radius_val += node_coords(rk_level, node_gid, dim)*node_coords(rk_level, node_gid, dim); - } // end for - radius_val = sqrt(radius_val); - - for(int dim=0; dim<2; dim++){ - if (radius_val > 1.0e-14){ - dir[dim] /= (radius_val); - } - else{ - dir[dim] = 0.0; - } - } // end for - - - node_vel(rk_level, node_gid, 0) = mat_fill(f_id).speed*dir[0]; - node_vel(rk_level, node_gid, 1) = mat_fill(f_id).speed*dir[1]; - if (num_dim == 3) node_vel(rk_level, node_gid, 2) = 0.0; - - break; - } - case VELOCITY_TYPE::spherical: - { - - // Setting up spherical - double dir[3]; - dir[0] = 0.0; - dir[1] = 0.0; - dir[2] = 0.0; - double radius_val = 0.0; - - for(int dim=0; dim<3; dim++){ - dir[dim] = node_coords(rk_level, node_gid, dim); - radius_val += node_coords(rk_level, node_gid, dim)*node_coords(rk_level, node_gid, dim); - } // end for - radius_val = sqrt(radius_val); - - for(int dim=0; dim<3; dim++){ - if (radius_val > 1.0e-14){ - dir[dim] /= (radius_val); - } - else{ - dir[dim] = 0.0; - } - } // end for - - node_vel(rk_level, node_gid, 0) = mat_fill(f_id).speed*dir[0]; - node_vel(rk_level, node_gid, 1) = mat_fill(f_id).speed*dir[1]; - if (num_dim == 3) node_vel(rk_level, node_gid, 2) = mat_fill(f_id).speed*dir[2]; - - break; - } - case VELOCITY_TYPE::radial_linear: - { - - break; - } - case VELOCITY_TYPE::spherical_linear: - { - - break; - } - case VELOCITY_TYPE::tg_vortex: - { - - node_vel(rk_level, node_gid, 0) = sin(PI * node_coords(rk_level,node_gid, 0)) * cos(PI * node_coords(rk_level,node_gid, 1)); - node_vel(rk_level, node_gid, 1) = -1.0*cos(PI * node_coords(rk_level,node_gid, 0)) * sin(PI * node_coords(rk_level,node_gid, 1)); - if (num_dim == 3) node_vel(rk_level, node_gid, 2) = 0.0; - - break; - } - } // end of switch - - }// end loop over nodes of element - - - if(mat_fill(f_id).velocity == VELOCITY_TYPE::tg_vortex) - { - elem_pres(elem_gid) = 0.25*( cos(2.0*PI*elem_coords[0]) + cos(2.0*PI*elem_coords[1]) ) + 1.0; - - // p = rho*ie*(gamma - 1) - size_t mat_id = f_id; - double gamma = global_vars(mat_id,0); // gamma value - elem_sie(rk_level, elem_gid) = - elem_pres(elem_gid)/(mat_fill(f_id).den*(gamma - 1.0)); - } // end if - - } // end if fill - - }); // end FOR_ALL_CLASS element loop - Kokkos::fence(); - - - } // end for loop over fills - - - // apply BC's to velocity - FEA_Module_SGH::boundary_velocity(*mesh, boundary, node_vel); - - - // calculate the corner massess if 2D - if(num_dim==2){ - - FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { - - // facial area of the corners - double corner_areas_array[4]; - - ViewCArrayKokkos corner_areas(&corner_areas_array[0],4); - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 4); - - get_area_weights2D(corner_areas, - elem_gid, - node_coords, - elem_node_gids, - rk_level); - - // loop over the corners of the element and calculate the mass - for (size_t corner_lid=0; corner_lid<4; corner_lid++){ - - size_t corner_gid = corners_in_elem(elem_gid, corner_lid); - corner_mass(corner_gid) = corner_areas(corner_lid)*elem_den(elem_gid); // node radius is added later - - } // end for over corners - }); - - } // end of - - - // calculate the nodal mass - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - - node_mass(node_gid) = 0.0; - - if(num_dim==3){ - - for(size_t elem_lid=0; elem_lidtopology_optimization_on||simparam->shape_optimization_on||simparam->num_dims==2){ - { - vec_array node_mass_interface = node_masses_distributed->getLocalView (Tpetra::Access::ReadWrite); - - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - node_mass_interface(node_gid,0) = node_mass(node_gid); - }); // end parallel for - } //end view scope - Kokkos::fence(); - //communicate ghost densities - comm_node_masses(); - - //this is forcing a copy to the device - //view scope - { - vec_array ghost_node_mass_interface = ghost_node_masses_distributed->getLocalView (Tpetra::Access::ReadWrite); - - - FOR_ALL_CLASS(node_gid, nlocal_nodes, nall_nodes, { - node_mass(node_gid) = ghost_node_mass_interface(node_gid-nlocal_nodes,0); - }); // end parallel for - } //end view scope - Kokkos::fence(); - } //endif - - //initialize if topology optimization is used - if(simparam->topology_optimization_on || simparam->shape_optimization_on){ - init_assembly(); - assemble_matrix(); - } - - // update host copies of arrays modified in this function - elem_den.update_host(); - elem_mass.update_host(); - elem_sie.update_host(); - elem_stress.update_host(); - elem_pres.update_host(); - elem_sspd.update_host(); - - return; - -} // end of setup - -/* ---------------------------------------------------------------------------- - solve function called by solver -------------------------------------------------------------------------------- */ - -void FEA_Module_SGH::module_cleanup(){ - cleanup_material_models(); -} - -/* ---------------------------------------------------------------------------- - Deallocate memory used for material models -------------------------------------------------------------------------------- */ - -void FEA_Module_SGH::cleanup_material_models() { - - const DCArrayKokkos material = simparam->material; - - // destroy strength model - destroy_strength_model(elem_strength, - material, - elem_mat_id, - state_vars, - global_vars, - elem_user_output_vars, - rnum_elem); - - // destroy eos model - destroy_eos_model(elem_eos, - material, - elem_mat_id, - state_vars, - global_vars, - elem_user_output_vars, - rnum_elem); - return; - -} // end cleanup_user_strength_model; - - -/** - * Determines which of the boundary patches are associated with which boundary. - * - * Modifies: bdy_patches_in_set -*/ -void FEA_Module_SGH::tag_bdys(const DCArrayKokkos &boundary, - mesh_t &mesh, - const DViewCArrayKokkos &node_coords){ - - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - size_t num_dim = simparam->num_dims; - int nboundary_patches = Explicit_Solver_Pointer_->nboundary_patches; - int num_nodes_in_patch = mesh.num_nodes_in_patch; - - //if (bdy_set == mesh.num_bdy_sets){ - // printf(" ERROR: number of boundary sets must be increased by %zu", - // bdy_set-mesh.num_bdy_sets+1); - // exit(0); - //} // end if - - //error and debug flag - //DCArrayKokkos print_flag(1, "print_flag"); - //print_flag.host(0) = false; - //print_flag.update_device(); - - - FOR_ALL_CLASS(bdy_set, 0, num_bdy_sets, { - - // tag boundaries - BOUNDARY_TYPE bc_type = boundary(bdy_set).surface.type; - double val = boundary(bdy_set).surface.plane_position; - - // save the boundary patches to this set that are on the plane, spheres, etc. - for (size_t bdy_patch_lid=0; bdy_patch_lid < nboundary_patches; bdy_patch_lid++){ - - // save the patch index - size_t bdy_patch_gid = bdy_patch_lid; - - - // check to see if this patch is on the specified plane - bool is_on_bdy = check_bdy(bdy_patch_gid, - num_dim, - num_nodes_in_patch, - bc_type, - val, - node_coords, - rk_level); // no=0, yes=1 - - //debug check - /* - for (size_t patch_node_lid=0; patch_node_lid &node_coords, - const size_t rk_level) const { - - // default bool is not on the boundary - size_t is_on_bdy = 0; - - // the patch coordinates - double these_patch_coords[3]; // Note: cannot allocated array with num_dim - - // loop over the nodes on the patch - for (size_t patch_node_lid=0; patch_node_lid < num_nodes_in_patch; patch_node_lid++) { - // get the nodal_gid for this node in the patch - //size_t node_gid = mesh.nodes_in_patch(patch_gid, patch_node_lid); - size_t node_gid = Local_Index_Boundary_Patches(patch_gid, patch_node_lid); - - for (size_t dim = 0; dim < num_dim; dim++){ - these_patch_coords[dim] = node_coords(rk_level, node_gid, dim); // (rk, node_gid, dim) - } - - if (bc_type == BOUNDARY_TYPE::x_plane) { - if ( fabs(these_patch_coords[0] - val) <= 1.0e-7 ) is_on_bdy += 1; - } - else if (bc_type == BOUNDARY_TYPE::y_plane) { - if ( fabs(these_patch_coords[1] - val) <= 1.0e-7 ) is_on_bdy += 1; - } - else if (bc_type == BOUNDARY_TYPE::z_plane) { - if ( fabs(these_patch_coords[2] - val) <= 1.0e-7 ) is_on_bdy += 1; - } - else if (bc_type == BOUNDARY_TYPE::cylinder){ - real_t R = sqrt(these_patch_coords[0]*these_patch_coords[0] + - these_patch_coords[1]*these_patch_coords[1]); - - if ( fabs(R - val) <= 1.0e-7 ) is_on_bdy += 1; - } - else if (bc_type == BOUNDARY_TYPE::sphere) { - real_t R = sqrt(these_patch_coords[0]*these_patch_coords[0] + - these_patch_coords[1]*these_patch_coords[1] + - these_patch_coords[2]*these_patch_coords[2]); - - if ( fabs(R - val) <= 1.0e-7 ) is_on_bdy += 1; - } - } - - // if all nodes in the patch are on the surface - return is_on_bdy == num_nodes_in_patch; -} // end method to check bdy - -/* ---------------------------------------------------------------------------- - Build set of nodes assigned to each boundary condition -------------------------------------------------------------------------------- */ - -void FEA_Module_SGH::build_boundry_node_sets(mesh_t &mesh){ - - // build boundary nodes in each boundary set - int nboundary_patches = Explicit_Solver_Pointer_->nboundary_patches; - int num_nodes_in_patch = mesh.num_nodes_in_patch; - num_bdy_nodes_in_set = mesh.num_bdy_nodes_in_set = DCArrayKokkos (num_bdy_sets, "num_bdy_nodes_in_set"); - CArrayKokkos temp_count_num_bdy_nodes_in_set(num_bdy_sets, nall_nodes, "temp_count_num_bdy_nodes_in_set"); - - DynamicRaggedRightArrayKokkos temp_nodes_in_set (mesh.num_bdy_sets, nboundary_patches*mesh.num_nodes_in_patch, "temp_nodes_in_set"); - - // Parallel loop over boundary sets on device - FOR_ALL_CLASS(bdy_set, 0, num_bdy_sets, { - - // finde the number of patches_in_set - size_t num_bdy_patches_in_set = bdy_patches_in_set.stride(bdy_set); - - num_bdy_nodes_in_set(bdy_set) = 0; - - // Loop over boundary patches in boundary set - for (size_t bdy_patch_gid = 0; bdy_patch_gid (mesh.num_bdy_nodes_in_set, "bdy_nodes_in_set"); - - FOR_ALL_CLASS (bdy_set, 0, num_bdy_sets, { - - // Loop over boundary patches in boundary set - for (size_t bdy_node_lid=0; bdy_node_liddynamic_options; - - const size_t rk_level = dynamic_options.rk_num_bins - 1; - time_value = dynamic_options.time_initial; - time_final = dynamic_options.time_final; - dt_max = dynamic_options.dt_max; - dt_min = dynamic_options.dt_min; - dt_cfl = dynamic_options.dt_cfl; - graphics_time = simparam->output_options.graphics_step; - graphics_dt_ival = simparam->output_options.graphics_step; - cycle_stop = dynamic_options.cycle_stop; - rk_num_stages = dynamic_options.rk_num_stages; - dt = dynamic_options.dt; - fuzz = dynamic_options.fuzz; - tiny = dynamic_options.tiny; - small = dynamic_options.small; - graphics_times = simparam->output_options.graphics_times; - graphics_id = simparam->output_options.graphics_id; - size_t num_bdy_nodes = mesh->num_bdy_nodes; - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos material = simparam->material; - int nTO_modules; - int old_max_forward_buffer; - size_t cycle; - const int num_dim = simparam->num_dims; - real_t objective_accumulation, global_objective_accumulation; - std::vector> FEA_Module_My_TO_Modules = simparam->FEA_Module_My_TO_Modules; - problem = Explicit_Solver_Pointer_->problem; //Pointer to ROL optimization problem object - ROL::Ptr> obj_pointer; - - //reset time accumulating objective and constraints - /* - for(int imodule = 0 ; imodule < FEA_Module_My_TO_Modules[my_fea_module_index_].size(); imodule++){ - current_module_index = FEA_Module_My_TO_Modules[my_fea_module_index_][imodule]; - //test if module needs reset - if(){ - - } - } - */ - //simple setup to just request KE for now; above loop to be expanded and used later for scanning modules - if(simparam->topology_optimization_on){ - obj_pointer = problem->getObjective(); - KineticEnergyMinimize_TopOpt& kinetic_energy_minimize_function = dynamic_cast(*obj_pointer); - kinetic_energy_minimize_function.objective_accumulation = 0; - global_objective_accumulation = objective_accumulation = 0; - kinetic_energy_objective = true; - if(max_time_steps +1 > forward_solve_velocity_data->size()){ - old_max_forward_buffer = forward_solve_velocity_data->size(); - time_data.resize(max_time_steps+1); - forward_solve_velocity_data->resize(max_time_steps+1); - forward_solve_coordinate_data->resize(max_time_steps+1); - forward_solve_internal_energy_data->resize(max_time_steps+1); - adjoint_vector_data->resize(max_time_steps+1); - phi_adjoint_vector_data->resize(max_time_steps+1); - psi_adjoint_vector_data->resize(max_time_steps+1); - //assign a multivector of corresponding size to each new timestep in the buffer - for(int istep = old_max_forward_buffer; istep < max_time_steps+1; istep++){ - (*forward_solve_velocity_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - (*forward_solve_coordinate_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - (*forward_solve_internal_energy_data)[istep] = Teuchos::rcp(new MV(all_element_map, 1)); - (*adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - (*phi_adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - (*psi_adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_element_map, 1)); - } - } - } - - if(simparam->topology_optimization_on) - nTO_modules = simparam->TO_Module_List.size(); - - int myrank = Explicit_Solver_Pointer_->myrank; - if(simparam->output_options.output_file_format==OUTPUT_FORMAT::vtk&&simparam->output_options.write_initial) - { - if(myrank==0) - printf("Writing outputs to file at %f \n", time_value); - - double comm_time1 = Explicit_Solver_Pointer_->CPU_Time(); - Explicit_Solver_Pointer_->write_outputs(); - double comm_time2 = Explicit_Solver_Pointer_->CPU_Time(); - Explicit_Solver_Pointer_->output_time += comm_time2 - comm_time1; - } - - CArrayKokkos node_extensive_mass(nall_nodes, "node_extensive_mass"); - - // extensive energy tallies over the mesh elements local to this MPI rank - double IE_t0 = 0.0; - double KE_t0 = 0.0; - double TE_t0 = 0.0; - - double IE_sum = 0.0; - double KE_sum = 0.0; - - double IE_loc_sum = 0.0; - double KE_loc_sum = 0.0; - - // extensive energy tallies over the entire mesh - double global_IE_t0 = 0.0; - double global_KE_t0 = 0.0; - double global_TE_t0 = 0.0; - - // ---- Calculate energy tallies ---- - double IE_tend = 0.0; - double KE_tend = 0.0; - double TE_tend = 0.0; - - double global_IE_tend = 0.0; - double global_KE_tend = 0.0; - double global_TE_tend = 0.0; - - int nlocal_elem_non_overlapping = Explicit_Solver_Pointer_->nlocal_elem_non_overlapping; - - // extensive IE - REDUCE_SUM_CLASS(elem_gid, 0, nlocal_elem_non_overlapping, IE_loc_sum, { - IE_loc_sum += elem_mass(elem_gid)*elem_sie(rk_level,elem_gid); - - }, IE_sum); - IE_t0 = IE_sum; - - MPI_Allreduce(&IE_t0,&global_IE_t0,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD); - - // extensive KE - REDUCE_SUM_CLASS(node_gid, 0, nlocal_nodes, KE_loc_sum, { - - double ke = 0; - for (size_t dim=0; dimtopology_optimization_on||simparam->shape_optimization_on){ - time_data[0] = 0; - //assign current velocity data to multivector - //view scope - { - vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView (Tpetra::Access::ReadWrite); - vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - node_velocities_interface(node_gid,idim) = node_vel(rk_level,node_gid,idim); - node_coords_interface(node_gid,idim) = node_coords(rk_level,node_gid,idim); - } - }); - } //end view scope - Kokkos::fence(); - - //communicate ghosts - double comm_time1 = Explicit_Solver_Pointer_->CPU_Time(); - - //active view scope; triggers host comms from updated data on device - { - const_host_vec_array node_velocities_host = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_host_vec_array node_coords_host = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - } - double comm_time2 = Explicit_Solver_Pointer_->CPU_Time(); - Explicit_Solver_Pointer_->dev2host_time += comm_time2-comm_time1; - - //communicate ghost velocities - Explicit_Solver_Pointer_->comm_velocities(); - Explicit_Solver_Pointer_->comm_coordinates(); - - - double comm_time3 = Explicit_Solver_Pointer_->CPU_Time(); - - //view scope - { - const_vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array ghost_node_velocities_interface = Explicit_Solver_Pointer_->ghost_node_velocities_distributed->getLocalView (Tpetra::Access::ReadOnly); - vec_array all_node_velocities_interface = Explicit_Solver_Pointer_->all_node_velocities_distributed->getLocalView (Tpetra::Access::ReadWrite); - vec_array element_internal_energy = element_internal_energy_distributed->getLocalView (Tpetra::Access::ReadWrite); - const_vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array ghost_node_coords_interface = Explicit_Solver_Pointer_->ghost_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - all_node_velocities_interface(node_gid,idim) = node_velocities_interface(node_gid,idim); - all_node_coords_interface(node_gid,idim) = node_coords_interface(node_gid,idim); - } - }); // end parallel for - Kokkos::fence(); - - FOR_ALL_CLASS(node_gid, nlocal_nodes, nlocal_nodes+nghost_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - all_node_velocities_interface(node_gid,idim) = ghost_node_velocities_interface(node_gid-nlocal_nodes,idim); - all_node_coords_interface(node_gid,idim) = ghost_node_coords_interface(node_gid-nlocal_nodes,idim); - } - }); // end parallel for - Kokkos::fence(); - - - //interface for element internal energies - FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { - element_internal_energy(elem_gid,0) = elem_sie(rk_level,elem_gid); - }); // end parallel for - Kokkos::fence(); - } //end view scope - - (*forward_solve_internal_energy_data)[0]->assign(*element_internal_energy_distributed); - (*forward_solve_velocity_data)[0]->assign(*Explicit_Solver_Pointer_->all_node_velocities_distributed); - (*forward_solve_coordinate_data)[0]->assign(*Explicit_Solver_Pointer_->all_node_coords_distributed); - } - - // loop over the max number of time integration cycles - for (cycle = 0; cycle < cycle_stop; cycle++) { - - // get the step - if(num_dim==2){ - get_timestep2D(*mesh, - node_coords, - node_vel, - elem_sspd, - elem_vol); - } - else { - get_timestep(*mesh, - node_coords, - node_vel, - elem_sspd, - elem_vol); - } // end if 2D - - double global_dt; - MPI_Allreduce(&dt,&global_dt,1,MPI_DOUBLE,MPI_MIN,MPI_COMM_WORLD); - dt = global_dt; - - // stop calculation if flag - //if (stop_calc == 1) break; - - - if(simparam->dynamic_options.output_time_sequence_level>=TIME_OUTPUT_LEVEL::high){ - if (cycle==0){ - if(myrank==0) - printf("cycle = %lu, time = %12.5e, time step = %12.5e \n", cycle, time_value, dt); - } - // print time step every 10 cycles - else if (cycle%20==0){ - if(myrank==0) - printf("cycle = %lu, time = %12.5e, time step = %12.5e \n", cycle, time_value, dt); - } // end if - } - - // --------------------------------------------------------------------- - // integrate the solution forward to t(n+1) via Runge Kutta (RK) method - // --------------------------------------------------------------------- - - // save the values at t_n - rk_init(node_coords, - node_vel, - elem_sie, - elem_stress, - rnum_elem, - nall_nodes); - - - - // integrate solution forward in time - for (size_t rk_stage = 0; rk_stage < rk_num_stages; rk_stage++){ - - - // ---- RK coefficient ---- - double rk_alpha = 1.0/((double)rk_num_stages - (double)rk_stage); - - // ---- Calculate velocity diveregence for the element ---- - if(num_dim==2){ - get_divergence2D(elem_div, - node_coords, - node_vel, - elem_vol); - } - else { - get_divergence(elem_div, - node_coords, - node_vel, - elem_vol); - } // end if 2D - - // ---- calculate the forces on the vertices and evolve stress (hypo model) ---- - if(num_dim==2){ - get_force_sgh2D(material, - *mesh, - node_coords, - node_vel, - elem_den, - elem_sie, - elem_pres, - elem_stress, - elem_sspd, - elem_vol, - elem_div, - elem_mat_id, - corner_force, - rk_alpha, - cycle); - } - else { - get_force_sgh(material, - *mesh, - node_coords, - node_vel, - elem_den, - elem_sie, - elem_pres, - elem_stress, - elem_sspd, - elem_vol, - elem_div, - elem_mat_id, - corner_force, - rk_alpha, - cycle); - } - - /* - debug block - if(myrank==1){ - std::cout << rk_alpha << " " << dt << std::endl; - for(int i = 0; i < nall_nodes; i++){ - double node_force[3]; - for (size_t dim = 0; dim < num_dim; dim++){ - node_force[dim] = 0.0; - } // end for dim - - // loop over all corners around the node and calculate the nodal force - for (size_t corner_lid=0; corner_lidall_node_map->getGlobalElement(i) << " " << corner_gid << " " << corner_force(corner_gid, 0) << " " << corner_force(corner_gid, 1) << " " << corner_force(corner_gid, 2) << std::endl; - // loop over dimension - for (size_t dim = 0; dim < num_dim; dim++){ - node_force[dim] += corner_force(corner_gid, dim); - } // end for dim - - } // end for corner_lid - //std::cout << Explicit_Solver_Pointer_->all_node_map->getGlobalElement(i) << " " << node_force[0] << " " << node_force[1] << " " << node_force[2] << std::endl; - //std::cout << Explicit_Solver_Pointer_->all_node_map->getGlobalElement(i) << " " << node_mass(i) << std::endl; - } - } - /* - //debug print vector values on a rank - /* - if(myrank==0) - for(int i = 0; i < nall_nodes; i++){ - std::cout << Explicit_Solver_Pointer_->all_node_map->getGlobalElement(i) << " " << node_vel(rk_level,i,0) << " " << node_vel(rk_level,i,1) << " " << node_vel(rk_level,i,2) << std::endl; - } - */ - - // ---- Update nodal velocities ---- // - update_velocity_sgh(rk_alpha, - node_vel, - node_mass, - corner_force); - - if(have_loading_conditions){ - applied_forces(material, - *mesh, - node_coords, - node_vel, - node_mass, - elem_den, - elem_vol, - elem_div, - elem_mat_id, - corner_force, - rk_alpha, - cycle); - } - - // ---- apply force boundary conditions to the boundary patches---- - boundary_velocity(*mesh, boundary, node_vel); - - //current interface has differing velocity arrays; this equates them until we unify memory - //first comm time interval point - double comm_time1 = Explicit_Solver_Pointer_->CPU_Time(); - //view scope - { - vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - node_velocities_interface(node_gid,idim) = node_vel(rk_level,node_gid,idim); - } - }); // end parallel for - } //end view scope - Kokkos::fence(); - - //active view scope - { - const_host_vec_array node_velocities_host = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView (Tpetra::Access::ReadOnly); - } - double comm_time2 = Explicit_Solver_Pointer_->CPU_Time(); - Explicit_Solver_Pointer_->dev2host_time += comm_time2-comm_time1; - //communicate ghost velocities - Explicit_Solver_Pointer_->comm_velocities(); - - double comm_time3 = Explicit_Solver_Pointer_->CPU_Time(); - //this is forcing a copy to the device - //view scope - { - vec_array ghost_node_velocities_interface = Explicit_Solver_Pointer_->ghost_node_velocities_distributed->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, nlocal_nodes, nall_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - node_vel(rk_level,node_gid,idim) = ghost_node_velocities_interface(node_gid-nlocal_nodes,idim); - } - - }); // end parallel for - } //end view scope - Kokkos::fence(); - - double comm_time4 = Explicit_Solver_Pointer_->CPU_Time(); - Explicit_Solver_Pointer_->host2dev_time += comm_time4-comm_time3; - Explicit_Solver_Pointer_->communication_time += comm_time4-comm_time1; - //debug print vector values on a rank - /* - if(myrank==0) - for(int i = 0; i < nall_nodes; i++){ - std::cout << Explicit_Solver_Pointer_->all_node_map->getGlobalElement(i) << " " << node_vel(rk_level,i,0) << " " << node_vel(rk_level,i,1) << " " << node_vel(rk_level,i,2) << std::endl; - } - */ - // ---- Update specific internal energy in the elements ---- - update_energy_sgh(rk_alpha, - *mesh, - node_vel, - node_coords, - elem_sie, - elem_mass, - corner_force); - - - // ---- Update nodal positions ---- - update_position_sgh(rk_alpha, - nall_nodes, - node_coords, - node_vel); - - - // ---- Calculate cell volume for next time step ---- - get_vol(); - - - - // ---- Calculate elem state (den, pres, sound speed, stress) for next time step ---- - if(num_dim==2){ - update_state2D(material, - *mesh, - node_coords, - node_vel, - elem_den, - elem_pres, - elem_stress, - elem_sspd, - elem_sie, - elem_vol, - elem_mass, - elem_mat_id, - rk_alpha, - cycle); - } - else{ - update_state(material, - *mesh, - node_coords, - node_vel, - elem_den, - elem_pres, - elem_stress, - elem_sspd, - elem_sie, - elem_vol, - elem_mass, - elem_mat_id, - rk_alpha, - cycle); - } - // ---- - // Notes on strength: - // 1) hyper-elastic strength models are called in update_state - // 2) hypo-elastic strength models are called in get_force - // 3) strength models must be added by the user in user_mat.cpp - - - // calculate the new corner masses if 2D - if(num_dim==2){ - - // calculate the nodal areal mass - FOR_ALL_CLASS(node_gid, 0, nall_nodes, { - - node_mass(node_gid) = 0.0; - - if (node_coords(rk_level,node_gid,1) > tiny){ - node_mass(node_gid) = node_extensive_mass(node_gid)/node_coords(rk_level,node_gid,1); - } - //if(cycle==0&&node_gid==1&&myrank==0) - //std::cout << "index " << node_gid << " on rank " << myrank << " node vel " << node_vel(rk_level,node_gid,0) << " " << node_mass(node_gid) << std::endl << std::flush; - - }); // end parallel for over node_gid - Kokkos::fence(); - - //current interface has differing density arrays; this equates them until we unify memory - //view scope - { - vec_array node_mass_interface = node_masses_distributed->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - node_mass_interface(node_gid,0) = node_mass(node_gid); - }); // end parallel for - } //end view scope - Kokkos::fence(); - //communicate ghost densities - comm_node_masses(); - - //this is forcing a copy to the device - //view scope - { - vec_array ghost_node_mass_interface = ghost_node_masses_distributed->getLocalView (Tpetra::Access::ReadWrite); - - FOR_ALL_CLASS(node_gid, nlocal_nodes, nall_nodes, { - node_mass(node_gid) = ghost_node_mass_interface(node_gid-nlocal_nodes,0); - }); // end parallel for - } //end view scope - Kokkos::fence(); - - - // ----------------------------------------------- - // Calcualte the areal mass for nodes on the axis - // ----------------------------------------------- - // The node order of the 2D element is - // - // J - // | - // 3---2 - // | | -- I - // 0---1 - /* - FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { - - // loop over the corners of the element and calculate the mass - for (size_t node_lid=0; node_lid<4; node_lid++){ - - size_t node_gid = nodes_in_elem(elem_gid, node_lid); - size_t node_minus_gid; - size_t node_plus_gid; - - - if (node_coords(rk_level,node_gid,1) < tiny){ - // node is on the axis - - // minus node - if (node_lid==0){ - node_minus_gid = nodes_in_elem(elem_gid, 3); - } else { - node_minus_gid = nodes_in_elem(elem_gid, node_lid-1); - } - - // plus node - if (node_lid==3){ - node_plus_gid = nodes_in_elem(elem_gid, 0); - } else { - node_plus_gid = nodes_in_elem(elem_gid, node_lid+1); - } - - node_mass(node_gid) = fmax(node_mass(node_plus_gid), node_mass(node_minus_gid))/2.0; - - } // end if - - } // end for over corners - - }); // end parallel for over elem_gid - Kokkos::fence(); - */ - - FOR_ALL_CLASS(node_bdy_gid, 0, num_bdy_nodes, { - //FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - size_t node_gid = bdy_nodes(node_bdy_gid); - - if (node_coords(rk_level,node_gid,1) < tiny){ - // node is on the axis - - for(size_t node_lid=0; node_lid < num_nodes_in_node(node_gid); node_lid++){ - - size_t node_neighbor_gid = nodes_in_node(node_gid, node_lid); - - // if the node is off the axis, use it's areal mass on the boundary - if (node_coords(rk_level,node_neighbor_gid,1) > tiny){ - node_mass(node_gid) = fmax(node_mass(node_gid), node_mass(node_neighbor_gid)/2.0); - } - - } // end for over neighboring nodes - - } // end if - - }); // end parallel for over elem_gid - - } // end of if 2D-RZ - - - } // end of RK loop - - // increment the time - Explicit_Solver_Pointer_->time_value = simparam->dynamic_options.time_value = time_value+=dt; - - if(simparam->topology_optimization_on||simparam->shape_optimization_on){ - if(cycle >= max_time_steps) - max_time_steps = cycle + 1; - - if(max_time_steps + 1 > forward_solve_velocity_data->size()){ - old_max_forward_buffer = forward_solve_velocity_data->size(); - time_data.resize(max_time_steps + BUFFER_GROW +1); - forward_solve_velocity_data->resize(max_time_steps + BUFFER_GROW +1); - forward_solve_coordinate_data->resize(max_time_steps + BUFFER_GROW +1); - forward_solve_internal_energy_data->resize(max_time_steps + BUFFER_GROW +1); - adjoint_vector_data->resize(max_time_steps + BUFFER_GROW +1); - phi_adjoint_vector_data->resize(max_time_steps + BUFFER_GROW +1); - psi_adjoint_vector_data->resize(max_time_steps + BUFFER_GROW +1); - //assign a multivector of corresponding size to each new timestep in the buffer - for(int istep = old_max_forward_buffer; istep < max_time_steps + BUFFER_GROW +1; istep++){ - (*forward_solve_velocity_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - (*forward_solve_coordinate_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - (*forward_solve_internal_energy_data)[istep] = Teuchos::rcp(new MV(all_element_map, 1)); - (*adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - (*phi_adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - (*psi_adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_element_map, 1)); - } - } - - - time_data[cycle+1] = dt + time_data[cycle]; - - - //assign current velocity data to multivector - //view scope - { - vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView (Tpetra::Access::ReadWrite); - vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - node_velocities_interface(node_gid,idim) = node_vel(rk_level,node_gid,idim); - node_coords_interface(node_gid,idim) = node_coords(rk_level,node_gid,idim); - } - }); - } //end view scope - Kokkos::fence(); - - //communicate ghosts - double comm_time1 = Explicit_Solver_Pointer_->CPU_Time(); - - //active view scope; triggers host comms from updated data on device - { - const_host_vec_array node_velocities_host = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_host_vec_array node_coords_host = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - } - double comm_time2 = Explicit_Solver_Pointer_->CPU_Time(); - Explicit_Solver_Pointer_->dev2host_time += comm_time2-comm_time1; - - //communicate ghost velocities - Explicit_Solver_Pointer_->comm_velocities(); - Explicit_Solver_Pointer_->comm_coordinates(); - - - double comm_time3 = Explicit_Solver_Pointer_->CPU_Time(); - - //view scope - { - const_vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array ghost_node_velocities_interface = Explicit_Solver_Pointer_->ghost_node_velocities_distributed->getLocalView (Tpetra::Access::ReadOnly); - vec_array all_node_velocities_interface = Explicit_Solver_Pointer_->all_node_velocities_distributed->getLocalView (Tpetra::Access::ReadWrite); - vec_array element_internal_energy = element_internal_energy_distributed->getLocalView (Tpetra::Access::ReadWrite); - const_vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array ghost_node_coords_interface = Explicit_Solver_Pointer_->ghost_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - all_node_velocities_interface(node_gid,idim) = node_velocities_interface(node_gid,idim); - all_node_coords_interface(node_gid,idim) = node_coords_interface(node_gid,idim); - } - }); // end parallel for - Kokkos::fence(); - - FOR_ALL_CLASS(node_gid, nlocal_nodes, nlocal_nodes+nghost_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - all_node_velocities_interface(node_gid,idim) = ghost_node_velocities_interface(node_gid-nlocal_nodes,idim); - all_node_coords_interface(node_gid,idim) = ghost_node_coords_interface(node_gid-nlocal_nodes,idim); - } - }); // end parallel for - Kokkos::fence(); - - //interface for element internal energies - FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { - element_internal_energy(elem_gid,0) = elem_sie(rk_level,elem_gid); - }); // end parallel for - Kokkos::fence(); - } //end view scope - - double comm_time4 = Explicit_Solver_Pointer_->CPU_Time(); - Explicit_Solver_Pointer_->host2dev_time += comm_time4-comm_time3; - Explicit_Solver_Pointer_->communication_time += comm_time4-comm_time1; - - - (*forward_solve_internal_energy_data)[cycle+1]->assign(*element_internal_energy_distributed); - (*forward_solve_velocity_data)[cycle+1]->assign(*Explicit_Solver_Pointer_->all_node_velocities_distributed); - (*forward_solve_coordinate_data)[cycle+1]->assign(*Explicit_Solver_Pointer_->all_node_coords_distributed); - - //kinetic energy accumulation - if(kinetic_energy_objective){ - const_vec_array node_velocities_interface = (*forward_solve_velocity_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array previous_node_velocities_interface = (*forward_solve_velocity_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - KE_loc_sum = 0.0; - KE_sum = 0.0; - // extensive KE - REDUCE_SUM_CLASS(node_gid, 0, nlocal_nodes, KE_loc_sum, { - - double ke = 0; - for (size_t dim=0; dim0){ - write = 1; - } - else if (cycle == cycle_stop) { - write = 1; - } - else if (time_value >= time_final&&simparam->output_options.write_final){ - write = 1; - } - else if (time_value >= graphics_time){ - write = 1; - } - - // write outputs - if (write == 1){ - //interface nodal coordinate data (note: this is not needed if using write_outputs()) - //view scope - { - vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - node_coords_interface(node_gid,idim) = node_coords(rk_level,node_gid,idim); - } - }); // end parallel for - } //end view scope - if(simparam->output_options.output_file_format==OUTPUT_FORMAT::vtk){ - if(myrank==0){ - printf("Writing outputs to file at %f \n", graphics_time); - } - - double comm_time1 = Explicit_Solver_Pointer_->CPU_Time(); - Explicit_Solver_Pointer_->write_outputs(); - - double comm_time2 = Explicit_Solver_Pointer_->CPU_Time(); - Explicit_Solver_Pointer_->output_time += comm_time2 - comm_time1; - } - - graphics_time = time_value + graphics_dt_ival; - } // end if - - - // end of calculation - if (time_value>=time_final) break; - - - } // end for cycle loop - - last_time_step = cycle; - - //simple setup to just calculate KE minimize objective for now - if(simparam->topology_optimization_on){ - KineticEnergyMinimize_TopOpt& kinetic_energy_minimize_function = dynamic_cast(*obj_pointer); - - //collect local objective values - MPI_Allreduce(&objective_accumulation,&global_objective_accumulation,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD); - kinetic_energy_minimize_function.objective_accumulation = global_objective_accumulation; - - if(myrank==0) - std::cout << "CURRENT TIME INTEGRAL OF KINETIC ENERGY " << global_objective_accumulation << std::endl; - } - - - auto time_2 = std::chrono::high_resolution_clock::now(); - auto time_difference = time_2 - time_1; - //double calc_time = std::chrono::duration_cast(diff).count(); - double calc_time = std::chrono::duration_cast(time_difference).count(); - if(myrank==0) - printf("\nCalculation time in seconds: %f \n", calc_time*1e-09); - - IE_loc_sum = 0.0; - KE_loc_sum = 0.0; - IE_sum = 0.0; - KE_sum = 0.0; - - // extensive IE - REDUCE_SUM_CLASS(elem_gid, 0, nlocal_elem_non_overlapping, IE_loc_sum, { - - IE_loc_sum += elem_mass(elem_gid)*elem_sie(rk_level,elem_gid); - - }, IE_sum); - IE_tend = IE_sum; - - //reduce over MPI ranks - MPI_Allreduce(&IE_tend,&global_IE_tend,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD); - - // extensive KE - REDUCE_SUM_CLASS(node_gid, 0, nlocal_nodes, KE_loc_sum, { - - double ke = 0; - for (size_t dim=0; dim mesh_in, const int my_fea_module_index = 0); - ~FEA_Module_SGH(); - - //initialize data for boundaries of the model and storage for boundary conditions and applied loads - void sgh_interface_setup(node_t &node, elem_t &elem, corner_t &corner); - - void setup(); - - void cleanup_material_models(); - - int solve(); - - void module_cleanup(); - - void sgh_solve(); - - void get_force_sgh(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_pres, - DViewCArrayKokkos &elem_stress, - const DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &elem_mat_id, - DViewCArrayKokkos &corner_force, - const double rk_alpha, - const size_t cycle); - - void get_force_vgradient_sgh(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &elem_mat_id, - const double rk_alpha, - const size_t cycle); - - void get_force_ugradient_sgh(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &elem_mat_id, - const double rk_alpha, - const size_t cycle); - - void get_force_egradient_sgh(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &elem_mat_id, - const double rk_alpha, - const size_t cycle); - - void get_force_dgradient_sgh(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &elem_mat_id, - const double rk_alpha, - const size_t cycle); - - void force_design_gradient_term(const_vec_array design_variables, vec_array design_gradients); - - - void get_force_sgh2D(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &elem_mat_id, - DViewCArrayKokkos &corner_force, - const double rk_alpha, - const size_t cycle); - - void update_position_sgh(double rk_alpha, - const size_t num_nodes, - DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel); - - void get_vol(); - - void get_vol_ugradient(const size_t gradient_node_id, const size_t gradient_dim); - - void init_assembly(); - - KOKKOS_INLINE_FUNCTION - void get_vol_hex(const DViewCArrayKokkos &elem_vol, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const; - - KOKKOS_FUNCTION - void get_vol_hex_ugradient(const ViewCArrayKokkos &elem_vol_gradients, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const; - - - KOKKOS_INLINE_FUNCTION - void get_vol_quad(const DViewCArrayKokkos &elem_vol, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const; - - - KOKKOS_FUNCTION - double get_area_quad(const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const; - - - KOKKOS_FUNCTION - void get_bmatrix(const ViewCArrayKokkos &B_matrix, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const; - - KOKKOS_FUNCTION - void get_bmatrix_gradients(const ViewCArrayKokkos &B_matrix_gradients, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const; - - - KOKKOS_FUNCTION - void get_bmatrix2D(const ViewCArrayKokkos &B_matrix, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const; - - KOKKOS_FUNCTION - void get_area_weights2D(const ViewCArrayKokkos &corner_areas, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const; - - - KOKKOS_INLINE_FUNCTION - double heron(const double x1, - const double y1, - const double x2, - const double y2, - const double x3, - const double y3) const; - - double average_element_density(const int nodes_per_elem, const CArray current_element_densities) const; - - void get_divergence(DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_vol); - - - void get_divergence2D(DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_vol); - - - KOKKOS_FUNCTION - void get_velgrad(ViewCArrayKokkos &vel_grad, - const ViewCArrayKokkos &elem_node_gids, - const DViewCArrayKokkos &node_vel, - const ViewCArrayKokkos &b_matrix, - const double elem_vol, - const size_t elem_gid, - const size_t rk_level) const; - - - KOKKOS_FUNCTION - void get_velgrad2D(ViewCArrayKokkos &vel_grad, - const ViewCArrayKokkos &elem_node_gids, - const DViewCArrayKokkos &node_vel, - const ViewCArrayKokkos &b_matrix, - const double elem_vol, - const double elem_area, - const size_t elem_gid, - const size_t rk_level) const; - - KOKKOS_INLINE_FUNCTION - void decompose_vel_grad(ViewCArrayKokkos &D_tensor, - ViewCArrayKokkos &W_tensor, - const ViewCArrayKokkos &vel_grad, - const ViewCArrayKokkos &elem_node_gids, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const double vol) const; - - - void update_velocity_sgh(double rk_alpha, - DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &node_mass, - const DViewCArrayKokkos &corner_force); - - void tag_bdys(const DCArrayKokkos &boundary, - mesh_t &mesh, - const DViewCArrayKokkos &node_coords); - - void boundary_velocity(const mesh_t &mesh, - const DCArrayKokkos &boundary, - DViewCArrayKokkos &node_vel); - - KOKKOS_INLINE_FUNCTION - bool check_bdy(const size_t patch_gid, - const int num_dim, - const int num_nodes_in_patch, - const BOUNDARY_TYPE this_bc_tag, - const double val, - const DViewCArrayKokkos &node_coords, - const size_t rk_level) const; - - void rk_init(DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &elem_sie, - DViewCArrayKokkos &elem_stress, - const size_t num_elems, - const size_t num_nodes); - - - void get_timestep(mesh_t &mesh, - DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &elem_sspd, - DViewCArrayKokkos &elem_vol); - - - void get_timestep2D(mesh_t &mesh, - DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &elem_sspd, - DViewCArrayKokkos &elem_vol); - - void update_energy_sgh(double rk_alpha, - const mesh_t &mesh, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_mass, - const DViewCArrayKokkos &corner_force); - - - void power_design_gradient_term(const_vec_array design_variables, vec_array design_gradients); - - void get_power_dgradient_sgh(double rk_alpha, - const mesh_t &mesh, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_mass, - const DViewCArrayKokkos &corner_force, - DCArrayKokkos elem_power_dgradients); - - void get_power_ugradient_sgh(double rk_alpha, - const mesh_t &mesh, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_mass, - const DViewCArrayKokkos &corner_force); - - void get_power_vgradient_sgh(double rk_alpha, - const mesh_t &mesh, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_mass, - const DViewCArrayKokkos &corner_force); - - void get_power_egradient_sgh(double rk_alpha, - const mesh_t &mesh, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_mass, - const DViewCArrayKokkos &corner_force); - - void update_state(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &elem_den, - DViewCArrayKokkos &elem_pres, - DViewCArrayKokkos &elem_stress, - DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_mass, - const DViewCArrayKokkos &elem_mat_id, - const double rk_alpha, - const size_t cycle); - - - void update_state2D(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &elem_den, - DViewCArrayKokkos &elem_pres, - DViewCArrayKokkos &elem_stress, - DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_mass, - const DViewCArrayKokkos &elem_mat_id, - const double rk_alpha, - const size_t cycle); - - void build_boundry_node_sets(mesh_t &mesh); - - void init_boundaries(); - - //initializes memory for arrays used in the global stiffness matrix assembly - void init_boundary_sets(int num_boundary_sets); - - void grow_boundary_sets(int num_boundary_sets); - - virtual void update_forward_solve(Teuchos::RCP zp); - - void comm_node_masses(); - - void comm_adjoint_vector(int cycle); - - void comm_phi_adjoint_vector(int cycle); - - void comm_variables(Teuchos::RCP zp); - - void read_conditions_ansys_dat(std::ifstream *in, std::streampos before_condition_header); - - //interfaces between user input and creating data structures for bcs - void generate_bcs(); - - void Displacement_Boundary_Conditions(); - - void init_output(); - - void compute_output(); - - void output_control(); - - void sort_output(Teuchos::RCP > sorted_map); - - void sort_element_output(Teuchos::RCP > sorted_map); - - void collect_output(Teuchos::RCP > global_reduce_map); - - void write_data(std::map &point_data_scalars_double, - std::map &point_data_vectors_double, - std::map &cell_data_scalars_double, - std::map &cell_data_scalars_int, - std::map > &cell_data_fields_double); - - void write_outputs (const mesh_t &mesh, - DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &node_mass, - DViewCArrayKokkos &elem_den, - DViewCArrayKokkos &elem_pres, - DViewCArrayKokkos &elem_stress, - DViewCArrayKokkos &elem_sspd, - DViewCArrayKokkos &elem_sie, - DViewCArrayKokkos &elem_vol, - DViewCArrayKokkos &elem_mass, - DViewCArrayKokkos &elem_mat_id); - - - void ensight(const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &node_mass, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_mass, - const DViewCArrayKokkos &elem_mat_id); - - - void state_file(const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &node_mass, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_mass, - const DViewCArrayKokkos &elem_mat_id); - - void node_density_constraints(host_vec_array node_densities_lower_bound); - - void compute_topology_optimization_adjoint_full(); //Force depends on node coords and velocity - - void compute_topology_optimization_gradient_full(Teuchos::RCP design_densities_distributed, Teuchos::RCP design_gradients_distributed); - - void boundary_adjoint(const mesh_t &mesh, - const DCArrayKokkos &boundary, - vec_array &node_adjoint, - vec_array &node_phi_adjoint, - vec_array &node_psi_adjoint); - - void applied_forces(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &node_mass, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &elem_mat_id, - DViewCArrayKokkos &corner_force, - const double rk_alpha, - const size_t cycle); - - bool have_loading_conditions; - bool nodal_density_flag; - real_t penalty_power; - - Simulation_Parameters_Explicit *simparam; - SGH_Parameters *module_params; - Explicit_Solver *Explicit_Solver_Pointer_; - - elements::ref_element *ref_elem; - - std::shared_ptr mesh; - //shallow copies of mesh class views - size_t num_nodes_in_elem; - // corner ids in node - RaggedRightArrayKokkos corners_in_node; - CArrayKokkos num_corners_in_node; - - // elem ids in node - RaggedRightArrayKokkos elems_in_node; - - // node ids in node - RaggedRightArrayKokkos nodes_in_node; - CArrayKokkos num_nodes_in_node; - - // node ids in elem - DCArrayKokkos nodes_in_elem; - - // corner ids in elem - CArrayKokkos corners_in_elem; - - // elem ids in elem - RaggedRightArrayKokkos elems_in_elem; - CArrayKokkos num_elems_in_elem; - - // patch ids in elem - CArrayKokkos patches_in_elem; - - // node ids in a patch - CArrayKokkos nodes_in_patch; - - // element ids in a patch - CArrayKokkos elems_in_patch; - - // bdy nodes - CArrayKokkos bdy_nodes; - - //Topology optimization filter variable - DCArrayKokkos relative_element_densities; - - //Local FEA data - host_elem_conn_array interface_nodes_in_elem; //host view of element connectivity to nodes - - //Global FEA data - Teuchos::RCP node_velocities_distributed; - Teuchos::RCP initial_node_coords_distributed; - Teuchos::RCP all_initial_node_coords_distributed; - Teuchos::RCP initial_node_velocities_distributed; - Teuchos::RCP all_node_velocities_distributed; - Teuchos::RCP all_cached_node_velocities_distributed; - Teuchos::RCP node_masses_distributed; - Teuchos::RCP ghost_node_masses_distributed; - Teuchos::RCP adjoint_vector_distributed; - Teuchos::RCP phi_adjoint_vector_distributed; - Teuchos::RCP psi_adjoint_vector_distributed; - Teuchos::RCP element_internal_energy_distributed; - Teuchos::RCP>> forward_solve_velocity_data; - Teuchos::RCP>> forward_solve_coordinate_data; - Teuchos::RCP>> forward_solve_internal_energy_data; - Teuchos::RCP>> adjoint_vector_data; - Teuchos::RCP>> phi_adjoint_vector_data; - Teuchos::RCP>> psi_adjoint_vector_data; - Teuchos::RCP force_gradient_design; - Teuchos::RCP force_gradient_position; - Teuchos::RCP force_gradient_velocity; - - //Local FEA data - DCArrayKokkos Global_Gradient_Matrix_Assembly_Map; //Maps element local nodes to columns on ragged right node connectivity graph - DCArrayKokkos Element_Gradient_Matrix_Assembly_Map; //Maps element-node pair to columns on ragged right node to element connectivity - RaggedRightArrayKokkos Graph_Matrix; //stores global indices - RaggedRightArrayKokkos DOF_Graph_Matrix; //stores global indices - RaggedRightArrayKokkos Force_Gradient_Positions; - RaggedRightArrayKokkos Force_Gradient_Velocities; - CArrayKokkos Force_Gradient_Energies; //transposed such that elem ids correspond to rows - RaggedRightArrayKokkos Power_Gradient_Positions; //transposed such that node dofs correspond to rows - RaggedRightArrayKokkos Power_Gradient_Velocities; //transposed such that node dofs correspond to rows - CArrayKokkos Power_Gradient_Energies; - DCArrayKokkos Gradient_Matrix_Strides; - DCArrayKokkos DOF_to_Elem_Matrix_Strides; - DCArrayKokkos Elem_to_Elem_Matrix_Strides; - DCArrayKokkos Graph_Matrix_Strides; - RaggedRightArrayKokkos Original_Gradient_Entries; - RaggedRightArrayKokkos Original_Gradient_Entry_Indices; - DCArrayKokkos Original_Gradient_Entries_Strides; - - //distributed matrices - Teuchos::RCP distributed_force_gradient_positions; - Teuchos::RCP distributed_force_gradient_velocities; - - std::vector time_data; - int max_time_steps, last_time_step; - - // --------------------------------------------------------------------- - // state data type declarations (must stay in scope for output after run) - // --------------------------------------------------------------------- - node_t node_interface; - elem_t elem_interface; - corner_t corner_interface; - - //Dual View wrappers - // Dual Views of the individual node struct variables - DViewCArrayKokkos node_coords; - DViewCArrayKokkos node_vel; - DViewCArrayKokkos node_mass; - - // Dual Views of the individual elem struct variables - DViewCArrayKokkos elem_den; - DViewCArrayKokkos elem_pres; - DViewCArrayKokkos elem_stress; // always 3D even in 2D-RZ - DViewCArrayKokkos elem_sspd; - DViewCArrayKokkos elem_sie; - DViewCArrayKokkos elem_vol; - DViewCArrayKokkos elem_div; - DViewCArrayKokkos elem_mass; - DViewCArrayKokkos elem_mat_id; - - // Element velocity gradient - DCArrayKokkos elem_vel_grad; - - // for storing global variables used in user material model - DCArrayKokkos global_vars; - - // for storing state variables used in user material model - DCArrayKokkos state_vars; - - //elem_user_output_vars allow users to output variables of interest per element - DCArrayKokkos elem_user_output_vars; - - //material models - DCArrayKokkos elem_eos; - DCArrayKokkos elem_strength; - - // Dual Views of the corner struct variables - DViewCArrayKokkos corner_force; - DViewCArrayKokkos corner_mass; - - //Boundary Conditions Data - DCArrayKokkos Local_Index_Boundary_Patches; - //CArray Patch_Nodes; - enum bc_type {NONE, POINT_LOADING_CONDITION, LINE_LOADING_CONDITION, SURFACE_LOADING_CONDITION}; - - //Boundary Conditions Data - int max_boundary_sets; - - //output dof data - //Global arrays with collected data used to print - int output_velocity_index, output_strain_index, output_stress_index; - - //parameters - double time_value, time_final, dt, dt_max, dt_min, dt_cfl, graphics_time, graphics_dt_ival; - size_t graphics_cyc_ival, cycle_stop, rk_num_stages, graphics_id; - double fuzz, tiny, small; - CArray graphics_times; - int rk_num_bins; - - //optimization flags - bool kinetic_energy_objective; -}; - -#endif // end HEADER_H diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/energy_sgh.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/energy_sgh.cpp deleted file mode 100644 index cf62a2b5d..000000000 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/energy_sgh.cpp +++ /dev/null @@ -1,57 +0,0 @@ - -#include "mesh.h" -#include "state.h" -#include "FEA_Module_SGH.h" - -void FEA_Module_SGH::update_energy_sgh(double rk_alpha, - const mesh_t &mesh, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_mass, - const DViewCArrayKokkos &corner_force){ - - const size_t rk_level = rk_num_bins - 1; - int num_dims = num_dim; - - // loop over all the elements in the mesh - FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { - - double elem_power = 0.0; - - // --- tally the contribution from each corner to the element --- - - // Loop over the nodes in the element - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - - size_t corner_lid = node_lid; - - // Get node global id for the local node id - size_t node_gid = nodes_in_elem(elem_gid, node_lid); - - // Get the corner global id for the local corner id - size_t corner_gid = corners_in_elem(elem_gid, corner_lid); - - double node_radius = 1; - if(num_dims==2){ - node_radius = node_coords(rk_level,node_gid,1); - } - - // calculate the Power=F dot V for this corner - for (size_t dim=0; dim -#include -#include -#include -#include -#include // fmin, fmax, abs note: fminl is long -#include -#include - -#include -#include -#include - -#include "elements.h" -#include "swage.h" -#include "matar.h" -#include "utilities.h" -#include "FEA_Module_SGH.h" -#include "Simulation_Parameters/Simulation_Parameters_Explicit.h" -#include "Simulation_Parameters/FEA_Module/SGH_Parameters.h" - - -// ----------------------------------------------------------------------------- -// This function calculates the corner force gradients w.r.t velocity; the -// current implementation assumes material.q2 = 0 -//------------------------------------------------------------------------------ -void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &elem_mat_id, - const double rk_alpha, - const size_t cycle - ){ - - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - const size_t num_dims = simparam->num_dims; - size_t num_corners = rnum_elem*num_nodes_in_elem; - - //initialize gradient matrix - FOR_ALL_CLASS(dof_gid, 0, nlocal_nodes*num_dims, { - for(int idof = 0; idof < Gradient_Matrix_Strides(dof_gid); idof++){ - Force_Gradient_Velocities(dof_gid,idof) = 0; - } - }); // end parallel for loop over nodes - Kokkos::fence(); - - //initialize buffer storage; not all components are explicitly set for this routine - FOR_ALL_CLASS(corner_gid, 0, num_corners, { - for (int dim = 0; dim < num_dims; dim++){ - //assign gradient of corner contribution of force to relevant matrix entries with non-zero node velocity gradient - for(int igradient = 0; igradient < num_nodes_in_elem; igradient++){ - for(int jdim = 0; jdim < num_dims; jdim++){ - corner_gradient_storage(corner_gid,dim,igradient,jdim) = 0; - } - } - } - }); // end parallel for loop over nodes - Kokkos::fence(); - - // --- calculate the forces acting on the nodes from the element --- - for (size_t elem_gid = 0; elem_gid < rnum_elem; elem_gid++){ - //FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { - - const size_t num_nodes_in_elem = 8; - real_t gradient_result[num_dims]; - // total Cauchy stress - double tau_array[9]; - double tau_gradient_array[9]; - - // corner area normals - double area_normal_array[24]; - - // estimate of shock direction - double shock_dir_array[3]; - - // the sums in the Riemann solver - double sum_array[4]; - - // corner shock impeadance x |corner area normal dot shock_dir| - double muc_array[8]; - double muc_gradient_array[8]; - - // Riemann velocity - double vel_star_array[3]; - double vel_star_gradient_array[3*num_nodes_in_elem]; - - // velocity gradient - double vel_grad_array[9]; - - // --- Create views of arrays to aid the force calculation --- - - ViewCArrayKokkos tau(tau_array, num_dims, num_dims); - ViewCArrayKokkos tau_gradient(tau_gradient_array, num_dims, num_dims); - ViewCArrayKokkos area_normal(area_normal_array, num_nodes_in_elem, num_dims); - ViewCArrayKokkos shock_dir(shock_dir_array, num_dims); - ViewCArrayKokkos sum(sum_array, 4); - ViewCArrayKokkos muc(muc_array, num_nodes_in_elem); - ViewCArrayKokkos muc_gradient(muc_gradient_array, num_nodes_in_elem); - ViewCArrayKokkos vel_star(vel_star_array, num_dims); - ViewCArrayKokkos vel_star_gradient(vel_star_gradient_array, num_nodes_in_elem, num_dims); - ViewCArrayKokkos vel_grad(vel_grad_array, num_dims, num_dims); - - EOSParent* eos_model = elem_eos(elem_gid).model; - - // --- abviatations of variables --- - - // element volume - double vol = elem_vol(elem_gid); - - // create a view of the stress_matrix - ViewCArrayKokkos stress(&elem_stress(rk_level, elem_gid, 0,0), 3, 3); - - - // cut out the node_gids for this element - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 8); - - - - // get the B matrix which are the OUTWARD corner area normals - get_bmatrix(area_normal, - elem_gid, - node_coords, - elem_node_gids, - rk_level); - - - // --- Calculate the velocity gradient --- - get_velgrad(vel_grad, - elem_node_gids, - node_vel, - area_normal, - vol, - elem_gid, - rk_level); - - - // the -1 is for the inward surface area normal, - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - for (size_t dim = 0; dim < num_dims; dim++){ - area_normal(node_lid, dim) = (-1.0)*area_normal(node_lid,dim); - } // end for - } // end for - - - - double div = elem_div(elem_gid); - - - // vel = [u,v,w] - // [du/dx, du/dy, du/dz] - // vel_grad = [dv/dx, dv/dy, dv/dz] - // [dw/dx, dw/dy, dw/dz] - double curl[3]; - curl[0] = vel_grad(2,1) - vel_grad(1,2); // dw/dy - dv/dz - curl[1] = vel_grad(0,2) - vel_grad(2,0); // du/dz - dw/dx - curl[2] = vel_grad(1,0) - vel_grad(0,1); // dv/dx - du/dy - - double mag_curl = sqrt(curl[0]*curl[0] + curl[1]*curl[1] + curl[2]*curl[2]); - - - // --- Calculate the Cauchy stress --- - for (size_t i = 0; i < 3; i++){ - for (size_t j = 0; j < 3; j++){ - tau(i, j) = stress(i,j); - // artificial viscosity can be added here to tau - } // end for - } //end for - - // add the pressure - for (int i = 0; i < num_dims; i++){ - tau(i, i) -= elem_pres(elem_gid); - } // end for - - - - - // ---- Multidirectional Approximate Riemann solver (MARS) ---- - // find the average velocity of the elem, it is an - // estimate of the Riemann velocity - - // initialize to Riemann velocity to zero - for (size_t dim = 0; dim < num_dims; dim++){ - vel_star(dim) = 0.0; - } - - // loop over nodes and calculate an average velocity, which is - // an estimate of Riemann velocity - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - - // Get node gloabl index and create view of nodal velocity - int node_gid = nodes_in_elem(elem_gid, node_lid); - - - ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); - - vel_star(0) += 0.125*vel(0); - vel_star(1) += 0.125*vel(1); - vel_star(2) += 0.125*vel(2); - - } // end for loop over nodes - - // find shock direction and shock impedance associated with each node - - // initialize sum term in MARS to zero - for (int i = 0; i < 4; i++){ - sum(i) = 0.0; - } - - double mag; // magnitude of the area normal - double mag_vel; // magnitude of velocity - - // loop over the nodes of the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { - - // Get global node id - size_t node_gid = nodes_in_elem(elem_gid, node_lid); - - // Create view of nodal velocity - ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); - - // Get an estimate of the shock direction. - mag_vel = sqrt( (vel(0) - vel_star(0) )*(vel(0) - vel_star(0) ) - + (vel(1) - vel_star(1) )*(vel(1) - vel_star(1) ) - + (vel(2) - vel_star(2) )*(vel(2) - vel_star(2) ) ); - - - if (mag_vel > small) { - - // estimate of the shock direction, a unit normal - for (int dim = 0; dim < num_dims; dim++){ - shock_dir(dim) = (vel(dim) - vel_star(dim)) / mag_vel; - } - } - - else { - - // if there is no velocity change, then use the surface area - // normal as the shock direction - mag = sqrt( area_normal(node_lid, 0)*area_normal(node_lid, 0) - + area_normal(node_lid, 1)*area_normal(node_lid, 1) - + area_normal(node_lid, 2)*area_normal(node_lid, 2) ); - - // estimate of the shock direction - for (int dim = 0; dim < num_dims; dim++){ - shock_dir(dim) = area_normal(node_lid, dim)/mag; - } - - } // end if mag_vel - - - // cell divergence indicates compression or expansions - size_t mat_id = elem_mat_id(elem_gid); - if (div < 0){ // element in compression - muc(node_lid) = elem_den(elem_gid) * - (material(mat_id).q1*elem_sspd(elem_gid) + material(mat_id).q2*mag_vel); - } - else { // element in expansion - muc(node_lid) = elem_den(elem_gid) * - (material(mat_id).q1ex*elem_sspd(elem_gid) + material(mat_id).q2ex*mag_vel); - } // end if on divergence sign - - - size_t use_shock_dir = 0; - double mu_term; - double mu_term_gradient; - - // Coding to use shock direction - if (use_shock_dir == 1){ - // this is denominator of the Riamann solver and the multiplier - // on velocity in the numerator. It filters on the shock - // direction - mu_term = muc(node_lid)* - fabs( shock_dir(0)*area_normal(node_lid,0) - + shock_dir(1)*area_normal(node_lid,1) - + shock_dir(2)*area_normal(node_lid,2) ); //code gradient for shock dir w.r.t velocity if using shock_dir - } - else { - // Using a full tensoral Riemann jump relation - mu_term = muc(node_lid) - * sqrt( area_normal(node_lid, 0)*area_normal(node_lid, 0) - + area_normal(node_lid, 1)*area_normal(node_lid, 1) - + area_normal(node_lid, 2)*area_normal(node_lid, 2) ); - } - - sum(0) += mu_term*vel(0); - sum(1) += mu_term*vel(1); - sum(2) += mu_term*vel(2); - sum(3) += mu_term; - - muc(node_lid) = mu_term; // the impeadance time surface area is stored here - - } // end for node_lid loop over nodes of the elem - - - - - // The Riemann velocity, called vel_star - if (sum(3) > fuzz) { - for (size_t i = 0; i < num_dims; i++) { - vel_star(i) = sum(i)/sum(3); - } - } - else { - for (int i = 0; i < num_dims; i++){ - vel_star(i) = 0.0; - } - } // end if - - - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { - // The Riemann velocity, called vel_star - if (sum(3) > fuzz) { - for (size_t i = 0; i < num_dims; i++) { - vel_star_gradient(node_lid,i) = muc(node_lid)/sum(3); - } - } - else { - for (int i = 0; i < num_dims; i++){ - vel_star_gradient(node_lid,i) = 0; - } - } // end if - } - - - - // ---- Calculate the shock detector for the Riemann-solver ---- - // - // The dissipation from the Riemann problem is limited by phi - // phi = (1. - max( 0., min( 1. , r_face ) ))^n - // where - // r_face = (C* div(u_+)/div(u_z)) - // The plus denotes the cell center divergence of a neighbor. - // The solution will be first order when phi=1 and have - // zero dissipation when phi=0. - // phi = 0 highest-order solution - // phi = 1 first order solution - // - - double phi = 0.0; // the shock detector - double r_face = 1.0; // the ratio on the face - double r_min = 1.0; // the min ratio for the cell - double r_coef = 0.9; // 0.9; the coefficient on the ratio - // (1=minmod and 2=superbee) - double n_coef = 1.0; // the power on the limiting coefficient - // (1=nominal, and n_coeff > 1 oscillatory) - - - // loop over the nieghboring cells - for (size_t elem_lid = 0; elem_lid < num_elems_in_elem(elem_gid); elem_lid++){ - - // Get global index for neighboring cell - size_t neighbor_gid = elems_in_elem(elem_gid, elem_lid); - - // calculate the velocity divergence in neighbor - double div_neighbor = elem_div(neighbor_gid); - - r_face = r_coef*(div_neighbor + small)/(div + small); - - // store the smallest face ratio - r_min = fmin(r_face, r_min); - - } // end for elem_lid - - - // calculate standard shock detector - phi = 1.0 - fmax(0.0, r_min); - phi = pow(phi, n_coef); - - // Mach number shock detector - double omega = 20.0;//20.0; // weighting factor on Mach number - double third = 1.0/3.0; - double c_length = pow(vol, third); // characteristic length - double alpha = fmin(1.0, omega * (c_length * fabs(div))/(elem_sspd(elem_gid) + fuzz) ); - - // use Mach based detector with standard shock detector - - // turn off dissipation in expansion - //alpha = fmax(-fabs(div0)/div0 * alpha, 0.0); // this should be if(div0<0) alpha=alpha else alpha=0 - - phi = alpha*phi; - - // curl limiter on Q - double phi_curl = fmin(1.0, 1.0*fabs(div)/(mag_curl + fuzz)); // disable Q when vorticity is high - //phi = phi_curl*phi; - phi = 1; - - // ---- Calculate the Riemann force on each node ---- - - // loop over the each node in the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { - - size_t corner_lid = node_lid; - size_t column_index; - - // Get corner gid - size_t corner_gid = corners_in_elem(elem_gid, corner_lid); - - // Get node gid - size_t node_gid = nodes_in_elem(elem_gid, node_lid); - - // loop over dimension - for (int dim = 0; dim < num_dims; dim++){ - //assign gradient of corner contribution of force to relevant matrix entries with non-zero node velocity gradient - for(int igradient = 0; igradient < num_nodes_in_elem; igradient++){ - size_t gradient_node_gid = nodes_in_elem(elem_gid, igradient); - //if(!map->isNodeLocalElement(gradient_node_gid)) continue; - column_index = num_dims*Global_Gradient_Matrix_Assembly_Map(elem_gid, igradient, node_lid); - if(node_lid==igradient){ - if(map->isNodeLocalElement(gradient_node_gid)){ - Force_Gradient_Velocities(gradient_node_gid*num_dims+dim, column_index+dim) += phi*muc(node_lid)*(vel_star_gradient(igradient,dim) - 1); - } - corner_gradient_storage(corner_gid,dim,igradient,dim) = phi*muc(node_lid)*(vel_star_gradient(igradient,dim) - 1); - } - else{ - if(map->isNodeLocalElement(gradient_node_gid)){ - Force_Gradient_Velocities(gradient_node_gid*num_dims+dim, column_index+dim) += phi*muc(node_lid)*(vel_star_gradient(igradient,dim)); - } - corner_gradient_storage(corner_gid,dim,igradient,dim) = phi*muc(node_lid)*(vel_star_gradient(igradient,dim)); - } - } - } // end loop over dimension - - } // end for loop over nodes in elem - - // --- Update Stress --- - // calculate the new stress at the next rk level, if it is a hypo model - size_t mat_id = elem_mat_id(elem_gid); - - - - //}); // end parallel for loop over elements - } - - /* - //accumulate node values from corner storage - force_gradient_design->putScalar(0); - - vec_array force_gradient_design_view = force_gradient_design->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { - size_t corner_id; - for(int icorner=0; icorner < num_corners_in_node(node_id); icorner++){ - corner_id = corners_in_node(node_id,icorner); - force_gradient_design_view(node_id,0) += corner_vector_storage(corner_id, 0); - force_gradient_design_view(node_id,1) += corner_vector_storage(corner_id, 1); - force_gradient_design_view(node_id,2) += corner_vector_storage(corner_id, 2); - } - }); // end parallel for - Kokkos::fence(); - */ - - return; - -} // end of routine - -// ----------------------------------------------------------------------------- -// This function calculates the corner forces and the evolves stress (hypo) -//------------------------------------------------------------------------------ -void FEA_Module_SGH::get_force_egradient_sgh(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &elem_mat_id, - const double rk_alpha, - const size_t cycle - ){ - - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - const size_t num_dims = simparam->num_dims; - - //initialize gradient matrix - FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { - //compute resulting row of force displacement gradient matrix transpose right multiplied by adjoint vector - for(int idof = 0; idof < num_nodes_in_elem*num_dims; idof++){ - Force_Gradient_Energies(elem_gid,idof) = 0; - } - }); // end parallel for - Kokkos::fence(); - - // --- calculate the forces acting on the nodes from the element --- - for (size_t elem_gid = 0; elem_gid < rnum_elem; elem_gid++){ - //FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { - - const size_t num_nodes_in_elem = 8; - real_t gradient_result[num_dims]; - // total Cauchy stress - double tau_array[9]; - double tau_gradient_array[9]; - - // corner area normals - double area_normal_array[24]; - - // estimate of shock direction - double shock_dir_array[3]; - - // the sums in the Riemann solver - double sum_array[4]; - double sum_gradient_array[4]; - - // corner shock impeadance x |corner area normal dot shock_dir| - double muc_array[8]; - double muc_gradient_array[8]; - - // Riemann velocity - double vel_star_array[3]; - double vel_star_gradient_array[3]; - - // velocity gradient - double vel_grad_array[9]; - - // --- Create views of arrays to aid the force calculation --- - - ViewCArrayKokkos tau(tau_array, num_dims, num_dims); - ViewCArrayKokkos tau_gradient(tau_gradient_array, num_dims, num_dims); - ViewCArrayKokkos area_normal(area_normal_array, num_nodes_in_elem, num_dims); - ViewCArrayKokkos shock_dir(shock_dir_array, num_dims); - ViewCArrayKokkos sum(sum_array, 4); - ViewCArrayKokkos sum_gradient(sum_gradient_array, 4); - ViewCArrayKokkos muc(muc_array, num_nodes_in_elem); - ViewCArrayKokkos muc_gradient(muc_gradient_array, num_nodes_in_elem); - ViewCArrayKokkos vel_star(vel_star_array, num_dims); - ViewCArrayKokkos vel_star_gradient(vel_star_gradient_array, num_dims); - ViewCArrayKokkos vel_grad(vel_grad_array, num_dims, num_dims); - - EOSParent* eos_model = elem_eos(elem_gid).model; - - // --- abviatations of variables --- - - // element volume - double vol = elem_vol(elem_gid); - - // create a view of the stress_matrix - ViewCArrayKokkos stress(&elem_stress(rk_level, elem_gid, 0,0), 3, 3); - - - // cut out the node_gids for this element - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 8); - - - - // get the B matrix which are the OUTWARD corner area normals - get_bmatrix(area_normal, - elem_gid, - node_coords, - elem_node_gids, - rk_level); - - - // --- Calculate the velocity gradient --- - get_velgrad(vel_grad, - elem_node_gids, - node_vel, - area_normal, - vol, - elem_gid, - rk_level); - - - // the -1 is for the inward surface area normal, - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - for (size_t dim = 0; dim < num_dims; dim++){ - area_normal(node_lid, dim) = (-1.0)*area_normal(node_lid,dim); - } // end for - } // end for - - - - double div = elem_div(elem_gid); - - - // vel = [u,v,w] - // [du/dx, du/dy, du/dz] - // vel_grad = [dv/dx, dv/dy, dv/dz] - // [dw/dx, dw/dy, dw/dz] - double curl[3]; - curl[0] = vel_grad(2,1) - vel_grad(1,2); // dw/dy - dv/dz - curl[1] = vel_grad(0,2) - vel_grad(2,0); // du/dz - dw/dx - curl[2] = vel_grad(1,0) - vel_grad(0,1); // dv/dx - du/dy - - double mag_curl = sqrt(curl[0]*curl[0] + curl[1]*curl[1] + curl[2]*curl[2]); - - - // --- Calculate the Cauchy stress --- - for (size_t i = 0; i < num_dims; i++){ - for (size_t j = 0; j < num_dims; j++){ - tau(i, j) = stress(i,j); - tau_gradient(i,j) = 0; - // artificial viscosity can be added here to tau - } // end for - } //end for - - // add the pressure - for (int i = 0; i < num_dims; i++){ - tau(i, i) -= elem_pres(elem_gid); - tau_gradient(i, i) = -eos_model->calc_pressure_gradient_internal_energy(elem_pres, - elem_stress, - elem_gid, - elem_mat_id(elem_gid), - state_vars, - global_vars, - elem_user_output_vars, - elem_sspd, - elem_den(elem_gid), - elem_sie(rk_level,elem_gid)); - } // end for - - - - - // ---- Multidirectional Approximate Riemann solver (MARS) ---- - // find the average velocity of the elem, it is an - // estimate of the Riemann velocity - - // initialize to Riemann velocity to zero - for (size_t dim = 0; dim < num_dims; dim++){ - vel_star(dim) = 0.0; - } - - // loop over nodes and calculate an average velocity, which is - // an estimate of Riemann velocity - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - - // Get node gloabl index and create view of nodal velocity - int node_gid = nodes_in_elem(elem_gid, node_lid); - - - ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); - - vel_star(0) += 0.125*vel(0); - vel_star(1) += 0.125*vel(1); - vel_star(2) += 0.125*vel(2); - - } // end for loop over nodes - - // find shock direction and shock impedance associated with each node - - // initialize sum term in MARS to zero - for (int i = 0; i < 4; i++){ - sum(i) = sum_gradient(i) = 0.0; - } - - double mag; // magnitude of the area normal - double mag_vel; // magnitude of velocity - - // --- Sound speed --- - real_t sound_speed_gradient_energy = - eos_model->calc_sound_speed_gradient_internal_energy(elem_pres, - elem_stress, - elem_gid, - elem_mat_id(elem_gid), - state_vars, - global_vars, - elem_user_output_vars, - elem_sspd, - elem_den(elem_gid), - elem_sie(rk_level,elem_gid)); - - // loop over the nodes of the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { - - // Get global node id - size_t node_gid = nodes_in_elem(elem_gid, node_lid); - - // Create view of nodal velocity - ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); - - // Get an estimate of the shock direction. - mag_vel = sqrt( (vel(0) - vel_star(0) )*(vel(0) - vel_star(0) ) - + (vel(1) - vel_star(1) )*(vel(1) - vel_star(1) ) - + (vel(2) - vel_star(2) )*(vel(2) - vel_star(2) ) ); - - - if (mag_vel > small) { - - // estimate of the shock direction, a unit normal - for (int dim = 0; dim < num_dims; dim++){ - shock_dir(dim) = (vel(dim) - vel_star(dim)) / mag_vel; - } - } - - else { - - // if there is no velocity change, then use the surface area - // normal as the shock direction - mag = sqrt( area_normal(node_lid, 0)*area_normal(node_lid, 0) - + area_normal(node_lid, 1)*area_normal(node_lid, 1) - + area_normal(node_lid, 2)*area_normal(node_lid, 2) ); - - // estimate of the shock direction - for (int dim = 0; dim < num_dims; dim++){ - shock_dir(dim) = area_normal(node_lid, dim)/mag; - } - - } // end if mag_vel - - - // cell divergence indicates compression or expansions - size_t mat_id = elem_mat_id(elem_gid); - if (div < 0){ // element in compression - muc(node_lid) = elem_den(elem_gid) * - (material(mat_id).q1*elem_sspd(elem_gid) + material(mat_id).q2*mag_vel); - muc_gradient(node_lid) = elem_den(elem_gid) * - (material(mat_id).q1*sound_speed_gradient_energy); - } - else { // element in expansion - muc(node_lid) = elem_den(elem_gid) * - (material(mat_id).q1ex*elem_sspd(elem_gid) + material(mat_id).q2ex*mag_vel); - muc_gradient(node_lid) = elem_den(elem_gid) * - (material(mat_id).q1ex*sound_speed_gradient_energy); - } // end if on divergence sign - - - size_t use_shock_dir = 0; - double mu_term; - double mu_term_gradient; - - // Coding to use shock direction - if (use_shock_dir == 1){ - // this is denominator of the Riamann solver and the multiplier - // on velocity in the numerator. It filters on the shock - // direction - mu_term = muc(node_lid)* - fabs( shock_dir(0)*area_normal(node_lid,0) - + shock_dir(1)*area_normal(node_lid,1) - + shock_dir(2)*area_normal(node_lid,2) ); - mu_term_gradient = muc_gradient(node_lid)* //amend if shock dir has dependence for gradient not captured here - fabs( shock_dir(0)*area_normal(node_lid,0) - + shock_dir(1)*area_normal(node_lid,1) - + shock_dir(2)*area_normal(node_lid,2) ); - } - else { - // Using a full tensoral Riemann jump relation - mu_term = muc(node_lid) - * sqrt( area_normal(node_lid, 0)*area_normal(node_lid, 0) - + area_normal(node_lid, 1)*area_normal(node_lid, 1) - + area_normal(node_lid, 2)*area_normal(node_lid, 2) ); - mu_term_gradient = muc_gradient(node_lid) - * sqrt( area_normal(node_lid, 0)*area_normal(node_lid, 0) - + area_normal(node_lid, 1)*area_normal(node_lid, 1) - + area_normal(node_lid, 2)*area_normal(node_lid, 2) ); - } - - sum(0) += mu_term*vel(0); - sum(1) += mu_term*vel(1); - sum(2) += mu_term*vel(2); - sum(3) += mu_term; - - sum_gradient(0) += mu_term_gradient*vel(0); - sum_gradient(1) += mu_term_gradient*vel(1); - sum_gradient(2) += mu_term_gradient*vel(2); - sum_gradient(3) += mu_term_gradient; - - muc(node_lid) = mu_term; // the impeadance time surface area is stored here - muc_gradient(node_lid) = mu_term_gradient; - - } // end for node_lid loop over nodes of the elem - - - - - // The Riemann velocity, called vel_star - if (sum(3) > fuzz) { - for (size_t i = 0; i < num_dims; i++) { - vel_star(i) = sum(i)/sum(3); - vel_star_gradient(i) = sum_gradient(i)/sum(3) - sum_gradient(3)*sum(i)/sum(3)/sum(3); - } - } - else { - for (int i = 0; i < num_dims; i++){ - vel_star(i) = 0.0; - vel_star_gradient(i) = 0; - } - } // end if - - - - // ---- Calculate the shock detector for the Riemann-solver ---- - // - // The dissipation from the Riemann problem is limited by phi - // phi = (1. - max( 0., min( 1. , r_face ) ))^n - // where - // r_face = (C* div(u_+)/div(u_z)) - // The plus denotes the cell center divergence of a neighbor. - // The solution will be first order when phi=1 and have - // zero dissipation when phi=0. - // phi = 0 highest-order solution - // phi = 1 first order solution - // - - double phi = 0.0; // the shock detector - double r_face = 1.0; // the ratio on the face - double r_min = 1.0; // the min ratio for the cell - double r_coef = 0.9; // 0.9; the coefficient on the ratio - // (1=minmod and 2=superbee) - double n_coef = 1.0; // the power on the limiting coefficient - // (1=nominal, and n_coeff > 1 oscillatory) - - - // loop over the nieghboring cells - for (size_t elem_lid = 0; elem_lid < num_elems_in_elem(elem_gid); elem_lid++){ - - // Get global index for neighboring cell - size_t neighbor_gid = elems_in_elem(elem_gid, elem_lid); - - // calculate the velocity divergence in neighbor - double div_neighbor = elem_div(neighbor_gid); - - r_face = r_coef*(div_neighbor + small)/(div + small); - - // store the smallest face ratio - r_min = fmin(r_face, r_min); - - } // end for elem_lid - - - // calculate standard shock detector - phi = 1.0 - fmax(0.0, r_min); - phi = pow(phi, n_coef); - - // Mach number shock detector - double omega = 20.0;//20.0; // weighting factor on Mach number - double third = 1.0/3.0; - double c_length = pow(vol, third); // characteristic length - double alpha = fmin(1.0, omega * (c_length * fabs(div))/(elem_sspd(elem_gid) + fuzz) ); - - // use Mach based detector with standard shock detector - - // turn off dissipation in expansion - //alpha = fmax(-fabs(div0)/div0 * alpha, 0.0); // this should be if(div0<0) alpha=alpha else alpha=0 - - phi = alpha*phi; - - // curl limiter on Q - double phi_curl = fmin(1.0, 1.0*fabs(div)/(mag_curl + fuzz)); // disable Q when vorticity is high - //phi = phi_curl*phi; - phi = 1; - - // ---- Calculate the Riemann force on each node ---- - - // loop over the each node in the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { - - size_t corner_lid = node_lid; - - // Get corner gid - size_t corner_gid = corners_in_elem(elem_gid, corner_lid); - - // Get node gid - size_t node_gid = nodes_in_elem(elem_gid, node_lid); - - // loop over dimension - for (int dim = 0; dim < num_dims; dim++){ - - Force_Gradient_Energies(elem_gid,node_lid*num_dims+dim) = corner_vector_storage(corner_gid, dim) = - area_normal(node_lid, 0)*tau_gradient(0, dim) - + area_normal(node_lid, 1)*tau_gradient(1, dim) - + area_normal(node_lid, 2)*tau_gradient(2, dim) - + phi*muc_gradient(node_lid)*(vel_star(dim) - node_vel(rk_level, node_gid, dim)) - + phi*muc(node_lid)*(vel_star_gradient(dim)); - - } // end loop over dimension - - } // end for loop over nodes in elem - - - - // --- Update Stress --- - // calculate the new stress at the next rk level, if it is a hypo model - - size_t mat_id = elem_mat_id(elem_gid); - - - } - //}); // end parallel for loop over elements - - /* - //accumulate node values from corner storage - force_gradient_design->putScalar(0); - - vec_array force_gradient_design_view = force_gradient_design->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { - size_t corner_id; - for(int icorner=0; icorner < num_corners_in_node(node_id); icorner++){ - corner_id = corners_in_node(node_id,icorner); - force_gradient_design_view(node_id,0) += corner_vector_storage(corner_id, 0); - force_gradient_design_view(node_id,1) += corner_vector_storage(corner_id, 1); - force_gradient_design_view(node_id,2) += corner_vector_storage(corner_id, 2); - } - }); // end parallel for - Kokkos::fence(); - */ - - return; - -} // end of routine - -// ----------------------------------------------------------------------------- -// This function calculates the corner forces and the evolves stress (hypo) -//------------------------------------------------------------------------------ -void FEA_Module_SGH::get_force_ugradient_sgh(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &elem_mat_id, - const double rk_alpha, - const size_t cycle - ){ - - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - const size_t num_dims = simparam->num_dims; - - //initialize gradient matrix - FOR_ALL_CLASS(dof_gid, 0, nlocal_nodes*num_dims, { - for(int idof = 0; idof < Gradient_Matrix_Strides(dof_gid); idof++){ - Force_Gradient_Positions(dof_gid,idof) = 0; - } - }); // end parallel for loop over nodes - Kokkos::fence(); - - // --- calculate the forces acting on the nodes from the element --- - for (size_t elem_gid = 0; elem_gid < rnum_elem; elem_gid++){ - //FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { - - const size_t num_nodes_in_elem = 8; - real_t gradient_result[num_dims]; - // total Cauchy stress - double tau_array[9]; - double tau_gradient_array[9*max_nodes_per_element*num_dims]; - - // corner area normals - double area_normal_gradients_array[max_nodes_per_element*max_nodes_per_element*num_dims*num_dims]; - double area_normal_array[24]; - - //volume data - double volume; - double volume_gradients_array[max_nodes_per_element*num_dims]; - - // estimate of shock direction - double shock_dir_array[3]; - - // the sums in the Riemann solver - double sum_array[4]; - double sum_gradient_array[4*max_nodes_per_element*num_dim]; - - // corner shock impeadance x |corner area normal dot shock_dir| - double muc_array[8]; - double muc_gradient_array[8*max_nodes_per_element*num_dims]; - - // Riemann velocity - double vel_star_array[3]; - double vel_star_gradient_array[num_dims*max_nodes_per_element*num_dims]; - - // velocity gradient - double vel_grad_array[9]; - - // --- Create views of arrays to aid the force calculation --- - - ViewCArrayKokkos tau(tau_array, num_dims, num_dims); - ViewCArrayKokkos tau_gradient(tau_gradient_array, num_dims, num_dims,max_nodes_per_element, num_dims); - ViewCArrayKokkos volume_gradients(volume_gradients_array, num_nodes_in_elem, num_dims); - ViewCArrayKokkos area_normal(area_normal_array, num_nodes_in_elem, num_dims); - ViewCArrayKokkos area_normal_gradients(area_normal_gradients_array, num_nodes_in_elem, num_dims, num_nodes_in_elem, num_dims); - ViewCArrayKokkos shock_dir(shock_dir_array, num_dims); - ViewCArrayKokkos sum(sum_array, 4); - ViewCArrayKokkos sum_gradient(sum_gradient_array, 4, max_nodes_per_element, num_dims); - ViewCArrayKokkos muc(muc_array, num_nodes_in_elem); - ViewCArrayKokkos muc_gradient(muc_gradient_array, num_nodes_in_elem, num_nodes_in_elem, num_dims); - ViewCArrayKokkos vel_star(vel_star_array, num_dims); - ViewCArrayKokkos vel_star_gradient(vel_star_gradient_array, num_dims, num_nodes_in_elem, num_dims); - ViewCArrayKokkos vel_grad(vel_grad_array, num_dims, num_dims); - - EOSParent* eos_model = elem_eos(elem_gid).model; - - // --- abviatations of variables --- - - // element volume - double vol = elem_vol(elem_gid); - - // create a view of the stress_matrix - ViewCArrayKokkos stress(&elem_stress(rk_level, elem_gid, 0,0), 3, 3); - - // cut out the node_gids for this element - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 8); - - //gradients of the element volume - get_vol_hex_ugradient(volume_gradients, elem_gid, node_coords, elem_node_gids, rk_level); - - - // //debug - // for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - // for (size_t dim = 0; dim < num_dims; dim++){ - // if(volume_gradients(node_lid, dim)>1 || volume_gradients(node_lid, dim) < -1) - // std::cout << volume_gradients(node_lid, dim) << " "; - // } // end for - // } // end for - // std::cout << std::endl; - - // get the B matrix which are the OUTWARD corner area normals - get_bmatrix(area_normal, - elem_gid, - node_coords, - elem_node_gids, - rk_level); - - get_bmatrix_gradients(area_normal_gradients, - elem_gid, - node_coords, - elem_node_gids, - rk_level); - - - // --- Calculate the velocity gradient --- - get_velgrad(vel_grad, - elem_node_gids, - node_vel, - area_normal, - vol, - elem_gid, - rk_level); - - - // the -1 is for the inward surface area normal, - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - for (size_t dim = 0; dim < num_dims; dim++){ - area_normal(node_lid, dim) = (-1.0)*area_normal(node_lid,dim); - } // end for - } // end for - - // the -1 is for the inward surface area normal, - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - for (size_t dim = 0; dim < num_dims; dim++){ - for (size_t igradient = 0; igradient < num_nodes_in_elem; igradient++){ - for (size_t jdim = 0; jdim < num_dims; jdim++){ - area_normal_gradients(node_lid, dim, igradient, jdim) = (-1.0)*area_normal_gradients(node_lid,dim, igradient, jdim); - } - } - } // end for - } // end for - - - - double div = elem_div(elem_gid); - - - // vel = [u,v,w] - // [du/dx, du/dy, du/dz] - // vel_grad = [dv/dx, dv/dy, dv/dz] - // [dw/dx, dw/dy, dw/dz] - double curl[3]; - curl[0] = vel_grad(2,1) - vel_grad(1,2); // dw/dy - dv/dz - curl[1] = vel_grad(0,2) - vel_grad(2,0); // du/dz - dw/dx - curl[2] = vel_grad(1,0) - vel_grad(0,1); // dv/dx - du/dy - - double mag_curl = sqrt(curl[0]*curl[0] + curl[1]*curl[1] + curl[2]*curl[2]); - - - // --- Calculate the Cauchy stress --- - for (size_t i = 0; i < 3; i++){ - for (size_t j = 0; j < 3; j++){ - tau(i, j) = stress(i,j); - // artificial viscosity can be added here to tau - } // end for - } //end for - - //initialize gradient array - for (size_t i = 0; i < 3; i++){ - for (size_t j = 0; j < 3; j++){ - for(int igradient = 0; igradient < num_nodes_in_elem; igradient++){ - tau_gradient(i, j, igradient, 0) = 0; - tau_gradient(i, j, igradient, 1) = 0; - if(num_dims==3) - tau_gradient(i, j, igradient, 2) = 0; - } - } // end for - } //end for - - // add the pressure - for (int i = 0; i < num_dims; i++){ - tau(i, i) -= elem_pres(elem_gid); - //current expression only works for ideal gas! generalize. - for(int igradient = 0; igradient < num_nodes_in_elem; igradient++){ - tau_gradient(i, i,igradient,0) = elem_pres(elem_gid)*volume_gradients(igradient,0)/vol; - tau_gradient(i, i,igradient,1) = elem_pres(elem_gid)*volume_gradients(igradient,1)/vol; - tau_gradient(i, i,igradient,2) = elem_pres(elem_gid)*volume_gradients(igradient,2)/vol; - } - } // end for - - - - - // ---- Multidirectional Approximate Riemann solver (MARS) ---- - // find the average velocity of the elem, it is an - // estimate of the Riemann velocity - - // initialize to Riemann velocity to zero - for (size_t dim = 0; dim < num_dims; dim++){ - vel_star(dim) = 0.0; - } - - // loop over nodes and calculate an average velocity, which is - // an estimate of Riemann velocity - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - - // Get node gloabl index and create view of nodal velocity - int node_gid = nodes_in_elem(elem_gid, node_lid); - - - ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); - - vel_star(0) += 0.125*vel(0); - vel_star(1) += 0.125*vel(1); - vel_star(2) += 0.125*vel(2); - - } // end for loop over nodes - - // find shock direction and shock impedance associated with each node - - // initialize sum term in MARS to zero - for (int i = 0; i < 4; i++){ - sum(i) = 0.0; - } - for (int i = 0; i < 4; i++){ - for (size_t igradient = 0; igradient < num_nodes_in_elem; igradient++) { - for (size_t jdim = 0; jdim < num_dims; jdim++) { - sum_gradient(i, igradient, jdim) = 0.0; - } - } - } - - double mag; // magnitude of the area normal - double mag_vel; // magnitude of velocity - - // loop over the nodes of the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { - - // Get global node id - size_t node_gid = nodes_in_elem(elem_gid, node_lid); - - // Create view of nodal velocity - ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); - - // Get an estimate of the shock direction. - mag_vel = sqrt( (vel(0) - vel_star(0) )*(vel(0) - vel_star(0) ) - + (vel(1) - vel_star(1) )*(vel(1) - vel_star(1) ) - + (vel(2) - vel_star(2) )*(vel(2) - vel_star(2) ) ); - - - if (mag_vel > small) { - - // estimate of the shock direction, a unit normal - for (int dim = 0; dim < num_dims; dim++){ - shock_dir(dim) = (vel(dim) - vel_star(dim)) / mag_vel; - } - } - - else { - - // if there is no velocity change, then use the surface area - // normal as the shock direction - mag = sqrt( area_normal(node_lid, 0)*area_normal(node_lid, 0) - + area_normal(node_lid, 1)*area_normal(node_lid, 1) - + area_normal(node_lid, 2)*area_normal(node_lid, 2) ); - - // estimate of the shock direction - for (int dim = 0; dim < num_dims; dim++){ - shock_dir(dim) = area_normal(node_lid, dim)/mag; - } - - } // end if mag_vel - - - // cell divergence indicates compression or expansions - size_t mat_id = elem_mat_id(elem_gid); - if (div < 0){ // element in compression - muc(node_lid) = elem_den(elem_gid) * - (material(mat_id).q1*elem_sspd(elem_gid) + material(mat_id).q2*mag_vel); - - for(int igradient = 0; igradient < num_nodes_in_elem; igradient++){ - muc_gradient(node_lid, igradient, 0) = -elem_den(elem_gid)*volume_gradients(igradient,0)/vol* - (material(mat_id).q1*elem_sspd(elem_gid)); - muc_gradient(node_lid, igradient, 1) = -elem_den(elem_gid)*volume_gradients(igradient,1)/vol* - (material(mat_id).q1*elem_sspd(elem_gid)); - muc_gradient(node_lid, igradient, 2) = -elem_den(elem_gid)*volume_gradients(igradient,2)/vol* - (material(mat_id).q1*elem_sspd(elem_gid)); - } - } - else { // element in expansion - muc(node_lid) = elem_den(elem_gid) * - (material(mat_id).q1ex*elem_sspd(elem_gid) + material(mat_id).q2ex*mag_vel); - for(int igradient = 0; igradient < num_nodes_in_elem; igradient++){ - muc_gradient(node_lid, igradient, 0) = -elem_den(elem_gid)*volume_gradients(igradient,0)/vol* - (material(mat_id).q1ex*elem_sspd(elem_gid)); - muc_gradient(node_lid, igradient, 1) = -elem_den(elem_gid)*volume_gradients(igradient,1)/vol* - (material(mat_id).q1ex*elem_sspd(elem_gid)); - muc_gradient(node_lid, igradient, 2) = -elem_den(elem_gid)*volume_gradients(igradient,2)/vol* - (material(mat_id).q1ex*elem_sspd(elem_gid)); - } - } // end if on divergence sign - - - size_t use_shock_dir = 0; - double mu_term; - double mu_term_gradient; - - // Coding to use shock direction - if (use_shock_dir == 1){ - // this is denominator of the Riamann solver and the multiplier - // on velocity in the numerator. It filters on the shock - // direction - mu_term = muc(node_lid)* - fabs( shock_dir(0)*area_normal(node_lid,0) - + shock_dir(1)*area_normal(node_lid,1) - + shock_dir(2)*area_normal(node_lid,2) ); - // muc_gradient(node_lid) = muc_gradient(node_lid)* - // fabs( shock_dir(0)*area_normal(node_lid,0) - // + shock_dir(1)*area_normal(node_lid,1) - // + shock_dir(2)*area_normal(node_lid,2) ); - // mu_term_gradient = muc(node_lid)* //amend if shock dir has dependence for gradient not captured here - // fabs( shock_dir(0)*area_normal_gradients(node_lid,0) - // + shock_dir(1)*area_normal_gradients(node_lid,1) - // + shock_dir(2)*area_normal_gradients(node_lid,2) ); - } - else { - // Using a full tensoral Riemann jump relation - mu_term = muc(node_lid) - * sqrt( area_normal(node_lid, 0)*area_normal(node_lid, 0) - + area_normal(node_lid, 1)*area_normal(node_lid, 1) - + area_normal(node_lid, 2)*area_normal(node_lid, 2) ); - for(int igradient = 0; igradient < num_nodes_in_elem; igradient++){ - for( int jdim = 0; jdim < num_dims; jdim++){ - mu_term_gradient = muc_gradient(node_lid,igradient,jdim) - * sqrt( area_normal(node_lid, 0)*area_normal(node_lid, 0) - + area_normal(node_lid, 1)*area_normal(node_lid, 1) - + area_normal(node_lid, 2)*area_normal(node_lid, 2) ); - - mu_term_gradient += muc(node_lid)*(area_normal(node_lid, 0)*area_normal_gradients(node_lid, 0,igradient,jdim) - + area_normal(node_lid, 1)*area_normal_gradients(node_lid, 1,igradient,jdim) - + area_normal(node_lid, 2)*area_normal_gradients(node_lid, 2,igradient,jdim))/ - sqrt( area_normal(node_lid, 0)*area_normal(node_lid, 0) - + area_normal(node_lid, 1)*area_normal(node_lid, 1) - + area_normal(node_lid, 2)*area_normal(node_lid, 2) ); - - muc_gradient(node_lid, igradient,jdim) = mu_term_gradient; - } - } - } - - sum(0) += mu_term*vel(0); - sum(1) += mu_term*vel(1); - sum(2) += mu_term*vel(2); - sum(3) += mu_term; - - //sum gradients - - for(int igradient = 0; igradient < num_nodes_in_elem; igradient++){ - for( int jdim = 0; jdim < num_dims; jdim++){ - sum_gradient(0,igradient,jdim) += muc_gradient(node_lid, igradient,jdim)*vel(0); - sum_gradient(1,igradient,jdim) += muc_gradient(node_lid, igradient,jdim)*vel(1); - sum_gradient(2,igradient,jdim) += muc_gradient(node_lid, igradient,jdim)*vel(2); - sum_gradient(3,igradient,jdim) += muc_gradient(node_lid, igradient,jdim); - } - } - - muc(node_lid) = mu_term; // the impeadance time surface area is stored here - - } // end for node_lid loop over nodes of the elem - - - // The Riemann velocity, called vel_star - if (sum(3) > fuzz) { - for (size_t i = 0; i < num_dims; i++) { - vel_star(i) = sum(i)/sum(3); - } - } - else { - for (int i = 0; i < num_dims; i++){ - vel_star(i) = 0.0; - } - } // end if - - //vel star gradients - if (sum(3) > fuzz) { - for (size_t i = 0; i < num_dims; i++) { - for(int igradient = 0; igradient < num_nodes_in_elem; igradient++){ - for( int jdim = 0; jdim < num_dims; jdim++){ - vel_star_gradient(i,igradient,jdim) = sum_gradient(i,igradient,jdim)/sum(3)-sum(i)/(sum(3)*sum(3))*sum_gradient(3,igradient,jdim); - } - } - } - } - else { - for (int i = 0; i < num_dims; i++){ - for(int igradient = 0; igradient < num_nodes_in_elem; igradient++){ - for( int jdim = 0; jdim < num_dims; jdim++){ - vel_star_gradient(i,igradient,jdim) = 0; - } - } - } - } // end if - - - // ---- Calculate the shock detector for the Riemann-solver ---- - // - // The dissipation from the Riemann problem is limited by phi - // phi = (1. - max( 0., min( 1. , r_face ) ))^n - // where - // r_face = (C* div(u_+)/div(u_z)) - // The plus denotes the cell center divergence of a neighbor. - // The solution will be first order when phi=1 and have - // zero dissipation when phi=0. - // phi = 0 highest-order solution - // phi = 1 first order solution - // - - double phi = 0.0; // the shock detector - double r_face = 1.0; // the ratio on the face - double r_min = 1.0; // the min ratio for the cell - double r_coef = 0.9; // 0.9; the coefficient on the ratio - // (1=minmod and 2=superbee) - double n_coef = 1.0; // the power on the limiting coefficient - // (1=nominal, and n_coeff > 1 oscillatory) - - - // loop over the nieghboring cells - for (size_t elem_lid = 0; elem_lid < num_elems_in_elem(elem_gid); elem_lid++){ - - // Get global index for neighboring cell - size_t neighbor_gid = elems_in_elem(elem_gid, elem_lid); - - // calculate the velocity divergence in neighbor - double div_neighbor = elem_div(neighbor_gid); - - r_face = r_coef*(div_neighbor + small)/(div + small); - - // store the smallest face ratio - r_min = fmin(r_face, r_min); - - } // end for elem_lid - - - // calculate standard shock detector - phi = 1.0 - fmax(0.0, r_min); - phi = pow(phi, n_coef); - - // Mach number shock detector - double omega = 20.0;//20.0; // weighting factor on Mach number - double third = 1.0/3.0; - double c_length = pow(vol, third); // characteristic length - double alpha = fmin(1.0, omega * (c_length * fabs(div))/(elem_sspd(elem_gid) + fuzz) ); - - // use Mach based detector with standard shock detector - - // turn off dissipation in expansion - //alpha = fmax(-fabs(div0)/div0 * alpha, 0.0); // this should be if(div0<0) alpha=alpha else alpha=0 - - phi = alpha*phi; - - // curl limiter on Q - double phi_curl = fmin(1.0, 1.0*fabs(div)/(mag_curl + fuzz)); // disable Q when vorticity is high - //phi = phi_curl*phi; - phi = 1; - - // ---- Calculate the Riemann force on each node ---- - - // loop over the each node in the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { - - size_t corner_lid = node_lid; - size_t column_index; - - // Get corner gid - size_t corner_gid = corners_in_elem(elem_gid, corner_lid); - - // Get node gid - size_t node_gid = nodes_in_elem(elem_gid, node_lid); - - // loop over dimension - for (int dim = 0; dim < num_dims; dim++){ - //assign gradient of corner contribution of force to relevant matrix entries with non-zero node velocity gradient - for(int igradient = 0; igradient < num_nodes_in_elem; igradient++){ - for(int jdim = 0; jdim < num_dims; jdim++){ - size_t gradient_node_gid = nodes_in_elem(elem_gid, igradient); - //if(!map->isNodeLocalElement(gradient_node_gid)) continue; - column_index = num_dims*Global_Gradient_Matrix_Assembly_Map(elem_gid, igradient, node_lid); - if(map->isNodeLocalElement(gradient_node_gid)){ - Force_Gradient_Positions(gradient_node_gid*num_dims+jdim, column_index+dim) += area_normal(node_lid, 0)*tau_gradient(0, dim, igradient, jdim) - + area_normal(node_lid, 1)*tau_gradient(1, dim, igradient, jdim) - + area_normal(node_lid, 2)*tau_gradient(2, dim, igradient, jdim) - + area_normal_gradients(node_lid, 0, igradient, jdim)*tau(0, dim) - + area_normal_gradients(node_lid, 1, igradient, jdim)*tau(1, dim) - + area_normal_gradients(node_lid, 2, igradient, jdim)*tau(2, dim) - + phi*muc_gradient(node_lid, igradient, jdim)*(vel_star(dim) - node_vel(rk_level, node_gid, dim)) - + phi*muc(node_lid)*(vel_star_gradient(dim, igradient, jdim)); - } - corner_gradient_storage(corner_gid,dim,igradient,jdim) = area_normal(node_lid, 0)*tau_gradient(0, dim, igradient, jdim) - + area_normal(node_lid, 1)*tau_gradient(1, dim, igradient, jdim) - + area_normal(node_lid, 2)*tau_gradient(2, dim, igradient, jdim) - + area_normal_gradients(node_lid, 0, igradient, jdim)*tau(0, dim) - + area_normal_gradients(node_lid, 1, igradient, jdim)*tau(1, dim) - + area_normal_gradients(node_lid, 2, igradient, jdim)*tau(2, dim) - + phi*muc_gradient(node_lid, igradient, jdim)*(vel_star(dim) - node_vel(rk_level, node_gid, dim)) - + phi*muc(node_lid)*(vel_star_gradient(dim, igradient, jdim)); - // if(map->isNodeLocalElement(gradient_node_gid)){ - // Force_Gradient_Positions(gradient_node_gid*num_dims+jdim, column_index+dim) += - // + area_normal_gradients(node_lid, 0, igradient, jdim)*tau(0, dim) - // + area_normal_gradients(node_lid, 1, igradient, jdim)*tau(1, dim) - // + area_normal_gradients(node_lid, 2, igradient, jdim)*tau(2, dim); - // } - // corner_gradient_storage(corner_gid,dim,igradient,jdim) = - // + area_normal_gradients(node_lid, 0, igradient, jdim)*tau(0, dim) - // + area_normal_gradients(node_lid, 1, igradient, jdim)*tau(1, dim) - // + area_normal_gradients(node_lid, 2, igradient, jdim)*tau(2, dim); - } - } - } // end loop over dimension - - } // end for loop over nodes in elem - - - - // --- Update Stress --- - // calculate the new stress at the next rk level, if it is a hypo model - - size_t mat_id = elem_mat_id(elem_gid); - - - } - //}); // end parallel for loop over elements - - /* - //accumulate node values from corner storage - force_gradient_design->putScalar(0); - - vec_array force_gradient_design_view = force_gradient_design->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { - size_t corner_id; - for(int icorner=0; icorner < num_corners_in_node(node_id); icorner++){ - corner_id = corners_in_node(node_id,icorner); - force_gradient_design_view(node_id,0) += corner_vector_storage(corner_id, 0); - force_gradient_design_view(node_id,1) += corner_vector_storage(corner_id, 1); - force_gradient_design_view(node_id,2) += corner_vector_storage(corner_id, 2); - } - }); // end parallel for - Kokkos::fence(); - */ - - return; - -} // end of routine - -// -------------------------------------------------------------------------------------------------------- -// Computes term objective derivative term involving gradient of power with respect to the design variable -//--------------------------------------------------------------------------------------------------------- - -void FEA_Module_SGH::force_design_gradient_term(const_vec_array design_variables, vec_array design_gradients){ - - size_t num_bdy_nodes = mesh->num_bdy_nodes; - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos material = simparam->material; - const int num_dim = simparam->num_dims; - size_t num_corners = rnum_elem*num_nodes_in_elem; - real_t global_dt; - bool element_constant_density = true; - size_t current_data_index, next_data_index; - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - CArrayKokkos current_element_adjoint = CArrayKokkos(num_nodes_in_elem,num_dim); - - //gradient contribution from gradient of Force vector with respect to design variable. - if(simparam->dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ - if(myrank==0){ - std::cout << "gradient term involving adjoint derivative" << std::endl; - } - } - - for (unsigned long cycle = 0; cycle < last_time_step+1; cycle++) { - //compute timestep from time data - global_dt = time_data[cycle+1] - time_data[cycle]; - //print - if(simparam->dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ - if (cycle==0){ - if(myrank==0) - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } - // print time step every 10 cycles - else if (cycle%20==0){ - if(myrank==0) - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } // end if - } - - - //view scope - { - const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_element_internal_energy = (*forward_solve_internal_energy_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_coord_vector = (*forward_solve_coordinate_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array next_velocity_vector = (*forward_solve_velocity_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array next_element_internal_energy = (*forward_solve_internal_energy_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array next_coord_vector = (*forward_solve_coordinate_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array next_adjoint_vector = (*adjoint_vector_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - - //first half of integration step calculation - FOR_ALL_CLASS(node_gid, 0, nall_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - node_vel(rk_level,node_gid,idim) = current_velocity_vector(node_gid,idim); - node_coords(rk_level,node_gid,idim) = current_coord_vector(node_gid,idim); - } - - }); // end parallel for - Kokkos::fence(); - - FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { - elem_sie(rk_level,elem_gid) = current_element_internal_energy(elem_gid,0); - }); // end parallel for - Kokkos::fence(); - - get_vol(); - - // ---- Calculate velocity diveregence for the element ---- - if(num_dim==2){ - get_divergence2D(elem_div, - node_coords, - node_vel, - elem_vol); - } - else { - get_divergence(elem_div, - node_coords, - node_vel, - elem_vol); - } // end if 2D - - // ---- Calculate elem state (den, pres, sound speed, stress) for next time step ---- - if(num_dim==2){ - update_state2D(material, - *mesh, - node_coords, - node_vel, - elem_den, - elem_pres, - elem_stress, - elem_sspd, - elem_sie, - elem_vol, - elem_mass, - elem_mat_id, - 1.0, - cycle); - } - else{ - update_state(material, - *mesh, - node_coords, - node_vel, - elem_den, - elem_pres, - elem_stress, - elem_sspd, - elem_sie, - elem_vol, - elem_mass, - elem_mat_id, - 1.0, - cycle); - } - - get_force_dgradient_sgh(material, - *mesh, - node_coords, - node_vel, - elem_den, - elem_sie, - elem_pres, - elem_stress, - elem_sspd, - elem_vol, - elem_div, - elem_mat_id, - 1.0, - cycle); - - //derivatives of forces at corners stored in corner_vector_storage buffer by previous routine - FOR_ALL_CLASS(elem_id, 0, rnum_elem, { - size_t node_id; - size_t corner_id; - real_t inner_product; - - inner_product = 0; - for(int ifill=0; ifill < num_nodes_in_elem; ifill++){ - node_id = nodes_in_elem(elem_id, ifill); - corner_id = elem_id*num_nodes_in_elem + ifill; - for(int idim=0; idim < num_dim; idim++){ - inner_product += corner_vector_storage(corner_id,idim)*current_adjoint_vector(node_id,idim); - } - } - - for (int inode = 0; inode < num_nodes_in_elem; inode++){ - //compute gradient of local element contribution to v^t*M*v product - corner_id = elem_id*num_nodes_in_elem + inode; - corner_value_storage(corner_id) = inner_product; - } - - }); // end parallel for - Kokkos::fence(); - - //accumulate node values from corner storage - //multiply - FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { - size_t corner_id; - for(int icorner=0; icorner < num_corners_in_node(node_id); icorner++){ - corner_id = corners_in_node(node_id,icorner); - design_gradients(node_id,0) += -corner_value_storage(corner_id)*global_dt; - } - }); // end parallel for - Kokkos::fence(); - - } //end view scope - } - -} - -// ----------------------------------------------------------------------------- -// This function calculates the corner forces and the evolves stress (hypo) -//------------------------------------------------------------------------------ -void FEA_Module_SGH::get_force_dgradient_sgh(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &elem_mat_id, - const double rk_alpha, - const size_t cycle - ) { - - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - const size_t num_dims = simparam->num_dims; - // --- calculate the forces acting on the nodes from the element --- - for (size_t elem_gid = 0; elem_gid < rnum_elem; elem_gid++){ - //FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { - - const size_t num_nodes_in_elem = 8; - real_t gradient_result[num_dims]; - // total Cauchy stress - double tau_array[9]; - double tau_gradient_array[9]; - - // corner area normals - double area_normal_array[24]; - - // estimate of shock direction - double shock_dir_array[3]; - - // the sums in the Riemann solver - double sum_array[4]; - double sum_gradient_array[4]; - - // corner shock impeadance x |corner area normal dot shock_dir| - double muc_array[8]; - double muc_gradient_array[8]; - - // Riemann velocity - double vel_star_array[3]; - double vel_star_gradient_array[3]; - - // velocity gradient - double vel_grad_array[9]; - - // --- Create views of arrays to aid the force calculation --- - - ViewCArrayKokkos tau(tau_array, num_dims, num_dims); - ViewCArrayKokkos tau_gradient(tau_gradient_array, num_dims, num_dims); - ViewCArrayKokkos area_normal(area_normal_array, num_nodes_in_elem, num_dims); - ViewCArrayKokkos shock_dir(shock_dir_array, num_dims); - ViewCArrayKokkos sum(sum_array, 4); - ViewCArrayKokkos sum_gradient(sum_gradient_array, 4); - ViewCArrayKokkos muc(muc_array, num_nodes_in_elem); - ViewCArrayKokkos muc_gradient(muc_gradient_array, num_nodes_in_elem); - ViewCArrayKokkos vel_star(vel_star_array, num_dims); - ViewCArrayKokkos vel_star_gradient(vel_star_gradient_array, num_dims); - ViewCArrayKokkos vel_grad(vel_grad_array, num_dims, num_dims); - - EOSParent* eos_model = elem_eos(elem_gid).model; - - // --- abviatations of variables --- - - // element volume - double vol = elem_vol(elem_gid); - - // create a view of the stress_matrix - ViewCArrayKokkos stress(&elem_stress(rk_level, elem_gid, 0,0), 3, 3); - - - // cut out the node_gids for this element - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 8); - - - - // get the B matrix which are the OUTWARD corner area normals - get_bmatrix(area_normal, - elem_gid, - node_coords, - elem_node_gids, - rk_level); - - - // --- Calculate the velocity gradient --- - get_velgrad(vel_grad, - elem_node_gids, - node_vel, - area_normal, - vol, - elem_gid, - rk_level); - - - // the -1 is for the inward surface area normal, - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - for (size_t dim = 0; dim < num_dims; dim++){ - area_normal(node_lid, dim) = (-1.0)*area_normal(node_lid,dim); - } // end for - } // end for - - - - double div = elem_div(elem_gid); - - - // vel = [u,v,w] - // [du/dx, du/dy, du/dz] - // vel_grad = [dv/dx, dv/dy, dv/dz] - // [dw/dx, dw/dy, dw/dz] - double curl[3]; - curl[0] = vel_grad(2,1) - vel_grad(1,2); // dw/dy - dv/dz - curl[1] = vel_grad(0,2) - vel_grad(2,0); // du/dz - dw/dx - curl[2] = vel_grad(1,0) - vel_grad(0,1); // dv/dx - du/dy - - double mag_curl = sqrt(curl[0]*curl[0] + curl[1]*curl[1] + curl[2]*curl[2]); - - - // --- Calculate the Cauchy stress --- - for (size_t i = 0; i < 3; i++){ - for (size_t j = 0; j < 3; j++){ - tau(i, j) = stress(i,j); - tau_gradient(i,j) = 0; - } // end for - } //end for - - // add the pressure - for (int i = 0; i < num_dims; i++){ - tau(i, i) -= elem_pres(elem_gid); - tau_gradient(i, i) -= elem_pres(elem_gid)/num_nodes_in_elem/relative_element_densities(elem_gid); - } // end for - - - - - // ---- Multidirectional Approximate Riemann solver (MARS) ---- - // find the average velocity of the elem, it is an - // estimate of the Riemann velocity - - // initialize to Riemann velocity to zero - for (size_t dim = 0; dim < num_dims; dim++){ - vel_star(dim) = 0.0; - } - - // loop over nodes and calculate an average velocity, which is - // an estimate of Riemann velocity - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - - // Get node gloabl index and create view of nodal velocity - int node_gid = nodes_in_elem(elem_gid, node_lid); - - - ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); - - vel_star(0) += 0.125*vel(0); - vel_star(1) += 0.125*vel(1); - vel_star(2) += 0.125*vel(2); - - } // end for loop over nodes - - // find shock direction and shock impedance associated with each node - - // initialize sum term in MARS to zero - for (int i = 0; i < 4; i++){ - sum(i) = sum_gradient(i) = 0.0; - } - - double mag; // magnitude of the area normal - double mag_vel; // magnitude of velocity - - // loop over the nodes of the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { - - // Get global node id - size_t node_gid = nodes_in_elem(elem_gid, node_lid); - - // Create view of nodal velocity - ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); - - // Get an estimate of the shock direction. - mag_vel = sqrt( (vel(0) - vel_star(0) )*(vel(0) - vel_star(0) ) - + (vel(1) - vel_star(1) )*(vel(1) - vel_star(1) ) - + (vel(2) - vel_star(2) )*(vel(2) - vel_star(2) ) ); - - - if (mag_vel > small) { - - // estimate of the shock direction, a unit normal - for (int dim = 0; dim < num_dims; dim++){ - shock_dir(dim) = (vel(dim) - vel_star(dim)) / mag_vel; - } - } - - else { - - // if there is no velocity change, then use the surface area - // normal as the shock direction - mag = sqrt( area_normal(node_lid, 0)*area_normal(node_lid, 0) - + area_normal(node_lid, 1)*area_normal(node_lid, 1) - + area_normal(node_lid, 2)*area_normal(node_lid, 2) ); - - // estimate of the shock direction - for (int dim = 0; dim < num_dims; dim++){ - shock_dir(dim) = area_normal(node_lid, dim)/mag; - } - - } // end if mag_vel - - - // cell divergence indicates compression or expansions - size_t mat_id = elem_mat_id(elem_gid); - if (div < 0){ // element in compression - muc(node_lid) = elem_den(elem_gid) * - (material(mat_id).q1*elem_sspd(elem_gid) + material(mat_id).q2*mag_vel); - - muc_gradient(node_lid) = elem_den(elem_gid)/relative_element_densities(elem_gid)/num_nodes_in_elem * - (material(mat_id).q1*elem_sspd(elem_gid) + material(mat_id).q2*mag_vel); - } - else { // element in expansion - muc(node_lid) = elem_den(elem_gid) * - (material(mat_id).q1ex*elem_sspd(elem_gid) + material(mat_id).q2ex*mag_vel); - muc_gradient(node_lid) = elem_den(elem_gid)/relative_element_densities(elem_gid)/num_nodes_in_elem * - (material(mat_id).q1ex*elem_sspd(elem_gid) + material(mat_id).q2ex*mag_vel); - } // end if on divergence sign - - - size_t use_shock_dir = 0; - double mu_term; - double mu_term_gradient; - - // Coding to use shock direction - if (use_shock_dir == 1){ - // this is denominator of the Riamann solver and the multiplier - // on velocity in the numerator. It filters on the shock - // direction - mu_term = muc(node_lid)* - fabs( shock_dir(0)*area_normal(node_lid,0) - + shock_dir(1)*area_normal(node_lid,1) - + shock_dir(2)*area_normal(node_lid,2) ); - mu_term_gradient = muc_gradient(node_lid)* //amend if shock dir has dependence for gradient not captured here - fabs( shock_dir(0)*area_normal(node_lid,0) - + shock_dir(1)*area_normal(node_lid,1) - + shock_dir(2)*area_normal(node_lid,2) ); - } - else { - // Using a full tensoral Riemann jump relation - mu_term = muc(node_lid) - * sqrt( area_normal(node_lid, 0)*area_normal(node_lid, 0) - + area_normal(node_lid, 1)*area_normal(node_lid, 1) - + area_normal(node_lid, 2)*area_normal(node_lid, 2) ); - mu_term_gradient = muc_gradient(node_lid) - * sqrt( area_normal(node_lid, 0)*area_normal(node_lid, 0) - + area_normal(node_lid, 1)*area_normal(node_lid, 1) - + area_normal(node_lid, 2)*area_normal(node_lid, 2) ); - } - - sum(0) += mu_term*vel(0); - sum(1) += mu_term*vel(1); - sum(2) += mu_term*vel(2); - sum(3) += mu_term; - - sum_gradient(0) += mu_term_gradient*vel(0); - sum_gradient(1) += mu_term_gradient*vel(1); - sum_gradient(2) += mu_term_gradient*vel(2); - sum_gradient(3) += mu_term_gradient; - - muc(node_lid) = mu_term; // the impeadance time surface area is stored here - muc_gradient(node_lid) = mu_term_gradient; - - } // end for node_lid loop over nodes of the elem - - - - - // The Riemann velocity, called vel_star - if (sum(3) > fuzz) { - for (size_t i = 0; i < num_dims; i++) { - vel_star(i) = sum(i)/sum(3); - vel_star_gradient(i) = sum_gradient(i)/sum(3) - sum_gradient(3)*sum(i)/sum(3)/sum(3); - } - } - else { - for (int i = 0; i < num_dims; i++){ - vel_star(i) = 0.0; - vel_star_gradient(i) = 0.0; - } - } // end if - - - - // ---- Calculate the shock detector for the Riemann-solver ---- - // - // The dissipation from the Riemann problem is limited by phi - // phi = (1. - max( 0., min( 1. , r_face ) ))^n - // where - // r_face = (C* div(u_+)/div(u_z)) - // The plus denotes the cell center divergence of a neighbor. - // The solution will be first order when phi=1 and have - // zero dissipation when phi=0. - // phi = 0 highest-order solution - // phi = 1 first order solution - // - - double phi = 0.0; // the shock detector - double r_face = 1.0; // the ratio on the face - double r_min = 1.0; // the min ratio for the cell - double r_coef = 0.9; // 0.9; the coefficient on the ratio - // (1=minmod and 2=superbee) - double n_coef = 1.0; // the power on the limiting coefficient - // (1=nominal, and n_coeff > 1 oscillatory) - - - // loop over the nieghboring cells - for (size_t elem_lid = 0; elem_lid < num_elems_in_elem(elem_gid); elem_lid++){ - - // Get global index for neighboring cell - size_t neighbor_gid = elems_in_elem(elem_gid, elem_lid); - - // calculate the velocity divergence in neighbor - double div_neighbor = elem_div(neighbor_gid); - - r_face = r_coef*(div_neighbor + small)/(div + small); - - // store the smallest face ratio - r_min = fmin(r_face, r_min); - - } // end for elem_lid - - - // calculate standard shock detector - phi = 1.0 - fmax(0.0, r_min); - phi = pow(phi, n_coef); - - // Mach number shock detector - double omega = 20.0;//20.0; // weighting factor on Mach number - double third = 1.0/3.0; - double c_length = pow(vol, third); // characteristic length - double alpha = fmin(1.0, omega * (c_length * fabs(div))/(elem_sspd(elem_gid) + fuzz) ); - - // use Mach based detector with standard shock detector - - // turn off dissipation in expansion - //alpha = fmax(-fabs(div0)/div0 * alpha, 0.0); // this should be if(div0<0) alpha=alpha else alpha=0 - - phi = alpha*phi; - - // curl limiter on Q - double phi_curl = fmin(1.0, 1.0*fabs(div)/(mag_curl + fuzz)); // disable Q when vorticity is high - //phi = phi_curl*phi; - phi = 1; - - // ---- Calculate the Riemann force on each node ---- - - // loop over the each node in the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { - - size_t corner_lid = node_lid; - - // Get corner gid - size_t corner_gid = corners_in_elem(elem_gid, corner_lid); - - // Get node gid - size_t node_gid = nodes_in_elem(elem_gid, node_lid); - - // loop over dimension - for (int dim = 0; dim < num_dims; dim++){ - - corner_vector_storage(corner_gid, dim) = - area_normal(node_lid, 0)*tau_gradient(0, dim) - + area_normal(node_lid, 1)*tau_gradient(1, dim) - + area_normal(node_lid, 2)*tau_gradient(2, dim) - + phi*muc_gradient(node_lid)*(vel_star(dim) - node_vel(rk_level, node_gid, dim)) - + phi*muc(node_lid)*(vel_star_gradient(dim)); - - } // end loop over dimension - - } // end for loop over nodes in elem - - - - // --- Update Stress --- - // calculate the new stress at the next rk level, if it is a hypo model - - size_t mat_id = elem_mat_id(elem_gid); - - - } - //}); // end parallel for loop over elements - - /* - //accumulate node values from corner storage - force_gradient_design->putScalar(0); - - vec_array force_gradient_design_view = force_gradient_design->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { - size_t corner_id; - for(int icorner=0; icorner < num_corners_in_node(node_id); icorner++){ - corner_id = corners_in_node(node_id,icorner); - force_gradient_design_view(node_id,0) += corner_vector_storage(corner_id, 0); - force_gradient_design_view(node_id,1) += corner_vector_storage(corner_id, 1); - force_gradient_design_view(node_id,2) += corner_vector_storage(corner_id, 2); - } - }); // end parallel for - Kokkos::fence(); - */ - - return; - -} // end of routine diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/geometry.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/geometry.cpp deleted file mode 100644 index bcbc38541..000000000 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/geometry.cpp +++ /dev/null @@ -1,1631 +0,0 @@ -// ----------------------------------------------------------------------------- -// This code handles the geometric information for the mesh for the SHG solver -//------------------------------------------------------------------------------ -#include "matar.h" -#include "state.h" -#include "FEA_Module_SGH.h" - -void FEA_Module_SGH::update_position_sgh(double rk_alpha, - const size_t num_nodes, - DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel){ - - const size_t rk_level = rk_num_bins - 1; - int num_dims = num_dim; - - // loop over all the nodes in the mesh - FOR_ALL_CLASS(node_gid, 0, num_nodes, { - - for (int dim = 0; dim < num_dims; dim++){ - double half_vel = (node_vel(rk_level, node_gid, dim) + node_vel(0, node_gid, dim))*0.5; - node_coords(rk_level, node_gid, dim) = node_coords(0, node_gid, dim) + rk_alpha*dt*half_vel; - } - - }); // end parallel for over nodes - -} // end subroutine - - -// ----------------------------------------------------------------------------- -// This function claculates -// B_p = J^{-T} \cdot (\nabla_{xi} \phi_p w -// where -// \phi_p is the basis function for vertex p -// w is the 1 gauss point for the cell (everything is evaluted at this point) -// J^{-T} is the inverse transpose of the Jacobi matrix -// \nabla_{xi} is the gradient opperator in the reference coordinates -// -// B_p is the OUTWARD corner area normal at node p -//------------------------------------------------------------------------------ -KOKKOS_FUNCTION -void FEA_Module_SGH::get_bmatrix(const ViewCArrayKokkos &B_matrix, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const { - - const size_t num_nodes = 8; - - double x_array[8]; - double y_array[8]; - double z_array[8]; - - // x, y, z coordinates of elem vertices - auto x = ViewCArrayKokkos (x_array, num_nodes); - auto y = ViewCArrayKokkos (y_array, num_nodes); - auto z = ViewCArrayKokkos (z_array, num_nodes); - - // get the coordinates of the nodes(rk,elem,node) in this element - for (int node_lid = 0; node_lid < num_nodes; node_lid++){ - x(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 0); - y(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 1); - z(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 2); - } // end for - - double twelth = 1./12.; - - B_matrix(0,0) = ( +y(1)*( -z(2) -z(3) +z(4) +z(5) ) - +y(2)*( +z(1) -z(3) ) - +y(3)*( +z(1) +z(2) -z(4) -z(7) ) - +y(4)*( -z(1) +z(3) -z(5) +z(7) ) - +y(5)*( -z(1) +z(4) ) - +y(7)*( +z(3) -z(4) ) )*twelth; - - B_matrix(1,0) = ( +y(0)*( +z(2) +z(3) -z(4) -z(5) ) - +y(2)*( -z(0) -z(3) +z(5) +z(6) ) - +y(3)*( -z(0) +z(2) ) - +y(4)*( +z(0) -z(5) ) - +y(5)*( +z(0) -z(2) +z(4) -z(6) ) - +y(6)*( -z(2) +z(5) ) )*twelth; - - B_matrix(2,0) = ( +y(0)*( -z(1) +z(3) ) - +y(1)*( +z(0) +z(3) -z(5) -z(6) ) - +y(3)*( -z(0) -z(1) +z(6) +z(7) ) - +y(5)*( +z(1) -z(6) ) - +y(6)*( +z(1) -z(3) +z(5) -z(7) ) - +y(7)*( -z(3) +z(6) ) )*twelth; - - B_matrix(3,0) = ( +y(0)*( -z(1) -z(2) +z(4) +z(7) ) - +y(1)*( +z(0) -z(2) ) - +y(2)*( +z(0) +z(1) -z(6) -z(7) ) - +y(4)*( -z(0) +z(7) ) - +y(6)*( +z(2) -z(7) ) - +y(7)*( -z(0) +z(2) -z(4) +z(6) ) )*twelth; - - B_matrix(4,0) = ( +y(0)*( +z(1) -z(3) +z(5) -z(7) ) - +y(1)*( -z(0) +z(5) ) - +y(3)*( +z(0) -z(7) ) - +y(5)*( -z(0) -z(1) +z(6) +z(7) ) - +y(6)*( -z(5) +z(7) ) - +y(7)*( +z(0) +z(3) -z(5) -z(6) ) )*twelth; - - B_matrix(5,0) = ( +y(0)*( +z(1) -z(4) ) - +y(1)*( -z(0) +z(2) -z(4) +z(6) ) - +y(2)*( -z(1) +z(6) ) - +y(4)*( +z(0) +z(1) -z(6) -z(7) ) - +y(6)*( -z(1) -z(2) +z(4) +z(7) ) - +y(7)*( +z(4) -z(6) ) )*twelth; - - B_matrix(6,0) = ( +y(1)*( +z(2) -z(5) ) - +y(2)*( -z(1) +z(3) -z(5) +z(7) ) - +y(3)*( -z(2) +z(7) ) - +y(4)*( +z(5) -z(7) ) - +y(5)*( +z(1) +z(2) -z(4) -z(7) ) - +y(7)*( -z(2) -z(3) +z(4) +z(5) ) )*twelth; - - B_matrix(7,0) = ( +y(0)*( -z(3) +z(4) ) - +y(2)*( +z(3) -z(6) ) - +y(3)*( +z(0) -z(2) +z(4) -z(6) ) - +y(4)*( -z(0) -z(3) +z(5) +z(6) ) - +y(5)*( -z(4) +z(6) ) - +y(6)*( +z(2) +z(3) -z(4) -z(5) ) )*twelth; - - B_matrix(0,1) = ( +z(1)*( -x(2) -x(3) +x(4) +x(5) ) - +z(2)*( +x(1) -x(3) ) - +z(3)*( +x(1) +x(2) -x(4) -x(7) ) - +z(4)*( -x(1) +x(3) -x(5) +x(7) ) - +z(5)*( -x(1) +x(4) ) - +z(7)*( +x(3) -x(4) ) )*twelth; - - B_matrix(1,1) = ( +z(0)*( +x(2) +x(3) -x(4) -x(5) ) - +z(2)*( -x(0) -x(3) +x(5) +x(6) ) - +z(3)*( -x(0) +x(2) ) - +z(4)*( +x(0) -x(5) ) - +z(5)*( +x(0) -x(2) +x(4) -x(6) ) - +z(6)*( -x(2) +x(5) ) )*twelth; - - B_matrix(2,1) = ( +z(0)*( -x(1) +x(3) ) - +z(1)*( +x(0) +x(3) -x(5) -x(6) ) - +z(3)*( -x(0) -x(1) +x(6) +x(7) ) - +z(5)*( +x(1) -x(6) ) - +z(6)*( +x(1) -x(3) +x(5) -x(7) ) - +z(7)*( -x(3) +x(6) ) )*twelth; - - B_matrix(3,1) = ( +z(0)*( -x(1) -x(2) +x(4) +x(7) ) - +z(1)*( +x(0) -x(2) ) - +z(2)*( +x(0) +x(1) -x(6) -x(7) ) - +z(4)*( -x(0) +x(7) ) - +z(6)*( +x(2) -x(7) ) - +z(7)*( -x(0) +x(2) -x(4) +x(6) ) )*twelth; - - B_matrix(4,1) = ( +z(0)*( +x(1) -x(3) +x(5) -x(7) ) - +z(1)*( -x(0) +x(5) ) - +z(3)*( +x(0) -x(7) ) - +z(5)*( -x(0) -x(1) +x(6) +x(7) ) - +z(6)*( -x(5) +x(7) ) - +z(7)*( +x(0) +x(3) -x(5) -x(6) ) )*twelth; - - B_matrix(5,1) = ( +z(0)*( +x(1) -x(4) ) - +z(1)*( -x(0) +x(2) -x(4) +x(6) ) - +z(2)*( -x(1) +x(6) ) - +z(4)*( +x(0) +x(1) -x(6) -x(7) ) - +z(6)*( -x(1) -x(2) +x(4) +x(7) ) - +z(7)*( +x(4) -x(6) ) )*twelth; - - B_matrix(6,1) = ( +z(1)*( +x(2) -x(5) ) - +z(2)*( -x(1) +x(3) -x(5) +x(7) ) - +z(3)*( -x(2) +x(7) ) - +z(4)*( +x(5) -x(7) ) - +z(5)*( +x(1) +x(2) -x(4) -x(7) ) - +z(7)*( -x(2) -x(3) +x(4) +x(5) ) )*twelth; - - B_matrix(7,1) = ( +z(0)*( -x(3) +x(4) ) - +z(2)*( +x(3) -x(6) ) - +z(3)*( +x(0) -x(2) +x(4) -x(6) ) - +z(4)*( -x(0) -x(3) +x(5) +x(6) ) - +z(5)*( -x(4) +x(6) ) - +z(6)*( +x(2) +x(3) -x(4) -x(5) ) )*twelth; - - B_matrix(0,2) = ( +x(1)*( -y(2) -y(3) +y(4) +y(5) ) - +x(2)*( +y(1) -y(3) ) - +x(3)*( +y(1) +y(2) -y(4) -y(7) ) - +x(4)*( -y(1) +y(3) -y(5) +y(7) ) - +x(5)*( -y(1) +y(4) ) - +x(7)*( +y(3) -y(4) ) )*twelth; - - B_matrix(1,2) = ( +x(0)*( +y(2) +y(3) -y(4) -y(5) ) - +x(2)*( -y(0) -y(3) +y(5) +y(6) ) - +x(3)*( -y(0) +y(2) ) - +x(4)*( +y(0) -y(5) ) - +x(5)*( +y(0) -y(2) +y(4) -y(6) ) - +x(6)*( -y(2) +y(5) ) )*twelth; - - B_matrix(2,2) = ( +x(0)*( -y(1) +y(3) ) - +x(1)*( +y(0) +y(3) -y(5) -y(6) ) - +x(3)*( -y(0) -y(1) +y(6) +y(7) ) - +x(5)*( +y(1) -y(6) ) - +x(6)*( +y(1) -y(3) +y(5) -y(7) ) - +x(7)*( -y(3) +y(6) ) )*twelth; - - B_matrix(3,2) = ( +x(0)*( -y(1) -y(2) +y(4) +y(7) ) - +x(1)*( +y(0) -y(2) ) - +x(2)*( +y(0) +y(1) -y(6) -y(7) ) - +x(4)*( -y(0) +y(7) ) - +x(6)*( +y(2) -y(7) ) - +x(7)*( -y(0) +y(2) -y(4) +y(6) ) )*twelth; - - B_matrix(4,2) = ( +x(0)*( +y(1) -y(3) +y(5) -y(7) ) - +x(1)*( -y(0) +y(5) ) - +x(3)*( +y(0) -y(7) ) - +x(5)*( -y(0) -y(1) +y(6) +y(7) ) - +x(6)*( -y(5) +y(7) ) - +x(7)*( +y(0) +y(3) -y(5) -y(6) ) )*twelth; - - B_matrix(5,2) = ( +x(0)*( +y(1) -y(4) ) - +x(1)*( -y(0) +y(2) -y(4) +y(6) ) - +x(2)*( -y(1) +y(6) ) - +x(4)*( +y(0) +y(1) -y(6) -y(7) ) - +x(6)*( -y(1) -y(2) +y(4) +y(7) ) - +x(7)*( +y(4) -y(6) ) )*twelth; - - B_matrix(6,2) = ( +x(1)*( +y(2) -y(5) ) - +x(2)*( -y(1) +y(3) -y(5) +y(7) ) - +x(3)*( -y(2) +y(7) ) - +x(4)*( +y(5) -y(7) ) - +x(5)*( +y(1) +y(2) -y(4) -y(7) ) - +x(7)*( -y(2) -y(3) +y(4) +y(5) ) )*twelth; - - B_matrix(7,2) = ( +x(0)*( -y(3) +y(4) ) - +x(2)*( +y(3) -y(6) ) - +x(3)*( +y(0) -y(2) +y(4) -y(6) ) - +x(4)*( -y(0) -y(3) +y(5) +y(6) ) - +x(5)*( -y(4) +y(6) ) - +x(6)*( +y(2) +y(3) -y(4) -y(5) ) )*twelth; - -} // end subroutine - - -/* ---------------------------------------------------------------------------- - Compute Volume of each finite element -------------------------------------------------------------------------------- */ - -void FEA_Module_SGH::get_vol(){ - - const size_t rk_level = rk_num_bins - 1; - const size_t num_dims = num_dim; - - if (num_dims == 2){ - FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { - - // cut out the node_gids for this element - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 4); - get_vol_quad(elem_vol, elem_gid, node_coords, elem_node_gids, rk_level); - - }); - Kokkos::fence(); - } - else { - FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { - - // cut out the node_gids for this element - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 8); - get_vol_hex(elem_vol, elem_gid, node_coords, elem_node_gids, rk_level); - - }); - Kokkos::fence(); - } // end if - - return; - -} // end subroutine - - -// Exact volume for a hex element -KOKKOS_INLINE_FUNCTION -void FEA_Module_SGH::get_vol_hex(const DViewCArrayKokkos &elem_vol, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const { - - const size_t num_nodes = 8; - - double x_array[8]; - double y_array[8]; - double z_array[8]; - - // x, y, z coordinates of elem vertices - auto x = ViewCArrayKokkos (x_array, num_nodes); - auto y = ViewCArrayKokkos (y_array, num_nodes); - auto z = ViewCArrayKokkos (z_array, num_nodes); - - // get the coordinates of the nodes(rk,elem,node) in this element - for (int node_lid = 0; node_lid < num_nodes; node_lid++){ - x(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 0); - y(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 1); - z(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 2); - } // end for - - double twelth = 1./12.; - - // element volume - elem_vol(elem_gid) = - (x(1)*(y(3)*(-z(0) + z(2)) + y(4)*( z(0) - z(5)) + y(0)*(z(2) + z(3) - z(4) - z(5)) + y(6)*(-z(2) + z(5)) + y(5)*(z(0) - z(2) + z(4) - z(6)) + y(2)*(-z(0) - z(3) + z(5) + z(6))) + - x(7)*(y(0)*(-z(3) + z(4)) + y(6)*( z(2) + z(3) - z(4) - z(5)) + y(2)*(z(3) - z(6)) + y(3)*(z(0) - z(2) + z(4) - z(6)) + y(5)*(-z(4) + z(6)) + y(4)*(-z(0) - z(3) + z(5) + z(6))) + - x(3)*(y(1)*( z(0) - z(2)) + y(7)*(-z(0) + z(2) - z(4) + z(6)) + y(6)*(z(2) - z(7)) + y(2)*(z(0) + z(1) - z(6) - z(7)) + y(4)*(-z(0) + z(7)) + y(0)*(-z(1) - z(2) + z(4) + z(7))) + - x(5)*(y(0)*( z(1) - z(4)) + y(7)*( z(4) - z(6)) + y(2)*(-z(1) + z(6)) + y(1)*(-z(0) + z(2) - z(4) + z(6)) + y(4)*(z(0) + z(1) - z(6) - z(7)) + y(6)*(-z(1) - z(2) + z(4) + z(7))) + - x(6)*(y(1)*( z(2) - z(5)) + y(7)*(-z(2) - z(3) + z(4) + z(5)) + y(5)*(z(1) + z(2) - z(4) - z(7)) + y(4)*(z(5) - z(7)) + y(3)*(-z(2) + z(7)) + y(2)*(-z(1) + z(3) - z(5) + z(7))) + - x(0)*(y(2)*( z(1) - z(3)) + y(7)*( z(3) - z(4)) + y(5)*(-z(1) + z(4)) + y(1)*(-z(2) - z(3) + z(4) + z(5)) + y(3)*(z(1) + z(2) - z(4) - z(7)) + y(4)*(-z(1) + z(3) - z(5) + z(7))) + - x(2)*(y(0)*(-z(1) + z(3)) + y(5)*( z(1) - z(6)) + y(1)*(z(0) + z(3) - z(5) - z(6)) + y(7)*(-z(3) + z(6)) + y(6)*(z(1) - z(3) + z(5) - z(7)) + y(3)*(-z(0) - z(1) + z(6) + z(7))) + - x(4)*(y(1)*(-z(0) + z(5)) + y(7)*( z(0) + z(3) - z(5) - z(6)) + y(3)*(z(0) - z(7)) + y(0)*(z(1) - z(3) + z(5) - z(7)) + y(6)*(-z(5) + z(7)) + y(5)*(-z(0) - z(1) + z(6) + z(7))))*twelth; - - return; - -} // end subroutine - - - -KOKKOS_FUNCTION -void FEA_Module_SGH::get_bmatrix2D(const ViewCArrayKokkos &B_matrix, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const { - - const size_t num_nodes = 4; - - double x_array[4]; - double y_array[4]; - - // x, y coordinates of elem vertices - auto x = ViewCArrayKokkos (x_array, num_nodes); - auto y = ViewCArrayKokkos (y_array, num_nodes); - - // get the coordinates of the nodes(rk,elem,node) in this element - for (int node_lid = 0; node_lid < num_nodes; node_lid++){ - x(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 0); - y(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 1); - } // end for - - /* ensight node order 0 1 2 3 - Flanaghan node order 3 4 1 2 - */ - - - B_matrix(0,0) = -0.5*(y(3)-y(1)); - - B_matrix(1,0) = -0.5*(y(0)-y(2)); - - B_matrix(2,0) = -0.5*(y(1)-y(3)); - - B_matrix(3,0) = -0.5*(y(2)-y(0)); - - - B_matrix(0,1) = -0.5*(x(1)-x(3)); - - B_matrix(1,1) = -0.5*(x(2)-x(0)); - - B_matrix(2,1) = -0.5*(x(3)-x(1)); - - B_matrix(3,1) = -0.5*(x(0)-x(2)); - - - // - /* - The Flanagan and Belytschko paper has: - x y - node 1: 0.5*(y2 - y4) , 0.5*(x4 - x2) - node 2: 0.5*(y3 - y1) , 0.5*(x1 - x3) - node 3: 0.5*(y4 - y2) , 0.5*(x2 - x4) - node 4: 0.5*(y1 - y3) , 0.5*(x3 - x1) - - Ensight order would be - - node 2: 0.5*(y3 - y1) , 0.5*(x1 - x3) - node 3: 0.5*(y0 - y2) , 0.5*(x2 - x0) - node 0: 0.5*(y1 - y3) , 0.5*(x3 - x1) - node 1: 0.5*(y2 - y0) , 0.5*(x0 - x2) - - */ - - return; - -} // end subroutine - - -// true volume of a quad in RZ coords -KOKKOS_INLINE_FUNCTION -void FEA_Module_SGH::get_vol_quad(const DViewCArrayKokkos &elem_vol, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const { - - // --- testing here --- - /* - double test_vol = 0.0; - // getting the corner facial area - double corner_areas_array[4]; - ViewCArrayKokkos corner_areas(&corner_areas_array[0],4); - - get_area_weights2D(corner_areas, - elem_gid, - node_coords, - elem_node_gids, - rk_level); - - - for(size_t node_lid=0; node_lid<4; node_lid++){ - double y = node_coords(rk_level, elem_node_gids(node_lid), 1); // node radius - test_vol += corner_areas(node_lid)*y; - } // end for - - test_vol matches the Barlow volume formula - */ - // ------------------- - - - elem_vol(elem_gid) = 0.0; - - const size_t num_nodes = 4; - - double x_array[4]; - double y_array[4]; - - // x, y coordinates of elem vertices - auto x = ViewCArrayKokkos (x_array, num_nodes); - auto y = ViewCArrayKokkos (y_array, num_nodes); - - // get the coordinates of the nodes(rk,elem,node) in this element - for (int node_lid = 0; node_lid < num_nodes; node_lid++){ - x(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 0); - y(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 1); - } // end for - - /* ensight node order 0 1 2 3 - Flanaghan node order 3 4 1 2 - */ - elem_vol(elem_gid) = - ( (y(2)+y(3)+y(0))*((y(2)-y(3))*(x(0)-x(3))-(y(0)-y(3))*(x(2)-x(3)) ) - + (y(0)+y(1)+y(2))*((y(0)-y(1))*(x(2)-x(1))-(y(2)-y(1))*(x(0)-x(1))) )/6.0; - - return; - -} // end subroutine - - -// element facial area -KOKKOS_FUNCTION -double FEA_Module_SGH::get_area_quad(const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const { - - double elem_area=0.0; - - const size_t num_nodes = 4; - - double x_array[4]; - double y_array[4]; - - // x, y coordinates of elem vertices - auto x = ViewCArrayKokkos (x_array, num_nodes); - auto y = ViewCArrayKokkos (y_array, num_nodes); - - // get the coordinates of the nodes(rk,elem,node) in this element - for (int node_lid = 0; node_lid < num_nodes; node_lid++){ - x(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 0); - y(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 1); - } // end for - - /* ensight node order 0 1 2 3 - Flanaghan node order 3 4 1 2 - */ - - // element facial area - elem_area = 0.5*((x(0)-x(2))*(y(1)-y(3))+(x(3)-x(1))*(y(0)-y(2))); - - return elem_area; - -} // end subroutine - - -KOKKOS_INLINE_FUNCTION -double FEA_Module_SGH::heron(const double x1, - const double y1, - const double x2, - const double y2, - const double x3, - const double y3) const -{ - double S,a,b,c,area; - - S=0.0; - a=sqrt((x2-x1)*(x2-x1)+(y2-y1)*(y2-y1)); - S+=a; - b=sqrt((x3-x2)*(x3-x2)+(y3-y2)*(y3-y2)); - S+=b; - c=sqrt((x3-x1)*(x3-x1)+(y3-y1)*(y3-y1)); - S+=c; - - S*=0.5; - area=sqrt(S*(S-a)*(S-b)*(S-c)); - - return area; -} - - - -KOKKOS_FUNCTION -void FEA_Module_SGH::get_area_weights2D(const ViewCArrayKokkos &corner_areas, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const { - - const size_t num_nodes = 4; - - double x_array[4]; - double y_array[4]; - - double rc,zc; - double A12,A23,A34,A41; - - // x, y coordinates of elem vertices - ViewCArrayKokkos x(x_array, num_nodes); - ViewCArrayKokkos y(y_array, num_nodes); - - // get the coordinates of the nodes(rk,elem,node) in this element - rc=zc=0.0; - for (int node_lid = 0; node_lid < num_nodes; node_lid++){ - x(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 0); - y(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 1); - rc+=0.25*y(node_lid); - zc+=0.25*x(node_lid); - } // end for - - /* ensight node order 0 1 2 3 - Barlow node order 1 2 3 4 - */ - - A12 = heron(x(0),y(0),zc,rc,x(1),y(1)); - A23 = heron(x(1),y(1),zc,rc,x(2),y(2)); - A34 = heron(x(2),y(2),zc,rc,x(3),y(3)); - A41 = heron(x(3),y(3),zc,rc,x(0),y(0)); - - corner_areas(0)=(5.*A41+5.*A12+A23+A34)/12.; - corner_areas(1)=(A41+5.*A12+5.*A23+A34)/12.; - corner_areas(2)=(A41+A12+5.*A23+5.*A34)/12.; - corner_areas(3)=(5.*A41+A12+A23+5.*A34)/12.; - - - - return; - -} // end subroutine - -/* ----------------------------------------------------------------------------------- - Compute Gradient of the Volume of each finite element with respect to displacement --------------------------------------------------------------------------------------- */ - -void FEA_Module_SGH::get_vol_ugradient(const size_t gradient_node_id, const size_t gradient_dim){ - - const size_t rk_level = rk_num_bins - 1; - const size_t num_dims = num_dim; - - if (num_dims == 2){ - FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { - - // cut out the node_gids for this element - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 4); - get_vol_quad(elem_vol, elem_gid, node_coords, elem_node_gids, rk_level); - - }); - Kokkos::fence(); - } - else { - FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { - // cut out the node_gids for this element - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 8); - //get_vol_hex_ugradient(elem_vol, elem_gid, node_coords, elem_node_gids, rk_level); - - }); - Kokkos::fence(); - } // end if - - return; - -} // end subroutine - - -// Exact volume for a hex element -KOKKOS_FUNCTION -void FEA_Module_SGH::get_vol_hex_ugradient(const ViewCArrayKokkos &elem_vol_gradients, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const { - - const size_t num_nodes = 8; - const size_t num_dims = num_dim; - double x_array[8]; - double y_array[8]; - double z_array[8]; - double gradient_result; - - // x, y, z coordinates of elem vertices - auto x = ViewCArrayKokkos (x_array, num_nodes); - auto y = ViewCArrayKokkos (y_array, num_nodes); - auto z = ViewCArrayKokkos (z_array, num_nodes); - - // get the coordinates of the nodes(rk,elem,node) in this element - for (int node_lid = 0; node_lid < num_nodes; node_lid++){ - x(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 0); - y(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 1); - z(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 2); - } // end for - - double twelth = 1./12.; - - // element volume gradient - for(int inode = 0; inode < 8; inode++){ - for(int idim = 0; idim < num_dims; idim++){ - switch(num_dims*inode + idim){ - case 0: - gradient_result = - ((y(2)*( z(1) - z(3)) + y(7)*( z(3) - z(4)) + y(5)*(-z(1) + z(4)) + y(1)*(-z(2) - z(3) + z(4) + z(5)) + y(3)*(z(1) + z(2) - z(4) - z(7)) + y(4)*(-z(1) + z(3) - z(5) + z(7))))*twelth; - break; - case 3: - gradient_result = - ((y(3)*(-z(0) + z(2)) + y(4)*( z(0) - z(5)) + y(0)*(z(2) + z(3) - z(4) - z(5)) + y(6)*(-z(2) + z(5)) + y(5)*(z(0) - z(2) + z(4) - z(6)) + y(2)*(-z(0) - z(3) + z(5) + z(6))))*twelth; - break; - - case 6: - gradient_result = - ((y(0)*(-z(1) + z(3)) + y(5)*( z(1) - z(6)) + y(1)*(z(0) + z(3) - z(5) - z(6)) + y(7)*(-z(3) + z(6)) + y(6)*(z(1) - z(3) + z(5) - z(7)) + y(3)*(-z(0) - z(1) + z(6) + z(7))))*twelth; - break; - case 9: - gradient_result = - ((y(1)*( z(0) - z(2)) + y(7)*(-z(0) + z(2) - z(4) + z(6)) + y(6)*(z(2) - z(7)) + y(2)*(z(0) + z(1) - z(6) - z(7)) + y(4)*(-z(0) + z(7)) + y(0)*(-z(1) - z(2) + z(4) + z(7))))*twelth; - break; - case 12: - gradient_result = - ((y(1)*(-z(0) + z(5)) + y(7)*( z(0) + z(3) - z(5) - z(6)) + y(3)*(z(0) - z(7)) + y(0)*(z(1) - z(3) + z(5) - z(7)) + y(6)*(-z(5) + z(7)) + y(5)*(-z(0) - z(1) + z(6) + z(7))))*twelth; - break; - case 15: - gradient_result = - ((y(0)*( z(1) - z(4)) + y(7)*( z(4) - z(6)) + y(2)*(-z(1) + z(6)) + y(1)*(-z(0) + z(2) - z(4) + z(6)) + y(4)*(z(0) + z(1) - z(6) - z(7)) + y(6)*(-z(1) - z(2) + z(4) + z(7))))*twelth; - break; - case 18: - gradient_result = - ((y(1)*( z(2) - z(5)) + y(7)*(-z(2) - z(3) + z(4) + z(5)) + y(5)*(z(1) + z(2) - z(4) - z(7)) + y(4)*(z(5) - z(7)) + y(3)*(-z(2) + z(7)) + y(2)*(-z(1) + z(3) - z(5) + z(7))))*twelth; - break; - case 21: - gradient_result = - ((y(0)*(-z(3) + z(4)) + y(6)*( z(2) + z(3) - z(4) - z(5)) + y(2)*(z(3) - z(6)) + y(3)*(z(0) - z(2) + z(4) - z(6)) + y(5)*(-z(4) + z(6)) + y(4)*(-z(0) - z(3) + z(5) + z(6))))*twelth; - break; - case 1: - gradient_result = - (x(1)*((z(2) + z(3) - z(4) - z(5))) + - x(7)*((-z(3) + z(4))) + - x(3)*((-z(1) - z(2) + z(4) + z(7))) + - x(5)*(( z(1) - z(4))) + - x(2)*((-z(1) + z(3))) + - x(4)*((z(1) - z(3) + z(5) - z(7))))*twelth; - break; - case 4: - gradient_result = - (x(3)*(( z(0) - z(2))) + - x(5)*((-z(0) + z(2) - z(4) + z(6)))+ - x(6)*(( z(2) - z(5))) + - x(0)*((-z(2) - z(3) + z(4) + z(5))) + - x(2)*((z(0) + z(3) - z(5) - z(6))) + - x(4)*((-z(0) + z(5))))*twelth; - break; - case 7: - gradient_result = - (x(1)*((-z(0) - z(3) + z(5) + z(6))) + - x(7)*((z(3) - z(6))) + - x(3)*((z(0) + z(1) - z(6) - z(7))) + - x(5)*((-z(1) + z(6))) + - x(6)*((-z(1) + z(3) - z(5) + z(7))) + - x(0)*(( z(1) - z(3))))*twelth; - break; - case 10: - gradient_result = - (x(1)*((-z(0) + z(2))) + - x(7)*((z(0) - z(2) + z(4) - z(6))) + - x(6)*((-z(2) + z(7))) + - x(0)*((z(1) + z(2) - z(4) - z(7))) + - x(2)*((-z(0) - z(1) + z(6) + z(7))) + - x(4)*((z(0) - z(7))))*twelth; - break; - case 13: - gradient_result = - (x(1)*(( z(0) - z(5))) + - x(7)*((-z(0) - z(3) + z(5) + z(6))) + - x(3)*((-z(0) + z(7))) + - x(5)*((z(0) + z(1) - z(6) - z(7))) + - x(6)*((z(5) - z(7))) + - x(0)*((-z(1) + z(3) - z(5) + z(7))))*twelth; - break; - case 16: - gradient_result = - (x(1)*((z(0) - z(2) + z(4) - z(6))) + - x(7)*((-z(4) + z(6))) + - x(6)*((z(1) + z(2) - z(4) - z(7))) + - x(0)*((-z(1) + z(4))) + - x(2)*(( z(1) - z(6))) + - x(4)*((-z(0) - z(1) + z(6) + z(7))))*twelth; - break; - case 19: - gradient_result = - (x(1)*((-z(2) + z(5))) + - x(7)*(( z(2) + z(3) - z(4) - z(5))) + - x(3)*((z(2) - z(7))) + - x(5)*((-z(1) - z(2) + z(4) + z(7))) + - x(2)*((z(1) - z(3) + z(5) - z(7))) + - x(4)*((-z(5) + z(7))))*twelth; - break; - case 22: - gradient_result = - (x(3)*((-z(0) + z(2) - z(4) + z(6))) + - x(5)*(( z(4) - z(6))) + - x(6)*((-z(2) - z(3) + z(4) + z(5))) + - x(0)*(( z(3) - z(4))) + - x(2)*((-z(3) + z(6))) + - x(4)*(( z(0) + z(3) - z(5) - z(6))))*twelth; - break; - case 2: - gradient_result = - (x(1)*(-y(3) + y(4) + y(5) - y(2)) + - x(7)*(y(3) - y(4)) + - x(3)*(y(1) - y(7) + y(2) - y(4)) + - x(5)*(-y(1) + y(4)) + - x(2)*(y(1) - y(3)) + - x(4)*(-y(1) + y(7) + y(3) - y(5)))*twelth; - break; - case 5: - gradient_result = - (x(3)*(y(2) - y(0)) + - x(5)*(y(0) - y(2) + y(4) - y(6)) + - x(6)*(y(5) - y(2)) + - x(0)*(y(2) - y(5) + y(3) - y(4)) + - x(2)*(-y(0) + y(5) + y(6) - y(3)) + - x(4)*(y(0) - y(5)))*twelth; - break; - case 8: - gradient_result = - (x(1)*(y(3) + y(0) - y(6) - y(5)) + - x(7)*(y(6) - y(3)) + - x(3)*(-y(1)+ y(7) + y(6) - y(0)) + - x(5)*(y(1) - y(6)) + - x(6)*(y(1) - y(7) + y(5) - y(3)) + - x(0)*(-y(1) + y(3)))*twelth; - break; - case 11: - gradient_result = - (x(1)*(y(0) - y(2)) + - x(7)*(-y(0) + y(6) + y(2) - y(4)) + - x(6)*(-y(7) + y(2)) + - x(0)*(-y(2) + y(7) - y(1) + y(4)) + - x(2)*(y(0) + y(1) - y(7) - y(6)) + - x(4)*(y(7) - y(0)))*twelth; - break; - case 14: - gradient_result = - (x(1)*(-y(0) + y(5)) + - x(7)*(y(0) - y(6) + y(3) - y(5)) + - x(3)*(-y(7)+ y(0)) + - x(5)*(-y(0) + y(7) - y(1)+ y(6)) + - x(6)*(y(7) - y(5)) + - x(0)*(-y(7) + y(5) + y(1) - y(3)))*twelth; - break; - case 17: - gradient_result = - (x(1)*(-y(4) - y(0) + y(6) + y(2)) + - x(7)*(-y(6) + y(4)) + - x(6)*(-y(1) + y(7)+ y(4) - y(2)) + - x(0)*(y(1) - y(4)) + - x(2)*(-y(1) + y(6)) + - x(4)*(y(1) - y(7) + y(0) - y(6)))*twelth; - break; - case 20: - gradient_result = - (x(1)*(-y(5) + y(2)) + - x(7)*(-y(2) - y(3) + y(5) + y(4)) + - x(3)*(y(7) - y(2)) + - x(5)*(-y(7) + y(2) + y(1) - y(4)) + - x(2)*(-y(5) - y(1) + y(7) + y(3)) + - x(4)*(-y(7) + y(5)))*twelth; - break; - case 23: - gradient_result = - (x(3)*(-y(6) - y(2) + y(4) + y(0)) + - x(5)*(-y(4) + y(6)) + - x(6)*(-y(5) - y(4) + y(3) + y(2)) + - x(0)*(-y(3) + y(4)) + - x(2)*(-y(6) + y(3)) + - x(4)*(-y(3) - y(0) + y(6) + y(5)))*twelth; - break; - } - elem_vol_gradients(inode, idim) = gradient_result; - } - } - return; - -} // end subroutine - -KOKKOS_FUNCTION -void FEA_Module_SGH::get_bmatrix_gradients(const ViewCArrayKokkos &B_matrix_gradients, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const { - - const size_t num_nodes = 8; - - double x_array[8]; - double y_array[8]; - double z_array[8]; - double gradient_terms_array[2*8]; - - // x, y, z coordinates of elem vertices - auto x = ViewCArrayKokkos (x_array, num_nodes); - auto y = ViewCArrayKokkos (y_array, num_nodes); - auto z = ViewCArrayKokkos (z_array, num_nodes); - auto gradient_terms = ViewCArrayKokkos (gradient_terms_array, 2, num_nodes); - - // get the coordinates of the nodes(rk,elem,node) in this element - for (int node_lid = 0; node_lid < num_nodes; node_lid++){ - x(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 0); - y(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 1); - z(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 2); - } // end for - - double twelth = 1./12.; - - - // B_matrix(0,0) = ( +y(1)*( -z(2) -z(3) +z(4) +z(5) ) - // +y(2)*( +z(1) -z(3) ) - // +y(3)*( +z(1) +z(2) -z(4) -z(7) ) - // +y(4)*( -z(1) +z(3) -z(5) +z(7) ) - // +y(5)*( -z(1) +z(4) ) - // +y(7)*( +z(3) -z(4) ) )*twelth; - - //y derivative - gradient_terms(0,0) = 0; - gradient_terms(0,1) = -z(2) - z(3) + z(4) + z(5); - gradient_terms(0,2) = z(1) - z(3); - gradient_terms(0,3) = z(1) + z(2) - z(4) - z(7); - gradient_terms(0,4) = -z(1) + z(3) - z(5) + z(7); - gradient_terms(0,5) = -z(1) + z(4); - gradient_terms(0,6) = 0; - gradient_terms(0,7) = z(3) - z(4); - - //z derivative - gradient_terms(1,0) = 0; - gradient_terms(1,1) = y(2) + y(3) - y(4) - y(5); - gradient_terms(1,2) = -y(1) + y(3); - gradient_terms(1,3) = -y(1) - y(2) +y(4) +y(7); - gradient_terms(1,4) = y(1) - y(3) + y(5) - y(7); - gradient_terms(1,5) = y(1) - y(4); - gradient_terms(1,6) = 0; - gradient_terms(1,7) = -y(3) + y(4); - - for(int inode = 0; inode < 8; inode++){ - B_matrix_gradients(0,0,inode,0) = 0; - B_matrix_gradients(0,0,inode,1) = gradient_terms(0,inode)*twelth; - B_matrix_gradients(0,0,inode,2) = gradient_terms(1,inode)*twelth; - } - - // B_matrix(1,0) = ( +y(0)*( +z(2) +z(3) -z(4) -z(5) ) - // +y(2)*( -z(0) -z(3) +z(5) +z(6) ) - // +y(3)*( -z(0) +z(2) ) - // +y(4)*( +z(0) -z(5) ) - // +y(5)*( +z(0) -z(2) +z(4) -z(6) ) - // +y(6)*( -z(2) +z(5) ) )*twelth; - - //y derivative - gradient_terms(0,0) = z(2) + z(3) - z(4) - z(5); - gradient_terms(0,1) = 0; - gradient_terms(0,2) = -z(0) -z(3) + z(5) + z(6); - gradient_terms(0,3) = -z(0) + z(2); - gradient_terms(0,4) = z(0) -z(5); - gradient_terms(0,5) = z(0) -z(2) + z(4) - z(6); - gradient_terms(0,6) = -z(2) + z(5); - gradient_terms(0,7) = 0; - - //z derivative - gradient_terms(1,0) = -y(2) - y(3) + y(4) + y(5); - gradient_terms(1,1) = 0; - gradient_terms(1,2) = y(0) + y(3) - y(5) - y(6); - gradient_terms(1,3) = y(0) - y(2); - gradient_terms(1,4) = -y(0) + y(5); - gradient_terms(1,5) = -y(0) + y(2) - y(4) + y(6); - gradient_terms(1,6) = y(2) - y(5); - gradient_terms(1,7) = 0; - - for(int inode = 0; inode < 8; inode++){ - B_matrix_gradients(1,0,inode,0) = 0; - B_matrix_gradients(1,0,inode,1) = gradient_terms(0,inode)*twelth; - B_matrix_gradients(1,0,inode,2) = gradient_terms(1,inode)*twelth; - } - - // B_matrix(2,0) = ( +y(0)*( -z(1) +z(3) ) - // +y(1)*( +z(0) +z(3) -z(5) -z(6) ) - // +y(3)*( -z(0) -z(1) +z(6) +z(7) ) - // +y(5)*( +z(1) -z(6) ) - // +y(6)*( +z(1) -z(3) +z(5) -z(7) ) - // +y(7)*( -z(3) +z(6) ) )*twelth; - - //y derivative - gradient_terms(0,0) = -z(1) + z(3); - gradient_terms(0,1) = z(0) + z(3) - z(5) - z(6); - gradient_terms(0,2) = 0; - gradient_terms(0,3) = -z(0) - z(1) + z(6) + z(7); - gradient_terms(0,4) = 0; - gradient_terms(0,5) = z(1) - z(6); - gradient_terms(0,6) = z(1) - z(3) + z(5) - z(7); - gradient_terms(0,7) = -z(3) + z(6); - - //z derivative - gradient_terms(1,0) = y(1) - y(3); - gradient_terms(1,1) = -y(0) - y(3) + y(5) + y(6); - gradient_terms(1,2) = 0; - gradient_terms(1,3) = y(0) + y(1) - y(6) - y(7); - gradient_terms(1,4) = 0; - gradient_terms(1,5) = -y(1) + y(6); - gradient_terms(1,6) = -y(1) + y(3) - y(5) + y(7); - gradient_terms(1,7) = y(3) - y(6); - - for(int inode = 0; inode < 8; inode++){ - B_matrix_gradients(2,0,inode,0) = 0; - B_matrix_gradients(2,0,inode,1) = gradient_terms(0,inode)*twelth; - B_matrix_gradients(2,0,inode,2) = gradient_terms(1,inode)*twelth; - } - - // B_matrix(3,0) = ( +y(0)*( -z(1) -z(2) +z(4) +z(7) ) - // +y(1)*( +z(0) -z(2) ) - // +y(2)*( +z(0) +z(1) -z(6) -z(7) ) - // +y(4)*( -z(0) +z(7) ) - // +y(6)*( +z(2) -z(7) ) - // +y(7)*( -z(0) +z(2) -z(4) +z(6) ) )*twelth; - - //y derivative - gradient_terms(0,0) = -z(1) - z(2) + z(4) + z(7); - gradient_terms(0,1) = z(0) - z(2); - gradient_terms(0,2) = z(0) + z(1) - z(6) - z(7); - gradient_terms(0,3) = 0; - gradient_terms(0,4) = -z(0) + z(7); - gradient_terms(0,5) = 0; - gradient_terms(0,6) = z(2) - z(7); - gradient_terms(0,7) = -z(0) + z(2) - z(4) + z(6); - - //z derivative - gradient_terms(1,0) = y(1) + y(2) - y(4) - y(7); - gradient_terms(1,1) = -y(0) + y(2); - gradient_terms(1,2) = -y(0) - y(1) + y(6) + y(7); - gradient_terms(1,3) = 0; - gradient_terms(1,4) = y(0) - y(7); - gradient_terms(1,5) = 0; - gradient_terms(1,6) = -y(2) + y(7); - gradient_terms(1,7) = y(0) - y(2) + y(4) - y(6); - - for(int inode = 0; inode < 8; inode++){ - B_matrix_gradients(3,0,inode,0) = 0; - B_matrix_gradients(3,0,inode,1) = gradient_terms(0,inode)*twelth; - B_matrix_gradients(3,0,inode,2) = gradient_terms(1,inode)*twelth; - } - - // B_matrix(4,0) = ( +y(0)*( +z(1) -z(3) +z(5) -z(7) ) - // +y(1)*( -z(0) +z(5) ) - // +y(3)*( +z(0) -z(7) ) - // +y(5)*( -z(0) -z(1) +z(6) +z(7) ) - // +y(6)*( -z(5) +z(7) ) - // +y(7)*( +z(0) +z(3) -z(5) -z(6) ) )*twelth; - - //y derivative - gradient_terms(0,0) = z(1) - z(3) + z(5) - z(7); - gradient_terms(0,1) = -z(0) + z(5); - gradient_terms(0,2) = 0; - gradient_terms(0,3) = z(0) - z(7); - gradient_terms(0,4) = 0; - gradient_terms(0,5) = -z(0) - z(1) + z(6) + z(7); - gradient_terms(0,6) = -z(5) + z(7); - gradient_terms(0,7) = z(0) + z(3) - z(5) - z(6); - - //z derivative - gradient_terms(1,0) = -y(1) + y(3) - y(5) + y(7); - gradient_terms(1,1) = y(0) - y(5); - gradient_terms(1,2) = 0; - gradient_terms(1,3) = -y(0) + y(7); - gradient_terms(1,4) = 0; - gradient_terms(1,5) = y(0) + y(1) - y(6) - y(7); - gradient_terms(1,6) = y(5) - y(7); - gradient_terms(1,7) = -y(0) - y(3) + y(5) + y(6); - - for(int inode = 0; inode < 8; inode++){ - B_matrix_gradients(4,0,inode,0) = 0; - B_matrix_gradients(4,0,inode,1) = gradient_terms(0,inode)*twelth; - B_matrix_gradients(4,0,inode,2) = gradient_terms(1,inode)*twelth; - } - - // B_matrix(5,0) = ( +y(0)*( +z(1) -z(4) ) - // +y(1)*( -z(0) +z(2) -z(4) +z(6) ) - // +y(2)*( -z(1) +z(6) ) - // +y(4)*( +z(0) +z(1) -z(6) -z(7) ) - // +y(6)*( -z(1) -z(2) +z(4) +z(7) ) - // +y(7)*( +z(4) -z(6) ) )*twelth; - - //y derivative - gradient_terms(0,0) = z(1) -z(4); - gradient_terms(0,1) = -z(0) +z(2) -z(4) +z(6); - gradient_terms(0,2) = -z(1) +z(6); - gradient_terms(0,3) = 0; - gradient_terms(0,4) = z(0) +z(1) -z(6) -z(7); - gradient_terms(0,5) = 0; - gradient_terms(0,6) = -z(1) -z(2) +z(4) +z(7); - gradient_terms(0,7) = z(4) -z(6); - - //z derivative - gradient_terms(1,0) = -y(1) + y(4); - gradient_terms(1,1) = y(0) - y(2) + y(4) - y(6); - gradient_terms(1,2) = y(1) - y(6); - gradient_terms(1,3) = 0; - gradient_terms(1,4) = -y(0) - y(1) + y(6) +y(7); - gradient_terms(1,5) = 0; - gradient_terms(1,6) = y(1) + y(2) - y(4) - y(7); - gradient_terms(1,7) = -y(4) + y(6); - - for(int inode = 0; inode < 8; inode++){ - B_matrix_gradients(5,0,inode,0) = 0; - B_matrix_gradients(5,0,inode,1) = gradient_terms(0,inode)*twelth; - B_matrix_gradients(5,0,inode,2) = gradient_terms(1,inode)*twelth; - } - - // B_matrix(6,0) = ( +y(1)*( +z(2) -z(5) ) - // +y(2)*( -z(1) +z(3) -z(5) +z(7) ) - // +y(3)*( -z(2) +z(7) ) - // +y(4)*( +z(5) -z(7) ) - // +y(5)*( +z(1) +z(2) -z(4) -z(7) ) - // +y(7)*( -z(2) -z(3) +z(4) +z(5) ) )*twelth; - - //y derivative - gradient_terms(0,0) = 0; - gradient_terms(0,1) = z(2) - z(5); - gradient_terms(0,2) = -z(1) + z(3) - z(5) + z(7); - gradient_terms(0,3) = -z(2) + z(7); - gradient_terms(0,4) = z(5) - z(7); - gradient_terms(0,5) = z(1) + z(2) - z(4) - z(7); - gradient_terms(0,6) = 0; - gradient_terms(0,7) = -z(2) - z(3) + z(4) + z(5); - - //z derivative - gradient_terms(1,0) = 0; - gradient_terms(1,1) = -y(2) + y(5); - gradient_terms(1,2) = y(1) - y(3) + y(5) - y(7); - gradient_terms(1,3) = y(2) - y(7); - gradient_terms(1,4) = -y(5) + y(7); - gradient_terms(1,5) = -y(1) - y(2) + y(4) + y(7); - gradient_terms(1,6) = 0; - gradient_terms(1,7) = y(2) + y(3) - y(4) - y(5); - - for(int inode = 0; inode < 8; inode++){ - B_matrix_gradients(6,0,inode,0) = 0; - B_matrix_gradients(6,0,inode,1) = gradient_terms(0,inode)*twelth; - B_matrix_gradients(6,0,inode,2) = gradient_terms(1,inode)*twelth; - } - - // B_matrix(7,0) = ( +y(0)*( -z(3) +z(4) ) - // +y(2)*( +z(3) -z(6) ) - // +y(3)*( +z(0) -z(2) +z(4) -z(6) ) - // +y(4)*( -z(0) -z(3) +z(5) +z(6) ) - // +y(5)*( -z(4) +z(6) ) - // +y(6)*( +z(2) +z(3) -z(4) -z(5) ) )*twelth; - - //y derivative - gradient_terms(0,0) = -z(3) +z(4); - gradient_terms(0,1) = 0; - gradient_terms(0,2) = z(3) -z(6); - gradient_terms(0,3) = z(0) -z(2) +z(4) -z(6); - gradient_terms(0,4) = -z(0) -z(3) +z(5) +z(6); - gradient_terms(0,5) = -z(4) +z(6); - gradient_terms(0,6) = z(2) +z(3) -z(4) -z(5); - gradient_terms(0,7) = 0; - - //z derivative - gradient_terms(1,0) = y(3) - y(4); - gradient_terms(1,1) = 0; - gradient_terms(1,2) = -y(3) + y(6); - gradient_terms(1,3) = -y(0) + y(2) - y(4) + y(6); - gradient_terms(1,4) = y(0) + y(3) - y(5) - y(6); - gradient_terms(1,5) = y(4) - y(6); - gradient_terms(1,6) = -y(2) - y(3) + y(4) + y(5); - gradient_terms(1,7) = 0; - - for(int inode = 0; inode < 8; inode++){ - B_matrix_gradients(7,0,inode,0) = 0; - B_matrix_gradients(7,0,inode,1) = gradient_terms(0,inode)*twelth; - B_matrix_gradients(7,0,inode,2) = gradient_terms(1,inode)*twelth; - } - - // B_matrix(0,1) = ( +z(1)*( -x(2) -x(3) +x(4) +x(5) ) - // +z(2)*( +x(1) -x(3) ) - // +z(3)*( +x(1) +x(2) -x(4) -x(7) ) - // +z(4)*( -x(1) +x(3) -x(5) +x(7) ) - // +z(5)*( -x(1) +x(4) ) - // +z(7)*( +x(3) -x(4) ) )*twelth; - - //x derivative - gradient_terms(0,0) = 0; - gradient_terms(0,1) = z(2) + z(3) - z(4) - z(5); - gradient_terms(0,2) = -z(1) + z(3); - gradient_terms(0,3) = -z(1) - z(2) + z(4) + z(7); - gradient_terms(0,4) = z(1) - z(3) + z(5) - z(7); - gradient_terms(0,5) = z(1) - z(4); - gradient_terms(0,6) = 0; - gradient_terms(0,7) = -z(3) + z(4); - - //z derivative - gradient_terms(1,0) = 0; - gradient_terms(1,1) = -x(2) - x(3) + x(4) + x(5); - gradient_terms(1,2) = x(1) - x(3); - gradient_terms(1,3) = x(1) + x(2) - x(4) - x(7); - gradient_terms(1,4) = -x(1) + x(3) - x(5) + x(7); - gradient_terms(1,5) = -x(1) + x(4); - gradient_terms(1,6) = 0; - gradient_terms(1,7) = x(3) - x(4); - - for(int inode = 0; inode < 8; inode++){ - B_matrix_gradients(0,1,inode,0) = gradient_terms(0,inode)*twelth; - B_matrix_gradients(0,1,inode,1) = 0; - B_matrix_gradients(0,1,inode,2) = gradient_terms(1,inode)*twelth; - } - - // B_matrix(1,1) = ( +z(0)*( +x(2) +x(3) -x(4) -x(5) ) - // +z(2)*( -x(0) -x(3) +x(5) +x(6) ) - // +z(3)*( -x(0) +x(2) ) - // +z(4)*( +x(0) -x(5) ) - // +z(5)*( +x(0) -x(2) +x(4) -x(6) ) - // +z(6)*( -x(2) +x(5) ) )*twelth; - - //x derivative - gradient_terms(0,0) = -z(2) - z(3) +z(4) + z(5); - gradient_terms(0,1) = 0; - gradient_terms(0,2) = z(0) + z(3) - z(5) - z(6); - gradient_terms(0,3) = z(0) - z(2); - gradient_terms(0,4) = -z(0) + z(5); - gradient_terms(0,5) = -z(0) + z(2) - z(4) + z(6); - gradient_terms(0,6) = z(2) - z(5); - gradient_terms(0,7) = 0; - - //z derivative - gradient_terms(1,0) = x(2) + x(3) - x(4) - x(5); - gradient_terms(1,1) = 0; - gradient_terms(1,2) = -x(0) -x(3) + x(5) + x(6); - gradient_terms(1,3) = -x(0) + x(2); - gradient_terms(1,4) = x(0) - x(5); - gradient_terms(1,5) = x(0) - x(2) + x(4) - x(6); - gradient_terms(1,6) = -x(2) + x(5); - gradient_terms(1,7) = 0; - - for(int inode = 0; inode < 8; inode++){ - B_matrix_gradients(1,1,inode,0) = gradient_terms(0,inode)*twelth; - B_matrix_gradients(1,1,inode,1) = 0; - B_matrix_gradients(1,1,inode,2) = gradient_terms(1,inode)*twelth; - } - - // B_matrix(2,1) = ( +z(0)*( -x(1) +x(3) ) - // +z(1)*( +x(0) +x(3) -x(5) -x(6) ) - // +z(3)*( -x(0) -x(1) +x(6) +x(7) ) - // +z(5)*( +x(1) -x(6) ) - // +z(6)*( +x(1) -x(3) +x(5) -x(7) ) - // +z(7)*( -x(3) +x(6) ) )*twelth; - - //x derivative - gradient_terms(0,0) = z(1) - z(3); - gradient_terms(0,1) = -z(0) - z(3) +z(5) + z(6); - gradient_terms(0,2) = 0; - gradient_terms(0,3) = z(0) + z(1) - z(6) - z(7); - gradient_terms(0,4) = 0; - gradient_terms(0,5) = -z(1) + z(6); - gradient_terms(0,6) = -z(1) + z(3) - z(5) + z(7); - gradient_terms(0,7) = z(3) - z(6); - - //z derivative - gradient_terms(1,0) = -x(1) + x(3); - gradient_terms(1,1) = x(0) + x(3) - x(5) - x(6); - gradient_terms(1,2) = 0; - gradient_terms(1,3) = -x(0) - x(1) + x(6) + x(7); - gradient_terms(1,4) = 0; - gradient_terms(1,5) = x(1) - x(6); - gradient_terms(1,6) = x(1) - x(3) + x(5) - x(7); - gradient_terms(1,7) = -x(3) + x(6); - - for(int inode = 0; inode < 8; inode++){ - B_matrix_gradients(2,1,inode,0) = gradient_terms(0,inode)*twelth; - B_matrix_gradients(2,1,inode,1) = 0; - B_matrix_gradients(2,1,inode,2) = gradient_terms(1,inode)*twelth; - } - - // B_matrix(3,1) = ( +z(0)*( -x(1) -x(2) +x(4) +x(7) ) - // +z(1)*( +x(0) -x(2) ) - // +z(2)*( +x(0) +x(1) -x(6) -x(7) ) - // +z(4)*( -x(0) +x(7) ) - // +z(6)*( +x(2) -x(7) ) - // +z(7)*( -x(0) +x(2) -x(4) +x(6) ) )*twelth; - - //x derivative - gradient_terms(0,0) = z(1) + z(2) - z(4) - z(7); - gradient_terms(0,1) = -z(0) + z(2); - gradient_terms(0,2) = -z(0) - z(1) + z(6) + z(7); - gradient_terms(0,3) = 0; - gradient_terms(0,4) = z(0) - z(7); - gradient_terms(0,5) = 0; - gradient_terms(0,6) = -z(2) + z(7); - gradient_terms(0,7) = z(0) - z(2) + z(4) - z(6); - - //z derivative - gradient_terms(1,0) = -x(1) - x(2) + x(4) + x(7); - gradient_terms(1,1) = x(0) - x(2); - gradient_terms(1,2) = x(0) + x(1) - x(6) - x(7); - gradient_terms(1,3) = 0; - gradient_terms(1,4) = -x(0) + x(7); - gradient_terms(1,5) = 0; - gradient_terms(1,6) = x(2) - x(7); - gradient_terms(1,7) = -x(0) + x(2) - x(4) + x(6); - - for(int inode = 0; inode < 8; inode++){ - B_matrix_gradients(3,1,inode,0) = gradient_terms(0,inode)*twelth; - B_matrix_gradients(3,1,inode,1) = 0; - B_matrix_gradients(3,1,inode,2) = gradient_terms(1,inode)*twelth; - } - - // B_matrix(4,1) = ( +z(0)*( +x(1) -x(3) +x(5) -x(7) ) - // +z(1)*( -x(0) +x(5) ) - // +z(3)*( +x(0) -x(7) ) - // +z(5)*( -x(0) -x(1) +x(6) +x(7) ) - // +z(6)*( -x(5) +x(7) ) - // +z(7)*( +x(0) +x(3) -x(5) -x(6) ) )*twelth; - - //x derivative - gradient_terms(0,0) = -z(1) + z(3) - z(5) + z(7); - gradient_terms(0,1) = z(0) - z(5); - gradient_terms(0,2) = 0; - gradient_terms(0,3) = -z(0) + z(7); - gradient_terms(0,4) = 0; - gradient_terms(0,5) = z(0) + z(1) - z(6) - z(7); - gradient_terms(0,6) = z(5) - z(7); - gradient_terms(0,7) = -z(0) - z(3) + z(5) + z(6); - - //z derivative - gradient_terms(1,0) = x(1) - x(3) + x(5) - x(7); - gradient_terms(1,1) = -x(0) + x(5); - gradient_terms(1,2) = 0; - gradient_terms(1,3) = x(0) -x(7); - gradient_terms(1,4) = 0; - gradient_terms(1,5) = -x(0) - x(1) + x(6) + x(7); - gradient_terms(1,6) = -x(5) + x(7); - gradient_terms(1,7) = x(0) + x(3) - x(5) - x(6); - - for(int inode = 0; inode < 8; inode++){ - B_matrix_gradients(4,1,inode,0) = gradient_terms(0,inode)*twelth; - B_matrix_gradients(4,1,inode,1) = 0; - B_matrix_gradients(4,1,inode,2) = gradient_terms(1,inode)*twelth; - } - - // B_matrix(5,1) = ( +z(0)*( +x(1) -x(4) ) - // +z(1)*( -x(0) +x(2) -x(4) +x(6) ) - // +z(2)*( -x(1) +x(6) ) - // +z(4)*( +x(0) +x(1) -x(6) -x(7) ) - // +z(6)*( -x(1) -x(2) +x(4) +x(7) ) - // +z(7)*( +x(4) -x(6) ) )*twelth; - - //x derivative - gradient_terms(0,0) = -z(1) + z(4); - gradient_terms(0,1) = z(0) - z(2) + z(4) - z(6); - gradient_terms(0,2) = z(1) - z(6); - gradient_terms(0,3) = 0; - gradient_terms(0,4) = -z(0) - z(1) + z(6) + z(7); - gradient_terms(0,5) = 0; - gradient_terms(0,6) = z(1) + z(2) - z(4) - z(7); - gradient_terms(0,7) = -z(4) + z(6); - - //z derivative - gradient_terms(1,0) = x(1) - x(4); - gradient_terms(1,1) = -x(0) + x(2) -x(4) + x(6); - gradient_terms(1,2) = -x(1) + x(6); - gradient_terms(1,3) = 0; - gradient_terms(1,4) = x(0) + x(1) - x(6) - x(7); - gradient_terms(1,5) = 0; - gradient_terms(1,6) = -x(1) - x(2) + x(4) + x(7); - gradient_terms(1,7) = x(4) - x(6); - - for(int inode = 0; inode < 8; inode++){ - B_matrix_gradients(5,1,inode,0) = gradient_terms(0,inode)*twelth; - B_matrix_gradients(5,1,inode,1) = 0; - B_matrix_gradients(5,1,inode,2) = gradient_terms(1,inode)*twelth; - } - - // B_matrix(6,1) = ( +z(1)*( +x(2) -x(5) ) - // +z(2)*( -x(1) +x(3) -x(5) +x(7) ) - // +z(3)*( -x(2) +x(7) ) - // +z(4)*( +x(5) -x(7) ) - // +z(5)*( +x(1) +x(2) -x(4) -x(7) ) - // +z(7)*( -x(2) -x(3) +x(4) +x(5) ) )*twelth; - - //x derivative - gradient_terms(0,0) = 0; - gradient_terms(0,1) = -z(2) + z(5); - gradient_terms(0,2) = z(1) - z(3) + z(5) - z(7); - gradient_terms(0,3) = z(2) - z(7); - gradient_terms(0,4) = -z(5) + z(7); - gradient_terms(0,5) = -z(1) - z(2) +z(4) +z(7); - gradient_terms(0,6) = 0; - gradient_terms(0,7) = z(2) + z(3) - z(4) - z(5); - - //z derivative - gradient_terms(1,0) = 0; - gradient_terms(1,1) = x(2) - x(5); - gradient_terms(1,2) = -x(1) + x(3) - x(5) + x(7); - gradient_terms(1,3) = -x(2) + x(7); - gradient_terms(1,4) = x(5) - x(7); - gradient_terms(1,5) = x(1) + x(2) - x(4) - x(7); - gradient_terms(1,6) = 0; - gradient_terms(1,7) = -x(2) - x(3) + x(4) + x(5); - - for(int inode = 0; inode < 8; inode++){ - B_matrix_gradients(6,1,inode,0) = gradient_terms(0,inode)*twelth; - B_matrix_gradients(6,1,inode,1) = 0; - B_matrix_gradients(6,1,inode,2) = gradient_terms(1,inode)*twelth; - } - - // B_matrix(7,1) = ( +z(0)*( -x(3) +x(4) ) - // +z(2)*( +x(3) -x(6) ) - // +z(3)*( +x(0) -x(2) +x(4) -x(6) ) - // +z(4)*( -x(0) -x(3) +x(5) +x(6) ) - // +z(5)*( -x(4) +x(6) ) - // +z(6)*( +x(2) +x(3) -x(4) -x(5) ) )*twelth; - - //x derivative - gradient_terms(0,0) = z(3) - z(4); - gradient_terms(0,1) = 0; - gradient_terms(0,2) = -z(3) + z(6); - gradient_terms(0,3) = -z(0) + z(2) - z(4) + z(6); - gradient_terms(0,4) = z(0) + z(3) - z(5) - z(6); - gradient_terms(0,5) = z(4) - z(6); - gradient_terms(0,6) = -z(2) - z(3) + z(4) + z(5); - gradient_terms(0,7) = 0; - - //z derivative - gradient_terms(1,0) = -x(3) + x(4); - gradient_terms(1,1) = 0; - gradient_terms(1,2) = x(3) - x(6); - gradient_terms(1,3) = x(0) - x(2) + x(4) - x(6); - gradient_terms(1,4) = -x(0) - x(3) + x(5) + x(6); - gradient_terms(1,5) = -x(4) + x(6); - gradient_terms(1,6) = x(2) + x(3) -x(4) - x(5); - gradient_terms(1,7) = 0; - - for(int inode = 0; inode < 8; inode++){ - B_matrix_gradients(7,1,inode,0) = gradient_terms(0,inode)*twelth; - B_matrix_gradients(7,1,inode,1) = 0; - B_matrix_gradients(7,1,inode,2) = gradient_terms(1,inode)*twelth; - } - - // B_matrix(0,2) = ( +x(1)*( -y(2) -y(3) +y(4) +y(5) ) - // +x(2)*( +y(1) -y(3) ) - // +x(3)*( +y(1) +y(2) -y(4) -y(7) ) - // +x(4)*( -y(1) +y(3) -y(5) +y(7) ) - // +x(5)*( -y(1) +y(4) ) - // +x(7)*( +y(3) -y(4) ) )*twelth; - - //x derivative - gradient_terms(0,0) = 0; - gradient_terms(0,1) = -y(2) - y(3) + y(4) + y(5); - gradient_terms(0,2) = y(1) - y(3); - gradient_terms(0,3) = y(1) + y(2) - y(4) - y(7); - gradient_terms(0,4) = -y(1) + y(3) - y(5) + y(7); - gradient_terms(0,5) = -y(1) + y(4); - gradient_terms(0,6) = 0; - gradient_terms(0,7) = y(3) - y(4); - - //y derivative - gradient_terms(1,0) = 0; - gradient_terms(1,1) = x(2) + x(3) - x(4) - x(5); - gradient_terms(1,2) = -x(1) + x(3); - gradient_terms(1,3) = -x(1) - x(2) + x(4) + x(7); - gradient_terms(1,4) = x(1) - x(3) + x(5) - x(7); - gradient_terms(1,5) = x(1) - x(4); - gradient_terms(1,6) = 0; - gradient_terms(1,7) = -x(3) + x(4); - - for(int inode = 0; inode < 8; inode++){ - B_matrix_gradients(0,2,inode,0) = gradient_terms(0,inode)*twelth; - B_matrix_gradients(0,2,inode,1) = gradient_terms(1,inode)*twelth; - B_matrix_gradients(0,2,inode,2) = 0; - } - - // B_matrix(1,2) = ( +x(0)*( +y(2) +y(3) -y(4) -y(5) ) - // +x(2)*( -y(0) -y(3) +y(5) +y(6) ) - // +x(3)*( -y(0) +y(2) ) - // +x(4)*( +y(0) -y(5) ) - // +x(5)*( +y(0) -y(2) +y(4) -y(6) ) - // +x(6)*( -y(2) +y(5) ) )*twelth; - - //x derivative - gradient_terms(0,0) = y(2) + y(3) - y(4) - y(5); - gradient_terms(0,1) = 0; - gradient_terms(0,2) = -y(0) - y(3) + y(5) + y(6); - gradient_terms(0,3) = -y(0) + y(2); - gradient_terms(0,4) = y(0) - y(5); - gradient_terms(0,5) = y(0) - y(2) + y(4) - y(6); - gradient_terms(0,6) = -y(2) + y(5); - gradient_terms(0,7) = 0; - - //y derivative - gradient_terms(1,0) = -x(2) - x(3) + x(4) + x(5); - gradient_terms(1,1) = 0; - gradient_terms(1,2) = x(0) + x(3) - x(5) - x(6); - gradient_terms(1,3) = x(0) - x(2); - gradient_terms(1,4) = -x(0) + x(5); - gradient_terms(1,5) = -x(0) + x(2) - x(4) + x(6); - gradient_terms(1,6) = x(2) - x(5); - gradient_terms(1,7) = 0; - - for(int inode = 0; inode < 8; inode++){ - B_matrix_gradients(1,2,inode,0) = gradient_terms(0,inode)*twelth; - B_matrix_gradients(1,2,inode,1) = gradient_terms(1,inode)*twelth; - B_matrix_gradients(1,2,inode,2) = 0; - } - - // B_matrix(2,2) = ( +x(0)*( -y(1) +y(3) ) - // +x(1)*( +y(0) +y(3) -y(5) -y(6) ) - // +x(3)*( -y(0) -y(1) +y(6) +y(7) ) - // +x(5)*( +y(1) -y(6) ) - // +x(6)*( +y(1) -y(3) +y(5) -y(7) ) - // +x(7)*( -y(3) +y(6) ) )*twelth; - - //x derivative - gradient_terms(0,0) = -y(1) + y(3); - gradient_terms(0,1) = y(0) + y(3) - y(5) - y(6); - gradient_terms(0,2) = 0; - gradient_terms(0,3) = -y(0) - y(1) + y(6) + y(7); - gradient_terms(0,4) = 0; - gradient_terms(0,5) = y(1) - y(6); - gradient_terms(0,6) = y(1) - y(3) + y(5) - y(7); - gradient_terms(0,7) = -y(3) + y(6); - - //y derivative - gradient_terms(1,0) = x(1) - x(3); - gradient_terms(1,1) = -x(0) - x(3) + x(5) + x(6); - gradient_terms(1,2) = 0; - gradient_terms(1,3) = x(0) + x(1) -x(6) - x(7); - gradient_terms(1,4) = 0; - gradient_terms(1,5) = -x(1) + x(6); - gradient_terms(1,6) = -x(1) + x(3) - x(5) + x(7); - gradient_terms(1,7) = x(3) - x(6); - - for(int inode = 0; inode < 8; inode++){ - B_matrix_gradients(2,2,inode,0) = gradient_terms(0,inode)*twelth; - B_matrix_gradients(2,2,inode,1) = gradient_terms(1,inode)*twelth; - B_matrix_gradients(2,2,inode,2) = 0; - } - - // B_matrix(3,2) = ( +x(0)*( -y(1) -y(2) +y(4) +y(7) ) - // +x(1)*( +y(0) -y(2) ) - // +x(2)*( +y(0) +y(1) -y(6) -y(7) ) - // +x(4)*( -y(0) +y(7) ) - // +x(6)*( +y(2) -y(7) ) - // +x(7)*( -y(0) +y(2) -y(4) +y(6) ) )*twelth; - - //x derivative - gradient_terms(0,0) = -y(1) - y(2) + y(4) + y(7); - gradient_terms(0,1) = y(0) - y(2); - gradient_terms(0,2) = y(0) + y(1) - y(6) - y(7); - gradient_terms(0,3) = 0; - gradient_terms(0,4) = -y(0) + y(7); - gradient_terms(0,5) = 0; - gradient_terms(0,6) = y(2) - y(7); - gradient_terms(0,7) = -y(0) + y(2) - y(4) + y(6); - - //y derivative - gradient_terms(1,0) = x(1) + x(2) - x(4) - x(7); - gradient_terms(1,1) = -x(0) + x(2); - gradient_terms(1,2) = -x(0) - x(1) + x(6) + x(7); - gradient_terms(1,3) = 0; - gradient_terms(1,4) = x(0) - x(7); - gradient_terms(1,5) = 0; - gradient_terms(1,6) = -x(2) + x(7); - gradient_terms(1,7) = x(0) - x(2) + x(4) - x(6); - - for(int inode = 0; inode < 8; inode++){ - B_matrix_gradients(3,2,inode,0) = gradient_terms(0,inode)*twelth; - B_matrix_gradients(3,2,inode,1) = gradient_terms(1,inode)*twelth; - B_matrix_gradients(3,2,inode,2) = 0; - } - - // B_matrix(4,2) = ( +x(0)*( +y(1) -y(3) +y(5) -y(7) ) - // +x(1)*( -y(0) +y(5) ) - // +x(3)*( +y(0) -y(7) ) - // +x(5)*( -y(0) -y(1) +y(6) +y(7) ) - // +x(6)*( -y(5) +y(7) ) - // +x(7)*( +y(0) +y(3) -y(5) -y(6) ) )*twelth; - - //x derivative - gradient_terms(0,0) = y(1) - y(3) + y(5) - y(7); - gradient_terms(0,1) = -y(0) + y(5); - gradient_terms(0,2) = 0; - gradient_terms(0,3) = y(0) - y(7); - gradient_terms(0,4) = 0; - gradient_terms(0,5) = -y(0) - y(1) + y(6) + y(7); - gradient_terms(0,6) = -y(5) + y(7); - gradient_terms(0,7) = y(0) + y(3) - y(5) - y(6); - - //y derivative - gradient_terms(1,0) = -x(1) + x(3) - x(4) + x(7); - gradient_terms(1,1) = x(0) - x(5); - gradient_terms(1,2) = 0; - gradient_terms(1,3) = -x(0) + x(7); - gradient_terms(1,4) = 0; - gradient_terms(1,5) = x(0) + x(1) - x(6) - x(7); - gradient_terms(1,6) = x(5) - x(7); - gradient_terms(1,7) = -x(0) - x(3) + x(5) + x(6); - - for(int inode = 0; inode < 8; inode++){ - B_matrix_gradients(4,2,inode,0) = gradient_terms(0,inode)*twelth; - B_matrix_gradients(4,2,inode,1) = gradient_terms(1,inode)*twelth; - B_matrix_gradients(4,2,inode,2) = 0; - } - - // B_matrix(5,2) = ( +x(0)*( +y(1) -y(4) ) - // +x(1)*( -y(0) +y(2) -y(4) +y(6) ) - // +x(2)*( -y(1) +y(6) ) - // +x(4)*( +y(0) +y(1) -y(6) -y(7) ) - // +x(6)*( -y(1) -y(2) +y(4) +y(7) ) - // +x(7)*( +y(4) -y(6) ) )*twelth; - - //x derivative - gradient_terms(0,0) = y(1) - y(4); - gradient_terms(0,1) = -y(0) + y(2) - y(4) + y(6); - gradient_terms(0,2) = -y(1) + y(6); - gradient_terms(0,3) = 0; - gradient_terms(0,4) = y(0) + y(1) - y(6) - y(7); - gradient_terms(0,5) = 0; - gradient_terms(0,6) = -y(1) - y(2) + y(4) + y(7); - gradient_terms(0,7) = y(4) - y(6); - - //y derivative - gradient_terms(1,0) = -x(1) + x(4); - gradient_terms(1,1) = x(0) - x(2) + x(4) - x(6); - gradient_terms(1,2) = x(1) - x(6); - gradient_terms(1,3) = 0; - gradient_terms(1,4) = -x(0) - x(1) + x(6) + x(7); - gradient_terms(1,5) = 0; - gradient_terms(1,6) = x(1) + x(2) - x(4) - x(7); - gradient_terms(1,7) = -x(4) + x(6); - - for(int inode = 0; inode < 8; inode++){ - B_matrix_gradients(5,2,inode,0) = gradient_terms(0,inode)*twelth; - B_matrix_gradients(5,2,inode,1) = gradient_terms(1,inode)*twelth; - B_matrix_gradients(5,2,inode,2) = 0; - } - - // B_matrix(6,2) = ( +x(1)*( +y(2) -y(5) ) - // +x(2)*( -y(1) +y(3) -y(5) +y(7) ) - // +x(3)*( -y(2) +y(7) ) - // +x(4)*( +y(5) -y(7) ) - // +x(5)*( +y(1) +y(2) -y(4) -y(7) ) - // +x(7)*( -y(2) -y(3) +y(4) +y(5) ) )*twelth; - - //x derivative - gradient_terms(0,0) = 0; - gradient_terms(0,1) = y(2) - y(5); - gradient_terms(0,2) = -y(1) + y(3) -y(5) + y(7); - gradient_terms(0,3) = -y(2) + y(7); - gradient_terms(0,4) = y(5) - y(7); - gradient_terms(0,5) = y(1) + y(2) - y(4) - y(7); - gradient_terms(0,6) = 0; - gradient_terms(0,7) = -y(2) - y(3) + y(4) + y(5); - - //y derivative - gradient_terms(1,0) = 0; - gradient_terms(1,1) = -x(2) + x(5); - gradient_terms(1,2) = x(1) - x(3) + x(5) - x(7); - gradient_terms(1,3) = x(2) - x(7); - gradient_terms(1,4) = -x(5) + x(7); - gradient_terms(1,5) = -x(1) - x(2) + x(4) + x(7); - gradient_terms(1,6) = 0; - gradient_terms(1,7) = x(2) + x(3) - x(4) - x(5); - - for(int inode = 0; inode < 8; inode++){ - B_matrix_gradients(6,2,inode,0) = gradient_terms(0,inode)*twelth; - B_matrix_gradients(6,2,inode,1) = gradient_terms(1,inode)*twelth; - B_matrix_gradients(6,2,inode,2) = 0; - } - - // B_matrix(7,2) = ( +x(0)*( -y(3) +y(4) ) - // +x(2)*( +y(3) -y(6) ) - // +x(3)*( +y(0) -y(2) +y(4) -y(6) ) - // +x(4)*( -y(0) -y(3) +y(5) +y(6) ) - // +x(5)*( -y(4) +y(6) ) - // +x(6)*( +y(2) +y(3) -y(4) -y(5) ) )*twelth; - - //x derivative - gradient_terms(0,0) = -y(3) + y(4); - gradient_terms(0,1) = 0; - gradient_terms(0,2) = y(3) - y(6); - gradient_terms(0,3) = y(0) - y(2) + y(4) - y(6); - gradient_terms(0,4) = -y(0) - y(3) + y(5) + y(6); - gradient_terms(0,5) = -y(4) + y(6); - gradient_terms(0,6) = y(2) + y(3) - y(4) - y(5); - gradient_terms(0,7) = 0; - - //y derivative - gradient_terms(1,0) = x(3) - x(4); - gradient_terms(1,1) = 0; - gradient_terms(1,2) = -x(3) + x(6); - gradient_terms(1,3) = -x(0) +x(2) - x(4) + x(6); - gradient_terms(1,4) = x(0) + x(3) - x(5) - x(6); - gradient_terms(1,5) = x(4) - x(6); - gradient_terms(1,6) = -x(2) - x(3) + x(4) + x(5); - gradient_terms(1,7) = 0; - - for(int inode = 0; inode < 8; inode++){ - B_matrix_gradients(7,2,inode,0) = gradient_terms(0,inode)*twelth; - B_matrix_gradients(7,2,inode,1) = gradient_terms(1,inode)*twelth; - B_matrix_gradients(7,2,inode,2) = 0; - } - -} // end subroutine diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/FEA_Module_SGH.h b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/FEA_Module_SGH.h new file mode 100644 index 000000000..a9cc158bd --- /dev/null +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/FEA_Module_SGH.h @@ -0,0 +1,700 @@ +/********************************************************************************************** + © 2020. Triad National Security, LLC. All rights reserved. + This program was produced under U.S. Government contract 89233218CNA000001 for Los Alamos + National Laboratory (LANL), which is operated by Triad National Security, LLC for the U.S. + Department of Energy/National Nuclear Security Administration. All rights in the program are + reserved by Triad National Security, LLC, and the U.S. Department of Energy/National Nuclear + Security Administration. The Government is granted for itself and others acting on its behalf a + nonexclusive, paid-up, irrevocable worldwide license in this material to reproduce, prepare + derivative works, distribute copies to the public, perform publicly and display publicly, and + to permit others to do so. + This program is open source under the BSD-3 License. + Redistribution and use in source and binary forms, with or without modification, are permitted + provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, this list of + conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, this list of + conditions and the following disclaimer in the documentation and/or other materials + provided with the distribution. + + 3. Neither the name of the copyright holder nor the names of its contributors may be used + to endorse or promote products derived from this software without specific prior + written permission. + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS + IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **********************************************************************************************/ + +#ifndef FEA_MODULE_SGH_H +#define FEA_MODULE_SGH_H + +#include "mesh.h" +#include "state.h" +#include "matar.h" +#include "elements.h" +#include "node_combination.h" +#include "FEA_Module.h" +#include "material_models.h" + +// Forward declarations +class Explicit_Solver; +class Solver; +class Simulation_Parameters_Explicit; +class SGH_Parameters; + +struct material_t; +struct boundary_t; + +class FEA_Module_SGH : public FEA_Module +{ +public: + + FEA_Module_SGH(SGH_Parameters& params, Solver* Solver_Pointer, std::shared_ptr mesh_in, const int my_fea_module_index = 0); + ~FEA_Module_SGH(); + + // initialize data for boundaries of the model and storage for boundary conditions and applied loads + void sgh_interface_setup(node_t& node, elem_t& elem, corner_t& corner); + + void setup(); + + void cleanup_material_models(); + + int solve(); + + void module_cleanup(); + + void sgh_solve(); + + /** + * \brief Calculate forces + * + * This function calculates the corner forces and the evolves stress + * according to the chosen material constitutive model + * + * \param material Array of material types + * \param mesh Simulation mesh + * \param node_coords View into nodal position array + * \param node_vel View into the nodal velocity array + * \param elem_den View into the element density array + * \param elem_sie View into the element specific internal energy array + * \param elem_pres View into the element pressure array + * \param elem_stress View into the element stress array + * \param elem_sspd View into the element sound speed array + * \param elem_vol View into the element volume array + * \param elem_div View into the element divergence array + * \param elem_mat_id View into the element material id array + * \param corner_force View into the corner forces + * \param rk_alpha Runge Kutta alpha value + * \param cycle Simulation cycle + * + */ + void get_force_sgh(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_pres, + DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + DViewCArrayKokkos& corner_force, + const double rk_alpha, + const size_t cycle); + + void get_force_vgradient_sgh(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_pres, + const DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + const double rk_alpha, + const size_t cycle); + + void get_force_ugradient_sgh(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_pres, + const DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + const double rk_alpha, + const size_t cycle); + + void get_force_egradient_sgh(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_pres, + const DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + const double rk_alpha, + const size_t cycle); + + void get_force_dgradient_sgh(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_pres, + const DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + const double rk_alpha, + const size_t cycle); + + void force_design_gradient_term(const_vec_array design_variables, vec_array design_gradients); + + void get_force_sgh2D(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_pres, + const DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + DViewCArrayKokkos& corner_force, + const double rk_alpha, + const size_t cycle); + + void update_position_sgh(double rk_alpha, + const size_t num_nodes, + DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel); + + void get_vol(); + + void get_vol_ugradient(const size_t gradient_node_id, const size_t gradient_dim); + + void init_assembly(); + + KOKKOS_INLINE_FUNCTION + void get_vol_hex(const DViewCArrayKokkos& elem_vol, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const; + + KOKKOS_FUNCTION + void get_vol_hex_ugradient(const ViewCArrayKokkos& elem_vol_gradients, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const; + + KOKKOS_INLINE_FUNCTION + void get_vol_quad(const DViewCArrayKokkos& elem_vol, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const; + + KOKKOS_FUNCTION + double get_area_quad(const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const; + + KOKKOS_FUNCTION + void get_bmatrix(const ViewCArrayKokkos& B_matrix, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const; + + KOKKOS_FUNCTION + void get_bmatrix_gradients(const ViewCArrayKokkos& B_matrix_gradients, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const; + + KOKKOS_FUNCTION + void get_bmatrix2D(const ViewCArrayKokkos& B_matrix, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const; + + KOKKOS_FUNCTION + void get_area_weights2D(const ViewCArrayKokkos& corner_areas, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const; + + KOKKOS_INLINE_FUNCTION + double heron(const double x1, + const double y1, + const double x2, + const double y2, + const double x3, + const double y3) const; + + double average_element_density(const int nodes_per_elem, const CArray current_element_densities) const; + + void get_divergence(DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_vol); + + void get_divergence2D(DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_vol); + + KOKKOS_FUNCTION + void get_velgrad(ViewCArrayKokkos& vel_grad, + const ViewCArrayKokkos& elem_node_gids, + const DViewCArrayKokkos& node_vel, + const ViewCArrayKokkos& b_matrix, + const double elem_vol, + const size_t elem_gid, + const size_t rk_level) const; + + KOKKOS_FUNCTION + void get_velgrad2D(ViewCArrayKokkos& vel_grad, + const ViewCArrayKokkos& elem_node_gids, + const DViewCArrayKokkos& node_vel, + const ViewCArrayKokkos& b_matrix, + const double elem_vol, + const double elem_area, + const size_t elem_gid, + const size_t rk_level) const; + + KOKKOS_INLINE_FUNCTION + void decompose_vel_grad(ViewCArrayKokkos& D_tensor, + ViewCArrayKokkos& W_tensor, + const ViewCArrayKokkos& vel_grad, + const ViewCArrayKokkos& elem_node_gids, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const double vol) const; + + void update_velocity_sgh(double rk_alpha, + DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_mass, + const DViewCArrayKokkos& corner_force); + + void tag_bdys(const DCArrayKokkos& boundary, + mesh_t& mesh, + const DViewCArrayKokkos& node_coords); + + void boundary_velocity(const mesh_t& mesh, + const DCArrayKokkos& boundary, + DViewCArrayKokkos& node_vel); + + KOKKOS_INLINE_FUNCTION + bool check_bdy(const size_t patch_gid, + const int num_dim, + const int num_nodes_in_patch, + const BOUNDARY_TYPE this_bc_tag, + const double val, + const DViewCArrayKokkos& node_coords, + const size_t rk_level) const; + + void rk_init(DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_sie, + DViewCArrayKokkos& elem_stress, + const size_t num_elems, + const size_t num_nodes); + + void get_timestep(mesh_t& mesh, + DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_sspd, + DViewCArrayKokkos& elem_vol); + + void get_timestep2D(mesh_t& mesh, + DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_sspd, + DViewCArrayKokkos& elem_vol); + + void update_energy_sgh(double rk_alpha, + const mesh_t& mesh, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& corner_force); + + void power_design_gradient_term(const_vec_array design_variables, vec_array design_gradients); + + void get_power_dgradient_sgh(double rk_alpha, + const mesh_t& mesh, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& corner_force, + DCArrayKokkos elem_power_dgradients); + + void get_power_ugradient_sgh(double rk_alpha, + const mesh_t& mesh, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& corner_force); + + void get_power_vgradient_sgh(double rk_alpha, + const mesh_t& mesh, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& corner_force); + + void get_power_egradient_sgh(double rk_alpha, + const mesh_t& mesh, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& corner_force); + + void update_state(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_den, + DViewCArrayKokkos& elem_pres, + DViewCArrayKokkos& elem_stress, + DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& elem_mat_id, + const double rk_alpha, + const size_t cycle); + + void update_state2D(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_den, + DViewCArrayKokkos& elem_pres, + DViewCArrayKokkos& elem_stress, + DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& elem_mat_id, + const double rk_alpha, + const size_t cycle); + + void build_boundry_node_sets(mesh_t& mesh); + + void init_boundaries(); + + // initializes memory for arrays used in the global stiffness matrix assembly + void init_boundary_sets(int num_boundary_sets); + + void grow_boundary_sets(int num_boundary_sets); + + virtual void update_forward_solve(Teuchos::RCP zp); + + void comm_node_masses(); + + void comm_adjoint_vector(int cycle); + + void comm_phi_adjoint_vector(int cycle); + + void comm_variables(Teuchos::RCP zp); + + void read_conditions_ansys_dat(std::ifstream* in, std::streampos before_condition_header); + + // interfaces between user input and creating data structures for bcs + void generate_bcs(); + + void Displacement_Boundary_Conditions(); + + void init_output(); + + void compute_output(); + + void output_control(); + + void sort_output(Teuchos::RCP> sorted_map); + + void sort_element_output(Teuchos::RCP> sorted_map); + + void collect_output(Teuchos::RCP> global_reduce_map); + + void write_data(std::map& point_data_scalars_double, + std::map& point_data_vectors_double, + std::map& cell_data_scalars_double, + std::map& cell_data_scalars_int, + std::map>& cell_data_fields_double); + + void write_outputs(const mesh_t& mesh, + DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& node_mass, + DViewCArrayKokkos& elem_den, + DViewCArrayKokkos& elem_pres, + DViewCArrayKokkos& elem_stress, + DViewCArrayKokkos& elem_sspd, + DViewCArrayKokkos& elem_sie, + DViewCArrayKokkos& elem_vol, + DViewCArrayKokkos& elem_mass, + DViewCArrayKokkos& elem_mat_id); + + void ensight(const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_mass, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_pres, + const DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& elem_mat_id); + + void state_file(const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_mass, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_pres, + const DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& elem_mat_id); + + void node_density_constraints(host_vec_array node_densities_lower_bound); + + void compute_topology_optimization_adjoint_full(); // Force depends on node coords and velocity + + void compute_topology_optimization_gradient_full(Teuchos::RCP design_densities_distributed, Teuchos::RCP design_gradients_distributed); + + void boundary_adjoint(const mesh_t& mesh, + const DCArrayKokkos& boundary, + vec_array& node_adjoint, + vec_array& node_phi_adjoint, + vec_array& node_psi_adjoint); + + void applied_forces(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_mass, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + DViewCArrayKokkos& corner_force, + const double rk_alpha, + const size_t cycle); + + bool have_loading_conditions; + bool nodal_density_flag; + real_t penalty_power; + + Simulation_Parameters_Explicit* simparam; + SGH_Parameters* module_params; + Explicit_Solver* Explicit_Solver_Pointer_; + + elements::ref_element* ref_elem; + + std::shared_ptr mesh; + // shallow copies of mesh class views + size_t num_nodes_in_elem; + // corner ids in node + RaggedRightArrayKokkos corners_in_node; + CArrayKokkos num_corners_in_node; + + // elem ids in node + RaggedRightArrayKokkos elems_in_node; + + // node ids in node + RaggedRightArrayKokkos nodes_in_node; + CArrayKokkos num_nodes_in_node; + + // node ids in elem + DCArrayKokkos nodes_in_elem; + + // corner ids in elem + CArrayKokkos corners_in_elem; + + // elem ids in elem + RaggedRightArrayKokkos elems_in_elem; + CArrayKokkos num_elems_in_elem; + + // patch ids in elem + CArrayKokkos patches_in_elem; + + // node ids in a patch + CArrayKokkos nodes_in_patch; + + // element ids in a patch + CArrayKokkos elems_in_patch; + + // bdy nodes + CArrayKokkos bdy_nodes; + + // Topology optimization filter variable + DCArrayKokkos relative_element_densities; + + // Local FEA data + host_elem_conn_array interface_nodes_in_elem; // host view of element connectivity to nodes + + // Global FEA data + Teuchos::RCP node_velocities_distributed; + Teuchos::RCP initial_node_coords_distributed; + Teuchos::RCP all_initial_node_coords_distributed; + Teuchos::RCP initial_node_velocities_distributed; + Teuchos::RCP all_node_velocities_distributed; + Teuchos::RCP all_cached_node_velocities_distributed; + Teuchos::RCP node_masses_distributed; + Teuchos::RCP ghost_node_masses_distributed; + Teuchos::RCP adjoint_vector_distributed; + Teuchos::RCP phi_adjoint_vector_distributed; + Teuchos::RCP psi_adjoint_vector_distributed; + Teuchos::RCP element_internal_energy_distributed; + Teuchos::RCP>> forward_solve_velocity_data; + Teuchos::RCP>> forward_solve_coordinate_data; + Teuchos::RCP>> forward_solve_internal_energy_data; + Teuchos::RCP>> adjoint_vector_data; + Teuchos::RCP>> phi_adjoint_vector_data; + Teuchos::RCP>> psi_adjoint_vector_data; + Teuchos::RCP force_gradient_design; + Teuchos::RCP force_gradient_position; + Teuchos::RCP force_gradient_velocity; + + // Local FEA data + DCArrayKokkos Global_Gradient_Matrix_Assembly_Map; // Maps element local nodes to columns on ragged right node connectivity graph + DCArrayKokkos Element_Gradient_Matrix_Assembly_Map; // Maps element-node pair to columns on ragged right node to element connectivity + RaggedRightArrayKokkos Graph_Matrix; // stores global indices + RaggedRightArrayKokkos DOF_Graph_Matrix; // stores global indices + RaggedRightArrayKokkos Force_Gradient_Positions; + RaggedRightArrayKokkos Force_Gradient_Velocities; + CArrayKokkos Force_Gradient_Energies; // transposed such that elem ids correspond to rows + RaggedRightArrayKokkos Power_Gradient_Positions; // transposed such that node dofs correspond to rows + RaggedRightArrayKokkos Power_Gradient_Velocities; // transposed such that node dofs correspond to rows + CArrayKokkos Power_Gradient_Energies; + DCArrayKokkos Gradient_Matrix_Strides; + DCArrayKokkos DOF_to_Elem_Matrix_Strides; + DCArrayKokkos Elem_to_Elem_Matrix_Strides; + DCArrayKokkos Graph_Matrix_Strides; + RaggedRightArrayKokkos Original_Gradient_Entries; + RaggedRightArrayKokkos Original_Gradient_Entry_Indices; + DCArrayKokkos Original_Gradient_Entries_Strides; + + // distributed matrices + Teuchos::RCP distributed_force_gradient_positions; + Teuchos::RCP distributed_force_gradient_velocities; + + std::vector time_data; + int max_time_steps, last_time_step; + + // --------------------------------------------------------------------- + // state data type declarations (must stay in scope for output after run) + // --------------------------------------------------------------------- + node_t node_interface; + elem_t elem_interface; + corner_t corner_interface; + + // Dual View wrappers + // Dual Views of the individual node struct variables + DViewCArrayKokkos node_coords; + DViewCArrayKokkos node_vel; + DViewCArrayKokkos node_mass; + + // Dual Views of the individual elem struct variables + DViewCArrayKokkos elem_den; + DViewCArrayKokkos elem_pres; + DViewCArrayKokkos elem_stress; // always 3D even in 2D-RZ + DViewCArrayKokkos elem_sspd; + DViewCArrayKokkos elem_sie; + DViewCArrayKokkos elem_vol; + DViewCArrayKokkos elem_div; + DViewCArrayKokkos elem_mass; + DViewCArrayKokkos elem_mat_id; + + // Element velocity gradient + DCArrayKokkos elem_vel_grad; + + // for storing global variables used in user material model + DCArrayKokkos global_vars; + + // for storing state variables used in user material model + DCArrayKokkos state_vars; + + // elem_user_output_vars allow users to output variables of interest per element + DCArrayKokkos elem_user_output_vars; + + // material models + DCArrayKokkos elem_eos; + DCArrayKokkos elem_strength; + + // Dual Views of the corner struct variables + DViewCArrayKokkos corner_force; + DViewCArrayKokkos corner_mass; + + // Boundary Conditions Data + DCArrayKokkos Local_Index_Boundary_Patches; + // CArray Patch_Nodes; + enum bc_type { NONE, POINT_LOADING_CONDITION, LINE_LOADING_CONDITION, SURFACE_LOADING_CONDITION }; + + // Boundary Conditions Data + int max_boundary_sets; + + // output dof data + // Global arrays with collected data used to print + int output_velocity_index, output_strain_index, output_stress_index; + + // parameters + double time_value, time_final, dt, dt_max, dt_min, dt_cfl, graphics_time, graphics_dt_ival; + size_t graphics_cyc_ival, cycle_stop, rk_num_stages, graphics_id; + double fuzz, tiny, small; + CArray graphics_times; + int rk_num_bins; + + // optimization flags + bool kinetic_energy_objective; +}; + +#endif // end HEADER_H diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/mesh.h b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/mesh.h similarity index 60% rename from src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/mesh.h rename to src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/mesh.h index 25111f2ed..d33862889 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/mesh.h +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/mesh.h @@ -1,7 +1,6 @@ #ifndef MESH_H #define MESH_H - #include "matar.h" #include "state.h" @@ -48,195 +47,181 @@ patch 6: [4,5,6,7] zeta-plus dir // sort in ascending order using bubble sort KOKKOS_INLINE_FUNCTION -void bubble_sort(size_t arr[], const size_t num){ - - for (size_t i=0; i<(num-1); i++){ - for (size_t j=0; j<(num-i-1); j++){ - - if (arr[j]>arr[j+1]){ +void bubble_sort(size_t arr[], const size_t num) +{ + for (size_t i = 0; i < (num - 1); i++) + { + for (size_t j = 0; j < (num - i - 1); j++) + { + if (arr[j] > arr[j + 1]) + { size_t temp = arr[j]; - arr[j] = arr[j+1]; - arr[j+1] = temp; + arr[j] = arr[j + 1]; + arr[j + 1] = temp; } // end if - } // end for j } // end for i } // end function - // mesh sizes and connectivity data structures -struct mesh_t { - +struct mesh_t +{ size_t num_dims; - + size_t num_nodes, num_local_nodes; - + size_t num_elems; size_t num_nodes_in_elem; size_t num_patches_in_elem; size_t num_corners; - + size_t num_patches; - + size_t num_bdy_patches; size_t num_bdy_nodes; size_t num_bdy_sets; size_t num_nodes_in_patch; - // ---- nodes ---- - + // corner ids in node - RaggedRightArrayKokkos corners_in_node; - CArrayKokkos num_corners_in_node; - + RaggedRightArrayKokkos corners_in_node; + CArrayKokkos num_corners_in_node; + // elem ids in node - RaggedRightArrayKokkos elems_in_node; - + RaggedRightArrayKokkos elems_in_node; + // node ids in node - RaggedRightArrayKokkos nodes_in_node; - CArrayKokkos num_nodes_in_node; - - + RaggedRightArrayKokkos nodes_in_node; + CArrayKokkos num_nodes_in_node; + // ---- elems ---- - + // node ids in elem - DCArrayKokkos nodes_in_elem; - + DCArrayKokkos nodes_in_elem; + // corner ids in elem - CArrayKokkos corners_in_elem; - + CArrayKokkos corners_in_elem; + // elem ids in elem - RaggedRightArrayKokkos elems_in_elem; - CArrayKokkos num_elems_in_elem; - + RaggedRightArrayKokkos elems_in_elem; + CArrayKokkos num_elems_in_elem; + // patch ids in elem - CArrayKokkos patches_in_elem; - - + CArrayKokkos patches_in_elem; + // ---- patches ---- - + // node ids in a patch - CArrayKokkos nodes_in_patch; - + CArrayKokkos nodes_in_patch; + // element ids in a patch - CArrayKokkos elems_in_patch; - - + CArrayKokkos elems_in_patch; + // ---- bdy ---- - + // bdy_patches - CArrayKokkos bdy_patches; - + CArrayKokkos bdy_patches; + // bdy nodes - CArrayKokkos bdy_nodes; - + CArrayKokkos bdy_nodes; + // patch ids in bdy set - DynamicRaggedRightArrayKokkos bdy_patches_in_set; - + DynamicRaggedRightArrayKokkos bdy_patches_in_set; + // node ids in bdy_patch set - RaggedRightArrayKokkos bdy_nodes_in_set; - DCArrayKokkos num_bdy_nodes_in_set; - + RaggedRightArrayKokkos bdy_nodes_in_set; + DCArrayKokkos num_bdy_nodes_in_set; + // initialization methods void initialize_nodes(const size_t num_nodes_inp) { num_nodes = num_nodes_inp; - + return; - }; // end method // initialization methods void initialize_local_nodes(const size_t num_nodes_inp) { num_local_nodes = num_nodes_inp; - + return; - }; // end method - - + // initialization methods void initialize_elems(const size_t num_elems_inp, const size_t num_dims_inp) { num_dims = num_dims_inp; num_nodes_in_elem = 1; - for (int dim=0; dim (num_elems, num_nodes_in_elem, "nodes_in_elem"); - corners_in_elem = CArrayKokkos (num_elems, num_nodes_in_elem, "corners_in_elem"); - + num_elems = num_elems_inp; + nodes_in_elem = DCArrayKokkos(num_elems, num_nodes_in_elem, "nodes_in_elem"); + corners_in_elem = CArrayKokkos(num_elems, num_nodes_in_elem, "corners_in_elem"); + return; - }; // end method - - + // initialization methods void initialize_corners(const size_t num_corners_inp) { num_corners = num_corners_inp; - + return; - }; // end method - - + // build the corner mesh connectivity arrays - void build_corner_connectivity(){ - - num_corners_in_node = CArrayKokkos (num_nodes); // stride sizes - + void build_corner_connectivity() + { + num_corners_in_node = CArrayKokkos(num_nodes); // stride sizes + // initializing the number of corners (node-cell pair) to be zero FOR_ALL_CLASS(node_gid, 0, num_nodes, { num_corners_in_node(node_gid) = 0; }); - - - for (size_t elem_gid = 0; elem_gid < num_elems; elem_gid++){ - FOR_ALL_CLASS(node_lid, 0, num_nodes_in_elem,{ - + + for (size_t elem_gid = 0; elem_gid < num_elems; elem_gid++) + { + FOR_ALL_CLASS(node_lid, 0, num_nodes_in_elem, { // get the global_id of the node size_t node_gid = nodes_in_elem(elem_gid, node_lid); // increment the number of corners attached to this point num_corners_in_node(node_gid) = num_corners_in_node(node_gid) + 1; - }); // end FOR_ALL over nodes in element } // end for elem_gid - - + // the stride sizes are the num_corners_in_node at the node - corners_in_node = RaggedRightArrayKokkos (num_corners_in_node, "corners_in_node"); + corners_in_node = RaggedRightArrayKokkos(num_corners_in_node, "corners_in_node"); - CArrayKokkos count_saved_corners_in_node(num_nodes, "count_saved_corners_in_node"); + CArrayKokkos count_saved_corners_in_node(num_nodes, "count_saved_corners_in_node"); // reset num_corners to zero FOR_ALL_CLASS(node_gid, 0, num_nodes, { count_saved_corners_in_node(node_gid) = 0; }); - - + // he elems_in_elem data type - elems_in_node = RaggedRightArrayKokkos (num_corners_in_node, "elems_in_node"); - + elems_in_node = RaggedRightArrayKokkos(num_corners_in_node, "elems_in_node"); + // populate the elems connected to a node list and corners in a node - for (size_t elem_gid = 0; elem_gid < num_elems; elem_gid++){ + for (size_t elem_gid = 0; elem_gid < num_elems; elem_gid++) + { FOR_ALL_CLASS(node_lid, 0, num_nodes_in_elem, { - // get the global_id of the node size_t node_gid = nodes_in_elem(elem_gid, node_lid); - + // the column index is the num corners saved size_t j = count_saved_corners_in_node(node_gid); // Save corner index to this node_gid - size_t corner_gid = node_lid + elem_gid*num_nodes_in_elem; + size_t corner_gid = node_lid + elem_gid * num_nodes_in_elem; corners_in_node(node_gid, j) = corner_gid; - + elems_in_node(node_gid, j) = elem_gid; // save the elem_gid // Save corner index to element @@ -245,133 +230,127 @@ struct mesh_t { // increment the number of corners saved to this node_gid count_saved_corners_in_node(node_gid) = count_saved_corners_in_node(node_gid) + 1; - }); // end FOR_ALL over nodes in element } // end for elem_gid - + return; - } // end of build_corner_connectivity - - + // build elem connectivity arrays - void build_elem_elem_connectivity(){ - + void build_elem_elem_connectivity() + { // find the max number of elems around a node size_t max_num_elems_in_node; size_t max_num_lcl; REDUCE_MAX_CLASS(node_gid, 0, num_nodes, max_num_lcl, { - // num_corners_in_node = num_elems_in_node size_t max_num = num_corners_in_node(node_gid); - - if (max_num > max_num_lcl) max_num_lcl = max_num; - + + if (max_num > max_num_lcl) + { + max_num_lcl = max_num; + } }, max_num_elems_in_node); // end parallel reduction on max Kokkos::fence(); - + // a temporary ragged array to save the elems around an elem - DynamicRaggedRightArrayKokkos temp_elems_in_elem(num_nodes, num_nodes_in_elem*max_num_elems_in_node, "temp_elems_in_elem"); - - num_elems_in_elem = CArrayKokkos (num_elems, "num_elems_in_elem"); + DynamicRaggedRightArrayKokkos temp_elems_in_elem(num_nodes, num_nodes_in_elem * max_num_elems_in_node, "temp_elems_in_elem"); + + num_elems_in_elem = CArrayKokkos(num_elems, "num_elems_in_elem"); FOR_ALL_CLASS(elem_gid, 0, num_elems, { num_elems_in_elem(elem_gid) = 0; }); Kokkos::fence(); - + // find and save neighboring elem_gids of an elem FOR_ALL_CLASS(elem_gid, 0, num_elems, { - for (int node_lid=0; node_lid (num_elems_in_elem, "elems_in_elem"); - + elems_in_elem = RaggedRightArrayKokkos(num_elems_in_elem, "elems_in_elem"); + FOR_ALL_CLASS(elem_gid, 0, num_elems, { - for (size_t i=0; i node_ordering_in_elem; // node lids in a patch - - num_nodes_in_patch = 2*(num_dims-1); // 2 (2D) or 4 (3D) - num_patches_in_elem = 2*num_dims; // 4 (2D) or 6 (3D) - + DViewCArrayKokkos node_ordering_in_elem; // node lids in a patch + + num_nodes_in_patch = 2 * (num_dims - 1); // 2 (2D) or 4 (3D) + num_patches_in_elem = 2 * num_dims; // 4 (2D) or 6 (3D) + size_t node_lids_in_patch_in_elem[24]; - - if(num_dims == 3) { - size_t temp_node_lids[24] = {0,4,7,3, - 1,2,6,5, - 0,1,5,4, - 2,3,7,6, - 0,3,2,1, - 4,5,6,7}; - - for (size_t i=0; i<24; i++){ + + if (num_dims == 3) + { + size_t temp_node_lids[24] = { 0, 4, 7, 3, + 1, 2, 6, 5, + 0, 1, 5, 4, + 2, 3, 7, 6, + 0, 3, 2, 1, + 4, 5, 6, 7 }; + + for (size_t i = 0; i < 24; i++) + { node_lids_in_patch_in_elem[i] = temp_node_lids[i]; } // end for i - } - else { + else + { // J // | // 3---2 @@ -379,447 +358,409 @@ struct mesh_t { // 0---1 // size_t temp_node_lids[8] = - {0,3, - 1,2, - 0,1, - 3,2}; - - for (size_t i=0; i<8; i++){ + { 0, 3, + 1, 2, + 0, 1, + 3, 2 }; + + for (size_t i = 0; i < 8; i++) + { node_lids_in_patch_in_elem[i] = temp_node_lids[i]; } // end for i - } // end if on dims - - node_ordering_in_elem = DViewCArrayKokkos (&node_lids_in_patch_in_elem[0],num_patches_in_elem,num_nodes_in_patch); - - + + node_ordering_in_elem = DViewCArrayKokkos(&node_lids_in_patch_in_elem[0], num_patches_in_elem, num_nodes_in_patch); + // for saviong the hash keys of the patches and then the nighboring elem_gid - CArrayKokkos hash_keys_in_elem (num_elems, num_patches_in_elem, "hash_keys_in_elem"); - + CArrayKokkos hash_keys_in_elem(num_elems, num_patches_in_elem, "hash_keys_in_elem"); + // for saving the adjacient patch_lid, which is the slide_lid - //CArrayKokkos neighboring_side_lids (num_elems, num_patches_in_elem); - + // CArrayKokkos neighboring_side_lids (num_elems, num_patches_in_elem); + // allocate memory for the patches in the elem - patches_in_elem = CArrayKokkos (num_elems, num_patches_in_elem, "patches_in_elem"); - + patches_in_elem = CArrayKokkos(num_elems, num_patches_in_elem, "patches_in_elem"); + // a temporary storaage for the patch_gids that are on the mesh boundary - CArrayKokkos temp_bdy_patches(num_elems*num_patches_in_elem, "temp_bdy_patches"); - + CArrayKokkos temp_bdy_patches(num_elems * num_patches_in_elem, "temp_bdy_patches"); + // step 1) calculate the hash values for each patch in the element FOR_ALL_CLASS(elem_gid, 0, num_elems, { - - for (size_t patch_lid = 0; patch_lid num_values(2, "num_values"); - + + DCArrayKokkos num_values(2, "num_values"); + // step 2: walk around the elements and save the elem pairs that have the same hash_key RUN_CLASS({ // serial execution on GPU - - size_t patch_gid = 0; + + size_t patch_gid = 0; size_t bdy_patch_gid = 0; - for (size_t elem_gid = 0; elem_gid (num_patches, 2, "elems_in_patch"); - nodes_in_patch = CArrayKokkos (num_patches, num_nodes_in_patch, "nodes_in_patch"); - + + elems_in_patch = CArrayKokkos(num_patches, 2, "elems_in_patch"); + nodes_in_patch = CArrayKokkos(num_patches, num_nodes_in_patch, "nodes_in_patch"); + // a temporary variable to help populate patch structures - CArrayKokkos num_elems_in_patch_saved (num_patches, "num_elems_in_patch_saved"); - + CArrayKokkos num_elems_in_patch_saved(num_patches, "num_elems_in_patch_saved"); + // initialize the number of elems in a patch saved to zero FOR_ALL_CLASS(patch_gid, 0, num_patches, { num_elems_in_patch_saved(patch_gid) = 0; }); - - for(size_t elem_gid=0; elem_gid (num_bdy_patches, "bdy_patches"); - + bdy_patches = CArrayKokkos(num_bdy_patches, "bdy_patches"); + FOR_ALL_CLASS(bdy_patch_gid, 0, num_bdy_patches, { bdy_patches(bdy_patch_gid) = temp_bdy_patches(bdy_patch_gid); }); // end FOR_ALL bdy_patch_gid - - - - + // find and store the boundary nodes - CArrayKokkos temp_bdy_nodes(num_nodes, "temp_bdy_nodes"); - CArrayKokkos hash_bdy_nodes(num_nodes, "hash_bdy_nodes"); - - FOR_ALL_CLASS (node_gid, 0, num_nodes, { + CArrayKokkos temp_bdy_nodes(num_nodes, "temp_bdy_nodes"); + CArrayKokkos hash_bdy_nodes(num_nodes, "hash_bdy_nodes"); + + FOR_ALL_CLASS(node_gid, 0, num_nodes, { hash_bdy_nodes(node_gid) = -1; }); // end for node_gid - + // Parallel loop over boundary patches - DCArrayKokkos num_bdy_nodes_saved(1); - + DCArrayKokkos num_bdy_nodes_saved(1); + RUN_CLASS({ num_bdy_nodes_saved(0) = 0; - for (size_t bdy_patch_gid=0; bdy_patch_gid (num_bdy_nodes, "bdy_nodes"); - - FOR_ALL_CLASS (node_gid, 0, num_bdy_nodes, { + + bdy_nodes = CArrayKokkos(num_bdy_nodes, "bdy_nodes"); + + FOR_ALL_CLASS(node_gid, 0, num_bdy_nodes, { bdy_nodes(node_gid) = temp_bdy_nodes(node_gid); }); // end for boundary node_gid - + printf("Num boundary nodes = %lu \n", num_bdy_nodes); - + return; - } // end patch connectivity method - - - - // build the patches - void build_node_node_connectivity(){ - - + + // build the node-node connectivity data + void build_node_node_connectivity() + { // find the max number of elems around a node size_t max_num_elems_in_node; size_t max_num_lcl; REDUCE_MAX_CLASS(node_gid, 0, num_nodes, max_num_lcl, { - // num_corners_in_node = num_elems_in_node size_t max_num = num_corners_in_node(node_gid); - - if (max_num > max_num_lcl) max_num_lcl = max_num; - + + if (max_num > max_num_lcl) + { + max_num_lcl = max_num; + } }, max_num_elems_in_node); // end parallel reduction on max Kokkos::fence(); - + // each elem corner will contribute 3 edges to the node. Those edges will likely be the same // ones from an adjacent element so it is a safe estimate to multiply by 3 - DynamicRaggedRightArrayKokkos temp_nodes_in_nodes(num_nodes, max_num_elems_in_node*3, "temp_nodes_in_nodes"); - - num_nodes_in_node = CArrayKokkos (num_nodes, "num_nodes_in_node"); - + DynamicRaggedRightArrayKokkos temp_nodes_in_nodes(num_nodes, max_num_elems_in_node * 3, "temp_nodes_in_nodes"); + + num_nodes_in_node = CArrayKokkos(num_nodes, "num_nodes_in_node"); + // walk over the patches and save the node node connectivity RUN_CLASS({ - if (num_dims==3){ - - for (size_t patch_gid=0; patch_gid (num_nodes_in_node, "nodes_in_node"); - + + nodes_in_node = RaggedRightArrayKokkos(num_nodes_in_node, "nodes_in_node"); + // save the connectivity FOR_ALL_CLASS(node_gid, 0, num_nodes, { - size_t num_saved = 0; - for (size_t node_lid=0; node_lid (num_bcs, num_bdy_patches, "bdy_patches_in_set"); - + num_bdy_sets = num_bcs; + bdy_patches_in_set = DynamicRaggedRightArrayKokkos(num_bcs, num_bdy_patches, "bdy_patches_in_set"); + return; - } // end of init_bdy_sets method - - - }; // end mesh_t -#endif \ No newline at end of file +#endif \ No newline at end of file diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/state.h b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/state.h new file mode 100644 index 000000000..8200848b6 --- /dev/null +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/state.h @@ -0,0 +1,80 @@ +#ifndef STATE_H +#define STATE_H + +#include "matar.h" + +using namespace mtr; + +// node_state +struct node_t +{ + // Position + DCArrayKokkos coords; + + // velocity + DCArrayKokkos vel; + + // mass at nodes + DCArrayKokkos mass; + + // Includes Ghost Positions + // DCArrayKokkos all_coords; + + // Includes Ghost velocities + // DCArrayKokkos all_vel; + + // Includes Ghost masses + // DCArrayKokkos all_mass; + + // initialization method (num_rk_storage_bins, num_nodes, num_dims) + void initialize(size_t num_rk, size_t num_nodes, size_t num_dims) + { + this->coords = DCArrayKokkos(num_rk, num_nodes, num_dims); + this->vel = DCArrayKokkos(num_rk, num_nodes, num_dims); + this->mass = DCArrayKokkos(num_nodes); + }; // end method +}; // end node_t + +// elem_state +struct elem_t +{ + CArray den; ///< element density + CArray pres; ///< element pressure + CArray stress; ///< element stress + CArray sspd; ///< element sound speed + CArray sie; ///< specific internal energy + CArray vol; ///< element volume + CArray div; ///< divergence of velocity + CArray mass; ///< element mass + CArray mat_id; ///< element material id + + // initialization method (num_rk_storage_bins, num_cells, num_dims) + void initialize(size_t num_rk, size_t num_elems, size_t num_dims) + { + this->den = CArray(num_elems); + this->pres = CArray(num_elems); + this->stress = CArray(num_rk, num_elems, num_dims, num_dims); + this->sspd = CArray(num_elems); + this->sie = CArray(num_rk, num_elems); + this->vol = CArray(num_elems); + this->div = CArray(num_elems); + this->mass = CArray(num_elems); + this->mat_id = CArray(num_elems); + }; // end method +}; // end elem_t + +// corner_state +struct corner_t +{ + CArray force; ///< Force acting on a corner + CArray mass; ///< Partitioned mass of the corner + + // initialization method (num_corners, num_dims) + void initialize(size_t num_corners, size_t num_dims) + { + this->force = CArray(num_corners, num_dims); + this->mass = CArray(num_corners); + }; // end method +}; // end corner_t + +#endif diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/momentum.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/momentum.cpp deleted file mode 100644 index 4f4ecd9fe..000000000 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/momentum.cpp +++ /dev/null @@ -1,417 +0,0 @@ - -#include "state.h" -#include "FEA_Module_SGH.h" - -// ----------------------------------------------------------------------------- -// This function evolves the velocity at the nodes of the mesh -//------------------------------------------------------------------------------ -void FEA_Module_SGH::update_velocity_sgh(double rk_alpha, - DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &node_mass, - const DViewCArrayKokkos &corner_force - ){ - - const size_t rk_level = rk_num_bins - 1; - const size_t num_dims = num_dim; - - // walk over the nodes to update the velocity - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - - double node_force[3]; - for (size_t dim = 0; dim < num_dims; dim++){ - node_force[dim] = 0.0; - } // end for dim - - // loop over all corners around the node and calculate the nodal force - for (size_t corner_lid=0; corner_lid &vel_grad, - const ViewCArrayKokkos &elem_node_gids, - const DViewCArrayKokkos &node_vel, - const ViewCArrayKokkos &b_matrix, - const double elem_vol, - const size_t elem_gid, - const size_t rk_level - ) const { - - const size_t num_nodes_in_elem = 8; - - double u_array[num_nodes_in_elem]; - double v_array[num_nodes_in_elem]; - double w_array[num_nodes_in_elem]; - ViewCArrayKokkos u(u_array, num_nodes_in_elem); // x-dir vel component - ViewCArrayKokkos v(v_array, num_nodes_in_elem); // y-dir vel component - ViewCArrayKokkos w(w_array, num_nodes_in_elem); // z-dir vel component - - // get the vertex velocities for the cell - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - - // Get node gid - size_t node_gid = elem_node_gids(node_lid); - - u(node_lid) = node_vel(rk_level, node_gid, 0); - v(node_lid) = node_vel(rk_level, node_gid, 1); - w(node_lid) = node_vel(rk_level, node_gid, 2); - - } // end for - - - // --- calculate the velocity gradient terms --- - double inverse_vol = 1.0/elem_vol; - - // x-dir - vel_grad(0,0) = (u(0)*b_matrix(0,0) + u(1)*b_matrix(1,0) - + u(2)*b_matrix(2,0) + u(3)*b_matrix(3,0) - + u(4)*b_matrix(4,0) + u(5)*b_matrix(5,0) - + u(6)*b_matrix(6,0) + u(7)*b_matrix(7,0))*inverse_vol; - - vel_grad(0,1) = (u(0)*b_matrix(0,1) + u(1)*b_matrix(1,1) - + u(2)*b_matrix(2,1) + u(3)*b_matrix(3,1) - + u(4)*b_matrix(4,1) + u(5)*b_matrix(5,1) - + u(6)*b_matrix(6,1) + u(7)*b_matrix(7,1))*inverse_vol; - - vel_grad(0,2) = (u(0)*b_matrix(0,2) + u(1)*b_matrix(1,2) - + u(2)*b_matrix(2,2) + u(3)*b_matrix(3,2) - + u(4)*b_matrix(4,2) + u(5)*b_matrix(5,2) - + u(6)*b_matrix(6,2) + u(7)*b_matrix(7,2))*inverse_vol; - - // y-dir - vel_grad(1,0) = (v(0)*b_matrix(0,0) + v(1)*b_matrix(1,0) - + v(2)*b_matrix(2,0) + v(3)*b_matrix(3,0) - + v(4)*b_matrix(4,0) + v(5)*b_matrix(5,0) - + v(6)*b_matrix(6,0) + v(7)*b_matrix(7,0))*inverse_vol; - - vel_grad(1,1) = (v(0)*b_matrix(0,1) + v(1)*b_matrix(1,1) - + v(2)*b_matrix(2,1) + v(3)*b_matrix(3,1) - + v(4)*b_matrix(4,1) + v(5)*b_matrix(5,1) - + v(6)*b_matrix(6,1) + v(7)*b_matrix(7,1))*inverse_vol; - vel_grad(1,2) = (v(0)*b_matrix(0,2) + v(1)*b_matrix(1,2) - + v(2)*b_matrix(2,2) + v(3)*b_matrix(3,2) - + v(4)*b_matrix(4,2) + v(5)*b_matrix(5,2) - + v(6)*b_matrix(6,2) + v(7)*b_matrix(7,2))*inverse_vol; - - // z-dir - vel_grad(2,0) = (w(0)*b_matrix(0,0) + w(1)*b_matrix(1,0) - + w(2)*b_matrix(2,0) + w(3)*b_matrix(3,0) - + w(4)*b_matrix(4,0) + w(5)*b_matrix(5,0) - + w(6)*b_matrix(6,0) + w(7)*b_matrix(7,0))*inverse_vol; - - vel_grad(2,1) = (w(0)*b_matrix(0,1) + w(1)*b_matrix(1,1) - + w(2)*b_matrix(2,1) + w(3)*b_matrix(3,1) - + w(4)*b_matrix(4,1) + w(5)*b_matrix(5,1) - + w(6)*b_matrix(6,1) + w(7)*b_matrix(7,1))*inverse_vol; - - vel_grad(2,2) = (w(0)*b_matrix(0,2) + w(1)*b_matrix(1,2) - + w(2)*b_matrix(2,2) + w(3)*b_matrix(3,2) - + w(4)*b_matrix(4,2) + w(5)*b_matrix(5,2) - + w(6)*b_matrix(6,2) + w(7)*b_matrix(7,2))*inverse_vol; - - - return; - -} // end function - - -// ----------------------------------------------------------------------------- -// This function calculates the velocity gradient -//------------------------------------------------------------------------------ -KOKKOS_FUNCTION -void FEA_Module_SGH::get_velgrad2D(ViewCArrayKokkos &vel_grad, - const ViewCArrayKokkos &elem_node_gids, - const DViewCArrayKokkos &node_vel, - const ViewCArrayKokkos &b_matrix, - const double elem_vol, - const double elem_area, - const size_t elem_gid, - const size_t rk_level - ) const { - - const size_t num_nodes_in_elem = 4; - - double u_array[num_nodes_in_elem]; - double v_array[num_nodes_in_elem]; - ViewCArrayKokkos u(u_array, num_nodes_in_elem); // x-dir vel component - ViewCArrayKokkos v(v_array, num_nodes_in_elem); // y-dir vel component - - // get the vertex velocities for the cell - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - - // Get node gid - size_t node_gid = elem_node_gids(node_lid); - - u(node_lid) = node_vel(rk_level, node_gid, 0); // x-comp - v(node_lid) = node_vel(rk_level, node_gid, 1); // y-comp - - } // end for - - // initialize to zero - for (size_t i=0; i<3; i++){ - for (size_t j=0; j<3; j++){ - vel_grad(i,j) = 0.0; - } - } - - - double mean_radius = elem_vol/elem_area; - double elem_vel_r = 0.25*(v(0) + v(1) + v(2) + v(3)); - - - // --- calculate the velocity gradient terms --- - double inverse_area = 1.0/elem_area; - - // x-dir - vel_grad(0,0) = (u(0)*b_matrix(0,0) + u(1)*b_matrix(1,0) - + u(2)*b_matrix(2,0) + u(3)*b_matrix(3,0))*inverse_area; - - vel_grad(0,1) = (u(0)*b_matrix(0,1) + u(1)*b_matrix(1,1) - + u(2)*b_matrix(2,1) + u(3)*b_matrix(3,1))*inverse_area; - - - // y-dir - vel_grad(1,0) = (v(0)*b_matrix(0,0) + v(1)*b_matrix(1,0) - + v(2)*b_matrix(2,0) + v(3)*b_matrix(3,0))*inverse_area; - - vel_grad(1,1) = (v(0)*b_matrix(0,1) + v(1)*b_matrix(1,1) - + v(2)*b_matrix(2,1) + v(3)*b_matrix(3,1))*inverse_area; - - vel_grad(2,2) = elem_vel_r/mean_radius; // + avg(vel_R)/R - - return; - -} // end function - - - - -// ----------------------------------------------------------------------------- -// This subroutine to calculate the velocity divergence in all elements -//------------------------------------------------------------------------------ -void FEA_Module_SGH::get_divergence(DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_vol - ){ - - const size_t rk_level = rk_num_bins - 1; - - // --- calculate the forces acting on the nodes from the element --- - FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { - - const size_t num_nodes_in_elem = 8; - const size_t num_dims = 3; - - double u_array[num_nodes_in_elem]; - double v_array[num_nodes_in_elem]; - double w_array[num_nodes_in_elem]; - ViewCArrayKokkos u(u_array, num_nodes_in_elem); // x-dir vel component - ViewCArrayKokkos v(v_array, num_nodes_in_elem); // y-dir vel component - ViewCArrayKokkos w(w_array, num_nodes_in_elem); // z-dir vel component - - // cut out the node_gids for this element - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 8); - - // The b_matrix are the outward corner area normals - double b_matrix_array[24]; - ViewCArrayKokkos b_matrix(b_matrix_array, num_nodes_in_elem, num_dims); - get_bmatrix(b_matrix, - elem_gid, - node_coords, - elem_node_gids, - rk_level); - - // get the vertex velocities for the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - - // Get node gid - size_t node_gid = elem_node_gids(node_lid); - - u(node_lid) = node_vel(rk_level, node_gid, 0); - v(node_lid) = node_vel(rk_level, node_gid, 1); - w(node_lid) = node_vel(rk_level, node_gid, 2); - - } // end for - - - // --- calculate the velocity divergence terms --- - double inverse_vol = 1.0/elem_vol(elem_gid); - - elem_div(elem_gid) = 0.0; - - // x-dir - elem_div(elem_gid) += (u(0)*b_matrix(0,0) + u(1)*b_matrix(1,0) - + u(2)*b_matrix(2,0) + u(3)*b_matrix(3,0) - + u(4)*b_matrix(4,0) + u(5)*b_matrix(5,0) - + u(6)*b_matrix(6,0) + u(7)*b_matrix(7,0))*inverse_vol; - - // y-dir - elem_div(elem_gid) += (v(0)*b_matrix(0,1) + v(1)*b_matrix(1,1) - + v(2)*b_matrix(2,1) + v(3)*b_matrix(3,1) - + v(4)*b_matrix(4,1) + v(5)*b_matrix(5,1) - + v(6)*b_matrix(6,1) + v(7)*b_matrix(7,1))*inverse_vol; - - // z-dir - elem_div(elem_gid) += (w(0)*b_matrix(0,2) + w(1)*b_matrix(1,2) - + w(2)*b_matrix(2,2) + w(3)*b_matrix(3,2) - + w(4)*b_matrix(4,2) + w(5)*b_matrix(5,2) - + w(6)*b_matrix(6,2) + w(7)*b_matrix(7,2))*inverse_vol; - - }); // end parallel for over elem_gid - - return; - -} // end subroutine - - -// ----------------------------------------------------------------------------- -// This subroutine to calculate the velocity divergence in all elements -//------------------------------------------------------------------------------ -void FEA_Module_SGH::get_divergence2D(DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_vol - ){ - - const size_t rk_level = rk_num_bins - 1; - - // --- calculate the forces acting on the nodes from the element --- - FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { - - const size_t num_nodes_in_elem = 4; - const size_t num_dims = 2; - - double u_array[num_nodes_in_elem]; - double v_array[num_nodes_in_elem]; - ViewCArrayKokkos u(u_array, num_nodes_in_elem); // x-dir vel component - ViewCArrayKokkos v(v_array, num_nodes_in_elem); // y-dir vel component - - - // true volume RZ - //double r_array[num_nodes_in_elem]; - //ViewCArrayKokkos r(r_array, num_nodes_in_elem); // r-dir coordinate - - - // cut out the node_gids for this element - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 4); - - // The b_matrix are the outward corner area normals - double b_matrix_array[24]; - ViewCArrayKokkos b_matrix(b_matrix_array, num_nodes_in_elem, num_dims); - get_bmatrix2D(b_matrix, - elem_gid, - node_coords, - elem_node_gids, - rk_level); - - // calculate the area of the quad - double elem_area = get_area_quad(elem_gid, node_coords, elem_node_gids, rk_level); - // true volume uses the elem_vol - - - // get the vertex velocities and node coordinate for the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - - // Get node gid - size_t node_gid = elem_node_gids(node_lid); - - u(node_lid) = node_vel(rk_level, node_gid, 0); - v(node_lid) = node_vel(rk_level, node_gid, 1); - - //r(node_lid) = node_coords(rk_level, node_gid, 1); // true volume RZ - - } // end for - - - // --- calculate the velocity divergence terms --- - double inverse_area = 1.0/elem_area; - - double mean_radius = elem_vol(elem_gid)/elem_area; - double elem_vel_r = 0.25*(v(0) + v(1) + v(2) + v(3)); - - - elem_div(elem_gid) = 0.0; - - // x-dir - elem_div(elem_gid) += (u(0)*b_matrix(0,0) - + u(1)*b_matrix(1,0) - + u(2)*b_matrix(2,0) - + u(3)*b_matrix(3,0))*inverse_area; - - // y-dir (i.e., r direction) - elem_div(elem_gid) += (v(0)*b_matrix(0,1) - + v(1)*b_matrix(1,1) - + v(2)*b_matrix(2,1) - + v(3)*b_matrix(3,1))*inverse_area - + elem_vel_r/mean_radius; // + avg(u_R)/R - - }); // end parallel for over elem_gid - - return; - -} // end subroutine - - -// The velocity gradient can be decomposed into symmetric and antisymmetric tensors -// L = vel_grad -// D = sym(L) -// W = antisym(L) -KOKKOS_INLINE_FUNCTION -void FEA_Module_SGH::decompose_vel_grad(ViewCArrayKokkos &D_tensor, - ViewCArrayKokkos &W_tensor, - const ViewCArrayKokkos &vel_grad, - const ViewCArrayKokkos &elem_node_gids, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const double vol - ) const { - - - // --- Calculate the velocity gradient --- - - const size_t num_dims = 3; - - // initialize to zero - for(size_t i=0; inum_bdy_nodes; - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos material = simparam->material; - const int num_dim = simparam->num_dims; - int num_corners = rnum_elem*num_nodes_in_elem; - real_t global_dt; - bool element_constant_density = true; - size_t current_data_index, next_data_index; - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - CArrayKokkos current_element_adjoint = CArrayKokkos(num_nodes_in_elem,num_dim); - DCArrayKokkos elem_power_dgradients(rnum_elem); - //gradient contribution from gradient of Force vector with respect to design variable. - if(simparam->dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ - if(myrank==0){ - std::cout << "gradient term involving adjoint derivative" << std::endl; - } - } - - for (unsigned long cycle = 0; cycle < last_time_step+1; cycle++) { - //compute timestep from time data - global_dt = time_data[cycle+1] - time_data[cycle]; - //print - if(simparam->dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ - if (cycle==0){ - if(myrank==0) - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } - // print time step every 10 cycles - else if (cycle%20==0){ - if(myrank==0) - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } // end if - } - - - //view scope - { - const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_element_internal_energy = (*forward_solve_internal_energy_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_coord_vector = (*forward_solve_coordinate_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_psi_adjoint_vector = (*psi_adjoint_vector_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array next_velocity_vector = (*forward_solve_velocity_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array next_element_internal_energy = (*forward_solve_internal_energy_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array next_coord_vector = (*forward_solve_coordinate_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array next_psi_adjoint_vector = (*psi_adjoint_vector_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - - //first half of integration step calculation - FOR_ALL_CLASS(node_gid, 0, nall_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - node_vel(rk_level,node_gid,idim) = current_velocity_vector(node_gid,idim); - node_coords(rk_level,node_gid,idim) = current_coord_vector(node_gid,idim); - } - - }); // end parallel for - Kokkos::fence(); - - FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { - elem_sie(rk_level,elem_gid) = current_element_internal_energy(elem_gid,0); - }); // end parallel for - Kokkos::fence(); - - get_vol(); - - // ---- Calculate velocity diveregence for the element ---- - if(num_dim==2){ - get_divergence2D(elem_div, - node_coords, - node_vel, - elem_vol); - } - else { - get_divergence(elem_div, - node_coords, - node_vel, - elem_vol); - } // end if 2D - - // ---- Calculate elem state (den, pres, sound speed, stress) for next time step ---- - if(num_dim==2){ - update_state2D(material, - *mesh, - node_coords, - node_vel, - elem_den, - elem_pres, - elem_stress, - elem_sspd, - elem_sie, - elem_vol, - elem_mass, - elem_mat_id, - 1.0, - cycle); - } - else{ - update_state(material, - *mesh, - node_coords, - node_vel, - elem_den, - elem_pres, - elem_stress, - elem_sspd, - elem_sie, - elem_vol, - elem_mass, - elem_mat_id, - 1.0, - cycle); - } - - // ---- calculate the forces on the vertices and evolve stress (hypo model) ---- - if(num_dim==2){ - get_force_sgh2D(material, - *mesh, - node_coords, - node_vel, - elem_den, - elem_sie, - elem_pres, - elem_stress, - elem_sspd, - elem_vol, - elem_div, - elem_mat_id, - corner_force, - 1.0, - cycle); - } - else { - get_force_sgh(material, - *mesh, - node_coords, - node_vel, - elem_den, - elem_sie, - elem_pres, - elem_stress, - elem_sspd, - elem_vol, - elem_div, - elem_mat_id, - corner_force, - 1.0, - cycle); - } - - get_force_dgradient_sgh(material, - *mesh, - node_coords, - node_vel, - elem_den, - elem_sie, - elem_pres, - elem_stress, - elem_sspd, - elem_vol, - elem_div, - elem_mat_id, - 1.0, - cycle); - - get_power_dgradient_sgh(1.0, - *mesh, - node_vel, - node_coords, - elem_sie, - elem_mass, - corner_force, - elem_power_dgradients); - - //derivatives of forces at corners stored in corner_vector_storage buffer by previous routine - FOR_ALL_CLASS(elem_id, 0, rnum_elem, { - size_t node_id; - size_t corner_id; - real_t inner_product; - - inner_product = current_psi_adjoint_vector(elem_id,0)*elem_power_dgradients(elem_id); - - for (int inode = 0; inode < num_nodes_in_elem; inode++){ - //compute gradient of local element contribution to v^t*M*v product - corner_id = elem_id*num_nodes_in_elem + inode; - corner_value_storage(corner_id) = inner_product; - } - - }); // end parallel for - Kokkos::fence(); - - //accumulate node values from corner storage - //multiply - FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { - size_t corner_id; - for(int icorner=0; icorner < num_corners_in_node(node_id); icorner++){ - corner_id = corners_in_node(node_id,icorner); - design_gradients(node_id,0) += -corner_value_storage(corner_id)*global_dt; - } - }); // end parallel for - Kokkos::fence(); - - } //end view scope - } - -} - -// --------------------------------------------------------------------------------------- -// This function calculates the gradient for element power with respect to design variable -//---------------------------------------------------------------------------------------- - -void FEA_Module_SGH::get_power_dgradient_sgh(double rk_alpha, - const mesh_t &mesh, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_mass, - const DViewCArrayKokkos &corner_force, - DCArrayKokkos elem_power_dgradients){ - - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - int num_dims = simparam->num_dims; - - // loop over all the elements in the mesh - FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { - - double elem_power = 0.0; - elem_power_dgradients(elem_gid) = 0; - - // --- tally the contribution from each corner to the element --- - - // Loop over the nodes in the element - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - - size_t corner_lid = node_lid; - - // Get node global id for the local node id - size_t node_gid = nodes_in_elem(elem_gid, node_lid); - - // Get the corner global id for the local corner id - size_t corner_gid = corners_in_elem(elem_gid, corner_lid); - - double node_radius = 1; - if(num_dims==2){ - node_radius = node_coords(rk_level,node_gid,1); - } - - // calculate the Power=F dot V for this corner - for (size_t dim=0; dim &node_vel, - const DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_mass, - const DViewCArrayKokkos &corner_force){ - - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - int num_dims = simparam->num_dims; - - //initialize gradient matrix - FOR_ALL_CLASS(dof_gid, 0, nlocal_nodes*num_dims, { - for(int idof = 0; idof < DOF_to_Elem_Matrix_Strides(dof_gid); idof++){ - Power_Gradient_Positions(dof_gid,idof) = 0; - } - }); // end parallel for loop over nodes - Kokkos::fence(); - - // loop over all the elements in the mesh - for (size_t elem_gid = 0; elem_gid < rnum_elem; elem_gid++){ - //FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { - - double elem_power = 0.0; - - // --- tally the contribution from each corner to the element --- - - // Loop over the nodes in the element - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - - size_t corner_lid = node_lid; - size_t column_id; - size_t gradient_node_id; - // Get node global id for the local node id - size_t node_gid = nodes_in_elem(elem_gid, node_lid); - - // Get the corner global id for the local corner id - size_t corner_gid = corners_in_elem(elem_gid, corner_lid); - - double node_radius = 1; - if(num_dims==2){ - node_radius = node_coords(rk_level,node_gid,1); - } - - // calculate the Power=F dot V for this corner - for (size_t dim=0; dimisNodeLocalElement(gradient_node_id)) continue; - Power_Gradient_Positions(gradient_node_id*num_dims+jdim, column_id) -= corner_gradient_storage(corner_gid,dim,igradient,jdim)*node_vel(rk_level, node_gid, dim)*node_radius; - } - } - } // end for dim - - } // end for node_lid - - //}); // end parallel loop over the elements - } - - return; -} // end subroutine - -// ----------------------------------------------------------------------------- -// This function calculates the gradient for element power with respect to velocity -//------------------------------------------------------------------------------ - -void FEA_Module_SGH::get_power_vgradient_sgh(double rk_alpha, - const mesh_t &mesh, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_mass, - const DViewCArrayKokkos &corner_force){ - - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - int num_dims = simparam->num_dims; - - //initialize gradient matrix - FOR_ALL_CLASS(dof_gid, 0, nlocal_nodes*num_dims, { - for(int idof = 0; idof < DOF_to_Elem_Matrix_Strides(dof_gid); idof++){ - Power_Gradient_Velocities(dof_gid,idof) = 0; - } - }); // end parallel for loop over nodes - Kokkos::fence(); - - // loop over all the elements in the mesh - for (size_t elem_gid = 0; elem_gid < rnum_elem; elem_gid++){ - //FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { - - double elem_power = 0.0; - - // --- tally the contribution from each corner to the element --- - - // Loop over the nodes in the element - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - - size_t corner_lid = node_lid; - size_t column_id; - size_t gradient_node_id; - // Get node global id for the local node id - size_t node_gid = nodes_in_elem(elem_gid, node_lid); - - // Get the corner global id for the local corner id - size_t corner_gid = corners_in_elem(elem_gid, corner_lid); - - double node_radius = 1; - if(num_dims==2){ - node_radius = node_coords(rk_level,node_gid,1); - } - - // calculate the Power=F dot V for this corner - for (size_t dim=0; dimisNodeLocalElement(gradient_node_id)) continue; - if(node_lid==igradient){ - Power_Gradient_Velocities(gradient_node_id*num_dims+dim, column_id) -= corner_gradient_storage(corner_gid,dim,igradient,dim)*node_vel(rk_level, node_gid, dim)*node_radius+ - corner_force(corner_gid, dim)*node_radius; - } - else{ - Power_Gradient_Velocities(gradient_node_id*num_dims+dim, column_id) -= corner_gradient_storage(corner_gid,dim,igradient,dim)*node_vel(rk_level, node_gid, dim)*node_radius; - } - } - } // end for dim - - } // end for node_lid - - //}); // end parallel loop over the elements - } - - return; -} // end subroutine - -// ----------------------------------------------------------------------------- -// This function calculates the gradient for element power with respect to energy -//------------------------------------------------------------------------------ - -void FEA_Module_SGH::get_power_egradient_sgh(double rk_alpha, - const mesh_t &mesh, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_mass, - const DViewCArrayKokkos &corner_force){ - - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - int num_dims = simparam->num_dims; - - //initialize gradient storage - FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { - Power_Gradient_Energies(elem_gid) = 0; - }); // end parallel loop over the elements - - // loop over all the elements in the mesh - FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { - - double elem_power = 0.0; - - // --- tally the contribution from each corner to the element --- - - // Loop over the nodes in the element - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - - size_t corner_lid = node_lid; - - // Get node global id for the local node id - size_t node_gid = nodes_in_elem(elem_gid, node_lid); - - // Get the corner global id for the local corner id - size_t corner_gid = corners_in_elem(elem_gid, corner_lid); - - double node_radius = 1; - if(num_dims==2){ - node_radius = node_coords(rk_level,node_gid,1); - } - - // calculate the Power=F dot V for this corner - for (size_t dim=0; dim -#include -#include -#include -#include -#include // fmin, fmax, abs note: fminl is long -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include "Tpetra_Details_makeColMap.hpp" -#include "Tpetra_Details_DefaultTypes.hpp" -#include "Tpetra_Details_FixedHashTable.hpp" -#include "Tpetra_Import.hpp" -#include "Tpetra_Import_Util2.hpp" - -#include "elements.h" -#include "swage.h" -#include "matar.h" -#include "utilities.h" -#include "node_combination.h" -#include "FEA_Module_SGH.h" -#include "Explicit_Solver.h" -#include "Simulation_Parameters/Simulation_Parameters_Explicit.h" -#include "Simulation_Parameters/FEA_Module/SGH_Parameters.h" - -//optimization -#include "ROL_Solver.hpp" -#include "Kinetic_Energy_Minimize.h" - -/* ---------------------------------------------------------------------- - Compute new system response due to the design variable update -------------------------------------------------------------------------- */ - -void FEA_Module_SGH::update_forward_solve(Teuchos::RCP zp){ - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - //local variable for host view in the dual view - int num_dim = simparam->num_dims; - int nodes_per_elem = max_nodes_per_element; - int local_node_index, current_row, current_column; - int max_stride = 0; - int current_module_index; - size_t access_index, row_access_index, row_counter; - GO global_index, global_dof_index; - LO local_dof_index; - const size_t num_fills = simparam->regions.size(); - const size_t rk_num_bins = simparam->dynamic_options.rk_num_bins; - const size_t num_bcs = module_params->boundary_conditions.size(); - const size_t num_materials = simparam->materials.size(); - real_t objective_accumulation; - - // --- Read in the nodes in the mesh --- - int myrank = Explicit_Solver_Pointer_->myrank; - int nranks = Explicit_Solver_Pointer_->nranks; - - const DCArrayKokkos mat_fill = simparam->mat_fill; - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos material = simparam->material; - CArray current_element_nodal_densities = CArray(num_nodes_in_elem); - - std::vector> FEA_Module_My_TO_Modules = simparam->FEA_Module_My_TO_Modules; - problem = Explicit_Solver_Pointer_->problem; //Pointer to ROL optimization problem object - ROL::Ptr> obj_pointer; - - //compute element averaged density ratios corresponding to nodal density design variables - {//view scope - const_host_vec_array all_node_densities = all_node_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); - //debug print - //std::cout << "NODE DENSITY TEST " << all_node_densities(0,0) << std::endl; - for(int elem_id = 0; elem_id < rnum_elem; elem_id++){ - for(int inode = 0; inode < num_nodes_in_elem; inode++){ - current_element_nodal_densities(inode) = all_node_densities(nodes_in_elem(elem_id,inode),0); - } - relative_element_densities.host(elem_id) = average_element_density(num_nodes_in_elem, current_element_nodal_densities); - }//for - } //view scope - //debug print - //std::cout << "ELEMENT RELATIVE DENSITY TEST " << relative_element_densities.host(0) << std::endl; - relative_element_densities.update_device(); - - //set density vector to the current value chosen by the optimizer - test_node_densities_distributed = zp; - - //reset nodal coordinates to initial values - node_coords_distributed->assign(*initial_node_coords_distributed); - - //comms for ghosts - Explicit_Solver_Pointer_->comm_coordinates(); - - //view scope - { - const_vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array ghost_node_coords_interface = Explicit_Solver_Pointer_->ghost_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - all_node_coords_interface(node_gid,idim) = node_coords_interface(node_gid,idim); - } - }); // end parallel for - Kokkos::fence(); - - FOR_ALL_CLASS(node_gid, nlocal_nodes, nlocal_nodes+nghost_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - all_node_coords_interface(node_gid,idim) = ghost_node_coords_interface(node_gid-nlocal_nodes,idim); - } - }); // end parallel for - Kokkos::fence(); - } //end view scope - - //reset velocities to initial conditions - node_velocities_distributed->assign(*initial_node_velocities_distributed); - - //reset time accumulating objective and constraints - /* - for(int imodule = 0 ; imodule < FEA_Module_My_TO_Modules[my_fea_module_index_].size(); imodule++){ - current_module_index = FEA_Module_My_TO_Modules[my_fea_module_index_][imodule]; - //test if module needs reset - if(){ - - } - } - */ - //simple setup to just request KE for now; above loop to be expanded and used later for scanning modules - obj_pointer = problem->getObjective(); - KineticEnergyMinimize_TopOpt& kinetic_energy_minimize_function = dynamic_cast(*obj_pointer); - kinetic_energy_minimize_function.objective_accumulation = 0; - - //interface trial density vector - - //interfacing of vectors(should be removed later once made compatible) - //view scope - { - host_vec_array interface_node_coords = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); - for(size_t ibin = 0; ibin < rk_num_bins; ibin++){ - //save node data to node.coords - //std::cout << "NODE DATA ON RANK " << myrank << std::endl; - if(num_dim==2){ - for(int inode = 0; inode < nall_nodes; inode++){ - //std::cout << "Node index " << inode+1 << " "; - node_coords.host(ibin,inode,0) = interface_node_coords(inode,0); - //std::cout << host_node_coords_state(0,inode,0)+1<< " "; - node_coords.host(ibin,inode,1) = interface_node_coords(inode,1); - //std::cout << host_node_coords_state(0,inode,1)+1<< " "; - } - } - else if(num_dim==3){ - for(int inode = 0; inode < nall_nodes; inode++){ - //std::cout << "Node index " << inode+1 << " "; - node_coords.host(ibin,inode,0) = interface_node_coords(inode,0); - //std::cout << host_node_coords_state(0,inode,0)+1<< " "; - node_coords.host(ibin,inode,1) = interface_node_coords(inode,1); - //std::cout << host_node_coords_state(0,inode,1)+1<< " "; - - node_coords.host(ibin,inode,2) = interface_node_coords(inode,2); - //std::cout << host_node_coords_state(0,inode,2)+1<< std::endl; - } - } - } - } //end view scope - - // save the node coords to the current RK value - for (size_t node_gid=0; node_gid < nall_nodes; node_gid++){ - - for(int rk=1; rkcalc_pressure(elem_pres, - elem_stress, - elem_gid, - elem_mat_id(elem_gid), - state_vars, - global_vars, - elem_user_output_vars, - elem_sspd, - elem_den(elem_gid), - elem_sie(rk_level,elem_gid)); - - // --- Sound speed --- - eos_model->calc_sound_speed(elem_pres, - elem_stress, - elem_gid, - elem_mat_id(elem_gid), - state_vars, - global_vars, - elem_user_output_vars, - elem_sspd, - elem_den(elem_gid), - elem_sie(rk_level,elem_gid)); - - // loop over the nodes of this element and apply velocity - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - - // get the mesh node index - size_t node_gid = nodes_in_elem(elem_gid, node_lid); - - - // --- Velocity --- - switch(mat_fill(f_id).velocity) - { - case VELOCITY_TYPE::cartesian: - { - - node_vel(rk_level, node_gid, 0) = mat_fill(f_id).u; - node_vel(rk_level, node_gid, 1) = mat_fill(f_id).v; - if (num_dim == 3) node_vel(rk_level, node_gid, 2) = mat_fill(f_id).w; - - - break; - } - case VELOCITY_TYPE::radial: - { - // Setting up cylindrical - double dir[2]; - dir[0] = 0.0; - dir[1] = 0.0; - double radius_val = 0.0; - - for(int dim=0; dim<2; dim++){ - dir[dim] = node_coords(rk_level, node_gid, dim); - radius_val += node_coords(rk_level, node_gid, dim)*node_coords(rk_level, node_gid, dim); - } // end for - radius_val = sqrt(radius_val); - - for(int dim=0; dim<2; dim++){ - if (radius_val > 1.0e-14){ - dir[dim] /= (radius_val); - } - else{ - dir[dim] = 0.0; - } - } // end for - - - node_vel(rk_level, node_gid, 0) = mat_fill(f_id).speed*dir[0]; - node_vel(rk_level, node_gid, 1) = mat_fill(f_id).speed*dir[1]; - if (num_dim == 3) node_vel(rk_level, node_gid, 2) = 0.0; - - break; - } - case VELOCITY_TYPE::spherical: - { - - // Setting up spherical - double dir[3]; - dir[0] = 0.0; - dir[1] = 0.0; - dir[2] = 0.0; - double radius_val = 0.0; - - for(int dim=0; dim<3; dim++){ - dir[dim] = node_coords(rk_level, node_gid, dim); - radius_val += node_coords(rk_level, node_gid, dim)*node_coords(rk_level, node_gid, dim); - } // end for - radius_val = sqrt(radius_val); - - for(int dim=0; dim<3; dim++){ - if (radius_val > 1.0e-14){ - dir[dim] /= (radius_val); - } - else{ - dir[dim] = 0.0; - } - } // end for - - node_vel(rk_level, node_gid, 0) = mat_fill(f_id).speed*dir[0]; - node_vel(rk_level, node_gid, 1) = mat_fill(f_id).speed*dir[1]; - if (num_dim == 3) node_vel(rk_level, node_gid, 2) = mat_fill(f_id).speed*dir[2]; - - break; - } - case VELOCITY_TYPE::radial_linear: - { - - break; - } - case VELOCITY_TYPE::spherical_linear: - { - - break; - } - case VELOCITY_TYPE::tg_vortex: - { - - node_vel(rk_level, node_gid, 0) = sin(PI * node_coords(rk_level,node_gid, 0)) * cos(PI * node_coords(rk_level,node_gid, 1)); - node_vel(rk_level, node_gid, 1) = -1.0*cos(PI * node_coords(rk_level,node_gid, 0)) * sin(PI * node_coords(rk_level,node_gid, 1)); - if (num_dim == 3) node_vel(rk_level, node_gid, 2) = 0.0; - - break; - } - } // end of switch - - }// end loop over nodes of element - - - if(mat_fill(f_id).velocity == VELOCITY_TYPE::tg_vortex) - { - elem_pres(elem_gid) = 0.25*( cos(2.0*PI*elem_coords[0]) + cos(2.0*PI*elem_coords[1]) ) + 1.0; - - // p = rho*ie*(gamma - 1) - size_t mat_id = f_id; - double gamma = global_vars(mat_id,0); // gamma value - elem_sie(rk_level, elem_gid) = - elem_pres(elem_gid)/(mat_fill(f_id).den*(gamma - 1.0)); - } // end if - - } // end if fill - - }); // end FOR_ALL_CLASS element loop - Kokkos::fence(); - - - } // end for loop over fills - }//end view scope - - - - // apply BC's to velocity - FEA_Module_SGH::boundary_velocity(*mesh, boundary, node_vel); - - - // calculate the corner massess if 2D - if(num_dim==2){ - - FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { - - // facial area of the corners - double corner_areas_array[4]; - - ViewCArrayKokkos corner_areas(&corner_areas_array[0],4); - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 4); - - get_area_weights2D(corner_areas, - elem_gid, - node_coords, - elem_node_gids, - rk_level); - - // loop over the corners of the element and calculate the mass - for (size_t corner_lid=0; corner_lid<4; corner_lid++){ - - size_t corner_gid = corners_in_elem(elem_gid, corner_lid); - corner_mass(corner_gid) = corner_areas(corner_lid)*elem_den(elem_gid); // node radius is added later - - } // end for over corners - }); - - } // end of - - - // calculate the nodal mass - FOR_ALL_CLASS(node_gid, 0, nall_nodes, { - - node_mass(node_gid) = 0.0; - - if(num_dim==3){ - - for(size_t elem_lid=0; elem_lid < num_corners_in_node(node_gid); elem_lid++){ - size_t elem_gid = elems_in_node(node_gid,elem_lid); - node_mass(node_gid) += 1.0/8.0*elem_mass(elem_gid); - } // end for elem_lid - - }// end if dims=3 - else { - - // 2D-RZ - for(size_t corner_lid=0; corner_lid < num_corners_in_node(node_gid); corner_lid++){ - - size_t corner_gid = corners_in_node(node_gid, corner_lid); - node_mass(node_gid) += corner_mass(corner_gid); // sans the radius so it is areal node mass - - corner_mass(corner_gid) *= node_coords(rk_level,node_gid,1); // true corner mass now - } // end for elem_lid - - } // end else - - }); // end FOR_ALL_CLASS - Kokkos::fence(); - - - //current interface has differing mass arrays; this equates them until we unify memory - //view scope - { - vec_array node_mass_interface = node_masses_distributed->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - node_mass_interface(node_gid,0) = node_mass(node_gid); - }); // end parallel for - } //end view scope - Kokkos::fence(); - //communicate ghost densities - comm_node_masses(); - - //this is forcing a copy to the device - //view scope - { - vec_array ghost_node_mass_interface = ghost_node_masses_distributed->getLocalView (Tpetra::Access::ReadWrite); - - FOR_ALL_CLASS(node_gid, nlocal_nodes, nall_nodes, { - node_mass(node_gid) = ghost_node_mass_interface(node_gid-nlocal_nodes,0); - }); // end parallel for - } //end view scope - Kokkos::fence(); - - //update stiffness matrix - if(simparam->topology_optimization_on||simparam->shape_optimization_on){ - //assemble_matrix(); - } - - // update host copies of arrays modified in this function - elem_den.update_host(); - elem_mass.update_host(); - elem_sie.update_host(); - elem_stress.update_host(); - elem_pres.update_host(); - elem_sspd.update_host(); - - //execute solve - sgh_solve(); - -} - -/* ------------------------------------------------------------------------------------------- - Compute average density of an element from nodal densities ----------------------------------------------------------------------------------------------- */ - -double FEA_Module_SGH::average_element_density(const int nodes_per_elem, const CArray current_element_densities) const -{ - double result = 0; - for(int i=0; i < nodes_per_elem; i++){ - result += current_element_densities(i)/nodes_per_elem; - } - - return result; -} - - -/* ------------------------------------------------------------------------------ - Coupled adjoint problem for the kinetic energy minimization problem ---------------------------------------------------------------------------------- */ - -void FEA_Module_SGH::compute_topology_optimization_adjoint_full(){ - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - size_t num_bdy_nodes = mesh->num_bdy_nodes; - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos material = simparam->material; - const int num_dim = simparam->num_dims; - real_t global_dt; - size_t current_data_index, next_data_index; - Teuchos::RCP previous_adjoint_vector_distributed, current_adjoint_vector_distributed, previous_velocity_vector_distributed, current_velocity_vector_distributed; - Teuchos::RCP previous_phi_adjoint_vector_distributed, current_phi_adjoint_vector_distributed; - //initialize first adjoint vector at last_time_step to 0 as the terminal value - (*adjoint_vector_data)[last_time_step+1]->putScalar(0); - (*phi_adjoint_vector_data)[last_time_step+1]->putScalar(0); - (*psi_adjoint_vector_data)[last_time_step+1]->putScalar(0); - - //solve terminal value problem, proceeds in time backward. For simplicity, we use the same timestep data from the forward solve. - //A linear interpolant is assumed between velocity data points; velocity midpoint is used to update the adjoint. - if(myrank==0) - std::cout << "Computing adjoint vector " << time_data.size() << std::endl; - - for (int cycle = last_time_step; cycle >= 0; cycle--) { - //compute timestep from time data - global_dt = time_data[cycle+1] - time_data[cycle]; - //print - if(simparam->dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ - if (cycle==last_time_step){ - if(myrank==0) - printf("cycle = %d, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } - // print time step every 20 cycles - else if (cycle%20==0){ - if(myrank==0) - printf("cycle = %d, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } // end if - } - //else if (cycle==1){ - //if(myrank==0) - //printf("cycle = %lu, time = %f, time step = %f \n", cycle-1, time_data[cycle-1], global_dt); - //} // end if - - //compute adjoint vector for this data point; use velocity midpoint - //view scope - { - - //set velocity, internal energy, and position for this timestep - const_vec_array previous_velocity_vector = (*forward_solve_velocity_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - - const_vec_array previous_coordinate_vector = (*forward_solve_coordinate_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_coordinate_vector = (*forward_solve_coordinate_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - - const_vec_array previous_element_internal_energy = (*forward_solve_internal_energy_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_element_internal_energy = (*forward_solve_internal_energy_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - - //interface of arrays for current implementation of force calculation - - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes+nghost_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - node_vel(rk_level,node_gid,idim) = previous_velocity_vector(node_gid,idim); - node_coords(rk_level,node_gid,idim) = previous_coordinate_vector(node_gid,idim); - } - }); - Kokkos::fence(); - - FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { - elem_sie(rk_level,elem_gid) = previous_element_internal_energy(elem_gid,0); - }); - Kokkos::fence(); - - //set state according to phase data at this timestep - - get_vol(); - - // ---- Calculate velocity diveregence for the element ---- - if(num_dim==2){ - get_divergence2D(elem_div, - node_coords, - node_vel, - elem_vol); - } - else { - get_divergence(elem_div, - node_coords, - node_vel, - elem_vol); - } // end if 2D - - // ---- Calculate elem state (den, pres, sound speed, stress) for next time step ---- - if(num_dim==2){ - update_state2D(material, - *mesh, - node_coords, - node_vel, - elem_den, - elem_pres, - elem_stress, - elem_sspd, - elem_sie, - elem_vol, - elem_mass, - elem_mat_id, - 1.0, - cycle); - } - else{ - update_state(material, - *mesh, - node_coords, - node_vel, - elem_den, - elem_pres, - elem_stress, - elem_sspd, - elem_sie, - elem_vol, - elem_mass, - elem_mat_id, - 1.0, - cycle); - } - - if(num_dim==2){ - get_force_sgh2D(material, - *mesh, - node_coords, - node_vel, - elem_den, - elem_sie, - elem_pres, - elem_stress, - elem_sspd, - elem_vol, - elem_div, - elem_mat_id, - corner_force, - 1.0, - cycle); - } - else { - get_force_sgh(material, - *mesh, - node_coords, - node_vel, - elem_den, - elem_sie, - elem_pres, - elem_stress, - elem_sspd, - elem_vol, - elem_div, - elem_mat_id, - corner_force, - 1.0, - cycle); - } - - //compute gradient matrices - get_force_egradient_sgh(material, - *mesh, - node_coords, - node_vel, - elem_den, - elem_sie, - elem_pres, - elem_stress, - elem_sspd, - elem_vol, - elem_div, - elem_mat_id, - 1.0, - cycle); - - get_power_egradient_sgh(1.0, - *mesh, - node_vel, - node_coords, - elem_sie, - elem_mass, - corner_force); - - get_force_vgradient_sgh(material, - *mesh, - node_coords, - node_vel, - elem_den, - elem_sie, - elem_pres, - elem_stress, - elem_sspd, - elem_vol, - elem_div, - elem_mat_id, - 1.0, - cycle); - - get_power_vgradient_sgh(1.0, - *mesh, - node_vel, - node_coords, - elem_sie, - elem_mass, - corner_force); - - get_force_ugradient_sgh(material, - *mesh, - node_coords, - node_vel, - elem_den, - elem_sie, - elem_pres, - elem_stress, - elem_sspd, - elem_vol, - elem_div, - elem_mat_id, - 1.0, - cycle); - - get_power_ugradient_sgh(1.0, - *mesh, - node_vel, - node_coords, - elem_sie, - elem_mass, - corner_force); - - - //force_gradient_velocity->describe(*fos,Teuchos::VERB_EXTREME); - const_vec_array previous_force_gradient_position = force_gradient_position->getLocalView (Tpetra::Access::ReadOnly); - //const_vec_array current_force_gradient_position = force_gradient_position->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array previous_force_gradient_velocity = force_gradient_velocity->getLocalView (Tpetra::Access::ReadOnly); - //const_vec_array current_force_gradient_velocity = force_gradient_velocity->getLocalView (Tpetra::Access::ReadOnly); - //compute gradient of force with respect to velocity - - const_vec_array previous_adjoint_vector = (*adjoint_vector_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array phi_previous_adjoint_vector = (*phi_adjoint_vector_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array psi_previous_adjoint_vector = (*psi_adjoint_vector_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - vec_array midpoint_adjoint_vector = adjoint_vector_distributed->getLocalView (Tpetra::Access::ReadWrite); - vec_array phi_midpoint_adjoint_vector = phi_adjoint_vector_distributed->getLocalView (Tpetra::Access::ReadWrite); - vec_array psi_midpoint_adjoint_vector = psi_adjoint_vector_distributed->getLocalView (Tpetra::Access::ReadWrite); - - //half step update for RK2 scheme; EQUATION 1 - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - real_t rate_of_change; - real_t matrix_contribution; - size_t dof_id; - size_t elem_id; - for (int idim = 0; idim < num_dim; idim++){ - //EQUATION 1 - matrix_contribution = 0; - //compute resulting row of force velocity gradient matrix transpose right multiplied by adjoint vector - for(int idof = 0; idof < Gradient_Matrix_Strides(node_gid*num_dim+idim); idof++){ - dof_id = DOF_Graph_Matrix(node_gid*num_dim+idim,idof); - matrix_contribution += previous_adjoint_vector(dof_id/num_dim,dof_id%num_dim)*Force_Gradient_Velocities(node_gid*num_dim+idim,idof); - } - - //compute resulting row of transpose of power gradient w.r.t velocity matrix right multiplied by psi adjoint vector - for(int ielem = 0; ielem < DOF_to_Elem_Matrix_Strides(node_gid*num_dim+idim); ielem++){ - elem_id = elems_in_node(node_gid,ielem); - matrix_contribution += psi_previous_adjoint_vector(elem_id,0)*Power_Gradient_Velocities(node_gid*num_dim+idim,ielem); - } - - rate_of_change = previous_velocity_vector(node_gid,idim)- - matrix_contribution/node_mass(node_gid)- - phi_previous_adjoint_vector(node_gid,idim)/node_mass(node_gid); - midpoint_adjoint_vector(node_gid,idim) = -rate_of_change*global_dt/2 + previous_adjoint_vector(node_gid,idim); - - } - }); // end parallel for - Kokkos::fence(); - - //apply BCs to adjoint vector, only matters for the momentum adjoint if using strictly velocity boundary conditions - boundary_adjoint(*mesh, boundary,midpoint_adjoint_vector, phi_midpoint_adjoint_vector, psi_midpoint_adjoint_vector); - - comm_adjoint_vector(cycle); - - //half step update for RK2 scheme; EQUATION 2 - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - real_t rate_of_change; - real_t matrix_contribution; - size_t dof_id; - size_t elem_id; - for (int idim = 0; idim < num_dim; idim++){ - - //EQUATION 2 - matrix_contribution = 0; - //compute resulting row of force displacement gradient matrix transpose right multiplied by adjoint vector - for(int idof = 0; idof < Gradient_Matrix_Strides(node_gid*num_dim+idim); idof++){ - dof_id = DOF_Graph_Matrix(node_gid*num_dim+idim,idof); - matrix_contribution += previous_adjoint_vector(dof_id/num_dim,dof_id%num_dim)*Force_Gradient_Positions(node_gid*num_dim+idim,idof); - } - - //compute resulting row of transpose of power gradient w.r.t displacement matrix right multiplied by psi adjoint vector - for(int ielem = 0; ielem < DOF_to_Elem_Matrix_Strides(node_gid*num_dim+idim); ielem++){ - elem_id = elems_in_node(node_gid,ielem); - matrix_contribution += psi_previous_adjoint_vector(elem_id,0)*Power_Gradient_Positions(node_gid*num_dim+idim,ielem); - } - - rate_of_change = -matrix_contribution; - //rate_of_change = -0.0000001*previous_adjoint_vector(node_gid,idim); - phi_midpoint_adjoint_vector(node_gid,idim) = -rate_of_change*global_dt/2 + phi_previous_adjoint_vector(node_gid,idim); - - } - }); // end parallel for - Kokkos::fence(); - - comm_phi_adjoint_vector(cycle); - - //phi_adjoint_vector_distributed->describe(*fos,Teuchos::VERB_EXTREME); - - //half step update for RK2 scheme; EQUATION 3 - FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { - real_t rate_of_change; - real_t matrix_contribution; - size_t dof_id; - size_t elem_id; - //EQUATION 3 - matrix_contribution = 0; - //compute resulting row of force displacement gradient matrix transpose right multiplied by adjoint vector - for(int idof = 0; idof < num_nodes_in_elem*num_dim; idof++){ - dof_id = nodes_in_elem(elem_gid,idof/num_dim)*num_dim + idof%num_dim; - matrix_contribution += previous_adjoint_vector(dof_id/num_dim,dof_id%num_dim)*Force_Gradient_Energies(elem_gid,idof); - } - rate_of_change = -(matrix_contribution + psi_previous_adjoint_vector(elem_gid,0)*Power_Gradient_Energies(elem_gid))/elem_mass(elem_gid); - //rate_of_change = -0.0000001*previous_adjoint_vector(node_gid,idim); - psi_midpoint_adjoint_vector(elem_gid,0) = -rate_of_change*global_dt/2 + psi_previous_adjoint_vector(elem_gid,0); - }); // end parallel for - Kokkos::fence(); - - //save for second half of RK - (*psi_adjoint_vector_data)[cycle]->assign(*psi_adjoint_vector_distributed); - - //swap names to get ghost nodes for the midpoint vectors - vec_array current_adjoint_vector = adjoint_vector_distributed->getLocalView (Tpetra::Access::ReadWrite); - vec_array phi_current_adjoint_vector = phi_adjoint_vector_distributed->getLocalView (Tpetra::Access::ReadWrite); - vec_array psi_current_adjoint_vector = psi_adjoint_vector_distributed->getLocalView (Tpetra::Access::ReadWrite); - midpoint_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView (Tpetra::Access::ReadWrite); - phi_midpoint_adjoint_vector = (*phi_adjoint_vector_data)[cycle]->getLocalView (Tpetra::Access::ReadWrite); - psi_midpoint_adjoint_vector = (*psi_adjoint_vector_data)[cycle]->getLocalView (Tpetra::Access::ReadWrite); - - - //full step update with midpoint gradient for RK2 scheme; EQUATION 1 - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - real_t rate_of_change; - real_t matrix_contribution; - size_t dof_id; - size_t elem_id; - for (int idim = 0; idim < num_dim; idim++){ - //EQUATION 1 - matrix_contribution = 0; - //compute resulting row of force velocity gradient matrix transpose right multiplied by adjoint vector - - for(int idof = 0; idof < Gradient_Matrix_Strides(node_gid*num_dim+idim); idof++){ - dof_id = DOF_Graph_Matrix(node_gid*num_dim+idim,idof); - matrix_contribution += midpoint_adjoint_vector(dof_id/num_dim,dof_id%num_dim)*Force_Gradient_Velocities(node_gid*num_dim+idim,idof); - } - - //compute resulting row of transpose of power gradient w.r.t velocity matrix right multiplied by psi adjoint vector - for(int ielem = 0; ielem < DOF_to_Elem_Matrix_Strides(node_gid*num_dim+idim); ielem++){ - elem_id = elems_in_node(node_gid,ielem); - matrix_contribution += psi_midpoint_adjoint_vector(elem_id,0)*Power_Gradient_Velocities(node_gid*num_dim+idim,ielem); - } - - rate_of_change = (previous_velocity_vector(node_gid,idim) + current_velocity_vector(node_gid,idim))/2- - matrix_contribution/node_mass(node_gid)- - phi_midpoint_adjoint_vector(node_gid,idim)/node_mass(node_gid); - current_adjoint_vector(node_gid,idim) = -rate_of_change*global_dt + previous_adjoint_vector(node_gid,idim); - } - }); // end parallel for - Kokkos::fence(); - - boundary_adjoint(*mesh, boundary,current_adjoint_vector, phi_current_adjoint_vector, psi_midpoint_adjoint_vector); - comm_adjoint_vector(cycle); - - //full step update with midpoint gradient for RK2 scheme; EQUATION 2 - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - real_t rate_of_change; - real_t matrix_contribution; - size_t dof_id; - size_t elem_id; - for (int idim = 0; idim < num_dim; idim++){ - - //EQUATION 2 - matrix_contribution = 0; - //compute resulting row of force displacement gradient matrix transpose right multiplied by adjoint vector - for(int idof = 0; idof < Gradient_Matrix_Strides(node_gid*num_dim+idim); idof++){ - dof_id = DOF_Graph_Matrix(node_gid*num_dim+idim,idof); - matrix_contribution += midpoint_adjoint_vector(dof_id/num_dim,dof_id%num_dim)*Force_Gradient_Positions(node_gid*num_dim+idim,idof); - } - - //compute resulting row of transpose of power gradient w.r.t displacement matrix right multiplied by psi adjoint vector - for(int ielem = 0; ielem < DOF_to_Elem_Matrix_Strides(node_gid*num_dim+idim); ielem++){ - elem_id = elems_in_node(node_gid,ielem); - matrix_contribution += psi_midpoint_adjoint_vector(elem_id,0)*Power_Gradient_Positions(node_gid*num_dim+idim,ielem); - } - - rate_of_change = -matrix_contribution; - //rate_of_change = -0.0000001*midpoint_adjoint_vector(node_gid,idim); - phi_current_adjoint_vector(node_gid,idim) = -rate_of_change*global_dt + phi_previous_adjoint_vector(node_gid,idim); - } - }); // end parallel for - Kokkos::fence(); - - comm_phi_adjoint_vector(cycle); - - //full step update for RK2 scheme; EQUATION 3 - FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { - real_t rate_of_change; - real_t matrix_contribution; - size_t dof_id; - size_t elem_id; - //EQUATION 3 - matrix_contribution = 0; - //compute resulting row of force displacement gradient matrix transpose right multiplied by adjoint vector - for(int idof = 0; idof < num_nodes_in_elem*num_dim; idof++){ - dof_id = nodes_in_elem(elem_gid,idof/num_dim)*num_dim + idof%num_dim; - matrix_contribution += midpoint_adjoint_vector(dof_id/num_dim,dof_id%num_dim)*Force_Gradient_Energies(elem_gid,idof); - } - rate_of_change = -(matrix_contribution + psi_midpoint_adjoint_vector(elem_gid,0)*Power_Gradient_Energies(elem_gid))/elem_mass(elem_gid); - //debug - //std::cout << "PSI RATE OF CHANGE " << rate_of_change << std::endl; - psi_current_adjoint_vector(elem_gid,0) = -rate_of_change*global_dt + psi_previous_adjoint_vector(elem_gid,0); - }); // end parallel for - Kokkos::fence(); - - //save data from time-step completion - (*psi_adjoint_vector_data)[cycle]->assign(*psi_adjoint_vector_distributed); - - } //end view scope - - //phi_adjoint_vector_distributed->describe(*fos,Teuchos::VERB_EXTREME); - - } -} - -/* ---------------------------------------------------------------------------- - Gradient for the (unsimplified) kinetic energy minimization problem -------------------------------------------------------------------------------- */ - -void FEA_Module_SGH::compute_topology_optimization_gradient_full(Teuchos::RCP design_densities_distributed, Teuchos::RCP design_gradients_distributed){ - - size_t num_bdy_nodes = mesh->num_bdy_nodes; - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos material = simparam->material; - const int num_dim = simparam->num_dims; - int num_corners = rnum_elem*num_nodes_in_elem; - real_t global_dt; - bool element_constant_density = true; - size_t current_data_index, next_data_index; - CArrayKokkos current_element_velocities = CArrayKokkos(num_nodes_in_elem,num_dim); - CArrayKokkos current_element_adjoint = CArrayKokkos(num_nodes_in_elem,num_dim); - - if(myrank==0) - std::cout << "Computing accumulated kinetic energy gradient" << std::endl; - - compute_topology_optimization_adjoint_full(); - - - - { //view scope - vec_array design_gradients = design_gradients_distributed->getLocalView (Tpetra::Access::ReadWrite); - const_vec_array design_densities = design_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); - //initialize design gradients - FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { - design_gradients(node_id,0) = 0; - }); // end parallel for - Kokkos::fence(); - - //gradient contribution from kinetic energy v(dM/drho)v product. - if(simparam->dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ - if(myrank==0){ - std::cout << "v*dM/drho*v term" << std::endl; - } - } - - for (unsigned long cycle = 0; cycle < last_time_step+1; cycle++) { - //compute timestep from time data - global_dt = time_data[cycle+1] - time_data[cycle]; - - //print - if(simparam->dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ - - if (cycle==0){ - if(myrank==0) - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } - // print time step every 10 cycles - else if (cycle%20==0){ - if(myrank==0) - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } // end if - } - //view scope - { - const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array next_velocity_vector = (*forward_solve_velocity_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - - FOR_ALL_CLASS(elem_id, 0, rnum_elem, { - size_t node_id; - size_t corner_id; - real_t inner_product; - //std::cout << elem_mass(elem_id) <dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ - if(myrank==0){ - std::cout << "gradient term involving adjoint derivative" << std::endl; - } - } - - for (unsigned long cycle = 0; cycle < last_time_step+1; cycle++) { - //compute timestep from time data - global_dt = time_data[cycle+1] - time_data[cycle]; - //print - if(simparam->dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ - - if (cycle==0){ - if(myrank==0) - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } - // print time step every 10 cycles - else if (cycle%20==0){ - if(myrank==0) - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } // end if - } - - //compute adjoint vector for this data point; use velocity midpoint - //view scope - { - const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array next_velocity_vector = (*forward_solve_velocity_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array next_adjoint_vector = (*adjoint_vector_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - - FOR_ALL_CLASS(elem_id, 0, rnum_elem, { - real_t lambda_dot_current; - real_t lambda_dot_next; - size_t node_id; - size_t corner_id; - real_t inner_product; - //std::cout << elem_mass(elem_id) <dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ - if(myrank==0){ - std::cout << "gradient term involving adjoint derivative" << std::endl; - } - } - - for (unsigned long cycle = 0; cycle < last_time_step+1; cycle++) { - //compute timestep from time data - global_dt = time_data[cycle+1] - time_data[cycle]; - //print - if(simparam->dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ - - if (cycle==0){ - if(myrank==0) - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } - // print time step every 10 cycles - else if (cycle%20==0){ - if(myrank==0) - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } // end if - } - - //compute adjoint vector for this data point; use velocity midpoint - //view scope - { - const_vec_array current_element_internal_energy = (*forward_solve_internal_energy_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_psi_adjoint_vector = (*psi_adjoint_vector_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array next_element_internal_energy = (*forward_solve_internal_energy_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array next_psi_adjoint_vector = (*psi_adjoint_vector_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - - FOR_ALL_CLASS(elem_id, 0, rnum_elem, { - real_t psi_dot_current; - real_t psi_dot_next; - size_t node_id; - size_t corner_id; - real_t inner_product; - //std::cout << elem_mass(elem_id) <getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_adjoint_vector = (*adjoint_vector_data)[0]->getLocalView (Tpetra::Access::ReadOnly); - - FOR_ALL_CLASS(elem_id, 0, rnum_elem, { - real_t lambda_dot; - size_t node_id; - size_t corner_id; - real_t inner_product; - //std::cout << elem_mass(elem_id) <getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_psi_adjoint_vector = (*psi_adjoint_vector_data)[0]->getLocalView (Tpetra::Access::ReadOnly); - - //(*psi_adjoint_vector_data)[100]->describe(*fos,Teuchos::VERB_EXTREME); - FOR_ALL_CLASS(elem_id, 0, rnum_elem, { - real_t lambda_dot; - size_t node_id; - size_t corner_id; - real_t inner_product; - //std::cout << elem_mass(elem_id) <getLocalView (Tpetra::Access::ReadWrite); - //const_host_vec_array host_design_variables = design_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); - vec_array design_gradients = design_gradients_distributed->getLocalView (Tpetra::Access::ReadWrite); - const_vec_array design_variables = design_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); - force_design_gradient_term(design_variables, design_gradients); - power_design_gradient_term(design_variables, design_gradients); - }//end view scope - -} - -/* ---------------------------------------------------------------------- - Initialize global vectors and array maps needed for matrix assembly -------------------------------------------------------------------------- */ -void FEA_Module_SGH::init_assembly(){ - int num_dim = simparam->num_dims; - //const_host_elem_conn_array nodes_in_elem = global_nodes_in_elem_distributed->getLocalView (Tpetra::Access::ReadOnly); - Gradient_Matrix_Strides = DCArrayKokkos (nlocal_nodes*num_dim, "Gradient_Matrix_Strides"); - DOF_to_Elem_Matrix_Strides = DCArrayKokkos (nlocal_nodes*num_dim, "Gradient_Matrix_Strides"); - Elem_to_Elem_Matrix_Strides = DCArrayKokkos (rnum_elem, "Gradient_Matrix_Strides"); - CArrayKokkos Graph_Fill(nall_nodes, "nall_nodes"); - CArrayKokkos current_row_nodes_scanned; - CArrayKokkos count_saved_corners_in_node(nall_nodes, "count_saved_corners_in_node"); - int local_node_index, current_column_index; - size_t max_stride = 0; - size_t nodes_per_element; - nodal_density_flag = simparam->nodal_density_flag; - penalty_power = simparam->optimization_options.simp_penalty_power; - - //allocate stride arrays - CArrayKokkos Graph_Matrix_Strides_initial(nlocal_nodes, "Graph_Matrix_Strides_initial"); - DCArrayKokkos Dual_Graph_Matrix_Strides_initial(nlocal_nodes, "Host_Graph_Matrix_Strides_initial"); - Graph_Matrix_Strides = DCArrayKokkos(nlocal_nodes, "Graph_Matrix_Strides"); - - //allocate storage for the sparse gradient matrix map for node to node connectivity - Global_Gradient_Matrix_Assembly_Map = DCArrayKokkos(rnum_elem, - max_nodes_per_element,max_nodes_per_element, "Global_Gradient_Matrix_Assembly_Map"); - - //allocate storage for the sparse gradient matrix map for node to element connectivity - Element_Gradient_Matrix_Assembly_Map = DCArrayKokkos(rnum_elem, - max_nodes_per_element, "Element_Gradient_Matrix_Assembly_Map"); - - //allocate array used to determine global node repeats in the sparse graph later - DCArrayKokkos node_indices_used(nall_nodes, "node_indices_used"); - - /*allocate array that stores which column the node index occured on for the current row - when removing repeats*/ - DCArrayKokkos column_index(nall_nodes, "column_index"); - - //initialize nlocal arrays - FOR_ALL_CLASS(inode, 0, nlocal_nodes, { - Graph_Matrix_Strides_initial(inode) = 0; - Graph_Matrix_Strides(inode) = 0; - Graph_Fill(inode) = 0; - }); // end parallel for - Kokkos::fence(); - - //initialize nall arrays - //initialize nlocal arrays - FOR_ALL_CLASS(inode, 0, nall_nodes, { - node_indices_used(inode) = 0; - column_index(inode) = 0; - count_saved_corners_in_node(inode) = 0; - }); // end parallel for - Kokkos::fence(); - - //count upper bound of strides for Sparse Pattern Graph by allowing repeats due to connectivity - if(num_dim == 2) - for (int ielem = 0; ielem < rnum_elem; ielem++){ - element_select->choose_2Delem_type(Element_Types(ielem), elem2D); - nodes_per_element = elem2D->num_nodes(); - for (int lnode = 0; lnode < nodes_per_element; lnode++){ - local_node_index = nodes_in_elem(ielem, lnode); - if(local_node_index < nlocal_nodes){ - Dual_Graph_Matrix_Strides_initial.host(local_node_index) += nodes_per_element; - } - } - } - - if(num_dim == 3) - for (int ielem = 0; ielem < rnum_elem; ielem++){ - element_select->choose_3Delem_type(Element_Types(ielem), elem); - nodes_per_element = elem->num_nodes(); - for (int lnode = 0; lnode < nodes_per_element; lnode++){ - local_node_index = nodes_in_elem(ielem, lnode); - if(local_node_index < nlocal_nodes){ - Dual_Graph_Matrix_Strides_initial.host(local_node_index) += nodes_per_element; - } - } - } - - Dual_Graph_Matrix_Strides_initial.update_device(); - - //equate strides for later - FOR_ALL_CLASS(inode, 0, nlocal_nodes, { - Graph_Matrix_Strides(inode) = Graph_Matrix_Strides_initial(inode) = Dual_Graph_Matrix_Strides_initial(inode); - }); // end parallel for - - //for (int inode = 0; inode < nlocal_nodes; inode++) - //std::cout << Graph_Matrix_Strides_initial(inode) << std::endl; - - //compute maximum stride - size_t update = 0; - REDUCE_MAX_CLASS(inode, 0, nlocal_nodes, update, { - if(update < Graph_Matrix_Strides_initial(inode)) - update = Graph_Matrix_Strides_initial(inode); - }, max_stride); - - //std::cout << "THE MAX STRIDE" << max_stride << std::endl; - //allocate array used in the repeat removal process - current_row_nodes_scanned = CArrayKokkos(max_stride, "current_row_nodes_scanned"); - - //allocate sparse graph with node repeats - RaggedRightArrayKokkos Repeat_Graph_Matrix(Graph_Matrix_Strides_initial); - RaggedRightArrayofVectorsKokkos Element_local_indices(Graph_Matrix_Strides_initial,num_dim); - - //Fill the initial Graph with repeats - if(num_dim == 2){ - for (int ielem = 0; ielem < rnum_elem; ielem++){ - element_select->choose_2Delem_type(Element_Types(ielem), elem2D); - nodes_per_element = elem2D->num_nodes(); - for (int lnode = 0; lnode < nodes_per_element; lnode++){ - local_node_index = nodes_in_elem(ielem, lnode); - if(local_node_index < nlocal_nodes){ - for (int jnode = 0; jnode < nodes_per_element; jnode++){ - current_column_index = Graph_Fill(local_node_index)+jnode; - Repeat_Graph_Matrix(local_node_index, current_column_index) = nodes_in_elem(ielem,jnode); - - //fill inverse map - Element_local_indices(local_node_index,current_column_index,0) = ielem; - Element_local_indices(local_node_index,current_column_index,1) = lnode; - Element_local_indices(local_node_index,current_column_index,2) = jnode; - - //fill forward map - Global_Gradient_Matrix_Assembly_Map(ielem,lnode,jnode) = current_column_index; - } - Graph_Fill(local_node_index) += nodes_per_element; - } - } - } - } - - if(num_dim == 3){ - for (int ielem = 0; ielem < rnum_elem; ielem++){ - element_select->choose_3Delem_type(Element_Types(ielem), elem); - nodes_per_element = elem->num_nodes(); - for (int lnode = 0; lnode < nodes_per_element; lnode++){ - local_node_index = nodes_in_elem(ielem, lnode); - if(local_node_index < nlocal_nodes){ - for (int jnode = 0; jnode < nodes_per_element; jnode++){ - current_column_index = Graph_Fill(local_node_index)+jnode; - Repeat_Graph_Matrix(local_node_index, current_column_index) = nodes_in_elem(ielem,jnode); - - //fill inverse map - Element_local_indices(local_node_index,current_column_index,0) = ielem; - Element_local_indices(local_node_index,current_column_index,1) = lnode; - Element_local_indices(local_node_index,current_column_index,2) = jnode; - - //fill forward map - Global_Gradient_Matrix_Assembly_Map(ielem,lnode,jnode) = current_column_index; - } - Graph_Fill(local_node_index) += nodes_per_element; - } - } - } - } - - //debug statement - //std::cout << "started run" << std::endl; - //std::cout << "Graph Matrix Strides Repeat on task " << myrank << std::endl; - //for (int inode = 0; inode < nlocal_nodes; inode++) - //std::cout << Graph_Matrix_Strides(inode) << std::endl; - RUN_CLASS({ - //remove repeats from the inital graph setup - int current_node; - //remove repeats from the inital graph setup - int current_element_index; - int element_row_index; - int element_column_index; - int current_stride; - int current_row_n_nodes_scanned; - for (int inode = 0; inode < nlocal_nodes; inode++){ - current_row_n_nodes_scanned = 0; - for (int istride = 0; istride < Graph_Matrix_Strides(inode); istride++){ - //convert global index in graph to its local index for the flagging array - current_node = Repeat_Graph_Matrix(inode,istride); - //debug - //if(current_node==-1) - //std::cout << "Graph Matrix node access on task " << myrank << std::endl; - //std::cout << Repeat_Graph_Matrix(inode,istride) << std::endl; - if(node_indices_used(current_node)){ - //set global assembly map index to the location in the graph matrix where this global node was first found - current_element_index = Element_local_indices(inode,istride,0); - element_row_index = Element_local_indices(inode,istride,1); - element_column_index = Element_local_indices(inode,istride,2); - Global_Gradient_Matrix_Assembly_Map(current_element_index,element_row_index, element_column_index) - = column_index(current_node); - - - //swap current node with the end of the current row and shorten the stride of the row - //first swap information about the inverse and forward maps - - current_stride = Graph_Matrix_Strides(inode); - if(istride!=current_stride-1){ - Element_local_indices(inode,istride,0) = Element_local_indices(inode,current_stride-1,0); - Element_local_indices(inode,istride,1) = Element_local_indices(inode,current_stride-1,1); - Element_local_indices(inode,istride,2) = Element_local_indices(inode,current_stride-1,2); - current_element_index = Element_local_indices(inode,istride,0); - element_row_index = Element_local_indices(inode,istride,1); - element_column_index = Element_local_indices(inode,istride,2); - - Global_Gradient_Matrix_Assembly_Map(current_element_index,element_row_index, element_column_index) - = istride; - - //now that the element map information has been copied, copy the global node index and delete the last index - - Repeat_Graph_Matrix(inode,istride) = Repeat_Graph_Matrix(inode,current_stride-1); - } - istride--; - Graph_Matrix_Strides(inode)--; - } - else{ - /*this node hasn't shown up in the row before; add it to the list of nodes - that have been scanned uniquely. Use this list to reset the flag array - afterwards without having to loop over all the nodes in the system*/ - node_indices_used(current_node) = 1; - column_index(current_node) = istride; - current_row_nodes_scanned(current_row_n_nodes_scanned) = current_node; - current_row_n_nodes_scanned++; - } - } - //reset nodes used list for the next row of the sparse list - for(int node_reset = 0; node_reset < current_row_n_nodes_scanned; node_reset++) - node_indices_used(current_row_nodes_scanned(node_reset)) = 0; - - } - }); - Kokkos::fence(); - - Graph_Matrix_Strides.update_host(); - //copy reduced content to non_repeat storage - Graph_Matrix = RaggedRightArrayKokkos(Graph_Matrix_Strides); - - FOR_ALL_CLASS(inode, 0, nlocal_nodes, { - for(int istride = 0; istride < Graph_Matrix_Strides(inode); istride++){ - Graph_Matrix(inode,istride) = Repeat_Graph_Matrix(inode,istride); - } - }); // end parallel for - - //deallocate repeat matrix - - /*At this stage the sparse graph should have unique global indices on each row. - The constructed Assembly map (to the global sparse matrix) - is used to loop over each element's local stiffness matrix in the assembly process.*/ - - //expand strides for stiffness matrix by multipling by dim - FOR_ALL_CLASS(idof, 0, num_dim*nlocal_nodes, { - Gradient_Matrix_Strides(idof) = num_dim*Graph_Matrix_Strides(idof/num_dim); - }); // end parallel for - - Gradient_Matrix_Strides.update_host(); - - //build inverse map for element gradient assembly - for (size_t elem_gid = 0; elem_gid < rnum_elem; elem_gid++){ - FOR_ALL_CLASS(node_lid, 0, num_nodes_in_elem, { - - // get the global_id of the node - size_t node_gid = nodes_in_elem(elem_gid, node_lid); - - // the column index is the num corners saved - size_t j = count_saved_corners_in_node(node_gid); - Element_Gradient_Matrix_Assembly_Map(elem_gid, node_lid) = j; - - // increment the number of corners saved to this node_gid - count_saved_corners_in_node(node_gid)++; - - }); // end FOR_ALL over nodes in element - Kokkos::fence(); - } // end for elem_gid - - DOF_Graph_Matrix = RaggedRightArrayKokkos(Gradient_Matrix_Strides); - Force_Gradient_Positions = RaggedRightArrayKokkos(Gradient_Matrix_Strides); - Force_Gradient_Velocities = RaggedRightArrayKokkos(Gradient_Matrix_Strides); - //needs different graph of node to elem rather than node to node - //DOF_to_Elem_Matrix_Strides.get_kokkos_dual_view().d_view = elems_in_node.mystrides_; - Kokkos::View node_to_elem_strides = elems_in_node.mystrides_; - DOF_to_Elem_Matrix_Strides = DCArrayKokkos(nlocal_nodes*num_dim); - FOR_ALL_CLASS(inode, 0, nlocal_nodes, { - for(int idim = 0; idim < num_dim; idim++){ - DOF_to_Elem_Matrix_Strides(inode*num_dim + idim) = count_saved_corners_in_node(inode); - } - }); // end parallel for - DOF_to_Elem_Matrix_Strides.update_host(); - Force_Gradient_Energies = CArrayKokkos(rnum_elem, num_nodes_in_elem*num_dim); - Power_Gradient_Energies = CArrayKokkos(rnum_elem); - Power_Gradient_Positions = RaggedRightArrayKokkos(DOF_to_Elem_Matrix_Strides); - Power_Gradient_Velocities = RaggedRightArrayKokkos(DOF_to_Elem_Matrix_Strides); - - //set stiffness Matrix Graph - //debug print - //std::cout << "DOF GRAPH MATRIX ENTRIES ON TASK " << myrank << std::endl; - FOR_ALL_CLASS(idof, 0, num_dim*nlocal_nodes, { - for (int istride = 0; istride < Gradient_Matrix_Strides(idof); istride++){ - DOF_Graph_Matrix(idof,istride) = Graph_Matrix(idof/num_dim,istride/num_dim)*num_dim + istride%num_dim; - } - }); // end parallel for - - - /* - //construct distributed gradient matrix from local kokkos data - //build column map for the global gradient matrix - Teuchos::RCP > colmap; - const Teuchos::RCP > dommap = local_dof_map; - - Tpetra::Details::makeColMap(colmap,dommap,DOF_Graph_Matrix.get_kokkos_view(), nullptr); - - size_t nnz = DOF_Graph_Matrix.size(); - - //debug print - //std::cout << "DOF GRAPH SIZE ON RANK " << myrank << " IS " << nnz << std::endl; - - //local indices in the graph using the constructed column map - CArrayKokkos gradient_local_indices(nnz, "gradient_local_indices"); - - //row offsets with compatible template arguments - Kokkos::View row_offsets = DOF_Graph_Matrix.start_index_; - row_pointers row_offsets_pass("row_offsets", nlocal_nodes*num_dim+1); - for(int ipass = 0; ipass < nlocal_nodes*num_dim + 1; ipass++){ - row_offsets_pass(ipass) = row_offsets(ipass); - } - - size_t entrycount = 0; - for(int irow = 0; irow < nlocal_nodes*num_dim; irow++){ - for(int istride = 0; istride < Gradient_Matrix_Strides(irow); istride++){ - gradient_local_indices(entrycount) = colmap->getLocalElement(DOF_Graph_Matrix(irow,istride)); - entrycount++; - } - } - - - //sort values and indices - Tpetra::Import_Util::sortCrsEntries(row_offsets_pass, gradient_local_indices.get_kokkos_view(), Force_Gradient_Positions.get_kokkos_view()); - Tpetra::Import_Util::sortCrsEntries(row_offsets_pass, gradient_local_indices.get_kokkos_view(), Force_Gradient_Velocities.get_kokkos_view()); - - //Teuchos::RCP crs_matrix_params = Teuchos::rcp(new Teuchos::ParameterList("crsmatrix")); - //crs_matrix_params->set("sorted", false); - distributed_force_gradient_positions = Teuchos::rcp(new MAT(local_dof_map, colmap, row_offsets_pass, gradient_local_indices.get_kokkos_view(), Force_Gradient_Positions.get_kokkos_view())); - distributed_force_gradient_positions->fillComplete(); - distributed_force_gradient_velocities = Teuchos::rcp(new MAT(local_dof_map, colmap, row_offsets_pass, gradient_local_indices.get_kokkos_view(), Force_Gradient_Velocities.get_kokkos_view())); - distributed_force_gradient_velocities->fillComplete(); - */ - //distributed_force_gradient_positions->describe(*fos,Teuchos::VERB_EXTREME); - //distributed_force_gradient_velocities->describe(*fos,Teuchos::VERB_EXTREME); -} - -/* ---------------------------------------------------------------------- - Enforce boundary conditions on the adjoint vectors -------------------------------------------------------------------------- */ - -void FEA_Module_SGH::boundary_adjoint(const mesh_t &mesh, - const DCArrayKokkos &boundary, - vec_array &node_adjoint, - vec_array &node_phi_adjoint, - vec_array &node_psi_adjoint){ - - //error and debug flag - //DCArrayKokkos print_flag(1, "print_flag"); - //print_flag.host(0) = false; - //print_flag.update_device(); - - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - int num_dims = simparam->num_dims; - // Loop over boundary sets - for (size_t bdy_set=0; bdy_setdoImport(*adjoint_vector_distributed, *importer, Tpetra::INSERT); -} - -/* ---------------------------------------------------------------------- - Communicate updated nodal adjoint vectors to ghost nodes -------------------------------------------------------------------------- */ - -void FEA_Module_SGH::comm_phi_adjoint_vector(int cycle){ - //comms to get ghosts - (*phi_adjoint_vector_data)[cycle]->doImport(*phi_adjoint_vector_distributed, *importer, Tpetra::INSERT); -} diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/FEA_Module_SGH.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/FEA_Module_SGH.cpp new file mode 100644 index 000000000..ac3a309ed --- /dev/null +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/FEA_Module_SGH.cpp @@ -0,0 +1,1725 @@ +/********************************************************************************************** + © 2020. Triad National Security, LLC. All rights reserved. + This program was produced under U.S. Government contract 89233218CNA000001 for Los Alamos + National Laboratory (LANL), which is operated by Triad National Security, LLC for the U.S. + Department of Energy/National Nuclear Security Administration. All rights in the program are + reserved by Triad National Security, LLC, and the U.S. Department of Energy/National Nuclear + Security Administration. The Government is granted for itself and others acting on its behalf a + nonexclusive, paid-up, irrevocable worldwide license in this material to reproduce, prepare + derivative works, distribute copies to the public, perform publicly and display publicly, and + to permit others to do so. + This program is open source under the BSD-3 License. + Redistribution and use in source and binary forms, with or without modification, are permitted + provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, this list of + conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, this list of + conditions and the following disclaimer in the documentation and/or other materials + provided with the distribution. + + 3. Neither the name of the copyright holder nor the names of its contributors may be used + to endorse or promote products derived from this software without specific prior + written permission. + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS + IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **********************************************************************************************/ + +#include +#include +#include +#include +#include +#include +#include // fmin, fmax, abs note: fminl is long +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include "Tpetra_Import.hpp" +#include "Tpetra_Import_Util2.hpp" + +#include "elements.h" +#include "swage.h" +#include "matar.h" +#include "utilities.h" +#include "node_combination.h" +#include "Simulation_Parameters/Simulation_Parameters_Explicit.h" +#include "Simulation_Parameters/FEA_Module/SGH_Parameters.h" +#include "FEA_Module_SGH.h" +#include "Explicit_Solver.h" + +// optimization +#include "ROL_Solver.hpp" +#include "Kinetic_Energy_Minimize.h" + +#define MAX_ELEM_NODES 8 +#define STRAIN_EPSILON 0.000000001 +#define DENSITY_EPSILON 0.0001 +#define BC_EPSILON 1.0e-6 +#define BUFFER_GROW 100 + +using namespace utils; + +FEA_Module_SGH::FEA_Module_SGH( + SGH_Parameters& params, Solver* Solver_Pointer, + std::shared_ptr mesh_in, const int my_fea_module_index) + : FEA_Module(Solver_Pointer) +{ + // assign interfacing index + my_fea_module_index_ = my_fea_module_index; + Module_Type = FEA_MODULE_TYPE::SGH; + + // recast solver pointer for non-base class access + Explicit_Solver_Pointer_ = dynamic_cast(Solver_Pointer); + simparam = &(Explicit_Solver_Pointer_->simparam); + module_params = ¶ms; + + // create ref element object + // ref_elem = new elements::ref_element(); + // create mesh objects + // init_mesh = new swage::mesh_t(simparam); + // mesh = new swage::mesh_t(simparam); + + mesh = mesh_in; + + // boundary condition data + max_boundary_sets = 0; + Local_Index_Boundary_Patches = Explicit_Solver_Pointer_->Local_Index_Boundary_Patches; + + // set Tpetra vector pointers + initial_node_velocities_distributed = Explicit_Solver_Pointer_->initial_node_velocities_distributed; + initial_node_coords_distributed = Explicit_Solver_Pointer_->initial_node_coords_distributed; + all_initial_node_coords_distributed = Explicit_Solver_Pointer_->all_initial_node_coords_distributed; + node_coords_distributed = Explicit_Solver_Pointer_->node_coords_distributed; + node_velocities_distributed = Explicit_Solver_Pointer_->node_velocities_distributed; + all_node_velocities_distributed = Explicit_Solver_Pointer_->all_node_velocities_distributed; + + // Switch for optimization solver + if (simparam->topology_optimization_on || simparam->shape_optimization_on) + { + all_cached_node_velocities_distributed = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + force_gradient_velocity = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + force_gradient_position = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + force_gradient_design = Teuchos::rcp(new MV(all_node_map, 1)); + corner_value_storage = Solver_Pointer->corner_value_storage; + corner_vector_storage = Solver_Pointer->corner_vector_storage; + corner_gradient_storage = Solver_Pointer->corner_gradient_storage; + relative_element_densities = DCArrayKokkos(rnum_elem, "relative_element_densities"); + adjoint_vector_distributed = Teuchos::rcp(new MV(map, simparam->num_dims)); + phi_adjoint_vector_distributed = Teuchos::rcp(new MV(map, simparam->num_dims)); + psi_adjoint_vector_distributed = Teuchos::rcp(new MV(all_element_map, 1)); + } + + if (simparam->topology_optimization_on || simparam->shape_optimization_on || simparam->num_dims == 2) + { + node_masses_distributed = Teuchos::rcp(new MV(map, 1)); + ghost_node_masses_distributed = Teuchos::rcp(new MV(ghost_node_map, 1)); + } + + // setup output + noutput = 0; + init_output(); + + // optimization flags + kinetic_energy_objective = false; + + // set parameters + Dynamic_Options dynamic_options = simparam->dynamic_options; + time_value = dynamic_options.time_value; + time_final = dynamic_options.time_final; + dt_max = dynamic_options.dt_max; + dt_min = dynamic_options.dt_min; + dt_cfl = dynamic_options.dt_cfl; + graphics_time = simparam->output_options.graphics_time; + graphics_dt_ival = simparam->output_options.graphics_dt_ival; + graphics_cyc_ival = simparam->output_options.graphics_cyc_ival; + cycle_stop = dynamic_options.cycle_stop; + rk_num_stages = dynamic_options.rk_num_stages; + dt = dynamic_options.dt; + fuzz = dynamic_options.fuzz; + tiny = dynamic_options.tiny; + small = dynamic_options.small; + graphics_times = simparam->output_options.graphics_times; + graphics_id = simparam->output_options.graphics_id; + rk_num_bins = simparam->dynamic_options.rk_num_bins; + + if (simparam->topology_optimization_on) + { + max_time_steps = BUFFER_GROW; + time_data.resize(max_time_steps + 1); + element_internal_energy_distributed = Teuchos::rcp(new MV(all_element_map, 1)); + forward_solve_velocity_data = Teuchos::rcp(new std::vector>(max_time_steps + 1)); + forward_solve_coordinate_data = Teuchos::rcp(new std::vector>(max_time_steps + 1)); + forward_solve_internal_energy_data = Teuchos::rcp(new std::vector>(max_time_steps + 1)); + adjoint_vector_data = Teuchos::rcp(new std::vector>(max_time_steps + 1)); + phi_adjoint_vector_data = Teuchos::rcp(new std::vector>(max_time_steps + 1)); + psi_adjoint_vector_data = Teuchos::rcp(new std::vector>(max_time_steps + 1)); + + // assign a multivector of corresponding size to each new timestep in the buffer + for (int istep = 0; istep < max_time_steps + 1; istep++) + { + (*forward_solve_velocity_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*forward_solve_coordinate_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*forward_solve_internal_energy_data)[istep] = Teuchos::rcp(new MV(all_element_map, 1)); + (*adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*phi_adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*psi_adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_element_map, 1)); + } + } + + have_loading_conditions = false; +} + +FEA_Module_SGH::~FEA_Module_SGH() +{ + // delete simparam; +} + +/* ---------------------------------------------------------------------- + Read ANSYS dat format mesh file +------------------------------------------------------------------------- */ +void FEA_Module_SGH::read_conditions_ansys_dat(std::ifstream* in, std::streampos before_condition_header) +{ + auto input_options = simparam->input_options.value(); + + char ch; + std::string skip_line, read_line, substring, token; + std::stringstream line_parse, line_parse2; + + int num_dim = simparam->num_dims; + int buffer_lines = 1000; + int max_word = 30; + int local_node_index, current_column_index; + int p_order = input_options.p_order; + int buffer_loop, buffer_iteration, buffer_iterations, scan_loop, nodes_per_element, words_per_line; + + size_t read_index_start, node_rid, elem_gid; + size_t strain_count; + + real_t unit_scaling = input_options.unit_scaling; + real_t dof_value; + + CArrayKokkos read_buffer; + CArrayKokkos read_buffer_indices; + + LO local_dof_id; + GO node_gid; + + host_vec_array node_densities; +} // end read_conditions_ansys_dat + +/* ---------------------------------------------------------------------- + Assign sets of element boundary surfaces corresponding to user BCs +------------------------------------------------------------------------- */ + +void FEA_Module_SGH::generate_bcs() +{ +} // end generate_bcs + +/* ---------------------------------------------------------------------- + Loop through applied boundary conditions and tag node ids to remove + necessary rows and columns from the assembled linear system +------------------------------------------------------------------------- */ + +void FEA_Module_SGH::Displacement_Boundary_Conditions() +{ +} + +/* ---------------------------------------------------------------------------- + Output field settings and file settings +------------------------------------------------------------------------------- */ + +void FEA_Module_SGH::output_control() +{ +} + +/* ---------------------------------------------------------------------------- + Initialize output data structures +------------------------------------------------------------------------------- */ + +void FEA_Module_SGH::init_output() +{ + // check user parameters for output + bool output_velocity_flag = simparam->output(FIELD::velocity); + bool output_strain_flag = simparam->output(FIELD::strain); + bool output_stress_flag = simparam->output(FIELD::stress); + int num_dim = simparam->num_dims; + int Brows; + if (num_dim == 3) + { + Brows = 6; + } + else + { + Brows = 3; + } + + // Implicit compliant code + if (output_velocity_flag) + { + // displacement_index is accessed by writers at the solver level for deformed output + output_velocity_index = noutput; + noutput += 1; + module_outputs.resize(noutput); + + vector_style.resize(noutput); + vector_style[noutput - 1] = DOF; + + output_vector_sizes.resize(noutput); + output_vector_sizes[noutput - 1] = num_dim; + + output_dof_names.resize(noutput); + output_dof_names[noutput - 1].resize(num_dim); + output_dof_names[noutput - 1][0] = "vx"; + output_dof_names[noutput - 1][1] = "vy"; + if (num_dim == 3) + { + output_dof_names[noutput - 1][2] = "vz"; + } + } + if (output_strain_flag) + { + output_strain_index = noutput; + noutput += 1; + module_outputs.resize(noutput); + + vector_style.resize(noutput); + vector_style[noutput - 1] = NODAL; + + output_vector_sizes.resize(noutput); + output_vector_sizes[noutput - 1] = Brows; + + output_dof_names.resize(noutput); + output_dof_names[noutput - 1].resize(Brows); + if (num_dim == 2) + { + output_dof_names[noutput - 1][0] = "strain_xx"; + output_dof_names[noutput - 1][1] = "strain_yy"; + output_dof_names[noutput - 1][2] = "strain_xy"; + } + if (num_dim == 3) + { + output_dof_names[noutput - 1][0] = "strain_xx"; + output_dof_names[noutput - 1][1] = "strain_yy"; + output_dof_names[noutput - 1][2] = "strain_zz"; + output_dof_names[noutput - 1][3] = "strain_xy"; + output_dof_names[noutput - 1][4] = "strain_xz"; + output_dof_names[noutput - 1][5] = "strain_yz"; + } + } + if (output_stress_flag) + { + output_stress_index = noutput; + noutput += 1; + module_outputs.resize(noutput); + + vector_style.resize(noutput); + vector_style[noutput - 1] = NODAL; + + output_vector_sizes.resize(noutput); + output_vector_sizes[noutput - 1] = Brows; + + output_dof_names.resize(noutput); + output_dof_names[noutput - 1].resize(Brows); + if (num_dim == 2) + { + output_dof_names[noutput - 1][0] = "stress_xx"; + output_dof_names[noutput - 1][1] = "stress_yy"; + output_dof_names[noutput - 1][3] = "stress_xy"; + } + if (num_dim == 3) + { + output_dof_names[noutput - 1][0] = "stress_xx"; + output_dof_names[noutput - 1][1] = "stress_yy"; + output_dof_names[noutput - 1][2] = "stress_zz"; + output_dof_names[noutput - 1][3] = "stress_xy"; + output_dof_names[noutput - 1][4] = "stress_xz"; + output_dof_names[noutput - 1][5] = "stress_yz"; + } + } +} + +/* ------------------------------------------------------------------------------------------- + Prompts sorting for elastic response output data. For now, nodal strains. +---------------------------------------------------------------------------------------------- */ + +void FEA_Module_SGH::sort_output(Teuchos::RCP> sorted_map) +{ +} + +/* ------------------------------------------------------------------------------------------- + populate requests this module makes for output data +---------------------------------------------------------------------------------------------- */ + +void FEA_Module_SGH::write_data(std::map& point_data_scalars_double, + std::map& point_data_vectors_double, + std::map& cell_data_scalars_double, + std::map& cell_data_scalars_int, + std::map>& cell_data_fields_double) +{ + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + + for (const auto& field_name : simparam->output_options.output_fields) + { + switch (field_name) + { + case FIELD::velocity: + // node "velocity" + node_vel.update_host(); + point_data_vectors_double["velocity"] = &node_vel.host(rk_level, 0, 0); + break; + + case FIELD::element_density: + // element "density" + elem_den.update_host(); + cell_data_scalars_double["element_density"] = elem_den.host_pointer(); + break; + + case FIELD::pressure: + // element "pressure" + elem_pres.update_host(); + cell_data_scalars_double["pressure"] = elem_pres.host_pointer(); + break; + + case FIELD::SIE: + // element "SIE" + elem_sie.update_host(); + cell_data_scalars_double["SIE"] = &elem_sie.host(rk_level, 0); + break; + + case FIELD::volume: + // element "volume" + elem_vol.update_host(); + cell_data_scalars_double["volume"] = elem_vol.host_pointer(); + break; + + case FIELD::mass: + // element "mass" + elem_mass.update_host(); + cell_data_scalars_double["mass"] = elem_mass.host_pointer(); + break; + + case FIELD::sound_speed: + // element "sspd" + elem_sspd.update_host(); + cell_data_scalars_double["sound_speed"] = elem_sspd.host_pointer(); + break; + + case FIELD::material_id: + // element "material_id" + elem_mat_id.update_host(); + cell_data_scalars_int["material_id"] = reinterpret_cast(elem_mat_id.host_pointer()); + break; + + case FIELD::user_vars: + // element "user_vars" + elem_user_output_vars.update_host(); + cell_data_fields_double["user_vars"] = std::make_pair(elem_user_output_vars.host_pointer(), + elem_user_output_vars.dims(1)); + case FIELD::stress: + // element "stress" + elem_stress.update_host(); + cell_data_fields_double["stress"] = std::make_pair(&elem_stress.host(rk_level, 0, 0, 0), 9); + break; + + default: + break; + } // end switch + } // end if + + // element "mat_id" //uncomment if needed (works fine) + // sgh_module->elem_mat_id.update_host(); + // cell_data_scalars_int["mat_id"] = reinterpret_cast(&sgh_module->elem_mat_id.host(0)); + + // element "user_output_vars" //uncomment if needed (works fine) + // sgh_module->elem_user_output_vars.update_host(); + // cell_data_fields_double["user_output_vars"] = std::make_pair(&sgh_module->elem_user_output_vars.host_pointer(), + // sgh_module->elem_user_output_vars.dims(1)); + + // element "stress" //uncomment if needed (works fine) + // sgh_module->elem_stress.update_host(); + // cell_data_fields_double["stress"] = std::make_pair(&sgh_module->elem_stress.host(rk_level,0,0,0), 9); +} + +/* ------------------------------------------------------------------------------------------- + Prompts sorting for elastic response output data. For now, nodal strains. +---------------------------------------------------------------------------------------------- */ + +void FEA_Module_SGH::sort_element_output(Teuchos::RCP> sorted_map) +{ + // interface element density data + { + host_vec_array Element_Densities = Global_Element_Densities->getLocalView(Tpetra::Access::ReadWrite); + elem_den.update_host(); + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + Element_Densities(ielem, 0) = elem_den.host(ielem); + } + } +} + +/* ------------------------------------------------------------------------------------------- + Prompts computation of elastic response output data. For now, nodal strains. +---------------------------------------------------------------------------------------------- */ + +void FEA_Module_SGH::collect_output(Teuchos::RCP> global_reduce_map) +{ +} + +/* ------------------------------------------------------------------------------------------- + Prompts computation of elastic response output data. For now, nodal strains. +---------------------------------------------------------------------------------------------- */ + +void FEA_Module_SGH::compute_output() +{ +} + +/* ---------------------------------------------------------------------- + Communicate updated nodal velocities to ghost nodes +------------------------------------------------------------------------- */ + +void FEA_Module_SGH::comm_node_masses() +{ + // debug print of design vector + // std::ostream &out = std::cout; + // Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + // if(myrank==0) + // *fos << "Density data :" << std::endl; + // node_densities_distributed->describe(*fos,Teuchos::VERB_EXTREME); + // *fos << std::endl; + // std::fflush(stdout); + + // communicate design densities + // create import object using local node indices map and all indices map + // Tpetra::Import importer(map, ghost_node_map); + + // comms to get ghosts + ghost_node_masses_distributed->doImport(*node_masses_distributed, *ghost_importer, Tpetra::INSERT); + // all_node_map->describe(*fos,Teuchos::VERB_EXTREME); + // all_node_velocities_distributed->describe(*fos,Teuchos::VERB_EXTREME); + + // update_count++; + // if(update_count==1){ + // MPI_Barrier(world); + // MPI_Abort(world,4); + // } +} + +/* ------------------------------------------------------------------------------------------- + Communicate ghosts using the current optimization design data +---------------------------------------------------------------------------------------------- */ + +void FEA_Module_SGH::comm_variables(Teuchos::RCP zp) +{ + if (simparam->topology_optimization_on) + { + // set density vector to the current value chosen by the optimizer + test_node_densities_distributed = zp; + + // debug print of design vector + // std::ostream &out = std::cout; + // Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + // if(myrank==0) + // *fos << "Density data :" << std::endl; + // node_densities_distributed->describe(*fos,Teuchos::VERB_EXTREME); + // *fos << std::endl; + // std::fflush(stdout); + + // communicate design densities + // create import object using local node indices map and all indices map + // Tpetra::Import importer(map, all_node_map); + + // comms to get ghosts + all_node_densities_distributed->doImport(*test_node_densities_distributed, *importer, Tpetra::INSERT); + } + else if (simparam->shape_optimization_on) + { + // clause to communicate boundary node data if the boundary nodes are ghosts on this rank + } +} + +/* ------------------------------------------------------------------------------------------- + enforce density constraints on nodes due to BCS +---------------------------------------------------------------------------------------------- */ +void FEA_Module_SGH::node_density_constraints(host_vec_array node_densities_lower_bound) +{ + const size_t num_dim = mesh->num_dims; + const_vec_array all_initial_node_coords = all_initial_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + const size_t num_lcs = module_params->loading_conditions.size(); + + const DCArrayKokkos mat_fill = simparam->mat_fill; + const DCArrayKokkos loading = module_params->loading; + + // debug check + // std::cout << "NUMBER OF LOADING CONDITIONS: " << num_lcs << std::endl; + + // walk over the nodes to update the velocity + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + double current_node_coords[3]; + double radius; + for (size_t dim = 0; dim < num_dim; dim++) + { + current_node_coords[dim] = all_initial_node_coords(node_gid, dim); + } // end for dim + radius = sqrt(current_node_coords[0] * current_node_coords[0] + + current_node_coords[1] * current_node_coords[1] + + current_node_coords[2] * current_node_coords[2]); + for (size_t ilc = 0; ilc < num_lcs; ilc++) + { + // debug check + // std::cout << "LOADING CONDITION VOLUME TYPE: " << to_string(loading(ilc).volume) << std::endl; + + bool fill_this = loading(ilc).volume.contains(current_node_coords); + if (fill_this) + { + node_densities_lower_bound(node_gid, 0) = 1; + } + } + }); // end for parallel for over nodes +} + +/* ---------------------------------------------------------------------------- + solve function called by solver +------------------------------------------------------------------------------- */ + +void FEA_Module_SGH::module_cleanup() +{ + cleanup_material_models(); +} + +/* ---------------------------------------------------------------------------- + Deallocate memory used for material models +------------------------------------------------------------------------------- */ + +void FEA_Module_SGH::cleanup_material_models() +{ + const DCArrayKokkos material = simparam->material; + + // destroy strength model + destroy_strength_model(elem_strength, + material, + elem_mat_id, + state_vars, + global_vars, + elem_user_output_vars, + rnum_elem); + + // destroy eos model + destroy_eos_model(elem_eos, + material, + elem_mat_id, + state_vars, + global_vars, + elem_user_output_vars, + rnum_elem); + return; +} // end cleanup_user_strength_model; + +/** + * Determines which of the boundary patches are associated with which boundary. + * + * Modifies: bdy_patches_in_set +*/ +void FEA_Module_SGH::tag_bdys(const DCArrayKokkos& boundary, + mesh_t& mesh, + const DViewCArrayKokkos& node_coords) +{ + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + size_t num_dim = simparam->num_dims; + int nboundary_patches = Explicit_Solver_Pointer_->nboundary_patches; + int num_nodes_in_patch = mesh.num_nodes_in_patch; + + // if (bdy_set == mesh.num_bdy_sets){ + // printf(" ERROR: number of boundary sets must be increased by %zu", + // bdy_set-mesh.num_bdy_sets+1); + // exit(0); + // } // end if + + // error and debug flag + // DCArrayKokkos print_flag(1, "print_flag"); + // print_flag.host(0) = false; + // print_flag.update_device(); + + FOR_ALL_CLASS(bdy_set, 0, num_bdy_sets, { + // tag boundaries + BOUNDARY_TYPE bc_type = boundary(bdy_set).surface.type; + double val = boundary(bdy_set).surface.plane_position; + + // save the boundary patches to this set that are on the plane, spheres, etc. + for (size_t bdy_patch_lid = 0; bdy_patch_lid < nboundary_patches; bdy_patch_lid++) + { + // save the patch index + size_t bdy_patch_gid = bdy_patch_lid; + + // check to see if this patch is on the specified plane + bool is_on_bdy = check_bdy(bdy_patch_gid, + num_dim, + num_nodes_in_patch, + bc_type, + val, + node_coords, + rk_level); // no=0, yes=1 + + // debug check + /* + for (size_t patch_node_lid=0; patch_node_lid& node_coords, + const size_t rk_level) const +{ + // default bool is not on the boundary + size_t is_on_bdy = 0; + + // the patch coordinates + double these_patch_coords[3]; // Note: cannot allocated array with num_dim + + // loop over the nodes on the patch + for (size_t patch_node_lid = 0; patch_node_lid < num_nodes_in_patch; patch_node_lid++) + { + // get the nodal_gid for this node in the patch + // size_t node_gid = mesh.nodes_in_patch(patch_gid, patch_node_lid); + size_t node_gid = Local_Index_Boundary_Patches(patch_gid, patch_node_lid); + + for (size_t dim = 0; dim < num_dim; dim++) + { + these_patch_coords[dim] = node_coords(rk_level, node_gid, dim); // (rk, node_gid, dim) + } + + if (bc_type == BOUNDARY_TYPE::x_plane) + { + if (fabs(these_patch_coords[0] - val) <= 1.0e-7) + { + is_on_bdy += 1; + } + } + else if (bc_type == BOUNDARY_TYPE::y_plane) + { + if (fabs(these_patch_coords[1] - val) <= 1.0e-7) + { + is_on_bdy += 1; + } + } + else if (bc_type == BOUNDARY_TYPE::z_plane) + { + if (fabs(these_patch_coords[2] - val) <= 1.0e-7) + { + is_on_bdy += 1; + } + } + else if (bc_type == BOUNDARY_TYPE::cylinder) + { + real_t R = sqrt(these_patch_coords[0] * these_patch_coords[0] + + these_patch_coords[1] * these_patch_coords[1]); + + if (fabs(R - val) <= 1.0e-7) + { + is_on_bdy += 1; + } + } + else if (bc_type == BOUNDARY_TYPE::sphere) + { + real_t R = sqrt(these_patch_coords[0] * these_patch_coords[0] + + these_patch_coords[1] * these_patch_coords[1] + + these_patch_coords[2] * these_patch_coords[2]); + + if (fabs(R - val) <= 1.0e-7) + { + is_on_bdy += 1; + } + } + } + + // if all nodes in the patch are on the surface + return is_on_bdy == num_nodes_in_patch; +} // end method to check bdy + +/* ---------------------------------------------------------------------------- + solve function called by solver +------------------------------------------------------------------------------- */ + +int FEA_Module_SGH::solve() +{ + sgh_solve(); + + return 0; +} + +/* ---------------------------------------------------------------------------- + SGH solver loop +------------------------------------------------------------------------------- */ + +void FEA_Module_SGH::sgh_solve() +{ + Dynamic_Options dynamic_options = simparam->dynamic_options; + + const size_t rk_level = dynamic_options.rk_num_bins - 1; + time_value = dynamic_options.time_initial; + time_final = dynamic_options.time_final; + dt_max = dynamic_options.dt_max; + dt_min = dynamic_options.dt_min; + dt_cfl = dynamic_options.dt_cfl; + graphics_time = simparam->output_options.graphics_step; + graphics_dt_ival = simparam->output_options.graphics_step; + cycle_stop = dynamic_options.cycle_stop; + rk_num_stages = dynamic_options.rk_num_stages; + dt = dynamic_options.dt; + fuzz = dynamic_options.fuzz; + tiny = dynamic_options.tiny; + small = dynamic_options.small; + graphics_times = simparam->output_options.graphics_times; + graphics_id = simparam->output_options.graphics_id; + size_t num_bdy_nodes = mesh->num_bdy_nodes; + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos material = simparam->material; + int nTO_modules; + int old_max_forward_buffer; + size_t cycle; + const int num_dim = simparam->num_dims; + real_t objective_accumulation, global_objective_accumulation; + std::vector> FEA_Module_My_TO_Modules = simparam->FEA_Module_My_TO_Modules; + problem = Explicit_Solver_Pointer_->problem; // Pointer to ROL optimization problem object + ROL::Ptr> obj_pointer; + + // reset time accumulating objective and constraints + /* + for(int imodule = 0 ; imodule < FEA_Module_My_TO_Modules[my_fea_module_index_].size(); imodule++){ + current_module_index = FEA_Module_My_TO_Modules[my_fea_module_index_][imodule]; + //test if module needs reset + if(){ + + } + } + */ + // simple setup to just request KE for now; above loop to be expanded and used later for scanning modules + if (simparam->topology_optimization_on) + { + obj_pointer = problem->getObjective(); + KineticEnergyMinimize_TopOpt& kinetic_energy_minimize_function = dynamic_cast(*obj_pointer); + kinetic_energy_minimize_function.objective_accumulation = 0; + global_objective_accumulation = objective_accumulation = 0; + kinetic_energy_objective = true; + if (max_time_steps + 1 > forward_solve_velocity_data->size()) + { + old_max_forward_buffer = forward_solve_velocity_data->size(); + time_data.resize(max_time_steps + 1); + forward_solve_velocity_data->resize(max_time_steps + 1); + forward_solve_coordinate_data->resize(max_time_steps + 1); + forward_solve_internal_energy_data->resize(max_time_steps + 1); + adjoint_vector_data->resize(max_time_steps + 1); + phi_adjoint_vector_data->resize(max_time_steps + 1); + psi_adjoint_vector_data->resize(max_time_steps + 1); + // assign a multivector of corresponding size to each new timestep in the buffer + for (int istep = old_max_forward_buffer; istep < max_time_steps + 1; istep++) + { + (*forward_solve_velocity_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*forward_solve_coordinate_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*forward_solve_internal_energy_data)[istep] = Teuchos::rcp(new MV(all_element_map, 1)); + (*adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*phi_adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*psi_adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_element_map, 1)); + } + } + } + + if (simparam->topology_optimization_on) + { + nTO_modules = simparam->TO_Module_List.size(); + } + + int myrank = Explicit_Solver_Pointer_->myrank; + if (simparam->output_options.output_file_format == OUTPUT_FORMAT::vtk && simparam->output_options.write_initial) + { + if (myrank == 0) + { + printf("Writing outputs to file at %f \n", time_value); + } + + double comm_time1 = Explicit_Solver_Pointer_->CPU_Time(); + Explicit_Solver_Pointer_->write_outputs(); + double comm_time2 = Explicit_Solver_Pointer_->CPU_Time(); + Explicit_Solver_Pointer_->output_time += comm_time2 - comm_time1; + } + + CArrayKokkos node_extensive_mass(nall_nodes, "node_extensive_mass"); + + // extensive energy tallies over the mesh elements local to this MPI rank + double IE_t0 = 0.0; + double KE_t0 = 0.0; + double TE_t0 = 0.0; + + double IE_sum = 0.0; + double KE_sum = 0.0; + + double IE_loc_sum = 0.0; + double KE_loc_sum = 0.0; + + // extensive energy tallies over the entire mesh + double global_IE_t0 = 0.0; + double global_KE_t0 = 0.0; + double global_TE_t0 = 0.0; + + // ---- Calculate energy tallies ---- + double IE_tend = 0.0; + double KE_tend = 0.0; + double TE_tend = 0.0; + + double global_IE_tend = 0.0; + double global_KE_tend = 0.0; + double global_TE_tend = 0.0; + + int nlocal_elem_non_overlapping = Explicit_Solver_Pointer_->nlocal_elem_non_overlapping; + + // extensive IE + REDUCE_SUM_CLASS(elem_gid, 0, nlocal_elem_non_overlapping, IE_loc_sum, { + IE_loc_sum += elem_mass(elem_gid) * elem_sie(rk_level, elem_gid); + }, IE_sum); + IE_t0 = IE_sum; + + MPI_Allreduce(&IE_t0, &global_IE_t0, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); + + // extensive KE + REDUCE_SUM_CLASS(node_gid, 0, nlocal_nodes, KE_loc_sum, { + double ke = 0; + for (size_t dim = 0; dim < num_dim; dim++) + { + ke += node_vel(rk_level, node_gid, dim) * node_vel(rk_level, node_gid, dim); // 1/2 at end + } // end for + + if (num_dim == 2) + { + KE_loc_sum += node_mass(node_gid) * node_coords(rk_level, node_gid, 1) * ke; + } + else + { + KE_loc_sum += node_mass(node_gid) * ke; + } + }, KE_sum); + Kokkos::fence(); + KE_t0 = 0.5 * KE_sum; + + MPI_Allreduce(&KE_t0, &global_KE_t0, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); + + // extensive TE + global_TE_t0 = global_IE_t0 + global_KE_t0; + TE_t0 = global_TE_t0; + KE_t0 = global_KE_t0; + IE_t0 = global_IE_t0; + + // save the nodal mass + FOR_ALL_CLASS(node_gid, 0, nall_nodes, { + double radius = 1.0; + if (num_dim == 2) + { + radius = node_coords(rk_level, node_gid, 1); + } + node_extensive_mass(node_gid) = node_mass(node_gid) * radius; + }); // end parallel for + + // a flag to exit the calculation + size_t stop_calc = 0; + + auto time_1 = std::chrono::high_resolution_clock::now(); + + // save initial data + if (simparam->topology_optimization_on || simparam->shape_optimization_on) + { + time_data[0] = 0; + // assign current velocity data to multivector + // view scope + { + vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); + vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + node_velocities_interface(node_gid, idim) = node_vel(rk_level, node_gid, idim); + node_coords_interface(node_gid, idim) = node_coords(rk_level, node_gid, idim); + } + }); + } // end view scope + Kokkos::fence(); + + // communicate ghosts + double comm_time1 = Explicit_Solver_Pointer_->CPU_Time(); + + // active view scope; triggers host comms from updated data on device + { + const_host_vec_array node_velocities_host = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_host_vec_array node_coords_host = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + } + double comm_time2 = Explicit_Solver_Pointer_->CPU_Time(); + Explicit_Solver_Pointer_->dev2host_time += comm_time2 - comm_time1; + + // communicate ghost velocities + Explicit_Solver_Pointer_->comm_velocities(); + Explicit_Solver_Pointer_->comm_coordinates(); + + double comm_time3 = Explicit_Solver_Pointer_->CPU_Time(); + + // view scope + { + const_vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array ghost_node_velocities_interface = Explicit_Solver_Pointer_->ghost_node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); + vec_array all_node_velocities_interface = Explicit_Solver_Pointer_->all_node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); + vec_array element_internal_energy = element_internal_energy_distributed->getLocalView(Tpetra::Access::ReadWrite); + const_vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array ghost_node_coords_interface = Explicit_Solver_Pointer_->ghost_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + all_node_velocities_interface(node_gid, idim) = node_velocities_interface(node_gid, idim); + all_node_coords_interface(node_gid, idim) = node_coords_interface(node_gid, idim); + } + }); // end parallel for + Kokkos::fence(); + + FOR_ALL_CLASS(node_gid, nlocal_nodes, nlocal_nodes + nghost_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + all_node_velocities_interface(node_gid, idim) = ghost_node_velocities_interface(node_gid - nlocal_nodes, idim); + all_node_coords_interface(node_gid, idim) = ghost_node_coords_interface(node_gid - nlocal_nodes, idim); + } + }); // end parallel for + Kokkos::fence(); + + // interface for element internal energies + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { + element_internal_energy(elem_gid, 0) = elem_sie(rk_level, elem_gid); + }); // end parallel for + Kokkos::fence(); + } // end view scope + + (*forward_solve_internal_energy_data)[0]->assign(*element_internal_energy_distributed); + (*forward_solve_velocity_data)[0]->assign(*Explicit_Solver_Pointer_->all_node_velocities_distributed); + (*forward_solve_coordinate_data)[0]->assign(*Explicit_Solver_Pointer_->all_node_coords_distributed); + } + + // loop over the max number of time integration cycles + for (cycle = 0; cycle < cycle_stop; cycle++) + { + // get the step + if (num_dim == 2) + { + get_timestep2D(*mesh, + node_coords, + node_vel, + elem_sspd, + elem_vol); + } + else + { + get_timestep(*mesh, + node_coords, + node_vel, + elem_sspd, + elem_vol); + } // end if 2D + + double global_dt; + MPI_Allreduce(&dt, &global_dt, 1, MPI_DOUBLE, MPI_MIN, MPI_COMM_WORLD); + dt = global_dt; + + // stop calculation if flag + // if (stop_calc == 1) break; + + if (simparam->dynamic_options.output_time_sequence_level >= TIME_OUTPUT_LEVEL::high) + { + if (cycle == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %12.5e, time step = %12.5e \n", cycle, time_value, dt); + } + } + // print time step every 10 cycles + else if (cycle % 20 == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %12.5e, time step = %12.5e \n", cycle, time_value, dt); + } + } // end if + } + + // --------------------------------------------------------------------- + // integrate the solution forward to t(n+1) via Runge Kutta (RK) method + // --------------------------------------------------------------------- + + // save the values at t_n + rk_init(node_coords, + node_vel, + elem_sie, + elem_stress, + rnum_elem, + nall_nodes); + + // integrate solution forward in time + for (size_t rk_stage = 0; rk_stage < rk_num_stages; rk_stage++) + { + // ---- RK coefficient ---- + double rk_alpha = 1.0 / ((double)rk_num_stages - (double)rk_stage); + + // ---- Calculate velocity diveregence for the element ---- + if (num_dim == 2) + { + get_divergence2D(elem_div, + node_coords, + node_vel, + elem_vol); + } + else + { + get_divergence(elem_div, + node_coords, + node_vel, + elem_vol); + } // end if 2D + + // ---- calculate the forces on the vertices and evolve stress (hypo model) ---- + if (num_dim == 2) + { + get_force_sgh2D(material, + *mesh, + node_coords, + node_vel, + elem_den, + elem_sie, + elem_pres, + elem_stress, + elem_sspd, + elem_vol, + elem_div, + elem_mat_id, + corner_force, + rk_alpha, + cycle); + } + else + { + get_force_sgh(material, + *mesh, + node_coords, + node_vel, + elem_den, + elem_sie, + elem_pres, + elem_stress, + elem_sspd, + elem_vol, + elem_div, + elem_mat_id, + corner_force, + rk_alpha, + cycle); + } + + /* + debug block + if(myrank==1){ + std::cout << rk_alpha << " " << dt << std::endl; + for(int i = 0; i < nall_nodes; i++){ + double node_force[3]; + for (size_t dim = 0; dim < num_dim; dim++){ + node_force[dim] = 0.0; + } // end for dim + + // loop over all corners around the node and calculate the nodal force + for (size_t corner_lid=0; corner_lidall_node_map->getGlobalElement(i) << " " << corner_gid << " " << corner_force(corner_gid, 0) << " " << corner_force(corner_gid, 1) << " " << corner_force(corner_gid, 2) << std::endl; + // loop over dimension + for (size_t dim = 0; dim < num_dim; dim++){ + node_force[dim] += corner_force(corner_gid, dim); + } // end for dim + + } // end for corner_lid + //std::cout << Explicit_Solver_Pointer_->all_node_map->getGlobalElement(i) << " " << node_force[0] << " " << node_force[1] << " " << node_force[2] << std::endl; + //std::cout << Explicit_Solver_Pointer_->all_node_map->getGlobalElement(i) << " " << node_mass(i) << std::endl; + } + } + /* + //debug print vector values on a rank + /* + if(myrank==0) + for(int i = 0; i < nall_nodes; i++){ + std::cout << Explicit_Solver_Pointer_->all_node_map->getGlobalElement(i) << " " << node_vel(rk_level,i,0) << " " << node_vel(rk_level,i,1) << " " << node_vel(rk_level,i,2) << std::endl; + } + */ + + // ---- Update nodal velocities ---- // + update_velocity_sgh(rk_alpha, + node_vel, + node_mass, + corner_force); + + if (have_loading_conditions) + { + applied_forces(material, + *mesh, + node_coords, + node_vel, + node_mass, + elem_den, + elem_vol, + elem_div, + elem_mat_id, + corner_force, + rk_alpha, + cycle); + } + + // ---- apply force boundary conditions to the boundary patches---- + boundary_velocity(*mesh, boundary, node_vel); + + // current interface has differing velocity arrays; this equates them until we unify memory + // first comm time interval point + double comm_time1 = Explicit_Solver_Pointer_->CPU_Time(); + // view scope + { + vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + node_velocities_interface(node_gid, idim) = node_vel(rk_level, node_gid, idim); + } + }); // end parallel for + } // end view scope + Kokkos::fence(); + + // active view scope + { + const_host_vec_array node_velocities_host = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); + } + double comm_time2 = Explicit_Solver_Pointer_->CPU_Time(); + Explicit_Solver_Pointer_->dev2host_time += comm_time2 - comm_time1; + // communicate ghost velocities + Explicit_Solver_Pointer_->comm_velocities(); + + double comm_time3 = Explicit_Solver_Pointer_->CPU_Time(); + // this is forcing a copy to the device + // view scope + { + vec_array ghost_node_velocities_interface = Explicit_Solver_Pointer_->ghost_node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_gid, nlocal_nodes, nall_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + node_vel(rk_level, node_gid, idim) = ghost_node_velocities_interface(node_gid - nlocal_nodes, idim); + } + }); // end parallel for + } // end view scope + Kokkos::fence(); + + double comm_time4 = Explicit_Solver_Pointer_->CPU_Time(); + Explicit_Solver_Pointer_->host2dev_time += comm_time4 - comm_time3; + Explicit_Solver_Pointer_->communication_time += comm_time4 - comm_time1; + // debug print vector values on a rank + /* + if(myrank==0) + for(int i = 0; i < nall_nodes; i++){ + std::cout << Explicit_Solver_Pointer_->all_node_map->getGlobalElement(i) << " " << node_vel(rk_level,i,0) << " " << node_vel(rk_level,i,1) << " " << node_vel(rk_level,i,2) << std::endl; + } + */ + // ---- Update specific internal energy in the elements ---- + update_energy_sgh(rk_alpha, + *mesh, + node_vel, + node_coords, + elem_sie, + elem_mass, + corner_force); + + // ---- Update nodal positions ---- + update_position_sgh(rk_alpha, + nall_nodes, + node_coords, + node_vel); + + // ---- Calculate cell volume for next time step ---- + get_vol(); + + // ---- Calculate elem state (den, pres, sound speed, stress) for next time step ---- + if (num_dim == 2) + { + update_state2D(material, + *mesh, + node_coords, + node_vel, + elem_den, + elem_pres, + elem_stress, + elem_sspd, + elem_sie, + elem_vol, + elem_mass, + elem_mat_id, + rk_alpha, + cycle); + } + else + { + update_state(material, + *mesh, + node_coords, + node_vel, + elem_den, + elem_pres, + elem_stress, + elem_sspd, + elem_sie, + elem_vol, + elem_mass, + elem_mat_id, + rk_alpha, + cycle); + } + // ---- + // Notes on strength: + // 1) hyper-elastic strength models are called in update_state + // 2) hypo-elastic strength models are called in get_force + // 3) strength models must be added by the user in user_mat.cpp + + // calculate the new corner masses if 2D + if (num_dim == 2) + { + // calculate the nodal areal mass + FOR_ALL_CLASS(node_gid, 0, nall_nodes, { + node_mass(node_gid) = 0.0; + + if (node_coords(rk_level, node_gid, 1) > tiny) + { + node_mass(node_gid) = node_extensive_mass(node_gid) / node_coords(rk_level, node_gid, 1); + } + // if(cycle==0&&node_gid==1&&myrank==0) + // std::cout << "index " << node_gid << " on rank " << myrank << " node vel " << node_vel(rk_level,node_gid,0) << " " << node_mass(node_gid) << std::endl << std::flush; + }); // end parallel for over node_gid + Kokkos::fence(); + + // current interface has differing density arrays; this equates them until we unify memory + // view scope + { + vec_array node_mass_interface = node_masses_distributed->getLocalView(Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + node_mass_interface(node_gid, 0) = node_mass(node_gid); + }); // end parallel for + } // end view scope + Kokkos::fence(); + // communicate ghost densities + comm_node_masses(); + + // this is forcing a copy to the device + // view scope + { + vec_array ghost_node_mass_interface = ghost_node_masses_distributed->getLocalView(Tpetra::Access::ReadWrite); + + FOR_ALL_CLASS(node_gid, nlocal_nodes, nall_nodes, { + node_mass(node_gid) = ghost_node_mass_interface(node_gid - nlocal_nodes, 0); + }); // end parallel for + } // end view scope + Kokkos::fence(); + + // ----------------------------------------------- + // Calcualte the areal mass for nodes on the axis + // ----------------------------------------------- + // The node order of the 2D element is + // + // J + // | + // 3---2 + // | | -- I + // 0---1 + /* + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { + + // loop over the corners of the element and calculate the mass + for (size_t node_lid=0; node_lid<4; node_lid++){ + + size_t node_gid = nodes_in_elem(elem_gid, node_lid); + size_t node_minus_gid; + size_t node_plus_gid; + + + if (node_coords(rk_level,node_gid,1) < tiny){ + // node is on the axis + + // minus node + if (node_lid==0){ + node_minus_gid = nodes_in_elem(elem_gid, 3); + } else { + node_minus_gid = nodes_in_elem(elem_gid, node_lid-1); + } + + // plus node + if (node_lid==3){ + node_plus_gid = nodes_in_elem(elem_gid, 0); + } else { + node_plus_gid = nodes_in_elem(elem_gid, node_lid+1); + } + + node_mass(node_gid) = fmax(node_mass(node_plus_gid), node_mass(node_minus_gid))/2.0; + + } // end if + + } // end for over corners + + }); // end parallel for over elem_gid + Kokkos::fence(); + */ + + FOR_ALL_CLASS(node_bdy_gid, 0, num_bdy_nodes, { + // FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + size_t node_gid = bdy_nodes(node_bdy_gid); + + if (node_coords(rk_level, node_gid, 1) < tiny) + { + // node is on the axis + + for (size_t node_lid = 0; node_lid < num_nodes_in_node(node_gid); node_lid++) + { + size_t node_neighbor_gid = nodes_in_node(node_gid, node_lid); + + // if the node is off the axis, use it's areal mass on the boundary + if (node_coords(rk_level, node_neighbor_gid, 1) > tiny) + { + node_mass(node_gid) = fmax(node_mass(node_gid), node_mass(node_neighbor_gid) / 2.0); + } + } // end for over neighboring nodes + } // end if + }); // end parallel for over elem_gid + } // end of if 2D-RZ + } // end of RK loop + + // increment the time + Explicit_Solver_Pointer_->time_value = simparam->dynamic_options.time_value = time_value += dt; + + if (simparam->topology_optimization_on || simparam->shape_optimization_on) + { + if (cycle >= max_time_steps) + { + max_time_steps = cycle + 1; + } + + if (max_time_steps + 1 > forward_solve_velocity_data->size()) + { + old_max_forward_buffer = forward_solve_velocity_data->size(); + time_data.resize(max_time_steps + BUFFER_GROW + 1); + forward_solve_velocity_data->resize(max_time_steps + BUFFER_GROW + 1); + forward_solve_coordinate_data->resize(max_time_steps + BUFFER_GROW + 1); + forward_solve_internal_energy_data->resize(max_time_steps + BUFFER_GROW + 1); + adjoint_vector_data->resize(max_time_steps + BUFFER_GROW + 1); + phi_adjoint_vector_data->resize(max_time_steps + BUFFER_GROW + 1); + psi_adjoint_vector_data->resize(max_time_steps + BUFFER_GROW + 1); + // assign a multivector of corresponding size to each new timestep in the buffer + for (int istep = old_max_forward_buffer; istep < max_time_steps + BUFFER_GROW + 1; istep++) + { + (*forward_solve_velocity_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*forward_solve_coordinate_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*forward_solve_internal_energy_data)[istep] = Teuchos::rcp(new MV(all_element_map, 1)); + (*adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*phi_adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*psi_adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_element_map, 1)); + } + } + + time_data[cycle + 1] = dt + time_data[cycle]; + + // assign current velocity data to multivector + // view scope + { + vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); + vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + node_velocities_interface(node_gid, idim) = node_vel(rk_level, node_gid, idim); + node_coords_interface(node_gid, idim) = node_coords(rk_level, node_gid, idim); + } + }); + } // end view scope + Kokkos::fence(); + + // communicate ghosts + double comm_time1 = Explicit_Solver_Pointer_->CPU_Time(); + + // active view scope; triggers host comms from updated data on device + { + const_host_vec_array node_velocities_host = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_host_vec_array node_coords_host = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + } + double comm_time2 = Explicit_Solver_Pointer_->CPU_Time(); + Explicit_Solver_Pointer_->dev2host_time += comm_time2 - comm_time1; + + // communicate ghost velocities + Explicit_Solver_Pointer_->comm_velocities(); + Explicit_Solver_Pointer_->comm_coordinates(); + + double comm_time3 = Explicit_Solver_Pointer_->CPU_Time(); + + // view scope + { + const_vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array ghost_node_velocities_interface = Explicit_Solver_Pointer_->ghost_node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); + vec_array all_node_velocities_interface = Explicit_Solver_Pointer_->all_node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); + vec_array element_internal_energy = element_internal_energy_distributed->getLocalView(Tpetra::Access::ReadWrite); + const_vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array ghost_node_coords_interface = Explicit_Solver_Pointer_->ghost_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + all_node_velocities_interface(node_gid, idim) = node_velocities_interface(node_gid, idim); + all_node_coords_interface(node_gid, idim) = node_coords_interface(node_gid, idim); + } + }); // end parallel for + Kokkos::fence(); + + FOR_ALL_CLASS(node_gid, nlocal_nodes, nlocal_nodes + nghost_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + all_node_velocities_interface(node_gid, idim) = ghost_node_velocities_interface(node_gid - nlocal_nodes, idim); + all_node_coords_interface(node_gid, idim) = ghost_node_coords_interface(node_gid - nlocal_nodes, idim); + } + }); // end parallel for + Kokkos::fence(); + + // interface for element internal energies + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { + element_internal_energy(elem_gid, 0) = elem_sie(rk_level, elem_gid); + }); // end parallel for + Kokkos::fence(); + } // end view scope + + double comm_time4 = Explicit_Solver_Pointer_->CPU_Time(); + Explicit_Solver_Pointer_->host2dev_time += comm_time4 - comm_time3; + Explicit_Solver_Pointer_->communication_time += comm_time4 - comm_time1; + + (*forward_solve_internal_energy_data)[cycle + 1]->assign(*element_internal_energy_distributed); + (*forward_solve_velocity_data)[cycle + 1]->assign(*Explicit_Solver_Pointer_->all_node_velocities_distributed); + (*forward_solve_coordinate_data)[cycle + 1]->assign(*Explicit_Solver_Pointer_->all_node_coords_distributed); + + // kinetic energy accumulation + if (kinetic_energy_objective) + { + const_vec_array node_velocities_interface = (*forward_solve_velocity_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array previous_node_velocities_interface = (*forward_solve_velocity_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + KE_loc_sum = 0.0; + KE_sum = 0.0; + // extensive KE + REDUCE_SUM_CLASS(node_gid, 0, nlocal_nodes, KE_loc_sum, { + double ke = 0; + for (size_t dim = 0; dim < num_dim; dim++) + { + // midpoint integration approximation + ke += (node_velocities_interface(node_gid, dim) + previous_node_velocities_interface(node_gid, dim)) * (node_velocities_interface(node_gid, + dim) + previous_node_velocities_interface(node_gid, dim)) / 4; // 1/2 at end + } // end for + + if (num_dim == 2) + { + KE_loc_sum += node_mass(node_gid) * node_coords(rk_level, node_gid, 1) * ke; + } + else + { + KE_loc_sum += node_mass(node_gid) * ke; + } + }, KE_sum); + Kokkos::fence(); + KE_sum = 0.5 * KE_sum; + objective_accumulation += KE_sum * dt; + } + } + + size_t write = 0; + if ((cycle + 1) % graphics_cyc_ival == 0 && cycle > 0) + { + write = 1; + } + else if (cycle == cycle_stop) + { + write = 1; + } + else if (time_value >= time_final && simparam->output_options.write_final) + { + write = 1; + } + else if (time_value >= graphics_time) + { + write = 1; + } + + // write outputs + if (write == 1) + { + // interface nodal coordinate data (note: this is not needed if using write_outputs()) + // view scope + { + vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + node_coords_interface(node_gid, idim) = node_coords(rk_level, node_gid, idim); + } + }); // end parallel for + } // end view scope + if (simparam->output_options.output_file_format == OUTPUT_FORMAT::vtk) + { + if (myrank == 0) + { + printf("Writing outputs to file at %f \n", graphics_time); + } + + double comm_time1 = Explicit_Solver_Pointer_->CPU_Time(); + Explicit_Solver_Pointer_->write_outputs(); + + double comm_time2 = Explicit_Solver_Pointer_->CPU_Time(); + Explicit_Solver_Pointer_->output_time += comm_time2 - comm_time1; + } + + graphics_time = time_value + graphics_dt_ival; + } // end if + + // end of calculation + if (time_value >= time_final) + { + break; + } + } // end for cycle loop + + last_time_step = cycle; + + // simple setup to just calculate KE minimize objective for now + if (simparam->topology_optimization_on) + { + KineticEnergyMinimize_TopOpt& kinetic_energy_minimize_function = dynamic_cast(*obj_pointer); + + // collect local objective values + MPI_Allreduce(&objective_accumulation, &global_objective_accumulation, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); + kinetic_energy_minimize_function.objective_accumulation = global_objective_accumulation; + + if (myrank == 0) + { + std::cout << "CURRENT TIME INTEGRAL OF KINETIC ENERGY " << global_objective_accumulation << std::endl; + } + } + + auto time_2 = std::chrono::high_resolution_clock::now(); + auto time_difference = time_2 - time_1; + // double calc_time = std::chrono::duration_cast(diff).count(); + double calc_time = std::chrono::duration_cast(time_difference).count(); + if (myrank == 0) + { + printf("\nCalculation time in seconds: %f \n", calc_time * 1e-09); + } + + IE_loc_sum = 0.0; + KE_loc_sum = 0.0; + IE_sum = 0.0; + KE_sum = 0.0; + + // extensive IE + REDUCE_SUM_CLASS(elem_gid, 0, nlocal_elem_non_overlapping, IE_loc_sum, { + IE_loc_sum += elem_mass(elem_gid) * elem_sie(rk_level, elem_gid); + }, IE_sum); + IE_tend = IE_sum; + + // reduce over MPI ranks + MPI_Allreduce(&IE_tend, &global_IE_tend, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); + + // extensive KE + REDUCE_SUM_CLASS(node_gid, 0, nlocal_nodes, KE_loc_sum, { + double ke = 0; + for (size_t dim = 0; dim < num_dim; dim++) + { + ke += node_vel(rk_level, node_gid, dim) * node_vel(rk_level, node_gid, dim); // 1/2 at end + } // end for + + if (num_dim == 2) + { + KE_loc_sum += node_mass(node_gid) * node_coords(rk_level, node_gid, 1) * ke; + } + else + { + KE_loc_sum += node_mass(node_gid) * ke; + } + }, KE_sum); + Kokkos::fence(); + KE_tend = 0.5 * KE_sum; + + // reduce over MPI ranks + MPI_Allreduce(&KE_tend, &global_KE_tend, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); + + // extensive TE + TE_tend = IE_tend + KE_tend; + KE_tend = global_KE_tend; + IE_tend = global_IE_tend; + + // extensive TE + TE_tend = IE_tend + KE_tend; + + // reduce over MPI ranks + + if (myrank == 0) + { + printf("Time=0: KE = %20.15f, IE = %20.15f, TE = %20.15f \n", KE_t0, IE_t0, TE_t0); + } + if (myrank == 0) + { + printf("Time=End: KE = %20.15f, IE = %20.15f, TE = %20.15f \n", KE_tend, IE_tend, TE_tend); + } + if (myrank == 0) + { + printf("total energy conservation error = %e \n\n", 100 * (TE_tend - TE_t0) / TE_t0); + } + + return; +} // end of SGH solve diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/boundary.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/boundary.cpp similarity index 50% rename from src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/boundary.cpp rename to src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/boundary.cpp index c94a8a961..5996ca246 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/boundary.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/boundary.cpp @@ -1,77 +1,67 @@ -// ------------------------------------------------------- -// This function applys the boundary condition -// to points on a list of patches created at setup -//-------------------------------------------------------- - #include "mesh.h" #include "state.h" #include "FEA_Module_SGH.h" #include "Simulation_Parameters/FEA_Module/Boundary_Conditions.h" -void FEA_Module_SGH::boundary_velocity(const mesh_t &mesh, - const DCArrayKokkos &boundary, - DViewCArrayKokkos &node_vel){ +void FEA_Module_SGH::boundary_velocity(const mesh_t& mesh, + const DCArrayKokkos& boundary, + DViewCArrayKokkos& node_vel) +{ + // error and debug flag + // DCArrayKokkos print_flag(1, "print_flag"); + // print_flag.host(0) = false; + // print_flag.update_device(); - //error and debug flag - //DCArrayKokkos print_flag(1, "print_flag"); - //print_flag.host(0) = false; - //print_flag.update_device(); - - const size_t rk_level = rk_num_bins - 1; - int num_dims = num_dim; + const size_t rk_level = rk_num_bins - 1; + int num_dims = num_dim; // Loop over boundary sets - for (size_t bdy_set=0; bdy_set& node_vel, + const DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& corner_force) +{ + const size_t rk_level = rk_num_bins - 1; + int num_dims = num_dim; + + // loop over all the elements in the mesh + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { + double elem_power = 0.0; + + // --- tally the contribution from each corner to the element --- + + // Loop over the nodes in the element + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + size_t corner_lid = node_lid; + + // Get node global id for the local node id + size_t node_gid = nodes_in_elem(elem_gid, node_lid); + + // Get the corner global id for the local corner id + size_t corner_gid = corners_in_elem(elem_gid, corner_lid); + + double node_radius = 1; + if (num_dims == 2) + { + node_radius = node_coords(rk_level, node_gid, 1); + } + + // calculate the Power=F dot V for this corner + for (size_t dim = 0; dim < num_dims; dim++) + { + double half_vel = (node_vel(rk_level, node_gid, dim) + node_vel(0, node_gid, dim)) * 0.5; + elem_power += corner_force(corner_gid, dim) * node_radius * half_vel; + } // end for dim + } // end for node_lid + + // update the specific energy + elem_sie(rk_level, elem_gid) = elem_sie(0, elem_gid) - + rk_alpha * dt / elem_mass(elem_gid) * elem_power; + }); // end parallel loop over the elements + + return; +} // end subroutine diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp new file mode 100644 index 000000000..9ef23e8c1 --- /dev/null +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp @@ -0,0 +1,2063 @@ + +#include "mesh.h" +#include "state.h" +#include +#include +#include +#include +#include +#include // fmin, fmax, abs note: fminl is long +#include +#include + +#include +#include +#include + +#include "elements.h" +#include "swage.h" +#include "matar.h" +#include "utilities.h" +#include "FEA_Module_SGH.h" +#include "Simulation_Parameters/Simulation_Parameters_Explicit.h" +#include "Simulation_Parameters/FEA_Module/SGH_Parameters.h" + +// ----------------------------------------------------------------------------- +// This function calculates the corner force gradients w.r.t velocity; the +// current implementation assumes material.q2 = 0 +// ------------------------------------------------------------------------------ +void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_pres, + const DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + const double rk_alpha, + const size_t cycle + ) +{ + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + const size_t num_dims = simparam->num_dims; + size_t num_corners = rnum_elem * num_nodes_in_elem; + + // initialize gradient matrix + FOR_ALL_CLASS(dof_gid, 0, nlocal_nodes * num_dims, { + for (int idof = 0; idof < Gradient_Matrix_Strides(dof_gid); idof++) + { + Force_Gradient_Velocities(dof_gid, idof) = 0; + } + }); // end parallel for loop over nodes + Kokkos::fence(); + + // initialize buffer storage; not all components are explicitly set for this routine + FOR_ALL_CLASS(corner_gid, 0, num_corners, { + for (int dim = 0; dim < num_dims; dim++) + { + // assign gradient of corner contribution of force to relevant matrix entries with non-zero node velocity gradient + for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) + { + for (int jdim = 0; jdim < num_dims; jdim++) + { + corner_gradient_storage(corner_gid, dim, igradient, jdim) = 0; + } + } + } + }); // end parallel for loop over nodes + Kokkos::fence(); + + // --- calculate the forces acting on the nodes from the element --- + for (size_t elem_gid = 0; elem_gid < rnum_elem; elem_gid++) + { + // FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { + + const size_t num_nodes_in_elem = 8; + real_t gradient_result[num_dims]; + /// total Cauchy stress + double tau_array[9]; + double tau_gradient_array[9]; + + /// corner area normals + double area_normal_array[24]; + + /// estimate of shock direction + double shock_dir_array[3]; + + /// the sums in the Riemann solver + double sum_array[4]; + + /// corner shock impeadance x |corner area normal dot shock_dir| + double muc_array[8]; + double muc_gradient_array[8]; + + /// Riemann velocity + double vel_star_array[3]; + double vel_star_gradient_array[3 * num_nodes_in_elem]; + + /// velocity gradient + double vel_grad_array[9]; + + // --- Create views of arrays to aid the force calculation --- + + ViewCArrayKokkos tau(tau_array, num_dims, num_dims); + ViewCArrayKokkos tau_gradient(tau_gradient_array, num_dims, num_dims); + ViewCArrayKokkos area_normal(area_normal_array, num_nodes_in_elem, num_dims); + ViewCArrayKokkos shock_dir(shock_dir_array, num_dims); + ViewCArrayKokkos sum(sum_array, 4); + ViewCArrayKokkos muc(muc_array, num_nodes_in_elem); + ViewCArrayKokkos muc_gradient(muc_gradient_array, num_nodes_in_elem); + ViewCArrayKokkos vel_star(vel_star_array, num_dims); + ViewCArrayKokkos vel_star_gradient(vel_star_gradient_array, num_nodes_in_elem, num_dims); + ViewCArrayKokkos vel_grad(vel_grad_array, num_dims, num_dims); + + EOSParent* eos_model = elem_eos(elem_gid).model; + + // --- abviatations of variables --- + + // element volume + double vol = elem_vol(elem_gid); + + // create a view of the stress_matrix + ViewCArrayKokkos stress(&elem_stress(rk_level, elem_gid, 0, 0), 3, 3); + + // cut out the node_gids for this element + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 8); + + // get the B matrix which are the OUTWARD corner area normals + get_bmatrix(area_normal, + elem_gid, + node_coords, + elem_node_gids, + rk_level); + + // --- Calculate the velocity gradient --- + get_velgrad(vel_grad, + elem_node_gids, + node_vel, + area_normal, + vol, + elem_gid, + rk_level); + + // the -1 is for the inward surface area normal, + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + for (size_t dim = 0; dim < num_dims; dim++) + { + area_normal(node_lid, dim) = (-1.0) * area_normal(node_lid, dim); + } // end for + } // end for + + double div = elem_div(elem_gid); + + // vel = [u,v,w] + // [du/dx, du/dy, du/dz] + // vel_grad = [dv/dx, dv/dy, dv/dz] + // [dw/dx, dw/dy, dw/dz] + double curl[3]; + curl[0] = vel_grad(2, 1) - vel_grad(1, 2); // dw/dy - dv/dz + curl[1] = vel_grad(0, 2) - vel_grad(2, 0); // du/dz - dw/dx + curl[2] = vel_grad(1, 0) - vel_grad(0, 1); // dv/dx - du/dy + + double mag_curl = sqrt(curl[0] * curl[0] + curl[1] * curl[1] + curl[2] * curl[2]); + + // --- Calculate the Cauchy stress --- + for (size_t i = 0; i < 3; i++) + { + for (size_t j = 0; j < 3; j++) + { + tau(i, j) = stress(i, j); + // artificial viscosity can be added here to tau + } // end for + } // end for + + // add the pressure + for (int i = 0; i < num_dims; i++) + { + tau(i, i) -= elem_pres(elem_gid); + } // end for + + // ---- Multidirectional Approximate Riemann solver (MARS) ---- + // find the average velocity of the elem, it is an + // estimate of the Riemann velocity + + // initialize to Riemann velocity to zero + for (size_t dim = 0; dim < num_dims; dim++) + { + vel_star(dim) = 0.0; + } + + // loop over nodes and calculate an average velocity, which is + // an estimate of Riemann velocity + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + // Get node gloabl index and create view of nodal velocity + int node_gid = nodes_in_elem(elem_gid, node_lid); + + ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); + + vel_star(0) += 0.125 * vel(0); + vel_star(1) += 0.125 * vel(1); + vel_star(2) += 0.125 * vel(2); + } // end for loop over nodes + + // find shock direction and shock impedance associated with each node + + // initialize sum term in MARS to zero + for (int i = 0; i < 4; i++) + { + sum(i) = 0.0; + } + + double mag; // magnitude of the area normal + double mag_vel; // magnitude of velocity + + // loop over the nodes of the elem + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + // Get global node id + size_t node_gid = nodes_in_elem(elem_gid, node_lid); + + // Create view of nodal velocity + ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); + + // Get an estimate of the shock direction. + mag_vel = sqrt( (vel(0) - vel_star(0) ) * (vel(0) - vel_star(0) ) + + (vel(1) - vel_star(1) ) * (vel(1) - vel_star(1) ) + + (vel(2) - vel_star(2) ) * (vel(2) - vel_star(2) ) ); + + if (mag_vel > small) + { + // estimate of the shock direction, a unit normal + for (int dim = 0; dim < num_dims; dim++) + { + shock_dir(dim) = (vel(dim) - vel_star(dim)) / mag_vel; + } + } + else + { + // if there is no velocity change, then use the surface area + // normal as the shock direction + mag = sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) + + area_normal(node_lid, 1) * area_normal(node_lid, 1) + + area_normal(node_lid, 2) * area_normal(node_lid, 2) ); + + // estimate of the shock direction + for (int dim = 0; dim < num_dims; dim++) + { + shock_dir(dim) = area_normal(node_lid, dim) / mag; + } + } // end if mag_vel + + // cell divergence indicates compression or expansions + size_t mat_id = elem_mat_id(elem_gid); + if (div < 0) // element in compression + { + muc(node_lid) = elem_den(elem_gid) * + (material(mat_id).q1 * elem_sspd(elem_gid) + material(mat_id).q2 * mag_vel); + } + else // element in expansion + { + muc(node_lid) = elem_den(elem_gid) * + (material(mat_id).q1ex * elem_sspd(elem_gid) + material(mat_id).q2ex * mag_vel); + } // end if on divergence sign + + size_t use_shock_dir = 0; + double mu_term; + double mu_term_gradient; + + // Coding to use shock direction + if (use_shock_dir == 1) + { + // this is denominator of the Riamann solver and the multiplier + // on velocity in the numerator. It filters on the shock + // direction + mu_term = muc(node_lid) * + fabs(shock_dir(0) * area_normal(node_lid, 0) + + shock_dir(1) * area_normal(node_lid, 1) + + shock_dir(2) * area_normal(node_lid, 2) ); // code gradient for shock dir w.r.t velocity if using shock_dir + } + else + { + // Using a full tensoral Riemann jump relation + mu_term = muc(node_lid) + * sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) + + area_normal(node_lid, 1) * area_normal(node_lid, 1) + + area_normal(node_lid, 2) * area_normal(node_lid, 2) ); + } + + sum(0) += mu_term * vel(0); + sum(1) += mu_term * vel(1); + sum(2) += mu_term * vel(2); + sum(3) += mu_term; + + muc(node_lid) = mu_term; // the impeadance time surface area is stored here + } // end for node_lid loop over nodes of the elem + + // The Riemann velocity, called vel_star + if (sum(3) > fuzz) + { + for (size_t i = 0; i < num_dims; i++) + { + vel_star(i) = sum(i) / sum(3); + } + } + else + { + for (int i = 0; i < num_dims; i++) + { + vel_star(i) = 0.0; + } + } // end if + + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + // The Riemann velocity, called vel_star + if (sum(3) > fuzz) + { + for (size_t i = 0; i < num_dims; i++) + { + vel_star_gradient(node_lid, i) = muc(node_lid) / sum(3); + } + } + else + { + for (int i = 0; i < num_dims; i++) + { + vel_star_gradient(node_lid, i) = 0; + } + } // end if + } + + // ---- Calculate the shock detector for the Riemann-solver ---- + // + // The dissipation from the Riemann problem is limited by phi + // phi = (1. - max( 0., min( 1. , r_face ) ))^n + // where + // r_face = (C* div(u_+)/div(u_z)) + // The plus denotes the cell center divergence of a neighbor. + // The solution will be first order when phi=1 and have + // zero dissipation when phi=0. + // phi = 0 highest-order solution + // phi = 1 first order solution + // + + double phi = 0.0; // the shock detector + double r_face = 1.0; // the ratio on the face + double r_min = 1.0; // the min ratio for the cell + double r_coef = 0.9; // 0.9; the coefficient on the ratio + // (1=minmod and 2=superbee) + double n_coef = 1.0; // the power on the limiting coefficient + // (1=nominal, and n_coeff > 1 oscillatory) + + // loop over the nieghboring cells + for (size_t elem_lid = 0; elem_lid < num_elems_in_elem(elem_gid); elem_lid++) + { + // Get global index for neighboring cell + size_t neighbor_gid = elems_in_elem(elem_gid, elem_lid); + + // calculate the velocity divergence in neighbor + double div_neighbor = elem_div(neighbor_gid); + + r_face = r_coef * (div_neighbor + small) / (div + small); + + // store the smallest face ratio + r_min = fmin(r_face, r_min); + } // end for elem_lid + + // calculate standard shock detector + phi = 1.0 - fmax(0.0, r_min); + phi = pow(phi, n_coef); + + // Mach number shock detector + double omega = 20.0; // 20.0; // weighting factor on Mach number + double third = 1.0 / 3.0; + double c_length = pow(vol, third); // characteristic length + double alpha = fmin(1.0, omega * (c_length * fabs(div)) / (elem_sspd(elem_gid) + fuzz) ); + + // use Mach based detector with standard shock detector + + // turn off dissipation in expansion + // alpha = fmax(-fabs(div0)/div0 * alpha, 0.0); // this should be if(div0<0) alpha=alpha else alpha=0 + + phi = alpha * phi; + + // curl limiter on Q + double phi_curl = fmin(1.0, 1.0 * fabs(div) / (mag_curl + fuzz)); // disable Q when vorticity is high + // phi = phi_curl*phi; + phi = 1; + + // ---- Calculate the Riemann force on each node ---- + + // loop over the each node in the elem + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + size_t corner_lid = node_lid; + size_t column_index; + + // Get corner gid + size_t corner_gid = corners_in_elem(elem_gid, corner_lid); + + // Get node gid + size_t node_gid = nodes_in_elem(elem_gid, node_lid); + + // loop over dimension + for (int dim = 0; dim < num_dims; dim++) + { + // assign gradient of corner contribution of force to relevant matrix entries with non-zero node velocity gradient + for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) + { + size_t gradient_node_gid = nodes_in_elem(elem_gid, igradient); + // if(!map->isNodeLocalElement(gradient_node_gid)) continue; + column_index = num_dims * Global_Gradient_Matrix_Assembly_Map(elem_gid, igradient, node_lid); + if (node_lid == igradient) + { + if (map->isNodeLocalElement(gradient_node_gid)) + { + Force_Gradient_Velocities(gradient_node_gid * num_dims + dim, column_index + dim) += phi * muc(node_lid) * (vel_star_gradient(igradient, dim) - 1); + } + corner_gradient_storage(corner_gid, dim, igradient, dim) = phi * muc(node_lid) * (vel_star_gradient(igradient, dim) - 1); + } + else + { + if (map->isNodeLocalElement(gradient_node_gid)) + { + Force_Gradient_Velocities(gradient_node_gid * num_dims + dim, column_index + dim) += phi * muc(node_lid) * (vel_star_gradient(igradient, dim)); + } + corner_gradient_storage(corner_gid, dim, igradient, dim) = phi * muc(node_lid) * (vel_star_gradient(igradient, dim)); + } + } + } // end loop over dimension + } // end for loop over nodes in elem + + // --- Update Stress --- + // calculate the new stress at the next rk level, if it is a hypo model + size_t mat_id = elem_mat_id(elem_gid); + + // }); // end parallel for loop over elements + } + + /* + //accumulate node values from corner storage + force_gradient_design->putScalar(0); + + vec_array force_gradient_design_view = force_gradient_design->getLocalView (Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { + size_t corner_id; + for(int icorner=0; icorner < num_corners_in_node(node_id); icorner++){ + corner_id = corners_in_node(node_id,icorner); + force_gradient_design_view(node_id,0) += corner_vector_storage(corner_id, 0); + force_gradient_design_view(node_id,1) += corner_vector_storage(corner_id, 1); + force_gradient_design_view(node_id,2) += corner_vector_storage(corner_id, 2); + } + }); // end parallel for + Kokkos::fence(); + */ + + return; +} // end of routine + +// ----------------------------------------------------------------------------- +// This function calculates the corner forces and the evolves stress (hypo) +// ------------------------------------------------------------------------------ +void FEA_Module_SGH::get_force_egradient_sgh(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_pres, + const DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + const double rk_alpha, + const size_t cycle + ) +{ + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + const size_t num_dims = simparam->num_dims; + + // initialize gradient matrix + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { + // compute resulting row of force displacement gradient matrix transpose right multiplied by adjoint vector + for (int idof = 0; idof < num_nodes_in_elem * num_dims; idof++) + { + Force_Gradient_Energies(elem_gid, idof) = 0; + } + }); // end parallel for + Kokkos::fence(); + + // --- calculate the forces acting on the nodes from the element --- + for (size_t elem_gid = 0; elem_gid < rnum_elem; elem_gid++) + { + // FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { + + const size_t num_nodes_in_elem = 8; + real_t gradient_result[num_dims]; + // total Cauchy stress + double tau_array[9]; + double tau_gradient_array[9]; + + // corner area normals + double area_normal_array[24]; + + // estimate of shock direction + double shock_dir_array[3]; + + // the sums in the Riemann solver + double sum_array[4]; + double sum_gradient_array[4]; + + // corner shock impeadance x |corner area normal dot shock_dir| + double muc_array[8]; + double muc_gradient_array[8]; + + // Riemann velocity + double vel_star_array[3]; + double vel_star_gradient_array[3]; + + // velocity gradient + double vel_grad_array[9]; + + // --- Create views of arrays to aid the force calculation --- + + ViewCArrayKokkos tau(tau_array, num_dims, num_dims); + ViewCArrayKokkos tau_gradient(tau_gradient_array, num_dims, num_dims); + ViewCArrayKokkos area_normal(area_normal_array, num_nodes_in_elem, num_dims); + ViewCArrayKokkos shock_dir(shock_dir_array, num_dims); + ViewCArrayKokkos sum(sum_array, 4); + ViewCArrayKokkos sum_gradient(sum_gradient_array, 4); + ViewCArrayKokkos muc(muc_array, num_nodes_in_elem); + ViewCArrayKokkos muc_gradient(muc_gradient_array, num_nodes_in_elem); + ViewCArrayKokkos vel_star(vel_star_array, num_dims); + ViewCArrayKokkos vel_star_gradient(vel_star_gradient_array, num_dims); + ViewCArrayKokkos vel_grad(vel_grad_array, num_dims, num_dims); + + EOSParent* eos_model = elem_eos(elem_gid).model; + + // --- abviatations of variables --- + + // element volume + double vol = elem_vol(elem_gid); + + // create a view of the stress_matrix + ViewCArrayKokkos stress(&elem_stress(rk_level, elem_gid, 0, 0), 3, 3); + + // cut out the node_gids for this element + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 8); + + // get the B matrix which are the OUTWARD corner area normals + get_bmatrix(area_normal, + elem_gid, + node_coords, + elem_node_gids, + rk_level); + + // --- Calculate the velocity gradient --- + get_velgrad(vel_grad, + elem_node_gids, + node_vel, + area_normal, + vol, + elem_gid, + rk_level); + + // the -1 is for the inward surface area normal, + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + for (size_t dim = 0; dim < num_dims; dim++) + { + area_normal(node_lid, dim) = (-1.0) * area_normal(node_lid, dim); + } // end for + } // end for + + double div = elem_div(elem_gid); + + // vel = [u,v,w] + // [du/dx, du/dy, du/dz] + // vel_grad = [dv/dx, dv/dy, dv/dz] + // [dw/dx, dw/dy, dw/dz] + double curl[3]; + curl[0] = vel_grad(2, 1) - vel_grad(1, 2); // dw/dy - dv/dz + curl[1] = vel_grad(0, 2) - vel_grad(2, 0); // du/dz - dw/dx + curl[2] = vel_grad(1, 0) - vel_grad(0, 1); // dv/dx - du/dy + + double mag_curl = sqrt(curl[0] * curl[0] + curl[1] * curl[1] + curl[2] * curl[2]); + + // --- Calculate the Cauchy stress --- + for (size_t i = 0; i < num_dims; i++) + { + for (size_t j = 0; j < num_dims; j++) + { + tau(i, j) = stress(i, j); + tau_gradient(i, j) = 0; + // artificial viscosity can be added here to tau + } // end for + } // end for + + // add the pressure + for (int i = 0; i < num_dims; i++) + { + tau(i, i) -= elem_pres(elem_gid); + tau_gradient(i, i) = -eos_model->calc_pressure_gradient_internal_energy(elem_pres, + elem_stress, + elem_gid, + elem_mat_id(elem_gid), + state_vars, + global_vars, + elem_user_output_vars, + elem_sspd, + elem_den(elem_gid), + elem_sie(rk_level, elem_gid)); + } // end for + + // ---- Multidirectional Approximate Riemann solver (MARS) ---- + // find the average velocity of the elem, it is an + // estimate of the Riemann velocity + + // initialize to Riemann velocity to zero + for (size_t dim = 0; dim < num_dims; dim++) + { + vel_star(dim) = 0.0; + } + + // loop over nodes and calculate an average velocity, which is + // an estimate of Riemann velocity + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + // Get node gloabl index and create view of nodal velocity + int node_gid = nodes_in_elem(elem_gid, node_lid); + + ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); + + vel_star(0) += 0.125 * vel(0); + vel_star(1) += 0.125 * vel(1); + vel_star(2) += 0.125 * vel(2); + } // end for loop over nodes + + // find shock direction and shock impedance associated with each node + + // initialize sum term in MARS to zero + for (int i = 0; i < 4; i++) + { + sum(i) = sum_gradient(i) = 0.0; + } + + double mag; // magnitude of the area normal + double mag_vel; // magnitude of velocity + + // --- Sound speed --- + real_t sound_speed_gradient_energy = + eos_model->calc_sound_speed_gradient_internal_energy(elem_pres, + elem_stress, + elem_gid, + elem_mat_id(elem_gid), + state_vars, + global_vars, + elem_user_output_vars, + elem_sspd, + elem_den(elem_gid), + elem_sie(rk_level, elem_gid)); + + // loop over the nodes of the elem + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + // Get global node id + size_t node_gid = nodes_in_elem(elem_gid, node_lid); + + // Create view of nodal velocity + ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); + + // Get an estimate of the shock direction. + mag_vel = sqrt( (vel(0) - vel_star(0) ) * (vel(0) - vel_star(0) ) + + (vel(1) - vel_star(1) ) * (vel(1) - vel_star(1) ) + + (vel(2) - vel_star(2) ) * (vel(2) - vel_star(2) ) ); + + if (mag_vel > small) + { + // estimate of the shock direction, a unit normal + for (int dim = 0; dim < num_dims; dim++) + { + shock_dir(dim) = (vel(dim) - vel_star(dim)) / mag_vel; + } + } + else + { + // if there is no velocity change, then use the surface area + // normal as the shock direction + mag = sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) + + area_normal(node_lid, 1) * area_normal(node_lid, 1) + + area_normal(node_lid, 2) * area_normal(node_lid, 2) ); + + // estimate of the shock direction + for (int dim = 0; dim < num_dims; dim++) + { + shock_dir(dim) = area_normal(node_lid, dim) / mag; + } + } // end if mag_vel + + // cell divergence indicates compression or expansions + size_t mat_id = elem_mat_id(elem_gid); + if (div < 0) // element in compression + { + muc(node_lid) = elem_den(elem_gid) * + (material(mat_id).q1 * elem_sspd(elem_gid) + material(mat_id).q2 * mag_vel); + muc_gradient(node_lid) = elem_den(elem_gid) * + (material(mat_id).q1 * sound_speed_gradient_energy); + } + else // element in expansion + { + muc(node_lid) = elem_den(elem_gid) * + (material(mat_id).q1ex * elem_sspd(elem_gid) + material(mat_id).q2ex * mag_vel); + muc_gradient(node_lid) = elem_den(elem_gid) * + (material(mat_id).q1ex * sound_speed_gradient_energy); + } // end if on divergence sign + + size_t use_shock_dir = 0; + double mu_term; + double mu_term_gradient; + + // Coding to use shock direction + if (use_shock_dir == 1) + { + // this is denominator of the Riamann solver and the multiplier + // on velocity in the numerator. It filters on the shock + // direction + mu_term = muc(node_lid) * + fabs(shock_dir(0) * area_normal(node_lid, 0) + + shock_dir(1) * area_normal(node_lid, 1) + + shock_dir(2) * area_normal(node_lid, 2) ); + mu_term_gradient = muc_gradient(node_lid) * // amend if shock dir has dependence for gradient not captured here + fabs(shock_dir(0) * area_normal(node_lid, 0) + + shock_dir(1) * area_normal(node_lid, 1) + + shock_dir(2) * area_normal(node_lid, 2) ); + } + else + { + // Using a full tensoral Riemann jump relation + mu_term = muc(node_lid) + * sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) + + area_normal(node_lid, 1) * area_normal(node_lid, 1) + + area_normal(node_lid, 2) * area_normal(node_lid, 2) ); + mu_term_gradient = muc_gradient(node_lid) + * sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) + + area_normal(node_lid, 1) * area_normal(node_lid, 1) + + area_normal(node_lid, 2) * area_normal(node_lid, 2) ); + } + + sum(0) += mu_term * vel(0); + sum(1) += mu_term * vel(1); + sum(2) += mu_term * vel(2); + sum(3) += mu_term; + + sum_gradient(0) += mu_term_gradient * vel(0); + sum_gradient(1) += mu_term_gradient * vel(1); + sum_gradient(2) += mu_term_gradient * vel(2); + sum_gradient(3) += mu_term_gradient; + + muc(node_lid) = mu_term; // the impeadance time surface area is stored here + muc_gradient(node_lid) = mu_term_gradient; + } // end for node_lid loop over nodes of the elem + + // The Riemann velocity, called vel_star + if (sum(3) > fuzz) + { + for (size_t i = 0; i < num_dims; i++) + { + vel_star(i) = sum(i) / sum(3); + vel_star_gradient(i) = sum_gradient(i) / sum(3) - sum_gradient(3) * sum(i) / sum(3) / sum(3); + } + } + else + { + for (int i = 0; i < num_dims; i++) + { + vel_star(i) = 0.0; + vel_star_gradient(i) = 0; + } + } // end if + + // ---- Calculate the shock detector for the Riemann-solver ---- + // + // The dissipation from the Riemann problem is limited by phi + // phi = (1. - max( 0., min( 1. , r_face ) ))^n + // where + // r_face = (C* div(u_+)/div(u_z)) + // The plus denotes the cell center divergence of a neighbor. + // The solution will be first order when phi=1 and have + // zero dissipation when phi=0. + // phi = 0 highest-order solution + // phi = 1 first order solution + // + + double phi = 0.0; // the shock detector + double r_face = 1.0; // the ratio on the face + double r_min = 1.0; // the min ratio for the cell + double r_coef = 0.9; // 0.9; the coefficient on the ratio + // (1=minmod and 2=superbee) + double n_coef = 1.0; // the power on the limiting coefficient + // (1=nominal, and n_coeff > 1 oscillatory) + + // loop over the nieghboring cells + for (size_t elem_lid = 0; elem_lid < num_elems_in_elem(elem_gid); elem_lid++) + { + // Get global index for neighboring cell + size_t neighbor_gid = elems_in_elem(elem_gid, elem_lid); + + // calculate the velocity divergence in neighbor + double div_neighbor = elem_div(neighbor_gid); + + r_face = r_coef * (div_neighbor + small) / (div + small); + + // store the smallest face ratio + r_min = fmin(r_face, r_min); + } // end for elem_lid + + // calculate standard shock detector + phi = 1.0 - fmax(0.0, r_min); + phi = pow(phi, n_coef); + + // Mach number shock detector + double omega = 20.0; // 20.0; // weighting factor on Mach number + double third = 1.0 / 3.0; + double c_length = pow(vol, third); // characteristic length + double alpha = fmin(1.0, omega * (c_length * fabs(div)) / (elem_sspd(elem_gid) + fuzz) ); + + // use Mach based detector with standard shock detector + + // turn off dissipation in expansion + // alpha = fmax(-fabs(div0)/div0 * alpha, 0.0); // this should be if(div0<0) alpha=alpha else alpha=0 + + phi = alpha * phi; + + // curl limiter on Q + double phi_curl = fmin(1.0, 1.0 * fabs(div) / (mag_curl + fuzz)); // disable Q when vorticity is high + // phi = phi_curl*phi; + phi = 1; + + // ---- Calculate the Riemann force on each node ---- + + // loop over the each node in the elem + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + size_t corner_lid = node_lid; + + // Get corner gid + size_t corner_gid = corners_in_elem(elem_gid, corner_lid); + + // Get node gid + size_t node_gid = nodes_in_elem(elem_gid, node_lid); + + // loop over dimension + for (int dim = 0; dim < num_dims; dim++) + { + Force_Gradient_Energies(elem_gid, node_lid * num_dims + dim) = corner_vector_storage(corner_gid, dim) = + area_normal(node_lid, 0) * tau_gradient(0, dim) + + area_normal(node_lid, 1) * tau_gradient(1, dim) + + area_normal(node_lid, 2) * tau_gradient(2, dim) + + phi * muc_gradient(node_lid) * (vel_star(dim) - node_vel(rk_level, node_gid, dim)) + + phi * muc(node_lid) * (vel_star_gradient(dim)); + } // end loop over dimension + } // end for loop over nodes in elem + + // --- Update Stress --- + // calculate the new stress at the next rk level, if it is a hypo model + + size_t mat_id = elem_mat_id(elem_gid); + } + // }); // end parallel for loop over elements + + /* + //accumulate node values from corner storage + force_gradient_design->putScalar(0); + + vec_array force_gradient_design_view = force_gradient_design->getLocalView (Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { + size_t corner_id; + for(int icorner=0; icorner < num_corners_in_node(node_id); icorner++){ + corner_id = corners_in_node(node_id,icorner); + force_gradient_design_view(node_id,0) += corner_vector_storage(corner_id, 0); + force_gradient_design_view(node_id,1) += corner_vector_storage(corner_id, 1); + force_gradient_design_view(node_id,2) += corner_vector_storage(corner_id, 2); + } + }); // end parallel for + Kokkos::fence(); + */ + + return; +} // end of routine + +// ----------------------------------------------------------------------------- +// This function calculates the corner forces and the evolves stress (hypo) +// ------------------------------------------------------------------------------ +void FEA_Module_SGH::get_force_ugradient_sgh(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_pres, + const DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + const double rk_alpha, + const size_t cycle + ) +{ + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + const size_t num_dims = simparam->num_dims; + + // initialize gradient matrix + FOR_ALL_CLASS(dof_gid, 0, nlocal_nodes * num_dims, { + for (int idof = 0; idof < Gradient_Matrix_Strides(dof_gid); idof++) + { + Force_Gradient_Positions(dof_gid, idof) = 0; + } + }); // end parallel for loop over nodes + Kokkos::fence(); + + // --- calculate the forces acting on the nodes from the element --- + for (size_t elem_gid = 0; elem_gid < rnum_elem; elem_gid++) + { + // FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { + + const size_t num_nodes_in_elem = 8; + real_t gradient_result[num_dims]; + // total Cauchy stress + double tau_array[9]; + double tau_gradient_array[9 * max_nodes_per_element * num_dims]; + + // corner area normals + double area_normal_gradients_array[max_nodes_per_element * max_nodes_per_element * num_dims * num_dims]; + double area_normal_array[24]; + + // volume data + double volume; + double volume_gradients_array[max_nodes_per_element * num_dims]; + + // estimate of shock direction + double shock_dir_array[3]; + + // the sums in the Riemann solver + double sum_array[4]; + double sum_gradient_array[4 * max_nodes_per_element * num_dim]; + + // corner shock impeadance x |corner area normal dot shock_dir| + double muc_array[8]; + double muc_gradient_array[8 * max_nodes_per_element * num_dims]; + + // Riemann velocity + double vel_star_array[3]; + double vel_star_gradient_array[num_dims * max_nodes_per_element * num_dims]; + + // velocity gradient + double vel_grad_array[9]; + + // --- Create views of arrays to aid the force calculation --- + + ViewCArrayKokkos tau(tau_array, num_dims, num_dims); + ViewCArrayKokkos tau_gradient(tau_gradient_array, num_dims, num_dims, max_nodes_per_element, num_dims); + ViewCArrayKokkos volume_gradients(volume_gradients_array, num_nodes_in_elem, num_dims); + ViewCArrayKokkos area_normal(area_normal_array, num_nodes_in_elem, num_dims); + ViewCArrayKokkos area_normal_gradients(area_normal_gradients_array, num_nodes_in_elem, num_dims, num_nodes_in_elem, num_dims); + ViewCArrayKokkos shock_dir(shock_dir_array, num_dims); + ViewCArrayKokkos sum(sum_array, 4); + ViewCArrayKokkos sum_gradient(sum_gradient_array, 4, max_nodes_per_element, num_dims); + ViewCArrayKokkos muc(muc_array, num_nodes_in_elem); + ViewCArrayKokkos muc_gradient(muc_gradient_array, num_nodes_in_elem, num_nodes_in_elem, num_dims); + ViewCArrayKokkos vel_star(vel_star_array, num_dims); + ViewCArrayKokkos vel_star_gradient(vel_star_gradient_array, num_dims, num_nodes_in_elem, num_dims); + ViewCArrayKokkos vel_grad(vel_grad_array, num_dims, num_dims); + + EOSParent* eos_model = elem_eos(elem_gid).model; + + // --- abviatations of variables --- + + // element volume + double vol = elem_vol(elem_gid); + + // create a view of the stress_matrix + ViewCArrayKokkos stress(&elem_stress(rk_level, elem_gid, 0, 0), 3, 3); + + // cut out the node_gids for this element + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 8); + + // gradients of the element volume + get_vol_hex_ugradient(volume_gradients, elem_gid, node_coords, elem_node_gids, rk_level); + + // //debug + // for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ + // for (size_t dim = 0; dim < num_dims; dim++){ + // if(volume_gradients(node_lid, dim)>1 || volume_gradients(node_lid, dim) < -1) + // std::cout << volume_gradients(node_lid, dim) << " "; + // } // end for + // } // end for + // std::cout << std::endl; + + // get the B matrix which are the OUTWARD corner area normals + get_bmatrix(area_normal, + elem_gid, + node_coords, + elem_node_gids, + rk_level); + + get_bmatrix_gradients(area_normal_gradients, + elem_gid, + node_coords, + elem_node_gids, + rk_level); + + // --- Calculate the velocity gradient --- + get_velgrad(vel_grad, + elem_node_gids, + node_vel, + area_normal, + vol, + elem_gid, + rk_level); + + // the -1 is for the inward surface area normal, + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + for (size_t dim = 0; dim < num_dims; dim++) + { + area_normal(node_lid, dim) = (-1.0) * area_normal(node_lid, dim); + } // end for + } // end for + + // the -1 is for the inward surface area normal, + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + for (size_t dim = 0; dim < num_dims; dim++) + { + for (size_t igradient = 0; igradient < num_nodes_in_elem; igradient++) + { + for (size_t jdim = 0; jdim < num_dims; jdim++) + { + area_normal_gradients(node_lid, dim, igradient, jdim) = (-1.0) * area_normal_gradients(node_lid, dim, igradient, jdim); + } + } + } // end for + } // end for + + double div = elem_div(elem_gid); + + // vel = [u,v,w] + // [du/dx, du/dy, du/dz] + // vel_grad = [dv/dx, dv/dy, dv/dz] + // [dw/dx, dw/dy, dw/dz] + double curl[3]; + curl[0] = vel_grad(2, 1) - vel_grad(1, 2); // dw/dy - dv/dz + curl[1] = vel_grad(0, 2) - vel_grad(2, 0); // du/dz - dw/dx + curl[2] = vel_grad(1, 0) - vel_grad(0, 1); // dv/dx - du/dy + + double mag_curl = sqrt(curl[0] * curl[0] + curl[1] * curl[1] + curl[2] * curl[2]); + + // --- Calculate the Cauchy stress --- + for (size_t i = 0; i < 3; i++) + { + for (size_t j = 0; j < 3; j++) + { + tau(i, j) = stress(i, j); + // artificial viscosity can be added here to tau + } // end for + } // end for + + // initialize gradient array + for (size_t i = 0; i < 3; i++) + { + for (size_t j = 0; j < 3; j++) + { + for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) + { + tau_gradient(i, j, igradient, 0) = 0; + tau_gradient(i, j, igradient, 1) = 0; + if (num_dims == 3) + { + tau_gradient(i, j, igradient, 2) = 0; + } + } + } // end for + } // end for + + // add the pressure + for (int i = 0; i < num_dims; i++) + { + tau(i, i) -= elem_pres(elem_gid); + // current expression only works for ideal gas! generalize. + for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) + { + tau_gradient(i, i, igradient, 0) = elem_pres(elem_gid) * volume_gradients(igradient, 0) / vol; + tau_gradient(i, i, igradient, 1) = elem_pres(elem_gid) * volume_gradients(igradient, 1) / vol; + tau_gradient(i, i, igradient, 2) = elem_pres(elem_gid) * volume_gradients(igradient, 2) / vol; + } + } // end for + + // ---- Multidirectional Approximate Riemann solver (MARS) ---- + // find the average velocity of the elem, it is an + // estimate of the Riemann velocity + + // initialize to Riemann velocity to zero + for (size_t dim = 0; dim < num_dims; dim++) + { + vel_star(dim) = 0.0; + } + + // loop over nodes and calculate an average velocity, which is + // an estimate of Riemann velocity + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + // Get node gloabl index and create view of nodal velocity + int node_gid = nodes_in_elem(elem_gid, node_lid); + + ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); + + vel_star(0) += 0.125 * vel(0); + vel_star(1) += 0.125 * vel(1); + vel_star(2) += 0.125 * vel(2); + } // end for loop over nodes + + // find shock direction and shock impedance associated with each node + + // initialize sum term in MARS to zero + for (int i = 0; i < 4; i++) + { + sum(i) = 0.0; + } + for (int i = 0; i < 4; i++) + { + for (size_t igradient = 0; igradient < num_nodes_in_elem; igradient++) + { + for (size_t jdim = 0; jdim < num_dims; jdim++) + { + sum_gradient(i, igradient, jdim) = 0.0; + } + } + } + + double mag; // magnitude of the area normal + double mag_vel; // magnitude of velocity + + // loop over the nodes of the elem + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + // Get global node id + size_t node_gid = nodes_in_elem(elem_gid, node_lid); + + // Create view of nodal velocity + ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); + + // Get an estimate of the shock direction. + mag_vel = sqrt( (vel(0) - vel_star(0) ) * (vel(0) - vel_star(0) ) + + (vel(1) - vel_star(1) ) * (vel(1) - vel_star(1) ) + + (vel(2) - vel_star(2) ) * (vel(2) - vel_star(2) ) ); + + if (mag_vel > small) + { + // estimate of the shock direction, a unit normal + for (int dim = 0; dim < num_dims; dim++) + { + shock_dir(dim) = (vel(dim) - vel_star(dim)) / mag_vel; + } + } + else + { + // if there is no velocity change, then use the surface area + // normal as the shock direction + mag = sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) + + area_normal(node_lid, 1) * area_normal(node_lid, 1) + + area_normal(node_lid, 2) * area_normal(node_lid, 2) ); + + // estimate of the shock direction + for (int dim = 0; dim < num_dims; dim++) + { + shock_dir(dim) = area_normal(node_lid, dim) / mag; + } + } // end if mag_vel + + // cell divergence indicates compression or expansions + size_t mat_id = elem_mat_id(elem_gid); + if (div < 0) // element in compression + { + muc(node_lid) = elem_den(elem_gid) * + (material(mat_id).q1 * elem_sspd(elem_gid) + material(mat_id).q2 * mag_vel); + + for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) + { + muc_gradient(node_lid, igradient, 0) = -elem_den(elem_gid) * volume_gradients(igradient, 0) / vol * + (material(mat_id).q1 * elem_sspd(elem_gid)); + muc_gradient(node_lid, igradient, 1) = -elem_den(elem_gid) * volume_gradients(igradient, 1) / vol * + (material(mat_id).q1 * elem_sspd(elem_gid)); + muc_gradient(node_lid, igradient, 2) = -elem_den(elem_gid) * volume_gradients(igradient, 2) / vol * + (material(mat_id).q1 * elem_sspd(elem_gid)); + } + } + else // element in expansion + { + muc(node_lid) = elem_den(elem_gid) * + (material(mat_id).q1ex * elem_sspd(elem_gid) + material(mat_id).q2ex * mag_vel); + for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) + { + muc_gradient(node_lid, igradient, 0) = -elem_den(elem_gid) * volume_gradients(igradient, 0) / vol * + (material(mat_id).q1ex * elem_sspd(elem_gid)); + muc_gradient(node_lid, igradient, 1) = -elem_den(elem_gid) * volume_gradients(igradient, 1) / vol * + (material(mat_id).q1ex * elem_sspd(elem_gid)); + muc_gradient(node_lid, igradient, 2) = -elem_den(elem_gid) * volume_gradients(igradient, 2) / vol * + (material(mat_id).q1ex * elem_sspd(elem_gid)); + } + } // end if on divergence sign + + size_t use_shock_dir = 0; + double mu_term; + double mu_term_gradient; + + // Coding to use shock direction + if (use_shock_dir == 1) + { + // this is denominator of the Riamann solver and the multiplier + // on velocity in the numerator. It filters on the shock + // direction + mu_term = muc(node_lid) * + fabs(shock_dir(0) * area_normal(node_lid, 0) + + shock_dir(1) * area_normal(node_lid, 1) + + shock_dir(2) * area_normal(node_lid, 2) ); + // muc_gradient(node_lid) = muc_gradient(node_lid)* + // fabs( shock_dir(0)*area_normal(node_lid,0) + // + shock_dir(1)*area_normal(node_lid,1) + // + shock_dir(2)*area_normal(node_lid,2) ); + // mu_term_gradient = muc(node_lid)* //amend if shock dir has dependence for gradient not captured here + // fabs( shock_dir(0)*area_normal_gradients(node_lid,0) + // + shock_dir(1)*area_normal_gradients(node_lid,1) + // + shock_dir(2)*area_normal_gradients(node_lid,2) ); + } + else + { + // Using a full tensoral Riemann jump relation + mu_term = muc(node_lid) + * sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) + + area_normal(node_lid, 1) * area_normal(node_lid, 1) + + area_normal(node_lid, 2) * area_normal(node_lid, 2) ); + for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) + { + for (int jdim = 0; jdim < num_dims; jdim++) + { + mu_term_gradient = muc_gradient(node_lid, igradient, jdim) + * sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) + + area_normal(node_lid, 1) * area_normal(node_lid, 1) + + area_normal(node_lid, 2) * area_normal(node_lid, 2) ); + + mu_term_gradient += muc(node_lid) * (area_normal(node_lid, 0) * area_normal_gradients(node_lid, 0, igradient, jdim) + + area_normal(node_lid, 1) * area_normal_gradients(node_lid, 1, igradient, jdim) + + area_normal(node_lid, 2) * area_normal_gradients(node_lid, 2, igradient, jdim)) / + sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) + + area_normal(node_lid, 1) * area_normal(node_lid, 1) + + area_normal(node_lid, 2) * area_normal(node_lid, 2) ); + + muc_gradient(node_lid, igradient, jdim) = mu_term_gradient; + } + } + } + + sum(0) += mu_term * vel(0); + sum(1) += mu_term * vel(1); + sum(2) += mu_term * vel(2); + sum(3) += mu_term; + + // sum gradients + + for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) + { + for (int jdim = 0; jdim < num_dims; jdim++) + { + sum_gradient(0, igradient, jdim) += muc_gradient(node_lid, igradient, jdim) * vel(0); + sum_gradient(1, igradient, jdim) += muc_gradient(node_lid, igradient, jdim) * vel(1); + sum_gradient(2, igradient, jdim) += muc_gradient(node_lid, igradient, jdim) * vel(2); + sum_gradient(3, igradient, jdim) += muc_gradient(node_lid, igradient, jdim); + } + } + + muc(node_lid) = mu_term; // the impeadance time surface area is stored here + } // end for node_lid loop over nodes of the elem + + // The Riemann velocity, called vel_star + if (sum(3) > fuzz) + { + for (size_t i = 0; i < num_dims; i++) + { + vel_star(i) = sum(i) / sum(3); + } + } + else + { + for (int i = 0; i < num_dims; i++) + { + vel_star(i) = 0.0; + } + } // end if + + // vel star gradients + if (sum(3) > fuzz) + { + for (size_t i = 0; i < num_dims; i++) + { + for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) + { + for (int jdim = 0; jdim < num_dims; jdim++) + { + vel_star_gradient(i, igradient, jdim) = sum_gradient(i, igradient, jdim) / sum(3) - sum(i) / (sum(3) * sum(3)) * sum_gradient(3, igradient, jdim); + } + } + } + } + else + { + for (int i = 0; i < num_dims; i++) + { + for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) + { + for (int jdim = 0; jdim < num_dims; jdim++) + { + vel_star_gradient(i, igradient, jdim) = 0; + } + } + } + } // end if + + // ---- Calculate the shock detector for the Riemann-solver ---- + // + // The dissipation from the Riemann problem is limited by phi + // phi = (1. - max( 0., min( 1. , r_face ) ))^n + // where + // r_face = (C* div(u_+)/div(u_z)) + // The plus denotes the cell center divergence of a neighbor. + // The solution will be first order when phi=1 and have + // zero dissipation when phi=0. + // phi = 0 highest-order solution + // phi = 1 first order solution + // + + double phi = 0.0; // the shock detector + double r_face = 1.0; // the ratio on the face + double r_min = 1.0; // the min ratio for the cell + double r_coef = 0.9; // 0.9; the coefficient on the ratio + // (1=minmod and 2=superbee) + double n_coef = 1.0; // the power on the limiting coefficient + // (1=nominal, and n_coeff > 1 oscillatory) + + // loop over the nieghboring cells + for (size_t elem_lid = 0; elem_lid < num_elems_in_elem(elem_gid); elem_lid++) + { + // Get global index for neighboring cell + size_t neighbor_gid = elems_in_elem(elem_gid, elem_lid); + + // calculate the velocity divergence in neighbor + double div_neighbor = elem_div(neighbor_gid); + + r_face = r_coef * (div_neighbor + small) / (div + small); + + // store the smallest face ratio + r_min = fmin(r_face, r_min); + } // end for elem_lid + + // calculate standard shock detector + phi = 1.0 - fmax(0.0, r_min); + phi = pow(phi, n_coef); + + // Mach number shock detector + double omega = 20.0; // 20.0; // weighting factor on Mach number + double third = 1.0 / 3.0; + double c_length = pow(vol, third); // characteristic length + double alpha = fmin(1.0, omega * (c_length * fabs(div)) / (elem_sspd(elem_gid) + fuzz) ); + + // use Mach based detector with standard shock detector + + // turn off dissipation in expansion + // alpha = fmax(-fabs(div0)/div0 * alpha, 0.0); // this should be if(div0<0) alpha=alpha else alpha=0 + + phi = alpha * phi; + + // curl limiter on Q + double phi_curl = fmin(1.0, 1.0 * fabs(div) / (mag_curl + fuzz)); // disable Q when vorticity is high + // phi = phi_curl*phi; + phi = 1; + + // ---- Calculate the Riemann force on each node ---- + + // loop over the each node in the elem + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + size_t corner_lid = node_lid; + size_t column_index; + + // Get corner gid + size_t corner_gid = corners_in_elem(elem_gid, corner_lid); + + // Get node gid + size_t node_gid = nodes_in_elem(elem_gid, node_lid); + + // loop over dimension + for (int dim = 0; dim < num_dims; dim++) + { + // assign gradient of corner contribution of force to relevant matrix entries with non-zero node velocity gradient + for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) + { + for (int jdim = 0; jdim < num_dims; jdim++) + { + size_t gradient_node_gid = nodes_in_elem(elem_gid, igradient); + // if(!map->isNodeLocalElement(gradient_node_gid)) continue; + column_index = num_dims * Global_Gradient_Matrix_Assembly_Map(elem_gid, igradient, node_lid); + if (map->isNodeLocalElement(gradient_node_gid)) + { + Force_Gradient_Positions(gradient_node_gid * num_dims + jdim, column_index + dim) += area_normal(node_lid, 0) * tau_gradient(0, dim, igradient, jdim) + + area_normal(node_lid, 1) * tau_gradient(1, dim, igradient, jdim) + + area_normal(node_lid, 2) * tau_gradient(2, dim, igradient, jdim) + + area_normal_gradients(node_lid, 0, igradient, jdim) * tau(0, dim) + + area_normal_gradients(node_lid, 1, igradient, jdim) * tau(1, dim) + + area_normal_gradients(node_lid, 2, igradient, jdim) * tau(2, dim) + + phi * muc_gradient(node_lid, igradient, + jdim) * (vel_star(dim) - node_vel(rk_level, node_gid, dim)) + + phi * muc(node_lid) * (vel_star_gradient(dim, igradient, jdim)); + } + corner_gradient_storage(corner_gid, dim, igradient, jdim) = area_normal(node_lid, 0) * tau_gradient(0, dim, igradient, jdim) + + area_normal(node_lid, 1) * tau_gradient(1, dim, igradient, jdim) + + area_normal(node_lid, 2) * tau_gradient(2, dim, igradient, jdim) + + area_normal_gradients(node_lid, 0, igradient, jdim) * tau(0, dim) + + area_normal_gradients(node_lid, 1, igradient, jdim) * tau(1, dim) + + area_normal_gradients(node_lid, 2, igradient, jdim) * tau(2, dim) + + phi * muc_gradient(node_lid, igradient, jdim) * (vel_star(dim) - node_vel(rk_level, node_gid, dim)) + + phi * muc(node_lid) * (vel_star_gradient(dim, igradient, jdim)); + // if(map->isNodeLocalElement(gradient_node_gid)){ + // Force_Gradient_Positions(gradient_node_gid*num_dims+jdim, column_index+dim) += + // + area_normal_gradients(node_lid, 0, igradient, jdim)*tau(0, dim) + // + area_normal_gradients(node_lid, 1, igradient, jdim)*tau(1, dim) + // + area_normal_gradients(node_lid, 2, igradient, jdim)*tau(2, dim); + // } + // corner_gradient_storage(corner_gid,dim,igradient,jdim) = + // + area_normal_gradients(node_lid, 0, igradient, jdim)*tau(0, dim) + // + area_normal_gradients(node_lid, 1, igradient, jdim)*tau(1, dim) + // + area_normal_gradients(node_lid, 2, igradient, jdim)*tau(2, dim); + } + } + } // end loop over dimension + } // end for loop over nodes in elem + + // --- Update Stress --- + // calculate the new stress at the next rk level, if it is a hypo model + + size_t mat_id = elem_mat_id(elem_gid); + } + // }); // end parallel for loop over elements + + /* + //accumulate node values from corner storage + force_gradient_design->putScalar(0); + + vec_array force_gradient_design_view = force_gradient_design->getLocalView (Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { + size_t corner_id; + for(int icorner=0; icorner < num_corners_in_node(node_id); icorner++){ + corner_id = corners_in_node(node_id,icorner); + force_gradient_design_view(node_id,0) += corner_vector_storage(corner_id, 0); + force_gradient_design_view(node_id,1) += corner_vector_storage(corner_id, 1); + force_gradient_design_view(node_id,2) += corner_vector_storage(corner_id, 2); + } + }); // end parallel for + Kokkos::fence(); + */ + + return; +} // end of routine + +// -------------------------------------------------------------------------------------------------------- +// Computes term objective derivative term involving gradient of power with respect to the design variable +// --------------------------------------------------------------------------------------------------------- + +void FEA_Module_SGH::force_design_gradient_term(const_vec_array design_variables, vec_array design_gradients) +{ + size_t num_bdy_nodes = mesh->num_bdy_nodes; + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos material = simparam->material; + const int num_dim = simparam->num_dims; + size_t num_corners = rnum_elem * num_nodes_in_elem; + real_t global_dt; + bool element_constant_density = true; + size_t current_data_index, next_data_index; + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + CArrayKokkos current_element_adjoint = CArrayKokkos(num_nodes_in_elem, num_dim); + + // gradient contribution from gradient of Force vector with respect to design variable. + if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) + { + if (myrank == 0) + { + std::cout << "gradient term involving adjoint derivative" << std::endl; + } + } + + for (unsigned long cycle = 0; cycle < last_time_step + 1; cycle++) + { + // compute timestep from time data + global_dt = time_data[cycle + 1] - time_data[cycle]; + // print + if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) + { + if (cycle == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } + // print time step every 10 cycles + else if (cycle % 20 == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } // end if + } + + // view scope + { + const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array current_element_internal_energy = (*forward_solve_internal_energy_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array current_coord_vector = (*forward_solve_coordinate_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_velocity_vector = (*forward_solve_velocity_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_element_internal_energy = (*forward_solve_internal_energy_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_coord_vector = (*forward_solve_coordinate_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_adjoint_vector = (*adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + + // first half of integration step calculation + FOR_ALL_CLASS(node_gid, 0, nall_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + node_vel(rk_level, node_gid, idim) = current_velocity_vector(node_gid, idim); + node_coords(rk_level, node_gid, idim) = current_coord_vector(node_gid, idim); + } + }); // end parallel for + Kokkos::fence(); + + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { + elem_sie(rk_level, elem_gid) = current_element_internal_energy(elem_gid, 0); + }); // end parallel for + Kokkos::fence(); + + get_vol(); + + // ---- Calculate velocity diveregence for the element ---- + if (num_dim == 2) + { + get_divergence2D(elem_div, + node_coords, + node_vel, + elem_vol); + } + else + { + get_divergence(elem_div, + node_coords, + node_vel, + elem_vol); + } // end if 2D + + // ---- Calculate elem state (den, pres, sound speed, stress) for next time step ---- + if (num_dim == 2) + { + update_state2D(material, + *mesh, + node_coords, + node_vel, + elem_den, + elem_pres, + elem_stress, + elem_sspd, + elem_sie, + elem_vol, + elem_mass, + elem_mat_id, + 1.0, + cycle); + } + else + { + update_state(material, + *mesh, + node_coords, + node_vel, + elem_den, + elem_pres, + elem_stress, + elem_sspd, + elem_sie, + elem_vol, + elem_mass, + elem_mat_id, + 1.0, + cycle); + } + + get_force_dgradient_sgh(material, + *mesh, + node_coords, + node_vel, + elem_den, + elem_sie, + elem_pres, + elem_stress, + elem_sspd, + elem_vol, + elem_div, + elem_mat_id, + 1.0, + cycle); + + // derivatives of forces at corners stored in corner_vector_storage buffer by previous routine + FOR_ALL_CLASS(elem_id, 0, rnum_elem, { + size_t node_id; + size_t corner_id; + real_t inner_product; + + inner_product = 0; + for (int ifill = 0; ifill < num_nodes_in_elem; ifill++) + { + node_id = nodes_in_elem(elem_id, ifill); + corner_id = elem_id * num_nodes_in_elem + ifill; + for (int idim = 0; idim < num_dim; idim++) + { + inner_product += corner_vector_storage(corner_id, idim) * current_adjoint_vector(node_id, idim); + } + } + + for (int inode = 0; inode < num_nodes_in_elem; inode++) + { + // compute gradient of local element contribution to v^t*M*v product + corner_id = elem_id * num_nodes_in_elem + inode; + corner_value_storage(corner_id) = inner_product; + } + }); // end parallel for + Kokkos::fence(); + + // accumulate node values from corner storage + // multiply + FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { + size_t corner_id; + for (int icorner = 0; icorner < num_corners_in_node(node_id); icorner++) + { + corner_id = corners_in_node(node_id, icorner); + design_gradients(node_id, 0) += -corner_value_storage(corner_id) * global_dt; + } + }); // end parallel for + Kokkos::fence(); + } // end view scope + } +} + +// ----------------------------------------------------------------------------- +// This function calculates the corner forces and the evolves stress (hypo) +// ------------------------------------------------------------------------------ +void FEA_Module_SGH::get_force_dgradient_sgh(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_pres, + const DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + const double rk_alpha, + const size_t cycle + ) +{ + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + const size_t num_dims = simparam->num_dims; + // --- calculate the forces acting on the nodes from the element --- + for (size_t elem_gid = 0; elem_gid < rnum_elem; elem_gid++) + { + // FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { + + const size_t num_nodes_in_elem = 8; + real_t gradient_result[num_dims]; + // total Cauchy stress + double tau_array[9]; + double tau_gradient_array[9]; + + // corner area normals + double area_normal_array[24]; + + // estimate of shock direction + double shock_dir_array[3]; + + // the sums in the Riemann solver + double sum_array[4]; + double sum_gradient_array[4]; + + // corner shock impeadance x |corner area normal dot shock_dir| + double muc_array[8]; + double muc_gradient_array[8]; + + // Riemann velocity + double vel_star_array[3]; + double vel_star_gradient_array[3]; + + // velocity gradient + double vel_grad_array[9]; + + // --- Create views of arrays to aid the force calculation --- + + ViewCArrayKokkos tau(tau_array, num_dims, num_dims); + ViewCArrayKokkos tau_gradient(tau_gradient_array, num_dims, num_dims); + ViewCArrayKokkos area_normal(area_normal_array, num_nodes_in_elem, num_dims); + ViewCArrayKokkos shock_dir(shock_dir_array, num_dims); + ViewCArrayKokkos sum(sum_array, 4); + ViewCArrayKokkos sum_gradient(sum_gradient_array, 4); + ViewCArrayKokkos muc(muc_array, num_nodes_in_elem); + ViewCArrayKokkos muc_gradient(muc_gradient_array, num_nodes_in_elem); + ViewCArrayKokkos vel_star(vel_star_array, num_dims); + ViewCArrayKokkos vel_star_gradient(vel_star_gradient_array, num_dims); + ViewCArrayKokkos vel_grad(vel_grad_array, num_dims, num_dims); + + EOSParent* eos_model = elem_eos(elem_gid).model; + + // --- abviatations of variables --- + + // element volume + double vol = elem_vol(elem_gid); + + // create a view of the stress_matrix + ViewCArrayKokkos stress(&elem_stress(rk_level, elem_gid, 0, 0), 3, 3); + + // cut out the node_gids for this element + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 8); + + // get the B matrix which are the OUTWARD corner area normals + get_bmatrix(area_normal, + elem_gid, + node_coords, + elem_node_gids, + rk_level); + + // --- Calculate the velocity gradient --- + get_velgrad(vel_grad, + elem_node_gids, + node_vel, + area_normal, + vol, + elem_gid, + rk_level); + + // the -1 is for the inward surface area normal, + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + for (size_t dim = 0; dim < num_dims; dim++) + { + area_normal(node_lid, dim) = (-1.0) * area_normal(node_lid, dim); + } // end for + } // end for + + double div = elem_div(elem_gid); + + // vel = [u,v,w] + // [du/dx, du/dy, du/dz] + // vel_grad = [dv/dx, dv/dy, dv/dz] + // [dw/dx, dw/dy, dw/dz] + double curl[3]; + curl[0] = vel_grad(2, 1) - vel_grad(1, 2); // dw/dy - dv/dz + curl[1] = vel_grad(0, 2) - vel_grad(2, 0); // du/dz - dw/dx + curl[2] = vel_grad(1, 0) - vel_grad(0, 1); // dv/dx - du/dy + + double mag_curl = sqrt(curl[0] * curl[0] + curl[1] * curl[1] + curl[2] * curl[2]); + + // --- Calculate the Cauchy stress --- + for (size_t i = 0; i < 3; i++) + { + for (size_t j = 0; j < 3; j++) + { + tau(i, j) = stress(i, j); + tau_gradient(i, j) = 0; + } // end for + } // end for + + // add the pressure + for (int i = 0; i < num_dims; i++) + { + tau(i, i) -= elem_pres(elem_gid); + tau_gradient(i, i) -= elem_pres(elem_gid) / num_nodes_in_elem / relative_element_densities(elem_gid); + } // end for + + // ---- Multidirectional Approximate Riemann solver (MARS) ---- + // find the average velocity of the elem, it is an + // estimate of the Riemann velocity + + // initialize to Riemann velocity to zero + for (size_t dim = 0; dim < num_dims; dim++) + { + vel_star(dim) = 0.0; + } + + // loop over nodes and calculate an average velocity, which is + // an estimate of Riemann velocity + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + // Get node gloabl index and create view of nodal velocity + int node_gid = nodes_in_elem(elem_gid, node_lid); + + ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); + + vel_star(0) += 0.125 * vel(0); + vel_star(1) += 0.125 * vel(1); + vel_star(2) += 0.125 * vel(2); + } // end for loop over nodes + + // find shock direction and shock impedance associated with each node + + // initialize sum term in MARS to zero + for (int i = 0; i < 4; i++) + { + sum(i) = sum_gradient(i) = 0.0; + } + + double mag; // magnitude of the area normal + double mag_vel; // magnitude of velocity + + // loop over the nodes of the elem + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + // Get global node id + size_t node_gid = nodes_in_elem(elem_gid, node_lid); + + // Create view of nodal velocity + ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); + + // Get an estimate of the shock direction. + mag_vel = sqrt( (vel(0) - vel_star(0) ) * (vel(0) - vel_star(0) ) + + (vel(1) - vel_star(1) ) * (vel(1) - vel_star(1) ) + + (vel(2) - vel_star(2) ) * (vel(2) - vel_star(2) ) ); + + if (mag_vel > small) + { + // estimate of the shock direction, a unit normal + for (int dim = 0; dim < num_dims; dim++) + { + shock_dir(dim) = (vel(dim) - vel_star(dim)) / mag_vel; + } + } + else + { + // if there is no velocity change, then use the surface area + // normal as the shock direction + mag = sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) + + area_normal(node_lid, 1) * area_normal(node_lid, 1) + + area_normal(node_lid, 2) * area_normal(node_lid, 2) ); + + // estimate of the shock direction + for (int dim = 0; dim < num_dims; dim++) + { + shock_dir(dim) = area_normal(node_lid, dim) / mag; + } + } // end if mag_vel + + // cell divergence indicates compression or expansions + size_t mat_id = elem_mat_id(elem_gid); + if (div < 0) // element in compression + { + muc(node_lid) = elem_den(elem_gid) * + (material(mat_id).q1 * elem_sspd(elem_gid) + material(mat_id).q2 * mag_vel); + + muc_gradient(node_lid) = elem_den(elem_gid) / relative_element_densities(elem_gid) / num_nodes_in_elem * + (material(mat_id).q1 * elem_sspd(elem_gid) + material(mat_id).q2 * mag_vel); + } + else // element in expansion + { + muc(node_lid) = elem_den(elem_gid) * + (material(mat_id).q1ex * elem_sspd(elem_gid) + material(mat_id).q2ex * mag_vel); + muc_gradient(node_lid) = elem_den(elem_gid) / relative_element_densities(elem_gid) / num_nodes_in_elem * + (material(mat_id).q1ex * elem_sspd(elem_gid) + material(mat_id).q2ex * mag_vel); + } // end if on divergence sign + + size_t use_shock_dir = 0; + double mu_term; + double mu_term_gradient; + + // Coding to use shock direction + if (use_shock_dir == 1) + { + // this is denominator of the Riamann solver and the multiplier + // on velocity in the numerator. It filters on the shock + // direction + mu_term = muc(node_lid) * + fabs(shock_dir(0) * area_normal(node_lid, 0) + + shock_dir(1) * area_normal(node_lid, 1) + + shock_dir(2) * area_normal(node_lid, 2) ); + mu_term_gradient = muc_gradient(node_lid) * // amend if shock dir has dependence for gradient not captured here + fabs(shock_dir(0) * area_normal(node_lid, 0) + + shock_dir(1) * area_normal(node_lid, 1) + + shock_dir(2) * area_normal(node_lid, 2) ); + } + else + { + // Using a full tensoral Riemann jump relation + mu_term = muc(node_lid) + * sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) + + area_normal(node_lid, 1) * area_normal(node_lid, 1) + + area_normal(node_lid, 2) * area_normal(node_lid, 2) ); + mu_term_gradient = muc_gradient(node_lid) + * sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) + + area_normal(node_lid, 1) * area_normal(node_lid, 1) + + area_normal(node_lid, 2) * area_normal(node_lid, 2) ); + } + + sum(0) += mu_term * vel(0); + sum(1) += mu_term * vel(1); + sum(2) += mu_term * vel(2); + sum(3) += mu_term; + + sum_gradient(0) += mu_term_gradient * vel(0); + sum_gradient(1) += mu_term_gradient * vel(1); + sum_gradient(2) += mu_term_gradient * vel(2); + sum_gradient(3) += mu_term_gradient; + + muc(node_lid) = mu_term; // the impeadance time surface area is stored here + muc_gradient(node_lid) = mu_term_gradient; + } // end for node_lid loop over nodes of the elem + + // The Riemann velocity, called vel_star + if (sum(3) > fuzz) + { + for (size_t i = 0; i < num_dims; i++) + { + vel_star(i) = sum(i) / sum(3); + vel_star_gradient(i) = sum_gradient(i) / sum(3) - sum_gradient(3) * sum(i) / sum(3) / sum(3); + } + } + else + { + for (int i = 0; i < num_dims; i++) + { + vel_star(i) = 0.0; + vel_star_gradient(i) = 0.0; + } + } // end if + + // ---- Calculate the shock detector for the Riemann-solver ---- + // + // The dissipation from the Riemann problem is limited by phi + // phi = (1. - max( 0., min( 1. , r_face ) ))^n + // where + // r_face = (C* div(u_+)/div(u_z)) + // The plus denotes the cell center divergence of a neighbor. + // The solution will be first order when phi=1 and have + // zero dissipation when phi=0. + // phi = 0 highest-order solution + // phi = 1 first order solution + // + + double phi = 0.0; // the shock detector + double r_face = 1.0; // the ratio on the face + double r_min = 1.0; // the min ratio for the cell + double r_coef = 0.9; // 0.9; the coefficient on the ratio + // (1=minmod and 2=superbee) + double n_coef = 1.0; // the power on the limiting coefficient + // (1=nominal, and n_coeff > 1 oscillatory) + + // loop over the nieghboring cells + for (size_t elem_lid = 0; elem_lid < num_elems_in_elem(elem_gid); elem_lid++) + { + // Get global index for neighboring cell + size_t neighbor_gid = elems_in_elem(elem_gid, elem_lid); + + // calculate the velocity divergence in neighbor + double div_neighbor = elem_div(neighbor_gid); + + r_face = r_coef * (div_neighbor + small) / (div + small); + + // store the smallest face ratio + r_min = fmin(r_face, r_min); + } // end for elem_lid + + // calculate standard shock detector + phi = 1.0 - fmax(0.0, r_min); + phi = pow(phi, n_coef); + + // Mach number shock detector + double omega = 20.0; // 20.0; // weighting factor on Mach number + double third = 1.0 / 3.0; + double c_length = pow(vol, third); // characteristic length + double alpha = fmin(1.0, omega * (c_length * fabs(div)) / (elem_sspd(elem_gid) + fuzz) ); + + // use Mach based detector with standard shock detector + + // turn off dissipation in expansion + // alpha = fmax(-fabs(div0)/div0 * alpha, 0.0); // this should be if(div0<0) alpha=alpha else alpha=0 + + phi = alpha * phi; + + // curl limiter on Q + double phi_curl = fmin(1.0, 1.0 * fabs(div) / (mag_curl + fuzz)); // disable Q when vorticity is high + // phi = phi_curl*phi; + phi = 1; + + // ---- Calculate the Riemann force on each node ---- + + // loop over the each node in the elem + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + size_t corner_lid = node_lid; + + // Get corner gid + size_t corner_gid = corners_in_elem(elem_gid, corner_lid); + + // Get node gid + size_t node_gid = nodes_in_elem(elem_gid, node_lid); + + // loop over dimension + for (int dim = 0; dim < num_dims; dim++) + { + corner_vector_storage(corner_gid, dim) = + area_normal(node_lid, 0) * tau_gradient(0, dim) + + area_normal(node_lid, 1) * tau_gradient(1, dim) + + area_normal(node_lid, 2) * tau_gradient(2, dim) + + phi * muc_gradient(node_lid) * (vel_star(dim) - node_vel(rk_level, node_gid, dim)) + + phi * muc(node_lid) * (vel_star_gradient(dim)); + } // end loop over dimension + } // end for loop over nodes in elem + + // --- Update Stress --- + // calculate the new stress at the next rk level, if it is a hypo model + + size_t mat_id = elem_mat_id(elem_gid); + } + // }); // end parallel for loop over elements + + /* + //accumulate node values from corner storage + force_gradient_design->putScalar(0); + + vec_array force_gradient_design_view = force_gradient_design->getLocalView (Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { + size_t corner_id; + for(int icorner=0; icorner < num_corners_in_node(node_id); icorner++){ + corner_id = corners_in_node(node_id,icorner); + force_gradient_design_view(node_id,0) += corner_vector_storage(corner_id, 0); + force_gradient_design_view(node_id,1) += corner_vector_storage(corner_id, 1); + force_gradient_design_view(node_id,2) += corner_vector_storage(corner_id, 2); + } + }); // end parallel for + Kokkos::fence(); + */ + + return; +} // end of routine diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/force_sgh.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_sgh.cpp similarity index 54% rename from src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/force_sgh.cpp rename to src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_sgh.cpp index 2ca9ef946..e4046cddf 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/force_sgh.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_sgh.cpp @@ -1,101 +1,94 @@ - #include "mesh.h" #include "state.h" #include "Simulation_Parameters/Simulation_Parameters_Explicit.h" #include "Simulation_Parameters/FEA_Module/SGH_Parameters.h" #include "FEA_Module_SGH.h" -// ----------------------------------------------------------------------------- -// This function calculates the corner forces and the evolves stress (hypo) -//------------------------------------------------------------------------------ -void FEA_Module_SGH::get_force_sgh(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_pres, - DViewCArrayKokkos &elem_stress, - const DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &elem_mat_id, - DViewCArrayKokkos &corner_force, - const double rk_alpha, - const size_t cycle - ){ - - const_vec_array initial_node_coords = initial_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - + +void FEA_Module_SGH::get_force_sgh(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_pres, + DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + DViewCArrayKokkos& corner_force, + const double rk_alpha, + const size_t cycle + ) +{ + const_vec_array initial_node_coords = initial_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + // check to see if any material model will be run on the host bool any_host_material_model_run = false; - for(int imat = 0; imat < material.size(); imat++){ - if(material.host(imat).strength_run_location == RUN_LOCATION::host){ + for (int imat = 0; imat < material.size(); imat++) + { + if (material.host(imat).strength_run_location == RUN_LOCATION::host) + { any_host_material_model_run = true; } - } - + } + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - + // --- calculate the forces acting on the nodes from the element --- - FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { - + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { const size_t num_dims = 3; const size_t num_nodes_in_elem = 8; - + // total Cauchy stress double tau_array[9]; - + // corner area normals double area_normal_array[24]; - + // estimate of shock direction double shock_dir_array[3]; - + // the sums in the Riemann solver double sum_array[4]; - + // corner shock impeadance x |corner area normal dot shock_dir| double muc_array[8]; - + // Riemann velocity double vel_star_array[3]; - + // velocity gradient double vel_grad_array[9]; - + // --- Create views of arrays to aid the force calculation --- - - ViewCArrayKokkos tau(tau_array, num_dims, num_dims); - ViewCArrayKokkos area_normal(area_normal_array, num_nodes_in_elem, num_dims); - ViewCArrayKokkos shock_dir(shock_dir_array, num_dims); - ViewCArrayKokkos sum(sum_array, 4); - ViewCArrayKokkos muc(muc_array, num_nodes_in_elem); - ViewCArrayKokkos vel_star(vel_star_array, num_dims); - ViewCArrayKokkos vel_grad(vel_grad_array, num_dims, num_dims); - - + + ViewCArrayKokkos tau(tau_array, num_dims, num_dims); + ViewCArrayKokkos area_normal(area_normal_array, num_nodes_in_elem, num_dims); + ViewCArrayKokkos shock_dir(shock_dir_array, num_dims); + ViewCArrayKokkos sum(sum_array, 4); + ViewCArrayKokkos muc(muc_array, num_nodes_in_elem); + ViewCArrayKokkos vel_star(vel_star_array, num_dims); + ViewCArrayKokkos vel_grad(vel_grad_array, num_dims, num_dims); + // --- abviatations of variables --- - + // element volume double vol = elem_vol(elem_gid); - + // create a view of the stress_matrix - ViewCArrayKokkos stress(&elem_stress(rk_level, elem_gid, 0,0), 3, 3); - - + ViewCArrayKokkos stress(&elem_stress(rk_level, elem_gid, 0, 0), 3, 3); + // cut out the node_gids for this element - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 8); - - - + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 8); + // get the B matrix which are the OUTWARD corner area normals get_bmatrix(area_normal, elem_gid, node_coords, elem_node_gids, rk_level); - - + // --- Calculate the velocity gradient --- get_velgrad(vel_grad, elem_node_gids, @@ -104,84 +97,86 @@ void FEA_Module_SGH::get_force_sgh(const DCArrayKokkos &material, vol, elem_gid, rk_level); - - // save vel_grad in elem_vel_grad - for (size_t i = 0; i < 3; i++) { - for (size_t j = 0; j < 3; j++) { - elem_vel_grad(elem_gid,i,j) = vel_grad(i,j); + + // save vel_grad in elem_vel_grad + for (size_t i = 0; i < 3; i++) + { + for (size_t j = 0; j < 3; j++) + { + elem_vel_grad(elem_gid, i, j) = vel_grad(i, j); } } - + // the -1 is for the inward surface area normal, - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - for (size_t dim = 0; dim < num_dims; dim++){ - area_normal(node_lid, dim) = (-1.0)*area_normal(node_lid,dim); + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + for (size_t dim = 0; dim < num_dims; dim++) + { + area_normal(node_lid, dim) = (-1.0) * area_normal(node_lid, dim); } // end for } // end for - - - + double div = elem_div(elem_gid); - - + // vel = [u,v,w] // [du/dx, du/dy, du/dz] // vel_grad = [dv/dx, dv/dy, dv/dz] // [dw/dx, dw/dy, dw/dz] double curl[3]; - curl[0] = vel_grad(2,1) - vel_grad(1,2); // dw/dy - dv/dz - curl[1] = vel_grad(0,2) - vel_grad(2,0); // du/dz - dw/dx - curl[2] = vel_grad(1,0) - vel_grad(0,1); // dv/dx - du/dy - - double mag_curl = sqrt(curl[0]*curl[0] + curl[1]*curl[1] + curl[2]*curl[2]); - - + curl[0] = vel_grad(2, 1) - vel_grad(1, 2); // dw/dy - dv/dz + curl[1] = vel_grad(0, 2) - vel_grad(2, 0); // du/dz - dw/dx + curl[2] = vel_grad(1, 0) - vel_grad(0, 1); // dv/dx - du/dy + + double mag_curl = sqrt(curl[0] * curl[0] + curl[1] * curl[1] + curl[2] * curl[2]); + // --- Calculate the Cauchy stress --- - for (size_t i = 0; i < 3; i++){ - for (size_t j = 0; j < 3; j++){ - tau(i, j) = stress(i,j); + for (size_t i = 0; i < 3; i++) + { + for (size_t j = 0; j < 3; j++) + { + tau(i, j) = stress(i, j); // artificial viscosity can be added here to tau } // end for - } //end for + } // end for // add the pressure - if(elem_pres(elem_gid)!=0){ - for (int i = 0; i < num_dims; i++){ + if (elem_pres(elem_gid) != 0) + { + for (int i = 0; i < num_dims; i++) + { tau(i, i) -= elem_pres(elem_gid); } // end for } - - - // ---- Multidirectional Approximate Riemann solver (MARS) ---- // find the average velocity of the elem, it is an // estimate of the Riemann velocity - + // initialize to Riemann velocity to zero - for (size_t dim = 0; dim < num_dims; dim++){ + for (size_t dim = 0; dim < num_dims; dim++) + { vel_star(dim) = 0.0; } // loop over nodes and calculate an average velocity, which is // an estimate of Riemann velocity - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { // Get node gloabl index and create view of nodal velocity int node_gid = nodes_in_elem(elem_gid, node_lid); - - ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); - - vel_star(0) += 0.125*vel(0); - vel_star(1) += 0.125*vel(1); - vel_star(2) += 0.125*vel(2); - + + ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); + + vel_star(0) += 0.125 * vel(0); + vel_star(1) += 0.125 * vel(1); + vel_star(2) += 0.125 * vel(2); } // end for loop over nodes // find shock direction and shock impedance associated with each node - + // initialize sum term in MARS to zero - for (int i = 0; i < 4; i++){ + for (int i = 0; i < 4; i++) + { sum(i) = 0.0; } @@ -190,102 +185,101 @@ void FEA_Module_SGH::get_force_sgh(const DCArrayKokkos &material, size_t mat_id = elem_mat_id(elem_gid); // loop over the nodes of the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { - + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { // Get global node id size_t node_gid = nodes_in_elem(elem_gid, node_lid); // Create view of nodal velocity - ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); + ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); // Get an estimate of the shock direction. - mag_vel = sqrt( (vel(0) - vel_star(0) )*(vel(0) - vel_star(0) ) - + (vel(1) - vel_star(1) )*(vel(1) - vel_star(1) ) - + (vel(2) - vel_star(2) )*(vel(2) - vel_star(2) ) ); + mag_vel = sqrt( (vel(0) - vel_star(0) ) * (vel(0) - vel_star(0) ) + + (vel(1) - vel_star(1) ) * (vel(1) - vel_star(1) ) + + (vel(2) - vel_star(2) ) * (vel(2) - vel_star(2) ) ); - - if (mag_vel > small) { - + if (mag_vel > small) + { // estimate of the shock direction, a unit normal - for (int dim = 0; dim < num_dims; dim++){ + for (int dim = 0; dim < num_dims; dim++) + { shock_dir(dim) = (vel(dim) - vel_star(dim)) / mag_vel; } } - - else { - + else + { // if there is no velocity change, then use the surface area // normal as the shock direction - mag = sqrt( area_normal(node_lid, 0)*area_normal(node_lid, 0) - + area_normal(node_lid, 1)*area_normal(node_lid, 1) - + area_normal(node_lid, 2)*area_normal(node_lid, 2) ); - + mag = sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) + + area_normal(node_lid, 1) * area_normal(node_lid, 1) + + area_normal(node_lid, 2) * area_normal(node_lid, 2) ); + // estimate of the shock direction - for (int dim = 0; dim < num_dims; dim++){ - shock_dir(dim) = area_normal(node_lid, dim)/mag; + for (int dim = 0; dim < num_dims; dim++) + { + shock_dir(dim) = area_normal(node_lid, dim) / mag; } - } // end if mag_vel - // cell divergence indicates compression or expansions - if (div < 0){ // element in compression + if (div < 0) // element in compression + { muc(node_lid) = elem_den(elem_gid) * - (material(mat_id).q1*elem_sspd(elem_gid) + material(mat_id).q2*mag_vel); + (material(mat_id).q1 * elem_sspd(elem_gid) + material(mat_id).q2 * mag_vel); } - else { // element in expansion + else // element in expansion + { muc(node_lid) = elem_den(elem_gid) * - (material(mat_id).q1ex*elem_sspd(elem_gid) + material(mat_id).q2ex*mag_vel); + (material(mat_id).q1ex * elem_sspd(elem_gid) + material(mat_id).q2ex * mag_vel); } // end if on divergence sign - size_t use_shock_dir = 0; double mu_term; - + // Coding to use shock direction - if (use_shock_dir == 1){ + if (use_shock_dir == 1) + { // this is denominator of the Riamann solver and the multiplier // on velocity in the numerator. It filters on the shock // direction - mu_term = muc(node_lid)* - fabs( shock_dir(0)*area_normal(node_lid,0) - + shock_dir(1)*area_normal(node_lid,1) - + shock_dir(2)*area_normal(node_lid,2) ); + mu_term = muc(node_lid) * + fabs(shock_dir(0) * area_normal(node_lid, 0) + + shock_dir(1) * area_normal(node_lid, 1) + + shock_dir(2) * area_normal(node_lid, 2) ); } - else { - // Using a full tensoral Riemann jump relation - mu_term = muc(node_lid) - * sqrt( area_normal(node_lid, 0)*area_normal(node_lid, 0) - + area_normal(node_lid, 1)*area_normal(node_lid, 1) - + area_normal(node_lid, 2)*area_normal(node_lid, 2) ); + else + { + // Using a full tensoral Riemann jump relation + mu_term = muc(node_lid) + * sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) + + area_normal(node_lid, 1) * area_normal(node_lid, 1) + + area_normal(node_lid, 2) * area_normal(node_lid, 2) ); } - - sum(0) += mu_term*vel(0); - sum(1) += mu_term*vel(1); - sum(2) += mu_term*vel(2); + + sum(0) += mu_term * vel(0); + sum(1) += mu_term * vel(1); + sum(2) += mu_term * vel(2); sum(3) += mu_term; muc(node_lid) = mu_term; // the impeadance time surface area is stored here - } // end for node_lid loop over nodes of the elem - - - // The Riemann velocity, called vel_star - if (sum(3) > fuzz) { - for (size_t i = 0; i < num_dims; i++) { - vel_star(i) = sum(i)/sum(3); + if (sum(3) > fuzz) + { + for (size_t i = 0; i < num_dims; i++) + { + vel_star(i) = sum(i) / sum(3); } } - else { - for (int i = 0; i < num_dims; i++){ + else + { + for (int i = 0; i < num_dims; i++) + { vel_star(i) = 0.0; } } // end if - - // ---- Calculate the shock detector for the Riemann-solver ---- // // The dissipation from the Riemann problem is limited by phi @@ -298,7 +292,7 @@ void FEA_Module_SGH::get_force_sgh(const DCArrayKokkos &material, // phi = 0 highest-order solution // phi = 1 first order solution // - + double phi = 0.0; // the shock detector double r_face = 1.0; // the ratio on the face double r_min = 1.0; // the min ratio for the cell @@ -307,88 +301,84 @@ void FEA_Module_SGH::get_force_sgh(const DCArrayKokkos &material, double n_coef = 1.0; // the power on the limiting coefficient // (1=nominal, and n_coeff > 1 oscillatory) - // loop over the nieghboring cells - for (size_t elem_lid = 0; elem_lid < num_elems_in_elem(elem_gid); elem_lid++){ - + for (size_t elem_lid = 0; elem_lid < num_elems_in_elem(elem_gid); elem_lid++) + { // Get global index for neighboring cell size_t neighbor_gid = elems_in_elem(elem_gid, elem_lid); - + // calculate the velocity divergence in neighbor double div_neighbor = elem_div(neighbor_gid); - r_face = r_coef*(div_neighbor + small)/(div + small); + r_face = r_coef * (div_neighbor + small) / (div + small); // store the smallest face ratio r_min = fmin(r_face, r_min); - } // end for elem_lid - // calculate standard shock detector phi = 1.0 - fmax(0.0, r_min); phi = pow(phi, n_coef); // Mach number shock detector - double omega = 20.0; // weighting factor on Mach number - double third = 1.0/3.0; + double omega = 20.0; // weighting factor on Mach number + double third = 1.0 / 3.0; double c_length = pow(vol, third); // characteristic length - double alpha = fmin(1.0, omega * (c_length * fabs(div))/(elem_sspd(elem_gid) + fuzz) ); + double alpha = fmin(1.0, omega * (c_length * fabs(div)) / (elem_sspd(elem_gid) + fuzz) ); // curl limiter on Q double omega_curl = 1.0; // increase this to increase robustness, but as it increases, additional dissipation will be introduce, blocking bending - double phi_curl = fmin(1.0, omega_curl*fabs(div)/(mag_curl + fuzz)); // disable Q when vorticity is high - + double phi_curl = fmin(1.0, omega_curl * fabs(div) / (mag_curl + fuzz)); // disable Q when vorticity is high + phi = 1.0; // for the future case of using a slope limiter approach - phi = fmin(phi_curl*phi, alpha*phi);// if noise arrises in simulation on really smooth flows, then try something like - phi = fmax(phi,0.001); // ensuring a very small amount of dissipation for stability and robustness + phi = fmin(phi_curl * phi, alpha * phi); // if noise arrises in simulation on really smooth flows, then try something like + phi = fmax(phi, 0.001); // ensuring a very small amount of dissipation for stability and robustness - if(material(mat_id).maximum_limiter){ - phi=1; + if (material(mat_id).maximum_limiter) + { + phi = 1; } - + // ---- Calculate the Riemann force on each node ---- // loop over the each node in the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { - + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { size_t corner_lid = node_lid; // Get corner gid size_t corner_gid = corners_in_elem(elem_gid, corner_lid); - + // Get node gid size_t node_gid = nodes_in_elem(elem_gid, node_lid); - + // loop over dimension - for (int dim = 0; dim < num_dims; dim++){ + for (int dim = 0; dim < num_dims; dim++) + { corner_force(corner_gid, dim) = - area_normal(node_lid, 0)*tau(0, dim) - + area_normal(node_lid, 1)*tau(1, dim) - + area_normal(node_lid, 2)*tau(2, dim) - + phi*muc(node_lid)*(vel_star(dim) - node_vel(rk_level, node_gid, dim)); - //test clause - //corner_force(corner_gid, dim) = -0.00001*node_vel(rk_level, node_gid, dim); - //corner_force(corner_gid, dim) = 0.0001*relative_element_densities(elem_gid)-0.00001*node_vel(rk_level, node_gid, dim); - + area_normal(node_lid, 0) * tau(0, dim) + + area_normal(node_lid, 1) * tau(1, dim) + + area_normal(node_lid, 2) * tau(2, dim) + + phi * muc(node_lid) * (vel_star(dim) - node_vel(rk_level, node_gid, dim)); + // test clause + // corner_force(corner_gid, dim) = -0.00001*node_vel(rk_level, node_gid, dim); + // corner_force(corner_gid, dim) = 0.0001*relative_element_densities(elem_gid)-0.00001*node_vel(rk_level, node_gid, dim); } // end loop over dimension - } // end for loop over nodes in elem - - + // --- Update Stress --- // calculate the new stress at the next rk level, if it is a hypo model - - // hypo elastic plastic model - if(material(mat_id).strength_type == STRENGTH_TYPE::hypo){ - - if(material(mat_id).strength_run_location == RUN_LOCATION::device){ + // hypo elastic plastic model + if (material(mat_id).strength_type == STRENGTH_TYPE::hypo) + { + if (material(mat_id).strength_run_location == RUN_LOCATION::device) + { // cut out the node_gids for this element - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 8); + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 8); StrengthParent* strength_model = elem_strength(elem_gid).model; - + // --- call strength model --- strength_model->calc_stress(elem_pres, elem_stress, @@ -399,52 +389,49 @@ void FEA_Module_SGH::get_force_sgh(const DCArrayKokkos &material, elem_user_output_vars, elem_sspd, elem_den(elem_gid), - elem_sie(rk_level,elem_gid), + elem_sie(rk_level, elem_gid), vel_grad, elem_node_gids, node_coords, node_vel, elem_vol(elem_gid), - dt, + dt, rk_alpha, cycle, rk_level); } // end logical for strength run location - } // end logical on hypo strength model - - }); // end parallel for loop over elements - - if (any_host_material_model_run == true) { + if (any_host_material_model_run == true) + { // update host elem_vel_grad.update_host(); // below host updates are commented out to save time because they are not used for // the current user model. if a user model uses any of them, please uncomment it - //elem_pres.update_host(); - //elem_den.upsate_host(); - //elem_sie.update_host(); - //node_coords.update_host(); - //node_vel.update_host(); - //elem_vol.update_host(); + // elem_pres.update_host(); + // elem_den.upsate_host(); + // elem_sie.update_host(); + // node_coords.update_host(); + // node_vel.update_host(); + // elem_vol.update_host(); // calling user strength model on host - for (size_t elem_gid = 0; elem_gid < mesh.num_elems; elem_gid++) { - + for (size_t elem_gid = 0; elem_gid < mesh.num_elems; elem_gid++) + { const size_t num_dims = 3; - size_t mat_id = elem_mat_id.host(elem_gid); + size_t mat_id = elem_mat_id.host(elem_gid); // hypo elastic plastic model - if(material.host(mat_id).strength_type == STRENGTH_TYPE::hypo){ - - if(material.host(mat_id).strength_run_location == RUN_LOCATION::host){ - + if (material.host(mat_id).strength_type == STRENGTH_TYPE::hypo) + { + if (material.host(mat_id).strength_run_location == RUN_LOCATION::host) + { // cut out the node_gids for this element - ViewCArrayKokkos elem_node_gids(&nodes_in_elem.host(elem_gid, 0), 8); + ViewCArrayKokkos elem_node_gids(&nodes_in_elem.host(elem_gid, 0), 8); - // cut out vel_grad - ViewCArrayKokkos vel_grad(&elem_vel_grad.host(elem_gid,0,0), num_dims, num_dims); + // cut out vel_grad + ViewCArrayKokkos vel_grad(&elem_vel_grad.host(elem_gid, 0, 0), num_dims, num_dims); StrengthParent* strength_model = elem_strength.host(elem_gid).model; @@ -458,7 +445,7 @@ void FEA_Module_SGH::get_force_sgh(const DCArrayKokkos &material, elem_user_output_vars, elem_sspd, elem_den.host(elem_gid), - elem_sie.host(rk_level,elem_gid), + elem_sie.host(rk_level, elem_gid), vel_grad, elem_node_gids, node_coords, @@ -468,90 +455,81 @@ void FEA_Module_SGH::get_force_sgh(const DCArrayKokkos &material, rk_alpha, cycle, rk_level); - } // end logical for strength run location - } // end logical on hypo strength model - - } // end for loop over elements + } // end for loop over elements // update device elem_stress.update_device(); - } // end for if host_material_model_run == true return; - } // end of routine - // ----------------------------------------------------------------------------- // This function calculates the corner forces and the evolves stress (hypo) -//------------------------------------------------------------------------------ -void FEA_Module_SGH::get_force_sgh2D(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &elem_mat_id, - DViewCArrayKokkos &corner_force, - const double rk_alpha, - const size_t cycle - ){ - +// ------------------------------------------------------------------------------ +void FEA_Module_SGH::get_force_sgh2D(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_pres, + const DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + DViewCArrayKokkos& corner_force, + const double rk_alpha, + const size_t cycle + ) +{ const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; // --- calculate the forces acting on the nodes from the element --- - FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { - + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { const size_t num_dims = 2; const size_t num_nodes_in_elem = 4; - + // total Cauchy stress double tau_array[9]; - + // corner area normals double area_normal_array[8]; // 4 corners and 2 directions - + // estimate of shock direction double shock_dir_array[2]; - + // the sums in the Riemann solver double sum_array[4]; - + // corner shock impeadance x |corner area normal dot shock_dir| double muc_array[4]; - + // Riemann velocity double vel_star_array[2]; - + // velocity gradient double vel_grad_array[9]; - + // --- Create views of arrays to aid the force calculation --- - - ViewCArrayKokkos tau(tau_array, 3, 3); - ViewCArrayKokkos area_normal(area_normal_array, num_nodes_in_elem, num_dims); - ViewCArrayKokkos shock_dir(shock_dir_array, num_dims); - ViewCArrayKokkos sum(sum_array, 4); - ViewCArrayKokkos muc(muc_array, num_nodes_in_elem); - ViewCArrayKokkos vel_star(vel_star_array, num_dims); - ViewCArrayKokkos vel_grad(vel_grad_array, 3, 3); - - + + ViewCArrayKokkos tau(tau_array, 3, 3); + ViewCArrayKokkos area_normal(area_normal_array, num_nodes_in_elem, num_dims); + ViewCArrayKokkos shock_dir(shock_dir_array, num_dims); + ViewCArrayKokkos sum(sum_array, 4); + ViewCArrayKokkos muc(muc_array, num_nodes_in_elem); + ViewCArrayKokkos vel_star(vel_star_array, num_dims); + ViewCArrayKokkos vel_grad(vel_grad_array, 3, 3); + // create a view of the stress_matrix - ViewCArrayKokkos stress(&elem_stress(rk_level, elem_gid, 0,0), 3, 3); - - + ViewCArrayKokkos stress(&elem_stress(rk_level, elem_gid, 0, 0), 3, 3); + // cut out the node_gids for this element - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 4); - + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 4); + // get the B matrix which are the OUTWARD corner area normals get_bmatrix2D(area_normal, elem_gid, @@ -559,11 +537,10 @@ void FEA_Module_SGH::get_force_sgh2D(const DCArrayKokkos &material, elem_node_gids, rk_level); // NOTE: I added a minux in bmatrix2D, it should be outward pointing now? - + // facial area of the element double elem_area = get_area_quad(elem_gid, node_coords, elem_node_gids, rk_level); - - + // --- Calculate the velocity gradient --- get_velgrad2D(vel_grad, elem_node_gids, @@ -573,71 +550,70 @@ void FEA_Module_SGH::get_force_sgh2D(const DCArrayKokkos &material, elem_area, elem_gid, rk_level); - - + // the -1 is for the inward surface area normal, - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - for (size_t dim = 0; dim < num_dims; dim++){ - area_normal(node_lid, dim) = (-1.0)*area_normal(node_lid,dim); + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + for (size_t dim = 0; dim < num_dims; dim++) + { + area_normal(node_lid, dim) = (-1.0) * area_normal(node_lid, dim); } // end for } // end for - - - + double div = elem_div(elem_gid); - - + // vel = [u,v] // [du/dx, du/dy] // vel_grad = [dv/dx, dv/dy] double curl; - curl = vel_grad(1,0) - vel_grad(0,1); // dv/dx - du/dy - + curl = vel_grad(1, 0) - vel_grad(0, 1); // dv/dx - du/dy + double mag_curl = curl; - - + // --- Calculate the Cauchy stress --- - for (size_t i = 0; i < 3; i++){ - for (size_t j = 0; j < 3; j++){ - tau(i, j) = stress(i,j); + for (size_t i = 0; i < 3; i++) + { + for (size_t j = 0; j < 3; j++) + { + tau(i, j) = stress(i, j); // artificial viscosity can be added here to tau } // end for - } //end for + } // end for // add the pressure - for (int i = 0; i < 3; i++){ + for (int i = 0; i < 3; i++) + { tau(i, i) -= elem_pres(elem_gid); } // end for - - // ---- Multidirectional Approximate Riemann solver (MARS) ---- // find the average velocity of the elem, it is an // estimate of the Riemann velocity - + // initialize to Riemann velocity to zero - for (size_t dim = 0; dim < num_dims; dim++){ + for (size_t dim = 0; dim < num_dims; dim++) + { vel_star(dim) = 0.0; } // loop over nodes and calculate an average velocity, which is // an estimate of Riemann velocity - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { // Get node gloabl index and create view of nodal velocity int node_gid = nodes_in_elem(elem_gid, node_lid); - - ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); - - vel_star(0) += 0.25*vel(0); - vel_star(1) += 0.25*vel(1); - + + ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); + + vel_star(0) += 0.25 * vel(0); + vel_star(1) += 0.25 * vel(1); } // end for loop over nodes // find shock direction and shock impedance associated with each node - + // initialize sum term in MARS to zero - for (int i = 0; i < 4; i++){ + for (int i = 0; i < 4; i++) + { sum(i) = 0.0; } @@ -645,98 +621,97 @@ void FEA_Module_SGH::get_force_sgh2D(const DCArrayKokkos &material, double mag_vel; // magnitude of velocity // loop over the nodes of the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { - + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { // Get global node id size_t node_gid = nodes_in_elem(elem_gid, node_lid); // Create view of nodal velocity - ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); + ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); // Get an estimate of the shock direction. - mag_vel = sqrt( (vel(0) - vel_star(0) )*(vel(0) - vel_star(0) ) - + (vel(1) - vel_star(1) )*(vel(1) - vel_star(1) ) ); + mag_vel = sqrt( (vel(0) - vel_star(0) ) * (vel(0) - vel_star(0) ) + + (vel(1) - vel_star(1) ) * (vel(1) - vel_star(1) ) ); - - if (mag_vel > small) { - + if (mag_vel > small) + { // estimate of the shock direction, a unit normal - for (int dim = 0; dim < num_dims; dim++){ + for (int dim = 0; dim < num_dims; dim++) + { shock_dir(dim) = (vel(dim) - vel_star(dim)) / mag_vel; } } - - else { - + else + { // if there is no velocity change, then use the surface area // normal as the shock direction - mag = sqrt( area_normal(node_lid, 0)*area_normal(node_lid, 0) - + area_normal(node_lid, 1)*area_normal(node_lid, 1) ); - + mag = sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) + + area_normal(node_lid, 1) * area_normal(node_lid, 1) ); + // estimate of the shock direction - for (int dim = 0; dim < num_dims; dim++){ - shock_dir(dim) = area_normal(node_lid, dim)/mag; + for (int dim = 0; dim < num_dims; dim++) + { + shock_dir(dim) = area_normal(node_lid, dim) / mag; } - } // end if mag_vel - // cell divergence indicates compression or expansions size_t mat_id = elem_mat_id(elem_gid); - if (div < 0){ // element in compression + if (div < 0) // element in compression + { muc(node_lid) = elem_den(elem_gid) * - (material(mat_id).q1*elem_sspd(elem_gid) + material(mat_id).q2*mag_vel); + (material(mat_id).q1 * elem_sspd(elem_gid) + material(mat_id).q2 * mag_vel); } - else { // element in expansion + else // element in expansion + { muc(node_lid) = elem_den(elem_gid) * - (material(mat_id).q1ex*elem_sspd(elem_gid) + material(mat_id).q2ex*mag_vel); + (material(mat_id).q1ex * elem_sspd(elem_gid) + material(mat_id).q2ex * mag_vel); } // end if on divergence sign - size_t use_shock_dir = 0; double mu_term; - + // Coding to use shock direction - if (use_shock_dir == 1){ + if (use_shock_dir == 1) + { // this is denominator of the Riamann solver and the multiplier // on velocity in the numerator. It filters on the shock // direction - mu_term = muc(node_lid)* - fabs( shock_dir(0)*area_normal(0) - + shock_dir(1)*area_normal(1) ); + mu_term = muc(node_lid) * + fabs(shock_dir(0) * area_normal(0) + + shock_dir(1) * area_normal(1) ); } - else { - // Using a full tensoral Riemann jump relation - mu_term = muc(node_lid) - * sqrt( area_normal(node_lid, 0)*area_normal(node_lid, 0) - + area_normal(node_lid, 1)*area_normal(node_lid, 1) ); + else + { + // Using a full tensoral Riemann jump relation + mu_term = muc(node_lid) + * sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) + + area_normal(node_lid, 1) * area_normal(node_lid, 1) ); } - - sum(0) += mu_term*vel(0); - sum(1) += mu_term*vel(1); + + sum(0) += mu_term * vel(0); + sum(1) += mu_term * vel(1); sum(3) += mu_term; muc(node_lid) = mu_term; // the impeadance time surface area is stored here - } // end for node_lid loop over nodes of the elem - - - // The Riemann velocity, called vel_star - if (sum(3) > fuzz) { - for (size_t i = 0; i < num_dims; i++) { - vel_star(i) = sum(i)/sum(3); + if (sum(3) > fuzz) + { + for (size_t i = 0; i < num_dims; i++) + { + vel_star(i) = sum(i) / sum(3); } } - else { - for (int i = 0; i < num_dims; i++){ + else + { + for (int i = 0; i < num_dims; i++) + { vel_star(i) = 0.0; } } // end if - - - + // ---- Calculate the shock detector for the Riemann-solver ---- // // The dissipation from the Riemann problem is limited by phi @@ -749,7 +724,7 @@ void FEA_Module_SGH::get_force_sgh2D(const DCArrayKokkos &material, // phi = 0 highest-order solution // phi = 1 first order solution // - + double phi = 0.0; // the shock detector double r_face = 1.0; // the ratio on the face double r_min = 1.0; // the min ratio for the cell @@ -758,95 +733,88 @@ void FEA_Module_SGH::get_force_sgh2D(const DCArrayKokkos &material, double n_coef = 1.0; // the power on the limiting coefficient // (1=nominal, and n_coeff > 1 oscillatory) - // loop over the nieghboring cells - for (size_t elem_lid = 0; elem_lid < num_elems_in_elem(elem_gid); elem_lid++){ - + for (size_t elem_lid = 0; elem_lid < num_elems_in_elem(elem_gid); elem_lid++) + { // Get global index for neighboring cell size_t neighbor_gid = elems_in_elem(elem_gid, elem_lid); - + // calculate the velocity divergence in neighbor double div_neighbor = elem_div(neighbor_gid); - r_face = r_coef*(div_neighbor + small)/(div + small); + r_face = r_coef * (div_neighbor + small) / (div + small); // store the smallest face ratio r_min = fmin(r_face, r_min); - } // end for elem_lid - // calculate standard shock detector phi = 1.0 - fmax(0.0, r_min); phi = pow(phi, n_coef); // Mach number shock detector - double omega = 20.0;//20.0; // weighting factor on Mach number + double omega = 20.0; // 20.0; // weighting factor on Mach number double c_length = sqrt(elem_area); // characteristic length - double alpha = fmin(1.0, omega * (c_length * fabs(div))/(elem_sspd(elem_gid) + fuzz) ); - + double alpha = fmin(1.0, omega * (c_length * fabs(div)) / (elem_sspd(elem_gid) + fuzz) ); + // use Mach based detector with standard shock detector // turn off dissipation in expansion - //alpha = fmax(-fabs(div0)/div0 * alpha, 0.0); // this should be if(div0<0) alpha=alpha else alpha=0 - - phi = alpha*phi; - + // alpha = fmax(-fabs(div0)/div0 * alpha, 0.0); // this should be if(div0<0) alpha=alpha else alpha=0 + + phi = alpha * phi; + // curl limiter on Q - double phi_curl = fmin(1.0, 4.0*fabs(div)/(mag_curl + fuzz)); // disable Q when vorticity is high - //phi = phi_curl*phi; + double phi_curl = fmin(1.0, 4.0 * fabs(div) / (mag_curl + fuzz)); // disable Q when vorticity is high + // phi = phi_curl*phi; phi = 1.0; // WARNING WARNING WARNING // ---- Calculate the Riemann force on each node ---- // loop over the each node in the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { - + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { size_t corner_lid = node_lid; // Get corner gid size_t corner_gid = corners_in_elem(elem_gid, corner_lid); - + // Get node gid size_t node_gid = nodes_in_elem(elem_gid, node_lid); - - // loop over dimension - for (int dim = 0; dim < num_dims; dim++){ + // loop over dimension + for (int dim = 0; dim < num_dims; dim++) + { corner_force(corner_gid, dim) = - area_normal(node_lid, 0)*tau(0, dim) - + area_normal(node_lid, 1)*tau(1, dim) - + phi*muc(node_lid)*(vel_star(dim) - node_vel(rk_level, node_gid, dim)); - + area_normal(node_lid, 0) * tau(0, dim) + + area_normal(node_lid, 1) * tau(1, dim) + + phi * muc(node_lid) * (vel_star(dim) - node_vel(rk_level, node_gid, dim)); } // end loop over dimension - - + // ---- add hoop stress terms ---- - - double node_radius = node_coords(rk_level,node_gid,1); - - if(node_radius>1e-14){ + + double node_radius = node_coords(rk_level, node_gid, 1); + + if (node_radius > 1e-14) + { // sigma_RZ / R_p - corner_force(corner_gid, 0) += tau(1,0)*elem_area*0.25/node_radius; - + corner_force(corner_gid, 0) += tau(1, 0) * elem_area * 0.25 / node_radius; + // (sigma_RR - sigma_theta) / R_p - corner_force(corner_gid, 1) += (tau(1,1) - tau(2,2))*elem_area*0.25/node_radius; + corner_force(corner_gid, 1) += (tau(1, 1) - tau(2, 2)) * elem_area * 0.25 / node_radius; } // end if radius >0 - } // end for loop over nodes in elem - - - + // --- Update Stress --- // calculate the new stress at the next rk level, if it is a hypo model - + size_t mat_id = elem_mat_id(elem_gid); - - // hypo elastic plastic model - if(material(mat_id).strength_type == STRENGTH_TYPE::hypo){ + // hypo elastic plastic model + if (material(mat_id).strength_type == STRENGTH_TYPE::hypo) + { // cut out the node_gids for this element - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 4); + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 4); StrengthParent* strength_model = elem_strength(elem_gid).model; @@ -869,45 +837,40 @@ void FEA_Module_SGH::get_force_sgh2D(const DCArrayKokkos &material, dt, rk_alpha, cycle, - rk_level); - + rk_level); } // end logical on hypo strength model - - }); // end parallel for loop over elements - return; - } // end of routine for 2D force and stress update // ----------------------------------------------------------------------------- // This function calculates the corner forces and the evolves stress (hypo) -//------------------------------------------------------------------------------ -void FEA_Module_SGH::applied_forces(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &node_mass, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &elem_mat_id, - DViewCArrayKokkos &corner_force, - const double rk_alpha, - const size_t cycle - ){ - - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - const size_t num_dim = mesh.num_dims; - const_vec_array all_initial_node_coords = all_initial_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - const size_t num_lcs = module_params->loading.size(); - - const DCArrayKokkos mat_fill = simparam->mat_fill; - const DCArrayKokkos loading = module_params->loading; - - //debug check - //std::cout << "NUMBER OF LOADING CONDITIONS: " << num_lcs << std::endl; +// ------------------------------------------------------------------------------ +void FEA_Module_SGH::applied_forces(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_mass, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + DViewCArrayKokkos& corner_force, + const double rk_alpha, + const size_t cycle + ) +{ + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + const size_t num_dim = mesh.num_dims; + const_vec_array all_initial_node_coords = all_initial_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + const size_t num_lcs = module_params->loading.size(); + + const DCArrayKokkos mat_fill = simparam->mat_fill; + const DCArrayKokkos loading = module_params->loading; + + // debug check + // std::cout << "NUMBER OF LOADING CONDITIONS: " << num_lcs << std::endl; // walk over the nodes to update the velocity FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { @@ -916,41 +879,44 @@ void FEA_Module_SGH::applied_forces(const DCArrayKokkos &material, double node_force[3]; double applied_force[3]; double radius; - for (size_t dim = 0; dim < num_dim; dim++){ + for (size_t dim = 0; dim < num_dim; dim++) + { node_force[dim] = 0.0; current_node_coords[dim] = all_initial_node_coords(node_gid, dim); } // end for dim - radius = sqrt(current_node_coords[0]*current_node_coords[0]+current_node_coords[1]*current_node_coords[1]+current_node_coords[2]*current_node_coords[2]); - for(size_t ilc=0; ilc < num_lcs; ilc++){ - //debug check - //std::cout << "LOADING CONDITION VOLUME TYPE: " << to_string(loading(ilc).volume) << std::endl; - - bool fill_this = loading(ilc).volume.contains(current_node_coords); - if(fill_this){ - // loop over all corners around the node and calculate the nodal force - for (size_t corner_lid=0; corner_lid& node_coords, + const DViewCArrayKokkos& node_vel) +{ + const size_t rk_level = rk_num_bins - 1; + int num_dims = num_dim; + + // loop over all the nodes in the mesh + FOR_ALL_CLASS(node_gid, 0, num_nodes, { + for (int dim = 0; dim < num_dims; dim++) + { + double half_vel = (node_vel(rk_level, node_gid, dim) + node_vel(0, node_gid, dim)) * 0.5; + node_coords(rk_level, node_gid, dim) = node_coords(0, node_gid, dim) + rk_alpha * dt * half_vel; + } + }); // end parallel for over nodes +} // end subroutine + +// ----------------------------------------------------------------------------- +// This function claculates +// B_p = J^{-T} \cdot (\nabla_{xi} \phi_p w +// where +// \phi_p is the basis function for vertex p +// w is the 1 gauss point for the cell (everything is evaluted at this point) +// J^{-T} is the inverse transpose of the Jacobi matrix +// \nabla_{xi} is the gradient opperator in the reference coordinates +// +// B_p is the OUTWARD corner area normal at node p +// ------------------------------------------------------------------------------ +KOKKOS_FUNCTION +void FEA_Module_SGH::get_bmatrix(const ViewCArrayKokkos& B_matrix, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const +{ + const size_t num_nodes = 8; + + double x_array[8]; + double y_array[8]; + double z_array[8]; + + // x, y, z coordinates of elem vertices + auto x = ViewCArrayKokkos(x_array, num_nodes); + auto y = ViewCArrayKokkos(y_array, num_nodes); + auto z = ViewCArrayKokkos(z_array, num_nodes); + + // get the coordinates of the nodes(rk,elem,node) in this element + for (int node_lid = 0; node_lid < num_nodes; node_lid++) + { + x(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 0); + y(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 1); + z(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 2); + } // end for + + double twelth = 1. / 12.; + + B_matrix(0, 0) = (+y(1) * (-z(2) - z(3) + z(4) + z(5) ) + + y(2) * (+z(1) - z(3) ) + + y(3) * (+z(1) + z(2) - z(4) - z(7) ) + + y(4) * (-z(1) + z(3) - z(5) + z(7) ) + + y(5) * (-z(1) + z(4) ) + + y(7) * (+z(3) - z(4) ) ) * twelth; + + B_matrix(1, 0) = (+y(0) * (+z(2) + z(3) - z(4) - z(5) ) + + y(2) * (-z(0) - z(3) + z(5) + z(6) ) + + y(3) * (-z(0) + z(2) ) + + y(4) * (+z(0) - z(5) ) + + y(5) * (+z(0) - z(2) + z(4) - z(6) ) + + y(6) * (-z(2) + z(5) ) ) * twelth; + + B_matrix(2, 0) = (+y(0) * (-z(1) + z(3) ) + + y(1) * (+z(0) + z(3) - z(5) - z(6) ) + + y(3) * (-z(0) - z(1) + z(6) + z(7) ) + + y(5) * (+z(1) - z(6) ) + + y(6) * (+z(1) - z(3) + z(5) - z(7) ) + + y(7) * (-z(3) + z(6) ) ) * twelth; + + B_matrix(3, 0) = (+y(0) * (-z(1) - z(2) + z(4) + z(7) ) + + y(1) * (+z(0) - z(2) ) + + y(2) * (+z(0) + z(1) - z(6) - z(7) ) + + y(4) * (-z(0) + z(7) ) + + y(6) * (+z(2) - z(7) ) + + y(7) * (-z(0) + z(2) - z(4) + z(6) ) ) * twelth; + + B_matrix(4, 0) = (+y(0) * (+z(1) - z(3) + z(5) - z(7) ) + + y(1) * (-z(0) + z(5) ) + + y(3) * (+z(0) - z(7) ) + + y(5) * (-z(0) - z(1) + z(6) + z(7) ) + + y(6) * (-z(5) + z(7) ) + + y(7) * (+z(0) + z(3) - z(5) - z(6) ) ) * twelth; + + B_matrix(5, 0) = (+y(0) * (+z(1) - z(4) ) + + y(1) * (-z(0) + z(2) - z(4) + z(6) ) + + y(2) * (-z(1) + z(6) ) + + y(4) * (+z(0) + z(1) - z(6) - z(7) ) + + y(6) * (-z(1) - z(2) + z(4) + z(7) ) + + y(7) * (+z(4) - z(6) ) ) * twelth; + + B_matrix(6, 0) = (+y(1) * (+z(2) - z(5) ) + + y(2) * (-z(1) + z(3) - z(5) + z(7) ) + + y(3) * (-z(2) + z(7) ) + + y(4) * (+z(5) - z(7) ) + + y(5) * (+z(1) + z(2) - z(4) - z(7) ) + + y(7) * (-z(2) - z(3) + z(4) + z(5) ) ) * twelth; + + B_matrix(7, 0) = (+y(0) * (-z(3) + z(4) ) + + y(2) * (+z(3) - z(6) ) + + y(3) * (+z(0) - z(2) + z(4) - z(6) ) + + y(4) * (-z(0) - z(3) + z(5) + z(6) ) + + y(5) * (-z(4) + z(6) ) + + y(6) * (+z(2) + z(3) - z(4) - z(5) ) ) * twelth; + + B_matrix(0, 1) = (+z(1) * (-x(2) - x(3) + x(4) + x(5) ) + + z(2) * (+x(1) - x(3) ) + + z(3) * (+x(1) + x(2) - x(4) - x(7) ) + + z(4) * (-x(1) + x(3) - x(5) + x(7) ) + + z(5) * (-x(1) + x(4) ) + + z(7) * (+x(3) - x(4) ) ) * twelth; + + B_matrix(1, 1) = (+z(0) * (+x(2) + x(3) - x(4) - x(5) ) + + z(2) * (-x(0) - x(3) + x(5) + x(6) ) + + z(3) * (-x(0) + x(2) ) + + z(4) * (+x(0) - x(5) ) + + z(5) * (+x(0) - x(2) + x(4) - x(6) ) + + z(6) * (-x(2) + x(5) ) ) * twelth; + + B_matrix(2, 1) = (+z(0) * (-x(1) + x(3) ) + + z(1) * (+x(0) + x(3) - x(5) - x(6) ) + + z(3) * (-x(0) - x(1) + x(6) + x(7) ) + + z(5) * (+x(1) - x(6) ) + + z(6) * (+x(1) - x(3) + x(5) - x(7) ) + + z(7) * (-x(3) + x(6) ) ) * twelth; + + B_matrix(3, 1) = (+z(0) * (-x(1) - x(2) + x(4) + x(7) ) + + z(1) * (+x(0) - x(2) ) + + z(2) * (+x(0) + x(1) - x(6) - x(7) ) + + z(4) * (-x(0) + x(7) ) + + z(6) * (+x(2) - x(7) ) + + z(7) * (-x(0) + x(2) - x(4) + x(6) ) ) * twelth; + + B_matrix(4, 1) = (+z(0) * (+x(1) - x(3) + x(5) - x(7) ) + + z(1) * (-x(0) + x(5) ) + + z(3) * (+x(0) - x(7) ) + + z(5) * (-x(0) - x(1) + x(6) + x(7) ) + + z(6) * (-x(5) + x(7) ) + + z(7) * (+x(0) + x(3) - x(5) - x(6) ) ) * twelth; + + B_matrix(5, 1) = (+z(0) * (+x(1) - x(4) ) + + z(1) * (-x(0) + x(2) - x(4) + x(6) ) + + z(2) * (-x(1) + x(6) ) + + z(4) * (+x(0) + x(1) - x(6) - x(7) ) + + z(6) * (-x(1) - x(2) + x(4) + x(7) ) + + z(7) * (+x(4) - x(6) ) ) * twelth; + + B_matrix(6, 1) = (+z(1) * (+x(2) - x(5) ) + + z(2) * (-x(1) + x(3) - x(5) + x(7) ) + + z(3) * (-x(2) + x(7) ) + + z(4) * (+x(5) - x(7) ) + + z(5) * (+x(1) + x(2) - x(4) - x(7) ) + + z(7) * (-x(2) - x(3) + x(4) + x(5) ) ) * twelth; + + B_matrix(7, 1) = (+z(0) * (-x(3) + x(4) ) + + z(2) * (+x(3) - x(6) ) + + z(3) * (+x(0) - x(2) + x(4) - x(6) ) + + z(4) * (-x(0) - x(3) + x(5) + x(6) ) + + z(5) * (-x(4) + x(6) ) + + z(6) * (+x(2) + x(3) - x(4) - x(5) ) ) * twelth; + + B_matrix(0, 2) = (+x(1) * (-y(2) - y(3) + y(4) + y(5) ) + + x(2) * (+y(1) - y(3) ) + + x(3) * (+y(1) + y(2) - y(4) - y(7) ) + + x(4) * (-y(1) + y(3) - y(5) + y(7) ) + + x(5) * (-y(1) + y(4) ) + + x(7) * (+y(3) - y(4) ) ) * twelth; + + B_matrix(1, 2) = (+x(0) * (+y(2) + y(3) - y(4) - y(5) ) + + x(2) * (-y(0) - y(3) + y(5) + y(6) ) + + x(3) * (-y(0) + y(2) ) + + x(4) * (+y(0) - y(5) ) + + x(5) * (+y(0) - y(2) + y(4) - y(6) ) + + x(6) * (-y(2) + y(5) ) ) * twelth; + + B_matrix(2, 2) = (+x(0) * (-y(1) + y(3) ) + + x(1) * (+y(0) + y(3) - y(5) - y(6) ) + + x(3) * (-y(0) - y(1) + y(6) + y(7) ) + + x(5) * (+y(1) - y(6) ) + + x(6) * (+y(1) - y(3) + y(5) - y(7) ) + + x(7) * (-y(3) + y(6) ) ) * twelth; + + B_matrix(3, 2) = (+x(0) * (-y(1) - y(2) + y(4) + y(7) ) + + x(1) * (+y(0) - y(2) ) + + x(2) * (+y(0) + y(1) - y(6) - y(7) ) + + x(4) * (-y(0) + y(7) ) + + x(6) * (+y(2) - y(7) ) + + x(7) * (-y(0) + y(2) - y(4) + y(6) ) ) * twelth; + + B_matrix(4, 2) = (+x(0) * (+y(1) - y(3) + y(5) - y(7) ) + + x(1) * (-y(0) + y(5) ) + + x(3) * (+y(0) - y(7) ) + + x(5) * (-y(0) - y(1) + y(6) + y(7) ) + + x(6) * (-y(5) + y(7) ) + + x(7) * (+y(0) + y(3) - y(5) - y(6) ) ) * twelth; + + B_matrix(5, 2) = (+x(0) * (+y(1) - y(4) ) + + x(1) * (-y(0) + y(2) - y(4) + y(6) ) + + x(2) * (-y(1) + y(6) ) + + x(4) * (+y(0) + y(1) - y(6) - y(7) ) + + x(6) * (-y(1) - y(2) + y(4) + y(7) ) + + x(7) * (+y(4) - y(6) ) ) * twelth; + + B_matrix(6, 2) = (+x(1) * (+y(2) - y(5) ) + + x(2) * (-y(1) + y(3) - y(5) + y(7) ) + + x(3) * (-y(2) + y(7) ) + + x(4) * (+y(5) - y(7) ) + + x(5) * (+y(1) + y(2) - y(4) - y(7) ) + + x(7) * (-y(2) - y(3) + y(4) + y(5) ) ) * twelth; + + B_matrix(7, 2) = (+x(0) * (-y(3) + y(4) ) + + x(2) * (+y(3) - y(6) ) + + x(3) * (+y(0) - y(2) + y(4) - y(6) ) + + x(4) * (-y(0) - y(3) + y(5) + y(6) ) + + x(5) * (-y(4) + y(6) ) + + x(6) * (+y(2) + y(3) - y(4) - y(5) ) ) * twelth; +} // end subroutine + +/* ---------------------------------------------------------------------------- + Compute Volume of each finite element +------------------------------------------------------------------------------- */ + +void FEA_Module_SGH::get_vol() +{ + const size_t rk_level = rk_num_bins - 1; + const size_t num_dims = num_dim; + + if (num_dims == 2) + { + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { + // cut out the node_gids for this element + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 4); + get_vol_quad(elem_vol, elem_gid, node_coords, elem_node_gids, rk_level); + }); + Kokkos::fence(); + } + else + { + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { + // cut out the node_gids for this element + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 8); + get_vol_hex(elem_vol, elem_gid, node_coords, elem_node_gids, rk_level); + }); + Kokkos::fence(); + } // end if + + return; +} // end subroutine + +// Exact volume for a hex element +KOKKOS_INLINE_FUNCTION +void FEA_Module_SGH::get_vol_hex(const DViewCArrayKokkos& elem_vol, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const +{ + const size_t num_nodes = 8; + + double x_array[8]; + double y_array[8]; + double z_array[8]; + + // x, y, z coordinates of elem vertices + auto x = ViewCArrayKokkos(x_array, num_nodes); + auto y = ViewCArrayKokkos(y_array, num_nodes); + auto z = ViewCArrayKokkos(z_array, num_nodes); + + // get the coordinates of the nodes(rk,elem,node) in this element + for (int node_lid = 0; node_lid < num_nodes; node_lid++) + { + x(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 0); + y(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 1); + z(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 2); + } // end for + + double twelth = 1. / 12.; + + // element volume + elem_vol(elem_gid) = + (x(1) * (y(3) * (-z(0) + z(2)) + y(4) * (z(0) - z(5)) + y(0) * (z(2) + z(3) - z(4) - z(5)) + y(6) * (-z(2) + z(5)) + y(5) * (z(0) - z(2) + z(4) - z(6)) + y(2) * (-z(0) - z(3) + z(5) + z(6))) + + x(7) * (y(0) * (-z(3) + z(4)) + y(6) * (z(2) + z(3) - z(4) - z(5)) + y(2) * (z(3) - z(6)) + y(3) * (z(0) - z(2) + z(4) - z(6)) + y(5) * (-z(4) + z(6)) + y(4) * (-z(0) - z(3) + z(5) + z(6))) + + x(3) * (y(1) * (z(0) - z(2)) + y(7) * (-z(0) + z(2) - z(4) + z(6)) + y(6) * (z(2) - z(7)) + y(2) * (z(0) + z(1) - z(6) - z(7)) + y(4) * (-z(0) + z(7)) + y(0) * (-z(1) - z(2) + z(4) + z(7))) + + x(5) * (y(0) * (z(1) - z(4)) + y(7) * (z(4) - z(6)) + y(2) * (-z(1) + z(6)) + y(1) * (-z(0) + z(2) - z(4) + z(6)) + y(4) * (z(0) + z(1) - z(6) - z(7)) + y(6) * (-z(1) - z(2) + z(4) + z(7))) + + x(6) * (y(1) * (z(2) - z(5)) + y(7) * (-z(2) - z(3) + z(4) + z(5)) + y(5) * (z(1) + z(2) - z(4) - z(7)) + y(4) * (z(5) - z(7)) + y(3) * (-z(2) + z(7)) + y(2) * (-z(1) + z(3) - z(5) + z(7))) + + x(0) * (y(2) * (z(1) - z(3)) + y(7) * (z(3) - z(4)) + y(5) * (-z(1) + z(4)) + y(1) * (-z(2) - z(3) + z(4) + z(5)) + y(3) * (z(1) + z(2) - z(4) - z(7)) + y(4) * (-z(1) + z(3) - z(5) + z(7))) + + x(2) * (y(0) * (-z(1) + z(3)) + y(5) * (z(1) - z(6)) + y(1) * (z(0) + z(3) - z(5) - z(6)) + y(7) * (-z(3) + z(6)) + y(6) * (z(1) - z(3) + z(5) - z(7)) + y(3) * (-z(0) - z(1) + z(6) + z(7))) + + x(4) * + (y(1) * (-z(0) + z(5)) + y(7) * (z(0) + z(3) - z(5) - z(6)) + y(3) * (z(0) - z(7)) + y(0) * (z(1) - z(3) + z(5) - z(7)) + y(6) * (-z(5) + z(7)) + y(5) * (-z(0) - z(1) + z(6) + z(7)))) * + twelth; + + return; +} // end subroutine + +KOKKOS_FUNCTION +void FEA_Module_SGH::get_bmatrix2D(const ViewCArrayKokkos& B_matrix, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const +{ + const size_t num_nodes = 4; + + double x_array[4]; + double y_array[4]; + + // x, y coordinates of elem vertices + auto x = ViewCArrayKokkos(x_array, num_nodes); + auto y = ViewCArrayKokkos(y_array, num_nodes); + + // get the coordinates of the nodes(rk,elem,node) in this element + for (int node_lid = 0; node_lid < num_nodes; node_lid++) + { + x(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 0); + y(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 1); + } // end for + + /* ensight node order 0 1 2 3 + Flanaghan node order 3 4 1 2 + */ + + B_matrix(0, 0) = -0.5 * (y(3) - y(1)); + + B_matrix(1, 0) = -0.5 * (y(0) - y(2)); + + B_matrix(2, 0) = -0.5 * (y(1) - y(3)); + + B_matrix(3, 0) = -0.5 * (y(2) - y(0)); + + B_matrix(0, 1) = -0.5 * (x(1) - x(3)); + + B_matrix(1, 1) = -0.5 * (x(2) - x(0)); + + B_matrix(2, 1) = -0.5 * (x(3) - x(1)); + + B_matrix(3, 1) = -0.5 * (x(0) - x(2)); + + // + /* + The Flanagan and Belytschko paper has: + x y + node 1: 0.5*(y2 - y4) , 0.5*(x4 - x2) + node 2: 0.5*(y3 - y1) , 0.5*(x1 - x3) + node 3: 0.5*(y4 - y2) , 0.5*(x2 - x4) + node 4: 0.5*(y1 - y3) , 0.5*(x3 - x1) + + Ensight order would be + + node 2: 0.5*(y3 - y1) , 0.5*(x1 - x3) + node 3: 0.5*(y0 - y2) , 0.5*(x2 - x0) + node 0: 0.5*(y1 - y3) , 0.5*(x3 - x1) + node 1: 0.5*(y2 - y0) , 0.5*(x0 - x2) + + */ + + return; +} // end subroutine + +// true volume of a quad in RZ coords +KOKKOS_INLINE_FUNCTION +void FEA_Module_SGH::get_vol_quad(const DViewCArrayKokkos& elem_vol, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const +{ + // --- testing here --- + /* + double test_vol = 0.0; + // getting the corner facial area + double corner_areas_array[4]; + ViewCArrayKokkos corner_areas(&corner_areas_array[0],4); + + get_area_weights2D(corner_areas, + elem_gid, + node_coords, + elem_node_gids, + rk_level); + + + for(size_t node_lid=0; node_lid<4; node_lid++){ + double y = node_coords(rk_level, elem_node_gids(node_lid), 1); // node radius + test_vol += corner_areas(node_lid)*y; + } // end for + + test_vol matches the Barlow volume formula + */ + // ------------------- + + elem_vol(elem_gid) = 0.0; + + const size_t num_nodes = 4; + + double x_array[4]; + double y_array[4]; + + // x, y coordinates of elem vertices + auto x = ViewCArrayKokkos(x_array, num_nodes); + auto y = ViewCArrayKokkos(y_array, num_nodes); + + // get the coordinates of the nodes(rk,elem,node) in this element + for (int node_lid = 0; node_lid < num_nodes; node_lid++) + { + x(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 0); + y(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 1); + } // end for + + /* ensight node order 0 1 2 3 + Flanaghan node order 3 4 1 2 + */ + elem_vol(elem_gid) = + ( (y(2) + y(3) + y(0)) * ((y(2) - y(3)) * (x(0) - x(3)) - (y(0) - y(3)) * (x(2) - x(3)) ) + + (y(0) + y(1) + y(2)) * ((y(0) - y(1)) * (x(2) - x(1)) - (y(2) - y(1)) * (x(0) - x(1))) ) / 6.0; + + return; +} // end subroutine + +// element facial area +KOKKOS_FUNCTION +double FEA_Module_SGH::get_area_quad(const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const +{ + double elem_area = 0.0; + + const size_t num_nodes = 4; + + double x_array[4]; + double y_array[4]; + + // x, y coordinates of elem vertices + auto x = ViewCArrayKokkos(x_array, num_nodes); + auto y = ViewCArrayKokkos(y_array, num_nodes); + + // get the coordinates of the nodes(rk,elem,node) in this element + for (int node_lid = 0; node_lid < num_nodes; node_lid++) + { + x(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 0); + y(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 1); + } // end for + + /* ensight node order 0 1 2 3 + Flanaghan node order 3 4 1 2 + */ + + // element facial area + elem_area = 0.5 * ((x(0) - x(2)) * (y(1) - y(3)) + (x(3) - x(1)) * (y(0) - y(2))); + + return elem_area; +} // end subroutine + +KOKKOS_INLINE_FUNCTION +double FEA_Module_SGH::heron(const double x1, + const double y1, + const double x2, + const double y2, + const double x3, + const double y3) const +{ + double S, a, b, c, area; + + S = 0.0; + a = sqrt((x2 - x1) * (x2 - x1) + (y2 - y1) * (y2 - y1)); + S += a; + b = sqrt((x3 - x2) * (x3 - x2) + (y3 - y2) * (y3 - y2)); + S += b; + c = sqrt((x3 - x1) * (x3 - x1) + (y3 - y1) * (y3 - y1)); + S += c; + + S *= 0.5; + area = sqrt(S * (S - a) * (S - b) * (S - c)); + + return area; +} + +KOKKOS_FUNCTION +void FEA_Module_SGH::get_area_weights2D(const ViewCArrayKokkos& corner_areas, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const +{ + const size_t num_nodes = 4; + + double x_array[4]; + double y_array[4]; + + double rc, zc; + double A12, A23, A34, A41; + + // x, y coordinates of elem vertices + ViewCArrayKokkos x(x_array, num_nodes); + ViewCArrayKokkos y(y_array, num_nodes); + + // get the coordinates of the nodes(rk,elem,node) in this element + rc = zc = 0.0; + for (int node_lid = 0; node_lid < num_nodes; node_lid++) + { + x(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 0); + y(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 1); + rc += 0.25 * y(node_lid); + zc += 0.25 * x(node_lid); + } // end for + + /* ensight node order 0 1 2 3 + Barlow node order 1 2 3 4 + */ + + A12 = heron(x(0), y(0), zc, rc, x(1), y(1)); + A23 = heron(x(1), y(1), zc, rc, x(2), y(2)); + A34 = heron(x(2), y(2), zc, rc, x(3), y(3)); + A41 = heron(x(3), y(3), zc, rc, x(0), y(0)); + + corner_areas(0) = (5. * A41 + 5. * A12 + A23 + A34) / 12.; + corner_areas(1) = (A41 + 5. * A12 + 5. * A23 + A34) / 12.; + corner_areas(2) = (A41 + A12 + 5. * A23 + 5. * A34) / 12.; + corner_areas(3) = (5. * A41 + A12 + A23 + 5. * A34) / 12.; + + return; +} // end subroutine + +/* ----------------------------------------------------------------------------------- + Compute Gradient of the Volume of each finite element with respect to displacement +-------------------------------------------------------------------------------------- */ + +void FEA_Module_SGH::get_vol_ugradient(const size_t gradient_node_id, const size_t gradient_dim) +{ + const size_t rk_level = rk_num_bins - 1; + const size_t num_dims = num_dim; + + if (num_dims == 2) + { + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { + // cut out the node_gids for this element + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 4); + get_vol_quad(elem_vol, elem_gid, node_coords, elem_node_gids, rk_level); + }); + Kokkos::fence(); + } + else + { + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { + // cut out the node_gids for this element + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 8); + // get_vol_hex_ugradient(elem_vol, elem_gid, node_coords, elem_node_gids, rk_level); + }); + Kokkos::fence(); + } // end if + + return; +} // end subroutine + +// Exact volume for a hex element +KOKKOS_FUNCTION +void FEA_Module_SGH::get_vol_hex_ugradient(const ViewCArrayKokkos& elem_vol_gradients, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const +{ + const size_t num_nodes = 8; + const size_t num_dims = num_dim; + double x_array[8]; + double y_array[8]; + double z_array[8]; + double gradient_result; + + // x, y, z coordinates of elem vertices + auto x = ViewCArrayKokkos(x_array, num_nodes); + auto y = ViewCArrayKokkos(y_array, num_nodes); + auto z = ViewCArrayKokkos(z_array, num_nodes); + + // get the coordinates of the nodes(rk,elem,node) in this element + for (int node_lid = 0; node_lid < num_nodes; node_lid++) + { + x(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 0); + y(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 1); + z(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 2); + } // end for + + double twelth = 1. / 12.; + + // element volume gradient + for (int inode = 0; inode < 8; inode++) + { + for (int idim = 0; idim < num_dims; idim++) + { + switch (num_dims * inode + idim) + { + case 0: + gradient_result = + ((y(2) * (z(1) - z(3)) + y(7) * (z(3) - z(4)) + y(5) * (-z(1) + z(4)) + y(1) * (-z(2) - z(3) + z(4) + z(5)) + y(3) * (z(1) + z(2) - z(4) - z(7)) + y(4) * + (-z(1) + z(3) - z(5) + z(7)))) * twelth; + break; + case 3: + gradient_result = + ((y(3) * (-z(0) + z(2)) + y(4) * (z(0) - z(5)) + y(0) * (z(2) + z(3) - z(4) - z(5)) + y(6) * (-z(2) + z(5)) + y(5) * (z(0) - z(2) + z(4) - z(6)) + y(2) * + (-z(0) - z(3) + z(5) + z(6)))) * twelth; + break; + + case 6: + gradient_result = + ((y(0) * (-z(1) + z(3)) + y(5) * (z(1) - z(6)) + y(1) * (z(0) + z(3) - z(5) - z(6)) + y(7) * (-z(3) + z(6)) + y(6) * (z(1) - z(3) + z(5) - z(7)) + y(3) * + (-z(0) - z(1) + z(6) + z(7)))) * twelth; + break; + case 9: + gradient_result = + ((y(1) * (z(0) - z(2)) + y(7) * (-z(0) + z(2) - z(4) + z(6)) + y(6) * (z(2) - z(7)) + y(2) * (z(0) + z(1) - z(6) - z(7)) + y(4) * (-z(0) + z(7)) + y(0) * + (-z(1) - z(2) + z(4) + z(7)))) * twelth; + break; + case 12: + gradient_result = + ((y(1) * (-z(0) + z(5)) + y(7) * (z(0) + z(3) - z(5) - z(6)) + y(3) * (z(0) - z(7)) + y(0) * (z(1) - z(3) + z(5) - z(7)) + y(6) * (-z(5) + z(7)) + y(5) * + (-z(0) - z(1) + z(6) + z(7)))) * twelth; + break; + case 15: + gradient_result = + ((y(0) * (z(1) - z(4)) + y(7) * (z(4) - z(6)) + y(2) * (-z(1) + z(6)) + y(1) * (-z(0) + z(2) - z(4) + z(6)) + y(4) * (z(0) + z(1) - z(6) - z(7)) + y(6) * + (-z(1) - z(2) + z(4) + z(7)))) * twelth; + break; + case 18: + gradient_result = + ((y(1) * (z(2) - z(5)) + y(7) * (-z(2) - z(3) + z(4) + z(5)) + y(5) * (z(1) + z(2) - z(4) - z(7)) + y(4) * (z(5) - z(7)) + y(3) * (-z(2) + z(7)) + y(2) * + (-z(1) + z(3) - z(5) + z(7)))) * twelth; + break; + case 21: + gradient_result = + ((y(0) * (-z(3) + z(4)) + y(6) * (z(2) + z(3) - z(4) - z(5)) + y(2) * (z(3) - z(6)) + y(3) * (z(0) - z(2) + z(4) - z(6)) + y(5) * (-z(4) + z(6)) + y(4) * + (-z(0) - z(3) + z(5) + z(6)))) * twelth; + break; + case 1: + gradient_result = + (x(1) * ((z(2) + z(3) - z(4) - z(5))) + + x(7) * ((-z(3) + z(4))) + + x(3) * ((-z(1) - z(2) + z(4) + z(7))) + + x(5) * ((z(1) - z(4))) + + x(2) * ((-z(1) + z(3))) + + x(4) * ((z(1) - z(3) + z(5) - z(7)))) * twelth; + break; + case 4: + gradient_result = + (x(3) * ((z(0) - z(2))) + + x(5) * ((-z(0) + z(2) - z(4) + z(6))) + + x(6) * ((z(2) - z(5))) + + x(0) * ((-z(2) - z(3) + z(4) + z(5))) + + x(2) * ((z(0) + z(3) - z(5) - z(6))) + + x(4) * ((-z(0) + z(5)))) * twelth; + break; + case 7: + gradient_result = + (x(1) * ((-z(0) - z(3) + z(5) + z(6))) + + x(7) * ((z(3) - z(6))) + + x(3) * ((z(0) + z(1) - z(6) - z(7))) + + x(5) * ((-z(1) + z(6))) + + x(6) * ((-z(1) + z(3) - z(5) + z(7))) + + x(0) * ((z(1) - z(3)))) * twelth; + break; + case 10: + gradient_result = + (x(1) * ((-z(0) + z(2))) + + x(7) * ((z(0) - z(2) + z(4) - z(6))) + + x(6) * ((-z(2) + z(7))) + + x(0) * ((z(1) + z(2) - z(4) - z(7))) + + x(2) * ((-z(0) - z(1) + z(6) + z(7))) + + x(4) * ((z(0) - z(7)))) * twelth; + break; + case 13: + gradient_result = + (x(1) * ((z(0) - z(5))) + + x(7) * ((-z(0) - z(3) + z(5) + z(6))) + + x(3) * ((-z(0) + z(7))) + + x(5) * ((z(0) + z(1) - z(6) - z(7))) + + x(6) * ((z(5) - z(7))) + + x(0) * ((-z(1) + z(3) - z(5) + z(7)))) * twelth; + break; + case 16: + gradient_result = + (x(1) * ((z(0) - z(2) + z(4) - z(6))) + + x(7) * ((-z(4) + z(6))) + + x(6) * ((z(1) + z(2) - z(4) - z(7))) + + x(0) * ((-z(1) + z(4))) + + x(2) * ((z(1) - z(6))) + + x(4) * ((-z(0) - z(1) + z(6) + z(7)))) * twelth; + break; + case 19: + gradient_result = + (x(1) * ((-z(2) + z(5))) + + x(7) * ((z(2) + z(3) - z(4) - z(5))) + + x(3) * ((z(2) - z(7))) + + x(5) * ((-z(1) - z(2) + z(4) + z(7))) + + x(2) * ((z(1) - z(3) + z(5) - z(7))) + + x(4) * ((-z(5) + z(7)))) * twelth; + break; + case 22: + gradient_result = + (x(3) * ((-z(0) + z(2) - z(4) + z(6))) + + x(5) * ((z(4) - z(6))) + + x(6) * ((-z(2) - z(3) + z(4) + z(5))) + + x(0) * ((z(3) - z(4))) + + x(2) * ((-z(3) + z(6))) + + x(4) * ((z(0) + z(3) - z(5) - z(6)))) * twelth; + break; + case 2: + gradient_result = + (x(1) * (-y(3) + y(4) + y(5) - y(2)) + + x(7) * (y(3) - y(4)) + + x(3) * (y(1) - y(7) + y(2) - y(4)) + + x(5) * (-y(1) + y(4)) + + x(2) * (y(1) - y(3)) + + x(4) * (-y(1) + y(7) + y(3) - y(5))) * twelth; + break; + case 5: + gradient_result = + (x(3) * (y(2) - y(0)) + + x(5) * (y(0) - y(2) + y(4) - y(6)) + + x(6) * (y(5) - y(2)) + + x(0) * (y(2) - y(5) + y(3) - y(4)) + + x(2) * (-y(0) + y(5) + y(6) - y(3)) + + x(4) * (y(0) - y(5))) * twelth; + break; + case 8: + gradient_result = + (x(1) * (y(3) + y(0) - y(6) - y(5)) + + x(7) * (y(6) - y(3)) + + x(3) * (-y(1) + y(7) + y(6) - y(0)) + + x(5) * (y(1) - y(6)) + + x(6) * (y(1) - y(7) + y(5) - y(3)) + + x(0) * (-y(1) + y(3))) * twelth; + break; + case 11: + gradient_result = + (x(1) * (y(0) - y(2)) + + x(7) * (-y(0) + y(6) + y(2) - y(4)) + + x(6) * (-y(7) + y(2)) + + x(0) * (-y(2) + y(7) - y(1) + y(4)) + + x(2) * (y(0) + y(1) - y(7) - y(6)) + + x(4) * (y(7) - y(0))) * twelth; + break; + case 14: + gradient_result = + (x(1) * (-y(0) + y(5)) + + x(7) * (y(0) - y(6) + y(3) - y(5)) + + x(3) * (-y(7) + y(0)) + + x(5) * (-y(0) + y(7) - y(1) + y(6)) + + x(6) * (y(7) - y(5)) + + x(0) * (-y(7) + y(5) + y(1) - y(3))) * twelth; + break; + case 17: + gradient_result = + (x(1) * (-y(4) - y(0) + y(6) + y(2)) + + x(7) * (-y(6) + y(4)) + + x(6) * (-y(1) + y(7) + y(4) - y(2)) + + x(0) * (y(1) - y(4)) + + x(2) * (-y(1) + y(6)) + + x(4) * (y(1) - y(7) + y(0) - y(6))) * twelth; + break; + case 20: + gradient_result = + (x(1) * (-y(5) + y(2)) + + x(7) * (-y(2) - y(3) + y(5) + y(4)) + + x(3) * (y(7) - y(2)) + + x(5) * (-y(7) + y(2) + y(1) - y(4)) + + x(2) * (-y(5) - y(1) + y(7) + y(3)) + + x(4) * (-y(7) + y(5))) * twelth; + break; + case 23: + gradient_result = + (x(3) * (-y(6) - y(2) + y(4) + y(0)) + + x(5) * (-y(4) + y(6)) + + x(6) * (-y(5) - y(4) + y(3) + y(2)) + + x(0) * (-y(3) + y(4)) + + x(2) * (-y(6) + y(3)) + + x(4) * (-y(3) - y(0) + y(6) + y(5))) * twelth; + break; + } + elem_vol_gradients(inode, idim) = gradient_result; + } + } + return; +} // end subroutine + +KOKKOS_FUNCTION +void FEA_Module_SGH::get_bmatrix_gradients(const ViewCArrayKokkos& B_matrix_gradients, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const +{ + const size_t num_nodes = 8; + + double x_array[8]; + double y_array[8]; + double z_array[8]; + double gradient_terms_array[2 * 8]; + + // x, y, z coordinates of elem vertices + auto x = ViewCArrayKokkos(x_array, num_nodes); + auto y = ViewCArrayKokkos(y_array, num_nodes); + auto z = ViewCArrayKokkos(z_array, num_nodes); + auto gradient_terms = ViewCArrayKokkos(gradient_terms_array, 2, num_nodes); + + // get the coordinates of the nodes(rk,elem,node) in this element + for (int node_lid = 0; node_lid < num_nodes; node_lid++) + { + x(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 0); + y(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 1); + z(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 2); + } // end for + + double twelth = 1. / 12.; + + // B_matrix(0,0) = ( +y(1)*( -z(2) -z(3) +z(4) +z(5) ) + // +y(2)*( +z(1) -z(3) ) + // +y(3)*( +z(1) +z(2) -z(4) -z(7) ) + // +y(4)*( -z(1) +z(3) -z(5) +z(7) ) + // +y(5)*( -z(1) +z(4) ) + // +y(7)*( +z(3) -z(4) ) )*twelth; + + // y derivative + gradient_terms(0, 0) = 0; + gradient_terms(0, 1) = -z(2) - z(3) + z(4) + z(5); + gradient_terms(0, 2) = z(1) - z(3); + gradient_terms(0, 3) = z(1) + z(2) - z(4) - z(7); + gradient_terms(0, 4) = -z(1) + z(3) - z(5) + z(7); + gradient_terms(0, 5) = -z(1) + z(4); + gradient_terms(0, 6) = 0; + gradient_terms(0, 7) = z(3) - z(4); + + // z derivative + gradient_terms(1, 0) = 0; + gradient_terms(1, 1) = y(2) + y(3) - y(4) - y(5); + gradient_terms(1, 2) = -y(1) + y(3); + gradient_terms(1, 3) = -y(1) - y(2) + y(4) + y(7); + gradient_terms(1, 4) = y(1) - y(3) + y(5) - y(7); + gradient_terms(1, 5) = y(1) - y(4); + gradient_terms(1, 6) = 0; + gradient_terms(1, 7) = -y(3) + y(4); + + for (int inode = 0; inode < 8; inode++) + { + B_matrix_gradients(0, 0, inode, 0) = 0; + B_matrix_gradients(0, 0, inode, 1) = gradient_terms(0, inode) * twelth; + B_matrix_gradients(0, 0, inode, 2) = gradient_terms(1, inode) * twelth; + } + + // B_matrix(1,0) = ( +y(0)*( +z(2) +z(3) -z(4) -z(5) ) + // +y(2)*( -z(0) -z(3) +z(5) +z(6) ) + // +y(3)*( -z(0) +z(2) ) + // +y(4)*( +z(0) -z(5) ) + // +y(5)*( +z(0) -z(2) +z(4) -z(6) ) + // +y(6)*( -z(2) +z(5) ) )*twelth; + + // y derivative + gradient_terms(0, 0) = z(2) + z(3) - z(4) - z(5); + gradient_terms(0, 1) = 0; + gradient_terms(0, 2) = -z(0) - z(3) + z(5) + z(6); + gradient_terms(0, 3) = -z(0) + z(2); + gradient_terms(0, 4) = z(0) - z(5); + gradient_terms(0, 5) = z(0) - z(2) + z(4) - z(6); + gradient_terms(0, 6) = -z(2) + z(5); + gradient_terms(0, 7) = 0; + + // z derivative + gradient_terms(1, 0) = -y(2) - y(3) + y(4) + y(5); + gradient_terms(1, 1) = 0; + gradient_terms(1, 2) = y(0) + y(3) - y(5) - y(6); + gradient_terms(1, 3) = y(0) - y(2); + gradient_terms(1, 4) = -y(0) + y(5); + gradient_terms(1, 5) = -y(0) + y(2) - y(4) + y(6); + gradient_terms(1, 6) = y(2) - y(5); + gradient_terms(1, 7) = 0; + + for (int inode = 0; inode < 8; inode++) + { + B_matrix_gradients(1, 0, inode, 0) = 0; + B_matrix_gradients(1, 0, inode, 1) = gradient_terms(0, inode) * twelth; + B_matrix_gradients(1, 0, inode, 2) = gradient_terms(1, inode) * twelth; + } + + // B_matrix(2,0) = ( +y(0)*( -z(1) +z(3) ) + // +y(1)*( +z(0) +z(3) -z(5) -z(6) ) + // +y(3)*( -z(0) -z(1) +z(6) +z(7) ) + // +y(5)*( +z(1) -z(6) ) + // +y(6)*( +z(1) -z(3) +z(5) -z(7) ) + // +y(7)*( -z(3) +z(6) ) )*twelth; + + // y derivative + gradient_terms(0, 0) = -z(1) + z(3); + gradient_terms(0, 1) = z(0) + z(3) - z(5) - z(6); + gradient_terms(0, 2) = 0; + gradient_terms(0, 3) = -z(0) - z(1) + z(6) + z(7); + gradient_terms(0, 4) = 0; + gradient_terms(0, 5) = z(1) - z(6); + gradient_terms(0, 6) = z(1) - z(3) + z(5) - z(7); + gradient_terms(0, 7) = -z(3) + z(6); + + // z derivative + gradient_terms(1, 0) = y(1) - y(3); + gradient_terms(1, 1) = -y(0) - y(3) + y(5) + y(6); + gradient_terms(1, 2) = 0; + gradient_terms(1, 3) = y(0) + y(1) - y(6) - y(7); + gradient_terms(1, 4) = 0; + gradient_terms(1, 5) = -y(1) + y(6); + gradient_terms(1, 6) = -y(1) + y(3) - y(5) + y(7); + gradient_terms(1, 7) = y(3) - y(6); + + for (int inode = 0; inode < 8; inode++) + { + B_matrix_gradients(2, 0, inode, 0) = 0; + B_matrix_gradients(2, 0, inode, 1) = gradient_terms(0, inode) * twelth; + B_matrix_gradients(2, 0, inode, 2) = gradient_terms(1, inode) * twelth; + } + + // B_matrix(3,0) = ( +y(0)*( -z(1) -z(2) +z(4) +z(7) ) + // +y(1)*( +z(0) -z(2) ) + // +y(2)*( +z(0) +z(1) -z(6) -z(7) ) + // +y(4)*( -z(0) +z(7) ) + // +y(6)*( +z(2) -z(7) ) + // +y(7)*( -z(0) +z(2) -z(4) +z(6) ) )*twelth; + + // y derivative + gradient_terms(0, 0) = -z(1) - z(2) + z(4) + z(7); + gradient_terms(0, 1) = z(0) - z(2); + gradient_terms(0, 2) = z(0) + z(1) - z(6) - z(7); + gradient_terms(0, 3) = 0; + gradient_terms(0, 4) = -z(0) + z(7); + gradient_terms(0, 5) = 0; + gradient_terms(0, 6) = z(2) - z(7); + gradient_terms(0, 7) = -z(0) + z(2) - z(4) + z(6); + + // z derivative + gradient_terms(1, 0) = y(1) + y(2) - y(4) - y(7); + gradient_terms(1, 1) = -y(0) + y(2); + gradient_terms(1, 2) = -y(0) - y(1) + y(6) + y(7); + gradient_terms(1, 3) = 0; + gradient_terms(1, 4) = y(0) - y(7); + gradient_terms(1, 5) = 0; + gradient_terms(1, 6) = -y(2) + y(7); + gradient_terms(1, 7) = y(0) - y(2) + y(4) - y(6); + + for (int inode = 0; inode < 8; inode++) + { + B_matrix_gradients(3, 0, inode, 0) = 0; + B_matrix_gradients(3, 0, inode, 1) = gradient_terms(0, inode) * twelth; + B_matrix_gradients(3, 0, inode, 2) = gradient_terms(1, inode) * twelth; + } + + // B_matrix(4,0) = ( +y(0)*( +z(1) -z(3) +z(5) -z(7) ) + // +y(1)*( -z(0) +z(5) ) + // +y(3)*( +z(0) -z(7) ) + // +y(5)*( -z(0) -z(1) +z(6) +z(7) ) + // +y(6)*( -z(5) +z(7) ) + // +y(7)*( +z(0) +z(3) -z(5) -z(6) ) )*twelth; + + // y derivative + gradient_terms(0, 0) = z(1) - z(3) + z(5) - z(7); + gradient_terms(0, 1) = -z(0) + z(5); + gradient_terms(0, 2) = 0; + gradient_terms(0, 3) = z(0) - z(7); + gradient_terms(0, 4) = 0; + gradient_terms(0, 5) = -z(0) - z(1) + z(6) + z(7); + gradient_terms(0, 6) = -z(5) + z(7); + gradient_terms(0, 7) = z(0) + z(3) - z(5) - z(6); + + // z derivative + gradient_terms(1, 0) = -y(1) + y(3) - y(5) + y(7); + gradient_terms(1, 1) = y(0) - y(5); + gradient_terms(1, 2) = 0; + gradient_terms(1, 3) = -y(0) + y(7); + gradient_terms(1, 4) = 0; + gradient_terms(1, 5) = y(0) + y(1) - y(6) - y(7); + gradient_terms(1, 6) = y(5) - y(7); + gradient_terms(1, 7) = -y(0) - y(3) + y(5) + y(6); + + for (int inode = 0; inode < 8; inode++) + { + B_matrix_gradients(4, 0, inode, 0) = 0; + B_matrix_gradients(4, 0, inode, 1) = gradient_terms(0, inode) * twelth; + B_matrix_gradients(4, 0, inode, 2) = gradient_terms(1, inode) * twelth; + } + + // B_matrix(5,0) = ( +y(0)*( +z(1) -z(4) ) + // +y(1)*( -z(0) +z(2) -z(4) +z(6) ) + // +y(2)*( -z(1) +z(6) ) + // +y(4)*( +z(0) +z(1) -z(6) -z(7) ) + // +y(6)*( -z(1) -z(2) +z(4) +z(7) ) + // +y(7)*( +z(4) -z(6) ) )*twelth; + + // y derivative + gradient_terms(0, 0) = z(1) - z(4); + gradient_terms(0, 1) = -z(0) + z(2) - z(4) + z(6); + gradient_terms(0, 2) = -z(1) + z(6); + gradient_terms(0, 3) = 0; + gradient_terms(0, 4) = z(0) + z(1) - z(6) - z(7); + gradient_terms(0, 5) = 0; + gradient_terms(0, 6) = -z(1) - z(2) + z(4) + z(7); + gradient_terms(0, 7) = z(4) - z(6); + + // z derivative + gradient_terms(1, 0) = -y(1) + y(4); + gradient_terms(1, 1) = y(0) - y(2) + y(4) - y(6); + gradient_terms(1, 2) = y(1) - y(6); + gradient_terms(1, 3) = 0; + gradient_terms(1, 4) = -y(0) - y(1) + y(6) + y(7); + gradient_terms(1, 5) = 0; + gradient_terms(1, 6) = y(1) + y(2) - y(4) - y(7); + gradient_terms(1, 7) = -y(4) + y(6); + + for (int inode = 0; inode < 8; inode++) + { + B_matrix_gradients(5, 0, inode, 0) = 0; + B_matrix_gradients(5, 0, inode, 1) = gradient_terms(0, inode) * twelth; + B_matrix_gradients(5, 0, inode, 2) = gradient_terms(1, inode) * twelth; + } + + // B_matrix(6,0) = ( +y(1)*( +z(2) -z(5) ) + // +y(2)*( -z(1) +z(3) -z(5) +z(7) ) + // +y(3)*( -z(2) +z(7) ) + // +y(4)*( +z(5) -z(7) ) + // +y(5)*( +z(1) +z(2) -z(4) -z(7) ) + // +y(7)*( -z(2) -z(3) +z(4) +z(5) ) )*twelth; + + // y derivative + gradient_terms(0, 0) = 0; + gradient_terms(0, 1) = z(2) - z(5); + gradient_terms(0, 2) = -z(1) + z(3) - z(5) + z(7); + gradient_terms(0, 3) = -z(2) + z(7); + gradient_terms(0, 4) = z(5) - z(7); + gradient_terms(0, 5) = z(1) + z(2) - z(4) - z(7); + gradient_terms(0, 6) = 0; + gradient_terms(0, 7) = -z(2) - z(3) + z(4) + z(5); + + // z derivative + gradient_terms(1, 0) = 0; + gradient_terms(1, 1) = -y(2) + y(5); + gradient_terms(1, 2) = y(1) - y(3) + y(5) - y(7); + gradient_terms(1, 3) = y(2) - y(7); + gradient_terms(1, 4) = -y(5) + y(7); + gradient_terms(1, 5) = -y(1) - y(2) + y(4) + y(7); + gradient_terms(1, 6) = 0; + gradient_terms(1, 7) = y(2) + y(3) - y(4) - y(5); + + for (int inode = 0; inode < 8; inode++) + { + B_matrix_gradients(6, 0, inode, 0) = 0; + B_matrix_gradients(6, 0, inode, 1) = gradient_terms(0, inode) * twelth; + B_matrix_gradients(6, 0, inode, 2) = gradient_terms(1, inode) * twelth; + } + + // B_matrix(7,0) = ( +y(0)*( -z(3) +z(4) ) + // +y(2)*( +z(3) -z(6) ) + // +y(3)*( +z(0) -z(2) +z(4) -z(6) ) + // +y(4)*( -z(0) -z(3) +z(5) +z(6) ) + // +y(5)*( -z(4) +z(6) ) + // +y(6)*( +z(2) +z(3) -z(4) -z(5) ) )*twelth; + + // y derivative + gradient_terms(0, 0) = -z(3) + z(4); + gradient_terms(0, 1) = 0; + gradient_terms(0, 2) = z(3) - z(6); + gradient_terms(0, 3) = z(0) - z(2) + z(4) - z(6); + gradient_terms(0, 4) = -z(0) - z(3) + z(5) + z(6); + gradient_terms(0, 5) = -z(4) + z(6); + gradient_terms(0, 6) = z(2) + z(3) - z(4) - z(5); + gradient_terms(0, 7) = 0; + + // z derivative + gradient_terms(1, 0) = y(3) - y(4); + gradient_terms(1, 1) = 0; + gradient_terms(1, 2) = -y(3) + y(6); + gradient_terms(1, 3) = -y(0) + y(2) - y(4) + y(6); + gradient_terms(1, 4) = y(0) + y(3) - y(5) - y(6); + gradient_terms(1, 5) = y(4) - y(6); + gradient_terms(1, 6) = -y(2) - y(3) + y(4) + y(5); + gradient_terms(1, 7) = 0; + + for (int inode = 0; inode < 8; inode++) + { + B_matrix_gradients(7, 0, inode, 0) = 0; + B_matrix_gradients(7, 0, inode, 1) = gradient_terms(0, inode) * twelth; + B_matrix_gradients(7, 0, inode, 2) = gradient_terms(1, inode) * twelth; + } + + // B_matrix(0,1) = ( +z(1)*( -x(2) -x(3) +x(4) +x(5) ) + // +z(2)*( +x(1) -x(3) ) + // +z(3)*( +x(1) +x(2) -x(4) -x(7) ) + // +z(4)*( -x(1) +x(3) -x(5) +x(7) ) + // +z(5)*( -x(1) +x(4) ) + // +z(7)*( +x(3) -x(4) ) )*twelth; + + // x derivative + gradient_terms(0, 0) = 0; + gradient_terms(0, 1) = z(2) + z(3) - z(4) - z(5); + gradient_terms(0, 2) = -z(1) + z(3); + gradient_terms(0, 3) = -z(1) - z(2) + z(4) + z(7); + gradient_terms(0, 4) = z(1) - z(3) + z(5) - z(7); + gradient_terms(0, 5) = z(1) - z(4); + gradient_terms(0, 6) = 0; + gradient_terms(0, 7) = -z(3) + z(4); + + // z derivative + gradient_terms(1, 0) = 0; + gradient_terms(1, 1) = -x(2) - x(3) + x(4) + x(5); + gradient_terms(1, 2) = x(1) - x(3); + gradient_terms(1, 3) = x(1) + x(2) - x(4) - x(7); + gradient_terms(1, 4) = -x(1) + x(3) - x(5) + x(7); + gradient_terms(1, 5) = -x(1) + x(4); + gradient_terms(1, 6) = 0; + gradient_terms(1, 7) = x(3) - x(4); + + for (int inode = 0; inode < 8; inode++) + { + B_matrix_gradients(0, 1, inode, 0) = gradient_terms(0, inode) * twelth; + B_matrix_gradients(0, 1, inode, 1) = 0; + B_matrix_gradients(0, 1, inode, 2) = gradient_terms(1, inode) * twelth; + } + + // B_matrix(1,1) = ( +z(0)*( +x(2) +x(3) -x(4) -x(5) ) + // +z(2)*( -x(0) -x(3) +x(5) +x(6) ) + // +z(3)*( -x(0) +x(2) ) + // +z(4)*( +x(0) -x(5) ) + // +z(5)*( +x(0) -x(2) +x(4) -x(6) ) + // +z(6)*( -x(2) +x(5) ) )*twelth; + + // x derivative + gradient_terms(0, 0) = -z(2) - z(3) + z(4) + z(5); + gradient_terms(0, 1) = 0; + gradient_terms(0, 2) = z(0) + z(3) - z(5) - z(6); + gradient_terms(0, 3) = z(0) - z(2); + gradient_terms(0, 4) = -z(0) + z(5); + gradient_terms(0, 5) = -z(0) + z(2) - z(4) + z(6); + gradient_terms(0, 6) = z(2) - z(5); + gradient_terms(0, 7) = 0; + + // z derivative + gradient_terms(1, 0) = x(2) + x(3) - x(4) - x(5); + gradient_terms(1, 1) = 0; + gradient_terms(1, 2) = -x(0) - x(3) + x(5) + x(6); + gradient_terms(1, 3) = -x(0) + x(2); + gradient_terms(1, 4) = x(0) - x(5); + gradient_terms(1, 5) = x(0) - x(2) + x(4) - x(6); + gradient_terms(1, 6) = -x(2) + x(5); + gradient_terms(1, 7) = 0; + + for (int inode = 0; inode < 8; inode++) + { + B_matrix_gradients(1, 1, inode, 0) = gradient_terms(0, inode) * twelth; + B_matrix_gradients(1, 1, inode, 1) = 0; + B_matrix_gradients(1, 1, inode, 2) = gradient_terms(1, inode) * twelth; + } + + // B_matrix(2,1) = ( +z(0)*( -x(1) +x(3) ) + // +z(1)*( +x(0) +x(3) -x(5) -x(6) ) + // +z(3)*( -x(0) -x(1) +x(6) +x(7) ) + // +z(5)*( +x(1) -x(6) ) + // +z(6)*( +x(1) -x(3) +x(5) -x(7) ) + // +z(7)*( -x(3) +x(6) ) )*twelth; + + // x derivative + gradient_terms(0, 0) = z(1) - z(3); + gradient_terms(0, 1) = -z(0) - z(3) + z(5) + z(6); + gradient_terms(0, 2) = 0; + gradient_terms(0, 3) = z(0) + z(1) - z(6) - z(7); + gradient_terms(0, 4) = 0; + gradient_terms(0, 5) = -z(1) + z(6); + gradient_terms(0, 6) = -z(1) + z(3) - z(5) + z(7); + gradient_terms(0, 7) = z(3) - z(6); + + // z derivative + gradient_terms(1, 0) = -x(1) + x(3); + gradient_terms(1, 1) = x(0) + x(3) - x(5) - x(6); + gradient_terms(1, 2) = 0; + gradient_terms(1, 3) = -x(0) - x(1) + x(6) + x(7); + gradient_terms(1, 4) = 0; + gradient_terms(1, 5) = x(1) - x(6); + gradient_terms(1, 6) = x(1) - x(3) + x(5) - x(7); + gradient_terms(1, 7) = -x(3) + x(6); + + for (int inode = 0; inode < 8; inode++) + { + B_matrix_gradients(2, 1, inode, 0) = gradient_terms(0, inode) * twelth; + B_matrix_gradients(2, 1, inode, 1) = 0; + B_matrix_gradients(2, 1, inode, 2) = gradient_terms(1, inode) * twelth; + } + + // B_matrix(3,1) = ( +z(0)*( -x(1) -x(2) +x(4) +x(7) ) + // +z(1)*( +x(0) -x(2) ) + // +z(2)*( +x(0) +x(1) -x(6) -x(7) ) + // +z(4)*( -x(0) +x(7) ) + // +z(6)*( +x(2) -x(7) ) + // +z(7)*( -x(0) +x(2) -x(4) +x(6) ) )*twelth; + + // x derivative + gradient_terms(0, 0) = z(1) + z(2) - z(4) - z(7); + gradient_terms(0, 1) = -z(0) + z(2); + gradient_terms(0, 2) = -z(0) - z(1) + z(6) + z(7); + gradient_terms(0, 3) = 0; + gradient_terms(0, 4) = z(0) - z(7); + gradient_terms(0, 5) = 0; + gradient_terms(0, 6) = -z(2) + z(7); + gradient_terms(0, 7) = z(0) - z(2) + z(4) - z(6); + + // z derivative + gradient_terms(1, 0) = -x(1) - x(2) + x(4) + x(7); + gradient_terms(1, 1) = x(0) - x(2); + gradient_terms(1, 2) = x(0) + x(1) - x(6) - x(7); + gradient_terms(1, 3) = 0; + gradient_terms(1, 4) = -x(0) + x(7); + gradient_terms(1, 5) = 0; + gradient_terms(1, 6) = x(2) - x(7); + gradient_terms(1, 7) = -x(0) + x(2) - x(4) + x(6); + + for (int inode = 0; inode < 8; inode++) + { + B_matrix_gradients(3, 1, inode, 0) = gradient_terms(0, inode) * twelth; + B_matrix_gradients(3, 1, inode, 1) = 0; + B_matrix_gradients(3, 1, inode, 2) = gradient_terms(1, inode) * twelth; + } + + // B_matrix(4,1) = ( +z(0)*( +x(1) -x(3) +x(5) -x(7) ) + // +z(1)*( -x(0) +x(5) ) + // +z(3)*( +x(0) -x(7) ) + // +z(5)*( -x(0) -x(1) +x(6) +x(7) ) + // +z(6)*( -x(5) +x(7) ) + // +z(7)*( +x(0) +x(3) -x(5) -x(6) ) )*twelth; + + // x derivative + gradient_terms(0, 0) = -z(1) + z(3) - z(5) + z(7); + gradient_terms(0, 1) = z(0) - z(5); + gradient_terms(0, 2) = 0; + gradient_terms(0, 3) = -z(0) + z(7); + gradient_terms(0, 4) = 0; + gradient_terms(0, 5) = z(0) + z(1) - z(6) - z(7); + gradient_terms(0, 6) = z(5) - z(7); + gradient_terms(0, 7) = -z(0) - z(3) + z(5) + z(6); + + // z derivative + gradient_terms(1, 0) = x(1) - x(3) + x(5) - x(7); + gradient_terms(1, 1) = -x(0) + x(5); + gradient_terms(1, 2) = 0; + gradient_terms(1, 3) = x(0) - x(7); + gradient_terms(1, 4) = 0; + gradient_terms(1, 5) = -x(0) - x(1) + x(6) + x(7); + gradient_terms(1, 6) = -x(5) + x(7); + gradient_terms(1, 7) = x(0) + x(3) - x(5) - x(6); + + for (int inode = 0; inode < 8; inode++) + { + B_matrix_gradients(4, 1, inode, 0) = gradient_terms(0, inode) * twelth; + B_matrix_gradients(4, 1, inode, 1) = 0; + B_matrix_gradients(4, 1, inode, 2) = gradient_terms(1, inode) * twelth; + } + + // B_matrix(5,1) = ( +z(0)*( +x(1) -x(4) ) + // +z(1)*( -x(0) +x(2) -x(4) +x(6) ) + // +z(2)*( -x(1) +x(6) ) + // +z(4)*( +x(0) +x(1) -x(6) -x(7) ) + // +z(6)*( -x(1) -x(2) +x(4) +x(7) ) + // +z(7)*( +x(4) -x(6) ) )*twelth; + + // x derivative + gradient_terms(0, 0) = -z(1) + z(4); + gradient_terms(0, 1) = z(0) - z(2) + z(4) - z(6); + gradient_terms(0, 2) = z(1) - z(6); + gradient_terms(0, 3) = 0; + gradient_terms(0, 4) = -z(0) - z(1) + z(6) + z(7); + gradient_terms(0, 5) = 0; + gradient_terms(0, 6) = z(1) + z(2) - z(4) - z(7); + gradient_terms(0, 7) = -z(4) + z(6); + + // z derivative + gradient_terms(1, 0) = x(1) - x(4); + gradient_terms(1, 1) = -x(0) + x(2) - x(4) + x(6); + gradient_terms(1, 2) = -x(1) + x(6); + gradient_terms(1, 3) = 0; + gradient_terms(1, 4) = x(0) + x(1) - x(6) - x(7); + gradient_terms(1, 5) = 0; + gradient_terms(1, 6) = -x(1) - x(2) + x(4) + x(7); + gradient_terms(1, 7) = x(4) - x(6); + + for (int inode = 0; inode < 8; inode++) + { + B_matrix_gradients(5, 1, inode, 0) = gradient_terms(0, inode) * twelth; + B_matrix_gradients(5, 1, inode, 1) = 0; + B_matrix_gradients(5, 1, inode, 2) = gradient_terms(1, inode) * twelth; + } + + // B_matrix(6,1) = ( +z(1)*( +x(2) -x(5) ) + // +z(2)*( -x(1) +x(3) -x(5) +x(7) ) + // +z(3)*( -x(2) +x(7) ) + // +z(4)*( +x(5) -x(7) ) + // +z(5)*( +x(1) +x(2) -x(4) -x(7) ) + // +z(7)*( -x(2) -x(3) +x(4) +x(5) ) )*twelth; + + // x derivative + gradient_terms(0, 0) = 0; + gradient_terms(0, 1) = -z(2) + z(5); + gradient_terms(0, 2) = z(1) - z(3) + z(5) - z(7); + gradient_terms(0, 3) = z(2) - z(7); + gradient_terms(0, 4) = -z(5) + z(7); + gradient_terms(0, 5) = -z(1) - z(2) + z(4) + z(7); + gradient_terms(0, 6) = 0; + gradient_terms(0, 7) = z(2) + z(3) - z(4) - z(5); + + // z derivative + gradient_terms(1, 0) = 0; + gradient_terms(1, 1) = x(2) - x(5); + gradient_terms(1, 2) = -x(1) + x(3) - x(5) + x(7); + gradient_terms(1, 3) = -x(2) + x(7); + gradient_terms(1, 4) = x(5) - x(7); + gradient_terms(1, 5) = x(1) + x(2) - x(4) - x(7); + gradient_terms(1, 6) = 0; + gradient_terms(1, 7) = -x(2) - x(3) + x(4) + x(5); + + for (int inode = 0; inode < 8; inode++) + { + B_matrix_gradients(6, 1, inode, 0) = gradient_terms(0, inode) * twelth; + B_matrix_gradients(6, 1, inode, 1) = 0; + B_matrix_gradients(6, 1, inode, 2) = gradient_terms(1, inode) * twelth; + } + + // B_matrix(7,1) = ( +z(0)*( -x(3) +x(4) ) + // +z(2)*( +x(3) -x(6) ) + // +z(3)*( +x(0) -x(2) +x(4) -x(6) ) + // +z(4)*( -x(0) -x(3) +x(5) +x(6) ) + // +z(5)*( -x(4) +x(6) ) + // +z(6)*( +x(2) +x(3) -x(4) -x(5) ) )*twelth; + + // x derivative + gradient_terms(0, 0) = z(3) - z(4); + gradient_terms(0, 1) = 0; + gradient_terms(0, 2) = -z(3) + z(6); + gradient_terms(0, 3) = -z(0) + z(2) - z(4) + z(6); + gradient_terms(0, 4) = z(0) + z(3) - z(5) - z(6); + gradient_terms(0, 5) = z(4) - z(6); + gradient_terms(0, 6) = -z(2) - z(3) + z(4) + z(5); + gradient_terms(0, 7) = 0; + + // z derivative + gradient_terms(1, 0) = -x(3) + x(4); + gradient_terms(1, 1) = 0; + gradient_terms(1, 2) = x(3) - x(6); + gradient_terms(1, 3) = x(0) - x(2) + x(4) - x(6); + gradient_terms(1, 4) = -x(0) - x(3) + x(5) + x(6); + gradient_terms(1, 5) = -x(4) + x(6); + gradient_terms(1, 6) = x(2) + x(3) - x(4) - x(5); + gradient_terms(1, 7) = 0; + + for (int inode = 0; inode < 8; inode++) + { + B_matrix_gradients(7, 1, inode, 0) = gradient_terms(0, inode) * twelth; + B_matrix_gradients(7, 1, inode, 1) = 0; + B_matrix_gradients(7, 1, inode, 2) = gradient_terms(1, inode) * twelth; + } + + // B_matrix(0,2) = ( +x(1)*( -y(2) -y(3) +y(4) +y(5) ) + // +x(2)*( +y(1) -y(3) ) + // +x(3)*( +y(1) +y(2) -y(4) -y(7) ) + // +x(4)*( -y(1) +y(3) -y(5) +y(7) ) + // +x(5)*( -y(1) +y(4) ) + // +x(7)*( +y(3) -y(4) ) )*twelth; + + // x derivative + gradient_terms(0, 0) = 0; + gradient_terms(0, 1) = -y(2) - y(3) + y(4) + y(5); + gradient_terms(0, 2) = y(1) - y(3); + gradient_terms(0, 3) = y(1) + y(2) - y(4) - y(7); + gradient_terms(0, 4) = -y(1) + y(3) - y(5) + y(7); + gradient_terms(0, 5) = -y(1) + y(4); + gradient_terms(0, 6) = 0; + gradient_terms(0, 7) = y(3) - y(4); + + // y derivative + gradient_terms(1, 0) = 0; + gradient_terms(1, 1) = x(2) + x(3) - x(4) - x(5); + gradient_terms(1, 2) = -x(1) + x(3); + gradient_terms(1, 3) = -x(1) - x(2) + x(4) + x(7); + gradient_terms(1, 4) = x(1) - x(3) + x(5) - x(7); + gradient_terms(1, 5) = x(1) - x(4); + gradient_terms(1, 6) = 0; + gradient_terms(1, 7) = -x(3) + x(4); + + for (int inode = 0; inode < 8; inode++) + { + B_matrix_gradients(0, 2, inode, 0) = gradient_terms(0, inode) * twelth; + B_matrix_gradients(0, 2, inode, 1) = gradient_terms(1, inode) * twelth; + B_matrix_gradients(0, 2, inode, 2) = 0; + } + + // B_matrix(1,2) = ( +x(0)*( +y(2) +y(3) -y(4) -y(5) ) + // +x(2)*( -y(0) -y(3) +y(5) +y(6) ) + // +x(3)*( -y(0) +y(2) ) + // +x(4)*( +y(0) -y(5) ) + // +x(5)*( +y(0) -y(2) +y(4) -y(6) ) + // +x(6)*( -y(2) +y(5) ) )*twelth; + + // x derivative + gradient_terms(0, 0) = y(2) + y(3) - y(4) - y(5); + gradient_terms(0, 1) = 0; + gradient_terms(0, 2) = -y(0) - y(3) + y(5) + y(6); + gradient_terms(0, 3) = -y(0) + y(2); + gradient_terms(0, 4) = y(0) - y(5); + gradient_terms(0, 5) = y(0) - y(2) + y(4) - y(6); + gradient_terms(0, 6) = -y(2) + y(5); + gradient_terms(0, 7) = 0; + + // y derivative + gradient_terms(1, 0) = -x(2) - x(3) + x(4) + x(5); + gradient_terms(1, 1) = 0; + gradient_terms(1, 2) = x(0) + x(3) - x(5) - x(6); + gradient_terms(1, 3) = x(0) - x(2); + gradient_terms(1, 4) = -x(0) + x(5); + gradient_terms(1, 5) = -x(0) + x(2) - x(4) + x(6); + gradient_terms(1, 6) = x(2) - x(5); + gradient_terms(1, 7) = 0; + + for (int inode = 0; inode < 8; inode++) + { + B_matrix_gradients(1, 2, inode, 0) = gradient_terms(0, inode) * twelth; + B_matrix_gradients(1, 2, inode, 1) = gradient_terms(1, inode) * twelth; + B_matrix_gradients(1, 2, inode, 2) = 0; + } + + // B_matrix(2,2) = ( +x(0)*( -y(1) +y(3) ) + // +x(1)*( +y(0) +y(3) -y(5) -y(6) ) + // +x(3)*( -y(0) -y(1) +y(6) +y(7) ) + // +x(5)*( +y(1) -y(6) ) + // +x(6)*( +y(1) -y(3) +y(5) -y(7) ) + // +x(7)*( -y(3) +y(6) ) )*twelth; + + // x derivative + gradient_terms(0, 0) = -y(1) + y(3); + gradient_terms(0, 1) = y(0) + y(3) - y(5) - y(6); + gradient_terms(0, 2) = 0; + gradient_terms(0, 3) = -y(0) - y(1) + y(6) + y(7); + gradient_terms(0, 4) = 0; + gradient_terms(0, 5) = y(1) - y(6); + gradient_terms(0, 6) = y(1) - y(3) + y(5) - y(7); + gradient_terms(0, 7) = -y(3) + y(6); + + // y derivative + gradient_terms(1, 0) = x(1) - x(3); + gradient_terms(1, 1) = -x(0) - x(3) + x(5) + x(6); + gradient_terms(1, 2) = 0; + gradient_terms(1, 3) = x(0) + x(1) - x(6) - x(7); + gradient_terms(1, 4) = 0; + gradient_terms(1, 5) = -x(1) + x(6); + gradient_terms(1, 6) = -x(1) + x(3) - x(5) + x(7); + gradient_terms(1, 7) = x(3) - x(6); + + for (int inode = 0; inode < 8; inode++) + { + B_matrix_gradients(2, 2, inode, 0) = gradient_terms(0, inode) * twelth; + B_matrix_gradients(2, 2, inode, 1) = gradient_terms(1, inode) * twelth; + B_matrix_gradients(2, 2, inode, 2) = 0; + } + + // B_matrix(3,2) = ( +x(0)*( -y(1) -y(2) +y(4) +y(7) ) + // +x(1)*( +y(0) -y(2) ) + // +x(2)*( +y(0) +y(1) -y(6) -y(7) ) + // +x(4)*( -y(0) +y(7) ) + // +x(6)*( +y(2) -y(7) ) + // +x(7)*( -y(0) +y(2) -y(4) +y(6) ) )*twelth; + + // x derivative + gradient_terms(0, 0) = -y(1) - y(2) + y(4) + y(7); + gradient_terms(0, 1) = y(0) - y(2); + gradient_terms(0, 2) = y(0) + y(1) - y(6) - y(7); + gradient_terms(0, 3) = 0; + gradient_terms(0, 4) = -y(0) + y(7); + gradient_terms(0, 5) = 0; + gradient_terms(0, 6) = y(2) - y(7); + gradient_terms(0, 7) = -y(0) + y(2) - y(4) + y(6); + + // y derivative + gradient_terms(1, 0) = x(1) + x(2) - x(4) - x(7); + gradient_terms(1, 1) = -x(0) + x(2); + gradient_terms(1, 2) = -x(0) - x(1) + x(6) + x(7); + gradient_terms(1, 3) = 0; + gradient_terms(1, 4) = x(0) - x(7); + gradient_terms(1, 5) = 0; + gradient_terms(1, 6) = -x(2) + x(7); + gradient_terms(1, 7) = x(0) - x(2) + x(4) - x(6); + + for (int inode = 0; inode < 8; inode++) + { + B_matrix_gradients(3, 2, inode, 0) = gradient_terms(0, inode) * twelth; + B_matrix_gradients(3, 2, inode, 1) = gradient_terms(1, inode) * twelth; + B_matrix_gradients(3, 2, inode, 2) = 0; + } + + // B_matrix(4,2) = ( +x(0)*( +y(1) -y(3) +y(5) -y(7) ) + // +x(1)*( -y(0) +y(5) ) + // +x(3)*( +y(0) -y(7) ) + // +x(5)*( -y(0) -y(1) +y(6) +y(7) ) + // +x(6)*( -y(5) +y(7) ) + // +x(7)*( +y(0) +y(3) -y(5) -y(6) ) )*twelth; + + // x derivative + gradient_terms(0, 0) = y(1) - y(3) + y(5) - y(7); + gradient_terms(0, 1) = -y(0) + y(5); + gradient_terms(0, 2) = 0; + gradient_terms(0, 3) = y(0) - y(7); + gradient_terms(0, 4) = 0; + gradient_terms(0, 5) = -y(0) - y(1) + y(6) + y(7); + gradient_terms(0, 6) = -y(5) + y(7); + gradient_terms(0, 7) = y(0) + y(3) - y(5) - y(6); + + // y derivative + gradient_terms(1, 0) = -x(1) + x(3) - x(4) + x(7); + gradient_terms(1, 1) = x(0) - x(5); + gradient_terms(1, 2) = 0; + gradient_terms(1, 3) = -x(0) + x(7); + gradient_terms(1, 4) = 0; + gradient_terms(1, 5) = x(0) + x(1) - x(6) - x(7); + gradient_terms(1, 6) = x(5) - x(7); + gradient_terms(1, 7) = -x(0) - x(3) + x(5) + x(6); + + for (int inode = 0; inode < 8; inode++) + { + B_matrix_gradients(4, 2, inode, 0) = gradient_terms(0, inode) * twelth; + B_matrix_gradients(4, 2, inode, 1) = gradient_terms(1, inode) * twelth; + B_matrix_gradients(4, 2, inode, 2) = 0; + } + + // B_matrix(5,2) = ( +x(0)*( +y(1) -y(4) ) + // +x(1)*( -y(0) +y(2) -y(4) +y(6) ) + // +x(2)*( -y(1) +y(6) ) + // +x(4)*( +y(0) +y(1) -y(6) -y(7) ) + // +x(6)*( -y(1) -y(2) +y(4) +y(7) ) + // +x(7)*( +y(4) -y(6) ) )*twelth; + + // x derivative + gradient_terms(0, 0) = y(1) - y(4); + gradient_terms(0, 1) = -y(0) + y(2) - y(4) + y(6); + gradient_terms(0, 2) = -y(1) + y(6); + gradient_terms(0, 3) = 0; + gradient_terms(0, 4) = y(0) + y(1) - y(6) - y(7); + gradient_terms(0, 5) = 0; + gradient_terms(0, 6) = -y(1) - y(2) + y(4) + y(7); + gradient_terms(0, 7) = y(4) - y(6); + + // y derivative + gradient_terms(1, 0) = -x(1) + x(4); + gradient_terms(1, 1) = x(0) - x(2) + x(4) - x(6); + gradient_terms(1, 2) = x(1) - x(6); + gradient_terms(1, 3) = 0; + gradient_terms(1, 4) = -x(0) - x(1) + x(6) + x(7); + gradient_terms(1, 5) = 0; + gradient_terms(1, 6) = x(1) + x(2) - x(4) - x(7); + gradient_terms(1, 7) = -x(4) + x(6); + + for (int inode = 0; inode < 8; inode++) + { + B_matrix_gradients(5, 2, inode, 0) = gradient_terms(0, inode) * twelth; + B_matrix_gradients(5, 2, inode, 1) = gradient_terms(1, inode) * twelth; + B_matrix_gradients(5, 2, inode, 2) = 0; + } + + // B_matrix(6,2) = ( +x(1)*( +y(2) -y(5) ) + // +x(2)*( -y(1) +y(3) -y(5) +y(7) ) + // +x(3)*( -y(2) +y(7) ) + // +x(4)*( +y(5) -y(7) ) + // +x(5)*( +y(1) +y(2) -y(4) -y(7) ) + // +x(7)*( -y(2) -y(3) +y(4) +y(5) ) )*twelth; + + // x derivative + gradient_terms(0, 0) = 0; + gradient_terms(0, 1) = y(2) - y(5); + gradient_terms(0, 2) = -y(1) + y(3) - y(5) + y(7); + gradient_terms(0, 3) = -y(2) + y(7); + gradient_terms(0, 4) = y(5) - y(7); + gradient_terms(0, 5) = y(1) + y(2) - y(4) - y(7); + gradient_terms(0, 6) = 0; + gradient_terms(0, 7) = -y(2) - y(3) + y(4) + y(5); + + // y derivative + gradient_terms(1, 0) = 0; + gradient_terms(1, 1) = -x(2) + x(5); + gradient_terms(1, 2) = x(1) - x(3) + x(5) - x(7); + gradient_terms(1, 3) = x(2) - x(7); + gradient_terms(1, 4) = -x(5) + x(7); + gradient_terms(1, 5) = -x(1) - x(2) + x(4) + x(7); + gradient_terms(1, 6) = 0; + gradient_terms(1, 7) = x(2) + x(3) - x(4) - x(5); + + for (int inode = 0; inode < 8; inode++) + { + B_matrix_gradients(6, 2, inode, 0) = gradient_terms(0, inode) * twelth; + B_matrix_gradients(6, 2, inode, 1) = gradient_terms(1, inode) * twelth; + B_matrix_gradients(6, 2, inode, 2) = 0; + } + + // B_matrix(7,2) = ( +x(0)*( -y(3) +y(4) ) + // +x(2)*( +y(3) -y(6) ) + // +x(3)*( +y(0) -y(2) +y(4) -y(6) ) + // +x(4)*( -y(0) -y(3) +y(5) +y(6) ) + // +x(5)*( -y(4) +y(6) ) + // +x(6)*( +y(2) +y(3) -y(4) -y(5) ) )*twelth; + + // x derivative + gradient_terms(0, 0) = -y(3) + y(4); + gradient_terms(0, 1) = 0; + gradient_terms(0, 2) = y(3) - y(6); + gradient_terms(0, 3) = y(0) - y(2) + y(4) - y(6); + gradient_terms(0, 4) = -y(0) - y(3) + y(5) + y(6); + gradient_terms(0, 5) = -y(4) + y(6); + gradient_terms(0, 6) = y(2) + y(3) - y(4) - y(5); + gradient_terms(0, 7) = 0; + + // y derivative + gradient_terms(1, 0) = x(3) - x(4); + gradient_terms(1, 1) = 0; + gradient_terms(1, 2) = -x(3) + x(6); + gradient_terms(1, 3) = -x(0) + x(2) - x(4) + x(6); + gradient_terms(1, 4) = x(0) + x(3) - x(5) - x(6); + gradient_terms(1, 5) = x(4) - x(6); + gradient_terms(1, 6) = -x(2) - x(3) + x(4) + x(5); + gradient_terms(1, 7) = 0; + + for (int inode = 0; inode < 8; inode++) + { + B_matrix_gradients(7, 2, inode, 0) = gradient_terms(0, inode) * twelth; + B_matrix_gradients(7, 2, inode, 1) = gradient_terms(1, inode) * twelth; + B_matrix_gradients(7, 2, inode, 2) = 0; + } +} // end subroutine diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/momentum.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/momentum.cpp new file mode 100644 index 000000000..03452ccd1 --- /dev/null +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/momentum.cpp @@ -0,0 +1,391 @@ + +#include "state.h" +#include "FEA_Module_SGH.h" + +// ----------------------------------------------------------------------------- +// This function evolves the velocity at the nodes of the mesh +// ------------------------------------------------------------------------------ +void FEA_Module_SGH::update_velocity_sgh(double rk_alpha, + DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_mass, + const DViewCArrayKokkos& corner_force + ) +{ + const size_t rk_level = rk_num_bins - 1; + const size_t num_dims = num_dim; + + // walk over the nodes to update the velocity + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + double node_force[3]; + for (size_t dim = 0; dim < num_dims; dim++) + { + node_force[dim] = 0.0; + } // end for dim + + // loop over all corners around the node and calculate the nodal force + for (size_t corner_lid = 0; corner_lid < num_corners_in_node(node_gid); corner_lid++) + { + // Get corner gid + size_t corner_gid = corners_in_node(node_gid, corner_lid); + + // loop over dimension + for (size_t dim = 0; dim < num_dims; dim++) + { + node_force[dim] += corner_force(corner_gid, dim); + } // end for dim + } // end for corner_lid + + // update the velocity + for (int dim = 0; dim < num_dims; dim++) + { + node_vel(rk_level, node_gid, dim) = node_vel(0, node_gid, dim) + + rk_alpha * dt * node_force[dim] / node_mass(node_gid); + } // end for dim + }); // end for parallel for over nodes + + return; +} // end subroutine update_velocity + +// ----------------------------------------------------------------------------- +// This function calculates the velocity gradient +// ------------------------------------------------------------------------------ +KOKKOS_FUNCTION +void FEA_Module_SGH::get_velgrad(ViewCArrayKokkos& vel_grad, + const ViewCArrayKokkos& elem_node_gids, + const DViewCArrayKokkos& node_vel, + const ViewCArrayKokkos& b_matrix, + const double elem_vol, + const size_t elem_gid, + const size_t rk_level + ) const +{ + const size_t num_nodes_in_elem = 8; + + double u_array[num_nodes_in_elem]; // x-dir vel component + double v_array[num_nodes_in_elem]; // y-dir vel component + double w_array[num_nodes_in_elem]; // z-dir vel component + + ViewCArrayKokkos u(u_array, num_nodes_in_elem); + ViewCArrayKokkos v(v_array, num_nodes_in_elem); + ViewCArrayKokkos w(w_array, num_nodes_in_elem); + + // get the vertex velocities for the cell + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + // Get node gid + size_t node_gid = elem_node_gids(node_lid); + + u(node_lid) = node_vel(rk_level, node_gid, 0); + v(node_lid) = node_vel(rk_level, node_gid, 1); + w(node_lid) = node_vel(rk_level, node_gid, 2); + } // end for + + // --- calculate the velocity gradient terms --- + double inverse_vol = 1.0 / elem_vol; + + // x-dir + vel_grad(0, 0) = (u(0) * b_matrix(0, 0) + u(1) * b_matrix(1, 0) + + u(2) * b_matrix(2, 0) + u(3) * b_matrix(3, 0) + + u(4) * b_matrix(4, 0) + u(5) * b_matrix(5, 0) + + u(6) * b_matrix(6, 0) + u(7) * b_matrix(7, 0)) * inverse_vol; + + vel_grad(0, 1) = (u(0) * b_matrix(0, 1) + u(1) * b_matrix(1, 1) + + u(2) * b_matrix(2, 1) + u(3) * b_matrix(3, 1) + + u(4) * b_matrix(4, 1) + u(5) * b_matrix(5, 1) + + u(6) * b_matrix(6, 1) + u(7) * b_matrix(7, 1)) * inverse_vol; + + vel_grad(0, 2) = (u(0) * b_matrix(0, 2) + u(1) * b_matrix(1, 2) + + u(2) * b_matrix(2, 2) + u(3) * b_matrix(3, 2) + + u(4) * b_matrix(4, 2) + u(5) * b_matrix(5, 2) + + u(6) * b_matrix(6, 2) + u(7) * b_matrix(7, 2)) * inverse_vol; + + // y-dir + vel_grad(1, 0) = (v(0) * b_matrix(0, 0) + v(1) * b_matrix(1, 0) + + v(2) * b_matrix(2, 0) + v(3) * b_matrix(3, 0) + + v(4) * b_matrix(4, 0) + v(5) * b_matrix(5, 0) + + v(6) * b_matrix(6, 0) + v(7) * b_matrix(7, 0)) * inverse_vol; + + vel_grad(1, 1) = (v(0) * b_matrix(0, 1) + v(1) * b_matrix(1, 1) + + v(2) * b_matrix(2, 1) + v(3) * b_matrix(3, 1) + + v(4) * b_matrix(4, 1) + v(5) * b_matrix(5, 1) + + v(6) * b_matrix(6, 1) + v(7) * b_matrix(7, 1)) * inverse_vol; + vel_grad(1, 2) = (v(0) * b_matrix(0, 2) + v(1) * b_matrix(1, 2) + + v(2) * b_matrix(2, 2) + v(3) * b_matrix(3, 2) + + v(4) * b_matrix(4, 2) + v(5) * b_matrix(5, 2) + + v(6) * b_matrix(6, 2) + v(7) * b_matrix(7, 2)) * inverse_vol; + + // z-dir + vel_grad(2, 0) = (w(0) * b_matrix(0, 0) + w(1) * b_matrix(1, 0) + + w(2) * b_matrix(2, 0) + w(3) * b_matrix(3, 0) + + w(4) * b_matrix(4, 0) + w(5) * b_matrix(5, 0) + + w(6) * b_matrix(6, 0) + w(7) * b_matrix(7, 0)) * inverse_vol; + + vel_grad(2, 1) = (w(0) * b_matrix(0, 1) + w(1) * b_matrix(1, 1) + + w(2) * b_matrix(2, 1) + w(3) * b_matrix(3, 1) + + w(4) * b_matrix(4, 1) + w(5) * b_matrix(5, 1) + + w(6) * b_matrix(6, 1) + w(7) * b_matrix(7, 1)) * inverse_vol; + + vel_grad(2, 2) = (w(0) * b_matrix(0, 2) + w(1) * b_matrix(1, 2) + + w(2) * b_matrix(2, 2) + w(3) * b_matrix(3, 2) + + w(4) * b_matrix(4, 2) + w(5) * b_matrix(5, 2) + + w(6) * b_matrix(6, 2) + w(7) * b_matrix(7, 2)) * inverse_vol; + + return; +} // end function + +// ----------------------------------------------------------------------------- +// This function calculates the velocity gradient +// ------------------------------------------------------------------------------ +KOKKOS_FUNCTION +void FEA_Module_SGH::get_velgrad2D(ViewCArrayKokkos& vel_grad, + const ViewCArrayKokkos& elem_node_gids, + const DViewCArrayKokkos& node_vel, + const ViewCArrayKokkos& b_matrix, + const double elem_vol, + const double elem_area, + const size_t elem_gid, + const size_t rk_level + ) const +{ + const size_t num_nodes_in_elem = 4; + + double u_array[num_nodes_in_elem]; + double v_array[num_nodes_in_elem]; + ViewCArrayKokkos u(u_array, num_nodes_in_elem); // x-dir vel component + ViewCArrayKokkos v(v_array, num_nodes_in_elem); // y-dir vel component + + // get the vertex velocities for the cell + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + // Get node gid + size_t node_gid = elem_node_gids(node_lid); + + u(node_lid) = node_vel(rk_level, node_gid, 0); // x-comp + v(node_lid) = node_vel(rk_level, node_gid, 1); // y-comp + } // end for + + // initialize to zero + for (size_t i = 0; i < 3; i++) + { + for (size_t j = 0; j < 3; j++) + { + vel_grad(i, j) = 0.0; + } + } + + double mean_radius = elem_vol / elem_area; + double elem_vel_r = 0.25 * (v(0) + v(1) + v(2) + v(3)); + + // --- calculate the velocity gradient terms --- + double inverse_area = 1.0 / elem_area; + + // x-dir + vel_grad(0, 0) = (u(0) * b_matrix(0, 0) + u(1) * b_matrix(1, 0) + + u(2) * b_matrix(2, 0) + u(3) * b_matrix(3, 0)) * inverse_area; + + vel_grad(0, 1) = (u(0) * b_matrix(0, 1) + u(1) * b_matrix(1, 1) + + u(2) * b_matrix(2, 1) + u(3) * b_matrix(3, 1)) * inverse_area; + + // y-dir + vel_grad(1, 0) = (v(0) * b_matrix(0, 0) + v(1) * b_matrix(1, 0) + + v(2) * b_matrix(2, 0) + v(3) * b_matrix(3, 0)) * inverse_area; + + vel_grad(1, 1) = (v(0) * b_matrix(0, 1) + v(1) * b_matrix(1, 1) + + v(2) * b_matrix(2, 1) + v(3) * b_matrix(3, 1)) * inverse_area; + + vel_grad(2, 2) = elem_vel_r / mean_radius; // + avg(vel_R)/R + + return; +} // end function + +// ----------------------------------------------------------------------------- +// This subroutine to calculate the velocity divergence in all elements +// ------------------------------------------------------------------------------ +void FEA_Module_SGH::get_divergence(DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_vol + ) +{ + const size_t rk_level = rk_num_bins - 1; + + // --- calculate the forces acting on the nodes from the element --- + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { + const size_t num_nodes_in_elem = 8; + const size_t num_dims = 3; + + double u_array[num_nodes_in_elem]; + double v_array[num_nodes_in_elem]; + double w_array[num_nodes_in_elem]; + ViewCArrayKokkos u(u_array, num_nodes_in_elem); // x-dir vel component + ViewCArrayKokkos v(v_array, num_nodes_in_elem); // y-dir vel component + ViewCArrayKokkos w(w_array, num_nodes_in_elem); // z-dir vel component + + // cut out the node_gids for this element + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 8); + + // The b_matrix are the outward corner area normals + double b_matrix_array[24]; + ViewCArrayKokkos b_matrix(b_matrix_array, num_nodes_in_elem, num_dims); + get_bmatrix(b_matrix, + elem_gid, + node_coords, + elem_node_gids, + rk_level); + + // get the vertex velocities for the elem + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + // Get node gid + size_t node_gid = elem_node_gids(node_lid); + + u(node_lid) = node_vel(rk_level, node_gid, 0); + v(node_lid) = node_vel(rk_level, node_gid, 1); + w(node_lid) = node_vel(rk_level, node_gid, 2); + } // end for + + // --- calculate the velocity divergence terms --- + double inverse_vol = 1.0 / elem_vol(elem_gid); + + elem_div(elem_gid) = 0.0; + + // x-dir + elem_div(elem_gid) += (u(0) * b_matrix(0, 0) + u(1) * b_matrix(1, 0) + + u(2) * b_matrix(2, 0) + u(3) * b_matrix(3, 0) + + u(4) * b_matrix(4, 0) + u(5) * b_matrix(5, 0) + + u(6) * b_matrix(6, 0) + u(7) * b_matrix(7, 0)) * inverse_vol; + + // y-dir + elem_div(elem_gid) += (v(0) * b_matrix(0, 1) + v(1) * b_matrix(1, 1) + + v(2) * b_matrix(2, 1) + v(3) * b_matrix(3, 1) + + v(4) * b_matrix(4, 1) + v(5) * b_matrix(5, 1) + + v(6) * b_matrix(6, 1) + v(7) * b_matrix(7, 1)) * inverse_vol; + + // z-dir + elem_div(elem_gid) += (w(0) * b_matrix(0, 2) + w(1) * b_matrix(1, 2) + + w(2) * b_matrix(2, 2) + w(3) * b_matrix(3, 2) + + w(4) * b_matrix(4, 2) + w(5) * b_matrix(5, 2) + + w(6) * b_matrix(6, 2) + w(7) * b_matrix(7, 2)) * inverse_vol; + }); // end parallel for over elem_gid + + return; +} // end subroutine + +// ----------------------------------------------------------------------------- +// This subroutine to calculate the velocity divergence in all elements +// ------------------------------------------------------------------------------ +void FEA_Module_SGH::get_divergence2D(DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_vol + ) +{ + const size_t rk_level = rk_num_bins - 1; + + // --- calculate the forces acting on the nodes from the element --- + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { + const size_t num_nodes_in_elem = 4; + const size_t num_dims = 2; + + double u_array[num_nodes_in_elem]; + double v_array[num_nodes_in_elem]; + ViewCArrayKokkos u(u_array, num_nodes_in_elem); // x-dir vel component + ViewCArrayKokkos v(v_array, num_nodes_in_elem); // y-dir vel component + + // true volume RZ + // double r_array[num_nodes_in_elem]; + // ViewCArrayKokkos r(r_array, num_nodes_in_elem); // r-dir coordinate + + // cut out the node_gids for this element + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 4); + + // The b_matrix are the outward corner area normals + double b_matrix_array[24]; + ViewCArrayKokkos b_matrix(b_matrix_array, num_nodes_in_elem, num_dims); + get_bmatrix2D(b_matrix, + elem_gid, + node_coords, + elem_node_gids, + rk_level); + + // calculate the area of the quad + double elem_area = get_area_quad(elem_gid, node_coords, elem_node_gids, rk_level); + // true volume uses the elem_vol + + // get the vertex velocities and node coordinate for the elem + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + // Get node gid + size_t node_gid = elem_node_gids(node_lid); + + u(node_lid) = node_vel(rk_level, node_gid, 0); + v(node_lid) = node_vel(rk_level, node_gid, 1); + + // r(node_lid) = node_coords(rk_level, node_gid, 1); // true volume RZ + } // end for + + // --- calculate the velocity divergence terms --- + double inverse_area = 1.0 / elem_area; + + double mean_radius = elem_vol(elem_gid) / elem_area; + double elem_vel_r = 0.25 * (v(0) + v(1) + v(2) + v(3)); + + elem_div(elem_gid) = 0.0; + + // x-dir + elem_div(elem_gid) += (u(0) * b_matrix(0, 0) + + u(1) * b_matrix(1, 0) + + u(2) * b_matrix(2, 0) + + u(3) * b_matrix(3, 0)) * inverse_area; + + // y-dir (i.e., r direction) + elem_div(elem_gid) += (v(0) * b_matrix(0, 1) + + v(1) * b_matrix(1, 1) + + v(2) * b_matrix(2, 1) + + v(3) * b_matrix(3, 1)) * inverse_area + + elem_vel_r / mean_radius; // + avg(u_R)/R + }); // end parallel for over elem_gid + + return; +} // end subroutine + +// The velocity gradient can be decomposed into symmetric and antisymmetric tensors +// L = vel_grad +// D = sym(L) +// W = antisym(L) +KOKKOS_INLINE_FUNCTION +void FEA_Module_SGH::decompose_vel_grad(ViewCArrayKokkos& D_tensor, + ViewCArrayKokkos& W_tensor, + const ViewCArrayKokkos& vel_grad, + const ViewCArrayKokkos& elem_node_gids, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const double vol + ) const +{ + // --- Calculate the velocity gradient --- + + const size_t num_dims = 3; + + // initialize to zero + for (size_t i = 0; i < num_dims; i++) + { + for (size_t j = 0; j < num_dims; j++) + { + D_tensor(i, j) = 0.0; + W_tensor(i, j) = 0.0; + } + } // end for + + for (size_t i = 0; i < num_dims; i++) + { + for (size_t j = 0; j < num_dims; j++) + { + D_tensor(i, j) = 0.5 * (vel_grad(i, j) + vel_grad(j, i)); + W_tensor(i, j) = 0.5 * (vel_grad(i, j) - vel_grad(j, i)); + } + } // end for + + return; +} // end function to calculate D and W diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/power_gradients_sgh.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/power_gradients_sgh.cpp new file mode 100644 index 000000000..ad7e648d5 --- /dev/null +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/power_gradients_sgh.cpp @@ -0,0 +1,495 @@ + +#include "mesh.h" +#include "state.h" +#include "FEA_Module_SGH.h" +#include "Simulation_Parameters/Simulation_Parameters_Explicit.h" +#include "Simulation_Parameters/FEA_Module/SGH_Parameters.h" + +// -------------------------------------------------------------------------------------------------------- +// Computes term objective derivative term involving gradient of power with respect to the design variable +// --------------------------------------------------------------------------------------------------------- + +void FEA_Module_SGH::power_design_gradient_term(const_vec_array design_variables, vec_array design_gradients) +{ + bool element_constant_density = true; + size_t current_data_index, next_data_index; + size_t num_bdy_nodes = mesh->num_bdy_nodes; + int num_corners = rnum_elem * num_nodes_in_elem; + real_t global_dt; + + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos material = simparam->material; + const int num_dim = simparam->num_dims; + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + + CArrayKokkos current_element_adjoint = CArrayKokkos(num_nodes_in_elem, num_dim); + + DCArrayKokkos elem_power_dgradients(rnum_elem); + + // gradient contribution from gradient of Force vector with respect to design variable. + if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) + { + if (myrank == 0) + { + std::cout << "gradient term involving adjoint derivative" << std::endl; + } + } + + for (unsigned long cycle = 0; cycle < last_time_step + 1; cycle++) + { + // compute timestep from time data + global_dt = time_data[cycle + 1] - time_data[cycle]; + // print + if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) + { + if (cycle == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } + // print time step every 10 cycles + else if (cycle % 20 == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } // end if + } + + // view scope + { + const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array current_element_internal_energy = (*forward_solve_internal_energy_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array current_coord_vector = (*forward_solve_coordinate_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array current_psi_adjoint_vector = (*psi_adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_velocity_vector = (*forward_solve_velocity_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_element_internal_energy = (*forward_solve_internal_energy_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_coord_vector = (*forward_solve_coordinate_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_psi_adjoint_vector = (*psi_adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + + // first half of integration step calculation + FOR_ALL_CLASS(node_gid, 0, nall_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + node_vel(rk_level, node_gid, idim) = current_velocity_vector(node_gid, idim); + node_coords(rk_level, node_gid, idim) = current_coord_vector(node_gid, idim); + } + }); // end parallel for + Kokkos::fence(); + + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { + elem_sie(rk_level, elem_gid) = current_element_internal_energy(elem_gid, 0); + }); // end parallel for + Kokkos::fence(); + + get_vol(); + + // ---- Calculate velocity diveregence for the element ---- + if (num_dim == 2) + { + get_divergence2D(elem_div, node_coords, node_vel, elem_vol); + } + else + { + get_divergence(elem_div, node_coords, node_vel, elem_vol); + } // end if 2D + + // ---- Calculate elem state (den, pres, sound speed, stress) for next time step ---- + if (num_dim == 2) + { + update_state2D(material, + *mesh, + node_coords, + node_vel, + elem_den, + elem_pres, + elem_stress, + elem_sspd, + elem_sie, + elem_vol, + elem_mass, + elem_mat_id, + 1.0, + cycle); + } + else + { + update_state(material, + *mesh, + node_coords, + node_vel, + elem_den, + elem_pres, + elem_stress, + elem_sspd, + elem_sie, + elem_vol, + elem_mass, + elem_mat_id, + 1.0, + cycle); + } + + // ---- calculate the forces on the vertices and evolve stress (hypo model) ---- + if (num_dim == 2) + { + get_force_sgh2D(material, + *mesh, + node_coords, + node_vel, + elem_den, + elem_sie, + elem_pres, + elem_stress, + elem_sspd, + elem_vol, + elem_div, + elem_mat_id, + corner_force, + 1.0, + cycle); + } + else + { + get_force_sgh(material, + *mesh, + node_coords, + node_vel, + elem_den, + elem_sie, + elem_pres, + elem_stress, + elem_sspd, + elem_vol, + elem_div, + elem_mat_id, + corner_force, + 1.0, + cycle); + } + + get_force_dgradient_sgh(material, + *mesh, + node_coords, + node_vel, + elem_den, + elem_sie, + elem_pres, + elem_stress, + elem_sspd, + elem_vol, + elem_div, + elem_mat_id, + 1.0, + cycle); + + get_power_dgradient_sgh(1.0, + *mesh, + node_vel, + node_coords, + elem_sie, + elem_mass, + corner_force, + elem_power_dgradients); + + // derivatives of forces at corners stored in corner_vector_storage buffer by previous routine + FOR_ALL_CLASS(elem_id, 0, rnum_elem, { + size_t node_id; + size_t corner_id; + real_t inner_product; + + inner_product = current_psi_adjoint_vector(elem_id, 0) * elem_power_dgradients(elem_id); + + for (int inode = 0; inode < num_nodes_in_elem; inode++) + { + // compute gradient of local element contribution to v^t*M*v product + corner_id = elem_id * num_nodes_in_elem + inode; + corner_value_storage(corner_id) = inner_product; + } + }); // end parallel for + Kokkos::fence(); + + // accumulate node values from corner storage + // multiply + FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { + size_t corner_id; + for (int icorner = 0; icorner < num_corners_in_node(node_id); icorner++) + { + corner_id = corners_in_node(node_id, icorner); + design_gradients(node_id, 0) += -corner_value_storage(corner_id) * global_dt; + } + }); // end parallel for + Kokkos::fence(); + } // end view scope + } +} + +// --------------------------------------------------------------------------------------- +// This function calculates the gradient for element power with respect to design variable +// ---------------------------------------------------------------------------------------- + +void FEA_Module_SGH::get_power_dgradient_sgh(double rk_alpha, + const mesh_t& mesh, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& corner_force, + DCArrayKokkos elem_power_dgradients) +{ + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + int num_dims = simparam->num_dims; + + // loop over all the elements in the mesh + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { + double elem_power = 0.0; + elem_power_dgradients(elem_gid) = 0; + + // --- tally the contribution from each corner to the element --- + + // Loop over the nodes in the element + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + size_t corner_lid = node_lid; + + // Get node global id for the local node id + size_t node_gid = nodes_in_elem(elem_gid, node_lid); + + // Get the corner global id for the local corner id + size_t corner_gid = corners_in_elem(elem_gid, corner_lid); + + double node_radius = 1; + if (num_dims == 2) + { + node_radius = node_coords(rk_level, node_gid, 1); + } + + // calculate the Power=F dot V for this corner + for (size_t dim = 0; dim < num_dims; dim++) + { + double half_vel = (node_vel(rk_level, node_gid, dim) + node_vel(0, node_gid, dim)) * 0.5; + elem_power_dgradients(elem_gid) -= corner_vector_storage(corner_gid, dim) * node_radius * node_vel(rk_level, node_gid, dim); + } // end for dim + } // end for node_lid + }); // end parallel loop over the elements + + return; +} // end subroutine + +// ----------------------------------------------------------------------------- +// This function calculates the gradient for element power with respect to position +// ------------------------------------------------------------------------------ + +void FEA_Module_SGH::get_power_ugradient_sgh(double rk_alpha, + const mesh_t& mesh, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& corner_force) +{ + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + int num_dims = simparam->num_dims; + + // initialize gradient matrix + FOR_ALL_CLASS(dof_gid, 0, nlocal_nodes * num_dims, { + for (int idof = 0; idof < DOF_to_Elem_Matrix_Strides(dof_gid); idof++) + { + Power_Gradient_Positions(dof_gid, idof) = 0; + } + }); // end parallel for loop over nodes + Kokkos::fence(); + + // loop over all the elements in the mesh + for (size_t elem_gid = 0; elem_gid < rnum_elem; elem_gid++) + { + // FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { + + double elem_power = 0.0; + + // --- tally the contribution from each corner to the element --- + + // Loop over the nodes in the element + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + size_t corner_lid = node_lid; + size_t column_id; + size_t gradient_node_id; + // Get node global id for the local node id + size_t node_gid = nodes_in_elem(elem_gid, node_lid); + + // Get the corner global id for the local corner id + size_t corner_gid = corners_in_elem(elem_gid, corner_lid); + + double node_radius = 1; + if (num_dims == 2) + { + node_radius = node_coords(rk_level, node_gid, 1); + } + + // calculate the Power=F dot V for this corner + for (size_t dim = 0; dim < num_dims; dim++) + { + for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) + { + for (size_t jdim = 0; jdim < num_dims; jdim++) + { + column_id = Element_Gradient_Matrix_Assembly_Map(elem_gid, igradient); + gradient_node_id = nodes_in_elem(elem_gid, igradient); + if (!map->isNodeLocalElement(gradient_node_id)) + { + continue; + } + Power_Gradient_Positions(gradient_node_id * num_dims + jdim, column_id) -= corner_gradient_storage(corner_gid, dim, igradient, jdim) * node_vel(rk_level, node_gid, + dim) * node_radius; + } + } + } // end for dim + } // end for node_lid + + // }); // end parallel loop over the elements + } + + return; +} // end subroutine + +// ----------------------------------------------------------------------------- +// This function calculates the gradient for element power with respect to velocity +// ------------------------------------------------------------------------------ + +void FEA_Module_SGH::get_power_vgradient_sgh(double rk_alpha, + const mesh_t& mesh, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& corner_force) +{ + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + int num_dims = simparam->num_dims; + + // initialize gradient matrix + FOR_ALL_CLASS(dof_gid, 0, nlocal_nodes * num_dims, { + for (int idof = 0; idof < DOF_to_Elem_Matrix_Strides(dof_gid); idof++) + { + Power_Gradient_Velocities(dof_gid, idof) = 0; + } + }); // end parallel for loop over nodes + Kokkos::fence(); + + // loop over all the elements in the mesh + for (size_t elem_gid = 0; elem_gid < rnum_elem; elem_gid++) + { + // FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { + + double elem_power = 0.0; + + // --- tally the contribution from each corner to the element --- + + // Loop over the nodes in the element + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + size_t corner_lid = node_lid; + size_t column_id; + size_t gradient_node_id; + // Get node global id for the local node id + size_t node_gid = nodes_in_elem(elem_gid, node_lid); + + // Get the corner global id for the local corner id + size_t corner_gid = corners_in_elem(elem_gid, corner_lid); + + double node_radius = 1; + if (num_dims == 2) + { + node_radius = node_coords(rk_level, node_gid, 1); + } + + // calculate the Power=F dot V for this corner + for (size_t dim = 0; dim < num_dims; dim++) + { + for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) + { + column_id = Element_Gradient_Matrix_Assembly_Map(elem_gid, igradient); + gradient_node_id = nodes_in_elem(elem_gid, igradient); + if (!map->isNodeLocalElement(gradient_node_id)) + { + continue; + } + if (node_lid == igradient) + { + Power_Gradient_Velocities(gradient_node_id * num_dims + dim, column_id) -= corner_gradient_storage(corner_gid, dim, igradient, dim) * node_vel(rk_level, node_gid, + dim) * node_radius + + corner_force(corner_gid, dim) * node_radius; + } + else + { + Power_Gradient_Velocities(gradient_node_id * num_dims + dim, column_id) -= corner_gradient_storage(corner_gid, dim, igradient, dim) * node_vel(rk_level, node_gid, + dim) * node_radius; + } + } + } // end for dim + } // end for node_lid + + // }); // end parallel loop over the elements + } + + return; +} // end subroutine + +// ----------------------------------------------------------------------------- +// This function calculates the gradient for element power with respect to energy +// ------------------------------------------------------------------------------ + +void FEA_Module_SGH::get_power_egradient_sgh(double rk_alpha, + const mesh_t& mesh, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& corner_force) +{ + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + int num_dims = simparam->num_dims; + + // initialize gradient storage + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { + Power_Gradient_Energies(elem_gid) = 0; + }); // end parallel loop over the elements + + // loop over all the elements in the mesh + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { + double elem_power = 0.0; + + // --- tally the contribution from each corner to the element --- + + // Loop over the nodes in the element + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + size_t corner_lid = node_lid; + + // Get node global id for the local node id + size_t node_gid = nodes_in_elem(elem_gid, node_lid); + + // Get the corner global id for the local corner id + size_t corner_gid = corners_in_elem(elem_gid, corner_lid); + + double node_radius = 1; + if (num_dims == 2) + { + node_radius = node_coords(rk_level, node_gid, 1); + } + + // calculate the Power=F dot V for this corner + for (size_t dim = 0; dim < num_dims; dim++) + { + double half_vel = (node_vel(rk_level, node_gid, dim) + node_vel(0, node_gid, dim)) * 0.5; + Power_Gradient_Energies(elem_gid) -= Force_Gradient_Energies(elem_gid, node_lid * num_dims + dim) * node_radius * node_vel(rk_level, node_gid, dim); + } // end for dim + } // end for node_lid + }); // end parallel loop over the elements + + return; +} // end subroutine diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/properties.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/properties.cpp similarity index 63% rename from src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/properties.cpp rename to src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/properties.cpp index 9719963c9..f415296de 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/properties.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/properties.cpp @@ -1,73 +1,71 @@ - + // ----------------------------------------------------------------------------- // This calls the models to update state -//------------------------------------------------------------------------------ +// ------------------------------------------------------------------------------ #include "state.h" #include "mesh.h" #include "FEA_Module_SGH.h" -void FEA_Module_SGH::update_state(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &elem_den, - DViewCArrayKokkos &elem_pres, - DViewCArrayKokkos &elem_stress, - DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_mass, - const DViewCArrayKokkos &elem_mat_id, - const double rk_alpha, - const size_t cycle - ){ - +void FEA_Module_SGH::update_state(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_den, + DViewCArrayKokkos& elem_pres, + DViewCArrayKokkos& elem_stress, + DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& elem_mat_id, + const double rk_alpha, + const size_t cycle + ) +{ const size_t rk_level = rk_num_bins - 1; - int num_dims = num_dim; - + int num_dims = num_dim; + // loop over all the elements in the mesh - FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { - + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { const size_t num_dims = num_dims; const size_t num_nodes_in_elem = num_nodes_in_elem; // cut out the node_gids for this element - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), num_nodes_in_elem); - + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), num_nodes_in_elem); + // --- Density --- - elem_den(elem_gid) = elem_mass(elem_gid)/elem_vol(elem_gid); - + elem_den(elem_gid) = elem_mass(elem_gid) / elem_vol(elem_gid); + size_t mat_id = elem_mat_id(elem_gid); - //initialize elem pressure + // initialize elem pressure elem_pres(elem_gid) = 0; - + // --- Stress --- // hyper elastic plastic model - if(material(mat_id).strength_type == STRENGTH_TYPE::hyper){ - + if (material(mat_id).strength_type == STRENGTH_TYPE::hyper) + { // cut out the node_gids for this element - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), num_nodes_in_elem); - + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), num_nodes_in_elem); + // --- Density --- - elem_den(elem_gid) = elem_mass(elem_gid)/elem_vol(elem_gid); - + elem_den(elem_gid) = elem_mass(elem_gid) / elem_vol(elem_gid); + // corner area normals double area_array[24]; - ViewCArrayKokkos area(area_array, num_nodes_in_elem, num_dims); - + ViewCArrayKokkos area(area_array, num_nodes_in_elem, num_dims); + // velocity gradient double vel_grad_array[9]; - ViewCArrayKokkos vel_grad(vel_grad_array, num_dims, num_dims); - + ViewCArrayKokkos vel_grad(vel_grad_array, num_dims, num_dims); + // get the B matrix which are the OUTWARD corner area normals get_bmatrix(area, elem_gid, node_coords, elem_node_gids, rk_level); - - + // --- Calculate the velocity gradient --- get_velgrad(vel_grad, elem_node_gids, @@ -76,7 +74,7 @@ void FEA_Module_SGH::update_state(const DCArrayKokkos &material, elem_vol(elem_gid), elem_gid, rk_level); - + StrengthParent* strength_model = elem_strength(elem_gid).model; // --- call strength model --- @@ -99,7 +97,6 @@ void FEA_Module_SGH::update_state(const DCArrayKokkos &material, rk_alpha, cycle, rk_level); - } // end logical on hyper strength model EOSParent* eos_model = elem_eos(elem_gid).model; @@ -114,7 +111,7 @@ void FEA_Module_SGH::update_state(const DCArrayKokkos &material, elem_user_output_vars, elem_sspd, elem_den(elem_gid), - elem_sie(rk_level,elem_gid)); + elem_sie(rk_level, elem_gid)); // --- Sound speed --- eos_model->calc_sound_speed(elem_pres, @@ -126,79 +123,73 @@ void FEA_Module_SGH::update_state(const DCArrayKokkos &material, elem_user_output_vars, elem_sspd, elem_den(elem_gid), - elem_sie(rk_level,elem_gid)); - + elem_sie(rk_level, elem_gid)); }); // end parallel for Kokkos::fence(); - + return; - } // end method to update state - - -void FEA_Module_SGH::update_state2D(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &elem_den, - DViewCArrayKokkos &elem_pres, - DViewCArrayKokkos &elem_stress, - DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_mass, - const DViewCArrayKokkos &elem_mat_id, - const double rk_alpha, - const size_t cycle - ){ - +void FEA_Module_SGH::update_state2D(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_den, + DViewCArrayKokkos& elem_pres, + DViewCArrayKokkos& elem_stress, + DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& elem_mat_id, + const double rk_alpha, + const size_t cycle + ) +{ const size_t rk_level = rk_num_bins - 1; - int num_dims = num_dim; - + int num_dims = num_dim; + // loop over all the elements in the mesh - FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { - + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { const size_t num_dims = num_dims; const size_t num_nodes_in_elem = num_nodes_in_elem; // cut out the node_gids for this element - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), num_nodes_in_elem); - + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), num_nodes_in_elem); + // --- Density --- - elem_den(elem_gid) = elem_mass(elem_gid)/elem_vol(elem_gid); - + elem_den(elem_gid) = elem_mass(elem_gid) / elem_vol(elem_gid); + size_t mat_id = elem_mat_id(elem_gid); - //initialize elem pressure + // initialize elem pressure elem_pres(elem_gid) = 0; - + // --- Stress --- // hyper elastic plastic model - if(material(mat_id).strength_type == STRENGTH_TYPE::hyper){ - + if (material(mat_id).strength_type == STRENGTH_TYPE::hyper) + { // cut out the node_gids for this element - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), num_nodes_in_elem); - + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), num_nodes_in_elem); + // --- Density --- - elem_den(elem_gid) = elem_mass(elem_gid)/elem_vol(elem_gid); - + elem_den(elem_gid) = elem_mass(elem_gid) / elem_vol(elem_gid); + // corner area normals double area_array[8]; - ViewCArrayKokkos area(area_array, num_nodes_in_elem, num_dims); - + ViewCArrayKokkos area(area_array, num_nodes_in_elem, num_dims); + // velocity gradient double vel_grad_array[4]; - ViewCArrayKokkos vel_grad(vel_grad_array, num_dims, num_dims); - + ViewCArrayKokkos vel_grad(vel_grad_array, num_dims, num_dims); + // get the B matrix which are the OUTWARD corner area normals get_bmatrix(area, elem_gid, node_coords, elem_node_gids, rk_level); - - + // --- Calculate the velocity gradient --- get_velgrad(vel_grad, elem_node_gids, @@ -207,7 +198,7 @@ void FEA_Module_SGH::update_state2D(const DCArrayKokkos &material, elem_vol(elem_gid), elem_gid, rk_level); - + StrengthParent* strength_model = elem_strength(elem_gid).model; // --- call strength model --- @@ -230,7 +221,6 @@ void FEA_Module_SGH::update_state2D(const DCArrayKokkos &material, rk_alpha, cycle, rk_level); - } // end logical on hyper strength model EOSParent* eos_model = elem_eos(elem_gid).model; @@ -245,7 +235,7 @@ void FEA_Module_SGH::update_state2D(const DCArrayKokkos &material, elem_user_output_vars, elem_sspd, elem_den(elem_gid), - elem_sie(rk_level,elem_gid)); + elem_sie(rk_level, elem_gid)); // --- Sound speed --- eos_model->calc_sound_speed(elem_pres, @@ -257,14 +247,9 @@ void FEA_Module_SGH::update_state2D(const DCArrayKokkos &material, elem_user_output_vars, elem_sspd, elem_den(elem_gid), - elem_sie(rk_level,elem_gid)); - + elem_sie(rk_level, elem_gid)); }); // end parallel for Kokkos::fence(); - + return; - } // end method to update state - - - diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/setup_sgh.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/setup_sgh.cpp new file mode 100644 index 000000000..dd85ffe5e --- /dev/null +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/setup_sgh.cpp @@ -0,0 +1,958 @@ +/* ---------------------------------------------------------------------------- + Setup SGH solver data +------------------------------------------------------------------------------- */ +#include "matar.h" +#include "state.h" +#include "FEA_Module_SGH.h" +#include "Simulation_Parameters/Simulation_Parameters_Explicit.h" +#include "Simulation_Parameters/FEA_Module/SGH_Parameters.h" +#include "Explicit_Solver.h" + +void FEA_Module_SGH::setup() +{ + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + const size_t num_fills = simparam->regions.size(); + const size_t rk_num_bins = simparam->dynamic_options.rk_num_bins; + const size_t num_bcs = module_params->boundary_conditions.size(); + const size_t num_materials = simparam->materials.size(); + const int num_dim = simparam->num_dims; + const size_t num_lcs = module_params->loading.size(); + if (num_lcs) + { + have_loading_conditions = true; + } + + // --------------------------------------------------------------------- + // obtain mesh data + // --------------------------------------------------------------------- + sgh_interface_setup(node_interface, elem_interface, corner_interface); + mesh->build_corner_connectivity(); + // debug print of corner ids + /* + if(myrank==1){ + for(int i = 0; i < mesh.num_nodes; i++){ + + // loop over all corners around the node and calculate the nodal force + for (size_t corner_lid=0; corner_lidgetGlobalElement(i) << " " << i << " " << all_node_map->getLocalElement(all_node_map->getGlobalElement(i)) << " " << corner_gid << " " << std::endl; + + } // end for corner_lid + //std::cout << explicit_solver_pointer->all_node_map->getGlobalElement(i) << " " << node_force[0] << " " << node_force[1] << " " << node_force[2] << std::endl; + //std::cout << explicit_solver_pointer->all_node_map->getGlobalElement(i) << " " << node_mass(i) << std::endl; + } + } + */ + /* + if(myrank==1){ + for(int i = 0; i < mesh.num_elems; i++){ + + // loop over all corners around the node and calculate the nodal force + for (size_t corner_lid=0; corner_lidgetGlobalElement(mesh.nodes_in_elem(i, corner_lid)) <<" " << corner_gid << " " << std::endl; + + } // end for corner_lid + //std::cout << explicit_solver_pointer->all_node_map->getGlobalElement(i) << " " << node_force[0] << " " << node_force[1] << " " << node_force[2] << std::endl; + //std::cout << explicit_solver_pointer->all_node_map->getGlobalElement(i) << " " << node_mass(i) << std::endl; + } + } + */ + mesh->build_elem_elem_connectivity(); + mesh->num_bdy_patches = nboundary_patches; + if (num_dim == 2) + { + mesh->build_patch_connectivity(); + mesh->build_node_node_connectivity(); + } + + // --------------------------------------------------------------------- + // allocate memory + // --------------------------------------------------------------------- + + // shorthand names + const size_t num_nodes = mesh->num_nodes; + const size_t num_elems = mesh->num_elems; + const size_t num_corners = mesh->num_corners; + + // --- make dual views of data on CPU and GPU --- + // Notes: + // Instead of using a struct of dual types like the mesh type, + // individual dual views will be made for all the state + // variables. The motivation is to reduce memory movement + // when passing state into a function. Passing a struct by + // reference will copy the meta data and pointers for the + // variables held inside the struct. Since all the mesh + // variables are typically used by most functions, a single + // mesh struct or passing the arrays will be roughly equivalent + // for memory movement. + + // create Dual Views of the individual node struct variables + node_coords = DViewCArrayKokkos(node_interface.coords.get_kokkos_dual_view().view_host().data(), rk_num_bins, num_nodes, num_dim); + node_vel = DViewCArrayKokkos(node_interface.vel.get_kokkos_dual_view().view_host().data(), rk_num_bins, num_nodes, num_dim); + node_mass = DViewCArrayKokkos(node_interface.mass.get_kokkos_dual_view().view_host().data(), num_nodes); + + // create Dual Views of the individual elem struct variables + elem_den = DViewCArrayKokkos(&elem_interface.den(0), num_elems); + elem_pres = DViewCArrayKokkos(&elem_interface.pres(0), num_elems); + elem_stress = DViewCArrayKokkos(&elem_interface.stress(0, 0, 0, 0), rk_num_bins, num_elems, 3, 3); // always 3D even in 2D-RZ + elem_sspd = DViewCArrayKokkos(&elem_interface.sspd(0), num_elems); + elem_sie = DViewCArrayKokkos(&elem_interface.sie(0, 0), rk_num_bins, num_elems); + elem_vol = DViewCArrayKokkos(&elem_interface.vol(0), num_elems); + elem_div = DViewCArrayKokkos(&elem_interface.div(0), num_elems); + elem_mass = DViewCArrayKokkos(&elem_interface.mass(0), num_elems); + elem_mat_id = DViewCArrayKokkos(&elem_interface.mat_id(0), num_elems); + + // create Dual Views of the corner struct variables + corner_force = DViewCArrayKokkos(&corner_interface.force(0, 0), num_corners, num_dim); + corner_mass = DViewCArrayKokkos(&corner_interface.mass(0), num_corners); + + // allocate elem_vel_grad + elem_vel_grad = DCArrayKokkos(num_elems, 3, 3); + + // allocate material models + elem_eos = DCArrayKokkos(num_elems); + elem_strength = DCArrayKokkos(num_elems); + + // --------------------------------------------------------------------- + // calculate geometry + // --------------------------------------------------------------------- + node_coords.update_device(); + Kokkos::fence(); + + get_vol(); + + // FEA_Module boundary condition variable + num_boundary_conditions = num_bcs; + + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos mat_fill = simparam->mat_fill; + const DCArrayKokkos material = simparam->material; + global_vars = simparam->global_vars; + state_vars = DCArrayKokkos(rnum_elem, simparam->max_num_state_vars); + elem_user_output_vars = DCArrayKokkos(rnum_elem, simparam->output_options.max_num_user_output_vars); + + // --- calculate bdy sets ---// + mesh->num_nodes_in_patch = 2 * (num_dim - 1); // 2 (2D) or 4 (3D) + mesh->num_patches_in_elem = 2 * num_dim; // 4 (2D) or 6 (3D) + mesh->init_bdy_sets(num_bcs); + num_bdy_sets = mesh->num_bdy_sets; + printf("Num BC's = %lu\n", num_bcs); + + // patch ids in bdy set + bdy_patches_in_set = mesh->bdy_patches_in_set; + if (num_dim == 2) + { + bdy_nodes = mesh->bdy_nodes; + } + + // tag boundary patches in the set + tag_bdys(boundary, *mesh, node_coords); + + build_boundry_node_sets(*mesh); + + // node ids in bdy_patch set + bdy_nodes_in_set = mesh->bdy_nodes_in_set; + num_bdy_nodes_in_set = mesh->num_bdy_nodes_in_set; + + // assign mesh views needed by the FEA module + + // elem ids in elem + elems_in_elem = mesh->elems_in_elem; + num_elems_in_elem = mesh->num_elems_in_elem; + + // corners + num_corners_in_node = mesh->num_corners_in_node; + corners_in_node = mesh->corners_in_node; + corners_in_elem = mesh->corners_in_elem; + + // elem-node conn & node-node conn + elems_in_node = mesh->elems_in_node; + if (num_dim == 2) + { + nodes_in_node = mesh->nodes_in_node; + num_nodes_in_node = mesh->num_nodes_in_node; + // patch conn + + patches_in_elem = mesh->patches_in_elem; + nodes_in_patch = mesh->nodes_in_patch; + elems_in_patch = mesh->elems_in_patch; + } + + // loop over BCs + for (size_t this_bdy = 0; this_bdy < num_bcs; this_bdy++) + { + RUN_CLASS({ + printf("Boundary Condition number %lu \n", this_bdy); + printf(" Num bdy patches in this set = %lu \n", bdy_patches_in_set.stride(this_bdy)); + printf(" Num bdy nodes in this set = %lu \n", bdy_nodes_in_set.stride(this_bdy)); + }); + Kokkos::fence(); + } // end for + + // elem_mat_id needs to be initialized before initialization of material models + for (int f_id = 0; f_id < num_fills; f_id++) + { + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { + elem_mat_id(elem_gid) = mat_fill(f_id).material_id; + }); + } + elem_mat_id.update_host(); + + // function for initializing state_vars + init_state_vars(material, + elem_mat_id, + state_vars, + global_vars, + elem_user_output_vars, + rnum_elem); + + // initialize strength model + init_strength_model(elem_strength, + material, + elem_mat_id, + state_vars, + global_vars, + elem_user_output_vars, + rnum_elem); + + // initialize eos model + init_eos_model(elem_eos, + material, + elem_mat_id, + state_vars, + global_vars, + elem_user_output_vars, + rnum_elem); + + // --- apply the fill instructions over each of the Elements---// + + // initialize if topology optimization is used + if (simparam->topology_optimization_on) + { + for (int elem_id = 0; elem_id < rnum_elem; elem_id++) + { + relative_element_densities.host(elem_id) = 1; + } // for + relative_element_densities.update_device(); + } + + // loop over the fill instructures + for (int f_id = 0; f_id < num_fills; f_id++) + { + // parallel loop over elements in mesh + // for (size_t elem_gid = 0; elem_gid <= rnum_elem; elem_gid++) { + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { + // calculate the coordinates and radius of the element + double elem_coords[3]; // note:initialization with a list won't work + elem_coords[0] = 0.0; + elem_coords[1] = 0.0; + elem_coords[2] = 0.0; + + // get the coordinates of the element center + for (int node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + elem_coords[0] += node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), 0); + elem_coords[1] += node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), 1); + if (num_dim == 3) + { + elem_coords[2] += node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), 2); + } + else + { + elem_coords[2] = 0.0; + } + } // end loop over nodes in element + elem_coords[0] = elem_coords[0] / num_nodes_in_elem; + elem_coords[1] = elem_coords[1] / num_nodes_in_elem; + elem_coords[2] = elem_coords[2] / num_nodes_in_elem; + + // default is not to fill the element + bool fill_this = mat_fill(f_id).volume.contains(elem_coords); + + // paint the material state on the element + if (fill_this) + { + // density + elem_den(elem_gid) = mat_fill(f_id).den; + + // mass + elem_mass(elem_gid) = elem_den(elem_gid) * elem_vol(elem_gid); + + // specific internal energy + elem_sie(rk_level, elem_gid) = mat_fill(f_id).sie; + + size_t mat_id = elem_mat_id(elem_gid); // short name + + // --- stress tensor --- + // always 3D even for 2D-RZ + for (size_t i = 0; i < 3; i++) + { + for (size_t j = 0; j < 3; j++) + { + elem_stress(rk_level, elem_gid, i, j) = 0.0; + } + } // end for + + // short form for clean code + EOSParent* eos_model = elem_eos(elem_gid).model; + // --- Pressure --- + eos_model->calc_pressure(elem_pres, + elem_stress, + elem_gid, + elem_mat_id(elem_gid), + state_vars, + global_vars, + elem_user_output_vars, + elem_sspd, + elem_den(elem_gid), + elem_sie(rk_level, elem_gid)); + + // --- Sound speed --- + eos_model->calc_sound_speed(elem_pres, + elem_stress, + elem_gid, + elem_mat_id(elem_gid), + state_vars, + global_vars, + elem_user_output_vars, + elem_sspd, + elem_den(elem_gid), + elem_sie(rk_level, elem_gid)); + + // loop over the nodes of this element and apply velocity + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + // get the mesh node index + size_t node_gid = nodes_in_elem(elem_gid, node_lid); + + // --- Velocity --- + switch (mat_fill(f_id).velocity) + { + case VELOCITY_TYPE::cartesian: + { + node_vel(rk_level, node_gid, 0) = mat_fill(f_id).u; + node_vel(rk_level, node_gid, 1) = mat_fill(f_id).v; + if (num_dim == 3) + { + node_vel(rk_level, node_gid, 2) = mat_fill(f_id).w; + } + + break; + } + case VELOCITY_TYPE::radial: + { + // Setting up cylindrical + double dir[2]; + dir[0] = 0.0; + dir[1] = 0.0; + double radius_val = 0.0; + + for (int dim = 0; dim < 2; dim++) + { + dir[dim] = node_coords(rk_level, node_gid, dim); + radius_val += node_coords(rk_level, node_gid, dim) * node_coords(rk_level, node_gid, dim); + } // end for + radius_val = sqrt(radius_val); + + for (int dim = 0; dim < 2; dim++) + { + if (radius_val > 1.0e-14) + { + dir[dim] /= (radius_val); + } + else + { + dir[dim] = 0.0; + } + } // end for + + node_vel(rk_level, node_gid, 0) = mat_fill(f_id).speed * dir[0]; + node_vel(rk_level, node_gid, 1) = mat_fill(f_id).speed * dir[1]; + if (num_dim == 3) + { + node_vel(rk_level, node_gid, 2) = 0.0; + } + + break; + } + case VELOCITY_TYPE::spherical: + { + // Setting up spherical + double dir[3]; + dir[0] = 0.0; + dir[1] = 0.0; + dir[2] = 0.0; + double radius_val = 0.0; + + for (int dim = 0; dim < 3; dim++) + { + dir[dim] = node_coords(rk_level, node_gid, dim); + radius_val += node_coords(rk_level, node_gid, dim) * node_coords(rk_level, node_gid, dim); + } // end for + radius_val = sqrt(radius_val); + + for (int dim = 0; dim < 3; dim++) + { + if (radius_val > 1.0e-14) + { + dir[dim] /= (radius_val); + } + else + { + dir[dim] = 0.0; + } + } // end for + + node_vel(rk_level, node_gid, 0) = mat_fill(f_id).speed * dir[0]; + node_vel(rk_level, node_gid, 1) = mat_fill(f_id).speed * dir[1]; + if (num_dim == 3) + { + node_vel(rk_level, node_gid, 2) = mat_fill(f_id).speed * dir[2]; + } + + break; + } + case VELOCITY_TYPE::radial_linear: + { + break; + } + case VELOCITY_TYPE::spherical_linear: + { + break; + } + case VELOCITY_TYPE::tg_vortex: + { + node_vel(rk_level, node_gid, 0) = sin(PI * node_coords(rk_level, node_gid, 0)) * cos(PI * node_coords(rk_level, node_gid, 1)); + node_vel(rk_level, node_gid, 1) = -1.0 * cos(PI * node_coords(rk_level, node_gid, 0)) * sin(PI * node_coords(rk_level, node_gid, 1)); + if (num_dim == 3) + { + node_vel(rk_level, node_gid, 2) = 0.0; + } + + break; + } + } // end of switch + } // end loop over nodes of element + + if (mat_fill(f_id).velocity == VELOCITY_TYPE::tg_vortex) + { + elem_pres(elem_gid) = 0.25 * (cos(2.0 * PI * elem_coords[0]) + cos(2.0 * PI * elem_coords[1]) ) + 1.0; + + // p = rho*ie*(gamma - 1) + size_t mat_id = f_id; + double gamma = global_vars(mat_id, 0); // gamma value + elem_sie(rk_level, elem_gid) = + elem_pres(elem_gid) / (mat_fill(f_id).den * (gamma - 1.0)); + } // end if + } // end if fill + }); // end FOR_ALL_CLASS element loop + Kokkos::fence(); + } // end for loop over fills + + // apply BC's to velocity + FEA_Module_SGH::boundary_velocity(*mesh, boundary, node_vel); + + // calculate the corner massess if 2D + if (num_dim == 2) + { + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { + // facial area of the corners + double corner_areas_array[4]; + + ViewCArrayKokkos corner_areas(&corner_areas_array[0], 4); + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 4); + + get_area_weights2D(corner_areas, + elem_gid, + node_coords, + elem_node_gids, + rk_level); + + // loop over the corners of the element and calculate the mass + for (size_t corner_lid = 0; corner_lid < 4; corner_lid++) + { + size_t corner_gid = corners_in_elem(elem_gid, corner_lid); + corner_mass(corner_gid) = corner_areas(corner_lid) * elem_den(elem_gid); // node radius is added later + } // end for over corners + }); + } // end of + + // calculate the nodal mass + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + node_mass(node_gid) = 0.0; + + if (num_dim == 3) + { + for (size_t elem_lid = 0; elem_lid < num_corners_in_node(node_gid); elem_lid++) + { + size_t elem_gid = elems_in_node(node_gid, elem_lid); + node_mass(node_gid) += 1.0 / 8.0 * elem_mass(elem_gid); + } // end for elem_lid + } // end if dims=3 + else + { + // 2D-RZ + for (size_t corner_lid = 0; corner_lid < num_corners_in_node(node_gid); corner_lid++) + { + size_t corner_gid = corners_in_node(node_gid, corner_lid); + node_mass(node_gid) += corner_mass(corner_gid); // sans the radius so it is areal node mass + + corner_mass(corner_gid) *= node_coords(rk_level, node_gid, 1); // true corner mass now + } // end for elem_lid + } // end else + }); // end FOR_ALL_CLASS + Kokkos::fence(); + + // current interface has differing mass arrays; this equates them until we unify memory + // view scope + if (simparam->topology_optimization_on || simparam->shape_optimization_on || simparam->num_dims == 2) + { + { + vec_array node_mass_interface = node_masses_distributed->getLocalView(Tpetra::Access::ReadWrite); + + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + node_mass_interface(node_gid, 0) = node_mass(node_gid); + }); // end parallel for + } // end view scope + Kokkos::fence(); + // communicate ghost densities + comm_node_masses(); + + // this is forcing a copy to the device + // view scope + { + vec_array ghost_node_mass_interface = ghost_node_masses_distributed->getLocalView(Tpetra::Access::ReadWrite); + + FOR_ALL_CLASS(node_gid, nlocal_nodes, nall_nodes, { + node_mass(node_gid) = ghost_node_mass_interface(node_gid - nlocal_nodes, 0); + }); // end parallel for + } // end view scope + Kokkos::fence(); + } // endif + + // initialize if topology optimization is used + if (simparam->topology_optimization_on || simparam->shape_optimization_on) + { + init_assembly(); + assemble_matrix(); + } + + // update host copies of arrays modified in this function + elem_den.update_host(); + elem_mass.update_host(); + elem_sie.update_host(); + elem_stress.update_host(); + elem_pres.update_host(); + elem_sspd.update_host(); + + return; +} // end of setup + +// ----------------------------------------------------------------------------- +// Interfaces read in data with the SGH solver data; currently a hack to streamline +// ------------------------------------------------------------------------------ +void FEA_Module_SGH::sgh_interface_setup(node_t& node, + elem_t& elem, + corner_t& corner) +{ + const size_t num_dim = simparam->num_dims; + const size_t rk_num_bins = simparam->dynamic_options.rk_num_bins; + + num_nodes_in_elem = 1; + for (int dim = 0; dim < num_dim; dim++) + { + num_nodes_in_elem *= 2; + } + + // --- Read in the nodes in the mesh --- + + nall_nodes = Explicit_Solver_Pointer_->nall_nodes; + int myrank = Explicit_Solver_Pointer_->myrank; + int nranks = Explicit_Solver_Pointer_->nranks; + // printf("Num nodes assigned to MPI rank %lu is %lu\n" , myrank, nall_nodes); + + // intialize node variables + mesh->initialize_nodes(nall_nodes); + mesh->initialize_local_nodes(Explicit_Solver_Pointer_->nlocal_nodes); + node.initialize(rk_num_bins, nall_nodes, num_dim); + // std::cout << "Bin counts " << rk_num_bins << " Node counts " << nall_nodes << " Num dim " << num_dim << std::endl; + + // view scope + { + host_vec_array interface_node_coords = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + // save node data to node.coords + // std::cout << "NODE DATA ON RANK " << myrank << std::endl; + if (num_dim == 2) + { + for (int inode = 0; inode < nall_nodes; inode++) + { + // std::cout << "Node index " << inode+1 << " "; + node.coords.host(0, inode, 0) = interface_node_coords(inode, 0); + // std::cout << host_node_coords_state(0,inode,0)+1<< " "; + node.coords.host(0, inode, 1) = interface_node_coords(inode, 1); + // std::cout << host_node_coords_state(0,inode,1)+1<< " "; + } + } + else if (num_dim == 3) + { + for (int inode = 0; inode < nall_nodes; inode++) + { + // std::cout << "Node index " << inode+1 << " "; + node.coords.host(0, inode, 0) = interface_node_coords(inode, 0); + // std::cout << host_node_coords_state(0,inode,0)+1<< " "; + node.coords.host(0, inode, 1) = interface_node_coords(inode, 1); + // std::cout << host_node_coords_state(0,inode,1)+1<< " "; + + node.coords.host(0, inode, 2) = interface_node_coords(inode, 2); + // std::cout << host_node_coords_state(0,inode,2)+1<< std::endl; + } + } + } // end view scope + // --- read in the elements in the mesh --- + + rnum_elem = Explicit_Solver_Pointer_->rnum_elem; + // printf("Num elems assigned to MPI rank %lu is %lu\n" , myrank, rnum_elem); + + // intialize elem variables + mesh->initialize_elems(rnum_elem, num_dim); + elem.initialize(rk_num_bins, nall_nodes, 3); // always 3D here, even for 2D + nodes_in_elem = mesh->nodes_in_elem; + // save data to nodes_in_elem.host + // CArrayKokkos host_mesh_nodes_in_elem(rnum_elem, num_nodes_in_elem); + // view scope + { + host_elem_conn_array interface_nodes_in_elem = Explicit_Solver_Pointer_->global_nodes_in_elem_distributed->getLocalView(Tpetra::Access::ReadWrite); + // save node data to node.coords + // std::cout << "ELEMENT CONNECTIVITY ON RANK " << myrank << std::endl; + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + // std::cout << "Element index " << ielem+1 << " "; + for (int inode = 0; inode < num_nodes_in_elem; inode++) + { + nodes_in_elem.host(ielem, inode) = Explicit_Solver_Pointer_->all_node_map->getLocalElement(interface_nodes_in_elem(ielem, inode)); + // debug print + // std::cout << nodes_in_elem.get_kokkos_dual_view().h_view(ielem*num_nodes_in_elem + inode)+1<< " "; + } + // std::cout << std::endl; + } + } + // update device side + nodes_in_elem.update_device(); + + // debug print + + // CArrayKokkos device_mesh_nodes_in_elem(rnum_elem, num_nodes_in_elem); + // device_mesh_nodes_in_elem.get_kokkos_view() = nodes_in_elem.get_kokkos_dual_view().d_view; + // host_mesh_nodes_in_elem.get_kokkos_view() = nodes_in_elem.get_kokkos_dual_view().view_host(); + /* + if(myrank==1){ + std::cout << "ELEMENT CONNECTIVITY ON RANK 1 in LOCAL INDICES" << myrank << std::endl; + for(int ielem = 0; ielem < rnum_elem; ielem++){ + std::cout << "Element index " << ielem+1 << " "; + for(int inode = 0; inode < num_nodes_in_elem; inode++){ + //debug print + //device_mesh_nodes_in_elem(ielem,inode) = Explicit_Solver_Pointer_->all_node_map->getLocalElement(interface_nodes_in_elem(ielem,inode)); + std::cout << nodes_in_elem(ielem, inode)+1<< " "; + } + std::cout << std::endl; + } + } + */ + /* + std::cout.flush(); + if(myrank==1){ + std::cout << "ELEMENT CONNECTIVITY ON RANK 1 in GLOBAL INDICES" << myrank << std::endl; + std::cout << "local node index of global index 275 on rank 1 " << Explicit_Solver_Pointer_->all_node_map->getLocalElement(275) << std::endl; + for(int ielem = 0; ielem < rnum_elem; ielem++){ + std::cout << ielem << " "; + for(int inode = 0; inode < num_nodes_in_elem; inode++){ + //debug print + //device_mesh_nodes_in_elem(ielem,inode) = Explicit_Solver_Pointer_->all_node_map->getLocalElement(interface_nodes_in_elem(ielem,inode)); + std::cout << Explicit_Solver_Pointer_->all_node_map->getGlobalElement(nodes_in_elem(ielem, inode))<< " "; + } + std::cout << std::endl; + } + } + std::cout.flush(); + */ + /* + size_t nall_nodes = Explicit_Solver_Pointer_->nall_nodes; + node.all_coords = DCArrayKokkos (rk_num_bins, nall_nodes, num_dim); + node.all_vel = DCArrayKokkos (rk_num_bins, nall_nodes, num_dim); + node.all_mass = DCArrayKokkos (nall_nodes); + + //save all data (nlocal +nghost) + CArrayKokkos host_all_node_coords_state(rk_num_bins, nall_nodes, num_dim); + host_vec_array interface_all_node_coords = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); + host_all_node_coords_state.get_kokkos_view() = node.all_coords.get_kokkos_dual_view().view_host(); + //host_node_coords_state = CArrayKokkos(rk_num_bins, nall_nodes, num_dim); + //host_all_node_coords_state.get_kokkos_view() = Kokkos::View("debug", rk_num_bins*nall_nodes*num_dim); + //save node data to node.coords + + //std::cout << "ALL NODE DATA ON RANK " << myrank << std::endl; + for(int inode = 0; inode < nall_nodes; inode++){ + //std::cout << "Node index " << inode+1 << " "; + node.all_coords.host(0,inode,0) = interface_all_node_coords(inode,0); + //std::cout << host_all_node_coords_state(0,inode,0)+1<< " "; + node.all_coords.host(0,inode,1) = interface_all_node_coords(inode,1); + //std::cout << host_all_node_coords_state(0,inode,1)+1<< " "; + node.all_coords.host(0,inode,2) = interface_all_node_coords(inode,2); + //std::cout << host_all_node_coords_state(0,inode,2)+1<< std::endl; + } + */ + + // save the node coords to the current RK value + for (size_t node_gid = 0; node_gid < nall_nodes; node_gid++) + { + for (int rk = 1; rk < rk_num_bins; rk++) + { + for (int dim = 0; dim < num_dim; dim++) + { + node.coords.host(rk, node_gid, dim) = node.coords.host(0, node_gid, dim); + } // end for dim + } // end for rk + } // end parallel for + + /* + // save the node coords to the current RK value + for (size_t node_gid=0; node_gidinitialize_corners(num_corners); + corner.initialize(num_corners, num_dim); + + /* + for(int inode = 0; inode < nall_nodes; inode++){ + std::cout << "Node index " << inode+1 << " "; + for(int rk=0; rknboundary_patches; + int num_nodes_in_patch = mesh.num_nodes_in_patch; + num_bdy_nodes_in_set = mesh.num_bdy_nodes_in_set = DCArrayKokkos(num_bdy_sets, "num_bdy_nodes_in_set"); + CArrayKokkos temp_count_num_bdy_nodes_in_set(num_bdy_sets, nall_nodes, "temp_count_num_bdy_nodes_in_set"); + + DynamicRaggedRightArrayKokkos temp_nodes_in_set(mesh.num_bdy_sets, nboundary_patches * mesh.num_nodes_in_patch, "temp_nodes_in_set"); + + // Parallel loop over boundary sets on device + FOR_ALL_CLASS(bdy_set, 0, num_bdy_sets, { + // finde the number of patches_in_set + size_t num_bdy_patches_in_set = bdy_patches_in_set.stride(bdy_set); + + num_bdy_nodes_in_set(bdy_set) = 0; + + // Loop over boundary patches in boundary set + for (size_t bdy_patch_gid = 0; bdy_patch_gid < num_bdy_patches_in_set; bdy_patch_gid++) + { + // get the global id for this boundary patch + size_t patch_gid = bdy_patches_in_set(bdy_set, bdy_patch_gid); + + // apply boundary condition at nodes on boundary + for (size_t node_lid = 0; node_lid < num_nodes_in_patch; node_lid++) + { + size_t node_gid = Local_Index_Boundary_Patches(patch_gid, node_lid); + + temp_count_num_bdy_nodes_in_set(bdy_set, node_gid) = -1; + } // end for node_lid + } // end for bdy_patch_gid + + // Loop over boundary patches in boundary set + for (size_t bdy_patch_gid = 0; bdy_patch_gid < num_bdy_patches_in_set; bdy_patch_gid++) + { + // get the global id for this boundary patch + size_t patch_gid = bdy_patches_in_set(bdy_set, bdy_patch_gid); + + // apply boundary condition at nodes on boundary + for (size_t node_lid = 0; node_lid < num_nodes_in_patch; node_lid++) + { + size_t node_gid = Local_Index_Boundary_Patches(patch_gid, node_lid); + + if (temp_count_num_bdy_nodes_in_set(bdy_set, node_gid) == -1) + { + size_t num_saved = num_bdy_nodes_in_set(bdy_set); + + num_bdy_nodes_in_set(bdy_set)++; + + // replace -1 with node_gid to denote the node was already saved + temp_count_num_bdy_nodes_in_set(bdy_set, node_gid) = node_gid; + + // increment the number of saved nodes, create memory + temp_nodes_in_set.stride(bdy_set)++; + temp_nodes_in_set(bdy_set, num_saved) = node_gid; + } // end if + } // end for node_lid + } // end for bdy_patch_gid + }); // end FOR_ALL_CLASS bdy_set + Kokkos::fence(); + + // allocate the RaggedRight bdy_nodes_in_set array + bdy_nodes_in_set = mesh.bdy_nodes_in_set = RaggedRightArrayKokkos(mesh.num_bdy_nodes_in_set, "bdy_nodes_in_set"); + + FOR_ALL_CLASS(bdy_set, 0, num_bdy_sets, { + // Loop over boundary patches in boundary set + for (size_t bdy_node_lid = 0; bdy_node_lid < num_bdy_nodes_in_set(bdy_set); bdy_node_lid++) + { + // save the bdy_node_gid + bdy_nodes_in_set(bdy_set, bdy_node_lid) = temp_nodes_in_set(bdy_set, bdy_node_lid); + } // end for + }); // end FOR_ALL_CLASS bdy_set + + // update the host side for the number nodes in a bdy_set + num_bdy_nodes_in_set.update_host(); + + return; +} // end method to build boundary nodes + +/* ---------------------------------------------------------------------------- + Initialize sets of element boundary surfaces and arrays for input conditions +------------------------------------------------------------------------------- */ +void FEA_Module_SGH::init_boundaries() +{ + max_boundary_sets = module_params->boundary_conditions.size(); + int num_dim = simparam->num_dims; + + // set the number of boundary sets + if (myrank == 0) + { + std::cout << "building boundary sets " << std::endl; + } + + // initialize to 1 since there must be at least 1 boundary set anyway; read in may occure later + if (max_boundary_sets == 0) + { + max_boundary_sets = 1; + } + // std::cout << "NUM BOUNDARY CONDITIONS ON RANK " << myrank << " FOR INIT " << num_boundary_conditions <(nall_nodes * num_dim, "Node_DOF_Boundary_Condition_Type"); + + // initialize + for (int init = 0; init < nall_nodes * num_dim; init++) + { + Node_DOF_Boundary_Condition_Type(init) = NONE; + } + + Number_DOF_BCS = 0; +} + +/* ---------------------------------------------------------------------- + initialize storage for element boundary surfaces corresponding to user BCs +------------------------------------------------------------------------- */ +void FEA_Module_SGH::init_boundary_sets(int num_sets) +{ + if (num_sets == 0) + { + std::cout << " Warning: number of boundary conditions = 0"; + return; + } + // initialize maximum + max_boundary_sets = num_sets; + // std::cout << " DEBUG PRINT "<(num_sets, "Boundary_Condition_Type_List"); + NBoundary_Condition_Patches = CArrayKokkos(num_sets, "NBoundary_Condition_Patches"); + // std::cout << "NBOUNDARY PATCHES ON RANK " << myrank << " FOR INIT IS " << nboundary_patches <(num_sets, nboundary_patches, "Boundary_Condition_Patches"); + + // initialize data + for (int iset = 0; iset < num_sets; iset++) + { + NBoundary_Condition_Patches(iset) = 0; + } + + // initialize + for (int ibdy = 0; ibdy < num_sets; ibdy++) + { + Boundary_Condition_Type_List(ibdy) = NONE; + } +} + +/* ---------------------------------------------------------------------------- + Grow boundary conditions sets of element boundary surfaces +------------------------------------------------------------------------------- */ + +void FEA_Module_SGH::grow_boundary_sets(int num_sets) +{ + int num_dim = simparam->num_dims; + + if (num_sets == 0) + { + std::cout << " Warning: number of boundary conditions being set to 0"; + return; + } + + // std::cout << " DEBUG PRINT "< max_boundary_sets) + { + // temporary storage for previous data + CArrayKokkos Temp_Boundary_Condition_Type_List = Boundary_Condition_Type_List; + CArrayKokkos Temp_NBoundary_Condition_Patches = NBoundary_Condition_Patches; + CArrayKokkos Temp_Boundary_Condition_Patches = Boundary_Condition_Patches; + + max_boundary_sets = num_sets + 5; // 5 is an arbitrary buffer + Boundary_Condition_Type_List = CArrayKokkos(max_boundary_sets, "Boundary_Condition_Type_List"); + NBoundary_Condition_Patches = CArrayKokkos(max_boundary_sets, "NBoundary_Condition_Patches"); + // std::cout << "NBOUNDARY PATCHES ON RANK " << myrank << " FOR GROW " << nboundary_patches <(max_boundary_sets, nboundary_patches, "Boundary_Condition_Patches"); + + // copy previous data back over + // std::cout << "NUM BOUNDARY CONDITIONS ON RANK " << myrank << " FOR COPY " << max_boundary_sets < +#include +#include +#include +#include +#include // fmin, fmax, abs note: fminl is long +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include "Tpetra_Details_makeColMap.hpp" +#include "Tpetra_Details_DefaultTypes.hpp" +#include "Tpetra_Details_FixedHashTable.hpp" +#include "Tpetra_Import.hpp" +#include "Tpetra_Import_Util2.hpp" + +#include "elements.h" +#include "swage.h" +#include "matar.h" +#include "utilities.h" +#include "node_combination.h" +#include "FEA_Module_SGH.h" +#include "Explicit_Solver.h" +#include "Simulation_Parameters/Simulation_Parameters_Explicit.h" +#include "Simulation_Parameters/FEA_Module/SGH_Parameters.h" + +// optimization +#include "ROL_Solver.hpp" +#include "Kinetic_Energy_Minimize.h" + +/* ---------------------------------------------------------------------- + Compute new system response due to the design variable update +------------------------------------------------------------------------- */ + +void FEA_Module_SGH::update_forward_solve(Teuchos::RCP zp) +{ + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + // local variable for host view in the dual view + int num_dim = simparam->num_dims; + int nodes_per_elem = max_nodes_per_element; + int local_node_index, current_row, current_column; + int max_stride = 0; + int current_module_index; + size_t access_index, row_access_index, row_counter; + GO global_index, global_dof_index; + LO local_dof_index; + + const size_t num_fills = simparam->regions.size(); + const size_t rk_num_bins = simparam->dynamic_options.rk_num_bins; + const size_t num_bcs = module_params->boundary_conditions.size(); + const size_t num_materials = simparam->materials.size(); + real_t objective_accumulation; + + // --- Read in the nodes in the mesh --- + int myrank = Explicit_Solver_Pointer_->myrank; + int nranks = Explicit_Solver_Pointer_->nranks; + + const DCArrayKokkos mat_fill = simparam->mat_fill; + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos material = simparam->material; + CArray current_element_nodal_densities = CArray(num_nodes_in_elem); + + std::vector> FEA_Module_My_TO_Modules = simparam->FEA_Module_My_TO_Modules; + problem = Explicit_Solver_Pointer_->problem; // Pointer to ROL optimization problem object + ROL::Ptr> obj_pointer; + + // compute element averaged density ratios corresponding to nodal density design variables + { // view scope + const_host_vec_array all_node_densities = all_node_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); + // debug print + // std::cout << "NODE DENSITY TEST " << all_node_densities(0,0) << std::endl; + for (int elem_id = 0; elem_id < rnum_elem; elem_id++) + { + for (int inode = 0; inode < num_nodes_in_elem; inode++) + { + current_element_nodal_densities(inode) = all_node_densities(nodes_in_elem(elem_id, inode), 0); + } + relative_element_densities.host(elem_id) = average_element_density(num_nodes_in_elem, current_element_nodal_densities); + } // for + } // view scope + // debug print + // std::cout << "ELEMENT RELATIVE DENSITY TEST " << relative_element_densities.host(0) << std::endl; + relative_element_densities.update_device(); + + // set density vector to the current value chosen by the optimizer + test_node_densities_distributed = zp; + + // reset nodal coordinates to initial values + node_coords_distributed->assign(*initial_node_coords_distributed); + + // comms for ghosts + Explicit_Solver_Pointer_->comm_coordinates(); + + // view scope + { + const_vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array ghost_node_coords_interface = Explicit_Solver_Pointer_->ghost_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + all_node_coords_interface(node_gid, idim) = node_coords_interface(node_gid, idim); + } + }); // end parallel for + Kokkos::fence(); + + FOR_ALL_CLASS(node_gid, nlocal_nodes, nlocal_nodes + nghost_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + all_node_coords_interface(node_gid, idim) = ghost_node_coords_interface(node_gid - nlocal_nodes, idim); + } + }); // end parallel for + Kokkos::fence(); + } // end view scope + + // reset velocities to initial conditions + node_velocities_distributed->assign(*initial_node_velocities_distributed); + + // reset time accumulating objective and constraints + /* + for(int imodule = 0 ; imodule < FEA_Module_My_TO_Modules[my_fea_module_index_].size(); imodule++){ + current_module_index = FEA_Module_My_TO_Modules[my_fea_module_index_][imodule]; + //test if module needs reset + if(){ + + } + } + */ + // simple setup to just request KE for now; above loop to be expanded and used later for scanning modules + obj_pointer = problem->getObjective(); + KineticEnergyMinimize_TopOpt& kinetic_energy_minimize_function = dynamic_cast(*obj_pointer); + kinetic_energy_minimize_function.objective_accumulation = 0; + + // interface trial density vector + + // interfacing of vectors(should be removed later once made compatible) + // view scope + { + host_vec_array interface_node_coords = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + for (size_t ibin = 0; ibin < rk_num_bins; ibin++) + { + // save node data to node.coords + // std::cout << "NODE DATA ON RANK " << myrank << std::endl; + if (num_dim == 2) + { + for (int inode = 0; inode < nall_nodes; inode++) + { + // std::cout << "Node index " << inode+1 << " "; + node_coords.host(ibin, inode, 0) = interface_node_coords(inode, 0); + // std::cout << host_node_coords_state(0,inode,0)+1<< " "; + node_coords.host(ibin, inode, 1) = interface_node_coords(inode, 1); + // std::cout << host_node_coords_state(0,inode,1)+1<< " "; + } + } + else if (num_dim == 3) + { + for (int inode = 0; inode < nall_nodes; inode++) + { + // std::cout << "Node index " << inode+1 << " "; + node_coords.host(ibin, inode, 0) = interface_node_coords(inode, 0); + // std::cout << host_node_coords_state(0,inode,0)+1<< " "; + node_coords.host(ibin, inode, 1) = interface_node_coords(inode, 1); + // std::cout << host_node_coords_state(0,inode,1)+1<< " "; + + node_coords.host(ibin, inode, 2) = interface_node_coords(inode, 2); + // std::cout << host_node_coords_state(0,inode,2)+1<< std::endl; + } + } + } + } // end view scope + + // save the node coords to the current RK value + for (size_t node_gid = 0; node_gid < nall_nodes; node_gid++) + { + for (int rk = 1; rk < rk_num_bins; rk++) + { + for (int dim = 0; dim < num_dim; dim++) + { + node_coords.host(rk, node_gid, dim) = node_coords.host(0, node_gid, dim); + } // end for dim + } // end for rk + } // end parallel for + + node_coords.update_device(); + + // setup that needs repeating + get_vol(); + // --- apply the fill instructions over the Elements---// + + // loop over the fill instructures + // view scope + { + for (int f_id = 0; f_id < num_fills; f_id++) + { + // parallel loop over elements in mesh + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { + // calculate the coordinates and radius of the element + double elem_coords[3]; // note:initialization with a list won't work + elem_coords[0] = 0.0; + elem_coords[1] = 0.0; + elem_coords[2] = 0.0; + + // get the coordinates of the element center + for (int node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + elem_coords[0] += node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), 0); + elem_coords[1] += node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), 1); + if (num_dim == 3) + { + elem_coords[2] += node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), 2); + } + else + { + elem_coords[2] = 0.0; + } + } // end loop over nodes in element + elem_coords[0] = elem_coords[0] / num_nodes_in_elem; + elem_coords[1] = elem_coords[1] / num_nodes_in_elem; + elem_coords[2] = elem_coords[2] / num_nodes_in_elem; + + // default is not to fill the element + bool fill_this = mat_fill(f_id).volume.contains(elem_coords); + + // paint the material state on the element + if (fill_this) + { + // density + elem_den(elem_gid) = mat_fill(f_id).den; + + // compute element average density from initial nodal density variables used as TO design variables + elem_den(elem_gid) = elem_den(elem_gid) * relative_element_densities(elem_gid); + + // mass + elem_mass(elem_gid) = elem_den(elem_gid) * elem_vol(elem_gid); + + // specific internal energy + elem_sie(rk_level, elem_gid) = mat_fill(f_id).sie; + + elem_mat_id(elem_gid) = mat_fill(f_id).material_id; + size_t mat_id = elem_mat_id(elem_gid); // short name + + // --- stress tensor --- + // always 3D even for 2D-RZ + for (size_t i = 0; i < 3; i++) + { + for (size_t j = 0; j < 3; j++) + { + elem_stress(rk_level, elem_gid, i, j) = 0.0; + } + } // end for + + // short form for clean code + EOSParent* eos_model = elem_eos(elem_gid).model; + + // --- Pressure --- + eos_model->calc_pressure(elem_pres, + elem_stress, + elem_gid, + elem_mat_id(elem_gid), + state_vars, + global_vars, + elem_user_output_vars, + elem_sspd, + elem_den(elem_gid), + elem_sie(rk_level, elem_gid)); + + // --- Sound speed --- + eos_model->calc_sound_speed(elem_pres, + elem_stress, + elem_gid, + elem_mat_id(elem_gid), + state_vars, + global_vars, + elem_user_output_vars, + elem_sspd, + elem_den(elem_gid), + elem_sie(rk_level, elem_gid)); + + // loop over the nodes of this element and apply velocity + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + // get the mesh node index + size_t node_gid = nodes_in_elem(elem_gid, node_lid); + + // --- Velocity --- + switch (mat_fill(f_id).velocity) + { + case VELOCITY_TYPE::cartesian: + { + node_vel(rk_level, node_gid, 0) = mat_fill(f_id).u; + node_vel(rk_level, node_gid, 1) = mat_fill(f_id).v; + if (num_dim == 3) + { + node_vel(rk_level, node_gid, 2) = mat_fill(f_id).w; + } + + break; + } + case VELOCITY_TYPE::radial: + { + // Setting up cylindrical + double dir[2]; + dir[0] = 0.0; + dir[1] = 0.0; + double radius_val = 0.0; + + for (int dim = 0; dim < 2; dim++) + { + dir[dim] = node_coords(rk_level, node_gid, dim); + radius_val += node_coords(rk_level, node_gid, dim) * node_coords(rk_level, node_gid, dim); + } // end for + radius_val = sqrt(radius_val); + + for (int dim = 0; dim < 2; dim++) + { + if (radius_val > 1.0e-14) + { + dir[dim] /= (radius_val); + } + else + { + dir[dim] = 0.0; + } + } // end for + + node_vel(rk_level, node_gid, 0) = mat_fill(f_id).speed * dir[0]; + node_vel(rk_level, node_gid, 1) = mat_fill(f_id).speed * dir[1]; + if (num_dim == 3) + { + node_vel(rk_level, node_gid, 2) = 0.0; + } + + break; + } + case VELOCITY_TYPE::spherical: + { + // Setting up spherical + double dir[3]; + dir[0] = 0.0; + dir[1] = 0.0; + dir[2] = 0.0; + double radius_val = 0.0; + + for (int dim = 0; dim < 3; dim++) + { + dir[dim] = node_coords(rk_level, node_gid, dim); + radius_val += node_coords(rk_level, node_gid, dim) * node_coords(rk_level, node_gid, dim); + } // end for + radius_val = sqrt(radius_val); + + for (int dim = 0; dim < 3; dim++) + { + if (radius_val > 1.0e-14) + { + dir[dim] /= (radius_val); + } + else + { + dir[dim] = 0.0; + } + } // end for + + node_vel(rk_level, node_gid, 0) = mat_fill(f_id).speed * dir[0]; + node_vel(rk_level, node_gid, 1) = mat_fill(f_id).speed * dir[1]; + if (num_dim == 3) + { + node_vel(rk_level, node_gid, 2) = mat_fill(f_id).speed * dir[2]; + } + + break; + } + case VELOCITY_TYPE::radial_linear: + { + break; + } + case VELOCITY_TYPE::spherical_linear: + { + break; + } + case VELOCITY_TYPE::tg_vortex: + { + node_vel(rk_level, node_gid, 0) = sin(PI * node_coords(rk_level, node_gid, 0)) * cos(PI * node_coords(rk_level, node_gid, 1)); + node_vel(rk_level, node_gid, 1) = -1.0 * cos(PI * node_coords(rk_level, node_gid, 0)) * sin(PI * node_coords(rk_level, node_gid, 1)); + if (num_dim == 3) + { + node_vel(rk_level, node_gid, 2) = 0.0; + } + + break; + } + } // end of switch + } // end loop over nodes of element + + if (mat_fill(f_id).velocity == VELOCITY_TYPE::tg_vortex) + { + elem_pres(elem_gid) = 0.25 * (cos(2.0 * PI * elem_coords[0]) + cos(2.0 * PI * elem_coords[1]) ) + 1.0; + + // p = rho*ie*(gamma - 1) + size_t mat_id = f_id; + double gamma = global_vars(mat_id, 0); // gamma value + elem_sie(rk_level, elem_gid) = + elem_pres(elem_gid) / (mat_fill(f_id).den * (gamma - 1.0)); + } // end if + } // end if fill + }); // end FOR_ALL_CLASS element loop + Kokkos::fence(); + } // end for loop over fills + } // end view scope + + // apply BC's to velocity + FEA_Module_SGH::boundary_velocity(*mesh, boundary, node_vel); + + // calculate the corner massess if 2D + if (num_dim == 2) + { + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { + // facial area of the corners + double corner_areas_array[4]; + + ViewCArrayKokkos corner_areas(&corner_areas_array[0], 4); + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 4); + + get_area_weights2D(corner_areas, + elem_gid, + node_coords, + elem_node_gids, + rk_level); + + // loop over the corners of the element and calculate the mass + for (size_t corner_lid = 0; corner_lid < 4; corner_lid++) + { + size_t corner_gid = corners_in_elem(elem_gid, corner_lid); + corner_mass(corner_gid) = corner_areas(corner_lid) * elem_den(elem_gid); // node radius is added later + } // end for over corners + }); + } // end of + + // calculate the nodal mass + FOR_ALL_CLASS(node_gid, 0, nall_nodes, { + node_mass(node_gid) = 0.0; + + if (num_dim == 3) + { + for (size_t elem_lid = 0; elem_lid < num_corners_in_node(node_gid); elem_lid++) + { + size_t elem_gid = elems_in_node(node_gid, elem_lid); + node_mass(node_gid) += 1.0 / 8.0 * elem_mass(elem_gid); + } // end for elem_lid + } // end if dims=3 + else + { + // 2D-RZ + for (size_t corner_lid = 0; corner_lid < num_corners_in_node(node_gid); corner_lid++) + { + size_t corner_gid = corners_in_node(node_gid, corner_lid); + node_mass(node_gid) += corner_mass(corner_gid); // sans the radius so it is areal node mass + + corner_mass(corner_gid) *= node_coords(rk_level, node_gid, 1); // true corner mass now + } // end for elem_lid + } // end else + }); // end FOR_ALL_CLASS + Kokkos::fence(); + + // current interface has differing mass arrays; this equates them until we unify memory + // view scope + { + vec_array node_mass_interface = node_masses_distributed->getLocalView(Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + node_mass_interface(node_gid, 0) = node_mass(node_gid); + }); // end parallel for + } // end view scope + Kokkos::fence(); + // communicate ghost densities + comm_node_masses(); + + // this is forcing a copy to the device + // view scope + { + vec_array ghost_node_mass_interface = ghost_node_masses_distributed->getLocalView(Tpetra::Access::ReadWrite); + + FOR_ALL_CLASS(node_gid, nlocal_nodes, nall_nodes, { + node_mass(node_gid) = ghost_node_mass_interface(node_gid - nlocal_nodes, 0); + }); // end parallel for + } // end view scope + Kokkos::fence(); + + // update stiffness matrix + if (simparam->topology_optimization_on || simparam->shape_optimization_on) + { + // assemble_matrix(); + } + + // update host copies of arrays modified in this function + elem_den.update_host(); + elem_mass.update_host(); + elem_sie.update_host(); + elem_stress.update_host(); + elem_pres.update_host(); + elem_sspd.update_host(); + + // execute solve + sgh_solve(); +} + +/* ------------------------------------------------------------------------------------------- + Compute average density of an element from nodal densities +---------------------------------------------------------------------------------------------- */ + +double FEA_Module_SGH::average_element_density(const int nodes_per_elem, const CArray current_element_densities) const +{ + double result = 0; + for (int i = 0; i < nodes_per_elem; i++) + { + result += current_element_densities(i) / nodes_per_elem; + } + + return result; +} + +/* ------------------------------------------------------------------------------ + Coupled adjoint problem for the kinetic energy minimization problem +--------------------------------------------------------------------------------- */ + +void FEA_Module_SGH::compute_topology_optimization_adjoint_full() +{ + const int num_dim = simparam->num_dims; + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos material = simparam->material; + + real_t global_dt; + size_t current_data_index, next_data_index; + size_t num_bdy_nodes = mesh->num_bdy_nodes; + + Teuchos::RCP previous_adjoint_vector_distributed, current_adjoint_vector_distributed; + Teuchos::RCP previous_velocity_vector_distributed, current_velocity_vector_distributed; + Teuchos::RCP previous_phi_adjoint_vector_distributed, current_phi_adjoint_vector_distributed; + + // initialize first adjoint vector at last_time_step to 0 as the terminal value + (*adjoint_vector_data)[last_time_step + 1]->putScalar(0); + (*phi_adjoint_vector_data)[last_time_step + 1]->putScalar(0); + (*psi_adjoint_vector_data)[last_time_step + 1]->putScalar(0); + + // solve terminal value problem, proceeds in time backward. For simplicity, we use the same timestep data from the forward solve. + // A linear interpolant is assumed between velocity data points; velocity midpoint is used to update the adjoint. + if (myrank == 0) + { + std::cout << "Computing adjoint vector " << time_data.size() << std::endl; + } + + for (int cycle = last_time_step; cycle >= 0; cycle--) + { + // compute timestep from time data + global_dt = time_data[cycle + 1] - time_data[cycle]; + + // print + if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) + { + if (cycle == last_time_step) + { + if (myrank == 0) + { + printf("cycle = %d, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } + // print time step every 20 cycles + else if (cycle % 20 == 0) + { + if (myrank == 0) + { + printf("cycle = %d, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } // end if + } + // else if (cycle==1){ + // if(myrank==0) + // printf("cycle = %lu, time = %f, time step = %f \n", cycle-1, time_data[cycle-1], global_dt); + // } // end if + + // compute adjoint vector for this data point; use velocity midpoint + // view scope + { + // set velocity, internal energy, and position for this timestep + const_vec_array previous_velocity_vector = (*forward_solve_velocity_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + + const_vec_array previous_coordinate_vector = (*forward_solve_coordinate_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array current_coordinate_vector = (*forward_solve_coordinate_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + + const_vec_array previous_element_internal_energy = (*forward_solve_internal_energy_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array current_element_internal_energy = (*forward_solve_internal_energy_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + + // interface of arrays for current implementation of force calculation + + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes + nghost_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + node_vel(rk_level, node_gid, idim) = previous_velocity_vector(node_gid, idim); + node_coords(rk_level, node_gid, idim) = previous_coordinate_vector(node_gid, idim); + } + }); + Kokkos::fence(); + + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { + elem_sie(rk_level, elem_gid) = previous_element_internal_energy(elem_gid, 0); + }); + Kokkos::fence(); + + // set state according to phase data at this timestep + + get_vol(); + + // ---- Calculate velocity diveregence for the element ---- + if (num_dim == 2) + { + get_divergence2D(elem_div, + node_coords, + node_vel, + elem_vol); + } + else + { + get_divergence(elem_div, + node_coords, + node_vel, + elem_vol); + } // end if 2D + + // ---- Calculate elem state (den, pres, sound speed, stress) for next time step ---- + if (num_dim == 2) + { + update_state2D(material, + *mesh, + node_coords, + node_vel, + elem_den, + elem_pres, + elem_stress, + elem_sspd, + elem_sie, + elem_vol, + elem_mass, + elem_mat_id, + 1.0, + cycle); + } + else + { + update_state(material, + *mesh, + node_coords, + node_vel, + elem_den, + elem_pres, + elem_stress, + elem_sspd, + elem_sie, + elem_vol, + elem_mass, + elem_mat_id, + 1.0, + cycle); + } + + if (num_dim == 2) + { + get_force_sgh2D(material, + *mesh, + node_coords, + node_vel, + elem_den, + elem_sie, + elem_pres, + elem_stress, + elem_sspd, + elem_vol, + elem_div, + elem_mat_id, + corner_force, + 1.0, + cycle); + } + else + { + get_force_sgh(material, + *mesh, + node_coords, + node_vel, + elem_den, + elem_sie, + elem_pres, + elem_stress, + elem_sspd, + elem_vol, + elem_div, + elem_mat_id, + corner_force, + 1.0, + cycle); + } + + // compute gradient matrices + get_force_egradient_sgh(material, + *mesh, + node_coords, + node_vel, + elem_den, + elem_sie, + elem_pres, + elem_stress, + elem_sspd, + elem_vol, + elem_div, + elem_mat_id, + 1.0, + cycle); + + get_power_egradient_sgh(1.0, + *mesh, + node_vel, + node_coords, + elem_sie, + elem_mass, + corner_force); + + get_force_vgradient_sgh(material, + *mesh, + node_coords, + node_vel, + elem_den, + elem_sie, + elem_pres, + elem_stress, + elem_sspd, + elem_vol, + elem_div, + elem_mat_id, + 1.0, + cycle); + + get_power_vgradient_sgh(1.0, + *mesh, + node_vel, + node_coords, + elem_sie, + elem_mass, + corner_force); + + get_force_ugradient_sgh(material, + *mesh, + node_coords, + node_vel, + elem_den, + elem_sie, + elem_pres, + elem_stress, + elem_sspd, + elem_vol, + elem_div, + elem_mat_id, + 1.0, + cycle); + + get_power_ugradient_sgh(1.0, + *mesh, + node_vel, + node_coords, + elem_sie, + elem_mass, + corner_force); + + // force_gradient_velocity->describe(*fos,Teuchos::VERB_EXTREME); + const_vec_array previous_force_gradient_position = force_gradient_position->getLocalView(Tpetra::Access::ReadOnly); + // const_vec_array current_force_gradient_position = force_gradient_position->getLocalView (Tpetra::Access::ReadOnly); + const_vec_array previous_force_gradient_velocity = force_gradient_velocity->getLocalView(Tpetra::Access::ReadOnly); + // const_vec_array current_force_gradient_velocity = force_gradient_velocity->getLocalView (Tpetra::Access::ReadOnly); + // compute gradient of force with respect to velocity + + const_vec_array previous_adjoint_vector = (*adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array phi_previous_adjoint_vector = (*phi_adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array psi_previous_adjoint_vector = (*psi_adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + vec_array midpoint_adjoint_vector = adjoint_vector_distributed->getLocalView(Tpetra::Access::ReadWrite); + vec_array phi_midpoint_adjoint_vector = phi_adjoint_vector_distributed->getLocalView(Tpetra::Access::ReadWrite); + vec_array psi_midpoint_adjoint_vector = psi_adjoint_vector_distributed->getLocalView(Tpetra::Access::ReadWrite); + + // half step update for RK2 scheme; EQUATION 1 + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + real_t rate_of_change; + real_t matrix_contribution; + size_t dof_id; + size_t elem_id; + for (int idim = 0; idim < num_dim; idim++) + { + // EQUATION 1 + matrix_contribution = 0; + // compute resulting row of force velocity gradient matrix transpose right multiplied by adjoint vector + for (int idof = 0; idof < Gradient_Matrix_Strides(node_gid * num_dim + idim); idof++) + { + dof_id = DOF_Graph_Matrix(node_gid * num_dim + idim, idof); + matrix_contribution += previous_adjoint_vector(dof_id / num_dim, dof_id % num_dim) * Force_Gradient_Velocities(node_gid * num_dim + idim, idof); + } + + // compute resulting row of transpose of power gradient w.r.t velocity matrix right multiplied by psi adjoint vector + for (int ielem = 0; ielem < DOF_to_Elem_Matrix_Strides(node_gid * num_dim + idim); ielem++) + { + elem_id = elems_in_node(node_gid, ielem); + matrix_contribution += psi_previous_adjoint_vector(elem_id, 0) * Power_Gradient_Velocities(node_gid * num_dim + idim, ielem); + } + + rate_of_change = previous_velocity_vector(node_gid, idim) - + matrix_contribution / node_mass(node_gid) - + phi_previous_adjoint_vector(node_gid, idim) / node_mass(node_gid); + midpoint_adjoint_vector(node_gid, idim) = -rate_of_change * global_dt / 2 + previous_adjoint_vector(node_gid, idim); + } + }); // end parallel for + Kokkos::fence(); + + // apply BCs to adjoint vector, only matters for the momentum adjoint if using strictly velocity boundary conditions + boundary_adjoint(*mesh, boundary, midpoint_adjoint_vector, phi_midpoint_adjoint_vector, psi_midpoint_adjoint_vector); + + comm_adjoint_vector(cycle); + + // half step update for RK2 scheme; EQUATION 2 + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + real_t rate_of_change; + real_t matrix_contribution; + size_t dof_id; + size_t elem_id; + for (int idim = 0; idim < num_dim; idim++) + { + // EQUATION 2 + matrix_contribution = 0; + // compute resulting row of force displacement gradient matrix transpose right multiplied by adjoint vector + for (int idof = 0; idof < Gradient_Matrix_Strides(node_gid * num_dim + idim); idof++) + { + dof_id = DOF_Graph_Matrix(node_gid * num_dim + idim, idof); + matrix_contribution += previous_adjoint_vector(dof_id / num_dim, dof_id % num_dim) * Force_Gradient_Positions(node_gid * num_dim + idim, idof); + } + + // compute resulting row of transpose of power gradient w.r.t displacement matrix right multiplied by psi adjoint vector + for (int ielem = 0; ielem < DOF_to_Elem_Matrix_Strides(node_gid * num_dim + idim); ielem++) + { + elem_id = elems_in_node(node_gid, ielem); + matrix_contribution += psi_previous_adjoint_vector(elem_id, 0) * Power_Gradient_Positions(node_gid * num_dim + idim, ielem); + } + + rate_of_change = -matrix_contribution; + // rate_of_change = -0.0000001*previous_adjoint_vector(node_gid,idim); + phi_midpoint_adjoint_vector(node_gid, idim) = -rate_of_change * global_dt / 2 + phi_previous_adjoint_vector(node_gid, idim); + } + }); // end parallel for + Kokkos::fence(); + + comm_phi_adjoint_vector(cycle); + + // phi_adjoint_vector_distributed->describe(*fos,Teuchos::VERB_EXTREME); + + // half step update for RK2 scheme; EQUATION 3 + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { + real_t rate_of_change; + real_t matrix_contribution; + size_t dof_id; + size_t elem_id; + // EQUATION 3 + matrix_contribution = 0; + // compute resulting row of force displacement gradient matrix transpose right multiplied by adjoint vector + for (int idof = 0; idof < num_nodes_in_elem * num_dim; idof++) + { + dof_id = nodes_in_elem(elem_gid, idof / num_dim) * num_dim + idof % num_dim; + matrix_contribution += previous_adjoint_vector(dof_id / num_dim, dof_id % num_dim) * Force_Gradient_Energies(elem_gid, idof); + } + rate_of_change = -(matrix_contribution + psi_previous_adjoint_vector(elem_gid, 0) * Power_Gradient_Energies(elem_gid)) / elem_mass(elem_gid); + // rate_of_change = -0.0000001*previous_adjoint_vector(node_gid,idim); + psi_midpoint_adjoint_vector(elem_gid, 0) = -rate_of_change * global_dt / 2 + psi_previous_adjoint_vector(elem_gid, 0); + }); // end parallel for + Kokkos::fence(); + + // save for second half of RK + (*psi_adjoint_vector_data)[cycle]->assign(*psi_adjoint_vector_distributed); + + // swap names to get ghost nodes for the midpoint vectors + vec_array current_adjoint_vector = adjoint_vector_distributed->getLocalView(Tpetra::Access::ReadWrite); + vec_array phi_current_adjoint_vector = phi_adjoint_vector_distributed->getLocalView(Tpetra::Access::ReadWrite); + vec_array psi_current_adjoint_vector = psi_adjoint_vector_distributed->getLocalView(Tpetra::Access::ReadWrite); + midpoint_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadWrite); + phi_midpoint_adjoint_vector = (*phi_adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadWrite); + psi_midpoint_adjoint_vector = (*psi_adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadWrite); + + // full step update with midpoint gradient for RK2 scheme; EQUATION 1 + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + real_t rate_of_change; + real_t matrix_contribution; + size_t dof_id; + size_t elem_id; + for (int idim = 0; idim < num_dim; idim++) + { + // EQUATION 1 + matrix_contribution = 0; + // compute resulting row of force velocity gradient matrix transpose right multiplied by adjoint vector + + for (int idof = 0; idof < Gradient_Matrix_Strides(node_gid * num_dim + idim); idof++) + { + dof_id = DOF_Graph_Matrix(node_gid * num_dim + idim, idof); + matrix_contribution += midpoint_adjoint_vector(dof_id / num_dim, dof_id % num_dim) * Force_Gradient_Velocities(node_gid * num_dim + idim, idof); + } + + // compute resulting row of transpose of power gradient w.r.t velocity matrix right multiplied by psi adjoint vector + for (int ielem = 0; ielem < DOF_to_Elem_Matrix_Strides(node_gid * num_dim + idim); ielem++) + { + elem_id = elems_in_node(node_gid, ielem); + matrix_contribution += psi_midpoint_adjoint_vector(elem_id, 0) * Power_Gradient_Velocities(node_gid * num_dim + idim, ielem); + } + + rate_of_change = (previous_velocity_vector(node_gid, idim) + current_velocity_vector(node_gid, idim)) / 2 - + matrix_contribution / node_mass(node_gid) - + phi_midpoint_adjoint_vector(node_gid, idim) / node_mass(node_gid); + current_adjoint_vector(node_gid, idim) = -rate_of_change * global_dt + previous_adjoint_vector(node_gid, idim); + } + }); // end parallel for + Kokkos::fence(); + + boundary_adjoint(*mesh, boundary, current_adjoint_vector, phi_current_adjoint_vector, psi_midpoint_adjoint_vector); + comm_adjoint_vector(cycle); + + // full step update with midpoint gradient for RK2 scheme; EQUATION 2 + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + real_t rate_of_change; + real_t matrix_contribution; + size_t dof_id; + size_t elem_id; + for (int idim = 0; idim < num_dim; idim++) + { + // EQUATION 2 + matrix_contribution = 0; + // compute resulting row of force displacement gradient matrix transpose right multiplied by adjoint vector + for (int idof = 0; idof < Gradient_Matrix_Strides(node_gid * num_dim + idim); idof++) + { + dof_id = DOF_Graph_Matrix(node_gid * num_dim + idim, idof); + matrix_contribution += midpoint_adjoint_vector(dof_id / num_dim, dof_id % num_dim) * Force_Gradient_Positions(node_gid * num_dim + idim, idof); + } + + // compute resulting row of transpose of power gradient w.r.t displacement matrix right multiplied by psi adjoint vector + for (int ielem = 0; ielem < DOF_to_Elem_Matrix_Strides(node_gid * num_dim + idim); ielem++) + { + elem_id = elems_in_node(node_gid, ielem); + matrix_contribution += psi_midpoint_adjoint_vector(elem_id, 0) * Power_Gradient_Positions(node_gid * num_dim + idim, ielem); + } + + rate_of_change = -matrix_contribution; + // rate_of_change = -0.0000001*midpoint_adjoint_vector(node_gid,idim); + phi_current_adjoint_vector(node_gid, idim) = -rate_of_change * global_dt + phi_previous_adjoint_vector(node_gid, idim); + } + }); // end parallel for + Kokkos::fence(); + + comm_phi_adjoint_vector(cycle); + + // full step update for RK2 scheme; EQUATION 3 + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { + real_t rate_of_change; + real_t matrix_contribution; + size_t dof_id; + size_t elem_id; + // EQUATION 3 + matrix_contribution = 0; + // compute resulting row of force displacement gradient matrix transpose right multiplied by adjoint vector + for (int idof = 0; idof < num_nodes_in_elem * num_dim; idof++) + { + dof_id = nodes_in_elem(elem_gid, idof / num_dim) * num_dim + idof % num_dim; + matrix_contribution += midpoint_adjoint_vector(dof_id / num_dim, dof_id % num_dim) * Force_Gradient_Energies(elem_gid, idof); + } + rate_of_change = -(matrix_contribution + psi_midpoint_adjoint_vector(elem_gid, 0) * Power_Gradient_Energies(elem_gid)) / elem_mass(elem_gid); + // debug + // std::cout << "PSI RATE OF CHANGE " << rate_of_change << std::endl; + psi_current_adjoint_vector(elem_gid, 0) = -rate_of_change * global_dt + psi_previous_adjoint_vector(elem_gid, 0); + }); // end parallel for + Kokkos::fence(); + + // save data from time-step completion + (*psi_adjoint_vector_data)[cycle]->assign(*psi_adjoint_vector_distributed); + } // end view scope + + // phi_adjoint_vector_distributed->describe(*fos,Teuchos::VERB_EXTREME); + } +} + +/* ---------------------------------------------------------------------------- + Gradient for the (unsimplified) kinetic energy minimization problem +------------------------------------------------------------------------------- */ + +void FEA_Module_SGH::compute_topology_optimization_gradient_full(Teuchos::RCP design_densities_distributed, Teuchos::RCP design_gradients_distributed) +{ + size_t num_bdy_nodes = mesh->num_bdy_nodes; + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos material = simparam->material; + const int num_dim = simparam->num_dims; + int num_corners = rnum_elem * num_nodes_in_elem; + real_t global_dt; + bool element_constant_density = true; + size_t current_data_index, next_data_index; + CArrayKokkos current_element_velocities = CArrayKokkos(num_nodes_in_elem, num_dim); + CArrayKokkos current_element_adjoint = CArrayKokkos(num_nodes_in_elem, num_dim); + + if (myrank == 0) + { + std::cout << "Computing accumulated kinetic energy gradient" << std::endl; + } + + compute_topology_optimization_adjoint_full(); + + { // view scope + vec_array design_gradients = design_gradients_distributed->getLocalView(Tpetra::Access::ReadWrite); + const_vec_array design_densities = design_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); + // initialize design gradients + FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { + design_gradients(node_id, 0) = 0; + }); // end parallel for + Kokkos::fence(); + + // gradient contribution from kinetic energy v(dM/drho)v product. + if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) + { + if (myrank == 0) + { + std::cout << "v*dM/drho*v term" << std::endl; + } + } + + for (unsigned long cycle = 0; cycle < last_time_step + 1; cycle++) + { + // compute timestep from time data + global_dt = time_data[cycle + 1] - time_data[cycle]; + + // print + if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) + { + if (cycle == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } + // print time step every 10 cycles + else if (cycle % 20 == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } // end if + } + // view scope + { + const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_velocity_vector = (*forward_solve_velocity_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + + FOR_ALL_CLASS(elem_id, 0, rnum_elem, { + size_t node_id; + size_t corner_id; + real_t inner_product; + // std::cout << elem_mass(elem_id) <dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) + { + if (myrank == 0) + { + std::cout << "gradient term involving adjoint derivative" << std::endl; + } + } + + for (unsigned long cycle = 0; cycle < last_time_step + 1; cycle++) + { + // compute timestep from time data + global_dt = time_data[cycle + 1] - time_data[cycle]; + // print + if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) + { + if (cycle == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } + // print time step every 10 cycles + else if (cycle % 20 == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } // end if + } + + // compute adjoint vector for this data point; use velocity midpoint + // view scope + { + const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_velocity_vector = (*forward_solve_velocity_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_adjoint_vector = (*adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + + FOR_ALL_CLASS(elem_id, 0, rnum_elem, { + real_t lambda_dot_current; + real_t lambda_dot_next; + size_t node_id; + size_t corner_id; + real_t inner_product; + // std::cout << elem_mass(elem_id) <dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) + { + if (myrank == 0) + { + std::cout << "gradient term involving adjoint derivative" << std::endl; + } + } + + for (unsigned long cycle = 0; cycle < last_time_step + 1; cycle++) + { + // compute timestep from time data + global_dt = time_data[cycle + 1] - time_data[cycle]; + // print + if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) + { + if (cycle == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } + // print time step every 10 cycles + else if (cycle % 20 == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } // end if + } + + // compute adjoint vector for this data point; use velocity midpoint + // view scope + { + const_vec_array current_element_internal_energy = (*forward_solve_internal_energy_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array current_psi_adjoint_vector = (*psi_adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_element_internal_energy = (*forward_solve_internal_energy_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_psi_adjoint_vector = (*psi_adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + + FOR_ALL_CLASS(elem_id, 0, rnum_elem, { + real_t psi_dot_current; + real_t psi_dot_next; + size_t node_id; + size_t corner_id; + real_t inner_product; + // std::cout << elem_mass(elem_id) <getLocalView(Tpetra::Access::ReadOnly); + const_vec_array current_adjoint_vector = (*adjoint_vector_data)[0]->getLocalView(Tpetra::Access::ReadOnly); + + FOR_ALL_CLASS(elem_id, 0, rnum_elem, { + real_t lambda_dot; + size_t node_id; + size_t corner_id; + real_t inner_product; + // std::cout << elem_mass(elem_id) <getLocalView(Tpetra::Access::ReadOnly); + const_vec_array current_psi_adjoint_vector = (*psi_adjoint_vector_data)[0]->getLocalView(Tpetra::Access::ReadOnly); + + // (*psi_adjoint_vector_data)[100]->describe(*fos,Teuchos::VERB_EXTREME); + FOR_ALL_CLASS(elem_id, 0, rnum_elem, { + real_t lambda_dot; + size_t node_id; + size_t corner_id; + real_t inner_product; + // std::cout << elem_mass(elem_id) <getLocalView (Tpetra::Access::ReadWrite); + // const_host_vec_array host_design_variables = design_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); + vec_array design_gradients = design_gradients_distributed->getLocalView(Tpetra::Access::ReadWrite); + const_vec_array design_variables = design_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); + force_design_gradient_term(design_variables, design_gradients); + power_design_gradient_term(design_variables, design_gradients); + } // end view scope +} + +/* ---------------------------------------------------------------------- + Initialize global vectors and array maps needed for matrix assembly +------------------------------------------------------------------------- */ +void FEA_Module_SGH::init_assembly() +{ + int num_dim = simparam->num_dims; + // const_host_elem_conn_array nodes_in_elem = global_nodes_in_elem_distributed->getLocalView (Tpetra::Access::ReadOnly); + Gradient_Matrix_Strides = DCArrayKokkos(nlocal_nodes * num_dim, "Gradient_Matrix_Strides"); + DOF_to_Elem_Matrix_Strides = DCArrayKokkos(nlocal_nodes * num_dim, "Gradient_Matrix_Strides"); + Elem_to_Elem_Matrix_Strides = DCArrayKokkos(rnum_elem, "Gradient_Matrix_Strides"); + CArrayKokkos Graph_Fill(nall_nodes, "nall_nodes"); + CArrayKokkos current_row_nodes_scanned; + CArrayKokkos count_saved_corners_in_node(nall_nodes, "count_saved_corners_in_node"); + int local_node_index, current_column_index; + size_t max_stride = 0; + size_t nodes_per_element; + nodal_density_flag = simparam->nodal_density_flag; + penalty_power = simparam->optimization_options.simp_penalty_power; + + // allocate stride arrays + CArrayKokkos Graph_Matrix_Strides_initial(nlocal_nodes, "Graph_Matrix_Strides_initial"); + DCArrayKokkos Dual_Graph_Matrix_Strides_initial(nlocal_nodes, "Host_Graph_Matrix_Strides_initial"); + Graph_Matrix_Strides = DCArrayKokkos(nlocal_nodes, "Graph_Matrix_Strides"); + + // allocate storage for the sparse gradient matrix map for node to node connectivity + Global_Gradient_Matrix_Assembly_Map = DCArrayKokkos(rnum_elem, + max_nodes_per_element, max_nodes_per_element, "Global_Gradient_Matrix_Assembly_Map"); + + // allocate storage for the sparse gradient matrix map for node to element connectivity + Element_Gradient_Matrix_Assembly_Map = DCArrayKokkos(rnum_elem, + max_nodes_per_element, "Element_Gradient_Matrix_Assembly_Map"); + + // allocate array used to determine global node repeats in the sparse graph later + DCArrayKokkos node_indices_used(nall_nodes, "node_indices_used"); + + /*allocate array that stores which column the node index occured on for the current row + when removing repeats*/ + DCArrayKokkos column_index(nall_nodes, "column_index"); + + // initialize nlocal arrays + FOR_ALL_CLASS(inode, 0, nlocal_nodes, { + Graph_Matrix_Strides_initial(inode) = 0; + Graph_Matrix_Strides(inode) = 0; + Graph_Fill(inode) = 0; + }); // end parallel for + Kokkos::fence(); + + // initialize nall arrays + // initialize nlocal arrays + FOR_ALL_CLASS(inode, 0, nall_nodes, { + node_indices_used(inode) = 0; + column_index(inode) = 0; + count_saved_corners_in_node(inode) = 0; + }); // end parallel for + Kokkos::fence(); + + // count upper bound of strides for Sparse Pattern Graph by allowing repeats due to connectivity + if (num_dim == 2) + { + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + element_select->choose_2Delem_type(Element_Types(ielem), elem2D); + nodes_per_element = elem2D->num_nodes(); + for (int lnode = 0; lnode < nodes_per_element; lnode++) + { + local_node_index = nodes_in_elem(ielem, lnode); + if (local_node_index < nlocal_nodes) + { + Dual_Graph_Matrix_Strides_initial.host(local_node_index) += nodes_per_element; + } + } + } + } + + if (num_dim == 3) + { + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + element_select->choose_3Delem_type(Element_Types(ielem), elem); + nodes_per_element = elem->num_nodes(); + for (int lnode = 0; lnode < nodes_per_element; lnode++) + { + local_node_index = nodes_in_elem(ielem, lnode); + if (local_node_index < nlocal_nodes) + { + Dual_Graph_Matrix_Strides_initial.host(local_node_index) += nodes_per_element; + } + } + } + } + + Dual_Graph_Matrix_Strides_initial.update_device(); + + // equate strides for later + FOR_ALL_CLASS(inode, 0, nlocal_nodes, { + Graph_Matrix_Strides(inode) = Graph_Matrix_Strides_initial(inode) = Dual_Graph_Matrix_Strides_initial(inode); + }); // end parallel for + + // for (int inode = 0; inode < nlocal_nodes; inode++) + // std::cout << Graph_Matrix_Strides_initial(inode) << std::endl; + + // compute maximum stride + size_t update = 0; + REDUCE_MAX_CLASS(inode, 0, nlocal_nodes, update, { + if (update < Graph_Matrix_Strides_initial(inode)) + { + update = Graph_Matrix_Strides_initial(inode); + } + }, max_stride); + + // std::cout << "THE MAX STRIDE" << max_stride << std::endl; + // allocate array used in the repeat removal process + current_row_nodes_scanned = CArrayKokkos(max_stride, "current_row_nodes_scanned"); + + // allocate sparse graph with node repeats + RaggedRightArrayKokkos Repeat_Graph_Matrix(Graph_Matrix_Strides_initial); + RaggedRightArrayofVectorsKokkos Element_local_indices(Graph_Matrix_Strides_initial, num_dim); + + // Fill the initial Graph with repeats + if (num_dim == 2) + { + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + element_select->choose_2Delem_type(Element_Types(ielem), elem2D); + nodes_per_element = elem2D->num_nodes(); + for (int lnode = 0; lnode < nodes_per_element; lnode++) + { + local_node_index = nodes_in_elem(ielem, lnode); + if (local_node_index < nlocal_nodes) + { + for (int jnode = 0; jnode < nodes_per_element; jnode++) + { + current_column_index = Graph_Fill(local_node_index) + jnode; + Repeat_Graph_Matrix(local_node_index, current_column_index) = nodes_in_elem(ielem, jnode); + + // fill inverse map + Element_local_indices(local_node_index, current_column_index, 0) = ielem; + Element_local_indices(local_node_index, current_column_index, 1) = lnode; + Element_local_indices(local_node_index, current_column_index, 2) = jnode; + + // fill forward map + Global_Gradient_Matrix_Assembly_Map(ielem, lnode, jnode) = current_column_index; + } + Graph_Fill(local_node_index) += nodes_per_element; + } + } + } + } + + if (num_dim == 3) + { + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + element_select->choose_3Delem_type(Element_Types(ielem), elem); + nodes_per_element = elem->num_nodes(); + for (int lnode = 0; lnode < nodes_per_element; lnode++) + { + local_node_index = nodes_in_elem(ielem, lnode); + if (local_node_index < nlocal_nodes) + { + for (int jnode = 0; jnode < nodes_per_element; jnode++) + { + current_column_index = Graph_Fill(local_node_index) + jnode; + Repeat_Graph_Matrix(local_node_index, current_column_index) = nodes_in_elem(ielem, jnode); + + // fill inverse map + Element_local_indices(local_node_index, current_column_index, 0) = ielem; + Element_local_indices(local_node_index, current_column_index, 1) = lnode; + Element_local_indices(local_node_index, current_column_index, 2) = jnode; + + // fill forward map + Global_Gradient_Matrix_Assembly_Map(ielem, lnode, jnode) = current_column_index; + } + Graph_Fill(local_node_index) += nodes_per_element; + } + } + } + } + + // debug statement + // std::cout << "started run" << std::endl; + // std::cout << "Graph Matrix Strides Repeat on task " << myrank << std::endl; + // for (int inode = 0; inode < nlocal_nodes; inode++) + // std::cout << Graph_Matrix_Strides(inode) << std::endl; + RUN_CLASS({ + // remove repeats from the inital graph setup + int current_node; + // remove repeats from the inital graph setup + int current_element_index; + int element_row_index; + int element_column_index; + int current_stride; + int current_row_n_nodes_scanned; + for (int inode = 0; inode < nlocal_nodes; inode++) + { + current_row_n_nodes_scanned = 0; + for (int istride = 0; istride < Graph_Matrix_Strides(inode); istride++) + { + // convert global index in graph to its local index for the flagging array + current_node = Repeat_Graph_Matrix(inode, istride); + // debug + // if(current_node==-1) + // std::cout << "Graph Matrix node access on task " << myrank << std::endl; + // std::cout << Repeat_Graph_Matrix(inode,istride) << std::endl; + if (node_indices_used(current_node)) + { + // set global assembly map index to the location in the graph matrix where this global node was first found + current_element_index = Element_local_indices(inode, istride, 0); + element_row_index = Element_local_indices(inode, istride, 1); + element_column_index = Element_local_indices(inode, istride, 2); + Global_Gradient_Matrix_Assembly_Map(current_element_index, element_row_index, element_column_index) + = column_index(current_node); + + // swap current node with the end of the current row and shorten the stride of the row + // first swap information about the inverse and forward maps + + current_stride = Graph_Matrix_Strides(inode); + if (istride != current_stride - 1) + { + Element_local_indices(inode, istride, 0) = Element_local_indices(inode, current_stride - 1, 0); + Element_local_indices(inode, istride, 1) = Element_local_indices(inode, current_stride - 1, 1); + Element_local_indices(inode, istride, 2) = Element_local_indices(inode, current_stride - 1, 2); + current_element_index = Element_local_indices(inode, istride, 0); + element_row_index = Element_local_indices(inode, istride, 1); + element_column_index = Element_local_indices(inode, istride, 2); + + Global_Gradient_Matrix_Assembly_Map(current_element_index, element_row_index, element_column_index) + = istride; + + // now that the element map information has been copied, copy the global node index and delete the last index + + Repeat_Graph_Matrix(inode, istride) = Repeat_Graph_Matrix(inode, current_stride - 1); + } + istride--; + Graph_Matrix_Strides(inode)--; + } + else + { + /*this node hasn't shown up in the row before; add it to the list of nodes + that have been scanned uniquely. Use this list to reset the flag array + afterwards without having to loop over all the nodes in the system*/ + node_indices_used(current_node) = 1; + column_index(current_node) = istride; + current_row_nodes_scanned(current_row_n_nodes_scanned) = current_node; + current_row_n_nodes_scanned++; + } + } + // reset nodes used list for the next row of the sparse list + for (int node_reset = 0; node_reset < current_row_n_nodes_scanned; node_reset++) + { + node_indices_used(current_row_nodes_scanned(node_reset)) = 0; + } + } + }); + Kokkos::fence(); + + Graph_Matrix_Strides.update_host(); + // copy reduced content to non_repeat storage + Graph_Matrix = RaggedRightArrayKokkos(Graph_Matrix_Strides); + + FOR_ALL_CLASS(inode, 0, nlocal_nodes, { + for (int istride = 0; istride < Graph_Matrix_Strides(inode); istride++) + { + Graph_Matrix(inode, istride) = Repeat_Graph_Matrix(inode, istride); + } + }); // end parallel for + + // deallocate repeat matrix + + /*At this stage the sparse graph should have unique global indices on each row. + The constructed Assembly map (to the global sparse matrix) + is used to loop over each element's local stiffness matrix in the assembly process.*/ + + // expand strides for stiffness matrix by multipling by dim + FOR_ALL_CLASS(idof, 0, num_dim * nlocal_nodes, { + Gradient_Matrix_Strides(idof) = num_dim * Graph_Matrix_Strides(idof / num_dim); + }); // end parallel for + + Gradient_Matrix_Strides.update_host(); + + // build inverse map for element gradient assembly + for (size_t elem_gid = 0; elem_gid < rnum_elem; elem_gid++) + { + FOR_ALL_CLASS(node_lid, 0, num_nodes_in_elem, { + // get the global_id of the node + size_t node_gid = nodes_in_elem(elem_gid, node_lid); + + // the column index is the num corners saved + size_t j = count_saved_corners_in_node(node_gid); + Element_Gradient_Matrix_Assembly_Map(elem_gid, node_lid) = j; + + // increment the number of corners saved to this node_gid + count_saved_corners_in_node(node_gid)++; + }); // end FOR_ALL over nodes in element + Kokkos::fence(); + } // end for elem_gid + + DOF_Graph_Matrix = RaggedRightArrayKokkos(Gradient_Matrix_Strides); + Force_Gradient_Positions = RaggedRightArrayKokkos(Gradient_Matrix_Strides); + Force_Gradient_Velocities = RaggedRightArrayKokkos(Gradient_Matrix_Strides); + // needs different graph of node to elem rather than node to node + // DOF_to_Elem_Matrix_Strides.get_kokkos_dual_view().d_view = elems_in_node.mystrides_; + Kokkos::View node_to_elem_strides = elems_in_node.mystrides_; + DOF_to_Elem_Matrix_Strides = DCArrayKokkos(nlocal_nodes * num_dim); + FOR_ALL_CLASS(inode, 0, nlocal_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + DOF_to_Elem_Matrix_Strides(inode * num_dim + idim) = count_saved_corners_in_node(inode); + } + }); // end parallel for + DOF_to_Elem_Matrix_Strides.update_host(); + Force_Gradient_Energies = CArrayKokkos(rnum_elem, num_nodes_in_elem * num_dim); + Power_Gradient_Energies = CArrayKokkos(rnum_elem); + Power_Gradient_Positions = RaggedRightArrayKokkos(DOF_to_Elem_Matrix_Strides); + Power_Gradient_Velocities = RaggedRightArrayKokkos(DOF_to_Elem_Matrix_Strides); + + // set stiffness Matrix Graph + // debug print + // std::cout << "DOF GRAPH MATRIX ENTRIES ON TASK " << myrank << std::endl; + FOR_ALL_CLASS(idof, 0, num_dim * nlocal_nodes, { + for (int istride = 0; istride < Gradient_Matrix_Strides(idof); istride++) + { + DOF_Graph_Matrix(idof, istride) = Graph_Matrix(idof / num_dim, istride / num_dim) * num_dim + istride % num_dim; + } + }); // end parallel for + + /* + //construct distributed gradient matrix from local kokkos data + //build column map for the global gradient matrix + Teuchos::RCP > colmap; + const Teuchos::RCP > dommap = local_dof_map; + + Tpetra::Details::makeColMap(colmap,dommap,DOF_Graph_Matrix.get_kokkos_view(), nullptr); + + size_t nnz = DOF_Graph_Matrix.size(); + + //debug print + //std::cout << "DOF GRAPH SIZE ON RANK " << myrank << " IS " << nnz << std::endl; + + //local indices in the graph using the constructed column map + CArrayKokkos gradient_local_indices(nnz, "gradient_local_indices"); + + //row offsets with compatible template arguments + Kokkos::View row_offsets = DOF_Graph_Matrix.start_index_; + row_pointers row_offsets_pass("row_offsets", nlocal_nodes*num_dim+1); + for(int ipass = 0; ipass < nlocal_nodes*num_dim + 1; ipass++){ + row_offsets_pass(ipass) = row_offsets(ipass); + } + + size_t entrycount = 0; + for(int irow = 0; irow < nlocal_nodes*num_dim; irow++){ + for(int istride = 0; istride < Gradient_Matrix_Strides(irow); istride++){ + gradient_local_indices(entrycount) = colmap->getLocalElement(DOF_Graph_Matrix(irow,istride)); + entrycount++; + } + } + + + //sort values and indices + Tpetra::Import_Util::sortCrsEntries(row_offsets_pass, gradient_local_indices.get_kokkos_view(), Force_Gradient_Positions.get_kokkos_view()); + Tpetra::Import_Util::sortCrsEntries(row_offsets_pass, gradient_local_indices.get_kokkos_view(), Force_Gradient_Velocities.get_kokkos_view()); + + //Teuchos::RCP crs_matrix_params = Teuchos::rcp(new Teuchos::ParameterList("crsmatrix")); + //crs_matrix_params->set("sorted", false); + distributed_force_gradient_positions = Teuchos::rcp(new MAT(local_dof_map, colmap, row_offsets_pass, gradient_local_indices.get_kokkos_view(), Force_Gradient_Positions.get_kokkos_view())); + distributed_force_gradient_positions->fillComplete(); + distributed_force_gradient_velocities = Teuchos::rcp(new MAT(local_dof_map, colmap, row_offsets_pass, gradient_local_indices.get_kokkos_view(), Force_Gradient_Velocities.get_kokkos_view())); + distributed_force_gradient_velocities->fillComplete(); + */ + // distributed_force_gradient_positions->describe(*fos,Teuchos::VERB_EXTREME); + // distributed_force_gradient_velocities->describe(*fos,Teuchos::VERB_EXTREME); +} + +/* ---------------------------------------------------------------------- + Enforce boundary conditions on the adjoint vectors +------------------------------------------------------------------------- */ + +void FEA_Module_SGH::boundary_adjoint(const mesh_t& mesh, + const DCArrayKokkos& boundary, + vec_array& node_adjoint, + vec_array& node_phi_adjoint, + vec_array& node_psi_adjoint) +{ + // error and debug flag + // DCArrayKokkos print_flag(1, "print_flag"); + // print_flag.host(0) = false; + // print_flag.update_device(); + + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + int num_dims = simparam->num_dims; + // Loop over boundary sets + for (size_t bdy_set = 0; bdy_set < num_bdy_sets; bdy_set++) + { + // Loop over boundary nodes in a boundary set + FOR_ALL_CLASS(bdy_node_lid, 0, num_bdy_nodes_in_set.host(bdy_set), { + // reflected (boundary array is on the device) + if (boundary(bdy_set).type == BOUNDARY_CONDITION_TYPE::reflected) + { + // directions with hydro_bc: + // x_plane = 0, + // y_plane = 1, + // z_plane = 2, + size_t direction = boundary(bdy_set).surface.planar_surface_index(); + + size_t bdy_node_gid = bdy_nodes_in_set(bdy_set, bdy_node_lid); + + // Set velocity to zero in that directdion + if (bdy_node_gid < nlocal_nodes) + { + node_adjoint(bdy_node_gid, direction) = 0.0; + } + // node_phi_adjoint(bdy_node_gid, direction) = 0.0; + } + else if (boundary(bdy_set).type == BOUNDARY_CONDITION_TYPE::fixed_position) + { + size_t bdy_node_gid = bdy_nodes_in_set(bdy_set, bdy_node_lid); + + // debug clause + // if(bdy_node_gid==549412) print_flag(0) = true; + + for (size_t dim = 0; dim < num_dims; dim++) + { + // Set velocity to zero + if (bdy_node_gid < nlocal_nodes) + { + node_adjoint(bdy_node_gid, dim) = 0.0; + } + // node_phi_adjoint(bdy_node_gid, dim) = 0.0; + } + } + }); // end for bdy_node_lid + } // end for bdy_set + + // debug check + // print_flag.update_host(); + // if(print_flag.host(0)) std::cout << "found boundary node with id 549412" << std::endl; + return; +} // end boundary_velocity function + +/* ---------------------------------------------------------------------- + Communicate updated nodal adjoint vectors to ghost nodes +------------------------------------------------------------------------- */ + +void FEA_Module_SGH::comm_adjoint_vector(int cycle) +{ + // comms to get ghosts + (*adjoint_vector_data)[cycle]->doImport(*adjoint_vector_distributed, *importer, Tpetra::INSERT); +} + +/* ---------------------------------------------------------------------- + Communicate updated nodal adjoint vectors to ghost nodes +------------------------------------------------------------------------- */ + +void FEA_Module_SGH::comm_phi_adjoint_vector(int cycle) +{ + // comms to get ghosts + (*phi_adjoint_vector_data)[cycle]->doImport(*phi_adjoint_vector_distributed, *importer, Tpetra::INSERT); +} diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/time_integration.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/time_integration.cpp new file mode 100644 index 000000000..a934f2932 --- /dev/null +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/time_integration.cpp @@ -0,0 +1,231 @@ + +#include "mesh.h" +#include "state.h" +#include "FEA_Module_SGH.h" + +// ----------------------------------------------------------------------------- +// This function saves the variables at rk_stage = 0, which is t_n +// ------------------------------------------------------------------------------ +void FEA_Module_SGH::rk_init(DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_sie, + DViewCArrayKokkos& elem_stress, + const size_t num_elems, + const size_t num_nodes) +{ + const size_t rk_level = rk_num_bins - 1; + int num_dims = num_dim; + // save elem quantities + FOR_ALL_CLASS(elem_gid, 0, num_elems, { + // stress is always 3D even with 2D-RZ + for (size_t i = 0; i < 3; i++) + { + for (size_t j = 0; j < 3; j++) + { + elem_stress(0, elem_gid, i, j) = elem_stress(rk_level, elem_gid, i, j); + } + } // end for + + elem_sie(0, elem_gid) = elem_sie(rk_level, elem_gid); + }); // end parallel for + + // save nodal quantities + FOR_ALL_CLASS(node_gid, 0, num_nodes, { + for (size_t i = 0; i < num_dims; i++) + { + node_coords(0, node_gid, i) = node_coords(rk_level, node_gid, i); + node_vel(0, node_gid, i) = node_vel(rk_level, node_gid, i); + } + }); // end parallel for + Kokkos::fence(); + + return; +} // end rk_init + +// ----------------------------------------------------------------------------- +// This function calculates the time step by finding the shortest distance +// between any two nodes in the mesh +// ------------------------------------------------------------------------------ +// WARNING WARNING : Only works for 3D, 8 node elements +void FEA_Module_SGH::get_timestep(mesh_t& mesh, + DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_sspd, + DViewCArrayKokkos& elem_vol) +{ + const size_t rk_level = rk_num_bins - 1; + + // increase dt by 10%, that is the largest dt value + dt = dt * 1.1; + int num_dims = num_dim; + double dt_lcl; + double min_dt_calc; + REDUCE_MIN_CLASS(elem_gid, 0, rnum_elem, dt_lcl, { + double coords0[24]; // element coords + ViewCArrayKokkos coords(coords0, 8, 3); + + double distance0[28]; // array for holding distances between each node + ViewCArrayKokkos dist(distance0, 28); + + // Getting the coordinates of the element + for (size_t node_lid = 0; node_lid < 8; node_lid++) + { + for (size_t dim = 0; dim < num_dims; dim++) + { + coords(node_lid, dim) = node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), dim); + } // end for dim + } // end for loop over node_lid + + // loop conditions needed for distance calculation + size_t countA = 0; + size_t countB = 1; + size_t a; + size_t b; + size_t loop = 0; + + // Only works for 3D + // Solving for the magnitude of distance between each node + for (size_t i = 0; i < 28; i++) + { + a = countA; + b = countB; + + // returns magnitude of distance between each node, 28 total options + dist(i) = fabs(sqrt((pow((coords(b, 0) - coords(a, 0)), 2.0) + + pow((coords(b, 1) - coords(a, 1)), 2.0) + + pow((coords(b, 2) - coords(a, 2)), 2.0)))); + + countB++; + countA++; + + // tricky indexing + if (countB > 7) + { + loop++; + countB = 1 + loop; + countA = 0; + } + } // endo for i + + double dist_min = dist(0); + + for (int i = 0; i < 28; ++i) + { + dist_min = fmin(dist(i), dist_min); + } + + // local dt calc based on CFL + double dt_lcl_ = dt_cfl * dist_min / (elem_sspd(elem_gid) + fuzz); + + // make dt be in bounds + dt_lcl_ = fmin(dt_lcl_, dt_max); // make dt small than dt_max + dt_lcl_ = fmax(dt_lcl_, dt_min); // make dt larger than dt_min + + if (dt_lcl_ < dt_lcl) + { + dt_lcl = dt_lcl_; + } + }, min_dt_calc); // end parallel reduction + Kokkos::fence(); + + // save the min dt + if (min_dt_calc < dt) + { + dt = min_dt_calc; + } + + // ensure time step hits the graphics time intervals + dt = fmin(dt, (graphics_time - time_value) + fuzz); + + // make dt be exact for final time + dt = fmin(dt, time_final - time_value); + + return; +} // end get_timestep + +// ----------------------------------------------------------------------------- +// This function calculates the time step by finding the shortest distance +// between any two nodes in the mesh +// ------------------------------------------------------------------------------ +// WARNING WARNING : Only works for 3D, 8 node elements +void FEA_Module_SGH::get_timestep2D(mesh_t& mesh, + DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_sspd, + DViewCArrayKokkos& elem_vol) +{ + const size_t rk_level = rk_num_bins - 1; + + // increase dt by 10%, that is the largest dt value + dt = dt * 1.1; + int num_dims = num_dim; + double dt_lcl; + double min_dt_calc; + + REDUCE_MIN_CLASS(elem_gid, 0, rnum_elem, dt_lcl, { + double coords0[8]; // element coords + ViewCArrayKokkos coords(coords0, 4, 2); + + double distance0[6]; // array for holding distances between each node + ViewCArrayKokkos dist(distance0, 6); + + // Getting the coordinates of the nodes of the element + for (size_t node_lid = 0; node_lid < 4; node_lid++) + { + for (size_t dim = 0; dim < num_dims; dim++) + { + coords(node_lid, dim) = node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), dim); + } // end for dim + } // end for loop over node_lid + + // Only works for 2D + // Solving for the magnitude of distance between each node + size_t count = 0; + for (size_t i = 0; i < 3; i++) + { + for (size_t j = i + 1; j <= 3; j++) + { + // returns magnitude of distance between each node, 6 total options + dist(count) = fabs( + sqrt(pow((coords(i, 0) - coords(j, 0)), 2.0) + + pow((coords(i, 1) - coords(j, 1)), 2.0) ) + ); + count++; + } // end for j + } // end for i + + double dist_min = dist(0); + + for (int i = 0; i < 6; ++i) + { + dist_min = fmin(dist(i), dist_min); + } + + // local dt calc based on CFL + double dt_lcl_ = dt_cfl * dist_min / (elem_sspd(elem_gid) + fuzz); + + // make dt be in bounds + dt_lcl_ = fmin(dt_lcl_, dt_max); // make dt small than dt_max + dt_lcl_ = fmax(dt_lcl_, dt_min); // make dt larger than dt_min + + if (dt_lcl_ < dt_lcl) + { + dt_lcl = dt_lcl_; + } + }, min_dt_calc); // end parallel reduction + Kokkos::fence(); + + // save the min dt + if (min_dt_calc < dt) + { + dt = min_dt_calc; + } + + // ensure time step hits the graphics time intervals + dt = fmin(dt, (graphics_time - time_value) + fuzz); + + // make dt be exact for final time + dt = fmin(dt, time_final - time_value); + + return; +} // end get_timestep2D diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/state.h b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/state.h deleted file mode 100644 index 439f28cc2..000000000 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/state.h +++ /dev/null @@ -1,108 +0,0 @@ -#ifndef STATE_H -#define STATE_H - - -#include "matar.h" - -using namespace mtr; - -// node_state -struct node_t { - - // Position - DCArrayKokkos coords; - - // velocity - DCArrayKokkos vel; - - // mass at nodes - DCArrayKokkos mass; - - // Includes Ghost Positions - //DCArrayKokkos all_coords; - - // Includes Ghost velocities - //DCArrayKokkos all_vel; - - // Includes Ghost masses - //DCArrayKokkos all_mass; - - - // initialization method (num_rk_storage_bins, num_nodes, num_dims) - void initialize(size_t num_rk, size_t num_nodes, size_t num_dims) - { - this->coords = DCArrayKokkos (num_rk, num_nodes, num_dims); - this->vel = DCArrayKokkos (num_rk, num_nodes, num_dims); - this->mass = DCArrayKokkos (num_nodes); - }; // end method - -}; // end node_t - - -// elem_state -struct elem_t { - - // den - CArray den; - - // pres - CArray pres; - - // stress - CArray stress; - - // sspd - CArray sspd; - - // sie - CArray sie; - - // vol - CArray vol; - - // divergence of velocity - CArray div; - - // mass of elem - CArray mass; - - // mat ids - CArray mat_id; - - // initialization method (num_rk_storage_bins, num_cells, num_dims) - void initialize(size_t num_rk, size_t num_elems, size_t num_dims) - { - this->den = CArray (num_elems); - this->pres = CArray (num_elems); - this->stress = CArray (num_rk, num_elems, num_dims, num_dims); - this->sspd = CArray (num_elems); - this->sie = CArray (num_rk, num_elems); - this->vol = CArray (num_elems); - this->div = CArray (num_elems); - this->mass = CArray (num_elems); - this->mat_id = CArray (num_elems); - }; // end method - -}; // end elem_t - - -// corner_state -struct corner_t { - - // force - CArray force; - - // mass of corner - CArray mass; - - - // initialization method (num_corners, num_dims) - void initialize(size_t num_corners, size_t num_dims) - { - this->force = CArray (num_corners, num_dims); - this->mass = CArray (num_corners); - }; // end method - -}; // end corner_t - -#endif diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/time_integration.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/time_integration.cpp deleted file mode 100644 index 0d8d75ae0..000000000 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/time_integration.cpp +++ /dev/null @@ -1,237 +0,0 @@ - -#include "mesh.h" -#include "state.h" -#include "FEA_Module_SGH.h" - -// ----------------------------------------------------------------------------- -// This function saves the variables at rk_stage = 0, which is t_n -//------------------------------------------------------------------------------ -void FEA_Module_SGH::rk_init(DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &elem_sie, - DViewCArrayKokkos &elem_stress, - const size_t num_elems, - const size_t num_nodes){ - - const size_t rk_level = rk_num_bins - 1; - int num_dims = num_dim; - // save elem quantities - FOR_ALL_CLASS(elem_gid, 0, num_elems, { - - // stress is always 3D even with 2D-RZ - for(size_t i=0; i<3; i++){ - for(size_t j=0; j<3; j++){ - elem_stress(0,elem_gid,i,j) = elem_stress(rk_level,elem_gid,i,j); - } - } // end for - - elem_sie(0,elem_gid) = elem_sie(rk_level,elem_gid); - - }); // end parallel for - - - // save nodal quantities - FOR_ALL_CLASS(node_gid, 0, num_nodes, { - - for(size_t i=0; i &node_coords, - DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &elem_sspd, - DViewCArrayKokkos &elem_vol){ - - - const size_t rk_level = rk_num_bins - 1; - - // increase dt by 10%, that is the largest dt value - dt = dt*1.1; - int num_dims = num_dim; - double dt_lcl; - double min_dt_calc; - REDUCE_MIN_CLASS(elem_gid, 0, rnum_elem, dt_lcl, { - - double coords0[24]; // element coords - ViewCArrayKokkos coords(coords0, 8, 3); - - - double distance0[28]; // array for holding distances between each node - ViewCArrayKokkos dist(distance0, 28); - - // Getting the coordinates of the element - for(size_t node_lid = 0; node_lid < 8; node_lid++){ - - for (size_t dim = 0; dim < num_dims; dim++){ - coords(node_lid, dim) = node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), dim); - } // end for dim - - } // end for loop over node_lid - - // loop conditions needed for distance calculation - size_t countA = 0; - size_t countB = 1; - size_t a; - size_t b; - size_t loop = 0; - - // Only works for 3D - // Solving for the magnitude of distance between each node - for (size_t i = 0; i < 28; i++){ - - a = countA; - b = countB; - - // returns magnitude of distance between each node, 28 total options - dist(i) = fabs(sqrt(( pow((coords(b, 0) - coords(a, 0)), 2.0) - + pow((coords(b, 1) - coords(a, 1)), 2.0) - + pow((coords(b, 2) - coords(a, 2)), 2.0)))); - - countB++; - countA++; - - // tricky indexing - if (countB > 7) { - loop++; - countB = 1 + loop; - countA = 0; - } - - } // endo for i - - - double dist_min = dist(0); - - for(int i = 0; i < 28; ++i){ - dist_min = fmin(dist(i), dist_min); - } - - // local dt calc based on CFL - double dt_lcl_ = dt_cfl*dist_min/(elem_sspd(elem_gid) + fuzz); - - // make dt be in bounds - dt_lcl_ = fmin(dt_lcl_, dt_max); // make dt small than dt_max - dt_lcl_ = fmax(dt_lcl_, dt_min); // make dt larger than dt_min - - - if (dt_lcl_ < dt_lcl) dt_lcl = dt_lcl_; - - }, min_dt_calc); // end parallel reduction - Kokkos::fence(); - - // save the min dt - if(min_dt_calc < dt) dt = min_dt_calc; - - // ensure time step hits the graphics time intervals - dt = fmin(dt, (graphics_time - time_value)+fuzz); - - // make dt be exact for final time - dt = fmin(dt, time_final-time_value); - - return; - -} // end get_timestep - -// ----------------------------------------------------------------------------- -// This function calculates the time step by finding the shortest distance -// between any two nodes in the mesh -//------------------------------------------------------------------------------ -// WARNING WARNING : Only works for 3D, 8 node elements -void FEA_Module_SGH::get_timestep2D(mesh_t &mesh, - DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &elem_sspd, - DViewCArrayKokkos &elem_vol){ - - const size_t rk_level = rk_num_bins - 1; - - // increase dt by 10%, that is the largest dt value - dt = dt*1.1; - int num_dims = num_dim; - double dt_lcl; - double min_dt_calc; - - REDUCE_MIN_CLASS(elem_gid, 0, rnum_elem, dt_lcl, { - - double coords0[8]; // element coords - ViewCArrayKokkos coords(coords0, 4, 2); - - - double distance0[6]; // array for holding distances between each node - ViewCArrayKokkos dist(distance0, 6); - - // Getting the coordinates of the nodes of the element - for(size_t node_lid = 0; node_lid < 4; node_lid++){ - - for (size_t dim = 0; dim < num_dims; dim++){ - coords(node_lid, dim) = node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), dim); - } // end for dim - - } // end for loop over node_lid - - - // Only works for 2D - // Solving for the magnitude of distance between each node - size_t count = 0; - for (size_t i = 0; i < 3; i++){ - for (size_t j=i+1; j<=3; j++){ - - // returns magnitude of distance between each node, 6 total options - dist(count) = fabs( - sqrt( pow((coords(i, 0) - coords(j, 0)), 2.0) - + pow((coords(i, 1) - coords(j, 1)), 2.0) ) - ); - count ++; - } // end for j - } // end for i - - - double dist_min = dist(0); - - for(int i = 0; i < 6; ++i){ - dist_min = fmin(dist(i), dist_min); - } - - // local dt calc based on CFL - double dt_lcl_ = dt_cfl*dist_min/(elem_sspd(elem_gid) + fuzz); - - // make dt be in bounds - dt_lcl_ = fmin(dt_lcl_, dt_max); // make dt small than dt_max - dt_lcl_ = fmax(dt_lcl_, dt_min); // make dt larger than dt_min - - - if (dt_lcl_ < dt_lcl) dt_lcl = dt_lcl_; - - }, min_dt_calc); // end parallel reduction - Kokkos::fence(); - - // save the min dt - if(min_dt_calc < dt) dt = min_dt_calc; - - // ensure time step hits the graphics time intervals - dt = fmin(dt, (graphics_time - time_value)+fuzz); - - // make dt be exact for final time - dt = fmin(dt, time_final-time_value); - - return; - -} // end get_timestep2D - - diff --git a/src/Parallel-Solvers/Parallel-Explicit/Topology_Optimization/Kinetic_Energy_Minimize.h b/src/Parallel-Solvers/Parallel-Explicit/Topology_Optimization/Kinetic_Energy_Minimize.h index ff255f1f0..4ac860c6b 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Topology_Optimization/Kinetic_Energy_Minimize.h +++ b/src/Parallel-Solvers/Parallel-Explicit/Topology_Optimization/Kinetic_Energy_Minimize.h @@ -11,14 +11,14 @@ This program is open source under the BSD-3 License. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - + 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - + 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - + 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. @@ -34,7 +34,7 @@ OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. **********************************************************************************************/ - + #ifndef KINETIC_ENERGY_MINIMIZE_TOPOPT_H #define KINETIC_ENERGY_MINIMIZE_TOPOPT_H @@ -61,312 +61,360 @@ #include "FEA_Module_Dynamic_Elasticity.h" #include "Explicit_Solver.h" -class KineticEnergyMinimize_TopOpt : public ROL::Objective { - - typedef Tpetra::Map<>::local_ordinal_type LO; - typedef Tpetra::Map<>::global_ordinal_type GO; - typedef Tpetra::Map<>::node_type Node; - typedef Tpetra::Map Map; - typedef Tpetra::MultiVector MV; - typedef ROL::Vector V; - typedef ROL::TpetraMultiVector ROL_MV; - - using traits = Kokkos::ViewTraits; - using array_layout = typename traits::array_layout; - using execution_space = typename traits::execution_space; - using device_type = typename traits::device_type; - using memory_traits = typename traits::memory_traits; - using global_size_t = Tpetra::global_size_t; - - typedef Kokkos::View values_array; - typedef Kokkos::View global_indices_array; - typedef Kokkos::View indices_array; - - //typedef Kokkos::DualView::t_dev vec_array; - typedef MV::dual_view_type::t_dev vec_array; - typedef MV::dual_view_type::t_host host_vec_array; - typedef Kokkos::View const_host_vec_array; - typedef Kokkos::View const_vec_array; - typedef MV::dual_view_type dual_vec_array; +class KineticEnergyMinimize_TopOpt : public ROL::Objective +{ +typedef Tpetra::Map<>::local_ordinal_type LO; +typedef Tpetra::Map<>::global_ordinal_type GO; +typedef Tpetra::Map<>::node_type Node; +typedef Tpetra::Map Map; +typedef Tpetra::MultiVector MV; +typedef ROL::Vector V; +typedef ROL::TpetraMultiVector ROL_MV; + +using traits = Kokkos::ViewTraits; +using array_layout = typename traits::array_layout; +using execution_space = typename traits::execution_space; +using device_type = typename traits::device_type; +using memory_traits = typename traits::memory_traits; +using global_size_t = Tpetra::global_size_t; + +typedef Kokkos::View values_array; +typedef Kokkos::View global_indices_array; +typedef Kokkos::View indices_array; + +// typedef Kokkos::DualView::t_dev vec_array; +typedef MV::dual_view_type::t_dev vec_array; +typedef MV::dual_view_type::t_host host_vec_array; +typedef Kokkos::View const_host_vec_array; +typedef Kokkos::View const_vec_array; +typedef MV::dual_view_type dual_vec_array; private: - Explicit_Solver *Explicit_Solver_Pointer_; - FEA_Module_SGH *FEM_SGH_; - FEA_Module_Dynamic_Elasticity *FEM_Dynamic_Elasticity_; - ROL::Ptr ROL_Force; - ROL::Ptr ROL_Velocities; - ROL::Ptr ROL_Gradients; + Explicit_Solver* Explicit_Solver_Pointer_; + FEA_Module_SGH* FEM_SGH_; + FEA_Module_Dynamic_Elasticity* FEM_Dynamic_Elasticity_; + ROL::Ptr ROL_Force; + ROL::Ptr ROL_Velocities; + ROL::Ptr ROL_Gradients; - bool useLC_; // Use linear form of energy. Otherwise use quadratic form. + bool useLC_; // Use linear form of energy. Otherwise use quadratic form. - ROL::Ptr getVector( const V& x ) { - return dynamic_cast(x).getVector(); - } + ROL::Ptr getVector(const V& x) + { + return dynamic_cast(x).getVector(); + } - ROL::Ptr getVector( V& x ) { - return dynamic_cast(x).getVector(); - } + ROL::Ptr getVector(V& x) + { + return dynamic_cast(x).getVector(); + } public: - bool nodal_density_flag_, time_accumulation; - int last_comm_step, last_solve_step, current_step; - size_t nvalid_modules; - std::vector valid_fea_modules; //modules that may interface with this objective function - FEA_MODULE_TYPE set_module_type; - //std::string my_fea_module = "SGH"; - real_t objective_accumulation; - - KineticEnergyMinimize_TopOpt(Explicit_Solver *Explicit_Solver_Pointer, bool nodal_density_flag) - : useLC_(true) { - Explicit_Solver_Pointer_ = Explicit_Solver_Pointer; - - valid_fea_modules.push_back(FEA_MODULE_TYPE::SGH); - valid_fea_modules.push_back(FEA_MODULE_TYPE::Dynamic_Elasticity); - nvalid_modules = valid_fea_modules.size(); - - const Simulation_Parameters& simparam = Explicit_Solver_Pointer_->simparam; - for (const auto& fea_module : Explicit_Solver_Pointer_->fea_modules) { - for(int ivalid = 0; ivalid < nvalid_modules; ivalid++){ - if(fea_module->Module_Type==FEA_MODULE_TYPE::SGH){ - FEM_SGH_ = dynamic_cast(fea_module); - set_module_type = FEA_MODULE_TYPE::SGH; - } - if(fea_module->Module_Type==FEA_MODULE_TYPE::Dynamic_Elasticity){ - FEM_Dynamic_Elasticity_ = dynamic_cast(fea_module); - set_module_type = FEA_MODULE_TYPE::Dynamic_Elasticity; - } + bool nodal_density_flag_, time_accumulation; + int last_comm_step, last_solve_step, current_step; + size_t nvalid_modules; + std::vector valid_fea_modules; // modules that may interface with this objective function + FEA_MODULE_TYPE set_module_type; + // std::string my_fea_module = "SGH"; + real_t objective_accumulation; + + KineticEnergyMinimize_TopOpt(Explicit_Solver* Explicit_Solver_Pointer, bool nodal_density_flag) + : useLC_(true) + { + Explicit_Solver_Pointer_ = Explicit_Solver_Pointer; + + valid_fea_modules.push_back(FEA_MODULE_TYPE::SGH); + valid_fea_modules.push_back(FEA_MODULE_TYPE::Dynamic_Elasticity); + nvalid_modules = valid_fea_modules.size(); + + const Simulation_Parameters& simparam = Explicit_Solver_Pointer_->simparam; + for (const auto& fea_module : Explicit_Solver_Pointer_->fea_modules) + { + for (int ivalid = 0; ivalid < nvalid_modules; ivalid++) + { + if (fea_module->Module_Type == FEA_MODULE_TYPE::SGH) + { + FEM_SGH_ = dynamic_cast(fea_module); + set_module_type = FEA_MODULE_TYPE::SGH; + } + if (fea_module->Module_Type == FEA_MODULE_TYPE::Dynamic_Elasticity) + { + FEM_Dynamic_Elasticity_ = dynamic_cast(fea_module); + set_module_type = FEA_MODULE_TYPE::Dynamic_Elasticity; + } + } + } + nodal_density_flag_ = nodal_density_flag; + last_comm_step = last_solve_step = -1; + current_step = 0; + time_accumulation = true; + objective_accumulation = 0; + + // ROL_Force = ROL::makePtr(FEM_->Global_Nodal_Forces); + if (set_module_type == FEA_MODULE_TYPE::SGH) + { + ROL_Velocities = ROL::makePtr(FEM_SGH_->node_velocities_distributed); + } + if (set_module_type == FEA_MODULE_TYPE::Dynamic_Elasticity) + { + ROL_Velocities = ROL::makePtr(FEM_Dynamic_Elasticity_->node_velocities_distributed); } - } - nodal_density_flag_ = nodal_density_flag; - last_comm_step = last_solve_step = -1; - current_step = 0; - time_accumulation = true; - objective_accumulation = 0; - - //ROL_Force = ROL::makePtr(FEM_->Global_Nodal_Forces); - if(set_module_type==FEA_MODULE_TYPE::SGH) - ROL_Velocities = ROL::makePtr(FEM_SGH_->node_velocities_distributed); - if(set_module_type==FEA_MODULE_TYPE::Dynamic_Elasticity) - ROL_Velocities = ROL::makePtr(FEM_Dynamic_Elasticity_->node_velocities_distributed); - - //real_t current_kinetic_energy = ROL_Velocities->dot(*ROL_Force)/2; - //std::cout.precision(10); - //if(FEM_->myrank==0) - //std::cout << "INITIAL KINETIC ENERGY " << current_kinetic_energy << std::endl; - } - - void update(const ROL::Vector &z, ROL::UpdateType type, int iter = -1 ) { - if(set_module_type==FEA_MODULE_TYPE::SGH) - update_sgh(z,type,iter); - if(set_module_type==FEA_MODULE_TYPE::Dynamic_Elasticity) - update_elasticity(z,type,iter); - } - - void update_elasticity(const ROL::Vector &z, ROL::UpdateType type, int iter = -1 ) { - - //debug - std::ostream &out = std::cout; - Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - - current_step++; - ROL::Ptr zp = getVector(z); - const_host_vec_array design_densities = zp->getLocalView (Tpetra::Access::ReadOnly); - - if (type == ROL::UpdateType::Initial) { - // This is the first call to update - //first linear solve was done in FEA class run function already - FEM_Dynamic_Elasticity_->elastic_solve(); - //initial design density data was already communicated for ghost nodes in init_design() - //decide to output current optimization state - FEM_Dynamic_Elasticity_->Explicit_Solver_Pointer_->write_outputs(); - } - else if (type == ROL::UpdateType::Accept) { + // real_t current_kinetic_energy = ROL_Velocities->dot(*ROL_Force)/2; + // std::cout.precision(10); + // if(FEM_->myrank==0) + // std::cout << "INITIAL KINETIC ENERGY " << current_kinetic_energy << std::endl; } - else if (type == ROL::UpdateType::Revert) { - // u_ was set to u=S(x) during a trial update - // and has been rejected as the new iterate - // Revert to cached value - // This is a new value of x - //communicate density variables for ghosts - FEM_Dynamic_Elasticity_->comm_variables(zp); - //update deformation variables - FEM_Dynamic_Elasticity_->update_forward_solve(zp); - if(Explicit_Solver_Pointer_->myrank==0) - *fos << "called Revert" << std::endl; - } - else if (type == ROL::UpdateType::Trial) { - // This is a new value of x - //communicate density variables for ghosts - FEM_Dynamic_Elasticity_->comm_variables(zp); - //update deformation variables - FEM_Dynamic_Elasticity_->update_forward_solve(zp); - if(Explicit_Solver_Pointer_->myrank==0) - *fos << "called Trial" << std::endl; - - //decide to output current optimization state - FEM_Dynamic_Elasticity_->Explicit_Solver_Pointer_->write_outputs(); - } - else { // ROL::UpdateType::Temp - // This is a new value of x used for, - // e.g., finite-difference checks - if(Explicit_Solver_Pointer_->myrank==0) - *fos << "called Temp" << std::endl; - FEM_Dynamic_Elasticity_->comm_variables(zp); - FEM_Dynamic_Elasticity_->update_forward_solve(zp); - } - } - - void update_sgh(const ROL::Vector &z, ROL::UpdateType type, int iter = -1 ) { - //debug - std::ostream &out = std::cout; - Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - - current_step++; - ROL::Ptr zp = getVector(z); - const_host_vec_array design_densities = zp->getLocalView (Tpetra::Access::ReadOnly); - - if (type == ROL::UpdateType::Initial) { - // This is the first call to update - //first linear solve was done in FEA class run function already - FEM_SGH_->sgh_solve(); - //initial design density data was already communicated for ghost nodes in init_design() - //decide to output current optimization state - FEM_SGH_->Explicit_Solver_Pointer_->write_outputs(); - } - else if (type == ROL::UpdateType::Accept) { + void update(const ROL::Vector& z, ROL::UpdateType type, int iter = -1) + { + if (set_module_type == FEA_MODULE_TYPE::SGH) + { + update_sgh(z, type, iter); + } + if (set_module_type == FEA_MODULE_TYPE::Dynamic_Elasticity) + { + update_elasticity(z, type, iter); + } } - else if (type == ROL::UpdateType::Revert) { - // u_ was set to u=S(x) during a trial update - // and has been rejected as the new iterate - // Revert to cached value - // This is a new value of x - //communicate density variables for ghosts - FEM_SGH_->comm_variables(zp); - //update deformation variables - FEM_SGH_->update_forward_solve(zp); - if(Explicit_Solver_Pointer_->myrank==0) - *fos << "called Revert" << std::endl; - } - else if (type == ROL::UpdateType::Trial) { - // This is a new value of x - //communicate density variables for ghosts - FEM_SGH_->comm_variables(zp); - //update deformation variables - FEM_SGH_->update_forward_solve(zp); - if(Explicit_Solver_Pointer_->myrank==0) - *fos << "called Trial" << std::endl; - - //decide to output current optimization state - FEM_SGH_->Explicit_Solver_Pointer_->write_outputs(); + + void update_elasticity(const ROL::Vector& z, ROL::UpdateType type, int iter = -1) + { + // debug + std::ostream& out = std::cout; + Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + + current_step++; + ROL::Ptr zp = getVector(z); + const_host_vec_array design_densities = zp->getLocalView(Tpetra::Access::ReadOnly); + + if (type == ROL::UpdateType::Initial) + { + // This is the first call to update + // first linear solve was done in FEA class run function already + FEM_Dynamic_Elasticity_->elastic_solve(); + // initial design density data was already communicated for ghost nodes in init_design() + // decide to output current optimization state + FEM_Dynamic_Elasticity_->Explicit_Solver_Pointer_->write_outputs(); + } + else if (type == ROL::UpdateType::Accept) + { + } + else if (type == ROL::UpdateType::Revert) + { + // u_ was set to u=S(x) during a trial update + // and has been rejected as the new iterate + // Revert to cached value + // This is a new value of x + // communicate density variables for ghosts + FEM_Dynamic_Elasticity_->comm_variables(zp); + // update deformation variables + FEM_Dynamic_Elasticity_->update_forward_solve(zp); + if (Explicit_Solver_Pointer_->myrank == 0) + { + *fos << "called Revert" << std::endl; + } + } + else if (type == ROL::UpdateType::Trial) + { + // This is a new value of x + // communicate density variables for ghosts + FEM_Dynamic_Elasticity_->comm_variables(zp); + // update deformation variables + FEM_Dynamic_Elasticity_->update_forward_solve(zp); + if (Explicit_Solver_Pointer_->myrank == 0) + { + *fos << "called Trial" << std::endl; + } + + // decide to output current optimization state + FEM_Dynamic_Elasticity_->Explicit_Solver_Pointer_->write_outputs(); + } + else // ROL::UpdateType::Temp + // This is a new value of x used for, + // e.g., finite-difference checks + { + if (Explicit_Solver_Pointer_->myrank == 0) + { + *fos << "called Temp" << std::endl; + } + FEM_Dynamic_Elasticity_->comm_variables(zp); + FEM_Dynamic_Elasticity_->update_forward_solve(zp); + } } - else { // ROL::UpdateType::Temp - // This is a new value of x used for, - // e.g., finite-difference checks - if(Explicit_Solver_Pointer_->myrank==0) - *fos << "called Temp" << std::endl; - FEM_SGH_->comm_variables(zp); - FEM_SGH_->update_forward_solve(zp); + + void update_sgh(const ROL::Vector& z, ROL::UpdateType type, int iter = -1) + { + // debug + std::ostream& out = std::cout; + Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + + current_step++; + ROL::Ptr zp = getVector(z); + const_host_vec_array design_densities = zp->getLocalView(Tpetra::Access::ReadOnly); + + if (type == ROL::UpdateType::Initial) + { + // This is the first call to update + // first linear solve was done in FEA class run function already + FEM_SGH_->sgh_solve(); + // initial design density data was already communicated for ghost nodes in init_design() + // decide to output current optimization state + FEM_SGH_->Explicit_Solver_Pointer_->write_outputs(); + } + else if (type == ROL::UpdateType::Accept) + { + } + else if (type == ROL::UpdateType::Revert) + { + // u_ was set to u=S(x) during a trial update + // and has been rejected as the new iterate + // Revert to cached value + // This is a new value of x + // communicate density variables for ghosts + FEM_SGH_->comm_variables(zp); + // update deformation variables + FEM_SGH_->update_forward_solve(zp); + if (Explicit_Solver_Pointer_->myrank == 0) + { + *fos << "called Revert" << std::endl; + } + } + else if (type == ROL::UpdateType::Trial) + { + // This is a new value of x + // communicate density variables for ghosts + FEM_SGH_->comm_variables(zp); + // update deformation variables + FEM_SGH_->update_forward_solve(zp); + if (Explicit_Solver_Pointer_->myrank == 0) + { + *fos << "called Trial" << std::endl; + } + + // decide to output current optimization state + FEM_SGH_->Explicit_Solver_Pointer_->write_outputs(); + } + else // ROL::UpdateType::Temp + // This is a new value of x used for, + // e.g., finite-difference checks + { + if (Explicit_Solver_Pointer_->myrank == 0) + { + *fos << "called Temp" << std::endl; + } + FEM_SGH_->comm_variables(zp); + FEM_SGH_->update_forward_solve(zp); + } } - } - - real_t value(const ROL::Vector &z, real_t &tol) { - //std::cout << "Started obj value on task " <myrank << std::endl; - ROL::Ptr zp = getVector(z); - real_t c = 0.0; - - //debug print - //std::ostream &out = std::cout; - //Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - //if(FEM_->myrank==0) - //*fos << "Value function z:" << std::endl; - //zp->describe(*fos,Teuchos::VERB_EXTREME); - //*fos << std::endl; - //std::fflush(stdout); - - const_host_vec_array design_densities = zp->getLocalView (Tpetra::Access::ReadOnly); - //communicate ghosts and solve for nodal degrees of freedom as a function of the current design variables - /* - if(last_comm_step!=current_step){ - FEM_->comm_variables(zp); - last_comm_step = current_step; + + real_t value(const ROL::Vector& z, real_t& tol) + { + // std::cout << "Started obj value on task " <myrank << std::endl; + ROL::Ptr zp = getVector(z); + real_t c = 0.0; + + // debug print + // std::ostream &out = std::cout; + // Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + // if(FEM_->myrank==0) + // *fos << "Value function z:" << std::endl; + // zp->describe(*fos,Teuchos::VERB_EXTREME); + // *fos << std::endl; + // std::fflush(stdout); + + const_host_vec_array design_densities = zp->getLocalView(Tpetra::Access::ReadOnly); + // communicate ghosts and solve for nodal degrees of freedom as a function of the current design variables + /* + if(last_comm_step!=current_step){ + FEM_->comm_variables(zp); + last_comm_step = current_step; + } + + if(last_solve_step!=current_step){ + //std::cout << "UPDATED velocities" << std::endl; + FEM_->update_linear_solve(zp); + last_solve_step = current_step; + } + */ + // debug print of velocities + // std::ostream &out = std::cout; + // Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + // if(FEM_->myrank==0) + // *fos << "Displacement data :" << std::endl; + // FEM_->node_velocities_distributed->describe(*fos,Teuchos::VERB_EXTREME); + // *fos << std::endl; + // std::fflush(stdout); + + // ROL_Force = ROL::makePtr(FEM_->Global_Nodal_Forces); + if (set_module_type == FEA_MODULE_TYPE::SGH) + { + ROL_Velocities = ROL::makePtr(FEM_SGH_->node_velocities_distributed); + } + if (set_module_type == FEA_MODULE_TYPE::Dynamic_Elasticity) + { + ROL_Velocities = ROL::makePtr(FEM_Dynamic_Elasticity_->node_velocities_distributed); + } + + std::cout.precision(10); + if (Explicit_Solver_Pointer_->myrank == 0) + { + std::cout << "CURRENT TIME INTEGRAL OF KINETIC ENERGY " << objective_accumulation << std::endl; + } + + // std::cout << "Ended obj value on task " <myrank << std::endl; + return objective_accumulation; } - - if(last_solve_step!=current_step){ - //std::cout << "UPDATED velocities" << std::endl; - FEM_->update_linear_solve(zp); - last_solve_step = current_step; + + // void gradient_1( ROL::Vector &g, const ROL::Vector &u, const ROL::Vector &z, real_t &tol ) { + // g.zero(); + // } + + void gradient(ROL::Vector& g, const ROL::Vector& z, real_t& tol) + { + // std::cout << "Started obj gradient on task " <myrank << std::endl; + // get Tpetra multivector pointer from the ROL vector + ROL::Ptr zp = getVector(z); + ROL::Ptr gp = getVector(g); + + // communicate ghosts and solve for nodal degrees of freedom as a function of the current design variables + // FEM_->gradient_print_sync=1; + // FEM_->gradient_print_sync=0; + // get local view of the data + + if (set_module_type == FEA_MODULE_TYPE::SGH) + { + FEM_SGH_->compute_topology_optimization_gradient_full(zp, gp); + } + if (set_module_type == FEA_MODULE_TYPE::Dynamic_Elasticity) + { + FEM_Dynamic_Elasticity_->compute_topology_optimization_gradient_full(zp, gp); + } + // debug print of gradient + // std::ostream &out = std::cout; + // Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + // if(FEM_->myrank==0) + // *fos << "Gradient data :" << std::endl; + // gp->describe(*fos,Teuchos::VERB_EXTREME); + // *fos << std::endl; + // std::fflush(stdout); + // for(int i = 0; i < FEM_->nlocal_nodes; i++){ + // objective_gradients(i,0) *= -1; + // } + + // std::cout << "Objective Gradient called"<< std::endl; + // debug print of design variables + // std::ostream &out = std::cout; + // Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + // if(FEM_->myrank==0) + // *fos << "Gradient data :" << std::endl; + // gp->describe(*fos,Teuchos::VERB_EXTREME); + + // *fos << std::endl; + // std::fflush(stdout); + // std::cout << "ended obj gradient on task " <myrank << std::endl; } - */ - //debug print of velocities - //std::ostream &out = std::cout; - //Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - //if(FEM_->myrank==0) - //*fos << "Displacement data :" << std::endl; - //FEM_->node_velocities_distributed->describe(*fos,Teuchos::VERB_EXTREME); - //*fos << std::endl; - //std::fflush(stdout); - - //ROL_Force = ROL::makePtr(FEM_->Global_Nodal_Forces); - if(set_module_type==FEA_MODULE_TYPE::SGH) - ROL_Velocities = ROL::makePtr(FEM_SGH_->node_velocities_distributed); - if(set_module_type==FEA_MODULE_TYPE::Dynamic_Elasticity) - ROL_Velocities = ROL::makePtr(FEM_Dynamic_Elasticity_->node_velocities_distributed); - - std::cout.precision(10); - if(Explicit_Solver_Pointer_->myrank==0) - std::cout << "CURRENT TIME INTEGRAL OF KINETIC ENERGY " << objective_accumulation << std::endl; - - //std::cout << "Ended obj value on task " <myrank << std::endl; - return objective_accumulation; - } - - //void gradient_1( ROL::Vector &g, const ROL::Vector &u, const ROL::Vector &z, real_t &tol ) { - //g.zero(); - //} - - void gradient( ROL::Vector &g, const ROL::Vector &z, real_t &tol ) { - //std::cout << "Started obj gradient on task " <myrank << std::endl; - //get Tpetra multivector pointer from the ROL vector - ROL::Ptr zp = getVector(z); - ROL::Ptr gp = getVector(g); - - //communicate ghosts and solve for nodal degrees of freedom as a function of the current design variables - //FEM_->gradient_print_sync=1; - //FEM_->gradient_print_sync=0; - //get local view of the data - - if(set_module_type==FEA_MODULE_TYPE::SGH) - FEM_SGH_->compute_topology_optimization_gradient_full(zp,gp); - if(set_module_type==FEA_MODULE_TYPE::Dynamic_Elasticity) - FEM_Dynamic_Elasticity_->compute_topology_optimization_gradient_full(zp,gp); - //debug print of gradient - //std::ostream &out = std::cout; - //Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - //if(FEM_->myrank==0) - //*fos << "Gradient data :" << std::endl; - //gp->describe(*fos,Teuchos::VERB_EXTREME); - //*fos << std::endl; - //std::fflush(stdout); - //for(int i = 0; i < FEM_->nlocal_nodes; i++){ - //objective_gradients(i,0) *= -1; - //} - - //std::cout << "Objective Gradient called"<< std::endl; - //debug print of design variables - //std::ostream &out = std::cout; - //Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - //if(FEM_->myrank==0) - //*fos << "Gradient data :" << std::endl; - //gp->describe(*fos,Teuchos::VERB_EXTREME); - - //*fos << std::endl; - //std::fflush(stdout); - //std::cout << "ended obj gradient on task " <myrank << std::endl; - } - }; #endif // end header guard diff --git a/src/Parallel-Solvers/Parallel-Explicit/example_simple.yaml b/src/Parallel-Solvers/Parallel-Explicit/example_simple.yaml index 86c77388a..c3c68db69 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/example_simple.yaml +++ b/src/Parallel-Solvers/Parallel-Explicit/example_simple.yaml @@ -10,7 +10,7 @@ mesh_generation_options: type: Box origin: [0, 0, 0] length: [1.2, 1.2, 1.2] - num_elems: [32, 32, 32] + num_elems: [40, 40, 40] output_options: timer_output_level: thorough @@ -87,7 +87,7 @@ regions: - volume: type: sphere radius1: 0.0 - radius2: 0.0375 + radius2: 0.03 material_id: 0 den: 1.0 ie: 0.25833839995946534 diff --git a/src/Parallel-Solvers/Parallel-Explicit/main.cpp b/src/Parallel-Solvers/Parallel-Explicit/main.cpp index 3e1ff63dc..8f0d8151b 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/main.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/main.cpp @@ -11,14 +11,14 @@ This program is open source under the BSD-3 License. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - + 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - + 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - + 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. @@ -33,8 +33,8 @@ WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - **********************************************************************************************/ - +**********************************************************************************************/ + #include #include #include @@ -47,37 +47,49 @@ #include #include -void solver_setup(const char* filename){ - Explicit_Solver solver( - Yaml::from_file_strict(filename) - ); - //checks for optional solver routines - if(solver.setup_flag) solver.solver_setup(); - // invoke solver's run function (should perform most of the computation) - solver.run(); - //invoke optional finalize function - if(solver.finalize_flag) solver.solver_finalize(); +void solver_setup(const char* filename) +{ + Explicit_Solver solver( + Yaml::from_file_strict(filename) + ); + // checks for optional solver routines + if (solver.setup_flag) + { + solver.solver_setup(); + } + // invoke solver's run function (should perform most of the computation) + solver.run(); + // invoke optional finalize function + if (solver.finalize_flag) + { + solver.solver_finalize(); + } } -int main(int argc, char *argv[]){ - //initialize MPI - MPI_Init(&argc, &argv); +int main(int argc, char* argv[]) +{ + // initialize MPI + MPI_Init(&argc, &argv); + + Kokkos::initialize(); + if (argc < 2) + { + int myrank; + MPI_Comm_rank(MPI_COMM_WORLD, &myrank); + if (myrank == 0) + { + std::cout << "Fierro requires a yaml setup file as a command line argument" << std::endl; + } + } + else + { + solver_setup(argv[1]); + } + + MPI_Barrier(MPI_COMM_WORLD); - Kokkos::initialize(); - if(argc<2){ - int myrank; - MPI_Comm_rank(MPI_COMM_WORLD,&myrank); - if(myrank==0) - std::cout << "Fierro requires a yaml setup file as a command line argument" << std::endl; - } - else{ - solver_setup(argv[1]); - } - - MPI_Barrier(MPI_COMM_WORLD); - - Kokkos::finalize(); - MPI_Finalize(); + Kokkos::finalize(); + MPI_Finalize(); - return 0; + return 0; } diff --git a/src/Parallel-Solvers/Parallel-Explicit/outputs.cpp b/src/Parallel-Solvers/Parallel-Explicit/outputs.cpp index 2c2cf1702..6c7261e1b 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/outputs.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/outputs.cpp @@ -8,716 +8,803 @@ typedef Kokkos::LayoutRight CArrayLayout; typedef Kokkos::LayoutLeft FArrayLayout; typedef Tpetra::MultiVector<>::dual_view_type::t_host::array_layout TpetraHostViewLayout; -MPI_Offset -mpi_get_file_position_shared( - const MPI_Comm comm, - const MPI_File file_parallel); - -void -write_string_stream_to_file_mpi_all( - const std::stringstream& str_stream, - MPI_Offset rank_offset_multiplier, - const MPI_File file_parallel, - const MPI_Comm comm); - -template -void -write_data_to_string_stream( - const T* ptr, - size_t dim0, - size_t dim1, - std::stringstream& str_stream); +MPI_Offset mpi_get_file_position_shared( + const MPI_Comm comm, + const MPI_File file_parallel); + +void write_string_stream_to_file_mpi_all( + const std::stringstream& str_stream, + MPI_Offset rank_offset_multiplier, + const MPI_File file_parallel, + const MPI_Comm comm); + +template +void write_data_to_string_stream( + const T* ptr, + size_t dim0, + size_t dim1, + std::stringstream& str_stream); template -Teuchos::RCP> -sort_data( - const SC* unsorted_data_ptr, - Teuchos::RCP> unsorted_map, - size_t dim1, - size_t num_global_unique_elements, - MPI_Comm comm, - Teuchos::RCP> &sorting_importer); +Teuchos::RCP> sort_data( + const SC* unsorted_data_ptr, + Teuchos::RCP> unsorted_map, + size_t dim1, + size_t num_global_unique_elements, + MPI_Comm comm, + Teuchos::RCP>& sorting_importer); template -void -sort_and_write_data_to_file_mpi_all( - const SC* unsorted_data_ptr, - Teuchos::RCP> unsorted_map, - size_t dim1, - size_t num_global_unique_elements, - MPI_Comm comm, - MPI_File file_parallel, - Teuchos::RCP> &sorting_importer); - -Teuchos::RCP> -get_cell_nodes( - const CArray & nodes_in_elem, - size_t num_dim, - Solver::node_ordering_convention active_node_ordering_convention); - -Teuchos::RCP> -calculate_elem_speed( - const Teuchos::RCP all_node_velocities_distributed, - const Teuchos::RCP global_nodes_in_elem_distributed, - const Teuchos::RCP ghost_node_velocities_distributed, - const Teuchos::RCP> ghost_importer); - -Teuchos::RCP> -calculate_elem_switch( - Teuchos::RCP> all_element_map); - -Teuchos::RCP> -get_elem_proc_id( - Teuchos::RCP> all_element_map, - size_t myrank); - -Teuchos::RCP> -get_elem_gid( - Teuchos::RCP> all_element_map); - -Teuchos::RCP> -get_design_density( - size_t rnum_nodes, - bool topology_optimization_on, - const Teuchos::RCP design_node_densities_distributed); - -void -Explicit_Solver::write_outputs() +void sort_and_write_data_to_file_mpi_all( + const SC* unsorted_data_ptr, + Teuchos::RCP> unsorted_map, + size_t dim1, + size_t num_global_unique_elements, + MPI_Comm comm, + MPI_File file_parallel, + Teuchos::RCP>& sorting_importer); + +Teuchos::RCP> get_cell_nodes( + const CArray& nodes_in_elem, + size_t num_dim, + Solver::node_ordering_convention active_node_ordering_convention); + +Teuchos::RCP> calculate_elem_speed( + const Teuchos::RCP all_node_velocities_distributed, + const Teuchos::RCP global_nodes_in_elem_distributed, + const Teuchos::RCP ghost_node_velocities_distributed, + const Teuchos::RCP> ghost_importer); + +Teuchos::RCP> calculate_elem_switch( + Teuchos::RCP> all_element_map); + +Teuchos::RCP> get_elem_proc_id( + Teuchos::RCP> all_element_map, + size_t myrank); + +Teuchos::RCP> get_elem_gid( + Teuchos::RCP> all_element_map); + +Teuchos::RCP> get_design_density( + size_t rnum_nodes, + bool topology_optimization_on, + const Teuchos::RCP design_node_densities_distributed); + +void Explicit_Solver::write_outputs() { - // No output for OUTPUT_FORMAT::none - if (simparam.output_options.output_file_format == OUTPUT_FORMAT::none) - return; + // No output for OUTPUT_FORMAT::none + if (simparam.output_options.output_file_format == OUTPUT_FORMAT::none) + { + return; + } - const size_t rk_level = simparam.dynamic_options.rk_num_bins - 1; - Teuchos::RCP> design_density; - Teuchos::RCP> elem_switch; - Teuchos::RCP> elem_proc_id; - Teuchos::RCP> elem_gid; - Teuchos::RCP> elem_speed; + const size_t rk_level = simparam.dynamic_options.rk_num_bins - 1; + Teuchos::RCP> design_density; + Teuchos::RCP> elem_switch; + Teuchos::RCP> elem_proc_id; + Teuchos::RCP> elem_gid; + Teuchos::RCP> elem_speed; - for (const FIELD& field_name : simparam.output_options.output_fields) { - switch (field_name) + for (const FIELD& field_name : simparam.output_options.output_fields) { - case FIELD::design_density: - // node "design_density" - design_density = get_design_density(map->getLocalNumElements(), + switch (field_name) + { + case FIELD::design_density: + // node "design_density" + design_density = get_design_density(map->getLocalNumElements(), simparam.topology_optimization_on, design_node_densities_distributed); - point_data_scalars_double["design_density"] = design_density->pointer(); - break; - - case FIELD::speed: - // element "speed" - elem_speed = calculate_elem_speed(all_node_velocities_distributed, global_nodes_in_elem_distributed, - ghost_node_velocities_distributed, ghost_importer); - cell_data_scalars_double["speed"] = elem_speed->pointer(); - break; - - case FIELD::element_switch: - // element "element_switch" - elem_switch = calculate_elem_switch(all_element_map); - cell_data_scalars_int["element_switch"] = elem_switch->pointer(); - break; - - case FIELD::processor_id: - // element "processor_id" - elem_proc_id = get_elem_proc_id(all_element_map, myrank); - cell_data_scalars_int["processor_id"] = elem_proc_id->pointer(); - break; - - case FIELD::element_id: - // element "element_id" - elem_gid = get_elem_gid(all_element_map); - cell_data_scalars_int["element_id"] = elem_gid->pointer(); - break; - - default: - break; - - } // end switch - } // end if - - for(int imodule = 0; imodule < nfea_modules; imodule++){ - fea_modules[imodule]->write_data(point_data_scalars_double, point_data_vectors_double, cell_data_scalars_double, cell_data_scalars_int, cell_data_fields_double); - } + point_data_scalars_double["design_density"] = design_density->pointer(); + break; + + case FIELD::speed: + // element "speed" + elem_speed = calculate_elem_speed( + all_node_velocities_distributed, + global_nodes_in_elem_distributed, + ghost_node_velocities_distributed, + ghost_importer); + cell_data_scalars_double["speed"] = elem_speed->pointer(); + break; + + case FIELD::element_switch: + // element "element_switch" + elem_switch = calculate_elem_switch(all_element_map); + cell_data_scalars_int["element_switch"] = elem_switch->pointer(); + break; + + case FIELD::processor_id: + // element "processor_id" + elem_proc_id = get_elem_proc_id(all_element_map, myrank); + cell_data_scalars_int["processor_id"] = elem_proc_id->pointer(); + break; + + case FIELD::element_id: + // element "element_id" + elem_gid = get_elem_gid(all_element_map); + cell_data_scalars_int["element_id"] = elem_gid->pointer(); + break; + + default: + break; + } // end switch + } // end if + + for (int imodule = 0; imodule < nfea_modules; imodule++) + { + fea_modules[imodule]->write_data( + point_data_scalars_double, + point_data_vectors_double, + cell_data_scalars_double, + cell_data_scalars_int, + cell_data_fields_double); + } - switch (simparam.output_options.output_file_format) - { + switch (simparam.output_options.output_file_format) + { case OUTPUT_FORMAT::vtk: - parallel_vtk_writer_new(); - break; + parallel_vtk_writer_new(); + break; case OUTPUT_FORMAT::vtu: - parallel_vtu_writer_new(); - break; + parallel_vtu_writer_new(); + break; default: - break; - }; + break; + } + ; } -void -Explicit_Solver::parallel_vtu_writer_new() +void Explicit_Solver::parallel_vtu_writer_new() { - /* to be added... */ - throw std::runtime_error("parallel_vtu_writer_new() not yet implemented. use parallel_vtk_writer_new()"); + /* to be added... */ + throw std::runtime_error("parallel_vtu_writer_new() not yet implemented. use parallel_vtk_writer_new()"); } -std::string -construct_file_name( - size_t file_count, - int displacement_module) +std::string construct_file_name( + size_t file_count, + int displacement_module) { - bool displace_geometry = false; - std::string current_file_name; - std::string base_file_name= "VTK"; - std::string base_file_name_undeformed= "VTK_undeformed"; - std::string file_extension= ".vtk"; - - if(displace_geometry && displacement_module>=0) - current_file_name = base_file_name_undeformed + std::to_string(file_count) + file_extension; - else - current_file_name = base_file_name + std::to_string(file_count) + file_extension; - return current_file_name; + bool displace_geometry = false; + std::string current_file_name; + std::string base_file_name = "VTK"; + std::string base_file_name_undeformed = "VTK_undeformed"; + std::string file_extension = ".vtk"; + + if (displace_geometry && displacement_module >= 0) + { + current_file_name = base_file_name_undeformed + std::to_string(file_count) + file_extension; + } + else + { + current_file_name = base_file_name + std::to_string(file_count) + file_extension; + } + return current_file_name; } -void -Explicit_Solver::parallel_vtk_writer_new() +void Explicit_Solver::parallel_vtk_writer_new() { + int num_dim = simparam.num_dims; + std::stringstream str_stream; + MPI_Offset current_offset; + MPI_File myfile_parallel; + bool displace_geometry = false; - int num_dim = simparam.num_dims; - std::stringstream str_stream; - MPI_Offset current_offset; - MPI_File myfile_parallel; - bool displace_geometry = false; - - std::string vtk_dir = "vtk/"; - std::string vtk_data_dir = vtk_dir + "data/"; - - // mkdir if needed - struct stat st; - if (myrank == 0) { - if (stat(vtk_dir.c_str(), &st) != 0) { - str_stream.str(""); - str_stream << "mkdir" << " " << vtk_dir; - system(str_stream.str().c_str()); - } - if (stat(vtk_data_dir.c_str(), &st) != 0) { - str_stream.str(""); - str_stream << "mkdir" << " " << vtk_data_dir; - system(str_stream.str().c_str()); - } - } - MPI_Barrier(world); - - //construct file name - std::string current_file_name = construct_file_name(simparam.output_options.graphics_id, displacement_module); - std::string file_path = vtk_data_dir + current_file_name; - //open mpi file - int err = MPI_File_open(MPI_COMM_WORLD, file_path.c_str(), - MPI_MODE_CREATE|MPI_MODE_EXCL|MPI_MODE_WRONLY, - MPI_INFO_NULL, &myfile_parallel); - //allows overwriting the file if it already existed in the directory - if (err != MPI_SUCCESS) { - if (myrank == 0){ - MPI_File_delete(file_path.c_str(),MPI_INFO_NULL); + std::string vtk_dir = "vtk/"; + std::string vtk_data_dir = vtk_dir + "data/"; + + // mkdir if needed + struct stat st; + if (myrank == 0) + { + if (stat(vtk_dir.c_str(), &st) != 0) + { + str_stream.str(""); + str_stream << "mkdir" << " " << vtk_dir; + system(str_stream.str().c_str()); + } + if (stat(vtk_data_dir.c_str(), &st) != 0) + { + str_stream.str(""); + str_stream << "mkdir" << " " << vtk_data_dir; + system(str_stream.str().c_str()); + } } - MPI_File_open(MPI_COMM_WORLD, file_path.c_str(), - MPI_MODE_CREATE|MPI_MODE_EXCL|MPI_MODE_WRONLY, + MPI_Barrier(world); + + // construct file name + std::string current_file_name = construct_file_name(simparam.output_options.graphics_id, displacement_module); + std::string file_path = vtk_data_dir + current_file_name; + // open mpi file + int err = MPI_File_open(MPI_COMM_WORLD, file_path.c_str(), + MPI_MODE_CREATE | MPI_MODE_EXCL | MPI_MODE_WRONLY, + MPI_INFO_NULL, &myfile_parallel); + // allows overwriting the file if it already existed in the directory + if (err != MPI_SUCCESS) + { + if (myrank == 0) + { + MPI_File_delete(file_path.c_str(), MPI_INFO_NULL); + } + MPI_File_open(MPI_COMM_WORLD, file_path.c_str(), + MPI_MODE_CREATE | MPI_MODE_EXCL | MPI_MODE_WRONLY, MPI_INFO_NULL, &myfile_parallel); - } - - /*************** write header of the vtk file ***************/ - str_stream.str(""); - str_stream << "# vtk DataFile Version 2.0" << std::endl; - str_stream << "Mesh for Fierro" << std::endl; - str_stream << "ASCII" << std::endl; - str_stream << "DATASET UNSTRUCTURED_GRID" << std::endl; - current_offset = mpi_get_file_position_shared(world, myfile_parallel); - if (myrank == 0) { - MPI_File_write_at(myfile_parallel, current_offset, str_stream.str().c_str(), str_stream.str().length(), MPI_CHAR, MPI_STATUS_IGNORE); - } - - - /*************** write POINTS ***************/ - str_stream.str(""); - str_stream << std::endl << "POINTS " << num_nodes << " float" << std::endl; - current_offset = mpi_get_file_position_shared(world, myfile_parallel); - if(myrank == 0) { - MPI_File_write_at(myfile_parallel, current_offset, str_stream.str().c_str(), str_stream.str().length(), MPI_CHAR, MPI_STATUS_IGNORE); - } - //sgh_module->node_coords.update_host(); - { //view scope - host_vec_array node_coords = node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); - double* coord_data = node_coords.data(); - sort_and_write_data_to_file_mpi_all ( - coord_data, map, num_dim, num_nodes, world, myfile_parallel, node_sorting_importer); - } - - - /*************** write CELLS ***************/ - str_stream.str(""); - str_stream << std::endl << "CELLS " << num_elem << " " << num_elem*(max_nodes_per_element+1) << std::endl; - current_offset = mpi_get_file_position_shared(world, myfile_parallel); - if(myrank == 0) { - MPI_File_write_at(myfile_parallel, current_offset, str_stream.str().c_str(), str_stream.str().length(), MPI_CHAR, MPI_STATUS_IGNORE); - } - CArray nodes_in_elem (rnum_elem, max_nodes_per_element); - { //view scope - auto host_view = global_nodes_in_elem_distributed->getLocalView (Tpetra::Access::ReadOnly); - for (size_t ielem = 0; ielem < nodes_in_elem.dims(0); ielem++) { - for (size_t inode = 0; inode < nodes_in_elem.dims(1); inode++) { - nodes_in_elem(ielem, inode) = host_view(ielem, inode); - } - } - } //end view scope - auto cell_data = get_cell_nodes(nodes_in_elem, num_dim, active_node_ordering_convention); - sort_and_write_data_to_file_mpi_all ( - cell_data->pointer(), all_element_map, cell_data->dims(1), num_elem, world, myfile_parallel, element_sorting_importer); - - - /*************** write CELL_TYPES ***************/ - str_stream.str(""); - str_stream << std::endl << "CELL_TYPES " << num_elem << std::endl; - current_offset = mpi_get_file_position_shared(world, myfile_parallel); - if(myrank == 0) { - MPI_File_write_at(myfile_parallel, current_offset, str_stream.str().c_str(), str_stream.str().length(), MPI_CHAR, MPI_STATUS_IGNORE); - } - CArray cell_type (all_element_map->getLocalNumElements(), 1); - for (int i = 0; i < cell_type.dims(0); i++) - for (int j = 0; j < cell_type.dims(1); j++) - cell_type(i,j) = 12; - sort_and_write_data_to_file_mpi_all ( - cell_type.pointer(), all_element_map, cell_type.dims(1), num_elem, world, myfile_parallel, element_sorting_importer); - - - /*************** write POINT_DATA ***************/ - str_stream.str(""); - str_stream << std::endl << "POINT_DATA " << num_nodes; - current_offset = mpi_get_file_position_shared(world, myfile_parallel); - if(myrank == 0) { - MPI_File_write_at(myfile_parallel, current_offset, str_stream.str().c_str(), - str_stream.str().length(), MPI_CHAR, MPI_STATUS_IGNORE); - } + } - //SCALARS float - for (auto it = point_data_scalars_double.begin(); it != point_data_scalars_double.end(); it++) { + /*************** write header of the vtk file ***************/ str_stream.str(""); - str_stream << std::endl << "SCALARS " << it->first << " float 1" << std::endl; - str_stream << "LOOKUP_TABLE default" << std::endl; + str_stream << "# vtk DataFile Version 2.0" << std::endl; + str_stream << "Mesh for Fierro" << std::endl; + str_stream << "ASCII" << std::endl; + str_stream << "DATASET UNSTRUCTURED_GRID" << std::endl; current_offset = mpi_get_file_position_shared(world, myfile_parallel); - if(myrank == 0) { - MPI_File_write_at(myfile_parallel, current_offset, str_stream.str().c_str(), - str_stream.str().length(), MPI_CHAR, MPI_STATUS_IGNORE); + if (myrank == 0) + { + MPI_File_write_at( + myfile_parallel, + current_offset, + str_stream.str().c_str(), + str_stream.str().length(), + MPI_CHAR, + MPI_STATUS_IGNORE); } - sort_and_write_data_to_file_mpi_all ( - it->second, map, 1, num_nodes, world, myfile_parallel, node_sorting_importer); - } - //VECTORS float - for (auto it = point_data_vectors_double.begin(); it != point_data_vectors_double.end(); it++) { + /*************** write POINTS ***************/ str_stream.str(""); - str_stream << std::endl << "VECTORS " << it->first << " float" << std::endl; + str_stream << std::endl << "POINTS " << num_nodes << " float" << std::endl; current_offset = mpi_get_file_position_shared(world, myfile_parallel); - if(myrank == 0) { - MPI_File_write_at(myfile_parallel, current_offset, str_stream.str().c_str(), - str_stream.str().length(), MPI_CHAR, MPI_STATUS_IGNORE); + if (myrank == 0) + { + MPI_File_write_at( + myfile_parallel, + current_offset, + str_stream.str().c_str(), + str_stream.str().length(), + MPI_CHAR, + MPI_STATUS_IGNORE); + } + // sgh_module->node_coords.update_host(); + { // view scope + host_vec_array node_coords = node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + double* coord_data = node_coords.data(); + sort_and_write_data_to_file_mpi_all( + coord_data, + map, + num_dim, + num_nodes, + world, + myfile_parallel, + node_sorting_importer); } - sort_and_write_data_to_file_mpi_all ( - it->second, map, num_dim, num_nodes, world, myfile_parallel, node_sorting_importer); - } - - - /*************** write CELL_DATA ***************/ - str_stream.str(""); - str_stream << std::endl << "CELL_DATA " << num_elem; - current_offset = mpi_get_file_position_shared(world, myfile_parallel); - if(myrank == 0) { - MPI_File_write_at(myfile_parallel, current_offset, str_stream.str().c_str(), - str_stream.str().length(), MPI_CHAR, MPI_STATUS_IGNORE); - } - //SCALARS float - for (auto it = cell_data_scalars_double.begin(); it != cell_data_scalars_double.end(); it++) { + /*************** write CELLS ***************/ str_stream.str(""); - str_stream << std::endl << "SCALARS " << it->first << " float 1" << std::endl; - str_stream << "LOOKUP_TABLE default" << std::endl; + str_stream << std::endl << "CELLS " << num_elem << " " << num_elem * (max_nodes_per_element + 1) << std::endl; current_offset = mpi_get_file_position_shared(world, myfile_parallel); - if(myrank == 0) { - MPI_File_write_at(myfile_parallel, current_offset, str_stream.str().c_str(), + if (myrank == 0) + { + MPI_File_write_at( + myfile_parallel, + current_offset, + str_stream.str().c_str(), + str_stream.str().length(), + MPI_CHAR, + MPI_STATUS_IGNORE); + } + CArray nodes_in_elem(rnum_elem, max_nodes_per_element); + { // view scope + auto host_view = global_nodes_in_elem_distributed->getLocalView(Tpetra::Access::ReadOnly); + for (size_t ielem = 0; ielem < nodes_in_elem.dims(0); ielem++) + { + for (size_t inode = 0; inode < nodes_in_elem.dims(1); inode++) + { + nodes_in_elem(ielem, inode) = host_view(ielem, inode); + } + } + } // end view scope + auto cell_data = get_cell_nodes(nodes_in_elem, num_dim, active_node_ordering_convention); + sort_and_write_data_to_file_mpi_all( + cell_data->pointer(), + all_element_map, + cell_data->dims(1), + num_elem, + world, + myfile_parallel, + element_sorting_importer); + + /*************** write CELL_TYPES ***************/ + str_stream.str(""); + str_stream << std::endl << "CELL_TYPES " << num_elem << std::endl; + current_offset = mpi_get_file_position_shared(world, myfile_parallel); + if (myrank == 0) + { + MPI_File_write_at( + myfile_parallel, + current_offset, + str_stream.str().c_str(), + str_stream.str().length(), + MPI_CHAR, + MPI_STATUS_IGNORE); + } + CArray cell_type(all_element_map->getLocalNumElements(), 1); + for (int i = 0; i < cell_type.dims(0); i++) + { + for (int j = 0; j < cell_type.dims(1); j++) + { + cell_type(i, j) = 12; + } + } + sort_and_write_data_to_file_mpi_all( + cell_type.pointer(), + all_element_map, + cell_type.dims(1), + num_elem, + world, + myfile_parallel, + element_sorting_importer); + + /*************** write POINT_DATA ***************/ + str_stream.str(""); + str_stream << std::endl << "POINT_DATA " << num_nodes; + current_offset = mpi_get_file_position_shared(world, myfile_parallel); + if (myrank == 0) + { + MPI_File_write_at(myfile_parallel, current_offset, str_stream.str().c_str(), + str_stream.str().length(), MPI_CHAR, MPI_STATUS_IGNORE); + } + + // SCALARS float + for (auto it = point_data_scalars_double.begin(); it != point_data_scalars_double.end(); it++) + { + str_stream.str(""); + str_stream << std::endl << "SCALARS " << it->first << " float 1" << std::endl; + str_stream << "LOOKUP_TABLE default" << std::endl; + current_offset = mpi_get_file_position_shared(world, myfile_parallel); + if (myrank == 0) + { + MPI_File_write_at(myfile_parallel, current_offset, str_stream.str().c_str(), str_stream.str().length(), MPI_CHAR, MPI_STATUS_IGNORE); + } + sort_and_write_data_to_file_mpi_all( + it->second, map, 1, num_nodes, world, myfile_parallel, node_sorting_importer); } - sort_and_write_data_to_file_mpi_all ( - it->second, all_element_map, 1, num_elem, world, myfile_parallel, element_sorting_importer); - } - //SCALARS int - for (auto it = cell_data_scalars_int.begin(); it != cell_data_scalars_int.end(); it++) { + // VECTORS float + for (auto it = point_data_vectors_double.begin(); it != point_data_vectors_double.end(); it++) + { + str_stream.str(""); + str_stream << std::endl << "VECTORS " << it->first << " float" << std::endl; + current_offset = mpi_get_file_position_shared(world, myfile_parallel); + if (myrank == 0) + { + MPI_File_write_at(myfile_parallel, current_offset, str_stream.str().c_str(), + str_stream.str().length(), MPI_CHAR, MPI_STATUS_IGNORE); + } + sort_and_write_data_to_file_mpi_all( + it->second, map, num_dim, num_nodes, world, myfile_parallel, node_sorting_importer); + } + + /*************** write CELL_DATA ***************/ str_stream.str(""); - str_stream << std::endl << "SCALARS " << it->first << " int 1" << std::endl; - str_stream << "LOOKUP_TABLE default" << std::endl; + str_stream << std::endl << "CELL_DATA " << num_elem; current_offset = mpi_get_file_position_shared(world, myfile_parallel); - if(myrank == 0) { - MPI_File_write_at(myfile_parallel, current_offset, str_stream.str().c_str(), + if (myrank == 0) + { + MPI_File_write_at(myfile_parallel, current_offset, str_stream.str().c_str(), + str_stream.str().length(), MPI_CHAR, MPI_STATUS_IGNORE); + } + + // SCALARS float + for (auto it = cell_data_scalars_double.begin(); it != cell_data_scalars_double.end(); it++) + { + str_stream.str(""); + str_stream << std::endl << "SCALARS " << it->first << " float 1" << std::endl; + str_stream << "LOOKUP_TABLE default" << std::endl; + current_offset = mpi_get_file_position_shared(world, myfile_parallel); + if (myrank == 0) + { + MPI_File_write_at(myfile_parallel, current_offset, str_stream.str().c_str(), str_stream.str().length(), MPI_CHAR, MPI_STATUS_IGNORE); + } + sort_and_write_data_to_file_mpi_all( + it->second, all_element_map, 1, num_elem, world, myfile_parallel, element_sorting_importer); } - sort_and_write_data_to_file_mpi_all ( + + // SCALARS int + for (auto it = cell_data_scalars_int.begin(); it != cell_data_scalars_int.end(); it++) + { + str_stream.str(""); + str_stream << std::endl << "SCALARS " << it->first << " int 1" << std::endl; + str_stream << "LOOKUP_TABLE default" << std::endl; + current_offset = mpi_get_file_position_shared(world, myfile_parallel); + if (myrank == 0) + { + MPI_File_write_at(myfile_parallel, current_offset, str_stream.str().c_str(), + str_stream.str().length(), MPI_CHAR, MPI_STATUS_IGNORE); + } + sort_and_write_data_to_file_mpi_all( it->second, all_element_map, 1, num_elem, world, myfile_parallel, element_sorting_importer); - } - - //FIELD - str_stream.str(""); - str_stream << std::endl << "FIELD FieldData " << cell_data_fields_double.size() << std::endl; - current_offset = mpi_get_file_position_shared(world, myfile_parallel); - if(myrank == 0) { - MPI_File_write_at(myfile_parallel, current_offset, str_stream.str().c_str(), - str_stream.str().length(), MPI_CHAR, MPI_STATUS_IGNORE); - } - for (auto it = cell_data_fields_double.begin(); it != cell_data_fields_double.end(); it++) { - auto data_name = it->first; - auto [data_ptr, data_num_comps] = it->second; // Structured binding C++17 + } + + // FIELD str_stream.str(""); - str_stream << data_name << " " << data_num_comps << " " << num_elem << " float" << std::endl; + str_stream << std::endl << "FIELD FieldData " << cell_data_fields_double.size() << std::endl; current_offset = mpi_get_file_position_shared(world, myfile_parallel); - if(myrank == 0) { - MPI_File_write_at(myfile_parallel, current_offset, str_stream.str().c_str(), - str_stream.str().length(), MPI_CHAR, MPI_STATUS_IGNORE); + if (myrank == 0) + { + MPI_File_write_at(myfile_parallel, current_offset, str_stream.str().c_str(), + str_stream.str().length(), MPI_CHAR, MPI_STATUS_IGNORE); } - sort_and_write_data_to_file_mpi_all ( + for (auto it = cell_data_fields_double.begin(); it != cell_data_fields_double.end(); it++) + { + auto data_name = it->first; + auto [data_ptr, data_num_comps] = it->second; // Structured binding C++17 + str_stream.str(""); + str_stream << data_name << " " << data_num_comps << " " << num_elem << " float" << std::endl; + current_offset = mpi_get_file_position_shared(world, myfile_parallel); + if (myrank == 0) + { + MPI_File_write_at(myfile_parallel, current_offset, str_stream.str().c_str(), + str_stream.str().length(), MPI_CHAR, MPI_STATUS_IGNORE); + } + sort_and_write_data_to_file_mpi_all( data_ptr, all_element_map, data_num_comps, num_elem, world, myfile_parallel, element_sorting_importer); - } - - MPI_Barrier(world); - MPI_File_sync(myfile_parallel); - MPI_File_close(&myfile_parallel); - - - /*************** write .vtk.series file ***************/ - simparam.output_options.graphics_times(simparam.output_options.graphics_id) = time_value; - if (myrank == 0) { - FILE *myfile; - std::string filename = vtk_dir + "outputs.vtk.series"; - myfile = fopen(filename.c_str(), "w"); - - fprintf(myfile, "{\n"); - fprintf(myfile, " \"file-series-version\" : \"1.0\",\n"); - fprintf(myfile, " \"files\" : [\n"); - for (int i = 0; i <= simparam.output_options.graphics_id; i++) { - std::string vtk_filename = construct_file_name(i, displacement_module); - fprintf(myfile, " { \"name\" : \"data/%s\", \"time\" : %12.5e },\n", - vtk_filename.c_str(), simparam.output_options.graphics_times(i)); } - fprintf(myfile, " ]\n"); - fprintf(myfile, "}\n"); - fclose(myfile); - } - - simparam.output_options.graphics_id++; + MPI_Barrier(world); + MPI_File_sync(myfile_parallel); + MPI_File_close(&myfile_parallel); -} + /*************** write .vtk.series file ***************/ + simparam.output_options.graphics_times(simparam.output_options.graphics_id) = time_value; + if (myrank == 0) + { + FILE* myfile; + std::string filename = vtk_dir + "outputs.vtk.series"; + myfile = fopen(filename.c_str(), "w"); + + fprintf(myfile, "{\n"); + fprintf(myfile, " \"file-series-version\" : \"1.0\",\n"); + fprintf(myfile, " \"files\" : [\n"); + for (int i = 0; i <= simparam.output_options.graphics_id; i++) + { + std::string vtk_filename = construct_file_name(i, displacement_module); + fprintf(myfile, " { \"name\" : \"data/%s\", \"time\" : %12.5e },\n", + vtk_filename.c_str(), simparam.output_options.graphics_times(i)); + } + fprintf(myfile, " ]\n"); + fprintf(myfile, "}\n"); + fclose(myfile); + } + + simparam.output_options.graphics_id++; +} -MPI_Offset -mpi_get_file_position_shared( - const MPI_Comm comm, - const MPI_File file_parallel) +MPI_Offset mpi_get_file_position_shared( + const MPI_Comm comm, + const MPI_File file_parallel) { - MPI_Offset position; + MPI_Offset position; - MPI_Barrier(comm); - MPI_File_seek_shared(file_parallel, 0, MPI_SEEK_END); - MPI_File_get_position_shared(file_parallel, &position); - MPI_Barrier(comm); - return position; + MPI_Barrier(comm); + MPI_File_seek_shared(file_parallel, 0, MPI_SEEK_END); + MPI_File_get_position_shared(file_parallel, &position); + MPI_Barrier(comm); + return position; } -void -write_string_stream_to_file_mpi_all( - const std::stringstream& str_stream, - MPI_Offset rank_offset_multiplier, - const MPI_File file_parallel, - const MPI_Comm comm) +void write_string_stream_to_file_mpi_all( + const std::stringstream& str_stream, + MPI_Offset rank_offset_multiplier, + const MPI_File file_parallel, + const MPI_Comm comm) { - /* `str_stream`: must have the same line length - * `rank_offset_multiplier`: should be the index of the first node or element in the rank - * `file_parallel`: the parallel mpi file to write to - * `comm`: MPI_Comm that should do the write - * */ - - MPI_Offset current_offset, file_offset; - current_offset = mpi_get_file_position_shared(comm, file_parallel); - std::string line; - std::getline(const_cast(str_stream), line); - auto line_length = line.length() + 1; // the +1 is for end of line character - file_offset = line_length * rank_offset_multiplier + current_offset; - MPI_Barrier(comm); - MPI_File_write_at_all(file_parallel, file_offset, str_stream.str().c_str(), + /* `str_stream`: must have the same line length + * `rank_offset_multiplier`: should be the index of the first node or element in the rank + * `file_parallel`: the parallel mpi file to write to + * `comm`: MPI_Comm that should do the write + * */ + + MPI_Offset current_offset, file_offset; + current_offset = mpi_get_file_position_shared(comm, file_parallel); + std::string line; + std::getline(const_cast(str_stream), line); + auto line_length = line.length() + 1; // the +1 is for end of line character + file_offset = line_length * rank_offset_multiplier + current_offset; + MPI_Barrier(comm); + MPI_File_write_at_all(file_parallel, file_offset, str_stream.str().c_str(), str_stream.str().length(), MPI_CHAR, MPI_STATUS_IGNORE); - MPI_Barrier(comm); - return; + MPI_Barrier(comm); + return; } -template +template void write_data_to_string_stream( - const T* ptr, - size_t dim0, - size_t dim1, - std::stringstream& str_stream) + const T* ptr, + size_t dim0, + size_t dim1, + std::stringstream& str_stream) { - size_t w; - size_t p; - //crude way to ensure that all lines have the same length. required for MPI IO - if constexpr (std::is_same::value or - std::is_same::value) { - w = 25; - p = 8; - } - else if constexpr (std::is_same::value or - std::is_same::value or - std::is_same::value or - std::is_same::value) { - w = 10; - p = 0; - } - else { - throw std::runtime_error(std::string("unknown datatype in function ") + + size_t w; + size_t p; + // crude way to ensure that all lines have the same length. required for MPI IO + if constexpr (std::is_same::value or + std::is_same::value) + { + w = 25; + p = 8; + } + else if constexpr (std::is_same::value or + std::is_same::value or + std::is_same::value or + std::is_same::value) + { + w = 10; + p = 0; + } + else + { + throw std::runtime_error(std::string("unknown datatype in function ") + std::string(__FUNCTION__) + std::string("in file ") + std::string(__FILE__)); - } + } - // create view of unsorted_data_ptr. not using matar here. array_layout is needed - Kokkos::View + // create view of unsorted_data_ptr. not using matar here. array_layout is needed + Kokkos::View data_view(ptr, dim0, dim1); - str_stream.str(""); - str_stream << std::fixed << std::setprecision(p); - for (size_t i = 0; i < dim0; i++) { - for (size_t j = 0; j < dim1; j++) { - str_stream << std::left << std::setw(w) << data_view(i,j) << " "; + str_stream.str(""); + str_stream << std::fixed << std::setprecision(p); + for (size_t i = 0; i < dim0; i++) + { + for (size_t j = 0; j < dim1; j++) + { + str_stream << std::left << std::setw(w) << data_view(i, j) << " "; + } + str_stream << std::endl; } - str_stream << std::endl; - } - return; + return; } template -Teuchos::RCP> -sort_data( - const SC* unsorted_data_ptr, - Teuchos::RCP> unsorted_map, - size_t dim1, - size_t num_global_unique_elements, - MPI_Comm comm, - Teuchos::RCP> &sorting_importer) +Teuchos::RCP> sort_data( + const SC* unsorted_data_ptr, + Teuchos::RCP> unsorted_map, + size_t dim1, + size_t num_global_unique_elements, + MPI_Comm comm, + Teuchos::RCP>& sorting_importer) { - - // create view of unsorted_data_ptr. not using matar here. array_layout is needed - Kokkos::View + // create view of unsorted_data_ptr. not using matar here. array_layout is needed + Kokkos::View unsorted_data(unsorted_data_ptr, unsorted_map->getLocalNumElements(), dim1); - // sorted_map - Teuchos::RCP> sorted_map = - Teuchos::rcp(new Tpetra::Map( + // sorted_map + Teuchos::RCP> sorted_map = + Teuchos::rcp(new Tpetra::Map( num_global_unique_elements, unsorted_map->getIndexBase(), unsorted_map->getComm())); - // sorted storage - Teuchos::RCP> sorted_storage = - Teuchos::rcp(new Tpetra::MultiVector(sorted_map, dim1)); + // sorted storage + Teuchos::RCP> sorted_storage = + Teuchos::rcp(new Tpetra::MultiVector(sorted_map, dim1)); - // unsorted storage - Teuchos::RCP> unsorted_storage = - Teuchos::rcp(new Tpetra::MultiVector(unsorted_map, dim1)); + // unsorted storage + Teuchos::RCP> unsorted_storage = + Teuchos::rcp(new Tpetra::MultiVector(unsorted_map, dim1)); - { //view scope - auto host_view = unsorted_storage->getLocalViewHost(Tpetra::Access::ReadWrite); - for(int i = 0; i < unsorted_map->getLocalNumElements(); i++) { - for (int j = 0; j < dim1; j++) { - host_view(i,j) = unsorted_data(i,j); - } - } // for - } //end view scope + { // view scope + auto host_view = unsorted_storage->getLocalViewHost(Tpetra::Access::ReadWrite); + for (int i = 0; i < unsorted_map->getLocalNumElements(); i++) + { + for (int j = 0; j < dim1; j++) + { + host_view(i, j) = unsorted_data(i, j); + } + } // for + } // end view scope - sorted_storage->doImport(*unsorted_storage, *sorting_importer, Tpetra::INSERT); + sorted_storage->doImport(*unsorted_storage, *sorting_importer, Tpetra::INSERT); - return sorted_storage; + return sorted_storage; } template -void -sort_and_write_data_to_file_mpi_all( - const SC* unsorted_data_ptr, - Teuchos::RCP> unsorted_map, - size_t dim1, - size_t num_global_unique_elements, - MPI_Comm comm, - MPI_File file_parallel, - Teuchos::RCP> &sorting_importer) +void sort_and_write_data_to_file_mpi_all( + const SC* unsorted_data_ptr, + Teuchos::RCP> unsorted_map, + size_t dim1, + size_t num_global_unique_elements, + MPI_Comm comm, + MPI_File file_parallel, + Teuchos::RCP>& sorting_importer) { - - auto sorted_data = - sort_data (unsorted_data_ptr, unsorted_map, dim1, num_global_unique_elements, comm, sorting_importer); - - { //view scope - auto const_host_view = sorted_data->getLocalViewHost(Tpetra::Access::ReadOnly); - std::stringstream str_stream; - write_data_to_string_stream (const_host_view.data(), sorted_data->getMap()->getLocalNumElements(), dim1, str_stream); - write_string_stream_to_file_mpi_all(str_stream, sorted_data->getMap()->getGlobalElement(0), file_parallel, comm); - } //end view scope + auto sorted_data = + sort_data( + unsorted_data_ptr, + unsorted_map, + dim1, + num_global_unique_elements, + comm, + sorting_importer); + + { // view scope + auto const_host_view = sorted_data->getLocalViewHost(Tpetra::Access::ReadOnly); + std::stringstream str_stream; + write_data_to_string_stream( + const_host_view.data(), + sorted_data->getMap()->getLocalNumElements(), + dim1, + str_stream); + write_string_stream_to_file_mpi_all( + str_stream, + sorted_data->getMap()->getGlobalElement(0), + file_parallel, + comm); + } // end view scope } -Teuchos::RCP> -get_cell_nodes( - const CArray & nodes_in_elem, - size_t num_dim, - Solver::node_ordering_convention active_node_ordering_convention) +Teuchos::RCP> get_cell_nodes( + const CArray& nodes_in_elem, + size_t num_dim, + Solver::node_ordering_convention active_node_ordering_convention) { - CArray convert_ijk_to_ensight(nodes_in_elem.dims(1)); - convert_ijk_to_ensight(0) = 0; - convert_ijk_to_ensight(1) = 1; - convert_ijk_to_ensight(2) = 3; - convert_ijk_to_ensight(3) = 2; - if(num_dim==3){ - convert_ijk_to_ensight(4) = 4; - convert_ijk_to_ensight(5) = 5; - convert_ijk_to_ensight(6) = 7; - convert_ijk_to_ensight(7) = 6; - } - - Teuchos::RCP> cell_data = - Teuchos::rcp( new CArray(nodes_in_elem.dims(0), nodes_in_elem.dims(1)+1) ); - - size_t temp_convert; - for (size_t ielem = 0; ielem < nodes_in_elem.dims(0); ielem++) { - (*cell_data)(ielem,0) = 8; - for (int ii = 0; ii < nodes_in_elem.dims(1); ii++) { - if(active_node_ordering_convention == Solver::node_ordering_convention::IJK) - temp_convert = convert_ijk_to_ensight(ii); - else - temp_convert = ii; - (*cell_data)(ielem,ii+1) = nodes_in_elem(ielem,temp_convert); - //(*cell_data)(ielem,ii+1) = nodes_in_elem(ielem,ii); - } - } - - return cell_data; -} - - -Teuchos::RCP> -calculate_elem_speed( - const Teuchos::RCP all_node_velocities_distributed, - const Teuchos::RCP global_nodes_in_elem_distributed, - const Teuchos::RCP ghost_node_velocities_distributed, - const Teuchos::RCP> ghost_importer) -{ - - size_t rnum_elems = global_nodes_in_elem_distributed->getLocalLength(); - size_t num_nodes_in_elem = global_nodes_in_elem_distributed->getNumVectors(); - size_t num_dims = all_node_velocities_distributed->getNumVectors(); - - Teuchos::RCP> elem_speed = - Teuchos::rcp(new CArray(rnum_elems)); - - auto nodes_in_elem_hview = global_nodes_in_elem_distributed->getLocalViewHost(Tpetra::Access::ReadOnly); - auto all_node_vel_hview = all_node_velocities_distributed->getLocalViewHost(Tpetra::Access::ReadOnly); - auto vector_map = all_node_velocities_distributed->getMap(); - for (size_t elem_gid = 0; elem_gid < rnum_elems; elem_gid++) { - double elem_vel[3]; - elem_vel[0] = 0.0; - elem_vel[1] = 0.0; - elem_vel[2] = 0.0; - // get the coordinates of the element center - for (int node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - size_t inode = vector_map->getLocalElement(nodes_in_elem_hview(elem_gid, node_lid)); - elem_vel[0] += all_node_vel_hview(inode, 0); - elem_vel[1] += all_node_vel_hview(inode, 1); - if (num_dims == 3){ - elem_vel[2] += all_node_vel_hview(inode, 2); - } - else { - elem_vel[2] = 0.0; - } - } // end loop over nodes in element + CArray convert_ijk_to_ensight(nodes_in_elem.dims(1)); + convert_ijk_to_ensight(0) = 0; + convert_ijk_to_ensight(1) = 1; + convert_ijk_to_ensight(2) = 3; + convert_ijk_to_ensight(3) = 2; + if (num_dim == 3) + { + convert_ijk_to_ensight(4) = 4; + convert_ijk_to_ensight(5) = 5; + convert_ijk_to_ensight(6) = 7; + convert_ijk_to_ensight(7) = 6; + } - elem_vel[0] = elem_vel[0]/num_nodes_in_elem; - elem_vel[1] = elem_vel[1]/num_nodes_in_elem; - elem_vel[2] = elem_vel[2]/num_nodes_in_elem; + Teuchos::RCP> cell_data = + Teuchos::rcp(new CArray(nodes_in_elem.dims(0), nodes_in_elem.dims(1) + 1) ); - double speed_sqrd = 0.0; - for (int dim=0; dim> calculate_elem_speed( + const Teuchos::RCP all_node_velocities_distributed, + const Teuchos::RCP global_nodes_in_elem_distributed, + const Teuchos::RCP ghost_node_velocities_distributed, + const Teuchos::RCP> ghost_importer) +{ + size_t rnum_elems = global_nodes_in_elem_distributed->getLocalLength(); + size_t num_nodes_in_elem = global_nodes_in_elem_distributed->getNumVectors(); + size_t num_dims = all_node_velocities_distributed->getNumVectors(); -} + Teuchos::RCP> elem_speed = + Teuchos::rcp(new CArray(rnum_elems)); + auto nodes_in_elem_hview = global_nodes_in_elem_distributed->getLocalViewHost(Tpetra::Access::ReadOnly); + auto all_node_vel_hview = all_node_velocities_distributed->getLocalViewHost(Tpetra::Access::ReadOnly); + auto vector_map = all_node_velocities_distributed->getMap(); + for (size_t elem_gid = 0; elem_gid < rnum_elems; elem_gid++) + { + double elem_vel[3]; + elem_vel[0] = 0.0; + elem_vel[1] = 0.0; + elem_vel[2] = 0.0; + // get the coordinates of the element center + for (int node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + size_t inode = vector_map->getLocalElement(nodes_in_elem_hview(elem_gid, node_lid)); + elem_vel[0] += all_node_vel_hview(inode, 0); + elem_vel[1] += all_node_vel_hview(inode, 1); + if (num_dims == 3) + { + elem_vel[2] += all_node_vel_hview(inode, 2); + } + else + { + elem_vel[2] = 0.0; + } + } // end loop over nodes in element + + elem_vel[0] = elem_vel[0] / num_nodes_in_elem; + elem_vel[1] = elem_vel[1] / num_nodes_in_elem; + elem_vel[2] = elem_vel[2] / num_nodes_in_elem; + + double speed_sqrd = 0.0; + for (int dim = 0; dim < num_dims; dim++) + { + speed_sqrd += elem_vel[dim] * elem_vel[dim]; + } + + (*elem_speed)(elem_gid) = sqrt(speed_sqrd); + } // end for + + return elem_speed; +} -Teuchos::RCP> -calculate_elem_switch( - Teuchos::RCP> all_element_map) +Teuchos::RCP> calculate_elem_switch( + Teuchos::RCP> all_element_map) { - Teuchos::RCP> elem_switch = - Teuchos::rcp(new CArray(all_element_map->getLocalNumElements())); + Teuchos::RCP> elem_switch = + Teuchos::rcp(new CArray(all_element_map->getLocalNumElements())); - for (size_t ielem = 0; ielem < elem_switch->dims(0); ielem++) { - (*elem_switch)(ielem) = all_element_map->getGlobalElement(ielem) % 2; - } - return elem_switch; + for (size_t ielem = 0; ielem < elem_switch->dims(0); ielem++) + { + (*elem_switch)(ielem) = all_element_map->getGlobalElement(ielem) % 2; + } + return elem_switch; } -Teuchos::RCP> -get_elem_proc_id( - Teuchos::RCP> all_element_map, - size_t myrank) +Teuchos::RCP> get_elem_proc_id( + Teuchos::RCP> all_element_map, + size_t myrank) { - Teuchos::RCP> elem_proc_id = - Teuchos::rcp(new CArray(all_element_map->getLocalNumElements())); + Teuchos::RCP> elem_proc_id = + Teuchos::rcp(new CArray(all_element_map->getLocalNumElements())); - for (size_t ielem = 0; ielem < elem_proc_id->dims(0); ielem++) { - (*elem_proc_id)(ielem) = myrank; - } - return elem_proc_id; + for (size_t ielem = 0; ielem < elem_proc_id->dims(0); ielem++) + { + (*elem_proc_id)(ielem) = myrank; + } + return elem_proc_id; } -Teuchos::RCP> -get_elem_gid( - Teuchos::RCP> all_element_map) +Teuchos::RCP> get_elem_gid( + Teuchos::RCP> all_element_map) { - Teuchos::RCP> elem_gid = - Teuchos::rcp(new CArray(all_element_map->getLocalNumElements())); + Teuchos::RCP> elem_gid = + Teuchos::rcp(new CArray(all_element_map->getLocalNumElements())); - for (size_t ielem = 0; ielem < elem_gid->dims(0); ielem++) { - (*elem_gid)(ielem) = all_element_map->getGlobalElement(ielem); - } - return elem_gid; + for (size_t ielem = 0; ielem < elem_gid->dims(0); ielem++) + { + (*elem_gid)(ielem) = all_element_map->getGlobalElement(ielem); + } + return elem_gid; } -Teuchos::RCP> -get_design_density( - size_t rnum_nodes, - bool topology_optimization_on, - const Teuchos::RCP design_node_densities_distributed) +Teuchos::RCP> get_design_density( + size_t rnum_nodes, + bool topology_optimization_on, + const Teuchos::RCP design_node_densities_distributed) { - Teuchos::RCP> design_density = - Teuchos::rcp(new CArray(rnum_nodes)); + Teuchos::RCP> design_density = + Teuchos::rcp(new CArray(rnum_nodes)); - if(topology_optimization_on) { - auto host_view = design_node_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); - for (size_t inode = 0; inode < rnum_nodes; inode++) { - (*design_density)(inode) = host_view(inode,0); + if (topology_optimization_on) + { + auto host_view = design_node_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); + for (size_t inode = 0; inode < rnum_nodes; inode++) + { + (*design_density)(inode) = host_view(inode, 0); + } } - } - else { - for (size_t inode = 0; inode < rnum_nodes; inode++) { - (*design_density)(inode) = 1.0; + else + { + for (size_t inode = 0; inode < rnum_nodes; inode++) + { + (*design_density)(inode) = 1.0; + } } - } - return design_density; + return design_density; } - diff --git a/src/Parallel-Solvers/Solver.cpp b/src/Parallel-Solvers/Solver.cpp index 56eff2302..4791db610 100644 --- a/src/Parallel-Solvers/Solver.cpp +++ b/src/Parallel-Solvers/Solver.cpp @@ -11,14 +11,14 @@ This program is open source under the BSD-3 License. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - + 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - + 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - + 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. @@ -41,7 +41,7 @@ #include #include #include -#include +#include #include // fmin, fmax, abs note: fminl is long #include #include @@ -71,13 +71,13 @@ #include "MeshBuilder.h" #include "MeshIO.h" -//Repartition Package +// Repartition Package #include #include #include #include -//debug and performance includes +// debug and performance includes #include #include #include @@ -89,2508 +89,2911 @@ #define DENSITY_EPSILON 0.0001 #define BC_EPSILON 1.0e-8 -Solver::Solver(Simulation_Parameters& _simparam) : simparam(_simparam) { - //default flags assume optional routines are off - setup_flag = finalize_flag = 0; - communication_time = dev2host_time = host2dev_time = output_time = 0; - last_print_step = -1; +Solver::Solver(Simulation_Parameters& _simparam) : simparam(_simparam) +{ + // default flags assume optional routines are off + setup_flag = finalize_flag = 0; + communication_time = dev2host_time = host2dev_time = output_time = 0; + last_print_step = -1; + + // FEA module data init + nfea_modules = 0; + displacement_module = -1; +} - //FEA module data init - nfea_modules = 0; - displacement_module = -1; +void Solver::exit_solver(int status) +{ + Kokkos::finalize(); + MPI_Barrier(MPI_COMM_WORLD); + MPI_Finalize(); + exit(status); } -void Solver::exit_solver(int status){ - Kokkos::finalize(); - MPI_Barrier(MPI_COMM_WORLD); - MPI_Finalize(); - exit(status); +Solver::~Solver() +{ + // destroy FEA modules + for (int imodule = 0; imodule < nfea_modules; imodule++) + { + delete fea_modules[imodule]; + } } -Solver::~Solver(){ - //destroy FEA modules - for(int imodule = 0; imodule < nfea_modules; imodule++){ - delete fea_modules[imodule]; - } +namespace elements +{ +namespace elem_types +{ +elem_type from_vtk(const int& vtk_elem) +{ + switch (vtk_elem) + { + case 9: + return elem_type::Quad4; + case 12: + return elem_type::Hex8; + case 70: + return elem_type::QuadN; + case 72: + return elem_type::HexN; + default: + throw std::runtime_error("Unsupported vtk element type: " + std::to_string(vtk_elem)); + } } +} // namespace elem_types +} // namespace elements +void Solver::generate_mesh(const std::shared_ptr& mesh_generation_options) +{ + auto mesh = MeshBuilder::build_mesh(mesh_generation_options); + switch (active_node_ordering_convention) + { + case ENSIGHT: + MeshIO::_Impl::reorder_columns(mesh.element_point_index, MeshIO::_Impl::ijk_to_fea().data()); + break; + case IJK: + // Already in IJK + break; + } -namespace elements { - namespace elem_types { - elem_type from_vtk(const int& vtk_elem) { - switch (vtk_elem) { - case 9: - return elem_type::Quad4; - case 12: - return elem_type::Hex8; - case 70: - return elem_type::QuadN; - case 72: - return elem_type::HexN; - default: - throw std::runtime_error("Unsupported vtk element type: " + std::to_string(vtk_elem)); + num_nodes = mesh.points.dims(0); + num_elem = mesh.element_point_index.dims(0); + map = Teuchos::rcp(new Tpetra::Map(num_nodes, 0, comm)); + + nlocal_nodes = map->getLocalNumElements(); + + node_coords_distributed = Teuchos::rcp(new MV(map, mesh.points.dims(1))); + { + host_vec_array node_coords = node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + for (long long int i = 0; i < num_nodes; i++) + { + // set global node id (ensight specific order) + // let map decide if this node id belongs locally; if yes store data + if (map->isNodeGlobalElement(i)) + { + // set local node index in this mpi rank + long long int node_rid = map->getLocalElement(i); + for (long long int j = 0; j < mesh.points.dims(1); j++) + { + node_coords(node_rid, j) = mesh.points(i, j); + } + } } } - } -} -void Solver::generate_mesh(const std::shared_ptr& mesh_generation_options) { - auto mesh = MeshBuilder::build_mesh(mesh_generation_options); - switch (active_node_ordering_convention) { - case ENSIGHT: - MeshIO::_Impl::reorder_columns(mesh.element_point_index, MeshIO::_Impl::ijk_to_fea().data()); - break; - case IJK: - // Already in IJK - break; - } - - num_nodes = mesh.points.dims(0); - num_elem = mesh.element_point_index.dims(0); - map = Teuchos::rcp( new Tpetra::Map(num_nodes, 0, comm)); - - nlocal_nodes = map->getLocalNumElements(); - - node_coords_distributed = Teuchos::rcp(new MV(map, mesh.points.dims(1))); - { - host_vec_array node_coords = node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); - for(long long int i = 0; i < num_nodes; i++){ - //set global node id (ensight specific order) - //let map decide if this node id belongs locally; if yes store data - if(map->isNodeGlobalElement(i)){ - //set local node index in this mpi rank - long long int node_rid = map->getLocalElement(i); - for (long long int j = 0; j < mesh.points.dims(1); j++) - node_coords(node_rid, j) = mesh.points(i, j); - } + repartition_nodes(); + + max_nodes_per_element = mesh.element_point_index.dims(1); + + // Figure out which elements belong to me. + std::vector global_indices_temp; + for (size_t i = 0; i < mesh.element_point_index.dims(0); i++) + { + for (size_t j = 0; j < mesh.element_point_index.dims(1); j++) + { + if (map->isNodeGlobalElement(mesh.element_point_index(i, j))) + { + global_indices_temp.push_back(i); + break; + } + } } - } - repartition_nodes(); - - max_nodes_per_element = mesh.element_point_index.dims(1); - - // Figure out which elements belong to me. - std::vector global_indices_temp; - for (size_t i = 0; i < mesh.element_point_index.dims(0); i++) { - for (size_t j = 0; j < mesh.element_point_index.dims(1); j++) { - if (map->isNodeGlobalElement(mesh.element_point_index(i, j))) { - global_indices_temp.push_back(i); + + rnum_elem = global_indices_temp.size(); + + Element_Types = CArrayKokkos(global_indices_temp.size()); + auto element_type = elements::elem_types::from_vtk(mesh.element_types(0)); // Not from elements. From up there ^ + switch (mesh.points.dims(1)) + { + case 2: + switch (element_type) + { + case elements::elem_types::elem_type::Quad4: + max_nodes_per_patch = 2; + break; + default: + throw std::runtime_error("Higher order meshes are unsupported."); + } + break; + case 3: + switch (element_type) + { + case elements::elem_types::elem_type::Hex8: + max_nodes_per_patch = 4; + break; + default: + throw std::runtime_error("Higher order meshes are unsupported."); + } break; - } } - } - rnum_elem = global_indices_temp.size(); + for (size_t i = 0; i < global_indices_temp.size(); i++) + { + Element_Types(i) = element_type; + } - Element_Types = CArrayKokkos(global_indices_temp.size()); - auto element_type = elements::elem_types::from_vtk(mesh.element_types(0)); // Not from elements. From up there ^ - switch (mesh.points.dims(1)) { - case 2: - switch (element_type) { - case elements::elem_types::elem_type::Quad4: - max_nodes_per_patch = 2; - break; - default: - throw std::runtime_error("Higher order meshes are unsupported."); - } - break; - case 3: - switch (element_type) { - case elements::elem_types::elem_type::Hex8: - max_nodes_per_patch = 4; - break; - default: - throw std::runtime_error("Higher order meshes are unsupported."); - } - break; - } - - for (size_t i = 0; i < global_indices_temp.size(); i++) - Element_Types(i) = element_type; - - //copy temporary element storage to multivector storage - dual_nodes_in_elem = dual_elem_conn_array("dual_nodes_in_elem", global_indices_temp.size(), mesh.element_point_index.dims(1)); - host_elem_conn_array nodes_in_elem = dual_nodes_in_elem.view_host(); - dual_nodes_in_elem.modify_host(); - - for (size_t i = 0; i < global_indices_temp.size(); i++) - for (size_t j = 0; j < mesh.element_point_index.dims(1); j++) - nodes_in_elem(i, j) = mesh.element_point_index(global_indices_temp[i], j); - - //view storage for all local elements connected to local nodes on this rank - Kokkos::DualView All_Element_Global_Indices("All_Element_Global_Indices", global_indices_temp.size()); - //copy temporary global indices storage to view storage - for(int i = 0; i < global_indices_temp.size(); i++) - All_Element_Global_Indices.h_view(i) = global_indices_temp[i]; - - //construct overlapping element map (since different ranks can own the same elements due to the local node map) - All_Element_Global_Indices.modify_host(); - All_Element_Global_Indices.sync_device(); - - all_element_map = Teuchos::rcp( new Tpetra::Map(Teuchos::OrdinalTraits::invalid(), All_Element_Global_Indices.d_view, 0, comm)); + // copy temporary element storage to multivector storage + dual_nodes_in_elem = dual_elem_conn_array("dual_nodes_in_elem", global_indices_temp.size(), mesh.element_point_index.dims(1)); + host_elem_conn_array nodes_in_elem = dual_nodes_in_elem.view_host(); + dual_nodes_in_elem.modify_host(); + + for (size_t i = 0; i < global_indices_temp.size(); i++) + { + for (size_t j = 0; j < mesh.element_point_index.dims(1); j++) + { + nodes_in_elem(i, j) = mesh.element_point_index(global_indices_temp[i], j); + } + } + + // view storage for all local elements connected to local nodes on this rank + Kokkos::DualView All_Element_Global_Indices("All_Element_Global_Indices", global_indices_temp.size()); + // copy temporary global indices storage to view storage + for (int i = 0; i < global_indices_temp.size(); i++) + { + All_Element_Global_Indices.h_view(i) = global_indices_temp[i]; + } + + // construct overlapping element map (since different ranks can own the same elements due to the local node map) + All_Element_Global_Indices.modify_host(); + All_Element_Global_Indices.sync_device(); + + all_element_map = Teuchos::rcp(new Tpetra::Map(Teuchos::OrdinalTraits::invalid(), All_Element_Global_Indices.d_view, 0, comm)); } /* ---------------------------------------------------------------------- Read Ensight format mesh file ------------------------------------------------------------------------- */ -void Solver::read_mesh_ensight(const char *MESH){ - - char ch; - int num_dim = simparam.num_dims; - Input_Options input_options = simparam.input_options.value(); - int p_order = input_options.p_order; - real_t unit_scaling = input_options.unit_scaling; - int local_node_index, current_column_index; - size_t strain_count; - std::string skip_line, read_line, substring; - std::stringstream line_parse; - CArrayKokkos read_buffer; - int buffer_loop, buffer_iteration, buffer_iterations, dof_limit, scan_loop; - size_t read_index_start, node_rid, elem_gid; - GO node_gid; - real_t dof_value; - bool zero_index_base = input_options.zero_index_base; - int negative_index_found = 0; - int global_negative_index_found = 0; - //Nodes_Per_Element_Type = elements::elem_types::Nodes_Per_Element_Type; - - //read the mesh - //PLACEHOLDER: ensight_format(MESH); - // abaqus_format(MESH); - // vtk_format(MESH) - - //task 0 reads file - if(myrank==0){ - std::cout << " NUM DIM is " << num_dim << std::endl; - in = new std::ifstream(); - in->open(MESH); - if (!(*in)) throw std::runtime_error(std::string("Can't open ") + MESH); - //skip 8 lines - for (int j = 1; j <= 8; j++) { - getline(*in, skip_line); - std::cout << skip_line << std::endl; - } //for - } - - // --- Read the number of nodes in the mesh --- // - if(myrank==0){ - getline(*in, read_line); - line_parse.str(read_line); - line_parse >> num_nodes; - std::cout << "declared node count: " << num_nodes << std::endl; - } - - //broadcast number of nodes - MPI_Bcast(&num_nodes,1,MPI_LONG_LONG_INT,0,world); - - //construct contiguous parallel row map now that we know the number of nodes - map = Teuchos::rcp( new Tpetra::Map(num_nodes,0,comm)); - //map->describe(*fos,Teuchos::VERB_EXTREME); - // set the vertices in the mesh read in - nlocal_nodes = map->getLocalNumElements(); - //populate local row offset data from global data - global_size_t min_gid = map->getMinGlobalIndex(); - global_size_t max_gid = map->getMaxGlobalIndex(); - global_size_t index_base = map->getIndexBase(); - //debug print - //std::cout << "local node count on task: " << " " << nlocal_nodes << std::endl; - - //allocate node storage with dual view - //dual_node_coords = dual_vec_array("dual_node_coords", nlocal_nodes,num_dim); - - //local variable for host view in the dual view - - node_coords_distributed = Teuchos::rcp(new MV(map, num_dim)); - - //scope ensures view is destroyed for now to avoid calling a device view with an active host view later - { - host_vec_array node_coords = node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); - //host_vec_array node_coords = dual_node_coords.view_host(); - //notify that the host view is going to be modified in the file readin - //dual_node_coords.modify_host(); - - //old swage method - //mesh->init_nodes(local_nrows); // add 1 for index starting at 1 - - std::cout << "Num nodes assigned to task " << myrank << " = " << nlocal_nodes << std::endl; - - // read the initial mesh coordinates - // x-coords - /*only task 0 reads in nodes and elements from the input file - stores node data in a buffer and communicates once the buffer cap is reached - or the data ends*/ - - words_per_line = input_options.words_per_line; - elem_words_per_line = input_options.elem_words_per_line; - - //allocate read buffer - read_buffer = CArrayKokkos(BUFFER_LINES,words_per_line,MAX_WORD); - - dof_limit = num_nodes; - buffer_iterations = dof_limit/BUFFER_LINES; - if(dof_limit%BUFFER_LINES!=0) buffer_iterations++; - - //x-coords - read_index_start = 0; - for(buffer_iteration = 0; buffer_iteration < buffer_iterations; buffer_iteration++){ - //pack buffer on rank 0 - if(myrank==0&&buffer_iteration> substring; - //debug print - //std::cout<<" "<< substring < read_buffer; + + GO node_gid; + real_t dof_value; + real_t unit_scaling = input_options.unit_scaling; + + // Nodes_Per_Element_Type = elements::elem_types::Nodes_Per_Element_Type; + + // read the mesh + // PLACEHOLDER: ensight_format(MESH); + // abaqus_format(MESH); + // vtk_format(MESH) + + // task 0 reads file + if (myrank == 0) + { + std::cout << " NUM DIM is " << num_dim << std::endl; + in = new std::ifstream(); + in->open(MESH); + if (!(*in)) + { + throw std::runtime_error(std::string("Can't open ") + MESH); } - } - } - else if(myrank==0){ - buffer_loop=0; - while(buffer_iteration*BUFFER_LINES+buffer_loop < num_nodes) { - getline(*in,read_line); - line_parse.clear(); + // skip 8 lines + for (int j = 1; j <= 8; j++) + { + getline(*in, skip_line); + std::cout << skip_line << std::endl; + } // for + } + + // --- Read the number of nodes in the mesh --- // + if (myrank == 0) + { + getline(*in, read_line); line_parse.str(read_line); - for(int iword = 0; iword < words_per_line; iword++){ - //read portions of the line into the substring variable - line_parse >> substring; - //assign the substring variable as a word of the read buffer - strcpy(&read_buffer(buffer_loop,iword,0),substring.c_str()); - } - buffer_loop++; - } - - } - - //broadcast buffer to all ranks; each rank will determine which nodes in the buffer belong - MPI_Bcast(read_buffer.pointer(),BUFFER_LINES*words_per_line*MAX_WORD,MPI_CHAR,0,world); - //broadcast how many nodes were read into this buffer iteration - MPI_Bcast(&buffer_loop,1,MPI_INT,0,world); - - //debug_print - //std::cout << "NODE BUFFER LOOP IS: " << buffer_loop << std::endl; - //for(int iprint=0; iprint < buffer_loop; iprint++) - //std::cout<<"buffer packing: " << std::string(&read_buffer(iprint,0,0)) << std::endl; - //return; - - //determine which data to store in the swage mesh members (the local node data) - //loop through read buffer - for(scan_loop = 0; scan_loop < buffer_loop; scan_loop++){ - //set global node id (ensight specific order) - node_gid = read_index_start + scan_loop; - //let map decide if this node id belongs locally; if yes store data - if(map->isNodeGlobalElement(node_gid)){ - //set local node index in this mpi rank - node_rid = map->getLocalElement(node_gid); - //extract nodal position from the read buffer - //for ensight format this is just one coordinate per line - dof_value = atof(&read_buffer(scan_loop,0,0)); - node_coords(node_rid, 0) = dof_value * unit_scaling; - } + line_parse >> num_nodes; + std::cout << "declared node count: " << num_nodes << std::endl; } - read_index_start+=BUFFER_LINES; - } - - - // y-coords - read_index_start = 0; - for(buffer_iteration = 0; buffer_iteration < buffer_iterations; buffer_iteration++){ - - //pack buffer on rank 0 - if(myrank==0&&buffer_iteration> substring; - //assign the substring variable as a word of the read buffer - strcpy(&read_buffer(buffer_loop,iword,0),substring.c_str()); + + // broadcast number of nodes + MPI_Bcast(&num_nodes, 1, MPI_LONG_LONG_INT, 0, world); + + // construct contiguous parallel row map now that we know the number of nodes + map = Teuchos::rcp(new Tpetra::Map(num_nodes, 0, comm)); + // map->describe(*fos,Teuchos::VERB_EXTREME); + // set the vertices in the mesh read in + nlocal_nodes = map->getLocalNumElements(); + // populate local row offset data from global data + global_size_t min_gid = map->getMinGlobalIndex(); + global_size_t max_gid = map->getMaxGlobalIndex(); + global_size_t index_base = map->getIndexBase(); + // debug print + // std::cout << "local node count on task: " << " " << nlocal_nodes << std::endl; + + // allocate node storage with dual view + // dual_node_coords = dual_vec_array("dual_node_coords", nlocal_nodes,num_dim); + + // local variable for host view in the dual view + + node_coords_distributed = Teuchos::rcp(new MV(map, num_dim)); + + // scope ensures view is destroyed for now to avoid calling a device view with an active host view later + { + host_vec_array node_coords = node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + // host_vec_array node_coords = dual_node_coords.view_host(); + // notify that the host view is going to be modified in the file readin + // dual_node_coords.modify_host(); + + // old swage method + // mesh->init_nodes(local_nrows); // add 1 for index starting at 1 + + std::cout << "Num nodes assigned to task " << myrank << " = " << nlocal_nodes << std::endl; + + // read the initial mesh coordinates + // x-coords + /*only task 0 reads in nodes and elements from the input file + stores node data in a buffer and communicates once the buffer cap is reached + or the data ends*/ + + words_per_line = input_options.words_per_line; + elem_words_per_line = input_options.elem_words_per_line; + + // allocate read buffer + read_buffer = CArrayKokkos(BUFFER_LINES, words_per_line, MAX_WORD); + + dof_limit = num_nodes; + buffer_iterations = dof_limit / BUFFER_LINES; + if (dof_limit % BUFFER_LINES != 0) + { + buffer_iterations++; } + + // x-coords + read_index_start = 0; + for (buffer_iteration = 0; buffer_iteration < buffer_iterations; buffer_iteration++) + { + // pack buffer on rank 0 + if (myrank == 0 && buffer_iteration < buffer_iterations - 1) + { + for (buffer_loop = 0; buffer_loop < BUFFER_LINES; buffer_loop++) + { + getline(*in, read_line); + line_parse.clear(); + line_parse.str(read_line); + + for (int iword = 0; iword < words_per_line; iword++) + { + // read portions of the line into the substring variable + line_parse >> substring; + // debug print + // std::cout<<" "<< substring <> substring; + // assign the substring variable as a word of the read buffer + strcpy(&read_buffer(buffer_loop, iword, 0), substring.c_str()); + } + buffer_loop++; + } + } + + // broadcast buffer to all ranks; each rank will determine which nodes in the buffer belong + MPI_Bcast(read_buffer.pointer(), BUFFER_LINES * words_per_line * MAX_WORD, MPI_CHAR, 0, world); + // broadcast how many nodes were read into this buffer iteration + MPI_Bcast(&buffer_loop, 1, MPI_INT, 0, world); + + // debug_print + // std::cout << "NODE BUFFER LOOP IS: " << buffer_loop << std::endl; + // for(int iprint=0; iprint < buffer_loop; iprint++) + // std::cout<<"buffer packing: " << std::string(&read_buffer(iprint,0,0)) << std::endl; + // return; + + // determine which data to store in the swage mesh members (the local node data) + // loop through read buffer + for (scan_loop = 0; scan_loop < buffer_loop; scan_loop++) + { + // set global node id (ensight specific order) + node_gid = read_index_start + scan_loop; + // let map decide if this node id belongs locally; if yes store data + if (map->isNodeGlobalElement(node_gid)) + { + // set local node index in this mpi rank + node_rid = map->getLocalElement(node_gid); + // extract nodal position from the read buffer + // for ensight format this is just one coordinate per line + dof_value = atof(&read_buffer(scan_loop, 0, 0)); + node_coords(node_rid, 0) = dof_value * unit_scaling; + } + } + read_index_start += BUFFER_LINES; + } + + // y-coords + read_index_start = 0; + for (buffer_iteration = 0; buffer_iteration < buffer_iterations; buffer_iteration++) + { + // pack buffer on rank 0 + if (myrank == 0 && buffer_iteration < buffer_iterations - 1) + { + for (buffer_loop = 0; buffer_loop < BUFFER_LINES; buffer_loop++) + { + getline(*in, read_line); + line_parse.clear(); + line_parse.str(read_line); + for (int iword = 0; iword < words_per_line; iword++) + { + // read portions of the line into the substring variable + line_parse >> substring; + // assign the substring variable as a word of the read buffer + strcpy(&read_buffer(buffer_loop, iword, 0), substring.c_str()); + } + } + } + else if (myrank == 0) + { + buffer_loop = 0; + while (buffer_iteration * BUFFER_LINES + buffer_loop < num_nodes) { + getline(*in, read_line); + line_parse.clear(); + line_parse.str(read_line); + for (int iword = 0; iword < words_per_line; iword++) + { + // read portions of the line into the substring variable + line_parse >> substring; + // assign the substring variable as a word of the read buffer + strcpy(&read_buffer(buffer_loop, iword, 0), substring.c_str()); + } + buffer_loop++; + // std::cout<<" "<< node_coords(node_gid, 0)<isNodeGlobalElement(node_gid)) + { + // set local node index in this mpi rank + node_rid = map->getLocalElement(node_gid); + // extract nodal position from the read buffer + // for ensight format this is just one coordinate per line + dof_value = atof(&read_buffer(scan_loop, 0, 0)); + node_coords(node_rid, 1) = dof_value * unit_scaling; + } + } + read_index_start += BUFFER_LINES; + } + + // z-coords + read_index_start = 0; + for (buffer_iteration = 0; buffer_iteration < buffer_iterations; buffer_iteration++) + { + // pack buffer on rank 0 + if (myrank == 0 && buffer_iteration < buffer_iterations - 1) + { + for (buffer_loop = 0; buffer_loop < BUFFER_LINES; buffer_loop++) + { + getline(*in, read_line); + line_parse.clear(); + line_parse.str(read_line); + for (int iword = 0; iword < words_per_line; iword++) + { + // read portions of the line into the substring variable + line_parse >> substring; + // assign the substring variable as a word of the read buffer + strcpy(&read_buffer(buffer_loop, iword, 0), substring.c_str()); + } + } + } + else if (myrank == 0) + { + buffer_loop = 0; + while (buffer_iteration * BUFFER_LINES + buffer_loop < num_nodes) { + getline(*in, read_line); + line_parse.clear(); + line_parse.str(read_line); + for (int iword = 0; iword < words_per_line; iword++) + { + // read portions of the line into the substring variable + line_parse >> substring; + // assign the substring variable as a word of the read buffer + strcpy(&read_buffer(buffer_loop, iword, 0), substring.c_str()); + } + buffer_loop++; + // std::cout<<" "<< node_coords(node_gid, 0)<isNodeGlobalElement(node_gid)) + { + // set local node index in this mpi rank + node_rid = map->getLocalElement(node_gid); + // extract nodal position from the read buffer + // for ensight format this is just one coordinate per line + dof_value = atof(&read_buffer(scan_loop, 0, 0)); + if (num_dim == 3) + { + node_coords(node_rid, 2) = dof_value * unit_scaling; + } + } + } + read_index_start += BUFFER_LINES; + } + } // end active view scope + // repartition node distribution + repartition_nodes(); + + // synchronize device data + // dual_node_coords.sync_device(); + // dual_node_coords.modify_device(); + + // debug print of nodal data + + // debug print nodal positions and indices + /* + std::cout << " ------------NODAL POSITIONS ON TASK " << myrank << " --------------"<getGlobalElement(inode) + 1 << " { "; + for (int istride = 0; istride < num_dim; istride++){ + std::cout << node_coords(inode,istride) << " , "; } + std::cout << " }"<< std::endl; } - else if(myrank==0){ - buffer_loop=0; - while(buffer_iteration*BUFFER_LINES+buffer_loop < num_nodes) { - getline(*in,read_line); - line_parse.clear(); - line_parse.str(read_line); - for(int iword = 0; iword < words_per_line; iword++){ - //read portions of the line into the substring variable - line_parse >> substring; - //assign the substring variable as a word of the read buffer - strcpy(&read_buffer(buffer_loop,iword,0),substring.c_str()); - } - buffer_loop++; - //std::cout<<" "<< node_coords(node_gid, 0)<isNodeGlobalElement(node_gid)){ - //set local node index in this mpi rank - node_rid = map->getLocalElement(node_gid); - //extract nodal position from the read buffer - //for ensight format this is just one coordinate per line - dof_value = atof(&read_buffer(scan_loop,0,0)); - node_coords(node_rid, 1) = dof_value * unit_scaling; - } + // check that local assignments match global total + + // read in element info (ensight file format is organized in element type sections) + // loop over this later for several element type sections + + num_elem = 0; + rnum_elem = 0; + CArrayKokkos node_store(elem_words_per_line); + + if (myrank == 0) + { + // skip element type name line + getline(*in, skip_line); + std::cout << skip_line << std::endl; } - read_index_start+=BUFFER_LINES; - } - - // z-coords - read_index_start = 0; - for(buffer_iteration = 0; buffer_iteration < buffer_iterations; buffer_iteration++){ - - //pack buffer on rank 0 - if(myrank==0&&buffer_iteration> substring; - //assign the substring variable as a word of the read buffer - strcpy(&read_buffer(buffer_loop,iword,0),substring.c_str()); + line_parse >> num_elem; + std::cout << "declared element count: " << num_elem << std::endl; + if (num_elem <= 0) + { + std::cout << "ERROR, NO ELEMENTS IN MESH" << std::endl; } - } - } - else if(myrank==0){ - buffer_loop=0; - while(buffer_iteration*BUFFER_LINES+buffer_loop < num_nodes) { - getline(*in,read_line); - line_parse.clear(); - line_parse.str(read_line); - for(int iword = 0; iword < words_per_line; iword++){ - //read portions of the line into the substring variable - line_parse >> substring; - //assign the substring variable as a word of the read buffer - strcpy(&read_buffer(buffer_loop,iword,0),substring.c_str()); - } - buffer_loop++; - //std::cout<<" "<< node_coords(node_gid, 0)<isNodeGlobalElement(node_gid)){ - //set local node index in this mpi rank - node_rid = map->getLocalElement(node_gid); - //extract nodal position from the read buffer - //for ensight format this is just one coordinate per line - dof_value = atof(&read_buffer(scan_loop,0,0)); - if(num_dim==3) - node_coords(node_rid, 2) = dof_value * unit_scaling; - } - } - read_index_start+=BUFFER_LINES; - } - } //end active view scope - //repartition node distribution - repartition_nodes(); - - //synchronize device data - //dual_node_coords.sync_device(); - //dual_node_coords.modify_device(); - - //debug print of nodal data - - //debug print nodal positions and indices - /* - std::cout << " ------------NODAL POSITIONS ON TASK " << myrank << " --------------"<getGlobalElement(inode) + 1 << " { "; - for (int istride = 0; istride < num_dim; istride++){ - std::cout << node_coords(inode,istride) << " , "; - } - std::cout << " }"<< std::endl; - } - */ - - //check that local assignments match global total - - - //read in element info (ensight file format is organized in element type sections) - //loop over this later for several element type sections - - num_elem = 0; - rnum_elem = 0; - CArrayKokkos node_store(elem_words_per_line); - - if(myrank==0){ - //skip element type name line - getline(*in, skip_line); - std::cout << skip_line << std::endl; - } - - // --- read the number of cells in the mesh --- - // --- Read the number of vertices in the mesh --- // - if(myrank==0){ - getline(*in, read_line); - line_parse.clear(); - line_parse.str(read_line); - line_parse >> num_elem; - std::cout << "declared element count: " << num_elem << std::endl; - if(num_elem <= 0) std::cout << "ERROR, NO ELEMENTS IN MESH" << std::endl; - } - - //broadcast number of elements - MPI_Bcast(&num_elem,1,MPI_LONG_LONG_INT,0,world); - - if(myrank == 0) - std::cout<<"before mesh initialization"<(BUFFER_LINES,elem_words_per_line,MAX_WORD); - - //calculate buffer iterations to read number of lines - buffer_iterations = num_elem/BUFFER_LINES; - int assign_flag; - - //dynamic buffer used to store elements before we know how many this rank needs - std::vector element_temp(BUFFER_LINES*elem_words_per_line); - std::vector global_indices_temp(BUFFER_LINES); - size_t buffer_max = BUFFER_LINES*elem_words_per_line; - size_t indices_buffer_max = BUFFER_LINES; - - if(num_elem%BUFFER_LINES!=0) buffer_iterations++; - read_index_start = 0; - //std::cout << "ELEMENT BUFFER ITERATIONS: " << buffer_iterations << std::endl; - rnum_elem = 0; - for(buffer_iteration = 0; buffer_iteration < buffer_iterations; buffer_iteration++){ - //pack buffer on rank 0 - if(myrank==0&&buffer_iteration> substring; - //assign the substring variable as a word of the read buffer - strcpy(&read_buffer(buffer_loop,iword,0),substring.c_str()); + // broadcast number of elements + MPI_Bcast(&num_elem, 1, MPI_LONG_LONG_INT, 0, world); + + if (myrank == 0) + { + std::cout << "before mesh initialization" << std::endl; + } + + // read in element connectivity + // we're gonna reallocate for the words per line expected for the element connectivity + read_buffer = CArrayKokkos(BUFFER_LINES, elem_words_per_line, MAX_WORD); + + // calculate buffer iterations to read number of lines + buffer_iterations = num_elem / BUFFER_LINES; + int assign_flag; + + // dynamic buffer used to store elements before we know how many this rank needs + std::vector element_temp(BUFFER_LINES * elem_words_per_line); + std::vector global_indices_temp(BUFFER_LINES); + size_t buffer_max = BUFFER_LINES * elem_words_per_line; + size_t indices_buffer_max = BUFFER_LINES; + + if (num_elem % BUFFER_LINES != 0) + { + buffer_iterations++; + } + read_index_start = 0; + // std::cout << "ELEMENT BUFFER ITERATIONS: " << buffer_iterations << std::endl; + rnum_elem = 0; + for (buffer_iteration = 0; buffer_iteration < buffer_iterations; buffer_iteration++) + { + // pack buffer on rank 0 + if (myrank == 0 && buffer_iteration < buffer_iterations - 1) + { + for (buffer_loop = 0; buffer_loop < BUFFER_LINES; buffer_loop++) + { + getline(*in, read_line); + line_parse.clear(); + line_parse.str(read_line); + for (int iword = 0; iword < elem_words_per_line; iword++) + { + // read portions of the line into the substring variable + line_parse >> substring; + // assign the substring variable as a word of the read buffer + strcpy(&read_buffer(buffer_loop, iword, 0), substring.c_str()); + } + } } - } + else if (myrank == 0) + { + buffer_loop = 0; + while (buffer_iteration * BUFFER_LINES + buffer_loop < num_elem) { + getline(*in, read_line); + line_parse.clear(); + line_parse.str(read_line); + for (int iword = 0; iword < elem_words_per_line; iword++) + { + // read portions of the line into the substring variable + line_parse >> substring; + // assign the substring variable as a word of the read buffer + strcpy(&read_buffer(buffer_loop, iword, 0), substring.c_str()); + } + buffer_loop++; + // std::cout<<" "<< node_coords(node_gid, 0)<isNodeGlobalElement(node_gid) && !assign_flag) + { + assign_flag = 1; + rnum_elem++; + } + } + else + { + if (map->isNodeGlobalElement(node_gid - 1) && !assign_flag) + { + assign_flag = 1; + rnum_elem++; + } + } + } + + if (assign_flag) + { + for (int inode = 0; inode < elem_words_per_line; inode++) + { + if ((rnum_elem - 1) * elem_words_per_line + inode >= buffer_max) + { + element_temp.resize((rnum_elem - 1) * elem_words_per_line + inode + BUFFER_LINES * elem_words_per_line); + buffer_max = (rnum_elem - 1) * elem_words_per_line + inode + BUFFER_LINES * elem_words_per_line; + } + element_temp[(rnum_elem - 1) * elem_words_per_line + inode] = node_store(inode); + // std::cout << "VECTOR STORAGE FOR ELEM " << rnum_elem << " ON TASK " << myrank << " NODE " << inode+1 << " IS " << node_store(inode) + 1 << std::endl; + } + // assign global element id to temporary list + if (rnum_elem - 1 >= indices_buffer_max) + { + global_indices_temp.resize(rnum_elem - 1 + BUFFER_LINES); + indices_buffer_max = rnum_elem - 1 + BUFFER_LINES; + } + global_indices_temp[rnum_elem - 1] = elem_gid; + } + } + read_index_start += BUFFER_LINES; } - else if(myrank==0){ - buffer_loop=0; - while(buffer_iteration*BUFFER_LINES+buffer_loop < num_elem) { - getline(*in,read_line); - line_parse.clear(); - line_parse.str(read_line); - for(int iword = 0; iword < elem_words_per_line; iword++){ - //read portions of the line into the substring variable - line_parse >> substring; - //assign the substring variable as a word of the read buffer - strcpy(&read_buffer(buffer_loop,iword,0),substring.c_str()); - } - buffer_loop++; - //std::cout<<" "<< node_coords(node_gid, 0)<close(); } - //broadcast buffer to all ranks; each rank will determine which nodes in the buffer belong - MPI_Bcast(read_buffer.pointer(),BUFFER_LINES*elem_words_per_line*MAX_WORD,MPI_CHAR,0,world); - //broadcast how many nodes were read into this buffer iteration - MPI_Bcast(&buffer_loop,1,MPI_INT,0,world); - - //store element connectivity that belongs to this rank - //loop through read buffer - for(scan_loop = 0; scan_loop < buffer_loop; scan_loop++){ - //set global node id (ensight specific order) - elem_gid = read_index_start + scan_loop; - //add this element to the local list if any of its nodes belong to this rank according to the map - //get list of nodes for each element line and check if they belong to the map - assign_flag = 0; - for(int inode = 0; inode < elem_words_per_line; inode++){ - //as we loop through the nodes belonging to this element we store them - //if any of these nodes belongs to this rank this list is used to store the element locally - node_gid = atoi(&read_buffer(scan_loop,inode,0)); - if(zero_index_base) - node_store(inode) = node_gid; //subtract 1 since file index start is 1 but code expects 0 + // std::cout << "RNUM ELEMENTS IS: " << rnum_elem << std::endl; + + Element_Types = CArrayKokkos(rnum_elem); + + elements::elem_types::elem_type mesh_element_type; + + if (simparam.num_dims == 2) + { + if (input_options.element_type == ELEMENT_TYPE::quad4) + { + mesh_element_type = elements::elem_types::Quad4; + max_nodes_per_patch = 2; + } + else if (input_options.element_type == ELEMENT_TYPE::quad8) + { + mesh_element_type = elements::elem_types::Quad8; + max_nodes_per_patch = 3; + } + else if (input_options.element_type == ELEMENT_TYPE::quad12) + { + mesh_element_type = elements::elem_types::Quad12; + max_nodes_per_patch = 4; + } else - node_store(inode) = node_gid - 1; //subtract 1 since file index start is 1 but code expects 0 - if(node_store(inode) < 0){ - negative_index_found = 1; - } - //first we add the elements to a dynamically allocated list - if(zero_index_base){ - if(map->isNodeGlobalElement(node_gid)&&!assign_flag){ - assign_flag = 1; - rnum_elem++; - } - } - else{ - if(map->isNodeGlobalElement(node_gid-1)&&!assign_flag){ - assign_flag = 1; - rnum_elem++; - } + { + if (myrank == 0) + { + std::cout << "ELEMENT TYPE UNRECOGNIZED" << std::endl; + } + exit_solver(0); } - } + element_select->choose_2Delem_type(mesh_element_type, elem2D); + max_nodes_per_element = elem2D->num_nodes(); + } - if(assign_flag){ - for(int inode = 0; inode < elem_words_per_line; inode++){ - if((rnum_elem-1)*elem_words_per_line + inode>=buffer_max){ - element_temp.resize((rnum_elem-1)*elem_words_per_line + inode + BUFFER_LINES*elem_words_per_line); - buffer_max = (rnum_elem-1)*elem_words_per_line + inode + BUFFER_LINES*elem_words_per_line; - } - element_temp[(rnum_elem-1)*elem_words_per_line + inode] = node_store(inode); - //std::cout << "VECTOR STORAGE FOR ELEM " << rnum_elem << " ON TASK " << myrank << " NODE " << inode+1 << " IS " << node_store(inode) + 1 << std::endl; - } - //assign global element id to temporary list - if(rnum_elem-1>=indices_buffer_max){ - global_indices_temp.resize(rnum_elem-1 + BUFFER_LINES); - indices_buffer_max = rnum_elem-1 + BUFFER_LINES; - } - global_indices_temp[rnum_elem-1] = elem_gid; - } + if (simparam.num_dims == 3) + { + if (input_options.element_type == ELEMENT_TYPE::hex8) + { + mesh_element_type = elements::elem_types::Hex8; + max_nodes_per_patch = 4; + } + else if (input_options.element_type == ELEMENT_TYPE::hex20) + { + mesh_element_type = elements::elem_types::Hex20; + max_nodes_per_patch = 8; + } + else if (input_options.element_type == ELEMENT_TYPE::hex32) + { + mesh_element_type = elements::elem_types::Hex32; + max_nodes_per_patch = 12; + } + else + { + if (myrank == 0) + { + std::cout << "ELEMENT TYPE UNRECOGNIZED" << std::endl; + } + exit_solver(0); + } + element_select->choose_3Delem_type(mesh_element_type, elem); + max_nodes_per_element = elem->num_nodes(); } - read_index_start+=BUFFER_LINES; - } - - // Close mesh input file - if(myrank==0) in->close(); - - //std::cout << "RNUM ELEMENTS IS: " << rnum_elem << std::endl; - - Element_Types = CArrayKokkos(rnum_elem); - - elements::elem_types::elem_type mesh_element_type; - - if(simparam.num_dims == 2){ - if(input_options.element_type == ELEMENT_TYPE::quad4){ - mesh_element_type = elements::elem_types::Quad4; - max_nodes_per_patch = 2; - } - else if(input_options.element_type == ELEMENT_TYPE::quad8){ - mesh_element_type = elements::elem_types::Quad8; - max_nodes_per_patch = 3; - } - else if(input_options.element_type == ELEMENT_TYPE::quad12){ - mesh_element_type = elements::elem_types::Quad12; - max_nodes_per_patch = 4; - } - else{ - if(myrank==0){ - std::cout << "ELEMENT TYPE UNRECOGNIZED" << std::endl; - } - exit_solver(0); + + // 1 type per mesh for now + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + Element_Types(ielem) = mesh_element_type; } - element_select->choose_2Delem_type(mesh_element_type, elem2D); - max_nodes_per_element = elem2D->num_nodes(); - } - if(simparam.num_dims == 3){ - if(input_options.element_type == ELEMENT_TYPE::hex8){ - mesh_element_type = elements::elem_types::Hex8; - max_nodes_per_patch = 4; + // copy temporary element storage to multivector storage + dual_nodes_in_elem = dual_elem_conn_array("dual_nodes_in_elem", rnum_elem, max_nodes_per_element); + host_elem_conn_array nodes_in_elem = dual_nodes_in_elem.view_host(); + dual_nodes_in_elem.modify_host(); + + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + for (int inode = 0; inode < elem_words_per_line; inode++) + { + nodes_in_elem(ielem, inode) = element_temp[ielem * elem_words_per_line + inode]; + } } - else if(input_options.element_type == ELEMENT_TYPE::hex20){ - mesh_element_type = elements::elem_types::Hex20; - max_nodes_per_patch = 8; + + // view storage for all local elements connected to local nodes on this rank + // DCArrayKokkos All_Element_Global_Indices(rnum_elem); + Kokkos::DualView All_Element_Global_Indices("All_Element_Global_Indices", rnum_elem); + // copy temporary global indices storage to view storage + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + All_Element_Global_Indices.h_view(ielem) = global_indices_temp[ielem]; + if (global_indices_temp[ielem] < 0) + { + negative_index_found = 1; + } } - else if(input_options.element_type == ELEMENT_TYPE::hex32){ - mesh_element_type = elements::elem_types::Hex32; - max_nodes_per_patch = 12; + + MPI_Allreduce(&negative_index_found, &global_negative_index_found, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD); + if (global_negative_index_found) + { + if (myrank == 0) + { + std::cout << "Node index less than or equal to zero detected; set \"zero_index_base: true\" under \"input_options\" in your yaml file if indices start at 0" << std::endl; + } + exit_solver(0); + } + // delete temporary element connectivity and index storage + std::vector().swap(element_temp); + std::vector().swap(global_indices_temp); + + All_Element_Global_Indices.modify_host(); + All_Element_Global_Indices.sync_device(); + + // debug print + /* + Kokkos::View All_Element_Global_Indices_pass("All_Element_Global_Indices_pass",rnum_elem); + deep_copy(All_Element_Global_Indices_pass, All_Element_Global_Indices.h_view); + std::cout << " ------------ELEMENT GLOBAL INDICES ON TASK " << myrank << " --------------"<(Teuchos::OrdinalTraits::invalid(), All_Element_Global_Indices.d_view, 0, comm)); + + // element type selection (subject to change) + // ---- Set Element Type ---- // + // allocate element type memory + // elements::elem_type_t* elem_choice; + + int NE = 1; // number of element types in problem + + // Convert ensight index system to the ijk finite element numbering convention + // for vertices in cell + if (active_node_ordering_convention == IJK) + { + CArrayKokkos convert_ensight_to_ijk(max_nodes_per_element); + CArrayKokkos tmp_ijk_indx(max_nodes_per_element); + convert_ensight_to_ijk(0) = 0; + convert_ensight_to_ijk(1) = 1; + convert_ensight_to_ijk(2) = 3; + convert_ensight_to_ijk(3) = 2; + convert_ensight_to_ijk(4) = 4; + convert_ensight_to_ijk(5) = 5; + convert_ensight_to_ijk(6) = 7; + convert_ensight_to_ijk(7) = 6; + + int nodes_per_element; + + if (num_dim == 2) + { + for (int cell_rid = 0; cell_rid < rnum_elem; cell_rid++) + { + // set nodes per element + element_select->choose_2Delem_type(Element_Types(cell_rid), elem2D); + nodes_per_element = elem2D->num_nodes(); + for (int node_lid = 0; node_lid < nodes_per_element; node_lid++) + { + tmp_ijk_indx(node_lid) = nodes_in_elem(cell_rid, convert_ensight_to_ijk(node_lid)); + } + + for (int node_lid = 0; node_lid < nodes_per_element; node_lid++) + { + nodes_in_elem(cell_rid, node_lid) = tmp_ijk_indx(node_lid); + } + } + } + + if (num_dim == 3) + { + for (int cell_rid = 0; cell_rid < rnum_elem; cell_rid++) + { + // set nodes per element + element_select->choose_3Delem_type(Element_Types(cell_rid), elem); + nodes_per_element = elem->num_nodes(); + for (int node_lid = 0; node_lid < nodes_per_element; node_lid++) + { + tmp_ijk_indx(node_lid) = nodes_in_elem(cell_rid, convert_ensight_to_ijk(node_lid)); + } + + for (int node_lid = 0; node_lid < nodes_per_element; node_lid++) + { + nodes_in_elem(cell_rid, node_lid) = tmp_ijk_indx(node_lid); + } + } + } } - else{ - if(myrank==0){ - std::cout << "ELEMENT TYPE UNRECOGNIZED" << std::endl; + // debug print element edof + /* + std::cout << " ------------ELEMENT EDOF ON TASK " << myrank << " --------------"<choose_3Delem_type(mesh_element_type, elem); - max_nodes_per_element = elem->num_nodes(); - } - - //1 type per mesh for now - for(int ielem = 0; ielem < rnum_elem; ielem++) - Element_Types(ielem) = mesh_element_type; - - //copy temporary element storage to multivector storage - dual_nodes_in_elem = dual_elem_conn_array("dual_nodes_in_elem", rnum_elem, max_nodes_per_element); - host_elem_conn_array nodes_in_elem = dual_nodes_in_elem.view_host(); - dual_nodes_in_elem.modify_host(); - - for(int ielem = 0; ielem < rnum_elem; ielem++) - for(int inode = 0; inode < elem_words_per_line; inode++){ - nodes_in_elem(ielem, inode) = element_temp[ielem*elem_words_per_line + inode]; - } - - //view storage for all local elements connected to local nodes on this rank - //DCArrayKokkos All_Element_Global_Indices(rnum_elem); - Kokkos::DualView All_Element_Global_Indices("All_Element_Global_Indices",rnum_elem); - //copy temporary global indices storage to view storage - for(int ielem = 0; ielem < rnum_elem; ielem++){ - All_Element_Global_Indices.h_view(ielem) = global_indices_temp[ielem]; - if(global_indices_temp[ielem]<0){ - negative_index_found = 1; - } - } - - MPI_Allreduce(&negative_index_found,&global_negative_index_found,1,MPI_INT,MPI_MAX,MPI_COMM_WORLD); - if(global_negative_index_found){ - if(myrank==0){ - std::cout << "Node index less than or equal to zero detected; set \"zero_index_base: true\" under \"input_options\" in your yaml file if indices start at 0" << std::endl; - } - exit_solver(0); - } - //delete temporary element connectivity and index storage - std::vector().swap(element_temp); - std::vector().swap(global_indices_temp); - - All_Element_Global_Indices.modify_host(); - All_Element_Global_Indices.sync_device(); - - //debug print - /* - Kokkos::View All_Element_Global_Indices_pass("All_Element_Global_Indices_pass",rnum_elem); - deep_copy(All_Element_Global_Indices_pass, All_Element_Global_Indices.h_view); - std::cout << " ------------ELEMENT GLOBAL INDICES ON TASK " << myrank << " --------------"<(Teuchos::OrdinalTraits::invalid(),All_Element_Global_Indices.d_view,0,comm)); - - //element type selection (subject to change) - // ---- Set Element Type ---- // - // allocate element type memory - //elements::elem_type_t* elem_choice; - - int NE = 1; // number of element types in problem - - - // Convert ensight index system to the ijk finite element numbering convention - // for vertices in cell - if(active_node_ordering_convention == IJK){ - CArrayKokkos convert_ensight_to_ijk(max_nodes_per_element); - CArrayKokkos tmp_ijk_indx(max_nodes_per_element); - convert_ensight_to_ijk(0) = 0; - convert_ensight_to_ijk(1) = 1; - convert_ensight_to_ijk(2) = 3; - convert_ensight_to_ijk(3) = 2; - convert_ensight_to_ijk(4) = 4; - convert_ensight_to_ijk(5) = 5; - convert_ensight_to_ijk(6) = 7; - convert_ensight_to_ijk(7) = 6; - - int nodes_per_element; - - if(num_dim==2) - for (int cell_rid = 0; cell_rid < rnum_elem; cell_rid++) { - //set nodes per element - element_select->choose_2Delem_type(Element_Types(cell_rid), elem2D); - nodes_per_element = elem2D->num_nodes(); - for (int node_lid = 0; node_lid < nodes_per_element; node_lid++){ - tmp_ijk_indx(node_lid) = nodes_in_elem(cell_rid, convert_ensight_to_ijk(node_lid)); - } - - for (int node_lid = 0; node_lid < nodes_per_element; node_lid++){ - nodes_in_elem(cell_rid, node_lid) = tmp_ijk_indx(node_lid); - } - } - - if(num_dim==3) - for (int cell_rid = 0; cell_rid < rnum_elem; cell_rid++) { - //set nodes per element - element_select->choose_3Delem_type(Element_Types(cell_rid), elem); - nodes_per_element = elem->num_nodes(); - for (int node_lid = 0; node_lid < nodes_per_element; node_lid++){ - tmp_ijk_indx(node_lid) = nodes_in_elem(cell_rid, convert_ensight_to_ijk(node_lid)); - } - - for (int node_lid = 0; node_lid < nodes_per_element; node_lid++){ - nodes_in_elem(cell_rid, node_lid) = tmp_ijk_indx(node_lid); - } - } - } - //debug print element edof - /* - std::cout << " ------------ELEMENT EDOF ON TASK " << myrank << " --------------"< read_buffer; - int buffer_loop, buffer_iteration, buffer_iterations, dof_limit, scan_loop; - size_t read_index_start, node_rid, elem_gid; - GO node_gid; - real_t dof_value; - bool zero_index_base = input_options.zero_index_base; - int negative_index_found = 0; - int global_negative_index_found = 0; - //Nodes_Per_Element_Type = elements::elem_types::Nodes_Per_Element_Type; - - //read the mesh - //PLACEHOLDER: ensight_format(MESH); - // abaqus_format(MESH); - // vtk_format(MESH) - - // --- Read the number of nodes in the mesh --- // - num_nodes = 0; - if(myrank==0){ - std::cout << " NUM DIM is " << num_dim << std::endl; - in = new std::ifstream(); - in->open(MESH); - - int i = 0; - bool found = false; - while (found==false) { - std::getline(*in, read_line); - line_parse.str(""); - line_parse.clear(); - line_parse << read_line; - line_parse >> substring; - - // looking for the following text: - // POINTS %d float - if(substring == "POINTS"){ - line_parse >> num_nodes; - std::cout << "declared node count: " << num_nodes << std::endl; - if(num_nodes <= 0) throw std::runtime_error("ERROR, NO NODES IN MESH"); - found=true; - } // end if - - - if (i>1000){ - throw std::runtime_error("ERROR: Failed to find POINTS"); - break; - } // end if - - i++; - } // end while - } // end if(myrank==0) - - //broadcast number of nodes - MPI_Bcast(&num_nodes,1,MPI_LONG_LONG_INT,0,world); - - //construct contiguous parallel row map now that we know the number of nodes - map = Teuchos::rcp( new Tpetra::Map(num_nodes,0,comm)); - //map->describe(*fos,Teuchos::VERB_EXTREME); - - // set the vertices in the mesh read in - nlocal_nodes = map->getLocalNumElements(); - //populate local row offset data from global data - global_size_t min_gid = map->getMinGlobalIndex(); - global_size_t max_gid = map->getMaxGlobalIndex(); - global_size_t index_base = map->getIndexBase(); - //debug print - //std::cout << "local node count on task: " << " " << nlocal_nodes << std::endl; - - //allocate node storage with dual view - //dual_node_coords = dual_vec_array("dual_node_coords", nlocal_nodes,num_dim); - - //local variable for host view in the dual view - - node_coords_distributed = Teuchos::rcp(new MV(map, num_dim)); - - //scope ensures view is destroyed for now to avoid calling a device view with an active host view later - { - host_vec_array node_coords = node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); - //host_vec_array node_coords = dual_node_coords.view_host(); - //notify that the host view is going to be modified in the file readin - //dual_node_coords.modify_host(); - - //old swage method - //mesh->init_nodes(local_nrows); // add 1 for index starting at 1 - - std::cout << "Num nodes assigned to task " << myrank << " = " << nlocal_nodes << std::endl; - - // read the initial mesh coordinates - // x-coords - /*only task 0 reads in nodes and elements from the input file - stores node data in a buffer and communicates once the buffer cap is reached - or the data ends*/ - - words_per_line = input_options.words_per_line; - elem_words_per_line = input_options.elem_words_per_line; - - //allocate read buffer - read_buffer = CArrayKokkos(BUFFER_LINES,words_per_line,MAX_WORD); - - dof_limit = num_nodes; - buffer_iterations = dof_limit/BUFFER_LINES; - if(dof_limit%BUFFER_LINES!=0) buffer_iterations++; - - //read coords, also density if restarting - read_index_start = 0; - for(buffer_iteration = 0; buffer_iteration < buffer_iterations; buffer_iteration++){ - //pack buffer on rank 0 - if(myrank==0&&buffer_iteration> substring; - //debug print - //std::cout<<" "<< substring < read_buffer; + + // Nodes_Per_Element_Type = elements::elem_types::Nodes_Per_Element_Type; + + // read the mesh + // PLACEHOLDER: ensight_format(MESH); + // abaqus_format(MESH); + // vtk_format(MESH) + + // --- Read the number of nodes in the mesh --- // + num_nodes = 0; + if (myrank == 0) + { + std::cout << " NUM DIM is " << num_dim << std::endl; + in = new std::ifstream(); + in->open(MESH); + + int i = 0; + bool found = false; + while (found == false) { + std::getline(*in, read_line); + line_parse.str(""); + line_parse.clear(); + line_parse << read_line; + line_parse >> substring; + + // looking for the following text: + // POINTS %d float + if (substring == "POINTS") + { + line_parse >> num_nodes; + std::cout << "declared node count: " << num_nodes << std::endl; + if (num_nodes <= 0) + { + throw std::runtime_error("ERROR, NO NODES IN MESH"); + } + found = true; + } // end if + + if (i > 1000) + { + throw std::runtime_error("ERROR: Failed to find POINTS"); + break; + } // end if + + i++; + } // end while + } // end if(myrank==0) + + // broadcast number of nodes + MPI_Bcast(&num_nodes, 1, MPI_LONG_LONG_INT, 0, world); + + // construct contiguous parallel row map now that we know the number of nodes + map = Teuchos::rcp(new Tpetra::Map(num_nodes, 0, comm)); + // map->describe(*fos,Teuchos::VERB_EXTREME); + + // set the vertices in the mesh read in + nlocal_nodes = map->getLocalNumElements(); + // populate local row offset data from global data + global_size_t min_gid = map->getMinGlobalIndex(); + global_size_t max_gid = map->getMaxGlobalIndex(); + global_size_t index_base = map->getIndexBase(); + // debug print + // std::cout << "local node count on task: " << " " << nlocal_nodes << std::endl; + + // allocate node storage with dual view + // dual_node_coords = dual_vec_array("dual_node_coords", nlocal_nodes,num_dim); + + // local variable for host view in the dual view + + node_coords_distributed = Teuchos::rcp(new MV(map, num_dim)); + + // scope ensures view is destroyed for now to avoid calling a device view with an active host view later + { + host_vec_array node_coords = node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + // host_vec_array node_coords = dual_node_coords.view_host(); + // notify that the host view is going to be modified in the file readin + // dual_node_coords.modify_host(); + + // old swage method + // mesh->init_nodes(local_nrows); // add 1 for index starting at 1 + + std::cout << "Num nodes assigned to task " << myrank << " = " << nlocal_nodes << std::endl; + + // read the initial mesh coordinates + // x-coords + /*only task 0 reads in nodes and elements from the input file + stores node data in a buffer and communicates once the buffer cap is reached + or the data ends*/ + + words_per_line = input_options.words_per_line; + elem_words_per_line = input_options.elem_words_per_line; + + // allocate read buffer + read_buffer = CArrayKokkos(BUFFER_LINES, words_per_line, MAX_WORD); + + dof_limit = num_nodes; + buffer_iterations = dof_limit / BUFFER_LINES; + if (dof_limit % BUFFER_LINES != 0) + { + buffer_iterations++; } + + // read coords, also density if restarting + read_index_start = 0; + for (buffer_iteration = 0; buffer_iteration < buffer_iterations; buffer_iteration++) + { + // pack buffer on rank 0 + if (myrank == 0 && buffer_iteration < buffer_iterations - 1) + { + for (buffer_loop = 0; buffer_loop < BUFFER_LINES; buffer_loop++) + { + getline(*in, read_line); + line_parse.clear(); + line_parse.str(read_line); + + for (int iword = 0; iword < words_per_line; iword++) + { + // read portions of the line into the substring variable + line_parse >> substring; + // debug print + // std::cout<<" "<< substring <> substring; + // debug print + // std::cout<<" "<< substring <isNodeGlobalElement(node_gid)) + { + // set local node index in this mpi rank + node_rid = map->getLocalElement(node_gid); + // extract nodal position from the read buffer + // for tecplot format this is the three coords in the same line + dof_value = atof(&read_buffer(scan_loop, 0, 0)); + node_coords(node_rid, 0) = dof_value * unit_scaling; + dof_value = atof(&read_buffer(scan_loop, 1, 0)); + node_coords(node_rid, 1) = dof_value * unit_scaling; + if (num_dim == 3) + { + dof_value = atof(&read_buffer(scan_loop, 2, 0)); + node_coords(node_rid, 2) = dof_value * unit_scaling; + } + // extract density if restarting + } + } + read_index_start += BUFFER_LINES; + } + } // end of coordinate readin + // repartition node distribution + repartition_nodes(); + + // synchronize device data + // dual_node_coords.sync_device(); + // dual_node_coords.modify_device(); + + // debug print of nodal data + + // debug print nodal positions and indices + /* + std::cout << " ------------NODAL POSITIONS ON TASK " << myrank << " --------------"<getGlobalElement(inode) + 1 << " { "; + for (int istride = 0; istride < num_dim; istride++){ + std::cout << node_coords(inode,istride) << " , "; } + std::cout << " }"<< std::endl; + } + */ + + // check that local assignments match global total + + // read in element info (ensight file format is organized in element type sections) + // loop over this later for several element type sections + + num_elem = 0; + rnum_elem = 0; + CArrayKokkos node_store(elem_words_per_line); + + // --- read the number of cells in the mesh --- + // --- Read the number of vertices in the mesh --- // + if (myrank == 0) + { + bool found = false; + int i = 0; + while (found == false) { + std::getline(*in, read_line); + line_parse.str(""); + line_parse.clear(); + line_parse << read_line; + line_parse >> substring; + + // looking for the following text: + // CELLS num_cells size + if (substring == "CELLS") + { + line_parse >> num_elem; + std::cout << "declared element count: " << num_elem << std::endl; + if (num_elem <= 0) + { + throw std::runtime_error("ERROR, NO ELEMENTS IN MESH"); + } + found = true; + } // end if + + if (i > 1000) + { + throw std::runtime_error("ERROR: Failed to find CELLS"); + break; + } // end if + + i++; + } // end while + } // end if(myrank==0) + + // broadcast number of elements + MPI_Bcast(&num_elem, 1, MPI_LONG_LONG_INT, 0, world); + + if (myrank == 0) + { + std::cout << "before mesh initialization" << std::endl; + } + + // read in element connectivity + // we're gonna reallocate for the words per line expected for the element connectivity + read_buffer = CArrayKokkos(BUFFER_LINES, elem_words_per_line, MAX_WORD); + + // calculate buffer iterations to read number of lines + buffer_iterations = num_elem / BUFFER_LINES; + int assign_flag; + + // dynamic buffer used to store elements before we know how many this rank needs + std::vector element_temp(BUFFER_LINES * elem_words_per_line); + std::vector global_indices_temp(BUFFER_LINES); + size_t buffer_max = BUFFER_LINES * elem_words_per_line; + size_t indices_buffer_max = BUFFER_LINES; + + if (num_elem % BUFFER_LINES != 0) + { + buffer_iterations++; + } + read_index_start = 0; + // std::cout << "ELEMENT BUFFER ITERATIONS: " << buffer_iterations << std::endl; + rnum_elem = 0; + for (buffer_iteration = 0; buffer_iteration < buffer_iterations; buffer_iteration++) + { + // pack buffer on rank 0 + if (myrank == 0 && buffer_iteration < buffer_iterations - 1) + { + for (buffer_loop = 0; buffer_loop < BUFFER_LINES; buffer_loop++) + { + getline(*in, read_line); + line_parse.clear(); + line_parse.str(read_line); + // disregard node count line since we're using one element type per mesh + line_parse >> substring; + for (int iword = 0; iword < elem_words_per_line; iword++) + { + // read portions of the line into the substring variable + line_parse >> substring; + // debug print + // std::cout<<" "<< substring; + // assign the substring variable as a word of the read buffer + strcpy(&read_buffer(buffer_loop, iword, 0), substring.c_str()); + } + // std::cout <> substring; + for (int iword = 0; iword < elem_words_per_line; iword++) + { + // read portions of the line into the substring variable + line_parse >> substring; + // debug print + // std::cout<<" "<< substring; + // assign the substring variable as a word of the read buffer + strcpy(&read_buffer(buffer_loop, iword, 0), substring.c_str()); + } + // std::cout <isNodeGlobalElement(node_gid) && !assign_flag) + { + assign_flag = 1; + rnum_elem++; + } + } + else + { + if (map->isNodeGlobalElement(node_gid - 1) && !assign_flag) + { + assign_flag = 1; + rnum_elem++; + } + } + } + + if (assign_flag) + { + for (int inode = 0; inode < elem_words_per_line; inode++) + { + if ((rnum_elem - 1) * elem_words_per_line + inode >= buffer_max) + { + element_temp.resize((rnum_elem - 1) * elem_words_per_line + inode + BUFFER_LINES * elem_words_per_line); + buffer_max = (rnum_elem - 1) * elem_words_per_line + inode + BUFFER_LINES * elem_words_per_line; + } + element_temp[(rnum_elem - 1) * elem_words_per_line + inode] = node_store(inode); + // std::cout << "VECTOR STORAGE FOR ELEM " << rnum_elem << " ON TASK " << myrank << " NODE " << inode+1 << " IS " << node_store(inode) + 1 << std::endl; + } + // assign global element id to temporary list + if (rnum_elem - 1 >= indices_buffer_max) + { + global_indices_temp.resize(rnum_elem - 1 + BUFFER_LINES); + indices_buffer_max = rnum_elem - 1 + BUFFER_LINES; + } + global_indices_temp[rnum_elem - 1] = elem_gid; + } + } + read_index_start += BUFFER_LINES; } - else if(myrank==0){ - buffer_loop=0; - while(buffer_iteration*BUFFER_LINES+buffer_loop < num_nodes) { - getline(*in,read_line); - line_parse.clear(); - line_parse.str(read_line); - for(int iword = 0; iword < words_per_line; iword++){ - //read portions of the line into the substring variable - line_parse >> substring; - //debug print - //std::cout<<" "<< substring <isNodeGlobalElement(node_gid)){ - //set local node index in this mpi rank - node_rid = map->getLocalElement(node_gid); - //extract nodal position from the read buffer - //for tecplot format this is the three coords in the same line - dof_value = atof(&read_buffer(scan_loop,0,0)); - node_coords(node_rid, 0) = dof_value * unit_scaling; - dof_value = atof(&read_buffer(scan_loop,1,0)); - node_coords(node_rid, 1) = dof_value * unit_scaling; - if(num_dim==3){ - dof_value = atof(&read_buffer(scan_loop,2,0)); - node_coords(node_rid, 2) = dof_value * unit_scaling; - } - //extract density if restarting - } - } - read_index_start+=BUFFER_LINES; - } - } //end of coordinate readin - //repartition node distribution - repartition_nodes(); - - //synchronize device data - //dual_node_coords.sync_device(); - //dual_node_coords.modify_device(); - - //debug print of nodal data - - //debug print nodal positions and indices - /* - std::cout << " ------------NODAL POSITIONS ON TASK " << myrank << " --------------"<getGlobalElement(inode) + 1 << " { "; - for (int istride = 0; istride < num_dim; istride++){ - std::cout << node_coords(inode,istride) << " , "; - } - std::cout << " }"<< std::endl; - } - */ - - //check that local assignments match global total - - - //read in element info (ensight file format is organized in element type sections) - //loop over this later for several element type sections - - num_elem = 0; - rnum_elem = 0; - CArrayKokkos node_store(elem_words_per_line); - - // --- read the number of cells in the mesh --- - // --- Read the number of vertices in the mesh --- // - if(myrank==0){ - bool found = false; - int i = 0; - while (found==false) { - std::getline(*in, read_line); - line_parse.str(""); - line_parse.clear(); - line_parse << read_line; - line_parse >> substring; - - // looking for the following text: - // CELLS num_cells size - if(substring == "CELLS"){ - line_parse >> num_elem; - std::cout << "declared element count: " << num_elem << std::endl; - if(num_elem <= 0) throw std::runtime_error("ERROR, NO ELEMENTS IN MESH"); - found=true; - } // end if - - - if (i>1000){ - throw std::runtime_error("ERROR: Failed to find CELLS"); - break; - } // end if - - i++; - } // end while - } // end if(myrank==0) - - //broadcast number of elements - MPI_Bcast(&num_elem,1,MPI_LONG_LONG_INT,0,world); - - if(myrank == 0) - std::cout<<"before mesh initialization"<(BUFFER_LINES,elem_words_per_line,MAX_WORD); - - //calculate buffer iterations to read number of lines - buffer_iterations = num_elem/BUFFER_LINES; - int assign_flag; - - //dynamic buffer used to store elements before we know how many this rank needs - std::vector element_temp(BUFFER_LINES*elem_words_per_line); - std::vector global_indices_temp(BUFFER_LINES); - size_t buffer_max = BUFFER_LINES*elem_words_per_line; - size_t indices_buffer_max = BUFFER_LINES; - - if(num_elem%BUFFER_LINES!=0) buffer_iterations++; - read_index_start = 0; - //std::cout << "ELEMENT BUFFER ITERATIONS: " << buffer_iterations << std::endl; - rnum_elem = 0; - for(buffer_iteration = 0; buffer_iteration < buffer_iterations; buffer_iteration++){ - //pack buffer on rank 0 - if(myrank==0&&buffer_iteration> substring; - for(int iword = 0; iword < elem_words_per_line; iword++){ - //read portions of the line into the substring variable - line_parse >> substring; - //debug print - //std::cout<<" "<< substring; - //assign the substring variable as a word of the read buffer - strcpy(&read_buffer(buffer_loop,iword,0),substring.c_str()); - } - //std::cout <> substring; - for(int iword = 0; iword < elem_words_per_line; iword++){ - //read portions of the line into the substring variable - line_parse >> substring; - //debug print - //std::cout<<" "<< substring; - //assign the substring variable as a word of the read buffer - strcpy(&read_buffer(buffer_loop,iword,0),substring.c_str()); - } - //std::cout <close(); } - //broadcast buffer to all ranks; each rank will determine which nodes in the buffer belong - MPI_Bcast(read_buffer.pointer(),BUFFER_LINES*elem_words_per_line*MAX_WORD,MPI_CHAR,0,world); - //broadcast how many nodes were read into this buffer iteration - MPI_Bcast(&buffer_loop,1,MPI_INT,0,world); - - //store element connectivity that belongs to this rank - //loop through read buffer - for(scan_loop = 0; scan_loop < buffer_loop; scan_loop++){ - //set global node id (ensight specific order) - elem_gid = read_index_start + scan_loop; - //add this element to the local list if any of its nodes belong to this rank according to the map - //get list of nodes for each element line and check if they belong to the map - assign_flag = 0; - for(int inode = 0; inode < elem_words_per_line; inode++){ - //as we loop through the nodes belonging to this element we store them - //if any of these nodes belongs to this rank this list is used to store the element locally - node_gid = atoi(&read_buffer(scan_loop,inode,0)); - if(zero_index_base) - node_store(inode) = node_gid; //subtract 1 since file index start is 1 but code expects 0 + // std::cout << "RNUM ELEMENTS IS: " << rnum_elem << std::endl; + + Element_Types = CArrayKokkos(rnum_elem); + + elements::elem_types::elem_type mesh_element_type; + + if (simparam.num_dims == 2) + { + if (input_options.element_type == ELEMENT_TYPE::quad4) + { + mesh_element_type = elements::elem_types::Quad4; + max_nodes_per_patch = 2; + } + else if (input_options.element_type == ELEMENT_TYPE::quad8) + { + mesh_element_type = elements::elem_types::Quad8; + max_nodes_per_patch = 3; + } + else if (input_options.element_type == ELEMENT_TYPE::quad12) + { + mesh_element_type = elements::elem_types::Quad12; + max_nodes_per_patch = 4; + } else - node_store(inode) = node_gid - 1; //subtract 1 since file index start is 1 but code expects 0 - if(node_store(inode) < 0){ - negative_index_found = 1; - } - //first we add the elements to a dynamically allocated list - if(zero_index_base){ - if(map->isNodeGlobalElement(node_gid)&&!assign_flag){ - assign_flag = 1; - rnum_elem++; - } - } - else{ - if(map->isNodeGlobalElement(node_gid-1)&&!assign_flag){ - assign_flag = 1; - rnum_elem++; - } + { + if (myrank == 0) + { + std::cout << "ELEMENT TYPE UNRECOGNIZED" << std::endl; + } + exit_solver(0); } - } + element_select->choose_2Delem_type(mesh_element_type, elem2D); + max_nodes_per_element = elem2D->num_nodes(); + } - if(assign_flag){ - for(int inode = 0; inode < elem_words_per_line; inode++){ - if((rnum_elem-1)*elem_words_per_line + inode>=buffer_max){ - element_temp.resize((rnum_elem-1)*elem_words_per_line + inode + BUFFER_LINES*elem_words_per_line); - buffer_max = (rnum_elem-1)*elem_words_per_line + inode + BUFFER_LINES*elem_words_per_line; - } - element_temp[(rnum_elem-1)*elem_words_per_line + inode] = node_store(inode); - //std::cout << "VECTOR STORAGE FOR ELEM " << rnum_elem << " ON TASK " << myrank << " NODE " << inode+1 << " IS " << node_store(inode) + 1 << std::endl; - } - //assign global element id to temporary list - if(rnum_elem-1>=indices_buffer_max){ - global_indices_temp.resize(rnum_elem-1 + BUFFER_LINES); - indices_buffer_max = rnum_elem-1 + BUFFER_LINES; - } - global_indices_temp[rnum_elem-1] = elem_gid; - } + if (simparam.num_dims == 3) + { + if (input_options.element_type == ELEMENT_TYPE::hex8) + { + mesh_element_type = elements::elem_types::Hex8; + max_nodes_per_patch = 4; + } + else if (input_options.element_type == ELEMENT_TYPE::hex20) + { + mesh_element_type = elements::elem_types::Hex20; + max_nodes_per_patch = 8; + } + else if (input_options.element_type == ELEMENT_TYPE::hex32) + { + mesh_element_type = elements::elem_types::Hex32; + max_nodes_per_patch = 12; + } + else + { + if (myrank == 0) + { + std::cout << "ELEMENT TYPE UNRECOGNIZED" << std::endl; + } + exit_solver(0); + } + element_select->choose_3Delem_type(mesh_element_type, elem); + max_nodes_per_element = elem->num_nodes(); } - read_index_start+=BUFFER_LINES; - } - - // Close mesh input file - if(myrank==0) - in->close(); - - //std::cout << "RNUM ELEMENTS IS: " << rnum_elem << std::endl; - - Element_Types = CArrayKokkos(rnum_elem); - - elements::elem_types::elem_type mesh_element_type; - - if(simparam.num_dims == 2){ - if(input_options.element_type == ELEMENT_TYPE::quad4){ - mesh_element_type = elements::elem_types::Quad4; - max_nodes_per_patch = 2; - } - else if(input_options.element_type == ELEMENT_TYPE::quad8){ - mesh_element_type = elements::elem_types::Quad8; - max_nodes_per_patch = 3; - } - else if(input_options.element_type == ELEMENT_TYPE::quad12){ - mesh_element_type = elements::elem_types::Quad12; - max_nodes_per_patch = 4; - } - else{ - if(myrank==0){ - std::cout << "ELEMENT TYPE UNRECOGNIZED" << std::endl; - } - exit_solver(0); + + // 1 type per mesh for now + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + Element_Types(ielem) = mesh_element_type; } - element_select->choose_2Delem_type(mesh_element_type, elem2D); - max_nodes_per_element = elem2D->num_nodes(); - } - if(simparam.num_dims == 3){ - if(input_options.element_type == ELEMENT_TYPE::hex8){ - mesh_element_type = elements::elem_types::Hex8; - max_nodes_per_patch = 4; + // copy temporary element storage to multivector storage + dual_nodes_in_elem = dual_elem_conn_array("dual_nodes_in_elem", rnum_elem, max_nodes_per_element); + host_elem_conn_array nodes_in_elem = dual_nodes_in_elem.view_host(); + dual_nodes_in_elem.modify_host(); + + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + for (int inode = 0; inode < elem_words_per_line; inode++) + { + nodes_in_elem(ielem, inode) = element_temp[ielem * elem_words_per_line + inode]; + } } - else if(input_options.element_type == ELEMENT_TYPE::hex20){ - mesh_element_type = elements::elem_types::Hex20; - max_nodes_per_patch = 8; + + // view storage for all local elements connected to local nodes on this rank + // DCArrayKokkos All_Element_Global_Indices(rnum_elem); + Kokkos::DualView All_Element_Global_Indices("All_Element_Global_Indices", rnum_elem); + // copy temporary global indices storage to view storage + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + All_Element_Global_Indices.h_view(ielem) = global_indices_temp[ielem]; + if (global_indices_temp[ielem] < 0) + { + negative_index_found = 1; + } } - else if(input_options.element_type == ELEMENT_TYPE::hex32){ - mesh_element_type = elements::elem_types::Hex32; - max_nodes_per_patch = 12; + + MPI_Allreduce(&negative_index_found, &global_negative_index_found, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD); + if (global_negative_index_found) + { + if (myrank == 0) + { + std::cout << "Node index less than or equal to zero detected; set \"zero_index_base: true\" under \"input_options\" in your yaml file if indices start at 0" << std::endl; + } + exit_solver(0); + } + + // delete temporary element connectivity and index storage + std::vector().swap(element_temp); + std::vector().swap(global_indices_temp); + + All_Element_Global_Indices.modify_host(); + All_Element_Global_Indices.sync_device(); + + // debug print + /* + Kokkos::View All_Element_Global_Indices_pass("All_Element_Global_Indices_pass",rnum_elem); + deep_copy(All_Element_Global_Indices_pass, All_Element_Global_Indices.h_view); + std::cout << " ------------ELEMENT GLOBAL INDICES ON TASK " << myrank << " --------------"<(Teuchos::OrdinalTraits::invalid(), All_Element_Global_Indices.d_view, 0, comm)); + + // element type selection (subject to change) + // ---- Set Element Type ---- // + // allocate element type memory + // elements::elem_type_t* elem_choice; + + int NE = 1; // number of element types in problem + + // Convert ensight index system to the ijk finite element numbering convention + // for vertices in cell + if (active_node_ordering_convention == IJK) + { + CArrayKokkos convert_ensight_to_ijk(max_nodes_per_element); + CArrayKokkos tmp_ijk_indx(max_nodes_per_element); + convert_ensight_to_ijk(0) = 0; + convert_ensight_to_ijk(1) = 1; + convert_ensight_to_ijk(2) = 3; + convert_ensight_to_ijk(3) = 2; + convert_ensight_to_ijk(4) = 4; + convert_ensight_to_ijk(5) = 5; + convert_ensight_to_ijk(6) = 7; + convert_ensight_to_ijk(7) = 6; + + int nodes_per_element; + + if (num_dim == 2) + { + for (int cell_rid = 0; cell_rid < rnum_elem; cell_rid++) + { + // set nodes per element + element_select->choose_2Delem_type(Element_Types(cell_rid), elem2D); + nodes_per_element = elem2D->num_nodes(); + for (int node_lid = 0; node_lid < nodes_per_element; node_lid++) + { + tmp_ijk_indx(node_lid) = nodes_in_elem(cell_rid, convert_ensight_to_ijk(node_lid)); + } + + for (int node_lid = 0; node_lid < nodes_per_element; node_lid++) + { + nodes_in_elem(cell_rid, node_lid) = tmp_ijk_indx(node_lid); + } + } + } + + if (num_dim == 3) + { + for (int cell_rid = 0; cell_rid < rnum_elem; cell_rid++) + { + // set nodes per element + element_select->choose_3Delem_type(Element_Types(cell_rid), elem); + nodes_per_element = elem->num_nodes(); + for (int node_lid = 0; node_lid < nodes_per_element; node_lid++) + { + tmp_ijk_indx(node_lid) = nodes_in_elem(cell_rid, convert_ensight_to_ijk(node_lid)); + } + + for (int node_lid = 0; node_lid < nodes_per_element; node_lid++) + { + nodes_in_elem(cell_rid, node_lid) = tmp_ijk_indx(node_lid); + } + } + } } - else{ - if(myrank==0){ - std::cout << "ELEMENT TYPE UNRECOGNIZED" << std::endl; + // debug print element edof + /* + std::cout << " ------------ELEMENT EDOF ON TASK " << myrank << " --------------"<choose_3Delem_type(mesh_element_type, elem); - max_nodes_per_element = elem->num_nodes(); - } - - //1 type per mesh for now - for(int ielem = 0; ielem < rnum_elem; ielem++) - Element_Types(ielem) = mesh_element_type; - - //copy temporary element storage to multivector storage - dual_nodes_in_elem = dual_elem_conn_array("dual_nodes_in_elem", rnum_elem, max_nodes_per_element); - host_elem_conn_array nodes_in_elem = dual_nodes_in_elem.view_host(); - dual_nodes_in_elem.modify_host(); - - for(int ielem = 0; ielem < rnum_elem; ielem++) - for(int inode = 0; inode < elem_words_per_line; inode++){ - nodes_in_elem(ielem, inode) = element_temp[ielem*elem_words_per_line + inode]; - } - - //view storage for all local elements connected to local nodes on this rank - //DCArrayKokkos All_Element_Global_Indices(rnum_elem); - Kokkos::DualView All_Element_Global_Indices("All_Element_Global_Indices",rnum_elem); - //copy temporary global indices storage to view storage - for(int ielem = 0; ielem < rnum_elem; ielem++){ - All_Element_Global_Indices.h_view(ielem) = global_indices_temp[ielem]; - if(global_indices_temp[ielem]<0){ - negative_index_found = 1; - } - } - - MPI_Allreduce(&negative_index_found,&global_negative_index_found,1,MPI_INT,MPI_MAX,MPI_COMM_WORLD); - if(global_negative_index_found){ - if(myrank==0){ - std::cout << "Node index less than or equal to zero detected; set \"zero_index_base: true\" under \"input_options\" in your yaml file if indices start at 0" << std::endl; - } - exit_solver(0); - } - - //delete temporary element connectivity and index storage - std::vector().swap(element_temp); - std::vector().swap(global_indices_temp); - - All_Element_Global_Indices.modify_host(); - All_Element_Global_Indices.sync_device(); - - //debug print - /* - Kokkos::View All_Element_Global_Indices_pass("All_Element_Global_Indices_pass",rnum_elem); - deep_copy(All_Element_Global_Indices_pass, All_Element_Global_Indices.h_view); - std::cout << " ------------ELEMENT GLOBAL INDICES ON TASK " << myrank << " --------------"<(Teuchos::OrdinalTraits::invalid(),All_Element_Global_Indices.d_view,0,comm)); - - //element type selection (subject to change) - // ---- Set Element Type ---- // - // allocate element type memory - //elements::elem_type_t* elem_choice; - - int NE = 1; // number of element types in problem - - - // Convert ensight index system to the ijk finite element numbering convention - // for vertices in cell - if(active_node_ordering_convention == IJK){ - CArrayKokkos convert_ensight_to_ijk(max_nodes_per_element); - CArrayKokkos tmp_ijk_indx(max_nodes_per_element); - convert_ensight_to_ijk(0) = 0; - convert_ensight_to_ijk(1) = 1; - convert_ensight_to_ijk(2) = 3; - convert_ensight_to_ijk(3) = 2; - convert_ensight_to_ijk(4) = 4; - convert_ensight_to_ijk(5) = 5; - convert_ensight_to_ijk(6) = 7; - convert_ensight_to_ijk(7) = 6; - - int nodes_per_element; - - if(num_dim==2) - for (int cell_rid = 0; cell_rid < rnum_elem; cell_rid++) { - //set nodes per element - element_select->choose_2Delem_type(Element_Types(cell_rid), elem2D); - nodes_per_element = elem2D->num_nodes(); - for (int node_lid = 0; node_lid < nodes_per_element; node_lid++){ - tmp_ijk_indx(node_lid) = nodes_in_elem(cell_rid, convert_ensight_to_ijk(node_lid)); - } - - for (int node_lid = 0; node_lid < nodes_per_element; node_lid++){ - nodes_in_elem(cell_rid, node_lid) = tmp_ijk_indx(node_lid); - } - } - - if(num_dim==3) - for (int cell_rid = 0; cell_rid < rnum_elem; cell_rid++) { - //set nodes per element - element_select->choose_3Delem_type(Element_Types(cell_rid), elem); - nodes_per_element = elem->num_nodes(); - for (int node_lid = 0; node_lid < nodes_per_element; node_lid++){ - tmp_ijk_indx(node_lid) = nodes_in_elem(cell_rid, convert_ensight_to_ijk(node_lid)); - } - - for (int node_lid = 0; node_lid < nodes_per_element; node_lid++){ - nodes_in_elem(cell_rid, node_lid) = tmp_ijk_indx(node_lid); - } - } - } - //debug print element edof - /* - std::cout << " ------------ELEMENT EDOF ON TASK " << myrank << " --------------"< read_buffer; - int buffer_loop, buffer_iteration, buffer_iterations, dof_limit, scan_loop; - size_t read_index_start, node_rid, elem_gid; - GO node_gid; - real_t dof_value; - host_vec_array node_densities; - bool zero_index_base = input_options.zero_index_base; - int negative_index_found = 0; - int global_negative_index_found = 0; - //Nodes_Per_Element_Type = elements::elem_types::Nodes_Per_Element_Type; - - //read the mesh - //PLACEHOLDER: ensight_format(MESH); - // abaqus_format(MESH); - // vtk_format(MESH) - - //task 0 reads file - if(myrank==0){ - in = new std::ifstream(); - in->open(MESH); - //skip 2 lines - for (int j = 1; j <= 2; j++) { - getline(*in, skip_line); - std::cout << skip_line << std::endl; - } //for - } - - - // --- Read the number of nodes in the mesh --- // - if(myrank==0){ - getline(*in, read_line); - line_parse.str(read_line); - //stop when the NODES= string is reached - while (!line_parse.eof()){ - line_parse >> substring; - if(!substring.compare("NODES=")){ - line_parse >> num_nodes; - } - if(!substring.compare("ELEMENTS=")){ - line_parse >> num_elem; - } - } //while - std::cout << "declared node count: " << num_nodes << std::endl; - std::cout << "declared element count: " << num_elem << std::endl; - if(num_elem <= 0) std::cout << "ERROR, NO ELEMENTS IN MESH!!!!" << std::endl; - } - - //broadcast number of nodes - MPI_Bcast(&num_nodes,1,MPI_LONG_LONG_INT,0,world); - - //construct contiguous parallel row map now that we know the number of nodes - map = Teuchos::rcp( new Tpetra::Map(num_nodes,0,comm)); - - // set the vertices in the mesh read in - nlocal_nodes = map->getLocalNumElements(); - //populate local row offset data from global data - global_size_t min_gid = map->getMinGlobalIndex(); - global_size_t max_gid = map->getMaxGlobalIndex(); - global_size_t index_base = map->getIndexBase(); - //debug print - //std::cout << "local node count on task: " << " " << nlocal_nodes << std::endl; - - //allocate node storage with dual view - //dual_node_coords = dual_vec_array("dual_node_coords", nlocal_nodes,num_dim); - //if(restart_file) - //dual_node_densities = dual_vec_array("dual_node_densities", nlocal_nodes,1); - - //local variable for host view in the dual view - node_coords_distributed = Teuchos::rcp(new MV(map, num_dim)); - //active view scrope - { - host_vec_array node_coords = node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); - if(restart_file){ - design_node_densities_distributed = Teuchos::rcp(new MV(map, 1)); - node_densities = design_node_densities_distributed->getLocalView (Tpetra::Access::ReadWrite); - } - //host_vec_array node_coords = dual_node_coords.view_host(); - //if(restart_file) - //node_densities = dual_node_densities.view_host(); - //notify that the host view is going to be modified in the file readin - //dual_node_coords.modify_host(); - //if(restart_file) - //dual_node_densities.modify_host(); - - //old swage method - //mesh->init_nodes(local_nrows); // add 1 for index starting at 1 - - std::cout << "Num nodes assigned to task " << myrank << " = " << nlocal_nodes << std::endl; - - // read the initial mesh coordinates - // x-coords - /*only task 0 reads in nodes and elements from the input file - stores node data in a buffer and communicates once the buffer cap is reached - or the data ends*/ - - words_per_line = input_options.words_per_line; - if(restart_file) words_per_line++; - elem_words_per_line = input_options.elem_words_per_line; - - //allocate read buffer - read_buffer = CArrayKokkos(BUFFER_LINES,words_per_line,MAX_WORD); - - dof_limit = num_nodes; - buffer_iterations = dof_limit/BUFFER_LINES; - if(dof_limit%BUFFER_LINES!=0) buffer_iterations++; - - //read coords, also density if restarting - read_index_start = 0; - for(buffer_iteration = 0; buffer_iteration < buffer_iterations; buffer_iteration++){ - //pack buffer on rank 0 - if(myrank==0&&buffer_iteration read_buffer; + int buffer_loop, buffer_iteration, buffer_iterations, dof_limit, scan_loop; + size_t read_index_start, node_rid, elem_gid; + GO node_gid; + real_t dof_value; + host_vec_array node_densities; + bool zero_index_base = input_options.zero_index_base; + int negative_index_found = 0; + int global_negative_index_found = 0; + // Nodes_Per_Element_Type = elements::elem_types::Nodes_Per_Element_Type; + + // read the mesh + // PLACEHOLDER: ensight_format(MESH); + // abaqus_format(MESH); + // vtk_format(MESH) + + // task 0 reads file + if (myrank == 0) + { + in = new std::ifstream(); + in->open(MESH); + // skip 2 lines + for (int j = 1; j <= 2; j++) + { + getline(*in, skip_line); + std::cout << skip_line << std::endl; + } // for + } + + // --- Read the number of nodes in the mesh --- // + if (myrank == 0) + { + getline(*in, read_line); line_parse.str(read_line); - - for(int iword = 0; iword < words_per_line; iword++){ - //read portions of the line into the substring variable - line_parse >> substring; - //debug print - //std::cout<<" "<< substring <> substring; + if (!substring.compare("NODES=")) + { + line_parse >> num_nodes; + } + if (!substring.compare("ELEMENTS=")) + { + line_parse >> num_elem; + } + } // while + std::cout << "declared node count: " << num_nodes << std::endl; + std::cout << "declared element count: " << num_elem << std::endl; + if (num_elem <= 0) + { + std::cout << "ERROR, NO ELEMENTS IN MESH!!!!" << std::endl; } - } } - else if(myrank==0){ - buffer_loop=0; - while(buffer_iteration*BUFFER_LINES+buffer_loop < num_nodes) { - getline(*in,read_line); - line_parse.clear(); - line_parse.str(read_line); - for(int iword = 0; iword < words_per_line; iword++){ - //read portions of the line into the substring variable - line_parse >> substring; - //assign the substring variable as a word of the read buffer - strcpy(&read_buffer(buffer_loop,iword,0),substring.c_str()); + + // broadcast number of nodes + MPI_Bcast(&num_nodes, 1, MPI_LONG_LONG_INT, 0, world); + + // construct contiguous parallel row map now that we know the number of nodes + map = Teuchos::rcp(new Tpetra::Map(num_nodes, 0, comm)); + + // set the vertices in the mesh read in + nlocal_nodes = map->getLocalNumElements(); + // populate local row offset data from global data + global_size_t min_gid = map->getMinGlobalIndex(); + global_size_t max_gid = map->getMaxGlobalIndex(); + global_size_t index_base = map->getIndexBase(); + // debug print + // std::cout << "local node count on task: " << " " << nlocal_nodes << std::endl; + + // allocate node storage with dual view + // dual_node_coords = dual_vec_array("dual_node_coords", nlocal_nodes,num_dim); + // if(restart_file) + // dual_node_densities = dual_vec_array("dual_node_densities", nlocal_nodes,1); + + // local variable for host view in the dual view + node_coords_distributed = Teuchos::rcp(new MV(map, num_dim)); + // active view scrope + { + host_vec_array node_coords = node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + if (restart_file) + { + design_node_densities_distributed = Teuchos::rcp(new MV(map, 1)); + node_densities = design_node_densities_distributed->getLocalView(Tpetra::Access::ReadWrite); + } + // host_vec_array node_coords = dual_node_coords.view_host(); + // if(restart_file) + // node_densities = dual_node_densities.view_host(); + // notify that the host view is going to be modified in the file readin + // dual_node_coords.modify_host(); + // if(restart_file) + // dual_node_densities.modify_host(); + + // old swage method + // mesh->init_nodes(local_nrows); // add 1 for index starting at 1 + + std::cout << "Num nodes assigned to task " << myrank << " = " << nlocal_nodes << std::endl; + + // read the initial mesh coordinates + // x-coords + /*only task 0 reads in nodes and elements from the input file + stores node data in a buffer and communicates once the buffer cap is reached + or the data ends*/ + + words_per_line = input_options.words_per_line; + if (restart_file) + { + words_per_line++; + } + elem_words_per_line = input_options.elem_words_per_line; + + // allocate read buffer + read_buffer = CArrayKokkos(BUFFER_LINES, words_per_line, MAX_WORD); + + dof_limit = num_nodes; + buffer_iterations = dof_limit / BUFFER_LINES; + if (dof_limit % BUFFER_LINES != 0) + { + buffer_iterations++; + } + + // read coords, also density if restarting + read_index_start = 0; + for (buffer_iteration = 0; buffer_iteration < buffer_iterations; buffer_iteration++) + { + // pack buffer on rank 0 + if (myrank == 0 && buffer_iteration < buffer_iterations - 1) + { + for (buffer_loop = 0; buffer_loop < BUFFER_LINES; buffer_loop++) + { + getline(*in, read_line); + line_parse.clear(); + line_parse.str(read_line); + + for (int iword = 0; iword < words_per_line; iword++) + { + // read portions of the line into the substring variable + line_parse >> substring; + // debug print + // std::cout<<" "<< substring <> substring; + // assign the substring variable as a word of the read buffer + strcpy(&read_buffer(buffer_loop, iword, 0), substring.c_str()); + } + buffer_loop++; + } + } + + // broadcast buffer to all ranks; each rank will determine which nodes in the buffer belong + MPI_Bcast(read_buffer.pointer(), BUFFER_LINES * words_per_line * MAX_WORD, MPI_CHAR, 0, world); + // broadcast how many nodes were read into this buffer iteration + MPI_Bcast(&buffer_loop, 1, MPI_INT, 0, world); + + // debug_print + // std::cout << "NODE BUFFER LOOP IS: " << buffer_loop << std::endl; + // for(int iprint=0; iprint < buffer_loop; iprint++) + // std::cout<<"buffer packing: " << std::string(&read_buffer(iprint,0,0)) << std::endl; + // return; + + // determine which data to store in the swage mesh members (the local node data) + // loop through read buffer + for (scan_loop = 0; scan_loop < buffer_loop; scan_loop++) + { + // set global node id (ensight specific order) + node_gid = read_index_start + scan_loop; + // let map decide if this node id belongs locally; if yes store data + if (map->isNodeGlobalElement(node_gid)) + { + // set local node index in this mpi rank + node_rid = map->getLocalElement(node_gid); + // extract nodal position from the read buffer + // for tecplot format this is the three coords in the same line + dof_value = atof(&read_buffer(scan_loop, 0, 0)); + node_coords(node_rid, 0) = dof_value * unit_scaling; + dof_value = atof(&read_buffer(scan_loop, 1, 0)); + node_coords(node_rid, 1) = dof_value * unit_scaling; + if (num_dim == 3) + { + dof_value = atof(&read_buffer(scan_loop, 2, 0)); + node_coords(node_rid, 2) = dof_value * unit_scaling; + } + if (restart_file) + { + dof_value = atof(&read_buffer(scan_loop, num_dim, 0)); + node_densities(node_rid, 0) = dof_value; + } + // extract density if restarting + } + } + read_index_start += BUFFER_LINES; } - buffer_loop++; - } - - } - - //broadcast buffer to all ranks; each rank will determine which nodes in the buffer belong - MPI_Bcast(read_buffer.pointer(),BUFFER_LINES*words_per_line*MAX_WORD,MPI_CHAR,0,world); - //broadcast how many nodes were read into this buffer iteration - MPI_Bcast(&buffer_loop,1,MPI_INT,0,world); - - //debug_print - //std::cout << "NODE BUFFER LOOP IS: " << buffer_loop << std::endl; - //for(int iprint=0; iprint < buffer_loop; iprint++) - //std::cout<<"buffer packing: " << std::string(&read_buffer(iprint,0,0)) << std::endl; - //return; - - //determine which data to store in the swage mesh members (the local node data) - //loop through read buffer - for(scan_loop = 0; scan_loop < buffer_loop; scan_loop++){ - //set global node id (ensight specific order) - node_gid = read_index_start + scan_loop; - //let map decide if this node id belongs locally; if yes store data - if(map->isNodeGlobalElement(node_gid)){ - //set local node index in this mpi rank - node_rid = map->getLocalElement(node_gid); - //extract nodal position from the read buffer - //for tecplot format this is the three coords in the same line - dof_value = atof(&read_buffer(scan_loop,0,0)); - node_coords(node_rid, 0) = dof_value * unit_scaling; - dof_value = atof(&read_buffer(scan_loop,1,0)); - node_coords(node_rid, 1) = dof_value * unit_scaling; - if(num_dim==3){ - dof_value = atof(&read_buffer(scan_loop,2,0)); - node_coords(node_rid, 2) = dof_value * unit_scaling; - } - if(restart_file){ - dof_value = atof(&read_buffer(scan_loop,num_dim,0)); - node_densities(node_rid, 0) = dof_value; - } - //extract density if restarting - } } - read_index_start+=BUFFER_LINES; - } - } - //repartition node distribution - repartition_nodes(); - - //synchronize device data - //dual_node_coords.sync_device(); - //dual_node_coords.modify_device(); - //if(restart_file){ - //dual_node_densities.sync_device(); - //dual_node_densities.modify_device(); - //} - - //debug print of nodal data - - //debug print nodal positions and indices - - //std::cout << " ------------NODAL POSITIONS ON TASK " << myrank << " --------------"<getGlobalElement(inode) + 1 << " { "; - //for (int istride = 0; istride < num_dim; istride++){ - //std::cout << node_coords(inode,istride) << " , "; - //} - //std::cout << node_densities(inode,0); - //std::cout << " }"<< std::endl; - //} - - - //check that local assignments match global total - - - //read in element info (supported tecplot format currently assumes one type) - - CArrayKokkos node_store(elem_words_per_line); - - //broadcast number of elements - MPI_Bcast(&num_elem,1,MPI_LONG_LONG_INT,0,world); - //std::cout<<"before initial mesh initialization"<(BUFFER_LINES,elem_words_per_line,MAX_WORD); - - //calculate buffer iterations to read number of lines - buffer_iterations = num_elem/BUFFER_LINES; - int assign_flag; - - //dynamic buffer used to store elements before we know how many this rank needs - std::vector element_temp(BUFFER_LINES*elem_words_per_line); - std::vector global_indices_temp(BUFFER_LINES); - size_t buffer_max = BUFFER_LINES*elem_words_per_line; - size_t indices_buffer_max = BUFFER_LINES; - - if(num_elem%BUFFER_LINES!=0) buffer_iterations++; - read_index_start = 0; - //std::cout << "ELEMENT BUFFER ITERATIONS: " << buffer_iterations << std::endl; - rnum_elem = 0; - //std::cout << "BUFFER ITERATIONS IS: " << buffer_iterations << std::endl; - for(buffer_iteration = 0; buffer_iteration < buffer_iterations; buffer_iteration++){ - //pack buffer on rank 0 - if(myrank==0&&buffer_iteration> substring; - //assign the substring variable as a word of the read buffer - strcpy(&read_buffer(buffer_loop,iword,0),substring.c_str()); + // repartition node distribution + repartition_nodes(); + + // synchronize device data + // dual_node_coords.sync_device(); + // dual_node_coords.modify_device(); + // if(restart_file){ + // dual_node_densities.sync_device(); + // dual_node_densities.modify_device(); + // } + + // debug print of nodal data + + // debug print nodal positions and indices + + // std::cout << " ------------NODAL POSITIONS ON TASK " << myrank << " --------------"<getGlobalElement(inode) + 1 << " { "; + // for (int istride = 0; istride < num_dim; istride++){ + // std::cout << node_coords(inode,istride) << " , "; + // } + // std::cout << node_densities(inode,0); + // std::cout << " }"<< std::endl; + // } + + // check that local assignments match global total + + // read in element info (supported tecplot format currently assumes one type) + + CArrayKokkos node_store(elem_words_per_line); + + // broadcast number of elements + MPI_Bcast(&num_elem, 1, MPI_LONG_LONG_INT, 0, world); + // std::cout<<"before initial mesh initialization"<(BUFFER_LINES, elem_words_per_line, MAX_WORD); + + // calculate buffer iterations to read number of lines + buffer_iterations = num_elem / BUFFER_LINES; + int assign_flag; + + // dynamic buffer used to store elements before we know how many this rank needs + std::vector element_temp(BUFFER_LINES * elem_words_per_line); + std::vector global_indices_temp(BUFFER_LINES); + size_t buffer_max = BUFFER_LINES * elem_words_per_line; + size_t indices_buffer_max = BUFFER_LINES; + + if (num_elem % BUFFER_LINES != 0) + { + buffer_iterations++; + } + read_index_start = 0; + // std::cout << "ELEMENT BUFFER ITERATIONS: " << buffer_iterations << std::endl; + rnum_elem = 0; + // std::cout << "BUFFER ITERATIONS IS: " << buffer_iterations << std::endl; + for (buffer_iteration = 0; buffer_iteration < buffer_iterations; buffer_iteration++) + { + // pack buffer on rank 0 + if (myrank == 0 && buffer_iteration < buffer_iterations - 1) + { + for (buffer_loop = 0; buffer_loop < BUFFER_LINES; buffer_loop++) + { + getline(*in, read_line); + line_parse.clear(); + line_parse.str(read_line); + for (int iword = 0; iword < elem_words_per_line; iword++) + { + // read portions of the line into the substring variable + line_parse >> substring; + // assign the substring variable as a word of the read buffer + strcpy(&read_buffer(buffer_loop, iword, 0), substring.c_str()); + } + } } - } + else if (myrank == 0) + { + buffer_loop = 0; + while (buffer_iteration * BUFFER_LINES + buffer_loop < num_elem) { + getline(*in, read_line); + line_parse.clear(); + line_parse.str(read_line); + for (int iword = 0; iword < elem_words_per_line; iword++) + { + // read portions of the line into the substring variable + line_parse >> substring; + // assign the substring variable as a word of the read buffer + strcpy(&read_buffer(buffer_loop, iword, 0), substring.c_str()); + } + buffer_loop++; + // std::cout<<" "<< node_coords(node_gid, 0)<isNodeGlobalElement(node_gid) && !assign_flag) + { + assign_flag = 1; + rnum_elem++; + } + } + else + { + if (map->isNodeGlobalElement(node_gid - 1) && !assign_flag) + { + assign_flag = 1; + rnum_elem++; + } + } + } + + if (assign_flag) + { + for (int inode = 0; inode < elem_words_per_line; inode++) + { + if ((rnum_elem - 1) * elem_words_per_line + inode >= buffer_max) + { + element_temp.resize((rnum_elem - 1) * elem_words_per_line + inode + BUFFER_LINES * elem_words_per_line); + buffer_max = (rnum_elem - 1) * elem_words_per_line + inode + BUFFER_LINES * elem_words_per_line; + } + element_temp[(rnum_elem - 1) * elem_words_per_line + inode] = node_store(inode); + // std::cout << "VECTOR STORAGE FOR ELEM " << rnum_elem << " ON TASK " << myrank << " NODE " << inode+1 << " IS " << node_store(inode) + 1 << std::endl; + } + // assign global element id to temporary list + if (rnum_elem - 1 >= indices_buffer_max) + { + global_indices_temp.resize(rnum_elem - 1 + BUFFER_LINES); + indices_buffer_max = rnum_elem - 1 + BUFFER_LINES; + } + global_indices_temp[rnum_elem - 1] = elem_gid; + } + } + read_index_start += BUFFER_LINES; } - else if(myrank==0){ - buffer_loop=0; - while(buffer_iteration*BUFFER_LINES+buffer_loop < num_elem) { - getline(*in,read_line); - line_parse.clear(); - line_parse.str(read_line); - for(int iword = 0; iword < elem_words_per_line; iword++){ - //read portions of the line into the substring variable - line_parse >> substring; - //assign the substring variable as a word of the read buffer - strcpy(&read_buffer(buffer_loop,iword,0),substring.c_str()); - } - buffer_loop++; - //std::cout<<" "<< node_coords(node_gid, 0)<close(); } - //broadcast buffer to all ranks; each rank will determine which nodes in the buffer belong - MPI_Bcast(read_buffer.pointer(),BUFFER_LINES*elem_words_per_line*MAX_WORD,MPI_CHAR,0,world); - //broadcast how many nodes were read into this buffer iteration - MPI_Bcast(&buffer_loop,1,MPI_INT,0,world); - - //store element connectivity that belongs to this rank - //loop through read buffer - for(scan_loop = 0; scan_loop < buffer_loop; scan_loop++){ - //set global node id (ensight specific order) - elem_gid = read_index_start + scan_loop; - //add this element to the local list if any of its nodes belong to this rank according to the map - //get list of nodes for each element line and check if they belong to the map - assign_flag = 0; - for(int inode = 0; inode < elem_words_per_line; inode++){ - //as we loop through the nodes belonging to this element we store them - //if any of these nodes belongs to this rank this list is used to store the element locally - node_gid = atoi(&read_buffer(scan_loop,inode,0)); - if(zero_index_base) - node_store(inode) = node_gid; //subtract 1 since file index start is 1 but code expects 0 + std::cout << "RNUM ELEMENTS IS: " << rnum_elem << std::endl; + // copy temporary element storage to multivector storage + Element_Types = CArrayKokkos(rnum_elem); + + elements::elem_types::elem_type mesh_element_type; + + if (simparam.num_dims == 2) + { + if (input_options.element_type == ELEMENT_TYPE::quad4) + { + mesh_element_type = elements::elem_types::Quad4; + max_nodes_per_patch = 2; + } + else if (input_options.element_type == ELEMENT_TYPE::quad8) + { + mesh_element_type = elements::elem_types::Quad8; + max_nodes_per_patch = 3; + } + else if (input_options.element_type == ELEMENT_TYPE::quad12) + { + mesh_element_type = elements::elem_types::Quad12; + max_nodes_per_patch = 4; + } else - node_store(inode) = node_gid - 1; //subtract 1 since file index start is 1 but code expects 0 - if(node_store(inode) < 0){ - negative_index_found = 1; - } - //first we add the elements to a dynamically allocated list - if(zero_index_base){ - if(map->isNodeGlobalElement(node_gid)&&!assign_flag){ - assign_flag = 1; - rnum_elem++; - } - } - else{ - if(map->isNodeGlobalElement(node_gid-1)&&!assign_flag){ - assign_flag = 1; - rnum_elem++; - } + { + if (myrank == 0) + { + std::cout << "ELEMENT TYPE UNRECOGNIZED" << std::endl; + } + exit_solver(0); } - } - - if(assign_flag){ - for(int inode = 0; inode < elem_words_per_line; inode++){ - if((rnum_elem-1)*elem_words_per_line + inode>=buffer_max){ - element_temp.resize((rnum_elem-1)*elem_words_per_line + inode + BUFFER_LINES*elem_words_per_line); - buffer_max = (rnum_elem-1)*elem_words_per_line + inode + BUFFER_LINES*elem_words_per_line; - } - element_temp[(rnum_elem-1)*elem_words_per_line + inode] = node_store(inode); - //std::cout << "VECTOR STORAGE FOR ELEM " << rnum_elem << " ON TASK " << myrank << " NODE " << inode+1 << " IS " << node_store(inode) + 1 << std::endl; - } - //assign global element id to temporary list - if(rnum_elem-1>=indices_buffer_max){ - global_indices_temp.resize(rnum_elem-1 + BUFFER_LINES); - indices_buffer_max = rnum_elem-1 + BUFFER_LINES; - } - global_indices_temp[rnum_elem-1] = elem_gid; - } + element_select->choose_2Delem_type(mesh_element_type, elem2D); + max_nodes_per_element = elem2D->num_nodes(); } - read_index_start+=BUFFER_LINES; - } - - // Close mesh input file - if(myrank==0) - in->close(); - - std::cout << "RNUM ELEMENTS IS: " << rnum_elem << std::endl; - //copy temporary element storage to multivector storage - Element_Types = CArrayKokkos(rnum_elem); - - elements::elem_types::elem_type mesh_element_type; - - if(simparam.num_dims == 2){ - if(input_options.element_type == ELEMENT_TYPE::quad4){ - mesh_element_type = elements::elem_types::Quad4; - max_nodes_per_patch = 2; - } - else if(input_options.element_type == ELEMENT_TYPE::quad8){ - mesh_element_type = elements::elem_types::Quad8; - max_nodes_per_patch = 3; - } - else if(input_options.element_type == ELEMENT_TYPE::quad12){ - mesh_element_type = elements::elem_types::Quad12; - max_nodes_per_patch = 4; - } - else{ - if(myrank==0){ - std::cout << "ELEMENT TYPE UNRECOGNIZED" << std::endl; - } - exit_solver(0); + + if (simparam.num_dims == 3) + { + if (input_options.element_type == ELEMENT_TYPE::hex8) + { + mesh_element_type = elements::elem_types::Hex8; + max_nodes_per_patch = 4; + } + else if (input_options.element_type == ELEMENT_TYPE::hex20) + { + mesh_element_type = elements::elem_types::Hex20; + max_nodes_per_patch = 8; + } + else if (input_options.element_type == ELEMENT_TYPE::hex32) + { + mesh_element_type = elements::elem_types::Hex32; + max_nodes_per_patch = 12; + } + else + { + if (myrank == 0) + { + std::cout << "ELEMENT TYPE UNRECOGNIZED" << std::endl; + } + exit_solver(0); + } + element_select->choose_3Delem_type(mesh_element_type, elem); + max_nodes_per_element = elem->num_nodes(); } - element_select->choose_2Delem_type(mesh_element_type, elem2D); - max_nodes_per_element = elem2D->num_nodes(); - } - if(simparam.num_dims == 3){ - if(input_options.element_type == ELEMENT_TYPE::hex8){ - mesh_element_type = elements::elem_types::Hex8; - max_nodes_per_patch = 4; + dual_nodes_in_elem = dual_elem_conn_array("dual_nodes_in_elem", rnum_elem, max_nodes_per_element); + host_elem_conn_array nodes_in_elem = dual_nodes_in_elem.view_host(); + dual_nodes_in_elem.modify_host(); + + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + for (int inode = 0; inode < elem_words_per_line; inode++) + { + nodes_in_elem(ielem, inode) = element_temp[ielem * elem_words_per_line + inode]; + } } - else if(input_options.element_type == ELEMENT_TYPE::hex20){ - mesh_element_type = elements::elem_types::Hex20; - max_nodes_per_patch = 8; + + // view storage for all local elements connected to local nodes on this rank + Kokkos::DualView All_Element_Global_Indices("All_Element_Global_Indices", rnum_elem); + // copy temporary global indices storage to view storage + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + All_Element_Global_Indices.h_view(ielem) = global_indices_temp[ielem]; + if (global_indices_temp[ielem] < 0) + { + negative_index_found = 1; + } } - else if(input_options.element_type == ELEMENT_TYPE::hex32){ - mesh_element_type = elements::elem_types::Hex32; - max_nodes_per_patch = 12; + + MPI_Allreduce(&negative_index_found, &global_negative_index_found, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD); + if (global_negative_index_found) + { + if (myrank == 0) + { + std::cout << "Node index less than or equal to zero detected; set \"zero_index_base: true\" under \"input_options\" in your yaml file if indices start at 0" << std::endl; + } + exit_solver(0); + } + + // delete temporary element connectivity and index storage + std::vector().swap(element_temp); + std::vector().swap(global_indices_temp); + + All_Element_Global_Indices.modify_host(); + All_Element_Global_Indices.sync_device(); + + // construct overlapping element map (since different ranks can own the same elements due to the local node map) + all_element_map = Teuchos::rcp(new Tpetra::Map(Teuchos::OrdinalTraits::invalid(), All_Element_Global_Indices.d_view, 0, comm)); + + // element type selection (subject to change) + // ---- Set Element Type ---- // + // allocate element type memory + // elements::elem_type_t* elem_choice; + + int NE = 1; // number of element types in problem + + // Convert ijk index system to the finite element numbering convention + // for vertices in cell + if (active_node_ordering_convention == IJK) + { + CArrayKokkos convert_ensight_to_ijk(max_nodes_per_element); + CArrayKokkos tmp_ijk_indx(max_nodes_per_element); + convert_ensight_to_ijk(0) = 0; + convert_ensight_to_ijk(1) = 1; + convert_ensight_to_ijk(2) = 3; + convert_ensight_to_ijk(3) = 2; + convert_ensight_to_ijk(4) = 4; + convert_ensight_to_ijk(5) = 5; + convert_ensight_to_ijk(6) = 7; + convert_ensight_to_ijk(7) = 6; + + int nodes_per_element; + + if (num_dim == 2) + { + for (int cell_rid = 0; cell_rid < rnum_elem; cell_rid++) + { + // set nodes per element + element_select->choose_2Delem_type(Element_Types(cell_rid), elem2D); + nodes_per_element = elem2D->num_nodes(); + for (int node_lid = 0; node_lid < nodes_per_element; node_lid++) + { + tmp_ijk_indx(node_lid) = nodes_in_elem(cell_rid, convert_ensight_to_ijk(node_lid)); + } + + for (int node_lid = 0; node_lid < nodes_per_element; node_lid++) + { + nodes_in_elem(cell_rid, node_lid) = tmp_ijk_indx(node_lid); + } + } + } + + if (num_dim == 3) + { + for (int cell_rid = 0; cell_rid < rnum_elem; cell_rid++) + { + // set nodes per element + element_select->choose_3Delem_type(Element_Types(cell_rid), elem); + nodes_per_element = elem->num_nodes(); + for (int node_lid = 0; node_lid < nodes_per_element; node_lid++) + { + tmp_ijk_indx(node_lid) = nodes_in_elem(cell_rid, convert_ensight_to_ijk(node_lid)); + } + + for (int node_lid = 0; node_lid < nodes_per_element; node_lid++) + { + nodes_in_elem(cell_rid, node_lid) = tmp_ijk_indx(node_lid); + } + } + } } - else{ - if(myrank==0){ - std::cout << "ELEMENT TYPE UNRECOGNIZED" << std::endl; - } - exit_solver(0); - } - element_select->choose_3Delem_type(mesh_element_type, elem); - max_nodes_per_element = elem->num_nodes(); - } - - dual_nodes_in_elem = dual_elem_conn_array("dual_nodes_in_elem", rnum_elem, max_nodes_per_element); - host_elem_conn_array nodes_in_elem = dual_nodes_in_elem.view_host(); - dual_nodes_in_elem.modify_host(); - - for(int ielem = 0; ielem < rnum_elem; ielem++) - for(int inode = 0; inode < elem_words_per_line; inode++){ - nodes_in_elem(ielem, inode) = element_temp[ielem*elem_words_per_line + inode]; - } - - //view storage for all local elements connected to local nodes on this rank - Kokkos::DualView All_Element_Global_Indices("All_Element_Global_Indices",rnum_elem); - //copy temporary global indices storage to view storage - for(int ielem = 0; ielem < rnum_elem; ielem++){ - All_Element_Global_Indices.h_view(ielem) = global_indices_temp[ielem]; - if(global_indices_temp[ielem]<0){ - negative_index_found = 1; - } - } - - MPI_Allreduce(&negative_index_found,&global_negative_index_found,1,MPI_INT,MPI_MAX,MPI_COMM_WORLD); - if(global_negative_index_found){ - if(myrank==0){ - std::cout << "Node index less than or equal to zero detected; set \"zero_index_base: true\" under \"input_options\" in your yaml file if indices start at 0" << std::endl; - } - exit_solver(0); - } - - //delete temporary element connectivity and index storage - std::vector().swap(element_temp); - std::vector().swap(global_indices_temp); - - All_Element_Global_Indices.modify_host(); - All_Element_Global_Indices.sync_device(); - - //construct overlapping element map (since different ranks can own the same elements due to the local node map) - all_element_map = Teuchos::rcp( new Tpetra::Map(Teuchos::OrdinalTraits::invalid(),All_Element_Global_Indices.d_view,0,comm)); - - - //element type selection (subject to change) - // ---- Set Element Type ---- // - // allocate element type memory - //elements::elem_type_t* elem_choice; - - int NE = 1; // number of element types in problem - - // Convert ijk index system to the finite element numbering convention - // for vertices in cell - if(active_node_ordering_convention == IJK){ - CArrayKokkos convert_ensight_to_ijk(max_nodes_per_element); - CArrayKokkos tmp_ijk_indx(max_nodes_per_element); - convert_ensight_to_ijk(0) = 0; - convert_ensight_to_ijk(1) = 1; - convert_ensight_to_ijk(2) = 3; - convert_ensight_to_ijk(3) = 2; - convert_ensight_to_ijk(4) = 4; - convert_ensight_to_ijk(5) = 5; - convert_ensight_to_ijk(6) = 7; - convert_ensight_to_ijk(7) = 6; - - int nodes_per_element; - - if(num_dim==2) - for (int cell_rid = 0; cell_rid < rnum_elem; cell_rid++) { - //set nodes per element - element_select->choose_2Delem_type(Element_Types(cell_rid), elem2D); - nodes_per_element = elem2D->num_nodes(); - for (int node_lid = 0; node_lid < nodes_per_element; node_lid++){ - tmp_ijk_indx(node_lid) = nodes_in_elem(cell_rid, convert_ensight_to_ijk(node_lid)); - } - - for (int node_lid = 0; node_lid < nodes_per_element; node_lid++){ - nodes_in_elem(cell_rid, node_lid) = tmp_ijk_indx(node_lid); - } - } - - if(num_dim==3) - for (int cell_rid = 0; cell_rid < rnum_elem; cell_rid++) { - //set nodes per element - element_select->choose_3Delem_type(Element_Types(cell_rid), elem); - nodes_per_element = elem->num_nodes(); - for (int node_lid = 0; node_lid < nodes_per_element; node_lid++){ - tmp_ijk_indx(node_lid) = nodes_in_elem(cell_rid, convert_ensight_to_ijk(node_lid)); - } - - for (int node_lid = 0; node_lid < nodes_per_element; node_lid++){ - nodes_in_elem(cell_rid, node_lid) = tmp_ijk_indx(node_lid); - } - } - } - - //debug print element edof - - //std::cout << " ------------ELEMENT EDOF ON TASK " << myrank << " --------------"< read_buffer; - int nodes_per_element; - GO node_gid; - - //construct input adapted needed by Zoltan2 problem - typedef Xpetra::MultiVector xvector_t; - typedef Zoltan2::XpetraMultiVectorAdapter inputAdapter_t; - typedef Zoltan2::EvaluatePartition quality_t; - - Teuchos::RCP xpetra_node_coords = Teuchos::rcp(new Xpetra::TpetraMultiVector(node_coords_distributed)); - Teuchos::RCP problem_adapter = Teuchos::rcp(new inputAdapter_t(xpetra_node_coords)); - - // Create parameters for an RCB problem - - double tolerance = 1.05; - - Teuchos::ParameterList params("Node Partition Params"); - params.set("debug_level", "basic_status"); - params.set("debug_procs", "0"); - params.set("error_check_level", "debug_mode_assertions"); - - //params.set("algorithm", "rcb"); - params.set("algorithm", "multijagged"); - params.set("imbalance_tolerance", tolerance ); - params.set("num_global_parts", nranks); - params.set("partitioning_objective", "minimize_cut_edge_count"); - - Teuchos::RCP > problem = - Teuchos::rcp(new Zoltan2::PartitioningProblem(&(*problem_adapter), ¶ms)); - - // Solve the problem - - problem->solve(); - - // create metric object where communicator is Teuchos default - - quality_t *metricObject1 = new quality_t(&(*problem_adapter), ¶ms, //problem1->getComm(), - &problem->getSolution()); - // Check the solution. - - if (myrank == 0) { - metricObject1->printMetrics(std::cout); - } - - if (myrank == 0){ - real_t imb = metricObject1->getObjectCountImbalance(); - if (imb <= tolerance) - std::cout << "pass: " << imb << std::endl; - else - std::cout << "fail: " << imb << std::endl; - std::cout << std::endl; - } - delete metricObject1; - - //migrate rows of the vector so they correspond to the partition recommended by Zoltan2 - Teuchos::RCP partitioned_node_coords_distributed = Teuchos::rcp(new MV(map,num_dim)); - Teuchos::RCP xpartitioned_node_coords_distributed = - Teuchos::rcp(new Xpetra::TpetraMultiVector(partitioned_node_coords_distributed)); - - problem_adapter->applyPartitioningSolution(*xpetra_node_coords, xpartitioned_node_coords_distributed, problem->getSolution()); - *partitioned_node_coords_distributed = Xpetra::toTpetra(*xpartitioned_node_coords_distributed); - Teuchos::RCP > partitioned_map = Teuchos::rcp(new Tpetra::Map(*(partitioned_node_coords_distributed->getMap()))); - Teuchos::RCP > partitioned_map_one_to_one; - partitioned_map_one_to_one = Tpetra::createOneToOne(partitioned_map); - Teuchos::RCP partitioned_node_coords_one_to_one_distributed = Teuchos::rcp(new MV(partitioned_map_one_to_one,num_dim)); - - Tpetra::Import importer_one_to_one(partitioned_map, partitioned_map_one_to_one); - partitioned_node_coords_one_to_one_distributed->doImport(*partitioned_node_coords_distributed, importer_one_to_one, Tpetra::INSERT); - node_coords_distributed = partitioned_node_coords_one_to_one_distributed; - partitioned_map = Teuchos::rcp(new Tpetra::Map(*partitioned_map_one_to_one)); - - //migrate density vector if this is a restart file read - if(simparam.restart_file){ - Teuchos::RCP partitioned_node_densities_distributed = Teuchos::rcp(new MV(partitioned_map, 1)); - - //create import object using local node indices map and all indices map - Tpetra::Import importer(map, partitioned_map); - - //comms to get ghosts - partitioned_node_densities_distributed->doImport(*design_node_densities_distributed, importer, Tpetra::INSERT); - design_node_densities_distributed = partitioned_node_densities_distributed; - } - - //update nlocal_nodes and node map - map = partitioned_map; - nlocal_nodes = map->getLocalNumElements(); - +void Solver::repartition_nodes() +{ + char ch; + int num_dim = simparam.num_dims; + int local_node_index, current_column_index; + size_t strain_count; + std::stringstream line_parse; + CArrayKokkos read_buffer; + int nodes_per_element; + GO node_gid; + + // construct input adapted needed by Zoltan2 problem + typedef Xpetra::MultiVector xvector_t; + typedef Zoltan2::XpetraMultiVectorAdapter inputAdapter_t; + typedef Zoltan2::EvaluatePartition quality_t; + + Teuchos::RCP xpetra_node_coords = Teuchos::rcp(new Xpetra::TpetraMultiVector(node_coords_distributed)); + Teuchos::RCP problem_adapter = Teuchos::rcp(new inputAdapter_t(xpetra_node_coords)); + + // Create parameters for an RCB problem + + double tolerance = 1.05; + + Teuchos::ParameterList params("Node Partition Params"); + params.set("debug_level", "basic_status"); + params.set("debug_procs", "0"); + params.set("error_check_level", "debug_mode_assertions"); + + // params.set("algorithm", "rcb"); + params.set("algorithm", "multijagged"); + params.set("imbalance_tolerance", tolerance); + params.set("num_global_parts", nranks); + params.set("partitioning_objective", "minimize_cut_edge_count"); + + Teuchos::RCP> problem = + Teuchos::rcp(new Zoltan2::PartitioningProblem(&(*problem_adapter), ¶ms)); + + // Solve the problem + + problem->solve(); + + // create metric object where communicator is Teuchos default + + quality_t* metricObject1 = new quality_t(&(*problem_adapter), ¶ms, // problem1->getComm(), + &problem->getSolution()); + // Check the solution. + + if (myrank == 0) + { + metricObject1->printMetrics(std::cout); + } + + if (myrank == 0) + { + real_t imb = metricObject1->getObjectCountImbalance(); + if (imb <= tolerance) + { + std::cout << "pass: " << imb << std::endl; + } + else + { + std::cout << "fail: " << imb << std::endl; + } + std::cout << std::endl; + } + delete metricObject1; + + // migrate rows of the vector so they correspond to the partition recommended by Zoltan2 + Teuchos::RCP partitioned_node_coords_distributed = Teuchos::rcp(new MV(map, num_dim)); + Teuchos::RCP xpartitioned_node_coords_distributed = + Teuchos::rcp(new Xpetra::TpetraMultiVector(partitioned_node_coords_distributed)); + + problem_adapter->applyPartitioningSolution(*xpetra_node_coords, xpartitioned_node_coords_distributed, problem->getSolution()); + *partitioned_node_coords_distributed = Xpetra::toTpetra(*xpartitioned_node_coords_distributed); + Teuchos::RCP> partitioned_map = Teuchos::rcp(new Tpetra::Map(*(partitioned_node_coords_distributed->getMap()))); + Teuchos::RCP> partitioned_map_one_to_one; + partitioned_map_one_to_one = Tpetra::createOneToOne(partitioned_map); + Teuchos::RCP partitioned_node_coords_one_to_one_distributed = Teuchos::rcp(new MV(partitioned_map_one_to_one, num_dim)); + + Tpetra::Import importer_one_to_one(partitioned_map, partitioned_map_one_to_one); + partitioned_node_coords_one_to_one_distributed->doImport(*partitioned_node_coords_distributed, importer_one_to_one, Tpetra::INSERT); + node_coords_distributed = partitioned_node_coords_one_to_one_distributed; + partitioned_map = Teuchos::rcp(new Tpetra::Map(*partitioned_map_one_to_one)); + + // migrate density vector if this is a restart file read + if (simparam.restart_file) + { + Teuchos::RCP partitioned_node_densities_distributed = Teuchos::rcp(new MV(partitioned_map, 1)); + + // create import object using local node indices map and all indices map + Tpetra::Import importer(map, partitioned_map); + + // comms to get ghosts + partitioned_node_densities_distributed->doImport(*design_node_densities_distributed, importer, Tpetra::INSERT); + design_node_densities_distributed = partitioned_node_densities_distributed; + } + + // update nlocal_nodes and node map + map = partitioned_map; + nlocal_nodes = map->getLocalNumElements(); } /* ---------------------------------------------------------------------- Initialize Ghost and Non-Overlapping Element Maps ------------------------------------------------------------------------- */ -void Solver::init_maps(){ - char ch; - int num_dim = simparam.num_dims; - int local_node_index, current_column_index; - int nodes_per_element; - GO node_gid; - host_elem_conn_array nodes_in_elem = dual_nodes_in_elem.view_host(); - - if(rnum_elem >= 1) { - - //Construct set of ghost nodes; start with a buffer with upper limit - size_t buffer_limit = 0; - if(num_dim==2) - for(int ielem = 0; ielem < rnum_elem; ielem++){ - element_select->choose_2Delem_type(Element_Types(ielem), elem2D); - buffer_limit += elem2D->num_nodes(); - } - - if(num_dim==3) - for(int ielem = 0; ielem < rnum_elem; ielem++){ - element_select->choose_3Delem_type(Element_Types(ielem), elem); - buffer_limit += elem->num_nodes(); - } - - CArrayKokkos ghost_node_buffer(buffer_limit); - std::set ghost_node_set; - - //search through local elements for global node indices not owned by this MPI rank - if(num_dim==2) - for (int cell_rid = 0; cell_rid < rnum_elem; cell_rid++) { - //set nodes per element - element_select->choose_2Delem_type(Element_Types(cell_rid), elem2D); - nodes_per_element = elem2D->num_nodes(); - for (int node_lid = 0; node_lid < nodes_per_element; node_lid++){ - node_gid = nodes_in_elem(cell_rid, node_lid); - if(!map->isNodeGlobalElement(node_gid)) ghost_node_set.insert(node_gid); - } +void Solver::init_maps() +{ + char ch; + int num_dim = simparam.num_dims; + int local_node_index, current_column_index; + int nodes_per_element; + GO node_gid; + host_elem_conn_array nodes_in_elem = dual_nodes_in_elem.view_host(); + + if (rnum_elem >= 1) + { + // Construct set of ghost nodes; start with a buffer with upper limit + size_t buffer_limit = 0; + if (num_dim == 2) + { + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + element_select->choose_2Delem_type(Element_Types(ielem), elem2D); + buffer_limit += elem2D->num_nodes(); + } + } + + if (num_dim == 3) + { + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + element_select->choose_3Delem_type(Element_Types(ielem), elem); + buffer_limit += elem->num_nodes(); + } + } + + CArrayKokkos ghost_node_buffer(buffer_limit); + std::set ghost_node_set; + + // search through local elements for global node indices not owned by this MPI rank + if (num_dim == 2) + { + for (int cell_rid = 0; cell_rid < rnum_elem; cell_rid++) + { + // set nodes per element + element_select->choose_2Delem_type(Element_Types(cell_rid), elem2D); + nodes_per_element = elem2D->num_nodes(); + for (int node_lid = 0; node_lid < nodes_per_element; node_lid++) + { + node_gid = nodes_in_elem(cell_rid, node_lid); + if (!map->isNodeGlobalElement(node_gid)) + { + ghost_node_set.insert(node_gid); + } + } + } + } + + if (num_dim == 3) + { + for (int cell_rid = 0; cell_rid < rnum_elem; cell_rid++) + { + // set nodes per element + element_select->choose_3Delem_type(Element_Types(cell_rid), elem); + nodes_per_element = elem->num_nodes(); + for (int node_lid = 0; node_lid < nodes_per_element; node_lid++) + { + node_gid = nodes_in_elem(cell_rid, node_lid); + if (!map->isNodeGlobalElement(node_gid)) + { + ghost_node_set.insert(node_gid); + } + } + } + } + + // by now the set contains, with no repeats, all the global node indices that are ghosts for this rank + // now pass the contents of the set over to a CArrayKokkos, then create a map to find local ghost indices from global ghost indices + nghost_nodes = ghost_node_set.size(); + ghost_nodes = Kokkos::DualView("ghost_nodes", nghost_nodes); + ghost_node_ranks = Kokkos::DualView("ghost_node_ranks", nghost_nodes); + int ighost = 0; + auto it = ghost_node_set.begin(); + while (it != ghost_node_set.end()) { + ghost_nodes.h_view(ighost++) = *it; + it++; + } + + // debug print of ghost nodes + // std::cout << " GHOST NODE SET ON TASK " << myrank << std::endl; + // for(int i = 0; i < nghost_nodes; i++) + // std::cout << "{" << i + 1 << "," << ghost_nodes(i) + 1 << "}" << std::endl; + + // find which mpi rank each ghost node belongs to and store the information in a CArrayKokkos + // allocate Teuchos Views since they are the only input available at the moment in the map definitions + Teuchos::ArrayView ghost_nodes_pass(ghost_nodes.h_view.data(), nghost_nodes); + Teuchos::ArrayView ghost_node_ranks_pass(ghost_node_ranks.h_view.data(), nghost_nodes); + map->getRemoteIndexList(ghost_nodes_pass, ghost_node_ranks_pass); + + // debug print of ghost nodes + // std::cout << " GHOST NODE MAP ON TASK " << myrank << std::endl; + // for(int i = 0; i < nghost_nodes; i++) + // std::cout << "{" << i + 1 << "," << global2local_map.get(ghost_nodes(i)) + 1 << "}" << std::endl; + } + + ghost_nodes.modify_host(); + ghost_nodes.sync_device(); + ghost_node_ranks.modify_host(); + ghost_node_ranks.sync_device(); + // create a Map for ghost node indices + ghost_node_map = Teuchos::rcp(new Tpetra::Map(Teuchos::OrdinalTraits::invalid(), ghost_nodes.d_view, 0, comm)); + + // Create reference element + // ref_elem->init(p_order, num_dim, elem->num_basis()); + // std::cout<<"done with ref elem"< all_node_indices(nall_nodes, "all_node_indices"); + Kokkos::DualView all_node_indices("all_node_indices", nall_nodes); + for (int i = 0; i < nall_nodes; i++) + { + if (i < nlocal_nodes) + { + all_node_indices.h_view(i) = map->getGlobalElement(i); + } + else + { + all_node_indices.h_view(i) = ghost_nodes.h_view(i - nlocal_nodes); + } + } + all_node_indices.modify_host(); + all_node_indices.sync_device(); + // debug print of node indices + // for(int inode=0; inode < index_counter; inode++) + // std::cout << " my_reduced_global_indices " << my_reduced_global_indices(inode) <(Teuchos::OrdinalTraits::invalid(), all_node_indices.d_view, 0, comm)); + + // remove elements from the local set so that each rank has a unique set of global ids + + // local elements belonging to the non-overlapping element distribution to each rank with buffer + Kokkos::DualView Initial_Element_Global_Indices("Initial_Element_Global_Indices", rnum_elem); + size_t nonoverlapping_count = 0; + int my_element_flag; + // loop through local element set + if (num_dim == 2) + { + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + element_select->choose_2Delem_type(Element_Types(ielem), elem2D); + nodes_per_element = elem2D->num_nodes(); + my_element_flag = 1; + for (int lnode = 0; lnode < nodes_per_element; lnode++) + { + node_gid = nodes_in_elem(ielem, lnode); + if (ghost_node_map->isNodeGlobalElement(node_gid)) + { + local_node_index = ghost_node_map->getLocalElement(node_gid); + if (ghost_node_ranks.h_view(local_node_index) < myrank) + { + my_element_flag = 0; + } + } + } + if (my_element_flag) + { + Initial_Element_Global_Indices.h_view(nonoverlapping_count++) = all_element_map->getGlobalElement(ielem); + } + } } - if(num_dim==3) - for (int cell_rid = 0; cell_rid < rnum_elem; cell_rid++) { - //set nodes per element - element_select->choose_3Delem_type(Element_Types(cell_rid), elem); - nodes_per_element = elem->num_nodes(); - for (int node_lid = 0; node_lid < nodes_per_element; node_lid++){ - node_gid = nodes_in_elem(cell_rid, node_lid); - if(!map->isNodeGlobalElement(node_gid)) ghost_node_set.insert(node_gid); - } + if (num_dim == 3) + { + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + element_select->choose_3Delem_type(Element_Types(ielem), elem); + nodes_per_element = elem->num_nodes(); + my_element_flag = 1; + for (int lnode = 0; lnode < nodes_per_element; lnode++) + { + node_gid = nodes_in_elem(ielem, lnode); + if (ghost_node_map->isNodeGlobalElement(node_gid)) + { + local_node_index = ghost_node_map->getLocalElement(node_gid); + if (ghost_node_ranks.h_view(local_node_index) < myrank) + { + my_element_flag = 0; + } + } + } + if (my_element_flag) + { + Initial_Element_Global_Indices.h_view(nonoverlapping_count++) = all_element_map->getGlobalElement(ielem); + } + } } - //by now the set contains, with no repeats, all the global node indices that are ghosts for this rank - //now pass the contents of the set over to a CArrayKokkos, then create a map to find local ghost indices from global ghost indices - nghost_nodes = ghost_node_set.size(); - ghost_nodes = Kokkos::DualView ("ghost_nodes", nghost_nodes); - ghost_node_ranks = Kokkos::DualView ("ghost_node_ranks", nghost_nodes); - int ighost = 0; - auto it = ghost_node_set.begin(); - while(it!=ghost_node_set.end()){ - ghost_nodes.h_view(ighost++) = *it; - it++; - } - - //debug print of ghost nodes - //std::cout << " GHOST NODE SET ON TASK " << myrank << std::endl; - //for(int i = 0; i < nghost_nodes; i++) - //std::cout << "{" << i + 1 << "," << ghost_nodes(i) + 1 << "}" << std::endl; - - //find which mpi rank each ghost node belongs to and store the information in a CArrayKokkos - //allocate Teuchos Views since they are the only input available at the moment in the map definitions - Teuchos::ArrayView ghost_nodes_pass(ghost_nodes.h_view.data(), nghost_nodes); - Teuchos::ArrayView ghost_node_ranks_pass(ghost_node_ranks.h_view.data(), nghost_nodes); - map->getRemoteIndexList(ghost_nodes_pass, ghost_node_ranks_pass); - - //debug print of ghost nodes - //std::cout << " GHOST NODE MAP ON TASK " << myrank << std::endl; - //for(int i = 0; i < nghost_nodes; i++) - //std::cout << "{" << i + 1 << "," << global2local_map.get(ghost_nodes(i)) + 1 << "}" << std::endl; - - } - - ghost_nodes.modify_host(); - ghost_nodes.sync_device(); - ghost_node_ranks.modify_host(); - ghost_node_ranks.sync_device(); - // create a Map for ghost node indices - ghost_node_map = Teuchos::rcp( new Tpetra::Map(Teuchos::OrdinalTraits::invalid(),ghost_nodes.d_view,0,comm)); - - // Create reference element - //ref_elem->init(p_order, num_dim, elem->num_basis()); - //std::cout<<"done with ref elem"< all_node_indices(nall_nodes, "all_node_indices"); - Kokkos::DualView all_node_indices("all_node_indices", nall_nodes); - for(int i = 0; i < nall_nodes; i++){ - if(igetGlobalElement(i); - else all_node_indices.h_view(i) = ghost_nodes.h_view(i-nlocal_nodes); - } - all_node_indices.modify_host(); - all_node_indices.sync_device(); - //debug print of node indices - //for(int inode=0; inode < index_counter; inode++) - //std::cout << " my_reduced_global_indices " << my_reduced_global_indices(inode) <(Teuchos::OrdinalTraits::invalid(),all_node_indices.d_view,0,comm)); - - //remove elements from the local set so that each rank has a unique set of global ids - - //local elements belonging to the non-overlapping element distribution to each rank with buffer - Kokkos::DualView Initial_Element_Global_Indices("Initial_Element_Global_Indices", rnum_elem); - size_t nonoverlapping_count = 0; - int my_element_flag; - //loop through local element set - if(num_dim == 2) - for (int ielem = 0; ielem < rnum_elem; ielem++){ - element_select->choose_2Delem_type(Element_Types(ielem), elem2D); - nodes_per_element = elem2D->num_nodes(); - my_element_flag = 1; - for (int lnode = 0; lnode < nodes_per_element; lnode++){ - node_gid = nodes_in_elem(ielem, lnode); - if(ghost_node_map->isNodeGlobalElement(node_gid)){ - local_node_index = ghost_node_map->getLocalElement(node_gid); - if(ghost_node_ranks.h_view(local_node_index) < myrank) my_element_flag = 0; - } + // copy over from buffer to compressed storage + Kokkos::DualView Element_Global_Indices("Element_Global_Indices", nonoverlapping_count); + for (int ibuffer = 0; ibuffer < nonoverlapping_count; ibuffer++) + { + Element_Global_Indices.h_view(ibuffer) = Initial_Element_Global_Indices.h_view(ibuffer); + } + nlocal_elem_non_overlapping = nonoverlapping_count; + Element_Global_Indices.modify_host(); + Element_Global_Indices.sync_device(); + // create nonoverlapping element map + element_map = Teuchos::rcp(new Tpetra::Map(Teuchos::OrdinalTraits::invalid(), Element_Global_Indices.d_view, 0, comm)); + + // sort element connectivity so nonoverlaps are sequentially found first + // define initial sorting of global indices + + // element_map->describe(*fos,Teuchos::VERB_EXTREME); + + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + Initial_Element_Global_Indices.h_view(ielem) = all_element_map->getGlobalElement(ielem); + } + + // re-sort so local elements in the nonoverlapping map are first in storage + CArrayKokkos Temp_Nodes(max_nodes_per_element); + GO temp_element_gid, current_element_gid; + int last_storage_index = rnum_elem - 1; + for (int ielem = 0; ielem < nlocal_elem_non_overlapping; ielem++) + { + current_element_gid = Initial_Element_Global_Indices.h_view(ielem); + // if this element is not part of the non overlap list then send it to the end of the storage and swap the element at the end + if (!element_map->isNodeGlobalElement(current_element_gid)) + { + temp_element_gid = current_element_gid; + for (int lnode = 0; lnode < max_nodes_per_element; lnode++) + { + Temp_Nodes(lnode) = nodes_in_elem(ielem, lnode); + } + Initial_Element_Global_Indices.h_view(ielem) = Initial_Element_Global_Indices.h_view(last_storage_index); + Initial_Element_Global_Indices.h_view(last_storage_index) = temp_element_gid; + for (int lnode = 0; lnode < max_nodes_per_element; lnode++) + { + nodes_in_elem(ielem, lnode) = nodes_in_elem(last_storage_index, lnode); + nodes_in_elem(last_storage_index, lnode) = Temp_Nodes(lnode); + } + last_storage_index--; + + // test if swapped element is also not part of the non overlap map; if so lower loop counter to repeat the above + temp_element_gid = Initial_Element_Global_Indices.h_view(ielem); + if (!element_map->isNodeGlobalElement(temp_element_gid)) + { + ielem--; + } + } } - if(my_element_flag){ - Initial_Element_Global_Indices.h_view(nonoverlapping_count++) = all_element_map->getGlobalElement(ielem); - } - } - - if(num_dim == 3) - for (int ielem = 0; ielem < rnum_elem; ielem++){ - element_select->choose_3Delem_type(Element_Types(ielem), elem); - nodes_per_element = elem->num_nodes(); - my_element_flag = 1; - for (int lnode = 0; lnode < nodes_per_element; lnode++){ - node_gid = nodes_in_elem(ielem, lnode); - if(ghost_node_map->isNodeGlobalElement(node_gid)){ - local_node_index = ghost_node_map->getLocalElement(node_gid); - if(ghost_node_ranks.h_view(local_node_index) < myrank) my_element_flag = 0; - } + // reset all element map to its re-sorted version + Initial_Element_Global_Indices.modify_host(); + Initial_Element_Global_Indices.sync_device(); + + all_element_map = Teuchos::rcp(new Tpetra::Map(Teuchos::OrdinalTraits::invalid(), Initial_Element_Global_Indices.d_view, 0, comm)); + // element_map->describe(*fos,Teuchos::VERB_EXTREME); + // all_element_map->describe(*fos,Teuchos::VERB_EXTREME); + + // all_element_map->describe(*fos,Teuchos::VERB_EXTREME); + // construct dof map that follows from the node map (used for distributed matrix and vector objects later) + Kokkos::DualView local_dof_indices("local_dof_indices", nlocal_nodes * num_dim); + for (int i = 0; i < nlocal_nodes; i++) + { + for (int j = 0; j < num_dim; j++) + { + local_dof_indices.h_view(i * num_dim + j) = map->getGlobalElement(i) * num_dim + j; + } } - if(my_element_flag){ - Initial_Element_Global_Indices.h_view(nonoverlapping_count++) = all_element_map->getGlobalElement(ielem); - } - } - - //copy over from buffer to compressed storage - Kokkos::DualView Element_Global_Indices("Element_Global_Indices",nonoverlapping_count); - for(int ibuffer = 0; ibuffer < nonoverlapping_count; ibuffer++){ - Element_Global_Indices.h_view(ibuffer) = Initial_Element_Global_Indices.h_view(ibuffer); - } - nlocal_elem_non_overlapping = nonoverlapping_count; - Element_Global_Indices.modify_host(); - Element_Global_Indices.sync_device(); - //create nonoverlapping element map - element_map = Teuchos::rcp( new Tpetra::Map(Teuchos::OrdinalTraits::invalid(),Element_Global_Indices.d_view,0,comm)); - - //sort element connectivity so nonoverlaps are sequentially found first - //define initial sorting of global indices - - //element_map->describe(*fos,Teuchos::VERB_EXTREME); - - for (int ielem = 0; ielem < rnum_elem; ielem++){ - Initial_Element_Global_Indices.h_view(ielem) = all_element_map->getGlobalElement(ielem); - } - - //re-sort so local elements in the nonoverlapping map are first in storage - CArrayKokkos Temp_Nodes(max_nodes_per_element); - GO temp_element_gid, current_element_gid; - int last_storage_index = rnum_elem - 1; - for (int ielem = 0; ielem < nlocal_elem_non_overlapping; ielem++){ - current_element_gid = Initial_Element_Global_Indices.h_view(ielem); - //if this element is not part of the non overlap list then send it to the end of the storage and swap the element at the end - if(!element_map->isNodeGlobalElement(current_element_gid)){ - temp_element_gid = current_element_gid; - for (int lnode = 0; lnode < max_nodes_per_element; lnode++){ - Temp_Nodes(lnode) = nodes_in_elem(ielem,lnode); - } - Initial_Element_Global_Indices.h_view(ielem) = Initial_Element_Global_Indices.h_view(last_storage_index); - Initial_Element_Global_Indices.h_view(last_storage_index) = temp_element_gid; - for (int lnode = 0; lnode < max_nodes_per_element; lnode++){ - nodes_in_elem(ielem, lnode) = nodes_in_elem(last_storage_index,lnode); - nodes_in_elem(last_storage_index,lnode) = Temp_Nodes(lnode); - } - last_storage_index--; - - //test if swapped element is also not part of the non overlap map; if so lower loop counter to repeat the above - temp_element_gid = Initial_Element_Global_Indices.h_view(ielem); - if(!element_map->isNodeGlobalElement(temp_element_gid)) ielem--; - } - } - //reset all element map to its re-sorted version - Initial_Element_Global_Indices.modify_host(); - Initial_Element_Global_Indices.sync_device(); - - all_element_map = Teuchos::rcp( new Tpetra::Map(Teuchos::OrdinalTraits::invalid(),Initial_Element_Global_Indices.d_view,0,comm)); - //element_map->describe(*fos,Teuchos::VERB_EXTREME); - //all_element_map->describe(*fos,Teuchos::VERB_EXTREME); - - //all_element_map->describe(*fos,Teuchos::VERB_EXTREME); - //construct dof map that follows from the node map (used for distributed matrix and vector objects later) - Kokkos::DualView local_dof_indices("local_dof_indices", nlocal_nodes*num_dim); - for(int i = 0; i < nlocal_nodes; i++){ - for(int j = 0; j < num_dim; j++) - local_dof_indices.h_view(i*num_dim + j) = map->getGlobalElement(i)*num_dim + j; - } - - local_dof_indices.modify_host(); - local_dof_indices.sync_device(); - local_dof_map = Teuchos::rcp( new Tpetra::Map(num_nodes*num_dim,local_dof_indices.d_view,0,comm) ); - - //construct dof map that follows from the all_node map (used for distributed matrix and vector objects later) - Kokkos::DualView all_dof_indices("all_dof_indices", nall_nodes*num_dim); - for(int i = 0; i < nall_nodes; i++){ - for(int j = 0; j < num_dim; j++) - all_dof_indices.h_view(i*num_dim + j) = all_node_map->getGlobalElement(i)*num_dim + j; - } - - all_dof_indices.modify_host(); - all_dof_indices.sync_device(); - //pass invalid global count so the map reduces the global count automatically - all_dof_map = Teuchos::rcp( new Tpetra::Map(Teuchos::OrdinalTraits::invalid(),all_dof_indices.d_view,0,comm) ); - - //debug print of map - //debug print - std::ostream &out = std::cout; - Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - //if(myrank==0) - //*fos << "Ghost Node Map :" << std::endl; - //all_node_map->describe(*fos,Teuchos::VERB_EXTREME); - //*fos << std::endl; - //std::fflush(stdout); - - //Count how many elements connect to each local node - node_nconn_distributed = Teuchos::rcp(new MCONN(map, 1)); - //active view scope - { - host_elem_conn_array node_nconn = node_nconn_distributed->getLocalView (Tpetra::Access::ReadWrite); - for(int inode = 0; inode < nlocal_nodes; inode++) - node_nconn(inode,0) = 0; - - for(int ielem = 0; ielem < rnum_elem; ielem++){ - for(int inode = 0; inode < nodes_per_element; inode++){ - node_gid = nodes_in_elem(ielem, inode); - if(map->isNodeGlobalElement(node_gid)) - node_nconn(map->getLocalElement(node_gid),0)++; - } + + local_dof_indices.modify_host(); + local_dof_indices.sync_device(); + local_dof_map = Teuchos::rcp(new Tpetra::Map(num_nodes * num_dim, local_dof_indices.d_view, 0, comm) ); + + // construct dof map that follows from the all_node map (used for distributed matrix and vector objects later) + Kokkos::DualView all_dof_indices("all_dof_indices", nall_nodes * num_dim); + for (int i = 0; i < nall_nodes; i++) + { + for (int j = 0; j < num_dim; j++) + { + all_dof_indices.h_view(i * num_dim + j) = all_node_map->getGlobalElement(i) * num_dim + j; + } + } + + all_dof_indices.modify_host(); + all_dof_indices.sync_device(); + // pass invalid global count so the map reduces the global count automatically + all_dof_map = Teuchos::rcp(new Tpetra::Map(Teuchos::OrdinalTraits::invalid(), all_dof_indices.d_view, 0, comm) ); + + // debug print of map + // debug print + std::ostream& out = std::cout; + Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + // if(myrank==0) + // *fos << "Ghost Node Map :" << std::endl; + // all_node_map->describe(*fos,Teuchos::VERB_EXTREME); + // *fos << std::endl; + // std::fflush(stdout); + + // Count how many elements connect to each local node + node_nconn_distributed = Teuchos::rcp(new MCONN(map, 1)); + // active view scope + { + host_elem_conn_array node_nconn = node_nconn_distributed->getLocalView(Tpetra::Access::ReadWrite); + for (int inode = 0; inode < nlocal_nodes; inode++) + { + node_nconn(inode, 0) = 0; + } + + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + for (int inode = 0; inode < nodes_per_element; inode++) + { + node_gid = nodes_in_elem(ielem, inode); + if (map->isNodeGlobalElement(node_gid)) + { + node_nconn(map->getLocalElement(node_gid), 0)++; + } + } + } + } + + // create distributed multivector of the local node data and all (local + ghost) node storage + all_node_coords_distributed = Teuchos::rcp(new MV(all_node_map, num_dim)); + ghost_node_coords_distributed = Teuchos::rcp(new MV(ghost_node_map, num_dim)); + + // debug print + // std::ostream &out = std::cout; + // Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + // if(myrank==0) + // *fos << "Node Data :" << std::endl; + // node_coords_distributed->describe(*fos,Teuchos::VERB_EXTREME); + // *fos << std::endl; + // std::fflush(stdout); + + // create import object using local node indices map and all indices map + comm_importer_setup(); + + // comms to get ghosts + all_node_coords_distributed->doImport(*node_coords_distributed, *importer, Tpetra::INSERT); + // all_node_nconn_distributed->doImport(*node_nconn_distributed, importer, Tpetra::INSERT); + + dual_nodes_in_elem.sync_device(); + dual_nodes_in_elem.modify_device(); + // construct distributed element connectivity multivector + global_nodes_in_elem_distributed = Teuchos::rcp(new MCONN(all_element_map, dual_nodes_in_elem)); + + // debug print + // std::ostream &out = std::cout; + // Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + // if(myrank==0) + // *fos << "Element Connectivity :" << std::endl; + // global_nodes_in_elem_distributed->describe(*fos,Teuchos::VERB_EXTREME); + // *fos << std::endl; + // std::fflush(stdout); + + // debug print + // std::ostream &out = std::cout; + // Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + // if(myrank==0) + // *fos << "Number of Elements Connected to Each Node :" << std::endl; + // all_node_nconn_distributed->describe(*fos,Teuchos::VERB_EXTREME); + // *fos << std::endl; + // std::fflush(stdout); + + // debug print + // std::ostream &out = std::cout; + // Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + // if(myrank==0) + // *fos << "Node Data with Ghosts :" << std::endl; + // all_node_coords_distributed->describe(*fos,Teuchos::VERB_EXTREME); + // *fos << std::endl; + // std::fflush(stdout); + + // debug print of views node indices + // std::cout << "Local View of All Nodes on Task " << myrank <getLocalNumElements(); inode++){ + // std::cout << "node "<getGlobalElement(inode) << " } " ; + // std::cout << dual_all_node_coords.view_host()(inode,0) << " " << dual_all_node_coords.view_host()(inode,1) << " " << dual_all_node_coords.view_host()(inode,2) << " " << std::endl; + // } + + // std::cout << "number of patches = " << mesh->num_patches() << std::endl; + if (myrank == 0) + { + std::cout << "End of map setup " << std::endl; } - } - - //create distributed multivector of the local node data and all (local + ghost) node storage - all_node_coords_distributed = Teuchos::rcp(new MV(all_node_map, num_dim)); - ghost_node_coords_distributed = Teuchos::rcp(new MV(ghost_node_map, num_dim)); - - //debug print - //std::ostream &out = std::cout; - //Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - //if(myrank==0) - //*fos << "Node Data :" << std::endl; - //node_coords_distributed->describe(*fos,Teuchos::VERB_EXTREME); - //*fos << std::endl; - //std::fflush(stdout); - - //create import object using local node indices map and all indices map - comm_importer_setup(); - - //comms to get ghosts - all_node_coords_distributed->doImport(*node_coords_distributed, *importer, Tpetra::INSERT); - //all_node_nconn_distributed->doImport(*node_nconn_distributed, importer, Tpetra::INSERT); - - dual_nodes_in_elem.sync_device(); - dual_nodes_in_elem.modify_device(); - //construct distributed element connectivity multivector - global_nodes_in_elem_distributed = Teuchos::rcp(new MCONN(all_element_map, dual_nodes_in_elem)); - - //debug print - //std::ostream &out = std::cout; - //Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - //if(myrank==0) - //*fos << "Element Connectivity :" << std::endl; - //global_nodes_in_elem_distributed->describe(*fos,Teuchos::VERB_EXTREME); - //*fos << std::endl; - //std::fflush(stdout); - - //debug print - //std::ostream &out = std::cout; - //Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - //if(myrank==0) - //*fos << "Number of Elements Connected to Each Node :" << std::endl; - //all_node_nconn_distributed->describe(*fos,Teuchos::VERB_EXTREME); - //*fos << std::endl; - //std::fflush(stdout); - - //debug print - //std::ostream &out = std::cout; - //Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - //if(myrank==0) - //*fos << "Node Data with Ghosts :" << std::endl; - //all_node_coords_distributed->describe(*fos,Teuchos::VERB_EXTREME); - //*fos << std::endl; - //std::fflush(stdout); - - //debug print of views node indices - //std::cout << "Local View of All Nodes on Task " << myrank <getLocalNumElements(); inode++){ - //std::cout << "node "<getGlobalElement(inode) << " } " ; - //std::cout << dual_all_node_coords.view_host()(inode,0) << " " << dual_all_node_coords.view_host()(inode,1) << " " << dual_all_node_coords.view_host()(inode,2) << " " << std::endl; - //} - - //std::cout << "number of patches = " << mesh->num_patches() << std::endl; - if(myrank == 0) - std::cout << "End of map setup " << std::endl; } /* ---------------------------------------------------------------------- Find boundary surface segments that belong to this MPI rank ------------------------------------------------------------------------- */ -void Solver::Get_Boundary_Patches(){ - size_t npatches_repeat, npatches, element_npatches, num_nodes_in_patch, node_gid; - int local_node_id; - int num_dim = simparam.num_dims; - CArray Surface_Nodes; - const_host_elem_conn_array nodes_in_elem = global_nodes_in_elem_distributed->getLocalView (Tpetra::Access::ReadOnly); - //Surface_Nodes = CArrayKokkos(4, "Surface_Nodes"); - - std::set my_patches; - //inititializes type for the pair variable (finding the iterator type is annoying) - std::pair::iterator, bool> current_combination; - std::set::iterator it; - - - CArrayKokkos convert_node_order(max_nodes_per_element); - if((active_node_ordering_convention == ENSIGHT && num_dim==3)||(active_node_ordering_convention == IJK && num_dim==2)){ - convert_node_order(0) = 0; - convert_node_order(1) = 1; - convert_node_order(2) = 3; - convert_node_order(3) = 2; - if(num_dim == 3){ - convert_node_order(4) = 4; - convert_node_order(5) = 5; - convert_node_order(6) = 7; - convert_node_order(7) = 6; - } - } - else if((active_node_ordering_convention == IJK && num_dim==3)||(active_node_ordering_convention == ENSIGHT && num_dim==2)){ - convert_node_order(0) = 0; - convert_node_order(1) = 1; - convert_node_order(2) = 2; - convert_node_order(3) = 3; - if(num_dim==3){ - convert_node_order(4) = 4; - convert_node_order(5) = 5; - convert_node_order(6) = 6; - convert_node_order(7) = 7; - } - } - - //compute the number of patches in this MPI rank with repeats for adjacent cells - npatches_repeat = 0; - - if(num_dim==2) - for(int ielem = 0; ielem < rnum_elem; ielem++){ - element_select->choose_2Delem_type(Element_Types(ielem), elem2D); - element_npatches = elem2D->nsurfaces; - npatches_repeat += element_npatches; - } - - else if(num_dim==3) - for(int ielem = 0; ielem < rnum_elem; ielem++){ - element_select->choose_3Delem_type(Element_Types(ielem), elem); - element_npatches = elem->nsurfaces; - npatches_repeat += element_npatches; - } - //std::cout << "Starting boundary patch allocation of size " << npatches_repeat << std::endl < Patch_Nodes(npatches_repeat, "Patch_Nodes"); - CArrayKokkos Patch_Boundary_Flags(npatches_repeat, "Patch_Boundary_Flags"); - if(myrank == 0) - std::cout << "Done with boundary patch allocation" << std::endl <choose_2Delem_type(Element_Types(ielem), elem2D); - element_npatches = elem2D->nsurfaces; - //loop through local surfaces - for(int isurface = 0; isurface < element_npatches; isurface++){ - num_nodes_in_patch = elem2D->surface_to_dof_lid.stride(isurface); - Surface_Nodes = CArray(num_nodes_in_patch); - for(int inode = 0; inode < num_nodes_in_patch; inode++){ - local_node_id = elem2D->surface_to_dof_lid(isurface,inode); - local_node_id = convert_node_order(local_node_id); - Surface_Nodes(inode) = nodes_in_elem(ielem, local_node_id); - } - Node_Combination temp(Surface_Nodes); - //construct Node Combination object for this surface - Patch_Nodes(npatches_repeat) = temp; - Patch_Nodes(npatches_repeat).patch_id = npatches_repeat; - Patch_Nodes(npatches_repeat).element_id = ielem; - Patch_Nodes(npatches_repeat).local_patch_id = isurface; - //test if this patch has already been added; if yes set boundary flags to 0 - current_combination = my_patches.insert(Patch_Nodes(npatches_repeat)); - //if the set determines this is a duplicate, access the original element's patch id and set flag to 0 - if(current_combination.second==false){ - //set original element flag to 0 - Patch_Boundary_Flags((*current_combination.first).patch_id) = 0; - //set this current flag to 0 for the duplicate as well - Patch_Boundary_Flags(npatches_repeat) = 0; - - } - npatches_repeat++; - } - } - - if(num_dim==3) - for(int ielem = 0; ielem < rnum_elem; ielem++){ - element_select->choose_3Delem_type(Element_Types(ielem), elem); - element_npatches = elem->nsurfaces; - //loop through local surfaces - for(int isurface = 0; isurface < element_npatches; isurface++){ - num_nodes_in_patch = elem->surface_to_dof_lid.stride(isurface); - //debug print - //std::cout << "NUMBER OF PATCH NODES FOR ELEMENT " << ielem+1 << " ON LOCAL SURFACE " << isurface+1 << " IS " << num_nodes_in_patch << std::endl; - Surface_Nodes = CArray(num_nodes_in_patch); - for(int inode = 0; inode < num_nodes_in_patch; inode++){ - local_node_id = elem->surface_to_dof_lid(isurface,inode); - local_node_id = convert_node_order(local_node_id); - Surface_Nodes(inode) = nodes_in_elem(ielem, local_node_id); - } - Node_Combination temp(Surface_Nodes); - //construct Node Combination object for this surface - Patch_Nodes(npatches_repeat) = temp; - Patch_Nodes(npatches_repeat).patch_id = npatches_repeat; - Patch_Nodes(npatches_repeat).element_id = ielem; - Patch_Nodes(npatches_repeat).local_patch_id = isurface; - //test if this patch has already been added; if yes set boundary flags to 0 - current_combination = my_patches.insert(Patch_Nodes(npatches_repeat)); - //if the set determines this is a duplicate access the original element's patch id and set flag to 0 - - if(current_combination.second==false){ - //set original element flag to 0 - Patch_Boundary_Flags((*current_combination.first).patch_id) = 0; - //set this current flag to 0 for the duplicate as well - Patch_Boundary_Flags(npatches_repeat) = 0; - } - npatches_repeat++; - } - } - //debug print of all patches - /* - std::cout << " ALL PATCHES " << npatches_repeat <isNodeGlobalElement(node_gid)){ - //if(ghost_node_ranks.h_view(global2local_map.get(node_gid)) Surface_Nodes; + const_host_elem_conn_array nodes_in_elem = global_nodes_in_elem_distributed->getLocalView(Tpetra::Access::ReadOnly); + // Surface_Nodes = CArrayKokkos(4, "Surface_Nodes"); + + std::set my_patches; + // inititializes type for the pair variable (finding the iterator type is annoying) + std::pair::iterator, bool> current_combination; + std::set::iterator it; + + CArrayKokkos convert_node_order(max_nodes_per_element); + if ((active_node_ordering_convention == ENSIGHT && num_dim == 3) || (active_node_ordering_convention == IJK && num_dim == 2)) + { + convert_node_order(0) = 0; + convert_node_order(1) = 1; + convert_node_order(2) = 3; + convert_node_order(3) = 2; + if (num_dim == 3) + { + convert_node_order(4) = 4; + convert_node_order(5) = 5; + convert_node_order(6) = 7; + convert_node_order(7) = 6; + } + } + else if ((active_node_ordering_convention == IJK && num_dim == 3) || (active_node_ordering_convention == ENSIGHT && num_dim == 2)) + { + convert_node_order(0) = 0; + convert_node_order(1) = 1; + convert_node_order(2) = 2; + convert_node_order(3) = 3; + if (num_dim == 3) + { + convert_node_order(4) = 4; + convert_node_order(5) = 5; + convert_node_order(6) = 6; + convert_node_order(7) = 7; + } + } - if(remote_count == num_nodes_in_patch) my_rank_flag = false; - //all nodes were remote - //if(remote_count == num_nodes_in_patch) my_rank_flag = false; + // compute the number of patches in this MPI rank with repeats for adjacent cells + npatches_repeat = 0; - //if at least one node was local - if(my_rank_flag){ - Boundary_Patches(nboundary_patches++) = Patch_Nodes(ipatch); - boundary_patch_to_index[Patch_Nodes(ipatch)] = nboundary_patches-1; - } + if (num_dim == 2) + { + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + element_select->choose_2Delem_type(Element_Types(ielem), elem2D); + element_npatches = elem2D->nsurfaces; + npatches_repeat += element_npatches; + } } - } - - //debug print of boundary patches - /* - std::cout << " BOUNDARY PATCHES ON TASK " << myrank << " = " << nboundary_patches <choose_3Delem_type(Element_Types(ielem), elem); + element_npatches = elem->nsurfaces; + npatches_repeat += element_npatches; + } + } + // std::cout << "Starting boundary patch allocation of size " << npatches_repeat << std::endl < Patch_Nodes(npatches_repeat, "Patch_Nodes"); + CArrayKokkos Patch_Boundary_Flags(npatches_repeat, "Patch_Boundary_Flags"); + if (myrank == 0) + { + std::cout << "Done with boundary patch allocation" << std::endl << std::flush; + } + // initialize boundary patch flags + for (int init = 0; init < npatches_repeat; init++) + { + Patch_Boundary_Flags(init) = 1; + } + + if (myrank == 0) + { + std::cout << "Done with boundary patch flags init" << std::endl << std::flush; + } + // use set of nodal combinations to find boundary set + // boundary patches will not try to add nodal combinations twice + // loop through elements in this rank to find boundary patches + npatches_repeat = 0; + + if (num_dim == 2) + { + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + element_select->choose_2Delem_type(Element_Types(ielem), elem2D); + element_npatches = elem2D->nsurfaces; + // loop through local surfaces + for (int isurface = 0; isurface < element_npatches; isurface++) + { + num_nodes_in_patch = elem2D->surface_to_dof_lid.stride(isurface); + Surface_Nodes = CArray(num_nodes_in_patch); + for (int inode = 0; inode < num_nodes_in_patch; inode++) + { + local_node_id = elem2D->surface_to_dof_lid(isurface, inode); + local_node_id = convert_node_order(local_node_id); + Surface_Nodes(inode) = nodes_in_elem(ielem, local_node_id); + } + Node_Combination temp(Surface_Nodes); + // construct Node Combination object for this surface + Patch_Nodes(npatches_repeat) = temp; + Patch_Nodes(npatches_repeat).patch_id = npatches_repeat; + Patch_Nodes(npatches_repeat).element_id = ielem; + Patch_Nodes(npatches_repeat).local_patch_id = isurface; + // test if this patch has already been added; if yes set boundary flags to 0 + current_combination = my_patches.insert(Patch_Nodes(npatches_repeat)); + // if the set determines this is a duplicate, access the original element's patch id and set flag to 0 + if (current_combination.second == false) + { + // set original element flag to 0 + Patch_Boundary_Flags((*current_combination.first).patch_id) = 0; + // set this current flag to 0 for the duplicate as well + Patch_Boundary_Flags(npatches_repeat) = 0; + } + npatches_repeat++; + } + } + } + + if (num_dim == 3) + { + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + element_select->choose_3Delem_type(Element_Types(ielem), elem); + element_npatches = elem->nsurfaces; + // loop through local surfaces + for (int isurface = 0; isurface < element_npatches; isurface++) + { + num_nodes_in_patch = elem->surface_to_dof_lid.stride(isurface); + // debug print + // std::cout << "NUMBER OF PATCH NODES FOR ELEMENT " << ielem+1 << " ON LOCAL SURFACE " << isurface+1 << " IS " << num_nodes_in_patch << std::endl; + Surface_Nodes = CArray(num_nodes_in_patch); + for (int inode = 0; inode < num_nodes_in_patch; inode++) + { + local_node_id = elem->surface_to_dof_lid(isurface, inode); + local_node_id = convert_node_order(local_node_id); + Surface_Nodes(inode) = nodes_in_elem(ielem, local_node_id); + } + Node_Combination temp(Surface_Nodes); + // construct Node Combination object for this surface + Patch_Nodes(npatches_repeat) = temp; + Patch_Nodes(npatches_repeat).patch_id = npatches_repeat; + Patch_Nodes(npatches_repeat).element_id = ielem; + Patch_Nodes(npatches_repeat).local_patch_id = isurface; + // test if this patch has already been added; if yes set boundary flags to 0 + current_combination = my_patches.insert(Patch_Nodes(npatches_repeat)); + // if the set determines this is a duplicate access the original element's patch id and set flag to 0 + + if (current_combination.second == false) + { + // set original element flag to 0 + Patch_Boundary_Flags((*current_combination.first).patch_id) = 0; + // set this current flag to 0 for the duplicate as well + Patch_Boundary_Flags(npatches_repeat) = 0; + } + npatches_repeat++; + } + } + } + // debug print of all patches + /* + std::cout << " ALL PATCHES " << npatches_repeat <isNodeGlobalElement(node_gid)) + { + // if(ghost_node_ranks.h_view(global2local_map.get(node_gid))(map, all_node_map)); - ghost_importer = Teuchos::rcp( new Tpetra::Import(map, ghost_node_map)); - dof_importer = Teuchos::rcp( new Tpetra::Import(local_dof_map, all_dof_map)); - - //output map and importers - sorted_map = Teuchos::rcp( new Tpetra::Map(num_nodes,0,comm)); - node_sorting_importer = Teuchos::rcp( new Tpetra::Import(map, sorted_map)); - //sorted element mapping - sorted_element_map = Teuchos::rcp( new Tpetra::Map(num_elem,0,comm)); - element_sorting_importer = Teuchos::rcp( new Tpetra::Import(all_element_map, sorted_element_map));; - +void Solver::comm_importer_setup() +{ + // create import object using local node indices map and ghost indices map + importer = Teuchos::rcp(new Tpetra::Import(map, all_node_map)); + ghost_importer = Teuchos::rcp(new Tpetra::Import(map, ghost_node_map)); + dof_importer = Teuchos::rcp(new Tpetra::Import(local_dof_map, all_dof_map)); + + // output map and importers + sorted_map = Teuchos::rcp(new Tpetra::Map(num_nodes, 0, comm)); + node_sorting_importer = Teuchos::rcp(new Tpetra::Import(map, sorted_map)); + // sorted element mapping + sorted_element_map = Teuchos::rcp(new Tpetra::Map(num_elem, 0, comm)); + element_sorting_importer = Teuchos::rcp(new Tpetra::Import(all_element_map, sorted_element_map));; } /* ---------------------------------------------------------------------- Communicate updated nodal coordinates to ghost nodes ------------------------------------------------------------------------- */ -void Solver::comm_coordinates(){ +void Solver::comm_coordinates() +{ + // create import object using local node indices map and ghost indices map + // Tpetra::Import importer(map, ghost_node_map); - //create import object using local node indices map and ghost indices map - //Tpetra::Import importer(map, ghost_node_map); - - //comms to get ghosts - ghost_node_coords_distributed->doImport(*node_coords_distributed, *ghost_importer, Tpetra::INSERT); - //all_node_map->describe(*fos,Teuchos::VERB_EXTREME); - //all_node_coords_distributed->describe(*fos,Teuchos::VERB_EXTREME); + // comms to get ghosts + ghost_node_coords_distributed->doImport(*node_coords_distributed, *ghost_importer, Tpetra::INSERT); + // all_node_map->describe(*fos,Teuchos::VERB_EXTREME); + // all_node_coords_distributed->describe(*fos,Teuchos::VERB_EXTREME); } /* ---------------------------------------------------------------------- @@ -2600,22 +3003,23 @@ void Solver::comm_coordinates(){ double Solver::CPU_Time() { - std::chrono::system_clock::time_point zero_time; - auto zero_time_duration = zero_time.time_since_epoch(); - auto time = std::chrono::system_clock::now(); - auto time_duration = time.time_since_epoch(); - //double calc_time = std::chrono::duration_cast(diff).count(); - double calc_time = std::chrono::duration_cast< std::chrono::nanoseconds>(time_duration-zero_time_duration).count(); - calc_time *= 1e-09; - - return calc_time; + std::chrono::system_clock::time_point zero_time; + auto zero_time_duration = zero_time.time_since_epoch(); + auto time = std::chrono::system_clock::now(); + auto time_duration = time.time_since_epoch(); + // double calc_time = std::chrono::duration_cast(diff).count(); + double calc_time = std::chrono::duration_cast(time_duration - zero_time_duration).count(); + calc_time *= 1e-09; + + return calc_time; } /* ---------------------------------------------------------------------- Clock variable initialization ------------------------------------------------------------------------- */ -void Solver::init_clock(){ - double current_cpu = 0; - initial_CPU_time = CPU_Time(); +void Solver::init_clock() +{ + double current_cpu = 0; + initial_CPU_time = CPU_Time(); } diff --git a/src/Parallel-Solvers/Solver.h b/src/Parallel-Solvers/Solver.h index df0bf88a9..29d23fb1c 100644 --- a/src/Parallel-Solvers/Solver.h +++ b/src/Parallel-Solvers/Solver.h @@ -11,14 +11,14 @@ This program is open source under the BSD-3 License. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - + 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - + 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - + 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. @@ -34,9 +34,9 @@ OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. **********************************************************************************************/ - + #ifndef FIERRO_SOLVER_H -#define FIERRO_SOLVER_H +#define FIERRO_SOLVER_H #include "utilities.h" #include "matar.h" @@ -64,222 +64,222 @@ using namespace mtr; -//forward declarations -namespace swage{ - class mesh_t; -} +// forward declarations +namespace swage +{ +class mesh_t; +} // namespace swage -namespace ROL{ - template - class Problem; -} +namespace ROL +{ +template +class Problem; +} // namespace ROL +class Solver +{ +public: -class Solver{ + // Trilinos type definitions + typedef Tpetra::Map<>::local_ordinal_type LO; + typedef Tpetra::Map<>::global_ordinal_type GO; -public: + typedef Tpetra::CrsMatrix MAT; + typedef const Tpetra::CrsMatrix const_MAT; + typedef Tpetra::MultiVector MV; + typedef Tpetra::MultiVector MCONN; - //Trilinos type definitions - typedef Tpetra::Map<>::local_ordinal_type LO; - typedef Tpetra::Map<>::global_ordinal_type GO; + typedef Kokkos::ViewTraits::size_type SizeType; + typedef Tpetra::Details::DefaultTypes::node_type node_type; + using traits = Kokkos::ViewTraits; - typedef Tpetra::CrsMatrix MAT; - typedef const Tpetra::CrsMatrix const_MAT; - typedef Tpetra::MultiVector MV; - typedef Tpetra::MultiVector MCONN; - - typedef Kokkos::ViewTraits::size_type SizeType; - typedef Tpetra::Details::DefaultTypes::node_type node_type; - using traits = Kokkos::ViewTraits; - - using array_layout = typename traits::array_layout; - using execution_space = typename traits::execution_space; - using device_type = typename traits::device_type; - using memory_traits = typename traits::memory_traits; - using global_size_t = Tpetra::global_size_t; - - typedef Kokkos::View values_array; - typedef Kokkos::View global_indices_array; - typedef Kokkos::View indices_array; - //typedef Kokkos::View row_pointers; - typedef MAT::local_graph_device_type::row_map_type::non_const_type row_pointers; - //typedef Kokkos::DualView::t_dev vec_array; - typedef MV::dual_view_type::t_dev vec_array; - typedef MV::dual_view_type::t_host host_vec_array; - typedef Kokkos::View const_host_vec_array; - typedef Kokkos::View const_vec_array; - typedef Kokkos::View const_host_ivec_array; - typedef Kokkos::View host_ivec_array; - typedef MV::dual_view_type dual_vec_array; - typedef MCONN::dual_view_type dual_elem_conn_array; - typedef MCONN::dual_view_type::t_host host_elem_conn_array; - typedef MCONN::dual_view_type::t_dev elem_conn_array; - typedef Kokkos::View const_host_elem_conn_array; - typedef Kokkos::View const_elem_conn_array; - - Solver(Simulation_Parameters& _simparam); - virtual ~Solver(); - - virtual void setup() {} - - virtual void run() = 0; - - virtual void solver_setup() {} - - virtual void solver_finalize() {} - - virtual void exit_solver(int status); - - virtual void generate_mesh(const std::shared_ptr& mesh_generation_options); - - virtual void read_mesh_ensight(const char *MESH); - - virtual void init_design() {} - - virtual void read_mesh_tecplot(const char *MESH); - - virtual void read_mesh_vtk(const char *MESH); - - virtual void repartition_nodes(); - - virtual void comm_importer_setup(); - - virtual void comm_coordinates(); - - virtual void tecplot_writer() {} - - virtual void parallel_tecplot_writer() {} - - virtual void parallel_vtk_writer() {} - - virtual void output_design(int current_step) {} - - //setup ghosts and element maps - virtual void init_maps(); - - //finds the boundary element surfaces in this model - virtual void Get_Boundary_Patches(); - - int setup_flag, finalize_flag; - - //MPI data - int myrank = 0; //index of this mpi rank in the world communicator - int nranks; //number of mpi ranks in the world communicator - MPI_Comm world; //stores the default communicator object (MPI_COMM_WORLD) - Teuchos::RCP> importer; //all node comms - Teuchos::RCP> ghost_importer; //ghost node comms - Teuchos::RCP> node_sorting_importer; //sorted node comms - Teuchos::RCP> element_sorting_importer; //sorted element comms - Teuchos::RCP> dof_importer; //ghost dof comms - - //class Simulation_Parameters *simparam; - Simulation_Parameters simparam; - - //set of enabled FEA modules - std::vector fea_module_types; - std::vector fea_modules; - std::vector fea_modules_modal_analysis; - std::set fea_module_must_read; - int nfea_modules; - int displacement_module; - - //Local FEA data - size_t nlocal_nodes; - size_t nlocal_elem_non_overlapping; //used for reduction calls of per element values - dual_vec_array dual_node_coords; //coordinates of the nodes - dual_vec_array dual_node_densities; //topology optimization design variable - dual_elem_conn_array dual_nodes_in_elem; //dual view of element connectivity to nodes - //host_elem_conn_array nodes_in_elem; //host view of element connectivity to nodes - CArrayKokkos Element_Types; - CArrayKokkos Nodes_Per_Element_Type; - CArrayKokkos corner_value_storage; - CArrayKokkos corner_vector_storage; - CArrayKokkos corner_gradient_storage; - size_t max_nodes_per_element, max_nodes_per_patch; - std::shared_ptr element_select; - std::shared_ptr ref_elem; - elements::Element2D *elem2D; - elements::Element3D *elem; - - //Ghost data on this MPI rank - size_t nghost_nodes; - Kokkos::DualView ghost_nodes; - Kokkos::DualView ghost_node_ranks; - - //Local FEA data including ghosts - size_t nall_nodes; - size_t rnum_elem; - - //Global FEA data - long long int num_nodes, num_elem; - Teuchos::RCP > comm; - Teuchos::RCP > map; //map of node indices - Teuchos::RCP > sorted_map; //sorted contiguous map of node indices - Teuchos::RCP > ghost_node_map; //map of node indices with ghosts on each rank - Teuchos::RCP > all_node_map; //map of node indices with ghosts on each rank - Teuchos::RCP > element_map; //non overlapping map of elements owned by each rank used in reduction ops - Teuchos::RCP > all_element_map; //overlapping map of elements connected to the local nodes in each rank - Teuchos::RCP > sorted_element_map; //sorted contiguous map of element indices owned by each rank used in parallel IO - Teuchos::RCP > local_dof_map; //map of local dofs (typically num_node_local*num_dim) - Teuchos::RCP > all_dof_map; //map of local and ghost dofs (typically num_node_all*num_dim) - Teuchos::RCP global_nodes_in_elem_distributed; //element to node connectivity table - Teuchos::RCP node_nconn_distributed; //how many elements a node is connected to - Teuchos::RCP node_coords_distributed; - Teuchos::RCP ghost_node_coords_distributed; - Teuchos::RCP initial_node_coords_distributed; - Teuchos::RCP all_initial_node_coords_distributed; - Teuchos::RCP all_node_coords_distributed; - Teuchos::RCP design_node_densities_distributed; - Teuchos::RCP filtered_node_densities_distributed; - Teuchos::RCP test_node_densities_distributed; - Teuchos::RCP all_node_densities_distributed; - Teuchos::RCP all_filtered_node_densities_distributed; - Teuchos::RCP lower_bound_node_densities_distributed; - Teuchos::RCP upper_bound_node_densities_distributed; - Teuchos::RCP Global_Element_Densities_Upper_Bound; - Teuchos::RCP Global_Element_Densities_Lower_Bound; - Teuchos::RCP Global_Element_Densities; - - //Distributions of data used to print - Teuchos::RCP collected_node_coords_distributed; - Teuchos::RCP collected_node_densities_distributed; - Teuchos::RCP collected_nodes_in_elem_distributed; - Teuchos::RCP sorted_node_coords_distributed; - Teuchos::RCP sorted_node_densities_distributed; - Teuchos::RCP sorted_nodes_in_elem_distributed; - Teuchos::RCP sorted_element_densities_distributed; - - //Boundary Conditions Data - //CArray Patch_Nodes; - size_t nboundary_patches; - size_t num_boundary_conditions; - int current_bdy_id; - CArrayKokkos Boundary_Patches; - std::map boundary_patch_to_index; //maps patches to corresponding patch index (inverse of Boundary Patches array) - - //file readin variables - std::ifstream *in = NULL; - std::streampos before_condition_header; - std::string filename; - int words_per_line, elem_words_per_line; - enum node_ordering_convention {IJK, ENSIGHT}; - node_ordering_convention active_node_ordering_convention; - - //file output variables - int file_index, nsteps_print; //file sequence index and print frequency in # of optimization steps - - //output stream - Teuchos::RCP fos; - int last_print_step; - - //debug and system functions/variables - double CPU_Time(); - void init_clock(); - double initial_CPU_time, communication_time, dev2host_time, host2dev_time, output_time; - - //Pointer to ROL Problem for optimization solves - Teuchos::RCP> problem; + using array_layout = typename traits::array_layout; + using execution_space = typename traits::execution_space; + using device_type = typename traits::device_type; + using memory_traits = typename traits::memory_traits; + using global_size_t = Tpetra::global_size_t; + typedef Kokkos::View values_array; + typedef Kokkos::View global_indices_array; + typedef Kokkos::View indices_array; + // typedef Kokkos::View row_pointers; + typedef MAT::local_graph_device_type::row_map_type::non_const_type row_pointers; + // typedef Kokkos::DualView::t_dev vec_array; + typedef MV::dual_view_type::t_dev vec_array; + typedef MV::dual_view_type::t_host host_vec_array; + typedef Kokkos::View const_host_vec_array; + typedef Kokkos::View const_vec_array; + typedef Kokkos::View const_host_ivec_array; + typedef Kokkos::View host_ivec_array; + typedef MV::dual_view_type dual_vec_array; + typedef MCONN::dual_view_type dual_elem_conn_array; + typedef MCONN::dual_view_type::t_host host_elem_conn_array; + typedef MCONN::dual_view_type::t_dev elem_conn_array; + typedef Kokkos::View const_host_elem_conn_array; + typedef Kokkos::View const_elem_conn_array; + + Solver(Simulation_Parameters& _simparam); + virtual ~Solver(); + + virtual void setup() {} + + virtual void run() = 0; + + virtual void solver_setup() {} + + virtual void solver_finalize() {} + + virtual void exit_solver(int status); + + virtual void generate_mesh(const std::shared_ptr& mesh_generation_options); + + virtual void read_mesh_ensight(const char* MESH); + + virtual void init_design() {} + + virtual void read_mesh_tecplot(const char* MESH); + + virtual void read_mesh_vtk(const char* MESH); + + virtual void repartition_nodes(); + + virtual void comm_importer_setup(); + + virtual void comm_coordinates(); + + virtual void tecplot_writer() {} + + virtual void parallel_tecplot_writer() {} + + virtual void parallel_vtk_writer() {} + + virtual void output_design(int current_step) {} + + // setup ghosts and element maps + virtual void init_maps(); + + // finds the boundary element surfaces in this model + virtual void Get_Boundary_Patches(); + + int setup_flag, finalize_flag; + + // MPI data + int myrank = 0; // index of this mpi rank in the world communicator + int nranks; // number of mpi ranks in the world communicator + MPI_Comm world; // stores the default communicator object (MPI_COMM_WORLD) + Teuchos::RCP> importer; // all node comms + Teuchos::RCP> ghost_importer; // ghost node comms + Teuchos::RCP> node_sorting_importer; // sorted node comms + Teuchos::RCP> element_sorting_importer; // sorted element comms + Teuchos::RCP> dof_importer; // ghost dof comms + + // class Simulation_Parameters *simparam; + Simulation_Parameters simparam; + + // set of enabled FEA modules + std::vector fea_module_types; + std::vector fea_modules; + std::vector fea_modules_modal_analysis; + std::set fea_module_must_read; + int nfea_modules; + int displacement_module; + + // Local FEA data + size_t nlocal_nodes; + size_t nlocal_elem_non_overlapping; // used for reduction calls of per element values + dual_vec_array dual_node_coords; // coordinates of the nodes + dual_vec_array dual_node_densities; // topology optimization design variable + dual_elem_conn_array dual_nodes_in_elem; // dual view of element connectivity to nodes + // host_elem_conn_array nodes_in_elem; //host view of element connectivity to nodes + CArrayKokkos Element_Types; + CArrayKokkos Nodes_Per_Element_Type; + CArrayKokkos corner_value_storage; + CArrayKokkos corner_vector_storage; + CArrayKokkos corner_gradient_storage; + size_t max_nodes_per_element, max_nodes_per_patch; + std::shared_ptr element_select; + std::shared_ptr ref_elem; + elements::Element2D* elem2D; + elements::Element3D* elem; + + // Ghost data on this MPI rank + size_t nghost_nodes; + Kokkos::DualView ghost_nodes; + Kokkos::DualView ghost_node_ranks; + + // Local FEA data including ghosts + size_t nall_nodes; + size_t rnum_elem; + + // Global FEA data + long long int num_nodes, num_elem; + Teuchos::RCP> comm; + Teuchos::RCP> map; // map of node indices + Teuchos::RCP> sorted_map; // sorted contiguous map of node indices + Teuchos::RCP> ghost_node_map; // map of node indices with ghosts on each rank + Teuchos::RCP> all_node_map; // map of node indices with ghosts on each rank + Teuchos::RCP> element_map; // non overlapping map of elements owned by each rank used in reduction ops + Teuchos::RCP> all_element_map; // overlapping map of elements connected to the local nodes in each rank + Teuchos::RCP> sorted_element_map; // sorted contiguous map of element indices owned by each rank used in parallel IO + Teuchos::RCP> local_dof_map; // map of local dofs (typically num_node_local*num_dim) + Teuchos::RCP> all_dof_map; // map of local and ghost dofs (typically num_node_all*num_dim) + Teuchos::RCP global_nodes_in_elem_distributed; // element to node connectivity table + Teuchos::RCP node_nconn_distributed; // how many elements a node is connected to + Teuchos::RCP node_coords_distributed; + Teuchos::RCP ghost_node_coords_distributed; + Teuchos::RCP initial_node_coords_distributed; + Teuchos::RCP all_initial_node_coords_distributed; + Teuchos::RCP all_node_coords_distributed; + Teuchos::RCP design_node_densities_distributed; + Teuchos::RCP filtered_node_densities_distributed; + Teuchos::RCP test_node_densities_distributed; + Teuchos::RCP all_node_densities_distributed; + Teuchos::RCP all_filtered_node_densities_distributed; + Teuchos::RCP lower_bound_node_densities_distributed; + Teuchos::RCP upper_bound_node_densities_distributed; + Teuchos::RCP Global_Element_Densities_Upper_Bound; + Teuchos::RCP Global_Element_Densities_Lower_Bound; + Teuchos::RCP Global_Element_Densities; + + // Distributions of data used to print + Teuchos::RCP collected_node_coords_distributed; + Teuchos::RCP collected_node_densities_distributed; + Teuchos::RCP collected_nodes_in_elem_distributed; + Teuchos::RCP sorted_node_coords_distributed; + Teuchos::RCP sorted_node_densities_distributed; + Teuchos::RCP sorted_nodes_in_elem_distributed; + Teuchos::RCP sorted_element_densities_distributed; + + // Boundary Conditions Data + // CArray Patch_Nodes; + size_t nboundary_patches; + size_t num_boundary_conditions; + int current_bdy_id; + CArrayKokkos Boundary_Patches; + std::map boundary_patch_to_index; // maps patches to corresponding patch index (inverse of Boundary Patches array) + + // file readin variables + std::ifstream* in = NULL; + std::streampos before_condition_header; + std::string filename; + int words_per_line, elem_words_per_line; + enum node_ordering_convention { IJK, ENSIGHT }; + node_ordering_convention active_node_ordering_convention; + + // file output variables + int file_index, nsteps_print; // file sequence index and print frequency in # of optimization steps + + // output stream + Teuchos::RCP fos; + int last_print_step; + + // debug and system functions/variables + double CPU_Time(); + void init_clock(); + double initial_CPU_time, communication_time, dev2host_time, host2dev_time, output_time; + + // Pointer to ROL Problem for optimization solves + Teuchos::RCP> problem; }; #endif // end Header Guard diff --git a/src/Parallel-Solvers/node_combination.cpp b/src/Parallel-Solvers/node_combination.cpp index a251ba180..89433fc4e 100644 --- a/src/Parallel-Solvers/node_combination.cpp +++ b/src/Parallel-Solvers/node_combination.cpp @@ -11,14 +11,14 @@ This program is open source under the BSD-3 License. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - + 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - + 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - + 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. @@ -34,43 +34,58 @@ OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. **********************************************************************************************/ - + #include "node_combination.h" #include "utilities.h" #include "matar.h" using namespace utils; -//overload < operator - bool operator< (const Node_Combination &object1, const Node_Combination &object2){ +// overload < operator +bool operator<(const Node_Combination& object1, const Node_Combination& object2) +{ int size1 = object1.node_set.size(); - //check if this node combination is identical - //first check size of the combination; if smaller evaluate to true - //the set using this is then ordered first according to size of the combinations - if(size1object2.node_set.size()) - return false; - + // check if this node combination is identical + // first check size of the combination; if smaller evaluate to true + // the set using this is then ordered first according to size of the combinations + if (size1 < object2.node_set.size()) + { + return true; + } + else if (size1 > object2.node_set.size()) + { + return false; + } + CArray sort_set1 = object1.sort_set; CArray sort_set2 = object2.sort_set; - for(int i = 0; i < size1; i++){ - sort_set1(i) = object1.node_set(i); - sort_set2(i) = object2.node_set(i); + for (int i = 0; i < size1; i++) + { + sort_set1(i) = object1.node_set(i); + sort_set2(i) = object2.node_set(i); } - //This part sorts for segments of the set where combinations have the same size - //define < using the sort of both combinations. If the first nonequal element of the lhs combination, w.r.t to - //the corresponding element of the rhs, is less than the respective element of the rhs < evaluates to true - std::sort(sort_set1.pointer(),sort_set1.pointer()+sort_set1.size()); - std::sort(sort_set2.pointer(),sort_set2.pointer()+sort_set2.size()); + // This part sorts for segments of the set where combinations have the same size + // define < using the sort of both combinations. If the first nonequal element of the lhs combination, w.r.t to + // the corresponding element of the rhs, is less than the respective element of the rhs < evaluates to true + std::sort(sort_set1.pointer(), sort_set1.pointer() + sort_set1.size()); + std::sort(sort_set2.pointer(), sort_set2.pointer() + sort_set2.size()); - //loop through the sorted nodes to check for < - for(int i = 0; i < size1; i++){ - if(sort_set1(i)::local_ordinal_type LO; - typedef Tpetra::Map<>::global_ordinal_type GO; - typedef Kokkos::ViewTraits::size_type SizeType; - using traits = Kokkos::ViewTraits; - - using array_layout = typename traits::array_layout; - using execution_space = typename traits::execution_space; - using device_type = typename traits::device_type; - using memory_traits = typename traits::memory_traits; - - CArray node_set, sort_set; - GO patch_id, element_id; - LO local_patch_id; - - //Default Constructor - Node_Combination(){} - - //Constructor with initialization - Node_Combination(CArray &nodes_init) { - node_set = nodes_init; - sort_set = CArray(node_set.size()); - } - - //Destructor - ~Node_Combination( ) {} - - //overload = operator - Node_Combination& operator= (const Node_Combination ¬_this){ - node_set = not_this.node_set; - sort_set = not_this.sort_set; - patch_id = not_this.patch_id; - element_id = not_this.element_id; - local_patch_id = not_this.local_patch_id; - return *this; - } - - //overload = operator - bool operator== (Node_Combination ¬_this){ - int this_size = this->node_set.size(); - //check if this node combination is identical - //first check size of the combination - if(this_size!=not_this.node_set.size()) - return false; - - CArray sort_set1 = this->sort_set; - CArray sort_set2 = not_this.sort_set; - for(int i = 0; i < this_size; i++){ - sort_set1(i) = this->node_set(i); - sort_set2(i) = not_this.node_set(i); + // Trilinos type definitions + typedef Tpetra::Map<>::local_ordinal_type LO; + typedef Tpetra::Map<>::global_ordinal_type GO; + typedef Kokkos::ViewTraits::size_type SizeType; + using traits = Kokkos::ViewTraits; + + using array_layout = typename traits::array_layout; + using execution_space = typename traits::execution_space; + using device_type = typename traits::device_type; + using memory_traits = typename traits::memory_traits; + + CArray node_set, sort_set; + GO patch_id, element_id; + LO local_patch_id; + + // Default Constructor + Node_Combination() {} + + // Constructor with initialization + Node_Combination(CArray& nodes_init) + { + node_set = nodes_init; + sort_set = CArray(node_set.size()); } - //check if the nodes in the set are the same; sort them to simplify - std::sort(sort_set1.pointer(),sort_set1.pointer()+sort_set1.size()); - std::sort(sort_set2.pointer(),sort_set2.pointer()+sort_set2.size());\ - - //loop through the sorted nodes to check for equivalence - for(int i = 0; i < this_size; i++) - if(sort_set1(i)!=sort_set2(i)) return false; - - return true; - - } + // Destructor + ~Node_Combination() {} + + // overload = operator + Node_Combination& operator=(const Node_Combination& not_this) + { + node_set = not_this.node_set; + sort_set = not_this.sort_set; + patch_id = not_this.patch_id; + element_id = not_this.element_id; + local_patch_id = not_this.local_patch_id; + return *this; + } + // overload = operator + bool operator==(Node_Combination& not_this) + { + int this_size = this->node_set.size(); + // check if this node combination is identical + // first check size of the combination + if (this_size != not_this.node_set.size()) + { + return false; + } + + CArray sort_set1 = this->sort_set; + CArray sort_set2 = not_this.sort_set; + for (int i = 0; i < this_size; i++) + { + sort_set1(i) = this->node_set(i); + sort_set2(i) = not_this.node_set(i); + } + + // check if the nodes in the set are the same; sort them to simplify + std::sort(sort_set1.pointer(), sort_set1.pointer() + sort_set1.size()); + std::sort(sort_set2.pointer(), sort_set2.pointer() + sort_set2.size()); \ + + // loop through the sorted nodes to check for equivalence + for (int i = 0; i < this_size; i++) + { + if (sort_set1(i) != sort_set2(i)) { return false; } } + + return true; + } }; #endif // end STATE_H From 420a721dbcdd71e7b07384651911c3dd94b9d877 Mon Sep 17 00:00:00 2001 From: Jacob Moore Date: Fri, 16 Feb 2024 15:31:49 -0600 Subject: [PATCH 02/64] STYLE: Tidying up and wrapping debug statements in #ifdefs --- scripts/build-fierro.sh | 2 +- .../FEA_Module_Dynamic_Elasticity.cpp | 209 +++++++------- .../FEA_Module_Dynamic_Elasticity.h | 3 +- .../elastic_optimization.cpp | 240 +++++++++------- .../Dynamic_Elastic_Solver/force_elastic.cpp | 136 ++++----- .../force_gradients_elastic.cpp | 26 +- .../Dynamic_Elastic_Solver/geometry.cpp | 9 +- .../Dynamic_Elastic_Solver/momentum.cpp | 4 + .../Eulerian_Solver/FEA_Module_Eulerian.cpp | 59 ++-- .../Parallel-Explicit/Explicit_Solver.cpp | 145 +++++----- .../SGH_Solver/include/mesh.h | 8 +- .../SGH_Solver/src/FEA_Module_SGH.cpp | 262 ++++++++++-------- .../SGH_Solver/src/force_gradients_sgh.cpp | 56 ++-- .../SGH_Solver/src/force_sgh.cpp | 9 +- .../SGH_Solver/src/geometry.cpp | 7 - .../SGH_Solver/src/momentum.cpp | 20 +- .../SGH_Solver/src/setup_sgh.cpp | 162 +++++------ .../SGH_Solver/src/sgh_optimization.cpp | 100 +++---- .../SGH_Solver/src/time_integration.cpp | 2 +- 19 files changed, 787 insertions(+), 672 deletions(-) diff --git a/scripts/build-fierro.sh b/scripts/build-fierro.sh index 6e76c4941..983c0b77a 100755 --- a/scripts/build-fierro.sh +++ b/scripts/build-fierro.sh @@ -28,7 +28,7 @@ show_help() { echo " explicit-evpfft builds the explicit solver with the EVPFFT material model" echo " explicit-ls-evpfft builds the explicit solver with the LS-EVPFFT material model" echo " explicit-evp builds the explicit solver with the EVP material model" - echo " implicit builds the explicit solver" + echo " implicit builds the implicit solver" echo " " echo " --kokkos_build_type The desired kokkos parallel backend to use. The default is 'serial'" echo " " diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.cpp b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.cpp index 652d7e797..fdabd0bcf 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.cpp @@ -135,23 +135,27 @@ FEA_Module_Dynamic_Elasticity::FEA_Module_Dynamic_Elasticity( // set parameters Dynamic_Options dynamic_options = simparam->dynamic_options; - time_value = dynamic_options.time_value; - time_final = dynamic_options.time_final; - dt_max = dynamic_options.dt_max; - dt_min = dynamic_options.dt_min; - dt_cfl = dynamic_options.dt_cfl; + + dt = dynamic_options.dt; + time_value = dynamic_options.time_value; + time_final = dynamic_options.time_final; + dt_max = dynamic_options.dt_max; + dt_min = dynamic_options.dt_min; + dt_cfl = dynamic_options.dt_cfl; + rk_num_bins = simparam->dynamic_options.rk_num_bins; + graphics_time = simparam->output_options.graphics_time; graphics_dt_ival = simparam->output_options.graphics_dt_ival; graphics_cyc_ival = simparam->output_options.graphics_cyc_ival; - cycle_stop = dynamic_options.cycle_stop; - rk_num_stages = dynamic_options.rk_num_stages; - dt = dynamic_options.dt; + graphics_times = simparam->output_options.graphics_times; + graphics_id = simparam->output_options.graphics_id; + + cycle_stop = dynamic_options.cycle_stop; + rk_num_stages = dynamic_options.rk_num_stages; + fuzz = dynamic_options.fuzz; tiny = dynamic_options.tiny; small = dynamic_options.small; - graphics_times = simparam->output_options.graphics_times; - graphics_id = simparam->output_options.graphics_id; - rk_num_bins = simparam->dynamic_options.rk_num_bins; if (simparam->topology_optimization_on) { @@ -182,25 +186,33 @@ FEA_Module_Dynamic_Elasticity::~FEA_Module_Dynamic_Elasticity() ------------------------------------------------------------------------- */ void FEA_Module_Dynamic_Elasticity::read_conditions_ansys_dat(std::ifstream* in, std::streampos before_condition_header) { - char ch; - int num_dim = simparam->num_dims; - int buffer_lines = 1000; - int max_word = 30; - auto input_options = simparam->input_options.value(); - int p_order = input_options.p_order; - real_t unit_scaling = input_options.unit_scaling; - int local_node_index, current_column_index; - size_t strain_count; - std::string skip_line, read_line, substring, token; - std::stringstream line_parse, line_parse2; + Input_Options input_options = simparam->input_options.value(); + + char ch; + std::string skip_line, read_line, substring, token; + std::stringstream line_parse, line_parse2; + + int num_dim = simparam->num_dims; + int buffer_lines = 1000; + int max_word = 30; + int local_node_index, current_column_index; + int p_order = input_options.p_order; + int buffer_loop, buffer_iteration, buffer_iterations, scan_loop, nodes_per_element, words_per_line; + + size_t strain_count; + size_t read_index_start; + size_t node_rid; + size_t elem_gid; + + real_t unit_scaling = input_options.unit_scaling; + CArrayKokkos read_buffer; CArrayKokkos read_buffer_indices; - int buffer_loop, buffer_iteration, buffer_iterations, scan_loop, nodes_per_element, words_per_line; - size_t read_index_start, node_rid, elem_gid; - LO local_dof_id; - GO node_gid; - real_t dof_value; - host_vec_array node_densities; + + LO local_dof_id; + GO node_gid; + real_t dof_value; + host_vec_array node_densities; } // end read_conditions_ansys_dat // ----------------------------------------------------------------------------- @@ -554,8 +566,10 @@ void FEA_Module_Dynamic_Elasticity::init_output() bool output_velocity_flag = simparam->output(FIELD::velocity); bool output_strain_flag = simparam->output(FIELD::strain); bool output_stress_flag = simparam->output(FIELD::stress); - int num_dim = simparam->num_dims; - int Brows; + + int num_dim = simparam->num_dims; + int Brows; + if (num_dim == 3) { Brows = 6; @@ -842,16 +856,14 @@ void FEA_Module_Dynamic_Elasticity::comm_variables(Teuchos::RCP zp) void FEA_Module_Dynamic_Elasticity::node_density_constraints(host_vec_array& node_densities_lower_bound) { - const size_t num_dim = mesh->num_dims; const_vec_array all_initial_node_coords = all_initial_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); - const size_t num_lcs = module_params->loading_conditions.size(); + + const size_t num_dim = mesh->num_dims; + const size_t num_lcs = module_params->loading_conditions.size(); const DCArrayKokkos mat_fill = simparam->mat_fill; const DCArrayKokkos loading = module_params->loading; - // debug check - // std::cout << "NUMBER OF LOADING CONDITIONS: " << num_lcs << std::endl; - // walk over the nodes to update the velocity FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { double current_node_coords[3]; @@ -872,7 +884,7 @@ void FEA_Module_Dynamic_Elasticity::node_density_constraints(host_vec_array& nod node_densities_lower_bound(node_gid, 0) = 1; } } - }); // end for parallel for over nodes + }); // end for parallel for over nodes } /* ---------------------------------------------------------------------------- @@ -882,12 +894,14 @@ void FEA_Module_Dynamic_Elasticity::node_density_constraints(host_vec_array& nod void FEA_Module_Dynamic_Elasticity::setup() { Dynamic_Options dynamic_options = simparam->dynamic_options; - const size_t rk_level = dynamic_options.rk_num_bins - 1; - const size_t num_fills = simparam->regions.size(); - const size_t rk_num_bins = dynamic_options.rk_num_bins; - const size_t num_bcs = module_params->boundary_conditions.size(); - const size_t num_materials = simparam->materials.size(); - const int num_dim = simparam->num_dims; + + const size_t rk_level = dynamic_options.rk_num_bins - 1; + const size_t num_fills = simparam->regions.size(); + const size_t rk_num_bins = dynamic_options.rk_num_bins; + const size_t num_bcs = module_params->boundary_conditions.size(); + const size_t num_materials = simparam->materials.size(); + + const int num_dim = simparam->num_dims; // --------------------------------------------------------------------- // obtain mesh data @@ -960,50 +974,28 @@ void FEA_Module_Dynamic_Elasticity::setup() // create Dual Views of the individual node struct variables node_coords = DViewCArrayKokkos(node_interface.coords.get_kokkos_dual_view().view_host().data(), rk_num_bins, num_nodes, num_dim); - - node_vel = DViewCArrayKokkos(node_interface.vel.get_kokkos_dual_view().view_host().data(), rk_num_bins, num_nodes, num_dim); - - node_mass = DViewCArrayKokkos(node_interface.mass.get_kokkos_dual_view().view_host().data(), num_nodes); + node_vel = DViewCArrayKokkos(node_interface.vel.get_kokkos_dual_view().view_host().data(), rk_num_bins, num_nodes, num_dim); + node_mass = DViewCArrayKokkos(node_interface.mass.get_kokkos_dual_view().view_host().data(), num_nodes); // create Dual Views of the individual elem struct variables - elem_den = DViewCArrayKokkos(&elem_interface.den(0), - num_elems); - - elem_pres = DViewCArrayKokkos(&elem_interface.pres(0), - num_elems); - + elem_den = DViewCArrayKokkos(&elem_interface.den(0), num_elems); + elem_pres = DViewCArrayKokkos(&elem_interface.pres(0), num_elems); elem_stress = DViewCArrayKokkos(&elem_interface.stress(0, 0, 0, 0), - rk_num_bins, - num_elems, - 3, - 3); // always 3D even in 2D-RZ - - elem_sspd = DViewCArrayKokkos(&elem_interface.sspd(0), - num_elems); - - elem_sie = DViewCArrayKokkos(&elem_interface.sie(0, 0), rk_num_bins, - num_elems); - - elem_vol = DViewCArrayKokkos(&elem_interface.vol(0), - num_elems); - - elem_div = DViewCArrayKokkos(&elem_interface.div(0), - num_elems); - - elem_mass = DViewCArrayKokkos(&elem_interface.mass(0), - num_elems); - - elem_mat_id = DViewCArrayKokkos(&elem_interface.mat_id(0), - num_elems); + num_elems, + 3, 3); // always 3D even in 2D-RZ + elem_sspd = DViewCArrayKokkos(&elem_interface.sspd(0), num_elems); + elem_sie = DViewCArrayKokkos(&elem_interface.sie(0, 0), + rk_num_bins, + num_elems); + elem_vol = DViewCArrayKokkos(&elem_interface.vol(0), num_elems); + elem_div = DViewCArrayKokkos(&elem_interface.div(0), num_elems); + elem_mass = DViewCArrayKokkos(&elem_interface.mass(0), num_elems); + elem_mat_id = DViewCArrayKokkos(&elem_interface.mat_id(0), num_elems); // create Dual Views of the corner struct variables - corner_force = DViewCArrayKokkos(&corner_interface.force(0, 0), - num_corners, - num_dim); - - corner_mass = DViewCArrayKokkos(&corner_interface.mass(0), - num_corners); + corner_force = DViewCArrayKokkos(&corner_interface.force(0, 0), num_corners, num_dim); + corner_mass = DViewCArrayKokkos(&corner_interface.mass(0), num_corners); // allocate elem_vel_grad elem_vel_grad = DCArrayKokkos(num_elems, 3, 3); @@ -1026,6 +1018,7 @@ void FEA_Module_Dynamic_Elasticity::setup() const DCArrayKokkos boundary = module_params->boundary; const DCArrayKokkos mat_fill = simparam->mat_fill; const DCArrayKokkos material = simparam->material; + global_vars = simparam->global_vars; state_vars = DCArrayKokkos(rnum_elem, simparam->max_num_state_vars); elem_user_output_vars = DCArrayKokkos(rnum_elem, simparam->output_options.max_num_user_output_vars); @@ -1093,7 +1086,7 @@ void FEA_Module_Dynamic_Elasticity::setup() { FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { elem_mat_id(elem_gid) = mat_fill(f_id).material_id; - }); + }); } elem_mat_id.update_host(); @@ -1411,7 +1404,7 @@ void FEA_Module_Dynamic_Elasticity::setup() FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { node_mass_interface(node_gid, 0) = node_mass(node_gid); - }); // end parallel for + }); // end parallel for } // end view scope Kokkos::fence(); // communicate ghost densities @@ -1424,7 +1417,7 @@ void FEA_Module_Dynamic_Elasticity::setup() FOR_ALL_CLASS(node_gid, nlocal_nodes, nall_nodes, { node_mass(node_gid) = ghost_node_mass_interface(node_gid - nlocal_nodes, 0); - }); // end parallel for + }); // end parallel for } // end view scope Kokkos::fence(); } // endif @@ -1549,7 +1542,7 @@ void FEA_Module_Dynamic_Elasticity::tag_bdys(const DCArrayKokkos& bo bdy_patches_in_set(bdy_set, index) = bdy_patch_gid; } // end if } // end for bdy_patch - // } + // } }); // end FOR_ALL_CLASS bdy_sets // debug check @@ -1737,6 +1730,8 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve() Dynamic_Options dynamic_options = simparam->dynamic_options; const size_t rk_level = dynamic_options.rk_num_bins - 1; + const int num_dim = simparam->num_dims; + time_value = dynamic_options.time_initial; time_final = dynamic_options.time_final; dt_max = dynamic_options.dt_max; @@ -1752,15 +1747,19 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve() small = dynamic_options.small; graphics_times = simparam->output_options.graphics_times; graphics_id = simparam->output_options.graphics_id; - size_t num_bdy_nodes = mesh->num_bdy_nodes; - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos material = simparam->material; - int nTO_modules; - int old_max_forward_buffer; - unsigned long cycle; - const int num_dim = simparam->num_dims; - real_t objective_accumulation, global_objective_accumulation; - std::vector> FEA_Module_My_TO_Modules = simparam->FEA_Module_My_TO_Modules; + size_t num_bdy_nodes = mesh->num_bdy_nodes; + + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos material = simparam->material; + + int nTO_modules; + int old_max_forward_buffer; + + unsigned long cycle; + + real_t objective_accumulation, global_objective_accumulation; + + std::vector> FEA_Module_My_TO_Modules = simparam->FEA_Module_My_TO_Modules; problem = Explicit_Solver_Pointer_->problem; // Pointer to ROL optimization problem object ROL::Ptr> obj_pointer; @@ -1915,7 +1914,7 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve() node_velocities_interface(node_gid, idim) = node_vel(rk_level, node_gid, idim); node_coords_interface(node_gid, idim) = node_coords(rk_level, node_gid, idim); } - }); + }); } // end view scope Kokkos::fence(); @@ -1950,7 +1949,7 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve() all_node_velocities_interface(node_gid, idim) = node_velocities_interface(node_gid, idim); all_node_coords_interface(node_gid, idim) = node_coords_interface(node_gid, idim); } - }); // end parallel for + }); // end parallel for Kokkos::fence(); FOR_ALL_CLASS(node_gid, nlocal_nodes, nlocal_nodes + nghost_nodes, { @@ -1959,7 +1958,7 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve() all_node_velocities_interface(node_gid, idim) = ghost_node_velocities_interface(node_gid - nlocal_nodes, idim); all_node_coords_interface(node_gid, idim) = ghost_node_coords_interface(node_gid - nlocal_nodes, idim); } - }); // end parallel for + }); // end parallel for Kokkos::fence(); } // end view scope @@ -2114,7 +2113,7 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve() { node_velocities_interface(node_gid, idim) = node_vel(rk_level, node_gid, idim); } - }); // end parallel for + }); // end parallel for } // end view scope Kokkos::fence(); @@ -2138,7 +2137,7 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve() { node_vel(rk_level, node_gid, idim) = ghost_node_velocities_interface(node_gid - nlocal_nodes, idim); } - }); // end parallel for + }); // end parallel for } // end view scope Kokkos::fence(); @@ -2225,7 +2224,7 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve() vec_array node_mass_interface = node_masses_distributed->getLocalView(Tpetra::Access::ReadWrite); FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { node_mass_interface(node_gid, 0) = node_mass(node_gid); - }); // end parallel for + }); // end parallel for } // end view scope Kokkos::fence(); // communicate ghost densities @@ -2238,7 +2237,7 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve() FOR_ALL_CLASS(node_gid, nlocal_nodes, nall_nodes, { node_mass(node_gid) = ghost_node_mass_interface(node_gid - nlocal_nodes, 0); - }); // end parallel for + }); // end parallel for } // end view scope Kokkos::fence(); @@ -2379,17 +2378,19 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve() { const_vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); const_vec_array ghost_node_velocities_interface = Explicit_Solver_Pointer_->ghost_node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); - vec_array all_node_velocities_interface = Explicit_Solver_Pointer_->all_node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); const_vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); const_vec_array ghost_node_coords_interface = Explicit_Solver_Pointer_->ghost_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + vec_array all_node_velocities_interface = Explicit_Solver_Pointer_->all_node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { for (int idim = 0; idim < num_dim; idim++) { all_node_velocities_interface(node_gid, idim) = node_velocities_interface(node_gid, idim); all_node_coords_interface(node_gid, idim) = node_coords_interface(node_gid, idim); } - }); // end parallel for + }); // end parallel for Kokkos::fence(); FOR_ALL_CLASS(node_gid, nlocal_nodes, nlocal_nodes + nghost_nodes, { @@ -2398,7 +2399,7 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve() all_node_velocities_interface(node_gid, idim) = ghost_node_velocities_interface(node_gid - nlocal_nodes, idim); all_node_coords_interface(node_gid, idim) = ghost_node_coords_interface(node_gid - nlocal_nodes, idim); } - }); // end parallel for + }); // end parallel for Kokkos::fence(); } // end view scope @@ -2434,7 +2435,7 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve() { KE_loc_sum += node_mass(node_gid) * ke; } - }, KE_sum); + }, KE_sum); Kokkos::fence(); KE_sum = 0.5 * KE_sum; objective_accumulation += KE_sum * dt; diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.h b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.h index a62ab5bc7..6c10c8504 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.h +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.h @@ -179,7 +179,8 @@ class FEA_Module_Dynamic_Elasticity : public FEA_Module const double x3, const double y3) const; - double average_element_density(const int nodes_per_elem, const CArray current_element_densities) const; + double average_element_density(const int nodes_per_elem, + const CArray current_element_densities) const; void get_divergence(DViewCArrayKokkos& elem_div, const mesh_t mesh, diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/elastic_optimization.cpp b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/elastic_optimization.cpp index 7f122066d..2805208a9 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/elastic_optimization.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/elastic_optimization.cpp @@ -264,27 +264,27 @@ void FEA_Module_Dynamic_Elasticity::update_forward_solve(Teuchos::RCP // --- Pressure --- eos_model->calc_pressure(elem_pres, - elem_stress, - elem_gid, - elem_mat_id(elem_gid), - state_vars, - global_vars, - elem_user_output_vars, - elem_sspd, - elem_den(elem_gid), - elem_sie(rk_level, elem_gid)); + elem_stress, + elem_gid, + elem_mat_id(elem_gid), + state_vars, + global_vars, + elem_user_output_vars, + elem_sspd, + elem_den(elem_gid), + elem_sie(rk_level, elem_gid)); // --- Sound speed --- eos_model->calc_sound_speed(elem_pres, - elem_stress, - elem_gid, - elem_mat_id(elem_gid), - state_vars, - global_vars, - elem_user_output_vars, - elem_sspd, - elem_den(elem_gid), - elem_sie(rk_level, elem_gid)); + elem_stress, + elem_gid, + elem_mat_id(elem_gid), + state_vars, + global_vars, + elem_user_output_vars, + elem_sspd, + elem_den(elem_gid), + elem_sie(rk_level, elem_gid)); // loop over the nodes of this element and apply velocity for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) @@ -412,7 +412,7 @@ void FEA_Module_Dynamic_Elasticity::update_forward_solve(Teuchos::RCP elem_pres(elem_gid) / (mat_fill(f_id).den * (gamma - 1.0)); } // end if } // end if fill - }); // end FOR_ALL_CLASS element loop + }); // end FOR_ALL_CLASS element loop Kokkos::fence(); } // end for loop over fills } // end view scope @@ -477,7 +477,7 @@ void FEA_Module_Dynamic_Elasticity::update_forward_solve(Teuchos::RCP vec_array node_mass_interface = node_masses_distributed->getLocalView(Tpetra::Access::ReadWrite); FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { node_mass_interface(node_gid, 0) = node_mass(node_gid); - }); // end parallel for + }); // end parallel for } // end view scope Kokkos::fence(); // communicate ghost densities @@ -490,7 +490,7 @@ void FEA_Module_Dynamic_Elasticity::update_forward_solve(Teuchos::RCP FOR_ALL_CLASS(node_gid, nlocal_nodes, nall_nodes, { node_mass(node_gid) = ghost_node_mass_interface(node_gid - nlocal_nodes, 0); - }); // end parallel for + }); // end parallel for } // end view scope Kokkos::fence(); @@ -526,13 +526,17 @@ double FEA_Module_Dynamic_Elasticity::average_element_density(const int nodes_pe void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_adjoint() { - size_t num_bdy_nodes = mesh->num_bdy_nodes; - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos material = simparam->material; - const int num_dim = simparam->num_dims; - real_t global_dt; - size_t current_data_index, next_data_index; - Teuchos::RCP previous_adjoint_vector_distributed, current_adjoint_vector_distributed, previous_velocity_vector_distributed, current_velocity_vector_distributed; + size_t num_bdy_nodes = mesh->num_bdy_nodes; + size_t current_data_index, next_data_index; + real_t global_dt; + + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos material = simparam->material; + const int num_dim = simparam->num_dims; + + Teuchos::RCP previous_adjoint_vector_distributed, current_adjoint_vector_distributed; + Teuchos::RCP previous_velocity_vector_distributed, current_velocity_vector_distributed; + // initialize first adjoint vector at last_time_step to 0 as the terminal value (*adjoint_vector_data)[last_time_step + 1]->putScalar(0); @@ -585,10 +589,12 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_adjoint() for (int idim = 0; idim < num_dim; idim++) { // cancellation of half from midpoint and 2 from adjoint equation already done - current_adjoint_vector(node_gid, idim) = -0.5 * (current_velocity_vector(node_gid, idim) + previous_velocity_vector(node_gid, idim)) * global_dt + previous_adjoint_vector(node_gid, - idim); + current_adjoint_vector(node_gid, idim) = + -0.5 * (current_velocity_vector(node_gid, idim) + + previous_velocity_vector(node_gid, idim)) * global_dt + + previous_adjoint_vector(node_gid, idim); } - }); // end parallel for + }); // end parallel for Kokkos::fence(); } // end view scope } @@ -600,16 +606,21 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_adjoint() void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_adjoint_full() { - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - size_t num_bdy_nodes = mesh->num_bdy_nodes; - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos material = simparam->material; - const int num_dim = simparam->num_dims; - const real_t damping_constant = module_params->damping_constant; - real_t global_dt; - size_t current_data_index, next_data_index; - Teuchos::RCP previous_adjoint_vector_distributed, current_adjoint_vector_distributed, previous_velocity_vector_distributed, current_velocity_vector_distributed; - Teuchos::RCP previous_phi_adjoint_vector_distributed, current_phi_adjoint_vector_distributed; + const int num_dim = simparam->num_dims; + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + const real_t damping_constant = module_params->damping_constant; + + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos material = simparam->material; + + real_t global_dt; + size_t num_bdy_nodes = mesh->num_bdy_nodes; + size_t current_data_index, next_data_index; + + Teuchos::RCP revious_adjoint_vector_distributed, current_adjoint_vector_distributed; + Teuchos::RCP previous_velocity_vector_distributed, current_velocity_vector_distributed; + Teuchos::RCP previous_phi_adjoint_vector_distributed, current_phi_adjoint_vector_distributed; + // initialize first adjoint vector at last_time_step to 0 as the terminal value (*adjoint_vector_data)[last_time_step + 1]->putScalar(0); (*phi_adjoint_vector_data)[last_time_step + 1]->putScalar(0); @@ -663,7 +674,7 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_adjoint_full() node_vel(rk_level, node_gid, idim) = previous_velocity_vector(node_gid, idim); node_coords(rk_level, node_gid, idim) = previous_coordinate_vector(node_gid, idim); } - }); + }); Kokkos::fence(); get_force_vgradient_elastic(material, @@ -689,9 +700,10 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_adjoint_full() // compute gradient of force with respect to velocity const_vec_array previous_adjoint_vector = (*adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); - const_vec_array phi_previous_adjoint_vector = (*phi_adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); - vec_array midpoint_adjoint_vector = adjoint_vector_distributed->getLocalView(Tpetra::Access::ReadWrite); - vec_array phi_midpoint_adjoint_vector = phi_adjoint_vector_distributed->getLocalView(Tpetra::Access::ReadWrite); + const_vec_array phi_previous_adjoint_vector = (*phi_adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + + vec_array midpoint_adjoint_vector = adjoint_vector_distributed->getLocalView(Tpetra::Access::ReadWrite); + vec_array phi_midpoint_adjoint_vector = phi_adjoint_vector_distributed->getLocalView(Tpetra::Access::ReadWrite); // half step update for RK2 scheme FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { @@ -713,22 +725,25 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_adjoint_full() matrix_contribution / node_mass(node_gid) - phi_previous_adjoint_vector(node_gid, idim) / node_mass(node_gid); midpoint_adjoint_vector(node_gid, idim) = -rate_of_change * global_dt / 2 + previous_adjoint_vector(node_gid, idim); + matrix_contribution = 0; // compute resulting row of force displacement gradient matrix transpose right multiplied by adjoint vector for (int idof = 0; idof < Gradient_Matrix_Strides(node_gid * num_dim + idim); idof++) { dof_id = DOF_Graph_Matrix(node_gid * num_dim + idim, idof); - matrix_contribution += -previous_adjoint_vector(dof_id / num_dim, dof_id % num_dim) * Force_Gradient_Positions(node_gid * num_dim + idim, idof); + matrix_contribution += -previous_adjoint_vector(dof_id / num_dim, dof_id % num_dim) + * Force_Gradient_Positions(node_gid * num_dim + idim, idof); } rate_of_change = -matrix_contribution; // rate_of_change = -0.0000001*previous_adjoint_vector(node_gid,idim); phi_midpoint_adjoint_vector(node_gid, idim) = -rate_of_change * global_dt / 2 + phi_previous_adjoint_vector(node_gid, idim); } - }); // end parallel for + }); // end parallel for Kokkos::fence(); boundary_adjoint(*mesh, boundary, midpoint_adjoint_vector, phi_midpoint_adjoint_vector); comm_adjoint_vectors(cycle); + // swap names to get ghost nodes for the midpoint vectors vec_array current_adjoint_vector = adjoint_vector_distributed->getLocalView(Tpetra::Access::ReadWrite); vec_array phi_current_adjoint_vector = phi_adjoint_vector_distributed->getLocalView(Tpetra::Access::ReadWrite); @@ -766,7 +781,7 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_adjoint_full() // rate_of_change = -0.0000001*midpoint_adjoint_vector(node_gid,idim); phi_current_adjoint_vector(node_gid, idim) = -rate_of_change * global_dt + phi_previous_adjoint_vector(node_gid, idim); } - }); // end parallel for + }); // end parallel for Kokkos::fence(); boundary_adjoint(*mesh, boundary, current_adjoint_vector, phi_current_adjoint_vector); @@ -783,15 +798,18 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_adjoint_full() void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient(const_vec_array design_variables, vec_array design_gradients) { - size_t num_bdy_nodes = mesh->num_bdy_nodes; - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos material = simparam->material; - const int num_dim = simparam->num_dims; - int num_corners = rnum_elem * num_nodes_in_elem; - real_t global_dt; - size_t current_data_index, next_data_index; - CArrayKokkos current_element_velocities = CArrayKokkos(num_nodes_in_elem, num_dim); - CArrayKokkos current_element_adjoint = CArrayKokkos(num_nodes_in_elem, num_dim); + size_t num_bdy_nodes = mesh->num_bdy_nodes; + size_t current_data_index, next_data_index; + + int num_corners = rnum_elem * num_nodes_in_elem; + real_t global_dt; + + const int num_dim = simparam->num_dims; + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos material = simparam->material; + + auto current_element_velocities = CArrayKokkos(num_nodes_in_elem, num_dim); + auto current_element_adjoint = CArrayKokkos(num_nodes_in_elem, num_dim); if (myrank == 0) { @@ -803,7 +821,7 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient(const // compute design gradients FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { design_gradients(node_id, 0) = 0; - }); // end parallel for + }); // end parallel for Kokkos::fence(); // gradient contribution from kinetic energy vMv product. @@ -874,7 +892,7 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient(const corner_id = elem_id * num_nodes_in_elem + inode; corner_value_storage(corner_id) = inner_product * global_dt; } - }); // end parallel for + }); // end parallel for Kokkos::fence(); // accumulate node values from corner storage @@ -886,7 +904,7 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient(const corner_id = corners_in_node(node_id, icorner); design_gradients(node_id, 0) += corner_value_storage(corner_id); } - }); // end parallel for + }); // end parallel for Kokkos::fence(); // test code @@ -928,7 +946,7 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient(const FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { design_gradients(node_id, 0) *= -0.5 / (double)num_nodes_in_elem / (double)num_nodes_in_elem; // design_gradients(node_id,0) =0.00001; - }); // end parallel for + }); // end parallel for Kokkos::fence(); // gradient contribution from Force vector. @@ -1007,7 +1025,7 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient(const corner_id = elem_id * num_nodes_in_elem + inode; corner_value_storage(corner_id) = -inner_product * global_dt / (double)num_nodes_in_elem; } - }); // end parallel for + }); // end parallel for Kokkos::fence(); // accumulate node values from corner storage @@ -1019,7 +1037,7 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient(const corner_id = corners_in_node(node_id, icorner); design_gradients(node_id, 0) += corner_value_storage(corner_id); } - }); // end parallel for + }); // end parallel for Kokkos::fence(); } // end view scope } @@ -1031,16 +1049,18 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient(const void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient_full(Teuchos::RCP design_densities_distributed, Teuchos::RCP design_gradients_distributed) { - size_t num_bdy_nodes = mesh->num_bdy_nodes; - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos material = simparam->material; - const int num_dim = simparam->num_dims; - int num_corners = rnum_elem * num_nodes_in_elem; - real_t global_dt; - bool element_constant_density = true; - size_t current_data_index, next_data_index; - CArrayKokkos current_element_velocities = CArrayKokkos(num_nodes_in_elem, num_dim); - CArrayKokkos current_element_adjoint = CArrayKokkos(num_nodes_in_elem, num_dim); + bool element_constant_density = true; + size_t num_bdy_nodes = mesh->num_bdy_nodes; + size_t current_data_index, next_data_index; + int num_corners = rnum_elem * num_nodes_in_elem; + real_t global_dt; + + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos material = simparam->material; + const int num_dim = simparam->num_dims; + + auto current_element_velocities = CArrayKokkos(num_nodes_in_elem, num_dim); + auto current_element_adjoint = CArrayKokkos(num_nodes_in_elem, num_dim); if (myrank == 0) { @@ -1055,7 +1075,7 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient_full( // compute design gradients FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { design_gradients(node_id, 0) = 0; - }); // end parallel for + }); // end parallel for Kokkos::fence(); // gradient contribution from kinetic energy v(dM/drho)v product. @@ -1135,7 +1155,7 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient_full( // division by design ratio recovers nominal element mass used in the gradient operator corner_value_storage(corner_id) = inner_product * global_dt / relative_element_densities(elem_id); } - }); // end parallel for + }); // end parallel for Kokkos::fence(); // accumulate node values from corner storage @@ -1147,7 +1167,7 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient_full( corner_id = corners_in_node(node_id, icorner); design_gradients(node_id, 0) += corner_value_storage(corner_id); } - }); // end parallel for + }); // end parallel for Kokkos::fence(); } // end view scope } @@ -1156,7 +1176,7 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient_full( FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { design_gradients(node_id, 0) *= 0.5 / (double)num_nodes_in_elem / (double)num_nodes_in_elem; // design_gradients(node_id,0) =0.00001; - }); // end parallel for + }); // end parallel for Kokkos::fence(); // gradient contribution from time derivative of adjoint \dot{lambda}(dM/drho)v product. @@ -1203,7 +1223,9 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient_full( const_vec_array next_phi_adjoint_vector = (*phi_adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); const real_t damping_constant = module_params->damping_constant; - FOR_ALL_CLASS(elem_id, 0, rnum_elem, { + + FOR_ALL_CLASS(elem_id, 0, rnum_elem, + { real_t lambda_dot_current; real_t lambda_dot_next; size_t node_id; @@ -1230,10 +1252,14 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient_full( for (int idim = 0; idim < num_dim; idim++) { // lambda_dot_current = lambda_dot_next = (next_adjoint_vector(node_id,idim)-current_adjoint_vector(node_id,idim))/global_dt; - lambda_dot_current = current_velocity_vector(node_id, idim) + damping_constant * current_adjoint_vector(node_id, - idim) / node_mass(node_id) - current_phi_adjoint_vector(node_id, idim) / node_mass(node_id); - lambda_dot_next = next_velocity_vector(node_id, idim) + damping_constant * next_adjoint_vector(node_id, idim) / node_mass(node_id) - next_phi_adjoint_vector(node_id, - idim) / node_mass(node_id); + lambda_dot_current = current_velocity_vector(node_id, idim) + + damping_constant * current_adjoint_vector(node_id, idim) / node_mass(node_id) + - current_phi_adjoint_vector(node_id, idim) / node_mass(node_id); + + lambda_dot_next = next_velocity_vector(node_id, idim) + + damping_constant * next_adjoint_vector(node_id, idim) / node_mass(node_id) + - next_phi_adjoint_vector(node_id, idim) / node_mass(node_id); + inner_product += elem_mass(elem_id) * (lambda_dot_current + lambda_dot_next) * current_element_velocities(ifill, idim) / 2; } } @@ -1245,7 +1271,7 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient_full( // division by design ratio recovers nominal element mass used in the gradient operator corner_value_storage(corner_id) = inner_product * global_dt / relative_element_densities(elem_id); } - }); // end parallel for + }); // end parallel for Kokkos::fence(); // accumulate node values from corner storage @@ -1257,7 +1283,7 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient_full( corner_id = corners_in_node(node_id, icorner); design_gradients(node_id, 0) += -corner_value_storage(corner_id) / (double)num_nodes_in_elem / (double)num_nodes_in_elem; } - }); // end parallel for + }); // end parallel for Kokkos::fence(); } // end view scope } @@ -1268,7 +1294,8 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient_full( const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[0]->getLocalView(Tpetra::Access::ReadOnly); const_vec_array current_adjoint_vector = (*adjoint_vector_data)[0]->getLocalView(Tpetra::Access::ReadOnly); - FOR_ALL_CLASS(elem_id, 0, rnum_elem, { + FOR_ALL_CLASS(elem_id, 0, rnum_elem, + { real_t lambda_dot; size_t node_id; size_t corner_id; @@ -1304,7 +1331,7 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient_full( // division by design ratio recovers nominal element mass used in the gradient operator corner_value_storage(corner_id) = inner_product / relative_element_densities(elem_id); } - }); // end parallel for + }); // end parallel for Kokkos::fence(); // accumulate node values from corner storage @@ -1316,7 +1343,7 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient_full( corner_id = corners_in_node(node_id, icorner); design_gradients(node_id, 0) += -corner_value_storage(corner_id) / (double)num_nodes_in_elem / (double)num_nodes_in_elem; } - }); // end parallel for + }); // end parallel for Kokkos::fence(); } // end view scope } // end view scope @@ -1335,14 +1362,18 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient_full( ------------------------------------------------------------------------- */ void FEA_Module_Dynamic_Elasticity::init_assembly() { - int num_dim = simparam->num_dims; + int num_dim = simparam->num_dims; + int local_node_index; + int current_column_index; + size_t max_stride = 0; + size_t nodes_per_element; + // const_host_elem_conn_array nodes_in_elem = global_nodes_in_elem_distributed->getLocalView (Tpetra::Access::ReadOnly); Gradient_Matrix_Strides = DCArrayKokkos(nlocal_nodes * num_dim, "Gradient_Matrix_Strides"); + CArrayKokkos Graph_Fill(nall_nodes, "nall_nodes"); CArrayKokkos current_row_nodes_scanned; - int local_node_index, current_column_index; - size_t max_stride = 0; - size_t nodes_per_element; + nodal_density_flag = simparam->nodal_density_flag; penalty_power = simparam->optimization_options.simp_penalty_power; @@ -1352,8 +1383,8 @@ void FEA_Module_Dynamic_Elasticity::init_assembly() Graph_Matrix_Strides = DCArrayKokkos(nlocal_nodes, "Graph_Matrix_Strides"); // allocate storage for the sparse stiffness matrix map used in the assembly process - Global_Stiffness_Matrix_Assembly_Map = Global_Gradient_Matrix_Assembly_Map = DCArrayKokkos(rnum_elem, - max_nodes_per_element, max_nodes_per_element, "Global_Gradient_Matrix_Assembly_Map"); + Global_Stiffness_Matrix_Assembly_Map = Global_Gradient_Matrix_Assembly_Map = DCArrayKokkos( + rnum_elem, max_nodes_per_element, max_nodes_per_element, "Global_Gradient_Matrix_Assembly_Map"); // allocate array used to determine global node repeats in the sparse graph later DCArrayKokkos node_indices_used(nall_nodes, "node_indices_used"); @@ -1367,7 +1398,7 @@ void FEA_Module_Dynamic_Elasticity::init_assembly() Graph_Matrix_Strides_initial(inode) = 0; Graph_Matrix_Strides(inode) = 0; Graph_Fill(inode) = 0; - }); // end parallel for + }); // end parallel for Kokkos::fence(); // initialize nall arrays @@ -1375,7 +1406,7 @@ void FEA_Module_Dynamic_Elasticity::init_assembly() FOR_ALL_CLASS(inode, 0, nall_nodes, { node_indices_used(inode) = 0; column_index(inode) = 0; - }); // end parallel for + }); // end parallel for Kokkos::fence(); // count upper bound of strides for Sparse Pattern Graph by allowing repeats due to connectivity @@ -1418,7 +1449,7 @@ void FEA_Module_Dynamic_Elasticity::init_assembly() // equate strides for later FOR_ALL_CLASS(inode, 0, nlocal_nodes, { Graph_Matrix_Strides(inode) = Graph_Matrix_Strides_initial(inode) = Dual_Graph_Matrix_Strides_initial(inode); - }); // end parallel for + }); // end parallel for // for (int inode = 0; inode < nlocal_nodes; inode++) // std::cout << Graph_Matrix_Strides_initial(inode) << std::endl; @@ -1430,7 +1461,7 @@ void FEA_Module_Dynamic_Elasticity::init_assembly() { update = Graph_Matrix_Strides_initial(inode); } - }, max_stride); + }, max_stride); // std::cout << "THE MAX STRIDE" << max_stride << std::endl; // allocate array used in the repeat removal process @@ -1515,6 +1546,7 @@ void FEA_Module_Dynamic_Elasticity::init_assembly() int element_column_index; int current_stride; int current_row_n_nodes_scanned; + for (int inode = 0; inode < nlocal_nodes; inode++) { current_row_n_nodes_scanned = 0; @@ -1575,7 +1607,7 @@ void FEA_Module_Dynamic_Elasticity::init_assembly() node_indices_used(current_row_nodes_scanned(node_reset)) = 0; } } - }); + }); Kokkos::fence(); Graph_Matrix_Strides.update_host(); @@ -1587,7 +1619,7 @@ void FEA_Module_Dynamic_Elasticity::init_assembly() { Graph_Matrix(inode, istride) = Repeat_Graph_Matrix(inode, istride); } - }); // end parallel for + }); // end parallel for // deallocate repeat matrix @@ -1598,12 +1630,14 @@ void FEA_Module_Dynamic_Elasticity::init_assembly() // expand strides for stiffness matrix by multipling by dim FOR_ALL_CLASS(idof, 0, num_dim * nlocal_nodes, { Gradient_Matrix_Strides(idof) = num_dim * Graph_Matrix_Strides(idof / num_dim); - }); // end parallel for + }); // end parallel for - Stiffness_Matrix = Force_Gradient_Positions = RaggedRightArrayKokkos(Gradient_Matrix_Strides); - Force_Gradient_Velocities = RaggedRightArrayKokkos(Gradient_Matrix_Strides); + Stiffness_Matrix = RaggedRightArrayKokkos(Gradient_Matrix_Strides); DOF_Graph_Matrix = RaggedRightArrayKokkos(Gradient_Matrix_Strides); + Force_Gradient_Positions = RaggedRightArrayKokkos(Gradient_Matrix_Strides); + Force_Gradient_Velocities = RaggedRightArrayKokkos(Gradient_Matrix_Strides); + // set stiffness Matrix Graph // debug print // std::cout << "DOF GRAPH MATRIX ENTRIES ON TASK " << myrank << std::endl; @@ -1612,7 +1646,7 @@ void FEA_Module_Dynamic_Elasticity::init_assembly() { DOF_Graph_Matrix(idof, istride) = Graph_Matrix(idof / num_dim, istride / num_dim) * num_dim + istride % num_dim; } - }); // end parallel for + }); // end parallel for Stiffness_Matrix_Strides = Gradient_Matrix_Strides; diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/force_elastic.cpp b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/force_elastic.cpp index 4ef144028..edf219c35 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/force_elastic.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/force_elastic.cpp @@ -93,14 +93,15 @@ void FEA_Module_Dynamic_Elasticity::applied_forces(const DCArrayKokkosdynamic_options.rk_num_bins - 1; - const size_t num_dim = mesh.num_dims; - const_vec_array all_initial_node_coords = all_initial_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); - const size_t num_lcs = module_params->loading.size(); + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + const size_t num_dim = mesh.num_dims; + const size_t num_lcs = module_params->loading.size(); const DCArrayKokkos mat_fill = simparam->mat_fill; const DCArrayKokkos loading = module_params->loading; + const_vec_array all_initial_node_coords = all_initial_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + // debug check // std::cout << "NUMBER OF LOADING CONDITIONS: " << num_lcs << std::endl; @@ -118,7 +119,10 @@ void FEA_Module_Dynamic_Elasticity::applied_forces(const DCArrayKokkos legendre_nodes_1D(num_gauss_points); // CArrayKokkos legendre_weights_1D(num_gauss_points); - CArray legendre_nodes_1D(num_gauss_points); - CArray legendre_weights_1D(num_gauss_points); - real_t pointer_quad_coordinate[num_dim]; - real_t pointer_quad_coordinate_weight[num_dim]; - real_t pointer_interpolated_point[num_dim]; - real_t pointer_JT_row1[num_dim]; - real_t pointer_JT_row2[num_dim]; - real_t pointer_JT_row3[num_dim]; + CArray legendre_nodes_1D(num_gauss_points); + CArray legendre_weights_1D(num_gauss_points); + + real_t pointer_quad_coordinate[num_dim]; + real_t pointer_quad_coordinate_weight[num_dim]; + real_t pointer_interpolated_point[num_dim]; + real_t pointer_JT_row1[num_dim]; + real_t pointer_JT_row2[num_dim]; + real_t pointer_JT_row3[num_dim]; + real_t pointer_basis_values[elem->num_basis()]; + real_t pointer_basis_derivative_s1[elem->num_basis()]; + real_t pointer_basis_derivative_s2[elem->num_basis()]; + real_t pointer_basis_derivative_s3[elem->num_basis()]; + ViewCArray quad_coordinate(pointer_quad_coordinate, num_dim); ViewCArray quad_coordinate_weight(pointer_quad_coordinate_weight, num_dim); ViewCArray interpolated_point(pointer_interpolated_point, num_dim); ViewCArray JT_row1(pointer_JT_row1, num_dim); ViewCArray JT_row2(pointer_JT_row2, num_dim); ViewCArray JT_row3(pointer_JT_row3, num_dim); + ViewCArray basis_values(pointer_basis_values, elem->num_basis()); + ViewCArray basis_derivative_s1(pointer_basis_derivative_s1, elem->num_basis()); + ViewCArray basis_derivative_s2(pointer_basis_derivative_s2, elem->num_basis()); + ViewCArray basis_derivative_s3(pointer_basis_derivative_s3, elem->num_basis()); - real_t pointer_basis_values[elem->num_basis()]; - real_t pointer_basis_derivative_s1[elem->num_basis()]; - real_t pointer_basis_derivative_s2[elem->num_basis()]; - real_t pointer_basis_derivative_s3[elem->num_basis()]; - ViewCArray basis_values(pointer_basis_values, elem->num_basis()); - ViewCArray basis_derivative_s1(pointer_basis_derivative_s1, elem->num_basis()); - ViewCArray basis_derivative_s2(pointer_basis_derivative_s2, elem->num_basis()); - ViewCArray basis_derivative_s3(pointer_basis_derivative_s3, elem->num_basis()); CArrayKokkos nodal_positions(elem->num_basis(), num_dim); CArrayKokkos nodal_density(elem->num_basis()); @@ -857,16 +855,18 @@ void FEA_Module_Dynamic_Elasticity::compute_stiffness_gradients(const_host_vec_a { Element_Densities = Global_Element_Densities->getLocalView(Tpetra::Access::ReadOnly); } - int num_dim = simparam->num_dims; - int nodes_per_elem = elem->num_basis(); - int num_gauss_points = simparam->num_gauss_points; - int z_quad, y_quad, x_quad, direct_product_count; + int num_dim = simparam->num_dims; + int nodes_per_elem = elem->num_basis(); + int num_gauss_points = simparam->num_gauss_points; + int z_quad, y_quad, x_quad, direct_product_count; + size_t local_node_id, local_dof_idx, local_dof_idy, local_dof_idz; const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; GO current_global_index; - real_t global_dt; - size_t current_data_index, next_data_index; + real_t global_dt; + size_t current_data_index, next_data_index; + CArrayKokkos current_element_adjoint = CArrayKokkos(num_nodes_in_elem * num_dim); direct_product_count = std::pow(num_gauss_points, num_dim); @@ -875,29 +875,31 @@ void FEA_Module_Dynamic_Elasticity::compute_stiffness_gradients(const_host_vec_a real_t inner_product, matrix_term, Jacobian, invJacobian, weight_multiply; // CArrayKokkos legendre_nodes_1D(num_gauss_points); // CArrayKokkos legendre_weights_1D(num_gauss_points); - CArray legendre_nodes_1D(num_gauss_points); - CArray legendre_weights_1D(num_gauss_points); - real_t pointer_quad_coordinate[num_dim]; - real_t pointer_quad_coordinate_weight[num_dim]; - real_t pointer_interpolated_point[num_dim]; - real_t pointer_JT_row1[num_dim]; - real_t pointer_JT_row2[num_dim]; - real_t pointer_JT_row3[num_dim]; + CArray legendre_nodes_1D(num_gauss_points); + CArray legendre_weights_1D(num_gauss_points); + + real_t pointer_quad_coordinate[num_dim]; + real_t pointer_quad_coordinate_weight[num_dim]; + real_t pointer_interpolated_point[num_dim]; + real_t pointer_JT_row1[num_dim]; + real_t pointer_JT_row2[num_dim]; + real_t pointer_JT_row3[num_dim]; + real_t pointer_basis_values[elem->num_basis()]; + real_t pointer_basis_derivative_s1[elem->num_basis()]; + real_t pointer_basis_derivative_s2[elem->num_basis()]; + real_t pointer_basis_derivative_s3[elem->num_basis()]; + ViewCArray quad_coordinate(pointer_quad_coordinate, num_dim); ViewCArray quad_coordinate_weight(pointer_quad_coordinate_weight, num_dim); ViewCArray interpolated_point(pointer_interpolated_point, num_dim); ViewCArray JT_row1(pointer_JT_row1, num_dim); ViewCArray JT_row2(pointer_JT_row2, num_dim); ViewCArray JT_row3(pointer_JT_row3, num_dim); + ViewCArray basis_values(pointer_basis_values, elem->num_basis()); + ViewCArray basis_derivative_s1(pointer_basis_derivative_s1, elem->num_basis()); + ViewCArray basis_derivative_s2(pointer_basis_derivative_s2, elem->num_basis()); + ViewCArray basis_derivative_s3(pointer_basis_derivative_s3, elem->num_basis()); - real_t pointer_basis_values[elem->num_basis()]; - real_t pointer_basis_derivative_s1[elem->num_basis()]; - real_t pointer_basis_derivative_s2[elem->num_basis()]; - real_t pointer_basis_derivative_s3[elem->num_basis()]; - ViewCArray basis_values(pointer_basis_values, elem->num_basis()); - ViewCArray basis_derivative_s1(pointer_basis_derivative_s1, elem->num_basis()); - ViewCArray basis_derivative_s2(pointer_basis_derivative_s2, elem->num_basis()); - ViewCArray basis_derivative_s3(pointer_basis_derivative_s3, elem->num_basis()); CArrayKokkos nodal_positions(elem->num_basis(), num_dim); CArrayKokkos current_nodal_displacements(elem->num_basis() * num_dim); CArrayKokkos nodal_density(elem->num_basis()); @@ -923,7 +925,8 @@ void FEA_Module_Dynamic_Elasticity::compute_stiffness_gradients(const_host_vec_a elements::legendre_weights_1D(legendre_weights_1D, num_gauss_points); Solver::node_ordering_convention active_node_ordering_convention = Explicit_Solver_Pointer_->active_node_ordering_convention; - real_t current_density = 1; + real_t current_density = 1; + CArrayKokkos convert_node_order(max_nodes_per_element); if ((active_node_ordering_convention == Solver::ENSIGHT && num_dim == 3) || (active_node_ordering_convention == Solver::IJK && num_dim == 2)) { @@ -1030,10 +1033,15 @@ void FEA_Module_Dynamic_Elasticity::compute_stiffness_gradients(const_host_vec_a nodal_positions(node_loop, 0) = all_initial_node_coords(local_node_id, 0); nodal_positions(node_loop, 1) = all_initial_node_coords(local_node_id, 1); - current_nodal_displacements(node_loop * num_dim) = 0.5 * (next_coordinate_vector(local_node_id, 0) - all_initial_node_coords(local_node_id, - 0) + current_coordinate_vector(local_node_id, 0) - all_initial_node_coords(local_node_id, 0)); - current_nodal_displacements(node_loop * num_dim + 1) = 0.5 * (next_coordinate_vector(local_node_id, 1) - all_initial_node_coords(local_node_id, - 1) + current_coordinate_vector(local_node_id, 1) - all_initial_node_coords(local_node_id, 1)); + + current_nodal_displacements(node_loop * num_dim) = 0.5 + * (next_coordinate_vector(local_node_id, 0) - all_initial_node_coords(local_node_id, 0) + + current_coordinate_vector(local_node_id, 0) - all_initial_node_coords(local_node_id, 0)); + + current_nodal_displacements(node_loop * num_dim + 1) = 0.5 + * (next_coordinate_vector(local_node_id, 1) - all_initial_node_coords(local_node_id, 1) + + current_coordinate_vector(local_node_id, 1) - all_initial_node_coords(local_node_id, 1)); + current_element_adjoint(node_loop * num_dim) = 0.5 * (current_adjoint_vector(local_node_id, 0) + next_adjoint_vector(local_node_id, 0)); current_element_adjoint(node_loop * num_dim + 1) = 0.5 * (current_adjoint_vector(local_node_id, 1) + next_adjoint_vector(local_node_id, 1)); @@ -1042,7 +1050,7 @@ void FEA_Module_Dynamic_Elasticity::compute_stiffness_gradients(const_host_vec_a nodal_positions(node_loop, 2) = all_initial_node_coords(local_node_id, 2); current_nodal_displacements(node_loop * num_dim + 2) = 0.5 * (next_coordinate_vector(local_node_id, 2) - all_initial_node_coords(local_node_id, 2) + current_coordinate_vector(local_node_id, 2) - all_initial_node_coords(local_node_id, 2)); - current_element_adjoint(node_loop * num_dim + 2) = 0.5 * (current_adjoint_vector(local_node_id, 2) + next_adjoint_vector(local_node_id, 2)); + current_element_adjoint(node_loop * num_dim + 2) = 0.5 * (current_adjoint_vector(local_node_id, 2) + next_adjoint_vector(local_node_id, 2)); } if (nodal_density_flag) diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/force_gradients_elastic.cpp b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/force_gradients_elastic.cpp index 5e8999635..b88a3895e 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/force_gradients_elastic.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/force_gradients_elastic.cpp @@ -768,15 +768,17 @@ void FEA_Module_Dynamic_Elasticity::get_force_ugradient_elastic(const DCArrayKok void FEA_Module_Dynamic_Elasticity::force_design_gradient_term(const_vec_array design_variables, vec_array design_gradients) { - size_t num_bdy_nodes = mesh->num_bdy_nodes; - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos material = simparam->material; - const int num_dim = simparam->num_dims; - int num_corners = rnum_elem * num_nodes_in_elem; - real_t global_dt; - bool element_constant_density = true; - size_t current_data_index, next_data_index; - CArrayKokkos current_element_adjoint = CArrayKokkos(num_nodes_in_elem, num_dim); + bool element_constant_density = true; + int num_corners = rnum_elem * num_nodes_in_elem; + size_t num_bdy_nodes = mesh->num_bdy_nodes; + size_t current_data_index, next_data_index; + real_t global_dt; + + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos material = simparam->material; + const int num_dim = simparam->num_dims; + + auto current_element_adjoint = CArrayKokkos(num_nodes_in_elem, num_dim); // gradient contribution from gradient of Force vector with respect to design variable. for (unsigned long cycle = 0; cycle < last_time_step + 1; cycle++) @@ -819,11 +821,11 @@ void FEA_Module_Dynamic_Elasticity::force_design_gradient_term(const_vec_array d for (int inode = 0; inode < num_nodes_in_elem; inode++) { node_id = nodes_in_elem(elem_id, inode); - current_element_adjoint(inode, 0) = (current_adjoint_vector(node_id, 0) + next_adjoint_vector(node_id, 0)) / 2; - current_element_adjoint(inode, 1) = (current_adjoint_vector(node_id, 1) + next_adjoint_vector(node_id, 1)) / 2; + current_element_adjoint(inode, 0) = 0.5 * (current_adjoint_vector(node_id, 0) + next_adjoint_vector(node_id, 0)); + current_element_adjoint(inode, 1) = 0.5 * (current_adjoint_vector(node_id, 1) + next_adjoint_vector(node_id, 1)); if (num_dim == 3) { - current_element_adjoint(inode, 2) = (current_adjoint_vector(node_id, 2) + next_adjoint_vector(node_id, 2)) / 2; + current_element_adjoint(inode, 2) = 0.5 * (current_adjoint_vector(node_id, 2) + next_adjoint_vector(node_id, 2)); } } diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/geometry.cpp b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/geometry.cpp index 7e4034a8a..0f19f6580 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/geometry.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/geometry.cpp @@ -335,19 +335,12 @@ void FEA_Module_Dynamic_Elasticity::get_bmatrix2D(const ViewCArrayKokkos */ B_matrix(0, 0) = -0.5 * (y(3) - y(1)); - B_matrix(1, 0) = -0.5 * (y(0) - y(2)); - B_matrix(2, 0) = -0.5 * (y(1) - y(3)); - B_matrix(3, 0) = -0.5 * (y(2) - y(0)); - B_matrix(0, 1) = -0.5 * (x(1) - x(3)); - B_matrix(1, 1) = -0.5 * (x(2) - x(0)); - B_matrix(2, 1) = -0.5 * (x(3) - x(1)); - B_matrix(3, 1) = -0.5 * (x(0) - x(2)); // @@ -425,7 +418,7 @@ void FEA_Module_Dynamic_Elasticity::get_vol_quad(const DViewCArrayKokkos */ elem_vol(elem_gid) = ( (y(2) + y(3) + y(0)) * ((y(2) - y(3)) * (x(0) - x(3)) - (y(0) - y(3)) * (x(2) - x(3)) ) - + (y(0) + y(1) + y(2)) * ((y(0) - y(1)) * (x(2) - x(1)) - (y(2) - y(1)) * (x(0) - x(1))) ) / 6.0; + + (y(0) + y(1) + y(2)) * ((y(0) - y(1)) * (x(2) - x(1)) - (y(2) - y(1)) * (x(0) - x(1))) ) / 6.0; return; } // end subroutine diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/momentum.cpp b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/momentum.cpp index 98e83b455..be4ca4b69 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/momentum.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/momentum.cpp @@ -69,6 +69,7 @@ void FEA_Module_Dynamic_Elasticity::get_velgrad(ViewCArrayKokkos& double u_array[num_nodes_in_elem]; double v_array[num_nodes_in_elem]; double w_array[num_nodes_in_elem]; + ViewCArrayKokkos u(u_array, num_nodes_in_elem); // x-dir vel component ViewCArrayKokkos v(v_array, num_nodes_in_elem); // y-dir vel component ViewCArrayKokkos w(w_array, num_nodes_in_elem); // z-dir vel component @@ -155,6 +156,7 @@ void FEA_Module_Dynamic_Elasticity::get_velgrad2D(ViewCArrayKokkos& double u_array[num_nodes_in_elem]; double v_array[num_nodes_in_elem]; + ViewCArrayKokkos u(u_array, num_nodes_in_elem); // x-dir vel component ViewCArrayKokkos v(v_array, num_nodes_in_elem); // y-dir vel component @@ -222,6 +224,7 @@ void FEA_Module_Dynamic_Elasticity::get_divergence(DViewCArrayKokkos& double u_array[num_nodes_in_elem]; double v_array[num_nodes_in_elem]; double w_array[num_nodes_in_elem]; + ViewCArrayKokkos u(u_array, num_nodes_in_elem); // x-dir vel component ViewCArrayKokkos v(v_array, num_nodes_in_elem); // y-dir vel component ViewCArrayKokkos w(w_array, num_nodes_in_elem); // z-dir vel component @@ -295,6 +298,7 @@ void FEA_Module_Dynamic_Elasticity::get_divergence2D(DViewCArrayKokkos& double u_array[num_nodes_in_elem]; double v_array[num_nodes_in_elem]; + ViewCArrayKokkos u(u_array, num_nodes_in_elem); // x-dir vel component ViewCArrayKokkos v(v_array, num_nodes_in_elem); // y-dir vel component diff --git a/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/FEA_Module_Eulerian.cpp b/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/FEA_Module_Eulerian.cpp index d6319aa08..ee4b9d012 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/FEA_Module_Eulerian.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/FEA_Module_Eulerian.cpp @@ -207,24 +207,29 @@ FEA_Module_Eulerian::~FEA_Module_Eulerian() ------------------------------------------------------------------------- */ void FEA_Module_Eulerian::read_conditions_ansys_dat(std::ifstream* in, std::streampos before_condition_header) { - char ch; - int num_dim = simparam->num_dims; - int buffer_lines = 1000; - int max_word = 30; - int p_order = simparam->p_order; - real_t unit_scaling = simparam->get_unit_scaling(); - int local_node_index, current_column_index; - size_t strain_count; - std::string skip_line, read_line, substring, token; - std::stringstream line_parse, line_parse2; + char ch; + std::string skip_line, read_line, substring, token; + std::stringstream line_parse, line_parse2; + + int num_dim = simparam->num_dims; + int buffer_lines = 1000; + int max_word = 30; + int p_order = simparam->p_order; + int local_node_index, current_column_index; + int buffer_loop, buffer_iteration, buffer_iterations, scan_loop, nodes_per_element, words_per_line; + + real_t unit_scaling = simparam->get_unit_scaling(); + + size_t strain_count; + size_t read_index_start, node_rid, elem_gid; + CArrayKokkos read_buffer; CArrayKokkos read_buffer_indices; - int buffer_loop, buffer_iteration, buffer_iterations, scan_loop, nodes_per_element, words_per_line; - size_t read_index_start, node_rid, elem_gid; - LO local_dof_id; - GO node_gid; - real_t dof_value; - host_vec_array node_densities; + + LO local_dof_id; + GO node_gid; + real_t dof_value; + host_vec_array node_densities; } // end read_conditions_ansys_dat /* ---------------------------------------------------------------------------- @@ -902,17 +907,19 @@ void FEA_Module_Eulerian::euler_solve() { const_vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); const_vec_array ghost_node_velocities_interface = Explicit_Solver_Pointer_->ghost_node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); - vec_array all_node_velocities_interface = Explicit_Solver_Pointer_->all_node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); const_vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); const_vec_array ghost_node_coords_interface = Explicit_Solver_Pointer_->ghost_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); - vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + + vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + vec_array all_node_velocities_interface = Explicit_Solver_Pointer_->all_node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { for (int idim = 0; idim < num_dim; idim++) { all_node_velocities_interface(node_gid, idim) = node_velocities_interface(node_gid, idim); all_node_coords_interface(node_gid, idim) = node_coords_interface(node_gid, idim); } - }); // end parallel for + }); // end parallel for Kokkos::fence(); FOR_ALL_CLASS(node_gid, nlocal_nodes, nlocal_nodes + nghost_nodes, { @@ -921,7 +928,7 @@ void FEA_Module_Eulerian::euler_solve() all_node_velocities_interface(node_gid, idim) = ghost_node_velocities_interface(node_gid - nlocal_nodes, idim); all_node_coords_interface(node_gid, idim) = ghost_node_coords_interface(node_gid - nlocal_nodes, idim); } - }); // end parallel for + }); // end parallel for Kokkos::fence(); } // end view scope @@ -1026,7 +1033,7 @@ void FEA_Module_Eulerian::euler_solve() { node_vel(rk_level, node_gid, idim) = ghost_node_velocities_interface(node_gid - nlocal_nodes, idim); } - }); // end parallel for + }); // end parallel for } // end view scope Kokkos::fence(); @@ -1193,7 +1200,7 @@ void FEA_Module_Eulerian::euler_solve() node_velocities_interface(node_gid, idim) = node_vel(rk_level, node_gid, idim); node_coords_interface(node_gid, idim) = node_coords(rk_level, node_gid, idim); } - }); + }); } // end view scope Kokkos::fence(); @@ -1228,7 +1235,7 @@ void FEA_Module_Eulerian::euler_solve() all_node_velocities_interface(node_gid, idim) = node_velocities_interface(node_gid, idim); all_node_coords_interface(node_gid, idim) = node_coords_interface(node_gid, idim); } - }); // end parallel for + }); // end parallel for Kokkos::fence(); FOR_ALL_CLASS(node_gid, nlocal_nodes, nlocal_nodes + nghost_nodes, { @@ -1237,7 +1244,7 @@ void FEA_Module_Eulerian::euler_solve() all_node_velocities_interface(node_gid, idim) = ghost_node_velocities_interface(node_gid - nlocal_nodes, idim); all_node_coords_interface(node_gid, idim) = ghost_node_coords_interface(node_gid - nlocal_nodes, idim); } - }); // end parallel for + }); // end parallel for Kokkos::fence(); } // end view scope @@ -1273,12 +1280,12 @@ void FEA_Module_Eulerian::euler_solve() { KE_loc_sum += node_mass(node_gid) * ke; } - }, KE_sum); + }, KE_sum); Kokkos::fence(); KE_sum = 0.5 * KE_sum; objective_accumulation += KE_sum * dt; } - } + } // end topology optimization if check size_t write = 0; if ((cycle + 1) % graphics_cyc_ival == 0 && cycle > 0) diff --git a/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp b/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp index 37e6d91bb..de08fcb8a 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp @@ -408,25 +408,35 @@ void Explicit_Solver::run() ------------------------------------------------------------------------- */ void Explicit_Solver::read_mesh_ansys_dat(const char* MESH) { - char ch; - int num_dim = simparam.num_dims; - Input_Options input_options = simparam.input_options.value(); - int p_order = input_options.p_order; - real_t unit_scaling = input_options.unit_scaling; - bool restart_file = simparam.restart_file; - int local_node_index, current_column_index; - size_t strain_count; - std::string skip_line, read_line, substring, token; - std::stringstream line_parse; + Input_Options input_options = simparam.input_options.value(); + + bool restart_file = simparam.restart_file; + bool zero_index_base = input_options.zero_index_base; + + char ch; + std::string skip_line, read_line, substring, token; + std::stringstream line_parse; + + int num_dim = simparam.num_dims; + int p_order = input_options.p_order; + int local_node_index, current_column_index; + int buffer_loop, buffer_iteration, buffer_iterations; + int dof_limit, scan_loop, nodes_per_element; + int negative_index_found = 0; + int global_negative_index_found = 0; + + real_t unit_scaling = input_options.unit_scaling; + real_t dof_value; + + size_t read_index_start, node_rid, elem_gid; + size_t strain_count; + CArrayKokkos read_buffer; - int buffer_loop, buffer_iteration, buffer_iterations, dof_limit, scan_loop, nodes_per_element; - size_t read_index_start, node_rid, elem_gid; - GO node_gid; - real_t dof_value; - host_vec_array node_densities; - bool zero_index_base = input_options.zero_index_base; - int negative_index_found = 0; - int global_negative_index_found = 0; + + GO node_gid; + + host_vec_array node_densities; + // Nodes_Per_Element_Type = elements::elem_types::Nodes_Per_Element_Type; // read the mesh @@ -1215,26 +1225,29 @@ void Explicit_Solver::setup_optimization_problem() std::vector TO_Module_My_FEA_Module = simparam.TO_Module_My_FEA_Module; // std::vector Multi_Objective_Modules = simparam->Multi_Objective_Modules; // std::vector Multi_Objective_Weights = simparam->Multi_Objective_Weights; - std::vector> Function_Arguments = simparam.Function_Arguments; - std::vector TO_Function_Type = simparam.TO_Function_Type; + std::vector> Function_Arguments = simparam.Function_Arguments; + std::vector TO_Function_Type = simparam.TO_Function_Type; + std::vector>> Multi_Objective_Terms; - std::string constraint_base, constraint_name; - std::stringstream number_union; - CArray Surface_Nodes; - GO current_node_index, current_element_index; - LO local_node_index, local_element_id; - int num_bdy_patches_in_set; - size_t node_id, patch_id, module_id; - int num_boundary_sets; - int local_surface_id; + std::string constraint_base, constraint_name; + std::stringstream number_union; + + CArray Surface_Nodes; + GO current_node_index, current_element_index; + LO local_node_index, local_element_id; + int num_bdy_patches_in_set; + size_t node_id, patch_id, module_id; + int num_boundary_sets; + int local_surface_id; + const_host_vec_array design_densities; typedef ROL::TpetraMultiVector ROL_MV; const_host_elem_conn_array nodes_in_elem = global_nodes_in_elem_distributed->getLocalView(Tpetra::Access::ReadOnly); // fill parameter list with desired algorithmic options or leave as default // Read optimization input parameter list. - std::string filename = "optimization_parameters.xml"; + std::string filename = "/var/tmp/repos/Fierro/Fierro/src/Parallel-Solvers/Parallel-Explicit/optimization_parameters.xml"; auto parlist = ROL::getParametersFromXmlFile(filename); // ROL::ParameterList parlist; @@ -2033,19 +2046,22 @@ void Explicit_Solver::comm_densities() void Explicit_Solver::parallel_tecplot_writer() { - int num_dim = simparam.num_dims; - std::string current_file_name; - std::string base_file_name = "TecplotTO"; - std::string base_file_name_undeformed = "TecplotTO_undeformed"; - std::stringstream current_line_stream; - std::string current_line; - std::string file_extension = ".dat"; - std::string file_count; + int num_dim = simparam.num_dims; + int time_step = 0; + int temp_convert; + int noutput, nvector; + + std::string current_file_name; + std::string base_file_name = "TecplotTO"; + std::string base_file_name_undeformed = "TecplotTO_undeformed"; + std::string current_line; + std::string file_extension = ".dat"; + std::string file_count; + std::stringstream count_temp; - int time_step = 0; - int temp_convert; - int noutput, nvector; - bool displace_geometry = false; + std::stringstream current_line_stream; + + bool displace_geometry = false; // Convert ijk index system to the finite element numbering convention // for vertices in cell @@ -2299,27 +2315,31 @@ void Explicit_Solver::parallel_tecplot_writer() void Explicit_Solver::parallel_vtk_writer() { - int num_dim = simparam.num_dims; - std::string current_file_name; - std::string base_file_name = "VTK"; - std::string base_file_name_undeformed = "VTK_undeformed"; - std::stringstream current_line_stream; - std::string current_line; - std::string file_extension = ".vtk"; - std::string file_count; + int num_dim = simparam.num_dims; + int time_step = 0; + int temp_convert; + int noutput, nvector; + int buffer_size_per_element_line, nlocal_elements; + int default_vector_count; + int buffer_size_per_node_line; + int nlocal_sorted_nodes; + bool displace_geometry = false; + + std::string current_file_name; + std::string base_file_name = "VTK"; + std::string base_file_name_undeformed = "VTK_undeformed"; + std::string current_line; + std::string file_extension = ".vtk"; + std::string file_count; + std::stringstream count_temp; - int time_step = 0; - int temp_convert; - int noutput, nvector; - bool displace_geometry = false; - MPI_Offset current_stream_position, header_stream_offset, file_stream_offset; - MPI_File myfile_parallel; - int buffer_size_per_element_line, nlocal_elements; - GO first_element_global_id; - int default_vector_count; - int buffer_size_per_node_line; - int nlocal_sorted_nodes; - GO first_node_global_id; + std::stringstream current_line_stream; + + MPI_Offset current_stream_position, header_stream_offset, file_stream_offset; + MPI_File myfile_parallel; + + GO first_element_global_id; + GO first_node_global_id; // Convert ijk index system to the finite element numbering convention // for vertices in cell @@ -2343,6 +2363,7 @@ void Explicit_Solver::parallel_vtk_writer() convert_ijk_to_ensight(1) = 1; convert_ijk_to_ensight(2) = 3; convert_ijk_to_ensight(3) = 2; + if (num_dim == 3) { convert_ijk_to_ensight(4) = 4; diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/mesh.h b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/mesh.h index d33862889..40e9417fc 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/mesh.h +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/mesh.h @@ -67,17 +67,13 @@ void bubble_sort(size_t arr[], const size_t num) struct mesh_t { size_t num_dims; - - size_t num_nodes, num_local_nodes; - + size_t num_nodes; + size_t num_local_nodes; size_t num_elems; size_t num_nodes_in_elem; size_t num_patches_in_elem; - size_t num_corners; - size_t num_patches; - size_t num_bdy_patches; size_t num_bdy_nodes; size_t num_bdy_sets; diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/FEA_Module_SGH.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/FEA_Module_SGH.cpp index ac3a309ed..d06ceae29 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/FEA_Module_SGH.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/FEA_Module_SGH.cpp @@ -77,6 +77,8 @@ #define BC_EPSILON 1.0e-6 #define BUFFER_GROW 100 +// #define DEBUG + using namespace utils; FEA_Module_SGH::FEA_Module_SGH( @@ -500,28 +502,35 @@ void FEA_Module_SGH::compute_output() void FEA_Module_SGH::comm_node_masses() { // debug print of design vector - // std::ostream &out = std::cout; - // Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - // if(myrank==0) - // *fos << "Density data :" << std::endl; - // node_densities_distributed->describe(*fos,Teuchos::VERB_EXTREME); - // *fos << std::endl; - // std::fflush(stdout); - - // communicate design densities - // create import object using local node indices map and all indices map - // Tpetra::Import importer(map, ghost_node_map); +#ifdef DEBUG + std::ostream& out = std::cout; + Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + if (myrank == 0) + { + *fos << "Density data :" << std::endl; + } + node_densities_distributed->describe(*fos, Teuchos::VERB_EXTREME); + *fos << std::endl; + std::fflush(stdout); + communicate design densities + create import object using local node indices map and all indices map + Tpetra::Import importer(map, ghost_node_map); +#endif // comms to get ghosts ghost_node_masses_distributed->doImport(*node_masses_distributed, *ghost_importer, Tpetra::INSERT); - // all_node_map->describe(*fos,Teuchos::VERB_EXTREME); - // all_node_velocities_distributed->describe(*fos,Teuchos::VERB_EXTREME); - - // update_count++; - // if(update_count==1){ - // MPI_Barrier(world); - // MPI_Abort(world,4); - // } + +#ifdef DEBUG + all_node_map->describe(*fos, Teuchos::VERB_EXTREME); + all_node_velocities_distributed->describe(*fos, Teuchos::VERB_EXTREME); + + update_count++; + if (update_count == 1) + { + MPI_Barrier(world); + MPI_Abort(world, 4); + } +#endif } /* ------------------------------------------------------------------------------------------- @@ -534,20 +543,22 @@ void FEA_Module_SGH::comm_variables(Teuchos::RCP zp) { // set density vector to the current value chosen by the optimizer test_node_densities_distributed = zp; - +#ifdef DEBUG // debug print of design vector - // std::ostream &out = std::cout; - // Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - // if(myrank==0) - // *fos << "Density data :" << std::endl; - // node_densities_distributed->describe(*fos,Teuchos::VERB_EXTREME); - // *fos << std::endl; - // std::fflush(stdout); + std::ostream& out = std::cout; + Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + if (myrank == 0) + { + *fos << "Density data :" << std::endl; + } + node_densities_distributed->describe(*fos, Teuchos::VERB_EXTREME); + *fos << std::endl; + std::fflush(stdout); // communicate design densities // create import object using local node indices map and all indices map - // Tpetra::Import importer(map, all_node_map); - + Tpetra::Import importer(map, all_node_map); +#endif // comms to get ghosts all_node_densities_distributed->doImport(*test_node_densities_distributed, *importer, Tpetra::INSERT); } @@ -570,8 +581,9 @@ void FEA_Module_SGH::node_density_constraints(host_vec_array node_densities_lowe const DCArrayKokkos loading = module_params->loading; // debug check - // std::cout << "NUMBER OF LOADING CONDITIONS: " << num_lcs << std::endl; - +#ifdef DEBUG + std::cout << "NUMBER OF LOADING CONDITIONS: " << num_lcs << std::endl; +#endif // walk over the nodes to update the velocity FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { double current_node_coords[3]; @@ -586,8 +598,9 @@ void FEA_Module_SGH::node_density_constraints(host_vec_array node_densities_lowe for (size_t ilc = 0; ilc < num_lcs; ilc++) { // debug check - // std::cout << "LOADING CONDITION VOLUME TYPE: " << to_string(loading(ilc).volume) << std::endl; - +#ifdef DEBUG + std::cout << "LOADING CONDITION VOLUME TYPE: " << to_string(loading(ilc).volume) << std::endl; +#endif bool fill_this = loading(ilc).volume.contains(current_node_coords); if (fill_this) { @@ -647,17 +660,19 @@ void FEA_Module_SGH::tag_bdys(const DCArrayKokkos& boundary, size_t num_dim = simparam->num_dims; int nboundary_patches = Explicit_Solver_Pointer_->nboundary_patches; int num_nodes_in_patch = mesh.num_nodes_in_patch; - - // if (bdy_set == mesh.num_bdy_sets){ - // printf(" ERROR: number of boundary sets must be increased by %zu", - // bdy_set-mesh.num_bdy_sets+1); - // exit(0); - // } // end if +#ifdef DEBUG + if (bdy_set == mesh.num_bdy_sets) + { + printf(" ERROR: number of boundary sets must be increased by %zu", + bdy_set - mesh.num_bdy_sets + 1); + exit(0); + } // end if // error and debug flag - // DCArrayKokkos print_flag(1, "print_flag"); - // print_flag.host(0) = false; - // print_flag.update_device(); + DCArrayKokkos print_flag(1, "print_flag"); + print_flag.host(0) = false; + print_flag.update_device(); +#endif FOR_ALL_CLASS(bdy_set, 0, num_bdy_sets, { // tag boundaries @@ -680,12 +695,16 @@ void FEA_Module_SGH::tag_bdys(const DCArrayKokkos& boundary, rk_level); // no=0, yes=1 // debug check - /* - for (size_t patch_node_lid=0; patch_node_lid& boundary, } // end for bdy_patch }); // end FOR_ALL_CLASS bdy_sets - // debug check - // print_flag.update_host(); - // if(print_flag.host(0)) std::cout << "found boundary node with id 549412" << std::endl; +#ifdef DEBUG + print_flag.update_host(); + if (print_flag.host(0)) + { + std::cout << "found boundary node with id 549412" << std::endl; + } +#endif return; } // end tag @@ -800,31 +823,38 @@ void FEA_Module_SGH::sgh_solve() { Dynamic_Options dynamic_options = simparam->dynamic_options; + const int num_dim = simparam->num_dims; const size_t rk_level = dynamic_options.rk_num_bins - 1; - time_value = dynamic_options.time_initial; - time_final = dynamic_options.time_final; - dt_max = dynamic_options.dt_max; - dt_min = dynamic_options.dt_min; - dt_cfl = dynamic_options.dt_cfl; + + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos material = simparam->material; + + time_value = dynamic_options.time_initial; + time_final = dynamic_options.time_final; + dt_max = dynamic_options.dt_max; + dt_min = dynamic_options.dt_min; + dt = dynamic_options.dt; + dt_cfl = dynamic_options.dt_cfl; + graphics_time = simparam->output_options.graphics_step; graphics_dt_ival = simparam->output_options.graphics_step; cycle_stop = dynamic_options.cycle_stop; rk_num_stages = dynamic_options.rk_num_stages; - dt = dynamic_options.dt; + graphics_times = simparam->output_options.graphics_times; + graphics_id = simparam->output_options.graphics_id; + fuzz = dynamic_options.fuzz; tiny = dynamic_options.tiny; small = dynamic_options.small; - graphics_times = simparam->output_options.graphics_times; - graphics_id = simparam->output_options.graphics_id; - size_t num_bdy_nodes = mesh->num_bdy_nodes; - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos material = simparam->material; - int nTO_modules; - int old_max_forward_buffer; - size_t cycle; - const int num_dim = simparam->num_dims; - real_t objective_accumulation, global_objective_accumulation; - std::vector> FEA_Module_My_TO_Modules = simparam->FEA_Module_My_TO_Modules; + + size_t num_bdy_nodes = mesh->num_bdy_nodes; + size_t cycle; + real_t objective_accumulation, global_objective_accumulation; + + int nTO_modules; + int old_max_forward_buffer; + + std::vector> FEA_Module_My_TO_Modules = simparam->FEA_Module_My_TO_Modules; problem = Explicit_Solver_Pointer_->problem; // Pointer to ROL optimization problem object ROL::Ptr> obj_pointer; @@ -983,7 +1013,7 @@ void FEA_Module_SGH::sgh_solve() node_velocities_interface(node_gid, idim) = node_vel(rk_level, node_gid, idim); node_coords_interface(node_gid, idim) = node_coords(rk_level, node_gid, idim); } - }); + }); } // end view scope Kokkos::fence(); @@ -1019,7 +1049,7 @@ void FEA_Module_SGH::sgh_solve() all_node_velocities_interface(node_gid, idim) = node_velocities_interface(node_gid, idim); all_node_coords_interface(node_gid, idim) = node_coords_interface(node_gid, idim); } - }); // end parallel for + }); // end parallel for Kokkos::fence(); FOR_ALL_CLASS(node_gid, nlocal_nodes, nlocal_nodes + nghost_nodes, { @@ -1028,13 +1058,13 @@ void FEA_Module_SGH::sgh_solve() all_node_velocities_interface(node_gid, idim) = ghost_node_velocities_interface(node_gid - nlocal_nodes, idim); all_node_coords_interface(node_gid, idim) = ghost_node_coords_interface(node_gid - nlocal_nodes, idim); } - }); // end parallel for + }); // end parallel for Kokkos::fence(); // interface for element internal energies FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { element_internal_energy(elem_gid, 0) = elem_sie(rk_level, elem_gid); - }); // end parallel for + }); // end parallel for Kokkos::fence(); } // end view scope @@ -1162,40 +1192,45 @@ void FEA_Module_SGH::sgh_solve() cycle); } - /* - debug block - if(myrank==1){ - std::cout << rk_alpha << " " << dt << std::endl; - for(int i = 0; i < nall_nodes; i++){ - double node_force[3]; - for (size_t dim = 0; dim < num_dim; dim++){ - node_force[dim] = 0.0; - } // end for dim - - // loop over all corners around the node and calculate the nodal force - for (size_t corner_lid=0; corner_lidall_node_map->getGlobalElement(i) << " " << corner_gid << " " << corner_force(corner_gid, 0) << " " << corner_force(corner_gid, 1) << " " << corner_force(corner_gid, 2) << std::endl; - // loop over dimension - for (size_t dim = 0; dim < num_dim; dim++){ - node_force[dim] += corner_force(corner_gid, dim); - } // end for dim - - } // end for corner_lid - //std::cout << Explicit_Solver_Pointer_->all_node_map->getGlobalElement(i) << " " << node_force[0] << " " << node_force[1] << " " << node_force[2] << std::endl; - //std::cout << Explicit_Solver_Pointer_->all_node_map->getGlobalElement(i) << " " << node_mass(i) << std::endl; - } +#ifdef DEBUG + if (myrank == 1) + { + std::cout << "rk_alpha = " << rk_alpha << ", dt = " << dt << std::endl; + for (int i = 0; i < nall_nodes; i++) + { + double node_force[3]; + for (size_t dim = 0; dim < num_dim; dim++) + { + node_force[dim] = 0.0; + } // end for dim + + // loop over all corners around the node and calculate the nodal force + for (size_t corner_lid = 0; corner_lid < mesh.num_corners_in_node(i); corner_lid++) + { + // Get corner gid + size_t corner_gid = mesh.corners_in_node(i, corner_lid); + std::cout << Explicit_Solver_Pointer_->all_node_map->getGlobalElement(i) << " " << corner_gid << " " << corner_force(corner_gid, 0) << " " << corner_force(corner_gid, + 1) << " " << corner_force(corner_gid, 2) << std::endl; + // loop over dimension + for (size_t dim = 0; dim < num_dim; dim++) + { + node_force[dim] += corner_force(corner_gid, dim); + } // end for dim + } // end for corner_lid + } } - /* - //debug print vector values on a rank - /* - if(myrank==0) - for(int i = 0; i < nall_nodes; i++){ - std::cout << Explicit_Solver_Pointer_->all_node_map->getGlobalElement(i) << " " << node_vel(rk_level,i,0) << " " << node_vel(rk_level,i,1) << " " << node_vel(rk_level,i,2) << std::endl; - } - */ + + // debug print vector values on a rank + + if (myrank == 0) + { + for (int i = 0; i < nall_nodes; i++) + { + std::cout << Explicit_Solver_Pointer_->all_node_map->getGlobalElement(i) << " " << node_vel(rk_level, i, 0) << " " << node_vel(rk_level, i, 1) << " " << node_vel(rk_level, i, + 2) << std::endl; + } + } +#endif // ---- Update nodal velocities ---- // update_velocity_sgh(rk_alpha, @@ -1263,13 +1298,18 @@ void FEA_Module_SGH::sgh_solve() double comm_time4 = Explicit_Solver_Pointer_->CPU_Time(); Explicit_Solver_Pointer_->host2dev_time += comm_time4 - comm_time3; Explicit_Solver_Pointer_->communication_time += comm_time4 - comm_time1; + +#ifdef DEBUG // debug print vector values on a rank - /* - if(myrank==0) - for(int i = 0; i < nall_nodes; i++){ - std::cout << Explicit_Solver_Pointer_->all_node_map->getGlobalElement(i) << " " << node_vel(rk_level,i,0) << " " << node_vel(rk_level,i,1) << " " << node_vel(rk_level,i,2) << std::endl; - } - */ + if (myrank == 0) + { + for (int i = 0; i < nall_nodes; i++) + { + std::cout << Explicit_Solver_Pointer_->all_node_map->getGlobalElement(i) << " " << node_vel(rk_level, i, 0) << " " << node_vel(rk_level, i, 1) << " " << node_vel(rk_level, i, + 2) << std::endl; + } + } +#endif // ---- Update specific internal energy in the elements ---- update_energy_sgh(rk_alpha, *mesh, @@ -1572,7 +1612,7 @@ void FEA_Module_SGH::sgh_solve() { KE_loc_sum += node_mass(node_gid) * ke; } - }, KE_sum); + }, KE_sum); Kokkos::fence(); KE_sum = 0.5 * KE_sum; objective_accumulation += KE_sum * dt; diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp index 9ef23e8c1..2d5310509 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp @@ -280,7 +280,7 @@ void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& ma mu_term = muc(node_lid) * fabs(shock_dir(0) * area_normal(node_lid, 0) + shock_dir(1) * area_normal(node_lid, 1) - + shock_dir(2) * area_normal(node_lid, 2) ); // code gradient for shock dir w.r.t velocity if using shock_dir + + shock_dir(2) * area_normal(node_lid, 2) ); // code gradient for shock dir w.r.t velocity if using shock_dir } else { @@ -994,14 +994,19 @@ void FEA_Module_SGH::get_force_ugradient_sgh(const DCArrayKokkos& ma // gradients of the element volume get_vol_hex_ugradient(volume_gradients, elem_gid, node_coords, elem_node_gids, rk_level); - // //debug - // for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - // for (size_t dim = 0; dim < num_dims; dim++){ - // if(volume_gradients(node_lid, dim)>1 || volume_gradients(node_lid, dim) < -1) - // std::cout << volume_gradients(node_lid, dim) << " "; - // } // end for - // } // end for - // std::cout << std::endl; +#ifdef DEBUG + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + for (size_t dim = 0; dim < num_dims; dim++) + { + if (volume_gradients(node_lid, dim) > 1 || volume_gradients(node_lid, dim) < -1) + { + std::cout << volume_gradients(node_lid, dim) << " "; + } + } // end for + } // end for + std::cout << std::endl; +#endif // get the B matrix which are the OUTWARD corner area normals get_bmatrix(area_normal, @@ -1483,16 +1488,21 @@ void FEA_Module_SGH::get_force_ugradient_sgh(const DCArrayKokkos& ma void FEA_Module_SGH::force_design_gradient_term(const_vec_array design_variables, vec_array design_gradients) { - size_t num_bdy_nodes = mesh->num_bdy_nodes; - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos material = simparam->material; - const int num_dim = simparam->num_dims; - size_t num_corners = rnum_elem * num_nodes_in_elem; - real_t global_dt; - bool element_constant_density = true; - size_t current_data_index, next_data_index; - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - CArrayKokkos current_element_adjoint = CArrayKokkos(num_nodes_in_elem, num_dim); + bool element_constant_density = true; + + size_t num_bdy_nodes = mesh->num_bdy_nodes; + size_t num_corners = rnum_elem * num_nodes_in_elem; + size_t current_data_index, next_data_index; + + real_t global_dt; + + const int num_dim = simparam->num_dims; + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos material = simparam->material; + + auto current_element_adjoint = CArrayKokkos(num_nodes_in_elem, num_dim); // gradient contribution from gradient of Force vector with respect to design variable. if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) @@ -1545,12 +1555,12 @@ void FEA_Module_SGH::force_design_gradient_term(const_vec_array design_variables node_vel(rk_level, node_gid, idim) = current_velocity_vector(node_gid, idim); node_coords(rk_level, node_gid, idim) = current_coord_vector(node_gid, idim); } - }); // end parallel for + }); // end parallel for Kokkos::fence(); FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { elem_sie(rk_level, elem_gid) = current_element_internal_energy(elem_gid, 0); - }); // end parallel for + }); // end parallel for Kokkos::fence(); get_vol(); @@ -1645,7 +1655,7 @@ void FEA_Module_SGH::force_design_gradient_term(const_vec_array design_variables corner_id = elem_id * num_nodes_in_elem + inode; corner_value_storage(corner_id) = inner_product; } - }); // end parallel for + }); // end parallel for Kokkos::fence(); // accumulate node values from corner storage @@ -1657,7 +1667,7 @@ void FEA_Module_SGH::force_design_gradient_term(const_vec_array design_variables corner_id = corners_in_node(node_id, icorner); design_gradients(node_id, 0) += -corner_value_storage(corner_id) * global_dt; } - }); // end parallel for + }); // end parallel for Kokkos::fence(); } // end view scope } diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_sgh.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_sgh.cpp index e4046cddf..1a338a1e0 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_sgh.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_sgh.cpp @@ -861,14 +861,15 @@ void FEA_Module_SGH::applied_forces(const DCArrayKokkos& material, const size_t cycle ) { - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - const size_t num_dim = mesh.num_dims; - const_vec_array all_initial_node_coords = all_initial_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); - const size_t num_lcs = module_params->loading.size(); + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + const size_t num_dim = mesh.num_dims; + const size_t num_lcs = module_params->loading.size(); const DCArrayKokkos mat_fill = simparam->mat_fill; const DCArrayKokkos loading = module_params->loading; + const_vec_array all_initial_node_coords = all_initial_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + // debug check // std::cout << "NUMBER OF LOADING CONDITIONS: " << num_lcs << std::endl; diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/geometry.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/geometry.cpp index b0501bf19..726c54a5b 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/geometry.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/geometry.cpp @@ -335,19 +335,12 @@ void FEA_Module_SGH::get_bmatrix2D(const ViewCArrayKokkos& B_matrix, */ B_matrix(0, 0) = -0.5 * (y(3) - y(1)); - B_matrix(1, 0) = -0.5 * (y(0) - y(2)); - B_matrix(2, 0) = -0.5 * (y(1) - y(3)); - B_matrix(3, 0) = -0.5 * (y(2) - y(0)); - B_matrix(0, 1) = -0.5 * (x(1) - x(3)); - B_matrix(1, 1) = -0.5 * (x(2) - x(0)); - B_matrix(2, 1) = -0.5 * (x(3) - x(1)); - B_matrix(3, 1) = -0.5 * (x(0) - x(2)); // diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/momentum.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/momentum.cpp index 03452ccd1..7adbfb401 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/momentum.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/momentum.cpp @@ -61,13 +61,13 @@ void FEA_Module_SGH::get_velgrad(ViewCArrayKokkos& vel_grad, { const size_t num_nodes_in_elem = 8; - double u_array[num_nodes_in_elem]; // x-dir vel component - double v_array[num_nodes_in_elem]; // y-dir vel component - double w_array[num_nodes_in_elem]; // z-dir vel component - - ViewCArrayKokkos u(u_array, num_nodes_in_elem); - ViewCArrayKokkos v(v_array, num_nodes_in_elem); - ViewCArrayKokkos w(w_array, num_nodes_in_elem); + double u_array[num_nodes_in_elem]; // x-dir vel component + double v_array[num_nodes_in_elem]; // y-dir vel component + double w_array[num_nodes_in_elem]; // z-dir vel component + + ViewCArrayKokkos u(u_array, num_nodes_in_elem); + ViewCArrayKokkos v(v_array, num_nodes_in_elem); + ViewCArrayKokkos w(w_array, num_nodes_in_elem); // get the vertex velocities for the cell for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) @@ -149,8 +149,9 @@ void FEA_Module_SGH::get_velgrad2D(ViewCArrayKokkos& vel_grad, { const size_t num_nodes_in_elem = 4; - double u_array[num_nodes_in_elem]; - double v_array[num_nodes_in_elem]; + double u_array[num_nodes_in_elem]; + double v_array[num_nodes_in_elem]; + ViewCArrayKokkos u(u_array, num_nodes_in_elem); // x-dir vel component ViewCArrayKokkos v(v_array, num_nodes_in_elem); // y-dir vel component @@ -217,6 +218,7 @@ void FEA_Module_SGH::get_divergence(DViewCArrayKokkos& elem_div, double u_array[num_nodes_in_elem]; double v_array[num_nodes_in_elem]; double w_array[num_nodes_in_elem]; + ViewCArrayKokkos u(u_array, num_nodes_in_elem); // x-dir vel component ViewCArrayKokkos v(v_array, num_nodes_in_elem); // y-dir vel component ViewCArrayKokkos w(w_array, num_nodes_in_elem); // z-dir vel component diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/setup_sgh.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/setup_sgh.cpp index dd85ffe5e..a58fac95c 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/setup_sgh.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/setup_sgh.cpp @@ -8,6 +8,8 @@ #include "Simulation_Parameters/FEA_Module/SGH_Parameters.h" #include "Explicit_Solver.h" +// #define DEBUG + void FEA_Module_SGH::setup() { const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; @@ -28,40 +30,36 @@ void FEA_Module_SGH::setup() sgh_interface_setup(node_interface, elem_interface, corner_interface); mesh->build_corner_connectivity(); // debug print of corner ids - /* - if(myrank==1){ - for(int i = 0; i < mesh.num_nodes; i++){ - +#ifdef DEBUG + if (myrank == 1) + { + for (int i = 0; i < mesh.num_nodes; i++) + { // loop over all corners around the node and calculate the nodal force - for (size_t corner_lid=0; corner_lidgetGlobalElement(i) << " " << i << " " << all_node_map->getLocalElement(all_node_map->getGlobalElement(i)) << " " << corner_gid << " " << std::endl; - + for (size_t corner_lid = 0; corner_lid < mesh.num_corners_in_node(i); corner_lid++) + { + // Get corner gid + size_t corner_gid = mesh.corners_in_node(i, corner_lid); + std::cout << map->getGlobalElement(i) << " " << i << " " << all_node_map->getLocalElement(all_node_map->getGlobalElement(i)) << " " << corner_gid << " " << std::endl; } // end for corner_lid - //std::cout << explicit_solver_pointer->all_node_map->getGlobalElement(i) << " " << node_force[0] << " " << node_force[1] << " " << node_force[2] << std::endl; - //std::cout << explicit_solver_pointer->all_node_map->getGlobalElement(i) << " " << node_mass(i) << std::endl; - } } - */ - /* - if(myrank==1){ - for(int i = 0; i < mesh.num_elems; i++){ + } + if (myrank == 1) + { + for (int i = 0; i < mesh.num_elems; i++) + { // loop over all corners around the node and calculate the nodal force - for (size_t corner_lid=0; corner_lidgetGlobalElement(mesh.nodes_in_elem(i, corner_lid)) <<" " << corner_gid << " " << std::endl; - + for (size_t corner_lid = 0; corner_lid < max_nodes_per_element; corner_lid++) + { + // Get corner gid + size_t corner_gid = mesh.corners_in_elem(i, corner_lid); + std::cout << i << " " << mesh.nodes_in_elem(i, corner_lid) << " " << all_node_map->getGlobalElement(mesh.nodes_in_elem(i, corner_lid)) << " " << corner_gid << " " << std::endl; } // end for corner_lid - //std::cout << explicit_solver_pointer->all_node_map->getGlobalElement(i) << " " << node_force[0] << " " << node_force[1] << " " << node_force[2] << std::endl; - //std::cout << explicit_solver_pointer->all_node_map->getGlobalElement(i) << " " << node_mass(i) << std::endl; - } } - */ + } +#endif + mesh->build_elem_elem_connectivity(); mesh->num_bdy_patches = nboundary_patches; if (num_dim == 2) @@ -517,7 +515,7 @@ void FEA_Module_SGH::setup() FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { node_mass_interface(node_gid, 0) = node_mass(node_gid); - }); // end parallel for + }); // end parallel for } // end view scope Kokkos::fence(); // communicate ghost densities @@ -530,7 +528,7 @@ void FEA_Module_SGH::setup() FOR_ALL_CLASS(node_gid, nlocal_nodes, nall_nodes, { node_mass(node_gid) = ghost_node_mass_interface(node_gid - nlocal_nodes, 0); - }); // end parallel for + }); // end parallel for } // end view scope Kokkos::fence(); } // endif @@ -635,76 +633,78 @@ void FEA_Module_SGH::sgh_interface_setup(node_t& node, for (int inode = 0; inode < num_nodes_in_elem; inode++) { nodes_in_elem.host(ielem, inode) = Explicit_Solver_Pointer_->all_node_map->getLocalElement(interface_nodes_in_elem(ielem, inode)); - // debug print - // std::cout << nodes_in_elem.get_kokkos_dual_view().h_view(ielem*num_nodes_in_elem + inode)+1<< " "; } - // std::cout << std::endl; } } // update device side nodes_in_elem.update_device(); // debug print +#ifdef DEBUG + CArrayKokkos device_mesh_nodes_in_elem(rnum_elem, num_nodes_in_elem); + device_mesh_nodes_in_elem.get_kokkos_view() = nodes_in_elem.get_kokkos_dual_view().d_view; + host_mesh_nodes_in_elem.get_kokkos_view() = nodes_in_elem.get_kokkos_dual_view().view_host(); - // CArrayKokkos device_mesh_nodes_in_elem(rnum_elem, num_nodes_in_elem); - // device_mesh_nodes_in_elem.get_kokkos_view() = nodes_in_elem.get_kokkos_dual_view().d_view; - // host_mesh_nodes_in_elem.get_kokkos_view() = nodes_in_elem.get_kokkos_dual_view().view_host(); - /* - if(myrank==1){ - std::cout << "ELEMENT CONNECTIVITY ON RANK 1 in LOCAL INDICES" << myrank << std::endl; - for(int ielem = 0; ielem < rnum_elem; ielem++){ - std::cout << "Element index " << ielem+1 << " "; - for(int inode = 0; inode < num_nodes_in_elem; inode++){ - //debug print - //device_mesh_nodes_in_elem(ielem,inode) = Explicit_Solver_Pointer_->all_node_map->getLocalElement(interface_nodes_in_elem(ielem,inode)); - std::cout << nodes_in_elem(ielem, inode)+1<< " "; + if (myrank == 1) + { + std::cout << "ELEMENT CONNECTIVITY ON RANK 1 in LOCAL INDICES" << myrank << std::endl; + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + std::cout << "Element index " << ielem + 1 << " "; + for (int inode = 0; inode < num_nodes_in_elem; inode++) + { + // debug print + device_mesh_nodes_in_elem(ielem, inode) = Explicit_Solver_Pointer_->all_node_map->getLocalElement(interface_nodes_in_elem(ielem, inode)); + std::cout << nodes_in_elem(ielem, inode) + 1 << " "; + } + std::cout << std::endl; } - std::cout << std::endl; - } } - */ - /* + std::cout.flush(); - if(myrank==1){ - std::cout << "ELEMENT CONNECTIVITY ON RANK 1 in GLOBAL INDICES" << myrank << std::endl; - std::cout << "local node index of global index 275 on rank 1 " << Explicit_Solver_Pointer_->all_node_map->getLocalElement(275) << std::endl; - for(int ielem = 0; ielem < rnum_elem; ielem++){ - std::cout << ielem << " "; - for(int inode = 0; inode < num_nodes_in_elem; inode++){ - //debug print - //device_mesh_nodes_in_elem(ielem,inode) = Explicit_Solver_Pointer_->all_node_map->getLocalElement(interface_nodes_in_elem(ielem,inode)); - std::cout << Explicit_Solver_Pointer_->all_node_map->getGlobalElement(nodes_in_elem(ielem, inode))<< " "; + if (myrank == 1) + { + std::cout << "ELEMENT CONNECTIVITY ON RANK 1 in GLOBAL INDICES" << myrank << std::endl; + std::cout << "local node index of global index 275 on rank 1 " << Explicit_Solver_Pointer_->all_node_map->getLocalElement(275) << std::endl; + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + std::cout << ielem << " "; + for (int inode = 0; inode < num_nodes_in_elem; inode++) + { + // debug print + device_mesh_nodes_in_elem(ielem, inode) = Explicit_Solver_Pointer_->all_node_map->getLocalElement(interface_nodes_in_elem(ielem, inode)); + std::cout << Explicit_Solver_Pointer_->all_node_map->getGlobalElement(nodes_in_elem(ielem, inode)) << " "; + } + std::cout << std::endl; } - std::cout << std::endl; - } } std::cout.flush(); - */ - /* + size_t nall_nodes = Explicit_Solver_Pointer_->nall_nodes; - node.all_coords = DCArrayKokkos (rk_num_bins, nall_nodes, num_dim); - node.all_vel = DCArrayKokkos (rk_num_bins, nall_nodes, num_dim); - node.all_mass = DCArrayKokkos (nall_nodes); + node.all_coords = DCArrayKokkos(rk_num_bins, nall_nodes, num_dim); + node.all_vel = DCArrayKokkos(rk_num_bins, nall_nodes, num_dim); + node.all_mass = DCArrayKokkos(nall_nodes); - //save all data (nlocal +nghost) + // save all data (nlocal +nghost) CArrayKokkos host_all_node_coords_state(rk_num_bins, nall_nodes, num_dim); - host_vec_array interface_all_node_coords = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); + host_vec_array interface_all_node_coords = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); host_all_node_coords_state.get_kokkos_view() = node.all_coords.get_kokkos_dual_view().view_host(); - //host_node_coords_state = CArrayKokkos(rk_num_bins, nall_nodes, num_dim); - //host_all_node_coords_state.get_kokkos_view() = Kokkos::View("debug", rk_num_bins*nall_nodes*num_dim); - //save node data to node.coords + host_node_coords_state = CArrayKokkos(rk_num_bins, nall_nodes, num_dim); + host_all_node_coords_state.get_kokkos_view() = Kokkos::View("debug", rk_num_bins * nall_nodes * num_dim); + // save node data to node.coords - //std::cout << "ALL NODE DATA ON RANK " << myrank << std::endl; - for(int inode = 0; inode < nall_nodes; inode++){ - //std::cout << "Node index " << inode+1 << " "; - node.all_coords.host(0,inode,0) = interface_all_node_coords(inode,0); - //std::cout << host_all_node_coords_state(0,inode,0)+1<< " "; - node.all_coords.host(0,inode,1) = interface_all_node_coords(inode,1); - //std::cout << host_all_node_coords_state(0,inode,1)+1<< " "; - node.all_coords.host(0,inode,2) = interface_all_node_coords(inode,2); - //std::cout << host_all_node_coords_state(0,inode,2)+1<< std::endl; + std::cout << "ALL NODE DATA ON RANK " << myrank << std::endl; + for (int inode = 0; inode < nall_nodes; inode++) + { + // std::cout << "Node index " << inode+1 << " "; + node.all_coords.host(0, inode, 0) = interface_all_node_coords(inode, 0); + // std::cout << host_all_node_coords_state(0,inode,0)+1<< " "; + node.all_coords.host(0, inode, 1) = interface_all_node_coords(inode, 1); + // std::cout << host_all_node_coords_state(0,inode,1)+1<< " "; + node.all_coords.host(0, inode, 2) = interface_all_node_coords(inode, 2); + // std::cout << host_all_node_coords_state(0,inode,2)+1<< std::endl; } - */ +#endif // save the node coords to the current RK value for (size_t node_gid = 0; node_gid < nall_nodes; node_gid++) @@ -932,7 +932,9 @@ void FEA_Module_SGH::grow_boundary_sets(int num_sets) Boundary_Condition_Patches = CArrayKokkos(max_boundary_sets, nboundary_patches, "Boundary_Condition_Patches"); // copy previous data back over - // std::cout << "NUM BOUNDARY CONDITIONS ON RANK " << myrank << " FOR COPY " << max_boundary_sets < zp) int max_stride = 0; int current_module_index; size_t access_index, row_access_index, row_counter; - GO global_index, global_dof_index; - LO local_dof_index; + + GO global_index, global_dof_index; + LO local_dof_index; const size_t num_fills = simparam->regions.size(); const size_t rk_num_bins = simparam->dynamic_options.rk_num_bins; @@ -68,7 +69,8 @@ void FEA_Module_SGH::update_forward_solve(Teuchos::RCP zp) const DCArrayKokkos mat_fill = simparam->mat_fill; const DCArrayKokkos boundary = module_params->boundary; const DCArrayKokkos material = simparam->material; - CArray current_element_nodal_densities = CArray(num_nodes_in_elem); + + CArray current_element_nodal_densities = CArray(num_nodes_in_elem); std::vector> FEA_Module_My_TO_Modules = simparam->FEA_Module_My_TO_Modules; problem = Explicit_Solver_Pointer_->problem; // Pointer to ROL optimization problem object @@ -88,7 +90,8 @@ void FEA_Module_SGH::update_forward_solve(Teuchos::RCP zp) relative_element_densities.host(elem_id) = average_element_density(num_nodes_in_elem, current_element_nodal_densities); } // for } // view scope - // debug print + // debug print + // std::cout << "ELEMENT RELATIVE DENSITY TEST " << relative_element_densities.host(0) << std::endl; relative_element_densities.update_device(); @@ -477,7 +480,7 @@ void FEA_Module_SGH::update_forward_solve(Teuchos::RCP zp) vec_array node_mass_interface = node_masses_distributed->getLocalView(Tpetra::Access::ReadWrite); FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { node_mass_interface(node_gid, 0) = node_mass(node_gid); - }); // end parallel for + }); // end parallel for } // end view scope Kokkos::fence(); // communicate ghost densities @@ -582,10 +585,6 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full() } } // end if } - // else if (cycle==1){ - // if(myrank==0) - // printf("cycle = %lu, time = %f, time step = %f \n", cycle-1, time_data[cycle-1], global_dt); - // } // end if // compute adjoint vector for this data point; use velocity midpoint // view scope @@ -789,9 +788,10 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full() const_vec_array previous_adjoint_vector = (*adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); const_vec_array phi_previous_adjoint_vector = (*phi_adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); const_vec_array psi_previous_adjoint_vector = (*psi_adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); - vec_array midpoint_adjoint_vector = adjoint_vector_distributed->getLocalView(Tpetra::Access::ReadWrite); - vec_array phi_midpoint_adjoint_vector = phi_adjoint_vector_distributed->getLocalView(Tpetra::Access::ReadWrite); - vec_array psi_midpoint_adjoint_vector = psi_adjoint_vector_distributed->getLocalView(Tpetra::Access::ReadWrite); + + vec_array midpoint_adjoint_vector = adjoint_vector_distributed->getLocalView(Tpetra::Access::ReadWrite); + vec_array phi_midpoint_adjoint_vector = phi_adjoint_vector_distributed->getLocalView(Tpetra::Access::ReadWrite); + vec_array psi_midpoint_adjoint_vector = psi_adjoint_vector_distributed->getLocalView(Tpetra::Access::ReadWrite); // half step update for RK2 scheme; EQUATION 1 FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { @@ -822,7 +822,7 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full() phi_previous_adjoint_vector(node_gid, idim) / node_mass(node_gid); midpoint_adjoint_vector(node_gid, idim) = -rate_of_change * global_dt / 2 + previous_adjoint_vector(node_gid, idim); } - }); // end parallel for + }); // end parallel for Kokkos::fence(); // apply BCs to adjoint vector, only matters for the momentum adjoint if using strictly velocity boundary conditions @@ -858,7 +858,7 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full() // rate_of_change = -0.0000001*previous_adjoint_vector(node_gid,idim); phi_midpoint_adjoint_vector(node_gid, idim) = -rate_of_change * global_dt / 2 + phi_previous_adjoint_vector(node_gid, idim); } - }); // end parallel for + }); // end parallel for Kokkos::fence(); comm_phi_adjoint_vector(cycle); @@ -882,7 +882,7 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full() rate_of_change = -(matrix_contribution + psi_previous_adjoint_vector(elem_gid, 0) * Power_Gradient_Energies(elem_gid)) / elem_mass(elem_gid); // rate_of_change = -0.0000001*previous_adjoint_vector(node_gid,idim); psi_midpoint_adjoint_vector(elem_gid, 0) = -rate_of_change * global_dt / 2 + psi_previous_adjoint_vector(elem_gid, 0); - }); // end parallel for + }); // end parallel for Kokkos::fence(); // save for second half of RK @@ -926,7 +926,7 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full() phi_midpoint_adjoint_vector(node_gid, idim) / node_mass(node_gid); current_adjoint_vector(node_gid, idim) = -rate_of_change * global_dt + previous_adjoint_vector(node_gid, idim); } - }); // end parallel for + }); // end parallel for Kokkos::fence(); boundary_adjoint(*mesh, boundary, current_adjoint_vector, phi_current_adjoint_vector, psi_midpoint_adjoint_vector); @@ -960,7 +960,7 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full() // rate_of_change = -0.0000001*midpoint_adjoint_vector(node_gid,idim); phi_current_adjoint_vector(node_gid, idim) = -rate_of_change * global_dt + phi_previous_adjoint_vector(node_gid, idim); } - }); // end parallel for + }); // end parallel for Kokkos::fence(); comm_phi_adjoint_vector(cycle); @@ -983,7 +983,7 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full() // debug // std::cout << "PSI RATE OF CHANGE " << rate_of_change << std::endl; psi_current_adjoint_vector(elem_gid, 0) = -rate_of_change * global_dt + psi_previous_adjoint_vector(elem_gid, 0); - }); // end parallel for + }); // end parallel for Kokkos::fence(); // save data from time-step completion @@ -1000,16 +1000,18 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full() void FEA_Module_SGH::compute_topology_optimization_gradient_full(Teuchos::RCP design_densities_distributed, Teuchos::RCP design_gradients_distributed) { - size_t num_bdy_nodes = mesh->num_bdy_nodes; - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos material = simparam->material; - const int num_dim = simparam->num_dims; - int num_corners = rnum_elem * num_nodes_in_elem; - real_t global_dt; - bool element_constant_density = true; - size_t current_data_index, next_data_index; - CArrayKokkos current_element_velocities = CArrayKokkos(num_nodes_in_elem, num_dim); - CArrayKokkos current_element_adjoint = CArrayKokkos(num_nodes_in_elem, num_dim); + bool element_constant_density = true; + size_t num_bdy_nodes = mesh->num_bdy_nodes; + size_t current_data_index, next_data_index; + int num_corners = rnum_elem * num_nodes_in_elem; + real_t global_dt; + + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos material = simparam->material; + const int num_dim = simparam->num_dims; + + auto current_element_velocities = CArrayKokkos(num_nodes_in_elem, num_dim); + auto current_element_adjoint = CArrayKokkos(num_nodes_in_elem, num_dim); if (myrank == 0) { @@ -1121,7 +1123,7 @@ void FEA_Module_SGH::compute_topology_optimization_gradient_full(Teuchos::RCP(rnum_elem, num_nodes_in_elem * num_dim); Power_Gradient_Energies = CArrayKokkos(rnum_elem); @@ -1726,7 +1726,7 @@ void FEA_Module_SGH::init_assembly() { DOF_Graph_Matrix(idof, istride) = Graph_Matrix(idof / num_dim, istride / num_dim) * num_dim + istride % num_dim; } - }); // end parallel for + }); // end parallel for /* //construct distributed gradient matrix from local kokkos data diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/time_integration.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/time_integration.cpp index a934f2932..aeaea1d51 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/time_integration.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/time_integration.cpp @@ -147,7 +147,7 @@ void FEA_Module_SGH::get_timestep(mesh_t& mesh, // This function calculates the time step by finding the shortest distance // between any two nodes in the mesh // ------------------------------------------------------------------------------ -// WARNING WARNING : Only works for 3D, 8 node elements +// WARNING WARNING : Only works for 2D, 4 node elements void FEA_Module_SGH::get_timestep2D(mesh_t& mesh, DViewCArrayKokkos& node_coords, DViewCArrayKokkos& node_vel, From 62478bcf722dddbcc194edcb6924e77a02a654e2 Mon Sep 17 00:00:00 2001 From: Jacob Moore Date: Fri, 16 Feb 2024 17:25:51 -0600 Subject: [PATCH 03/64] COMP: Adding MSU machine type to automate building on HPCC machines --- scripts/build-fierro.sh | 8 +++++--- scripts/machines/msu-env.sh | 34 ++++++++++++++++++++++++++++++++++ scripts/trilinos-install.sh | 27 +++++++++++++++++++++++++++ 3 files changed, 66 insertions(+), 3 deletions(-) create mode 100644 scripts/machines/msu-env.sh diff --git a/scripts/build-fierro.sh b/scripts/build-fierro.sh index 983c0b77a..8432a75e4 100755 --- a/scripts/build-fierro.sh +++ b/scripts/build-fierro.sh @@ -43,6 +43,7 @@ show_help() { echo " darwin The darwin cluster at LANL. Uses module loads for software" echo " linux A general linux machine (that does not use modules)" echo " mac A Mac computer. This option does not allow for cuda and hip builds, and build_cores will be set to 1" + echo " msu A linux computer managed by the HPCC group at Mississippi State University" echo " " echo " --heffte_build_type The build type for the heffte installation. The default is 'fftw'" echo " " @@ -67,7 +68,7 @@ valid_build_action=("full-app" "set-env" "install-trilinos" "install-hdf5" "inst valid_solver=("all" "explicit" "explicit-evpfft" "explicit-ls-evpfft" "explicit-evp" "implicit") valid_kokkos_build_types=("serial" "openmp" "pthreads" "cuda" "hip") valid_heffte_build_types=("fftw" "cufft" "rocfft") -valid_machines=("darwin" "chicoma" "linux" "mac") +valid_machines=("darwin" "chicoma" "linux" "mac" "msu") # Parse command line arguments for arg in "$@"; do @@ -168,6 +169,7 @@ echo "Building based on these argument options:" echo "Build action - ${build_action}" echo "Solver - ${solver}" echo "Kokkos backend - ${kokkos_build_type}" +echo "Machine - ${machine}" if [ "${solver}" = "explicit-evpfft" ] || [ "${solver}" = "explicit-ls-evpfft" ]; then echo "HEFFTE - ${heffte_build_type}" fi @@ -180,14 +182,14 @@ source setup-env.sh ${machine} ${kokkos_build_type} ${build_cores} # Next, do action based on args if [ "$build_action" = "full-app" ]; then - source trilinos-install.sh ${kokkos_build_type} + source trilinos-install.sh ${kokkos_build_type} ${machine} if [ "$solver" = "explicit-evpfft" ] || [ "${solver}" = "explicit-ls-evpfft" ]; then source hdf5-install.sh source heffte-install.sh ${heffte_build_type} ${machine} fi source cmake_build.sh ${solver} elif [ "$build_action" = "install-trilinos" ]; then - source trilinos-install.sh ${kokkos_build_type} + source trilinos-install.sh ${kokkos_build_type} ${machine} elif [ "$build_action" = "install-hdf5" ]; then source hdf5-install.sh elif [ "$build_action" = "install-heffte" ]; then diff --git a/scripts/machines/msu-env.sh b/scripts/machines/msu-env.sh new file mode 100644 index 000000000..50adf7286 --- /dev/null +++ b/scripts/machines/msu-env.sh @@ -0,0 +1,34 @@ +#!/bin/bash -e +### Load environment modules here +### Assign names as relevant + +# arg number is '4' because it's based on the original build-fierro script args +kokkos_build_type="$1" + +mygcc="gcc/12.2.0" +mycuda="cuda/11.7.0" +mympi="openmpi/4.1.4" +mypython="python/3.10.8" +mymkl="intel-oneapi-mkl/2022.2.1" +myfftw="fftw/3.3.10" + + +echo "** Purging modules and loading those necessary for Fierro **" +module purge + +module load ${mygcc} +module load ${mympi} +module load ${mypython} +module load ${mymkl} +module load ${myfftw} + +if [ "$kokkos_build_type" = "cuda" ]; then + module load ${mycuda} +elif [ "$kokkos_build_type" = "hip" ]; then + echo "Error: MSU cannot build with Kokkos HIP backend since we do not have rocm" + show_help + return 1 +fi + +module load cmake +module -t list diff --git a/scripts/trilinos-install.sh b/scripts/trilinos-install.sh index ed6f1c6cb..579416312 100644 --- a/scripts/trilinos-install.sh +++ b/scripts/trilinos-install.sh @@ -1,10 +1,12 @@ #!/bin/bash -e kokkos_build_type="${1}" +machine="${2}" # If all arguments are valid, you can use them in your script as needed echo "Trilinos Kokkos Build Type: $kokkos_build_type" + #check if Trilinos directory exists, git clone Trilinos if it doesn't [ -d "${TRILINOS_SOURCE_DIR}" ] && echo "Directory Trilinos exists, skipping Trilinos download" @@ -68,12 +70,35 @@ OPENMP_ADDITIONS=( -D Trilinos_ENABLE_OpenMP=ON ) +# Flags for building with MKL, which is supported at MSU HPCC +MSU_ADDITIONS=( +-D BLAS_LIBRARY_NAMES="libmkl_rt.so" +-D BLAS_LIBRARY_DIRS="/apps/spack-managed/gcc-11.3.1/intel-oneapi-mkl-2022.2.1-7l7jlsd56x2kljiskrcvsoenmq4y3cu7/mkl/2022.2.1/lib/intel64" +-D LAPACK_LIBRARY_NAMES="libmkl_rt.so" +-D LAPACK_LIBRARY_DIRS="/apps/spack-managed/gcc-11.3.1/intel-oneapi-mkl-2022.2.1-7l7jlsd56x2kljiskrcvsoenmq4y3cu7/mkl/2022.2.1/lib/intel64" +-D TPL_ENABLE_MKL:BOOL=ON +-D MKL_LIBRARY_DIRS:FILEPATH="/apps/spack-managed/gcc-11.3.1/intel-oneapi-mkl-2022.2.1-7l7jlsd56x2kljiskrcvsoenmq4y3cu7/mkl/2022.2.1/lib/intel64" +-D MKL_LIBRARY_NAMES:STRING="mkl_rt" +-D MKL_INCLUDE_DIRS:FILEPATH="/apps/spack-managed/gcc-11.3.1/intel-oneapi-mkl-2022.2.1-7l7jlsd56x2kljiskrcvsoenmq4y3cu7/mkl/2022.2.1/include" +) + # Configure kokkos using CMake cmake_options=( -D CMAKE_BUILD_TYPE=Release -D Trilinos_MUST_FIND_ALL_TPL_LIBS=TRUE -D CMAKE_CXX_STANDARD=17 -D TPL_ENABLE_MPI=ON +) + +echo "**** Machine = ${machine} ****" +if [ "$machine" = "msu" ]; then + echo "**** WARNING: Verify MKL path in trilinos-install.sh ****" + cmake_options+=( + ${MSU_ADDITIONS[@]} + ) +fi + +cmake_options+=( -D Trilinos_ENABLE_Kokkos=ON ${ADDITIONS[@]} -D Trilinos_ENABLE_Amesos2=ON @@ -90,6 +115,8 @@ ${ADDITIONS[@]} -D CMAKE_INSTALL_PREFIX=${TRILINOS_INSTALL_DIR} ) + + if [ "$kokkos_build_type" = "openmp" ]; then cmake_options+=( ${OPENMP_ADDITIONS[@]} From c18ef90cd92483707a7064e2c94fedf49d0daf8b Mon Sep 17 00:00:00 2001 From: Jacob Moore <133793250+jacob-moore22@users.noreply.github.com> Date: Fri, 16 Feb 2024 17:39:21 -0600 Subject: [PATCH 04/64] Create ubuntu_build.yml Trying my hand at automated building using github workflows. --- .github/workflows/ubuntu_build.yml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 .github/workflows/ubuntu_build.yml diff --git a/.github/workflows/ubuntu_build.yml b/.github/workflows/ubuntu_build.yml new file mode 100644 index 000000000..c500c2da3 --- /dev/null +++ b/.github/workflows/ubuntu_build.yml @@ -0,0 +1,26 @@ +name: C/C++ CI + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - name: configure + run: ./configure + + - uses: actions/checkout@v2 + - name: Run script file + run: | + chmod +x ./scripts/build-fierro.sh + ./scripts/build-fierro.sh --build_action=full-app --solver=all --kokkos_build_type=openmp --build_cores=2 + shell: bash From eb634c03ea88f524b5c3b3b345550845a372f8d1 Mon Sep 17 00:00:00 2001 From: Jacob Moore <133793250+jacob-moore22@users.noreply.github.com> Date: Fri, 16 Feb 2024 17:41:27 -0600 Subject: [PATCH 05/64] Rename ubuntu_build.yml to ubuntu-build.yaml --- .github/workflows/{ubuntu_build.yml => ubuntu-build.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{ubuntu_build.yml => ubuntu-build.yaml} (100%) diff --git a/.github/workflows/ubuntu_build.yml b/.github/workflows/ubuntu-build.yaml similarity index 100% rename from .github/workflows/ubuntu_build.yml rename to .github/workflows/ubuntu-build.yaml From 8d7227dd75113f19a6ef1ee053820410464dc68f Mon Sep 17 00:00:00 2001 From: Jacob Moore <133793250+jacob-moore22@users.noreply.github.com> Date: Fri, 16 Feb 2024 17:42:05 -0600 Subject: [PATCH 06/64] Update ubuntu-build.yaml --- .github/workflows/ubuntu-build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu-build.yaml b/.github/workflows/ubuntu-build.yaml index c500c2da3..21c554453 100644 --- a/.github/workflows/ubuntu-build.yaml +++ b/.github/workflows/ubuntu-build.yaml @@ -1,4 +1,4 @@ -name: C/C++ CI +name: Ubuntu Build on: push: From 7bbedf0dd3ff1797b6f97d7d1061650c85906fbe Mon Sep 17 00:00:00 2001 From: Jacob Moore <133793250+jacob-moore22@users.noreply.github.com> Date: Fri, 16 Feb 2024 17:54:05 -0600 Subject: [PATCH 07/64] COMP: Update ubuntu-build.yaml Update build automation script. --- .github/workflows/ubuntu-build.yaml | 30 ++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ubuntu-build.yaml b/.github/workflows/ubuntu-build.yaml index 21c554453..1e518ef42 100644 --- a/.github/workflows/ubuntu-build.yaml +++ b/.github/workflows/ubuntu-build.yaml @@ -15,11 +15,31 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive - - name: configure - run: ./configure - - - uses: actions/checkout@v2 - - name: Run script file + - name: update the package list + run: + sudo apt-get update + shell: bash + - name: Install build tools + run: + sudo apt-get install build-essential + shell: bash + - name: Install LaPack + run: + sudo apt-get install libblas-dev liblapack-dev + shell: bash + - name: Install MPI + run: | + sudo apt-get install libopenmpi-dev openmpi-bin + echo "MPI_OPTS=--oversubscribe" >> $GITHUB_ENV + - name: Install cmake + run: + sudo apt-get -y install cmake + shell: bash + - name: Install python + run: + sudo apt-get install python3.6 + shell: bash + - name: Run build script run: | chmod +x ./scripts/build-fierro.sh ./scripts/build-fierro.sh --build_action=full-app --solver=all --kokkos_build_type=openmp --build_cores=2 From e9cd127c0a685dfc9d1aecd52b2f474e13af5902 Mon Sep 17 00:00:00 2001 From: yencal Date: Sun, 18 Feb 2024 13:02:54 -0700 Subject: [PATCH 08/64] Redesigned Material Model Interface --- scripts/build-fierro.sh | 5 +- scripts/cmake_build.sh | 6 - src/EVP/src/CMakeLists.txt | 4 - src/EVP/src/UserEOSModel.cpp | 63 -- src/EVP/src/UserEOSModel.h | 48 -- src/EVP/src/UserStrengthModel.h | 42 -- src/EVP/src/init_state_vars.cpp | 420 ----------- .../example_input_files/taylor_anvil.yaml | 12 +- .../tension_single_element.yaml | 14 +- src/EVPFFT/src/CMakeLists.txt | 28 +- .../Fierro-EVPFFT-Link/FierroEVPFFTLink.cpp | 147 ++++ .../src/Fierro-EVPFFT-Link/FierroEVPFFTLink.h | 53 ++ .../src/Fierro-EVPFFT-Link/UserEOSModel.cpp | 52 -- .../src/Fierro-EVPFFT-Link/UserEOSModel.h | 50 -- .../Fierro-EVPFFT-Link/UserStrengthModel.cpp | 99 --- .../Fierro-EVPFFT-Link/UserStrengthModel.h | 46 -- .../Fierro-EVPFFT-Link/init_state_vars.cpp | 26 - src/LS-EVPFFT/src/CMakeLists.txt | 28 +- .../FierroLSEVPFFTLink.cpp | 146 ++++ .../FierroLSEVPFFTLink.h | 53 ++ .../Fierro-LS-EVPFFT-Link/UserEOSModel.cpp | 52 -- .../src/Fierro-LS-EVPFFT-Link/UserEOSModel.h | 50 -- .../UserStrengthModel.cpp | 99 --- .../Fierro-LS-EVPFFT-Link/UserStrengthModel.h | 46 -- .../Fierro-LS-EVPFFT-Link/init_state_vars.cpp | 26 - .../Material-Models/CMakeLists.txt | 35 +- .../Ideal-Gas-Models/CMakeLists.txt | 4 - .../Ideal-Gas-Models/IdealGasEOSModel.cpp | 119 --- .../Ideal-Gas-Models/IdealGasEOSModel.h | 92 --- .../IdealGasStrengthModel.cpp | 49 -- .../Ideal-Gas-Models/IdealGasStrengthModel.h | 43 -- .../Ideal-Gas-Models/init_state_vars.cpp | 26 - .../User-Material-Models/.gitignore | 9 - .../User-Material-Models/CMakeLists.txt | 4 - .../User-Material-Models/UserEOSModel.cpp | 51 -- .../User-Material-Models/UserEOSModel.h | 48 -- .../UserStrengthModel.cpp | 43 -- .../User-Material-Models/UserStrengthModel.h | 42 -- .../User-Material-Models/init_state_vars.cpp | 26 - .../Material-Models/eos/CMakeLists.txt | 3 + .../eos/Constant/CMakeLists.txt | 4 + .../eos/Constant/ConstantEOSModel.cpp | 183 +++++ .../eos/Constant/ConstantEOSModel.h | 120 +++ .../eos/IdealGas/CMakeLists.txt | 4 + .../eos/IdealGas/IdealGasEOSModel.cpp | 206 +++++ .../eos/IdealGas/IdealGasEOSModel.h | 120 +++ .../eos/UserDefined/CMakeLists.txt | 4 + .../eos/UserDefined/UserDefinedEOSModel.cpp | 158 ++++ .../eos/UserDefined/UserDefinedEOSModel.h | 120 +++ .../Material-Models/material_models.cpp | 512 +++++++------ .../Material-Models/material_models.h | 143 ++-- .../Material-Models/strength/CMakeLists.txt | 5 + .../strength/EVP/CMakeLists.txt | 4 + .../strength/EVP/EVPStrengthModel.cpp | 703 ++++++++++++++++++ .../strength/EVP/EVPStrengthModel.h | 54 ++ .../strength/EVP}/chg_basis.cpp | 0 .../Material-Models/strength/EVP}/evpal.cpp | 0 .../EVP/example_input_files/ta_el.sx | 0 .../EVP/example_input_files/ta_pl.sx | 0 .../strength/EVP/user_mat.cpp} | 243 ------ .../Material-Models/strength/EVP}/user_mat.h | 0 .../strength/EVPFFT/CMakeLists.txt | 12 + .../strength/EVPFFT/EVPFFTStrengthModel.cpp | 105 +++ .../strength/EVPFFT/EVPFFTStrengthModel.h | 54 ++ .../strength/LSEVPFFT/CMakeLists.txt | 12 + .../LSEVPFFT/LSEVPFFTStrengthModel.cpp | 104 +++ .../strength/LSEVPFFT/LSEVPFFTStrengthModel.h | 54 ++ .../strength/UserDefined/CMakeLists.txt | 4 + .../UserDefined/UserDefinedStrengthModel.cpp | 79 ++ .../UserDefined/UserDefinedStrengthModel.h | 54 ++ .../strength/VUMAT/CMakeLists.txt | 33 + .../Material-Models/strength/VUMAT/VUMAT.cpp | 202 +++++ .../Material-Models/strength/VUMAT/VUMAT.h | 108 +++ .../strength/VUMAT/VUMATStrengthModel.cpp | 129 ++++ .../strength/VUMAT/VUMATStrengthModel.h | 53 ++ .../strength/VUMAT/lib_mod.mod | Bin 0 -> 1466 bytes .../strength/VUMAT/vpsc7_mod.mod | Bin 0 -> 5950 bytes .../strength/VUMAT/vpsc_type_def.mod | Bin 0 -> 4157 bytes .../Material-Models/strength/VUMAT/vumat.f90 | 40 + .../Material-Models/strength/VUMAT/vumat.h | 18 + .../FEA_Module_Dynamic_Elasticity.cpp | 57 +- .../FEA_Module_Dynamic_Elasticity.h | 6 +- .../elastic_optimization.cpp | 21 +- .../Dynamic_Elastic_Solver/properties.cpp | 62 +- .../Eulerian_Solver/FEA_Module_Eulerian.cpp | 6 +- .../Eulerian_Solver/FEA_Module_Eulerian.h | 6 +- .../Simulation_Parameters_Eulerian.h | 34 +- .../SGH_Solver/FEA_Module_SGH.cpp | 56 +- .../SGH_Solver/FEA_Module_SGH.h | 7 +- .../SGH_Solver/force_gradients_sgh.cpp | 24 +- .../SGH_Solver/force_sgh.cpp | 39 +- .../SGH_Solver/properties.cpp | 62 +- .../SGH_Solver/sgh_optimization.cpp | 21 +- .../Parallel-Explicit/example_simple.yaml | 2 +- .../Parallel-Explicit/example_simple_opt.yaml | 2 +- .../Simulation_Parameters/Material.h | 22 +- .../Simulation_Parameters.h | 39 +- 97 files changed, 3853 insertions(+), 2466 deletions(-) delete mode 100644 src/EVP/src/CMakeLists.txt delete mode 100644 src/EVP/src/UserEOSModel.cpp delete mode 100644 src/EVP/src/UserEOSModel.h delete mode 100644 src/EVP/src/UserStrengthModel.h delete mode 100644 src/EVP/src/init_state_vars.cpp create mode 100644 src/EVPFFT/src/Fierro-EVPFFT-Link/FierroEVPFFTLink.cpp create mode 100644 src/EVPFFT/src/Fierro-EVPFFT-Link/FierroEVPFFTLink.h delete mode 100644 src/EVPFFT/src/Fierro-EVPFFT-Link/UserEOSModel.cpp delete mode 100644 src/EVPFFT/src/Fierro-EVPFFT-Link/UserEOSModel.h delete mode 100644 src/EVPFFT/src/Fierro-EVPFFT-Link/UserStrengthModel.cpp delete mode 100644 src/EVPFFT/src/Fierro-EVPFFT-Link/UserStrengthModel.h delete mode 100644 src/EVPFFT/src/Fierro-EVPFFT-Link/init_state_vars.cpp create mode 100644 src/LS-EVPFFT/src/Fierro-LS-EVPFFT-Link/FierroLSEVPFFTLink.cpp create mode 100644 src/LS-EVPFFT/src/Fierro-LS-EVPFFT-Link/FierroLSEVPFFTLink.h delete mode 100644 src/LS-EVPFFT/src/Fierro-LS-EVPFFT-Link/UserEOSModel.cpp delete mode 100644 src/LS-EVPFFT/src/Fierro-LS-EVPFFT-Link/UserEOSModel.h delete mode 100644 src/LS-EVPFFT/src/Fierro-LS-EVPFFT-Link/UserStrengthModel.cpp delete mode 100644 src/LS-EVPFFT/src/Fierro-LS-EVPFFT-Link/UserStrengthModel.h delete mode 100644 src/LS-EVPFFT/src/Fierro-LS-EVPFFT-Link/init_state_vars.cpp delete mode 100644 src/Parallel-Solvers/Material-Models/Ideal-Gas-Models/CMakeLists.txt delete mode 100644 src/Parallel-Solvers/Material-Models/Ideal-Gas-Models/IdealGasEOSModel.cpp delete mode 100644 src/Parallel-Solvers/Material-Models/Ideal-Gas-Models/IdealGasEOSModel.h delete mode 100644 src/Parallel-Solvers/Material-Models/Ideal-Gas-Models/IdealGasStrengthModel.cpp delete mode 100644 src/Parallel-Solvers/Material-Models/Ideal-Gas-Models/IdealGasStrengthModel.h delete mode 100644 src/Parallel-Solvers/Material-Models/Ideal-Gas-Models/init_state_vars.cpp delete mode 100644 src/Parallel-Solvers/Material-Models/User-Material-Models/.gitignore delete mode 100644 src/Parallel-Solvers/Material-Models/User-Material-Models/CMakeLists.txt delete mode 100644 src/Parallel-Solvers/Material-Models/User-Material-Models/UserEOSModel.cpp delete mode 100644 src/Parallel-Solvers/Material-Models/User-Material-Models/UserEOSModel.h delete mode 100644 src/Parallel-Solvers/Material-Models/User-Material-Models/UserStrengthModel.cpp delete mode 100644 src/Parallel-Solvers/Material-Models/User-Material-Models/UserStrengthModel.h delete mode 100644 src/Parallel-Solvers/Material-Models/User-Material-Models/init_state_vars.cpp create mode 100644 src/Parallel-Solvers/Material-Models/eos/CMakeLists.txt create mode 100644 src/Parallel-Solvers/Material-Models/eos/Constant/CMakeLists.txt create mode 100644 src/Parallel-Solvers/Material-Models/eos/Constant/ConstantEOSModel.cpp create mode 100644 src/Parallel-Solvers/Material-Models/eos/Constant/ConstantEOSModel.h create mode 100644 src/Parallel-Solvers/Material-Models/eos/IdealGas/CMakeLists.txt create mode 100644 src/Parallel-Solvers/Material-Models/eos/IdealGas/IdealGasEOSModel.cpp create mode 100644 src/Parallel-Solvers/Material-Models/eos/IdealGas/IdealGasEOSModel.h create mode 100644 src/Parallel-Solvers/Material-Models/eos/UserDefined/CMakeLists.txt create mode 100644 src/Parallel-Solvers/Material-Models/eos/UserDefined/UserDefinedEOSModel.cpp create mode 100644 src/Parallel-Solvers/Material-Models/eos/UserDefined/UserDefinedEOSModel.h create mode 100644 src/Parallel-Solvers/Material-Models/strength/CMakeLists.txt create mode 100644 src/Parallel-Solvers/Material-Models/strength/EVP/CMakeLists.txt create mode 100644 src/Parallel-Solvers/Material-Models/strength/EVP/EVPStrengthModel.cpp create mode 100644 src/Parallel-Solvers/Material-Models/strength/EVP/EVPStrengthModel.h rename src/{EVP/src => Parallel-Solvers/Material-Models/strength/EVP}/chg_basis.cpp (100%) rename src/{EVP/src => Parallel-Solvers/Material-Models/strength/EVP}/evpal.cpp (100%) rename src/{ => Parallel-Solvers/Material-Models/strength}/EVP/example_input_files/ta_el.sx (100%) rename src/{ => Parallel-Solvers/Material-Models/strength}/EVP/example_input_files/ta_pl.sx (100%) rename src/{EVP/src/UserStrengthModel.cpp => Parallel-Solvers/Material-Models/strength/EVP/user_mat.cpp} (55%) rename src/{EVP/src => Parallel-Solvers/Material-Models/strength/EVP}/user_mat.h (100%) create mode 100644 src/Parallel-Solvers/Material-Models/strength/EVPFFT/CMakeLists.txt create mode 100644 src/Parallel-Solvers/Material-Models/strength/EVPFFT/EVPFFTStrengthModel.cpp create mode 100644 src/Parallel-Solvers/Material-Models/strength/EVPFFT/EVPFFTStrengthModel.h create mode 100644 src/Parallel-Solvers/Material-Models/strength/LSEVPFFT/CMakeLists.txt create mode 100644 src/Parallel-Solvers/Material-Models/strength/LSEVPFFT/LSEVPFFTStrengthModel.cpp create mode 100644 src/Parallel-Solvers/Material-Models/strength/LSEVPFFT/LSEVPFFTStrengthModel.h create mode 100644 src/Parallel-Solvers/Material-Models/strength/UserDefined/CMakeLists.txt create mode 100644 src/Parallel-Solvers/Material-Models/strength/UserDefined/UserDefinedStrengthModel.cpp create mode 100644 src/Parallel-Solvers/Material-Models/strength/UserDefined/UserDefinedStrengthModel.h create mode 100644 src/Parallel-Solvers/Material-Models/strength/VUMAT/CMakeLists.txt create mode 100644 src/Parallel-Solvers/Material-Models/strength/VUMAT/VUMAT.cpp create mode 100644 src/Parallel-Solvers/Material-Models/strength/VUMAT/VUMAT.h create mode 100644 src/Parallel-Solvers/Material-Models/strength/VUMAT/VUMATStrengthModel.cpp create mode 100644 src/Parallel-Solvers/Material-Models/strength/VUMAT/VUMATStrengthModel.h create mode 100644 src/Parallel-Solvers/Material-Models/strength/VUMAT/lib_mod.mod create mode 100644 src/Parallel-Solvers/Material-Models/strength/VUMAT/vpsc7_mod.mod create mode 100644 src/Parallel-Solvers/Material-Models/strength/VUMAT/vpsc_type_def.mod create mode 100644 src/Parallel-Solvers/Material-Models/strength/VUMAT/vumat.f90 create mode 100644 src/Parallel-Solvers/Material-Models/strength/VUMAT/vumat.h diff --git a/scripts/build-fierro.sh b/scripts/build-fierro.sh index 6e76c4941..4534ee5c8 100755 --- a/scripts/build-fierro.sh +++ b/scripts/build-fierro.sh @@ -2,7 +2,7 @@ show_help() { echo "Usage: source $(basename "$BASH_SOURCE") [OPTION]" echo "Valid options:" - echo " --solver=. Default is 'explicit'" + echo " --solver=. Default is 'explicit'" echo " --kokkos_build_type=. Default is 'serial'" echo " --build_action=. Default is 'full-app'" echo " --machine=. Default is 'linux'" @@ -27,7 +27,6 @@ show_help() { echo " explicit builds the explicit solver" echo " explicit-evpfft builds the explicit solver with the EVPFFT material model" echo " explicit-ls-evpfft builds the explicit solver with the LS-EVPFFT material model" - echo " explicit-evp builds the explicit solver with the EVP material model" echo " implicit builds the explicit solver" echo " " echo " --kokkos_build_type The desired kokkos parallel backend to use. The default is 'serial'" @@ -64,7 +63,7 @@ build_cores="1" # Define arrays of valid options valid_build_action=("full-app" "set-env" "install-trilinos" "install-hdf5" "install-heffte" "fierro") -valid_solver=("all" "explicit" "explicit-evpfft" "explicit-ls-evpfft" "explicit-evp" "implicit") +valid_solver=("all" "explicit" "explicit-evpfft" "explicit-ls-evpfft" "implicit") valid_kokkos_build_types=("serial" "openmp" "pthreads" "cuda" "hip") valid_heffte_build_types=("fftw" "cufft" "rocfft") valid_machines=("darwin" "chicoma" "linux" "mac") diff --git a/scripts/cmake_build.sh b/scripts/cmake_build.sh index 04d9c2a26..b69cf57d8 100644 --- a/scripts/cmake_build.sh +++ b/scripts/cmake_build.sh @@ -87,12 +87,6 @@ elif [ "$solver" = "explicit-evpfft" ] || [ "$solver" = "explicit-ls-evpfft" ]; -D USE_FFTW=ON ) fi -elif [ "$solver" = "explicit-evp" ]; then - cmake_options+=( - -D BUILD_PARALLEL_EXPLICIT_SOLVER=ON - -D BUILD_IMPLICIT_SOLVER=OFF - -D BUILD_EVP_FIERRO=ON - ) else cmake_options+=( -D BUILD_PARALLEL_EXPLICIT_SOLVER=ON diff --git a/src/EVP/src/CMakeLists.txt b/src/EVP/src/CMakeLists.txt deleted file mode 100644 index f850a3a67..000000000 --- a/src/EVP/src/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -set(SRC_Files UserEOSModel.cpp UserStrengthModel.cpp init_state_vars.cpp chg_basis.cpp evpal.cpp) -add_library(user_material_models OBJECT ${SRC_Files}) -target_include_directories(user_material_models PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) -target_link_libraries(user_material_models Elements) \ No newline at end of file diff --git a/src/EVP/src/UserEOSModel.cpp b/src/EVP/src/UserEOSModel.cpp deleted file mode 100644 index b010a5f97..000000000 --- a/src/EVP/src/UserEOSModel.cpp +++ /dev/null @@ -1,63 +0,0 @@ -#include "UserEOSModel.h" - -KOKKOS_FUNCTION -UserEOSModel::UserEOSModel( - const DCArrayKokkos &material, - const DCArrayKokkos &elem_state_vars, - const DCArrayKokkos &global_vars, - const DCArrayKokkos &elem_user_output_vars, - const size_t mat_id, - const size_t elem_gid) -{ -} - -KOKKOS_FUNCTION -UserEOSModel::~UserEOSModel() -{ -} - -KOKKOS_FUNCTION -int UserEOSModel::calc_sound_speed( - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const size_t elem_gid, - const size_t mat_id, - const DCArrayKokkos &elem_state_vars, - const DCArrayKokkos &global_vars, - const DCArrayKokkos &elem_user_output_vars, - const DViewCArrayKokkos &elem_sspd, - const double den, - const double sie) -{ - elem_sspd(elem_gid) = 0.357;//1.0e-8; // sound speed - - return 0; -} - -KOKKOS_FUNCTION -int UserEOSModel::calc_pressure( - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const size_t elem_gid, - const size_t mat_id, - const DCArrayKokkos &elem_state_vars, - const DCArrayKokkos &global_vars, - const DCArrayKokkos &elem_user_output_vars, - const DViewCArrayKokkos &elem_sspd, - const double den, - const double sie) -{ - const int num_dims = 3; - - elem_pres(elem_gid) = 0.0; // pressure - - // pressure = 1/3tr(stress) - for (int i=0; i &material, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, - const DCArrayKokkos &elem_user_output_vars, - const size_t mat_id, - const size_t elem_gid); - - KOKKOS_FUNCTION - ~UserEOSModel(); - - KOKKOS_FUNCTION - int calc_sound_speed( - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const size_t elem_gid, - const size_t mat_id, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, - const DCArrayKokkos &elem_user_output_vars, - const DViewCArrayKokkos &elem_sspd, - const double den, - const double sie) override; - - KOKKOS_FUNCTION - int calc_pressure( - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const size_t elem_gid, - const size_t mat_id, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, - const DCArrayKokkos &elem_user_output_vars, - const DViewCArrayKokkos &elem_sspd, - const double den, - const double sie) override; - -}; diff --git a/src/EVP/src/UserStrengthModel.h b/src/EVP/src/UserStrengthModel.h deleted file mode 100644 index 34c99ee8e..000000000 --- a/src/EVP/src/UserStrengthModel.h +++ /dev/null @@ -1,42 +0,0 @@ -#pragma once - -#include "material_models.h" - - -/* UserStrengthModel */ -class UserStrengthModel : public StrengthParent { -public: - KOKKOS_FUNCTION - UserStrengthModel( - const DCArrayKokkos &material, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, - const DCArrayKokkos &elem_user_output_vars, - const size_t mat_id, - const size_t elem_gid); - - KOKKOS_FUNCTION - ~UserStrengthModel(); - - KOKKOS_FUNCTION - int calc_stress( - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const size_t elem_gid, - const size_t mat_id, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, - const DCArrayKokkos &elem_user_output_vars, - const DViewCArrayKokkos &elem_sspd, - const double den, - const double sie, - const ViewCArrayKokkos &vel_grad, - const ViewCArrayKokkos &elem_node_gids, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const double vol, - const double dt, - const double rk_alpha, - const size_t cycle, - const size_t rk_level) override; -}; diff --git a/src/EVP/src/init_state_vars.cpp b/src/EVP/src/init_state_vars.cpp deleted file mode 100644 index bf6154b32..000000000 --- a/src/EVP/src/init_state_vars.cpp +++ /dev/null @@ -1,420 +0,0 @@ -// ----------------------------------------------------------------------------- -// This code contains the initialization of state vars for supplied models -//------------------------------------------------------------------------------ -#include -#include -#include -#include -#include -#include -#include -#include - -#include "user_mat.h" -#include "Simulation_Parameters/Material.h" - -#define CLEAR_LINE(ifstream) ( ifstream.ignore(std::numeric_limits::max(), '\n') ); -// for string delimiter parsing -std::vector split (std::string s, std::string delimiter); - - //This goes in input.cpp -// material(0).num_state_vars = 163; // actual num_state_vars -// material(0).eos_model = user_eos_model; // EOS model is required -// material(0).strength_type = model::hypo; -// material(0).strength_setup = model_init::input; -// material(0).strength_model = user_strength_model; - -// ----------------------------------------------------------------------------- -// The function to read in the state vars for a user supplied model -//------------------------------------------------------------------------------ - - -void init_state_vars( - const DCArrayKokkos &material, - const DViewCArrayKokkos &elem_mat_id, - DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, - const DCArrayKokkos &elem_user_output_vars, - const size_t num_elems) -{ - const real_t SQR2_ = 1.41421356237309; - const real_t RSQ2_ = 0.70710678118654744; - const real_t RSQ3_ = 0.57735026918962584; - const real_t RSQ6_ = 0.40824829046386304; - - CMatrix cc66v(6,6); - CMatrix cdim(3); - CMatrix isn(24,3); //assumed BCC with 2 modes here - CMatrix isb(24,3); //assumed BCC with 2 modes here - CMatrix sn(3); - CMatrix sb(3); - CMatrix aux5(5); - CMatrix aux33(3,3); - // Accessed within state_vars for loop - DFMatrixKokkos dnca(3,24); //assumed BCC with 2 modes here - DFMatrixKokkos dbca(3,24); //assumed BCC with 2 modes here - DCMatrixKokkos cc66(6,6); - DCMatrixKokkos B_basis(3,3,6); - // FMatrix schca(5,12*2); //assumed BCC with 2 modes here - std::ifstream ur1; - int iso,nmodesx,nmodes,modex,nsmx,nrsx,nsysx; - real_t gamd0x,tau0xf,tau0xb,tau1x,thet0x,thet1x; //real_t may be something else in fierro - real_t hselfx,hlatex,snor,qnor,prod,ph,th,om; - std::string junk,icryst; - - real_t stress_conv = 1.0e-5; - - for (int i = 1; i <= 3; i++) { - for (int j = 1; j <= 3; j++) { - for (int k = 1; k <= 6; k++) { - B_basis.host(i,j,k) = 0.0; - } - } - } - B_basis.host(1,1,2) = -RSQ6_; - B_basis.host(2,2,2) = -RSQ6_; - B_basis.host(3,3,2) = real_t(2.0)*RSQ6_; - - B_basis.host(1,1,1) = -RSQ2_; - B_basis.host(2,2,1) = RSQ2_; - - B_basis.host(2,3,3) = RSQ2_; - B_basis.host(3,2,3) = RSQ2_; - - B_basis.host(1,3,4) = RSQ2_; - B_basis.host(3,1,4) = RSQ2_; - - B_basis.host(1,2,5) = RSQ2_; - B_basis.host(2,1,5) = RSQ2_; - - B_basis.host(1,1,6) = RSQ3_; - B_basis.host(2,2,6) = RSQ3_; - B_basis.host(3,3,6) = RSQ3_; - B_basis.update_device(); - - ur1.open("./ta_el.sx");//("./cuel1.txt"); - // read el txt file here - ur1 >> iso; CLEAR_LINE(ur1); - - if (iso == 0) { - for (int i = 1; i <= 6; i++) { - for (int j = 1; j <= 6; j++) { - ur1 >> cc66v(i,j); - } - CLEAR_LINE(ur1); - } - } - - // CMatrix dde(3,3); - // CMatrix xid4(3,3,3,3); - // CMatrix cc3333(3,3,3,3); - - // for (int i = 1; i <= 3; i++) { - // for (int j = 1; j <= 3; j++) { - // dde(i,j) = 0.0; - // if (i == j) dde(i,j) = 1.0; - // } - // } - - // for (int i = 1; i <= 3; i++) { - // for (int j = 1; j <= 3; j++) { - // for (int k = 1; k <= 3; k++) { - // for (int l = 1; l <= 3; l++) { - // xid4(i,j,k,l) = (dde(i,k)*dde(j,l) + dde(i,l)*dde(j,k)) / real_t(2.0); - // } - // } - // } - // } - // real_t tmu = 119800.0 / (real_t(2.0)*(real_t(1.0) + 0.35)); - // real_t tla = real_t(2.0)*tmu*0.35 / (real_t(1.0) - real_t(2.0)*0.35); - - // printf("%g %g\n",tmu,tla ); - - // for (int i = 1; i <= 3; i++) { - // for (int j = 1; j <= 3; j++) { - // for (int k = 1; k <= 3; k++) { - // for (int l = 1; l <= 3; l++) { - // cc3333(i,j,k,l) = tla*dde(i,j)*dde(k,l) + real_t(2.0)*tmu*xid4(i,j,k,l); - // printf("%d %d %d %d %g \n",i,j,k,l,cc3333(i,j,k,l) ); - // } - // } - // } - // } - // chg_basis_4(cc66.pointer(), cc3333.pointer(), 4, 6, B_basis.pointer()); - - int i1,i2,j1,j2; - int ijv_[6*2] = {1,2,3,2,1,1,1,2,3,3,3,2}; - CMatrix cc3333(3,3,3,3); - ViewFMatrixKokkos ijv(ijv_,6,2); - for (int i = 1; i <= 6; i++) { - i1 = ijv(i,1); - i2 = ijv(i,2); - for (int j = 1; j <= 6; j++) { - j1 = ijv(j,1); - j2 = ijv(j,2); - cc3333(i1,i2,j1,j2) = cc66v(i,j); - cc3333(i2,i1,j1,j2) = cc66v(i,j); - cc3333(i1,i2,j2,j1) = cc66v(i,j); - cc3333(i2,i1,j2,j1) = cc66v(i,j); - } - } - - for (int i = 1; i <= 3; i++) { - for (int j = 1; j <= 3; j++) { - for (int k = 1; k <= 3; k++) { - for (int l = 1; l <= 3; l++) { - printf("%d %d %d %d %g \n", i,j,k,l,cc3333(i,j,k,l)); - } - } - } - } - - chg_basis_4(cc66.host.pointer(), cc3333.pointer(), 4, 6, B_basis.host.pointer()); - - cc66.update_device(); - - // for (int i = 1; i <= 6; i++) { - // printf("cc=%.8f %.8f %.8f %.8f %.8f %.8f\n", cc66(i,1),cc66(i,2),cc66(i,3),cc66(i,4),cc66(i,5),cc66(i,6)); - // } - - - ur1.close(); - - ur1.open("./ta_pl.sx");//("./cupl3.txt"); - // read pl txt file here, up to slip systems - ur1 >> junk; CLEAR_LINE(ur1); - ur1 >> icryst; CLEAR_LINE(ur1); - for (int i = 1; i <= 3; i++) { - ur1 >> cdim(i); - } - CLEAR_LINE(ur1); - - ur1 >> nmodesx; CLEAR_LINE(ur1); - ur1 >> nmodes; CLEAR_LINE(ur1); - // temporary check until general crystal file read in is implemented - // if (nmodes > 1) { - // printf("nmodes IN IS > 1\n"); - // printf("ONLY 1 SLIP MODE ALLOWED FOR NOW\n"); - // exit(1); - // } - ur1 >> junk; - CLEAR_LINE(ur1); - - ur1 >> junk; CLEAR_LINE(ur1); - ur1 >> modex >> nsmx >> nrsx >> gamd0x; CLEAR_LINE(ur1); - ur1 >> tau0xf >> tau0xb >> tau1x >> thet0x >> thet1x; CLEAR_LINE(ur1); - ur1 >> hselfx >> hlatex; CLEAR_LINE(ur1); - ur1 >> junk; CLEAR_LINE(ur1); - - if (thet0x < thet1x) { - printf("INITIAL HARDENING LOWER THAN FINAL HARDENING\n"); - exit(1); - } - // - // CASE TAU1=0 CORRESPONDS TO LINEAR HARDENING AND IS INDEPENDENT OF TAU0. - // AVOID DIVISION BY ZERO - if (tau1x <= 1.0e-6) { - tau1x = 1.0e-6; - thet0x = thet1x; - } - - for (int j = 1; j <= nsmx; j++) { - for (int k = 1; k <= 3; k++) ur1 >> isn(j,k); - for (int k = 1; k <= 3; k++) ur1 >> isb(j,k); - CLEAR_LINE(ur1); - } - - for (int js = 1; js <= nsmx; js++) { - nsysx = js; - // - // DEFINES RATE SENSITIVITY AND CRSS FOR EACH SYSTEM IN THE MODE - // - for (int m = 1; m <= 3; m++) { - sn(m) = isn(js,m) / cdim(m); - sb(m) = isb(js,m) * cdim(m); - } - // - // *** NORMALIZES SYSTEM VECTORS AND CHECKS NORMALITY - // - snor = sqrt( sn(1)*sn(1) + sn(2)*sn(2) + sn(3)*sn(3) ); - qnor = sqrt( sb(1)*sb(1) + sb(2)*sb(2) + sb(3)*sb(3) ); - prod = 0.0; - for (int j = 1; j <= 3; j++) { - dnca.host(j,nsysx) = sn(j) / snor; - dbca.host(j,nsysx) = sb(j) / qnor; - if (abs(dnca.host(j,nsysx)) < 1.e-03) dnca.host(j,nsysx) = 0.0; - if (abs(dbca.host(j,nsysx)) < 1.e-03) dbca.host(j,nsysx) = 0.0; - prod += dnca.host(j,nsysx) * dbca.host(j,nsysx); - } - - if (prod >= 1.0e-3) { - printf("SYSTEM %d IS NOT ORTHOGONAL !!\n", js); - printf("%d %d %d,", isb(js,1), isb(js,2), isb(js,3)); - printf("%d %d %d", isn(js,1), isn(js,2), isn(js,3)); - exit(1); - } - // - // DEFINE SCHMID VECTOR IN CRYSTAL AXES FOR EACH SYSTEM - // - //for (int i = 1; i <= 3; i++) { - // for (int j = 1; j <= 3; j++) { - // aux33(i,j) = (dnca(i,nsysx)*dbca(j,nsysx) + dnca(j,nsysx)*dbca(i,nsysx)) / 2.0; - // } - //} - - //cb.chg_basis_2(aux5.pointer(), aux33.pointer(), 2, 5, cb.B_basis_host_pointer()); //heavy modification needed here - - //for (int i = 1; i <= 5; i++) { - // schca(i,nsysx) = aux5(i); - //} - } // end for js - - // second set of slip systems for Ta BCC - ur1 >> junk; CLEAR_LINE(ur1); - ur1 >> junk; CLEAR_LINE(ur1); - ur1 >> junk; CLEAR_LINE(ur1); - ur1 >> junk; CLEAR_LINE(ur1); - ur1 >> junk; CLEAR_LINE(ur1); - - for (int j = nsmx+1; j <= nsmx*2; j++) { - for (int k = 1; k <= 3; k++) ur1 >> isn(j,k); - for (int k = 1; k <= 3; k++) ur1 >> isb(j,k); - CLEAR_LINE(ur1); - } - for (int js = nsmx+1; js <= nsmx*2; js++) { - nsysx = js; - // - // DEFINES RATE SENSITIVITY AND CRSS FOR EACH SYSTEM IN THE MODE - // - for (int m = 1; m <= 3; m++) { - sn(m) = isn(js,m) / cdim(m); - sb(m) = isb(js,m) * cdim(m); - } - // - // *** NORMALIZES SYSTEM VECTORS AND CHECKS NORMALITY - // - snor = sqrt( sn(1)*sn(1) + sn(2)*sn(2) + sn(3)*sn(3) ); - qnor = sqrt( sb(1)*sb(1) + sb(2)*sb(2) + sb(3)*sb(3) ); - prod = 0.0; - for (int j = 1; j <= 3; j++) { - dnca.host(j,nsysx) = sn(j) / snor; - dbca.host(j,nsysx) = sb(j) / qnor; - if (abs(dnca.host(j,nsysx)) < 1.e-03) dnca.host(j,nsysx) = 0.0; - if (abs(dbca.host(j,nsysx)) < 1.e-03) dbca.host(j,nsysx) = 0.0; - prod += dnca.host(j,nsysx) * dbca.host(j,nsysx); - } - - if (prod >= 1.0e-3) { - printf("SYSTEM %d IS NOT ORTHOGONAL !!\n", js); - printf("%d %d %d,", isb(js,1), isb(js,2), isb(js,3)); - printf("%d %d %d", isn(js,1), isn(js,2), isn(js,3)); - exit(1); - } - } - dnca.update_device(); - dbca.update_device(); - - nsmx = nsmx*2; - ur1.close(); - - //===================================================================== - - for (size_t elem_gid = 0; elem_gid> ph >> th >> om; CLEAR_LINE(ur1); - // single crystal test - ph = 0.0; th = 0.0; om = 0.0; //001 - state_vars(elem_gid,257) = ph; - state_vars(elem_gid,258) = th; - state_vars(elem_gid,259) = om; - - // 260-313 Basis change array values - c = 260; - for (int i = 1; i <= 3; i++) { - for (int j = 1; j <= 3; j++) { - for (int k = 1; k <= 6; k++) { - state_vars(elem_gid,c) = B_basis(i,j,k); - c += 1; - } - } - } - state_vars(elem_gid,314) = grain_id; - //}); - } - Kokkos::fence(); - state_vars.update_host(); - - printf("user_mat_init completed\n"); - - return; -} diff --git a/src/EVPFFT/example_input_files/taylor_anvil.yaml b/src/EVPFFT/example_input_files/taylor_anvil.yaml index 8ab8e24e4..e0c1c5792 100644 --- a/src/EVPFFT/example_input_files/taylor_anvil.yaml +++ b/src/EVPFFT/example_input_files/taylor_anvil.yaml @@ -51,9 +51,9 @@ fea_module_parameters: materials: - id: 0 - eos_model: user_eos_model + eos_model: constant eos_run_location: device - strength_model: user_strength_model + strength_model: evpfft strength_type: hypo strength_run_location: host q1: 0.01 # 0.01 to 0.1 @@ -61,10 +61,12 @@ materials: q1ex: 0.01 # 0.01 to 0.1 q2ex: 0.0 maximum_limiter: true - # num_state_vars: 5 - global_vars: + # num_strength_state_vars: 5 + eos_global_vars: # for constant eos model specify sound_speed and pressure + - 2400000 #[mm/s] ref sound speed + - 0.0 #[N/mm2] pressure + strength_global_vars: # evpfft only uses one strength_global_vars - 0.0001 #udotAccTh - - 3000000 #[mm/s] ref sound speed regions: - volume: diff --git a/src/EVPFFT/example_input_files/tension_single_element.yaml b/src/EVPFFT/example_input_files/tension_single_element.yaml index cab96c036..5bd797ca3 100644 --- a/src/EVPFFT/example_input_files/tension_single_element.yaml +++ b/src/EVPFFT/example_input_files/tension_single_element.yaml @@ -47,19 +47,21 @@ fea_module_parameters: materials: - id: 0 - eos_model: user_eos_model + eos_model: constant eos_run_location: device - strength_model: user_strength_model + strength_model: evpfft strength_type: hypo strength_run_location: host q1: 1.0 q2: 1.3333 q1ex: 1.0 q2ex: 0.0 - # num_state_vars: 5 - global_vars: - - 0.0 #0.0001 #udotAccTh - - 100000 #2400000 #[mm/s] ref sound speed + # num_strength_state_vars: 5 + eos_global_vars: # for constant eos model specify sound_speed and pressure + - 2400000 #[mm/s] ref sound speed + - 0.0 #[N/mm2] pressure + strength_global_vars: # evpfft only used one strength_global_vars + - 0.0001 #udotAccTh regions: - volume: diff --git a/src/EVPFFT/src/CMakeLists.txt b/src/EVPFFT/src/CMakeLists.txt index 64629e900..73d548596 100644 --- a/src/EVPFFT/src/CMakeLists.txt +++ b/src/EVPFFT/src/CMakeLists.txt @@ -2,25 +2,25 @@ cmake_minimum_required(VERSION 3.17) set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED TRUE) -if(NOT "${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}") +if(BUILD_EVPFFT_FIERRO) add_definitions(-DBUILD_EVPFFT_FIERRO=1) endif() if (BUILD_EVPFFT_FIERRO) - set(This user_material_models) + set(This evpfft-fierro) else() set(This evpfft) project (${This}) -enable_language(C) - -# CMAKE_BUILD_TYPE: -# 1. Release: `-O3 -DNDEBUG` -# 2. Debug: `-O0 -g` -# 3. RelWithDebInfo: `-O2 -g -DNDEBUG` -# 4. MinSizeRel: `-Os -DNDEBUG` -if (NOT CMAKE_BUILD_TYPE) - set(CMAKE_BUILD_TYPE RelWithDebInfo) -endif(NOT CMAKE_BUILD_TYPE) + enable_language(C) + + # CMAKE_BUILD_TYPE: + # 1. Release: `-O3 -DNDEBUG` + # 2. Debug: `-O0 -g` + # 3. RelWithDebInfo: `-O2 -g -DNDEBUG` + # 4. MinSizeRel: `-Os -DNDEBUG` + if (NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE RelWithDebInfo) + endif(NOT CMAKE_BUILD_TYPE) endif() # Set default values for EVPFFT options @@ -112,9 +112,7 @@ set( Sources if (BUILD_EVPFFT_FIERRO) list(APPEND Sources - Fierro-EVPFFT-Link/UserStrengthModel.cpp - Fierro-EVPFFT-Link/UserEOSModel.cpp - Fierro-EVPFFT-Link/init_state_vars.cpp) + Fierro-EVPFFT-Link/FierroEVPFFTLink.cpp) add_library(${This} STATIC ${Sources}) target_link_libraries(${This} PRIVATE ${Trilinos_LIBRARIES} ${Trilinos_TPL_LIBRARIES} Elements) diff --git a/src/EVPFFT/src/Fierro-EVPFFT-Link/FierroEVPFFTLink.cpp b/src/EVPFFT/src/Fierro-EVPFFT-Link/FierroEVPFFTLink.cpp new file mode 100644 index 000000000..bcdb5fdf0 --- /dev/null +++ b/src/EVPFFT/src/Fierro-EVPFFT-Link/FierroEVPFFTLink.cpp @@ -0,0 +1,147 @@ +#include "FierroEVPFFTLink.h" +#include +#include "evpfft.h" + + +MPI_Comm evpfft_mpi_comm = MPI_COMM_NULL; +// to hold all evpfft in each element +std::vector> elem_evpfft; + +namespace FierroEVPFFTLink +{ + void init_strength_state_vars( + const DCArrayKokkos &material, + const DViewCArrayKokkos &elem_mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const size_t num_elems) + { + // First, lets create a new communicator with each rank having its own communicator containing only itself. + if (evpfft_mpi_comm == MPI_COMM_NULL) { + int global_rank; + MPI_Comm_rank(MPI_COMM_WORLD, &global_rank); + MPI_Comm_split(MPI_COMM_WORLD, global_rank, global_rank, &evpfft_mpi_comm); + } + //... + + // assign correct size to elem_evpfft, all element are nullptr during initialization + elem_evpfft = std::vector> (num_elems, nullptr); + + for (size_t elem_gid = 0; elem_gid < num_elems; elem_gid++) { + + size_t mat_id = elem_mat_id.host(elem_gid); + + // only fill the elem_evpfft of elements that use this model + if (material.host(mat_id).strength_model == STRENGTH_MODEL::evpfft) { + + // evpfft only runs on host so check to see + if (material.host(mat_id).strength_run_location == RUN_LOCATION::device) { + throw std::runtime_error("EVPFFT only runs on Host"); + } + + // Input files for for multiple materials should be names as evpfft1.in, evpfft2.in, etc. + std::string filename = "evpfft" + std::to_string(mat_id+1) + ".in"; + + real_t stress_scale = 1.0; // 1.0e-5; // used to convert MPa to MegaBar + real_t time_scale = 1.0; // 1.0e+6; // used to convert second to microsecond + + CommandLineArgs cmd; + cmd.input_filename = filename; //"evpfft.in"; + cmd.micro_filetype = 0; + cmd.check_cmd_args(); + + // create EVPFFT model in element that used evpfft + elem_evpfft[elem_gid] = std::make_shared(evpfft_mpi_comm, + cmd, + stress_scale, + time_scale); + + } // end if (material.host(mat_id).strength_model... + + } // end for (size_t elem_gid = 0... + + return; + } + + void calc_stress ( + const DViewCArrayKokkos &elem_pres, + const DViewCArrayKokkos &elem_stress, + const size_t elem_gid, + const size_t mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const DViewCArrayKokkos &elem_sspd, + const double den, + const double sie, + const ViewCArrayKokkos &vel_grad, + const ViewCArrayKokkos &elem_node_gids, + const DViewCArrayKokkos &node_coords, + const DViewCArrayKokkos &node_vel, + const double vol, + const double dt, + const double rk_alpha, + const size_t cycle, + const size_t rk_level, + const double time) + { + + if (elem_evpfft[elem_gid] == nullptr) + { + throw std::runtime_error("EVPFFT not initialized in this element"); + } + + real_t dt_rk = dt; // since using rk_num_stages = 1 + + // Note EVPFFT uses F-layout while Fierro uses C-layout + FArray Fvel_grad(3,3); + FArray Fstress(3,3); + // Transpose vel_grad + for (int i = 0; i < 3; i++) { + for (int j = 0; j < 3; j++) { + Fvel_grad(i,j) = vel_grad(i,j); + Fstress(i,j) = elem_stress.host(rk_level,elem_gid,i,j); + } + } + + double udotAccTh = strength_global_vars.host(mat_id,0); // Linear Aprox. Threshold + elem_evpfft[elem_gid]->solve(Fvel_grad.pointer(), Fstress.pointer(), dt_rk, cycle, elem_gid, udotAccTh); + + // Transpose stress. Not needed, stress is symmetric. But why not. + for (int i = 0; i < 3; i++) { + for (int j = 0; j < 3; j++) { + elem_stress.host(rk_level,elem_gid,i,j) = Fstress(i,j); + } + } + + // write into elem_state_vars for output + // evm,evmp,dvm,dvmp,svm + elem_user_output_vars.host(elem_gid,0) = elem_evpfft[elem_gid]->evm; + elem_user_output_vars.host(elem_gid,1) = elem_evpfft[elem_gid]->evmp; + elem_user_output_vars.host(elem_gid,2) = elem_evpfft[elem_gid]->dvm; + elem_user_output_vars.host(elem_gid,3) = elem_evpfft[elem_gid]->dvmp; + elem_user_output_vars.host(elem_gid,4) = elem_evpfft[elem_gid]->svm; + + return; + } + + void destroy( + const DCArrayKokkos &material, + const DViewCArrayKokkos &elem_mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const size_t num_elems) + { + return; + } + +} // end namespace FierroEVPFFTLink + diff --git a/src/EVPFFT/src/Fierro-EVPFFT-Link/FierroEVPFFTLink.h b/src/EVPFFT/src/Fierro-EVPFFT-Link/FierroEVPFFTLink.h new file mode 100644 index 000000000..281078052 --- /dev/null +++ b/src/EVPFFT/src/Fierro-EVPFFT-Link/FierroEVPFFTLink.h @@ -0,0 +1,53 @@ +#pragma once + +#include "Simulation_Parameters/Material.h" +#include "matar.h" +using namespace mtr; + +namespace FierroEVPFFTLink +{ + void init_strength_state_vars( + const DCArrayKokkos &material, + const DViewCArrayKokkos &elem_mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const size_t num_elems); + + void calc_stress ( + const DViewCArrayKokkos &elem_pres, + const DViewCArrayKokkos &elem_stress, + const size_t elem_gid, + const size_t mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const DViewCArrayKokkos &elem_sspd, + const double den, + const double sie, + const ViewCArrayKokkos &vel_grad, + const ViewCArrayKokkos &elem_node_gids, + const DViewCArrayKokkos &node_coords, + const DViewCArrayKokkos &node_vel, + const double vol, + const double dt, + const double rk_alpha, + const size_t cycle, + const size_t rk_level, + const double time); + + void destroy( + const DCArrayKokkos &material, + const DViewCArrayKokkos &elem_mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const size_t num_elems); + +} // end namespace FierroEVPFFTLink diff --git a/src/EVPFFT/src/Fierro-EVPFFT-Link/UserEOSModel.cpp b/src/EVPFFT/src/Fierro-EVPFFT-Link/UserEOSModel.cpp deleted file mode 100644 index 2d1aaa82f..000000000 --- a/src/EVPFFT/src/Fierro-EVPFFT-Link/UserEOSModel.cpp +++ /dev/null @@ -1,52 +0,0 @@ -#include "UserEOSModel.h" - -KOKKOS_FUNCTION -UserEOSModel::UserEOSModel( - const DCArrayKokkos &material, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, - const DCArrayKokkos &elem_user_output_vars, - const size_t mat_id, - const size_t elem_gid) -{ - sound_speed = global_vars(mat_id,1); -} - -KOKKOS_FUNCTION -UserEOSModel::~UserEOSModel() -{ -} - -KOKKOS_FUNCTION -int UserEOSModel::calc_sound_speed( - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const size_t elem_gid, - const size_t mat_id, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, - const DCArrayKokkos &elem_user_output_vars, - const DViewCArrayKokkos &elem_sspd, - const double den, - const double sie) -{ - elem_sspd(elem_gid) = sound_speed; // sound speed - return 0; -} - -KOKKOS_FUNCTION -int UserEOSModel::calc_pressure( - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const size_t elem_gid, - const size_t mat_id, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, - const DCArrayKokkos &elem_user_output_vars, - const DViewCArrayKokkos &elem_sspd, - const double den, - const double sie) -{ - elem_pres(elem_gid) = 0.0; // pressure - return 0; -} diff --git a/src/EVPFFT/src/Fierro-EVPFFT-Link/UserEOSModel.h b/src/EVPFFT/src/Fierro-EVPFFT-Link/UserEOSModel.h deleted file mode 100644 index 301b790b2..000000000 --- a/src/EVPFFT/src/Fierro-EVPFFT-Link/UserEOSModel.h +++ /dev/null @@ -1,50 +0,0 @@ -#pragma once - -#include "material_models.h" - - -/* UserEOSModel */ -class UserEOSModel : public EOSParent { -public: - - KOKKOS_FUNCTION - UserEOSModel( - const DCArrayKokkos &material, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, - const DCArrayKokkos &elem_user_output_vars, - const size_t mat_id, - const size_t elem_gid); - - KOKKOS_FUNCTION - ~UserEOSModel(); - - KOKKOS_FUNCTION - int calc_sound_speed( - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const size_t elem_gid, - const size_t mat_id, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, - const DCArrayKokkos &elem_user_output_vars, - const DViewCArrayKokkos &elem_sspd, - const double den, - const double sie) override; - - KOKKOS_FUNCTION - int calc_pressure( - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const size_t elem_gid, - const size_t mat_id, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, - const DCArrayKokkos &elem_user_output_vars, - const DViewCArrayKokkos &elem_sspd, - const double den, - const double sie) override; - -private: - double sound_speed; -}; diff --git a/src/EVPFFT/src/Fierro-EVPFFT-Link/UserStrengthModel.cpp b/src/EVPFFT/src/Fierro-EVPFFT-Link/UserStrengthModel.cpp deleted file mode 100644 index a78594e00..000000000 --- a/src/EVPFFT/src/Fierro-EVPFFT-Link/UserStrengthModel.cpp +++ /dev/null @@ -1,99 +0,0 @@ -#include "UserStrengthModel.h" -#include -#include "evpfft.h" - -MPI_Comm evpfft_mpi_comm = MPI_COMM_NULL; - -KOKKOS_FUNCTION -UserStrengthModel::UserStrengthModel( - const DCArrayKokkos &material, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, - const DCArrayKokkos &elem_user_output_vars, - const size_t mat_id, - const size_t elem_gid) -{ - // First, lets create a new communicator with each rank having its own communicator containing only itself. - if (evpfft_mpi_comm == MPI_COMM_NULL) { - int global_rank; - MPI_Comm_rank(MPI_COMM_WORLD, &global_rank); - MPI_Comm_split(MPI_COMM_WORLD, global_rank, global_rank, &evpfft_mpi_comm); - } - //... - - // Input files for for multiple materials should be names as evpfft1.in, evpfft2.in, etc. - std::string filename = "evpfft" + std::to_string(mat_id+1) + ".in"; - - real_t stress_scale = 1.0; // 1.0e-5; // used to convert MPa to MegaBar - real_t time_scale = 1.0; // 1.0e+6; // used to convert second to microsecond - - CommandLineArgs cmd; - cmd.input_filename = filename; //"evpfft.in"; - cmd.micro_filetype = 0; - cmd.check_cmd_args(); - - evpfft_ptr = new EVPFFT(evpfft_mpi_comm, cmd,stress_scale,time_scale); -} - -KOKKOS_FUNCTION -UserStrengthModel::~UserStrengthModel() -{ - delete evpfft_ptr; -} - -KOKKOS_FUNCTION -int UserStrengthModel::calc_stress( - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const size_t elem_gid, - const size_t mat_id, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, - const DCArrayKokkos &elem_user_output_vars, - const DViewCArrayKokkos &elem_sspd, - const double den, - const double sie, - const ViewCArrayKokkos &vel_grad, - const ViewCArrayKokkos &elem_node_gids, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const double vol, - const double dt, - const double rk_alpha, - const size_t cycle, - const size_t rk_level) -{ - real_t dt_rk = dt; // since using rk_num_stages = 1 - - // Note EVPFFT uses F-layout while Fierro uses C-layout - FArray Fvel_grad(3,3); - FArray Fstress(3,3); - // Transpose vel_grad - for (int i = 0; i < 3; i++) { - for (int j = 0; j < 3; j++) { - Fvel_grad(i,j) = vel_grad(i,j); - Fstress(i,j) = elem_stress.host(rk_level,elem_gid,i,j); - } - } - - double udotAccTh = global_vars.host(mat_id,0); // Linear Aprox. Threshold - evpfft_ptr->solve(Fvel_grad.pointer(), Fstress.pointer(), dt_rk, cycle, elem_gid, udotAccTh); - - // Transpose stress. Not needed, stress is symmetric. But why not. - for (int i = 0; i < 3; i++) { - for (int j = 0; j < 3; j++) { - elem_stress.host(rk_level,elem_gid,i,j) = Fstress(i,j); - } - } - - // write into elem_state_vars for output - // evm,evmp,dvm,dvmp,svm - elem_user_output_vars.host(elem_gid,0) = evpfft_ptr->evm; - elem_user_output_vars.host(elem_gid,1) = evpfft_ptr->evmp; - elem_user_output_vars.host(elem_gid,2) = evpfft_ptr->dvm; - elem_user_output_vars.host(elem_gid,3) = evpfft_ptr->dvmp; - elem_user_output_vars.host(elem_gid,4) = evpfft_ptr->svm; - - return 0; -} - diff --git a/src/EVPFFT/src/Fierro-EVPFFT-Link/UserStrengthModel.h b/src/EVPFFT/src/Fierro-EVPFFT-Link/UserStrengthModel.h deleted file mode 100644 index 72b765ae1..000000000 --- a/src/EVPFFT/src/Fierro-EVPFFT-Link/UserStrengthModel.h +++ /dev/null @@ -1,46 +0,0 @@ -#pragma once - -#include "material_models.h" - -class EVPFFT; - -/* UserStrengthModel */ -class UserStrengthModel : public StrengthParent { -public: - KOKKOS_FUNCTION - UserStrengthModel( - const DCArrayKokkos &material, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, - const DCArrayKokkos &elem_user_output_vars, - const size_t mat_id, - const size_t elem_gid); - - KOKKOS_FUNCTION - ~UserStrengthModel(); - - KOKKOS_FUNCTION - int calc_stress( - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const size_t elem_gid, - const size_t mat_id, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, - const DCArrayKokkos &elem_user_output_vars, - const DViewCArrayKokkos &elem_sspd, - const double den, - const double sie, - const ViewCArrayKokkos &vel_grad, - const ViewCArrayKokkos &elem_node_gids, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const double vol, - const double dt, - const double rk_alpha, - const size_t cycle, - const size_t rk_level) override; - -public: - EVPFFT *evpfft_ptr; -}; diff --git a/src/EVPFFT/src/Fierro-EVPFFT-Link/init_state_vars.cpp b/src/EVPFFT/src/Fierro-EVPFFT-Link/init_state_vars.cpp deleted file mode 100644 index f7cde4d41..000000000 --- a/src/EVPFFT/src/Fierro-EVPFFT-Link/init_state_vars.cpp +++ /dev/null @@ -1,26 +0,0 @@ -#include "Simulation_Parameters/Material.h" -#include "matar.h" -using namespace mtr; - - -void init_state_vars( - const DCArrayKokkos &material, - const DViewCArrayKokkos &elem_mat_id, - DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, - const DCArrayKokkos &elem_user_output_vars, - const size_t num_elems) -{ - - // initialize to zero - for (size_t elem_gid = 0; elem_gid +#include "evpfft.h" + + +MPI_Comm evpfft_mpi_comm = MPI_COMM_NULL; +// to hold all evpfft in each element +std::vector> elem_evpfft; + +namespace FierroLSEVPFFTLink +{ + void init_strength_state_vars( + const DCArrayKokkos &material, + const DViewCArrayKokkos &elem_mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const size_t num_elems) + { + // First, lets create a new communicator with each rank having its own communicator containing only itself. + if (evpfft_mpi_comm == MPI_COMM_NULL) { + int global_rank; + MPI_Comm_rank(MPI_COMM_WORLD, &global_rank); + MPI_Comm_split(MPI_COMM_WORLD, global_rank, global_rank, &evpfft_mpi_comm); + } + //... + + // assign correct size to elem_evpfft, all element are nullptr during initialization + elem_evpfft = std::vector> (num_elems, nullptr); + + for (size_t elem_gid = 0; elem_gid < num_elems; elem_gid++) { + + size_t mat_id = elem_mat_id.host(elem_gid); + + // only fill the elem_evpfft of elements that use this model + if (material.host(mat_id).strength_model == STRENGTH_MODEL::ls_evpfft) { + + // evpfft only runs on host so check to see + if (material.host(mat_id).strength_run_location == RUN_LOCATION::device) { + throw std::runtime_error("EVPFFT only runs on Host"); + } + + // Input files for for multiple materials should be names as evpfft1.in, evpfft2.in, etc. + std::string filename = "evpfft" + std::to_string(mat_id+1) + ".in"; + + real_t stress_scale = 1.0; // 1.0e-5; // used to convert MPa to MegaBar + real_t time_scale = 1.0; // 1.0e+6; // used to convert second to microsecond + + CommandLineArgs cmd; + cmd.input_filename = filename; //"evpfft.in"; + cmd.micro_filetype = 0; + cmd.check_cmd_args(); + + // create EVPFFT model in element that used evpfft + elem_evpfft[elem_gid] = std::make_shared(evpfft_mpi_comm, + cmd, + stress_scale, + time_scale); + + } // end if (material.host(mat_id).strength_model... + + } // end for (size_t elem_gid = 0... + + return; + } + + void calc_stress ( + const DViewCArrayKokkos &elem_pres, + const DViewCArrayKokkos &elem_stress, + const size_t elem_gid, + const size_t mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const DViewCArrayKokkos &elem_sspd, + const double den, + const double sie, + const ViewCArrayKokkos &vel_grad, + const ViewCArrayKokkos &elem_node_gids, + const DViewCArrayKokkos &node_coords, + const DViewCArrayKokkos &node_vel, + const double vol, + const double dt, + const double rk_alpha, + const size_t cycle, + const size_t rk_level, + const double time) + { + if (elem_evpfft[elem_gid] == nullptr) + { + throw std::runtime_error("LSEVPFFT not initialized in this element"); + } + + real_t dt_rk = dt; // since using rk_num_stages = 1 + + // Note EVPFFT uses F-layout while Fierro uses C-layout + FArray Fvel_grad(3,3); + FArray Fstress(3,3); + // Transpose vel_grad + for (int i = 0; i < 3; i++) { + for (int j = 0; j < 3; j++) { + Fvel_grad(i,j) = vel_grad(i,j); + Fstress(i,j) = elem_stress.host(rk_level,elem_gid,i,j); + } + } + + double udotAccTh = strength_global_vars.host(mat_id,0); // Linear Aprox. Threshold + elem_evpfft[elem_gid]->solve(Fvel_grad.pointer(), Fstress.pointer(), dt_rk, cycle, elem_gid, udotAccTh); + + // Transpose stress. Not needed, stress is symmetric. But why not. + for (int i = 0; i < 3; i++) { + for (int j = 0; j < 3; j++) { + elem_stress.host(rk_level,elem_gid,i,j) = Fstress(i,j); + } + } + + // write into elem_state_vars for output + // evm,evmp,dvm,dvmp,svm + elem_user_output_vars.host(elem_gid,0) = elem_evpfft[elem_gid]->evm; + elem_user_output_vars.host(elem_gid,1) = elem_evpfft[elem_gid]->evmp; + elem_user_output_vars.host(elem_gid,2) = elem_evpfft[elem_gid]->dvm; + elem_user_output_vars.host(elem_gid,3) = elem_evpfft[elem_gid]->dvmp; + elem_user_output_vars.host(elem_gid,4) = elem_evpfft[elem_gid]->svm; + + return; + } + + void destroy( + const DCArrayKokkos &material, + const DViewCArrayKokkos &elem_mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const size_t num_elems) + { + return; + } + +} // end namespace FierroLSEVPFFTLink + diff --git a/src/LS-EVPFFT/src/Fierro-LS-EVPFFT-Link/FierroLSEVPFFTLink.h b/src/LS-EVPFFT/src/Fierro-LS-EVPFFT-Link/FierroLSEVPFFTLink.h new file mode 100644 index 000000000..4f4d414bb --- /dev/null +++ b/src/LS-EVPFFT/src/Fierro-LS-EVPFFT-Link/FierroLSEVPFFTLink.h @@ -0,0 +1,53 @@ +#pragma once + +#include "Simulation_Parameters/Material.h" +#include "matar.h" +using namespace mtr; + +namespace FierroLSEVPFFTLink +{ + void init_strength_state_vars( + const DCArrayKokkos &material, + const DViewCArrayKokkos &elem_mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const size_t num_elems); + + void calc_stress ( + const DViewCArrayKokkos &elem_pres, + const DViewCArrayKokkos &elem_stress, + const size_t elem_gid, + const size_t mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const DViewCArrayKokkos &elem_sspd, + const double den, + const double sie, + const ViewCArrayKokkos &vel_grad, + const ViewCArrayKokkos &elem_node_gids, + const DViewCArrayKokkos &node_coords, + const DViewCArrayKokkos &node_vel, + const double vol, + const double dt, + const double rk_alpha, + const size_t cycle, + const size_t rk_level, + const double time); + + void destroy( + const DCArrayKokkos &material, + const DViewCArrayKokkos &elem_mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const size_t num_elems); + +} // end namespace FierroLSEVPFFTLink diff --git a/src/LS-EVPFFT/src/Fierro-LS-EVPFFT-Link/UserEOSModel.cpp b/src/LS-EVPFFT/src/Fierro-LS-EVPFFT-Link/UserEOSModel.cpp deleted file mode 100644 index 2d1aaa82f..000000000 --- a/src/LS-EVPFFT/src/Fierro-LS-EVPFFT-Link/UserEOSModel.cpp +++ /dev/null @@ -1,52 +0,0 @@ -#include "UserEOSModel.h" - -KOKKOS_FUNCTION -UserEOSModel::UserEOSModel( - const DCArrayKokkos &material, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, - const DCArrayKokkos &elem_user_output_vars, - const size_t mat_id, - const size_t elem_gid) -{ - sound_speed = global_vars(mat_id,1); -} - -KOKKOS_FUNCTION -UserEOSModel::~UserEOSModel() -{ -} - -KOKKOS_FUNCTION -int UserEOSModel::calc_sound_speed( - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const size_t elem_gid, - const size_t mat_id, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, - const DCArrayKokkos &elem_user_output_vars, - const DViewCArrayKokkos &elem_sspd, - const double den, - const double sie) -{ - elem_sspd(elem_gid) = sound_speed; // sound speed - return 0; -} - -KOKKOS_FUNCTION -int UserEOSModel::calc_pressure( - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const size_t elem_gid, - const size_t mat_id, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, - const DCArrayKokkos &elem_user_output_vars, - const DViewCArrayKokkos &elem_sspd, - const double den, - const double sie) -{ - elem_pres(elem_gid) = 0.0; // pressure - return 0; -} diff --git a/src/LS-EVPFFT/src/Fierro-LS-EVPFFT-Link/UserEOSModel.h b/src/LS-EVPFFT/src/Fierro-LS-EVPFFT-Link/UserEOSModel.h deleted file mode 100644 index 301b790b2..000000000 --- a/src/LS-EVPFFT/src/Fierro-LS-EVPFFT-Link/UserEOSModel.h +++ /dev/null @@ -1,50 +0,0 @@ -#pragma once - -#include "material_models.h" - - -/* UserEOSModel */ -class UserEOSModel : public EOSParent { -public: - - KOKKOS_FUNCTION - UserEOSModel( - const DCArrayKokkos &material, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, - const DCArrayKokkos &elem_user_output_vars, - const size_t mat_id, - const size_t elem_gid); - - KOKKOS_FUNCTION - ~UserEOSModel(); - - KOKKOS_FUNCTION - int calc_sound_speed( - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const size_t elem_gid, - const size_t mat_id, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, - const DCArrayKokkos &elem_user_output_vars, - const DViewCArrayKokkos &elem_sspd, - const double den, - const double sie) override; - - KOKKOS_FUNCTION - int calc_pressure( - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const size_t elem_gid, - const size_t mat_id, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, - const DCArrayKokkos &elem_user_output_vars, - const DViewCArrayKokkos &elem_sspd, - const double den, - const double sie) override; - -private: - double sound_speed; -}; diff --git a/src/LS-EVPFFT/src/Fierro-LS-EVPFFT-Link/UserStrengthModel.cpp b/src/LS-EVPFFT/src/Fierro-LS-EVPFFT-Link/UserStrengthModel.cpp deleted file mode 100644 index a78594e00..000000000 --- a/src/LS-EVPFFT/src/Fierro-LS-EVPFFT-Link/UserStrengthModel.cpp +++ /dev/null @@ -1,99 +0,0 @@ -#include "UserStrengthModel.h" -#include -#include "evpfft.h" - -MPI_Comm evpfft_mpi_comm = MPI_COMM_NULL; - -KOKKOS_FUNCTION -UserStrengthModel::UserStrengthModel( - const DCArrayKokkos &material, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, - const DCArrayKokkos &elem_user_output_vars, - const size_t mat_id, - const size_t elem_gid) -{ - // First, lets create a new communicator with each rank having its own communicator containing only itself. - if (evpfft_mpi_comm == MPI_COMM_NULL) { - int global_rank; - MPI_Comm_rank(MPI_COMM_WORLD, &global_rank); - MPI_Comm_split(MPI_COMM_WORLD, global_rank, global_rank, &evpfft_mpi_comm); - } - //... - - // Input files for for multiple materials should be names as evpfft1.in, evpfft2.in, etc. - std::string filename = "evpfft" + std::to_string(mat_id+1) + ".in"; - - real_t stress_scale = 1.0; // 1.0e-5; // used to convert MPa to MegaBar - real_t time_scale = 1.0; // 1.0e+6; // used to convert second to microsecond - - CommandLineArgs cmd; - cmd.input_filename = filename; //"evpfft.in"; - cmd.micro_filetype = 0; - cmd.check_cmd_args(); - - evpfft_ptr = new EVPFFT(evpfft_mpi_comm, cmd,stress_scale,time_scale); -} - -KOKKOS_FUNCTION -UserStrengthModel::~UserStrengthModel() -{ - delete evpfft_ptr; -} - -KOKKOS_FUNCTION -int UserStrengthModel::calc_stress( - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const size_t elem_gid, - const size_t mat_id, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, - const DCArrayKokkos &elem_user_output_vars, - const DViewCArrayKokkos &elem_sspd, - const double den, - const double sie, - const ViewCArrayKokkos &vel_grad, - const ViewCArrayKokkos &elem_node_gids, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const double vol, - const double dt, - const double rk_alpha, - const size_t cycle, - const size_t rk_level) -{ - real_t dt_rk = dt; // since using rk_num_stages = 1 - - // Note EVPFFT uses F-layout while Fierro uses C-layout - FArray Fvel_grad(3,3); - FArray Fstress(3,3); - // Transpose vel_grad - for (int i = 0; i < 3; i++) { - for (int j = 0; j < 3; j++) { - Fvel_grad(i,j) = vel_grad(i,j); - Fstress(i,j) = elem_stress.host(rk_level,elem_gid,i,j); - } - } - - double udotAccTh = global_vars.host(mat_id,0); // Linear Aprox. Threshold - evpfft_ptr->solve(Fvel_grad.pointer(), Fstress.pointer(), dt_rk, cycle, elem_gid, udotAccTh); - - // Transpose stress. Not needed, stress is symmetric. But why not. - for (int i = 0; i < 3; i++) { - for (int j = 0; j < 3; j++) { - elem_stress.host(rk_level,elem_gid,i,j) = Fstress(i,j); - } - } - - // write into elem_state_vars for output - // evm,evmp,dvm,dvmp,svm - elem_user_output_vars.host(elem_gid,0) = evpfft_ptr->evm; - elem_user_output_vars.host(elem_gid,1) = evpfft_ptr->evmp; - elem_user_output_vars.host(elem_gid,2) = evpfft_ptr->dvm; - elem_user_output_vars.host(elem_gid,3) = evpfft_ptr->dvmp; - elem_user_output_vars.host(elem_gid,4) = evpfft_ptr->svm; - - return 0; -} - diff --git a/src/LS-EVPFFT/src/Fierro-LS-EVPFFT-Link/UserStrengthModel.h b/src/LS-EVPFFT/src/Fierro-LS-EVPFFT-Link/UserStrengthModel.h deleted file mode 100644 index 72b765ae1..000000000 --- a/src/LS-EVPFFT/src/Fierro-LS-EVPFFT-Link/UserStrengthModel.h +++ /dev/null @@ -1,46 +0,0 @@ -#pragma once - -#include "material_models.h" - -class EVPFFT; - -/* UserStrengthModel */ -class UserStrengthModel : public StrengthParent { -public: - KOKKOS_FUNCTION - UserStrengthModel( - const DCArrayKokkos &material, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, - const DCArrayKokkos &elem_user_output_vars, - const size_t mat_id, - const size_t elem_gid); - - KOKKOS_FUNCTION - ~UserStrengthModel(); - - KOKKOS_FUNCTION - int calc_stress( - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const size_t elem_gid, - const size_t mat_id, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, - const DCArrayKokkos &elem_user_output_vars, - const DViewCArrayKokkos &elem_sspd, - const double den, - const double sie, - const ViewCArrayKokkos &vel_grad, - const ViewCArrayKokkos &elem_node_gids, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const double vol, - const double dt, - const double rk_alpha, - const size_t cycle, - const size_t rk_level) override; - -public: - EVPFFT *evpfft_ptr; -}; diff --git a/src/LS-EVPFFT/src/Fierro-LS-EVPFFT-Link/init_state_vars.cpp b/src/LS-EVPFFT/src/Fierro-LS-EVPFFT-Link/init_state_vars.cpp deleted file mode 100644 index f7cde4d41..000000000 --- a/src/LS-EVPFFT/src/Fierro-LS-EVPFFT-Link/init_state_vars.cpp +++ /dev/null @@ -1,26 +0,0 @@ -#include "Simulation_Parameters/Material.h" -#include "matar.h" -using namespace mtr; - - -void init_state_vars( - const DCArrayKokkos &material, - const DViewCArrayKokkos &elem_mat_id, - DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, - const DCArrayKokkos &elem_user_output_vars, - const size_t num_elems) -{ - - // initialize to zero - for (size_t elem_gid = 0; elem_gid &material, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, - const DCArrayKokkos &elem_user_output_vars, - const size_t mat_id, - const size_t elem_gid) -{ - gamma_ = global_vars(mat_id,0); - csmin_ = global_vars(mat_id,1); - c_v_ = global_vars(mat_id,2); -} - -KOKKOS_FUNCTION -IdealGasEOSModel::~IdealGasEOSModel() -{ -} - -KOKKOS_FUNCTION -int IdealGasEOSModel::calc_sound_speed( - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const size_t elem_gid, - const size_t mat_id, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, - const DCArrayKokkos &elem_user_output_vars, - const DViewCArrayKokkos &elem_sspd, - const double den, - const double sie) -{ - // sound speed - elem_sspd(elem_gid) = sqrt(gamma_*(gamma_ - 1.0)*sie); - - // ensure soundspeed is great than min specified - if (elem_sspd(elem_gid) < csmin_){ - elem_sspd(elem_gid) = csmin_; - } // end if - - return 0; -} - -KOKKOS_FUNCTION -double IdealGasEOSModel::calc_sound_speed_gradient_internal_energy( - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const size_t elem_gid, - const size_t mat_id, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, - const DCArrayKokkos &elem_user_output_vars, - const DViewCArrayKokkos &elem_sspd, - const double den, - const double sie){ - // sound speed gradient - //compute sound speed to decide on avoiding sinfularity near sie=0 - if(sqrt(gamma_*(gamma_ - 1.0)*sie) < csmin_) - return 0; - else - return 0.5*gamma_*(gamma_ - 1.0)/sqrt(gamma_*(gamma_ - 1.0)*sie); -} - -KOKKOS_FUNCTION -int IdealGasEOSModel::calc_pressure( - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const size_t elem_gid, - const size_t mat_id, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, - const DCArrayKokkos &elem_user_output_vars, - const DViewCArrayKokkos &elem_sspd, - const double den, - const double sie) -{ - // pressure - elem_pres(elem_gid) = (gamma_ - 1.0)*sie*den; - - return 0; -} - -KOKKOS_FUNCTION -double IdealGasEOSModel::calc_pressure_gradient_density( - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const size_t elem_gid, - const size_t mat_id, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, - const DCArrayKokkos &elem_user_output_vars, - const DViewCArrayKokkos &elem_sspd, - const double den, - const double sie) -{ - // pressure - return (gamma_ - 1.0)*sie; -} - -KOKKOS_FUNCTION -double IdealGasEOSModel::calc_pressure_gradient_internal_energy( - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const size_t elem_gid, - const size_t mat_id, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, - const DCArrayKokkos &elem_user_output_vars, - const DViewCArrayKokkos &elem_sspd, - const double den, - const double sie) -{ - // pressure - return (gamma_ - 1.0)*den; -} - - diff --git a/src/Parallel-Solvers/Material-Models/Ideal-Gas-Models/IdealGasEOSModel.h b/src/Parallel-Solvers/Material-Models/Ideal-Gas-Models/IdealGasEOSModel.h deleted file mode 100644 index 3e2ade35b..000000000 --- a/src/Parallel-Solvers/Material-Models/Ideal-Gas-Models/IdealGasEOSModel.h +++ /dev/null @@ -1,92 +0,0 @@ -#pragma once - -#include "material_models.h" - - -/* IdealGasEOSModel */ -class IdealGasEOSModel : public EOSParent { -public: - - KOKKOS_FUNCTION - IdealGasEOSModel( - const DCArrayKokkos &material, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, - const DCArrayKokkos &elem_user_output_vars, - const size_t mat_id, - const size_t elem_gid); - - KOKKOS_FUNCTION - ~IdealGasEOSModel(); - - KOKKOS_FUNCTION - int calc_sound_speed( - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const size_t elem_gid, - const size_t mat_id, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, - const DCArrayKokkos &elem_user_output_vars, - const DViewCArrayKokkos &elem_sspd, - const double den, - const double sie) override; - - KOKKOS_FUNCTION - double calc_sound_speed_gradient_internal_energy( - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const size_t elem_gid, - const size_t mat_id, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, - const DCArrayKokkos &elem_user_output_vars, - const DViewCArrayKokkos &elem_sspd, - const double den, - const double sie) override; - - KOKKOS_FUNCTION - int calc_pressure( - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const size_t elem_gid, - const size_t mat_id, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, - const DCArrayKokkos &elem_user_output_vars, - const DViewCArrayKokkos &elem_sspd, - const double den, - const double sie) override; - - KOKKOS_FUNCTION - double calc_pressure_gradient_density( - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const size_t elem_gid, - const size_t mat_id, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, - const DCArrayKokkos &elem_user_output_vars, - const DViewCArrayKokkos &elem_sspd, - const double den, - const double sie) override; - - KOKKOS_FUNCTION - double calc_pressure_gradient_internal_energy( - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const size_t elem_gid, - const size_t mat_id, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, - const DCArrayKokkos &elem_user_output_vars, - const DViewCArrayKokkos &elem_sspd, - const double den, - const double sie) override; - -private: - double gamma_; - double csmin_; - double c_v_; -}; - diff --git a/src/Parallel-Solvers/Material-Models/Ideal-Gas-Models/IdealGasStrengthModel.cpp b/src/Parallel-Solvers/Material-Models/Ideal-Gas-Models/IdealGasStrengthModel.cpp deleted file mode 100644 index 5c061f70f..000000000 --- a/src/Parallel-Solvers/Material-Models/Ideal-Gas-Models/IdealGasStrengthModel.cpp +++ /dev/null @@ -1,49 +0,0 @@ -#include "IdealGasStrengthModel.h" - -KOKKOS_FUNCTION -IdealGasStrengthModel::IdealGasStrengthModel( - const DCArrayKokkos &material, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, - const DCArrayKokkos &elem_user_output_vars, - const size_t mat_id, - const size_t elem_gid) -{ -} - -KOKKOS_FUNCTION -IdealGasStrengthModel::~IdealGasStrengthModel() -{ -} - -KOKKOS_FUNCTION -int IdealGasStrengthModel::calc_stress( - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const size_t elem_gid, - const size_t mat_id, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, - const DCArrayKokkos &elem_user_output_vars, - const DViewCArrayKokkos &elem_sspd, - const double den, - const double sie, - const ViewCArrayKokkos &vel_grad, - const ViewCArrayKokkos &elem_node_gids, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const double vol, - const double dt, - const double rk_alpha, - const size_t cycle, - const size_t rk_level) -{ - - for (int i = 0; i < 3; i++) { - for (int j = 0; j < 3; j++) { - elem_stress(rk_level,elem_gid,i,j) = 0.0; - } - } - - return 0; -} diff --git a/src/Parallel-Solvers/Material-Models/Ideal-Gas-Models/IdealGasStrengthModel.h b/src/Parallel-Solvers/Material-Models/Ideal-Gas-Models/IdealGasStrengthModel.h deleted file mode 100644 index fa1c9b1dd..000000000 --- a/src/Parallel-Solvers/Material-Models/Ideal-Gas-Models/IdealGasStrengthModel.h +++ /dev/null @@ -1,43 +0,0 @@ -#pragma once - -#include "material_models.h" - - -/* IdealGasStrengthModel */ -class IdealGasStrengthModel : public StrengthParent { -public: - KOKKOS_FUNCTION - IdealGasStrengthModel( - const DCArrayKokkos &material, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, - const DCArrayKokkos &elem_user_output_vars, - const size_t mat_id, - const size_t elem_gid); - - KOKKOS_FUNCTION - ~IdealGasStrengthModel(); - - KOKKOS_FUNCTION - int calc_stress( - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const size_t elem_gid, - const size_t mat_id, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, - const DCArrayKokkos &elem_user_output_vars, - const DViewCArrayKokkos &elem_sspd, - const double den, - const double sie, - const ViewCArrayKokkos &vel_grad, - const ViewCArrayKokkos &elem_node_gids, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const double vol, - const double dt, - const double rk_alpha, - const size_t cycle, - const size_t rk_level) override; -}; - diff --git a/src/Parallel-Solvers/Material-Models/Ideal-Gas-Models/init_state_vars.cpp b/src/Parallel-Solvers/Material-Models/Ideal-Gas-Models/init_state_vars.cpp deleted file mode 100644 index f7cde4d41..000000000 --- a/src/Parallel-Solvers/Material-Models/Ideal-Gas-Models/init_state_vars.cpp +++ /dev/null @@ -1,26 +0,0 @@ -#include "Simulation_Parameters/Material.h" -#include "matar.h" -using namespace mtr; - - -void init_state_vars( - const DCArrayKokkos &material, - const DViewCArrayKokkos &elem_mat_id, - DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, - const DCArrayKokkos &elem_user_output_vars, - const size_t num_elems) -{ - - // initialize to zero - for (size_t elem_gid = 0; elem_gid &material, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, - const DCArrayKokkos &elem_user_output_vars, - const size_t mat_id, - const size_t elem_gid) -{ -} - -KOKKOS_FUNCTION -UserEOSModel::~UserEOSModel() -{ -} - -KOKKOS_FUNCTION -int UserEOSModel::calc_sound_speed( - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const size_t elem_gid, - const size_t mat_id, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, - const DCArrayKokkos &elem_user_output_vars, - const DViewCArrayKokkos &elem_sspd, - const double den, - const double sie) -{ - return 0; -} - -KOKKOS_FUNCTION -int UserEOSModel::calc_pressure( - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const size_t elem_gid, - const size_t mat_id, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, - const DCArrayKokkos &elem_user_output_vars, - const DViewCArrayKokkos &elem_sspd, - const double den, - const double sie) -{ - return 0; -} - - diff --git a/src/Parallel-Solvers/Material-Models/User-Material-Models/UserEOSModel.h b/src/Parallel-Solvers/Material-Models/User-Material-Models/UserEOSModel.h deleted file mode 100644 index 4e6691af7..000000000 --- a/src/Parallel-Solvers/Material-Models/User-Material-Models/UserEOSModel.h +++ /dev/null @@ -1,48 +0,0 @@ -#pragma once - -#include "material_models.h" - - -/* UserEOSModel */ -class UserEOSModel : public EOSParent { -public: - - KOKKOS_FUNCTION - UserEOSModel( - const DCArrayKokkos &material, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, - const DCArrayKokkos &elem_user_output_vars, - const size_t mat_id, - const size_t elem_gid); - - KOKKOS_FUNCTION - ~UserEOSModel(); - - KOKKOS_FUNCTION - int calc_sound_speed( - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const size_t elem_gid, - const size_t mat_id, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, - const DCArrayKokkos &elem_user_output_vars, - const DViewCArrayKokkos &elem_sspd, - const double den, - const double sie) override; - - KOKKOS_FUNCTION - int calc_pressure( - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const size_t elem_gid, - const size_t mat_id, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, - const DCArrayKokkos &elem_user_output_vars, - const DViewCArrayKokkos &elem_sspd, - const double den, - const double sie) override; - -}; diff --git a/src/Parallel-Solvers/Material-Models/User-Material-Models/UserStrengthModel.cpp b/src/Parallel-Solvers/Material-Models/User-Material-Models/UserStrengthModel.cpp deleted file mode 100644 index 1bf1ee2bc..000000000 --- a/src/Parallel-Solvers/Material-Models/User-Material-Models/UserStrengthModel.cpp +++ /dev/null @@ -1,43 +0,0 @@ -#include "UserStrengthModel.h" - -KOKKOS_FUNCTION -UserStrengthModel::UserStrengthModel( - const DCArrayKokkos &material, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, - const DCArrayKokkos &elem_user_output_vars, - const size_t mat_id, - const size_t elem_gid) -{ -} - -KOKKOS_FUNCTION -UserStrengthModel::~UserStrengthModel() -{ -} - -KOKKOS_FUNCTION -int UserStrengthModel::calc_stress( - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const size_t elem_gid, - const size_t mat_id, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, - const DCArrayKokkos &elem_user_output_vars, - const DViewCArrayKokkos &elem_sspd, - const double den, - const double sie, - const ViewCArrayKokkos &vel_grad, - const ViewCArrayKokkos &elem_node_gids, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const double vol, - const double dt, - const double rk_alpha, - const size_t cycle, - const size_t rk_level) -{ - return 0; -} - diff --git a/src/Parallel-Solvers/Material-Models/User-Material-Models/UserStrengthModel.h b/src/Parallel-Solvers/Material-Models/User-Material-Models/UserStrengthModel.h deleted file mode 100644 index 34c99ee8e..000000000 --- a/src/Parallel-Solvers/Material-Models/User-Material-Models/UserStrengthModel.h +++ /dev/null @@ -1,42 +0,0 @@ -#pragma once - -#include "material_models.h" - - -/* UserStrengthModel */ -class UserStrengthModel : public StrengthParent { -public: - KOKKOS_FUNCTION - UserStrengthModel( - const DCArrayKokkos &material, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, - const DCArrayKokkos &elem_user_output_vars, - const size_t mat_id, - const size_t elem_gid); - - KOKKOS_FUNCTION - ~UserStrengthModel(); - - KOKKOS_FUNCTION - int calc_stress( - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const size_t elem_gid, - const size_t mat_id, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, - const DCArrayKokkos &elem_user_output_vars, - const DViewCArrayKokkos &elem_sspd, - const double den, - const double sie, - const ViewCArrayKokkos &vel_grad, - const ViewCArrayKokkos &elem_node_gids, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const double vol, - const double dt, - const double rk_alpha, - const size_t cycle, - const size_t rk_level) override; -}; diff --git a/src/Parallel-Solvers/Material-Models/User-Material-Models/init_state_vars.cpp b/src/Parallel-Solvers/Material-Models/User-Material-Models/init_state_vars.cpp deleted file mode 100644 index f7cde4d41..000000000 --- a/src/Parallel-Solvers/Material-Models/User-Material-Models/init_state_vars.cpp +++ /dev/null @@ -1,26 +0,0 @@ -#include "Simulation_Parameters/Material.h" -#include "matar.h" -using namespace mtr; - - -void init_state_vars( - const DCArrayKokkos &material, - const DViewCArrayKokkos &elem_mat_id, - DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, - const DCArrayKokkos &elem_user_output_vars, - const size_t num_elems) -{ - - // initialize to zero - for (size_t elem_gid = 0; elem_gid &material, + const DViewCArrayKokkos &elem_mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const size_t num_elems) + { + /* + In this function, initialize the eos_state_vars according to the location this model needs it. + DO NOT eos_state_vars.update_host or eos_state_vars.update_device. + This is because different elements might have different models, run location, and + initialization location leading to overwriting of data. + */ + + FOR_ALL(elem_gid, 0, num_elems, { + size_t mat_id = elem_mat_id(elem_gid); + size_t num_eos_state_vars = material(mat_id).num_eos_state_vars; + + // only fill the state_vars of elements that use this model + if (material.host(mat_id).eos_model == EOS_MODEL::constant) { + for(size_t var=0; var &elem_pres, + const DViewCArrayKokkos &elem_stress, + const size_t elem_gid, + const size_t mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const DViewCArrayKokkos &elem_sspd, + const double den, + const double sie) + { + double sound_speed = eos_global_vars(mat_id,0); + double pressure = eos_global_vars(mat_id,1); + + // sound speed + elem_sspd(elem_gid) = sound_speed; + + return; + } + + KOKKOS_FUNCTION + double calc_sound_speed_gradient_density( + const DViewCArrayKokkos &elem_pres, + const DViewCArrayKokkos &elem_stress, + const size_t elem_gid, + const size_t mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const DViewCArrayKokkos &elem_sspd, + const double den, + const double sie) + { + double sound_speed = eos_global_vars(mat_id,0); + double pressure = eos_global_vars(mat_id,1); + + return 0.0; + } + + KOKKOS_FUNCTION + double calc_sound_speed_gradient_internal_energy( + const DViewCArrayKokkos &elem_pres, + const DViewCArrayKokkos &elem_stress, + const size_t elem_gid, + const size_t mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const DViewCArrayKokkos &elem_sspd, + const double den, + const double sie) + { + double sound_speed = eos_global_vars(mat_id,0); + double pressure = eos_global_vars(mat_id,1); + + return 0.0; + } + + KOKKOS_FUNCTION + void calc_pressure( + const DViewCArrayKokkos &elem_pres, + const DViewCArrayKokkos &elem_stress, + const size_t elem_gid, + const size_t mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const DViewCArrayKokkos &elem_sspd, + const double den, + const double sie) + { + double sound_speed = eos_global_vars(mat_id,0); + double pressure = eos_global_vars(mat_id,1); + + // pressure + elem_pres(elem_gid) = pressure; + + return; + } + + KOKKOS_FUNCTION + double calc_pressure_gradient_density( + const DViewCArrayKokkos &elem_pres, + const DViewCArrayKokkos &elem_stress, + const size_t elem_gid, + const size_t mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const DViewCArrayKokkos &elem_sspd, + const double den, + const double sie) + { + double sound_speed = eos_global_vars(mat_id,0); + double pressure = eos_global_vars(mat_id,1); + + return 0.0; + } + + KOKKOS_FUNCTION + double calc_pressure_gradient_internal_energy( + const DViewCArrayKokkos &elem_pres, + const DViewCArrayKokkos &elem_stress, + const size_t elem_gid, + const size_t mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const DViewCArrayKokkos &elem_sspd, + const double den, + const double sie) + { + double sound_speed = eos_global_vars(mat_id,0); + double pressure = eos_global_vars(mat_id,1); + + return 0.0; + } + + void destroy( + const DCArrayKokkos &material, + const DViewCArrayKokkos &elem_mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const size_t num_elems) + { + + } + +} // end namespace ConstantEOSModel \ No newline at end of file diff --git a/src/Parallel-Solvers/Material-Models/eos/Constant/ConstantEOSModel.h b/src/Parallel-Solvers/Material-Models/eos/Constant/ConstantEOSModel.h new file mode 100644 index 000000000..3ff5d3d3b --- /dev/null +++ b/src/Parallel-Solvers/Material-Models/eos/Constant/ConstantEOSModel.h @@ -0,0 +1,120 @@ +#pragma once + +#include "Simulation_Parameters/Material.h" +#include "matar.h" +using namespace mtr; + +namespace ConstantEOSModel +{ + void init_eos_state_vars( + const DCArrayKokkos &material, + const DViewCArrayKokkos &elem_mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const size_t num_elems); + + KOKKOS_FUNCTION + void calc_sound_speed( + const DViewCArrayKokkos &elem_pres, + const DViewCArrayKokkos &elem_stress, + const size_t elem_gid, + const size_t mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const DViewCArrayKokkos &elem_sspd, + const double den, + const double sie); + + KOKKOS_FUNCTION + double calc_sound_speed_gradient_density( + const DViewCArrayKokkos &elem_pres, + const DViewCArrayKokkos &elem_stress, + const size_t elem_gid, + const size_t mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const DViewCArrayKokkos &elem_sspd, + const double den, + const double sie); + + KOKKOS_FUNCTION + double calc_sound_speed_gradient_internal_energy( + const DViewCArrayKokkos &elem_pres, + const DViewCArrayKokkos &elem_stress, + const size_t elem_gid, + const size_t mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const DViewCArrayKokkos &elem_sspd, + const double den, + const double sie); + + KOKKOS_FUNCTION + void calc_pressure( + const DViewCArrayKokkos &elem_pres, + const DViewCArrayKokkos &elem_stress, + const size_t elem_gid, + const size_t mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const DViewCArrayKokkos &elem_sspd, + const double den, + const double sie); + + KOKKOS_FUNCTION + double calc_pressure_gradient_density( + const DViewCArrayKokkos &elem_pres, + const DViewCArrayKokkos &elem_stress, + const size_t elem_gid, + const size_t mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const DViewCArrayKokkos &elem_sspd, + const double den, + const double sie); + + KOKKOS_FUNCTION + double calc_pressure_gradient_internal_energy( + const DViewCArrayKokkos &elem_pres, + const DViewCArrayKokkos &elem_stress, + const size_t elem_gid, + const size_t mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const DViewCArrayKokkos &elem_sspd, + const double den, + const double sie); + + void destroy( + const DCArrayKokkos &material, + const DViewCArrayKokkos &elem_mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const size_t num_elems); + +} // end namespace ConstantEOSModel + diff --git a/src/Parallel-Solvers/Material-Models/eos/IdealGas/CMakeLists.txt b/src/Parallel-Solvers/Material-Models/eos/IdealGas/CMakeLists.txt new file mode 100644 index 000000000..7ac4e737b --- /dev/null +++ b/src/Parallel-Solvers/Material-Models/eos/IdealGas/CMakeLists.txt @@ -0,0 +1,4 @@ +set(SRC_Files IdealGasEOSModel.cpp) +add_library(IdealGasEOSModel OBJECT ${SRC_Files}) +target_include_directories(IdealGasEOSModel PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) +target_link_libraries(IdealGasEOSModel Elements) \ No newline at end of file diff --git a/src/Parallel-Solvers/Material-Models/eos/IdealGas/IdealGasEOSModel.cpp b/src/Parallel-Solvers/Material-Models/eos/IdealGas/IdealGasEOSModel.cpp new file mode 100644 index 000000000..fecdb6b5a --- /dev/null +++ b/src/Parallel-Solvers/Material-Models/eos/IdealGas/IdealGasEOSModel.cpp @@ -0,0 +1,206 @@ +#include "IdealGasEOSModel.h" + +namespace IdealGasEOSModel +{ + void init_eos_state_vars( + const DCArrayKokkos &material, + const DViewCArrayKokkos &elem_mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const size_t num_elems) + { + /* + In this function, initialize the eos_state_vars according to the location this model needs it. + DO NOT eos_state_vars.update_host or eos_state_vars.update_device. + This is because different elements might have different models, run location, and + initialization location leading to overwriting of data. + */ + + // Note: + // gamma = eos_global_vars(mat_id,0); + // csmin = eos_global_vars(mat_id,1); + // c_v = eos_global_vars(mat_id,2); + + FOR_ALL(elem_gid, 0, num_elems, { + size_t mat_id = elem_mat_id(elem_gid); + size_t num_eos_state_vars = material(mat_id).num_eos_state_vars; + + // only fill the state_vars of elements that use this model + if (material.host(mat_id).eos_model == EOS_MODEL::ideal_gas) { + for(size_t var=0; var &elem_pres, + const DViewCArrayKokkos &elem_stress, + const size_t elem_gid, + const size_t mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const DViewCArrayKokkos &elem_sspd, + const double den, + const double sie) + { + double gamma = eos_global_vars(mat_id,0); + double csmin = eos_global_vars(mat_id,1); + double c_v = eos_global_vars(mat_id,2); + + // sound speed + elem_sspd(elem_gid) = sqrt(gamma*(gamma - 1.0)*sie); + + // ensure soundspeed is great than min specified + if (elem_sspd(elem_gid) < csmin){ + elem_sspd(elem_gid) = csmin; + } // end if + + return; + } + + KOKKOS_FUNCTION + double calc_sound_speed_gradient_density( + const DViewCArrayKokkos &elem_pres, + const DViewCArrayKokkos &elem_stress, + const size_t elem_gid, + const size_t mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const DViewCArrayKokkos &elem_sspd, + const double den, + const double sie) + { + double gamma = eos_global_vars(mat_id,0); + double csmin = eos_global_vars(mat_id,1); + double c_v = eos_global_vars(mat_id,2); + + return 0.0; + } + + KOKKOS_FUNCTION + double calc_sound_speed_gradient_internal_energy( + const DViewCArrayKokkos &elem_pres, + const DViewCArrayKokkos &elem_stress, + const size_t elem_gid, + const size_t mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const DViewCArrayKokkos &elem_sspd, + const double den, + const double sie) + { + double gamma = eos_global_vars(mat_id,0); + double csmin = eos_global_vars(mat_id,1); + double c_v = eos_global_vars(mat_id,2); + + // sound speed gradient + //compute sound speed to decide on avoiding sinfularity near sie=0 + if(sqrt(gamma*(gamma - 1.0)*sie) < csmin) + return 0; + else + return 0.5*gamma*(gamma - 1.0)/sqrt(gamma*(gamma - 1.0)*sie); + } + + KOKKOS_FUNCTION + void calc_pressure( + const DViewCArrayKokkos &elem_pres, + const DViewCArrayKokkos &elem_stress, + const size_t elem_gid, + const size_t mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const DViewCArrayKokkos &elem_sspd, + const double den, + const double sie) + { + double gamma = eos_global_vars(mat_id,0); + double csmin = eos_global_vars(mat_id,1); + double c_v = eos_global_vars(mat_id,2); + + // pressure + elem_pres(elem_gid) = (gamma - 1.0)*sie*den; + + return; + } + + KOKKOS_FUNCTION + double calc_pressure_gradient_density( + const DViewCArrayKokkos &elem_pres, + const DViewCArrayKokkos &elem_stress, + const size_t elem_gid, + const size_t mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const DViewCArrayKokkos &elem_sspd, + const double den, + const double sie) + { + double gamma = eos_global_vars(mat_id,0); + double csmin = eos_global_vars(mat_id,1); + double c_v = eos_global_vars(mat_id,2); + + // pressure + return (gamma - 1.0)*sie; + } + + KOKKOS_FUNCTION + double calc_pressure_gradient_internal_energy( + const DViewCArrayKokkos &elem_pres, + const DViewCArrayKokkos &elem_stress, + const size_t elem_gid, + const size_t mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const DViewCArrayKokkos &elem_sspd, + const double den, + const double sie) + { + double gamma = eos_global_vars(mat_id,0); + double csmin = eos_global_vars(mat_id,1); + double c_v = eos_global_vars(mat_id,2); + + // pressure + return (gamma - 1.0)*den; + } + + void destroy( + const DCArrayKokkos &material, + const DViewCArrayKokkos &elem_mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const size_t num_elems) + { + + } + +} // end namespace IdealGasEOSModel \ No newline at end of file diff --git a/src/Parallel-Solvers/Material-Models/eos/IdealGas/IdealGasEOSModel.h b/src/Parallel-Solvers/Material-Models/eos/IdealGas/IdealGasEOSModel.h new file mode 100644 index 000000000..34cdf5d5e --- /dev/null +++ b/src/Parallel-Solvers/Material-Models/eos/IdealGas/IdealGasEOSModel.h @@ -0,0 +1,120 @@ +#pragma once + +#include "Simulation_Parameters/Material.h" +#include "matar.h" +using namespace mtr; + +namespace IdealGasEOSModel +{ + void init_eos_state_vars( + const DCArrayKokkos &material, + const DViewCArrayKokkos &elem_mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const size_t num_elems); + + KOKKOS_FUNCTION + void calc_sound_speed( + const DViewCArrayKokkos &elem_pres, + const DViewCArrayKokkos &elem_stress, + const size_t elem_gid, + const size_t mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const DViewCArrayKokkos &elem_sspd, + const double den, + const double sie); + + KOKKOS_FUNCTION + double calc_sound_speed_gradient_density( + const DViewCArrayKokkos &elem_pres, + const DViewCArrayKokkos &elem_stress, + const size_t elem_gid, + const size_t mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const DViewCArrayKokkos &elem_sspd, + const double den, + const double sie); + + KOKKOS_FUNCTION + double calc_sound_speed_gradient_internal_energy( + const DViewCArrayKokkos &elem_pres, + const DViewCArrayKokkos &elem_stress, + const size_t elem_gid, + const size_t mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const DViewCArrayKokkos &elem_sspd, + const double den, + const double sie); + + KOKKOS_FUNCTION + void calc_pressure( + const DViewCArrayKokkos &elem_pres, + const DViewCArrayKokkos &elem_stress, + const size_t elem_gid, + const size_t mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const DViewCArrayKokkos &elem_sspd, + const double den, + const double sie); + + KOKKOS_FUNCTION + double calc_pressure_gradient_density( + const DViewCArrayKokkos &elem_pres, + const DViewCArrayKokkos &elem_stress, + const size_t elem_gid, + const size_t mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const DViewCArrayKokkos &elem_sspd, + const double den, + const double sie); + + KOKKOS_FUNCTION + double calc_pressure_gradient_internal_energy( + const DViewCArrayKokkos &elem_pres, + const DViewCArrayKokkos &elem_stress, + const size_t elem_gid, + const size_t mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const DViewCArrayKokkos &elem_sspd, + const double den, + const double sie); + + void destroy( + const DCArrayKokkos &material, + const DViewCArrayKokkos &elem_mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const size_t num_elems); + +} // end namespace IdealGasEOS + diff --git a/src/Parallel-Solvers/Material-Models/eos/UserDefined/CMakeLists.txt b/src/Parallel-Solvers/Material-Models/eos/UserDefined/CMakeLists.txt new file mode 100644 index 000000000..7c8d1d0db --- /dev/null +++ b/src/Parallel-Solvers/Material-Models/eos/UserDefined/CMakeLists.txt @@ -0,0 +1,4 @@ +set(SRC_Files UserDefinedEOSModel.cpp) +add_library(UserDefinedEOSModel OBJECT ${SRC_Files}) +target_include_directories(UserDefinedEOSModel PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) +target_link_libraries(UserDefinedEOSModel Elements) \ No newline at end of file diff --git a/src/Parallel-Solvers/Material-Models/eos/UserDefined/UserDefinedEOSModel.cpp b/src/Parallel-Solvers/Material-Models/eos/UserDefined/UserDefinedEOSModel.cpp new file mode 100644 index 000000000..27005ddd3 --- /dev/null +++ b/src/Parallel-Solvers/Material-Models/eos/UserDefined/UserDefinedEOSModel.cpp @@ -0,0 +1,158 @@ +#include "UserDefinedEOSModel.h" + +namespace UserDefinedEOSModel +{ + void init_eos_state_vars( + const DCArrayKokkos &material, + const DViewCArrayKokkos &elem_mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const size_t num_elems) + { + /* + In this function, initialize the eos_state_vars according to the location this model needs it. + DO NOT eos_state_vars.update_host or eos_state_vars.update_device. + This is because different elements might have different models, run location, and + initialization location leading to overwriting of data. + */ + + FOR_ALL(elem_gid, 0, num_elems, { + size_t mat_id = elem_mat_id(elem_gid); + size_t num_eos_state_vars = material(mat_id).num_eos_state_vars; + + // only fill the state_vars of elements that use this model + if (material.host(mat_id).eos_model == EOS_MODEL::user_defined) { + for(size_t var=0; var &elem_pres, + const DViewCArrayKokkos &elem_stress, + const size_t elem_gid, + const size_t mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const DViewCArrayKokkos &elem_sspd, + const double den, + const double sie) + { + return; + } + + KOKKOS_FUNCTION + double calc_sound_speed_gradient_density( + const DViewCArrayKokkos &elem_pres, + const DViewCArrayKokkos &elem_stress, + const size_t elem_gid, + const size_t mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const DViewCArrayKokkos &elem_sspd, + const double den, + const double sie) + { + return 0.0; + } + + KOKKOS_FUNCTION + double calc_sound_speed_gradient_internal_energy( + const DViewCArrayKokkos &elem_pres, + const DViewCArrayKokkos &elem_stress, + const size_t elem_gid, + const size_t mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const DViewCArrayKokkos &elem_sspd, + const double den, + const double sie) + { + return 0.0; + } + + KOKKOS_FUNCTION + void calc_pressure( + const DViewCArrayKokkos &elem_pres, + const DViewCArrayKokkos &elem_stress, + const size_t elem_gid, + const size_t mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const DViewCArrayKokkos &elem_sspd, + const double den, + const double sie) + { + return; + } + + KOKKOS_FUNCTION + double calc_pressure_gradient_density( + const DViewCArrayKokkos &elem_pres, + const DViewCArrayKokkos &elem_stress, + const size_t elem_gid, + const size_t mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const DViewCArrayKokkos &elem_sspd, + const double den, + const double sie) + { + return 0.0; + } + + KOKKOS_FUNCTION + double calc_pressure_gradient_internal_energy( + const DViewCArrayKokkos &elem_pres, + const DViewCArrayKokkos &elem_stress, + const size_t elem_gid, + const size_t mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const DViewCArrayKokkos &elem_sspd, + const double den, + const double sie) + { + return 0.0; + } + + void destroy( + const DCArrayKokkos &material, + const DViewCArrayKokkos &elem_mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const size_t num_elems) + { + + } +} // end namespace UserDefinedEOSModel \ No newline at end of file diff --git a/src/Parallel-Solvers/Material-Models/eos/UserDefined/UserDefinedEOSModel.h b/src/Parallel-Solvers/Material-Models/eos/UserDefined/UserDefinedEOSModel.h new file mode 100644 index 000000000..ccd491f33 --- /dev/null +++ b/src/Parallel-Solvers/Material-Models/eos/UserDefined/UserDefinedEOSModel.h @@ -0,0 +1,120 @@ +#pragma once + +#include "Simulation_Parameters/Material.h" +#include "matar.h" +using namespace mtr; + +namespace UserDefinedEOSModel +{ + void init_eos_state_vars( + const DCArrayKokkos &material, + const DViewCArrayKokkos &elem_mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const size_t num_elems); + + KOKKOS_FUNCTION + void calc_sound_speed( + const DViewCArrayKokkos &elem_pres, + const DViewCArrayKokkos &elem_stress, + const size_t elem_gid, + const size_t mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const DViewCArrayKokkos &elem_sspd, + const double den, + const double sie); + + KOKKOS_FUNCTION + double calc_sound_speed_gradient_density( + const DViewCArrayKokkos &elem_pres, + const DViewCArrayKokkos &elem_stress, + const size_t elem_gid, + const size_t mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const DViewCArrayKokkos &elem_sspd, + const double den, + const double sie); + + KOKKOS_FUNCTION + double calc_sound_speed_gradient_internal_energy( + const DViewCArrayKokkos &elem_pres, + const DViewCArrayKokkos &elem_stress, + const size_t elem_gid, + const size_t mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const DViewCArrayKokkos &elem_sspd, + const double den, + const double sie); + + KOKKOS_FUNCTION + void calc_pressure( + const DViewCArrayKokkos &elem_pres, + const DViewCArrayKokkos &elem_stress, + const size_t elem_gid, + const size_t mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const DViewCArrayKokkos &elem_sspd, + const double den, + const double sie); + + KOKKOS_FUNCTION + double calc_pressure_gradient_density( + const DViewCArrayKokkos &elem_pres, + const DViewCArrayKokkos &elem_stress, + const size_t elem_gid, + const size_t mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const DViewCArrayKokkos &elem_sspd, + const double den, + const double sie); + + KOKKOS_FUNCTION + double calc_pressure_gradient_internal_energy( + const DViewCArrayKokkos &elem_pres, + const DViewCArrayKokkos &elem_stress, + const size_t elem_gid, + const size_t mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const DViewCArrayKokkos &elem_sspd, + const double den, + const double sie); + + void destroy( + const DCArrayKokkos &material, + const DViewCArrayKokkos &elem_mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const size_t num_elems); + +} // end namespace UserDefinedEOSModel + diff --git a/src/Parallel-Solvers/Material-Models/material_models.cpp b/src/Parallel-Solvers/Material-Models/material_models.cpp index 48bafbf85..54bf91ea4 100644 --- a/src/Parallel-Solvers/Material-Models/material_models.cpp +++ b/src/Parallel-Solvers/Material-Models/material_models.cpp @@ -1,68 +1,116 @@ #include "material_models.h" +#include + #include "IdealGasEOSModel.h" -#include "IdealGasStrengthModel.h" -#include "UserEOSModel.h" -#include "UserStrengthModel.h" +#include "ConstantEOSModel.h" +#include "UserDefinedEOSModel.h" +#include "UserDefinedStrengthModel.h" +#include "VUMATStrengthModel.h" +#include "EVPStrengthModel.h" +#include "EVPFFTStrengthModel.h" +#include "LSEVPFFTStrengthModel.h" + +void init_state_vars( + const DCArrayKokkos &material, + const DViewCArrayKokkos &elem_mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const size_t num_elems) +{ + /* + This function initializes eos_state_vars and strength_state_vars by calling + the init_eos_state_vars or init_strength_state_vars of the models. + TODO: split into two different functions + */ -/* EOSParent */ -KOKKOS_FUNCTION -EOSParent::EOSParent() {} + // get all strength models + std::unordered_set strength_models; + for (size_t elem_gid = 0; elem_gid -P* allocate_memory(RUN_LOCATION run_loc) -{ - P* ptr = nullptr; - if (run_loc == RUN_LOCATION::device) { - ptr = (C*)Kokkos::kokkos_malloc(sizeof(C)); - } - else if (run_loc == RUN_LOCATION::host) { - ptr = (C*)Kokkos::kokkos_malloc(sizeof(C)); - } - else { - throw std::runtime_error("run_loc should be device or host"); - } - return ptr; -} + case STRENGTH_MODEL::evp: + EVPStrengthModel::init_strength_state_vars(material, elem_mat_id, eos_state_vars, + strength_state_vars, eos_global_vars, strength_global_vars, elem_user_output_vars, num_elems); + break; -template -void free_memory(P* model_ptr, RUN_LOCATION run_loc) -{ - static_assert(std::is_same::value or std::is_same::value, - "Invalid type. Type must be EOSParent or StrengthParent"); + case STRENGTH_MODEL::evpfft: + EVPFFTStrengthModel::init_strength_state_vars(material, elem_mat_id, eos_state_vars, + strength_state_vars, eos_global_vars, strength_global_vars, elem_user_output_vars, num_elems); + break; - if (model_ptr == nullptr) - return; + case STRENGTH_MODEL::ls_evpfft: + LSEVPFFTStrengthModel::init_strength_state_vars(material, elem_mat_id, eos_state_vars, + strength_state_vars, eos_global_vars, strength_global_vars, elem_user_output_vars, num_elems); + break; - if (run_loc == RUN_LOCATION::device) { - Kokkos::kokkos_free(model_ptr); - model_ptr = nullptr; - } - else if (run_loc == RUN_LOCATION::host) { - Kokkos::kokkos_free(model_ptr); - model_ptr = nullptr; - } - else { - throw std::runtime_error("run_loc should be device or host"); + default: + break; + } // end switch + + } // end for + + + // get all eos models + std::unordered_set eos_models; + for (size_t elem_gid = 0; elem_gid &elem_strength, const DCArrayKokkos &material, const DViewCArrayKokkos &elem_mat_id, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, const DCArrayKokkos &elem_user_output_vars, const size_t num_elems) { @@ -70,67 +118,72 @@ void init_strength_model( Material model strength should be initialized here. */ - // Allocate memory on GPU or CPU depending on model run_location + // do not update elem_eos or elem_strength device or host, + // the functions will be placed in the right location + + // assign the function pointers of the strength models for (size_t elem_gid = 0; elem_gid (run_loc); + case STRENGTH_MODEL::user_defined: + if (run_loc == RUN_LOCATION::host) { + elem_strength.host(elem_gid).calc_stress = UserDefinedStrengthModel::calc_stress; + } else { + RUN({ + elem_strength(elem_gid).calc_stress = UserDefinedStrengthModel::calc_stress; + }); + } + break; + + case STRENGTH_MODEL::vumat: + if (run_loc == RUN_LOCATION::host) { + elem_strength.host(elem_gid).calc_stress = VUMATStrengthModel::calc_stress; + } else { + RUN({ + elem_strength(elem_gid).calc_stress = VUMATStrengthModel::calc_stress; + }); + } break; - case STRENGTH_MODEL::user_strength_model: - elem_strength.host(elem_gid).model = allocate_memory (run_loc); + + case STRENGTH_MODEL::evp: + if (run_loc == RUN_LOCATION::host) { + elem_strength.host(elem_gid).calc_stress = EVPStrengthModel::calc_stress; + } else { + RUN({ + elem_strength(elem_gid).calc_stress = EVPStrengthModel::calc_stress; + }); + } break; + + case STRENGTH_MODEL::evpfft: + if (run_loc == RUN_LOCATION::host) { + elem_strength.host(elem_gid).calc_stress = EVPFFTStrengthModel::calc_stress; + } else { + RUN({ + elem_strength(elem_gid).calc_stress = EVPFFTStrengthModel::calc_stress; + }); + } + break; + + case STRENGTH_MODEL::ls_evpfft: + if (run_loc == RUN_LOCATION::host) { + elem_strength.host(elem_gid).calc_stress = LSEVPFFTStrengthModel::calc_stress; + } else { + RUN({ + elem_strength(elem_gid).calc_stress = LSEVPFFTStrengthModel::calc_stress; + }); + } + break; + default: break; } // end switch - } - elem_strength.update_device(); - - // Create model on GPU using `placement new` for models that run on device - FOR_ALL(elem_gid, 0, num_elems, { - size_t mat_id = elem_mat_id(elem_gid); - auto run_loc = material(mat_id).strength_run_location; - auto strength_model = material(mat_id).strength_model; - if (run_loc == RUN_LOCATION::device) { - switch (strength_model) { - case STRENGTH_MODEL::ideal_gas: - new ((IdealGasStrengthModel*)elem_strength(elem_gid).model) - IdealGasStrengthModel(material, state_vars, global_vars, elem_user_output_vars, mat_id, elem_gid); - break; - case STRENGTH_MODEL::user_strength_model: - new ((UserStrengthModel*)elem_strength(elem_gid).model) - UserStrengthModel(material, state_vars, global_vars, elem_user_output_vars, mat_id, elem_gid); - break; - default: - break; - } // end switch - } // end if - }); - Kokkos::fence(); - - - // Create model on CPU using `placement new` for models that run on host - for (size_t elem_gid = 0; elem_gid &elem_eos, const DCArrayKokkos &material, const DViewCArrayKokkos &elem_mat_id, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, const DCArrayKokkos &elem_user_output_vars, const size_t num_elems) { @@ -148,67 +203,82 @@ void init_eos_model( Material EOS model should be initialized here. */ - // Allocate memory on GPU or CPU depending on model run_location + // do not update elem_eos or elem_strength device or host, + // the functions will be placed in the right location + + // assign the function pointers of the eos models for (size_t elem_gid = 0; elem_gid (run_loc); + case EOS_MODEL::constant: + if (run_loc == RUN_LOCATION::host) { + elem_eos.host(elem_gid).calc_pressure = ConstantEOSModel::calc_pressure; + elem_eos.host(elem_gid).calc_pressure_gradient_density = ConstantEOSModel::calc_pressure_gradient_density; + elem_eos.host(elem_gid).calc_pressure_gradient_internal_energy = ConstantEOSModel::calc_pressure_gradient_internal_energy; + elem_eos.host(elem_gid).calc_sound_speed = ConstantEOSModel::calc_sound_speed; + elem_eos.host(elem_gid).calc_sound_speed_gradient_density = ConstantEOSModel::calc_sound_speed_gradient_density; + elem_eos.host(elem_gid).calc_sound_speed_gradient_internal_energy = ConstantEOSModel::calc_sound_speed_gradient_internal_energy; + } else { + RUN({ + elem_eos(elem_gid).calc_pressure = ConstantEOSModel::calc_pressure; + elem_eos(elem_gid).calc_pressure_gradient_density = ConstantEOSModel::calc_pressure_gradient_density; + elem_eos(elem_gid).calc_pressure_gradient_internal_energy = ConstantEOSModel::calc_pressure_gradient_internal_energy; + elem_eos(elem_gid).calc_sound_speed = ConstantEOSModel::calc_sound_speed; + elem_eos(elem_gid).calc_sound_speed_gradient_density = ConstantEOSModel::calc_sound_speed_gradient_density; + elem_eos(elem_gid).calc_sound_speed_gradient_internal_energy = ConstantEOSModel::calc_sound_speed_gradient_internal_energy; + }); + } break; - case EOS_MODEL::user_eos_model: - elem_eos.host(elem_gid).model = allocate_memory (run_loc); + + case EOS_MODEL::ideal_gas: + if (run_loc == RUN_LOCATION::host) { + elem_eos.host(elem_gid).calc_pressure = IdealGasEOSModel::calc_pressure; + elem_eos.host(elem_gid).calc_pressure_gradient_density = IdealGasEOSModel::calc_pressure_gradient_density; + elem_eos.host(elem_gid).calc_pressure_gradient_internal_energy = IdealGasEOSModel::calc_pressure_gradient_internal_energy; + elem_eos.host(elem_gid).calc_sound_speed = IdealGasEOSModel::calc_sound_speed; + elem_eos.host(elem_gid).calc_sound_speed_gradient_density = IdealGasEOSModel::calc_sound_speed_gradient_density; + elem_eos.host(elem_gid).calc_sound_speed_gradient_internal_energy = IdealGasEOSModel::calc_sound_speed_gradient_internal_energy; + } else { + RUN({ + elem_eos(elem_gid).calc_pressure = IdealGasEOSModel::calc_pressure; + elem_eos(elem_gid).calc_pressure_gradient_density = IdealGasEOSModel::calc_pressure_gradient_density; + elem_eos(elem_gid).calc_pressure_gradient_internal_energy = IdealGasEOSModel::calc_pressure_gradient_internal_energy; + elem_eos(elem_gid).calc_sound_speed = IdealGasEOSModel::calc_sound_speed; + elem_eos(elem_gid).calc_sound_speed_gradient_density = IdealGasEOSModel::calc_sound_speed_gradient_density; + elem_eos(elem_gid).calc_sound_speed_gradient_internal_energy = IdealGasEOSModel::calc_sound_speed_gradient_internal_energy; + }); + } break; + + case EOS_MODEL::user_defined: + if (run_loc == RUN_LOCATION::host) { + elem_eos.host(elem_gid).calc_pressure = UserDefinedEOSModel::calc_pressure; + elem_eos.host(elem_gid).calc_pressure_gradient_density = UserDefinedEOSModel::calc_pressure_gradient_density; + elem_eos.host(elem_gid).calc_pressure_gradient_internal_energy = UserDefinedEOSModel::calc_pressure_gradient_internal_energy; + elem_eos.host(elem_gid).calc_sound_speed = UserDefinedEOSModel::calc_sound_speed; + elem_eos.host(elem_gid).calc_sound_speed_gradient_density = UserDefinedEOSModel::calc_sound_speed_gradient_density; + elem_eos.host(elem_gid).calc_sound_speed_gradient_internal_energy = UserDefinedEOSModel::calc_sound_speed_gradient_internal_energy; + } else { + RUN({ + elem_eos(elem_gid).calc_pressure = UserDefinedEOSModel::calc_pressure; + elem_eos(elem_gid).calc_pressure_gradient_density = UserDefinedEOSModel::calc_pressure_gradient_density; + elem_eos(elem_gid).calc_pressure_gradient_internal_energy = UserDefinedEOSModel::calc_pressure_gradient_internal_energy; + elem_eos(elem_gid).calc_sound_speed = UserDefinedEOSModel::calc_sound_speed; + elem_eos(elem_gid).calc_sound_speed_gradient_density = UserDefinedEOSModel::calc_sound_speed_gradient_density; + elem_eos(elem_gid).calc_sound_speed_gradient_internal_energy = UserDefinedEOSModel::calc_sound_speed_gradient_internal_energy; + }); + } + break; + default: break; } // end switch - } - elem_eos.update_device(); - - // Create model on GPU using `placement new` for models that run on device - FOR_ALL(elem_gid, 0, num_elems, { - size_t mat_id = elem_mat_id(elem_gid); - auto run_loc = material(mat_id).eos_run_location; - auto eos_model = material(mat_id).eos_model; - if (run_loc == RUN_LOCATION::device) { - switch (eos_model) { - case EOS_MODEL::ideal_gas: - new ((IdealGasEOSModel*)elem_eos(elem_gid).model) - IdealGasEOSModel(material, state_vars, global_vars, elem_user_output_vars, mat_id, elem_gid); - break; - case EOS_MODEL::user_eos_model: - new ((UserEOSModel*)elem_eos(elem_gid).model) - UserEOSModel(material, state_vars, global_vars, elem_user_output_vars, mat_id, elem_gid); - break; - default: - break; - } // end switch - } // end if - }); - Kokkos::fence(); - - - // Create model on CPU using `placement new` for models that run on host - for (size_t elem_gid = 0; elem_gid &elem_strength, const DCArrayKokkos &material, const DViewCArrayKokkos &elem_mat_id, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, - const DCArrayKokkos elem_user_output_vars, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, const size_t num_elems) { /* - All memory cleanup related to the user material model should be done in this fuction. + All memory cleanup related to the strength model should be done in this fuction. Fierro calls `destroy_strength_model` at the end of a simulation. */ - // Destroy GPU model - FOR_ALL(elem_gid, 0, num_elems, { - size_t mat_id = elem_mat_id(elem_gid); - auto run_loc = material(mat_id).strength_run_location; - if (run_loc == RUN_LOCATION::device and elem_strength(elem_gid).model != nullptr) { - elem_strength(elem_gid).model->~StrengthParent(); - } - }); - Kokkos::fence(); - - // Destroy CPU model - for (size_t elem_gid = 0; elem_gid < num_elems; elem_gid++) { + // get all strength models + std::unordered_set strength_models; + for (size_t elem_gid = 0; elem_gid~StrengthParent(); - } + strength_models.insert(material.host(mat_id).strength_model); } - // Free CPU memory - for (size_t elem_gid = 0; elem_gid < num_elems; elem_gid++) { - size_t mat_id = elem_mat_id.host(elem_gid); - auto run_loc = material.host(mat_id).strength_run_location; - if (elem_strength.host(elem_gid).model != nullptr) { - free_memory(elem_strength.host(elem_gid).model, run_loc); - } - } + // call the destroy functions of all strength models + for (auto strength_model : strength_models) { + + switch (strength_model) { + case STRENGTH_MODEL::user_defined: + UserDefinedStrengthModel::destroy(material, elem_mat_id, eos_state_vars, + strength_state_vars, eos_global_vars, strength_global_vars, elem_user_output_vars, num_elems); + break; + + case STRENGTH_MODEL::vumat: + VUMATStrengthModel::destroy(material, elem_mat_id, eos_state_vars, + strength_state_vars, eos_global_vars, strength_global_vars, elem_user_output_vars, num_elems); + break; + + case STRENGTH_MODEL::evp: + EVPStrengthModel::destroy(material, elem_mat_id, eos_state_vars, + strength_state_vars, eos_global_vars, strength_global_vars, elem_user_output_vars, num_elems); + break; + + case STRENGTH_MODEL::evpfft: + EVPFFTStrengthModel::destroy(material, elem_mat_id, eos_state_vars, + strength_state_vars, eos_global_vars, strength_global_vars, elem_user_output_vars, num_elems); + break; + + case STRENGTH_MODEL::ls_evpfft: + LSEVPFFTStrengthModel::destroy(material, elem_mat_id, eos_state_vars, + strength_state_vars, eos_global_vars, strength_global_vars, elem_user_output_vars, num_elems); + break; + + default: + break; + } // end switch + + } // end for return; } // end destroy_strength_model @@ -262,43 +348,49 @@ void destroy_eos_model( DCArrayKokkos &elem_eos, const DCArrayKokkos &material, const DViewCArrayKokkos &elem_mat_id, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, const DCArrayKokkos &elem_user_output_vars, const size_t num_elems) { /* - All memory cleanup related to the user material model should be done in this fuction. + All memory cleanup related to the eos model should be done in this fuction. Fierro calls `destroy_eos_model()` at the end of a simulation. */ - // Destroy GPU model - FOR_ALL(elem_gid, 0, num_elems, { - size_t mat_id = elem_mat_id(elem_gid); - auto run_loc = material(mat_id).eos_run_location; - if (run_loc == RUN_LOCATION::device and elem_eos(elem_gid).model != nullptr) { - elem_eos(elem_gid).model->~EOSParent(); - } - }); - Kokkos::fence(); - - // Destroy CPU model - for (size_t elem_gid = 0; elem_gid < num_elems; elem_gid++) { + // get all eos models + std::unordered_set eos_models; + for (size_t elem_gid = 0; elem_gid~EOSParent(); - } + eos_models.insert(material.host(mat_id).eos_model); } - // Free CPU memory - for (size_t elem_gid = 0; elem_gid < num_elems; elem_gid++) { - size_t mat_id = elem_mat_id.host(elem_gid); - auto run_loc = material.host(mat_id).eos_run_location; - if (elem_eos.host(elem_gid).model != nullptr) { - free_memory(elem_eos.host(elem_gid).model, run_loc); - } - } + // call the destroy functions of all eos models + for (auto eos_model : eos_models) { + + switch (eos_model) { + case EOS_MODEL::constant: + ConstantEOSModel::destroy(material, elem_mat_id, eos_state_vars, + strength_state_vars, eos_global_vars, strength_global_vars, elem_user_output_vars, num_elems); + break; + + case EOS_MODEL::ideal_gas: + IdealGasEOSModel::destroy(material, elem_mat_id, eos_state_vars, + strength_state_vars, eos_global_vars, strength_global_vars, elem_user_output_vars, num_elems); + break; + + case EOS_MODEL::user_defined: + UserDefinedEOSModel::destroy(material, elem_mat_id, eos_state_vars, + strength_state_vars, eos_global_vars, strength_global_vars, elem_user_output_vars, num_elems); + break; + + default: + break; + } // end switch + + } // end for return; } // end destroy_eos_model diff --git a/src/Parallel-Solvers/Material-Models/material_models.h b/src/Parallel-Solvers/Material-Models/material_models.h index 36bb996fb..3b27c1d20 100644 --- a/src/Parallel-Solvers/Material-Models/material_models.h +++ b/src/Parallel-Solvers/Material-Models/material_models.h @@ -7,114 +7,102 @@ using namespace mtr; struct eos_t; struct strength_t; -/* EOSParent */ -class EOSParent { -public: - - KOKKOS_FUNCTION - EOSParent(); - - KOKKOS_FUNCTION - virtual ~EOSParent(); - - KOKKOS_FUNCTION - virtual int calc_sound_speed( +struct eos_t { + void (*calc_sound_speed)( const DViewCArrayKokkos &elem_pres, const DViewCArrayKokkos &elem_stress, const size_t elem_gid, const size_t mat_id, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, const DCArrayKokkos &elem_user_output_vars, const DViewCArrayKokkos &elem_sspd, const double den, - const double sie) = 0; + const double sie) = nullptr; - KOKKOS_FUNCTION - virtual double calc_sound_speed_gradient_density( + double (*calc_sound_speed_gradient_density)( const DViewCArrayKokkos &elem_pres, const DViewCArrayKokkos &elem_stress, const size_t elem_gid, const size_t mat_id, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, const DCArrayKokkos &elem_user_output_vars, const DViewCArrayKokkos &elem_sspd, const double den, - const double sie){ return 0;} + const double sie) = nullptr; - KOKKOS_FUNCTION - virtual double calc_sound_speed_gradient_internal_energy( + double (*calc_sound_speed_gradient_internal_energy)( const DViewCArrayKokkos &elem_pres, const DViewCArrayKokkos &elem_stress, const size_t elem_gid, const size_t mat_id, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, const DCArrayKokkos &elem_user_output_vars, const DViewCArrayKokkos &elem_sspd, const double den, - const double sie){ return 0;} + const double sie) = nullptr; - KOKKOS_FUNCTION - virtual int calc_pressure( + void (*calc_pressure)( const DViewCArrayKokkos &elem_pres, const DViewCArrayKokkos &elem_stress, const size_t elem_gid, const size_t mat_id, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, const DCArrayKokkos &elem_user_output_vars, const DViewCArrayKokkos &elem_sspd, const double den, - const double sie) = 0; + const double sie) = nullptr; - KOKKOS_FUNCTION - virtual double calc_pressure_gradient_density( + double (*calc_pressure_gradient_density)( const DViewCArrayKokkos &elem_pres, const DViewCArrayKokkos &elem_stress, const size_t elem_gid, const size_t mat_id, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, const DCArrayKokkos &elem_user_output_vars, const DViewCArrayKokkos &elem_sspd, const double den, - const double sie){ return 0;} + const double sie) = nullptr; - KOKKOS_FUNCTION - virtual double calc_pressure_gradient_internal_energy( + double (*calc_pressure_gradient_internal_energy)( const DViewCArrayKokkos &elem_pres, const DViewCArrayKokkos &elem_stress, const size_t elem_gid, const size_t mat_id, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, const DCArrayKokkos &elem_user_output_vars, const DViewCArrayKokkos &elem_sspd, const double den, - const double sie){ return 0;} + const double sie) = nullptr; }; - -/* sterngth_parent */ -class StrengthParent { -public: - - KOKKOS_FUNCTION - StrengthParent(); - - KOKKOS_FUNCTION - virtual ~StrengthParent(); - - KOKKOS_FUNCTION - virtual int calc_stress( +struct strength_t { + void (*calc_stress) ( const DViewCArrayKokkos &elem_pres, const DViewCArrayKokkos &elem_stress, const size_t elem_gid, const size_t mat_id, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, const DCArrayKokkos &elem_user_output_vars, const DViewCArrayKokkos &elem_sspd, const double den, @@ -127,24 +115,17 @@ class StrengthParent { const double dt, const double rk_alpha, const size_t cycle, - const size_t rk_level) = 0; - -}; - - -struct eos_t { - EOSParent *model = nullptr; -}; - -struct strength_t { - StrengthParent *model = nullptr; + const size_t rk_level, + const double time) = nullptr; }; void init_state_vars( const DCArrayKokkos &material, const DViewCArrayKokkos &elem_mat_id, - DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, const DCArrayKokkos &elem_user_output_vars, const size_t num_elems); @@ -152,8 +133,10 @@ void init_strength_model( DCArrayKokkos &elem_strength, const DCArrayKokkos &material, const DViewCArrayKokkos &elem_mat_id, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, const DCArrayKokkos &elem_user_output_vars, const size_t num_elems); @@ -161,8 +144,10 @@ void init_eos_model( DCArrayKokkos &elem_eos, const DCArrayKokkos &material, const DViewCArrayKokkos &elem_mat_id, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, const DCArrayKokkos &elem_user_output_vars, const size_t num_elems); @@ -170,17 +155,21 @@ void destroy_strength_model( DCArrayKokkos &elem_strength, const DCArrayKokkos &material, const DViewCArrayKokkos &elem_mat_id, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, - const DCArrayKokkos elem_user_output_vars, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, const size_t num_elems); void destroy_eos_model( DCArrayKokkos &elem_eos, const DCArrayKokkos &material, const DViewCArrayKokkos &elem_mat_id, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, const DCArrayKokkos &elem_user_output_vars, const size_t num_elems); diff --git a/src/Parallel-Solvers/Material-Models/strength/CMakeLists.txt b/src/Parallel-Solvers/Material-Models/strength/CMakeLists.txt new file mode 100644 index 000000000..cfe7607ce --- /dev/null +++ b/src/Parallel-Solvers/Material-Models/strength/CMakeLists.txt @@ -0,0 +1,5 @@ +add_subdirectory(EVP) +add_subdirectory(EVPFFT) +add_subdirectory(LSEVPFFT) +add_subdirectory(UserDefined) +add_subdirectory(VUMAT) \ No newline at end of file diff --git a/src/Parallel-Solvers/Material-Models/strength/EVP/CMakeLists.txt b/src/Parallel-Solvers/Material-Models/strength/EVP/CMakeLists.txt new file mode 100644 index 000000000..0330a66ac --- /dev/null +++ b/src/Parallel-Solvers/Material-Models/strength/EVP/CMakeLists.txt @@ -0,0 +1,4 @@ +set(SRC_Files EVPStrengthModel.cpp chg_basis.cpp evpal.cpp user_mat.cpp) +add_library(EVPStrengthModel OBJECT ${SRC_Files}) +target_include_directories(EVPStrengthModel PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) +target_link_libraries(EVPStrengthModel Elements) \ No newline at end of file diff --git a/src/Parallel-Solvers/Material-Models/strength/EVP/EVPStrengthModel.cpp b/src/Parallel-Solvers/Material-Models/strength/EVP/EVPStrengthModel.cpp new file mode 100644 index 000000000..945efff99 --- /dev/null +++ b/src/Parallel-Solvers/Material-Models/strength/EVP/EVPStrengthModel.cpp @@ -0,0 +1,703 @@ +#include "EVPStrengthModel.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "user_mat.h" + +#define CLEAR_LINE(ifstream) ( ifstream.ignore(std::numeric_limits::max(), '\n') ); +// for string delimiter parsing +std::vector split (std::string s, std::string delimiter); + + //This goes in input.cpp +// material(0).num_state_vars = 163; // actual num_state_vars +// material(0).eos_model = user_eos_model; // EOS model is required +// material(0).strength_type = model::hypo; +// material(0).strength_setup = model_init::input; +// material(0).strength_model = user_strength_model; + +// ----------------------------------------------------------------------------- +// The function to read in the state vars for a user supplied model + +namespace EVPStrengthModel +{ + void init_strength_state_vars( + const DCArrayKokkos &material, + const DViewCArrayKokkos &elem_mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const size_t num_elems) + { + /* + In this function, initialize the strength_state_vars according to the location this model needs it. + DO NOT strength_state_vars.update_host or strength_state_vars.update_device. + This is because different elements might have different models, run location, and + initialization location leading to overwriting of data. + */ + + // for the reason given in the opening comment we will create a state_var, + // populate it as needed and copy needed data to strength_state_vars + DCArrayKokkos state_vars (strength_state_vars.dims(0), strength_state_vars.dims(1)); + + + const real_t SQR2_ = 1.41421356237309; + const real_t RSQ2_ = 0.70710678118654744; + const real_t RSQ3_ = 0.57735026918962584; + const real_t RSQ6_ = 0.40824829046386304; + + CMatrix cc66v(6,6); + CMatrix cdim(3); + CMatrix isn(24,3); //assumed BCC with 2 modes here + CMatrix isb(24,3); //assumed BCC with 2 modes here + CMatrix sn(3); + CMatrix sb(3); + CMatrix aux5(5); + CMatrix aux33(3,3); + // Accessed within state_vars for loop + DFMatrixKokkos dnca(3,24); //assumed BCC with 2 modes here + DFMatrixKokkos dbca(3,24); //assumed BCC with 2 modes here + DCMatrixKokkos cc66(6,6); + DCMatrixKokkos B_basis(3,3,6); + // FMatrix schca(5,12*2); //assumed BCC with 2 modes here + std::ifstream ur1; + int iso,nmodesx,nmodes,modex,nsmx,nrsx,nsysx; + real_t gamd0x,tau0xf,tau0xb,tau1x,thet0x,thet1x; //real_t may be something else in fierro + real_t hselfx,hlatex,snor,qnor,prod,ph,th,om; + std::string junk,icryst; + + real_t stress_conv = 1.0e-5; + + for (int i = 1; i <= 3; i++) { + for (int j = 1; j <= 3; j++) { + for (int k = 1; k <= 6; k++) { + B_basis.host(i,j,k) = 0.0; + } + } + } + B_basis.host(1,1,2) = -RSQ6_; + B_basis.host(2,2,2) = -RSQ6_; + B_basis.host(3,3,2) = real_t(2.0)*RSQ6_; + + B_basis.host(1,1,1) = -RSQ2_; + B_basis.host(2,2,1) = RSQ2_; + + B_basis.host(2,3,3) = RSQ2_; + B_basis.host(3,2,3) = RSQ2_; + + B_basis.host(1,3,4) = RSQ2_; + B_basis.host(3,1,4) = RSQ2_; + + B_basis.host(1,2,5) = RSQ2_; + B_basis.host(2,1,5) = RSQ2_; + + B_basis.host(1,1,6) = RSQ3_; + B_basis.host(2,2,6) = RSQ3_; + B_basis.host(3,3,6) = RSQ3_; + B_basis.update_device(); + + ur1.open("./ta_el.sx");//("./cuel1.txt"); + // read el txt file here + ur1 >> iso; CLEAR_LINE(ur1); + + if (iso == 0) { + for (int i = 1; i <= 6; i++) { + for (int j = 1; j <= 6; j++) { + ur1 >> cc66v(i,j); + } + CLEAR_LINE(ur1); + } + } + + // CMatrix dde(3,3); + // CMatrix xid4(3,3,3,3); + // CMatrix cc3333(3,3,3,3); + + // for (int i = 1; i <= 3; i++) { + // for (int j = 1; j <= 3; j++) { + // dde(i,j) = 0.0; + // if (i == j) dde(i,j) = 1.0; + // } + // } + + // for (int i = 1; i <= 3; i++) { + // for (int j = 1; j <= 3; j++) { + // for (int k = 1; k <= 3; k++) { + // for (int l = 1; l <= 3; l++) { + // xid4(i,j,k,l) = (dde(i,k)*dde(j,l) + dde(i,l)*dde(j,k)) / real_t(2.0); + // } + // } + // } + // } + // real_t tmu = 119800.0 / (real_t(2.0)*(real_t(1.0) + 0.35)); + // real_t tla = real_t(2.0)*tmu*0.35 / (real_t(1.0) - real_t(2.0)*0.35); + + // printf("%g %g\n",tmu,tla ); + + // for (int i = 1; i <= 3; i++) { + // for (int j = 1; j <= 3; j++) { + // for (int k = 1; k <= 3; k++) { + // for (int l = 1; l <= 3; l++) { + // cc3333(i,j,k,l) = tla*dde(i,j)*dde(k,l) + real_t(2.0)*tmu*xid4(i,j,k,l); + // printf("%d %d %d %d %g \n",i,j,k,l,cc3333(i,j,k,l) ); + // } + // } + // } + // } + // chg_basis_4(cc66.pointer(), cc3333.pointer(), 4, 6, B_basis.pointer()); + + int i1,i2,j1,j2; + int ijv_[6*2] = {1,2,3,2,1,1,1,2,3,3,3,2}; + CMatrix cc3333(3,3,3,3); + ViewFMatrixKokkos ijv(ijv_,6,2); + for (int i = 1; i <= 6; i++) { + i1 = ijv(i,1); + i2 = ijv(i,2); + for (int j = 1; j <= 6; j++) { + j1 = ijv(j,1); + j2 = ijv(j,2); + cc3333(i1,i2,j1,j2) = cc66v(i,j); + cc3333(i2,i1,j1,j2) = cc66v(i,j); + cc3333(i1,i2,j2,j1) = cc66v(i,j); + cc3333(i2,i1,j2,j1) = cc66v(i,j); + } + } + + for (int i = 1; i <= 3; i++) { + for (int j = 1; j <= 3; j++) { + for (int k = 1; k <= 3; k++) { + for (int l = 1; l <= 3; l++) { + printf("%d %d %d %d %g \n", i,j,k,l,cc3333(i,j,k,l)); + } + } + } + } + + chg_basis_4(cc66.host.pointer(), cc3333.pointer(), 4, 6, B_basis.host.pointer()); + + cc66.update_device(); + + // for (int i = 1; i <= 6; i++) { + // printf("cc=%.8f %.8f %.8f %.8f %.8f %.8f\n", cc66(i,1),cc66(i,2),cc66(i,3),cc66(i,4),cc66(i,5),cc66(i,6)); + // } + + + ur1.close(); + + ur1.open("./ta_pl.sx");//("./cupl3.txt"); + // read pl txt file here, up to slip systems + ur1 >> junk; CLEAR_LINE(ur1); + ur1 >> icryst; CLEAR_LINE(ur1); + for (int i = 1; i <= 3; i++) { + ur1 >> cdim(i); + } + CLEAR_LINE(ur1); + + ur1 >> nmodesx; CLEAR_LINE(ur1); + ur1 >> nmodes; CLEAR_LINE(ur1); + // temporary check until general crystal file read in is implemented + // if (nmodes > 1) { + // printf("nmodes IN IS > 1\n"); + // printf("ONLY 1 SLIP MODE ALLOWED FOR NOW\n"); + // exit(1); + // } + ur1 >> junk; + CLEAR_LINE(ur1); + + ur1 >> junk; CLEAR_LINE(ur1); + ur1 >> modex >> nsmx >> nrsx >> gamd0x; CLEAR_LINE(ur1); + ur1 >> tau0xf >> tau0xb >> tau1x >> thet0x >> thet1x; CLEAR_LINE(ur1); + ur1 >> hselfx >> hlatex; CLEAR_LINE(ur1); + ur1 >> junk; CLEAR_LINE(ur1); + + if (thet0x < thet1x) { + printf("INITIAL HARDENING LOWER THAN FINAL HARDENING\n"); + exit(1); + } + // + // CASE TAU1=0 CORRESPONDS TO LINEAR HARDENING AND IS INDEPENDENT OF TAU0. + // AVOID DIVISION BY ZERO + if (tau1x <= 1.0e-6) { + tau1x = 1.0e-6; + thet0x = thet1x; + } + + for (int j = 1; j <= nsmx; j++) { + for (int k = 1; k <= 3; k++) ur1 >> isn(j,k); + for (int k = 1; k <= 3; k++) ur1 >> isb(j,k); + CLEAR_LINE(ur1); + } + + for (int js = 1; js <= nsmx; js++) { + nsysx = js; + // + // DEFINES RATE SENSITIVITY AND CRSS FOR EACH SYSTEM IN THE MODE + // + for (int m = 1; m <= 3; m++) { + sn(m) = isn(js,m) / cdim(m); + sb(m) = isb(js,m) * cdim(m); + } + // + // *** NORMALIZES SYSTEM VECTORS AND CHECKS NORMALITY + // + snor = sqrt( sn(1)*sn(1) + sn(2)*sn(2) + sn(3)*sn(3) ); + qnor = sqrt( sb(1)*sb(1) + sb(2)*sb(2) + sb(3)*sb(3) ); + prod = 0.0; + for (int j = 1; j <= 3; j++) { + dnca.host(j,nsysx) = sn(j) / snor; + dbca.host(j,nsysx) = sb(j) / qnor; + if (abs(dnca.host(j,nsysx)) < 1.e-03) dnca.host(j,nsysx) = 0.0; + if (abs(dbca.host(j,nsysx)) < 1.e-03) dbca.host(j,nsysx) = 0.0; + prod += dnca.host(j,nsysx) * dbca.host(j,nsysx); + } + + if (prod >= 1.0e-3) { + printf("SYSTEM %d IS NOT ORTHOGONAL !!\n", js); + printf("%d %d %d,", isb(js,1), isb(js,2), isb(js,3)); + printf("%d %d %d", isn(js,1), isn(js,2), isn(js,3)); + exit(1); + } + // + // DEFINE SCHMID VECTOR IN CRYSTAL AXES FOR EACH SYSTEM + // + //for (int i = 1; i <= 3; i++) { + // for (int j = 1; j <= 3; j++) { + // aux33(i,j) = (dnca(i,nsysx)*dbca(j,nsysx) + dnca(j,nsysx)*dbca(i,nsysx)) / 2.0; + // } + //} + + //cb.chg_basis_2(aux5.pointer(), aux33.pointer(), 2, 5, cb.B_basis_host_pointer()); //heavy modification needed here + + //for (int i = 1; i <= 5; i++) { + // schca(i,nsysx) = aux5(i); + //} + } // end for js + + // second set of slip systems for Ta BCC + ur1 >> junk; CLEAR_LINE(ur1); + ur1 >> junk; CLEAR_LINE(ur1); + ur1 >> junk; CLEAR_LINE(ur1); + ur1 >> junk; CLEAR_LINE(ur1); + ur1 >> junk; CLEAR_LINE(ur1); + + for (int j = nsmx+1; j <= nsmx*2; j++) { + for (int k = 1; k <= 3; k++) ur1 >> isn(j,k); + for (int k = 1; k <= 3; k++) ur1 >> isb(j,k); + CLEAR_LINE(ur1); + } + for (int js = nsmx+1; js <= nsmx*2; js++) { + nsysx = js; + // + // DEFINES RATE SENSITIVITY AND CRSS FOR EACH SYSTEM IN THE MODE + // + for (int m = 1; m <= 3; m++) { + sn(m) = isn(js,m) / cdim(m); + sb(m) = isb(js,m) * cdim(m); + } + // + // *** NORMALIZES SYSTEM VECTORS AND CHECKS NORMALITY + // + snor = sqrt( sn(1)*sn(1) + sn(2)*sn(2) + sn(3)*sn(3) ); + qnor = sqrt( sb(1)*sb(1) + sb(2)*sb(2) + sb(3)*sb(3) ); + prod = 0.0; + for (int j = 1; j <= 3; j++) { + dnca.host(j,nsysx) = sn(j) / snor; + dbca.host(j,nsysx) = sb(j) / qnor; + if (abs(dnca.host(j,nsysx)) < 1.e-03) dnca.host(j,nsysx) = 0.0; + if (abs(dbca.host(j,nsysx)) < 1.e-03) dbca.host(j,nsysx) = 0.0; + prod += dnca.host(j,nsysx) * dbca.host(j,nsysx); + } + + if (prod >= 1.0e-3) { + printf("SYSTEM %d IS NOT ORTHOGONAL !!\n", js); + printf("%d %d %d,", isb(js,1), isb(js,2), isb(js,3)); + printf("%d %d %d", isn(js,1), isn(js,2), isn(js,3)); + exit(1); + } + } + dnca.update_device(); + dbca.update_device(); + + nsmx = nsmx*2; + ur1.close(); + + //===================================================================== + + for (size_t elem_gid = 0; elem_gid> ph >> th >> om; CLEAR_LINE(ur1); + // single crystal test + ph = 0.0; th = 0.0; om = 0.0; //001 + state_vars(elem_gid,257) = ph; + state_vars(elem_gid,258) = th; + state_vars(elem_gid,259) = om; + + // 260-313 Basis change array values + c = 260; + for (int i = 1; i <= 3; i++) { + for (int j = 1; j <= 3; j++) { + for (int k = 1; k <= 6; k++) { + state_vars(elem_gid,c) = B_basis(i,j,k); + c += 1; + } + } + } + state_vars(elem_gid,314) = grain_id; + //}); + } + Kokkos::fence(); + state_vars.update_host(); + + // note, dont use strength_state_vars update device or host is not encouranged + // this could lead to data been overwritten if multiple strength model are used + // within the same simulation of different elements + + // write data to device side of strength_state_vars + FOR_ALL(elem_gid, 0, num_elems, { + size_t mat_id = elem_mat_id(elem_gid); + size_t num_strength_state_vars = material(mat_id).num_strength_state_vars; + // only fill the state_vars of elements that use this model + if (material(mat_id).strength_model == STRENGTH_MODEL::evp) { + for(size_t var=0; var &elem_pres, + const DViewCArrayKokkos &elem_stress, + const size_t elem_gid, + const size_t mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const DViewCArrayKokkos &elem_sspd, + const double den, + const double sie, + const ViewCArrayKokkos &vel_grad, + const ViewCArrayKokkos &elem_node_gids, + const DViewCArrayKokkos &node_coords, + const DViewCArrayKokkos &node_vel, + const double vol, + const double dt, + const double rk_alpha, + const size_t cycle, + const size_t rk_level, + const double time) + { + // ChgBasis cb; + + // ----------------------------------------------------------------------------- + // The user must coding goes here + //------------------------------------------------------------------------------ + real_t dt_rk = dt/2.0; // can't use rk_alpha unless sum(rk_alpha)=1. + + real_t aux5_[5]; + real_t aux33_[3*3]; + real_t ag_[3*3]; + real_t strain_[3*3]; + real_t D_[3*3]; + real_t ept_[3*3]; + real_t edotp_[3*3]; + + ViewCMatrixKokkos aux5(aux5_,5); + ViewCMatrixKokkos aux33(aux33_,3,3); + ViewCMatrixKokkos ag(ag_,3,3); + ViewCMatrixKokkos strain(strain_,3,3); + ViewCMatrixKokkos D(D_,3,3); + ViewCMatrixKokkos ept(ept_,3,3); + ViewCMatrixKokkos edotp(edotp_,3,3); + + // Cauchy deviatoric stress tensor, stress(rk,elem,i,j) + // create a slice of the stress tensor + ViewCMatrixKokkos stress_n(&elem_stress(0, elem_gid, 0, 0), 3, 3); // stress at previous timestep + ViewCMatrixKokkos stress(&elem_stress(1, elem_gid, 0, 0), 3, 3); // stress at this timestep + + + // Decompress strength_state_vars (this may become its own function later) + // ----------------------------------------------------------------------------- + // read basis change matrix + ViewCMatrixKokkos B_basis(&strength_state_vars(elem_gid,260),3,3,6); + + ViewCMatrixKokkos cc66(&strength_state_vars(elem_gid, 0), 6, 6); // crystal elastic stiffness 0-35 + // try inverting crystal stiffness before rotate + real_t sc66_[6*6]; + ViewCMatrixKokkos sc66(sc66_,6,6); + for (int ii = 1; ii <= 6; ii++) { + for (int jj = 1; jj <= 6; jj++) { + sc66(ii,jj) = cc66(ii,jj); + } + } + inverse_gj(sc66.pointer(), 6); + + int nsmx = strength_state_vars(elem_gid, 36); + //if (nsmx == 12){ + const int nsm = 12*2; + //} + real_t nrs = strength_state_vars(elem_gid, 37); + real_t gamd0 = strength_state_vars(elem_gid, 38); + ViewCMatrixKokkos voceParam(&strength_state_vars(elem_gid, 39),5); + real_t hself = strength_state_vars(elem_gid, 44); + real_t hlate = strength_state_vars(elem_gid, 45); + + // create slip shear rate array + real_t gamdot_[nsm]; + ViewCMatrixKokkos gamdot(gamdot_,nsm); + // build H array, needs more work for more than 1 mode + real_t hard_[nsm*nsm]; + ViewCMatrixKokkos hard(hard_,nsm,nsm); + for (int i = 1; i <= nsm; i++) { + for (int j = 1; j <= nsm; j++) { + hard(i,j) = hlate; + } + hard(i,i) = hself; + } + + // ViewFMatrixKokkos schca(&strength_state_vars(elem_gid,46), 5, nsm); // crystal plastic Schmid tensors 46-105 + ViewFMatrixKokkos dnca(&strength_state_vars(elem_gid,46), 3, nsm); // crystal slip normal vectors 46-81 + ViewFMatrixKokkos dbca(&strength_state_vars(elem_gid,118), 3, nsm); // crystal slip normal vectors 46-81 + + // calculate schmid tensor + real_t schca_[5*nsm]; + ViewFMatrixKokkos schca(schca_,5,nsm); + for (int js = 1; js <= nsm; js++) { + for (int i = 1; i <= 3; i++) { + for (int j = 1; j <= 3; j++) { + aux33(i,j) = (dnca(i,js)*dbca(j,js) + dnca(j,js)*dbca(i,js)) / 2.0; + } + } + + chg_basis_2(aux5.pointer(), aux33.pointer(), 2, 5, B_basis.pointer()); //heavy modification needed here + + for (int i = 1; i <= 5; i++) { + schca(i,js) = aux5(i); + } + } + + ViewCMatrixKokkos crss(&strength_state_vars(elem_gid,190), 2, nsm); // forward and backward critical resolved shear stress 118-141 + + ViewCMatrixKokkos strain_n(&strength_state_vars(elem_gid,238), 3, 3); // strain at previous timestep 142-150 + ViewCMatrixKokkos ept_n(&strength_state_vars(elem_gid,247), 3, 3); // plastic strain at previous timestep 151-159 + real_t gacumgr = strength_state_vars(elem_gid,256); // accumulated slip shear at previous timestep + // read local bunge euler angles and generate rotation matrix + real_t ph = strength_state_vars(elem_gid,257);//*(4.0*atan(1.0))/180.0; + real_t th = strength_state_vars(elem_gid,258);//*(4.0*atan(1.0))/180.0; + real_t tm = strength_state_vars(elem_gid,259);//*(4.0*atan(1.0))/180.0; + euler(2, ph, th, tm, ag.pointer()); + // ----------------------------------------------------------------------------- + // END Decompress strength_state_vars + // CUUUUUUDDAAAAAA + for (int is = 1; is <= nsm; is++) { + gamdot(is) = 0.0; + } + // Update total strain from velgrad, also initialie some stuff to 0.0 cause CUDA + // D = 1/2(vel_grad + vel_grad^T) + for (int i = 1; i <= 3; i++) { + for (int j = 1; j <= 3; j++) { + D(i, j) = 0.5 * (vel_grad(i-1, j-1) + vel_grad(j-1, i-1)); //must convert between array and matrix + strain(i,j) = strain_n(i,j); + ept(i,j) = ept_n(i,j); + } + } + real_t ddnorm = 0.0; + for (int i = 1; i <= 3; i++) { + for (int j = 1; j <= 3; j++) { + ddnorm += dt_rk*D(i,j)*dt_rk*D(i,j); + } + } + ddnorm = sqrt(ddnorm); + + if (ddnorm >= 1.0e-14) { //strain increment too small, treat as no deformation + update_strain(strain.pointer(), strain_n.pointer(), D.pointer(), dt_rk); + + // Update local crystal elastic stiffness + real_t cg66_[6*6]; + real_t sg66_[6*6]; + ViewCMatrixKokkos cg66(cg66_,6,6); + ViewCMatrixKokkos sg66(sg66_,6,6); + update_stiff(cg66.pointer(), cc66.pointer(), ag.pointer(), B_basis.pointer()); + update_stiff(sg66.pointer(), sc66.pointer(), ag.pointer(), B_basis.pointer()); + + // Update local Schmid tensors + real_t sch_[5*nsm]; + ViewFMatrixKokkos sch(sch_,5,nsm); + update_schmid(sch.pointer(), schca.pointer(), ag.pointer(), nsm, B_basis.pointer()); + + // Evaluate crystal elsto-viscoplastic model + evpal(stress.pointer(), edotp.pointer(), gamdot.pointer(), stress_n.pointer(), strain.pointer(), + ept_n.pointer(), cg66.pointer(), sch.pointer(), crss.pointer(), gamd0, nrs, nsm, dt_rk, cycle, + B_basis.pointer(), elem_gid); //gacumgr, voceParam.pointer(), hard.pointer() + + // Update plastic strain + update_strain(ept.pointer(), ept_n.pointer(), edotp.pointer(), dt_rk); + + // Update hardening + harden(crss.pointer(), gacumgr, gamdot.pointer(), voceParam.pointer(), hard.pointer(), nsm, dt_rk); + + // Overwrite stress to evolve forward-euler instead of RK + for (int i = 1; i <= 3; i++) { + for (int j = 1; j <= 3; j++) { + stress_n(i,j) = stress(i,j); + } + } + + } + + // Update crystal orientation. Always done because no deformation does not mean no rotation + update_orient(ag.pointer(), vel_grad.pointer(), gamdot.pointer(), dnca.pointer(), dbca.pointer(), nsm, dt_rk); + + // Recompress strength_state_vars (this may become its own function later) + // // Overwrite stress to evolve forward-euler instead of RK + // for (int i = 1; i <= 3; i++) { + // for (int j = 1; j <= 3; j++) { + // stress_n(i,j) = stress(i,j); + // } + // } + // 0-117 should not be changing + // 118-141 crss updated via pointer + // 142-150 strain + for (int i = 1; i <= 3; i++) { + for (int j = 1; j <= 3; j++) { + strain_n(i,j) = strain(i,j); + } + } + // 151-159 plastic strain + for (int i = 1; i <= 3; i++) { + for (int j = 1; j <= 3; j++) { + ept_n(i,j) = ept(i,j); + } + } + // 160 accumulated total plastic slip + strength_state_vars(elem_gid,256)=gacumgr; + // 161-163 calc new euler angles + euler(1, ph, th, tm, ag.pointer()); + strength_state_vars(elem_gid,257) = ph; + strength_state_vars(elem_gid,258) = th; + strength_state_vars(elem_gid,259) = tm; + // END Recompress strength_state_vars + + for (int i = 1; i <= 3; i++) { + for (int j = 1; j <= 3; j++) { + if (isnan(stress(i,j))) { + printf("nan stress at elem %d cycle %d\n",elem_gid,cycle); + exit(1); + } + } + } + + return; + } + + void destroy( + const DCArrayKokkos &material, + const DViewCArrayKokkos &elem_mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const size_t num_elems) + { + + } + +} // end namespace EVPStrengthModel \ No newline at end of file diff --git a/src/Parallel-Solvers/Material-Models/strength/EVP/EVPStrengthModel.h b/src/Parallel-Solvers/Material-Models/strength/EVP/EVPStrengthModel.h new file mode 100644 index 000000000..e505b3629 --- /dev/null +++ b/src/Parallel-Solvers/Material-Models/strength/EVP/EVPStrengthModel.h @@ -0,0 +1,54 @@ +#pragma once + +#include "Simulation_Parameters/Material.h" +#include "matar.h" +using namespace mtr; + +namespace EVPStrengthModel +{ + void init_strength_state_vars( + const DCArrayKokkos &material, + const DViewCArrayKokkos &elem_mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const size_t num_elems); + + KOKKOS_FUNCTION + void calc_stress ( + const DViewCArrayKokkos &elem_pres, + const DViewCArrayKokkos &elem_stress, + const size_t elem_gid, + const size_t mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const DViewCArrayKokkos &elem_sspd, + const double den, + const double sie, + const ViewCArrayKokkos &vel_grad, + const ViewCArrayKokkos &elem_node_gids, + const DViewCArrayKokkos &node_coords, + const DViewCArrayKokkos &node_vel, + const double vol, + const double dt, + const double rk_alpha, + const size_t cycle, + const size_t rk_level, + const double time); + + void destroy( + const DCArrayKokkos &material, + const DViewCArrayKokkos &elem_mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const size_t num_elems); + +} // end namespace EVPStrengthModel \ No newline at end of file diff --git a/src/EVP/src/chg_basis.cpp b/src/Parallel-Solvers/Material-Models/strength/EVP/chg_basis.cpp similarity index 100% rename from src/EVP/src/chg_basis.cpp rename to src/Parallel-Solvers/Material-Models/strength/EVP/chg_basis.cpp diff --git a/src/EVP/src/evpal.cpp b/src/Parallel-Solvers/Material-Models/strength/EVP/evpal.cpp similarity index 100% rename from src/EVP/src/evpal.cpp rename to src/Parallel-Solvers/Material-Models/strength/EVP/evpal.cpp diff --git a/src/EVP/example_input_files/ta_el.sx b/src/Parallel-Solvers/Material-Models/strength/EVP/example_input_files/ta_el.sx similarity index 100% rename from src/EVP/example_input_files/ta_el.sx rename to src/Parallel-Solvers/Material-Models/strength/EVP/example_input_files/ta_el.sx diff --git a/src/EVP/example_input_files/ta_pl.sx b/src/Parallel-Solvers/Material-Models/strength/EVP/example_input_files/ta_pl.sx similarity index 100% rename from src/EVP/example_input_files/ta_pl.sx rename to src/Parallel-Solvers/Material-Models/strength/EVP/example_input_files/ta_pl.sx diff --git a/src/EVP/src/UserStrengthModel.cpp b/src/Parallel-Solvers/Material-Models/strength/EVP/user_mat.cpp similarity index 55% rename from src/EVP/src/UserStrengthModel.cpp rename to src/Parallel-Solvers/Material-Models/strength/EVP/user_mat.cpp index 332cdba70..c1ef9108e 100644 --- a/src/EVP/src/UserStrengthModel.cpp +++ b/src/Parallel-Solvers/Material-Models/strength/EVP/user_mat.cpp @@ -1,248 +1,5 @@ -#include "UserStrengthModel.h" #include "user_mat.h" -#include -KOKKOS_FUNCTION -UserStrengthModel::UserStrengthModel( - const DCArrayKokkos &material, - const DCArrayKokkos &state_vars, - const DCArrayKokkos &global_vars, - const DCArrayKokkos &elem_user_output_vars, - const size_t mat_id, - const size_t elem_gid) -{ -} - -KOKKOS_FUNCTION -UserStrengthModel::~UserStrengthModel() -{ -} - -KOKKOS_FUNCTION -int UserStrengthModel::calc_stress( - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const size_t elem_gid, - const size_t mat_id, - const DCArrayKokkos &elem_state_vars, - const DCArrayKokkos &global_vars, - const DCArrayKokkos &elem_user_output_vars, - const DViewCArrayKokkos &elem_sspd, - const double den, - const double sie, - const ViewCArrayKokkos &vel_grad, - const ViewCArrayKokkos &elem_node_gids, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const double vol, - const double dt, - const double rk_alpha, - const size_t cycle, - const size_t rk_level) -{ - // ChgBasis cb; - - // ----------------------------------------------------------------------------- - // The user must coding goes here - //------------------------------------------------------------------------------ - real_t dt_rk = dt/2.0; // can't use rk_alpha unless sum(rk_alpha)=1. - - real_t aux5_[5]; - real_t aux33_[3*3]; - real_t ag_[3*3]; - real_t strain_[3*3]; - real_t D_[3*3]; - real_t ept_[3*3]; - real_t edotp_[3*3]; - - ViewCMatrixKokkos aux5(aux5_,5); - ViewCMatrixKokkos aux33(aux33_,3,3); - ViewCMatrixKokkos ag(ag_,3,3); - ViewCMatrixKokkos strain(strain_,3,3); - ViewCMatrixKokkos D(D_,3,3); - ViewCMatrixKokkos ept(ept_,3,3); - ViewCMatrixKokkos edotp(edotp_,3,3); - - // Cauchy deviatoric stress tensor, stress(rk,elem,i,j) - // create a slice of the stress tensor - ViewCMatrixKokkos stress_n(&elem_stress(0, elem_gid, 0, 0), 3, 3); // stress at previous timestep - ViewCMatrixKokkos stress(&elem_stress(1, elem_gid, 0, 0), 3, 3); // stress at this timestep - - - // Decompress elem_state_vars (this may become its own function later) - // ----------------------------------------------------------------------------- - // read basis change matrix - ViewCMatrixKokkos B_basis(&elem_state_vars(elem_gid,260),3,3,6); - - ViewCMatrixKokkos cc66(&elem_state_vars(elem_gid, 0), 6, 6); // crystal elastic stiffness 0-35 - // try inverting crystal stiffness before rotate - real_t sc66_[6*6]; - ViewCMatrixKokkos sc66(sc66_,6,6); - for (int ii = 1; ii <= 6; ii++) { - for (int jj = 1; jj <= 6; jj++) { - sc66(ii,jj) = cc66(ii,jj); - } - } - inverse_gj(sc66.pointer(), 6); - - int nsmx = elem_state_vars(elem_gid, 36); - //if (nsmx == 12){ - const int nsm = 12*2; - //} - real_t nrs = elem_state_vars(elem_gid, 37); - real_t gamd0 = elem_state_vars(elem_gid, 38); - ViewCMatrixKokkos voceParam(&elem_state_vars(elem_gid, 39),5); - real_t hself = elem_state_vars(elem_gid, 44); - real_t hlate = elem_state_vars(elem_gid, 45); - - // create slip shear rate array - real_t gamdot_[nsm]; - ViewCMatrixKokkos gamdot(gamdot_,nsm); - // build H array, needs more work for more than 1 mode - real_t hard_[nsm*nsm]; - ViewCMatrixKokkos hard(hard_,nsm,nsm); - for (int i = 1; i <= nsm; i++) { - for (int j = 1; j <= nsm; j++) { - hard(i,j) = hlate; - } - hard(i,i) = hself; - } - - // ViewFMatrixKokkos schca(&elem_state_vars(elem_gid,46), 5, nsm); // crystal plastic Schmid tensors 46-105 - ViewFMatrixKokkos dnca(&elem_state_vars(elem_gid,46), 3, nsm); // crystal slip normal vectors 46-81 - ViewFMatrixKokkos dbca(&elem_state_vars(elem_gid,118), 3, nsm); // crystal slip normal vectors 46-81 - - // calculate schmid tensor - real_t schca_[5*nsm]; - ViewFMatrixKokkos schca(schca_,5,nsm); - for (int js = 1; js <= nsm; js++) { - for (int i = 1; i <= 3; i++) { - for (int j = 1; j <= 3; j++) { - aux33(i,j) = (dnca(i,js)*dbca(j,js) + dnca(j,js)*dbca(i,js)) / 2.0; - } - } - - chg_basis_2(aux5.pointer(), aux33.pointer(), 2, 5, B_basis.pointer()); //heavy modification needed here - - for (int i = 1; i <= 5; i++) { - schca(i,js) = aux5(i); - } - } - - ViewCMatrixKokkos crss(&elem_state_vars(elem_gid,190), 2, nsm); // forward and backward critical resolved shear stress 118-141 - - ViewCMatrixKokkos strain_n(&elem_state_vars(elem_gid,238), 3, 3); // strain at previous timestep 142-150 - ViewCMatrixKokkos ept_n(&elem_state_vars(elem_gid,247), 3, 3); // plastic strain at previous timestep 151-159 - real_t gacumgr = elem_state_vars(elem_gid,256); // accumulated slip shear at previous timestep - // read local bunge euler angles and generate rotation matrix - real_t ph = elem_state_vars(elem_gid,257);//*(4.0*atan(1.0))/180.0; - real_t th = elem_state_vars(elem_gid,258);//*(4.0*atan(1.0))/180.0; - real_t tm = elem_state_vars(elem_gid,259);//*(4.0*atan(1.0))/180.0; - euler(2, ph, th, tm, ag.pointer()); - // ----------------------------------------------------------------------------- - // END Decompress elem_state_vars - // CUUUUUUDDAAAAAA - for (int is = 1; is <= nsm; is++) { - gamdot(is) = 0.0; - } - // Update total strain from velgrad, also initialie some stuff to 0.0 cause CUDA - // D = 1/2(vel_grad + vel_grad^T) - for (int i = 1; i <= 3; i++) { - for (int j = 1; j <= 3; j++) { - D(i, j) = 0.5 * (vel_grad(i-1, j-1) + vel_grad(j-1, i-1)); //must convert between array and matrix - strain(i,j) = strain_n(i,j); - ept(i,j) = ept_n(i,j); - } - } - real_t ddnorm = 0.0; - for (int i = 1; i <= 3; i++) { - for (int j = 1; j <= 3; j++) { - ddnorm += dt_rk*D(i,j)*dt_rk*D(i,j); - } - } - ddnorm = sqrt(ddnorm); - - if (ddnorm >= 1.0e-14) { //strain increment too small, treat as no deformation - update_strain(strain.pointer(), strain_n.pointer(), D.pointer(), dt_rk); - - // Update local crystal elastic stiffness - real_t cg66_[6*6]; - real_t sg66_[6*6]; - ViewCMatrixKokkos cg66(cg66_,6,6); - ViewCMatrixKokkos sg66(sg66_,6,6); - update_stiff(cg66.pointer(), cc66.pointer(), ag.pointer(), B_basis.pointer()); - update_stiff(sg66.pointer(), sc66.pointer(), ag.pointer(), B_basis.pointer()); - - // Update local Schmid tensors - real_t sch_[5*nsm]; - ViewFMatrixKokkos sch(sch_,5,nsm); - update_schmid(sch.pointer(), schca.pointer(), ag.pointer(), nsm, B_basis.pointer()); - - // Evaluate crystal elsto-viscoplastic model - evpal(stress.pointer(), edotp.pointer(), gamdot.pointer(), stress_n.pointer(), strain.pointer(), - ept_n.pointer(), cg66.pointer(), sch.pointer(), crss.pointer(), gamd0, nrs, nsm, dt_rk, cycle, - B_basis.pointer(), elem_gid); //gacumgr, voceParam.pointer(), hard.pointer() - - // Update plastic strain - update_strain(ept.pointer(), ept_n.pointer(), edotp.pointer(), dt_rk); - - // Update hardening - harden(crss.pointer(), gacumgr, gamdot.pointer(), voceParam.pointer(), hard.pointer(), nsm, dt_rk); - - // Overwrite stress to evolve forward-euler instead of RK - for (int i = 1; i <= 3; i++) { - for (int j = 1; j <= 3; j++) { - stress_n(i,j) = stress(i,j); - } - } - - } - - // Update crystal orientation. Always done because no deformation does not mean no rotation - update_orient(ag.pointer(), vel_grad.pointer(), gamdot.pointer(), dnca.pointer(), dbca.pointer(), nsm, dt_rk); - - // Recompress elem_state_vars (this may become its own function later) - // // Overwrite stress to evolve forward-euler instead of RK - // for (int i = 1; i <= 3; i++) { - // for (int j = 1; j <= 3; j++) { - // stress_n(i,j) = stress(i,j); - // } - // } - // 0-117 should not be changing - // 118-141 crss updated via pointer - // 142-150 strain - for (int i = 1; i <= 3; i++) { - for (int j = 1; j <= 3; j++) { - strain_n(i,j) = strain(i,j); - } - } - // 151-159 plastic strain - for (int i = 1; i <= 3; i++) { - for (int j = 1; j <= 3; j++) { - ept_n(i,j) = ept(i,j); - } - } - // 160 accumulated total plastic slip - elem_state_vars(elem_gid,256)=gacumgr; - // 161-163 calc new euler angles - euler(1, ph, th, tm, ag.pointer()); - elem_state_vars(elem_gid,257) = ph; - elem_state_vars(elem_gid,258) = th; - elem_state_vars(elem_gid,259) = tm; - // END Recompress elem_state_vars - - for (int i = 1; i <= 3; i++) { - for (int j = 1; j <= 3; j++) { - if (isnan(stress(i,j))) { - printf("nan stress at elem %d cycle %d\n",elem_gid,cycle); - exit(1); - } - } - } - return 0; - -} // end of user mat -//===================================================================== // euler <-> rotation matrix transformation function KOKKOS_FUNCTION void euler(int iopt, real_t &ph, real_t &th, real_t &tm, real_t *a_){ diff --git a/src/EVP/src/user_mat.h b/src/Parallel-Solvers/Material-Models/strength/EVP/user_mat.h similarity index 100% rename from src/EVP/src/user_mat.h rename to src/Parallel-Solvers/Material-Models/strength/EVP/user_mat.h diff --git a/src/Parallel-Solvers/Material-Models/strength/EVPFFT/CMakeLists.txt b/src/Parallel-Solvers/Material-Models/strength/EVPFFT/CMakeLists.txt new file mode 100644 index 000000000..07dd1fc5c --- /dev/null +++ b/src/Parallel-Solvers/Material-Models/strength/EVPFFT/CMakeLists.txt @@ -0,0 +1,12 @@ +if(BUILD_EVPFFT_FIERRO) + add_definitions(-DBUILD_EVPFFT_FIERRO=1) + add_subdirectory(../../../../EVPFFT/src EVPFFT) +endif() + +set(SRC_Files EVPFFTStrengthModel.cpp) +add_library(EVPFFTStrengthModel OBJECT ${SRC_Files}) +target_include_directories(EVPFFTStrengthModel PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) +target_link_libraries(EVPFFTStrengthModel Elements) +if(BUILD_EVPFFT_FIERRO) + target_link_libraries(EVPFFTStrengthModel evpfft-fierro) +endif() \ No newline at end of file diff --git a/src/Parallel-Solvers/Material-Models/strength/EVPFFT/EVPFFTStrengthModel.cpp b/src/Parallel-Solvers/Material-Models/strength/EVPFFT/EVPFFTStrengthModel.cpp new file mode 100644 index 000000000..a71c2d9eb --- /dev/null +++ b/src/Parallel-Solvers/Material-Models/strength/EVPFFT/EVPFFTStrengthModel.cpp @@ -0,0 +1,105 @@ +#include "EVPFFTStrengthModel.h" +#if BUILD_EVPFFT_FIERRO + #include "FierroEVPFFTLink.h" +#endif + +namespace EVPFFTStrengthModel +{ + void init_strength_state_vars( + const DCArrayKokkos &material, + const DViewCArrayKokkos &elem_mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const size_t num_elems) + { + /* + In this function, initialize the strength_state_vars according to the location this model needs it. + DO NOT strength_state_vars.update_host or strength_state_vars.update_device. + This is because different elements might have different models, run location, and + initialization location leading to overwriting of data. + */ + +#if BUILD_EVPFFT_FIERRO + FierroEVPFFTLink::init_strength_state_vars(material, + elem_mat_id, + eos_state_vars, + strength_state_vars, + eos_global_vars, + strength_global_vars, + elem_user_output_vars, + num_elems); +#endif + + return; + } + + KOKKOS_FUNCTION + void calc_stress ( + const DViewCArrayKokkos &elem_pres, + const DViewCArrayKokkos &elem_stress, + const size_t elem_gid, + const size_t mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const DViewCArrayKokkos &elem_sspd, + const double den, + const double sie, + const ViewCArrayKokkos &vel_grad, + const ViewCArrayKokkos &elem_node_gids, + const DViewCArrayKokkos &node_coords, + const DViewCArrayKokkos &node_vel, + const double vol, + const double dt, + const double rk_alpha, + const size_t cycle, + const size_t rk_level, + const double time) + { + +#if BUILD_EVPFFT_FIERRO + FierroEVPFFTLink::calc_stress(elem_pres, + elem_stress, + elem_gid, + mat_id, + eos_state_vars, + strength_state_vars, + eos_global_vars, + strength_global_vars, + elem_user_output_vars, + elem_sspd, + den, + sie, + vel_grad, + elem_node_gids, + node_coords, + node_vel,vol, + dt, + rk_alpha, + cycle, + rk_level, + time); +#endif + + return; + } + + void destroy( + const DCArrayKokkos &material, + const DViewCArrayKokkos &elem_mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const size_t num_elems) + { + return; + } + +} // end namespace EVPFFTStrengthModel \ No newline at end of file diff --git a/src/Parallel-Solvers/Material-Models/strength/EVPFFT/EVPFFTStrengthModel.h b/src/Parallel-Solvers/Material-Models/strength/EVPFFT/EVPFFTStrengthModel.h new file mode 100644 index 000000000..3b85cd2e7 --- /dev/null +++ b/src/Parallel-Solvers/Material-Models/strength/EVPFFT/EVPFFTStrengthModel.h @@ -0,0 +1,54 @@ +#pragma once + +#include "Simulation_Parameters/Material.h" +#include "matar.h" +using namespace mtr; + +namespace EVPFFTStrengthModel +{ + void init_strength_state_vars( + const DCArrayKokkos &material, + const DViewCArrayKokkos &elem_mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const size_t num_elems); + + KOKKOS_FUNCTION + void calc_stress ( + const DViewCArrayKokkos &elem_pres, + const DViewCArrayKokkos &elem_stress, + const size_t elem_gid, + const size_t mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const DViewCArrayKokkos &elem_sspd, + const double den, + const double sie, + const ViewCArrayKokkos &vel_grad, + const ViewCArrayKokkos &elem_node_gids, + const DViewCArrayKokkos &node_coords, + const DViewCArrayKokkos &node_vel, + const double vol, + const double dt, + const double rk_alpha, + const size_t cycle, + const size_t rk_level, + const double time); + + void destroy( + const DCArrayKokkos &material, + const DViewCArrayKokkos &elem_mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const size_t num_elems); + +} // end namespace EVPFFTStrengthModel \ No newline at end of file diff --git a/src/Parallel-Solvers/Material-Models/strength/LSEVPFFT/CMakeLists.txt b/src/Parallel-Solvers/Material-Models/strength/LSEVPFFT/CMakeLists.txt new file mode 100644 index 000000000..7127e17c0 --- /dev/null +++ b/src/Parallel-Solvers/Material-Models/strength/LSEVPFFT/CMakeLists.txt @@ -0,0 +1,12 @@ +if(BUILD_LS_EVPFFT_FIERRO) + add_definitions(-DBUILD_LS_EVPFFT_FIERRO=1) + add_subdirectory(../../../../LS-EVPFFT/src LS-EVPFFT) +endif() + +set(SRC_Files LSEVPFFTStrengthModel.cpp) +add_library(LSEVPFFTStrengthModel OBJECT ${SRC_Files}) +target_include_directories(LSEVPFFTStrengthModel PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) +target_link_libraries(LSEVPFFTStrengthModel Elements) +if(BUILD_LS_EVPFFT_FIERRO) + target_link_libraries(LSEVPFFTStrengthModel ls-evpfft-fierro) +endif() \ No newline at end of file diff --git a/src/Parallel-Solvers/Material-Models/strength/LSEVPFFT/LSEVPFFTStrengthModel.cpp b/src/Parallel-Solvers/Material-Models/strength/LSEVPFFT/LSEVPFFTStrengthModel.cpp new file mode 100644 index 000000000..45786ad1d --- /dev/null +++ b/src/Parallel-Solvers/Material-Models/strength/LSEVPFFT/LSEVPFFTStrengthModel.cpp @@ -0,0 +1,104 @@ +#include "LSEVPFFTStrengthModel.h" +#if BUILD_LS_EVPFFT_FIERRO + #include "FierroLSEVPFFTLink.h" +#endif + +namespace LSEVPFFTStrengthModel +{ + void init_strength_state_vars( + const DCArrayKokkos &material, + const DViewCArrayKokkos &elem_mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const size_t num_elems) + { + /* + In this function, initialize the strength_state_vars according to the location this model needs it. + DO NOT strength_state_vars.update_host or strength_state_vars.update_device. + This is because different elements might have different models, run location, and + initialization location leading to overwriting of data. + */ + +#if BUILD_LS_EVPFFT_FIERRO + FierroLSEVPFFTLink::init_strength_state_vars(material, + elem_mat_id, + eos_state_vars, + strength_state_vars, + eos_global_vars, + strength_global_vars, + elem_user_output_vars, + num_elems); +#endif + + return; + } + + void calc_stress ( + const DViewCArrayKokkos &elem_pres, + const DViewCArrayKokkos &elem_stress, + const size_t elem_gid, + const size_t mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const DViewCArrayKokkos &elem_sspd, + const double den, + const double sie, + const ViewCArrayKokkos &vel_grad, + const ViewCArrayKokkos &elem_node_gids, + const DViewCArrayKokkos &node_coords, + const DViewCArrayKokkos &node_vel, + const double vol, + const double dt, + const double rk_alpha, + const size_t cycle, + const size_t rk_level, + const double time) + { + +#if BUILD_LS_EVPFFT_FIERRO + FierroLSEVPFFTLink::calc_stress(elem_pres, + elem_stress, + elem_gid, + mat_id, + eos_state_vars, + strength_state_vars, + eos_global_vars, + strength_global_vars, + elem_user_output_vars, + elem_sspd, + den, + sie, + vel_grad, + elem_node_gids, + node_coords, + node_vel,vol, + dt, + rk_alpha, + cycle, + rk_level, + time); +#endif + + return; + } + + void destroy( + const DCArrayKokkos &material, + const DViewCArrayKokkos &elem_mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const size_t num_elems) + { + return; + } + +} // end namespace LSEVPFFTStrengthModel \ No newline at end of file diff --git a/src/Parallel-Solvers/Material-Models/strength/LSEVPFFT/LSEVPFFTStrengthModel.h b/src/Parallel-Solvers/Material-Models/strength/LSEVPFFT/LSEVPFFTStrengthModel.h new file mode 100644 index 000000000..6e2735901 --- /dev/null +++ b/src/Parallel-Solvers/Material-Models/strength/LSEVPFFT/LSEVPFFTStrengthModel.h @@ -0,0 +1,54 @@ +#pragma once + +#include "Simulation_Parameters/Material.h" +#include "matar.h" +using namespace mtr; + +namespace LSEVPFFTStrengthModel +{ + void init_strength_state_vars( + const DCArrayKokkos &material, + const DViewCArrayKokkos &elem_mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const size_t num_elems); + + KOKKOS_FUNCTION + void calc_stress ( + const DViewCArrayKokkos &elem_pres, + const DViewCArrayKokkos &elem_stress, + const size_t elem_gid, + const size_t mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const DViewCArrayKokkos &elem_sspd, + const double den, + const double sie, + const ViewCArrayKokkos &vel_grad, + const ViewCArrayKokkos &elem_node_gids, + const DViewCArrayKokkos &node_coords, + const DViewCArrayKokkos &node_vel, + const double vol, + const double dt, + const double rk_alpha, + const size_t cycle, + const size_t rk_level, + const double time); + + void destroy( + const DCArrayKokkos &material, + const DViewCArrayKokkos &elem_mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const size_t num_elems); + +} // end namespace LSEVPFFTStrengthModel \ No newline at end of file diff --git a/src/Parallel-Solvers/Material-Models/strength/UserDefined/CMakeLists.txt b/src/Parallel-Solvers/Material-Models/strength/UserDefined/CMakeLists.txt new file mode 100644 index 000000000..0c87c3ff3 --- /dev/null +++ b/src/Parallel-Solvers/Material-Models/strength/UserDefined/CMakeLists.txt @@ -0,0 +1,4 @@ +set(SRC_Files UserDefinedStrengthModel.cpp) +add_library(UserDefinedStrengthModel OBJECT ${SRC_Files}) +target_include_directories(UserDefinedStrengthModel PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) +target_link_libraries(UserDefinedStrengthModel Elements) \ No newline at end of file diff --git a/src/Parallel-Solvers/Material-Models/strength/UserDefined/UserDefinedStrengthModel.cpp b/src/Parallel-Solvers/Material-Models/strength/UserDefined/UserDefinedStrengthModel.cpp new file mode 100644 index 000000000..f7e8435b5 --- /dev/null +++ b/src/Parallel-Solvers/Material-Models/strength/UserDefined/UserDefinedStrengthModel.cpp @@ -0,0 +1,79 @@ +#include "UserDefinedStrengthModel.h" + +namespace UserDefinedStrengthModel +{ + void init_strength_state_vars( + const DCArrayKokkos &material, + const DViewCArrayKokkos &elem_mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const size_t num_elems) + { + /* + In this function, initialize the strength_state_vars according to the location this model needs it. + DO NOT strength_state_vars.update_host or strength_state_vars.update_device. + This is because different elements might have different models, run location, and + initialization location leading to overwriting of data. + */ + + FOR_ALL(elem_gid, 0, num_elems, { + size_t mat_id = elem_mat_id(elem_gid); + size_t num_strength_state_vars = material(mat_id).num_strength_state_vars; + + // only fill the state_vars of elements that use this model + if (material(mat_id).strength_model == STRENGTH_MODEL::user_defined) { + for(size_t var=0; var &elem_pres, + const DViewCArrayKokkos &elem_stress, + const size_t elem_gid, + const size_t mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const DViewCArrayKokkos &elem_sspd, + const double den, + const double sie, + const ViewCArrayKokkos &vel_grad, + const ViewCArrayKokkos &elem_node_gids, + const DViewCArrayKokkos &node_coords, + const DViewCArrayKokkos &node_vel, + const double vol, + const double dt, + const double rk_alpha, + const size_t cycle, + const size_t rk_level, + const double time) + { + return; + } + + void destroy( + const DCArrayKokkos &material, + const DViewCArrayKokkos &elem_mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const size_t num_elems) + { + + } + +} // end namespace UserDefinedStrengthModel \ No newline at end of file diff --git a/src/Parallel-Solvers/Material-Models/strength/UserDefined/UserDefinedStrengthModel.h b/src/Parallel-Solvers/Material-Models/strength/UserDefined/UserDefinedStrengthModel.h new file mode 100644 index 000000000..d455f91df --- /dev/null +++ b/src/Parallel-Solvers/Material-Models/strength/UserDefined/UserDefinedStrengthModel.h @@ -0,0 +1,54 @@ +#pragma once + +#include "Simulation_Parameters/Material.h" +#include "matar.h" +using namespace mtr; + +namespace UserDefinedStrengthModel +{ + void init_strength_state_vars( + const DCArrayKokkos &material, + const DViewCArrayKokkos &elem_mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const size_t num_elems); + + KOKKOS_FUNCTION + void calc_stress ( + const DViewCArrayKokkos &elem_pres, + const DViewCArrayKokkos &elem_stress, + const size_t elem_gid, + const size_t mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const DViewCArrayKokkos &elem_sspd, + const double den, + const double sie, + const ViewCArrayKokkos &vel_grad, + const ViewCArrayKokkos &elem_node_gids, + const DViewCArrayKokkos &node_coords, + const DViewCArrayKokkos &node_vel, + const double vol, + const double dt, + const double rk_alpha, + const size_t cycle, + const size_t rk_level, + const double time); + + void destroy( + const DCArrayKokkos &material, + const DViewCArrayKokkos &elem_mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const size_t num_elems); + +} // end namespace UserDefinedStrengthModel \ No newline at end of file diff --git a/src/Parallel-Solvers/Material-Models/strength/VUMAT/CMakeLists.txt b/src/Parallel-Solvers/Material-Models/strength/VUMAT/CMakeLists.txt new file mode 100644 index 000000000..2d0ee71aa --- /dev/null +++ b/src/Parallel-Solvers/Material-Models/strength/VUMAT/CMakeLists.txt @@ -0,0 +1,33 @@ +enable_language(Fortran) + +# Set compiler flags for Fortran +# set(CMAKE_Fortran_FLAGS "-ffixed-form") # Set Fortran77 compiler flags +if (CMAKE_Fortran_COMPILER_ID MATCHES "GNU") + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fdefault-real-8 -fdefault-double-8") +elseif (CMAKE_Fortran_COMPILER_ID MATCHES "Intel") + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -r8") +elseif (CMAKE_Fortran_COMPILER_ID MATCHES "PGI") + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -r8") +elseif (CMAKE_Fortran_COMPILER_ID MATCHES "XL") + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -qrealsize=8") +elseif (CMAKE_Fortran_COMPILER_ID MATCHES "Clang") + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fdefault-real-8 -fdefault-double-8") +endif() + + +# Set source files +set(VUMAT_CXX_SRC VUMATStrengthModel.cpp VUMAT.cpp) +set(VUMAT_FORTRAN_SRC vumat.f90) + +add_library(VUMATStrengthModel OBJECT ${VUMAT_CXX_SRC} ${VUMAT_FORTRAN_SRC}) +target_include_directories(VUMATStrengthModel PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) +target_link_libraries(VUMATStrengthModel PRIVATE Elements) + +# For if the vumat needs OpenMP +find_package(OpenMP REQUIRED COMPONENTS Fortran) +target_link_libraries(VUMATStrengthModel PRIVATE OpenMP::OpenMP_Fortran) + +# Link against fortran library. might need to add other compilers +if (CMAKE_Fortran_COMPILER_ID MATCHES "GNU") + target_link_libraries(VUMATStrengthModel PRIVATE gfortran) +endif() \ No newline at end of file diff --git a/src/Parallel-Solvers/Material-Models/strength/VUMAT/VUMAT.cpp b/src/Parallel-Solvers/Material-Models/strength/VUMAT/VUMAT.cpp new file mode 100644 index 000000000..b7f418cbf --- /dev/null +++ b/src/Parallel-Solvers/Material-Models/strength/VUMAT/VUMAT.cpp @@ -0,0 +1,202 @@ +#include "VUMAT.h" +#include "vumat.h" + + +// Below are the list of variables calculated or that have values. +// Variables which are "dummy" are not currently calculated yet. If your vumat +// needs these dummy variables please calculated them in calc_stress before the +// call to vumat_ and update this list; + +// nblock = 1 +// ndir = 3 +// nshr = 3 +// nstatev = strength_state_vars.dims(1) // from fierro yaml +// nfieldv = 0; // Fierro does not currently support user-defined external field variables +// nprops = strength_global_vars.dims(1); // from fierro yaml +// lanneal - 0; // Fierro does not currently support annealing process +// stepTime - calculated in calc_stress before calling vumat_ +// totalTime - calculated in calc_stress before calling vumat_ +// dt - calculated in calc_stress before calling vumat_ +// cmname - string(mat_id) // mat_id from fierro +// coordMp - dummy +// charLength - dummy +// props - strength_global_vars.host(mat_id,0); // from fierro yaml +// density - dummy +// strainInc - calculated in calc_stress before calling vumat_ +// relSpinInc - calculated in calc_stress before calling vumat_ +// tempOld - dummy +// stretchOld - dummy +// defgradOld - dummy +// fieldOld - dummy +// stressOld - assigned in calc_stress before calling vumat_ +// stateOld - assigned in calc_stress before calling vumat_ +// enerInternOld - dummy +// enerInelasOld - dummy +// tempNew - dummy +// stretchNew - dummy +// defgradNew - dummy +// fieldNew - dummy +// stressNew - updated stress after call to vumat_ +// stateNew - updated state variables after call to vumat_ +// enerInternNew - dummy +// enerInelasNew - dummy + +VUMAT::VUMAT( + const DCArrayKokkos &material, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const size_t mat_id, + const size_t elem_gid) +{ + + auto run_loc = material.host(mat_id).strength_run_location; + if (run_loc == RUN_LOCATION::device) { + throw std::runtime_error("VUMAT only runs on Host"); + } + + // check that the user remembered to enter number of nstatev and nprops in the yaml + if (strength_state_vars.dims(1) == 0) { + throw std::runtime_error("VUMAT: num_strength_state_vars must be specified = abaqus nstatev"); + } + + nblock = 1; // num of gauss integration points in the element + ndir = 3; // for 3D (ndir: direct stress components) + nshr = 3; // for 3D (nshr: shear stress components) + nstatev = strength_state_vars.dims(1); + nfieldv = 0; // Fierro does not currently support user-defined external field variables + nprops = strength_global_vars.dims(1); + lanneal = 0; // Fierro does not currently support annealing process + + // This long process in needed to pass string to fortran + std::string mat_id_str = std::to_string(mat_id); + std::strncpy(cmname, mat_id_str.c_str(), mat_id_str.size()); + cmname[79] = '\0'; // Ensure null termination + + // Allocate Arrays + coordMp = FMatrix (nblock, ndir); + charLength = FMatrix (nblock); + props = ViewFMatrix (&strength_global_vars.host(mat_id,0), nprops); + density = FMatrix (nblock); + strainInc = FMatrix (nblock, ndir+nshr); + relSpinInc = FMatrix (nblock, nshr); + tempOld = FMatrix (nblock); + stretchOld = FMatrix (nblock, ndir+nshr); + defgradOld = FMatrix (nblock,ndir+2*nshr); + fieldOld = FMatrix (nblock, nfieldv); + stressOld = FMatrix (nblock, ndir+nshr); + stateOld = FMatrix (nblock, nstatev); + enerInternOld = FMatrix (nblock); + enerInelasOld = FMatrix (nblock); + tempNew = FMatrix (nblock); + stretchNew = FMatrix (nblock, ndir+nshr); + defgradNew = FMatrix (nblock,ndir+2*nshr); + fieldNew = FMatrix (nblock, nfieldv); + stressNew = FMatrix (nblock, ndir+nshr); + stateNew = ViewFMatrix (&strength_state_vars.host(elem_gid,0), nblock, nstatev); + enerInternNew = FMatrix (nblock); + enerInelasNew = FMatrix (nblock); + +} + +// VUMAT::~VUMAT() +// {} + +int VUMAT::calc_stress( + const DViewCArrayKokkos &elem_pres, + const DViewCArrayKokkos &elem_stress, + const size_t elem_gid, + const size_t mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const DViewCArrayKokkos &elem_sspd, + const double den, + const double sie, + const ViewCArrayKokkos &vel_grad, + const ViewCArrayKokkos &elem_node_gids, + const DViewCArrayKokkos &node_coords, + const DViewCArrayKokkos &node_vel, + const double vol, + const double dt_in, + const double rk_alpha, + const size_t cycle, + const size_t rk_level, + const double time) +{ + + //set dt + dt = dt_in; + + // slice stress view from elem_stress + ViewFMatrix stress_view (&elem_stress.host(rk_level,elem_gid,0,0), 3, 3); + + // stateOld: + for (int i = 1; i <= nblock; i++) { + for (int j = 1; j <= nstatev; j++) { + stateOld(i,j) = stateNew(i,j); + } + } + + // write vel_grad to Fortran array layout. + for (size_t i = 1; i <= 3; i++) { + for (size_t j = 1; j <= 3; j++) { + Fvel_grad(i,j) = vel_grad(i-1, j-1); + } + } + + // decompose vel_grad + for(size_t i = 1; i <= 3; i++) { + for(size_t j = 1; j <= 3; j++) { + D(i,j) = 0.5*(Fvel_grad(i,j) + Fvel_grad(j,i)); + W(i,j) = 0.5*(Fvel_grad(i,j) - Fvel_grad(j,i)); + } + } + + // calculate strainInc + for (int i = 1; i <= (ndir+nshr); i++) { + strainInc(1,i) = D(ind(i,1), ind(i,2)) * dt; + } + + // calculate relSpinInc + relSpinInc(1,1) = W(3,2) * dt; + relSpinInc(1,2) = W(1,3) * dt; + relSpinInc(1,3) = W(2,1) * dt; + + // stressOld: + + for (int i = 1; i <= (ndir+nshr); i++) { + stressOld(1,i) = stress_view(ind(i,1), ind(i,2)); + } + + //--------------------------------------------------------------------------------------------------- + //--------------------------------------------call vumat--------------------------------------------- + //--------------------------------------------------------------------------------------------------- + vumat_( + &nblock, &ndir, &nshr, &nstatev, &nfieldv, &nprops, &lanneal, + &stepTime, &totalTime, &dt, cmname, coordMp.pointer(), charLength.pointer(), + props.pointer(), density.pointer(), strainInc.pointer(), relSpinInc.pointer(), + tempOld.pointer(), stretchOld.pointer(), defgradOld.pointer(), fieldOld.pointer(), + stressOld.pointer(), stateOld.pointer(), enerInternOld.pointer(), enerInelasOld.pointer(), + tempNew.pointer(), stretchNew.pointer(), defgradNew.pointer(), fieldNew.pointer(), + stressNew.pointer(), stateNew.pointer(), enerInternNew.pointer(), enerInelasNew.pointer() ); + //--------------------------------------------------------------------------------------------------- + + + // stressNew: + for (int i = 1; i <= (ndir+nshr); i++) { + stress_view(ind(i,1), ind(i,2)) = stressNew(1,i); + stress_view(ind(i,2), ind(i,1)) = stressNew(1,i); + } + + // update stepTime, totalTime + stepTime += dt; + totalTime += dt; + + return 0; +} + diff --git a/src/Parallel-Solvers/Material-Models/strength/VUMAT/VUMAT.h b/src/Parallel-Solvers/Material-Models/strength/VUMAT/VUMAT.h new file mode 100644 index 000000000..7d4bb3e70 --- /dev/null +++ b/src/Parallel-Solvers/Material-Models/strength/VUMAT/VUMAT.h @@ -0,0 +1,108 @@ +#pragma once + +#include "Simulation_Parameters/Material.h" +#include "matar.h" +using namespace mtr; + + +/* VUMAT */ +class VUMAT { +public: + + VUMAT( + const DCArrayKokkos &material, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const size_t mat_id, + const size_t elem_gid); + + + // ~VUMAT(); + + int calc_stress( + const DViewCArrayKokkos &elem_pres, + const DViewCArrayKokkos &elem_stress, + const size_t elem_gid, + const size_t mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const DViewCArrayKokkos &elem_sspd, + const double den, + const double sie, + const ViewCArrayKokkos &vel_grad, + const ViewCArrayKokkos &elem_node_gids, + const DViewCArrayKokkos &node_coords, + const DViewCArrayKokkos &node_vel, + const double vol, + const double dt, + const double rk_alpha, + const size_t cycle, + const size_t rk_level, + const double time); + + +private: + + // Abaqus VUMAT parameters + int nblock; + int ndir; + int nshr; + int nstatev; + int nfieldv; + int nprops; + int lanneal; + double stepTime = 0.0; // reduction is done on this + double totalTime = 0.0; // reduction is done on this + double dt; + char cmname[80]; // fortran is expecting 80 char length + + // Abaqus VUMAT arrays parameters + FMatrix coordMp; + FMatrix charLength; + ViewFMatrix props; + FMatrix density; + FMatrix strainInc; + FMatrix relSpinInc; + FMatrix tempOld; + FMatrix stretchOld; + FMatrix defgradOld; + FMatrix fieldOld; + FMatrix stressOld; + FMatrix stateOld; + FMatrix enerInternOld; + FMatrix enerInelasOld; + FMatrix tempNew; + FMatrix stretchNew; + FMatrix defgradNew; + FMatrix fieldNew; + FMatrix stressNew; + ViewFMatrix stateNew; + FMatrix enerInternNew; + FMatrix enerInelasNew; + + // Other parameters for intermediate calculations + int ind_[6*2] = {1,2,3,1,2,3,1,2,3,2,3,1}; + ViewFMatrixKokkos ind {ind_,6,2}; + + FMatrix Fvel_grad {3,3}; // vel_grad in Fortran array layout + FMatrix D {3,3}; // symmetric part (D) of the velocity gradient (L) + FMatrix W {3,3}; // skew symmetric part (W) of the velocity gradient (L) + + FMatrix F_old {3,3}; // old deformation gradient + FMatrix R_old {3,3}; // old rotation matrix + FMatrix U_old {3,3}; // old right stretch tensor + + FMatrix F_new {3,3}; // new deformation gradient + FMatrix R_new {3,3}; // new rotation matrix + FMatrix U_new {3,3}; // new right stretch tensor + + +}; + + diff --git a/src/Parallel-Solvers/Material-Models/strength/VUMAT/VUMATStrengthModel.cpp b/src/Parallel-Solvers/Material-Models/strength/VUMAT/VUMATStrengthModel.cpp new file mode 100644 index 000000000..1f4db7350 --- /dev/null +++ b/src/Parallel-Solvers/Material-Models/strength/VUMAT/VUMATStrengthModel.cpp @@ -0,0 +1,129 @@ +#include "VUMATStrengthModel.h" +#include "VUMAT.h" +#include +#include + + +// to hold all vumat in each element +std::vector> elem_vumat; + +namespace VUMATStrengthModel +{ + void init_strength_state_vars( + const DCArrayKokkos &material, + const DViewCArrayKokkos &elem_mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const size_t num_elems) + { + /* + In this function, initialize the strength_state_vars according to the location this model needs it. + DO NOT strength_state_vars.update_host or strength_state_vars.update_device. + This is because different elements might have different models, run location, and + initialization location leading to overwriting of data. + */ + + // assign correct size to elem_vumat, all element are nullptr during initialization + elem_vumat = std::vector> (num_elems, nullptr); + + for (size_t elem_gid = 0; elem_gid < num_elems; elem_gid++) { + + size_t mat_id = elem_mat_id.host(elem_gid); + + // only fill the elem_vumat of elements that use this model + if (material.host(mat_id).strength_model == STRENGTH_MODEL::vumat) { + + // currently vumat only runs on host so check to see + if (material.host(mat_id).strength_run_location == RUN_LOCATION::device) { + throw std::runtime_error("VUMAT only runs on Host"); + } + + // create VUMAT model in element that used vumat + elem_vumat[elem_gid] = std::make_shared(material, + eos_state_vars, + strength_state_vars, + eos_global_vars, + strength_global_vars, + elem_user_output_vars, + mat_id, + elem_gid); + + } // end if (material.host(mat_id).strength_model... + + } // end for (size_t elem_gid = 0... + + return; + } + + void calc_stress ( + const DViewCArrayKokkos &elem_pres, + const DViewCArrayKokkos &elem_stress, + const size_t elem_gid, + const size_t mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const DViewCArrayKokkos &elem_sspd, + const double den, + const double sie, + const ViewCArrayKokkos &vel_grad, + const ViewCArrayKokkos &elem_node_gids, + const DViewCArrayKokkos &node_coords, + const DViewCArrayKokkos &node_vel, + const double vol, + const double dt, + const double rk_alpha, + const size_t cycle, + const size_t rk_level, + const double time) + { + if (elem_vumat[elem_gid] == nullptr) + { + throw std::runtime_error("VUMAT not initialized in this element"); + } + + elem_vumat[elem_gid]->calc_stress(elem_pres, + elem_stress, + elem_gid, + mat_id, + eos_state_vars, + strength_state_vars, + eos_global_vars, + strength_global_vars, + elem_user_output_vars, + elem_sspd, + den, + sie, + vel_grad, + elem_node_gids, + node_coords, + node_vel, + vol, + dt, + rk_alpha, + cycle, + rk_level, + time); + + return; + } + + void destroy( + const DCArrayKokkos &material, + const DViewCArrayKokkos &elem_mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const size_t num_elems) + { + + } + +} // end namespace VUMATStrengthModel \ No newline at end of file diff --git a/src/Parallel-Solvers/Material-Models/strength/VUMAT/VUMATStrengthModel.h b/src/Parallel-Solvers/Material-Models/strength/VUMAT/VUMATStrengthModel.h new file mode 100644 index 000000000..d567fe4c6 --- /dev/null +++ b/src/Parallel-Solvers/Material-Models/strength/VUMAT/VUMATStrengthModel.h @@ -0,0 +1,53 @@ +#pragma once + +#include "Simulation_Parameters/Material.h" +#include "matar.h" +using namespace mtr; + +namespace VUMATStrengthModel +{ + void init_strength_state_vars( + const DCArrayKokkos &material, + const DViewCArrayKokkos &elem_mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const size_t num_elems); + + void calc_stress ( + const DViewCArrayKokkos &elem_pres, + const DViewCArrayKokkos &elem_stress, + const size_t elem_gid, + const size_t mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const DViewCArrayKokkos &elem_sspd, + const double den, + const double sie, + const ViewCArrayKokkos &vel_grad, + const ViewCArrayKokkos &elem_node_gids, + const DViewCArrayKokkos &node_coords, + const DViewCArrayKokkos &node_vel, + const double vol, + const double dt, + const double rk_alpha, + const size_t cycle, + const size_t rk_level, + const double time); + + void destroy( + const DCArrayKokkos &material, + const DViewCArrayKokkos &elem_mat_id, + const DCArrayKokkos &eos_state_vars, + const DCArrayKokkos &strength_state_vars, + const DCArrayKokkos &eos_global_vars, + const DCArrayKokkos &strength_global_vars, + const DCArrayKokkos &elem_user_output_vars, + const size_t num_elems); + +} // end namespace VUMATStrengthModel \ No newline at end of file diff --git a/src/Parallel-Solvers/Material-Models/strength/VUMAT/lib_mod.mod b/src/Parallel-Solvers/Material-Models/strength/VUMAT/lib_mod.mod new file mode 100644 index 0000000000000000000000000000000000000000..19779ed7c1d494821227e84bbd2fdeaf37b6ed77 GIT binary patch literal 1466 zcmV;r1x5NFiwFP!000001MOSea;iuaeebW>Zji7i+J5=_dQrl{Pa$v4&<&TLG5$ zCyvIJVn;qGS||$H6QJ?w(v^ZsfarfF-M7p(I$sScHsmU;>y0OafE+cIm#7U<3xdA-RCuf>6c6^BF#B9oS5`j^3t?!Z`0&9<75WLo1=;^cO@K-T3QYoZ zeUsDlV$=MhnSt~7&kV^NIV7``!8J+NHaJgbhSFfF$qXDrvNsvMmWGqr+{zW;F2FD* zkfjVYR7kc+gcVB!v%-CCet4*$xQPHe9f;Hy4MgGqC2BH94$0{fRg+|0-xrw@wKSM& z0+FCAfoNYipDYj+ZzBknS}Rv3k$sC~5p5p?-sExiH1I1jN_maUAGryvTZhAB0U<0c z#h$k2^Tekjicbu`AHx=&^G36A!m(4UPg&^a@$>768w5!*u=ZUOZI8ZN4X+b-mjkxM zT|@3_c(R+cie^ZdCW+xi{1Co0#V6m|?psz$%qK0sqn2e{t+}rik*poXu&fTe$MIkp z!R!NDU9)oB*Xrxymu5D;>`TBYzHE&zlD^68h*ic`JztLkUC);?{aCwfy;|3{P3LCq zQj)#2hoEi?w>Q>)SuQrc5zWqH>|{)?#iPw~Uz_4;wh3hPuPwt50!Cpt2$RQZN^d9k z@Z#G!;1U{zi^-Rw@rcGc;x3URVrXkpbw+`Xt&{}JL#ovosVv{ByJhWFj zJn<_h(?@AS)L5CcDw-e!S#}1R5H|{DTk;SZUG06t?^LGA441&1Ec(36sMjjpT7_RY Uc@0b^D0xPH0d=WB8Q(qt0DwcoY5)KL literal 0 HcmV?d00001 diff --git a/src/Parallel-Solvers/Material-Models/strength/VUMAT/vpsc7_mod.mod b/src/Parallel-Solvers/Material-Models/strength/VUMAT/vpsc7_mod.mod new file mode 100644 index 0000000000000000000000000000000000000000..97ab6f94ff3976c4c739eb7ed94d38da1446fc7b GIT binary patch literal 5950 zcmV-E7s2QsiwFP!000001MOW)SK~;s-si8-&ajw!B&D89H@NF^h60X(np-O$l`TL$ z*yc*YQ1k02LJwP#Gxe|}r=U4go-SNA`XVyoi+n^xe*5!!dN&*RCv+FmgTG!%o7bfi{?CIglM`}sDs}N^`+H6a{1sj&BX9Mvo`t_(+>tce zyae;;@3V#bFp^aGB>a20;ru&=e^+;>YA%y1)A8)%Cxw{&8`2H@Uha>5n(l>$CLVw*QqS86cgQ z(#ZAwwLg+h@LS&|Q%S>rga4&JqZW6%O@Ha1mxaekm5P zg><4wGMW6}&E>_}#hr9=c71hwH-;yden|$C8^vGWeSuVbLyx{uEp(-hrSpsLldIc{ z>#I{~U;I<)=Kc?m1^{Khga1o^jseLD68LY%%ul!P%95$shO8^}lu$Got}&OIOl}Mb zOPrf){N!|YcK!VZ?Z=iz!Db2Cnoh=-l9hmue%Rt8eb|ixqqL}ca`cCE{%8ek_9bxU zuR@*O&a|hI2mv0cr$IJ8(olm8 zeC#1cZf!8W*+AX@_^QaiW$1{|!Bv5ygV^>4Ik+mY_@cx&M2{};{QI-_*UJ;%dFdX& z?4M5OPUT>U3L(J>_*kVAy%cM9-Ec&TR3VPMFM)wRh?XJbEFSqfRyjz!2!cD1q+F#K zbwdAlDN0g`@(A1nfhU+duk)whBXY1dkHAq7fL<5tu*r~JK_OP@hGdq=`NuJk5jou*V#NQV?Mr@__5H_k<0+ zT*!bM@pNweyeQx#4JUNHg+~gZXZ~0+p{p6mdykU4O{eTv*?Uf;bk9k%sG(C!J0)Q2 zHzoU{?e;16Hk~L0&{gnzxqitmz)9Hx{BVEg7$n5TRWxraYyglp&)>R60i!ajtI7CPSs(I8;G`m#U(6agp#0cZ$76$n5_08}6VBRWC|MC^xZ z@%c$HC>`h7@zIHpEpW|_b-otya9xfTdCqFHoV`2?L7?EYDTWX}&D`f2Fi7VFtgAT` zOXpn#2$~1stwewT67)h)fcst%5Bmh6Cm_6)KoCX(B63YZjI0AIdcHefZ`_K5l7}I-l{Kd2LCd@kwWK7=)-N0SdJ{IMK?F|%7czz!gLrc4 zGG-qjkTQSGz12n_2INdX*of4F87;0w0*IE@{;NO)YMf;ey6*x(08_5qXR#Qxc(LKF zX7lwb^w-r%AMKB6IZ%KtSC5runcW6G`f-ejhlH!S^AQd-O%u=H`B-ai*|rIbb@)lkb-7b`Hp7b%Yy~8CJ>t|XxY(mVw?$Dduqjn+1FXU4Y6t!zJWsPlu5MGX8Hp;K%2G_!32HoV3Gl9 zQ$=5Ta_SYGh|?w}VtNZbN1^1vERx=c-qs?Nbl601<8D)uD^@<@q$L}*8gbl?!t}tZ z+y=4fE_SqGwW!KJs2urY%tvhqM053HES_u(Mvu;G5U{dKYFoe&+)0t;6lz!jRdl6? z3^B=hetB|AvhjwJo`rBBHaU*+5zJMQAwW5AZy#V)s>oO-09ZeWVj)_U>2(MZRgocB zl`QXuq)trF+_>DF;h`uLlX~jxc#B*zby&7+E3$29R6iwP?zd$X^d0Ar!gE2>n`OwK zS$8n$21c6GoOpXfC)>Xw{!u57yw!}V@eDJaJIhyfWZ1? zEMUe0$ddhRvGvPX!0fNXnKOT78Y&sg>RZ7|=J#bd@Y*~D3!2}bD`1M=V^yNU=Kr|< zdZ?wt3f4Beug|VuR`EK3q8$Z`oSk=XzcQEPrijgsgZon}1cryQlIK0Jq!Mgjn?y=f z^8Co3y9`LJ5NqasP;y{3fGSc2i<^TIW+DTqC{&{A=EtYsi{VleM54!V9hTL>i4E&m z)cn4j^P|v^GE|hPx_O>Nqanw3tZ;T8!ncQ#fgeJ2#|mde2`oS4(t8CfoDpYG-pW9D zh{r1De}`|ulCfctFNG9)T%lm4^D2CEH|{E|JdfsLJ<2{M7CXN^mM@INok%KJ==@fg zC6UGnUA+x=hSSIMJSYD=(bo=H9dFYh}of@$_Q+$K|9}__+8UJI+6D$3G^K*^j4p7k|QA;9hs* z>9@&U3^P@y%201kz(gw40}7**3R89ANzcO2hQg>N!qn)b*|RiSLuqJV}k549XGXI8L63LUy9;B&o~5saYAZ zB#5x7hbNC{Jmdax_w!~F?Yd!On%R>R75bMKPsC#k(IC?OWnEM{Os|7Bd%gJ!}^fqhx?ZHGA2CRm*5pGmIC+h~qE}-qU z%(H-y7w792On2FRO|kapz}>*Pz5io+eSddxH6fGB39Jt8AkyC4Pbbp&@?Ybhx3ld! z5P!w22zh4@@f4n1|Cmgt7v~Wf+>BvPe|s}ILxH86n=|w@IkG!!;xc>iR!+W)lwzrS zW_zgVsDyuUiT!@d9Mp8@X zQTe0d$qfEK!kwJsK(!e1T4TB=32RLEcN@wz#%A!C!!16C*SyZL1&3p6nL%@SdfC91 z!?C-J5Dv$NF-#6;%?118aO?)d;&9ed^e7yT{a#ocZhtNg$38494!4!R8;4^Ll`js@ za=0n;IK2M#C@n}FyTuHe#M5%$Es1027$GE%&0tt0&Rh!jN8;G+g+=15_1{rQ9Q(R3 zNZhtQ8Hw9W5?AGYv;Ev8u3~SLFA~p@xXL5(1`P@hXK@wVzYLni(-g0i#Z_zqBZ9?M zZ0W*e@%}b47%Z-0uND@Im!BZ^$KopXW!YzORW?5ri>ufzg~{Uf-;KqwqsSMF=U80h zv3T9xeRfuA6DC&_vSsoi5_y^{uqASA&?1D$u`3Ic$orcJvxppfv#^M~zuox?BF9cG z3?f&I&qd_eY=uSSiuJc6a{G(Ob40F24Ou9crfBH(Ml@Ig1)@ry_B@ zjC{X7zrUQsPIf*yyX1ISj<{H^05}l(iK^<7sv7VzIdCzb)qQqD7h+Yo5r)=|&{CO= z65B!~C|_^`R1{3A#xN-drnZHdrhy8=smC}C4o)n0eECQYXYm~l>-62t`Z=14H?s^W zqjAzkwsSe{JIG*3tl!`4onk%bi>(d?LN;|FQF*oHTW+Mk7Qy?BO`^g2p5Vm3=`lSmbhf9fZ4W53f9N%GyS~~&6zPAdz`DyhqGnL3hIi60( zKW8xNlj-byax=L)pIn{&YyuiV7Gcx)S_N#W)o`N6-Xv^3(IY9q`b%DwIng7LKw2u0 z`hg@;WL0CxM3O;*NeG{}yL8X$Oz^yW8VN8}SqNe`8XYs8P}u@FwnP;fUJATac(GSt z6gOU9s#9c!Kr6T<>$_{;b64T)F)q~=68&hj3w%Nq6ZFcSWKbnj+1Rt8R##c&3>CNa zVq^j>RgIP$R8;{Fg&#F|>F_e(#h&?4+;|CUywrAX|y~k#YPpRljr6C7T@<9ney z3*Gnd)putPqL#o^i)~W_+te(0foPA+K(p0NGIWm>CwGTUMS`x?&L!`4qG1mBiu+brBLuqZyMyPkd_s8}5 zPu}@*^`U3wG&*wc6KVi68o=v@8hryrD++6KN49L~NDVsBku|`K4UQSx3V&qFr8NdO zvS7dlaEuY}N}y;(VJ*(cR%EzSQ?4&;$=H(2j$F&JjTeQ~1WPtm_HL0Z^MM@4tWlS+>|qXZfG7HQ9dV1-Q}XOU1pa81G*DQgNRurn}>%M9X$V zE$16{?{s@uODq*{JV|%dV?O5b6(f(K@ZJly=Qqw>Ybsx1!!7;aQ%9TY>S>MRXF- z!H_hy*Axie8RRY9g9oFCqnssK2(;vGRe=!n=+JjA);WMV=C9%U;h`HC&9#8HUE}O_ z?Jr#aP!NRUnsv0IWMb$6O+t6BnfU^vNoc8O>f0lxtZQlqQPknooq|yqG*QHI44Tll zmoT^2%GcbAX?HZfg`kao`QG=X-(AoA7V6F28TqB&ta^(fv=vJo7(%hbd~POj*a)|X zq1cK&n#{C!<2EQVeanw*>tF5A2iXpN-$ZQi*JvHaF_9MQ)TZ8>tW&YS^?VJJ&n+UO zZn_a0QFH5Uxi)Niu?J&Zuq`&$RCJ|-_M?J{m{EPlK=4?s-3-G#ezd{1%#7WPB9u?E zn_(o!1}vjB<-JX>Rs(Y2Y6)92d+V3bTkm48+0}mFf#zyzd#dRS(t-Pdre54M?Hp0V zO;gh<>?@l&>uEY`Yr2`7n2)6t^%5P}3e*c`|E%!B(ys9wXw2AucNP2&7T$6=m@qyq z*NZ`6iti0y4hoYIVH(!h!D?ycK8No6U?AE1q0H*~kgFkIW2RV+1q9F2v4iXC(H{;T z7AuLyq{R#{96&K81;YWfD~VXY?v@w!L9^%9Q%7v=8E>@r0jAkzJH-xlxrqoCM~R$> zU_}hgb1~(y~4w0&RCwg z7q&XXbRhMq8Y6582cDD(WHQ25OHcz2Ti`z2u!T+9G6qcBE!&T-gzKBk zM+xhrcYx`ZlT*Y}7Nv2(Ic@SCEm3e$$%(yDEErb5l-#e9F-Z7bDY-SrU|Ydm8ijZ` zKzU)t&yPJhJ=%$^!JF4$%fV0;r-8r2^qkr-J^wrC*yvry22aOW1~Ave=~(-0cKlfv za#*E-$LL*|O6wSeiRs>h_q@$!!{!!Z5m|d{-OeB#xYpI|;-+!uh#GDhVdY~_(=^q*vlQWU;4DoMX+>u-+d6PfXJG=Z?<|7E)}QcEMR1<{x#B2y>$IV`#N?;F`|D1X|x&Fm;;c zEVc7XkL4^Dt_EXS1E9H>8I-%PAozqfCod8J3l!Ok+20I@p0Ki{7&0mQXal_LGRV`kIh;P!um4o=6RlML2-lU+~!gYH1t`=WjhkrHm zSHIy~`ODO>vx`Lv*OFoBqMW<@3*XW6mu%r%#@^*sDAe{&t1bHNrBOTV+gq73$hVgr zMcD9H#g_=hzh!$<&yEl9j-2@B89BP)WqY3s{z!MXp%DC8+iqy$PkQ$H4t&GNUyOlw zX=r;FP~b1ZdntCHGdDk^ziM~m!tQtB_+@SgkmEZ8l1}`XaiQlrh+{{5b-&3xzDtiwFP!000001MOW~bK1xfexF~VH|&Q!rO_2V5?e`8YhxG2-jgRv0ohiT zi;pB@dw>1umJlGA8A*T`Ih#tHn2XTY-P2$9T)O%8^`<}QcY4ZnK7O6}%EAw$U_Mh? z)NCoE(D!10tUQMEsj_&TdhvfA9hID&oh!%qOR`_i3H%B_XRToNI3LCzFMdmDEncEg z^80Y?KeiMN-U+YA3(o60yiQs#;ru1yyu-jxh++_l_ZepL8Of6)e=^~`%US`>el(@k zE-JSNWm3maz(Mwg*sL)yVq{-u6xrvS9ZleS8ngW zfHVMT^BVkd=*kl_wy;gHGMkg&wLy}@_(hZi^3 zw>W+rr4=ovpsl{!xl-Co@X3P}K2pcHprD)Q7CAs9;QZ;)Ri0uvAvJ79oV5F(3qZq?`uX_((+!GVrlYjNDqmzMFx% ze*0?3zan%*s&O^o=qMd~jWn(XEIt^@&w!j zfhQVyucPOW7TH^yC*UXuK(FI@T*c&MA7vimL^{TyDBw;6NEJ=iWpN#%Q8$N1@cssbz2N2fN8-0~R&EWNK#22J z*oCm6g`+Qn7!a^tLscRG9RW~@01O16Nd#ab09_&gE7?MbWUPl8>Hf)fP&&=M)2$OJ zOW=l+>U=Ka=BAoV^1Rt%I((T(K|nEWN+HD0Lw`~N201zbn_3RV)O(i!LeGN;W->qk z2?jAJz<)1@hjoH5mmq?fL=e^zMBjH5YJv&#;gMbQs$3QFk48( zfSegd3z>SblgYJA0Lj!ke3ghmOHWzE{<}mFz?3t8A{T>^PBwzsa5SIA;k@|J$Kl(w z94Nq+v!}vgxw#B_^6eNKHwkAW?=x&@+BWXN^Rbp(vOOd$mfhF1lP$yN0<6r9O$mAh z3dj8B6X>-KOdorKb9~LwY8zrg{Ok+%X_}+eG|Mg`ss?Puw=oyT6d30u!b8E0INs2P zU=+W_&!I0^?QGDHH4$zkIG581rCPX~H2n;t)lNq1D1Q4KqZPO5rt`61$OLlF3bvhO zJ25>8S~|7j!mMkr-ho&(i{C(@Wy&ND(q{I6?4eDECNM#tYnY^g+SJgOp1pEKC-St3 z2Qh=OnWIp$=P8ofh~6<0lzOv?-oe$T^{y+#r1bV=zEZ~}Lts+vKK3!uub^iUxtd6SoCPjDKq zDaW%AF2p9!qn}~8QWXM}_xAP)W~EfcG6BH)NfZmwN~G5&L{zFmuo6t~nxsxVp1JV3 zXNH@iP(0LAX2o0Plk}IR6&2_Go)}|(DY&&^E2xsBz+t!WAQTF z`$b0GG*&N5E1X`PSn@_ASs3uZQXEWufFK8ev3X)j4g_$vp8!B&{W2CXV*#XKJzH-5 zG8Qn0^LXfuUYP@xG-mcqv6A_H8aKQ)k77ad`=kJ->@`-J6gL0khx4YE4pXdc_FpG{ zD5`jEK+z7xB4_VC*saVZxfHS5c5r=ag~0GADtX=kOA5i(wMayhlIJIW?$RK&LaZ5u zkzm8B0ac_Fi<=_>Gm!>V6w;)+`RVy%++1pcNc0rXV^JNP+^~*C&F|9@KMFPJgbGcn zn@7uN)a2NX70&)+{Prj?d=sKORyZR{VEHDO-czh_Mx2qjm4WmSk5$fp$8XV;v0#xe zg_OHpL9x<#7Qgume-;<+M{}{BWStU=o!_3s6C-&$k`xP_-?p+OQaJ?GXlaZ=-gK-A z!nx#~6RaK%9w%Pm=9%1s5frP8QI+$!*=&iW$kC{L;;!U#_fvXLQn8$GszUZwXnjtE zbtcTeCGO=?4}|Cq0xJ+!KUiLG@~twp{ul&71HDcQ}tXfs#h*bh$)V zrJa(LgbpjaljW~nq>X8{La_pT6pePf!8C1JgoWbZNq!U+VU;^GI^vF=r5TpmHnejB zH1bf;@cP)BdPzxqI2UZ{;jq}dg)}LDf!<2*$Gm`W?@$3!dWWMnh2)K5J*$Bl`=XAY z2kCZw0Xayw%J$5&;kq?4hWM@9qwChp5*ab|o}{IGp<=TO$bpJIYJB5$z_zkcD-OVG z&5{FDo4bJQuh|31{HXaNPqB7JwZ9d6T*YFy?{PF9`+UV3znWs}Pd_R^65fFYi90y< z&w+V?0$Z&ohtF_oo@JxvQjYBLMm~PfJf~z+MzUZ2t=Vl|Yuna2q1gImh>7+vm<91L zmwOM3zgC7kbo$-Smn*ka_~>4zj`PP|=fO>eeWyR@eubyNy#}5BH+PW2%+#qd)SDA< zDHZAgg`q-WioWoqW?}S-!e|0vN_5h!SsJ6FG`c{VGKV^A7Ki1?B5@3XI3;JaHA`ew zl*mMh7P0rEG6`gIlnTf@axB3s4};E^WmcL&rdC-Yl%)_hSBN6#uThA0MIjuH5G9c z-@3`F8#bmHK6^=_zqo+c@GM8U%Ol-n{+0ZJ&wZu8uaZs!S+Iq=+6A<#^X#Rr7UDlV; z_HGV*95{FPU-~!qgKp0y?v)F(g8@X^+xxz&bgusDe7_s6=7IPT!)?gx@sOtQ1{Qi8H6d^^rJsdts3{YyNi<634zS3=((RzZi)-OcK}B-C_G{ zlemVxQGSznj>I({iC1V)us@4y*#4!_ES??l%2-^(CNMHsT*H0y-BOq=?)=+W96O5qX7L=08$1>-ySmRlm0E?#DJ3f= zZz7RrX9ZS7jtyF*5IJ^bVG?Q0$`EgE938M!lriI@Kv!7K_!=Y$2L3XtUk6$2CzJXYRs`s5`o&8K-wpgh^m^wkcljf1X2H` zc!!_~ZlJ5#S?N!Jl`hd0&-NMtSKCF$Mtgm8dB1b)ti%+N0gjP~5e7IsiUad$oQhOB zVW#Q&rgnwPqC48%I0ajJVp38_&^F93bdzZbJdXpyCKHm1(ptE3P%at&a&!5eHy13P zpp?oe$eE_wfT|5YPI3i`wk@o~wX7&EYlhlT%Ni8GX>7|nly6zFw0hGlYrp_DO!%=< z%Ytaz!nQeTk*n$yl>1+ejHqXGNM6NKVPh(>X2D4?skDb~Wp)DaW+*Um8+5<^> z9PD{K3xH`viasXNtDA4#3wVJpkuDz>Ax&DmZ8%Gkd>+wR$`)`gho+(Bon;%&5uBwe zB4gWGj6)r`q_Z%AmUkBHiN>_jQ)`mb2K$b(ZqgY3a~(ww%R2 zy0cV7gnha4HZ%`);F8Y51X|u%u)!6}S@Q53;4G%jahCd!?&&PpC1}%GlIhVAouzE` zc{Vhyyt8b>IfAoPMbzGLmO~x5q_Z%AR&W-F<1BW=oCONcD!US60S+Inhh>Q<1F=&pot!!Fn1RmcwVUxKG9CfRACsVY{vzCNl#EtQn&nuBkpg16>j zQcDxOcS{0l>4G=!3P&x26m~1YxAB{rWMUC~8?SB1wnz@=uG`|+l=DW0D7H?D`!cAC zuzy2N7;cU5mNd4?yYY@V8th7jN}+C literal 0 HcmV?d00001 diff --git a/src/Parallel-Solvers/Material-Models/strength/VUMAT/vumat.f90 b/src/Parallel-Solvers/Material-Models/strength/VUMAT/vumat.f90 new file mode 100644 index 000000000..198783a6d --- /dev/null +++ b/src/Parallel-Solvers/Material-Models/strength/VUMAT/vumat.f90 @@ -0,0 +1,40 @@ +! +!************************************************************************ +!************************************************************************ +!************************************************************************ +! + subroutine vumat( & + nblock, ndir, nshr, nstatev, nfieldv, nprops, lanneal, & + stepTime, totalTime, dt, cmname, coordMp, charLength, & + props, density, strainInc, relSpinInc, & + tempOld, stretchOld, defgradOld, fieldOld, & + stressOld, stateOld, enerInternOld, enerInelasOld, & + tempNew, stretchNew, defgradNew, fieldNew, & + stressNew, stateNew, enerInternNew, enerInelasNew ) + +! include 'vaba_param.inc' + dimension propsv(nprops), density(nblock), coordMp(nblock,*), & + charLength(nblock), strainInc(nblock,ndir+nshr), & + relSpinInc(nblock,nshr), tempOld(nblock), & + stretchOld(nblock,ndir+nshr), & + defgradOld(nblock,ndir+nshr+nshr), & + fieldOld(nblock,nfieldv), stressOld(nblock,ndir+nshr), & + stateOld(nblock,nstatev), enerInternOld(nblock), & + enerInelasOld(nblock), tempNew(nblock), & + stretchNew(nblock,ndir+nshr), & + defgradNew(nblock,ndir+nshr+nshr), & + fieldNew(nblock,nfieldv), & + stressNew(nblock,ndir+nshr), stateNew(nblock,nstatev), & + enerInternNew(nblock), enerInelasNew(nblock) + + character*80 cmname + + +! loop over points + do 100 km = 1,nblock + 100 continue + + return + end + + diff --git a/src/Parallel-Solvers/Material-Models/strength/VUMAT/vumat.h b/src/Parallel-Solvers/Material-Models/strength/VUMAT/vumat.h new file mode 100644 index 000000000..58f60eb87 --- /dev/null +++ b/src/Parallel-Solvers/Material-Models/strength/VUMAT/vumat.h @@ -0,0 +1,18 @@ +#pragma once + +#ifdef __cplusplus +#define EXTERNC extern "C" +#else +#define EXTERNC +#endif + + +EXTERNC +void vumat_( + int* nblock, int* ndir, int* nshr, int* nstatev, int* nfieldv, int* nprops, int* lanneal, + double* stepTime, double* totalTime, double* dt, const char* cmname, double* coordMp, double* charLength, + double* props, double* density, double* strainInc, double* relSpinInc, + double* tempOld, double* stretchOld, double* defgradOld, double* fieldOld, + double* stressOld, double* stateOld, double* enerInternOld, double* enerInelasOld, + double* tempNew, double* stretchNew, double* defgradNew, double* fieldNew, + double* stressNew, double* stateNew, double* enerInternNew, double* enerInelasNew ); \ No newline at end of file diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.cpp b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.cpp index e09c67741..c5b60c44e 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.cpp @@ -979,8 +979,10 @@ void FEA_Module_Dynamic_Elasticity::setup(){ const DCArrayKokkos boundary = module_params->boundary; const DCArrayKokkos mat_fill = simparam->mat_fill; const DCArrayKokkos material = simparam->material; - global_vars = simparam->global_vars; - state_vars = DCArrayKokkos (rnum_elem, simparam->max_num_state_vars); + eos_global_vars = simparam->eos_global_vars; + strength_global_vars = simparam->strength_global_vars; + eos_state_vars = DCArrayKokkos (rnum_elem, simparam->max_num_eos_state_vars); + strength_state_vars = DCArrayKokkos (rnum_elem, simparam->max_num_strength_state_vars); elem_user_output_vars = DCArrayKokkos (rnum_elem, simparam->output_options.max_num_user_output_vars); //--- calculate bdy sets ---// @@ -1050,8 +1052,10 @@ void FEA_Module_Dynamic_Elasticity::setup(){ // function for initializing state_vars init_state_vars(material, elem_mat_id, - state_vars, - global_vars, + eos_state_vars, + strength_state_vars, + eos_global_vars, + strength_global_vars, elem_user_output_vars, rnum_elem); @@ -1059,8 +1063,10 @@ void FEA_Module_Dynamic_Elasticity::setup(){ init_strength_model(elem_strength, material, elem_mat_id, - state_vars, - global_vars, + eos_state_vars, + strength_state_vars, + eos_global_vars, + strength_global_vars, elem_user_output_vars, rnum_elem); @@ -1068,8 +1074,10 @@ void FEA_Module_Dynamic_Elasticity::setup(){ init_eos_model(elem_eos, material, elem_mat_id, - state_vars, - global_vars, + eos_state_vars, + strength_state_vars, + eos_global_vars, + strength_global_vars, elem_user_output_vars, rnum_elem); @@ -1134,29 +1142,30 @@ void FEA_Module_Dynamic_Elasticity::setup(){ elem_stress(rk_level,elem_gid,i,j) = 0.0; } } // end for - - // short form for clean code - EOSParent * eos_model = elem_eos(elem_gid).model; // --- Pressure --- - eos_model->calc_pressure(elem_pres, + elem_eos(elem_gid).calc_pressure(elem_pres, elem_stress, elem_gid, elem_mat_id(elem_gid), - state_vars, - global_vars, + eos_state_vars, + strength_state_vars, + eos_global_vars, + strength_global_vars, elem_user_output_vars, elem_sspd, elem_den(elem_gid), elem_sie(rk_level,elem_gid)); // --- Sound speed --- - eos_model->calc_sound_speed(elem_pres, + elem_eos(elem_gid).calc_sound_speed(elem_pres, elem_stress, elem_gid, elem_mat_id(elem_gid), - state_vars, - global_vars, + eos_state_vars, + strength_state_vars, + eos_global_vars, + strength_global_vars, elem_user_output_vars, elem_sspd, elem_den(elem_gid), @@ -1273,7 +1282,7 @@ void FEA_Module_Dynamic_Elasticity::setup(){ // p = rho*ie*(gamma - 1) size_t mat_id = f_id; - double gamma = global_vars(mat_id,0); // gamma value + double gamma = eos_global_vars(mat_id,0); // gamma value elem_sie(rk_level, elem_gid) = elem_pres(elem_gid)/(mat_fill(f_id).den*(gamma - 1.0)); } // end if @@ -1414,8 +1423,10 @@ void FEA_Module_Dynamic_Elasticity::cleanup_material_models() { destroy_strength_model(elem_strength, material, elem_mat_id, - state_vars, - global_vars, + eos_state_vars, + strength_state_vars, + eos_global_vars, + strength_global_vars, elem_user_output_vars, rnum_elem); @@ -1423,8 +1434,10 @@ void FEA_Module_Dynamic_Elasticity::cleanup_material_models() { destroy_eos_model(elem_eos, material, elem_mat_id, - state_vars, - global_vars, + eos_state_vars, + strength_state_vars, + eos_global_vars, + strength_global_vars, elem_user_output_vars, rnum_elem); return; diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.h b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.h index fc8ac9d3a..ac1c0e6a2 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.h +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.h @@ -558,10 +558,12 @@ class FEA_Module_Dynamic_Elasticity: public FEA_Module { DCArrayKokkos elem_vel_grad; // for storing global variables used in user material model - DCArrayKokkos global_vars; + DCArrayKokkos eos_global_vars; + DCArrayKokkos strength_global_vars; // for storing state variables used in user material model - DCArrayKokkos state_vars; + DCArrayKokkos eos_state_vars; + DCArrayKokkos strength_state_vars; //elem_user_output_vars allow users to output variables of interest per element DCArrayKokkos elem_user_output_vars; diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/elastic_optimization.cpp b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/elastic_optimization.cpp index b974a47f5..2917794d1 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/elastic_optimization.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/elastic_optimization.cpp @@ -243,29 +243,30 @@ void FEA_Module_Dynamic_Elasticity::update_forward_solve(Teuchos::RCP elem_stress(rk_level,elem_gid,i,j) = 0.0; } } // end for - - // short form for clean code - EOSParent * eos_model = elem_eos(elem_gid).model; // --- Pressure --- - eos_model->calc_pressure(elem_pres, + elem_eos(elem_gid).calc_pressure(elem_pres, elem_stress, elem_gid, elem_mat_id(elem_gid), - state_vars, - global_vars, + eos_state_vars, + strength_state_vars, + eos_global_vars, + strength_global_vars, elem_user_output_vars, elem_sspd, elem_den(elem_gid), elem_sie(rk_level,elem_gid)); // --- Sound speed --- - eos_model->calc_sound_speed(elem_pres, + elem_eos(elem_gid).calc_sound_speed(elem_pres, elem_stress, elem_gid, elem_mat_id(elem_gid), - state_vars, - global_vars, + eos_state_vars, + strength_state_vars, + eos_global_vars, + strength_global_vars, elem_user_output_vars, elem_sspd, elem_den(elem_gid), @@ -382,7 +383,7 @@ void FEA_Module_Dynamic_Elasticity::update_forward_solve(Teuchos::RCP // p = rho*ie*(gamma - 1) size_t mat_id = f_id; - double gamma = global_vars(mat_id,0); // gamma value + double gamma = eos_global_vars(mat_id,0); // gamma value elem_sie(rk_level, elem_gid) = elem_pres(elem_gid)/(mat_fill(f_id).den*(gamma - 1.0)); } // end if diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/properties.cpp b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/properties.cpp index 9eef6b49e..3bc753d82 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/properties.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/properties.cpp @@ -76,8 +76,6 @@ void FEA_Module_Dynamic_Elasticity::update_state(const DCArrayKokkos calc_stress(elem_pres, + elem_strength(elem_gid).calc_stress(elem_pres, elem_stress, elem_gid, mat_id, - state_vars, - global_vars, + eos_state_vars, + strength_state_vars, + eos_global_vars, + strength_global_vars, elem_user_output_vars, elem_sspd, elem_den(elem_gid), @@ -105,31 +105,34 @@ void FEA_Module_Dynamic_Elasticity::update_state(const DCArrayKokkos calc_pressure(elem_pres, + elem_eos(elem_gid).calc_pressure(elem_pres, elem_stress, elem_gid, elem_mat_id(elem_gid), - state_vars, - global_vars, + eos_state_vars, + strength_state_vars, + eos_global_vars, + strength_global_vars, elem_user_output_vars, elem_sspd, elem_den(elem_gid), elem_sie(rk_level,elem_gid)); // --- Sound speed --- - eos_model->calc_sound_speed(elem_pres, + elem_eos(elem_gid).calc_sound_speed(elem_pres, elem_stress, elem_gid, elem_mat_id(elem_gid), - state_vars, - global_vars, + eos_state_vars, + strength_state_vars, + eos_global_vars, + strength_global_vars, elem_user_output_vars, elem_sspd, elem_den(elem_gid), @@ -214,16 +217,16 @@ void FEA_Module_Dynamic_Elasticity::update_state2D(const DCArrayKokkos calc_stress(elem_pres, + elem_strength(elem_gid).calc_stress(elem_pres, elem_stress, elem_gid, mat_id, - state_vars, - global_vars, + eos_state_vars, + strength_state_vars, + eos_global_vars, + strength_global_vars, elem_user_output_vars, elem_sspd, elem_den(elem_gid), @@ -236,31 +239,34 @@ void FEA_Module_Dynamic_Elasticity::update_state2D(const DCArrayKokkos calc_pressure(elem_pres, + elem_eos(elem_gid).calc_pressure(elem_pres, elem_stress, elem_gid, elem_mat_id(elem_gid), - state_vars, - global_vars, + eos_state_vars, + strength_state_vars, + eos_global_vars, + strength_global_vars, elem_user_output_vars, elem_sspd, elem_den(elem_gid), elem_sie(rk_level,elem_gid)); // --- Sound speed --- - eos_model->calc_sound_speed(elem_pres, + elem_eos(elem_gid).calc_sound_speed(elem_pres, elem_stress, elem_gid, elem_mat_id(elem_gid), - state_vars, - global_vars, + eos_state_vars, + strength_state_vars, + eos_global_vars, + strength_global_vars, elem_user_output_vars, elem_sspd, elem_den(elem_gid), diff --git a/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/FEA_Module_Eulerian.cpp b/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/FEA_Module_Eulerian.cpp index 6602f6c77..a8607660a 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/FEA_Module_Eulerian.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/FEA_Module_Eulerian.cpp @@ -573,8 +573,10 @@ void FEA_Module_Eulerian::setup(){ const DCArrayKokkos mat_fill = simparam->mat_fill; const DCArrayKokkos boundary = simparam->boundary; const DCArrayKokkos material = simparam->material; - global_vars = simparam->global_vars; - state_vars = DCArrayKokkos (rnum_elem, simparam->max_num_state_vars); + eos_global_vars = simparam->eos_global_vars; + strength_global_vars = simparam->strength_global_vars; + eos_state_vars = DCArrayKokkos (rnum_elem, simparam->max_num_eos_state_vars); + strength_state_vars = DCArrayKokkos (rnum_elem, simparam->max_num_strength_state_vars); //--- calculate bdy sets ---// mesh.num_nodes_in_patch = 2*(num_dim-1); // 2 (2D) or 4 (3D) diff --git a/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/FEA_Module_Eulerian.h b/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/FEA_Module_Eulerian.h index 3841c6062..07181c7cb 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/FEA_Module_Eulerian.h +++ b/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/FEA_Module_Eulerian.h @@ -269,10 +269,12 @@ class FEA_Module_Eulerian: public FEA_Module{ DCArrayKokkos elem_vel_grad; // for storing global variables used in user material model - DCArrayKokkos global_vars; + DCArrayKokkos eos_global_vars; + DCArrayKokkos strength_global_vars; // for storing global variables used in user material model - DCArrayKokkos state_vars; + DCArrayKokkos eos_state_vars; + DCArrayKokkos strength_state_vars; // Dual Views of the corner struct variables DViewCArrayKokkos corner_force; diff --git a/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/Simulation_Parameters_Eulerian.h b/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/Simulation_Parameters_Eulerian.h index ace620341..48f2d12fb 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/Simulation_Parameters_Eulerian.h +++ b/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/Simulation_Parameters_Eulerian.h @@ -69,24 +69,32 @@ struct Simulation_Parameters_Eulerian : Simulation_Parameters { //Non-serialized fields int num_gauss_points = 2; - size_t max_num_global_vars; + size_t max_num_eos_global_vars; + size_t max_num_strength_global_vars; size_t rk_num_bins; double time_value = 0.0; - DCArrayKokkos global_vars; + DCArrayKokkos eos_global_vars; + DCArrayKokkos strength_global_vars; DCArrayKokkos mat_fill; DCArrayKokkos material; DCArrayKokkos boundary; std::vector gravity_vector {9.81, 0., 0.}; - void init_material_variable_arrays(size_t nglobal_vars) { - global_vars = DCArrayKokkos (material_options.size(), nglobal_vars); + void init_material_variable_arrays(size_t num_eos_global_vars, size_t num_strength_global_vars) { + eos_global_vars = DCArrayKokkos (material_options.size(), num_eos_global_vars); + strength_global_vars = DCArrayKokkos (material_options.size(), num_strength_global_vars); for (size_t i = 0; i < material_options.size(); i++) { auto mat = material_options[i]; - for (size_t j = 0; j < mat.global_vars.size(); j++) - global_vars.host(i, j) = mat.global_vars[j]; + for (size_t j = 0; j < mat.eos_global_vars.size(); j++) { + eos_global_vars.host(i, j) = mat.eos_global_vars[j]; + } + + for (size_t j = 0; j < mat.strength_global_vars.size(); j++) { + strength_global_vars.host(i, j) = mat.strength_global_vars[j]; + } } } @@ -96,11 +104,14 @@ struct Simulation_Parameters_Eulerian : Simulation_Parameters { array.host(i) = *(T*)&vec[i]; } void derive_kokkos_arrays() { - max_num_global_vars = 0; - for (auto mo : material_options) - max_num_global_vars = std::max(max_num_global_vars, mo.global_vars.size()); + max_num_eos_global_vars = 0; + max_num_strength_global_vars = 0; + for (auto mo : material_options) { + max_num_eos_global_vars = std::max(max_num_eos_global_vars, mo.eos_global_vars.size()); + max_num_strength_global_vars = std::max(max_num_strength_global_vars, mo.strength_global_vars.size()); + } - init_material_variable_arrays(max_num_global_vars); + init_material_variable_arrays(max_num_eos_global_vars, max_num_strength_global_vars); from_vector(mat_fill, region_options); from_vector(material, material_options); @@ -110,7 +121,8 @@ struct Simulation_Parameters_Eulerian : Simulation_Parameters { mat_fill.update_device(); boundary.update_device(); material.update_device(); - global_vars.update_device(); + eos_global_vars.update_device(); + strength_global_vars.update_device(); } diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/FEA_Module_SGH.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/FEA_Module_SGH.cpp index f36f7c0f3..e974ff2f7 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/FEA_Module_SGH.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/FEA_Module_SGH.cpp @@ -1019,8 +1019,10 @@ void FEA_Module_SGH::setup(){ const DCArrayKokkos boundary = module_params->boundary; const DCArrayKokkos mat_fill = simparam->mat_fill; const DCArrayKokkos material = simparam->material; - global_vars = simparam->global_vars; - state_vars = DCArrayKokkos (rnum_elem, simparam->max_num_state_vars); + eos_global_vars = simparam->eos_global_vars; + strength_global_vars = simparam->strength_global_vars; + eos_state_vars = DCArrayKokkos (rnum_elem, simparam->max_num_eos_state_vars); + strength_state_vars = DCArrayKokkos (rnum_elem, simparam->max_num_strength_state_vars); elem_user_output_vars = DCArrayKokkos (rnum_elem, simparam->output_options.max_num_user_output_vars); //--- calculate bdy sets ---// @@ -1090,8 +1092,10 @@ void FEA_Module_SGH::setup(){ // function for initializing state_vars init_state_vars(material, elem_mat_id, - state_vars, - global_vars, + eos_state_vars, + strength_state_vars, + eos_global_vars, + strength_global_vars, elem_user_output_vars, rnum_elem); @@ -1099,8 +1103,10 @@ void FEA_Module_SGH::setup(){ init_strength_model(elem_strength, material, elem_mat_id, - state_vars, - global_vars, + eos_state_vars, + strength_state_vars, + eos_global_vars, + strength_global_vars, elem_user_output_vars, rnum_elem); @@ -1108,8 +1114,10 @@ void FEA_Module_SGH::setup(){ init_eos_model(elem_eos, material, elem_mat_id, - state_vars, - global_vars, + eos_state_vars, + strength_state_vars, + eos_global_vars, + strength_global_vars, elem_user_output_vars, rnum_elem); @@ -1176,27 +1184,29 @@ void FEA_Module_SGH::setup(){ } } // end for - // short form for clean code - EOSParent * eos_model = elem_eos(elem_gid).model; // --- Pressure --- - eos_model->calc_pressure(elem_pres, + elem_eos(elem_gid).calc_pressure(elem_pres, elem_stress, elem_gid, elem_mat_id(elem_gid), - state_vars, - global_vars, + eos_state_vars, + strength_state_vars, + eos_global_vars, + strength_global_vars, elem_user_output_vars, elem_sspd, elem_den(elem_gid), elem_sie(rk_level,elem_gid)); // --- Sound speed --- - eos_model->calc_sound_speed(elem_pres, + elem_eos(elem_gid).calc_sound_speed(elem_pres, elem_stress, elem_gid, elem_mat_id(elem_gid), - state_vars, - global_vars, + eos_state_vars, + strength_state_vars, + eos_global_vars, + strength_global_vars, elem_user_output_vars, elem_sspd, elem_den(elem_gid), @@ -1313,7 +1323,7 @@ void FEA_Module_SGH::setup(){ // p = rho*ie*(gamma - 1) size_t mat_id = f_id; - double gamma = global_vars(mat_id,0); // gamma value + double gamma = eos_global_vars(mat_id,0); // gamma value elem_sie(rk_level, elem_gid) = elem_pres(elem_gid)/(mat_fill(f_id).den*(gamma - 1.0)); } // end if @@ -1454,8 +1464,10 @@ void FEA_Module_SGH::cleanup_material_models() { destroy_strength_model(elem_strength, material, elem_mat_id, - state_vars, - global_vars, + eos_state_vars, + strength_state_vars, + eos_global_vars, + strength_global_vars, elem_user_output_vars, rnum_elem); @@ -1463,8 +1475,10 @@ void FEA_Module_SGH::cleanup_material_models() { destroy_eos_model(elem_eos, material, elem_mat_id, - state_vars, - global_vars, + eos_state_vars, + strength_state_vars, + eos_global_vars, + strength_global_vars, elem_user_output_vars, rnum_elem); return; diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/FEA_Module_SGH.h b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/FEA_Module_SGH.h index b0cfb5d3d..41d2ca627 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/FEA_Module_SGH.h +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/FEA_Module_SGH.h @@ -655,10 +655,13 @@ class FEA_Module_SGH: public FEA_Module{ DCArrayKokkos elem_vel_grad; // for storing global variables used in user material model - DCArrayKokkos global_vars; + DCArrayKokkos eos_global_vars; + DCArrayKokkos strength_global_vars; // for storing state variables used in user material model - DCArrayKokkos state_vars; + DCArrayKokkos eos_state_vars; + DCArrayKokkos strength_state_vars; + //elem_user_output_vars allow users to output variables of interest per element DCArrayKokkos elem_user_output_vars; diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/force_gradients_sgh.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/force_gradients_sgh.cpp index be2231656..9b05751fd 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/force_gradients_sgh.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/force_gradients_sgh.cpp @@ -111,8 +111,6 @@ void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos &m ViewCArrayKokkos vel_star_gradient(vel_star_gradient_array, num_nodes_in_elem, num_dims); ViewCArrayKokkos vel_grad(vel_grad_array, num_dims, num_dims); - EOSParent* eos_model = elem_eos(elem_gid).model; - // --- abviatations of variables --- // element volume @@ -539,8 +537,6 @@ void FEA_Module_SGH::get_force_egradient_sgh(const DCArrayKokkos &m ViewCArrayKokkos vel_star_gradient(vel_star_gradient_array, num_dims); ViewCArrayKokkos vel_grad(vel_grad_array, num_dims, num_dims); - EOSParent* eos_model = elem_eos(elem_gid).model; - // --- abviatations of variables --- // element volume @@ -609,12 +605,14 @@ void FEA_Module_SGH::get_force_egradient_sgh(const DCArrayKokkos &m // add the pressure for (int i = 0; i < num_dims; i++){ tau(i, i) -= elem_pres(elem_gid); - tau_gradient(i, i) = -eos_model->calc_pressure_gradient_internal_energy(elem_pres, + tau_gradient(i, i) = -elem_eos(elem_gid).calc_pressure_gradient_internal_energy(elem_pres, elem_stress, elem_gid, elem_mat_id(elem_gid), - state_vars, - global_vars, + eos_state_vars, + strength_state_vars, + eos_global_vars, + strength_global_vars, elem_user_output_vars, elem_sspd, elem_den(elem_gid), @@ -661,12 +659,14 @@ void FEA_Module_SGH::get_force_egradient_sgh(const DCArrayKokkos &m // --- Sound speed --- real_t sound_speed_gradient_energy = - eos_model->calc_sound_speed_gradient_internal_energy(elem_pres, + elem_eos(elem_gid).calc_sound_speed_gradient_internal_energy(elem_pres, elem_stress, elem_gid, elem_mat_id(elem_gid), - state_vars, - global_vars, + eos_state_vars, + strength_state_vars, + eos_global_vars, + strength_global_vars, elem_user_output_vars, elem_sspd, elem_den(elem_gid), @@ -993,8 +993,6 @@ void FEA_Module_SGH::get_force_ugradient_sgh(const DCArrayKokkos &m ViewCArrayKokkos vel_star_gradient(vel_star_gradient_array, num_dims, num_nodes_in_elem, num_dims); ViewCArrayKokkos vel_grad(vel_grad_array, num_dims, num_dims); - EOSParent* eos_model = elem_eos(elem_gid).model; - // --- abviatations of variables --- // element volume @@ -1710,8 +1708,6 @@ void FEA_Module_SGH::get_force_dgradient_sgh(const DCArrayKokkos &m ViewCArrayKokkos vel_star_gradient(vel_star_gradient_array, num_dims); ViewCArrayKokkos vel_grad(vel_grad_array, num_dims, num_dims); - EOSParent* eos_model = elem_eos(elem_gid).model; - // --- abviatations of variables --- // element volume diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/force_sgh.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/force_sgh.cpp index 2ca9ef946..54d8434d6 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/force_sgh.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/force_sgh.cpp @@ -386,16 +386,16 @@ void FEA_Module_SGH::get_force_sgh(const DCArrayKokkos &material, // cut out the node_gids for this element ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 8); - - StrengthParent* strength_model = elem_strength(elem_gid).model; // --- call strength model --- - strength_model->calc_stress(elem_pres, + elem_strength(elem_gid).calc_stress(elem_pres, elem_stress, elem_gid, mat_id, - state_vars, - global_vars, + eos_state_vars, + strength_state_vars, + eos_global_vars, + strength_global_vars, elem_user_output_vars, elem_sspd, elem_den(elem_gid), @@ -408,7 +408,8 @@ void FEA_Module_SGH::get_force_sgh(const DCArrayKokkos &material, dt, rk_alpha, cycle, - rk_level); + rk_level, + time_value); } // end logical for strength run location } // end logical on hypo strength model @@ -446,15 +447,15 @@ void FEA_Module_SGH::get_force_sgh(const DCArrayKokkos &material, // cut out vel_grad ViewCArrayKokkos vel_grad(&elem_vel_grad.host(elem_gid,0,0), num_dims, num_dims); - StrengthParent* strength_model = elem_strength.host(elem_gid).model; - // --- call strength model --- - strength_model->calc_stress(elem_pres, + elem_strength(elem_gid).calc_stress(elem_pres, elem_stress, elem_gid, mat_id, - state_vars, - global_vars, + eos_state_vars, + strength_state_vars, + eos_global_vars, + strength_global_vars, elem_user_output_vars, elem_sspd, elem_den.host(elem_gid), @@ -467,7 +468,8 @@ void FEA_Module_SGH::get_force_sgh(const DCArrayKokkos &material, dt, rk_alpha, cycle, - rk_level); + rk_level, + time_value); } // end logical for strength run location @@ -848,15 +850,15 @@ void FEA_Module_SGH::get_force_sgh2D(const DCArrayKokkos &material, // cut out the node_gids for this element ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 4); - StrengthParent* strength_model = elem_strength(elem_gid).model; - // --- call strength model --- - strength_model->calc_stress(elem_pres, + elem_strength(elem_gid).calc_stress(elem_pres, elem_stress, elem_gid, mat_id, - state_vars, - global_vars, + eos_state_vars, + strength_state_vars, + eos_global_vars, + strength_global_vars, elem_user_output_vars, elem_sspd, elem_den(elem_gid), @@ -869,7 +871,8 @@ void FEA_Module_SGH::get_force_sgh2D(const DCArrayKokkos &material, dt, rk_alpha, cycle, - rk_level); + rk_level, + time_value); } // end logical on hypo strength model diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/properties.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/properties.cpp index 9719963c9..2c85b0517 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/properties.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/properties.cpp @@ -76,16 +76,16 @@ void FEA_Module_SGH::update_state(const DCArrayKokkos &material, elem_vol(elem_gid), elem_gid, rk_level); - - StrengthParent* strength_model = elem_strength(elem_gid).model; // --- call strength model --- - strength_model->calc_stress(elem_pres, + elem_strength(elem_gid).calc_stress(elem_pres, elem_stress, elem_gid, mat_id, - state_vars, - global_vars, + eos_state_vars, + strength_state_vars, + eos_global_vars, + strength_global_vars, elem_user_output_vars, elem_sspd, elem_den(elem_gid), @@ -98,31 +98,34 @@ void FEA_Module_SGH::update_state(const DCArrayKokkos &material, dt, rk_alpha, cycle, - rk_level); + rk_level, + time_value); } // end logical on hyper strength model - EOSParent* eos_model = elem_eos(elem_gid).model; - // --- Pressure --- - eos_model->calc_pressure(elem_pres, + elem_eos(elem_gid).calc_pressure(elem_pres, elem_stress, elem_gid, elem_mat_id(elem_gid), - state_vars, - global_vars, + eos_state_vars, + strength_state_vars, + eos_global_vars, + strength_global_vars, elem_user_output_vars, elem_sspd, elem_den(elem_gid), elem_sie(rk_level,elem_gid)); // --- Sound speed --- - eos_model->calc_sound_speed(elem_pres, + elem_eos(elem_gid).calc_sound_speed(elem_pres, elem_stress, elem_gid, elem_mat_id(elem_gid), - state_vars, - global_vars, + eos_state_vars, + strength_state_vars, + eos_global_vars, + strength_global_vars, elem_user_output_vars, elem_sspd, elem_den(elem_gid), @@ -207,16 +210,16 @@ void FEA_Module_SGH::update_state2D(const DCArrayKokkos &material, elem_vol(elem_gid), elem_gid, rk_level); - - StrengthParent* strength_model = elem_strength(elem_gid).model; // --- call strength model --- - strength_model->calc_stress(elem_pres, + elem_strength(elem_gid).calc_stress(elem_pres, elem_stress, elem_gid, mat_id, - state_vars, - global_vars, + eos_state_vars, + strength_state_vars, + eos_global_vars, + strength_global_vars, elem_user_output_vars, elem_sspd, elem_den(elem_gid), @@ -229,31 +232,34 @@ void FEA_Module_SGH::update_state2D(const DCArrayKokkos &material, dt, rk_alpha, cycle, - rk_level); + rk_level, + time_value); } // end logical on hyper strength model - EOSParent* eos_model = elem_eos(elem_gid).model; - // --- Pressure --- - eos_model->calc_pressure(elem_pres, + elem_eos(elem_gid).calc_pressure(elem_pres, elem_stress, elem_gid, elem_mat_id(elem_gid), - state_vars, - global_vars, + eos_state_vars, + strength_state_vars, + eos_global_vars, + strength_global_vars, elem_user_output_vars, elem_sspd, elem_den(elem_gid), elem_sie(rk_level,elem_gid)); // --- Sound speed --- - eos_model->calc_sound_speed(elem_pres, + elem_eos(elem_gid).calc_sound_speed(elem_pres, elem_stress, elem_gid, elem_mat_id(elem_gid), - state_vars, - global_vars, + eos_state_vars, + strength_state_vars, + eos_global_vars, + strength_global_vars, elem_user_output_vars, elem_sspd, elem_den(elem_gid), diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/sgh_optimization.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/sgh_optimization.cpp index e15e848d2..10fb822b9 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/sgh_optimization.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/sgh_optimization.cpp @@ -243,29 +243,30 @@ void FEA_Module_SGH::update_forward_solve(Teuchos::RCP zp){ elem_stress(rk_level,elem_gid,i,j) = 0.0; } } // end for - - // short form for clean code - EOSParent * eos_model = elem_eos(elem_gid).model; // --- Pressure --- - eos_model->calc_pressure(elem_pres, + elem_eos(elem_gid).calc_pressure(elem_pres, elem_stress, elem_gid, elem_mat_id(elem_gid), - state_vars, - global_vars, + eos_state_vars, + strength_state_vars, + eos_global_vars, + strength_global_vars, elem_user_output_vars, elem_sspd, elem_den(elem_gid), elem_sie(rk_level,elem_gid)); // --- Sound speed --- - eos_model->calc_sound_speed(elem_pres, + elem_eos(elem_gid).calc_sound_speed(elem_pres, elem_stress, elem_gid, elem_mat_id(elem_gid), - state_vars, - global_vars, + eos_state_vars, + strength_state_vars, + eos_global_vars, + strength_global_vars, elem_user_output_vars, elem_sspd, elem_den(elem_gid), @@ -382,7 +383,7 @@ void FEA_Module_SGH::update_forward_solve(Teuchos::RCP zp){ // p = rho*ie*(gamma - 1) size_t mat_id = f_id; - double gamma = global_vars(mat_id,0); // gamma value + double gamma = eos_global_vars(mat_id,0); // gamma value elem_sie(rk_level, elem_gid) = elem_pres(elem_gid)/(mat_fill(f_id).den*(gamma - 1.0)); } // end if diff --git a/src/Parallel-Solvers/Parallel-Explicit/example_simple.yaml b/src/Parallel-Solvers/Parallel-Explicit/example_simple.yaml index 86c77388a..362b4568a 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/example_simple.yaml +++ b/src/Parallel-Solvers/Parallel-Explicit/example_simple.yaml @@ -67,7 +67,7 @@ materials: q2: 0 q1ex: 1.0 q2ex: 0.0 - global_vars: + eos_global_vars: - 1.666666666666667 - 1.0E-14 - 1.0 diff --git a/src/Parallel-Solvers/Parallel-Explicit/example_simple_opt.yaml b/src/Parallel-Solvers/Parallel-Explicit/example_simple_opt.yaml index ae9ad2194..dd9233a6f 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/example_simple_opt.yaml +++ b/src/Parallel-Solvers/Parallel-Explicit/example_simple_opt.yaml @@ -61,7 +61,7 @@ materials: q2: 0 q1ex: 1.0 q2ex: 0.0 - global_vars: + eos_global_vars: - 1.666666666666667 - 1.0E-14 - 1.0 diff --git a/src/Parallel-Solvers/Simulation_Parameters/Material.h b/src/Parallel-Solvers/Simulation_Parameters/Material.h index 0fe57cddf..7406c0ace 100644 --- a/src/Parallel-Solvers/Simulation_Parameters/Material.h +++ b/src/Parallel-Solvers/Simulation_Parameters/Material.h @@ -1,8 +1,8 @@ #pragma once #include "yaml-serializable.h" -SERIALIZABLE_ENUM(EOS_MODEL, ideal_gas, user_eos_model) -SERIALIZABLE_ENUM(STRENGTH_MODEL, none, ideal_gas, user_strength_model) +SERIALIZABLE_ENUM(EOS_MODEL, ideal_gas, user_defined, constant) +SERIALIZABLE_ENUM(STRENGTH_MODEL, none, user_defined, vumat, evp, evpfft, ls_evpfft) SERIALIZABLE_ENUM(STRENGTH_TYPE, none, hypo, hyper) SERIALIZABLE_ENUM(RUN_LOCATION, device, host) @@ -32,18 +32,22 @@ struct material_t { double q2ex = 0.0; bool maximum_limiter = false; - size_t num_state_vars = 0; + size_t num_eos_state_vars = 0; + size_t num_strength_state_vars = 0; // Non-serialized fields - size_t num_global_vars = 0; + size_t num_eos_global_vars = 0; + size_t num_strength_global_vars = 0; }; struct Material : Yaml::DerivedFields, material_t { size_t id; - std::vector global_vars; + std::vector eos_global_vars; + std::vector strength_global_vars; void derive() { - num_global_vars = global_vars.size(); + num_eos_global_vars = eos_global_vars.size(); + num_strength_global_vars = strength_global_vars.size(); if(linear_cell_modulus) SIMP_modulus = false; } }; @@ -52,8 +56,10 @@ IMPL_YAML_SERIALIZABLE_FOR(Material, id, eos_model, strength_model, strength_type, strength_run_location, eos_run_location, q1, q2, q1ex, q2ex, maximum_limiter, - num_state_vars, - global_vars, + num_eos_state_vars, + num_strength_state_vars, + eos_global_vars, + strength_global_vars, elastic_modulus, poisson_ratio, density, initial_temperature, thermal_conductivity, specific_internal_energy_rate, expansion_coefficients, diff --git a/src/Parallel-Solvers/Simulation_Parameters/Simulation_Parameters.h b/src/Parallel-Solvers/Simulation_Parameters/Simulation_Parameters.h index 9d2260530..6b58e0771 100644 --- a/src/Parallel-Solvers/Simulation_Parameters/Simulation_Parameters.h +++ b/src/Parallel-Solvers/Simulation_Parameters/Simulation_Parameters.h @@ -24,7 +24,8 @@ struct Simulation_Parameters std::optional input_options; std::optional> mesh_generation_options; Output_Options output_options; - size_t max_num_state_vars = 0; + size_t max_num_eos_state_vars = 0; + size_t max_num_strength_state_vars = 0; std::vector regions; std::vector materials; @@ -56,28 +57,42 @@ struct Simulation_Parameters DCArrayKokkos mat_fill; DCArrayKokkos material; - DCArrayKokkos global_vars; + DCArrayKokkos eos_global_vars; + DCArrayKokkos strength_global_vars; void init_material_variable_arrays() { - size_t max_global_vars = 0; - for (const auto& mat : materials) - max_global_vars = std::max(mat.global_vars.size(), max_global_vars); + size_t max_eos_global_vars = 0; + size_t max_strength_global_vars = 0; + for (const auto& mat : materials) { + max_eos_global_vars = std::max(mat.eos_global_vars.size(), max_eos_global_vars); + max_strength_global_vars = std::max(mat.strength_global_vars.size(), max_strength_global_vars); + } - global_vars = DCArrayKokkos (materials.size(), max_global_vars); + eos_global_vars = DCArrayKokkos (materials.size(), max_eos_global_vars); + strength_global_vars = DCArrayKokkos (materials.size(), max_strength_global_vars); for (size_t i = 0; i < materials.size(); i++) { auto mat = materials[i]; - for (size_t j = 0; j < mat.global_vars.size(); j++) - global_vars.host(i, j) = mat.global_vars[j]; + for (size_t j = 0; j < mat.eos_global_vars.size(); j++) { + eos_global_vars.host(i, j) = mat.eos_global_vars[j]; + } + eos_global_vars.update_device(); + + for (size_t j = 0; j < mat.strength_global_vars.size(); j++) { + strength_global_vars.host(i, j) = mat.strength_global_vars[j]; + } } - global_vars.update_device(); + strength_global_vars.update_device(); } void derive_max_num_state_vars() { - max_num_state_vars = 0; - for (const auto& mat : materials) - max_num_state_vars = std::max(mat.num_state_vars, max_num_state_vars); + max_num_eos_state_vars = 0; + max_num_strength_state_vars = 0; + for (const auto& mat : materials) { + max_num_eos_state_vars = std::max(mat.num_eos_state_vars, max_num_eos_state_vars); + max_num_strength_state_vars = std::max(mat.num_strength_state_vars, max_num_strength_state_vars); + } } void derive_objective_module() { From d27ac394a5186d3630aa6b8d65d8648d56ec767e Mon Sep 17 00:00:00 2001 From: Jacob Moore Date: Mon, 19 Feb 2024 15:03:03 -0600 Subject: [PATCH 09/64] ENH: Basic integrated testing functionality. WIP --- .../Solver-Inputs/SGH_Sedov_12x12x12.yaml | 99 + .../SGH/Sedov_12x12x12/vtk/data/VTK0.vtk | 18252 ++++++++++++++++ .../SGH/Sedov_12x12x12/vtk/outputs.vtk.series | 6 + python/Test-scripts/test_fierro.py | 128 + 4 files changed, 18485 insertions(+) create mode 100644 python/Test-scripts/Solver-Inputs/SGH_Sedov_12x12x12.yaml create mode 100644 python/Test-scripts/standard-results/SGH/Sedov_12x12x12/vtk/data/VTK0.vtk create mode 100644 python/Test-scripts/standard-results/SGH/Sedov_12x12x12/vtk/outputs.vtk.series create mode 100644 python/Test-scripts/test_fierro.py diff --git a/python/Test-scripts/Solver-Inputs/SGH_Sedov_12x12x12.yaml b/python/Test-scripts/Solver-Inputs/SGH_Sedov_12x12x12.yaml new file mode 100644 index 000000000..a8604d82b --- /dev/null +++ b/python/Test-scripts/Solver-Inputs/SGH_Sedov_12x12x12.yaml @@ -0,0 +1,99 @@ +num_dims: 3 +dynamic_options: + time_final: 1.0 + dt_min: 1.e-8 + dt_max: 1.e-2 + dt_start: 1.e-5 + cycle_stop: 2000000 + +mesh_generation_options: + type: Box + origin: [0, 0, 0] + length: [1.2, 1.2, 1.2] + num_elems: [12, 12, 12] + +output_options: + timer_output_level: thorough + output_file_format: vtk + graphics_step: 1.0 + write_initial: false + +fea_module_parameters: + - type: SGH + material_id: 0 + boundary_conditions: + # Tag X plane + - surface: + type: x_plane + plane_position: 0.0 + type: reflected + + # Tag Y plane + - surface: + type: y_plane + plane_position: 0.0 + type: reflected + + # Tag Z plane + - surface: + type: z_plane + plane_position: 0.0 + type: reflected + + # Tag X plane + - surface: + type: x_plane + plane_position: 1.2 + type: reflected + + # Tag Y plane + - surface: + type: y_plane + plane_position: 1.2 + type: reflected + + # Tag Z plane + - surface: + type: z_plane + plane_position: 1.2 + type: reflected + +materials: + - id: 0 + eos_model: ideal_gas + strength_model: none + elastic_modulus: 10 + poisson_ratio: 0.3 + q1: 1.0 + q2: 0 + q1ex: 1.0 + q2ex: 0.0 + global_vars: + - 1.666666666666667 + - 1.0E-14 + - 1.0 + +regions: + - volume: + type: global + material_id: 0 + den: 1.0 + sie: 1.e-10 + + velocity: cartesian + u: 0.0 + v: 0.0 + w: 0.0 + # energy source initial conditions + - volume: + type: sphere + radius1: 0.0 + radius2: 0.1 + material_id: 0 + den: 1.0 + ie: 0.5 + + velocity: cartesian + u: 0.0 + v: 0.0 + w: 0.0 diff --git a/python/Test-scripts/standard-results/SGH/Sedov_12x12x12/vtk/data/VTK0.vtk b/python/Test-scripts/standard-results/SGH/Sedov_12x12x12/vtk/data/VTK0.vtk new file mode 100644 index 000000000..f5dc9b531 --- /dev/null +++ b/python/Test-scripts/standard-results/SGH/Sedov_12x12x12/vtk/data/VTK0.vtk @@ -0,0 +1,18252 @@ +# vtk DataFile Version 2.0 +Mesh for Fierro +ASCII +DATASET UNSTRUCTURED_GRID + +POINTS 2197 float +0.00000000 0.00000000 0.00000000 +0.53776191 0.00000000 0.00000000 +0.60205274 0.00000000 0.00000000 +0.69594450 0.00000000 0.00000000 +0.77599848 0.00000000 0.00000000 +0.84746937 0.00000000 0.00000000 +0.91204792 0.00000000 0.00000000 +0.96962515 0.00000000 0.00000000 +1.01908611 0.00000000 0.00000000 +1.05975035 0.00000000 0.00000000 +1.10930635 0.00000000 0.00000000 +1.11637892 0.00000000 0.00000000 +1.20000000 0.00000000 0.00000000 +0.00000000 0.53776191 0.00000000 +0.44435254 0.44435254 0.00000000 +0.61155544 0.26939912 0.00000000 +0.69846681 0.22064358 0.00000000 +0.77688181 0.18976175 0.00000000 +0.84748313 0.16967335 0.00000000 +0.91018232 0.15517265 0.00000000 +0.96483786 0.14332892 0.00000000 +1.01357716 0.13287109 0.00000000 +1.05399342 0.12283560 0.00000000 +1.09851927 0.11378558 0.00000000 +1.11184016 0.10152648 0.00000000 +1.20000000 0.10000052 0.00000000 +0.00000000 0.60205274 0.00000000 +0.26939912 0.61155544 0.00000000 +0.54203263 0.54203263 0.00000000 +0.67307734 0.41180871 0.00000000 +0.75701522 0.35501801 0.00000000 +0.82700772 0.31714318 0.00000000 +0.88817242 0.29071456 0.00000000 +0.94191103 0.27030805 0.00000000 +0.98673239 0.25193669 0.00000000 +1.02993988 0.23617460 0.00000000 +1.06148154 0.21674437 0.00000000 +1.10311427 0.20081847 0.00000000 +1.20000000 0.20000021 0.00000000 +0.00000000 0.69594450 0.00000000 +0.22064358 0.69846681 0.00000000 +0.41180871 0.67307734 0.00000000 +0.60391464 0.60391464 0.00000000 +0.71080419 0.50149623 0.00000000 +0.78868751 0.44974512 0.00000000 +0.85270186 0.41230124 0.00000000 +0.90696809 0.38368634 0.00000000 +0.95438813 0.35904395 0.00000000 +0.99927356 0.33668601 0.00000000 +1.02383514 0.30867229 0.00000000 +1.10029021 0.30009933 0.00000000 +1.20000000 0.30000001 0.00000000 +0.00000000 0.77599848 0.00000000 +0.18976175 0.77688181 0.00000000 +0.35501801 0.75701522 0.00000000 +0.50149623 0.71080419 0.00000000 +0.64970821 0.64970821 0.00000000 +0.73928284 0.56858842 0.00000000 +0.81006029 0.52387144 0.00000000 +0.86770511 0.48767065 0.00000000 +0.92067108 0.45894538 0.00000000 +0.95881249 0.42696076 0.00000000 +1.00427903 0.40189866 0.00000000 +1.10000820 0.40000322 0.00000000 +1.20000000 0.40000000 0.00000000 +0.00000000 0.84746937 0.00000000 +0.16967335 0.84748313 0.00000000 +0.31714318 0.82700772 0.00000000 +0.44974512 0.78868751 0.00000000 +0.56858842 0.73928284 0.00000000 +0.68717236 0.68717236 0.00000000 +0.76182433 0.62063065 0.00000000 +0.82630414 0.58266570 0.00000000 +0.87929698 0.54716000 0.00000000 +0.91747243 0.50972929 0.00000000 +1.00030406 0.50015749 0.00000000 +1.10000004 0.50000002 0.00000000 +1.20000000 0.50000000 0.00000000 +0.00000000 0.91204792 0.00000000 +0.15517265 0.91018232 0.00000000 +0.29071456 0.88817242 0.00000000 +0.41230124 0.85270186 0.00000000 +0.52387144 0.81006029 0.00000000 +0.62063065 0.76182433 0.00000000 +0.71717747 0.71717747 0.00000000 +0.78380908 0.66428258 0.00000000 +0.82986250 0.62120277 0.00000000 +0.90158256 0.60103535 0.00000000 +1.00000345 0.60000199 0.00000000 +1.10000000 0.60000000 0.00000000 +1.20000000 0.60000000 0.00000000 +0.00000000 0.96962515 0.00000000 +0.14332892 0.96483786 0.00000000 +0.27030805 0.94191103 0.00000000 +0.38368634 0.90696809 0.00000000 +0.48767065 0.86770511 0.00000000 +0.58266570 0.82630414 0.00000000 +0.66428258 0.78380908 0.00000000 +0.73913964 0.73913964 0.00000000 +0.80329722 0.70269171 0.00000000 +0.90002831 0.70002079 0.00000000 +1.00000000 0.70000000 0.00000000 +1.10000000 0.70000000 0.00000000 +1.20000000 0.70000000 0.00000000 +0.00000000 1.01908611 0.00000000 +0.13287109 1.01357716 0.00000000 +0.25193669 0.98673239 0.00000000 +0.35904395 0.95438813 0.00000000 +0.45894538 0.92067108 0.00000000 +0.54716000 0.87929698 0.00000000 +0.62120277 0.82986250 0.00000000 +0.70269171 0.80329722 0.00000000 +0.80011294 0.80011294 0.00000000 +0.90000003 0.80000002 0.00000000 +1.00000000 0.80000000 0.00000000 +1.10000000 0.80000000 0.00000000 +1.20000000 0.80000000 0.00000000 +0.00000000 1.05975035 0.00000000 +0.12283560 1.05399342 0.00000000 +0.23617460 1.02993988 0.00000000 +0.33668601 0.99927356 0.00000000 +0.42696076 0.95881249 0.00000000 +0.50972929 0.91747243 0.00000000 +0.60103535 0.90158256 0.00000000 +0.70002079 0.90002831 0.00000000 +0.80000002 0.90000003 0.00000000 +0.90000000 0.90000000 0.00000000 +1.00000000 0.90000000 0.00000000 +1.10000000 0.90000000 0.00000000 +1.20000000 0.90000000 0.00000000 +0.00000000 1.10930635 0.00000000 +0.11378558 1.09851927 0.00000000 +0.21674437 1.06148154 0.00000000 +0.30867229 1.02383514 0.00000000 +0.40189866 1.00427903 0.00000000 +0.50015749 1.00030406 0.00000000 +0.60000199 1.00000345 0.00000000 +0.70000000 1.00000000 0.00000000 +0.80000000 1.00000000 0.00000000 +0.90000000 1.00000000 0.00000000 +1.00000000 1.00000000 0.00000000 +1.10000000 1.00000000 0.00000000 +1.20000000 1.00000000 0.00000000 +0.00000000 1.11637892 0.00000000 +0.10152648 1.11184016 0.00000000 +0.20081847 1.10311427 0.00000000 +0.30009933 1.10029021 0.00000000 +0.40000322 1.10000820 0.00000000 +0.50000002 1.10000004 0.00000000 +0.60000000 1.10000000 0.00000000 +0.70000000 1.10000000 0.00000000 +0.80000000 1.10000000 0.00000000 +0.90000000 1.10000000 0.00000000 +1.00000000 1.10000000 0.00000000 +1.10000000 1.10000000 0.00000000 +1.20000000 1.10000000 0.00000000 +0.00000000 1.20000000 0.00000000 +0.10000052 1.20000000 0.00000000 +0.20000021 1.20000000 0.00000000 +0.30000001 1.20000000 0.00000000 +0.40000000 1.20000000 0.00000000 +0.50000000 1.20000000 0.00000000 +0.60000000 1.20000000 0.00000000 +0.70000000 1.20000000 0.00000000 +0.80000000 1.20000000 0.00000000 +0.90000000 1.20000000 0.00000000 +1.00000000 1.20000000 0.00000000 +1.10000000 1.20000000 0.00000000 +1.20000000 1.20000000 0.00000000 +0.00000000 0.00000000 0.53776191 +0.44435254 0.00000000 0.44435254 +0.61155544 0.00000000 0.26939912 +0.69846681 0.00000000 0.22064358 +0.77688181 0.00000000 0.18976175 +0.84748313 0.00000000 0.16967335 +0.91018232 0.00000000 0.15517265 +0.96483786 0.00000000 0.14332892 +1.01357716 0.00000000 0.13287109 +1.05399342 0.00000000 0.12283560 +1.09851927 0.00000000 0.11378558 +1.11184016 0.00000000 0.10152648 +1.20000000 0.00000000 0.10000052 +0.00000000 0.44435254 0.44435254 +0.40158516 0.40158516 0.40158516 +0.60529265 0.27963723 0.27963723 +0.69636507 0.21528261 0.21528261 +0.77402488 0.18567451 0.18567451 +0.84371201 0.16592356 0.16592356 +0.90540812 0.15186355 0.15186355 +0.95897612 0.14058988 0.14058988 +1.00656530 0.13066508 0.13066508 +1.04745861 0.12143171 0.12143171 +1.08708267 0.11202892 0.11202892 +1.10834589 0.10110022 0.10110022 +1.20000000 0.10000032 0.10000032 +0.00000000 0.61155544 0.26939912 +0.27963723 0.60529265 0.27963723 +0.52500961 0.52500961 0.25920378 +0.66381197 0.41240417 0.21097409 +0.74994845 0.35105276 0.17683110 +0.82057257 0.31369245 0.15782758 +0.88178372 0.28735387 0.14416070 +0.93524877 0.26714900 0.13370188 +0.98023176 0.24916134 0.12455660 +1.02428904 0.23383545 0.11668098 +1.05283153 0.21405368 0.10675696 +1.10216182 0.20056038 0.10028399 +1.20000000 0.20000013 0.10000006 +0.00000000 0.69846681 0.22064358 +0.21528261 0.69636507 0.21528261 +0.41240417 0.66381197 0.21097409 +0.59221847 0.59221847 0.19712823 +0.70343685 0.50080222 0.17093147 +0.78235981 0.44678136 0.15018738 +0.84712657 0.40975168 0.13725219 +0.90155981 0.38108807 0.12715273 +0.94953048 0.35679227 0.11879272 +0.99386503 0.33427106 0.11124799 +1.01968452 0.30706878 0.10223985 +1.10020126 0.30006789 0.10002269 +1.20000000 0.30000000 0.10000000 +0.00000000 0.77688181 0.18976175 +0.18567451 0.77402488 0.18567451 +0.35105276 0.74994845 0.17683110 +0.50080222 0.70343685 0.17093147 +0.64119052 0.64119052 0.16128471 +0.73369463 0.56719040 0.14458578 +0.80467853 0.52099842 0.13108583 +0.86286900 0.48524650 0.12165362 +0.91652877 0.45674548 0.11417539 +0.95313141 0.42424467 0.10595209 +1.00335672 0.40148402 0.10037558 +1.10000555 0.40000217 0.10000050 +1.20000000 0.40000000 0.10000000 +0.00000000 0.84748313 0.16967335 +0.16592356 0.84371201 0.16592356 +0.31369245 0.82057257 0.15782758 +0.44678136 0.78235981 0.15018738 +0.56719040 0.73369463 0.14458578 +0.68053012 0.68053012 0.13738574 +0.75739272 0.61883699 0.12555538 +0.82242886 0.58018583 0.11646131 +0.87441298 0.54424669 0.10890351 +0.91472437 0.50818514 0.10161500 +1.00022657 0.50011671 0.10002438 +1.10000003 0.50000001 0.10000000 +1.20000000 0.50000000 0.10000000 +0.00000000 0.91018232 0.15517265 +0.15186355 0.90540812 0.15186355 +0.28735387 0.88178372 0.14416070 +0.40975168 0.84712657 0.13725219 +0.52099842 0.80467853 0.13108583 +0.61883699 0.75739272 0.12555538 +0.71254499 0.71254499 0.11987896 +0.78003954 0.66232491 0.11125308 +0.82642640 0.61873750 0.10310482 +0.90125052 0.60081694 0.10014389 +1.00000241 0.60000138 0.10000021 +1.10000000 0.60000000 0.10000000 +1.20000000 0.60000000 0.10000000 +0.00000000 0.96483786 0.14332892 +0.14058988 0.95897612 0.14058988 +0.26714900 0.93524877 0.13370188 +0.38108807 0.90155981 0.12715273 +0.48524650 0.86286900 0.12165362 +0.58018583 0.82242886 0.11646131 +0.66232491 0.78003954 0.11125308 +0.73398435 0.73398435 0.10513418 +0.80271483 0.70223247 0.10034158 +0.90002082 0.70001527 0.10000221 +1.00000000 0.70000000 0.10000000 +1.10000000 0.70000000 0.10000000 +1.20000000 0.70000000 0.10000000 +0.00000000 1.01357716 0.13287109 +0.13066508 1.00656530 0.13066508 +0.24916134 0.98023176 0.12455660 +0.35679227 0.94953048 0.11879272 +0.45674548 0.91652877 0.11417539 +0.54424669 0.87441298 0.10890351 +0.61873750 0.82642640 0.10310482 +0.70223247 0.80271483 0.10034158 +0.80007988 0.80007988 0.10001129 +0.90000002 0.80000002 0.10000000 +1.00000000 0.80000000 0.10000000 +1.10000000 0.80000000 0.10000000 +1.20000000 0.80000000 0.10000000 +0.00000000 1.05399342 0.12283560 +0.12143171 1.04745861 0.12143171 +0.23383545 1.02428904 0.11668098 +0.33427106 0.99386503 0.11124799 +0.42424467 0.95313141 0.10595209 +0.50818514 0.91472437 0.10161500 +0.60081694 0.90125052 0.10014389 +0.70001527 0.90002082 0.10000221 +0.80000002 0.90000002 0.10000000 +0.90000000 0.90000000 0.10000000 +1.00000000 0.90000000 0.10000000 +1.10000000 0.90000000 0.10000000 +1.20000000 0.90000000 0.10000000 +0.00000000 1.09851927 0.11378558 +0.11202892 1.08708267 0.11202892 +0.21405368 1.05283153 0.10675696 +0.30706878 1.01968452 0.10223985 +0.40148402 1.00335672 0.10037558 +0.50011671 1.00022657 0.10002438 +0.60000138 1.00000241 0.10000021 +0.70000000 1.00000000 0.10000000 +0.80000000 1.00000000 0.10000000 +0.90000000 1.00000000 0.10000000 +1.00000000 1.00000000 0.10000000 +1.10000000 1.00000000 0.10000000 +1.20000000 1.00000000 0.10000000 +0.00000000 1.11184016 0.10152648 +0.10110022 1.10834589 0.10110022 +0.20056038 1.10216182 0.10028399 +0.30006789 1.10020126 0.10002269 +0.40000217 1.10000555 0.10000050 +0.50000001 1.10000003 0.10000000 +0.60000000 1.10000000 0.10000000 +0.70000000 1.10000000 0.10000000 +0.80000000 1.10000000 0.10000000 +0.90000000 1.10000000 0.10000000 +1.00000000 1.10000000 0.10000000 +1.10000000 1.10000000 0.10000000 +1.20000000 1.10000000 0.10000000 +0.00000000 1.20000000 0.10000052 +0.10000032 1.20000000 0.10000032 +0.20000013 1.20000000 0.10000006 +0.30000000 1.20000000 0.10000000 +0.40000000 1.20000000 0.10000000 +0.50000000 1.20000000 0.10000000 +0.60000000 1.20000000 0.10000000 +0.70000000 1.20000000 0.10000000 +0.80000000 1.20000000 0.10000000 +0.90000000 1.20000000 0.10000000 +1.00000000 1.20000000 0.10000000 +1.10000000 1.20000000 0.10000000 +1.20000000 1.20000000 0.10000000 +0.00000000 0.00000000 0.60205274 +0.26939912 0.00000000 0.61155544 +0.54203263 0.00000000 0.54203263 +0.67307734 0.00000000 0.41180871 +0.75701522 0.00000000 0.35501801 +0.82700772 0.00000000 0.31714318 +0.88817242 0.00000000 0.29071456 +0.94191103 0.00000000 0.27030805 +0.98673239 0.00000000 0.25193669 +1.02993988 0.00000000 0.23617460 +1.06148154 0.00000000 0.21674437 +1.10311427 0.00000000 0.20081847 +1.20000000 0.00000000 0.20000021 +0.00000000 0.26939912 0.61155544 +0.27963723 0.27963723 0.60529265 +0.52500961 0.25920378 0.52500961 +0.66381197 0.21097409 0.41240417 +0.74994845 0.17683110 0.35105276 +0.82057257 0.15782758 0.31369245 +0.88178372 0.14416070 0.28735387 +0.93524877 0.13370188 0.26714900 +0.98023176 0.12455660 0.24916134 +1.02428904 0.11668098 0.23383545 +1.05283153 0.10675696 0.21405368 +1.10216182 0.10028399 0.20056038 +1.20000000 0.10000006 0.20000013 +0.00000000 0.54203263 0.54203263 +0.25920378 0.52500961 0.52500961 +0.46801317 0.46801317 0.46801317 +0.62471435 0.40259738 0.40259738 +0.72359708 0.34252623 0.34252623 +0.79895918 0.30473274 0.30473274 +0.86136115 0.27879357 0.27879357 +0.91526121 0.25903270 0.25903270 +0.96161127 0.24190248 0.24190248 +1.00671014 0.22701033 0.22701033 +1.03110077 0.20766246 0.20766246 +1.10062140 0.20015072 0.20015072 +1.20000000 0.20000002 0.20000002 +0.00000000 0.67307734 0.41180871 +0.21097409 0.66381197 0.41240417 +0.40259738 0.62471435 0.40259738 +0.55777243 0.55777243 0.37130158 +0.67799702 0.49245949 0.33036339 +0.76188336 0.43867714 0.29400223 +0.82950195 0.40204041 0.26861035 +0.88510792 0.37365783 0.24924309 +0.93512570 0.35040098 0.23356272 +0.97657491 0.32704436 0.21784235 +1.01034139 0.30359058 0.20235762 +1.10006116 0.30001958 0.20001291 +1.20000000 0.30000000 0.20000000 +0.00000000 0.75701522 0.35501801 +0.17683110 0.74994845 0.35105276 +0.34252623 0.72359708 0.34252623 +0.49245949 0.67799702 0.33036339 +0.61746997 0.61746997 0.30925925 +0.71543727 0.55957630 0.28223638 +0.78816087 0.51279397 0.25768821 +0.84899828 0.47825791 0.23944822 +0.90311681 0.44979284 0.22492296 +0.93777007 0.41704032 0.20834384 +1.00154830 0.40067510 0.20034203 +1.10000159 0.40000060 0.20000028 +1.20000000 0.40000000 0.20000000 +0.00000000 0.82700772 0.31714318 +0.15782758 0.82057257 0.31369245 +0.30473274 0.79895918 0.30473274 +0.43867714 0.76188336 0.29400223 +0.55957630 0.71543727 0.28223638 +0.66254081 0.66254081 0.26630201 +0.74412487 0.61203591 0.24659868 +0.81100840 0.57321991 0.22993157 +0.85996942 0.53560066 0.21419343 +0.90841516 0.50466014 0.20186981 +1.00008969 0.50004540 0.20001849 +1.10000001 0.50000000 0.20000000 +1.20000000 0.50000000 0.20000000 +0.00000000 0.88817242 0.29071456 +0.14416070 0.88178372 0.28735387 +0.27879357 0.86136115 0.27879357 +0.40204041 0.82950195 0.26861035 +0.51279397 0.78816087 0.25768821 +0.61203591 0.74412487 0.24659868 +0.69988798 0.69988798 0.23467848 +0.76772981 0.65451792 0.21891426 +0.81767238 0.61254377 0.20417332 +0.90059787 0.60038862 0.20013549 +1.00000077 0.60000044 0.20000014 +1.10000000 0.60000000 0.20000000 +1.20000000 0.60000000 0.20000000 +0.00000000 0.94191103 0.27030805 +0.13370188 0.93524877 0.26714900 +0.25903270 0.91526121 0.25903270 +0.37365783 0.88510792 0.24924309 +0.47825791 0.84899828 0.23944822 +0.57321991 0.81100840 0.22993157 +0.65451792 0.76772981 0.21891426 +0.72157041 0.72157041 0.20635303 +0.80144842 0.70121158 0.20037105 +0.90000795 0.70000581 0.20000164 +1.00000000 0.70000000 0.20000000 +1.10000000 0.70000000 0.20000000 +1.20000000 0.70000000 0.20000000 +0.00000000 0.98673239 0.25193669 +0.12455660 0.98023176 0.24916134 +0.24190248 0.96161127 0.24190248 +0.35040098 0.93512570 0.23356272 +0.44979284 0.90311681 0.22492296 +0.53560066 0.85996942 0.21419343 +0.61254377 0.81767238 0.20417332 +0.70121158 0.80144842 0.20037105 +0.80002681 0.80002681 0.20000706 +0.90000001 0.80000001 0.20000000 +1.00000000 0.80000000 0.20000000 +1.10000000 0.80000000 0.20000000 +1.20000000 0.80000000 0.20000000 +0.00000000 1.02993988 0.23617460 +0.11668098 1.02428904 0.23383545 +0.22701033 1.00671014 0.22701033 +0.32704436 0.97657491 0.21784235 +0.41704032 0.93777007 0.20834384 +0.50466014 0.90841516 0.20186981 +0.60038862 0.90059787 0.20013549 +0.70000581 0.90000795 0.20000164 +0.80000001 0.90000001 0.20000000 +0.90000000 0.90000000 0.20000000 +1.00000000 0.90000000 0.20000000 +1.10000000 0.90000000 0.20000000 +1.20000000 0.90000000 0.20000000 +0.00000000 1.06148154 0.21674437 +0.10675696 1.05283153 0.21405368 +0.20766246 1.03110077 0.20766246 +0.30359058 1.01034139 0.20235762 +0.40067510 1.00154830 0.20034203 +0.50004540 1.00008969 0.20001849 +0.60000044 1.00000077 0.20000014 +0.70000000 1.00000000 0.20000000 +0.80000000 1.00000000 0.20000000 +0.90000000 1.00000000 0.20000000 +1.00000000 1.00000000 0.20000000 +1.10000000 1.00000000 0.20000000 +1.20000000 1.00000000 0.20000000 +0.00000000 1.10311427 0.20081847 +0.10028399 1.10216182 0.20056038 +0.20015072 1.10062140 0.20015072 +0.30001958 1.10006116 0.20001291 +0.40000060 1.10000159 0.20000028 +0.50000000 1.10000001 0.20000000 +0.60000000 1.10000000 0.20000000 +0.70000000 1.10000000 0.20000000 +0.80000000 1.10000000 0.20000000 +0.90000000 1.10000000 0.20000000 +1.00000000 1.10000000 0.20000000 +1.10000000 1.10000000 0.20000000 +1.20000000 1.10000000 0.20000000 +0.00000000 1.20000000 0.20000021 +0.10000006 1.20000000 0.20000013 +0.20000002 1.20000000 0.20000002 +0.30000000 1.20000000 0.20000000 +0.40000000 1.20000000 0.20000000 +0.50000000 1.20000000 0.20000000 +0.60000000 1.20000000 0.20000000 +0.70000000 1.20000000 0.20000000 +0.80000000 1.20000000 0.20000000 +0.90000000 1.20000000 0.20000000 +1.00000000 1.20000000 0.20000000 +1.10000000 1.20000000 0.20000000 +1.20000000 1.20000000 0.20000000 +0.00000000 0.00000000 0.69594450 +0.22064358 0.00000000 0.69846681 +0.41180871 0.00000000 0.67307734 +0.60391464 0.00000000 0.60391464 +0.71080419 0.00000000 0.50149623 +0.78868751 0.00000000 0.44974512 +0.85270186 0.00000000 0.41230124 +0.90696809 0.00000000 0.38368634 +0.95438813 0.00000000 0.35904395 +0.99927356 0.00000000 0.33668601 +1.02383514 0.00000000 0.30867229 +1.10029021 0.00000000 0.30009933 +1.20000000 0.00000000 0.30000001 +0.00000000 0.22064358 0.69846681 +0.21528261 0.21528261 0.69636507 +0.41240417 0.21097409 0.66381197 +0.59221847 0.19712823 0.59221847 +0.70343685 0.17093147 0.50080222 +0.78235981 0.15018738 0.44678136 +0.84712657 0.13725219 0.40975168 +0.90155981 0.12715273 0.38108807 +0.94953048 0.11879272 0.35679227 +0.99386503 0.11124799 0.33427106 +1.01968452 0.10223985 0.30706878 +1.10020126 0.10002269 0.30006789 +1.20000000 0.10000000 0.30000000 +0.00000000 0.41180871 0.67307734 +0.21097409 0.41240417 0.66381197 +0.40259738 0.40259738 0.62471435 +0.55777243 0.37130158 0.55777243 +0.67799702 0.33036339 0.49245949 +0.76188336 0.29400223 0.43867714 +0.82950195 0.26861035 0.40204041 +0.88510792 0.24924309 0.37365783 +0.93512570 0.23356272 0.35040098 +0.97657491 0.21784235 0.32704436 +1.01034139 0.20235762 0.30359058 +1.10006116 0.20001291 0.30001958 +1.20000000 0.20000000 0.30000000 +0.00000000 0.60391464 0.60391464 +0.19712823 0.59221847 0.59221847 +0.37130158 0.55777243 0.55777243 +0.51573703 0.51573703 0.51573703 +0.63468021 0.47026829 0.47026829 +0.72882359 0.42634679 0.42634679 +0.80031590 0.39037523 0.39037523 +0.85896581 0.36286548 0.36286548 +0.91301195 0.34124155 0.34124155 +0.94805255 0.31627253 0.31627253 +1.00287541 0.30096251 0.30096251 +1.10000642 0.30000187 0.30000187 +1.20000000 0.30000000 0.30000000 +0.00000000 0.71080419 0.50149623 +0.17093147 0.70343685 0.50080222 +0.33036339 0.67799702 0.49245949 +0.47026829 0.63468021 0.47026829 +0.58727758 0.58727758 0.43999065 +0.68544422 0.54231789 0.40765797 +0.76328972 0.50064654 0.37623684 +0.82731919 0.46741977 0.35081514 +0.87928885 0.43790516 0.32839894 +0.91846729 0.40823181 0.30610938 +1.00037561 0.40015870 0.30011969 +1.10000015 0.40000005 0.30000004 +1.20000000 0.40000000 0.30000000 +0.00000000 0.78868751 0.44974512 +0.15018738 0.78235981 0.44678136 +0.29400223 0.76188336 0.43867714 +0.42634679 0.72882359 0.42634679 +0.54231789 0.68544422 0.40765797 +0.63909979 0.63909979 0.38385260 +0.72252282 0.59824110 0.36006378 +0.79135631 0.56136548 0.33729954 +0.83815065 0.52260788 0.31343113 +0.90288377 0.50158998 0.30096622 +1.00001657 0.50000815 0.30000484 +1.10000000 0.50000000 0.30000000 +1.20000000 0.50000000 0.30000000 +0.00000000 0.85270186 0.41230124 +0.13725219 0.84712657 0.40975168 +0.26861035 0.82950195 0.40204041 +0.39037523 0.80031590 0.39037523 +0.50064654 0.76328972 0.37623684 +0.59824110 0.72252282 0.36006378 +0.68192803 0.68192803 0.34163275 +0.74630639 0.63839074 0.31941918 +0.80785710 0.60564149 0.30285894 +0.90015817 0.60010177 0.30005217 +1.00000009 0.60000005 0.30000003 +1.10000000 0.60000000 0.30000000 +1.20000000 0.60000000 0.30000000 +0.00000000 0.90696809 0.38368634 +0.12715273 0.90155981 0.38108807 +0.24924309 0.88510792 0.37365783 +0.36286548 0.85896581 0.36286548 +0.46741977 0.82731919 0.35081514 +0.56136548 0.79135631 0.33729954 +0.63839074 0.74630639 0.31941918 +0.70939520 0.70939520 0.30409247 +0.80044005 0.70037406 0.30016913 +0.90000138 0.70000101 0.30000042 +1.00000000 0.70000000 0.30000000 +1.10000000 0.70000000 0.30000000 +1.20000000 0.70000000 0.30000000 +0.00000000 0.95438813 0.35904395 +0.11879272 0.94953048 0.35679227 +0.23356272 0.93512570 0.35040098 +0.34124155 0.91301195 0.34124155 +0.43790516 0.87928885 0.32839894 +0.52260788 0.83815065 0.31343113 +0.60564149 0.80785710 0.30285894 +0.70037406 0.80044005 0.30016913 +0.80000384 0.80000384 0.30000142 +0.90000000 0.80000000 0.30000000 +1.00000000 0.80000000 0.30000000 +1.10000000 0.80000000 0.30000000 +1.20000000 0.80000000 0.30000000 +0.00000000 0.99927356 0.33668601 +0.11124799 0.99386503 0.33427106 +0.21784235 0.97657491 0.32704436 +0.31627253 0.94805255 0.31627253 +0.40823181 0.91846729 0.30610938 +0.50158998 0.90288377 0.30096622 +0.60010177 0.90015817 0.30005217 +0.70000101 0.90000138 0.30000042 +0.80000000 0.90000000 0.30000000 +0.90000000 0.90000000 0.30000000 +1.00000000 0.90000000 0.30000000 +1.10000000 0.90000000 0.30000000 +1.20000000 0.90000000 0.30000000 +0.00000000 1.02383514 0.30867229 +0.10223985 1.01968452 0.30706878 +0.20235762 1.01034139 0.30359058 +0.30096251 1.00287541 0.30096251 +0.40015870 1.00037561 0.30011969 +0.50000815 1.00001657 0.30000484 +0.60000005 1.00000009 0.30000003 +0.70000000 1.00000000 0.30000000 +0.80000000 1.00000000 0.30000000 +0.90000000 1.00000000 0.30000000 +1.00000000 1.00000000 0.30000000 +1.10000000 1.00000000 0.30000000 +1.20000000 1.00000000 0.30000000 +0.00000000 1.10029021 0.30009933 +0.10002269 1.10020126 0.30006789 +0.20001291 1.10006116 0.30001958 +0.30000187 1.10000642 0.30000187 +0.40000005 1.10000015 0.30000004 +0.50000000 1.10000000 0.30000000 +0.60000000 1.10000000 0.30000000 +0.70000000 1.10000000 0.30000000 +0.80000000 1.10000000 0.30000000 +0.90000000 1.10000000 0.30000000 +1.00000000 1.10000000 0.30000000 +1.10000000 1.10000000 0.30000000 +1.20000000 1.10000000 0.30000000 +0.00000000 1.20000000 0.30000001 +0.10000000 1.20000000 0.30000000 +0.20000000 1.20000000 0.30000000 +0.30000000 1.20000000 0.30000000 +0.40000000 1.20000000 0.30000000 +0.50000000 1.20000000 0.30000000 +0.60000000 1.20000000 0.30000000 +0.70000000 1.20000000 0.30000000 +0.80000000 1.20000000 0.30000000 +0.90000000 1.20000000 0.30000000 +1.00000000 1.20000000 0.30000000 +1.10000000 1.20000000 0.30000000 +1.20000000 1.20000000 0.30000000 +0.00000000 0.00000000 0.77599848 +0.18976175 0.00000000 0.77688181 +0.35501801 0.00000000 0.75701522 +0.50149623 0.00000000 0.71080419 +0.64970821 0.00000000 0.64970821 +0.73928284 0.00000000 0.56858842 +0.81006029 0.00000000 0.52387144 +0.86770511 0.00000000 0.48767065 +0.92067108 0.00000000 0.45894538 +0.95881249 0.00000000 0.42696076 +1.00427903 0.00000000 0.40189866 +1.10000820 0.00000000 0.40000322 +1.20000000 0.00000000 0.40000000 +0.00000000 0.18976175 0.77688181 +0.18567451 0.18567451 0.77402488 +0.35105276 0.17683110 0.74994845 +0.50080222 0.17093147 0.70343685 +0.64119052 0.16128471 0.64119052 +0.73369463 0.14458578 0.56719040 +0.80467853 0.13108583 0.52099842 +0.86286900 0.12165362 0.48524650 +0.91652877 0.11417539 0.45674548 +0.95313141 0.10595209 0.42424467 +1.00335672 0.10037558 0.40148402 +1.10000555 0.10000050 0.40000217 +1.20000000 0.10000000 0.40000000 +0.00000000 0.35501801 0.75701522 +0.17683110 0.35105276 0.74994845 +0.34252623 0.34252623 0.72359708 +0.49245949 0.33036339 0.67799702 +0.61746997 0.30925925 0.61746997 +0.71543727 0.28223638 0.55957630 +0.78816087 0.25768821 0.51279397 +0.84899828 0.23944822 0.47825791 +0.90311681 0.22492296 0.44979284 +0.93777007 0.20834384 0.41704032 +1.00154830 0.20034203 0.40067510 +1.10000159 0.20000028 0.40000060 +1.20000000 0.20000000 0.40000000 +0.00000000 0.50149623 0.71080419 +0.17093147 0.50080222 0.70343685 +0.33036339 0.49245949 0.67799702 +0.47026829 0.47026829 0.63468021 +0.58727758 0.43999065 0.58727758 +0.68544422 0.40765797 0.54231789 +0.76328972 0.37623684 0.50064654 +0.82731919 0.35081514 0.46741977 +0.87928885 0.32839894 0.43790516 +0.91846729 0.30610938 0.40823181 +1.00037561 0.30011969 0.40015870 +1.10000015 0.30000004 0.40000005 +1.20000000 0.30000000 0.40000000 +0.00000000 0.64970821 0.64970821 +0.16128471 0.64119052 0.64119052 +0.30925925 0.61746997 0.61746997 +0.43999065 0.58727758 0.58727758 +0.55432823 0.55432823 0.55432823 +0.64933117 0.51735904 0.51735904 +0.73125472 0.48386966 0.48386966 +0.80015564 0.45385923 0.45385923 +0.84633911 0.42186536 0.42186536 +0.90474355 0.40210769 0.40210769 +1.00003915 0.40001565 0.40001565 +1.10000000 0.40000000 0.40000000 +1.20000000 0.40000000 0.40000000 +0.00000000 0.73928284 0.56858842 +0.14458578 0.73369463 0.56719040 +0.28223638 0.71543727 0.55957630 +0.40765797 0.68544422 0.54231789 +0.51735904 0.64933117 0.51735904 +0.61430584 0.61430584 0.49101416 +0.69689697 0.57911331 0.46343371 +0.76247824 0.54287421 0.43423088 +0.81599699 0.50953910 0.40759358 +0.90052141 0.50028459 0.40022936 +1.00000107 0.50000051 0.40000040 +1.10000000 0.50000000 0.40000000 +1.20000000 0.50000000 0.40000000 +0.00000000 0.81006029 0.52387144 +0.13108583 0.80467853 0.52099842 +0.25768821 0.78816087 0.51279397 +0.37623684 0.76328972 0.50064654 +0.48386966 0.73125472 0.48386966 +0.57911331 0.69689697 0.46343371 +0.65872595 0.65872595 0.43876962 +0.72246813 0.61888951 0.41248886 +0.80192020 0.60140266 0.40095016 +0.90001954 0.60001237 0.40000825 +1.00000000 0.60000000 0.40000000 +1.10000000 0.60000000 0.40000000 +1.20000000 0.60000000 0.40000000 +0.00000000 0.86770511 0.48767065 +0.12165362 0.86286900 0.48524650 +0.23944822 0.84899828 0.47825791 +0.35081514 0.82731919 0.46741977 +0.45385923 0.80015564 0.45385923 +0.54287421 0.76247824 0.43423088 +0.61888951 0.72246813 0.41248886 +0.70244509 0.70244509 0.40141285 +0.80006445 0.70005521 0.40003218 +0.90000008 0.70000006 0.40000003 +1.00000000 0.70000000 0.40000000 +1.10000000 0.70000000 0.40000000 +1.20000000 0.70000000 0.40000000 +0.00000000 0.92067108 0.45894538 +0.11417539 0.91652877 0.45674548 +0.22492296 0.90311681 0.44979284 +0.32839894 0.87928885 0.43790516 +0.42186536 0.84633911 0.42186536 +0.50953910 0.81599699 0.40759358 +0.60140266 0.80192020 0.40095016 +0.70005521 0.80006445 0.40003218 +0.80000021 0.80000021 0.40000010 +0.90000000 0.80000000 0.40000000 +1.00000000 0.80000000 0.40000000 +1.10000000 0.80000000 0.40000000 +1.20000000 0.80000000 0.40000000 +0.00000000 0.95881249 0.42696076 +0.10595209 0.95313141 0.42424467 +0.20834384 0.93777007 0.41704032 +0.30610938 0.91846729 0.40823181 +0.40210769 0.90474355 0.40210769 +0.50028459 0.90052141 0.40022936 +0.60001237 0.90001954 0.40000825 +0.70000006 0.90000008 0.40000003 +0.80000000 0.90000000 0.40000000 +0.90000000 0.90000000 0.40000000 +1.00000000 0.90000000 0.40000000 +1.10000000 0.90000000 0.40000000 +1.20000000 0.90000000 0.40000000 +0.00000000 1.00427903 0.40189866 +0.10037558 1.00335672 0.40148402 +0.20034203 1.00154830 0.40067510 +0.30011969 1.00037561 0.40015870 +0.40001565 1.00003915 0.40001565 +0.50000051 1.00000107 0.40000040 +0.60000000 1.00000000 0.40000000 +0.70000000 1.00000000 0.40000000 +0.80000000 1.00000000 0.40000000 +0.90000000 1.00000000 0.40000000 +1.00000000 1.00000000 0.40000000 +1.10000000 1.00000000 0.40000000 +1.20000000 1.00000000 0.40000000 +0.00000000 1.10000820 0.40000322 +0.10000050 1.10000555 0.40000217 +0.20000028 1.10000159 0.40000060 +0.30000004 1.10000015 0.40000005 +0.40000000 1.10000000 0.40000000 +0.50000000 1.10000000 0.40000000 +0.60000000 1.10000000 0.40000000 +0.70000000 1.10000000 0.40000000 +0.80000000 1.10000000 0.40000000 +0.90000000 1.10000000 0.40000000 +1.00000000 1.10000000 0.40000000 +1.10000000 1.10000000 0.40000000 +1.20000000 1.10000000 0.40000000 +0.00000000 1.20000000 0.40000000 +0.10000000 1.20000000 0.40000000 +0.20000000 1.20000000 0.40000000 +0.30000000 1.20000000 0.40000000 +0.40000000 1.20000000 0.40000000 +0.50000000 1.20000000 0.40000000 +0.60000000 1.20000000 0.40000000 +0.70000000 1.20000000 0.40000000 +0.80000000 1.20000000 0.40000000 +0.90000000 1.20000000 0.40000000 +1.00000000 1.20000000 0.40000000 +1.10000000 1.20000000 0.40000000 +1.20000000 1.20000000 0.40000000 +0.00000000 0.00000000 0.84746937 +0.16967335 0.00000000 0.84748313 +0.31714318 0.00000000 0.82700772 +0.44974512 0.00000000 0.78868751 +0.56858842 0.00000000 0.73928284 +0.68717236 0.00000000 0.68717236 +0.76182433 0.00000000 0.62063065 +0.82630414 0.00000000 0.58266570 +0.87929698 0.00000000 0.54716000 +0.91747243 0.00000000 0.50972929 +1.00030406 0.00000000 0.50015749 +1.10000004 0.00000000 0.50000002 +1.20000000 0.00000000 0.50000000 +0.00000000 0.16967335 0.84748313 +0.16592356 0.16592356 0.84371201 +0.31369245 0.15782758 0.82057257 +0.44678136 0.15018738 0.78235981 +0.56719040 0.14458578 0.73369463 +0.68053012 0.13738574 0.68053012 +0.75739272 0.12555538 0.61883699 +0.82242886 0.11646131 0.58018583 +0.87441298 0.10890351 0.54424669 +0.91472437 0.10161500 0.50818514 +1.00022657 0.10002438 0.50011671 +1.10000003 0.10000000 0.50000001 +1.20000000 0.10000000 0.50000000 +0.00000000 0.31714318 0.82700772 +0.15782758 0.31369245 0.82057257 +0.30473274 0.30473274 0.79895918 +0.43867714 0.29400223 0.76188336 +0.55957630 0.28223638 0.71543727 +0.66254081 0.26630201 0.66254081 +0.74412487 0.24659868 0.61203591 +0.81100840 0.22993157 0.57321991 +0.85996942 0.21419343 0.53560066 +0.90841516 0.20186981 0.50466014 +1.00008969 0.20001849 0.50004540 +1.10000001 0.20000000 0.50000000 +1.20000000 0.20000000 0.50000000 +0.00000000 0.44974512 0.78868751 +0.15018738 0.44678136 0.78235981 +0.29400223 0.43867714 0.76188336 +0.42634679 0.42634679 0.72882359 +0.54231789 0.40765797 0.68544422 +0.63909979 0.38385260 0.63909979 +0.72252282 0.36006378 0.59824110 +0.79135631 0.33729954 0.56136548 +0.83815065 0.31343113 0.52260788 +0.90288377 0.30096622 0.50158998 +1.00001657 0.30000484 0.50000815 +1.10000000 0.30000000 0.50000000 +1.20000000 0.30000000 0.50000000 +0.00000000 0.56858842 0.73928284 +0.14458578 0.56719040 0.73369463 +0.28223638 0.55957630 0.71543727 +0.40765797 0.54231789 0.68544422 +0.51735904 0.51735904 0.64933117 +0.61430584 0.49101416 0.61430584 +0.69689697 0.46343371 0.57911331 +0.76247824 0.43423088 0.54287421 +0.81599699 0.40759358 0.50953910 +0.90052141 0.40022936 0.50028459 +1.00000107 0.40000040 0.50000051 +1.10000000 0.40000000 0.50000000 +1.20000000 0.40000000 0.50000000 +0.00000000 0.68717236 0.68717236 +0.13738574 0.68053012 0.68053012 +0.26630201 0.66254081 0.66254081 +0.38385260 0.63909979 0.63909979 +0.49101416 0.61430584 0.61430584 +0.58559764 0.58559764 0.58559764 +0.66643254 0.55462614 0.55462614 +0.72846227 0.51973761 0.51973761 +0.80311288 0.50189877 0.50189877 +0.90004039 0.50002147 0.50002147 +1.00000001 0.50000001 0.50000001 +1.10000000 0.50000000 0.50000000 +1.20000000 0.50000000 0.50000000 +0.00000000 0.76182433 0.62063065 +0.12555538 0.75739272 0.61883699 +0.24659868 0.74412487 0.61203591 +0.36006378 0.72252282 0.59824110 +0.46343371 0.69689697 0.57911331 +0.55462614 0.66643254 0.55462614 +0.62980716 0.62980716 0.52456904 +0.70574649 0.60486862 0.50404594 +0.80022001 0.60016189 0.50013560 +0.90000077 0.60000048 0.50000040 +1.00000000 0.60000000 0.50000000 +1.10000000 0.60000000 0.50000000 +1.20000000 0.60000000 0.50000000 +0.00000000 0.82630414 0.58266570 +0.11646131 0.82242886 0.58018583 +0.22993157 0.81100840 0.57321991 +0.33729954 0.79135631 0.56136548 +0.43423088 0.76247824 0.54287421 +0.51973761 0.72846227 0.51973761 +0.60486862 0.70574649 0.50404594 +0.70032266 0.70032266 0.50023117 +0.80000353 0.70000302 0.50000213 +0.90000000 0.70000000 0.50000000 +1.00000000 0.70000000 0.50000000 +1.10000000 0.70000000 0.50000000 +1.20000000 0.70000000 0.50000000 +0.00000000 0.87929698 0.54716000 +0.10890351 0.87441298 0.54424669 +0.21419343 0.85996942 0.53560066 +0.31343113 0.83815065 0.52260788 +0.40759358 0.81599699 0.50953910 +0.50189877 0.80311288 0.50189877 +0.60016189 0.80022001 0.50013560 +0.70000302 0.80000353 0.50000213 +0.80000000 0.80000000 0.50000000 +0.90000000 0.80000000 0.50000000 +1.00000000 0.80000000 0.50000000 +1.10000000 0.80000000 0.50000000 +1.20000000 0.80000000 0.50000000 +0.00000000 0.91747243 0.50972929 +0.10161500 0.91472437 0.50818514 +0.20186981 0.90841516 0.50466014 +0.30096622 0.90288377 0.50158998 +0.40022936 0.90052141 0.50028459 +0.50002147 0.90004039 0.50002147 +0.60000048 0.90000077 0.50000040 +0.70000000 0.90000000 0.50000000 +0.80000000 0.90000000 0.50000000 +0.90000000 0.90000000 0.50000000 +1.00000000 0.90000000 0.50000000 +1.10000000 0.90000000 0.50000000 +1.20000000 0.90000000 0.50000000 +0.00000000 1.00030406 0.50015749 +0.10002438 1.00022657 0.50011671 +0.20001849 1.00008969 0.50004540 +0.30000484 1.00001657 0.50000815 +0.40000040 1.00000107 0.50000051 +0.50000001 1.00000001 0.50000001 +0.60000000 1.00000000 0.50000000 +0.70000000 1.00000000 0.50000000 +0.80000000 1.00000000 0.50000000 +0.90000000 1.00000000 0.50000000 +1.00000000 1.00000000 0.50000000 +1.10000000 1.00000000 0.50000000 +1.20000000 1.00000000 0.50000000 +0.00000000 1.10000004 0.50000002 +0.10000000 1.10000003 0.50000001 +0.20000000 1.10000001 0.50000000 +0.30000000 1.10000000 0.50000000 +0.40000000 1.10000000 0.50000000 +0.50000000 1.10000000 0.50000000 +0.60000000 1.10000000 0.50000000 +0.70000000 1.10000000 0.50000000 +0.80000000 1.10000000 0.50000000 +0.90000000 1.10000000 0.50000000 +1.00000000 1.10000000 0.50000000 +1.10000000 1.10000000 0.50000000 +1.20000000 1.10000000 0.50000000 +0.00000000 1.20000000 0.50000000 +0.10000000 1.20000000 0.50000000 +0.20000000 1.20000000 0.50000000 +0.30000000 1.20000000 0.50000000 +0.40000000 1.20000000 0.50000000 +0.50000000 1.20000000 0.50000000 +0.60000000 1.20000000 0.50000000 +0.70000000 1.20000000 0.50000000 +0.80000000 1.20000000 0.50000000 +0.90000000 1.20000000 0.50000000 +1.00000000 1.20000000 0.50000000 +1.10000000 1.20000000 0.50000000 +1.20000000 1.20000000 0.50000000 +0.00000000 0.00000000 0.91204792 +0.15517265 0.00000000 0.91018232 +0.29071456 0.00000000 0.88817242 +0.41230124 0.00000000 0.85270186 +0.52387144 0.00000000 0.81006029 +0.62063065 0.00000000 0.76182433 +0.71717747 0.00000000 0.71717747 +0.78380908 0.00000000 0.66428258 +0.82986250 0.00000000 0.62120277 +0.90158256 0.00000000 0.60103535 +1.00000345 0.00000000 0.60000199 +1.10000000 0.00000000 0.60000000 +1.20000000 0.00000000 0.60000000 +0.00000000 0.15517265 0.91018232 +0.15186355 0.15186355 0.90540812 +0.28735387 0.14416070 0.88178372 +0.40975168 0.13725219 0.84712657 +0.52099842 0.13108583 0.80467853 +0.61883699 0.12555538 0.75739272 +0.71254499 0.11987896 0.71254499 +0.78003954 0.11125308 0.66232491 +0.82642640 0.10310482 0.61873750 +0.90125052 0.10014389 0.60081694 +1.00000241 0.10000021 0.60000138 +1.10000000 0.10000000 0.60000000 +1.20000000 0.10000000 0.60000000 +0.00000000 0.29071456 0.88817242 +0.14416070 0.28735387 0.88178372 +0.27879357 0.27879357 0.86136115 +0.40204041 0.26861035 0.82950195 +0.51279397 0.25768821 0.78816087 +0.61203591 0.24659868 0.74412487 +0.69988798 0.23467848 0.69988798 +0.76772981 0.21891426 0.65451792 +0.81767238 0.20417332 0.61254377 +0.90059787 0.20013549 0.60038862 +1.00000077 0.20000014 0.60000044 +1.10000000 0.20000000 0.60000000 +1.20000000 0.20000000 0.60000000 +0.00000000 0.41230124 0.85270186 +0.13725219 0.40975168 0.84712657 +0.26861035 0.40204041 0.82950195 +0.39037523 0.39037523 0.80031590 +0.50064654 0.37623684 0.76328972 +0.59824110 0.36006378 0.72252282 +0.68192803 0.34163275 0.68192803 +0.74630639 0.31941918 0.63839074 +0.80785710 0.30285894 0.60564149 +0.90015817 0.30005217 0.60010177 +1.00000009 0.30000003 0.60000005 +1.10000000 0.30000000 0.60000000 +1.20000000 0.30000000 0.60000000 +0.00000000 0.52387144 0.81006029 +0.13108583 0.52099842 0.80467853 +0.25768821 0.51279397 0.78816087 +0.37623684 0.50064654 0.76328972 +0.48386966 0.48386966 0.73125472 +0.57911331 0.46343371 0.69689697 +0.65872595 0.43876962 0.65872595 +0.72246813 0.41248886 0.61888951 +0.80192020 0.40095016 0.60140266 +0.90001954 0.40000825 0.60001237 +1.00000000 0.40000000 0.60000000 +1.10000000 0.40000000 0.60000000 +1.20000000 0.40000000 0.60000000 +0.00000000 0.62063065 0.76182433 +0.12555538 0.61883699 0.75739272 +0.24659868 0.61203591 0.74412487 +0.36006378 0.59824110 0.72252282 +0.46343371 0.57911331 0.69689697 +0.55462614 0.55462614 0.66643254 +0.62980716 0.52456904 0.62980716 +0.70574649 0.50404594 0.60486862 +0.80022001 0.50013560 0.60016189 +0.90000077 0.50000040 0.60000048 +1.00000000 0.50000000 0.60000000 +1.10000000 0.50000000 0.60000000 +1.20000000 0.50000000 0.60000000 +0.00000000 0.71717747 0.71717747 +0.11987896 0.71254499 0.71254499 +0.23467848 0.69988798 0.69988798 +0.34163275 0.68192803 0.68192803 +0.43876962 0.65872595 0.65872595 +0.52456904 0.62980716 0.62980716 +0.60700341 0.60700341 0.60700341 +0.70056154 0.60047943 0.60047943 +0.80000815 0.60000594 0.60000594 +0.90000001 0.60000000 0.60000000 +1.00000000 0.60000000 0.60000000 +1.10000000 0.60000000 0.60000000 +1.20000000 0.60000000 0.60000000 +0.00000000 0.78380908 0.66428258 +0.11125308 0.78003954 0.66232491 +0.21891426 0.76772981 0.65451792 +0.31941918 0.74630639 0.63839074 +0.41248886 0.72246813 0.61888951 +0.50404594 0.70574649 0.60486862 +0.60047943 0.70056154 0.60047943 +0.70001539 0.70001539 0.60001308 +0.80000004 0.70000003 0.60000003 +0.90000000 0.70000000 0.60000000 +1.00000000 0.70000000 0.60000000 +1.10000000 0.70000000 0.60000000 +1.20000000 0.70000000 0.60000000 +0.00000000 0.82986250 0.62120277 +0.10310482 0.82642640 0.61873750 +0.20417332 0.81767238 0.61254377 +0.30285894 0.80785710 0.60564149 +0.40095016 0.80192020 0.60140266 +0.50013560 0.80022001 0.60016189 +0.60000594 0.80000815 0.60000594 +0.70000003 0.80000004 0.60000003 +0.80000000 0.80000000 0.60000000 +0.90000000 0.80000000 0.60000000 +1.00000000 0.80000000 0.60000000 +1.10000000 0.80000000 0.60000000 +1.20000000 0.80000000 0.60000000 +0.00000000 0.90158256 0.60103535 +0.10014389 0.90125052 0.60081694 +0.20013549 0.90059787 0.60038862 +0.30005217 0.90015817 0.60010177 +0.40000825 0.90001954 0.60001237 +0.50000040 0.90000077 0.60000048 +0.60000000 0.90000001 0.60000000 +0.70000000 0.90000000 0.60000000 +0.80000000 0.90000000 0.60000000 +0.90000000 0.90000000 0.60000000 +1.00000000 0.90000000 0.60000000 +1.10000000 0.90000000 0.60000000 +1.20000000 0.90000000 0.60000000 +0.00000000 1.00000345 0.60000199 +0.10000021 1.00000241 0.60000138 +0.20000014 1.00000077 0.60000044 +0.30000003 1.00000009 0.60000005 +0.40000000 1.00000000 0.60000000 +0.50000000 1.00000000 0.60000000 +0.60000000 1.00000000 0.60000000 +0.70000000 1.00000000 0.60000000 +0.80000000 1.00000000 0.60000000 +0.90000000 1.00000000 0.60000000 +1.00000000 1.00000000 0.60000000 +1.10000000 1.00000000 0.60000000 +1.20000000 1.00000000 0.60000000 +0.00000000 1.10000000 0.60000000 +0.10000000 1.10000000 0.60000000 +0.20000000 1.10000000 0.60000000 +0.30000000 1.10000000 0.60000000 +0.40000000 1.10000000 0.60000000 +0.50000000 1.10000000 0.60000000 +0.60000000 1.10000000 0.60000000 +0.70000000 1.10000000 0.60000000 +0.80000000 1.10000000 0.60000000 +0.90000000 1.10000000 0.60000000 +1.00000000 1.10000000 0.60000000 +1.10000000 1.10000000 0.60000000 +1.20000000 1.10000000 0.60000000 +0.00000000 1.20000000 0.60000000 +0.10000000 1.20000000 0.60000000 +0.20000000 1.20000000 0.60000000 +0.30000000 1.20000000 0.60000000 +0.40000000 1.20000000 0.60000000 +0.50000000 1.20000000 0.60000000 +0.60000000 1.20000000 0.60000000 +0.70000000 1.20000000 0.60000000 +0.80000000 1.20000000 0.60000000 +0.90000000 1.20000000 0.60000000 +1.00000000 1.20000000 0.60000000 +1.10000000 1.20000000 0.60000000 +1.20000000 1.20000000 0.60000000 +0.00000000 0.00000000 0.96962515 +0.14332892 0.00000000 0.96483786 +0.27030805 0.00000000 0.94191103 +0.38368634 0.00000000 0.90696809 +0.48767065 0.00000000 0.86770511 +0.58266570 0.00000000 0.82630414 +0.66428258 0.00000000 0.78380908 +0.73913964 0.00000000 0.73913964 +0.80329722 0.00000000 0.70269171 +0.90002831 0.00000000 0.70002079 +1.00000000 0.00000000 0.70000000 +1.10000000 0.00000000 0.70000000 +1.20000000 0.00000000 0.70000000 +0.00000000 0.14332892 0.96483786 +0.14058988 0.14058988 0.95897612 +0.26714900 0.13370188 0.93524877 +0.38108807 0.12715273 0.90155981 +0.48524650 0.12165362 0.86286900 +0.58018583 0.11646131 0.82242886 +0.66232491 0.11125308 0.78003954 +0.73398435 0.10513418 0.73398435 +0.80271483 0.10034158 0.70223247 +0.90002082 0.10000221 0.70001527 +1.00000000 0.10000000 0.70000000 +1.10000000 0.10000000 0.70000000 +1.20000000 0.10000000 0.70000000 +0.00000000 0.27030805 0.94191103 +0.13370188 0.26714900 0.93524877 +0.25903270 0.25903270 0.91526121 +0.37365783 0.24924309 0.88510792 +0.47825791 0.23944822 0.84899828 +0.57321991 0.22993157 0.81100840 +0.65451792 0.21891426 0.76772981 +0.72157041 0.20635303 0.72157041 +0.80144842 0.20037105 0.70121158 +0.90000795 0.20000164 0.70000581 +1.00000000 0.20000000 0.70000000 +1.10000000 0.20000000 0.70000000 +1.20000000 0.20000000 0.70000000 +0.00000000 0.38368634 0.90696809 +0.12715273 0.38108807 0.90155981 +0.24924309 0.37365783 0.88510792 +0.36286548 0.36286548 0.85896581 +0.46741977 0.35081514 0.82731919 +0.56136548 0.33729954 0.79135631 +0.63839074 0.31941918 0.74630639 +0.70939520 0.30409247 0.70939520 +0.80044005 0.30016913 0.70037406 +0.90000138 0.30000042 0.70000101 +1.00000000 0.30000000 0.70000000 +1.10000000 0.30000000 0.70000000 +1.20000000 0.30000000 0.70000000 +0.00000000 0.48767065 0.86770511 +0.12165362 0.48524650 0.86286900 +0.23944822 0.47825791 0.84899828 +0.35081514 0.46741977 0.82731919 +0.45385923 0.45385923 0.80015564 +0.54287421 0.43423088 0.76247824 +0.61888951 0.41248886 0.72246813 +0.70244509 0.40141285 0.70244509 +0.80006445 0.40003218 0.70005521 +0.90000008 0.40000003 0.70000006 +1.00000000 0.40000000 0.70000000 +1.10000000 0.40000000 0.70000000 +1.20000000 0.40000000 0.70000000 +0.00000000 0.58266570 0.82630414 +0.11646131 0.58018583 0.82242886 +0.22993157 0.57321991 0.81100840 +0.33729954 0.56136548 0.79135631 +0.43423088 0.54287421 0.76247824 +0.51973761 0.51973761 0.72846227 +0.60486862 0.50404594 0.70574649 +0.70032266 0.50023117 0.70032266 +0.80000353 0.50000213 0.70000302 +0.90000000 0.50000000 0.70000000 +1.00000000 0.50000000 0.70000000 +1.10000000 0.50000000 0.70000000 +1.20000000 0.50000000 0.70000000 +0.00000000 0.66428258 0.78380908 +0.11125308 0.66232491 0.78003954 +0.21891426 0.65451792 0.76772981 +0.31941918 0.63839074 0.74630639 +0.41248886 0.61888951 0.72246813 +0.50404594 0.60486862 0.70574649 +0.60047943 0.60047943 0.70056154 +0.70001539 0.60001308 0.70001539 +0.80000004 0.60000003 0.70000003 +0.90000000 0.60000000 0.70000000 +1.00000000 0.60000000 0.70000000 +1.10000000 0.60000000 0.70000000 +1.20000000 0.60000000 0.70000000 +0.00000000 0.73913964 0.73913964 +0.10513418 0.73398435 0.73398435 +0.20635303 0.72157041 0.72157041 +0.30409247 0.70939520 0.70939520 +0.40141285 0.70244509 0.70244509 +0.50023117 0.70032266 0.70032266 +0.60001308 0.70001539 0.70001539 +0.70000010 0.70000010 0.70000010 +0.80000000 0.70000000 0.70000000 +0.90000000 0.70000000 0.70000000 +1.00000000 0.70000000 0.70000000 +1.10000000 0.70000000 0.70000000 +1.20000000 0.70000000 0.70000000 +0.00000000 0.80329722 0.70269171 +0.10034158 0.80271483 0.70223247 +0.20037105 0.80144842 0.70121158 +0.30016913 0.80044005 0.70037406 +0.40003218 0.80006445 0.70005521 +0.50000213 0.80000353 0.70000302 +0.60000003 0.80000004 0.70000003 +0.70000000 0.80000000 0.70000000 +0.80000000 0.80000000 0.70000000 +0.90000000 0.80000000 0.70000000 +1.00000000 0.80000000 0.70000000 +1.10000000 0.80000000 0.70000000 +1.20000000 0.80000000 0.70000000 +0.00000000 0.90002831 0.70002079 +0.10000221 0.90002082 0.70001527 +0.20000164 0.90000795 0.70000581 +0.30000042 0.90000138 0.70000101 +0.40000003 0.90000008 0.70000006 +0.50000000 0.90000000 0.70000000 +0.60000000 0.90000000 0.70000000 +0.70000000 0.90000000 0.70000000 +0.80000000 0.90000000 0.70000000 +0.90000000 0.90000000 0.70000000 +1.00000000 0.90000000 0.70000000 +1.10000000 0.90000000 0.70000000 +1.20000000 0.90000000 0.70000000 +0.00000000 1.00000000 0.70000000 +0.10000000 1.00000000 0.70000000 +0.20000000 1.00000000 0.70000000 +0.30000000 1.00000000 0.70000000 +0.40000000 1.00000000 0.70000000 +0.50000000 1.00000000 0.70000000 +0.60000000 1.00000000 0.70000000 +0.70000000 1.00000000 0.70000000 +0.80000000 1.00000000 0.70000000 +0.90000000 1.00000000 0.70000000 +1.00000000 1.00000000 0.70000000 +1.10000000 1.00000000 0.70000000 +1.20000000 1.00000000 0.70000000 +0.00000000 1.10000000 0.70000000 +0.10000000 1.10000000 0.70000000 +0.20000000 1.10000000 0.70000000 +0.30000000 1.10000000 0.70000000 +0.40000000 1.10000000 0.70000000 +0.50000000 1.10000000 0.70000000 +0.60000000 1.10000000 0.70000000 +0.70000000 1.10000000 0.70000000 +0.80000000 1.10000000 0.70000000 +0.90000000 1.10000000 0.70000000 +1.00000000 1.10000000 0.70000000 +1.10000000 1.10000000 0.70000000 +1.20000000 1.10000000 0.70000000 +0.00000000 1.20000000 0.70000000 +0.10000000 1.20000000 0.70000000 +0.20000000 1.20000000 0.70000000 +0.30000000 1.20000000 0.70000000 +0.40000000 1.20000000 0.70000000 +0.50000000 1.20000000 0.70000000 +0.60000000 1.20000000 0.70000000 +0.70000000 1.20000000 0.70000000 +0.80000000 1.20000000 0.70000000 +0.90000000 1.20000000 0.70000000 +1.00000000 1.20000000 0.70000000 +1.10000000 1.20000000 0.70000000 +1.20000000 1.20000000 0.70000000 +0.00000000 0.00000000 1.01908611 +0.13287109 0.00000000 1.01357716 +0.25193669 0.00000000 0.98673239 +0.35904395 0.00000000 0.95438813 +0.45894538 0.00000000 0.92067108 +0.54716000 0.00000000 0.87929698 +0.62120277 0.00000000 0.82986250 +0.70269171 0.00000000 0.80329722 +0.80011294 0.00000000 0.80011294 +0.90000003 0.00000000 0.80000002 +1.00000000 0.00000000 0.80000000 +1.10000000 0.00000000 0.80000000 +1.20000000 0.00000000 0.80000000 +0.00000000 0.13287109 1.01357716 +0.13066508 0.13066508 1.00656530 +0.24916134 0.12455660 0.98023176 +0.35679227 0.11879272 0.94953048 +0.45674548 0.11417539 0.91652877 +0.54424669 0.10890351 0.87441298 +0.61873750 0.10310482 0.82642640 +0.70223247 0.10034158 0.80271483 +0.80007988 0.10001129 0.80007988 +0.90000002 0.10000000 0.80000002 +1.00000000 0.10000000 0.80000000 +1.10000000 0.10000000 0.80000000 +1.20000000 0.10000000 0.80000000 +0.00000000 0.25193669 0.98673239 +0.12455660 0.24916134 0.98023176 +0.24190248 0.24190248 0.96161127 +0.35040098 0.23356272 0.93512570 +0.44979284 0.22492296 0.90311681 +0.53560066 0.21419343 0.85996942 +0.61254377 0.20417332 0.81767238 +0.70121158 0.20037105 0.80144842 +0.80002681 0.20000706 0.80002681 +0.90000001 0.20000000 0.80000001 +1.00000000 0.20000000 0.80000000 +1.10000000 0.20000000 0.80000000 +1.20000000 0.20000000 0.80000000 +0.00000000 0.35904395 0.95438813 +0.11879272 0.35679227 0.94953048 +0.23356272 0.35040098 0.93512570 +0.34124155 0.34124155 0.91301195 +0.43790516 0.32839894 0.87928885 +0.52260788 0.31343113 0.83815065 +0.60564149 0.30285894 0.80785710 +0.70037406 0.30016913 0.80044005 +0.80000384 0.30000142 0.80000384 +0.90000000 0.30000000 0.80000000 +1.00000000 0.30000000 0.80000000 +1.10000000 0.30000000 0.80000000 +1.20000000 0.30000000 0.80000000 +0.00000000 0.45894538 0.92067108 +0.11417539 0.45674548 0.91652877 +0.22492296 0.44979284 0.90311681 +0.32839894 0.43790516 0.87928885 +0.42186536 0.42186536 0.84633911 +0.50953910 0.40759358 0.81599699 +0.60140266 0.40095016 0.80192020 +0.70005521 0.40003218 0.80006445 +0.80000021 0.40000010 0.80000021 +0.90000000 0.40000000 0.80000000 +1.00000000 0.40000000 0.80000000 +1.10000000 0.40000000 0.80000000 +1.20000000 0.40000000 0.80000000 +0.00000000 0.54716000 0.87929698 +0.10890351 0.54424669 0.87441298 +0.21419343 0.53560066 0.85996942 +0.31343113 0.52260788 0.83815065 +0.40759358 0.50953910 0.81599699 +0.50189877 0.50189877 0.80311288 +0.60016189 0.50013560 0.80022001 +0.70000302 0.50000213 0.80000353 +0.80000000 0.50000000 0.80000000 +0.90000000 0.50000000 0.80000000 +1.00000000 0.50000000 0.80000000 +1.10000000 0.50000000 0.80000000 +1.20000000 0.50000000 0.80000000 +0.00000000 0.62120277 0.82986250 +0.10310482 0.61873750 0.82642640 +0.20417332 0.61254377 0.81767238 +0.30285894 0.60564149 0.80785710 +0.40095016 0.60140266 0.80192020 +0.50013560 0.60016189 0.80022001 +0.60000594 0.60000594 0.80000815 +0.70000003 0.60000003 0.80000004 +0.80000000 0.60000000 0.80000000 +0.90000000 0.60000000 0.80000000 +1.00000000 0.60000000 0.80000000 +1.10000000 0.60000000 0.80000000 +1.20000000 0.60000000 0.80000000 +0.00000000 0.70269171 0.80329722 +0.10034158 0.70223247 0.80271483 +0.20037105 0.70121158 0.80144842 +0.30016913 0.70037406 0.80044005 +0.40003218 0.70005521 0.80006445 +0.50000213 0.70000302 0.80000353 +0.60000003 0.70000003 0.80000004 +0.70000000 0.70000000 0.80000000 +0.80000000 0.70000000 0.80000000 +0.90000000 0.70000000 0.80000000 +1.00000000 0.70000000 0.80000000 +1.10000000 0.70000000 0.80000000 +1.20000000 0.70000000 0.80000000 +0.00000000 0.80011294 0.80011294 +0.10001129 0.80007988 0.80007988 +0.20000706 0.80002681 0.80002681 +0.30000142 0.80000384 0.80000384 +0.40000010 0.80000021 0.80000021 +0.50000000 0.80000000 0.80000000 +0.60000000 0.80000000 0.80000000 +0.70000000 0.80000000 0.80000000 +0.80000000 0.80000000 0.80000000 +0.90000000 0.80000000 0.80000000 +1.00000000 0.80000000 0.80000000 +1.10000000 0.80000000 0.80000000 +1.20000000 0.80000000 0.80000000 +0.00000000 0.90000003 0.80000002 +0.10000000 0.90000002 0.80000002 +0.20000000 0.90000001 0.80000001 +0.30000000 0.90000000 0.80000000 +0.40000000 0.90000000 0.80000000 +0.50000000 0.90000000 0.80000000 +0.60000000 0.90000000 0.80000000 +0.70000000 0.90000000 0.80000000 +0.80000000 0.90000000 0.80000000 +0.90000000 0.90000000 0.80000000 +1.00000000 0.90000000 0.80000000 +1.10000000 0.90000000 0.80000000 +1.20000000 0.90000000 0.80000000 +0.00000000 1.00000000 0.80000000 +0.10000000 1.00000000 0.80000000 +0.20000000 1.00000000 0.80000000 +0.30000000 1.00000000 0.80000000 +0.40000000 1.00000000 0.80000000 +0.50000000 1.00000000 0.80000000 +0.60000000 1.00000000 0.80000000 +0.70000000 1.00000000 0.80000000 +0.80000000 1.00000000 0.80000000 +0.90000000 1.00000000 0.80000000 +1.00000000 1.00000000 0.80000000 +1.10000000 1.00000000 0.80000000 +1.20000000 1.00000000 0.80000000 +0.00000000 1.10000000 0.80000000 +0.10000000 1.10000000 0.80000000 +0.20000000 1.10000000 0.80000000 +0.30000000 1.10000000 0.80000000 +0.40000000 1.10000000 0.80000000 +0.50000000 1.10000000 0.80000000 +0.60000000 1.10000000 0.80000000 +0.70000000 1.10000000 0.80000000 +0.80000000 1.10000000 0.80000000 +0.90000000 1.10000000 0.80000000 +1.00000000 1.10000000 0.80000000 +1.10000000 1.10000000 0.80000000 +1.20000000 1.10000000 0.80000000 +0.00000000 1.20000000 0.80000000 +0.10000000 1.20000000 0.80000000 +0.20000000 1.20000000 0.80000000 +0.30000000 1.20000000 0.80000000 +0.40000000 1.20000000 0.80000000 +0.50000000 1.20000000 0.80000000 +0.60000000 1.20000000 0.80000000 +0.70000000 1.20000000 0.80000000 +0.80000000 1.20000000 0.80000000 +0.90000000 1.20000000 0.80000000 +1.00000000 1.20000000 0.80000000 +1.10000000 1.20000000 0.80000000 +1.20000000 1.20000000 0.80000000 +0.00000000 0.00000000 1.05975035 +0.12283560 0.00000000 1.05399342 +0.23617460 0.00000000 1.02993988 +0.33668601 0.00000000 0.99927356 +0.42696076 0.00000000 0.95881249 +0.50972929 0.00000000 0.91747243 +0.60103535 0.00000000 0.90158256 +0.70002079 0.00000000 0.90002831 +0.80000002 0.00000000 0.90000003 +0.90000000 0.00000000 0.90000000 +1.00000000 0.00000000 0.90000000 +1.10000000 0.00000000 0.90000000 +1.20000000 0.00000000 0.90000000 +0.00000000 0.12283560 1.05399342 +0.12143171 0.12143171 1.04745861 +0.23383545 0.11668098 1.02428904 +0.33427106 0.11124799 0.99386503 +0.42424467 0.10595209 0.95313141 +0.50818514 0.10161500 0.91472437 +0.60081694 0.10014389 0.90125052 +0.70001527 0.10000221 0.90002082 +0.80000002 0.10000000 0.90000002 +0.90000000 0.10000000 0.90000000 +1.00000000 0.10000000 0.90000000 +1.10000000 0.10000000 0.90000000 +1.20000000 0.10000000 0.90000000 +0.00000000 0.23617460 1.02993988 +0.11668098 0.23383545 1.02428904 +0.22701033 0.22701033 1.00671014 +0.32704436 0.21784235 0.97657491 +0.41704032 0.20834384 0.93777007 +0.50466014 0.20186981 0.90841516 +0.60038862 0.20013549 0.90059787 +0.70000581 0.20000164 0.90000795 +0.80000001 0.20000000 0.90000001 +0.90000000 0.20000000 0.90000000 +1.00000000 0.20000000 0.90000000 +1.10000000 0.20000000 0.90000000 +1.20000000 0.20000000 0.90000000 +0.00000000 0.33668601 0.99927356 +0.11124799 0.33427106 0.99386503 +0.21784235 0.32704436 0.97657491 +0.31627253 0.31627253 0.94805255 +0.40823181 0.30610938 0.91846729 +0.50158998 0.30096622 0.90288377 +0.60010177 0.30005217 0.90015817 +0.70000101 0.30000042 0.90000138 +0.80000000 0.30000000 0.90000000 +0.90000000 0.30000000 0.90000000 +1.00000000 0.30000000 0.90000000 +1.10000000 0.30000000 0.90000000 +1.20000000 0.30000000 0.90000000 +0.00000000 0.42696076 0.95881249 +0.10595209 0.42424467 0.95313141 +0.20834384 0.41704032 0.93777007 +0.30610938 0.40823181 0.91846729 +0.40210769 0.40210769 0.90474355 +0.50028459 0.40022936 0.90052141 +0.60001237 0.40000825 0.90001954 +0.70000006 0.40000003 0.90000008 +0.80000000 0.40000000 0.90000000 +0.90000000 0.40000000 0.90000000 +1.00000000 0.40000000 0.90000000 +1.10000000 0.40000000 0.90000000 +1.20000000 0.40000000 0.90000000 +0.00000000 0.50972929 0.91747243 +0.10161500 0.50818514 0.91472437 +0.20186981 0.50466014 0.90841516 +0.30096622 0.50158998 0.90288377 +0.40022936 0.50028459 0.90052141 +0.50002147 0.50002147 0.90004039 +0.60000048 0.50000040 0.90000077 +0.70000000 0.50000000 0.90000000 +0.80000000 0.50000000 0.90000000 +0.90000000 0.50000000 0.90000000 +1.00000000 0.50000000 0.90000000 +1.10000000 0.50000000 0.90000000 +1.20000000 0.50000000 0.90000000 +0.00000000 0.60103535 0.90158256 +0.10014389 0.60081694 0.90125052 +0.20013549 0.60038862 0.90059787 +0.30005217 0.60010177 0.90015817 +0.40000825 0.60001237 0.90001954 +0.50000040 0.60000048 0.90000077 +0.60000000 0.60000000 0.90000001 +0.70000000 0.60000000 0.90000000 +0.80000000 0.60000000 0.90000000 +0.90000000 0.60000000 0.90000000 +1.00000000 0.60000000 0.90000000 +1.10000000 0.60000000 0.90000000 +1.20000000 0.60000000 0.90000000 +0.00000000 0.70002079 0.90002831 +0.10000221 0.70001527 0.90002082 +0.20000164 0.70000581 0.90000795 +0.30000042 0.70000101 0.90000138 +0.40000003 0.70000006 0.90000008 +0.50000000 0.70000000 0.90000000 +0.60000000 0.70000000 0.90000000 +0.70000000 0.70000000 0.90000000 +0.80000000 0.70000000 0.90000000 +0.90000000 0.70000000 0.90000000 +1.00000000 0.70000000 0.90000000 +1.10000000 0.70000000 0.90000000 +1.20000000 0.70000000 0.90000000 +0.00000000 0.80000002 0.90000003 +0.10000000 0.80000002 0.90000002 +0.20000000 0.80000001 0.90000001 +0.30000000 0.80000000 0.90000000 +0.40000000 0.80000000 0.90000000 +0.50000000 0.80000000 0.90000000 +0.60000000 0.80000000 0.90000000 +0.70000000 0.80000000 0.90000000 +0.80000000 0.80000000 0.90000000 +0.90000000 0.80000000 0.90000000 +1.00000000 0.80000000 0.90000000 +1.10000000 0.80000000 0.90000000 +1.20000000 0.80000000 0.90000000 +0.00000000 0.90000000 0.90000000 +0.10000000 0.90000000 0.90000000 +0.20000000 0.90000000 0.90000000 +0.30000000 0.90000000 0.90000000 +0.40000000 0.90000000 0.90000000 +0.50000000 0.90000000 0.90000000 +0.60000000 0.90000000 0.90000000 +0.70000000 0.90000000 0.90000000 +0.80000000 0.90000000 0.90000000 +0.90000000 0.90000000 0.90000000 +1.00000000 0.90000000 0.90000000 +1.10000000 0.90000000 0.90000000 +1.20000000 0.90000000 0.90000000 +0.00000000 1.00000000 0.90000000 +0.10000000 1.00000000 0.90000000 +0.20000000 1.00000000 0.90000000 +0.30000000 1.00000000 0.90000000 +0.40000000 1.00000000 0.90000000 +0.50000000 1.00000000 0.90000000 +0.60000000 1.00000000 0.90000000 +0.70000000 1.00000000 0.90000000 +0.80000000 1.00000000 0.90000000 +0.90000000 1.00000000 0.90000000 +1.00000000 1.00000000 0.90000000 +1.10000000 1.00000000 0.90000000 +1.20000000 1.00000000 0.90000000 +0.00000000 1.10000000 0.90000000 +0.10000000 1.10000000 0.90000000 +0.20000000 1.10000000 0.90000000 +0.30000000 1.10000000 0.90000000 +0.40000000 1.10000000 0.90000000 +0.50000000 1.10000000 0.90000000 +0.60000000 1.10000000 0.90000000 +0.70000000 1.10000000 0.90000000 +0.80000000 1.10000000 0.90000000 +0.90000000 1.10000000 0.90000000 +1.00000000 1.10000000 0.90000000 +1.10000000 1.10000000 0.90000000 +1.20000000 1.10000000 0.90000000 +0.00000000 1.20000000 0.90000000 +0.10000000 1.20000000 0.90000000 +0.20000000 1.20000000 0.90000000 +0.30000000 1.20000000 0.90000000 +0.40000000 1.20000000 0.90000000 +0.50000000 1.20000000 0.90000000 +0.60000000 1.20000000 0.90000000 +0.70000000 1.20000000 0.90000000 +0.80000000 1.20000000 0.90000000 +0.90000000 1.20000000 0.90000000 +1.00000000 1.20000000 0.90000000 +1.10000000 1.20000000 0.90000000 +1.20000000 1.20000000 0.90000000 +0.00000000 0.00000000 1.10930635 +0.11378558 0.00000000 1.09851927 +0.21674437 0.00000000 1.06148154 +0.30867229 0.00000000 1.02383514 +0.40189866 0.00000000 1.00427903 +0.50015749 0.00000000 1.00030406 +0.60000199 0.00000000 1.00000345 +0.70000000 0.00000000 1.00000000 +0.80000000 0.00000000 1.00000000 +0.90000000 0.00000000 1.00000000 +1.00000000 0.00000000 1.00000000 +1.10000000 0.00000000 1.00000000 +1.20000000 0.00000000 1.00000000 +0.00000000 0.11378558 1.09851927 +0.11202892 0.11202892 1.08708267 +0.21405368 0.10675696 1.05283153 +0.30706878 0.10223985 1.01968452 +0.40148402 0.10037558 1.00335672 +0.50011671 0.10002438 1.00022657 +0.60000138 0.10000021 1.00000241 +0.70000000 0.10000000 1.00000000 +0.80000000 0.10000000 1.00000000 +0.90000000 0.10000000 1.00000000 +1.00000000 0.10000000 1.00000000 +1.10000000 0.10000000 1.00000000 +1.20000000 0.10000000 1.00000000 +0.00000000 0.21674437 1.06148154 +0.10675696 0.21405368 1.05283153 +0.20766246 0.20766246 1.03110077 +0.30359058 0.20235762 1.01034139 +0.40067510 0.20034203 1.00154830 +0.50004540 0.20001849 1.00008969 +0.60000044 0.20000014 1.00000077 +0.70000000 0.20000000 1.00000000 +0.80000000 0.20000000 1.00000000 +0.90000000 0.20000000 1.00000000 +1.00000000 0.20000000 1.00000000 +1.10000000 0.20000000 1.00000000 +1.20000000 0.20000000 1.00000000 +0.00000000 0.30867229 1.02383514 +0.10223985 0.30706878 1.01968452 +0.20235762 0.30359058 1.01034139 +0.30096251 0.30096251 1.00287541 +0.40015870 0.30011969 1.00037561 +0.50000815 0.30000484 1.00001657 +0.60000005 0.30000003 1.00000009 +0.70000000 0.30000000 1.00000000 +0.80000000 0.30000000 1.00000000 +0.90000000 0.30000000 1.00000000 +1.00000000 0.30000000 1.00000000 +1.10000000 0.30000000 1.00000000 +1.20000000 0.30000000 1.00000000 +0.00000000 0.40189866 1.00427903 +0.10037558 0.40148402 1.00335672 +0.20034203 0.40067510 1.00154830 +0.30011969 0.40015870 1.00037561 +0.40001565 0.40001565 1.00003915 +0.50000051 0.40000040 1.00000107 +0.60000000 0.40000000 1.00000000 +0.70000000 0.40000000 1.00000000 +0.80000000 0.40000000 1.00000000 +0.90000000 0.40000000 1.00000000 +1.00000000 0.40000000 1.00000000 +1.10000000 0.40000000 1.00000000 +1.20000000 0.40000000 1.00000000 +0.00000000 0.50015749 1.00030406 +0.10002438 0.50011671 1.00022657 +0.20001849 0.50004540 1.00008969 +0.30000484 0.50000815 1.00001657 +0.40000040 0.50000051 1.00000107 +0.50000001 0.50000001 1.00000001 +0.60000000 0.50000000 1.00000000 +0.70000000 0.50000000 1.00000000 +0.80000000 0.50000000 1.00000000 +0.90000000 0.50000000 1.00000000 +1.00000000 0.50000000 1.00000000 +1.10000000 0.50000000 1.00000000 +1.20000000 0.50000000 1.00000000 +0.00000000 0.60000199 1.00000345 +0.10000021 0.60000138 1.00000241 +0.20000014 0.60000044 1.00000077 +0.30000003 0.60000005 1.00000009 +0.40000000 0.60000000 1.00000000 +0.50000000 0.60000000 1.00000000 +0.60000000 0.60000000 1.00000000 +0.70000000 0.60000000 1.00000000 +0.80000000 0.60000000 1.00000000 +0.90000000 0.60000000 1.00000000 +1.00000000 0.60000000 1.00000000 +1.10000000 0.60000000 1.00000000 +1.20000000 0.60000000 1.00000000 +0.00000000 0.70000000 1.00000000 +0.10000000 0.70000000 1.00000000 +0.20000000 0.70000000 1.00000000 +0.30000000 0.70000000 1.00000000 +0.40000000 0.70000000 1.00000000 +0.50000000 0.70000000 1.00000000 +0.60000000 0.70000000 1.00000000 +0.70000000 0.70000000 1.00000000 +0.80000000 0.70000000 1.00000000 +0.90000000 0.70000000 1.00000000 +1.00000000 0.70000000 1.00000000 +1.10000000 0.70000000 1.00000000 +1.20000000 0.70000000 1.00000000 +0.00000000 0.80000000 1.00000000 +0.10000000 0.80000000 1.00000000 +0.20000000 0.80000000 1.00000000 +0.30000000 0.80000000 1.00000000 +0.40000000 0.80000000 1.00000000 +0.50000000 0.80000000 1.00000000 +0.60000000 0.80000000 1.00000000 +0.70000000 0.80000000 1.00000000 +0.80000000 0.80000000 1.00000000 +0.90000000 0.80000000 1.00000000 +1.00000000 0.80000000 1.00000000 +1.10000000 0.80000000 1.00000000 +1.20000000 0.80000000 1.00000000 +0.00000000 0.90000000 1.00000000 +0.10000000 0.90000000 1.00000000 +0.20000000 0.90000000 1.00000000 +0.30000000 0.90000000 1.00000000 +0.40000000 0.90000000 1.00000000 +0.50000000 0.90000000 1.00000000 +0.60000000 0.90000000 1.00000000 +0.70000000 0.90000000 1.00000000 +0.80000000 0.90000000 1.00000000 +0.90000000 0.90000000 1.00000000 +1.00000000 0.90000000 1.00000000 +1.10000000 0.90000000 1.00000000 +1.20000000 0.90000000 1.00000000 +0.00000000 1.00000000 1.00000000 +0.10000000 1.00000000 1.00000000 +0.20000000 1.00000000 1.00000000 +0.30000000 1.00000000 1.00000000 +0.40000000 1.00000000 1.00000000 +0.50000000 1.00000000 1.00000000 +0.60000000 1.00000000 1.00000000 +0.70000000 1.00000000 1.00000000 +0.80000000 1.00000000 1.00000000 +0.90000000 1.00000000 1.00000000 +1.00000000 1.00000000 1.00000000 +1.10000000 1.00000000 1.00000000 +1.20000000 1.00000000 1.00000000 +0.00000000 1.10000000 1.00000000 +0.10000000 1.10000000 1.00000000 +0.20000000 1.10000000 1.00000000 +0.30000000 1.10000000 1.00000000 +0.40000000 1.10000000 1.00000000 +0.50000000 1.10000000 1.00000000 +0.60000000 1.10000000 1.00000000 +0.70000000 1.10000000 1.00000000 +0.80000000 1.10000000 1.00000000 +0.90000000 1.10000000 1.00000000 +1.00000000 1.10000000 1.00000000 +1.10000000 1.10000000 1.00000000 +1.20000000 1.10000000 1.00000000 +0.00000000 1.20000000 1.00000000 +0.10000000 1.20000000 1.00000000 +0.20000000 1.20000000 1.00000000 +0.30000000 1.20000000 1.00000000 +0.40000000 1.20000000 1.00000000 +0.50000000 1.20000000 1.00000000 +0.60000000 1.20000000 1.00000000 +0.70000000 1.20000000 1.00000000 +0.80000000 1.20000000 1.00000000 +0.90000000 1.20000000 1.00000000 +1.00000000 1.20000000 1.00000000 +1.10000000 1.20000000 1.00000000 +1.20000000 1.20000000 1.00000000 +0.00000000 0.00000000 1.11637892 +0.10152648 0.00000000 1.11184016 +0.20081847 0.00000000 1.10311427 +0.30009933 0.00000000 1.10029021 +0.40000322 0.00000000 1.10000820 +0.50000002 0.00000000 1.10000004 +0.60000000 0.00000000 1.10000000 +0.70000000 0.00000000 1.10000000 +0.80000000 0.00000000 1.10000000 +0.90000000 0.00000000 1.10000000 +1.00000000 0.00000000 1.10000000 +1.10000000 0.00000000 1.10000000 +1.20000000 0.00000000 1.10000000 +0.00000000 0.10152648 1.11184016 +0.10110022 0.10110022 1.10834589 +0.20056038 0.10028399 1.10216182 +0.30006789 0.10002269 1.10020126 +0.40000217 0.10000050 1.10000555 +0.50000001 0.10000000 1.10000003 +0.60000000 0.10000000 1.10000000 +0.70000000 0.10000000 1.10000000 +0.80000000 0.10000000 1.10000000 +0.90000000 0.10000000 1.10000000 +1.00000000 0.10000000 1.10000000 +1.10000000 0.10000000 1.10000000 +1.20000000 0.10000000 1.10000000 +0.00000000 0.20081847 1.10311427 +0.10028399 0.20056038 1.10216182 +0.20015072 0.20015072 1.10062140 +0.30001958 0.20001291 1.10006116 +0.40000060 0.20000028 1.10000159 +0.50000000 0.20000000 1.10000001 +0.60000000 0.20000000 1.10000000 +0.70000000 0.20000000 1.10000000 +0.80000000 0.20000000 1.10000000 +0.90000000 0.20000000 1.10000000 +1.00000000 0.20000000 1.10000000 +1.10000000 0.20000000 1.10000000 +1.20000000 0.20000000 1.10000000 +0.00000000 0.30009933 1.10029021 +0.10002269 0.30006789 1.10020126 +0.20001291 0.30001958 1.10006116 +0.30000187 0.30000187 1.10000642 +0.40000005 0.30000004 1.10000015 +0.50000000 0.30000000 1.10000000 +0.60000000 0.30000000 1.10000000 +0.70000000 0.30000000 1.10000000 +0.80000000 0.30000000 1.10000000 +0.90000000 0.30000000 1.10000000 +1.00000000 0.30000000 1.10000000 +1.10000000 0.30000000 1.10000000 +1.20000000 0.30000000 1.10000000 +0.00000000 0.40000322 1.10000820 +0.10000050 0.40000217 1.10000555 +0.20000028 0.40000060 1.10000159 +0.30000004 0.40000005 1.10000015 +0.40000000 0.40000000 1.10000000 +0.50000000 0.40000000 1.10000000 +0.60000000 0.40000000 1.10000000 +0.70000000 0.40000000 1.10000000 +0.80000000 0.40000000 1.10000000 +0.90000000 0.40000000 1.10000000 +1.00000000 0.40000000 1.10000000 +1.10000000 0.40000000 1.10000000 +1.20000000 0.40000000 1.10000000 +0.00000000 0.50000002 1.10000004 +0.10000000 0.50000001 1.10000003 +0.20000000 0.50000000 1.10000001 +0.30000000 0.50000000 1.10000000 +0.40000000 0.50000000 1.10000000 +0.50000000 0.50000000 1.10000000 +0.60000000 0.50000000 1.10000000 +0.70000000 0.50000000 1.10000000 +0.80000000 0.50000000 1.10000000 +0.90000000 0.50000000 1.10000000 +1.00000000 0.50000000 1.10000000 +1.10000000 0.50000000 1.10000000 +1.20000000 0.50000000 1.10000000 +0.00000000 0.60000000 1.10000000 +0.10000000 0.60000000 1.10000000 +0.20000000 0.60000000 1.10000000 +0.30000000 0.60000000 1.10000000 +0.40000000 0.60000000 1.10000000 +0.50000000 0.60000000 1.10000000 +0.60000000 0.60000000 1.10000000 +0.70000000 0.60000000 1.10000000 +0.80000000 0.60000000 1.10000000 +0.90000000 0.60000000 1.10000000 +1.00000000 0.60000000 1.10000000 +1.10000000 0.60000000 1.10000000 +1.20000000 0.60000000 1.10000000 +0.00000000 0.70000000 1.10000000 +0.10000000 0.70000000 1.10000000 +0.20000000 0.70000000 1.10000000 +0.30000000 0.70000000 1.10000000 +0.40000000 0.70000000 1.10000000 +0.50000000 0.70000000 1.10000000 +0.60000000 0.70000000 1.10000000 +0.70000000 0.70000000 1.10000000 +0.80000000 0.70000000 1.10000000 +0.90000000 0.70000000 1.10000000 +1.00000000 0.70000000 1.10000000 +1.10000000 0.70000000 1.10000000 +1.20000000 0.70000000 1.10000000 +0.00000000 0.80000000 1.10000000 +0.10000000 0.80000000 1.10000000 +0.20000000 0.80000000 1.10000000 +0.30000000 0.80000000 1.10000000 +0.40000000 0.80000000 1.10000000 +0.50000000 0.80000000 1.10000000 +0.60000000 0.80000000 1.10000000 +0.70000000 0.80000000 1.10000000 +0.80000000 0.80000000 1.10000000 +0.90000000 0.80000000 1.10000000 +1.00000000 0.80000000 1.10000000 +1.10000000 0.80000000 1.10000000 +1.20000000 0.80000000 1.10000000 +0.00000000 0.90000000 1.10000000 +0.10000000 0.90000000 1.10000000 +0.20000000 0.90000000 1.10000000 +0.30000000 0.90000000 1.10000000 +0.40000000 0.90000000 1.10000000 +0.50000000 0.90000000 1.10000000 +0.60000000 0.90000000 1.10000000 +0.70000000 0.90000000 1.10000000 +0.80000000 0.90000000 1.10000000 +0.90000000 0.90000000 1.10000000 +1.00000000 0.90000000 1.10000000 +1.10000000 0.90000000 1.10000000 +1.20000000 0.90000000 1.10000000 +0.00000000 1.00000000 1.10000000 +0.10000000 1.00000000 1.10000000 +0.20000000 1.00000000 1.10000000 +0.30000000 1.00000000 1.10000000 +0.40000000 1.00000000 1.10000000 +0.50000000 1.00000000 1.10000000 +0.60000000 1.00000000 1.10000000 +0.70000000 1.00000000 1.10000000 +0.80000000 1.00000000 1.10000000 +0.90000000 1.00000000 1.10000000 +1.00000000 1.00000000 1.10000000 +1.10000000 1.00000000 1.10000000 +1.20000000 1.00000000 1.10000000 +0.00000000 1.10000000 1.10000000 +0.10000000 1.10000000 1.10000000 +0.20000000 1.10000000 1.10000000 +0.30000000 1.10000000 1.10000000 +0.40000000 1.10000000 1.10000000 +0.50000000 1.10000000 1.10000000 +0.60000000 1.10000000 1.10000000 +0.70000000 1.10000000 1.10000000 +0.80000000 1.10000000 1.10000000 +0.90000000 1.10000000 1.10000000 +1.00000000 1.10000000 1.10000000 +1.10000000 1.10000000 1.10000000 +1.20000000 1.10000000 1.10000000 +0.00000000 1.20000000 1.10000000 +0.10000000 1.20000000 1.10000000 +0.20000000 1.20000000 1.10000000 +0.30000000 1.20000000 1.10000000 +0.40000000 1.20000000 1.10000000 +0.50000000 1.20000000 1.10000000 +0.60000000 1.20000000 1.10000000 +0.70000000 1.20000000 1.10000000 +0.80000000 1.20000000 1.10000000 +0.90000000 1.20000000 1.10000000 +1.00000000 1.20000000 1.10000000 +1.10000000 1.20000000 1.10000000 +1.20000000 1.20000000 1.10000000 +0.00000000 0.00000000 1.20000000 +0.10000052 0.00000000 1.20000000 +0.20000021 0.00000000 1.20000000 +0.30000001 0.00000000 1.20000000 +0.40000000 0.00000000 1.20000000 +0.50000000 0.00000000 1.20000000 +0.60000000 0.00000000 1.20000000 +0.70000000 0.00000000 1.20000000 +0.80000000 0.00000000 1.20000000 +0.90000000 0.00000000 1.20000000 +1.00000000 0.00000000 1.20000000 +1.10000000 0.00000000 1.20000000 +1.20000000 0.00000000 1.20000000 +0.00000000 0.10000052 1.20000000 +0.10000032 0.10000032 1.20000000 +0.20000013 0.10000006 1.20000000 +0.30000000 0.10000000 1.20000000 +0.40000000 0.10000000 1.20000000 +0.50000000 0.10000000 1.20000000 +0.60000000 0.10000000 1.20000000 +0.70000000 0.10000000 1.20000000 +0.80000000 0.10000000 1.20000000 +0.90000000 0.10000000 1.20000000 +1.00000000 0.10000000 1.20000000 +1.10000000 0.10000000 1.20000000 +1.20000000 0.10000000 1.20000000 +0.00000000 0.20000021 1.20000000 +0.10000006 0.20000013 1.20000000 +0.20000002 0.20000002 1.20000000 +0.30000000 0.20000000 1.20000000 +0.40000000 0.20000000 1.20000000 +0.50000000 0.20000000 1.20000000 +0.60000000 0.20000000 1.20000000 +0.70000000 0.20000000 1.20000000 +0.80000000 0.20000000 1.20000000 +0.90000000 0.20000000 1.20000000 +1.00000000 0.20000000 1.20000000 +1.10000000 0.20000000 1.20000000 +1.20000000 0.20000000 1.20000000 +0.00000000 0.30000001 1.20000000 +0.10000000 0.30000000 1.20000000 +0.20000000 0.30000000 1.20000000 +0.30000000 0.30000000 1.20000000 +0.40000000 0.30000000 1.20000000 +0.50000000 0.30000000 1.20000000 +0.60000000 0.30000000 1.20000000 +0.70000000 0.30000000 1.20000000 +0.80000000 0.30000000 1.20000000 +0.90000000 0.30000000 1.20000000 +1.00000000 0.30000000 1.20000000 +1.10000000 0.30000000 1.20000000 +1.20000000 0.30000000 1.20000000 +0.00000000 0.40000000 1.20000000 +0.10000000 0.40000000 1.20000000 +0.20000000 0.40000000 1.20000000 +0.30000000 0.40000000 1.20000000 +0.40000000 0.40000000 1.20000000 +0.50000000 0.40000000 1.20000000 +0.60000000 0.40000000 1.20000000 +0.70000000 0.40000000 1.20000000 +0.80000000 0.40000000 1.20000000 +0.90000000 0.40000000 1.20000000 +1.00000000 0.40000000 1.20000000 +1.10000000 0.40000000 1.20000000 +1.20000000 0.40000000 1.20000000 +0.00000000 0.50000000 1.20000000 +0.10000000 0.50000000 1.20000000 +0.20000000 0.50000000 1.20000000 +0.30000000 0.50000000 1.20000000 +0.40000000 0.50000000 1.20000000 +0.50000000 0.50000000 1.20000000 +0.60000000 0.50000000 1.20000000 +0.70000000 0.50000000 1.20000000 +0.80000000 0.50000000 1.20000000 +0.90000000 0.50000000 1.20000000 +1.00000000 0.50000000 1.20000000 +1.10000000 0.50000000 1.20000000 +1.20000000 0.50000000 1.20000000 +0.00000000 0.60000000 1.20000000 +0.10000000 0.60000000 1.20000000 +0.20000000 0.60000000 1.20000000 +0.30000000 0.60000000 1.20000000 +0.40000000 0.60000000 1.20000000 +0.50000000 0.60000000 1.20000000 +0.60000000 0.60000000 1.20000000 +0.70000000 0.60000000 1.20000000 +0.80000000 0.60000000 1.20000000 +0.90000000 0.60000000 1.20000000 +1.00000000 0.60000000 1.20000000 +1.10000000 0.60000000 1.20000000 +1.20000000 0.60000000 1.20000000 +0.00000000 0.70000000 1.20000000 +0.10000000 0.70000000 1.20000000 +0.20000000 0.70000000 1.20000000 +0.30000000 0.70000000 1.20000000 +0.40000000 0.70000000 1.20000000 +0.50000000 0.70000000 1.20000000 +0.60000000 0.70000000 1.20000000 +0.70000000 0.70000000 1.20000000 +0.80000000 0.70000000 1.20000000 +0.90000000 0.70000000 1.20000000 +1.00000000 0.70000000 1.20000000 +1.10000000 0.70000000 1.20000000 +1.20000000 0.70000000 1.20000000 +0.00000000 0.80000000 1.20000000 +0.10000000 0.80000000 1.20000000 +0.20000000 0.80000000 1.20000000 +0.30000000 0.80000000 1.20000000 +0.40000000 0.80000000 1.20000000 +0.50000000 0.80000000 1.20000000 +0.60000000 0.80000000 1.20000000 +0.70000000 0.80000000 1.20000000 +0.80000000 0.80000000 1.20000000 +0.90000000 0.80000000 1.20000000 +1.00000000 0.80000000 1.20000000 +1.10000000 0.80000000 1.20000000 +1.20000000 0.80000000 1.20000000 +0.00000000 0.90000000 1.20000000 +0.10000000 0.90000000 1.20000000 +0.20000000 0.90000000 1.20000000 +0.30000000 0.90000000 1.20000000 +0.40000000 0.90000000 1.20000000 +0.50000000 0.90000000 1.20000000 +0.60000000 0.90000000 1.20000000 +0.70000000 0.90000000 1.20000000 +0.80000000 0.90000000 1.20000000 +0.90000000 0.90000000 1.20000000 +1.00000000 0.90000000 1.20000000 +1.10000000 0.90000000 1.20000000 +1.20000000 0.90000000 1.20000000 +0.00000000 1.00000000 1.20000000 +0.10000000 1.00000000 1.20000000 +0.20000000 1.00000000 1.20000000 +0.30000000 1.00000000 1.20000000 +0.40000000 1.00000000 1.20000000 +0.50000000 1.00000000 1.20000000 +0.60000000 1.00000000 1.20000000 +0.70000000 1.00000000 1.20000000 +0.80000000 1.00000000 1.20000000 +0.90000000 1.00000000 1.20000000 +1.00000000 1.00000000 1.20000000 +1.10000000 1.00000000 1.20000000 +1.20000000 1.00000000 1.20000000 +0.00000000 1.10000000 1.20000000 +0.10000000 1.10000000 1.20000000 +0.20000000 1.10000000 1.20000000 +0.30000000 1.10000000 1.20000000 +0.40000000 1.10000000 1.20000000 +0.50000000 1.10000000 1.20000000 +0.60000000 1.10000000 1.20000000 +0.70000000 1.10000000 1.20000000 +0.80000000 1.10000000 1.20000000 +0.90000000 1.10000000 1.20000000 +1.00000000 1.10000000 1.20000000 +1.10000000 1.10000000 1.20000000 +1.20000000 1.10000000 1.20000000 +0.00000000 1.20000000 1.20000000 +0.10000000 1.20000000 1.20000000 +0.20000000 1.20000000 1.20000000 +0.30000000 1.20000000 1.20000000 +0.40000000 1.20000000 1.20000000 +0.50000000 1.20000000 1.20000000 +0.60000000 1.20000000 1.20000000 +0.70000000 1.20000000 1.20000000 +0.80000000 1.20000000 1.20000000 +0.90000000 1.20000000 1.20000000 +1.00000000 1.20000000 1.20000000 +1.10000000 1.20000000 1.20000000 +1.20000000 1.20000000 1.20000000 + +CELLS 1728 15552 +8 0 1 14 13 169 170 183 182 +8 1 2 15 14 170 171 184 183 +8 2 3 16 15 171 172 185 184 +8 3 4 17 16 172 173 186 185 +8 4 5 18 17 173 174 187 186 +8 5 6 19 18 174 175 188 187 +8 6 7 20 19 175 176 189 188 +8 7 8 21 20 176 177 190 189 +8 8 9 22 21 177 178 191 190 +8 9 10 23 22 178 179 192 191 +8 10 11 24 23 179 180 193 192 +8 11 12 25 24 180 181 194 193 +8 13 14 27 26 182 183 196 195 +8 14 15 28 27 183 184 197 196 +8 15 16 29 28 184 185 198 197 +8 16 17 30 29 185 186 199 198 +8 17 18 31 30 186 187 200 199 +8 18 19 32 31 187 188 201 200 +8 19 20 33 32 188 189 202 201 +8 20 21 34 33 189 190 203 202 +8 21 22 35 34 190 191 204 203 +8 22 23 36 35 191 192 205 204 +8 23 24 37 36 192 193 206 205 +8 24 25 38 37 193 194 207 206 +8 26 27 40 39 195 196 209 208 +8 27 28 41 40 196 197 210 209 +8 28 29 42 41 197 198 211 210 +8 29 30 43 42 198 199 212 211 +8 30 31 44 43 199 200 213 212 +8 31 32 45 44 200 201 214 213 +8 32 33 46 45 201 202 215 214 +8 33 34 47 46 202 203 216 215 +8 34 35 48 47 203 204 217 216 +8 35 36 49 48 204 205 218 217 +8 36 37 50 49 205 206 219 218 +8 37 38 51 50 206 207 220 219 +8 39 40 53 52 208 209 222 221 +8 40 41 54 53 209 210 223 222 +8 41 42 55 54 210 211 224 223 +8 42 43 56 55 211 212 225 224 +8 43 44 57 56 212 213 226 225 +8 44 45 58 57 213 214 227 226 +8 45 46 59 58 214 215 228 227 +8 46 47 60 59 215 216 229 228 +8 47 48 61 60 216 217 230 229 +8 48 49 62 61 217 218 231 230 +8 49 50 63 62 218 219 232 231 +8 50 51 64 63 219 220 233 232 +8 52 53 66 65 221 222 235 234 +8 53 54 67 66 222 223 236 235 +8 54 55 68 67 223 224 237 236 +8 55 56 69 68 224 225 238 237 +8 56 57 70 69 225 226 239 238 +8 57 58 71 70 226 227 240 239 +8 58 59 72 71 227 228 241 240 +8 59 60 73 72 228 229 242 241 +8 60 61 74 73 229 230 243 242 +8 61 62 75 74 230 231 244 243 +8 62 63 76 75 231 232 245 244 +8 63 64 77 76 232 233 246 245 +8 65 66 79 78 234 235 248 247 +8 66 67 80 79 235 236 249 248 +8 67 68 81 80 236 237 250 249 +8 68 69 82 81 237 238 251 250 +8 69 70 83 82 238 239 252 251 +8 70 71 84 83 239 240 253 252 +8 71 72 85 84 240 241 254 253 +8 72 73 86 85 241 242 255 254 +8 73 74 87 86 242 243 256 255 +8 74 75 88 87 243 244 257 256 +8 75 76 89 88 244 245 258 257 +8 76 77 90 89 245 246 259 258 +8 78 79 92 91 247 248 261 260 +8 79 80 93 92 248 249 262 261 +8 80 81 94 93 249 250 263 262 +8 81 82 95 94 250 251 264 263 +8 82 83 96 95 251 252 265 264 +8 83 84 97 96 252 253 266 265 +8 84 85 98 97 253 254 267 266 +8 85 86 99 98 254 255 268 267 +8 86 87 100 99 255 256 269 268 +8 87 88 101 100 256 257 270 269 +8 88 89 102 101 257 258 271 270 +8 89 90 103 102 258 259 272 271 +8 91 92 105 104 260 261 274 273 +8 92 93 106 105 261 262 275 274 +8 93 94 107 106 262 263 276 275 +8 94 95 108 107 263 264 277 276 +8 95 96 109 108 264 265 278 277 +8 96 97 110 109 265 266 279 278 +8 97 98 111 110 266 267 280 279 +8 98 99 112 111 267 268 281 280 +8 99 100 113 112 268 269 282 281 +8 100 101 114 113 269 270 283 282 +8 101 102 115 114 270 271 284 283 +8 102 103 116 115 271 272 285 284 +8 104 105 118 117 273 274 287 286 +8 105 106 119 118 274 275 288 287 +8 106 107 120 119 275 276 289 288 +8 107 108 121 120 276 277 290 289 +8 108 109 122 121 277 278 291 290 +8 109 110 123 122 278 279 292 291 +8 110 111 124 123 279 280 293 292 +8 111 112 125 124 280 281 294 293 +8 112 113 126 125 281 282 295 294 +8 113 114 127 126 282 283 296 295 +8 114 115 128 127 283 284 297 296 +8 115 116 129 128 284 285 298 297 +8 117 118 131 130 286 287 300 299 +8 118 119 132 131 287 288 301 300 +8 119 120 133 132 288 289 302 301 +8 120 121 134 133 289 290 303 302 +8 121 122 135 134 290 291 304 303 +8 122 123 136 135 291 292 305 304 +8 123 124 137 136 292 293 306 305 +8 124 125 138 137 293 294 307 306 +8 125 126 139 138 294 295 308 307 +8 126 127 140 139 295 296 309 308 +8 127 128 141 140 296 297 310 309 +8 128 129 142 141 297 298 311 310 +8 130 131 144 143 299 300 313 312 +8 131 132 145 144 300 301 314 313 +8 132 133 146 145 301 302 315 314 +8 133 134 147 146 302 303 316 315 +8 134 135 148 147 303 304 317 316 +8 135 136 149 148 304 305 318 317 +8 136 137 150 149 305 306 319 318 +8 137 138 151 150 306 307 320 319 +8 138 139 152 151 307 308 321 320 +8 139 140 153 152 308 309 322 321 +8 140 141 154 153 309 310 323 322 +8 141 142 155 154 310 311 324 323 +8 143 144 157 156 312 313 326 325 +8 144 145 158 157 313 314 327 326 +8 145 146 159 158 314 315 328 327 +8 146 147 160 159 315 316 329 328 +8 147 148 161 160 316 317 330 329 +8 148 149 162 161 317 318 331 330 +8 149 150 163 162 318 319 332 331 +8 150 151 164 163 319 320 333 332 +8 151 152 165 164 320 321 334 333 +8 152 153 166 165 321 322 335 334 +8 153 154 167 166 322 323 336 335 +8 154 155 168 167 323 324 337 336 +8 169 170 183 182 338 339 352 351 +8 170 171 184 183 339 340 353 352 +8 171 172 185 184 340 341 354 353 +8 172 173 186 185 341 342 355 354 +8 173 174 187 186 342 343 356 355 +8 174 175 188 187 343 344 357 356 +8 175 176 189 188 344 345 358 357 +8 176 177 190 189 345 346 359 358 +8 177 178 191 190 346 347 360 359 +8 178 179 192 191 347 348 361 360 +8 179 180 193 192 348 349 362 361 +8 180 181 194 193 349 350 363 362 +8 182 183 196 195 351 352 365 364 +8 183 184 197 196 352 353 366 365 +8 184 185 198 197 353 354 367 366 +8 185 186 199 198 354 355 368 367 +8 186 187 200 199 355 356 369 368 +8 187 188 201 200 356 357 370 369 +8 188 189 202 201 357 358 371 370 +8 189 190 203 202 358 359 372 371 +8 190 191 204 203 359 360 373 372 +8 191 192 205 204 360 361 374 373 +8 192 193 206 205 361 362 375 374 +8 193 194 207 206 362 363 376 375 +8 195 196 209 208 364 365 378 377 +8 196 197 210 209 365 366 379 378 +8 197 198 211 210 366 367 380 379 +8 198 199 212 211 367 368 381 380 +8 199 200 213 212 368 369 382 381 +8 200 201 214 213 369 370 383 382 +8 201 202 215 214 370 371 384 383 +8 202 203 216 215 371 372 385 384 +8 203 204 217 216 372 373 386 385 +8 204 205 218 217 373 374 387 386 +8 205 206 219 218 374 375 388 387 +8 206 207 220 219 375 376 389 388 +8 208 209 222 221 377 378 391 390 +8 209 210 223 222 378 379 392 391 +8 210 211 224 223 379 380 393 392 +8 211 212 225 224 380 381 394 393 +8 212 213 226 225 381 382 395 394 +8 213 214 227 226 382 383 396 395 +8 214 215 228 227 383 384 397 396 +8 215 216 229 228 384 385 398 397 +8 216 217 230 229 385 386 399 398 +8 217 218 231 230 386 387 400 399 +8 218 219 232 231 387 388 401 400 +8 219 220 233 232 388 389 402 401 +8 221 222 235 234 390 391 404 403 +8 222 223 236 235 391 392 405 404 +8 223 224 237 236 392 393 406 405 +8 224 225 238 237 393 394 407 406 +8 225 226 239 238 394 395 408 407 +8 226 227 240 239 395 396 409 408 +8 227 228 241 240 396 397 410 409 +8 228 229 242 241 397 398 411 410 +8 229 230 243 242 398 399 412 411 +8 230 231 244 243 399 400 413 412 +8 231 232 245 244 400 401 414 413 +8 232 233 246 245 401 402 415 414 +8 234 235 248 247 403 404 417 416 +8 235 236 249 248 404 405 418 417 +8 236 237 250 249 405 406 419 418 +8 237 238 251 250 406 407 420 419 +8 238 239 252 251 407 408 421 420 +8 239 240 253 252 408 409 422 421 +8 240 241 254 253 409 410 423 422 +8 241 242 255 254 410 411 424 423 +8 242 243 256 255 411 412 425 424 +8 243 244 257 256 412 413 426 425 +8 244 245 258 257 413 414 427 426 +8 245 246 259 258 414 415 428 427 +8 247 248 261 260 416 417 430 429 +8 248 249 262 261 417 418 431 430 +8 249 250 263 262 418 419 432 431 +8 250 251 264 263 419 420 433 432 +8 251 252 265 264 420 421 434 433 +8 252 253 266 265 421 422 435 434 +8 253 254 267 266 422 423 436 435 +8 254 255 268 267 423 424 437 436 +8 255 256 269 268 424 425 438 437 +8 256 257 270 269 425 426 439 438 +8 257 258 271 270 426 427 440 439 +8 258 259 272 271 427 428 441 440 +8 260 261 274 273 429 430 443 442 +8 261 262 275 274 430 431 444 443 +8 262 263 276 275 431 432 445 444 +8 263 264 277 276 432 433 446 445 +8 264 265 278 277 433 434 447 446 +8 265 266 279 278 434 435 448 447 +8 266 267 280 279 435 436 449 448 +8 267 268 281 280 436 437 450 449 +8 268 269 282 281 437 438 451 450 +8 269 270 283 282 438 439 452 451 +8 270 271 284 283 439 440 453 452 +8 271 272 285 284 440 441 454 453 +8 273 274 287 286 442 443 456 455 +8 274 275 288 287 443 444 457 456 +8 275 276 289 288 444 445 458 457 +8 276 277 290 289 445 446 459 458 +8 277 278 291 290 446 447 460 459 +8 278 279 292 291 447 448 461 460 +8 279 280 293 292 448 449 462 461 +8 280 281 294 293 449 450 463 462 +8 281 282 295 294 450 451 464 463 +8 282 283 296 295 451 452 465 464 +8 283 284 297 296 452 453 466 465 +8 284 285 298 297 453 454 467 466 +8 286 287 300 299 455 456 469 468 +8 287 288 301 300 456 457 470 469 +8 288 289 302 301 457 458 471 470 +8 289 290 303 302 458 459 472 471 +8 290 291 304 303 459 460 473 472 +8 291 292 305 304 460 461 474 473 +8 292 293 306 305 461 462 475 474 +8 293 294 307 306 462 463 476 475 +8 294 295 308 307 463 464 477 476 +8 295 296 309 308 464 465 478 477 +8 296 297 310 309 465 466 479 478 +8 297 298 311 310 466 467 480 479 +8 299 300 313 312 468 469 482 481 +8 300 301 314 313 469 470 483 482 +8 301 302 315 314 470 471 484 483 +8 302 303 316 315 471 472 485 484 +8 303 304 317 316 472 473 486 485 +8 304 305 318 317 473 474 487 486 +8 305 306 319 318 474 475 488 487 +8 306 307 320 319 475 476 489 488 +8 307 308 321 320 476 477 490 489 +8 308 309 322 321 477 478 491 490 +8 309 310 323 322 478 479 492 491 +8 310 311 324 323 479 480 493 492 +8 312 313 326 325 481 482 495 494 +8 313 314 327 326 482 483 496 495 +8 314 315 328 327 483 484 497 496 +8 315 316 329 328 484 485 498 497 +8 316 317 330 329 485 486 499 498 +8 317 318 331 330 486 487 500 499 +8 318 319 332 331 487 488 501 500 +8 319 320 333 332 488 489 502 501 +8 320 321 334 333 489 490 503 502 +8 321 322 335 334 490 491 504 503 +8 322 323 336 335 491 492 505 504 +8 323 324 337 336 492 493 506 505 +8 338 339 352 351 507 508 521 520 +8 339 340 353 352 508 509 522 521 +8 340 341 354 353 509 510 523 522 +8 341 342 355 354 510 511 524 523 +8 342 343 356 355 511 512 525 524 +8 343 344 357 356 512 513 526 525 +8 344 345 358 357 513 514 527 526 +8 345 346 359 358 514 515 528 527 +8 346 347 360 359 515 516 529 528 +8 347 348 361 360 516 517 530 529 +8 348 349 362 361 517 518 531 530 +8 349 350 363 362 518 519 532 531 +8 351 352 365 364 520 521 534 533 +8 352 353 366 365 521 522 535 534 +8 353 354 367 366 522 523 536 535 +8 354 355 368 367 523 524 537 536 +8 355 356 369 368 524 525 538 537 +8 356 357 370 369 525 526 539 538 +8 357 358 371 370 526 527 540 539 +8 358 359 372 371 527 528 541 540 +8 359 360 373 372 528 529 542 541 +8 360 361 374 373 529 530 543 542 +8 361 362 375 374 530 531 544 543 +8 362 363 376 375 531 532 545 544 +8 364 365 378 377 533 534 547 546 +8 365 366 379 378 534 535 548 547 +8 366 367 380 379 535 536 549 548 +8 367 368 381 380 536 537 550 549 +8 368 369 382 381 537 538 551 550 +8 369 370 383 382 538 539 552 551 +8 370 371 384 383 539 540 553 552 +8 371 372 385 384 540 541 554 553 +8 372 373 386 385 541 542 555 554 +8 373 374 387 386 542 543 556 555 +8 374 375 388 387 543 544 557 556 +8 375 376 389 388 544 545 558 557 +8 377 378 391 390 546 547 560 559 +8 378 379 392 391 547 548 561 560 +8 379 380 393 392 548 549 562 561 +8 380 381 394 393 549 550 563 562 +8 381 382 395 394 550 551 564 563 +8 382 383 396 395 551 552 565 564 +8 383 384 397 396 552 553 566 565 +8 384 385 398 397 553 554 567 566 +8 385 386 399 398 554 555 568 567 +8 386 387 400 399 555 556 569 568 +8 387 388 401 400 556 557 570 569 +8 388 389 402 401 557 558 571 570 +8 390 391 404 403 559 560 573 572 +8 391 392 405 404 560 561 574 573 +8 392 393 406 405 561 562 575 574 +8 393 394 407 406 562 563 576 575 +8 394 395 408 407 563 564 577 576 +8 395 396 409 408 564 565 578 577 +8 396 397 410 409 565 566 579 578 +8 397 398 411 410 566 567 580 579 +8 398 399 412 411 567 568 581 580 +8 399 400 413 412 568 569 582 581 +8 400 401 414 413 569 570 583 582 +8 401 402 415 414 570 571 584 583 +8 403 404 417 416 572 573 586 585 +8 404 405 418 417 573 574 587 586 +8 405 406 419 418 574 575 588 587 +8 406 407 420 419 575 576 589 588 +8 407 408 421 420 576 577 590 589 +8 408 409 422 421 577 578 591 590 +8 409 410 423 422 578 579 592 591 +8 410 411 424 423 579 580 593 592 +8 411 412 425 424 580 581 594 593 +8 412 413 426 425 581 582 595 594 +8 413 414 427 426 582 583 596 595 +8 414 415 428 427 583 584 597 596 +8 416 417 430 429 585 586 599 598 +8 417 418 431 430 586 587 600 599 +8 418 419 432 431 587 588 601 600 +8 419 420 433 432 588 589 602 601 +8 420 421 434 433 589 590 603 602 +8 421 422 435 434 590 591 604 603 +8 422 423 436 435 591 592 605 604 +8 423 424 437 436 592 593 606 605 +8 424 425 438 437 593 594 607 606 +8 425 426 439 438 594 595 608 607 +8 426 427 440 439 595 596 609 608 +8 427 428 441 440 596 597 610 609 +8 429 430 443 442 598 599 612 611 +8 430 431 444 443 599 600 613 612 +8 431 432 445 444 600 601 614 613 +8 432 433 446 445 601 602 615 614 +8 433 434 447 446 602 603 616 615 +8 434 435 448 447 603 604 617 616 +8 435 436 449 448 604 605 618 617 +8 436 437 450 449 605 606 619 618 +8 437 438 451 450 606 607 620 619 +8 438 439 452 451 607 608 621 620 +8 439 440 453 452 608 609 622 621 +8 440 441 454 453 609 610 623 622 +8 442 443 456 455 611 612 625 624 +8 443 444 457 456 612 613 626 625 +8 444 445 458 457 613 614 627 626 +8 445 446 459 458 614 615 628 627 +8 446 447 460 459 615 616 629 628 +8 447 448 461 460 616 617 630 629 +8 448 449 462 461 617 618 631 630 +8 449 450 463 462 618 619 632 631 +8 450 451 464 463 619 620 633 632 +8 451 452 465 464 620 621 634 633 +8 452 453 466 465 621 622 635 634 +8 453 454 467 466 622 623 636 635 +8 455 456 469 468 624 625 638 637 +8 456 457 470 469 625 626 639 638 +8 457 458 471 470 626 627 640 639 +8 458 459 472 471 627 628 641 640 +8 459 460 473 472 628 629 642 641 +8 460 461 474 473 629 630 643 642 +8 461 462 475 474 630 631 644 643 +8 462 463 476 475 631 632 645 644 +8 463 464 477 476 632 633 646 645 +8 464 465 478 477 633 634 647 646 +8 465 466 479 478 634 635 648 647 +8 466 467 480 479 635 636 649 648 +8 468 469 482 481 637 638 651 650 +8 469 470 483 482 638 639 652 651 +8 470 471 484 483 639 640 653 652 +8 471 472 485 484 640 641 654 653 +8 472 473 486 485 641 642 655 654 +8 473 474 487 486 642 643 656 655 +8 474 475 488 487 643 644 657 656 +8 475 476 489 488 644 645 658 657 +8 476 477 490 489 645 646 659 658 +8 477 478 491 490 646 647 660 659 +8 478 479 492 491 647 648 661 660 +8 479 480 493 492 648 649 662 661 +8 481 482 495 494 650 651 664 663 +8 482 483 496 495 651 652 665 664 +8 483 484 497 496 652 653 666 665 +8 484 485 498 497 653 654 667 666 +8 485 486 499 498 654 655 668 667 +8 486 487 500 499 655 656 669 668 +8 487 488 501 500 656 657 670 669 +8 488 489 502 501 657 658 671 670 +8 489 490 503 502 658 659 672 671 +8 490 491 504 503 659 660 673 672 +8 491 492 505 504 660 661 674 673 +8 492 493 506 505 661 662 675 674 +8 507 508 521 520 676 677 690 689 +8 508 509 522 521 677 678 691 690 +8 509 510 523 522 678 679 692 691 +8 510 511 524 523 679 680 693 692 +8 511 512 525 524 680 681 694 693 +8 512 513 526 525 681 682 695 694 +8 513 514 527 526 682 683 696 695 +8 514 515 528 527 683 684 697 696 +8 515 516 529 528 684 685 698 697 +8 516 517 530 529 685 686 699 698 +8 517 518 531 530 686 687 700 699 +8 518 519 532 531 687 688 701 700 +8 520 521 534 533 689 690 703 702 +8 521 522 535 534 690 691 704 703 +8 522 523 536 535 691 692 705 704 +8 523 524 537 536 692 693 706 705 +8 524 525 538 537 693 694 707 706 +8 525 526 539 538 694 695 708 707 +8 526 527 540 539 695 696 709 708 +8 527 528 541 540 696 697 710 709 +8 528 529 542 541 697 698 711 710 +8 529 530 543 542 698 699 712 711 +8 530 531 544 543 699 700 713 712 +8 531 532 545 544 700 701 714 713 +8 533 534 547 546 702 703 716 715 +8 534 535 548 547 703 704 717 716 +8 535 536 549 548 704 705 718 717 +8 536 537 550 549 705 706 719 718 +8 537 538 551 550 706 707 720 719 +8 538 539 552 551 707 708 721 720 +8 539 540 553 552 708 709 722 721 +8 540 541 554 553 709 710 723 722 +8 541 542 555 554 710 711 724 723 +8 542 543 556 555 711 712 725 724 +8 543 544 557 556 712 713 726 725 +8 544 545 558 557 713 714 727 726 +8 546 547 560 559 715 716 729 728 +8 547 548 561 560 716 717 730 729 +8 548 549 562 561 717 718 731 730 +8 549 550 563 562 718 719 732 731 +8 550 551 564 563 719 720 733 732 +8 551 552 565 564 720 721 734 733 +8 552 553 566 565 721 722 735 734 +8 553 554 567 566 722 723 736 735 +8 554 555 568 567 723 724 737 736 +8 555 556 569 568 724 725 738 737 +8 556 557 570 569 725 726 739 738 +8 557 558 571 570 726 727 740 739 +8 559 560 573 572 728 729 742 741 +8 560 561 574 573 729 730 743 742 +8 561 562 575 574 730 731 744 743 +8 562 563 576 575 731 732 745 744 +8 563 564 577 576 732 733 746 745 +8 564 565 578 577 733 734 747 746 +8 565 566 579 578 734 735 748 747 +8 566 567 580 579 735 736 749 748 +8 567 568 581 580 736 737 750 749 +8 568 569 582 581 737 738 751 750 +8 569 570 583 582 738 739 752 751 +8 570 571 584 583 739 740 753 752 +8 572 573 586 585 741 742 755 754 +8 573 574 587 586 742 743 756 755 +8 574 575 588 587 743 744 757 756 +8 575 576 589 588 744 745 758 757 +8 576 577 590 589 745 746 759 758 +8 577 578 591 590 746 747 760 759 +8 578 579 592 591 747 748 761 760 +8 579 580 593 592 748 749 762 761 +8 580 581 594 593 749 750 763 762 +8 581 582 595 594 750 751 764 763 +8 582 583 596 595 751 752 765 764 +8 583 584 597 596 752 753 766 765 +8 585 586 599 598 754 755 768 767 +8 586 587 600 599 755 756 769 768 +8 587 588 601 600 756 757 770 769 +8 588 589 602 601 757 758 771 770 +8 589 590 603 602 758 759 772 771 +8 590 591 604 603 759 760 773 772 +8 591 592 605 604 760 761 774 773 +8 592 593 606 605 761 762 775 774 +8 593 594 607 606 762 763 776 775 +8 594 595 608 607 763 764 777 776 +8 595 596 609 608 764 765 778 777 +8 596 597 610 609 765 766 779 778 +8 598 599 612 611 767 768 781 780 +8 599 600 613 612 768 769 782 781 +8 600 601 614 613 769 770 783 782 +8 601 602 615 614 770 771 784 783 +8 602 603 616 615 771 772 785 784 +8 603 604 617 616 772 773 786 785 +8 604 605 618 617 773 774 787 786 +8 605 606 619 618 774 775 788 787 +8 606 607 620 619 775 776 789 788 +8 607 608 621 620 776 777 790 789 +8 608 609 622 621 777 778 791 790 +8 609 610 623 622 778 779 792 791 +8 611 612 625 624 780 781 794 793 +8 612 613 626 625 781 782 795 794 +8 613 614 627 626 782 783 796 795 +8 614 615 628 627 783 784 797 796 +8 615 616 629 628 784 785 798 797 +8 616 617 630 629 785 786 799 798 +8 617 618 631 630 786 787 800 799 +8 618 619 632 631 787 788 801 800 +8 619 620 633 632 788 789 802 801 +8 620 621 634 633 789 790 803 802 +8 621 622 635 634 790 791 804 803 +8 622 623 636 635 791 792 805 804 +8 624 625 638 637 793 794 807 806 +8 625 626 639 638 794 795 808 807 +8 626 627 640 639 795 796 809 808 +8 627 628 641 640 796 797 810 809 +8 628 629 642 641 797 798 811 810 +8 629 630 643 642 798 799 812 811 +8 630 631 644 643 799 800 813 812 +8 631 632 645 644 800 801 814 813 +8 632 633 646 645 801 802 815 814 +8 633 634 647 646 802 803 816 815 +8 634 635 648 647 803 804 817 816 +8 635 636 649 648 804 805 818 817 +8 637 638 651 650 806 807 820 819 +8 638 639 652 651 807 808 821 820 +8 639 640 653 652 808 809 822 821 +8 640 641 654 653 809 810 823 822 +8 641 642 655 654 810 811 824 823 +8 642 643 656 655 811 812 825 824 +8 643 644 657 656 812 813 826 825 +8 644 645 658 657 813 814 827 826 +8 645 646 659 658 814 815 828 827 +8 646 647 660 659 815 816 829 828 +8 647 648 661 660 816 817 830 829 +8 648 649 662 661 817 818 831 830 +8 650 651 664 663 819 820 833 832 +8 651 652 665 664 820 821 834 833 +8 652 653 666 665 821 822 835 834 +8 653 654 667 666 822 823 836 835 +8 654 655 668 667 823 824 837 836 +8 655 656 669 668 824 825 838 837 +8 656 657 670 669 825 826 839 838 +8 657 658 671 670 826 827 840 839 +8 658 659 672 671 827 828 841 840 +8 659 660 673 672 828 829 842 841 +8 660 661 674 673 829 830 843 842 +8 661 662 675 674 830 831 844 843 +8 676 677 690 689 845 846 859 858 +8 677 678 691 690 846 847 860 859 +8 678 679 692 691 847 848 861 860 +8 679 680 693 692 848 849 862 861 +8 680 681 694 693 849 850 863 862 +8 681 682 695 694 850 851 864 863 +8 682 683 696 695 851 852 865 864 +8 683 684 697 696 852 853 866 865 +8 684 685 698 697 853 854 867 866 +8 685 686 699 698 854 855 868 867 +8 686 687 700 699 855 856 869 868 +8 687 688 701 700 856 857 870 869 +8 689 690 703 702 858 859 872 871 +8 690 691 704 703 859 860 873 872 +8 691 692 705 704 860 861 874 873 +8 692 693 706 705 861 862 875 874 +8 693 694 707 706 862 863 876 875 +8 694 695 708 707 863 864 877 876 +8 695 696 709 708 864 865 878 877 +8 696 697 710 709 865 866 879 878 +8 697 698 711 710 866 867 880 879 +8 698 699 712 711 867 868 881 880 +8 699 700 713 712 868 869 882 881 +8 700 701 714 713 869 870 883 882 +8 702 703 716 715 871 872 885 884 +8 703 704 717 716 872 873 886 885 +8 704 705 718 717 873 874 887 886 +8 705 706 719 718 874 875 888 887 +8 706 707 720 719 875 876 889 888 +8 707 708 721 720 876 877 890 889 +8 708 709 722 721 877 878 891 890 +8 709 710 723 722 878 879 892 891 +8 710 711 724 723 879 880 893 892 +8 711 712 725 724 880 881 894 893 +8 712 713 726 725 881 882 895 894 +8 713 714 727 726 882 883 896 895 +8 715 716 729 728 884 885 898 897 +8 716 717 730 729 885 886 899 898 +8 717 718 731 730 886 887 900 899 +8 718 719 732 731 887 888 901 900 +8 719 720 733 732 888 889 902 901 +8 720 721 734 733 889 890 903 902 +8 721 722 735 734 890 891 904 903 +8 722 723 736 735 891 892 905 904 +8 723 724 737 736 892 893 906 905 +8 724 725 738 737 893 894 907 906 +8 725 726 739 738 894 895 908 907 +8 726 727 740 739 895 896 909 908 +8 728 729 742 741 897 898 911 910 +8 729 730 743 742 898 899 912 911 +8 730 731 744 743 899 900 913 912 +8 731 732 745 744 900 901 914 913 +8 732 733 746 745 901 902 915 914 +8 733 734 747 746 902 903 916 915 +8 734 735 748 747 903 904 917 916 +8 735 736 749 748 904 905 918 917 +8 736 737 750 749 905 906 919 918 +8 737 738 751 750 906 907 920 919 +8 738 739 752 751 907 908 921 920 +8 739 740 753 752 908 909 922 921 +8 741 742 755 754 910 911 924 923 +8 742 743 756 755 911 912 925 924 +8 743 744 757 756 912 913 926 925 +8 744 745 758 757 913 914 927 926 +8 745 746 759 758 914 915 928 927 +8 746 747 760 759 915 916 929 928 +8 747 748 761 760 916 917 930 929 +8 748 749 762 761 917 918 931 930 +8 749 750 763 762 918 919 932 931 +8 750 751 764 763 919 920 933 932 +8 751 752 765 764 920 921 934 933 +8 752 753 766 765 921 922 935 934 +8 754 755 768 767 923 924 937 936 +8 755 756 769 768 924 925 938 937 +8 756 757 770 769 925 926 939 938 +8 757 758 771 770 926 927 940 939 +8 758 759 772 771 927 928 941 940 +8 759 760 773 772 928 929 942 941 +8 760 761 774 773 929 930 943 942 +8 761 762 775 774 930 931 944 943 +8 762 763 776 775 931 932 945 944 +8 763 764 777 776 932 933 946 945 +8 764 765 778 777 933 934 947 946 +8 765 766 779 778 934 935 948 947 +8 767 768 781 780 936 937 950 949 +8 768 769 782 781 937 938 951 950 +8 769 770 783 782 938 939 952 951 +8 770 771 784 783 939 940 953 952 +8 771 772 785 784 940 941 954 953 +8 772 773 786 785 941 942 955 954 +8 773 774 787 786 942 943 956 955 +8 774 775 788 787 943 944 957 956 +8 775 776 789 788 944 945 958 957 +8 776 777 790 789 945 946 959 958 +8 777 778 791 790 946 947 960 959 +8 778 779 792 791 947 948 961 960 +8 780 781 794 793 949 950 963 962 +8 781 782 795 794 950 951 964 963 +8 782 783 796 795 951 952 965 964 +8 783 784 797 796 952 953 966 965 +8 784 785 798 797 953 954 967 966 +8 785 786 799 798 954 955 968 967 +8 786 787 800 799 955 956 969 968 +8 787 788 801 800 956 957 970 969 +8 788 789 802 801 957 958 971 970 +8 789 790 803 802 958 959 972 971 +8 790 791 804 803 959 960 973 972 +8 791 792 805 804 960 961 974 973 +8 793 794 807 806 962 963 976 975 +8 794 795 808 807 963 964 977 976 +8 795 796 809 808 964 965 978 977 +8 796 797 810 809 965 966 979 978 +8 797 798 811 810 966 967 980 979 +8 798 799 812 811 967 968 981 980 +8 799 800 813 812 968 969 982 981 +8 800 801 814 813 969 970 983 982 +8 801 802 815 814 970 971 984 983 +8 802 803 816 815 971 972 985 984 +8 803 804 817 816 972 973 986 985 +8 804 805 818 817 973 974 987 986 +8 806 807 820 819 975 976 989 988 +8 807 808 821 820 976 977 990 989 +8 808 809 822 821 977 978 991 990 +8 809 810 823 822 978 979 992 991 +8 810 811 824 823 979 980 993 992 +8 811 812 825 824 980 981 994 993 +8 812 813 826 825 981 982 995 994 +8 813 814 827 826 982 983 996 995 +8 814 815 828 827 983 984 997 996 +8 815 816 829 828 984 985 998 997 +8 816 817 830 829 985 986 999 998 +8 817 818 831 830 986 987 1000 999 +8 819 820 833 832 988 989 1002 1001 +8 820 821 834 833 989 990 1003 1002 +8 821 822 835 834 990 991 1004 1003 +8 822 823 836 835 991 992 1005 1004 +8 823 824 837 836 992 993 1006 1005 +8 824 825 838 837 993 994 1007 1006 +8 825 826 839 838 994 995 1008 1007 +8 826 827 840 839 995 996 1009 1008 +8 827 828 841 840 996 997 1010 1009 +8 828 829 842 841 997 998 1011 1010 +8 829 830 843 842 998 999 1012 1011 +8 830 831 844 843 999 1000 1013 1012 +8 845 846 859 858 1014 1015 1028 1027 +8 846 847 860 859 1015 1016 1029 1028 +8 847 848 861 860 1016 1017 1030 1029 +8 848 849 862 861 1017 1018 1031 1030 +8 849 850 863 862 1018 1019 1032 1031 +8 850 851 864 863 1019 1020 1033 1032 +8 851 852 865 864 1020 1021 1034 1033 +8 852 853 866 865 1021 1022 1035 1034 +8 853 854 867 866 1022 1023 1036 1035 +8 854 855 868 867 1023 1024 1037 1036 +8 855 856 869 868 1024 1025 1038 1037 +8 856 857 870 869 1025 1026 1039 1038 +8 858 859 872 871 1027 1028 1041 1040 +8 859 860 873 872 1028 1029 1042 1041 +8 860 861 874 873 1029 1030 1043 1042 +8 861 862 875 874 1030 1031 1044 1043 +8 862 863 876 875 1031 1032 1045 1044 +8 863 864 877 876 1032 1033 1046 1045 +8 864 865 878 877 1033 1034 1047 1046 +8 865 866 879 878 1034 1035 1048 1047 +8 866 867 880 879 1035 1036 1049 1048 +8 867 868 881 880 1036 1037 1050 1049 +8 868 869 882 881 1037 1038 1051 1050 +8 869 870 883 882 1038 1039 1052 1051 +8 871 872 885 884 1040 1041 1054 1053 +8 872 873 886 885 1041 1042 1055 1054 +8 873 874 887 886 1042 1043 1056 1055 +8 874 875 888 887 1043 1044 1057 1056 +8 875 876 889 888 1044 1045 1058 1057 +8 876 877 890 889 1045 1046 1059 1058 +8 877 878 891 890 1046 1047 1060 1059 +8 878 879 892 891 1047 1048 1061 1060 +8 879 880 893 892 1048 1049 1062 1061 +8 880 881 894 893 1049 1050 1063 1062 +8 881 882 895 894 1050 1051 1064 1063 +8 882 883 896 895 1051 1052 1065 1064 +8 884 885 898 897 1053 1054 1067 1066 +8 885 886 899 898 1054 1055 1068 1067 +8 886 887 900 899 1055 1056 1069 1068 +8 887 888 901 900 1056 1057 1070 1069 +8 888 889 902 901 1057 1058 1071 1070 +8 889 890 903 902 1058 1059 1072 1071 +8 890 891 904 903 1059 1060 1073 1072 +8 891 892 905 904 1060 1061 1074 1073 +8 892 893 906 905 1061 1062 1075 1074 +8 893 894 907 906 1062 1063 1076 1075 +8 894 895 908 907 1063 1064 1077 1076 +8 895 896 909 908 1064 1065 1078 1077 +8 897 898 911 910 1066 1067 1080 1079 +8 898 899 912 911 1067 1068 1081 1080 +8 899 900 913 912 1068 1069 1082 1081 +8 900 901 914 913 1069 1070 1083 1082 +8 901 902 915 914 1070 1071 1084 1083 +8 902 903 916 915 1071 1072 1085 1084 +8 903 904 917 916 1072 1073 1086 1085 +8 904 905 918 917 1073 1074 1087 1086 +8 905 906 919 918 1074 1075 1088 1087 +8 906 907 920 919 1075 1076 1089 1088 +8 907 908 921 920 1076 1077 1090 1089 +8 908 909 922 921 1077 1078 1091 1090 +8 910 911 924 923 1079 1080 1093 1092 +8 911 912 925 924 1080 1081 1094 1093 +8 912 913 926 925 1081 1082 1095 1094 +8 913 914 927 926 1082 1083 1096 1095 +8 914 915 928 927 1083 1084 1097 1096 +8 915 916 929 928 1084 1085 1098 1097 +8 916 917 930 929 1085 1086 1099 1098 +8 917 918 931 930 1086 1087 1100 1099 +8 918 919 932 931 1087 1088 1101 1100 +8 919 920 933 932 1088 1089 1102 1101 +8 920 921 934 933 1089 1090 1103 1102 +8 921 922 935 934 1090 1091 1104 1103 +8 923 924 937 936 1092 1093 1106 1105 +8 924 925 938 937 1093 1094 1107 1106 +8 925 926 939 938 1094 1095 1108 1107 +8 926 927 940 939 1095 1096 1109 1108 +8 927 928 941 940 1096 1097 1110 1109 +8 928 929 942 941 1097 1098 1111 1110 +8 929 930 943 942 1098 1099 1112 1111 +8 930 931 944 943 1099 1100 1113 1112 +8 931 932 945 944 1100 1101 1114 1113 +8 932 933 946 945 1101 1102 1115 1114 +8 933 934 947 946 1102 1103 1116 1115 +8 934 935 948 947 1103 1104 1117 1116 +8 936 937 950 949 1105 1106 1119 1118 +8 937 938 951 950 1106 1107 1120 1119 +8 938 939 952 951 1107 1108 1121 1120 +8 939 940 953 952 1108 1109 1122 1121 +8 940 941 954 953 1109 1110 1123 1122 +8 941 942 955 954 1110 1111 1124 1123 +8 942 943 956 955 1111 1112 1125 1124 +8 943 944 957 956 1112 1113 1126 1125 +8 944 945 958 957 1113 1114 1127 1126 +8 945 946 959 958 1114 1115 1128 1127 +8 946 947 960 959 1115 1116 1129 1128 +8 947 948 961 960 1116 1117 1130 1129 +8 949 950 963 962 1118 1119 1132 1131 +8 950 951 964 963 1119 1120 1133 1132 +8 951 952 965 964 1120 1121 1134 1133 +8 952 953 966 965 1121 1122 1135 1134 +8 953 954 967 966 1122 1123 1136 1135 +8 954 955 968 967 1123 1124 1137 1136 +8 955 956 969 968 1124 1125 1138 1137 +8 956 957 970 969 1125 1126 1139 1138 +8 957 958 971 970 1126 1127 1140 1139 +8 958 959 972 971 1127 1128 1141 1140 +8 959 960 973 972 1128 1129 1142 1141 +8 960 961 974 973 1129 1130 1143 1142 +8 962 963 976 975 1131 1132 1145 1144 +8 963 964 977 976 1132 1133 1146 1145 +8 964 965 978 977 1133 1134 1147 1146 +8 965 966 979 978 1134 1135 1148 1147 +8 966 967 980 979 1135 1136 1149 1148 +8 967 968 981 980 1136 1137 1150 1149 +8 968 969 982 981 1137 1138 1151 1150 +8 969 970 983 982 1138 1139 1152 1151 +8 970 971 984 983 1139 1140 1153 1152 +8 971 972 985 984 1140 1141 1154 1153 +8 972 973 986 985 1141 1142 1155 1154 +8 973 974 987 986 1142 1143 1156 1155 +8 975 976 989 988 1144 1145 1158 1157 +8 976 977 990 989 1145 1146 1159 1158 +8 977 978 991 990 1146 1147 1160 1159 +8 978 979 992 991 1147 1148 1161 1160 +8 979 980 993 992 1148 1149 1162 1161 +8 980 981 994 993 1149 1150 1163 1162 +8 981 982 995 994 1150 1151 1164 1163 +8 982 983 996 995 1151 1152 1165 1164 +8 983 984 997 996 1152 1153 1166 1165 +8 984 985 998 997 1153 1154 1167 1166 +8 985 986 999 998 1154 1155 1168 1167 +8 986 987 1000 999 1155 1156 1169 1168 +8 988 989 1002 1001 1157 1158 1171 1170 +8 989 990 1003 1002 1158 1159 1172 1171 +8 990 991 1004 1003 1159 1160 1173 1172 +8 991 992 1005 1004 1160 1161 1174 1173 +8 992 993 1006 1005 1161 1162 1175 1174 +8 993 994 1007 1006 1162 1163 1176 1175 +8 994 995 1008 1007 1163 1164 1177 1176 +8 995 996 1009 1008 1164 1165 1178 1177 +8 996 997 1010 1009 1165 1166 1179 1178 +8 997 998 1011 1010 1166 1167 1180 1179 +8 998 999 1012 1011 1167 1168 1181 1180 +8 999 1000 1013 1012 1168 1169 1182 1181 +8 1014 1015 1028 1027 1183 1184 1197 1196 +8 1015 1016 1029 1028 1184 1185 1198 1197 +8 1016 1017 1030 1029 1185 1186 1199 1198 +8 1017 1018 1031 1030 1186 1187 1200 1199 +8 1018 1019 1032 1031 1187 1188 1201 1200 +8 1019 1020 1033 1032 1188 1189 1202 1201 +8 1020 1021 1034 1033 1189 1190 1203 1202 +8 1021 1022 1035 1034 1190 1191 1204 1203 +8 1022 1023 1036 1035 1191 1192 1205 1204 +8 1023 1024 1037 1036 1192 1193 1206 1205 +8 1024 1025 1038 1037 1193 1194 1207 1206 +8 1025 1026 1039 1038 1194 1195 1208 1207 +8 1027 1028 1041 1040 1196 1197 1210 1209 +8 1028 1029 1042 1041 1197 1198 1211 1210 +8 1029 1030 1043 1042 1198 1199 1212 1211 +8 1030 1031 1044 1043 1199 1200 1213 1212 +8 1031 1032 1045 1044 1200 1201 1214 1213 +8 1032 1033 1046 1045 1201 1202 1215 1214 +8 1033 1034 1047 1046 1202 1203 1216 1215 +8 1034 1035 1048 1047 1203 1204 1217 1216 +8 1035 1036 1049 1048 1204 1205 1218 1217 +8 1036 1037 1050 1049 1205 1206 1219 1218 +8 1037 1038 1051 1050 1206 1207 1220 1219 +8 1038 1039 1052 1051 1207 1208 1221 1220 +8 1040 1041 1054 1053 1209 1210 1223 1222 +8 1041 1042 1055 1054 1210 1211 1224 1223 +8 1042 1043 1056 1055 1211 1212 1225 1224 +8 1043 1044 1057 1056 1212 1213 1226 1225 +8 1044 1045 1058 1057 1213 1214 1227 1226 +8 1045 1046 1059 1058 1214 1215 1228 1227 +8 1046 1047 1060 1059 1215 1216 1229 1228 +8 1047 1048 1061 1060 1216 1217 1230 1229 +8 1048 1049 1062 1061 1217 1218 1231 1230 +8 1049 1050 1063 1062 1218 1219 1232 1231 +8 1050 1051 1064 1063 1219 1220 1233 1232 +8 1051 1052 1065 1064 1220 1221 1234 1233 +8 1053 1054 1067 1066 1222 1223 1236 1235 +8 1054 1055 1068 1067 1223 1224 1237 1236 +8 1055 1056 1069 1068 1224 1225 1238 1237 +8 1056 1057 1070 1069 1225 1226 1239 1238 +8 1057 1058 1071 1070 1226 1227 1240 1239 +8 1058 1059 1072 1071 1227 1228 1241 1240 +8 1059 1060 1073 1072 1228 1229 1242 1241 +8 1060 1061 1074 1073 1229 1230 1243 1242 +8 1061 1062 1075 1074 1230 1231 1244 1243 +8 1062 1063 1076 1075 1231 1232 1245 1244 +8 1063 1064 1077 1076 1232 1233 1246 1245 +8 1064 1065 1078 1077 1233 1234 1247 1246 +8 1066 1067 1080 1079 1235 1236 1249 1248 +8 1067 1068 1081 1080 1236 1237 1250 1249 +8 1068 1069 1082 1081 1237 1238 1251 1250 +8 1069 1070 1083 1082 1238 1239 1252 1251 +8 1070 1071 1084 1083 1239 1240 1253 1252 +8 1071 1072 1085 1084 1240 1241 1254 1253 +8 1072 1073 1086 1085 1241 1242 1255 1254 +8 1073 1074 1087 1086 1242 1243 1256 1255 +8 1074 1075 1088 1087 1243 1244 1257 1256 +8 1075 1076 1089 1088 1244 1245 1258 1257 +8 1076 1077 1090 1089 1245 1246 1259 1258 +8 1077 1078 1091 1090 1246 1247 1260 1259 +8 1079 1080 1093 1092 1248 1249 1262 1261 +8 1080 1081 1094 1093 1249 1250 1263 1262 +8 1081 1082 1095 1094 1250 1251 1264 1263 +8 1082 1083 1096 1095 1251 1252 1265 1264 +8 1083 1084 1097 1096 1252 1253 1266 1265 +8 1084 1085 1098 1097 1253 1254 1267 1266 +8 1085 1086 1099 1098 1254 1255 1268 1267 +8 1086 1087 1100 1099 1255 1256 1269 1268 +8 1087 1088 1101 1100 1256 1257 1270 1269 +8 1088 1089 1102 1101 1257 1258 1271 1270 +8 1089 1090 1103 1102 1258 1259 1272 1271 +8 1090 1091 1104 1103 1259 1260 1273 1272 +8 1092 1093 1106 1105 1261 1262 1275 1274 +8 1093 1094 1107 1106 1262 1263 1276 1275 +8 1094 1095 1108 1107 1263 1264 1277 1276 +8 1095 1096 1109 1108 1264 1265 1278 1277 +8 1096 1097 1110 1109 1265 1266 1279 1278 +8 1097 1098 1111 1110 1266 1267 1280 1279 +8 1098 1099 1112 1111 1267 1268 1281 1280 +8 1099 1100 1113 1112 1268 1269 1282 1281 +8 1100 1101 1114 1113 1269 1270 1283 1282 +8 1101 1102 1115 1114 1270 1271 1284 1283 +8 1102 1103 1116 1115 1271 1272 1285 1284 +8 1103 1104 1117 1116 1272 1273 1286 1285 +8 1105 1106 1119 1118 1274 1275 1288 1287 +8 1106 1107 1120 1119 1275 1276 1289 1288 +8 1107 1108 1121 1120 1276 1277 1290 1289 +8 1108 1109 1122 1121 1277 1278 1291 1290 +8 1109 1110 1123 1122 1278 1279 1292 1291 +8 1110 1111 1124 1123 1279 1280 1293 1292 +8 1111 1112 1125 1124 1280 1281 1294 1293 +8 1112 1113 1126 1125 1281 1282 1295 1294 +8 1113 1114 1127 1126 1282 1283 1296 1295 +8 1114 1115 1128 1127 1283 1284 1297 1296 +8 1115 1116 1129 1128 1284 1285 1298 1297 +8 1116 1117 1130 1129 1285 1286 1299 1298 +8 1118 1119 1132 1131 1287 1288 1301 1300 +8 1119 1120 1133 1132 1288 1289 1302 1301 +8 1120 1121 1134 1133 1289 1290 1303 1302 +8 1121 1122 1135 1134 1290 1291 1304 1303 +8 1122 1123 1136 1135 1291 1292 1305 1304 +8 1123 1124 1137 1136 1292 1293 1306 1305 +8 1124 1125 1138 1137 1293 1294 1307 1306 +8 1125 1126 1139 1138 1294 1295 1308 1307 +8 1126 1127 1140 1139 1295 1296 1309 1308 +8 1127 1128 1141 1140 1296 1297 1310 1309 +8 1128 1129 1142 1141 1297 1298 1311 1310 +8 1129 1130 1143 1142 1298 1299 1312 1311 +8 1131 1132 1145 1144 1300 1301 1314 1313 +8 1132 1133 1146 1145 1301 1302 1315 1314 +8 1133 1134 1147 1146 1302 1303 1316 1315 +8 1134 1135 1148 1147 1303 1304 1317 1316 +8 1135 1136 1149 1148 1304 1305 1318 1317 +8 1136 1137 1150 1149 1305 1306 1319 1318 +8 1137 1138 1151 1150 1306 1307 1320 1319 +8 1138 1139 1152 1151 1307 1308 1321 1320 +8 1139 1140 1153 1152 1308 1309 1322 1321 +8 1140 1141 1154 1153 1309 1310 1323 1322 +8 1141 1142 1155 1154 1310 1311 1324 1323 +8 1142 1143 1156 1155 1311 1312 1325 1324 +8 1144 1145 1158 1157 1313 1314 1327 1326 +8 1145 1146 1159 1158 1314 1315 1328 1327 +8 1146 1147 1160 1159 1315 1316 1329 1328 +8 1147 1148 1161 1160 1316 1317 1330 1329 +8 1148 1149 1162 1161 1317 1318 1331 1330 +8 1149 1150 1163 1162 1318 1319 1332 1331 +8 1150 1151 1164 1163 1319 1320 1333 1332 +8 1151 1152 1165 1164 1320 1321 1334 1333 +8 1152 1153 1166 1165 1321 1322 1335 1334 +8 1153 1154 1167 1166 1322 1323 1336 1335 +8 1154 1155 1168 1167 1323 1324 1337 1336 +8 1155 1156 1169 1168 1324 1325 1338 1337 +8 1157 1158 1171 1170 1326 1327 1340 1339 +8 1158 1159 1172 1171 1327 1328 1341 1340 +8 1159 1160 1173 1172 1328 1329 1342 1341 +8 1160 1161 1174 1173 1329 1330 1343 1342 +8 1161 1162 1175 1174 1330 1331 1344 1343 +8 1162 1163 1176 1175 1331 1332 1345 1344 +8 1163 1164 1177 1176 1332 1333 1346 1345 +8 1164 1165 1178 1177 1333 1334 1347 1346 +8 1165 1166 1179 1178 1334 1335 1348 1347 +8 1166 1167 1180 1179 1335 1336 1349 1348 +8 1167 1168 1181 1180 1336 1337 1350 1349 +8 1168 1169 1182 1181 1337 1338 1351 1350 +8 1183 1184 1197 1196 1352 1353 1366 1365 +8 1184 1185 1198 1197 1353 1354 1367 1366 +8 1185 1186 1199 1198 1354 1355 1368 1367 +8 1186 1187 1200 1199 1355 1356 1369 1368 +8 1187 1188 1201 1200 1356 1357 1370 1369 +8 1188 1189 1202 1201 1357 1358 1371 1370 +8 1189 1190 1203 1202 1358 1359 1372 1371 +8 1190 1191 1204 1203 1359 1360 1373 1372 +8 1191 1192 1205 1204 1360 1361 1374 1373 +8 1192 1193 1206 1205 1361 1362 1375 1374 +8 1193 1194 1207 1206 1362 1363 1376 1375 +8 1194 1195 1208 1207 1363 1364 1377 1376 +8 1196 1197 1210 1209 1365 1366 1379 1378 +8 1197 1198 1211 1210 1366 1367 1380 1379 +8 1198 1199 1212 1211 1367 1368 1381 1380 +8 1199 1200 1213 1212 1368 1369 1382 1381 +8 1200 1201 1214 1213 1369 1370 1383 1382 +8 1201 1202 1215 1214 1370 1371 1384 1383 +8 1202 1203 1216 1215 1371 1372 1385 1384 +8 1203 1204 1217 1216 1372 1373 1386 1385 +8 1204 1205 1218 1217 1373 1374 1387 1386 +8 1205 1206 1219 1218 1374 1375 1388 1387 +8 1206 1207 1220 1219 1375 1376 1389 1388 +8 1207 1208 1221 1220 1376 1377 1390 1389 +8 1209 1210 1223 1222 1378 1379 1392 1391 +8 1210 1211 1224 1223 1379 1380 1393 1392 +8 1211 1212 1225 1224 1380 1381 1394 1393 +8 1212 1213 1226 1225 1381 1382 1395 1394 +8 1213 1214 1227 1226 1382 1383 1396 1395 +8 1214 1215 1228 1227 1383 1384 1397 1396 +8 1215 1216 1229 1228 1384 1385 1398 1397 +8 1216 1217 1230 1229 1385 1386 1399 1398 +8 1217 1218 1231 1230 1386 1387 1400 1399 +8 1218 1219 1232 1231 1387 1388 1401 1400 +8 1219 1220 1233 1232 1388 1389 1402 1401 +8 1220 1221 1234 1233 1389 1390 1403 1402 +8 1222 1223 1236 1235 1391 1392 1405 1404 +8 1223 1224 1237 1236 1392 1393 1406 1405 +8 1224 1225 1238 1237 1393 1394 1407 1406 +8 1225 1226 1239 1238 1394 1395 1408 1407 +8 1226 1227 1240 1239 1395 1396 1409 1408 +8 1227 1228 1241 1240 1396 1397 1410 1409 +8 1228 1229 1242 1241 1397 1398 1411 1410 +8 1229 1230 1243 1242 1398 1399 1412 1411 +8 1230 1231 1244 1243 1399 1400 1413 1412 +8 1231 1232 1245 1244 1400 1401 1414 1413 +8 1232 1233 1246 1245 1401 1402 1415 1414 +8 1233 1234 1247 1246 1402 1403 1416 1415 +8 1235 1236 1249 1248 1404 1405 1418 1417 +8 1236 1237 1250 1249 1405 1406 1419 1418 +8 1237 1238 1251 1250 1406 1407 1420 1419 +8 1238 1239 1252 1251 1407 1408 1421 1420 +8 1239 1240 1253 1252 1408 1409 1422 1421 +8 1240 1241 1254 1253 1409 1410 1423 1422 +8 1241 1242 1255 1254 1410 1411 1424 1423 +8 1242 1243 1256 1255 1411 1412 1425 1424 +8 1243 1244 1257 1256 1412 1413 1426 1425 +8 1244 1245 1258 1257 1413 1414 1427 1426 +8 1245 1246 1259 1258 1414 1415 1428 1427 +8 1246 1247 1260 1259 1415 1416 1429 1428 +8 1248 1249 1262 1261 1417 1418 1431 1430 +8 1249 1250 1263 1262 1418 1419 1432 1431 +8 1250 1251 1264 1263 1419 1420 1433 1432 +8 1251 1252 1265 1264 1420 1421 1434 1433 +8 1252 1253 1266 1265 1421 1422 1435 1434 +8 1253 1254 1267 1266 1422 1423 1436 1435 +8 1254 1255 1268 1267 1423 1424 1437 1436 +8 1255 1256 1269 1268 1424 1425 1438 1437 +8 1256 1257 1270 1269 1425 1426 1439 1438 +8 1257 1258 1271 1270 1426 1427 1440 1439 +8 1258 1259 1272 1271 1427 1428 1441 1440 +8 1259 1260 1273 1272 1428 1429 1442 1441 +8 1261 1262 1275 1274 1430 1431 1444 1443 +8 1262 1263 1276 1275 1431 1432 1445 1444 +8 1263 1264 1277 1276 1432 1433 1446 1445 +8 1264 1265 1278 1277 1433 1434 1447 1446 +8 1265 1266 1279 1278 1434 1435 1448 1447 +8 1266 1267 1280 1279 1435 1436 1449 1448 +8 1267 1268 1281 1280 1436 1437 1450 1449 +8 1268 1269 1282 1281 1437 1438 1451 1450 +8 1269 1270 1283 1282 1438 1439 1452 1451 +8 1270 1271 1284 1283 1439 1440 1453 1452 +8 1271 1272 1285 1284 1440 1441 1454 1453 +8 1272 1273 1286 1285 1441 1442 1455 1454 +8 1274 1275 1288 1287 1443 1444 1457 1456 +8 1275 1276 1289 1288 1444 1445 1458 1457 +8 1276 1277 1290 1289 1445 1446 1459 1458 +8 1277 1278 1291 1290 1446 1447 1460 1459 +8 1278 1279 1292 1291 1447 1448 1461 1460 +8 1279 1280 1293 1292 1448 1449 1462 1461 +8 1280 1281 1294 1293 1449 1450 1463 1462 +8 1281 1282 1295 1294 1450 1451 1464 1463 +8 1282 1283 1296 1295 1451 1452 1465 1464 +8 1283 1284 1297 1296 1452 1453 1466 1465 +8 1284 1285 1298 1297 1453 1454 1467 1466 +8 1285 1286 1299 1298 1454 1455 1468 1467 +8 1287 1288 1301 1300 1456 1457 1470 1469 +8 1288 1289 1302 1301 1457 1458 1471 1470 +8 1289 1290 1303 1302 1458 1459 1472 1471 +8 1290 1291 1304 1303 1459 1460 1473 1472 +8 1291 1292 1305 1304 1460 1461 1474 1473 +8 1292 1293 1306 1305 1461 1462 1475 1474 +8 1293 1294 1307 1306 1462 1463 1476 1475 +8 1294 1295 1308 1307 1463 1464 1477 1476 +8 1295 1296 1309 1308 1464 1465 1478 1477 +8 1296 1297 1310 1309 1465 1466 1479 1478 +8 1297 1298 1311 1310 1466 1467 1480 1479 +8 1298 1299 1312 1311 1467 1468 1481 1480 +8 1300 1301 1314 1313 1469 1470 1483 1482 +8 1301 1302 1315 1314 1470 1471 1484 1483 +8 1302 1303 1316 1315 1471 1472 1485 1484 +8 1303 1304 1317 1316 1472 1473 1486 1485 +8 1304 1305 1318 1317 1473 1474 1487 1486 +8 1305 1306 1319 1318 1474 1475 1488 1487 +8 1306 1307 1320 1319 1475 1476 1489 1488 +8 1307 1308 1321 1320 1476 1477 1490 1489 +8 1308 1309 1322 1321 1477 1478 1491 1490 +8 1309 1310 1323 1322 1478 1479 1492 1491 +8 1310 1311 1324 1323 1479 1480 1493 1492 +8 1311 1312 1325 1324 1480 1481 1494 1493 +8 1313 1314 1327 1326 1482 1483 1496 1495 +8 1314 1315 1328 1327 1483 1484 1497 1496 +8 1315 1316 1329 1328 1484 1485 1498 1497 +8 1316 1317 1330 1329 1485 1486 1499 1498 +8 1317 1318 1331 1330 1486 1487 1500 1499 +8 1318 1319 1332 1331 1487 1488 1501 1500 +8 1319 1320 1333 1332 1488 1489 1502 1501 +8 1320 1321 1334 1333 1489 1490 1503 1502 +8 1321 1322 1335 1334 1490 1491 1504 1503 +8 1322 1323 1336 1335 1491 1492 1505 1504 +8 1323 1324 1337 1336 1492 1493 1506 1505 +8 1324 1325 1338 1337 1493 1494 1507 1506 +8 1326 1327 1340 1339 1495 1496 1509 1508 +8 1327 1328 1341 1340 1496 1497 1510 1509 +8 1328 1329 1342 1341 1497 1498 1511 1510 +8 1329 1330 1343 1342 1498 1499 1512 1511 +8 1330 1331 1344 1343 1499 1500 1513 1512 +8 1331 1332 1345 1344 1500 1501 1514 1513 +8 1332 1333 1346 1345 1501 1502 1515 1514 +8 1333 1334 1347 1346 1502 1503 1516 1515 +8 1334 1335 1348 1347 1503 1504 1517 1516 +8 1335 1336 1349 1348 1504 1505 1518 1517 +8 1336 1337 1350 1349 1505 1506 1519 1518 +8 1337 1338 1351 1350 1506 1507 1520 1519 +8 1352 1353 1366 1365 1521 1522 1535 1534 +8 1353 1354 1367 1366 1522 1523 1536 1535 +8 1354 1355 1368 1367 1523 1524 1537 1536 +8 1355 1356 1369 1368 1524 1525 1538 1537 +8 1356 1357 1370 1369 1525 1526 1539 1538 +8 1357 1358 1371 1370 1526 1527 1540 1539 +8 1358 1359 1372 1371 1527 1528 1541 1540 +8 1359 1360 1373 1372 1528 1529 1542 1541 +8 1360 1361 1374 1373 1529 1530 1543 1542 +8 1361 1362 1375 1374 1530 1531 1544 1543 +8 1362 1363 1376 1375 1531 1532 1545 1544 +8 1363 1364 1377 1376 1532 1533 1546 1545 +8 1365 1366 1379 1378 1534 1535 1548 1547 +8 1366 1367 1380 1379 1535 1536 1549 1548 +8 1367 1368 1381 1380 1536 1537 1550 1549 +8 1368 1369 1382 1381 1537 1538 1551 1550 +8 1369 1370 1383 1382 1538 1539 1552 1551 +8 1370 1371 1384 1383 1539 1540 1553 1552 +8 1371 1372 1385 1384 1540 1541 1554 1553 +8 1372 1373 1386 1385 1541 1542 1555 1554 +8 1373 1374 1387 1386 1542 1543 1556 1555 +8 1374 1375 1388 1387 1543 1544 1557 1556 +8 1375 1376 1389 1388 1544 1545 1558 1557 +8 1376 1377 1390 1389 1545 1546 1559 1558 +8 1378 1379 1392 1391 1547 1548 1561 1560 +8 1379 1380 1393 1392 1548 1549 1562 1561 +8 1380 1381 1394 1393 1549 1550 1563 1562 +8 1381 1382 1395 1394 1550 1551 1564 1563 +8 1382 1383 1396 1395 1551 1552 1565 1564 +8 1383 1384 1397 1396 1552 1553 1566 1565 +8 1384 1385 1398 1397 1553 1554 1567 1566 +8 1385 1386 1399 1398 1554 1555 1568 1567 +8 1386 1387 1400 1399 1555 1556 1569 1568 +8 1387 1388 1401 1400 1556 1557 1570 1569 +8 1388 1389 1402 1401 1557 1558 1571 1570 +8 1389 1390 1403 1402 1558 1559 1572 1571 +8 1391 1392 1405 1404 1560 1561 1574 1573 +8 1392 1393 1406 1405 1561 1562 1575 1574 +8 1393 1394 1407 1406 1562 1563 1576 1575 +8 1394 1395 1408 1407 1563 1564 1577 1576 +8 1395 1396 1409 1408 1564 1565 1578 1577 +8 1396 1397 1410 1409 1565 1566 1579 1578 +8 1397 1398 1411 1410 1566 1567 1580 1579 +8 1398 1399 1412 1411 1567 1568 1581 1580 +8 1399 1400 1413 1412 1568 1569 1582 1581 +8 1400 1401 1414 1413 1569 1570 1583 1582 +8 1401 1402 1415 1414 1570 1571 1584 1583 +8 1402 1403 1416 1415 1571 1572 1585 1584 +8 1404 1405 1418 1417 1573 1574 1587 1586 +8 1405 1406 1419 1418 1574 1575 1588 1587 +8 1406 1407 1420 1419 1575 1576 1589 1588 +8 1407 1408 1421 1420 1576 1577 1590 1589 +8 1408 1409 1422 1421 1577 1578 1591 1590 +8 1409 1410 1423 1422 1578 1579 1592 1591 +8 1410 1411 1424 1423 1579 1580 1593 1592 +8 1411 1412 1425 1424 1580 1581 1594 1593 +8 1412 1413 1426 1425 1581 1582 1595 1594 +8 1413 1414 1427 1426 1582 1583 1596 1595 +8 1414 1415 1428 1427 1583 1584 1597 1596 +8 1415 1416 1429 1428 1584 1585 1598 1597 +8 1417 1418 1431 1430 1586 1587 1600 1599 +8 1418 1419 1432 1431 1587 1588 1601 1600 +8 1419 1420 1433 1432 1588 1589 1602 1601 +8 1420 1421 1434 1433 1589 1590 1603 1602 +8 1421 1422 1435 1434 1590 1591 1604 1603 +8 1422 1423 1436 1435 1591 1592 1605 1604 +8 1423 1424 1437 1436 1592 1593 1606 1605 +8 1424 1425 1438 1437 1593 1594 1607 1606 +8 1425 1426 1439 1438 1594 1595 1608 1607 +8 1426 1427 1440 1439 1595 1596 1609 1608 +8 1427 1428 1441 1440 1596 1597 1610 1609 +8 1428 1429 1442 1441 1597 1598 1611 1610 +8 1430 1431 1444 1443 1599 1600 1613 1612 +8 1431 1432 1445 1444 1600 1601 1614 1613 +8 1432 1433 1446 1445 1601 1602 1615 1614 +8 1433 1434 1447 1446 1602 1603 1616 1615 +8 1434 1435 1448 1447 1603 1604 1617 1616 +8 1435 1436 1449 1448 1604 1605 1618 1617 +8 1436 1437 1450 1449 1605 1606 1619 1618 +8 1437 1438 1451 1450 1606 1607 1620 1619 +8 1438 1439 1452 1451 1607 1608 1621 1620 +8 1439 1440 1453 1452 1608 1609 1622 1621 +8 1440 1441 1454 1453 1609 1610 1623 1622 +8 1441 1442 1455 1454 1610 1611 1624 1623 +8 1443 1444 1457 1456 1612 1613 1626 1625 +8 1444 1445 1458 1457 1613 1614 1627 1626 +8 1445 1446 1459 1458 1614 1615 1628 1627 +8 1446 1447 1460 1459 1615 1616 1629 1628 +8 1447 1448 1461 1460 1616 1617 1630 1629 +8 1448 1449 1462 1461 1617 1618 1631 1630 +8 1449 1450 1463 1462 1618 1619 1632 1631 +8 1450 1451 1464 1463 1619 1620 1633 1632 +8 1451 1452 1465 1464 1620 1621 1634 1633 +8 1452 1453 1466 1465 1621 1622 1635 1634 +8 1453 1454 1467 1466 1622 1623 1636 1635 +8 1454 1455 1468 1467 1623 1624 1637 1636 +8 1456 1457 1470 1469 1625 1626 1639 1638 +8 1457 1458 1471 1470 1626 1627 1640 1639 +8 1458 1459 1472 1471 1627 1628 1641 1640 +8 1459 1460 1473 1472 1628 1629 1642 1641 +8 1460 1461 1474 1473 1629 1630 1643 1642 +8 1461 1462 1475 1474 1630 1631 1644 1643 +8 1462 1463 1476 1475 1631 1632 1645 1644 +8 1463 1464 1477 1476 1632 1633 1646 1645 +8 1464 1465 1478 1477 1633 1634 1647 1646 +8 1465 1466 1479 1478 1634 1635 1648 1647 +8 1466 1467 1480 1479 1635 1636 1649 1648 +8 1467 1468 1481 1480 1636 1637 1650 1649 +8 1469 1470 1483 1482 1638 1639 1652 1651 +8 1470 1471 1484 1483 1639 1640 1653 1652 +8 1471 1472 1485 1484 1640 1641 1654 1653 +8 1472 1473 1486 1485 1641 1642 1655 1654 +8 1473 1474 1487 1486 1642 1643 1656 1655 +8 1474 1475 1488 1487 1643 1644 1657 1656 +8 1475 1476 1489 1488 1644 1645 1658 1657 +8 1476 1477 1490 1489 1645 1646 1659 1658 +8 1477 1478 1491 1490 1646 1647 1660 1659 +8 1478 1479 1492 1491 1647 1648 1661 1660 +8 1479 1480 1493 1492 1648 1649 1662 1661 +8 1480 1481 1494 1493 1649 1650 1663 1662 +8 1482 1483 1496 1495 1651 1652 1665 1664 +8 1483 1484 1497 1496 1652 1653 1666 1665 +8 1484 1485 1498 1497 1653 1654 1667 1666 +8 1485 1486 1499 1498 1654 1655 1668 1667 +8 1486 1487 1500 1499 1655 1656 1669 1668 +8 1487 1488 1501 1500 1656 1657 1670 1669 +8 1488 1489 1502 1501 1657 1658 1671 1670 +8 1489 1490 1503 1502 1658 1659 1672 1671 +8 1490 1491 1504 1503 1659 1660 1673 1672 +8 1491 1492 1505 1504 1660 1661 1674 1673 +8 1492 1493 1506 1505 1661 1662 1675 1674 +8 1493 1494 1507 1506 1662 1663 1676 1675 +8 1495 1496 1509 1508 1664 1665 1678 1677 +8 1496 1497 1510 1509 1665 1666 1679 1678 +8 1497 1498 1511 1510 1666 1667 1680 1679 +8 1498 1499 1512 1511 1667 1668 1681 1680 +8 1499 1500 1513 1512 1668 1669 1682 1681 +8 1500 1501 1514 1513 1669 1670 1683 1682 +8 1501 1502 1515 1514 1670 1671 1684 1683 +8 1502 1503 1516 1515 1671 1672 1685 1684 +8 1503 1504 1517 1516 1672 1673 1686 1685 +8 1504 1505 1518 1517 1673 1674 1687 1686 +8 1505 1506 1519 1518 1674 1675 1688 1687 +8 1506 1507 1520 1519 1675 1676 1689 1688 +8 1521 1522 1535 1534 1690 1691 1704 1703 +8 1522 1523 1536 1535 1691 1692 1705 1704 +8 1523 1524 1537 1536 1692 1693 1706 1705 +8 1524 1525 1538 1537 1693 1694 1707 1706 +8 1525 1526 1539 1538 1694 1695 1708 1707 +8 1526 1527 1540 1539 1695 1696 1709 1708 +8 1527 1528 1541 1540 1696 1697 1710 1709 +8 1528 1529 1542 1541 1697 1698 1711 1710 +8 1529 1530 1543 1542 1698 1699 1712 1711 +8 1530 1531 1544 1543 1699 1700 1713 1712 +8 1531 1532 1545 1544 1700 1701 1714 1713 +8 1532 1533 1546 1545 1701 1702 1715 1714 +8 1534 1535 1548 1547 1703 1704 1717 1716 +8 1535 1536 1549 1548 1704 1705 1718 1717 +8 1536 1537 1550 1549 1705 1706 1719 1718 +8 1537 1538 1551 1550 1706 1707 1720 1719 +8 1538 1539 1552 1551 1707 1708 1721 1720 +8 1539 1540 1553 1552 1708 1709 1722 1721 +8 1540 1541 1554 1553 1709 1710 1723 1722 +8 1541 1542 1555 1554 1710 1711 1724 1723 +8 1542 1543 1556 1555 1711 1712 1725 1724 +8 1543 1544 1557 1556 1712 1713 1726 1725 +8 1544 1545 1558 1557 1713 1714 1727 1726 +8 1545 1546 1559 1558 1714 1715 1728 1727 +8 1547 1548 1561 1560 1716 1717 1730 1729 +8 1548 1549 1562 1561 1717 1718 1731 1730 +8 1549 1550 1563 1562 1718 1719 1732 1731 +8 1550 1551 1564 1563 1719 1720 1733 1732 +8 1551 1552 1565 1564 1720 1721 1734 1733 +8 1552 1553 1566 1565 1721 1722 1735 1734 +8 1553 1554 1567 1566 1722 1723 1736 1735 +8 1554 1555 1568 1567 1723 1724 1737 1736 +8 1555 1556 1569 1568 1724 1725 1738 1737 +8 1556 1557 1570 1569 1725 1726 1739 1738 +8 1557 1558 1571 1570 1726 1727 1740 1739 +8 1558 1559 1572 1571 1727 1728 1741 1740 +8 1560 1561 1574 1573 1729 1730 1743 1742 +8 1561 1562 1575 1574 1730 1731 1744 1743 +8 1562 1563 1576 1575 1731 1732 1745 1744 +8 1563 1564 1577 1576 1732 1733 1746 1745 +8 1564 1565 1578 1577 1733 1734 1747 1746 +8 1565 1566 1579 1578 1734 1735 1748 1747 +8 1566 1567 1580 1579 1735 1736 1749 1748 +8 1567 1568 1581 1580 1736 1737 1750 1749 +8 1568 1569 1582 1581 1737 1738 1751 1750 +8 1569 1570 1583 1582 1738 1739 1752 1751 +8 1570 1571 1584 1583 1739 1740 1753 1752 +8 1571 1572 1585 1584 1740 1741 1754 1753 +8 1573 1574 1587 1586 1742 1743 1756 1755 +8 1574 1575 1588 1587 1743 1744 1757 1756 +8 1575 1576 1589 1588 1744 1745 1758 1757 +8 1576 1577 1590 1589 1745 1746 1759 1758 +8 1577 1578 1591 1590 1746 1747 1760 1759 +8 1578 1579 1592 1591 1747 1748 1761 1760 +8 1579 1580 1593 1592 1748 1749 1762 1761 +8 1580 1581 1594 1593 1749 1750 1763 1762 +8 1581 1582 1595 1594 1750 1751 1764 1763 +8 1582 1583 1596 1595 1751 1752 1765 1764 +8 1583 1584 1597 1596 1752 1753 1766 1765 +8 1584 1585 1598 1597 1753 1754 1767 1766 +8 1586 1587 1600 1599 1755 1756 1769 1768 +8 1587 1588 1601 1600 1756 1757 1770 1769 +8 1588 1589 1602 1601 1757 1758 1771 1770 +8 1589 1590 1603 1602 1758 1759 1772 1771 +8 1590 1591 1604 1603 1759 1760 1773 1772 +8 1591 1592 1605 1604 1760 1761 1774 1773 +8 1592 1593 1606 1605 1761 1762 1775 1774 +8 1593 1594 1607 1606 1762 1763 1776 1775 +8 1594 1595 1608 1607 1763 1764 1777 1776 +8 1595 1596 1609 1608 1764 1765 1778 1777 +8 1596 1597 1610 1609 1765 1766 1779 1778 +8 1597 1598 1611 1610 1766 1767 1780 1779 +8 1599 1600 1613 1612 1768 1769 1782 1781 +8 1600 1601 1614 1613 1769 1770 1783 1782 +8 1601 1602 1615 1614 1770 1771 1784 1783 +8 1602 1603 1616 1615 1771 1772 1785 1784 +8 1603 1604 1617 1616 1772 1773 1786 1785 +8 1604 1605 1618 1617 1773 1774 1787 1786 +8 1605 1606 1619 1618 1774 1775 1788 1787 +8 1606 1607 1620 1619 1775 1776 1789 1788 +8 1607 1608 1621 1620 1776 1777 1790 1789 +8 1608 1609 1622 1621 1777 1778 1791 1790 +8 1609 1610 1623 1622 1778 1779 1792 1791 +8 1610 1611 1624 1623 1779 1780 1793 1792 +8 1612 1613 1626 1625 1781 1782 1795 1794 +8 1613 1614 1627 1626 1782 1783 1796 1795 +8 1614 1615 1628 1627 1783 1784 1797 1796 +8 1615 1616 1629 1628 1784 1785 1798 1797 +8 1616 1617 1630 1629 1785 1786 1799 1798 +8 1617 1618 1631 1630 1786 1787 1800 1799 +8 1618 1619 1632 1631 1787 1788 1801 1800 +8 1619 1620 1633 1632 1788 1789 1802 1801 +8 1620 1621 1634 1633 1789 1790 1803 1802 +8 1621 1622 1635 1634 1790 1791 1804 1803 +8 1622 1623 1636 1635 1791 1792 1805 1804 +8 1623 1624 1637 1636 1792 1793 1806 1805 +8 1625 1626 1639 1638 1794 1795 1808 1807 +8 1626 1627 1640 1639 1795 1796 1809 1808 +8 1627 1628 1641 1640 1796 1797 1810 1809 +8 1628 1629 1642 1641 1797 1798 1811 1810 +8 1629 1630 1643 1642 1798 1799 1812 1811 +8 1630 1631 1644 1643 1799 1800 1813 1812 +8 1631 1632 1645 1644 1800 1801 1814 1813 +8 1632 1633 1646 1645 1801 1802 1815 1814 +8 1633 1634 1647 1646 1802 1803 1816 1815 +8 1634 1635 1648 1647 1803 1804 1817 1816 +8 1635 1636 1649 1648 1804 1805 1818 1817 +8 1636 1637 1650 1649 1805 1806 1819 1818 +8 1638 1639 1652 1651 1807 1808 1821 1820 +8 1639 1640 1653 1652 1808 1809 1822 1821 +8 1640 1641 1654 1653 1809 1810 1823 1822 +8 1641 1642 1655 1654 1810 1811 1824 1823 +8 1642 1643 1656 1655 1811 1812 1825 1824 +8 1643 1644 1657 1656 1812 1813 1826 1825 +8 1644 1645 1658 1657 1813 1814 1827 1826 +8 1645 1646 1659 1658 1814 1815 1828 1827 +8 1646 1647 1660 1659 1815 1816 1829 1828 +8 1647 1648 1661 1660 1816 1817 1830 1829 +8 1648 1649 1662 1661 1817 1818 1831 1830 +8 1649 1650 1663 1662 1818 1819 1832 1831 +8 1651 1652 1665 1664 1820 1821 1834 1833 +8 1652 1653 1666 1665 1821 1822 1835 1834 +8 1653 1654 1667 1666 1822 1823 1836 1835 +8 1654 1655 1668 1667 1823 1824 1837 1836 +8 1655 1656 1669 1668 1824 1825 1838 1837 +8 1656 1657 1670 1669 1825 1826 1839 1838 +8 1657 1658 1671 1670 1826 1827 1840 1839 +8 1658 1659 1672 1671 1827 1828 1841 1840 +8 1659 1660 1673 1672 1828 1829 1842 1841 +8 1660 1661 1674 1673 1829 1830 1843 1842 +8 1661 1662 1675 1674 1830 1831 1844 1843 +8 1662 1663 1676 1675 1831 1832 1845 1844 +8 1664 1665 1678 1677 1833 1834 1847 1846 +8 1665 1666 1679 1678 1834 1835 1848 1847 +8 1666 1667 1680 1679 1835 1836 1849 1848 +8 1667 1668 1681 1680 1836 1837 1850 1849 +8 1668 1669 1682 1681 1837 1838 1851 1850 +8 1669 1670 1683 1682 1838 1839 1852 1851 +8 1670 1671 1684 1683 1839 1840 1853 1852 +8 1671 1672 1685 1684 1840 1841 1854 1853 +8 1672 1673 1686 1685 1841 1842 1855 1854 +8 1673 1674 1687 1686 1842 1843 1856 1855 +8 1674 1675 1688 1687 1843 1844 1857 1856 +8 1675 1676 1689 1688 1844 1845 1858 1857 +8 1690 1691 1704 1703 1859 1860 1873 1872 +8 1691 1692 1705 1704 1860 1861 1874 1873 +8 1692 1693 1706 1705 1861 1862 1875 1874 +8 1693 1694 1707 1706 1862 1863 1876 1875 +8 1694 1695 1708 1707 1863 1864 1877 1876 +8 1695 1696 1709 1708 1864 1865 1878 1877 +8 1696 1697 1710 1709 1865 1866 1879 1878 +8 1697 1698 1711 1710 1866 1867 1880 1879 +8 1698 1699 1712 1711 1867 1868 1881 1880 +8 1699 1700 1713 1712 1868 1869 1882 1881 +8 1700 1701 1714 1713 1869 1870 1883 1882 +8 1701 1702 1715 1714 1870 1871 1884 1883 +8 1703 1704 1717 1716 1872 1873 1886 1885 +8 1704 1705 1718 1717 1873 1874 1887 1886 +8 1705 1706 1719 1718 1874 1875 1888 1887 +8 1706 1707 1720 1719 1875 1876 1889 1888 +8 1707 1708 1721 1720 1876 1877 1890 1889 +8 1708 1709 1722 1721 1877 1878 1891 1890 +8 1709 1710 1723 1722 1878 1879 1892 1891 +8 1710 1711 1724 1723 1879 1880 1893 1892 +8 1711 1712 1725 1724 1880 1881 1894 1893 +8 1712 1713 1726 1725 1881 1882 1895 1894 +8 1713 1714 1727 1726 1882 1883 1896 1895 +8 1714 1715 1728 1727 1883 1884 1897 1896 +8 1716 1717 1730 1729 1885 1886 1899 1898 +8 1717 1718 1731 1730 1886 1887 1900 1899 +8 1718 1719 1732 1731 1887 1888 1901 1900 +8 1719 1720 1733 1732 1888 1889 1902 1901 +8 1720 1721 1734 1733 1889 1890 1903 1902 +8 1721 1722 1735 1734 1890 1891 1904 1903 +8 1722 1723 1736 1735 1891 1892 1905 1904 +8 1723 1724 1737 1736 1892 1893 1906 1905 +8 1724 1725 1738 1737 1893 1894 1907 1906 +8 1725 1726 1739 1738 1894 1895 1908 1907 +8 1726 1727 1740 1739 1895 1896 1909 1908 +8 1727 1728 1741 1740 1896 1897 1910 1909 +8 1729 1730 1743 1742 1898 1899 1912 1911 +8 1730 1731 1744 1743 1899 1900 1913 1912 +8 1731 1732 1745 1744 1900 1901 1914 1913 +8 1732 1733 1746 1745 1901 1902 1915 1914 +8 1733 1734 1747 1746 1902 1903 1916 1915 +8 1734 1735 1748 1747 1903 1904 1917 1916 +8 1735 1736 1749 1748 1904 1905 1918 1917 +8 1736 1737 1750 1749 1905 1906 1919 1918 +8 1737 1738 1751 1750 1906 1907 1920 1919 +8 1738 1739 1752 1751 1907 1908 1921 1920 +8 1739 1740 1753 1752 1908 1909 1922 1921 +8 1740 1741 1754 1753 1909 1910 1923 1922 +8 1742 1743 1756 1755 1911 1912 1925 1924 +8 1743 1744 1757 1756 1912 1913 1926 1925 +8 1744 1745 1758 1757 1913 1914 1927 1926 +8 1745 1746 1759 1758 1914 1915 1928 1927 +8 1746 1747 1760 1759 1915 1916 1929 1928 +8 1747 1748 1761 1760 1916 1917 1930 1929 +8 1748 1749 1762 1761 1917 1918 1931 1930 +8 1749 1750 1763 1762 1918 1919 1932 1931 +8 1750 1751 1764 1763 1919 1920 1933 1932 +8 1751 1752 1765 1764 1920 1921 1934 1933 +8 1752 1753 1766 1765 1921 1922 1935 1934 +8 1753 1754 1767 1766 1922 1923 1936 1935 +8 1755 1756 1769 1768 1924 1925 1938 1937 +8 1756 1757 1770 1769 1925 1926 1939 1938 +8 1757 1758 1771 1770 1926 1927 1940 1939 +8 1758 1759 1772 1771 1927 1928 1941 1940 +8 1759 1760 1773 1772 1928 1929 1942 1941 +8 1760 1761 1774 1773 1929 1930 1943 1942 +8 1761 1762 1775 1774 1930 1931 1944 1943 +8 1762 1763 1776 1775 1931 1932 1945 1944 +8 1763 1764 1777 1776 1932 1933 1946 1945 +8 1764 1765 1778 1777 1933 1934 1947 1946 +8 1765 1766 1779 1778 1934 1935 1948 1947 +8 1766 1767 1780 1779 1935 1936 1949 1948 +8 1768 1769 1782 1781 1937 1938 1951 1950 +8 1769 1770 1783 1782 1938 1939 1952 1951 +8 1770 1771 1784 1783 1939 1940 1953 1952 +8 1771 1772 1785 1784 1940 1941 1954 1953 +8 1772 1773 1786 1785 1941 1942 1955 1954 +8 1773 1774 1787 1786 1942 1943 1956 1955 +8 1774 1775 1788 1787 1943 1944 1957 1956 +8 1775 1776 1789 1788 1944 1945 1958 1957 +8 1776 1777 1790 1789 1945 1946 1959 1958 +8 1777 1778 1791 1790 1946 1947 1960 1959 +8 1778 1779 1792 1791 1947 1948 1961 1960 +8 1779 1780 1793 1792 1948 1949 1962 1961 +8 1781 1782 1795 1794 1950 1951 1964 1963 +8 1782 1783 1796 1795 1951 1952 1965 1964 +8 1783 1784 1797 1796 1952 1953 1966 1965 +8 1784 1785 1798 1797 1953 1954 1967 1966 +8 1785 1786 1799 1798 1954 1955 1968 1967 +8 1786 1787 1800 1799 1955 1956 1969 1968 +8 1787 1788 1801 1800 1956 1957 1970 1969 +8 1788 1789 1802 1801 1957 1958 1971 1970 +8 1789 1790 1803 1802 1958 1959 1972 1971 +8 1790 1791 1804 1803 1959 1960 1973 1972 +8 1791 1792 1805 1804 1960 1961 1974 1973 +8 1792 1793 1806 1805 1961 1962 1975 1974 +8 1794 1795 1808 1807 1963 1964 1977 1976 +8 1795 1796 1809 1808 1964 1965 1978 1977 +8 1796 1797 1810 1809 1965 1966 1979 1978 +8 1797 1798 1811 1810 1966 1967 1980 1979 +8 1798 1799 1812 1811 1967 1968 1981 1980 +8 1799 1800 1813 1812 1968 1969 1982 1981 +8 1800 1801 1814 1813 1969 1970 1983 1982 +8 1801 1802 1815 1814 1970 1971 1984 1983 +8 1802 1803 1816 1815 1971 1972 1985 1984 +8 1803 1804 1817 1816 1972 1973 1986 1985 +8 1804 1805 1818 1817 1973 1974 1987 1986 +8 1805 1806 1819 1818 1974 1975 1988 1987 +8 1807 1808 1821 1820 1976 1977 1990 1989 +8 1808 1809 1822 1821 1977 1978 1991 1990 +8 1809 1810 1823 1822 1978 1979 1992 1991 +8 1810 1811 1824 1823 1979 1980 1993 1992 +8 1811 1812 1825 1824 1980 1981 1994 1993 +8 1812 1813 1826 1825 1981 1982 1995 1994 +8 1813 1814 1827 1826 1982 1983 1996 1995 +8 1814 1815 1828 1827 1983 1984 1997 1996 +8 1815 1816 1829 1828 1984 1985 1998 1997 +8 1816 1817 1830 1829 1985 1986 1999 1998 +8 1817 1818 1831 1830 1986 1987 2000 1999 +8 1818 1819 1832 1831 1987 1988 2001 2000 +8 1820 1821 1834 1833 1989 1990 2003 2002 +8 1821 1822 1835 1834 1990 1991 2004 2003 +8 1822 1823 1836 1835 1991 1992 2005 2004 +8 1823 1824 1837 1836 1992 1993 2006 2005 +8 1824 1825 1838 1837 1993 1994 2007 2006 +8 1825 1826 1839 1838 1994 1995 2008 2007 +8 1826 1827 1840 1839 1995 1996 2009 2008 +8 1827 1828 1841 1840 1996 1997 2010 2009 +8 1828 1829 1842 1841 1997 1998 2011 2010 +8 1829 1830 1843 1842 1998 1999 2012 2011 +8 1830 1831 1844 1843 1999 2000 2013 2012 +8 1831 1832 1845 1844 2000 2001 2014 2013 +8 1833 1834 1847 1846 2002 2003 2016 2015 +8 1834 1835 1848 1847 2003 2004 2017 2016 +8 1835 1836 1849 1848 2004 2005 2018 2017 +8 1836 1837 1850 1849 2005 2006 2019 2018 +8 1837 1838 1851 1850 2006 2007 2020 2019 +8 1838 1839 1852 1851 2007 2008 2021 2020 +8 1839 1840 1853 1852 2008 2009 2022 2021 +8 1840 1841 1854 1853 2009 2010 2023 2022 +8 1841 1842 1855 1854 2010 2011 2024 2023 +8 1842 1843 1856 1855 2011 2012 2025 2024 +8 1843 1844 1857 1856 2012 2013 2026 2025 +8 1844 1845 1858 1857 2013 2014 2027 2026 +8 1859 1860 1873 1872 2028 2029 2042 2041 +8 1860 1861 1874 1873 2029 2030 2043 2042 +8 1861 1862 1875 1874 2030 2031 2044 2043 +8 1862 1863 1876 1875 2031 2032 2045 2044 +8 1863 1864 1877 1876 2032 2033 2046 2045 +8 1864 1865 1878 1877 2033 2034 2047 2046 +8 1865 1866 1879 1878 2034 2035 2048 2047 +8 1866 1867 1880 1879 2035 2036 2049 2048 +8 1867 1868 1881 1880 2036 2037 2050 2049 +8 1868 1869 1882 1881 2037 2038 2051 2050 +8 1869 1870 1883 1882 2038 2039 2052 2051 +8 1870 1871 1884 1883 2039 2040 2053 2052 +8 1872 1873 1886 1885 2041 2042 2055 2054 +8 1873 1874 1887 1886 2042 2043 2056 2055 +8 1874 1875 1888 1887 2043 2044 2057 2056 +8 1875 1876 1889 1888 2044 2045 2058 2057 +8 1876 1877 1890 1889 2045 2046 2059 2058 +8 1877 1878 1891 1890 2046 2047 2060 2059 +8 1878 1879 1892 1891 2047 2048 2061 2060 +8 1879 1880 1893 1892 2048 2049 2062 2061 +8 1880 1881 1894 1893 2049 2050 2063 2062 +8 1881 1882 1895 1894 2050 2051 2064 2063 +8 1882 1883 1896 1895 2051 2052 2065 2064 +8 1883 1884 1897 1896 2052 2053 2066 2065 +8 1885 1886 1899 1898 2054 2055 2068 2067 +8 1886 1887 1900 1899 2055 2056 2069 2068 +8 1887 1888 1901 1900 2056 2057 2070 2069 +8 1888 1889 1902 1901 2057 2058 2071 2070 +8 1889 1890 1903 1902 2058 2059 2072 2071 +8 1890 1891 1904 1903 2059 2060 2073 2072 +8 1891 1892 1905 1904 2060 2061 2074 2073 +8 1892 1893 1906 1905 2061 2062 2075 2074 +8 1893 1894 1907 1906 2062 2063 2076 2075 +8 1894 1895 1908 1907 2063 2064 2077 2076 +8 1895 1896 1909 1908 2064 2065 2078 2077 +8 1896 1897 1910 1909 2065 2066 2079 2078 +8 1898 1899 1912 1911 2067 2068 2081 2080 +8 1899 1900 1913 1912 2068 2069 2082 2081 +8 1900 1901 1914 1913 2069 2070 2083 2082 +8 1901 1902 1915 1914 2070 2071 2084 2083 +8 1902 1903 1916 1915 2071 2072 2085 2084 +8 1903 1904 1917 1916 2072 2073 2086 2085 +8 1904 1905 1918 1917 2073 2074 2087 2086 +8 1905 1906 1919 1918 2074 2075 2088 2087 +8 1906 1907 1920 1919 2075 2076 2089 2088 +8 1907 1908 1921 1920 2076 2077 2090 2089 +8 1908 1909 1922 1921 2077 2078 2091 2090 +8 1909 1910 1923 1922 2078 2079 2092 2091 +8 1911 1912 1925 1924 2080 2081 2094 2093 +8 1912 1913 1926 1925 2081 2082 2095 2094 +8 1913 1914 1927 1926 2082 2083 2096 2095 +8 1914 1915 1928 1927 2083 2084 2097 2096 +8 1915 1916 1929 1928 2084 2085 2098 2097 +8 1916 1917 1930 1929 2085 2086 2099 2098 +8 1917 1918 1931 1930 2086 2087 2100 2099 +8 1918 1919 1932 1931 2087 2088 2101 2100 +8 1919 1920 1933 1932 2088 2089 2102 2101 +8 1920 1921 1934 1933 2089 2090 2103 2102 +8 1921 1922 1935 1934 2090 2091 2104 2103 +8 1922 1923 1936 1935 2091 2092 2105 2104 +8 1924 1925 1938 1937 2093 2094 2107 2106 +8 1925 1926 1939 1938 2094 2095 2108 2107 +8 1926 1927 1940 1939 2095 2096 2109 2108 +8 1927 1928 1941 1940 2096 2097 2110 2109 +8 1928 1929 1942 1941 2097 2098 2111 2110 +8 1929 1930 1943 1942 2098 2099 2112 2111 +8 1930 1931 1944 1943 2099 2100 2113 2112 +8 1931 1932 1945 1944 2100 2101 2114 2113 +8 1932 1933 1946 1945 2101 2102 2115 2114 +8 1933 1934 1947 1946 2102 2103 2116 2115 +8 1934 1935 1948 1947 2103 2104 2117 2116 +8 1935 1936 1949 1948 2104 2105 2118 2117 +8 1937 1938 1951 1950 2106 2107 2120 2119 +8 1938 1939 1952 1951 2107 2108 2121 2120 +8 1939 1940 1953 1952 2108 2109 2122 2121 +8 1940 1941 1954 1953 2109 2110 2123 2122 +8 1941 1942 1955 1954 2110 2111 2124 2123 +8 1942 1943 1956 1955 2111 2112 2125 2124 +8 1943 1944 1957 1956 2112 2113 2126 2125 +8 1944 1945 1958 1957 2113 2114 2127 2126 +8 1945 1946 1959 1958 2114 2115 2128 2127 +8 1946 1947 1960 1959 2115 2116 2129 2128 +8 1947 1948 1961 1960 2116 2117 2130 2129 +8 1948 1949 1962 1961 2117 2118 2131 2130 +8 1950 1951 1964 1963 2119 2120 2133 2132 +8 1951 1952 1965 1964 2120 2121 2134 2133 +8 1952 1953 1966 1965 2121 2122 2135 2134 +8 1953 1954 1967 1966 2122 2123 2136 2135 +8 1954 1955 1968 1967 2123 2124 2137 2136 +8 1955 1956 1969 1968 2124 2125 2138 2137 +8 1956 1957 1970 1969 2125 2126 2139 2138 +8 1957 1958 1971 1970 2126 2127 2140 2139 +8 1958 1959 1972 1971 2127 2128 2141 2140 +8 1959 1960 1973 1972 2128 2129 2142 2141 +8 1960 1961 1974 1973 2129 2130 2143 2142 +8 1961 1962 1975 1974 2130 2131 2144 2143 +8 1963 1964 1977 1976 2132 2133 2146 2145 +8 1964 1965 1978 1977 2133 2134 2147 2146 +8 1965 1966 1979 1978 2134 2135 2148 2147 +8 1966 1967 1980 1979 2135 2136 2149 2148 +8 1967 1968 1981 1980 2136 2137 2150 2149 +8 1968 1969 1982 1981 2137 2138 2151 2150 +8 1969 1970 1983 1982 2138 2139 2152 2151 +8 1970 1971 1984 1983 2139 2140 2153 2152 +8 1971 1972 1985 1984 2140 2141 2154 2153 +8 1972 1973 1986 1985 2141 2142 2155 2154 +8 1973 1974 1987 1986 2142 2143 2156 2155 +8 1974 1975 1988 1987 2143 2144 2157 2156 +8 1976 1977 1990 1989 2145 2146 2159 2158 +8 1977 1978 1991 1990 2146 2147 2160 2159 +8 1978 1979 1992 1991 2147 2148 2161 2160 +8 1979 1980 1993 1992 2148 2149 2162 2161 +8 1980 1981 1994 1993 2149 2150 2163 2162 +8 1981 1982 1995 1994 2150 2151 2164 2163 +8 1982 1983 1996 1995 2151 2152 2165 2164 +8 1983 1984 1997 1996 2152 2153 2166 2165 +8 1984 1985 1998 1997 2153 2154 2167 2166 +8 1985 1986 1999 1998 2154 2155 2168 2167 +8 1986 1987 2000 1999 2155 2156 2169 2168 +8 1987 1988 2001 2000 2156 2157 2170 2169 +8 1989 1990 2003 2002 2158 2159 2172 2171 +8 1990 1991 2004 2003 2159 2160 2173 2172 +8 1991 1992 2005 2004 2160 2161 2174 2173 +8 1992 1993 2006 2005 2161 2162 2175 2174 +8 1993 1994 2007 2006 2162 2163 2176 2175 +8 1994 1995 2008 2007 2163 2164 2177 2176 +8 1995 1996 2009 2008 2164 2165 2178 2177 +8 1996 1997 2010 2009 2165 2166 2179 2178 +8 1997 1998 2011 2010 2166 2167 2180 2179 +8 1998 1999 2012 2011 2167 2168 2181 2180 +8 1999 2000 2013 2012 2168 2169 2182 2181 +8 2000 2001 2014 2013 2169 2170 2183 2182 +8 2002 2003 2016 2015 2171 2172 2185 2184 +8 2003 2004 2017 2016 2172 2173 2186 2185 +8 2004 2005 2018 2017 2173 2174 2187 2186 +8 2005 2006 2019 2018 2174 2175 2188 2187 +8 2006 2007 2020 2019 2175 2176 2189 2188 +8 2007 2008 2021 2020 2176 2177 2190 2189 +8 2008 2009 2022 2021 2177 2178 2191 2190 +8 2009 2010 2023 2022 2178 2179 2192 2191 +8 2010 2011 2024 2023 2179 2180 2193 2192 +8 2011 2012 2025 2024 2180 2181 2194 2193 +8 2012 2013 2026 2025 2181 2182 2195 2194 +8 2013 2014 2027 2026 2182 2183 2196 2195 + +CELL_TYPES 1728 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 + +POINT_DATA 2197 +VECTORS velocity float +0.00000000 0.00000000 0.00000000 +0.15129286 0.00000000 0.00000000 +0.15323531 0.00000000 0.00000000 +0.18052624 0.00000000 0.00000000 +0.19922659 0.00000000 0.00000000 +0.21962195 0.00000000 0.00000000 +0.24196835 0.00000000 0.00000000 +0.26345823 0.00000000 0.00000000 +0.27808604 0.00000000 0.00000000 +0.35128500 0.00000000 0.00000000 +0.25113715 0.00000000 0.00000000 +0.38181582 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.15129286 0.00000000 +0.12109740 0.12109740 0.00000000 +0.16706916 0.08253341 0.00000000 +0.18507825 0.07255161 0.00000000 +0.20510401 0.06409840 0.00000000 +0.22557832 0.05784992 0.00000000 +0.24812410 0.05315190 0.00000000 +0.26935568 0.05031494 0.00000000 +0.29184205 0.05226268 0.00000000 +0.32796197 0.04779386 0.00000000 +0.34155637 0.05854756 0.00000000 +0.29245201 0.03622370 0.00000000 +0.00000000 0.00004101 0.00000000 +0.00000000 0.15323531 0.00000000 +0.08253341 0.16706916 0.00000000 +0.15129872 0.15129872 0.00000000 +0.18661483 0.12387973 0.00000000 +0.20744356 0.11091403 0.00000000 +0.23059254 0.09877480 0.00000000 +0.25292777 0.08951714 0.00000000 +0.27422715 0.08372217 0.00000000 +0.30646293 0.08576194 0.00000000 +0.28483792 0.07437695 0.00000000 +0.43878482 0.12180165 0.00000000 +0.08015231 0.02176584 0.00000000 +0.00000000 0.00001641 0.00000000 +0.00000000 0.18052624 0.00000000 +0.07255161 0.18507825 0.00000000 +0.12387973 0.18661483 0.00000000 +0.17156448 0.17156448 0.00000000 +0.20161696 0.14816375 0.00000000 +0.22568042 0.13415464 0.00000000 +0.24815988 0.12082245 0.00000000 +0.27809616 0.11779382 0.00000000 +0.28558967 0.10768783 0.00000000 +0.34061828 0.12922569 0.00000000 +0.32820362 0.11853629 0.00000000 +0.00927076 0.00327665 0.00000000 +0.00000000 0.00000048 0.00000000 +0.00000000 0.19922659 0.00000000 +0.06409840 0.20510401 0.00000000 +0.11091403 0.20744356 0.00000000 +0.14816375 0.20161696 0.00000000 +0.18839642 0.18839642 0.00000000 +0.21255306 0.16465196 0.00000000 +0.24407471 0.15550298 0.00000000 +0.26798333 0.14539135 0.00000000 +0.26576622 0.13184502 0.00000000 +0.39499686 0.18176438 0.00000000 +0.09442587 0.04187089 0.00000000 +0.00039903 0.00015905 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.21962195 0.00000000 +0.05784992 0.22557832 0.00000000 +0.09877480 0.23059254 0.00000000 +0.13415464 0.22568042 0.00000000 +0.16465196 0.21255306 0.00000000 +0.19805544 0.19805544 0.00000000 +0.23584271 0.18550201 0.00000000 +0.22778616 0.15266555 0.00000000 +0.33580943 0.20439457 0.00000000 +0.24988166 0.13969963 0.00000000 +0.00901370 0.00475284 0.00000000 +0.00000239 0.00000109 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.24196835 0.00000000 +0.05315190 0.24812410 0.00000000 +0.08951714 0.25292777 0.00000000 +0.12082245 0.24815988 0.00000000 +0.15550298 0.24407471 0.00000000 +0.18550201 0.23584271 0.00000000 +0.19670448 0.19670448 0.00000000 +0.27828749 0.22263260 0.00000000 +0.29186871 0.20940311 0.00000000 +0.03569997 0.02352921 0.00000000 +0.00016711 0.00009720 0.00000000 +0.00000001 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.26345823 0.00000000 +0.05031494 0.26935568 0.00000000 +0.08372217 0.27422715 0.00000000 +0.11779382 0.27809616 0.00000000 +0.14539135 0.26798333 0.00000000 +0.15266555 0.22778616 0.00000000 +0.22263260 0.27828749 0.00000000 +0.28020504 0.28020504 0.00000000 +0.06343861 0.05159365 0.00000000 +0.00103623 0.00076839 0.00000000 +0.00000017 0.00000011 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.27808604 0.00000000 +0.05226268 0.29184205 0.00000000 +0.08576194 0.30646293 0.00000000 +0.10768783 0.28558967 0.00000000 +0.13184502 0.26576622 0.00000000 +0.20439457 0.33580943 0.00000000 +0.20940311 0.29186871 0.00000000 +0.05159365 0.06343861 0.00000000 +0.00335127 0.00335127 0.00000000 +0.00000143 0.00000119 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.35128500 0.00000000 +0.04779386 0.32796197 0.00000000 +0.07437695 0.28483792 0.00000000 +0.12922569 0.34061828 0.00000000 +0.18176438 0.39499686 0.00000000 +0.13969963 0.24988166 0.00000000 +0.02352921 0.03569997 0.00000000 +0.00076839 0.00103623 0.00000000 +0.00000119 0.00000143 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.25113715 0.00000000 +0.05854756 0.34155637 0.00000000 +0.12180165 0.43878482 0.00000000 +0.11853629 0.32820362 0.00000000 +0.04187089 0.09442587 0.00000000 +0.00475284 0.00901370 0.00000000 +0.00009720 0.00016711 0.00000000 +0.00000011 0.00000017 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.38181582 0.00000000 +0.03622370 0.29245201 0.00000000 +0.02176584 0.08015231 0.00000000 +0.00327665 0.00927076 0.00000000 +0.00015905 0.00039903 0.00000000 +0.00000109 0.00000239 0.00000000 +0.00000000 0.00000001 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00004101 0.00000000 0.00000000 +0.00001641 0.00000000 0.00000000 +0.00000048 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.15129286 +0.12109740 0.00000000 0.12109740 +0.16706916 0.00000000 0.08253341 +0.18507825 0.00000000 0.07255161 +0.20510401 0.00000000 0.06409840 +0.22557832 0.00000000 0.05784992 +0.24812410 0.00000000 0.05315190 +0.26935568 0.00000000 0.05031494 +0.29184205 0.00000000 0.05226268 +0.32796197 0.00000000 0.04779386 +0.34155637 0.00000000 0.05854756 +0.29245201 0.00000000 0.03622370 +0.00000000 0.00000000 0.00004101 +0.00000000 0.12109740 0.12109740 +0.11194433 0.11194433 0.11194433 +0.16763493 0.08447348 0.08447348 +0.18818094 0.06881720 0.06881720 +0.20793413 0.06125917 0.06125917 +0.22912333 0.05484865 0.05484865 +0.25170547 0.05018152 0.05018152 +0.27224313 0.04698754 0.04698754 +0.30077655 0.04897950 0.04897950 +0.30800102 0.04250931 0.04250931 +0.40165640 0.06440826 0.06440826 +0.21450057 0.02766431 0.02766431 +0.00000000 0.00002538 0.00002538 +0.00000000 0.16706916 0.08253341 +0.08447348 0.16763493 0.08447348 +0.14671206 0.14671206 0.07661738 +0.18541418 0.12388847 0.06505718 +0.20769496 0.10849415 0.05558558 +0.23095378 0.09698467 0.04975907 +0.25306158 0.08754438 0.04486165 +0.27552492 0.08255333 0.04216991 +0.30537290 0.08302718 0.04225252 +0.28730540 0.07481661 0.03779717 +0.42864392 0.11485281 0.05668998 +0.05712571 0.01532577 0.00770589 +0.00000000 0.00001000 0.00000477 +0.00000000 0.18507825 0.07255161 +0.06881720 0.18818094 0.06881720 +0.12388847 0.18541418 0.06505718 +0.16862699 0.16862699 0.05971093 +0.20078594 0.14850555 0.05284890 +0.22513003 0.13297281 0.04626450 +0.24786893 0.12034648 0.04185713 +0.27864928 0.11728339 0.04033039 +0.27970791 0.10485952 0.03571615 +0.35622202 0.13373729 0.04466998 +0.29339533 0.10428998 0.03244443 +0.00661588 0.00230056 0.00077655 +0.00000000 0.00000030 0.00000008 +0.00000000 0.20510401 0.06409840 +0.06125917 0.20793413 0.06125917 +0.10849415 0.20769496 0.05558558 +0.14850555 0.20078594 0.05284890 +0.18594880 0.18594880 0.04921526 +0.21173705 0.16466959 0.04398982 +0.24448037 0.15544591 0.04078769 +0.26429520 0.14306836 0.03713853 +0.27336582 0.13597825 0.03482393 +0.38848962 0.17779283 0.04530383 +0.07612365 0.03370150 0.00840840 +0.00027532 0.00010882 0.00002590 +0.00000000 0.00000000 0.00000000 +0.00000000 0.22557832 0.05784992 +0.05484865 0.22912333 0.05484865 +0.09698467 0.23095378 0.04975907 +0.13297281 0.22513003 0.04626450 +0.16466959 0.21173705 0.04398982 +0.19891678 0.19891678 0.04227551 +0.23331275 0.18492540 0.03964287 +0.22853000 0.15384842 0.03262551 +0.34080524 0.20669146 0.04322289 +0.22244465 0.12399210 0.02378979 +0.00688265 0.00360923 0.00076146 +0.00000159 0.00000072 0.00000012 +0.00000000 0.00000000 0.00000000 +0.00000000 0.24812410 0.05315190 +0.05018152 0.25170547 0.05018152 +0.08754438 0.25306158 0.04486165 +0.12034648 0.24786893 0.04185713 +0.15544591 0.24448037 0.04078769 +0.18492540 0.23331275 0.03964287 +0.19466515 0.19466515 0.03375700 +0.28234193 0.22783320 0.04143619 +0.27650887 0.19774171 0.03245509 +0.02890315 0.01903327 0.00332244 +0.00011812 0.00006852 0.00001094 +0.00000001 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.26935568 0.05031494 +0.04698754 0.27224313 0.04698754 +0.08255333 0.27552492 0.04216991 +0.11728339 0.27864928 0.04033039 +0.14306836 0.26429520 0.03713853 +0.15384842 0.22853000 0.03262551 +0.22783320 0.28234193 0.04143619 +0.26686098 0.26686098 0.04015244 +0.05361946 0.04401921 0.00666538 +0.00077884 0.00057648 0.00008618 +0.00000012 0.00000008 0.00000001 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.29184205 0.05226268 +0.04897950 0.30077655 0.04897950 +0.08302718 0.30537290 0.04225252 +0.10485952 0.27970791 0.03571615 +0.13597825 0.27336582 0.03482393 +0.20669146 0.34080524 0.04322289 +0.19774171 0.27650887 0.03245509 +0.04401921 0.05361946 0.00666538 +0.00242573 0.00242573 0.00035678 +0.00000100 0.00000083 0.00000009 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.32796197 0.04779386 +0.04250931 0.30800102 0.04250931 +0.07481661 0.28730540 0.03779717 +0.13373729 0.35622202 0.04466998 +0.17779283 0.38848962 0.04530383 +0.12399210 0.22244465 0.02378979 +0.01903327 0.02890315 0.00332244 +0.00057648 0.00077884 0.00008618 +0.00000083 0.00000100 0.00000009 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.34155637 0.05854756 +0.06440826 0.40165640 0.06440826 +0.11485281 0.42864392 0.05668998 +0.10428998 0.29339533 0.03244443 +0.03370150 0.07612365 0.00840840 +0.00360923 0.00688265 0.00076146 +0.00006852 0.00011812 0.00001094 +0.00000008 0.00000012 0.00000001 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.29245201 0.03622370 +0.02766431 0.21450057 0.02766431 +0.01532577 0.05712571 0.00770589 +0.00230056 0.00661588 0.00077655 +0.00010882 0.00027532 0.00002590 +0.00000072 0.00000159 0.00000012 +0.00000000 0.00000001 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.00004101 +0.00002538 0.00000000 0.00002538 +0.00001000 0.00000000 0.00000477 +0.00000030 0.00000000 0.00000008 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.15323531 +0.08253341 0.00000000 0.16706916 +0.15129872 0.00000000 0.15129872 +0.18661483 0.00000000 0.12387973 +0.20744356 0.00000000 0.11091403 +0.23059254 0.00000000 0.09877480 +0.25292777 0.00000000 0.08951714 +0.27422715 0.00000000 0.08372217 +0.30646293 0.00000000 0.08576194 +0.28483792 0.00000000 0.07437695 +0.43878482 0.00000000 0.12180165 +0.08015231 0.00000000 0.02176584 +0.00000000 0.00000000 0.00001641 +0.00000000 0.08253341 0.16706916 +0.08447348 0.08447348 0.16763493 +0.14671206 0.07661738 0.14671206 +0.18541418 0.06505718 0.12388847 +0.20769496 0.05558558 0.10849415 +0.23095378 0.04975907 0.09698467 +0.25306158 0.04486165 0.08754438 +0.27552492 0.04216991 0.08255333 +0.30537290 0.04225252 0.08302718 +0.28730540 0.03779717 0.07481661 +0.42864392 0.05668998 0.11485281 +0.05712571 0.00770589 0.01532577 +0.00000000 0.00000477 0.00001000 +0.00000000 0.15129872 0.15129872 +0.07661738 0.14671206 0.14671206 +0.13524621 0.13524621 0.13524621 +0.17594823 0.11939755 0.11939755 +0.20477920 0.10411901 0.10411901 +0.22897201 0.09274519 0.09274519 +0.25098717 0.08373253 0.08373253 +0.27829336 0.08042743 0.08042743 +0.29232812 0.07512978 0.07512978 +0.32324693 0.08311482 0.08311482 +0.35807989 0.08667964 0.08667964 +0.01778447 0.00447407 0.00447407 +0.00000000 0.00000178 0.00000178 +0.00000000 0.18661483 0.12387973 +0.06505718 0.18541418 0.12388847 +0.11939755 0.17594823 0.11939755 +0.16106826 0.16106826 0.11029892 +0.19508786 0.14601426 0.09944481 +0.22159286 0.13001978 0.08845597 +0.24729509 0.11936413 0.08111633 +0.27596054 0.11453460 0.07738642 +0.26793238 0.09981428 0.06718600 +0.39079255 0.14162973 0.09486673 +0.18972336 0.06513445 0.04209815 +0.00221287 0.00072722 0.00048335 +0.00000000 0.00000006 0.00000004 +0.00000000 0.20744356 0.11091403 +0.05558558 0.20769496 0.10849415 +0.10411901 0.20477920 0.10411901 +0.14601426 0.19508786 0.09944481 +0.17985915 0.17985915 0.09249949 +0.20934788 0.16423396 0.08491019 +0.24378950 0.15508359 0.08008805 +0.25175385 0.13583805 0.06946288 +0.29942689 0.14900888 0.07502505 +0.35117617 0.15836951 0.07863162 +0.03790964 0.01665534 0.00841665 +0.00008359 0.00003212 0.00001515 +0.00000000 0.00000000 0.00000000 +0.00000000 0.23059254 0.09877480 +0.04975907 0.23095378 0.09698467 +0.09274519 0.22897201 0.09274519 +0.13001978 0.22159286 0.08845597 +0.16423396 0.20934788 0.08491019 +0.19957410 0.19957410 0.08257578 +0.22329079 0.17940160 0.07444742 +0.23871448 0.16256273 0.06735030 +0.34498469 0.20762137 0.08568348 +0.14694051 0.08145713 0.03193184 +0.00293995 0.00151442 0.00062559 +0.00000044 0.00000020 0.00000007 +0.00000000 0.00000000 0.00000000 +0.00000000 0.25292777 0.08951714 +0.04486165 0.25306158 0.08754438 +0.08373253 0.25098717 0.08373253 +0.11936413 0.24729509 0.08111633 +0.15508359 0.24378950 0.08008805 +0.17940160 0.22329079 0.07444742 +0.19769699 0.19769699 0.06773175 +0.29264331 0.24083547 0.08576193 +0.22308379 0.15910566 0.05178263 +0.01485261 0.00974796 0.00339455 +0.00003945 0.00002269 0.00000711 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.27422715 0.08372217 +0.04216991 0.27552492 0.08255333 +0.08042743 0.27829336 0.08042743 +0.11453460 0.27596054 0.07738642 +0.13583805 0.25175385 0.06946288 +0.16256273 0.23871448 0.06735030 +0.24083547 0.29264331 0.08576193 +0.21988464 0.21988464 0.06371968 +0.03079889 0.02582956 0.00786198 +0.00031769 0.00023404 0.00006772 +0.00000004 0.00000003 0.00000001 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.30646293 0.08576194 +0.04225252 0.30537290 0.08302718 +0.07512978 0.29232812 0.07512978 +0.09981428 0.26793238 0.06718600 +0.14900888 0.29942689 0.07502505 +0.20762137 0.34498469 0.08568348 +0.15910566 0.22308379 0.05178263 +0.02582956 0.03079889 0.00786198 +0.00088028 0.00088028 0.00023977 +0.00000033 0.00000028 0.00000006 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.28483792 0.07437695 +0.03779717 0.28730540 0.07481661 +0.08311482 0.32324693 0.08311482 +0.14162973 0.39079255 0.09486673 +0.15836951 0.35117617 0.07863162 +0.08145713 0.14694051 0.03193184 +0.00974796 0.01485261 0.00339455 +0.00023404 0.00031769 0.00006772 +0.00000028 0.00000033 0.00000006 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.43878482 0.12180165 +0.05668998 0.42864392 0.11485281 +0.08667964 0.35807989 0.08667964 +0.06513445 0.18972336 0.04209815 +0.01665534 0.03790964 0.00841665 +0.00151442 0.00293995 0.00062559 +0.00002269 0.00003945 0.00000711 +0.00000003 0.00000004 0.00000001 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.08015231 0.02176584 +0.00770589 0.05712571 0.01532577 +0.00447407 0.01778447 0.00447407 +0.00072722 0.00221287 0.00048335 +0.00003212 0.00008359 0.00001515 +0.00000020 0.00000044 0.00000007 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.00001641 +0.00000477 0.00000000 0.00001000 +0.00000178 0.00000000 0.00000178 +0.00000006 0.00000000 0.00000004 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.18052624 +0.07255161 0.00000000 0.18507825 +0.12387973 0.00000000 0.18661483 +0.17156448 0.00000000 0.17156448 +0.20161696 0.00000000 0.14816375 +0.22568042 0.00000000 0.13415464 +0.24815988 0.00000000 0.12082245 +0.27809616 0.00000000 0.11779382 +0.28558967 0.00000000 0.10768783 +0.34061828 0.00000000 0.12922569 +0.32820362 0.00000000 0.11853629 +0.00927076 0.00000000 0.00327665 +0.00000000 0.00000000 0.00000048 +0.00000000 0.07255161 0.18507825 +0.06881720 0.06881720 0.18818094 +0.12388847 0.06505718 0.18541418 +0.16862699 0.05971093 0.16862699 +0.20078594 0.05284890 0.14850555 +0.22513003 0.04626450 0.13297281 +0.24786893 0.04185713 0.12034648 +0.27864928 0.04033039 0.11728339 +0.27970791 0.03571615 0.10485952 +0.35622202 0.04466998 0.13373729 +0.29339533 0.03244443 0.10428998 +0.00661588 0.00077655 0.00230056 +0.00000000 0.00000008 0.00000030 +0.00000000 0.12387973 0.18661483 +0.06505718 0.12388847 0.18541418 +0.11939755 0.11939755 0.17594823 +0.16106826 0.11029892 0.16106826 +0.19508786 0.09944481 0.14601426 +0.22159286 0.08845597 0.13001978 +0.24729509 0.08111633 0.11936413 +0.27596054 0.07738642 0.11453460 +0.26793238 0.06718600 0.09981428 +0.39079255 0.09486673 0.14162973 +0.18972336 0.04209815 0.06513445 +0.00221287 0.00048335 0.00072722 +0.00000000 0.00000004 0.00000006 +0.00000000 0.17156448 0.17156448 +0.05971093 0.16862699 0.16862699 +0.11029892 0.16106826 0.16106826 +0.15140484 0.15140484 0.15140484 +0.18473471 0.13903198 0.13903198 +0.21415373 0.12654261 0.12654261 +0.24571981 0.11846918 0.11846918 +0.26056065 0.10657151 0.10657151 +0.28272496 0.10587090 0.10587090 +0.37633366 0.12899042 0.12899042 +0.06562444 0.02195203 0.02195203 +0.00028331 0.00008430 0.00008430 +0.00000000 0.00000000 0.00000000 +0.00000000 0.20161696 0.14816375 +0.05284890 0.20078594 0.14850555 +0.09944481 0.19508786 0.14601426 +0.13903198 0.18473471 0.13903198 +0.17280630 0.17280630 0.13037117 +0.20593067 0.16267392 0.12321924 +0.23536902 0.15087424 0.11447137 +0.23783199 0.12929001 0.09792657 +0.34029210 0.16756172 0.12658670 +0.24698436 0.10955503 0.08071337 +0.01040427 0.00446799 0.00337684 +0.00000867 0.00000318 0.00000229 +0.00000000 0.00000000 0.00000000 +0.00000000 0.22568042 0.13415464 +0.04626450 0.22513003 0.13297281 +0.08845597 0.22159286 0.13001978 +0.12654261 0.21415373 0.12654261 +0.16267392 0.20593067 0.12321924 +0.19476779 0.19476779 0.11779403 +0.20768059 0.16858893 0.10276335 +0.26752042 0.18489086 0.11274378 +0.31261553 0.18628836 0.11184057 +0.05968429 0.03299667 0.01991438 +0.00062314 0.00031089 0.00018728 +0.00000005 0.00000002 0.00000001 +0.00000000 0.00000000 0.00000000 +0.00000000 0.24815988 0.12082245 +0.04185713 0.24786893 0.12034648 +0.08111633 0.24729509 0.11936413 +0.11846918 0.24571981 0.11846918 +0.15087424 0.23536902 0.11447137 +0.16858893 0.20768059 0.10276335 +0.21943567 0.21943567 0.11144995 +0.28722793 0.24046303 0.12382638 +0.12780462 0.09195493 0.04575108 +0.00443936 0.00288967 0.00149160 +0.00000523 0.00000297 0.00000139 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.27809616 0.11779382 +0.04033039 0.27864928 0.11728339 +0.07738642 0.27596054 0.11453460 +0.10657151 0.26056065 0.10657151 +0.12929001 0.23783199 0.09792657 +0.18489086 0.26752042 0.11274378 +0.24046303 0.28722793 0.12382638 +0.13479603 0.13479603 0.05758260 +0.01050494 0.00897675 0.00406497 +0.00006176 0.00004529 0.00001896 +0.00000001 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.28558967 0.10768783 +0.03571615 0.27970791 0.10485952 +0.06718600 0.26793238 0.09981428 +0.10587090 0.28272496 0.10587090 +0.16756172 0.34029210 0.12658670 +0.18628836 0.31261553 0.11184057 +0.09195493 0.12780462 0.04575108 +0.00897675 0.01050494 0.00406497 +0.00014707 0.00014707 0.00005548 +0.00000005 0.00000004 0.00000001 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.34061828 0.12922569 +0.04466998 0.35622202 0.13373729 +0.09486673 0.39079255 0.14162973 +0.12899042 0.37633366 0.12899042 +0.10955503 0.24698436 0.08071337 +0.03299667 0.05968429 0.01991438 +0.00288967 0.00443936 0.00149160 +0.00004529 0.00006176 0.00001896 +0.00000004 0.00000005 0.00000001 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.32820362 0.11853629 +0.03244443 0.29339533 0.10428998 +0.04209815 0.18972336 0.06513445 +0.02195203 0.06562444 0.02195203 +0.00446799 0.01040427 0.00337684 +0.00031089 0.00062314 0.00018728 +0.00000297 0.00000523 0.00000139 +0.00000000 0.00000001 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00927076 0.00327665 +0.00077655 0.00661588 0.00230056 +0.00048335 0.00221287 0.00072722 +0.00008430 0.00028331 0.00008430 +0.00000318 0.00000867 0.00000229 +0.00000002 0.00000005 0.00000001 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.00000048 +0.00000008 0.00000000 0.00000030 +0.00000004 0.00000000 0.00000006 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.19922659 +0.06409840 0.00000000 0.20510401 +0.11091403 0.00000000 0.20744356 +0.14816375 0.00000000 0.20161696 +0.18839642 0.00000000 0.18839642 +0.21255306 0.00000000 0.16465196 +0.24407471 0.00000000 0.15550298 +0.26798333 0.00000000 0.14539135 +0.26576622 0.00000000 0.13184502 +0.39499686 0.00000000 0.18176438 +0.09442587 0.00000000 0.04187089 +0.00039903 0.00000000 0.00015905 +0.00000000 0.00000000 0.00000000 +0.00000000 0.06409840 0.20510401 +0.06125917 0.06125917 0.20793413 +0.10849415 0.05558558 0.20769496 +0.14850555 0.05284890 0.20078594 +0.18594880 0.04921526 0.18594880 +0.21173705 0.04398982 0.16466959 +0.24448037 0.04078769 0.15544591 +0.26429520 0.03713853 0.14306836 +0.27336582 0.03482393 0.13597825 +0.38848962 0.04530383 0.17779283 +0.07612365 0.00840840 0.03370150 +0.00027532 0.00002590 0.00010882 +0.00000000 0.00000000 0.00000000 +0.00000000 0.11091403 0.20744356 +0.05558558 0.10849415 0.20769496 +0.10411901 0.10411901 0.20477920 +0.14601426 0.09944481 0.19508786 +0.17985915 0.09249949 0.17985915 +0.20934788 0.08491019 0.16423396 +0.24378950 0.08008805 0.15508359 +0.25175385 0.06946288 0.13583805 +0.29942689 0.07502505 0.14900888 +0.35117617 0.07863162 0.15836951 +0.03790964 0.00841665 0.01665534 +0.00008359 0.00001515 0.00003212 +0.00000000 0.00000000 0.00000000 +0.00000000 0.14816375 0.20161696 +0.05284890 0.14850555 0.20078594 +0.09944481 0.14601426 0.19508786 +0.13903198 0.13903198 0.18473471 +0.17280630 0.13037117 0.17280630 +0.20593067 0.12321924 0.16267392 +0.23536902 0.11447137 0.15087424 +0.23783199 0.09792657 0.12929001 +0.34029210 0.12658670 0.16756172 +0.24698436 0.08071337 0.10955503 +0.01040427 0.00337684 0.00446799 +0.00000867 0.00000229 0.00000318 +0.00000000 0.00000000 0.00000000 +0.00000000 0.18839642 0.18839642 +0.04921526 0.18594880 0.18594880 +0.09249949 0.17985915 0.17985915 +0.13037117 0.17280630 0.17280630 +0.16611896 0.16611896 0.16611896 +0.19930738 0.15831015 0.15831015 +0.21424615 0.13877265 0.13877265 +0.25662287 0.14200517 0.14200517 +0.32983805 0.15821767 0.15821767 +0.09124718 0.04040967 0.04040967 +0.00132483 0.00054062 0.00054062 +0.00000020 0.00000007 0.00000007 +0.00000000 0.00000000 0.00000000 +0.00000000 0.21255306 0.16465196 +0.04398982 0.21173705 0.16466959 +0.08491019 0.20934788 0.16423396 +0.12321924 0.20593067 0.16267392 +0.15831015 0.19930738 0.15831015 +0.18007570 0.18007570 0.14379405 +0.21044762 0.17305007 0.13879825 +0.29860258 0.20935749 0.16849324 +0.20675213 0.12296526 0.09726934 +0.01280627 0.00705675 0.00569046 +0.00004980 0.00002382 0.00001877 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.24407471 0.15550298 +0.04078769 0.24448037 0.15544591 +0.08008805 0.24378950 0.15508359 +0.11447137 0.23536902 0.15087424 +0.13877265 0.21424615 0.13877265 +0.17305007 0.21044762 0.13879825 +0.25132266 0.25132266 0.16847290 +0.22475615 0.18922115 0.12425884 +0.03895527 0.02854692 0.01923163 +0.00066014 0.00042264 0.00028438 +0.00000021 0.00000012 0.00000008 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.26798333 0.14539135 +0.03713853 0.26429520 0.14306836 +0.06946288 0.25175385 0.13583805 +0.09792657 0.23783199 0.12929001 +0.14200517 0.25662287 0.14200517 +0.20935749 0.29860258 0.16849324 +0.18922115 0.22475615 0.12425884 +0.04653578 0.04653578 0.02666223 +0.00183246 0.00157772 0.00092690 +0.00000432 0.00000317 0.00000174 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.26576622 0.13184502 +0.03482393 0.27336582 0.13597825 +0.07502505 0.29942689 0.14900888 +0.12658670 0.34029210 0.16756172 +0.15821767 0.32983805 0.15821767 +0.12296526 0.20675213 0.09726934 +0.02854692 0.03895527 0.01923163 +0.00157772 0.00183246 0.00092690 +0.00000966 0.00000966 0.00000463 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.39499686 0.18176438 +0.04530383 0.38848962 0.17779283 +0.07863162 0.35117617 0.15836951 +0.08071337 0.24698436 0.10955503 +0.04040967 0.09124718 0.04040967 +0.00705675 0.01280627 0.00569046 +0.00042264 0.00066014 0.00028438 +0.00000317 0.00000432 0.00000174 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 -0.00000000 +0.00000000 0.09442587 0.04187089 +0.00840840 0.07612365 0.03370150 +0.00841665 0.03790964 0.01665534 +0.00337684 0.01040427 0.00446799 +0.00054062 0.00132483 0.00054062 +0.00002382 0.00004980 0.00001877 +0.00000012 0.00000021 0.00000008 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00039903 0.00015905 +0.00002590 0.00027532 0.00010882 +0.00001515 0.00008359 0.00003212 +0.00000229 0.00000867 0.00000318 +0.00000007 0.00000020 0.00000007 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 -0.00000000 +0.00000000 -0.00000000 -0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.21962195 +0.05784992 0.00000000 0.22557832 +0.09877480 0.00000000 0.23059254 +0.13415464 0.00000000 0.22568042 +0.16465196 0.00000000 0.21255306 +0.19805544 0.00000000 0.19805544 +0.23584271 0.00000000 0.18550201 +0.22778616 0.00000000 0.15266555 +0.33580943 0.00000000 0.20439457 +0.24988166 0.00000000 0.13969963 +0.00901370 0.00000000 0.00475284 +0.00000239 0.00000000 0.00000109 +0.00000000 0.00000000 0.00000000 +0.00000000 0.05784992 0.22557832 +0.05484865 0.05484865 0.22912333 +0.09698467 0.04975907 0.23095378 +0.13297281 0.04626450 0.22513003 +0.16466959 0.04398982 0.21173705 +0.19891678 0.04227551 0.19891678 +0.23331275 0.03964287 0.18492540 +0.22853000 0.03262551 0.15384842 +0.34080524 0.04322289 0.20669146 +0.22244465 0.02378979 0.12399210 +0.00688265 0.00076146 0.00360923 +0.00000159 0.00000012 0.00000072 +0.00000000 0.00000000 0.00000000 +0.00000000 0.09877480 0.23059254 +0.04975907 0.09698467 0.23095378 +0.09274519 0.09274519 0.22897201 +0.13001978 0.08845597 0.22159286 +0.16423396 0.08491019 0.20934788 +0.19957410 0.08257578 0.19957410 +0.22329079 0.07444742 0.17940160 +0.23871448 0.06735030 0.16256273 +0.34498469 0.08568348 0.20762137 +0.14694051 0.03193184 0.08145713 +0.00293995 0.00062559 0.00151442 +0.00000044 0.00000007 0.00000020 +0.00000000 0.00000000 0.00000000 +0.00000000 0.13415464 0.22568042 +0.04626450 0.13297281 0.22513003 +0.08845597 0.13001978 0.22159286 +0.12654261 0.12654261 0.21415373 +0.16267392 0.12321924 0.20593067 +0.19476779 0.11779403 0.19476779 +0.20768059 0.10276335 0.16858893 +0.26752042 0.11274378 0.18489086 +0.31261553 0.11184057 0.18628836 +0.05968429 0.01991438 0.03299667 +0.00062314 0.00018728 0.00031089 +0.00000005 0.00000001 0.00000002 +0.00000000 0.00000000 0.00000000 +0.00000000 0.16465196 0.21255306 +0.04398982 0.16466959 0.21173705 +0.08491019 0.16423396 0.20934788 +0.12321924 0.16267392 0.20593067 +0.15831015 0.15831015 0.19930738 +0.18007570 0.14379405 0.18007570 +0.21044762 0.13879825 0.17305007 +0.29860258 0.16849324 0.20935749 +0.20675213 0.09726934 0.12296526 +0.01280627 0.00569046 0.00705675 +0.00004980 0.00001877 0.00002382 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.19805544 0.19805544 +0.04227551 0.19891678 0.19891678 +0.08257578 0.19957410 0.19957410 +0.11779403 0.19476779 0.19476779 +0.14379405 0.18007570 0.18007570 +0.17328577 0.17328577 0.17328577 +0.24761344 0.20555657 0.20555657 +0.24707821 0.17159958 0.17159958 +0.05937251 0.03616259 0.03616259 +0.00125378 0.00067638 0.00067638 +0.00000080 0.00000037 0.00000037 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.23584271 0.18550201 +0.03964287 0.23331275 0.18492540 +0.07444742 0.22329079 0.17940160 +0.10276335 0.20768059 0.16858893 +0.13879825 0.21044762 0.17305007 +0.20555657 0.24761344 0.20555657 +0.22620541 0.22620541 0.18680260 +0.09448152 0.07989574 0.06604124 +0.00548834 0.00406603 0.00340836 +0.00003419 0.00002144 0.00001768 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.22778616 0.15266555 +0.03262551 0.22853000 0.15384842 +0.06735030 0.23871448 0.16256273 +0.11274378 0.26752042 0.18489086 +0.16849324 0.29860258 0.20935749 +0.17159958 0.24707821 0.17159958 +0.07989574 0.09448152 0.06604124 +0.00763276 0.00763276 0.00547505 +0.00013148 0.00011296 0.00008002 +0.00000008 0.00000006 0.00000004 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.33580943 0.20439457 +0.04322289 0.34080524 0.20669146 +0.08568348 0.34498469 0.20762137 +0.11184057 0.31261553 0.18628836 +0.09726934 0.20675213 0.12296526 +0.03616259 0.05937251 0.03616259 +0.00406603 0.00548834 0.00340836 +0.00011296 0.00013148 0.00008002 +0.00000018 0.00000018 0.00000011 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.24988166 0.13969963 +0.02378979 0.22244465 0.12399210 +0.03193184 0.14694051 0.08145713 +0.01991438 0.05968429 0.03299667 +0.00569046 0.01280627 0.00705675 +0.00067638 0.00125378 0.00067638 +0.00002144 0.00003419 0.00001768 +0.00000006 0.00000008 0.00000004 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00901370 0.00475284 +0.00076146 0.00688265 0.00360923 +0.00062559 0.00293995 0.00151442 +0.00018728 0.00062314 0.00031089 +0.00001877 0.00004980 0.00002382 +0.00000037 0.00000080 0.00000037 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000239 0.00000109 +0.00000012 0.00000159 0.00000072 +0.00000007 0.00000044 0.00000020 +0.00000001 0.00000005 0.00000002 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 -0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.24196835 +0.05315190 0.00000000 0.24812410 +0.08951714 0.00000000 0.25292777 +0.12082245 0.00000000 0.24815988 +0.15550298 0.00000000 0.24407471 +0.18550201 0.00000000 0.23584271 +0.19670448 0.00000000 0.19670448 +0.27828749 0.00000000 0.22263260 +0.29186871 0.00000000 0.20940311 +0.03569997 0.00000000 0.02352921 +0.00016711 0.00000000 0.00009720 +0.00000001 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.05315190 0.24812410 +0.05018152 0.05018152 0.25170547 +0.08754438 0.04486165 0.25306158 +0.12034648 0.04185713 0.24786893 +0.15544591 0.04078769 0.24448037 +0.18492540 0.03964287 0.23331275 +0.19466515 0.03375700 0.19466515 +0.28234193 0.04143619 0.22783320 +0.27650887 0.03245509 0.19774171 +0.02890315 0.00332244 0.01903327 +0.00011812 0.00001094 0.00006852 +0.00000001 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.08951714 0.25292777 +0.04486165 0.08754438 0.25306158 +0.08373253 0.08373253 0.25098717 +0.11936413 0.08111633 0.24729509 +0.15508359 0.08008805 0.24378950 +0.17940160 0.07444742 0.22329079 +0.19769699 0.06773175 0.19769699 +0.29264331 0.08576193 0.24083547 +0.22308379 0.05178263 0.15910566 +0.01485261 0.00339455 0.00974796 +0.00003945 0.00000711 0.00002269 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.12082245 0.24815988 +0.04185713 0.12034648 0.24786893 +0.08111633 0.11936413 0.24729509 +0.11846918 0.11846918 0.24571981 +0.15087424 0.11447137 0.23536902 +0.16858893 0.10276335 0.20768059 +0.21943567 0.11144995 0.21943567 +0.28722793 0.12382638 0.24046303 +0.12780462 0.04575108 0.09195493 +0.00443936 0.00149160 0.00288967 +0.00000523 0.00000139 0.00000297 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.15550298 0.24407471 +0.04078769 0.15544591 0.24448037 +0.08008805 0.15508359 0.24378950 +0.11447137 0.15087424 0.23536902 +0.13877265 0.13877265 0.21424615 +0.17305007 0.13879825 0.21044762 +0.25132266 0.16847290 0.25132266 +0.22475615 0.12425884 0.18922115 +0.03895527 0.01923163 0.02854692 +0.00066014 0.00028438 0.00042264 +0.00000021 0.00000008 0.00000012 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.18550201 0.23584271 +0.03964287 0.18492540 0.23331275 +0.07444742 0.17940160 0.22329079 +0.10276335 0.16858893 0.20768059 +0.13879825 0.17305007 0.21044762 +0.20555657 0.20555657 0.24761344 +0.22620541 0.18680260 0.22620541 +0.09448152 0.06604124 0.07989574 +0.00548834 0.00340836 0.00406603 +0.00003419 0.00001768 0.00002144 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.19670448 0.19670448 +0.03375700 0.19466515 0.19466515 +0.06773175 0.19769699 0.19769699 +0.11144995 0.21943567 0.21943567 +0.16847290 0.25132266 0.25132266 +0.18680260 0.22620541 0.22620541 +0.10426029 0.10426029 0.10426029 +0.01253251 0.01070927 0.01070927 +0.00027520 0.00020192 0.00020192 +0.00000027 0.00000017 0.00000017 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.27828749 0.22263260 +0.04143619 0.28234193 0.22783320 +0.08576193 0.29264331 0.24083547 +0.12382638 0.28722793 0.24046303 +0.12425884 0.22475615 0.18922115 +0.06604124 0.09448152 0.07989574 +0.01070927 0.01253251 0.01070927 +0.00047962 0.00047962 0.00040935 +0.00000196 0.00000168 0.00000141 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.29186871 0.20940311 +0.03245509 0.27650887 0.19774171 +0.05178263 0.22308379 0.15910566 +0.04575108 0.12780462 0.09195493 +0.01923163 0.03895527 0.02854692 +0.00340836 0.00548834 0.00406603 +0.00020192 0.00027520 0.00020192 +0.00000168 0.00000196 0.00000141 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.03569997 0.02352921 +0.00332244 0.02890315 0.01903327 +0.00339455 0.01485261 0.00974796 +0.00149160 0.00443936 0.00288967 +0.00028438 0.00066014 0.00042264 +0.00001768 0.00003419 0.00002144 +0.00000017 0.00000027 0.00000017 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00016711 0.00009720 +0.00001094 0.00011812 0.00006852 +0.00000711 0.00003945 0.00002269 +0.00000139 0.00000523 0.00000297 +0.00000008 0.00000021 0.00000012 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000001 0.00000000 +0.00000000 0.00000001 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 -0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.26345823 +0.05031494 0.00000000 0.26935568 +0.08372217 0.00000000 0.27422715 +0.11779382 0.00000000 0.27809616 +0.14539135 0.00000000 0.26798333 +0.15266555 0.00000000 0.22778616 +0.22263260 0.00000000 0.27828749 +0.28020504 0.00000000 0.28020504 +0.06343861 0.00000000 0.05159365 +0.00103623 0.00000000 0.00076839 +0.00000017 0.00000000 0.00000011 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.05031494 0.26935568 +0.04698754 0.04698754 0.27224313 +0.08255333 0.04216991 0.27552492 +0.11728339 0.04033039 0.27864928 +0.14306836 0.03713853 0.26429520 +0.15384842 0.03262551 0.22853000 +0.22783320 0.04143619 0.28234193 +0.26686098 0.04015244 0.26686098 +0.05361946 0.00666538 0.04401921 +0.00077884 0.00008618 0.00057648 +0.00000012 0.00000001 0.00000008 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.08372217 0.27422715 +0.04216991 0.08255333 0.27552492 +0.08042743 0.08042743 0.27829336 +0.11453460 0.07738642 0.27596054 +0.13583805 0.06946288 0.25175385 +0.16256273 0.06735030 0.23871448 +0.24083547 0.08576193 0.29264331 +0.21988464 0.06371968 0.21988464 +0.03079889 0.00786198 0.02582956 +0.00031769 0.00006772 0.00023404 +0.00000004 0.00000001 0.00000003 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.11779382 0.27809616 +0.04033039 0.11728339 0.27864928 +0.07738642 0.11453460 0.27596054 +0.10657151 0.10657151 0.26056065 +0.12929001 0.09792657 0.23783199 +0.18489086 0.11274378 0.26752042 +0.24046303 0.12382638 0.28722793 +0.13479603 0.05758260 0.13479603 +0.01050494 0.00406497 0.00897675 +0.00006176 0.00001896 0.00004529 +0.00000001 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.14539135 0.26798333 +0.03713853 0.14306836 0.26429520 +0.06946288 0.13583805 0.25175385 +0.09792657 0.12929001 0.23783199 +0.14200517 0.14200517 0.25662287 +0.20935749 0.16849324 0.29860258 +0.18922115 0.12425884 0.22475615 +0.04653578 0.02666223 0.04653578 +0.00183246 0.00092690 0.00157772 +0.00000432 0.00000174 0.00000317 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.15266555 0.22778616 +0.03262551 0.15384842 0.22853000 +0.06735030 0.16256273 0.23871448 +0.11274378 0.18489086 0.26752042 +0.16849324 0.20935749 0.29860258 +0.17159958 0.17159958 0.24707821 +0.07989574 0.06604124 0.09448152 +0.00763276 0.00547505 0.00763276 +0.00013148 0.00008002 0.00011296 +0.00000008 0.00000004 0.00000006 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.22263260 0.27828749 +0.04143619 0.22783320 0.28234193 +0.08576193 0.24083547 0.29264331 +0.12382638 0.24046303 0.28722793 +0.12425884 0.18922115 0.22475615 +0.06604124 0.07989574 0.09448152 +0.01070927 0.01070927 0.01253251 +0.00047962 0.00040935 0.00047962 +0.00000196 0.00000141 0.00000168 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.28020504 0.28020504 +0.04015244 0.26686098 0.26686098 +0.06371968 0.21988464 0.21988464 +0.05758260 0.13479603 0.13479603 +0.02666223 0.04653578 0.04653578 +0.00547505 0.00763276 0.00763276 +0.00040935 0.00047962 0.00047962 +0.00000479 0.00000479 0.00000479 +0.00000001 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.06343861 0.05159365 +0.00666538 0.05361946 0.04401921 +0.00786198 0.03079889 0.02582956 +0.00406497 0.01050494 0.00897675 +0.00092690 0.00183246 0.00157772 +0.00008002 0.00013148 0.00011296 +0.00000141 0.00000196 0.00000168 +0.00000000 0.00000001 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00103623 0.00076839 +0.00008618 0.00077884 0.00057648 +0.00006772 0.00031769 0.00023404 +0.00001896 0.00006176 0.00004529 +0.00000174 0.00000432 0.00000317 +0.00000004 0.00000008 0.00000006 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000017 0.00000011 +0.00000001 0.00000012 0.00000008 +0.00000001 0.00000004 0.00000003 +0.00000000 0.00000001 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 -0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.27808604 +0.05226268 0.00000000 0.29184205 +0.08576194 0.00000000 0.30646293 +0.10768783 0.00000000 0.28558967 +0.13184502 0.00000000 0.26576622 +0.20439457 0.00000000 0.33580943 +0.20940311 0.00000000 0.29186871 +0.05159365 0.00000000 0.06343861 +0.00335127 0.00000000 0.00335127 +0.00000143 0.00000000 0.00000119 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.05226268 0.29184205 +0.04897950 0.04897950 0.30077655 +0.08302718 0.04225252 0.30537290 +0.10485952 0.03571615 0.27970791 +0.13597825 0.03482393 0.27336582 +0.20669146 0.04322289 0.34080524 +0.19774171 0.03245509 0.27650887 +0.04401921 0.00666538 0.05361946 +0.00242573 0.00035678 0.00242573 +0.00000100 0.00000009 0.00000083 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.08576194 0.30646293 +0.04225252 0.08302718 0.30537290 +0.07512978 0.07512978 0.29232812 +0.09981428 0.06718600 0.26793238 +0.14900888 0.07502505 0.29942689 +0.20762137 0.08568348 0.34498469 +0.15910566 0.05178263 0.22308379 +0.02582956 0.00786198 0.03079889 +0.00088028 0.00023977 0.00088028 +0.00000033 0.00000006 0.00000028 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.10768783 0.28558967 +0.03571615 0.10485952 0.27970791 +0.06718600 0.09981428 0.26793238 +0.10587090 0.10587090 0.28272496 +0.16756172 0.12658670 0.34029210 +0.18628836 0.11184057 0.31261553 +0.09195493 0.04575108 0.12780462 +0.00897675 0.00406497 0.01050494 +0.00014707 0.00005548 0.00014707 +0.00000005 0.00000001 0.00000004 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.13184502 0.26576622 +0.03482393 0.13597825 0.27336582 +0.07502505 0.14900888 0.29942689 +0.12658670 0.16756172 0.34029210 +0.15821767 0.15821767 0.32983805 +0.12296526 0.09726934 0.20675213 +0.02854692 0.01923163 0.03895527 +0.00157772 0.00092690 0.00183246 +0.00000966 0.00000463 0.00000966 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.20439457 0.33580943 +0.04322289 0.20669146 0.34080524 +0.08568348 0.20762137 0.34498469 +0.11184057 0.18628836 0.31261553 +0.09726934 0.12296526 0.20675213 +0.03616259 0.03616259 0.05937251 +0.00406603 0.00340836 0.00548834 +0.00011296 0.00008002 0.00013148 +0.00000018 0.00000011 0.00000018 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.20940311 0.29186871 +0.03245509 0.19774171 0.27650887 +0.05178263 0.15910566 0.22308379 +0.04575108 0.09195493 0.12780462 +0.01923163 0.02854692 0.03895527 +0.00340836 0.00406603 0.00548834 +0.00020192 0.00020192 0.00027520 +0.00000168 0.00000141 0.00000196 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.05159365 0.06343861 +0.00666538 0.04401921 0.05361946 +0.00786198 0.02582956 0.03079889 +0.00406497 0.00897675 0.01050494 +0.00092690 0.00157772 0.00183246 +0.00008002 0.00011296 0.00013148 +0.00000141 0.00000168 0.00000196 +0.00000000 0.00000000 0.00000001 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 -0.00000000 +0.00000000 0.00335127 0.00335127 +0.00035678 0.00242573 0.00242573 +0.00023977 0.00088028 0.00088028 +0.00005548 0.00014707 0.00014707 +0.00000463 0.00000966 0.00000966 +0.00000011 0.00000018 0.00000018 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000143 0.00000119 +0.00000009 0.00000100 0.00000083 +0.00000006 0.00000033 0.00000028 +0.00000001 0.00000005 0.00000004 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 -0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 -0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 -0.00000000 -0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.35128500 +0.04779386 0.00000000 0.32796197 +0.07437695 0.00000000 0.28483792 +0.12922569 0.00000000 0.34061828 +0.18176438 0.00000000 0.39499686 +0.13969963 0.00000000 0.24988166 +0.02352921 0.00000000 0.03569997 +0.00076839 0.00000000 0.00103623 +0.00000119 0.00000000 0.00000143 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.04779386 0.32796197 +0.04250931 0.04250931 0.30800102 +0.07481661 0.03779717 0.28730540 +0.13373729 0.04466998 0.35622202 +0.17779283 0.04530383 0.38848962 +0.12399210 0.02378979 0.22244465 +0.01903327 0.00332244 0.02890315 +0.00057648 0.00008618 0.00077884 +0.00000083 0.00000009 0.00000100 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.07437695 0.28483792 +0.03779717 0.07481661 0.28730540 +0.08311482 0.08311482 0.32324693 +0.14162973 0.09486673 0.39079255 +0.15836951 0.07863162 0.35117617 +0.08145713 0.03193184 0.14694051 +0.00974796 0.00339455 0.01485261 +0.00023404 0.00006772 0.00031769 +0.00000028 0.00000006 0.00000033 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.12922569 0.34061828 +0.04466998 0.13373729 0.35622202 +0.09486673 0.14162973 0.39079255 +0.12899042 0.12899042 0.37633366 +0.10955503 0.08071337 0.24698436 +0.03299667 0.01991438 0.05968429 +0.00288967 0.00149160 0.00443936 +0.00004529 0.00001896 0.00006176 +0.00000004 0.00000001 0.00000005 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.18176438 0.39499686 +0.04530383 0.17779283 0.38848962 +0.07863162 0.15836951 0.35117617 +0.08071337 0.10955503 0.24698436 +0.04040967 0.04040967 0.09124718 +0.00705675 0.00569046 0.01280627 +0.00042264 0.00028438 0.00066014 +0.00000317 0.00000174 0.00000432 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.13969963 0.24988166 +0.02378979 0.12399210 0.22244465 +0.03193184 0.08145713 0.14694051 +0.01991438 0.03299667 0.05968429 +0.00569046 0.00705675 0.01280627 +0.00067638 0.00067638 0.00125378 +0.00002144 0.00001768 0.00003419 +0.00000006 0.00000004 0.00000008 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.02352921 0.03569997 +0.00332244 0.01903327 0.02890315 +0.00339455 0.00974796 0.01485261 +0.00149160 0.00288967 0.00443936 +0.00028438 0.00042264 0.00066014 +0.00001768 0.00002144 0.00003419 +0.00000017 0.00000017 0.00000027 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00076839 0.00103623 +0.00008618 0.00057648 0.00077884 +0.00006772 0.00023404 0.00031769 +0.00001896 0.00004529 0.00006176 +0.00000174 0.00000317 0.00000432 +0.00000004 0.00000006 0.00000008 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000119 0.00000143 +0.00000009 0.00000083 0.00000100 +0.00000006 0.00000028 0.00000033 +0.00000001 0.00000004 0.00000005 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 -0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 -0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.25113715 +0.05854756 0.00000000 0.34155637 +0.12180165 0.00000000 0.43878482 +0.11853629 0.00000000 0.32820362 +0.04187089 0.00000000 0.09442587 +0.00475284 0.00000000 0.00901370 +0.00009720 0.00000000 0.00016711 +0.00000011 0.00000000 0.00000017 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.05854756 0.34155637 +0.06440826 0.06440826 0.40165640 +0.11485281 0.05668998 0.42864392 +0.10428998 0.03244443 0.29339533 +0.03370150 0.00840840 0.07612365 +0.00360923 0.00076146 0.00688265 +0.00006852 0.00001094 0.00011812 +0.00000008 0.00000001 0.00000012 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.12180165 0.43878482 +0.05668998 0.11485281 0.42864392 +0.08667964 0.08667964 0.35807989 +0.06513445 0.04209815 0.18972336 +0.01665534 0.00841665 0.03790964 +0.00151442 0.00062559 0.00293995 +0.00002269 0.00000711 0.00003945 +0.00000003 0.00000001 0.00000004 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.11853629 0.32820362 +0.03244443 0.10428998 0.29339533 +0.04209815 0.06513445 0.18972336 +0.02195203 0.02195203 0.06562444 +0.00446799 0.00337684 0.01040427 +0.00031089 0.00018728 0.00062314 +0.00000297 0.00000139 0.00000523 +0.00000000 0.00000000 0.00000001 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.04187089 0.09442587 +0.00840840 0.03370150 0.07612365 +0.00841665 0.01665534 0.03790964 +0.00337684 0.00446799 0.01040427 +0.00054062 0.00054062 0.00132483 +0.00002382 0.00001877 0.00004980 +0.00000012 0.00000008 0.00000021 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00475284 0.00901370 +0.00076146 0.00360923 0.00688265 +0.00062559 0.00151442 0.00293995 +0.00018728 0.00031089 0.00062314 +0.00001877 0.00002382 0.00004980 +0.00000037 0.00000037 0.00000080 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00009720 0.00016711 +0.00001094 0.00006852 0.00011812 +0.00000711 0.00002269 0.00003945 +0.00000139 0.00000297 0.00000523 +0.00000008 0.00000012 0.00000021 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000011 0.00000017 +0.00000001 0.00000008 0.00000012 +0.00000001 0.00000003 0.00000004 +0.00000000 0.00000000 0.00000001 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 -0.00000000 -0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.38181582 +0.03622370 0.00000000 0.29245201 +0.02176584 0.00000000 0.08015231 +0.00327665 0.00000000 0.00927076 +0.00015905 0.00000000 0.00039903 +0.00000109 0.00000000 0.00000239 +0.00000000 0.00000000 0.00000001 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.03622370 0.29245201 +0.02766431 0.02766431 0.21450057 +0.01532577 0.00770589 0.05712571 +0.00230056 0.00077655 0.00661588 +0.00010882 0.00002590 0.00027532 +0.00000072 0.00000012 0.00000159 +0.00000000 0.00000000 0.00000001 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.02176584 0.08015231 +0.00770589 0.01532577 0.05712571 +0.00447407 0.00447407 0.01778447 +0.00072722 0.00048335 0.00221287 +0.00003212 0.00001515 0.00008359 +0.00000020 0.00000007 0.00000044 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00327665 0.00927076 +0.00077655 0.00230056 0.00661588 +0.00048335 0.00072722 0.00221287 +0.00008430 0.00008430 0.00028331 +0.00000318 0.00000229 0.00000867 +0.00000002 0.00000001 0.00000005 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00015905 0.00039903 +0.00002590 0.00010882 0.00027532 +0.00001515 0.00003212 0.00008359 +0.00000229 0.00000318 0.00000867 +0.00000007 0.00000007 0.00000020 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000109 0.00000239 +0.00000012 0.00000072 0.00000159 +0.00000007 0.00000020 0.00000044 +0.00000001 0.00000002 0.00000005 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000001 +0.00000000 0.00000000 0.00000001 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 -0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 -0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 -0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 -0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.00000000 +0.00004101 0.00000000 0.00000000 +0.00001641 0.00000000 0.00000000 +0.00000048 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00004101 0.00000000 +0.00002538 0.00002538 0.00000000 +0.00001000 0.00000477 0.00000000 +0.00000030 0.00000008 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00001641 0.00000000 +0.00000477 0.00001000 0.00000000 +0.00000178 0.00000178 0.00000000 +0.00000006 0.00000004 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000048 0.00000000 +0.00000008 0.00000030 0.00000000 +0.00000004 0.00000006 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 + +CELL_DATA 1728 +SCALARS SIE float 1 +LOOKUP_TABLE default +11.96970310 +1.95000345 +0.68059159 +0.42679413 +0.31956721 +0.25884397 +0.21675030 +0.18791042 +0.17026011 +0.12799510 +0.15550975 +0.00010527 +1.95000345 +1.28520793 +0.55813255 +0.32283128 +0.23218801 +0.18530510 +0.15376736 +0.13124374 +0.11335403 +0.09892888 +0.04855330 +0.00002110 +0.68059159 +0.55813255 +0.42481935 +0.24743849 +0.17598073 +0.13956042 +0.11737831 +0.10329029 +0.08427341 +0.09175639 +0.00607623 +0.00000013 +0.42679413 +0.32283128 +0.24743849 +0.21151579 +0.14386465 +0.11536829 +0.10075016 +0.08506177 +0.08084794 +0.04458657 +0.00024090 +0.00000000 +0.31956721 +0.23218801 +0.17598073 +0.14386465 +0.12803331 +0.09721032 +0.08233922 +0.07372138 +0.07780606 +0.00588790 +0.00000049 +0.00000000 +0.25884397 +0.18530510 +0.13956042 +0.11536829 +0.09721032 +0.08964409 +0.06747199 +0.07294920 +0.01873712 +0.00009796 +0.00000000 +0.00000000 +0.21675030 +0.15376736 +0.11737831 +0.10075016 +0.08233922 +0.06747199 +0.07085256 +0.02962047 +0.00074974 +0.00000002 +0.00000000 +0.00000000 +0.18791042 +0.13124374 +0.10329029 +0.08506177 +0.07372138 +0.07294920 +0.02962047 +0.00298788 +0.00000029 +0.00000000 +0.00000000 +0.00000000 +0.17026011 +0.11335403 +0.08427341 +0.08084794 +0.07780606 +0.01873712 +0.00074974 +0.00000029 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.12799510 +0.09892888 +0.09175639 +0.04458657 +0.00588790 +0.00009796 +0.00000002 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.15550975 +0.04855330 +0.00607623 +0.00024090 +0.00000049 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00010527 +0.00002110 +0.00000013 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +1.95000345 +1.28520793 +0.55813255 +0.32283128 +0.23218801 +0.18530510 +0.15376736 +0.13124374 +0.11335403 +0.09892888 +0.04855330 +0.00002110 +1.28520793 +0.79807819 +0.46034581 +0.26637332 +0.19434385 +0.15332891 +0.12794963 +0.11231419 +0.09231118 +0.09251083 +0.01764984 +0.00000355 +0.55813255 +0.46034581 +0.31839076 +0.22030975 +0.15722323 +0.12704809 +0.10856596 +0.09405160 +0.08071688 +0.08188588 +0.00236939 +0.00000002 +0.32283128 +0.26637332 +0.22030975 +0.17199862 +0.13291264 +0.10597756 +0.09412617 +0.07907574 +0.08021050 +0.02804596 +0.00007821 +0.00000000 +0.23218801 +0.19434385 +0.15722323 +0.13291264 +0.11010145 +0.09350691 +0.07665995 +0.07183175 +0.06574932 +0.00305855 +0.00000013 +0.00000000 +0.18530510 +0.15332891 +0.12704809 +0.10597756 +0.09350691 +0.07775026 +0.06706071 +0.07144920 +0.01193449 +0.00003456 +0.00000000 +0.00000000 +0.15376736 +0.12794963 +0.10856596 +0.09412617 +0.07665995 +0.06706071 +0.06813314 +0.02153258 +0.00035406 +0.00000000 +0.00000000 +0.00000000 +0.13124374 +0.11231419 +0.09405160 +0.07907574 +0.07183175 +0.07144920 +0.02153258 +0.00124650 +0.00000009 +0.00000000 +0.00000000 +0.00000000 +0.11335403 +0.09231118 +0.08071688 +0.08021050 +0.06574932 +0.01193449 +0.00035406 +0.00000009 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.09892888 +0.09251083 +0.08188588 +0.02804596 +0.00305855 +0.00003456 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.04855330 +0.01764984 +0.00236939 +0.00007821 +0.00000013 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00002110 +0.00000355 +0.00000002 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.68059159 +0.55813255 +0.42481935 +0.24743849 +0.17598073 +0.13956042 +0.11737831 +0.10329029 +0.08427341 +0.09175639 +0.00607623 +0.00000013 +0.55813255 +0.46034581 +0.31839076 +0.22030975 +0.15722323 +0.12704809 +0.10856596 +0.09405160 +0.08071688 +0.08188588 +0.00236939 +0.00000002 +0.42481935 +0.31839076 +0.22944566 +0.17527137 +0.13735527 +0.11084416 +0.09659286 +0.08275941 +0.07860557 +0.04355621 +0.00030393 +0.00000000 +0.24743849 +0.22030975 +0.17527137 +0.13789615 +0.11555518 +0.09723650 +0.08277858 +0.07262965 +0.07833357 +0.00983444 +0.00000628 +0.00000000 +0.17598073 +0.15722323 +0.13735527 +0.11555518 +0.09525889 +0.08377748 +0.07027104 +0.07119741 +0.03692349 +0.00072355 +0.00000001 +0.00000000 +0.13956042 +0.12704809 +0.11084416 +0.09723650 +0.08377748 +0.06818122 +0.06616563 +0.05853347 +0.00451546 +0.00000354 +0.00000000 +0.00000000 +0.11737831 +0.10856596 +0.09659286 +0.08277858 +0.07027104 +0.06616563 +0.06025065 +0.00972799 +0.00006690 +0.00000000 +0.00000000 +0.00000000 +0.10329029 +0.09405160 +0.08275941 +0.07262965 +0.07119741 +0.05853347 +0.00972799 +0.00020339 +0.00000001 +0.00000000 +0.00000000 +0.00000000 +0.08427341 +0.08071688 +0.07860557 +0.07833357 +0.03692349 +0.00451546 +0.00006690 +0.00000001 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.09175639 +0.08188588 +0.04355621 +0.00983444 +0.00072355 +0.00000354 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00607623 +0.00236939 +0.00030393 +0.00000628 +0.00000001 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000013 +0.00000002 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.42679413 +0.32283128 +0.24743849 +0.21151579 +0.14386465 +0.11536829 +0.10075016 +0.08506177 +0.08084794 +0.04458657 +0.00024090 +0.00000000 +0.32283128 +0.26637332 +0.22030975 +0.17199862 +0.13291264 +0.10597756 +0.09412617 +0.07907574 +0.08021050 +0.02804596 +0.00007821 +0.00000000 +0.24743849 +0.22030975 +0.17527137 +0.13789615 +0.11555518 +0.09723650 +0.08277858 +0.07262965 +0.07833357 +0.00983444 +0.00000628 +0.00000000 +0.21151579 +0.17199862 +0.13789615 +0.11829215 +0.09811864 +0.08655123 +0.07309248 +0.07001552 +0.04925510 +0.00151216 +0.00000006 +0.00000000 +0.14386465 +0.13291264 +0.11555518 +0.09811864 +0.08753088 +0.07339695 +0.06683084 +0.07014299 +0.01118626 +0.00005207 +0.00000000 +0.00000000 +0.11536829 +0.10597756 +0.09723650 +0.08655123 +0.07339695 +0.06558138 +0.06847082 +0.02693209 +0.00080374 +0.00000007 +0.00000000 +0.00000000 +0.10075016 +0.09412617 +0.08277858 +0.07309248 +0.06683084 +0.06847082 +0.03245148 +0.00225449 +0.00000332 +0.00000000 +0.00000000 +0.00000000 +0.08506177 +0.07907574 +0.07262965 +0.07001552 +0.07014299 +0.02693209 +0.00225449 +0.00001003 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.08084794 +0.08021050 +0.07833357 +0.04925510 +0.01118626 +0.00080374 +0.00000332 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.04458657 +0.02804596 +0.00983444 +0.00151216 +0.00005207 +0.00000007 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00024090 +0.00007821 +0.00000628 +0.00000006 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.31956721 +0.23218801 +0.17598073 +0.14386465 +0.12803331 +0.09721032 +0.08233922 +0.07372138 +0.07780606 +0.00588790 +0.00000049 +0.00000000 +0.23218801 +0.19434385 +0.15722323 +0.13291264 +0.11010145 +0.09350691 +0.07665995 +0.07183175 +0.06574932 +0.00305855 +0.00000013 +0.00000000 +0.17598073 +0.15722323 +0.13735527 +0.11555518 +0.09525889 +0.08377748 +0.07027104 +0.07119741 +0.03692349 +0.00072355 +0.00000001 +0.00000000 +0.14386465 +0.13291264 +0.11555518 +0.09811864 +0.08753088 +0.07339695 +0.06683084 +0.07014299 +0.01118626 +0.00005207 +0.00000000 +0.00000000 +0.12803331 +0.11010145 +0.09525889 +0.08753088 +0.07550642 +0.06670611 +0.06800883 +0.03625927 +0.00145810 +0.00000040 +0.00000000 +0.00000000 +0.09721032 +0.09350691 +0.08377748 +0.07339695 +0.06670611 +0.06780607 +0.05087100 +0.00582269 +0.00003846 +0.00000000 +0.00000000 +0.00000000 +0.08233922 +0.07665995 +0.07027104 +0.06683084 +0.06800883 +0.05087100 +0.00845280 +0.00018220 +0.00000002 +0.00000000 +0.00000000 +0.00000000 +0.07372138 +0.07183175 +0.07119741 +0.07014299 +0.03625927 +0.00582269 +0.00018220 +0.00000008 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.07780606 +0.06574932 +0.03692349 +0.01118626 +0.00145810 +0.00003846 +0.00000002 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00588790 +0.00305855 +0.00072355 +0.00005207 +0.00000040 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000049 +0.00000013 +0.00000001 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.25884397 +0.18530510 +0.13956042 +0.11536829 +0.09721032 +0.08964409 +0.06747199 +0.07294920 +0.01873712 +0.00009796 +0.00000000 +0.00000000 +0.18530510 +0.15332891 +0.12704809 +0.10597756 +0.09350691 +0.07775026 +0.06706071 +0.07144920 +0.01193449 +0.00003456 +0.00000000 +0.00000000 +0.13956042 +0.12704809 +0.11084416 +0.09723650 +0.08377748 +0.06818122 +0.06616563 +0.05853347 +0.00451546 +0.00000354 +0.00000000 +0.00000000 +0.11536829 +0.10597756 +0.09723650 +0.08655123 +0.07339695 +0.06558138 +0.06847082 +0.02693209 +0.00080374 +0.00000007 +0.00000000 +0.00000000 +0.09721032 +0.09350691 +0.08377748 +0.07339695 +0.06670611 +0.06780607 +0.05087100 +0.00582269 +0.00003846 +0.00000000 +0.00000000 +0.00000000 +0.08964409 +0.07775026 +0.06818122 +0.06558138 +0.06780607 +0.05775727 +0.01246553 +0.00037585 +0.00000012 +0.00000000 +0.00000000 +0.00000000 +0.06747199 +0.06706071 +0.06616563 +0.06847082 +0.05087100 +0.01246553 +0.00074932 +0.00000165 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.07294920 +0.07144920 +0.05853347 +0.02693209 +0.00582269 +0.00037585 +0.00000165 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.01873712 +0.01193449 +0.00451546 +0.00080374 +0.00003846 +0.00000012 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00009796 +0.00003456 +0.00000354 +0.00000007 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.21675030 +0.15376736 +0.11737831 +0.10075016 +0.08233922 +0.06747199 +0.07085256 +0.02962047 +0.00074974 +0.00000002 +0.00000000 +0.00000000 +0.15376736 +0.12794963 +0.10856596 +0.09412617 +0.07665995 +0.06706071 +0.06813314 +0.02153258 +0.00035406 +0.00000000 +0.00000000 +0.00000000 +0.11737831 +0.10856596 +0.09659286 +0.08277858 +0.07027104 +0.06616563 +0.06025065 +0.00972799 +0.00006690 +0.00000000 +0.00000000 +0.00000000 +0.10075016 +0.09412617 +0.08277858 +0.07309248 +0.06683084 +0.06847082 +0.03245148 +0.00225449 +0.00000332 +0.00000000 +0.00000000 +0.00000000 +0.08233922 +0.07665995 +0.07027104 +0.06683084 +0.06800883 +0.05087100 +0.00845280 +0.00018220 +0.00000002 +0.00000000 +0.00000000 +0.00000000 +0.06747199 +0.06706071 +0.06616563 +0.06847082 +0.05087100 +0.01246553 +0.00074932 +0.00000165 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.07085256 +0.06813314 +0.06025065 +0.03245148 +0.00845280 +0.00074932 +0.00000578 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.02962047 +0.02153258 +0.00972799 +0.00225449 +0.00018220 +0.00000165 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00074974 +0.00035406 +0.00006690 +0.00000332 +0.00000002 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000002 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.18791042 +0.13124374 +0.10329029 +0.08506177 +0.07372138 +0.07294920 +0.02962047 +0.00298788 +0.00000029 +0.00000000 +0.00000000 +0.00000000 +0.13124374 +0.11231419 +0.09405160 +0.07907574 +0.07183175 +0.07144920 +0.02153258 +0.00124650 +0.00000009 +0.00000000 +0.00000000 +0.00000000 +0.10329029 +0.09405160 +0.08275941 +0.07262965 +0.07119741 +0.05853347 +0.00972799 +0.00020339 +0.00000001 +0.00000000 +0.00000000 +0.00000000 +0.08506177 +0.07907574 +0.07262965 +0.07001552 +0.07014299 +0.02693209 +0.00225449 +0.00001003 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.07372138 +0.07183175 +0.07119741 +0.07014299 +0.03625927 +0.00582269 +0.00018220 +0.00000008 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.07294920 +0.07144920 +0.05853347 +0.02693209 +0.00582269 +0.00037585 +0.00000165 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.02962047 +0.02153258 +0.00972799 +0.00225449 +0.00018220 +0.00000165 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00298788 +0.00124650 +0.00020339 +0.00001003 +0.00000008 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000029 +0.00000009 +0.00000001 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.17026011 +0.11335403 +0.08427341 +0.08084794 +0.07780606 +0.01873712 +0.00074974 +0.00000029 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.11335403 +0.09231118 +0.08071688 +0.08021050 +0.06574932 +0.01193449 +0.00035406 +0.00000009 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.08427341 +0.08071688 +0.07860557 +0.07833357 +0.03692349 +0.00451546 +0.00006690 +0.00000001 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.08084794 +0.08021050 +0.07833357 +0.04925510 +0.01118626 +0.00080374 +0.00000332 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.07780606 +0.06574932 +0.03692349 +0.01118626 +0.00145810 +0.00003846 +0.00000002 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.01873712 +0.01193449 +0.00451546 +0.00080374 +0.00003846 +0.00000012 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00074974 +0.00035406 +0.00006690 +0.00000332 +0.00000002 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000029 +0.00000009 +0.00000001 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.12799510 +0.09892888 +0.09175639 +0.04458657 +0.00588790 +0.00009796 +0.00000002 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.09892888 +0.09251083 +0.08188588 +0.02804596 +0.00305855 +0.00003456 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.09175639 +0.08188588 +0.04355621 +0.00983444 +0.00072355 +0.00000354 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.04458657 +0.02804596 +0.00983444 +0.00151216 +0.00005207 +0.00000007 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00588790 +0.00305855 +0.00072355 +0.00005207 +0.00000040 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00009796 +0.00003456 +0.00000354 +0.00000007 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000002 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.15550975 +0.04855330 +0.00607623 +0.00024090 +0.00000049 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.04855330 +0.01764984 +0.00236939 +0.00007821 +0.00000013 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00607623 +0.00236939 +0.00030393 +0.00000628 +0.00000001 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00024090 +0.00007821 +0.00000628 +0.00000006 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000049 +0.00000013 +0.00000001 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00010527 +0.00002110 +0.00000013 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00002110 +0.00000355 +0.00000002 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000013 +0.00000002 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 + +SCALARS element_density float 1 +LOOKUP_TABLE default +0.01059896 +0.06028598 +0.18275629 +0.30874663 +0.44891561 +0.61765647 +0.83598131 +1.10711539 +1.54571030 +1.64889211 +6.74836785 +1.12321467 +0.06028598 +0.09454520 +0.20220073 +0.37708219 +0.56522550 +0.78305577 +1.05302032 +1.41794008 +1.78819213 +2.55789130 +3.34525938 +1.06814764 +0.18275629 +0.20220073 +0.28684617 +0.45987870 +0.72089895 +0.99970957 +1.34702602 +1.80964877 +2.13732651 +4.92673785 +1.71008559 +1.01698812 +0.30874663 +0.37708219 +0.45987870 +0.60254824 +0.85104435 +1.20884303 +1.64579808 +1.99289978 +3.09972265 +3.69851764 +1.18071033 +1.00161247 +0.44891561 +0.56522550 +0.72089895 +0.85104435 +1.07338231 +1.47642710 +1.91607685 +2.48729715 +5.02047567 +1.74250470 +1.02797014 +1.00004667 +0.61765647 +0.78305577 +0.99970957 +1.20884303 +1.47642710 +1.84572537 +2.19363884 +4.46617088 +2.56923277 +1.14090797 +1.00196071 +1.00000022 +0.83598131 +1.05302032 +1.34702602 +1.64579808 +1.91607685 +2.19363884 +3.52467473 +3.18444579 +1.30733317 +1.01151158 +1.00002255 +1.00000000 +1.10711539 +1.41794008 +1.80964877 +1.99289978 +2.48729715 +4.46617088 +3.18444579 +1.54735240 +1.02717675 +1.00020750 +1.00000002 +1.00000000 +1.54571030 +1.78819213 +2.13732651 +3.09972265 +5.02047567 +2.56923277 +1.30733317 +1.02717675 +1.00093673 +1.00000020 +1.00000000 +1.00000000 +1.64889211 +2.55789130 +4.92673785 +3.69851764 +1.74250470 +1.14090797 +1.01151158 +1.00020750 +1.00000020 +1.00000000 +1.00000000 +1.00000000 +6.74836785 +3.34525938 +1.71008559 +1.18071033 +1.02797014 +1.00196071 +1.00002255 +1.00000002 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.12321467 +1.06814764 +1.01698812 +1.00161247 +1.00004667 +1.00000022 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +0.06028598 +0.09454520 +0.20220073 +0.37708219 +0.56522550 +0.78305577 +1.05302032 +1.41794008 +1.78819213 +2.55789130 +3.34525938 +1.06814764 +0.09454520 +0.13707712 +0.25363317 +0.43653918 +0.65601811 +0.91535002 +1.22495668 +1.65556195 +1.96413896 +3.79775217 +2.24473568 +1.03810236 +0.20220073 +0.25363317 +0.35047190 +0.53841255 +0.78764869 +1.08616431 +1.46680475 +1.91035799 +2.37151402 +5.21745587 +1.45588968 +1.00962730 +0.37708219 +0.43653918 +0.53841255 +0.68630633 +0.95276250 +1.30058326 +1.74284788 +2.08683938 +3.68913863 +2.96038837 +1.12195513 +1.00091153 +0.56522550 +0.65601811 +0.78764869 +0.95276250 +1.18653545 +1.60397360 +1.96337513 +2.79495075 +4.78716176 +1.54063711 +1.01848518 +1.00002536 +0.78305577 +0.91535002 +1.08616431 +1.30058326 +1.60397360 +1.90461577 +2.40758071 +4.82522598 +2.20193738 +1.10012297 +1.00121571 +1.00000012 +1.05302032 +1.22495668 +1.46680475 +1.74284788 +1.96337513 +2.40758071 +4.16408894 +2.80143960 +1.23126187 +1.00773513 +1.00001270 +1.00000000 +1.41794008 +1.65556195 +1.91035799 +2.08683938 +2.79495075 +4.82522598 +2.80143960 +1.37979954 +1.01932031 +1.00012613 +1.00000001 +1.00000000 +1.78819213 +1.96413896 +2.37151402 +3.68913863 +4.78716176 +2.20193738 +1.23126187 +1.01932031 +1.00054429 +1.00000011 +1.00000000 +1.00000000 +2.55789130 +3.79775217 +5.21745587 +2.96038837 +1.54063711 +1.10012297 +1.00773513 +1.00012613 +1.00000011 +1.00000000 +1.00000000 +1.00000000 +3.34525938 +2.24473568 +1.45588968 +1.12195513 +1.01848518 +1.00121571 +1.00001270 +1.00000001 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.06814764 +1.03810236 +1.00962730 +1.00091153 +1.00002536 +1.00000012 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +0.18275629 +0.20220073 +0.28684617 +0.45987870 +0.72089895 +0.99970957 +1.34702602 +1.80964877 +2.13732651 +4.92673785 +1.71008559 +1.01698812 +0.20220073 +0.25363317 +0.35047190 +0.53841255 +0.78764869 +1.08616431 +1.46680475 +1.91035799 +2.37151402 +5.21745587 +1.45588968 +1.00962730 +0.28684617 +0.35047190 +0.48503526 +0.67607895 +0.92000432 +1.23670716 +1.67408651 +2.03637404 +3.17013000 +3.74258616 +1.19708047 +1.00259320 +0.45987870 +0.53841255 +0.67607895 +0.85444168 +1.11636939 +1.47792318 +1.89455570 +2.36028267 +4.85497505 +2.02148892 +1.05398342 +1.00025353 +0.72089895 +0.78764869 +0.92000432 +1.11636939 +1.39929604 +1.78482881 +2.11214051 +3.62383286 +3.59078365 +1.29006240 +1.00767405 +1.00000659 +0.99970957 +1.08616431 +1.23670716 +1.47792318 +1.78482881 +2.05001846 +3.03428814 +4.69553640 +1.70798913 +1.04930793 +1.00043503 +1.00000003 +1.34702602 +1.46680475 +1.67408651 +1.89455570 +2.11214051 +3.03428814 +4.69121168 +2.11845737 +1.12863248 +1.00334449 +1.00000367 +1.00000000 +1.80964877 +1.91035799 +2.03637404 +2.36028267 +3.62383286 +4.69553640 +2.11845737 +1.19206755 +1.00928909 +1.00004347 +1.00000000 +1.00000000 +2.13732651 +2.37151402 +3.17013000 +4.85497505 +3.59078365 +1.70798913 +1.12863248 +1.00928909 +1.00016737 +1.00000004 +1.00000000 +1.00000000 +4.92673785 +5.21745587 +3.74258616 +2.02148892 +1.29006240 +1.04930793 +1.00334449 +1.00004347 +1.00000004 +1.00000000 +1.00000000 +1.00000000 +1.71008559 +1.45588968 +1.19708047 +1.05398342 +1.00767405 +1.00043503 +1.00000367 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.01698812 +1.00962730 +1.00259320 +1.00025353 +1.00000659 +1.00000003 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +0.30874663 +0.37708219 +0.45987870 +0.60254824 +0.85104435 +1.20884303 +1.64579808 +1.99289978 +3.09972265 +3.69851764 +1.18071033 +1.00161247 +0.37708219 +0.43653918 +0.53841255 +0.68630633 +0.95276250 +1.30058326 +1.74284788 +2.08683938 +3.68913863 +2.96038837 +1.12195513 +1.00091153 +0.45987870 +0.53841255 +0.67607895 +0.85444168 +1.11636939 +1.47792318 +1.89455570 +2.36028267 +4.85497505 +2.02148892 +1.05398342 +1.00025353 +0.60254824 +0.68630633 +0.85444168 +1.06799788 +1.34884632 +1.72646359 +2.05676383 +3.23815309 +4.23041524 +1.39968435 +1.01389135 +1.00002607 +0.85104435 +0.95276250 +1.11636939 +1.34884632 +1.66271014 +1.95104280 +2.56601603 +4.89786332 +2.19501261 +1.11057793 +1.00176949 +1.00000061 +1.20884303 +1.30058326 +1.47792318 +1.72646359 +1.95104280 +2.43868827 +4.31104219 +3.18754429 +1.32061019 +1.01554183 +1.00007703 +1.00000000 +1.64579808 +1.74284788 +1.89455570 +2.05676383 +2.56601603 +4.31104219 +3.53444834 +1.52323853 +1.04972809 +1.00084205 +1.00000045 +1.00000000 +1.99289978 +2.08683938 +2.36028267 +3.23815309 +4.89786332 +3.18754429 +1.52323853 +1.07104992 +1.00268739 +1.00000730 +1.00000000 +1.00000000 +3.09972265 +3.68913863 +4.85497505 +4.23041524 +2.19501261 +1.32061019 +1.04972809 +1.00268739 +1.00002353 +1.00000001 +1.00000000 +1.00000000 +3.69851764 +2.96038837 +2.02148892 +1.39968435 +1.11057793 +1.01554183 +1.00084205 +1.00000730 +1.00000001 +1.00000000 +1.00000000 +1.00000000 +1.18071033 +1.12195513 +1.05398342 +1.01389135 +1.00176949 +1.00007703 +1.00000045 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00161247 +1.00091153 +1.00025353 +1.00002607 +1.00000061 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +0.44891561 +0.56522550 +0.72089895 +0.85104435 +1.07338231 +1.47642710 +1.91607685 +2.48729715 +5.02047567 +1.74250470 +1.02797014 +1.00004667 +0.56522550 +0.65601811 +0.78764869 +0.95276250 +1.18653545 +1.60397360 +1.96337513 +2.79495075 +4.78716176 +1.54063711 +1.01848518 +1.00002536 +0.72089895 +0.78764869 +0.92000432 +1.11636939 +1.39929604 +1.78482881 +2.11214051 +3.62383286 +3.59078365 +1.29006240 +1.00767405 +1.00000659 +0.85104435 +0.95276250 +1.11636939 +1.34884632 +1.66271014 +1.95104280 +2.56601603 +4.89786332 +2.19501261 +1.11057793 +1.00176949 +1.00000061 +1.07338231 +1.18653545 +1.39929604 +1.66271014 +1.90393974 +2.29248911 +3.94722752 +3.71408451 +1.41865170 +1.02533557 +1.00018095 +1.00000001 +1.47642710 +1.60397360 +1.78482881 +1.95104280 +2.29248911 +3.58481503 +4.46756253 +1.85074347 +1.10400191 +1.00273473 +1.00000498 +1.00000000 +1.91607685 +1.96337513 +2.11214051 +2.56601603 +3.94722752 +4.46756253 +2.05894465 +1.18392652 +1.01153195 +1.00010268 +1.00000002 +1.00000000 +2.48729715 +2.79495075 +3.62383286 +4.89786332 +3.71408451 +1.85074347 +1.18392652 +1.01718902 +1.00039108 +1.00000045 +1.00000000 +1.00000000 +5.02047567 +4.78716176 +3.59078365 +2.19501261 +1.41865170 +1.10400191 +1.01153195 +1.00039108 +1.00000128 +1.00000000 +1.00000000 +1.00000000 +1.74250470 +1.54063711 +1.29006240 +1.11057793 +1.02533557 +1.00273473 +1.00010268 +1.00000045 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.02797014 +1.01848518 +1.00767405 +1.00176949 +1.00018095 +1.00000498 +1.00000002 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00004667 +1.00002536 +1.00000659 +1.00000061 +1.00000001 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +0.61765647 +0.78305577 +0.99970957 +1.20884303 +1.47642710 +1.84572537 +2.19363884 +4.46617088 +2.56923277 +1.14090797 +1.00196071 +1.00000022 +0.78305577 +0.91535002 +1.08616431 +1.30058326 +1.60397360 +1.90461577 +2.40758071 +4.82522598 +2.20193738 +1.10012297 +1.00121571 +1.00000012 +0.99970957 +1.08616431 +1.23670716 +1.47792318 +1.78482881 +2.05001846 +3.03428814 +4.69553640 +1.70798913 +1.04930793 +1.00043503 +1.00000003 +1.20884303 +1.30058326 +1.47792318 +1.72646359 +1.95104280 +2.43868827 +4.31104219 +3.18754429 +1.32061019 +1.01554183 +1.00007703 +1.00000000 +1.47642710 +1.60397360 +1.78482881 +1.95104280 +2.29248911 +3.58481503 +4.46756253 +1.85074347 +1.10400191 +1.00273473 +1.00000498 +1.00000000 +1.84572537 +1.90461577 +2.05001846 +2.43868827 +3.58481503 +4.69561911 +2.34453450 +1.24335023 +1.01858146 +1.00021178 +1.00000007 +1.00000000 +2.19363884 +2.40758071 +3.03428814 +4.31104219 +4.46756253 +2.34453450 +1.32099117 +1.04035687 +1.00132267 +1.00000413 +1.00000000 +1.00000000 +4.46617088 +4.82522598 +4.69553640 +3.18754429 +1.85074347 +1.24335023 +1.04035687 +1.00224842 +1.00002182 +1.00000001 +1.00000000 +1.00000000 +2.56923277 +2.20193738 +1.70798913 +1.32061019 +1.10400191 +1.01858146 +1.00132267 +1.00002182 +1.00000002 +1.00000000 +1.00000000 +1.00000000 +1.14090797 +1.10012297 +1.04930793 +1.01554183 +1.00273473 +1.00021178 +1.00000413 +1.00000001 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00196071 +1.00121571 +1.00043503 +1.00007703 +1.00000498 +1.00000007 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000022 +1.00000012 +1.00000003 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +0.83598131 +1.05302032 +1.34702602 +1.64579808 +1.91607685 +2.19363884 +3.52467473 +3.18444579 +1.30733317 +1.01151158 +1.00002255 +1.00000000 +1.05302032 +1.22495668 +1.46680475 +1.74284788 +1.96337513 +2.40758071 +4.16408894 +2.80143960 +1.23126187 +1.00773513 +1.00001270 +1.00000000 +1.34702602 +1.46680475 +1.67408651 +1.89455570 +2.11214051 +3.03428814 +4.69121168 +2.11845737 +1.12863248 +1.00334449 +1.00000367 +1.00000000 +1.64579808 +1.74284788 +1.89455570 +2.05676383 +2.56601603 +4.31104219 +3.53444834 +1.52323853 +1.04972809 +1.00084205 +1.00000045 +1.00000000 +1.91607685 +1.96337513 +2.11214051 +2.56601603 +3.94722752 +4.46756253 +2.05894465 +1.18392652 +1.01153195 +1.00010268 +1.00000002 +1.00000000 +2.19363884 +2.40758071 +3.03428814 +4.31104219 +4.46756253 +2.34453450 +1.32099117 +1.04035687 +1.00132267 +1.00000413 +1.00000000 +1.00000000 +3.52467473 +4.16408894 +4.69121168 +3.53444834 +2.05894465 +1.32099117 +1.05868539 +1.00389046 +1.00005026 +1.00000003 +1.00000000 +1.00000000 +3.18444579 +2.80143960 +2.11845737 +1.52323853 +1.18392652 +1.04035687 +1.00389046 +1.00011001 +1.00000025 +1.00000000 +1.00000000 +1.00000000 +1.30733317 +1.23126187 +1.12863248 +1.04972809 +1.01153195 +1.00132267 +1.00005026 +1.00000025 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.01151158 +1.00773513 +1.00334449 +1.00084205 +1.00010268 +1.00000413 +1.00000003 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00002255 +1.00001270 +1.00000367 +1.00000045 +1.00000002 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.10711539 +1.41794008 +1.80964877 +1.99289978 +2.48729715 +4.46617088 +3.18444579 +1.54735240 +1.02717675 +1.00020750 +1.00000002 +1.00000000 +1.41794008 +1.65556195 +1.91035799 +2.08683938 +2.79495075 +4.82522598 +2.80143960 +1.37979954 +1.01932031 +1.00012613 +1.00000001 +1.00000000 +1.80964877 +1.91035799 +2.03637404 +2.36028267 +3.62383286 +4.69553640 +2.11845737 +1.19206755 +1.00928909 +1.00004347 +1.00000000 +1.00000000 +1.99289978 +2.08683938 +2.36028267 +3.23815309 +4.89786332 +3.18754429 +1.52323853 +1.07104992 +1.00268739 +1.00000730 +1.00000000 +1.00000000 +2.48729715 +2.79495075 +3.62383286 +4.89786332 +3.71408451 +1.85074347 +1.18392652 +1.01718902 +1.00039108 +1.00000045 +1.00000000 +1.00000000 +4.46617088 +4.82522598 +4.69553640 +3.18754429 +1.85074347 +1.24335023 +1.04035687 +1.00224842 +1.00002182 +1.00000001 +1.00000000 +1.00000000 +3.18444579 +2.80143960 +2.11845737 +1.52323853 +1.18392652 +1.04035687 +1.00389046 +1.00011001 +1.00000025 +1.00000000 +1.00000000 +1.00000000 +1.54735240 +1.37979954 +1.19206755 +1.07104992 +1.01718902 +1.00224842 +1.00011001 +1.00000075 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.02717675 +1.01932031 +1.00928909 +1.00268739 +1.00039108 +1.00002182 +1.00000025 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00020750 +1.00012613 +1.00004347 +1.00000730 +1.00000045 +1.00000001 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000002 +1.00000001 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.54571030 +1.78819213 +2.13732651 +3.09972265 +5.02047567 +2.56923277 +1.30733317 +1.02717675 +1.00093673 +1.00000020 +1.00000000 +1.00000000 +1.78819213 +1.96413896 +2.37151402 +3.68913863 +4.78716176 +2.20193738 +1.23126187 +1.01932031 +1.00054429 +1.00000011 +1.00000000 +1.00000000 +2.13732651 +2.37151402 +3.17013000 +4.85497505 +3.59078365 +1.70798913 +1.12863248 +1.00928909 +1.00016737 +1.00000004 +1.00000000 +1.00000000 +3.09972265 +3.68913863 +4.85497505 +4.23041524 +2.19501261 +1.32061019 +1.04972809 +1.00268739 +1.00002353 +1.00000001 +1.00000000 +1.00000000 +5.02047567 +4.78716176 +3.59078365 +2.19501261 +1.41865170 +1.10400191 +1.01153195 +1.00039108 +1.00000128 +1.00000000 +1.00000000 +1.00000000 +2.56923277 +2.20193738 +1.70798913 +1.32061019 +1.10400191 +1.01858146 +1.00132267 +1.00002182 +1.00000002 +1.00000000 +1.00000000 +1.00000000 +1.30733317 +1.23126187 +1.12863248 +1.04972809 +1.01153195 +1.00132267 +1.00005026 +1.00000025 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.02717675 +1.01932031 +1.00928909 +1.00268739 +1.00039108 +1.00002182 +1.00000025 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00093673 +1.00054429 +1.00016737 +1.00002353 +1.00000128 +1.00000002 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000020 +1.00000011 +1.00000004 +1.00000001 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.64889211 +2.55789130 +4.92673785 +3.69851764 +1.74250470 +1.14090797 +1.01151158 +1.00020750 +1.00000020 +1.00000000 +1.00000000 +1.00000000 +2.55789130 +3.79775217 +5.21745587 +2.96038837 +1.54063711 +1.10012297 +1.00773513 +1.00012613 +1.00000011 +1.00000000 +1.00000000 +1.00000000 +4.92673785 +5.21745587 +3.74258616 +2.02148892 +1.29006240 +1.04930793 +1.00334449 +1.00004347 +1.00000004 +1.00000000 +1.00000000 +1.00000000 +3.69851764 +2.96038837 +2.02148892 +1.39968435 +1.11057793 +1.01554183 +1.00084205 +1.00000730 +1.00000001 +1.00000000 +1.00000000 +1.00000000 +1.74250470 +1.54063711 +1.29006240 +1.11057793 +1.02533557 +1.00273473 +1.00010268 +1.00000045 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.14090797 +1.10012297 +1.04930793 +1.01554183 +1.00273473 +1.00021178 +1.00000413 +1.00000001 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.01151158 +1.00773513 +1.00334449 +1.00084205 +1.00010268 +1.00000413 +1.00000003 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00020750 +1.00012613 +1.00004347 +1.00000730 +1.00000045 +1.00000001 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000020 +1.00000011 +1.00000004 +1.00000001 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +6.74836785 +3.34525938 +1.71008559 +1.18071033 +1.02797014 +1.00196071 +1.00002255 +1.00000002 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +3.34525938 +2.24473568 +1.45588968 +1.12195513 +1.01848518 +1.00121571 +1.00001270 +1.00000001 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.71008559 +1.45588968 +1.19708047 +1.05398342 +1.00767405 +1.00043503 +1.00000367 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.18071033 +1.12195513 +1.05398342 +1.01389135 +1.00176949 +1.00007703 +1.00000045 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.02797014 +1.01848518 +1.00767405 +1.00176949 +1.00018095 +1.00000498 +1.00000002 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00196071 +1.00121571 +1.00043503 +1.00007703 +1.00000498 +1.00000007 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00002255 +1.00001270 +1.00000367 +1.00000045 +1.00000002 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000002 +1.00000001 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.12321467 +1.06814764 +1.01698812 +1.00161247 +1.00004667 +1.00000022 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.06814764 +1.03810236 +1.00962730 +1.00091153 +1.00002536 +1.00000012 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.01698812 +1.00962730 +1.00259320 +1.00025353 +1.00000659 +1.00000003 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00161247 +1.00091153 +1.00025353 +1.00002607 +1.00000061 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00004667 +1.00002536 +1.00000659 +1.00000061 +1.00000001 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000022 +1.00000012 +1.00000003 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 + +SCALARS mass float 1 +LOOKUP_TABLE default +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 + +SCALARS pressure float 1 +LOOKUP_TABLE default +0.08457759 +0.07837191 +0.08292159 +0.08784750 +0.09563914 +0.10658444 +0.12079947 +0.13869234 +0.17544854 +0.14070008 +0.69962468 +0.00007882 +0.07837191 +0.08100683 +0.07523654 +0.08115595 +0.08749239 +0.09673615 +0.10794677 +0.12406384 +0.13513252 +0.16869955 +0.10828226 +0.00001502 +0.08292159 +0.07523654 +0.08123854 +0.07586113 +0.08457621 +0.09301326 +0.10540776 +0.12461277 +0.12007987 +0.30137312 +0.00692725 +0.00000009 +0.08784750 +0.08115595 +0.07586113 +0.08496564 +0.08162346 +0.09297477 +0.11054294 +0.11301305 +0.16707078 +0.10993614 +0.00018963 +0.00000000 +0.09563914 +0.08749239 +0.08457621 +0.08162346 +0.09161912 +0.09568263 +0.10517885 +0.12224465 +0.26041562 +0.00683980 +0.00000034 +0.00000000 +0.10658444 +0.09673615 +0.09301326 +0.09297477 +0.09568263 +0.11030559 +0.09867278 +0.21720238 +0.03209334 +0.00007451 +0.00000000 +0.00000000 +0.12079947 +0.10794677 +0.10540776 +0.11054294 +0.10517885 +0.09867278 +0.16648816 +0.06288320 +0.00065344 +0.00000001 +0.00000000 +0.00000000 +0.13869234 +0.12406384 +0.12461277 +0.11301305 +0.12224465 +0.21720238 +0.06288320 +0.00308220 +0.00000020 +0.00000000 +0.00000000 +0.00000000 +0.17544854 +0.13513252 +0.12007987 +0.16707078 +0.26041562 +0.03209334 +0.00065344 +0.00000020 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.14070008 +0.16869955 +0.30137312 +0.10993614 +0.00683980 +0.00007451 +0.00000001 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.69962468 +0.10828226 +0.00692725 +0.00018963 +0.00000034 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00007882 +0.00001502 +0.00000009 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.07837191 +0.08100683 +0.07523654 +0.08115595 +0.08749239 +0.09673615 +0.10794677 +0.12406384 +0.13513252 +0.16869955 +0.10828226 +0.00001502 +0.08100683 +0.07293218 +0.07783931 +0.07752159 +0.08499539 +0.09356641 +0.10448850 +0.12396206 +0.12087465 +0.23422214 +0.02641282 +0.00000246 +0.07523654 +0.07783931 +0.07439134 +0.07907836 +0.08255778 +0.09199674 +0.10616337 +0.11978148 +0.12761413 +0.28482399 +0.00229972 +0.00000001 +0.08115595 +0.07752159 +0.07907836 +0.07869583 +0.08442278 +0.09188843 +0.10936506 +0.11001225 +0.19727178 +0.05535130 +0.00005850 +0.00000000 +0.08749239 +0.08499539 +0.08255778 +0.08442278 +0.08709285 +0.09998841 +0.10034150 +0.13384413 +0.20983509 +0.00314141 +0.00000009 +0.00000000 +0.09673615 +0.09356641 +0.09199674 +0.09188843 +0.09998841 +0.09872292 +0.10763605 +0.22983901 +0.01751933 +0.00002535 +0.00000000 +0.00000000 +0.10794677 +0.10448850 +0.10616337 +0.10936506 +0.10034150 +0.10763605 +0.18914162 +0.04021482 +0.00029063 +0.00000000 +0.00000000 +0.00000000 +0.12406384 +0.12396206 +0.11978148 +0.11001225 +0.13384413 +0.22983901 +0.04021482 +0.00114662 +0.00000006 +0.00000000 +0.00000000 +0.00000000 +0.13513252 +0.12087465 +0.12761413 +0.19727178 +0.20983509 +0.01751933 +0.00029063 +0.00000006 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.16869955 +0.23422214 +0.28482399 +0.05535130 +0.00314141 +0.00002535 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.10828226 +0.02641282 +0.00229972 +0.00005850 +0.00000009 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00001502 +0.00000246 +0.00000001 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.08292159 +0.07523654 +0.08123854 +0.07586113 +0.08457621 +0.09301326 +0.10540776 +0.12461277 +0.12007987 +0.30137312 +0.00692725 +0.00000009 +0.07523654 +0.07783931 +0.07439134 +0.07907836 +0.08255778 +0.09199674 +0.10616337 +0.11978148 +0.12761413 +0.28482399 +0.00229972 +0.00000001 +0.08123854 +0.07439134 +0.07419282 +0.07899819 +0.08424496 +0.09138784 +0.10780320 +0.11235274 +0.16612659 +0.10867524 +0.00024256 +0.00000000 +0.07586113 +0.07907836 +0.07899819 +0.07854948 +0.08600151 +0.09580539 +0.10455243 +0.11428434 +0.25353834 +0.01325347 +0.00000441 +0.00000000 +0.08457621 +0.08255778 +0.08424496 +0.08600151 +0.08886359 +0.09968564 +0.09894821 +0.17200500 +0.08838950 +0.00062229 +0.00000000 +0.00000000 +0.09301326 +0.09199674 +0.09138784 +0.09580539 +0.09968564 +0.09318184 +0.13384373 +0.18323069 +0.00514157 +0.00000248 +0.00000000 +0.00000000 +0.10540776 +0.10616337 +0.10780320 +0.10455243 +0.09894821 +0.13384373 +0.18843237 +0.01373889 +0.00005033 +0.00000000 +0.00000000 +0.00000000 +0.12461277 +0.11978148 +0.11235274 +0.11428434 +0.17200500 +0.18323069 +0.01373889 +0.00016164 +0.00000001 +0.00000000 +0.00000000 +0.00000000 +0.12007987 +0.12761413 +0.16612659 +0.25353834 +0.08838950 +0.00514157 +0.00005033 +0.00000001 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.30137312 +0.28482399 +0.10867524 +0.01325347 +0.00062229 +0.00000248 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00692725 +0.00229972 +0.00024256 +0.00000441 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000009 +0.00000001 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.08784750 +0.08115595 +0.07586113 +0.08496564 +0.08162346 +0.09297477 +0.11054294 +0.11301305 +0.16707078 +0.10993614 +0.00018963 +0.00000000 +0.08115595 +0.07752159 +0.07907836 +0.07869583 +0.08442278 +0.09188843 +0.10936506 +0.11001225 +0.19727178 +0.05535130 +0.00005850 +0.00000000 +0.07586113 +0.07907836 +0.07899819 +0.07854948 +0.08600151 +0.09580539 +0.10455243 +0.11428434 +0.25353834 +0.01325347 +0.00000441 +0.00000000 +0.08496564 +0.07869583 +0.07854948 +0.08422384 +0.08823131 +0.09961837 +0.10022264 +0.15114731 +0.13891303 +0.00141103 +0.00000004 +0.00000000 +0.08162346 +0.08442278 +0.08600151 +0.08823131 +0.09702565 +0.09546706 +0.11432600 +0.22903386 +0.01636932 +0.00003855 +0.00000000 +0.00000000 +0.09297477 +0.09188843 +0.09580539 +0.09961837 +0.09546706 +0.10662170 +0.19678706 +0.05723148 +0.00070762 +0.00000005 +0.00000000 +0.00000000 +0.11054294 +0.10936506 +0.10455243 +0.10022264 +0.11432600 +0.19678706 +0.07646540 +0.00228942 +0.00000232 +0.00000000 +0.00000000 +0.00000000 +0.11301305 +0.11001225 +0.11428434 +0.15114731 +0.22903386 +0.05723148 +0.00228942 +0.00000716 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.16707078 +0.19727178 +0.25353834 +0.13891303 +0.01636932 +0.00070762 +0.00000232 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.10993614 +0.05535130 +0.01325347 +0.00141103 +0.00003855 +0.00000005 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00018963 +0.00005850 +0.00000441 +0.00000004 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.09563914 +0.08749239 +0.08457621 +0.08162346 +0.09161912 +0.09568263 +0.10517885 +0.12224465 +0.26041562 +0.00683980 +0.00000034 +0.00000000 +0.08749239 +0.08499539 +0.08255778 +0.08442278 +0.08709285 +0.09998841 +0.10034150 +0.13384413 +0.20983509 +0.00314141 +0.00000009 +0.00000000 +0.08457621 +0.08255778 +0.08424496 +0.08600151 +0.08886359 +0.09968564 +0.09894821 +0.17200500 +0.08838950 +0.00062229 +0.00000000 +0.00000000 +0.08162346 +0.08442278 +0.08600151 +0.08823131 +0.09702565 +0.09546706 +0.11432600 +0.22903386 +0.01636932 +0.00003855 +0.00000000 +0.00000000 +0.09161912 +0.08709285 +0.08886359 +0.09702565 +0.09583979 +0.10194868 +0.17896421 +0.08978000 +0.00137903 +0.00000027 +0.00000000 +0.00000000 +0.09568263 +0.09998841 +0.09968564 +0.09546706 +0.10194868 +0.16204816 +0.15151291 +0.00718420 +0.00002830 +0.00000000 +0.00000000 +0.00000000 +0.10517885 +0.10034150 +0.09894821 +0.11432600 +0.17896421 +0.15151291 +0.01160257 +0.00014381 +0.00000001 +0.00000000 +0.00000000 +0.00000000 +0.12224465 +0.13384413 +0.17200500 +0.22903386 +0.08978000 +0.00718420 +0.00014381 +0.00000005 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.26041562 +0.20983509 +0.08838950 +0.01636932 +0.00137903 +0.00002830 +0.00000001 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00683980 +0.00314141 +0.00062229 +0.00003855 +0.00000027 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000034 +0.00000009 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.10658444 +0.09673615 +0.09301326 +0.09297477 +0.09568263 +0.11030559 +0.09867278 +0.21720238 +0.03209334 +0.00007451 +0.00000000 +0.00000000 +0.09673615 +0.09356641 +0.09199674 +0.09188843 +0.09998841 +0.09872292 +0.10763605 +0.22983901 +0.01751933 +0.00002535 +0.00000000 +0.00000000 +0.09301326 +0.09199674 +0.09138784 +0.09580539 +0.09968564 +0.09318184 +0.13384373 +0.18323069 +0.00514157 +0.00000248 +0.00000000 +0.00000000 +0.09297477 +0.09188843 +0.09580539 +0.09961837 +0.09546706 +0.10662170 +0.19678706 +0.05723148 +0.00070762 +0.00000005 +0.00000000 +0.00000000 +0.09568263 +0.09998841 +0.09968564 +0.09546706 +0.10194868 +0.16204816 +0.15151291 +0.00718420 +0.00002830 +0.00000000 +0.00000000 +0.00000000 +0.11030559 +0.09872292 +0.09318184 +0.10662170 +0.16204816 +0.18080410 +0.01948391 +0.00031154 +0.00000008 +0.00000000 +0.00000000 +0.00000000 +0.09867278 +0.10763605 +0.13384373 +0.19678706 +0.15151291 +0.01948391 +0.00065989 +0.00000114 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.21720238 +0.22983901 +0.18323069 +0.05723148 +0.00718420 +0.00031154 +0.00000114 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.03209334 +0.01751933 +0.00514157 +0.00070762 +0.00002830 +0.00000008 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00007451 +0.00002535 +0.00000248 +0.00000005 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.12079947 +0.10794677 +0.10540776 +0.11054294 +0.10517885 +0.09867278 +0.16648816 +0.06288320 +0.00065344 +0.00000001 +0.00000000 +0.00000000 +0.10794677 +0.10448850 +0.10616337 +0.10936506 +0.10034150 +0.10763605 +0.18914162 +0.04021482 +0.00029063 +0.00000000 +0.00000000 +0.00000000 +0.10540776 +0.10616337 +0.10780320 +0.10455243 +0.09894821 +0.13384373 +0.18843237 +0.01373889 +0.00005033 +0.00000000 +0.00000000 +0.00000000 +0.11054294 +0.10936506 +0.10455243 +0.10022264 +0.11432600 +0.19678706 +0.07646540 +0.00228942 +0.00000232 +0.00000000 +0.00000000 +0.00000000 +0.10517885 +0.10034150 +0.09894821 +0.11432600 +0.17896421 +0.15151291 +0.01160257 +0.00014381 +0.00000001 +0.00000000 +0.00000000 +0.00000000 +0.09867278 +0.10763605 +0.13384373 +0.19678706 +0.15151291 +0.01948391 +0.00065989 +0.00000114 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.16648816 +0.18914162 +0.18843237 +0.07646540 +0.01160257 +0.00065989 +0.00000408 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.06288320 +0.04021482 +0.01373889 +0.00228942 +0.00014381 +0.00000114 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00065344 +0.00029063 +0.00005033 +0.00000232 +0.00000001 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000001 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.13869234 +0.12406384 +0.12461277 +0.11301305 +0.12224465 +0.21720238 +0.06288320 +0.00308220 +0.00000020 +0.00000000 +0.00000000 +0.00000000 +0.12406384 +0.12396206 +0.11978148 +0.11001225 +0.13384413 +0.22983901 +0.04021482 +0.00114662 +0.00000006 +0.00000000 +0.00000000 +0.00000000 +0.12461277 +0.11978148 +0.11235274 +0.11428434 +0.17200500 +0.18323069 +0.01373889 +0.00016164 +0.00000001 +0.00000000 +0.00000000 +0.00000000 +0.11301305 +0.11001225 +0.11428434 +0.15114731 +0.22903386 +0.05723148 +0.00228942 +0.00000716 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.12224465 +0.13384413 +0.17200500 +0.22903386 +0.08978000 +0.00718420 +0.00014381 +0.00000005 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.21720238 +0.22983901 +0.18323069 +0.05723148 +0.00718420 +0.00031154 +0.00000114 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.06288320 +0.04021482 +0.01373889 +0.00228942 +0.00014381 +0.00000114 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00308220 +0.00114662 +0.00016164 +0.00000716 +0.00000005 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000020 +0.00000006 +0.00000001 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.17544854 +0.13513252 +0.12007987 +0.16707078 +0.26041562 +0.03209334 +0.00065344 +0.00000020 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.13513252 +0.12087465 +0.12761413 +0.19727178 +0.20983509 +0.01751933 +0.00029063 +0.00000006 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.12007987 +0.12761413 +0.16612659 +0.25353834 +0.08838950 +0.00514157 +0.00005033 +0.00000001 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.16707078 +0.19727178 +0.25353834 +0.13891303 +0.01636932 +0.00070762 +0.00000232 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.26041562 +0.20983509 +0.08838950 +0.01636932 +0.00137903 +0.00002830 +0.00000001 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.03209334 +0.01751933 +0.00514157 +0.00070762 +0.00002830 +0.00000008 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00065344 +0.00029063 +0.00005033 +0.00000232 +0.00000001 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000020 +0.00000006 +0.00000001 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.14070008 +0.16869955 +0.30137312 +0.10993614 +0.00683980 +0.00007451 +0.00000001 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.16869955 +0.23422214 +0.28482399 +0.05535130 +0.00314141 +0.00002535 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.30137312 +0.28482399 +0.10867524 +0.01325347 +0.00062229 +0.00000248 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.10993614 +0.05535130 +0.01325347 +0.00141103 +0.00003855 +0.00000005 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00683980 +0.00314141 +0.00062229 +0.00003855 +0.00000027 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00007451 +0.00002535 +0.00000248 +0.00000005 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000001 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.69962468 +0.10828226 +0.00692725 +0.00018963 +0.00000034 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.10828226 +0.02641282 +0.00229972 +0.00005850 +0.00000009 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00692725 +0.00229972 +0.00024256 +0.00000441 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00018963 +0.00005850 +0.00000441 +0.00000004 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000034 +0.00000009 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00007882 +0.00001502 +0.00000009 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00001502 +0.00000246 +0.00000001 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000009 +0.00000001 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 + +SCALARS sound_speed float 1 +LOOKUP_TABLE default +3.64687128 +1.47196145 +0.86960501 +0.68863321 +0.59588143 +0.53628762 +0.49074807 +0.45693474 +0.43494586 +0.37711640 +0.41567850 +0.01081493 +1.47196145 +1.19499322 +0.78749431 +0.59891687 +0.50792389 +0.45375605 +0.41334323 +0.38187220 +0.35489283 +0.33154333 +0.23226734 +0.00484158 +0.86960501 +0.78749431 +0.68703821 +0.52433926 +0.44219243 +0.39378564 +0.36113757 +0.33877278 +0.30600184 +0.31929852 +0.08216671 +0.00037403 +0.68863321 +0.59891687 +0.52433926 +0.48478608 +0.39981197 +0.35803210 +0.33458126 +0.30742979 +0.29971827 +0.22257725 +0.01636066 +0.00001481 +0.59588143 +0.50792389 +0.44219243 +0.39981197 +0.37717268 +0.32865098 +0.30246987 +0.28620385 +0.29402581 +0.08088334 +0.00073983 +0.00001055 +0.53628762 +0.45375605 +0.39378564 +0.35803210 +0.32865098 +0.31560188 +0.27380444 +0.28470101 +0.14428797 +0.01043263 +0.00001580 +0.00001054 +0.49074807 +0.41334323 +0.36113757 +0.33458126 +0.30246987 +0.27380444 +0.28057988 +0.18141565 +0.02886257 +0.00013649 +0.00001054 +0.00001054 +0.45693474 +0.38187220 +0.33877278 +0.30742979 +0.28620385 +0.28470101 +0.18141565 +0.05761829 +0.00057044 +0.00001061 +0.00001054 +0.00001054 +0.43494586 +0.35489283 +0.30600184 +0.29971827 +0.29402581 +0.14428797 +0.02886257 +0.00057044 +0.00001166 +0.00001054 +0.00001054 +0.00001054 +0.37711640 +0.33154333 +0.31929852 +0.22257725 +0.08088334 +0.01043263 +0.00013649 +0.00001061 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.41567850 +0.23226734 +0.08216671 +0.01636066 +0.00073983 +0.00001580 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.01081493 +0.00484158 +0.00037403 +0.00001481 +0.00001055 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +1.47196145 +1.19499322 +0.78749431 +0.59891687 +0.50792389 +0.45375605 +0.41334323 +0.38187220 +0.35489283 +0.33154333 +0.23226734 +0.00484158 +1.19499322 +0.94167592 +0.71518902 +0.54403158 +0.46469088 +0.41275350 +0.37704941 +0.35326129 +0.32026235 +0.32060850 +0.14003905 +0.00198558 +0.78749431 +0.71518902 +0.59478358 +0.49476117 +0.41796229 +0.37571871 +0.34731663 +0.32326735 +0.29947524 +0.30163606 +0.05130944 +0.00015391 +0.59891687 +0.54403158 +0.49476117 +0.43716082 +0.38429248 +0.34315135 +0.32339547 +0.29641515 +0.29853439 +0.17652814 +0.00932219 +0.00001221 +0.50792389 +0.46469088 +0.41796229 +0.38429248 +0.34976413 +0.32232991 +0.29185223 +0.28251204 +0.27028652 +0.05829573 +0.00037442 +0.00001054 +0.45375605 +0.41275350 +0.37571871 +0.34315135 +0.32232991 +0.29392036 +0.27296868 +0.28175875 +0.11515441 +0.00619698 +0.00001289 +0.00001054 +0.41334323 +0.37704941 +0.34731663 +0.32339547 +0.29185223 +0.27296868 +0.27514266 +0.15467737 +0.01983436 +0.00007437 +0.00001054 +0.00001054 +0.38187220 +0.35326129 +0.32326735 +0.29641515 +0.28251204 +0.28175875 +0.15467737 +0.03721566 +0.00032433 +0.00001057 +0.00001054 +0.00001054 +0.35489283 +0.32026235 +0.29947524 +0.29853439 +0.27028652 +0.11515441 +0.01983436 +0.00032433 +0.00001099 +0.00001054 +0.00001054 +0.00001054 +0.33154333 +0.32060850 +0.30163606 +0.17652814 +0.05829573 +0.00619698 +0.00007437 +0.00001057 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.23226734 +0.14003905 +0.05130944 +0.00932219 +0.00037442 +0.00001289 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00484158 +0.00198558 +0.00015391 +0.00001221 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.86960501 +0.78749431 +0.68703821 +0.52433926 +0.44219243 +0.39378564 +0.36113757 +0.33877278 +0.30600184 +0.31929852 +0.08216671 +0.00037403 +0.78749431 +0.71518902 +0.59478358 +0.49476117 +0.41796229 +0.37571871 +0.34731663 +0.32326735 +0.29947524 +0.30163606 +0.05130944 +0.00015391 +0.68703821 +0.59478358 +0.50491546 +0.44130032 +0.39066221 +0.35094184 +0.32760556 +0.30324065 +0.29553262 +0.21999042 +0.01837672 +0.00002368 +0.52433926 +0.49476117 +0.44130032 +0.39143064 +0.35832198 +0.32869523 +0.30327579 +0.28407677 +0.29502084 +0.10453302 +0.00264075 +0.00001071 +0.44219243 +0.41796229 +0.39066221 +0.35832198 +0.32533553 +0.30510013 +0.27942608 +0.28126185 +0.20254900 +0.02835399 +0.00008876 +0.00001054 +0.39378564 +0.37571871 +0.35094184 +0.32869523 +0.30510013 +0.27523973 +0.27114086 +0.25502390 +0.07083202 +0.00198369 +0.00001092 +0.00001054 +0.36113757 +0.34731663 +0.32760556 +0.30327579 +0.27942608 +0.27114086 +0.25873764 +0.10396576 +0.00862150 +0.00002511 +0.00001054 +0.00001054 +0.33877278 +0.32326735 +0.30324065 +0.28407677 +0.28126185 +0.25502390 +0.10396576 +0.01503287 +0.00009890 +0.00001055 +0.00001054 +0.00001054 +0.30600184 +0.29947524 +0.29553262 +0.29502084 +0.20254900 +0.07083202 +0.00862150 +0.00009890 +0.00001059 +0.00001054 +0.00001054 +0.00001054 +0.31929852 +0.30163606 +0.21999042 +0.10453302 +0.02835399 +0.00198369 +0.00002511 +0.00001055 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.08216671 +0.05130944 +0.01837672 +0.00264075 +0.00008876 +0.00001092 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00037403 +0.00015391 +0.00002368 +0.00001071 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.68863321 +0.59891687 +0.52433926 +0.48478608 +0.39981197 +0.35803210 +0.33458126 +0.30742979 +0.29971827 +0.22257725 +0.01636066 +0.00001481 +0.59891687 +0.54403158 +0.49476117 +0.43716082 +0.38429248 +0.34315135 +0.32339547 +0.29641515 +0.29853439 +0.17652814 +0.00932219 +0.00001221 +0.52433926 +0.49476117 +0.44130032 +0.39143064 +0.35832198 +0.32869523 +0.30327579 +0.28407677 +0.29502084 +0.10453302 +0.00264075 +0.00001071 +0.48478608 +0.43716082 +0.39143064 +0.36254065 +0.33018285 +0.31010972 +0.28498046 +0.27891759 +0.23393994 +0.04098991 +0.00025368 +0.00001054 +0.39981197 +0.38429248 +0.35832198 +0.33018285 +0.31185979 +0.28557340 +0.27250044 +0.27917138 +0.11148623 +0.00760629 +0.00001593 +0.00001054 +0.35803210 +0.34315135 +0.32869523 +0.31010972 +0.28557340 +0.26994111 +0.27582365 +0.17298712 +0.02988380 +0.00027286 +0.00001056 +0.00001054 +0.33458126 +0.32339547 +0.30327579 +0.28498046 +0.27250044 +0.27582365 +0.18988735 +0.05004989 +0.00191934 +0.00001171 +0.00001054 +0.00001054 +0.30742979 +0.29641515 +0.28407677 +0.27891759 +0.27917138 +0.17298712 +0.05004989 +0.00333814 +0.00001997 +0.00001054 +0.00001054 +0.00001054 +0.29971827 +0.29853439 +0.29502084 +0.23393994 +0.11148623 +0.02988380 +0.00191934 +0.00001997 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.22257725 +0.17652814 +0.10453302 +0.04098991 +0.00760629 +0.00027286 +0.00001171 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.01636066 +0.00932219 +0.00264075 +0.00025368 +0.00001593 +0.00001056 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001481 +0.00001221 +0.00001071 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.59588143 +0.50792389 +0.44219243 +0.39981197 +0.37717268 +0.32865098 +0.30246987 +0.28620385 +0.29402581 +0.08088334 +0.00073983 +0.00001055 +0.50792389 +0.46469088 +0.41796229 +0.38429248 +0.34976413 +0.32232991 +0.29185223 +0.28251204 +0.27028652 +0.05829573 +0.00037442 +0.00001054 +0.44219243 +0.41796229 +0.39066221 +0.35832198 +0.32533553 +0.30510013 +0.27942608 +0.28126185 +0.20254900 +0.02835399 +0.00008876 +0.00001054 +0.39981197 +0.38429248 +0.35832198 +0.33018285 +0.31185979 +0.28557340 +0.27250044 +0.27917138 +0.11148623 +0.00760629 +0.00001593 +0.00001054 +0.37717268 +0.34976413 +0.32533553 +0.31185979 +0.28964811 +0.27224602 +0.27489155 +0.20071891 +0.04025066 +0.00066411 +0.00001062 +0.00001054 +0.32865098 +0.32232991 +0.30510013 +0.28557340 +0.27224602 +0.27448148 +0.23774636 +0.08043416 +0.00653668 +0.00002130 +0.00001054 +0.00001054 +0.30246987 +0.29185223 +0.27942608 +0.27250044 +0.27489155 +0.23774636 +0.09691234 +0.01422822 +0.00015379 +0.00001056 +0.00001054 +0.00001054 +0.28620385 +0.28251204 +0.28126185 +0.27917138 +0.20071891 +0.08043416 +0.01422822 +0.00029793 +0.00001080 +0.00001054 +0.00001054 +0.00001054 +0.29402581 +0.27028652 +0.20254900 +0.11148623 +0.04025066 +0.00653668 +0.00015379 +0.00001080 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.08088334 +0.05829573 +0.02835399 +0.00760629 +0.00066411 +0.00002130 +0.00001056 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00073983 +0.00037442 +0.00008876 +0.00001593 +0.00001062 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001055 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.53628762 +0.45375605 +0.39378564 +0.35803210 +0.32865098 +0.31560188 +0.27380444 +0.28470101 +0.14428797 +0.01043263 +0.00001580 +0.00001054 +0.45375605 +0.41275350 +0.37571871 +0.34315135 +0.32232991 +0.29392036 +0.27296868 +0.28175875 +0.11515441 +0.00619698 +0.00001289 +0.00001054 +0.39378564 +0.37571871 +0.35094184 +0.32869523 +0.30510013 +0.27523973 +0.27114086 +0.25502390 +0.07083202 +0.00198369 +0.00001092 +0.00001054 +0.35803210 +0.34315135 +0.32869523 +0.31010972 +0.28557340 +0.26994111 +0.27582365 +0.17298712 +0.02988380 +0.00027286 +0.00001056 +0.00001054 +0.32865098 +0.32232991 +0.30510013 +0.28557340 +0.27224602 +0.27448148 +0.23774636 +0.08043416 +0.00653668 +0.00002130 +0.00001054 +0.00001054 +0.31560188 +0.29392036 +0.27523973 +0.26994111 +0.27448148 +0.25332735 +0.11768853 +0.02043551 +0.00035919 +0.00001063 +0.00001054 +0.00001054 +0.27380444 +0.27296868 +0.27114086 +0.27582365 +0.23774636 +0.11768853 +0.02885434 +0.00135211 +0.00001308 +0.00001054 +0.00001054 +0.00001054 +0.28470101 +0.28175875 +0.25502390 +0.17298712 +0.08043416 +0.02043551 +0.00135211 +0.00001727 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.14428797 +0.11515441 +0.07083202 +0.02988380 +0.00653668 +0.00035919 +0.00001308 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.01043263 +0.00619698 +0.00198369 +0.00027286 +0.00002130 +0.00001063 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001580 +0.00001289 +0.00001092 +0.00001056 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.49074807 +0.41334323 +0.36113757 +0.33458126 +0.30246987 +0.27380444 +0.28057988 +0.18141565 +0.02886257 +0.00013649 +0.00001054 +0.00001054 +0.41334323 +0.37704941 +0.34731663 +0.32339547 +0.29185223 +0.27296868 +0.27514266 +0.15467737 +0.01983436 +0.00007437 +0.00001054 +0.00001054 +0.36113757 +0.34731663 +0.32760556 +0.30327579 +0.27942608 +0.27114086 +0.25873764 +0.10396576 +0.00862150 +0.00002511 +0.00001054 +0.00001054 +0.33458126 +0.32339547 +0.30327579 +0.28498046 +0.27250044 +0.27582365 +0.18988735 +0.05004989 +0.00191934 +0.00001171 +0.00001054 +0.00001054 +0.30246987 +0.29185223 +0.27942608 +0.27250044 +0.27489155 +0.23774636 +0.09691234 +0.01422822 +0.00015379 +0.00001056 +0.00001054 +0.00001054 +0.27380444 +0.27296868 +0.27114086 +0.27582365 +0.23774636 +0.11768853 +0.02885434 +0.00135211 +0.00001308 +0.00001054 +0.00001054 +0.00001054 +0.28057988 +0.27514266 +0.25873764 +0.18988735 +0.09691234 +0.02885434 +0.00253416 +0.00002930 +0.00001055 +0.00001054 +0.00001054 +0.00001054 +0.18141565 +0.15467737 +0.10396576 +0.05004989 +0.01422822 +0.00135211 +0.00002930 +0.00001056 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.02886257 +0.01983436 +0.00862150 +0.00191934 +0.00015379 +0.00001308 +0.00001055 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00013649 +0.00007437 +0.00002511 +0.00001171 +0.00001056 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.45693474 +0.38187220 +0.33877278 +0.30742979 +0.28620385 +0.28470101 +0.18141565 +0.05761829 +0.00057044 +0.00001061 +0.00001054 +0.00001054 +0.38187220 +0.35326129 +0.32326735 +0.29641515 +0.28251204 +0.28175875 +0.15467737 +0.03721566 +0.00032433 +0.00001057 +0.00001054 +0.00001054 +0.33877278 +0.32326735 +0.30324065 +0.28407677 +0.28126185 +0.25502390 +0.10396576 +0.01503287 +0.00009890 +0.00001055 +0.00001054 +0.00001054 +0.30742979 +0.29641515 +0.28407677 +0.27891759 +0.27917138 +0.17298712 +0.05004989 +0.00333814 +0.00001997 +0.00001054 +0.00001054 +0.00001054 +0.28620385 +0.28251204 +0.28126185 +0.27917138 +0.20071891 +0.08043416 +0.01422822 +0.00029793 +0.00001080 +0.00001054 +0.00001054 +0.00001054 +0.28470101 +0.28175875 +0.25502390 +0.17298712 +0.08043416 +0.02043551 +0.00135211 +0.00001727 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.18141565 +0.15467737 +0.10396576 +0.05004989 +0.01422822 +0.00135211 +0.00002930 +0.00001056 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.05761829 +0.03721566 +0.01503287 +0.00333814 +0.00029793 +0.00001727 +0.00001056 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00057044 +0.00032433 +0.00009890 +0.00001997 +0.00001080 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001061 +0.00001057 +0.00001055 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.43494586 +0.35489283 +0.30600184 +0.29971827 +0.29402581 +0.14428797 +0.02886257 +0.00057044 +0.00001166 +0.00001054 +0.00001054 +0.00001054 +0.35489283 +0.32026235 +0.29947524 +0.29853439 +0.27028652 +0.11515441 +0.01983436 +0.00032433 +0.00001099 +0.00001054 +0.00001054 +0.00001054 +0.30600184 +0.29947524 +0.29553262 +0.29502084 +0.20254900 +0.07083202 +0.00862150 +0.00009890 +0.00001059 +0.00001054 +0.00001054 +0.00001054 +0.29971827 +0.29853439 +0.29502084 +0.23393994 +0.11148623 +0.02988380 +0.00191934 +0.00001997 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.29402581 +0.27028652 +0.20254900 +0.11148623 +0.04025066 +0.00653668 +0.00015379 +0.00001080 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.14428797 +0.11515441 +0.07083202 +0.02988380 +0.00653668 +0.00035919 +0.00001308 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.02886257 +0.01983436 +0.00862150 +0.00191934 +0.00015379 +0.00001308 +0.00001055 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00057044 +0.00032433 +0.00009890 +0.00001997 +0.00001080 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001166 +0.00001099 +0.00001059 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.37711640 +0.33154333 +0.31929852 +0.22257725 +0.08088334 +0.01043263 +0.00013649 +0.00001061 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.33154333 +0.32060850 +0.30163606 +0.17652814 +0.05829573 +0.00619698 +0.00007437 +0.00001057 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.31929852 +0.30163606 +0.21999042 +0.10453302 +0.02835399 +0.00198369 +0.00002511 +0.00001055 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.22257725 +0.17652814 +0.10453302 +0.04098991 +0.00760629 +0.00027286 +0.00001171 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.08088334 +0.05829573 +0.02835399 +0.00760629 +0.00066411 +0.00002130 +0.00001056 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.01043263 +0.00619698 +0.00198369 +0.00027286 +0.00002130 +0.00001063 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00013649 +0.00007437 +0.00002511 +0.00001171 +0.00001056 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001061 +0.00001057 +0.00001055 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.41567850 +0.23226734 +0.08216671 +0.01636066 +0.00073983 +0.00001580 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.23226734 +0.14003905 +0.05130944 +0.00932219 +0.00037442 +0.00001289 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.08216671 +0.05130944 +0.01837672 +0.00264075 +0.00008876 +0.00001092 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.01636066 +0.00932219 +0.00264075 +0.00025368 +0.00001593 +0.00001056 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00073983 +0.00037442 +0.00008876 +0.00001593 +0.00001062 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001580 +0.00001289 +0.00001092 +0.00001056 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.01081493 +0.00484158 +0.00037403 +0.00001481 +0.00001055 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00484158 +0.00198558 +0.00015391 +0.00001221 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00037403 +0.00015391 +0.00002368 +0.00001071 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001481 +0.00001221 +0.00001071 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001055 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 + +SCALARS volume float 1 +LOOKUP_TABLE default +0.09434889 +0.01658761 +0.00547177 +0.00323890 +0.00222759 +0.00161902 +0.00119620 +0.00090325 +0.00064695 +0.00060647 +0.00014818 +0.00089030 +0.01658761 +0.01057695 +0.00494558 +0.00265194 +0.00176921 +0.00127705 +0.00094965 +0.00070525 +0.00055922 +0.00039095 +0.00029893 +0.00093620 +0.00547177 +0.00494558 +0.00348619 +0.00217449 +0.00138716 +0.00100029 +0.00074238 +0.00055259 +0.00046787 +0.00020297 +0.00058477 +0.00098330 +0.00323890 +0.00265194 +0.00217449 +0.00165962 +0.00117503 +0.00082724 +0.00060761 +0.00050178 +0.00032261 +0.00027038 +0.00084695 +0.00099839 +0.00222759 +0.00176921 +0.00138716 +0.00117503 +0.00093163 +0.00067731 +0.00052190 +0.00040204 +0.00019918 +0.00057389 +0.00097279 +0.00099995 +0.00161902 +0.00127705 +0.00100029 +0.00082724 +0.00067731 +0.00054179 +0.00045586 +0.00022391 +0.00038922 +0.00087649 +0.00099804 +0.00100000 +0.00119620 +0.00094965 +0.00074238 +0.00060761 +0.00052190 +0.00045586 +0.00028371 +0.00031403 +0.00076492 +0.00098862 +0.00099998 +0.00100000 +0.00090325 +0.00070525 +0.00055259 +0.00050178 +0.00040204 +0.00022391 +0.00031403 +0.00064627 +0.00097354 +0.00099979 +0.00100000 +0.00100000 +0.00064695 +0.00055922 +0.00046787 +0.00032261 +0.00019918 +0.00038922 +0.00076492 +0.00097354 +0.00099906 +0.00100000 +0.00100000 +0.00100000 +0.00060647 +0.00039095 +0.00020297 +0.00027038 +0.00057389 +0.00087649 +0.00098862 +0.00099979 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00014818 +0.00029893 +0.00058477 +0.00084695 +0.00097279 +0.00099804 +0.00099998 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00089030 +0.00093620 +0.00098330 +0.00099839 +0.00099995 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.01658761 +0.01057695 +0.00494558 +0.00265194 +0.00176921 +0.00127705 +0.00094965 +0.00070525 +0.00055922 +0.00039095 +0.00029893 +0.00093620 +0.01057695 +0.00729516 +0.00394270 +0.00229075 +0.00152435 +0.00109248 +0.00081636 +0.00060402 +0.00050913 +0.00026331 +0.00044549 +0.00096330 +0.00494558 +0.00394270 +0.00285330 +0.00185731 +0.00126960 +0.00092067 +0.00068175 +0.00052346 +0.00042167 +0.00019166 +0.00068687 +0.00099046 +0.00265194 +0.00229075 +0.00185731 +0.00145708 +0.00104958 +0.00076889 +0.00057377 +0.00047919 +0.00027107 +0.00033779 +0.00089130 +0.00099909 +0.00176921 +0.00152435 +0.00126960 +0.00104958 +0.00084279 +0.00062345 +0.00050933 +0.00035779 +0.00020889 +0.00064908 +0.00098185 +0.00099997 +0.00127705 +0.00109248 +0.00092067 +0.00076889 +0.00062345 +0.00052504 +0.00041535 +0.00020724 +0.00045415 +0.00090899 +0.00099879 +0.00100000 +0.00094965 +0.00081636 +0.00068175 +0.00057377 +0.00050933 +0.00041535 +0.00024015 +0.00035696 +0.00081217 +0.00099232 +0.00099999 +0.00100000 +0.00070525 +0.00060402 +0.00052346 +0.00047919 +0.00035779 +0.00020724 +0.00035696 +0.00072474 +0.00098105 +0.00099987 +0.00100000 +0.00100000 +0.00055922 +0.00050913 +0.00042167 +0.00027107 +0.00020889 +0.00045415 +0.00081217 +0.00098105 +0.00099946 +0.00100000 +0.00100000 +0.00100000 +0.00039095 +0.00026331 +0.00019166 +0.00033779 +0.00064908 +0.00090899 +0.00099232 +0.00099987 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00029893 +0.00044549 +0.00068687 +0.00089130 +0.00098185 +0.00099879 +0.00099999 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00093620 +0.00096330 +0.00099046 +0.00099909 +0.00099997 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00547177 +0.00494558 +0.00348619 +0.00217449 +0.00138716 +0.00100029 +0.00074238 +0.00055259 +0.00046787 +0.00020297 +0.00058477 +0.00098330 +0.00494558 +0.00394270 +0.00285330 +0.00185731 +0.00126960 +0.00092067 +0.00068175 +0.00052346 +0.00042167 +0.00019166 +0.00068687 +0.00099046 +0.00348619 +0.00285330 +0.00206171 +0.00147912 +0.00108695 +0.00080860 +0.00059734 +0.00049107 +0.00031544 +0.00026719 +0.00083537 +0.00099741 +0.00217449 +0.00185731 +0.00147912 +0.00117035 +0.00089576 +0.00067663 +0.00052783 +0.00042368 +0.00020597 +0.00049468 +0.00094878 +0.00099975 +0.00138716 +0.00126960 +0.00108695 +0.00089576 +0.00071465 +0.00056028 +0.00047345 +0.00027595 +0.00027849 +0.00077516 +0.00099238 +0.00099999 +0.00100029 +0.00092067 +0.00080860 +0.00067663 +0.00056028 +0.00048780 +0.00032957 +0.00021297 +0.00058548 +0.00095301 +0.00099957 +0.00100000 +0.00074238 +0.00068175 +0.00059734 +0.00052783 +0.00047345 +0.00032957 +0.00021316 +0.00047204 +0.00088603 +0.00099667 +0.00100000 +0.00100000 +0.00055259 +0.00052346 +0.00049107 +0.00042368 +0.00027595 +0.00021297 +0.00047204 +0.00083888 +0.00099080 +0.00099996 +0.00100000 +0.00100000 +0.00046787 +0.00042167 +0.00031544 +0.00020597 +0.00027849 +0.00058548 +0.00088603 +0.00099080 +0.00099983 +0.00100000 +0.00100000 +0.00100000 +0.00020297 +0.00019166 +0.00026719 +0.00049468 +0.00077516 +0.00095301 +0.00099667 +0.00099996 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00058477 +0.00068687 +0.00083537 +0.00094878 +0.00099238 +0.00099957 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00098330 +0.00099046 +0.00099741 +0.00099975 +0.00099999 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00323890 +0.00265194 +0.00217449 +0.00165962 +0.00117503 +0.00082724 +0.00060761 +0.00050178 +0.00032261 +0.00027038 +0.00084695 +0.00099839 +0.00265194 +0.00229075 +0.00185731 +0.00145708 +0.00104958 +0.00076889 +0.00057377 +0.00047919 +0.00027107 +0.00033779 +0.00089130 +0.00099909 +0.00217449 +0.00185731 +0.00147912 +0.00117035 +0.00089576 +0.00067663 +0.00052783 +0.00042368 +0.00020597 +0.00049468 +0.00094878 +0.00099975 +0.00165962 +0.00145708 +0.00117035 +0.00093633 +0.00074137 +0.00057922 +0.00048620 +0.00030882 +0.00023638 +0.00071445 +0.00098630 +0.00099997 +0.00117503 +0.00104958 +0.00089576 +0.00074137 +0.00060143 +0.00051255 +0.00038971 +0.00020417 +0.00045558 +0.00090043 +0.00099823 +0.00100000 +0.00082724 +0.00076889 +0.00067663 +0.00057922 +0.00051255 +0.00041006 +0.00023196 +0.00031372 +0.00075723 +0.00098470 +0.00099992 +0.00100000 +0.00060761 +0.00057377 +0.00052783 +0.00048620 +0.00038971 +0.00023196 +0.00028293 +0.00065650 +0.00095263 +0.00099916 +0.00100000 +0.00100000 +0.00050178 +0.00047919 +0.00042368 +0.00030882 +0.00020417 +0.00031372 +0.00065650 +0.00093366 +0.00099732 +0.00099999 +0.00100000 +0.00100000 +0.00032261 +0.00027107 +0.00020597 +0.00023638 +0.00045558 +0.00075723 +0.00095263 +0.00099732 +0.00099998 +0.00100000 +0.00100000 +0.00100000 +0.00027038 +0.00033779 +0.00049468 +0.00071445 +0.00090043 +0.00098470 +0.00099916 +0.00099999 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00084695 +0.00089130 +0.00094878 +0.00098630 +0.00099823 +0.00099992 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00099839 +0.00099909 +0.00099975 +0.00099997 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00222759 +0.00176921 +0.00138716 +0.00117503 +0.00093163 +0.00067731 +0.00052190 +0.00040204 +0.00019918 +0.00057389 +0.00097279 +0.00099995 +0.00176921 +0.00152435 +0.00126960 +0.00104958 +0.00084279 +0.00062345 +0.00050933 +0.00035779 +0.00020889 +0.00064908 +0.00098185 +0.00099997 +0.00138716 +0.00126960 +0.00108695 +0.00089576 +0.00071465 +0.00056028 +0.00047345 +0.00027595 +0.00027849 +0.00077516 +0.00099238 +0.00099999 +0.00117503 +0.00104958 +0.00089576 +0.00074137 +0.00060143 +0.00051255 +0.00038971 +0.00020417 +0.00045558 +0.00090043 +0.00099823 +0.00100000 +0.00093163 +0.00084279 +0.00071465 +0.00060143 +0.00052523 +0.00043621 +0.00025334 +0.00026925 +0.00070489 +0.00097529 +0.00099982 +0.00100000 +0.00067731 +0.00062345 +0.00056028 +0.00051255 +0.00043621 +0.00027895 +0.00022384 +0.00054032 +0.00090580 +0.00099727 +0.00100000 +0.00100000 +0.00052190 +0.00050933 +0.00047345 +0.00038971 +0.00025334 +0.00022384 +0.00048569 +0.00084465 +0.00098860 +0.00099990 +0.00100000 +0.00100000 +0.00040204 +0.00035779 +0.00027595 +0.00020417 +0.00026925 +0.00054032 +0.00084465 +0.00098310 +0.00099961 +0.00100000 +0.00100000 +0.00100000 +0.00019918 +0.00020889 +0.00027849 +0.00045558 +0.00070489 +0.00090580 +0.00098860 +0.00099961 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00057389 +0.00064908 +0.00077516 +0.00090043 +0.00097529 +0.00099727 +0.00099990 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00097279 +0.00098185 +0.00099238 +0.00099823 +0.00099982 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00099995 +0.00099997 +0.00099999 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00161902 +0.00127705 +0.00100029 +0.00082724 +0.00067731 +0.00054179 +0.00045586 +0.00022391 +0.00038922 +0.00087649 +0.00099804 +0.00100000 +0.00127705 +0.00109248 +0.00092067 +0.00076889 +0.00062345 +0.00052504 +0.00041535 +0.00020724 +0.00045415 +0.00090899 +0.00099879 +0.00100000 +0.00100029 +0.00092067 +0.00080860 +0.00067663 +0.00056028 +0.00048780 +0.00032957 +0.00021297 +0.00058548 +0.00095301 +0.00099957 +0.00100000 +0.00082724 +0.00076889 +0.00067663 +0.00057922 +0.00051255 +0.00041006 +0.00023196 +0.00031372 +0.00075723 +0.00098470 +0.00099992 +0.00100000 +0.00067731 +0.00062345 +0.00056028 +0.00051255 +0.00043621 +0.00027895 +0.00022384 +0.00054032 +0.00090580 +0.00099727 +0.00100000 +0.00100000 +0.00054179 +0.00052504 +0.00048780 +0.00041006 +0.00027895 +0.00021296 +0.00042652 +0.00080428 +0.00098176 +0.00099979 +0.00100000 +0.00100000 +0.00045586 +0.00041535 +0.00032957 +0.00023196 +0.00022384 +0.00042652 +0.00075701 +0.00096121 +0.00099868 +0.00100000 +0.00100000 +0.00100000 +0.00022391 +0.00020724 +0.00021297 +0.00031372 +0.00054032 +0.00080428 +0.00096121 +0.00099776 +0.00099998 +0.00100000 +0.00100000 +0.00100000 +0.00038922 +0.00045415 +0.00058548 +0.00075723 +0.00090580 +0.00098176 +0.00099868 +0.00099998 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00087649 +0.00090899 +0.00095301 +0.00098470 +0.00099727 +0.00099979 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00099804 +0.00099879 +0.00099957 +0.00099992 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00119620 +0.00094965 +0.00074238 +0.00060761 +0.00052190 +0.00045586 +0.00028371 +0.00031403 +0.00076492 +0.00098862 +0.00099998 +0.00100000 +0.00094965 +0.00081636 +0.00068175 +0.00057377 +0.00050933 +0.00041535 +0.00024015 +0.00035696 +0.00081217 +0.00099232 +0.00099999 +0.00100000 +0.00074238 +0.00068175 +0.00059734 +0.00052783 +0.00047345 +0.00032957 +0.00021316 +0.00047204 +0.00088603 +0.00099667 +0.00100000 +0.00100000 +0.00060761 +0.00057377 +0.00052783 +0.00048620 +0.00038971 +0.00023196 +0.00028293 +0.00065650 +0.00095263 +0.00099916 +0.00100000 +0.00100000 +0.00052190 +0.00050933 +0.00047345 +0.00038971 +0.00025334 +0.00022384 +0.00048569 +0.00084465 +0.00098860 +0.00099990 +0.00100000 +0.00100000 +0.00045586 +0.00041535 +0.00032957 +0.00023196 +0.00022384 +0.00042652 +0.00075701 +0.00096121 +0.00099868 +0.00100000 +0.00100000 +0.00100000 +0.00028371 +0.00024015 +0.00021316 +0.00028293 +0.00048569 +0.00075701 +0.00094457 +0.00099612 +0.00099995 +0.00100000 +0.00100000 +0.00100000 +0.00031403 +0.00035696 +0.00047204 +0.00065650 +0.00084465 +0.00096121 +0.00099612 +0.00099989 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00076492 +0.00081217 +0.00088603 +0.00095263 +0.00098860 +0.00099868 +0.00099995 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00098862 +0.00099232 +0.00099667 +0.00099916 +0.00099990 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00099998 +0.00099999 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00090325 +0.00070525 +0.00055259 +0.00050178 +0.00040204 +0.00022391 +0.00031403 +0.00064627 +0.00097354 +0.00099979 +0.00100000 +0.00100000 +0.00070525 +0.00060402 +0.00052346 +0.00047919 +0.00035779 +0.00020724 +0.00035696 +0.00072474 +0.00098105 +0.00099987 +0.00100000 +0.00100000 +0.00055259 +0.00052346 +0.00049107 +0.00042368 +0.00027595 +0.00021297 +0.00047204 +0.00083888 +0.00099080 +0.00099996 +0.00100000 +0.00100000 +0.00050178 +0.00047919 +0.00042368 +0.00030882 +0.00020417 +0.00031372 +0.00065650 +0.00093366 +0.00099732 +0.00099999 +0.00100000 +0.00100000 +0.00040204 +0.00035779 +0.00027595 +0.00020417 +0.00026925 +0.00054032 +0.00084465 +0.00098310 +0.00099961 +0.00100000 +0.00100000 +0.00100000 +0.00022391 +0.00020724 +0.00021297 +0.00031372 +0.00054032 +0.00080428 +0.00096121 +0.00099776 +0.00099998 +0.00100000 +0.00100000 +0.00100000 +0.00031403 +0.00035696 +0.00047204 +0.00065650 +0.00084465 +0.00096121 +0.00099612 +0.00099989 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00064627 +0.00072474 +0.00083888 +0.00093366 +0.00098310 +0.00099776 +0.00099989 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00097354 +0.00098105 +0.00099080 +0.00099732 +0.00099961 +0.00099998 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00099979 +0.00099987 +0.00099996 +0.00099999 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00064695 +0.00055922 +0.00046787 +0.00032261 +0.00019918 +0.00038922 +0.00076492 +0.00097354 +0.00099906 +0.00100000 +0.00100000 +0.00100000 +0.00055922 +0.00050913 +0.00042167 +0.00027107 +0.00020889 +0.00045415 +0.00081217 +0.00098105 +0.00099946 +0.00100000 +0.00100000 +0.00100000 +0.00046787 +0.00042167 +0.00031544 +0.00020597 +0.00027849 +0.00058548 +0.00088603 +0.00099080 +0.00099983 +0.00100000 +0.00100000 +0.00100000 +0.00032261 +0.00027107 +0.00020597 +0.00023638 +0.00045558 +0.00075723 +0.00095263 +0.00099732 +0.00099998 +0.00100000 +0.00100000 +0.00100000 +0.00019918 +0.00020889 +0.00027849 +0.00045558 +0.00070489 +0.00090580 +0.00098860 +0.00099961 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00038922 +0.00045415 +0.00058548 +0.00075723 +0.00090580 +0.00098176 +0.00099868 +0.00099998 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00076492 +0.00081217 +0.00088603 +0.00095263 +0.00098860 +0.00099868 +0.00099995 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00097354 +0.00098105 +0.00099080 +0.00099732 +0.00099961 +0.00099998 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00099906 +0.00099946 +0.00099983 +0.00099998 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00060647 +0.00039095 +0.00020297 +0.00027038 +0.00057389 +0.00087649 +0.00098862 +0.00099979 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00039095 +0.00026331 +0.00019166 +0.00033779 +0.00064908 +0.00090899 +0.00099232 +0.00099987 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00020297 +0.00019166 +0.00026719 +0.00049468 +0.00077516 +0.00095301 +0.00099667 +0.00099996 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00027038 +0.00033779 +0.00049468 +0.00071445 +0.00090043 +0.00098470 +0.00099916 +0.00099999 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00057389 +0.00064908 +0.00077516 +0.00090043 +0.00097529 +0.00099727 +0.00099990 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00087649 +0.00090899 +0.00095301 +0.00098470 +0.00099727 +0.00099979 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00098862 +0.00099232 +0.00099667 +0.00099916 +0.00099990 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00099979 +0.00099987 +0.00099996 +0.00099999 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00014818 +0.00029893 +0.00058477 +0.00084695 +0.00097279 +0.00099804 +0.00099998 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00029893 +0.00044549 +0.00068687 +0.00089130 +0.00098185 +0.00099879 +0.00099999 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00058477 +0.00068687 +0.00083537 +0.00094878 +0.00099238 +0.00099957 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00084695 +0.00089130 +0.00094878 +0.00098630 +0.00099823 +0.00099992 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00097279 +0.00098185 +0.00099238 +0.00099823 +0.00099982 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00099804 +0.00099879 +0.00099957 +0.00099992 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00099998 +0.00099999 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00089030 +0.00093620 +0.00098330 +0.00099839 +0.00099995 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00093620 +0.00096330 +0.00099046 +0.00099909 +0.00099997 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00098330 +0.00099046 +0.00099741 +0.00099975 +0.00099999 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00099839 +0.00099909 +0.00099975 +0.00099997 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00099995 +0.00099997 +0.00099999 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 + +FIELD FieldData 0 diff --git a/python/Test-scripts/standard-results/SGH/Sedov_12x12x12/vtk/outputs.vtk.series b/python/Test-scripts/standard-results/SGH/Sedov_12x12x12/vtk/outputs.vtk.series new file mode 100644 index 000000000..e75ac9e43 --- /dev/null +++ b/python/Test-scripts/standard-results/SGH/Sedov_12x12x12/vtk/outputs.vtk.series @@ -0,0 +1,6 @@ +{ + "file-series-version" : "1.0", + "files" : [ + { "name" : "data/VTK0.vtk", "time" : 1.00000e+00 }, + ] +} diff --git a/python/Test-scripts/test_fierro.py b/python/Test-scripts/test_fierro.py new file mode 100644 index 000000000..db50cc0f2 --- /dev/null +++ b/python/Test-scripts/test_fierro.py @@ -0,0 +1,128 @@ +#!/usr/bin/python3 + +import os +import os.path +import sys +import math + +ground_truth_path = "" + + +# print("you passed in", len(sys.argv), "argument(s)") +# print("those arguments are: ", str(sys.argv)) + +# print("hello", sys.argv[0]) + +# os.system('pwd') + +executable = "./../../build-fierro-openmp/bin/fierro-parallel-explicit" +sim_input = "Solver-Inputs/SGH_Sedov_12x12x12.yaml" + +# Run simulation +os.system(executable + ' ' + sim_input) + + +# Functions for reading results from vtk file + +def extract_vector_data(filename, keyword): + data = [] + found_keyword = False + + with open(filename, 'r') as file: + for line in file: + if found_keyword: + if line.strip(): # If the line is not blank + # Split each line into fields. + fields = line.split() + xx = float(fields[0]) + yy = float(fields[1]) + zz = float(fields[2]) + # Group and append to results. + vec = [xx, yy, zz] + data.append(vec) + else: + break # Exit the loop when encountering a blank line + elif keyword in line: + found_keyword = True + + return data + +def extract_scalar_data(filename, keyword): + data = [] + found_keyword = False + + with open(filename, 'r') as file: + for line in file: + if found_keyword: + if line.strip(): # If the line is not blank + fields = line.split() + if fields[0] != "LOOKUP_TABLE": + data.append(float(fields[0])) + else: + break # Exit the loop when encountering a blank line + elif keyword in line: + found_keyword = True + # file.next() # skip first line "LOOKUP_TABLE" + + return data + + + +def percent_difference_scalars(array1, array2): + if len(array1) != len(array2): + raise ValueError("Arrays must have the same length") + + percent_diff = [] + for i in range(len(array1)): + diff = array2[i] - array1[i] + percent_diff.append((diff / array1[i]) * 100 if array1[i] != 0 else 0) + + return percent_diff + +def percent_difference_vectors(array1, array2): + if len(array1) != len(array2): + raise ValueError("Arrays must have the same length") + + percent_diff = [] + for i in range(len(array1)): + if len(array1[i]) != 3 or len(array2[i]) != 3: + raise ValueError("Subarrays must have length 3") + + diff = [array2[i][j] - array1[i][j] for j in range(3)] + percent_diff.append([(diff[j] / array1[i][j]) * 100 if array1[i][j] != 0 else 0 for j in range(3)]) + + return percent_diff + +def magnitude(array): + mag = math.sqrt(sum(x**2 for x in array)) + return mag + + +# Read ground truth results for sedov +GT_filename = "standard-results/SGH/Sedov_12x12x12/vtk/data/VTK0.vtk" + +velocity_keyword = "VECTORS velocity float" +position_keyword = "POINTS 2197 float" +SIE_keyword = "SCALARS SIE float 1" +density_keyword = "SCALARS element_density float 1" + +GT_positions = extract_vector_data(GT_filename, position_keyword) +GT_velocities = extract_vector_data(GT_filename, velocity_keyword) +GT_SIE = extract_scalar_data(GT_filename, SIE_keyword) +GT_densities = extract_scalar_data(GT_filename, density_keyword) + + +# Read simulation results +results_filename = "vtk/data/VTK0.vtk" + +results_positions = extract_vector_data(results_filename, position_keyword) +results_velocities = extract_vector_data(results_filename, velocity_keyword) +results_SIE = extract_scalar_data(results_filename, SIE_keyword) +results_densities = extract_scalar_data(results_filename, density_keyword) + + +density_diff = percent_difference_scalars(GT_densities, results_densities) + +print("Density difference: ") +print(magnitude(density_diff)) + From 27a11bf462dde26624e98e8c3b4dfab72ac2051f Mon Sep 17 00:00:00 2001 From: yencal Date: Mon, 19 Feb 2024 15:55:12 -0700 Subject: [PATCH 10/64] removed the use of shared_ptr --- .../Fierro-EVPFFT-Link/FierroEVPFFTLink.cpp | 19 ++++++++++----- .../FierroLSEVPFFTLink.cpp | 20 +++++++++++----- .../strength/VUMAT/VUMATStrengthModel.cpp | 24 ++++++++++--------- 3 files changed, 40 insertions(+), 23 deletions(-) diff --git a/src/EVPFFT/src/Fierro-EVPFFT-Link/FierroEVPFFTLink.cpp b/src/EVPFFT/src/Fierro-EVPFFT-Link/FierroEVPFFTLink.cpp index bcdb5fdf0..8c1cffca4 100644 --- a/src/EVPFFT/src/Fierro-EVPFFT-Link/FierroEVPFFTLink.cpp +++ b/src/EVPFFT/src/Fierro-EVPFFT-Link/FierroEVPFFTLink.cpp @@ -5,7 +5,7 @@ MPI_Comm evpfft_mpi_comm = MPI_COMM_NULL; // to hold all evpfft in each element -std::vector> elem_evpfft; +std::vector elem_evpfft; namespace FierroEVPFFTLink { @@ -19,6 +19,8 @@ namespace FierroEVPFFTLink const DCArrayKokkos &elem_user_output_vars, const size_t num_elems) { + printf("Executing FierroEVPFFTLink::init_strength_state_vars ...\n"); + // First, lets create a new communicator with each rank having its own communicator containing only itself. if (evpfft_mpi_comm == MPI_COMM_NULL) { int global_rank; @@ -28,7 +30,7 @@ namespace FierroEVPFFTLink //... // assign correct size to elem_evpfft, all element are nullptr during initialization - elem_evpfft = std::vector> (num_elems, nullptr); + elem_evpfft = std::vector (num_elems, nullptr); for (size_t elem_gid = 0; elem_gid < num_elems; elem_gid++) { @@ -54,10 +56,10 @@ namespace FierroEVPFFTLink cmd.check_cmd_args(); // create EVPFFT model in element that used evpfft - elem_evpfft[elem_gid] = std::make_shared(evpfft_mpi_comm, - cmd, - stress_scale, - time_scale); + elem_evpfft[elem_gid] = new EVPFFT(evpfft_mpi_comm, + cmd, + stress_scale, + time_scale); } // end if (material.host(mat_id).strength_model... @@ -140,6 +142,11 @@ namespace FierroEVPFFTLink const DCArrayKokkos &elem_user_output_vars, const size_t num_elems) { + printf("Executing FierroEVPFFTLink::destroy ...\n"); + + for (size_t elem_gid = 0; elem_gid < num_elems; elem_gid++) { + delete elem_evpfft[elem_gid]; + } return; } diff --git a/src/LS-EVPFFT/src/Fierro-LS-EVPFFT-Link/FierroLSEVPFFTLink.cpp b/src/LS-EVPFFT/src/Fierro-LS-EVPFFT-Link/FierroLSEVPFFTLink.cpp index 1d9168deb..a556aaeb6 100644 --- a/src/LS-EVPFFT/src/Fierro-LS-EVPFFT-Link/FierroLSEVPFFTLink.cpp +++ b/src/LS-EVPFFT/src/Fierro-LS-EVPFFT-Link/FierroLSEVPFFTLink.cpp @@ -5,7 +5,7 @@ MPI_Comm evpfft_mpi_comm = MPI_COMM_NULL; // to hold all evpfft in each element -std::vector> elem_evpfft; +std::vector elem_evpfft; namespace FierroLSEVPFFTLink { @@ -19,6 +19,8 @@ namespace FierroLSEVPFFTLink const DCArrayKokkos &elem_user_output_vars, const size_t num_elems) { + printf("Executing FierroLSEVPFFTLink::init_strength_state_vars ...\n"); + // First, lets create a new communicator with each rank having its own communicator containing only itself. if (evpfft_mpi_comm == MPI_COMM_NULL) { int global_rank; @@ -28,7 +30,7 @@ namespace FierroLSEVPFFTLink //... // assign correct size to elem_evpfft, all element are nullptr during initialization - elem_evpfft = std::vector> (num_elems, nullptr); + elem_evpfft = std::vector (num_elems, nullptr); for (size_t elem_gid = 0; elem_gid < num_elems; elem_gid++) { @@ -54,10 +56,10 @@ namespace FierroLSEVPFFTLink cmd.check_cmd_args(); // create EVPFFT model in element that used evpfft - elem_evpfft[elem_gid] = std::make_shared(evpfft_mpi_comm, - cmd, - stress_scale, - time_scale); + elem_evpfft[elem_gid] = new EVPFFT(evpfft_mpi_comm, + cmd, + stress_scale, + time_scale); } // end if (material.host(mat_id).strength_model... @@ -139,6 +141,12 @@ namespace FierroLSEVPFFTLink const DCArrayKokkos &elem_user_output_vars, const size_t num_elems) { + printf("Executing FierroEVPFFTLink::destroy ...\n"); + + for (size_t elem_gid = 0; elem_gid < num_elems; elem_gid++) { + delete elem_evpfft[elem_gid]; + } + return; } diff --git a/src/Parallel-Solvers/Material-Models/strength/VUMAT/VUMATStrengthModel.cpp b/src/Parallel-Solvers/Material-Models/strength/VUMAT/VUMATStrengthModel.cpp index 1f4db7350..8a4b45330 100644 --- a/src/Parallel-Solvers/Material-Models/strength/VUMAT/VUMATStrengthModel.cpp +++ b/src/Parallel-Solvers/Material-Models/strength/VUMAT/VUMATStrengthModel.cpp @@ -5,7 +5,7 @@ // to hold all vumat in each element -std::vector> elem_vumat; +std::vector elem_vumat; namespace VUMATStrengthModel { @@ -27,7 +27,7 @@ namespace VUMATStrengthModel */ // assign correct size to elem_vumat, all element are nullptr during initialization - elem_vumat = std::vector> (num_elems, nullptr); + elem_vumat = std::vector (num_elems, nullptr); for (size_t elem_gid = 0; elem_gid < num_elems; elem_gid++) { @@ -42,14 +42,14 @@ namespace VUMATStrengthModel } // create VUMAT model in element that used vumat - elem_vumat[elem_gid] = std::make_shared(material, - eos_state_vars, - strength_state_vars, - eos_global_vars, - strength_global_vars, - elem_user_output_vars, - mat_id, - elem_gid); + elem_vumat[elem_gid] = new VUMAT(material, + eos_state_vars, + strength_state_vars, + eos_global_vars, + strength_global_vars, + elem_user_output_vars, + mat_id, + elem_gid); } // end if (material.host(mat_id).strength_model... @@ -123,7 +123,9 @@ namespace VUMATStrengthModel const DCArrayKokkos &elem_user_output_vars, const size_t num_elems) { - + for (size_t elem_gid = 0; elem_gid < num_elems; elem_gid++) { + delete elem_vumat[elem_gid]; + } } } // end namespace VUMATStrengthModel \ No newline at end of file From c448cb04d1e31a628c8ec1589797541fc8d545fe Mon Sep 17 00:00:00 2001 From: Jacob Moore <133793250+jacob-moore22@users.noreply.github.com> Date: Tue, 20 Feb 2024 08:18:06 -0600 Subject: [PATCH 11/64] COMP: Only test build on PR --- .github/workflows/ubuntu-build.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ubuntu-build.yaml b/.github/workflows/ubuntu-build.yaml index 1e518ef42..6844f33a6 100644 --- a/.github/workflows/ubuntu-build.yaml +++ b/.github/workflows/ubuntu-build.yaml @@ -1,8 +1,6 @@ name: Ubuntu Build on: - push: - branches: [ "main" ] pull_request: branches: [ "main" ] From b5d861218a38e23eced028fcdd1f384f7b4eaa10 Mon Sep 17 00:00:00 2001 From: Jacob Moore <133793250+jacob-moore22@users.noreply.github.com> Date: Tue, 20 Feb 2024 12:45:21 -0600 Subject: [PATCH 12/64] COMP: Update ubuntu-build.yaml to only run manually for now. --- .github/workflows/ubuntu-build.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ubuntu-build.yaml b/.github/workflows/ubuntu-build.yaml index 6844f33a6..2a712e011 100644 --- a/.github/workflows/ubuntu-build.yaml +++ b/.github/workflows/ubuntu-build.yaml @@ -1,8 +1,7 @@ name: Ubuntu Build -on: - pull_request: - branches: [ "main" ] +on: workflow_dispatch + jobs: build: From be8e9572f125385ba853fb9030bf27ada9f3dd94 Mon Sep 17 00:00:00 2001 From: yencal Date: Wed, 21 Feb 2024 06:36:01 -0700 Subject: [PATCH 13/64] added namespace EVP to avoid funciton conflicts --- .../Material-Models/strength/EVP/EVPStrengthModel.cpp | 5 +++++ .../Material-Models/strength/EVP/chg_basis.cpp | 8 ++++++++ .../Material-Models/strength/EVP/evpal.cpp | 7 +++++++ .../Material-Models/strength/EVP/user_mat.cpp | 6 ++++++ .../Material-Models/strength/EVP/user_mat.h | 6 ++++++ 5 files changed, 32 insertions(+) diff --git a/src/Parallel-Solvers/Material-Models/strength/EVP/EVPStrengthModel.cpp b/src/Parallel-Solvers/Material-Models/strength/EVP/EVPStrengthModel.cpp index 945efff99..76e26509c 100644 --- a/src/Parallel-Solvers/Material-Models/strength/EVP/EVPStrengthModel.cpp +++ b/src/Parallel-Solvers/Material-Models/strength/EVP/EVPStrengthModel.cpp @@ -43,6 +43,8 @@ namespace EVPStrengthModel initialization location leading to overwriting of data. */ + using namespace EVP; + // for the reason given in the opening comment we will create a state_var, // populate it as needed and copy needed data to strength_state_vars DCArrayKokkos state_vars (strength_state_vars.dims(0), strength_state_vars.dims(1)); @@ -484,6 +486,9 @@ namespace EVPStrengthModel const size_t rk_level, const double time) { + + using namespace EVP; + // ChgBasis cb; // ----------------------------------------------------------------------------- diff --git a/src/Parallel-Solvers/Material-Models/strength/EVP/chg_basis.cpp b/src/Parallel-Solvers/Material-Models/strength/EVP/chg_basis.cpp index aed6194a6..6bb22db1c 100644 --- a/src/Parallel-Solvers/Material-Models/strength/EVP/chg_basis.cpp +++ b/src/Parallel-Solvers/Material-Models/strength/EVP/chg_basis.cpp @@ -44,6 +44,11 @@ // } + +// using namespace to avoid collision of functions that are the same in other models like EVPFFT or LS-EVPFFT +namespace EVP +{ + KOKKOS_FUNCTION void chg_basis_1(real_t *CE2_, real_t *C2_, int IOPT, int KDIM, real_t *B_basis_) //ptr) const { @@ -132,6 +137,9 @@ void chg_basis_4(real_t *CE4_, real_t *C4_, int IOPT, int KDIM, real_t *B_basis_ } } + +} // end namespace EVP + // KOKKOS_FUNCTION // real_t* ChgBasis::B_basis_device_pointer() const // { diff --git a/src/Parallel-Solvers/Material-Models/strength/EVP/evpal.cpp b/src/Parallel-Solvers/Material-Models/strength/EVP/evpal.cpp index eaf622dab..4e99e2116 100644 --- a/src/Parallel-Solvers/Material-Models/strength/EVP/evpal.cpp +++ b/src/Parallel-Solvers/Material-Models/strength/EVP/evpal.cpp @@ -1,5 +1,10 @@ #include "user_mat.h" + +// using namespace to avoid collision of functions that are the same in other models like EVPFFT or LS-EVPFFT +namespace EVP +{ + // evaluate elasto-viscoplastic crystal plasticity model using Newton-Raphson solve KOKKOS_FUNCTION void evpal(real_t *stress_, real_t *edotp_, real_t *gamdot_, real_t *stress_n_, real_t *strain_, @@ -494,3 +499,5 @@ void lubksb(real_t *a_, int n, int np, int *indx_, real_t *b_) } // end loop 14 } + +} // end namespace EVP diff --git a/src/Parallel-Solvers/Material-Models/strength/EVP/user_mat.cpp b/src/Parallel-Solvers/Material-Models/strength/EVP/user_mat.cpp index c1ef9108e..7f2c68b77 100644 --- a/src/Parallel-Solvers/Material-Models/strength/EVP/user_mat.cpp +++ b/src/Parallel-Solvers/Material-Models/strength/EVP/user_mat.cpp @@ -1,5 +1,9 @@ #include "user_mat.h" +// using namespace to avoid collision of functions that are the same in other models like EVPFFT or LS-EVPFFT +namespace EVP +{ + // euler <-> rotation matrix transformation function KOKKOS_FUNCTION void euler(int iopt, real_t &ph, real_t &th, real_t &tm, real_t *a_){ @@ -381,3 +385,5 @@ void harden(real_t *crss_, real_t &gacumgr, real_t *gamdot_, real_t *voceParam_, return; } // end harden + +} // end namespace EVP diff --git a/src/Parallel-Solvers/Material-Models/strength/EVP/user_mat.h b/src/Parallel-Solvers/Material-Models/strength/EVP/user_mat.h index c20f8408c..4fc38b7d0 100644 --- a/src/Parallel-Solvers/Material-Models/strength/EVP/user_mat.h +++ b/src/Parallel-Solvers/Material-Models/strength/EVP/user_mat.h @@ -17,6 +17,10 @@ using real_t = double; // void set_b_basis(); +// using namespace to avoid collision of functions that are the same in other models like EVPFFT or LS-EVPFFT +namespace EVP +{ + KOKKOS_FUNCTION void chg_basis_1(real_t *CE2_, real_t *C2_, int IOPT, int KDIM, real_t *B_basis_);//ptr) const; @@ -73,3 +77,5 @@ void orient(real_t *a_, real_t *c_); KOKKOS_FUNCTION void harden(real_t *crss_, real_t &gacumgr, real_t *gamdot_, real_t *voceParam_, real_t *hard_, const int nsm, const real_t dt); + +} // end namespace EVP \ No newline at end of file From ff0472a19bf0d195c12613d5e23755f1f5842756 Mon Sep 17 00:00:00 2001 From: Daniel Dunning Date: Wed, 21 Feb 2024 12:16:40 -0700 Subject: [PATCH 14/64] FEATURE: added script modifications to build with Cuda --- lib/Elements | 2 +- scripts/build-fierro.sh | 4 ++-- scripts/cmake_build.sh | 13 +++++++++++++ 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/lib/Elements b/lib/Elements index 5bfc94f5e..0de655cd2 160000 --- a/lib/Elements +++ b/lib/Elements @@ -1 +1 @@ -Subproject commit 5bfc94f5e176d54ff2ef51f93541440be0c17510 +Subproject commit 0de655cd260e60e4f037a78561dddfb9c8757fc7 diff --git a/scripts/build-fierro.sh b/scripts/build-fierro.sh index 6e76c4941..17293bd68 100755 --- a/scripts/build-fierro.sh +++ b/scripts/build-fierro.sh @@ -185,7 +185,7 @@ if [ "$build_action" = "full-app" ]; then source hdf5-install.sh source heffte-install.sh ${heffte_build_type} ${machine} fi - source cmake_build.sh ${solver} + source cmake_build.sh ${solver} ${heffte_build_type} ${kokkos_build_type} elif [ "$build_action" = "install-trilinos" ]; then source trilinos-install.sh ${kokkos_build_type} elif [ "$build_action" = "install-hdf5" ]; then @@ -193,7 +193,7 @@ elif [ "$build_action" = "install-hdf5" ]; then elif [ "$build_action" = "install-heffte" ]; then source heffte-install.sh ${heffte_build_type} ${machine} elif [ "$build_action" = "fierro" ]; then - source cmake_build.sh ${solver} ${heffte_build_type} + source cmake_build.sh ${solver} ${heffte_build_type} ${kokkos_build_type} else echo "No build action, only setup the environment." fi diff --git a/scripts/cmake_build.sh b/scripts/cmake_build.sh index 04d9c2a26..cca151dd9 100644 --- a/scripts/cmake_build.sh +++ b/scripts/cmake_build.sh @@ -2,6 +2,7 @@ solver="${1}" heffte_build_type="${2}" +kokkos_build_type="${3}" #inititialize submodules if they aren't downloaded [ -d "${libdir}/Elements/elements" ] && echo "Elements submodule exists" @@ -100,6 +101,18 @@ else ) fi +if [ "$kokkos_build_type" = "cuda" ]; then + export OMPI_CXX=${TRILINOS_SOURCE_DIR}/packages/kokkos/bin/nvcc_wrapper + cmake_options+=( + -D CMAKE_CXX_COMPILER=${TRILINOS_SOURCE_DIR}/packages/kokkos/bin/nvcc_wrapper + ) +elif [ "$kokkos_build_type" = "hip" ]; then + export OMPI_CXX=hipcc + cmake_options+=( + -D CMAKE_CXX_COMPILER=hipcc + ) +fi + # Print CMake options for reference echo "CMake Options: ${cmake_options[@]}" From feb5929851231a87a1a7ef0b9f8162a73ab6afe7 Mon Sep 17 00:00:00 2001 From: yencal Date: Wed, 21 Feb 2024 12:41:15 -0700 Subject: [PATCH 15/64] updated EVPFFT and LS-EVPFFT README.md --- src/EVPFFT/README.md | 14 +------------- src/EVPFFT/src/evpfft.cpp | 8 +++++++- src/LS-EVPFFT/README.md | 14 +------------- 3 files changed, 9 insertions(+), 27 deletions(-) diff --git a/src/EVPFFT/README.md b/src/EVPFFT/README.md index 26602d3b3..2a2b73b62 100644 --- a/src/EVPFFT/README.md +++ b/src/EVPFFT/README.md @@ -105,19 +105,7 @@ mpirun -np 4 evpfft --infile=example_evpfft_standalone_inputfile.txt To use EVPFFT as a user material model for Fierro, build Fierro with the following cmake flags `-DBUILD_EVPFFT_FIERRO=ON`, choose the FFT backend with `-D=ON`, and use `-DABSOLUTE_NO_OUTPUT=ON` to specify that EVPFFT does not provide output per element which can be cumbersome and memory intensive. It is best to modify EVPFFT to output for a few elements of interst. -The yaml file for Fierro should include: - -``` -material_options: - - eos_model: user_eos_model - eos_run_location: device - strength_model: user_strength_model - strength_type: hypo - strength_run_location: host - global_vars: - - 0.0001 #udotAccTh (EVPFFT accumulated strain threshold before full solve) - - 3000000 #[mm/s] (EVPFFT material sound speed) -``` +See `example_input_files/taylor_anvil.yaml` for the yaml file setup when using EVPFFT with Fierro. Also, input files for each material should be defined as `evpfft1.in`, `evpfft2.in`, etc. and should be provided in the directory Fierro is being run from. diff --git a/src/EVPFFT/src/evpfft.cpp b/src/EVPFFT/src/evpfft.cpp index 9355b7ae9..f030df3b1 100644 --- a/src/EVPFFT/src/evpfft.cpp +++ b/src/EVPFFT/src/evpfft.cpp @@ -148,8 +148,14 @@ void EVPFFT::init_after_reading_input_data() init_xk_gb(); init_disgradmacro(); init_ept(); - init_sg(); init_evm(); + +// the variables initialized in the funcitons below are reduced into +// and should be done once, hence the need for #if guard since the variables +// needs to be initialized after udot and dt are know from fierro +#ifndef BUILD_EVPFFT_FIERRO + init_sg(); +#endif } void EVPFFT::init_xk_gb() diff --git a/src/LS-EVPFFT/README.md b/src/LS-EVPFFT/README.md index 85831347b..2c6d40dd8 100644 --- a/src/LS-EVPFFT/README.md +++ b/src/LS-EVPFFT/README.md @@ -105,19 +105,7 @@ mpirun -np 4 ls-evpfft --infile=example_ls-evpfft_standalone_inputfile.txt To use LS-EVPFFT as a user material model for Fierro, build Fierro with the following cmake flags `-DBUILD_LS_EVPFFT_FIERRO=ON`, choose the FFT backend with `-D=ON`, and use `-DABSOLUTE_NO_OUTPUT=ON` to specify that LS-EVPFFT does not provide output per element which can be cumbersome and memory intensive. It is best to modify LS-EVPFFT to output for a few elements of interst. -The yaml file for Fierro should include: - -``` -material_options: - - eos_model: user_eos_model - eos_run_location: device - strength_model: user_strength_model - strength_type: hypo - strength_run_location: host - global_vars: - - 0.0001 #udotAccTh (LS-EVPFFT accumulated strain threshold before full solve) - - 3000000 #[mm/s] (LS-EVPFFT material sound speed) -``` +See `example_input_files/taylor_anvil.yaml` for the yaml file setup when using EVPFFT with Fierro. Also, input files for each material should be defined as `evpfft1.in`, `evpfft2.in`, etc. and should be provided in the directory Fierro is being run from. From 43616625e52547b1de9feef505db39f06d10e1f6 Mon Sep 17 00:00:00 2001 From: Jacob Moore Date: Wed, 21 Feb 2024 14:29:40 -0600 Subject: [PATCH 16/64] STYLE: Tidying up python test script --- python/Test-scripts/test_fierro.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python/Test-scripts/test_fierro.py b/python/Test-scripts/test_fierro.py index db50cc0f2..6baab8cb2 100644 --- a/python/Test-scripts/test_fierro.py +++ b/python/Test-scripts/test_fierro.py @@ -5,7 +5,6 @@ import sys import math -ground_truth_path = "" # print("you passed in", len(sys.argv), "argument(s)") From d0e35d9f245b366d7832fd090b68140591a16ff4 Mon Sep 17 00:00:00 2001 From: Jacob Moore Date: Thu, 15 Feb 2024 15:40:37 -0600 Subject: [PATCH 17/64] STYLE: Uncrustify Parallel-Explicit solvers --- src/Parallel-Solvers/FEA_Module.cpp | 509 +- src/Parallel-Solvers/FEA_Module.h | 545 +- src/Parallel-Solvers/FEA_Module_Inertial.cpp | 3627 +++++------ src/Parallel-Solvers/FEA_Module_Inertial.h | 79 +- .../Parallel-Explicit/CMakeLists.txt | 2 +- .../FEA_Module_Dynamic_Elasticity.cpp | 2982 +++++----- .../FEA_Module_Dynamic_Elasticity.h | 1069 ++-- .../Dynamic_Elastic_Solver/boundary.cpp | 80 +- .../elastic_optimization.cpp | 2833 ++++----- .../Dynamic_Elastic_Solver/force_elastic.cpp | 2208 +++---- .../force_gradients_elastic.cpp | 1280 ++-- .../Dynamic_Elastic_Solver/geometry.cpp | 682 ++- .../Dynamic_Elastic_Solver/mesh.h | 761 ++- .../Dynamic_Elastic_Solver/momentum.cpp | 485 +- .../Dynamic_Elastic_Solver/properties.cpp | 193 +- .../Dynamic_Elastic_Solver/state.h | 104 +- .../time_integration.cpp | 236 +- .../Eulerian_Solver/FEA_Module_Eulerian.cpp | 1823 +++--- .../Eulerian_Solver/FEA_Module_Eulerian.h | 488 +- .../Simulation_Parameters_Eulerian.h | 162 +- .../Eulerian_Solver/eulerian_geometry.cpp | 42 +- .../eulerian_time_integration.cpp | 240 +- .../Parallel-Explicit/Explicit_Solver.cpp | 5260 ++++++++-------- .../Parallel-Explicit/Explicit_Solver.h | 179 +- .../SGH_Solver/CMakeLists.txt | 20 +- .../SGH_Solver/FEA_Module_SGH.cpp | 2615 -------- .../SGH_Solver/FEA_Module_SGH.h | 697 --- .../SGH_Solver/energy_sgh.cpp | 57 - .../SGH_Solver/force_gradients_sgh.cpp | 2059 ------- .../Parallel-Explicit/SGH_Solver/geometry.cpp | 1631 ----- .../SGH_Solver/include/FEA_Module_SGH.h | 700 +++ .../SGH_Solver/{ => include}/mesh.h | 763 ++- .../SGH_Solver/include/state.h | 80 + .../Parallel-Explicit/SGH_Solver/momentum.cpp | 417 -- .../SGH_Solver/power_gradients_sgh.cpp | 469 -- .../SGH_Solver/sgh_optimization.cpp | 1925 ------ .../SGH_Solver/src/FEA_Module_SGH.cpp | 1725 ++++++ .../SGH_Solver/{ => src}/boundary.cpp | 80 +- .../SGH_Solver/src/energy_sgh.cpp | 54 + .../SGH_Solver/src/force_gradients_sgh.cpp | 2063 +++++++ .../SGH_Solver/{ => src}/force_sgh.cpp | 888 ++- .../SGH_Solver/src/geometry.cpp | 1643 +++++ .../SGH_Solver/src/momentum.cpp | 391 ++ .../SGH_Solver/src/power_gradients_sgh.cpp | 495 ++ .../SGH_Solver/{ => src}/properties.cpp | 179 +- .../SGH_Solver/src/setup_sgh.cpp | 958 +++ .../SGH_Solver/src/sgh_optimization.cpp | 1862 ++++++ .../SGH_Solver/src/time_integration.cpp | 231 + .../Parallel-Explicit/SGH_Solver/state.h | 108 - .../SGH_Solver/time_integration.cpp | 237 - .../Kinetic_Energy_Minimize.h | 634 +- .../Parallel-Explicit/example_simple.yaml | 4 +- .../Parallel-Explicit/main.cpp | 80 +- .../Parallel-Explicit/outputs.cpp | 1279 ++-- src/Parallel-Solvers/Solver.cpp | 5288 +++++++++-------- src/Parallel-Solvers/Solver.h | 440 +- src/Parallel-Solvers/node_combination.cpp | 73 +- src/Parallel-Solvers/node_combination.h | 147 +- 58 files changed, 28787 insertions(+), 27374 deletions(-) delete mode 100644 src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/FEA_Module_SGH.cpp delete mode 100644 src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/FEA_Module_SGH.h delete mode 100644 src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/energy_sgh.cpp delete mode 100644 src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/force_gradients_sgh.cpp delete mode 100644 src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/geometry.cpp create mode 100644 src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/FEA_Module_SGH.h rename src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/{ => include}/mesh.h (60%) create mode 100644 src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/state.h delete mode 100644 src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/momentum.cpp delete mode 100644 src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/power_gradients_sgh.cpp delete mode 100644 src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/sgh_optimization.cpp create mode 100644 src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/FEA_Module_SGH.cpp rename src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/{ => src}/boundary.cpp (50%) create mode 100644 src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/energy_sgh.cpp create mode 100644 src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp rename src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/{ => src}/force_sgh.cpp (54%) create mode 100644 src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/geometry.cpp create mode 100644 src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/momentum.cpp create mode 100644 src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/power_gradients_sgh.cpp rename src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/{ => src}/properties.cpp (63%) create mode 100644 src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/setup_sgh.cpp create mode 100644 src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/sgh_optimization.cpp create mode 100644 src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/time_integration.cpp delete mode 100644 src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/state.h delete mode 100644 src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/time_integration.cpp diff --git a/src/Parallel-Solvers/FEA_Module.cpp b/src/Parallel-Solvers/FEA_Module.cpp index 6544cf279..3cf5165ca 100644 --- a/src/Parallel-Solvers/FEA_Module.cpp +++ b/src/Parallel-Solvers/FEA_Module.cpp @@ -11,14 +11,14 @@ This program is open source under the BSD-3 License. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - + 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - + 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - + 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. @@ -43,91 +43,93 @@ #define BC_EPSILON 1.0e-8 using namespace utils; -FEA_Module::FEA_Module(Solver *Solver_Pointer) { - - Solver_Pointer_ = Solver_Pointer; - simparam = &Solver_Pointer->simparam; - - num_dim = simparam->num_dims; - num_gauss_points = simparam->num_gauss_points; - - //obtain global and local node and element counts - num_nodes = Solver_Pointer->num_nodes; - num_elem = Solver_Pointer->num_elem; - nall_nodes = Solver_Pointer->nall_nodes; - rnum_elem = Solver_Pointer->rnum_elem; - nlocal_nodes = Solver_Pointer->nlocal_nodes; - nghost_nodes = Solver_Pointer->nghost_nodes; - max_nodes_per_element = Solver_Pointer->max_nodes_per_element; - - hessvec_count = update_count = 0; - linear_solve_time = hessvec_time = hessvec_linear_time = 0; - last_compute_step = -1; - - Matrix_alloc=0; - gradient_print_sync = 0; - //RCP pointer to *this (Parallel Nonlinear Solver Object) - //FEM_pass = Teuchos::rcp(this); - - //Trilinos output stream - std::ostream &out = std::cout; - fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - (*fos).setOutputToRootOnly(0); - - //MPI Data copy - myrank = Solver_Pointer->myrank; - nranks = Solver_Pointer->nranks; - world = Solver_Pointer->world; - importer = Solver_Pointer->importer; - ghost_importer = Solver_Pointer->ghost_importer; - node_sorting_importer = Solver_Pointer->node_sorting_importer; - element_sorting_importer = Solver_Pointer->element_sorting_importer; - dof_importer = Solver_Pointer->dof_importer; - - //obtain node and element maps - comm = Solver_Pointer->comm; - map = Solver_Pointer->map; //map of node indices - ghost_node_map = Solver_Pointer->ghost_node_map; //map of node indices with ghosts on each rank - all_node_map = Solver_Pointer->all_node_map; //map of node indices with ghosts on each rank - element_map = Solver_Pointer->element_map; //non overlapping map of elements owned by each rank used in reduction ops - all_element_map = Solver_Pointer->all_element_map; //overlapping map of elements connected to the local nodes in each rank - local_dof_map = Solver_Pointer->local_dof_map; //map of local dofs (typically num_node_local*num_dim) - all_dof_map = Solver_Pointer->all_dof_map; //map of local and ghost dofs (typically num_node_all*num_dim) - - //obtain mesh coordinates, densities, and element connectivity - global_nodes_in_elem_distributed = Solver_Pointer->global_nodes_in_elem_distributed; //element to node connectivity table - node_nconn_distributed = Solver_Pointer->node_nconn_distributed; //how many elements a node is connected to - node_coords_distributed = Solver_Pointer->node_coords_distributed; - all_node_coords_distributed = Solver_Pointer->all_node_coords_distributed; - initial_node_coords_distributed = Solver_Pointer->node_coords_distributed; - all_initial_node_coords_distributed = Solver_Pointer->all_node_coords_distributed; - design_node_densities_distributed = Solver_Pointer->design_node_densities_distributed; - filtered_node_densities_distributed = Solver_Pointer->filtered_node_densities_distributed; - test_node_densities_distributed = Solver_Pointer->test_node_densities_distributed; - all_node_densities_distributed = Solver_Pointer->all_node_densities_distributed; - all_filtered_node_densities_distributed = Solver_Pointer->all_filtered_node_densities_distributed; - Global_Element_Densities = Solver_Pointer->Global_Element_Densities; - Element_Types = Solver_Pointer->Element_Types; - - //element select data - element_select = Solver_Pointer->element_select; - element_select->choose_3Delem_type(Element_Types(0), elem); - - //obtain boundary condition and loading data - nboundary_patches = Solver_Pointer->nboundary_patches; - Boundary_Patches = Solver_Pointer->Boundary_Patches; - node_specified_bcs = false; - //initialize for default - num_boundary_conditions = 0; - - //flag init - body_term_flag = nonzero_bc_flag = false; - - //output data - noutput = 0; +FEA_Module::FEA_Module(Solver* Solver_Pointer) +{ + Solver_Pointer_ = Solver_Pointer; + simparam = &Solver_Pointer->simparam; + + num_dim = simparam->num_dims; + num_gauss_points = simparam->num_gauss_points; + + // obtain global and local node and element counts + num_nodes = Solver_Pointer->num_nodes; + num_elem = Solver_Pointer->num_elem; + nall_nodes = Solver_Pointer->nall_nodes; + rnum_elem = Solver_Pointer->rnum_elem; + nlocal_nodes = Solver_Pointer->nlocal_nodes; + nghost_nodes = Solver_Pointer->nghost_nodes; + max_nodes_per_element = Solver_Pointer->max_nodes_per_element; + + hessvec_count = update_count = 0; + linear_solve_time = hessvec_time = hessvec_linear_time = 0; + last_compute_step = -1; + + Matrix_alloc = 0; + gradient_print_sync = 0; + // RCP pointer to *this (Parallel Nonlinear Solver Object) + // FEM_pass = Teuchos::rcp(this); + + // Trilinos output stream + std::ostream& out = std::cout; + fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + (*fos).setOutputToRootOnly(0); + + // MPI Data copy + myrank = Solver_Pointer->myrank; + nranks = Solver_Pointer->nranks; + world = Solver_Pointer->world; + importer = Solver_Pointer->importer; + ghost_importer = Solver_Pointer->ghost_importer; + node_sorting_importer = Solver_Pointer->node_sorting_importer; + element_sorting_importer = Solver_Pointer->element_sorting_importer; + dof_importer = Solver_Pointer->dof_importer; + + // obtain node and element maps + comm = Solver_Pointer->comm; + map = Solver_Pointer->map; // map of node indices + ghost_node_map = Solver_Pointer->ghost_node_map; // map of node indices with ghosts on each rank + all_node_map = Solver_Pointer->all_node_map; // map of node indices with ghosts on each rank + element_map = Solver_Pointer->element_map; // non overlapping map of elements owned by each rank used in reduction ops + all_element_map = Solver_Pointer->all_element_map; // overlapping map of elements connected to the local nodes in each rank + local_dof_map = Solver_Pointer->local_dof_map; // map of local dofs (typically num_node_local*num_dim) + all_dof_map = Solver_Pointer->all_dof_map; // map of local and ghost dofs (typically num_node_all*num_dim) + + // obtain mesh coordinates, densities, and element connectivity + global_nodes_in_elem_distributed = Solver_Pointer->global_nodes_in_elem_distributed; // element to node connectivity table + node_nconn_distributed = Solver_Pointer->node_nconn_distributed; // how many elements a node is connected to + node_coords_distributed = Solver_Pointer->node_coords_distributed; + all_node_coords_distributed = Solver_Pointer->all_node_coords_distributed; + initial_node_coords_distributed = Solver_Pointer->node_coords_distributed; + all_initial_node_coords_distributed = Solver_Pointer->all_node_coords_distributed; + design_node_densities_distributed = Solver_Pointer->design_node_densities_distributed; + filtered_node_densities_distributed = Solver_Pointer->filtered_node_densities_distributed; + test_node_densities_distributed = Solver_Pointer->test_node_densities_distributed; + all_node_densities_distributed = Solver_Pointer->all_node_densities_distributed; + all_filtered_node_densities_distributed = Solver_Pointer->all_filtered_node_densities_distributed; + Global_Element_Densities = Solver_Pointer->Global_Element_Densities; + Element_Types = Solver_Pointer->Element_Types; + + // element select data + element_select = Solver_Pointer->element_select; + element_select->choose_3Delem_type(Element_Types(0), elem); + + // obtain boundary condition and loading data + nboundary_patches = Solver_Pointer->nboundary_patches; + Boundary_Patches = Solver_Pointer->Boundary_Patches; + node_specified_bcs = false; + // initialize for default + num_boundary_conditions = 0; + + // flag init + body_term_flag = nonzero_bc_flag = false; + + // output data + noutput = 0; } -FEA_Module::~FEA_Module() {} +FEA_Module::~FEA_Module() +{ +} /* ---------------------------------------------------------------------- find which boundary patches correspond to the given BC. @@ -135,98 +137,106 @@ FEA_Module::~FEA_Module() {} val = plane value, cylinder radius, shell radius ------------------------------------------------------------------------- */ -void FEA_Module::tag_boundaries(int bc_tag, real_t val, int bdy_set, real_t *patch_limits){ - - int is_on_set; - /* - if (bdy_set == num_bdy_sets_){ - std::cout << " ERROR: number of boundary sets must be increased by " - << bdy_set-num_bdy_sets_+1 << std::endl; - exit(0); - } - */ - - //test patch limits for feasibility - if(patch_limits != NULL){ - //test for upper bounds being greater than lower bounds - if(patch_limits[1] <= patch_limits[0]) std::cout << " Warning: patch limits for boundary condition are infeasible " << patch_limits[0] << " and " << patch_limits[1] << std::endl; - if(patch_limits[3] <= patch_limits[2]) std::cout << " Warning: patch limits for boundary condition are infeasible " << patch_limits[2] << " and " << patch_limits[3] << std::endl; - } - - // save the boundary vertices to this set that are on the plane - int counter = 0; - for (int iboundary_patch = 0; iboundary_patch < nboundary_patches; iboundary_patch++) { - - // check to see if this patch is on the specified plane - is_on_set = check_boundary(Boundary_Patches(iboundary_patch), bc_tag, val, patch_limits); // no=0, yes=1 - - if (is_on_set == 1){ - Boundary_Condition_Patches(bdy_set,counter) = iboundary_patch; - counter ++; +void FEA_Module::tag_boundaries(int bc_tag, real_t val, int bdy_set, real_t* patch_limits) +{ + int is_on_set; + /* + if (bdy_set == num_bdy_sets_){ + std::cout << " ERROR: number of boundary sets must be increased by " + << bdy_set-num_bdy_sets_+1 << std::endl; + exit(0); + } + */ + + // test patch limits for feasibility + if (patch_limits != NULL) + { + // test for upper bounds being greater than lower bounds + if (patch_limits[1] <= patch_limits[0]) + { + std::cout << " Warning: patch limits for boundary condition are infeasible " << patch_limits[0] << " and " << patch_limits[1] << std::endl; + } + if (patch_limits[3] <= patch_limits[2]) + { + std::cout << " Warning: patch limits for boundary condition are infeasible " << patch_limits[2] << " and " << patch_limits[3] << std::endl; + } } - } // end for bdy_patch - - // save the number of bdy patches in the set - NBoundary_Condition_Patches(bdy_set) = counter; - - *fos << " tagged boundary patches " << std::endl; + + // save the boundary vertices to this set that are on the plane + int counter = 0; + for (int iboundary_patch = 0; iboundary_patch < nboundary_patches; iboundary_patch++) + { + // check to see if this patch is on the specified plane + is_on_set = check_boundary(Boundary_Patches(iboundary_patch), bc_tag, val, patch_limits); // no=0, yes=1 + + if (is_on_set == 1) + { + Boundary_Condition_Patches(bdy_set, counter) = iboundary_patch; + counter++; + } + } // end for bdy_patch + + // save the number of bdy patches in the set + NBoundary_Condition_Patches(bdy_set) = counter; + + *fos << " tagged boundary patches " << std::endl; } /* ------------------------------------------------------------------------------------------- Communicate ghosts using the current optimization design data ---------------------------------------------------------------------------------------------- */ -void FEA_Module::comm_densities(Teuchos::RCP zp){ - - //set density vector to the current value chosen by the optimizer - test_node_densities_distributed = zp; - - //debug print of design vector - //std::ostream &out = std::cout; - //Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - //if(myrank==0) - //*fos << "Density data :" << std::endl; - //node_densities_distributed->describe(*fos,Teuchos::VERB_EXTREME); - //*fos << std::endl; - //std::fflush(stdout); - - //communicate design densities - //create import object using local node indices map and all indices map - Tpetra::Import importer(map, all_node_map); - - //comms to get ghosts - all_node_densities_distributed->doImport(*test_node_densities_distributed, importer, Tpetra::INSERT); - - //update_count++; - //if(update_count==1){ - //MPI_Barrier(world); - //MPI_Abort(world,4); - //} +void FEA_Module::comm_densities(Teuchos::RCP zp) +{ + // set density vector to the current value chosen by the optimizer + test_node_densities_distributed = zp; + + // debug print of design vector + // std::ostream &out = std::cout; + // Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + // if(myrank==0) + // *fos << "Density data :" << std::endl; + // node_densities_distributed->describe(*fos,Teuchos::VERB_EXTREME); + // *fos << std::endl; + // std::fflush(stdout); + + // communicate design densities + // create import object using local node indices map and all indices map + Tpetra::Import importer(map, all_node_map); + + // comms to get ghosts + all_node_densities_distributed->doImport(*test_node_densities_distributed, importer, Tpetra::INSERT); + + // update_count++; + // if(update_count==1){ + // MPI_Barrier(world); + // MPI_Abort(world,4); + // } } -void FEA_Module::comm_filtered_densities(){ - - //debug print of design vector - //std::ostream &out = std::cout; - //Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - //if(myrank==0) - //*fos << "Density data :" << std::endl; - //node_densities_distributed->describe(*fos,Teuchos::VERB_EXTREME); - //*fos << std::endl; - //std::fflush(stdout); - - //communicate design densities - //create import object using local node indices map and all indices map - Tpetra::Import importer(map, all_node_map); - - //comms to get ghosts - all_filtered_node_densities_distributed->doImport(*filtered_node_densities_distributed, importer, Tpetra::INSERT); - - //update_count++; - //if(update_count==1){ - //MPI_Barrier(world); - //MPI_Abort(world,4); - //} +void FEA_Module::comm_filtered_densities() +{ + // debug print of design vector + // std::ostream &out = std::cout; + // Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + // if(myrank==0) + // *fos << "Density data :" << std::endl; + // node_densities_distributed->describe(*fos,Teuchos::VERB_EXTREME); + // *fos << std::endl; + // std::fflush(stdout); + + // communicate design densities + // create import object using local node indices map and all indices map + Tpetra::Import importer(map, all_node_map); + + // comms to get ghosts + all_filtered_node_densities_distributed->doImport(*filtered_node_densities_distributed, importer, Tpetra::INSERT); + + // update_count++; + // if(update_count==1){ + // MPI_Barrier(world); + // MPI_Abort(world,4); + // } } /* ---------------------------------------------------------------------- @@ -235,89 +245,116 @@ void FEA_Module::comm_filtered_densities(){ val = plane value, radius, radius ------------------------------------------------------------------------- */ -int FEA_Module::check_boundary(Node_Combination &Patch_Nodes, int bc_tag, real_t val, real_t *patch_limits){ - - int is_on_set = 1; - const_host_vec_array all_node_coords = all_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - - //Nodes on the Patch - auto node_list = Patch_Nodes.node_set; - int num_dim = simparam->num_dims; - size_t nnodes = node_list.size(); - size_t node_rid; - real_t node_coord[num_dim]; - int dim_other1, dim_other2; - CArrayKokkos node_on_flags(nnodes, "node_on_flags"); - - //initialize - for(int inode = 0; inode < nnodes; inode++) node_on_flags(inode) = 0; - - if(bc_tag==0){ - dim_other1 = 1; - dim_other2 = 2; - } - else if(bc_tag==1){ - dim_other1 = 0; - dim_other2 = 2; - } - else if(bc_tag==2){ - dim_other1 = 0; - dim_other2 = 1; - } - - - //test for planes - if(bc_tag < 3) - for(int inode = 0; inode < nnodes; inode++){ - - node_rid = all_node_map->getLocalElement(node_list(inode)); - for(int init=0; init < num_dim; init++){ - node_coord[init] = all_node_coords(node_rid,init); +int FEA_Module::check_boundary(Node_Combination& Patch_Nodes, int bc_tag, real_t val, real_t* patch_limits) +{ + int is_on_set = 1; + const_host_vec_array all_node_coords = all_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + + // Nodes on the Patch + auto node_list = Patch_Nodes.node_set; + int num_dim = simparam->num_dims; + size_t nnodes = node_list.size(); + size_t node_rid; + real_t node_coord[num_dim]; + int dim_other1, dim_other2; + CArrayKokkos node_on_flags(nnodes, "node_on_flags"); + + // initialize + for (int inode = 0; inode < nnodes; inode++) + { + node_on_flags(inode) = 0; + } + + if (bc_tag == 0) + { + dim_other1 = 1; + dim_other2 = 2; } - if ( fabs(node_coord[bc_tag] - val) <= BC_EPSILON){ node_on_flags(inode) = 1; - - //test if within patch segment if user specified - if(patch_limits!=NULL){ - if (node_coord[dim_other1] - patch_limits[0] <= -BC_EPSILON) node_on_flags(inode) = 0; - if (node_coord[dim_other1] - patch_limits[1] >= BC_EPSILON) node_on_flags(inode) = 0; - if (node_coord[dim_other2] - patch_limits[2] <= -BC_EPSILON) node_on_flags(inode) = 0; - if (node_coord[dim_other2] - patch_limits[3] >= BC_EPSILON) node_on_flags(inode) = 0; - } + else if (bc_tag == 1) + { + dim_other1 = 0; + dim_other2 = 2; } - //debug print of node id and node coord - //std::cout << "node coords on task " << myrank << " for node " << node_rid << std::endl; - //std::cout << "coord " <getLocalElement(node_list(inode)); + for (int init = 0; init < num_dim; init++) + { + node_coord[init] = all_node_coords(node_rid, init); + } + if (fabs(node_coord[bc_tag] - val) <= BC_EPSILON) + { + node_on_flags(inode) = 1; + + // test if within patch segment if user specified + if (patch_limits != NULL) + { + if (node_coord[dim_other1] - patch_limits[0] <= -BC_EPSILON) + { + node_on_flags(inode) = 0; + } + if (node_coord[dim_other1] - patch_limits[1] >= BC_EPSILON) + { + node_on_flags(inode) = 0; + } + if (node_coord[dim_other2] - patch_limits[2] <= -BC_EPSILON) + { + node_on_flags(inode) = 0; + } + if (node_coord[dim_other2] - patch_limits[3] >= BC_EPSILON) + { + node_on_flags(inode) = 0; + } + } + } + // debug print of node id and node coord + // std::cout << "node coords on task " << myrank << " for node " << node_rid << std::endl; + // std::cout << "coord " < #include #include -//#include "Tpetra_Details_makeColMap.hpp" +// #include "Tpetra_Details_makeColMap.hpp" #include "Tpetra_Details_DefaultTypes.hpp" #include "utilities.h" #include "matar.h" #include "elements.h" #include "node_combination.h" -//#include "Simulation_Parameters/Simulation_Parameters.h" +// #include "Simulation_Parameters/Simulation_Parameters.h" using namespace mtr; -//forward declare +// forward declare class Solver; -//forward declarations -namespace ROL{ - template - class Problem; -} +// forward declarations +namespace ROL +{ +template +class Problem; +} // namespace ROL class Simulation_Parameters; enum class FEA_MODULE_TYPE; enum class BOUNDARY_TYPE; -class FEA_Module{ - +class FEA_Module +{ public: - FEA_Module(Solver *Solver_Pointer); - virtual ~FEA_Module(); + FEA_Module(Solver* Solver_Pointer); + virtual ~FEA_Module(); + + // Trilinos type definitions + typedef Tpetra::Map<>::local_ordinal_type LO; + typedef Tpetra::Map<>::global_ordinal_type GO; + + typedef Tpetra::CrsMatrix MAT; + typedef const Tpetra::CrsMatrix const_MAT; + typedef Tpetra::MultiVector MV; + typedef Tpetra::MultiVector MCONN; + + typedef Kokkos::ViewTraits::size_type SizeType; + typedef Tpetra::Details::DefaultTypes::node_type node_type; + using traits = Kokkos::ViewTraits; + + using array_layout = typename traits::array_layout; + using execution_space = typename traits::execution_space; + using device_type = typename traits::device_type; + using memory_traits = typename traits::memory_traits; + using global_size_t = Tpetra::global_size_t; + + typedef Kokkos::View values_array; + typedef Kokkos::View global_indices_array; + typedef Kokkos::View indices_array; + // typedef Kokkos::View row_pointers; + typedef MAT::local_graph_device_type::row_map_type::non_const_type row_pointers; + // typedef Kokkos::DualView::t_dev vec_array; + typedef MV::dual_view_type::t_dev vec_array; + typedef MV::dual_view_type::t_host host_vec_array; + typedef Kokkos::View const_host_vec_array; + typedef Kokkos::View const_vec_array; + typedef MV::dual_view_type dual_vec_array; + typedef MCONN::dual_view_type dual_elem_conn_array; + typedef MCONN::dual_view_type::t_host host_elem_conn_array; + typedef MCONN::dual_view_type::t_dev elem_conn_array; + typedef Kokkos::View const_host_elem_conn_array; + typedef Kokkos::View const_elem_conn_array; + typedef Kokkos::View const_host_int_array; + typedef Kokkos::View const_host_bool_array; + + // initializes memory for arrays used in the global stiffness matrix assembly + // initialize data for boundaries of the model and storage for boundary conditions and applied loads + virtual void init_boundaries() {} + + // initializes memory for arrays used in the global stiffness matrix assembly + virtual void init_boundary_sets(int num_boundary_sets) {} + + virtual void init_assembly() {} + + virtual void assemble_matrix() {} + + virtual void assemble_vector() {} + + // interfaces between user input and creating data structures for bcs + virtual void generate_bcs() {} + + // interfaces between user input and creating data structures for applied loads + virtual void generate_applied_loads() {} + + virtual void setup() {} + + virtual void module_cleanup() {} + + virtual int solve() { return 0; } + + virtual int eigensolve() { return 0; } + + virtual void read_conditions_ansys_dat(std::ifstream* in, std::streampos before_condition_header) {} + + virtual void linear_solver_parameters() {} + + virtual void comm_variables(Teuchos::RCP zp) {} + + virtual void comm_densities(Teuchos::RCP zp); + + virtual void comm_filtered_densities(); + + virtual void update_linear_solve(Teuchos::RCP zp, int compute_step) {} + + virtual void update_forward_solve(Teuchos::RCP zp) {} + + virtual void local_matrix(int ielem, CArrayKokkos& Local_Matrix) {} + + virtual void local_matrix_multiply(int ielem, CArrayKokkos& Local_Matrix) {} + + virtual void Element_Material_Properties(size_t ielem, real_t& Element_Modulus, real_t& Poisson_Ratio, real_t density) {} + + virtual void Gradient_Element_Material_Properties(size_t ielem, real_t& Element_Modulus, real_t& Poisson_Ratio, real_t density) {} + + virtual void Concavity_Element_Material_Properties(size_t ielem, real_t& Element_Modulus, real_t& Poisson_Ratio, real_t density) {} + + virtual void Body_Term(size_t ielem, real_t density, real_t* forces) {} + + virtual void Gradient_Body_Term(size_t ielem, real_t density, real_t* forces) {} + + virtual void tag_boundaries(int this_bc_tag, real_t val, int bdy_set, real_t* patch_limits = NULL); + + virtual int check_boundary(Node_Combination& Patch_Nodes, int this_bc_tag, real_t val, real_t* patch_limits); + + virtual void compute_output() {} + + virtual void output_control() {} + + virtual void write_data(std::map& point_data_scalars_double, + std::map& point_data_vectors_double, + std::map& cell_data_scalars_double, + std::map& cell_data_scalars_int, + std::map>& cell_data_fields_double) {} + + virtual void sort_output(Teuchos::RCP> sorted_map) {} // node data outputs - //Trilinos type definitions - typedef Tpetra::Map<>::local_ordinal_type LO; - typedef Tpetra::Map<>::global_ordinal_type GO; + virtual void sort_element_output(Teuchos::RCP> sorted_element_map) {} // element data outputs - typedef Tpetra::CrsMatrix MAT; - typedef const Tpetra::CrsMatrix const_MAT; - typedef Tpetra::MultiVector MV; - typedef Tpetra::MultiVector MCONN; + virtual void collect_output(Teuchos::RCP> global_reduce_map) {} - typedef Kokkos::ViewTraits::size_type SizeType; - typedef Tpetra::Details::DefaultTypes::node_type node_type; - using traits = Kokkos::ViewTraits; - - using array_layout = typename traits::array_layout; - using execution_space = typename traits::execution_space; - using device_type = typename traits::device_type; - using memory_traits = typename traits::memory_traits; - using global_size_t = Tpetra::global_size_t; - - typedef Kokkos::View values_array; - typedef Kokkos::View global_indices_array; - typedef Kokkos::View indices_array; - //typedef Kokkos::View row_pointers; - typedef MAT::local_graph_device_type::row_map_type::non_const_type row_pointers; - //typedef Kokkos::DualView::t_dev vec_array; - typedef MV::dual_view_type::t_dev vec_array; - typedef MV::dual_view_type::t_host host_vec_array; - typedef Kokkos::View const_host_vec_array; - typedef Kokkos::View const_vec_array; - typedef MV::dual_view_type dual_vec_array; - typedef MCONN::dual_view_type dual_elem_conn_array; - typedef MCONN::dual_view_type::t_host host_elem_conn_array; - typedef MCONN::dual_view_type::t_dev elem_conn_array; - typedef Kokkos::View const_host_elem_conn_array; - typedef Kokkos::View const_elem_conn_array; - typedef Kokkos::View const_host_int_array; - typedef Kokkos::View const_host_bool_array; - - //initializes memory for arrays used in the global stiffness matrix assembly - //initialize data for boundaries of the model and storage for boundary conditions and applied loads - virtual void init_boundaries() {} - - //initializes memory for arrays used in the global stiffness matrix assembly - virtual void init_boundary_sets(int num_boundary_sets) {} + virtual void node_density_constraints(host_vec_array node_densities_lower_bound) {} - virtual void init_assembly() {} + virtual void compute_topology_optimization_adjoint() {} // Force does not depend on node coords and velocity - virtual void assemble_matrix() {} + virtual void compute_topology_optimization_adjoint_full() {} // Force depends on node coords and velocity - virtual void assemble_vector() {} + virtual void compute_topology_optimization_gradient(const_vec_array design_densities, vec_array gradients) {} - //interfaces between user input and creating data structures for bcs - virtual void generate_bcs() {} - - //interfaces between user input and creating data structures for applied loads - virtual void generate_applied_loads() {} + virtual void compute_topology_optimization_gradient_full(Teuchos::RCP design_densities_distributed, Teuchos::RCP design_gradients_distributed) {} - virtual void setup() {} + // interfacing information + FEA_MODULE_TYPE Module_Type; + int last_compute_step; - virtual void module_cleanup() {} + // output stream + Teuchos::RCP fos; - virtual int solve() {return 0;} + std::shared_ptr element_select; + elements::Element3D* elem; + elements::Element2D* elem2D; - virtual int eigensolve() {return 0;} + Simulation_Parameters* simparam; + Solver* Solver_Pointer_; + int num_dim; + int num_gauss_points; + int my_fea_module_index_; - virtual void read_conditions_ansys_dat(std::ifstream *in, std::streampos before_condition_header) {} + // Local FEA data + size_t nlocal_nodes; + dual_vec_array dual_node_coords; // coordinates of the nodes + dual_vec_array dual_node_densities; // topology optimization design variable + dual_elem_conn_array dual_nodes_in_elem; // dual view of element connectivity to nodes + host_elem_conn_array nodes_in_elem; // host view of element connectivity to nodes + CArrayKokkos Element_Types; + CArrayKokkos Nodes_Per_Element_Type; + CArrayKokkos corner_value_storage; + CArrayKokkos corner_vector_storage; + CArrayKokkos corner_gradient_storage; - virtual void linear_solver_parameters() {} + // Ghost data on this MPI rank + size_t nghost_nodes; + CArrayKokkos ghost_nodes; + CArrayKokkos ghost_node_ranks; - virtual void comm_variables(Teuchos::RCP zp) {} + // Local FEA data including ghosts + size_t nall_nodes; + size_t rnum_elem; - virtual void comm_densities(Teuchos::RCP zp); - - virtual void comm_filtered_densities(); - - virtual void update_linear_solve(Teuchos::RCP zp, int compute_step) {} - - virtual void update_forward_solve(Teuchos::RCP zp) {} - - virtual void local_matrix(int ielem, CArrayKokkos &Local_Matrix) {} - - virtual void local_matrix_multiply(int ielem, CArrayKokkos &Local_Matrix) {} - - virtual void Element_Material_Properties(size_t ielem, real_t &Element_Modulus, real_t &Poisson_Ratio, real_t density) {} - - virtual void Gradient_Element_Material_Properties(size_t ielem, real_t &Element_Modulus, real_t &Poisson_Ratio, real_t density) {} - - virtual void Concavity_Element_Material_Properties(size_t ielem, real_t &Element_Modulus, real_t &Poisson_Ratio, real_t density) {} - - virtual void Body_Term(size_t ielem, real_t density, real_t *forces) {} - - virtual void Gradient_Body_Term(size_t ielem, real_t density, real_t *forces) {} - - virtual void tag_boundaries(int this_bc_tag, real_t val, int bdy_set, real_t *patch_limits = NULL); - - virtual int check_boundary(Node_Combination &Patch_Nodes, int this_bc_tag, real_t val, real_t *patch_limits); - - virtual void compute_output(){} - - virtual void output_control(){} - - virtual void write_data(std::map &point_data_scalars_double, - std::map &point_data_vectors_double, - std::map &cell_data_scalars_double, - std::map &cell_data_scalars_int, - std::map > &cell_data_fields_double){} - - virtual void sort_output(Teuchos::RCP> sorted_map){} //node data outputs - - virtual void sort_element_output(Teuchos::RCP> sorted_element_map){} //element data outputs - - virtual void collect_output(Teuchos::RCP> global_reduce_map){} - - virtual void node_density_constraints(host_vec_array node_densities_lower_bound){} - - virtual void compute_topology_optimization_adjoint() {} //Force does not depend on node coords and velocity - - virtual void compute_topology_optimization_adjoint_full() {} //Force depends on node coords and velocity - - virtual void compute_topology_optimization_gradient(const_vec_array design_densities, vec_array gradients) {} - - virtual void compute_topology_optimization_gradient_full(Teuchos::RCP design_densities_distributed, Teuchos::RCP design_gradients_distributed) {} - - //interfacing information - FEA_MODULE_TYPE Module_Type; - int last_compute_step; - - //output stream - Teuchos::RCP fos; - - std::shared_ptr element_select; - elements::Element3D *elem; - elements::Element2D *elem2D; - - - Simulation_Parameters *simparam; - Solver *Solver_Pointer_; - int num_dim; - int num_gauss_points; - int my_fea_module_index_; - - //Local FEA data - size_t nlocal_nodes; - dual_vec_array dual_node_coords; //coordinates of the nodes - dual_vec_array dual_node_densities; //topology optimization design variable - dual_elem_conn_array dual_nodes_in_elem; //dual view of element connectivity to nodes - host_elem_conn_array nodes_in_elem; //host view of element connectivity to nodes - CArrayKokkos Element_Types; - CArrayKokkos Nodes_Per_Element_Type; - CArrayKokkos corner_value_storage; - CArrayKokkos corner_vector_storage; - CArrayKokkos corner_gradient_storage; - - //Ghost data on this MPI rank - size_t nghost_nodes; - CArrayKokkos ghost_nodes; - CArrayKokkos ghost_node_ranks; - - //Local FEA data including ghosts - size_t nall_nodes; - size_t rnum_elem; - - //Global FEA data - long long int num_nodes, num_elem; - Teuchos::RCP > comm; - Teuchos::RCP > map; //map of node indices - Teuchos::RCP > ghost_node_map; //map of node indices with ghosts on each rank - Teuchos::RCP > all_node_map; //map of node indices with ghosts on each rank - Teuchos::RCP > element_map; //non overlapping map of elements owned by each rank used in reduction ops - Teuchos::RCP > all_element_map; //overlapping map of elements connected to the local nodes in each rank - Teuchos::RCP > local_dof_map; //map of local dofs (typically num_node_local*num_dim) - Teuchos::RCP > all_dof_map; //map of local and ghost dofs (typically num_node_all*num_dim) - Teuchos::RCP global_nodes_in_elem_distributed; //element to node connectivity table - Teuchos::RCP node_nconn_distributed; //how many elements a node is connected to - Teuchos::RCP node_coords_distributed; - Teuchos::RCP all_node_coords_distributed; - Teuchos::RCP initial_node_coords_distributed; - Teuchos::RCP all_initial_node_coords_distributed; - Teuchos::RCP design_node_densities_distributed; - Teuchos::RCP filtered_node_densities_distributed; - Teuchos::RCP test_node_densities_distributed; - Teuchos::RCP all_node_densities_distributed; - Teuchos::RCP all_filtered_node_densities_distributed; - Teuchos::RCP Global_Element_Densities; - - //Boundary Conditions Data - //CArray Patch_Nodes; - size_t nboundary_patches; - size_t num_boundary_conditions; - int current_bdy_id; - CArrayKokkos Boundary_Patches; - CArrayKokkos Boundary_Condition_Patches; //set of patches corresponding to each boundary condition - CArrayKokkos NBoundary_Condition_Patches; - CArrayKokkos Boundary_Condition_Patches_strides; - - //element selection parameters and data - size_t max_nodes_per_element; - - //determines if rhs gets a contribution from bcs - bool nonzero_bc_flag; - - //body force parameters - bool body_term_flag; - real_t *gravity_vector; - - //lists what kind of boundary condition the nodal DOF is subjected to if any - CArrayKokkos Node_DOF_Boundary_Condition_Type; - //lists what kind of boundary condition each boundary set is assigned to - CArrayKokkos Boundary_Condition_Type_List; - - //number of displacement boundary conditions acting on nodes; used to size the reduced global stiffness map - size_t Number_DOF_BCS; - - //MPI data - int myrank; //index of this mpi rank in the world communicator - int nranks; //number of mpi ranks in the world communicator - MPI_Comm world; //stores the default communicator object (MPI_COMM_WORLD) - Teuchos::RCP> importer; //all node comms - Teuchos::RCP> ghost_importer; //ghost node comms - Teuchos::RCP> node_sorting_importer; //sorted node comms - Teuchos::RCP> element_sorting_importer; //sorted element comms - Teuchos::RCP> dof_importer; //ghost dof comms - - //! mapping used to get local ghost index from the global ID. - //typedef ::Tpetra::Details::FixedHashTable - //global_to_local_table_host_type; - - //global_to_local_table_host_type global2local_map; - //CArrayKokkos active_ranks; - - //Pertains to local mesh information being stored as prescribed by the row map - global_size_t min_gid; - global_size_t max_gid; - global_size_t index_base; - - //allocation flags to avoid repeat MV and global matrix construction - int Matrix_alloc; - - //debug flags - int gradient_print_sync; - - //Topology Optimization parameter - int penalty_power; - bool nodal_density_flag; - - //runtime and counters for performance output - double linear_solve_time, hessvec_time, hessvec_linear_time; - int update_count, hessvec_count; - - //nodal DOF output data - enum vector_styles {NODAL, DOF}; //multivector can store as ndof by 1 or nnode by vector_size - int noutput; - int displacement_index; - std::vector> output_dof_names; - std::vector module_outputs; - std::vector vector_style; - std::vector output_vector_sizes; - - //Pointer to ROL Problem for optimization solves - Teuchos::RCP> problem; - - //Explicit BC data kept for now until bc data is refactored/consolidated - // node ids in bdy_patch set - RaggedRightArrayKokkos bdy_nodes_in_set; - DCArrayKokkos num_bdy_nodes_in_set; - bool node_specified_bcs; //currently happens with ansys import - - // patch ids in bdy set - size_t num_bdy_sets; - DynamicRaggedRightArrayKokkos bdy_patches_in_set; - + // Global FEA data + long long int num_nodes, num_elem; + Teuchos::RCP> comm; + Teuchos::RCP> map; // map of node indices + Teuchos::RCP> ghost_node_map; // map of node indices with ghosts on each rank + Teuchos::RCP> all_node_map; // map of node indices with ghosts on each rank + Teuchos::RCP> element_map; // non overlapping map of elements owned by each rank used in reduction ops + Teuchos::RCP> all_element_map; // overlapping map of elements connected to the local nodes in each rank + Teuchos::RCP> local_dof_map; // map of local dofs (typically num_node_local*num_dim) + Teuchos::RCP> all_dof_map; // map of local and ghost dofs (typically num_node_all*num_dim) + Teuchos::RCP global_nodes_in_elem_distributed; // element to node connectivity table + Teuchos::RCP node_nconn_distributed; // how many elements a node is connected to + Teuchos::RCP node_coords_distributed; + Teuchos::RCP all_node_coords_distributed; + Teuchos::RCP initial_node_coords_distributed; + Teuchos::RCP all_initial_node_coords_distributed; + Teuchos::RCP design_node_densities_distributed; + Teuchos::RCP filtered_node_densities_distributed; + Teuchos::RCP test_node_densities_distributed; + Teuchos::RCP all_node_densities_distributed; + Teuchos::RCP all_filtered_node_densities_distributed; + Teuchos::RCP Global_Element_Densities; + + // Boundary Conditions Data + // CArray Patch_Nodes; + size_t nboundary_patches; + size_t num_boundary_conditions; + int current_bdy_id; + CArrayKokkos Boundary_Patches; + CArrayKokkos Boundary_Condition_Patches; // set of patches corresponding to each boundary condition + CArrayKokkos NBoundary_Condition_Patches; + CArrayKokkos Boundary_Condition_Patches_strides; + + // element selection parameters and data + size_t max_nodes_per_element; + + // determines if rhs gets a contribution from bcs + bool nonzero_bc_flag; + + // body force parameters + bool body_term_flag; + real_t* gravity_vector; + + // lists what kind of boundary condition the nodal DOF is subjected to if any + CArrayKokkos Node_DOF_Boundary_Condition_Type; + // lists what kind of boundary condition each boundary set is assigned to + CArrayKokkos Boundary_Condition_Type_List; + + // number of displacement boundary conditions acting on nodes; used to size the reduced global stiffness map + size_t Number_DOF_BCS; + + // MPI data + int myrank; // index of this mpi rank in the world communicator + int nranks; // number of mpi ranks in the world communicator + MPI_Comm world; // stores the default communicator object (MPI_COMM_WORLD) + Teuchos::RCP> importer; // all node comms + Teuchos::RCP> ghost_importer; // ghost node comms + Teuchos::RCP> node_sorting_importer; // sorted node comms + Teuchos::RCP> element_sorting_importer; // sorted element comms + Teuchos::RCP> dof_importer; // ghost dof comms + + // ! mapping used to get local ghost index from the global ID. + // typedef ::Tpetra::Details::FixedHashTable + // global_to_local_table_host_type; + + // global_to_local_table_host_type global2local_map; + // CArrayKokkos active_ranks; + + // Pertains to local mesh information being stored as prescribed by the row map + global_size_t min_gid; + global_size_t max_gid; + global_size_t index_base; + + // allocation flags to avoid repeat MV and global matrix construction + int Matrix_alloc; + + // debug flags + int gradient_print_sync; + + // Topology Optimization parameter + int penalty_power; + bool nodal_density_flag; + + // runtime and counters for performance output + double linear_solve_time, hessvec_time, hessvec_linear_time; + int update_count, hessvec_count; + + // nodal DOF output data + enum vector_styles { NODAL, DOF }; // multivector can store as ndof by 1 or nnode by vector_size + int noutput; + int displacement_index; + std::vector> output_dof_names; + std::vector module_outputs; + std::vector vector_style; + std::vector output_vector_sizes; + + // Pointer to ROL Problem for optimization solves + Teuchos::RCP> problem; + + // Explicit BC data kept for now until bc data is refactored/consolidated + // node ids in bdy_patch set + RaggedRightArrayKokkos bdy_nodes_in_set; + DCArrayKokkos num_bdy_nodes_in_set; + bool node_specified_bcs; // currently happens with ansys import + + // patch ids in bdy set + size_t num_bdy_sets; + DynamicRaggedRightArrayKokkos bdy_patches_in_set; }; #endif // end HEADER_H diff --git a/src/Parallel-Solvers/FEA_Module_Inertial.cpp b/src/Parallel-Solvers/FEA_Module_Inertial.cpp index ee7e29549..e84945ea7 100644 --- a/src/Parallel-Solvers/FEA_Module_Inertial.cpp +++ b/src/Parallel-Solvers/FEA_Module_Inertial.cpp @@ -11,14 +11,14 @@ This program is open source under the BSD-3 License. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - + 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - + 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - + 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. @@ -39,7 +39,7 @@ #include #include #include -#include +#include #include // fmin, fmax, abs note: fminl is long #include #include @@ -74,1795 +74,2090 @@ using namespace utils; - FEA_Module_Inertial::FEA_Module_Inertial( - Inertial_Parameters& params, Solver *Solver_Pointer, - const int my_fea_module_index) - : FEA_Module(Solver_Pointer) { - - //assign interfacing index - my_fea_module_index_ = my_fea_module_index; - Module_Type = FEA_MODULE_TYPE::Inertial; - - //acquire base class data from existing simparam in solver (gets yaml options etc.) - module_params = ¶ms; - simparam = &(Solver_Pointer->simparam); - - //TO parameters - nodal_density_flag = simparam->nodal_density_flag; - - //property initialization flags - mass_init = false; - com_init[0] = com_init[1] = com_init[2] = false; - - //property update counters - mass_update = com_update[0] = com_update[1] = com_update[2] = -1; - - //RCP initialization - mass_gradients_distributed = Teuchos::null; - center_of_mass_gradients_distributed = Teuchos::null; - - //construct per element inertial property vectors - Global_Element_Masses = Teuchos::rcp(new MV(element_map, 1)); - Global_Element_Volumes = Teuchos::rcp(new MV(element_map, 1)); - Global_Element_Moments_x = Teuchos::rcp(new MV(element_map, 1)); - Global_Element_Moments_y = Teuchos::rcp(new MV(element_map, 1)); - Global_Element_Moments_z = Teuchos::rcp(new MV(element_map, 1)); - Global_Element_Moments_of_Inertia_xx = Teuchos::rcp(new MV(element_map, 1)); - Global_Element_Moments_of_Inertia_yy = Teuchos::rcp(new MV(element_map, 1)); - Global_Element_Moments_of_Inertia_zz = Teuchos::rcp(new MV(element_map, 1)); - Global_Element_Moments_of_Inertia_xy = Teuchos::rcp(new MV(element_map, 1)); - Global_Element_Moments_of_Inertia_xz = Teuchos::rcp(new MV(element_map, 1)); - Global_Element_Moments_of_Inertia_yz = Teuchos::rcp(new MV(element_map, 1)); + Inertial_Parameters& params, Solver* Solver_Pointer, + const int my_fea_module_index) + : FEA_Module(Solver_Pointer) +{ + // assign interfacing index + my_fea_module_index_ = my_fea_module_index; + Module_Type = FEA_MODULE_TYPE::Inertial; + + // acquire base class data from existing simparam in solver (gets yaml options etc.) + module_params = ¶ms; + simparam = &(Solver_Pointer->simparam); + + // TO parameters + nodal_density_flag = simparam->nodal_density_flag; + + // property initialization flags + mass_init = false; + com_init[0] = com_init[1] = com_init[2] = false; + + // property update counters + mass_update = com_update[0] = com_update[1] = com_update[2] = -1; + + // RCP initialization + mass_gradients_distributed = Teuchos::null; + center_of_mass_gradients_distributed = Teuchos::null; + + // construct per element inertial property vectors + Global_Element_Masses = Teuchos::rcp(new MV(element_map, 1)); + Global_Element_Volumes = Teuchos::rcp(new MV(element_map, 1)); + Global_Element_Moments_x = Teuchos::rcp(new MV(element_map, 1)); + Global_Element_Moments_y = Teuchos::rcp(new MV(element_map, 1)); + Global_Element_Moments_z = Teuchos::rcp(new MV(element_map, 1)); + Global_Element_Moments_of_Inertia_xx = Teuchos::rcp(new MV(element_map, 1)); + Global_Element_Moments_of_Inertia_yy = Teuchos::rcp(new MV(element_map, 1)); + Global_Element_Moments_of_Inertia_zz = Teuchos::rcp(new MV(element_map, 1)); + Global_Element_Moments_of_Inertia_xy = Teuchos::rcp(new MV(element_map, 1)); + Global_Element_Moments_of_Inertia_xz = Teuchos::rcp(new MV(element_map, 1)); + Global_Element_Moments_of_Inertia_yz = Teuchos::rcp(new MV(element_map, 1)); } -FEA_Module_Inertial::~FEA_Module_Inertial(){ } +FEA_Module_Inertial::~FEA_Module_Inertial() +{ +} /* ---------------------------------------------------------------------- Compute the mass of each element; estimated with quadrature ------------------------------------------------------------------------- */ -void FEA_Module_Inertial::compute_element_masses(const_host_vec_array design_densities, bool max_flag, bool use_initial_coords){ - //local number of uniquely assigned elements - size_t nonoverlap_nelements = element_map->getLocalNumElements(); - //initialize memory for volume storage - host_vec_array Element_Masses = Global_Element_Masses->getLocalView(Tpetra::Access::ReadWrite); - if(!nodal_density_flag) compute_element_volumes(); - const_host_vec_array Element_Volumes = Global_Element_Volumes->getLocalView(Tpetra::Access::ReadOnly); - //local variable for host view in the dual view - const_host_vec_array all_node_coords; - if(use_initial_coords){ - all_node_coords = all_initial_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - } - else{ - all_node_coords = all_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - } - const_host_vec_array all_design_densities; - if(nodal_density_flag) - all_design_densities = all_node_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_host_elem_conn_array nodes_in_elem = global_nodes_in_elem_distributed->getLocalView (Tpetra::Access::ReadOnly); - int nodes_per_elem = elem->num_basis(); - int z_quad,y_quad,x_quad, direct_product_count; - size_t local_node_id; - LO ielem; - GO global_element_index; - - real_t Jacobian, current_density, weight_multiply; - //CArrayKokkos legendre_nodes_1D(num_gauss_points); - //CArrayKokkos legendre_weights_1D(num_gauss_points); - CArray legendre_nodes_1D(num_gauss_points); - CArray legendre_weights_1D(num_gauss_points); - real_t pointer_quad_coordinate[num_dim]; - real_t pointer_quad_coordinate_weight[num_dim]; - real_t pointer_interpolated_point[num_dim]; - real_t pointer_JT_row1[num_dim]; - real_t pointer_JT_row2[num_dim]; - real_t pointer_JT_row3[num_dim]; - ViewCArray quad_coordinate(pointer_quad_coordinate,num_dim); - ViewCArray quad_coordinate_weight(pointer_quad_coordinate_weight,num_dim); - ViewCArray interpolated_point(pointer_interpolated_point,num_dim); - ViewCArray JT_row1(pointer_JT_row1,num_dim); - ViewCArray JT_row2(pointer_JT_row2,num_dim); - ViewCArray JT_row3(pointer_JT_row3,num_dim); - - real_t pointer_basis_values[elem->num_basis()]; - real_t pointer_basis_derivative_s1[elem->num_basis()]; - real_t pointer_basis_derivative_s2[elem->num_basis()]; - real_t pointer_basis_derivative_s3[elem->num_basis()]; - ViewCArray basis_values(pointer_basis_values,elem->num_basis()); - ViewCArray basis_derivative_s1(pointer_basis_derivative_s1,elem->num_basis()); - ViewCArray basis_derivative_s2(pointer_basis_derivative_s2,elem->num_basis()); - ViewCArray basis_derivative_s3(pointer_basis_derivative_s3,elem->num_basis()); - CArrayKokkos nodal_positions(elem->num_basis(),num_dim); - CArrayKokkos nodal_density(elem->num_basis()); - - //initialize weights - elements::legendre_nodes_1D(legendre_nodes_1D,num_gauss_points); - elements::legendre_weights_1D(legendre_weights_1D,num_gauss_points); - - Solver::node_ordering_convention active_node_ordering_convention = Solver_Pointer_->active_node_ordering_convention; - CArrayKokkos convert_node_order(max_nodes_per_element); - if((active_node_ordering_convention == Solver::ENSIGHT && num_dim==3)||(active_node_ordering_convention == Solver::IJK && num_dim==2)){ - convert_node_order(0) = 0; - convert_node_order(1) = 1; - convert_node_order(2) = 3; - convert_node_order(3) = 2; - if(num_dim == 3){ - convert_node_order(4) = 4; - convert_node_order(5) = 5; - convert_node_order(6) = 7; - convert_node_order(7) = 6; +void FEA_Module_Inertial::compute_element_masses(const_host_vec_array design_densities, bool max_flag, bool use_initial_coords) +{ + // local number of uniquely assigned elements + size_t nonoverlap_nelements = element_map->getLocalNumElements(); + // initialize memory for volume storage + host_vec_array Element_Masses = Global_Element_Masses->getLocalView(Tpetra::Access::ReadWrite); + if (!nodal_density_flag) + { + compute_element_volumes(); } - } - else{ - convert_node_order(0) = 0; - convert_node_order(1) = 1; - convert_node_order(2) = 2; - convert_node_order(3) = 3; - if(num_dim==3){ - convert_node_order(4) = 4; - convert_node_order(5) = 5; - convert_node_order(6) = 6; - convert_node_order(7) = 7; + const_host_vec_array Element_Volumes = Global_Element_Volumes->getLocalView(Tpetra::Access::ReadOnly); + // local variable for host view in the dual view + const_host_vec_array all_node_coords; + if (use_initial_coords) + { + all_node_coords = all_initial_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); } - } - - //loop over elements and use quadrature rule to compute volume from Jacobian determinant - for(int nonoverlapping_ielem = 0; nonoverlapping_ielem < nonoverlap_nelements; nonoverlapping_ielem++){ - global_element_index = element_map->getGlobalElement(nonoverlapping_ielem); - ielem = all_element_map->getLocalElement(global_element_index); - if(nodal_density_flag){ - //acquire set of nodes for this local element - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - local_node_id = all_node_map->getLocalElement(nodes_in_elem(ielem, convert_node_order(node_loop))); - nodal_positions(node_loop,0) = all_node_coords(local_node_id,0); - nodal_positions(node_loop,1) = all_node_coords(local_node_id,1); - nodal_positions(node_loop,2) = all_node_coords(local_node_id,2); - if(nodal_density_flag) nodal_density(node_loop) = all_design_densities(local_node_id,0); - /* - if(myrank==1&&nodal_positions(node_loop,2)>10000000){ - std::cout << " LOCAL MATRIX DEBUG ON TASK " << myrank << std::endl; - std::cout << node_loop+1 <<" " << local_node_id <<" "<< nodes_in_elem(ielem, node_loop) << " "<< nodal_positions(node_loop,2) << std::endl; - std::fflush(stdout); - } - */ - //std::cout << local_node_id << " " << nodes_in_elem(ielem, node_loop) << " " - //<< nodal_positions(node_loop,0) << " " << nodal_positions(node_loop,1) << " "<< nodal_positions(node_loop,2) << " " << nodal_density(node_loop) <getLocalView(Tpetra::Access::ReadOnly); } - - //debug print of index - //std::cout << "nonoverlap element id on TASK " << myrank << " is " << nonoverlapping_ielem << std::endl; - //std::fflush(stdout); - - //initialize element mass - Element_Masses(nonoverlapping_ielem,0) = 0; - - if(Element_Types(ielem)==elements::elem_types::Hex8){ - direct_product_count = std::pow(num_gauss_points,num_dim); + const_host_vec_array all_design_densities; + if (nodal_density_flag) + { + all_design_densities = all_node_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); } - - //loop over quadrature points - for(int iquad=0; iquad < direct_product_count; iquad++){ - - //set current quadrature point - if(num_dim==3) z_quad = iquad/(num_gauss_points*num_gauss_points); - y_quad = (iquad % (num_gauss_points*num_gauss_points))/num_gauss_points; - x_quad = iquad % num_gauss_points; - quad_coordinate(0) = legendre_nodes_1D(x_quad); - quad_coordinate(1) = legendre_nodes_1D(y_quad); - if(num_dim==3) - quad_coordinate(2) = legendre_nodes_1D(z_quad); - - //set current quadrature weight - quad_coordinate_weight(0) = legendre_weights_1D(x_quad); - quad_coordinate_weight(1) = legendre_weights_1D(y_quad); - if(num_dim==3) - quad_coordinate_weight(2) = legendre_weights_1D(z_quad); - else - quad_coordinate_weight(2) = 1; - weight_multiply = quad_coordinate_weight(0)*quad_coordinate_weight(1)*quad_coordinate_weight(2); - - //compute shape functions at this point for the element type - elem->basis(basis_values,quad_coordinate); - - //compute all the necessary coordinates and derivatives at this point - - //compute shape function derivatives - elem->partial_xi_basis(basis_derivative_s1,quad_coordinate); - elem->partial_eta_basis(basis_derivative_s2,quad_coordinate); - elem->partial_mu_basis(basis_derivative_s3,quad_coordinate); - - //compute derivatives of x,y,z w.r.t the s,t,w isoparametric space needed by JT (Transpose of the Jacobian) - //derivative of x,y,z w.r.t s - JT_row1(0) = 0; - JT_row1(1) = 0; - JT_row1(2) = 0; - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - JT_row1(0) += nodal_positions(node_loop,0)*basis_derivative_s1(node_loop); - JT_row1(1) += nodal_positions(node_loop,1)*basis_derivative_s1(node_loop); - JT_row1(2) += nodal_positions(node_loop,2)*basis_derivative_s1(node_loop); - } - - //derivative of x,y,z w.r.t t - JT_row2(0) = 0; - JT_row2(1) = 0; - JT_row2(2) = 0; - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - JT_row2(0) += nodal_positions(node_loop,0)*basis_derivative_s2(node_loop); - JT_row2(1) += nodal_positions(node_loop,1)*basis_derivative_s2(node_loop); - JT_row2(2) += nodal_positions(node_loop,2)*basis_derivative_s2(node_loop); - } - - //derivative of x,y,z w.r.t w - JT_row3(0) = 0; - JT_row3(1) = 0; - JT_row3(2) = 0; - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - JT_row3(0) += nodal_positions(node_loop,0)*basis_derivative_s3(node_loop); - JT_row3(1) += nodal_positions(node_loop,1)*basis_derivative_s3(node_loop); - JT_row3(2) += nodal_positions(node_loop,2)*basis_derivative_s3(node_loop); - //debug print - /*if(myrank==1&&nodal_positions(node_loop,2)*basis_derivative_s3(node_loop)<-10000000){ - std::cout << " ELEMENT VOLUME JACOBIAN DEBUG ON TASK " << myrank << std::endl; - std::cout << node_loop+1 << " " << JT_row3(2) << " "<< nodal_positions(node_loop,2) <<" "<< basis_derivative_s3(node_loop) << std::endl; - std::fflush(stdout); - }*/ - } - - - //compute the determinant of the Jacobian - Jacobian = JT_row1(0)*(JT_row2(1)*JT_row3(2)-JT_row3(1)*JT_row2(2))- - JT_row1(1)*(JT_row2(0)*JT_row3(2)-JT_row3(0)*JT_row2(2))+ - JT_row1(2)*(JT_row2(0)*JT_row3(1)-JT_row3(0)*JT_row2(1)); - if(Jacobian<0) Jacobian = -Jacobian; - - //compute density - current_density = 0; - if(max_flag){ - current_density = 1; - } - else{ - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - current_density += nodal_density(node_loop)*basis_values(node_loop); + const_host_elem_conn_array nodes_in_elem = global_nodes_in_elem_distributed->getLocalView(Tpetra::Access::ReadOnly); + int nodes_per_elem = elem->num_basis(); + int z_quad, y_quad, x_quad, direct_product_count; + size_t local_node_id; + LO ielem; + GO global_element_index; + + real_t Jacobian, current_density, weight_multiply; + // CArrayKokkos legendre_nodes_1D(num_gauss_points); + // CArrayKokkos legendre_weights_1D(num_gauss_points); + CArray legendre_nodes_1D(num_gauss_points); + CArray legendre_weights_1D(num_gauss_points); + real_t pointer_quad_coordinate[num_dim]; + real_t pointer_quad_coordinate_weight[num_dim]; + real_t pointer_interpolated_point[num_dim]; + real_t pointer_JT_row1[num_dim]; + real_t pointer_JT_row2[num_dim]; + real_t pointer_JT_row3[num_dim]; + ViewCArray quad_coordinate(pointer_quad_coordinate, num_dim); + ViewCArray quad_coordinate_weight(pointer_quad_coordinate_weight, num_dim); + ViewCArray interpolated_point(pointer_interpolated_point, num_dim); + ViewCArray JT_row1(pointer_JT_row1, num_dim); + ViewCArray JT_row2(pointer_JT_row2, num_dim); + ViewCArray JT_row3(pointer_JT_row3, num_dim); + + real_t pointer_basis_values[elem->num_basis()]; + real_t pointer_basis_derivative_s1[elem->num_basis()]; + real_t pointer_basis_derivative_s2[elem->num_basis()]; + real_t pointer_basis_derivative_s3[elem->num_basis()]; + ViewCArray basis_values(pointer_basis_values, elem->num_basis()); + ViewCArray basis_derivative_s1(pointer_basis_derivative_s1, elem->num_basis()); + ViewCArray basis_derivative_s2(pointer_basis_derivative_s2, elem->num_basis()); + ViewCArray basis_derivative_s3(pointer_basis_derivative_s3, elem->num_basis()); + CArrayKokkos nodal_positions(elem->num_basis(), num_dim); + CArrayKokkos nodal_density(elem->num_basis()); + + // initialize weights + elements::legendre_nodes_1D(legendre_nodes_1D, num_gauss_points); + elements::legendre_weights_1D(legendre_weights_1D, num_gauss_points); + + Solver::node_ordering_convention active_node_ordering_convention = Solver_Pointer_->active_node_ordering_convention; + CArrayKokkos convert_node_order(max_nodes_per_element); + if ((active_node_ordering_convention == Solver::ENSIGHT && num_dim == 3) || (active_node_ordering_convention == Solver::IJK && num_dim == 2)) + { + convert_node_order(0) = 0; + convert_node_order(1) = 1; + convert_node_order(2) = 3; + convert_node_order(3) = 2; + if (num_dim == 3) + { + convert_node_order(4) = 4; + convert_node_order(5) = 5; + convert_node_order(6) = 7; + convert_node_order(7) = 6; } - } - - Element_Masses(nonoverlapping_ielem,0) += current_density*weight_multiply*Jacobian; } + else + { + convert_node_order(0) = 0; + convert_node_order(1) = 1; + convert_node_order(2) = 2; + convert_node_order(3) = 3; + if (num_dim == 3) + { + convert_node_order(4) = 4; + convert_node_order(5) = 5; + convert_node_order(6) = 6; + convert_node_order(7) = 7; + } } - else{ - Element_Masses(nonoverlapping_ielem,0) = Element_Volumes(nonoverlapping_ielem,0)*design_densities(nonoverlapping_ielem,0); + + // loop over elements and use quadrature rule to compute volume from Jacobian determinant + for (int nonoverlapping_ielem = 0; nonoverlapping_ielem < nonoverlap_nelements; nonoverlapping_ielem++) + { + global_element_index = element_map->getGlobalElement(nonoverlapping_ielem); + ielem = all_element_map->getLocalElement(global_element_index); + if (nodal_density_flag) + { + // acquire set of nodes for this local element + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + local_node_id = all_node_map->getLocalElement(nodes_in_elem(ielem, convert_node_order(node_loop))); + nodal_positions(node_loop, 0) = all_node_coords(local_node_id, 0); + nodal_positions(node_loop, 1) = all_node_coords(local_node_id, 1); + nodal_positions(node_loop, 2) = all_node_coords(local_node_id, 2); + if (nodal_density_flag) + { + nodal_density(node_loop) = all_design_densities(local_node_id, 0); + } + /* + if(myrank==1&&nodal_positions(node_loop,2)>10000000){ + std::cout << " LOCAL MATRIX DEBUG ON TASK " << myrank << std::endl; + std::cout << node_loop+1 <<" " << local_node_id <<" "<< nodes_in_elem(ielem, node_loop) << " "<< nodal_positions(node_loop,2) << std::endl; + std::fflush(stdout); + } + */ + // std::cout << local_node_id << " " << nodes_in_elem(ielem, node_loop) << " " + // << nodal_positions(node_loop,0) << " " << nodal_positions(node_loop,1) << " "<< nodal_positions(node_loop,2) << " " << nodal_density(node_loop) <basis(basis_values, quad_coordinate); + + // compute all the necessary coordinates and derivatives at this point + + // compute shape function derivatives + elem->partial_xi_basis(basis_derivative_s1, quad_coordinate); + elem->partial_eta_basis(basis_derivative_s2, quad_coordinate); + elem->partial_mu_basis(basis_derivative_s3, quad_coordinate); + + // compute derivatives of x,y,z w.r.t the s,t,w isoparametric space needed by JT (Transpose of the Jacobian) + // derivative of x,y,z w.r.t s + JT_row1(0) = 0; + JT_row1(1) = 0; + JT_row1(2) = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row1(0) += nodal_positions(node_loop, 0) * basis_derivative_s1(node_loop); + JT_row1(1) += nodal_positions(node_loop, 1) * basis_derivative_s1(node_loop); + JT_row1(2) += nodal_positions(node_loop, 2) * basis_derivative_s1(node_loop); + } + + // derivative of x,y,z w.r.t t + JT_row2(0) = 0; + JT_row2(1) = 0; + JT_row2(2) = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row2(0) += nodal_positions(node_loop, 0) * basis_derivative_s2(node_loop); + JT_row2(1) += nodal_positions(node_loop, 1) * basis_derivative_s2(node_loop); + JT_row2(2) += nodal_positions(node_loop, 2) * basis_derivative_s2(node_loop); + } + + // derivative of x,y,z w.r.t w + JT_row3(0) = 0; + JT_row3(1) = 0; + JT_row3(2) = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row3(0) += nodal_positions(node_loop, 0) * basis_derivative_s3(node_loop); + JT_row3(1) += nodal_positions(node_loop, 1) * basis_derivative_s3(node_loop); + JT_row3(2) += nodal_positions(node_loop, 2) * basis_derivative_s3(node_loop); + // debug print + /*if(myrank==1&&nodal_positions(node_loop,2)*basis_derivative_s3(node_loop)<-10000000){ + std::cout << " ELEMENT VOLUME JACOBIAN DEBUG ON TASK " << myrank << std::endl; + std::cout << node_loop+1 << " " << JT_row3(2) << " "<< nodal_positions(node_loop,2) <<" "<< basis_derivative_s3(node_loop) << std::endl; + std::fflush(stdout); + }*/ + } + + // compute the determinant of the Jacobian + Jacobian = JT_row1(0) * (JT_row2(1) * JT_row3(2) - JT_row3(1) * JT_row2(2)) - + JT_row1(1) * (JT_row2(0) * JT_row3(2) - JT_row3(0) * JT_row2(2)) + + JT_row1(2) * (JT_row2(0) * JT_row3(1) - JT_row3(0) * JT_row2(1)); + if (Jacobian < 0) + { + Jacobian = -Jacobian; + } + + // compute density + current_density = 0; + if (max_flag) + { + current_density = 1; + } + else + { + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + current_density += nodal_density(node_loop) * basis_values(node_loop); + } + } + + Element_Masses(nonoverlapping_ielem, 0) += current_density * weight_multiply * Jacobian; + } + } + else + { + Element_Masses(nonoverlapping_ielem, 0) = Element_Volumes(nonoverlapping_ielem, 0) * design_densities(nonoverlapping_ielem, 0); + } } - } - - //std::ostream &out = std::cout; - //Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - //if(myrank==0) - //*fos << "Global Element Masses:" << std::endl; - //Global_Element_Masses->describe(*fos,Teuchos::VERB_EXTREME); - //*fos << std::endl; + + // std::ostream &out = std::cout; + // Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + // if(myrank==0) + // *fos << "Global Element Masses:" << std::endl; + // Global_Element_Masses->describe(*fos,Teuchos::VERB_EXTREME); + // *fos << std::endl; } /* ---------------------------------------------------------------------- Compute the gradients of mass function with respect to nodal densities ------------------------------------------------------------------------- */ -void FEA_Module_Inertial::compute_nodal_gradients(const_host_vec_array design_variables, host_vec_array design_gradients, bool use_initial_coords){ - //local number of uniquely assigned elements - size_t nonoverlap_nelements = element_map->getLocalNumElements(); - //local variable for host view in the dual view - const_host_vec_array all_node_coords; - if(use_initial_coords){ - all_node_coords = all_initial_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - } - else{ - all_node_coords = all_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - } - const_host_elem_conn_array nodes_in_elem = global_nodes_in_elem_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_host_vec_array all_node_densities; - if(nodal_density_flag) - all_node_densities = all_node_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); - int nodes_per_elem = elem->num_basis(); - int z_quad,y_quad,x_quad, direct_product_count; - size_t local_node_id; - LO ielem; - GO global_element_index; - - real_t Jacobian, weight_multiply; - //CArrayKokkos legendre_nodes_1D(num_gauss_points); - //CArrayKokkos legendre_weights_1D(num_gauss_points); - CArray legendre_nodes_1D(num_gauss_points); - CArray legendre_weights_1D(num_gauss_points); - real_t pointer_quad_coordinate[num_dim]; - real_t pointer_quad_coordinate_weight[num_dim]; - real_t pointer_interpolated_point[num_dim]; - real_t pointer_JT_row1[num_dim]; - real_t pointer_JT_row2[num_dim]; - real_t pointer_JT_row3[num_dim]; - ViewCArray quad_coordinate(pointer_quad_coordinate,num_dim); - ViewCArray quad_coordinate_weight(pointer_quad_coordinate_weight,num_dim); - ViewCArray interpolated_point(pointer_interpolated_point,num_dim); - ViewCArray JT_row1(pointer_JT_row1,num_dim); - ViewCArray JT_row2(pointer_JT_row2,num_dim); - ViewCArray JT_row3(pointer_JT_row3,num_dim); - - real_t pointer_basis_values[elem->num_basis()]; - real_t pointer_basis_derivative_s1[elem->num_basis()]; - real_t pointer_basis_derivative_s2[elem->num_basis()]; - real_t pointer_basis_derivative_s3[elem->num_basis()]; - ViewCArray basis_values(pointer_basis_values,elem->num_basis()); - ViewCArray basis_derivative_s1(pointer_basis_derivative_s1,elem->num_basis()); - ViewCArray basis_derivative_s2(pointer_basis_derivative_s2,elem->num_basis()); - ViewCArray basis_derivative_s3(pointer_basis_derivative_s3,elem->num_basis()); - CArrayKokkos nodal_positions(elem->num_basis(),num_dim); - CArrayKokkos nodal_density(elem->num_basis()); - - //initialize weights - elements::legendre_nodes_1D(legendre_nodes_1D,num_gauss_points); - elements::legendre_weights_1D(legendre_weights_1D,num_gauss_points); - - Solver::node_ordering_convention active_node_ordering_convention = Solver_Pointer_->active_node_ordering_convention; - CArrayKokkos convert_node_order(max_nodes_per_element); - if((active_node_ordering_convention == Solver::ENSIGHT && num_dim==3)||(active_node_ordering_convention == Solver::IJK && num_dim==2)){ - convert_node_order(0) = 0; - convert_node_order(1) = 1; - convert_node_order(2) = 3; - convert_node_order(3) = 2; - if(num_dim == 3){ - convert_node_order(4) = 4; - convert_node_order(5) = 5; - convert_node_order(6) = 7; - convert_node_order(7) = 6; +void FEA_Module_Inertial::compute_nodal_gradients(const_host_vec_array design_variables, host_vec_array design_gradients, bool use_initial_coords) +{ + // local number of uniquely assigned elements + size_t nonoverlap_nelements = element_map->getLocalNumElements(); + // local variable for host view in the dual view + const_host_vec_array all_node_coords; + if (use_initial_coords) + { + all_node_coords = all_initial_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); } - } - else{ - convert_node_order(0) = 0; - convert_node_order(1) = 1; - convert_node_order(2) = 2; - convert_node_order(3) = 3; - if(num_dim==3){ - convert_node_order(4) = 4; - convert_node_order(5) = 5; - convert_node_order(6) = 6; - convert_node_order(7) = 7; + else + { + all_node_coords = all_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); } - } - - //initialize design gradients to 0 - for(int init = 0; init < nlocal_nodes; init++) - design_gradients(init,0) = 0; - - //loop over elements and use quadrature rule to compute volume from Jacobian determinant - for(int ielem = 0; ielem < rnum_elem; ielem++){ - //acquire set of nodes for this local element - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - local_node_id = all_node_map->getLocalElement(nodes_in_elem(ielem, convert_node_order(node_loop))); - nodal_positions(node_loop,0) = all_node_coords(local_node_id,0); - nodal_positions(node_loop,1) = all_node_coords(local_node_id,1); - nodal_positions(node_loop,2) = all_node_coords(local_node_id,2); - if(nodal_density_flag) nodal_density(node_loop) = all_node_densities(local_node_id,0); - /* - if(myrank==1&&nodal_positions(node_loop,2)>10000000){ - std::cout << " LOCAL MATRIX DEBUG ON TASK " << myrank << std::endl; - std::cout << node_loop+1 <<" " << local_node_id <<" "<< nodes_in_elem(ielem, node_loop) << " "<< nodal_positions(node_loop,2) << std::endl; - std::fflush(stdout); - } - */ - //std::cout << local_node_id << " " << nodes_in_elem(ielem, node_loop) << " " << nodal_positions(node_loop,0) << " " << nodal_positions(node_loop,1) << " "<< nodal_positions(node_loop,2) <getLocalView(Tpetra::Access::ReadOnly); + const_host_vec_array all_node_densities; + if (nodal_density_flag) + { + all_node_densities = all_node_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); } - - if(Element_Types(ielem)==elements::elem_types::Hex8){ - direct_product_count = std::pow(num_gauss_points,num_dim); + int nodes_per_elem = elem->num_basis(); + int z_quad, y_quad, x_quad, direct_product_count; + size_t local_node_id; + LO ielem; + GO global_element_index; + + real_t Jacobian, weight_multiply; + // CArrayKokkos legendre_nodes_1D(num_gauss_points); + // CArrayKokkos legendre_weights_1D(num_gauss_points); + CArray legendre_nodes_1D(num_gauss_points); + CArray legendre_weights_1D(num_gauss_points); + real_t pointer_quad_coordinate[num_dim]; + real_t pointer_quad_coordinate_weight[num_dim]; + real_t pointer_interpolated_point[num_dim]; + real_t pointer_JT_row1[num_dim]; + real_t pointer_JT_row2[num_dim]; + real_t pointer_JT_row3[num_dim]; + ViewCArray quad_coordinate(pointer_quad_coordinate, num_dim); + ViewCArray quad_coordinate_weight(pointer_quad_coordinate_weight, num_dim); + ViewCArray interpolated_point(pointer_interpolated_point, num_dim); + ViewCArray JT_row1(pointer_JT_row1, num_dim); + ViewCArray JT_row2(pointer_JT_row2, num_dim); + ViewCArray JT_row3(pointer_JT_row3, num_dim); + + real_t pointer_basis_values[elem->num_basis()]; + real_t pointer_basis_derivative_s1[elem->num_basis()]; + real_t pointer_basis_derivative_s2[elem->num_basis()]; + real_t pointer_basis_derivative_s3[elem->num_basis()]; + ViewCArray basis_values(pointer_basis_values, elem->num_basis()); + ViewCArray basis_derivative_s1(pointer_basis_derivative_s1, elem->num_basis()); + ViewCArray basis_derivative_s2(pointer_basis_derivative_s2, elem->num_basis()); + ViewCArray basis_derivative_s3(pointer_basis_derivative_s3, elem->num_basis()); + CArrayKokkos nodal_positions(elem->num_basis(), num_dim); + CArrayKokkos nodal_density(elem->num_basis()); + + // initialize weights + elements::legendre_nodes_1D(legendre_nodes_1D, num_gauss_points); + elements::legendre_weights_1D(legendre_weights_1D, num_gauss_points); + + Solver::node_ordering_convention active_node_ordering_convention = Solver_Pointer_->active_node_ordering_convention; + CArrayKokkos convert_node_order(max_nodes_per_element); + if ((active_node_ordering_convention == Solver::ENSIGHT && num_dim == 3) || (active_node_ordering_convention == Solver::IJK && num_dim == 2)) + { + convert_node_order(0) = 0; + convert_node_order(1) = 1; + convert_node_order(2) = 3; + convert_node_order(3) = 2; + if (num_dim == 3) + { + convert_node_order(4) = 4; + convert_node_order(5) = 5; + convert_node_order(6) = 7; + convert_node_order(7) = 6; + } } - - //loop over quadrature points - for(int iquad=0; iquad < direct_product_count; iquad++){ - - //set current quadrature point - if(num_dim==3) z_quad = iquad/(num_gauss_points*num_gauss_points); - y_quad = (iquad % (num_gauss_points*num_gauss_points))/num_gauss_points; - x_quad = iquad % num_gauss_points; - quad_coordinate(0) = legendre_nodes_1D(x_quad); - quad_coordinate(1) = legendre_nodes_1D(y_quad); - if(num_dim==3) - quad_coordinate(2) = legendre_nodes_1D(z_quad); - - //set current quadrature weight - quad_coordinate_weight(0) = legendre_weights_1D(x_quad); - quad_coordinate_weight(1) = legendre_weights_1D(y_quad); - if(num_dim==3) - quad_coordinate_weight(2) = legendre_weights_1D(z_quad); - else - quad_coordinate_weight(2) = 1; - weight_multiply = quad_coordinate_weight(0)*quad_coordinate_weight(1)*quad_coordinate_weight(2); - - //compute shape functions at this point for the element type - elem->basis(basis_values,quad_coordinate); - - //compute all the necessary coordinates and derivatives at this point - - //compute shape function derivatives - elem->partial_xi_basis(basis_derivative_s1,quad_coordinate); - elem->partial_eta_basis(basis_derivative_s2,quad_coordinate); - elem->partial_mu_basis(basis_derivative_s3,quad_coordinate); - - //compute derivatives of x,y,z w.r.t the s,t,w isoparametric space needed by JT (Transpose of the Jacobian) - //derivative of x,y,z w.r.t s - JT_row1(0) = 0; - JT_row1(1) = 0; - JT_row1(2) = 0; - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - JT_row1(0) += nodal_positions(node_loop,0)*basis_derivative_s1(node_loop); - JT_row1(1) += nodal_positions(node_loop,1)*basis_derivative_s1(node_loop); - JT_row1(2) += nodal_positions(node_loop,2)*basis_derivative_s1(node_loop); - } - - //derivative of x,y,z w.r.t t - JT_row2(0) = 0; - JT_row2(1) = 0; - JT_row2(2) = 0; - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - JT_row2(0) += nodal_positions(node_loop,0)*basis_derivative_s2(node_loop); - JT_row2(1) += nodal_positions(node_loop,1)*basis_derivative_s2(node_loop); - JT_row2(2) += nodal_positions(node_loop,2)*basis_derivative_s2(node_loop); - } - - //derivative of x,y,z w.r.t w - JT_row3(0) = 0; - JT_row3(1) = 0; - JT_row3(2) = 0; - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - JT_row3(0) += nodal_positions(node_loop,0)*basis_derivative_s3(node_loop); - JT_row3(1) += nodal_positions(node_loop,1)*basis_derivative_s3(node_loop); - JT_row3(2) += nodal_positions(node_loop,2)*basis_derivative_s3(node_loop); - //debug print - /*if(myrank==1&&nodal_positions(node_loop,2)*basis_derivative_s3(node_loop)<-10000000){ - std::cout << " ELEMENT VOLUME JACOBIAN DEBUG ON TASK " << myrank << std::endl; - std::cout << node_loop+1 << " " << JT_row3(2) << " "<< nodal_positions(node_loop,2) <<" "<< basis_derivative_s3(node_loop) << std::endl; - std::fflush(stdout); - }*/ - } - - - //compute the determinant of the Jacobian - Jacobian = JT_row1(0)*(JT_row2(1)*JT_row3(2)-JT_row3(1)*JT_row2(2))- - JT_row1(1)*(JT_row2(0)*JT_row3(2)-JT_row3(0)*JT_row2(2))+ - JT_row1(2)*(JT_row2(0)*JT_row3(1)-JT_row3(0)*JT_row2(1)); - if(Jacobian<0) Jacobian = -Jacobian; - - //assign contribution to every local node this element has - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - if(map->isNodeGlobalElement(nodes_in_elem(ielem, node_loop))){ - local_node_id = map->getLocalElement(nodes_in_elem(ielem, node_loop)); - design_gradients(local_node_id,0)+=weight_multiply*basis_values(node_loop)*Jacobian; + else + { + convert_node_order(0) = 0; + convert_node_order(1) = 1; + convert_node_order(2) = 2; + convert_node_order(3) = 3; + if (num_dim == 3) + { + convert_node_order(4) = 4; + convert_node_order(5) = 5; + convert_node_order(6) = 6; + convert_node_order(7) = 7; } - } } - - } + // initialize design gradients to 0 + for (int init = 0; init < nlocal_nodes; init++) + { + design_gradients(init, 0) = 0; + } + + // loop over elements and use quadrature rule to compute volume from Jacobian determinant + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + // acquire set of nodes for this local element + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + local_node_id = all_node_map->getLocalElement(nodes_in_elem(ielem, convert_node_order(node_loop))); + nodal_positions(node_loop, 0) = all_node_coords(local_node_id, 0); + nodal_positions(node_loop, 1) = all_node_coords(local_node_id, 1); + nodal_positions(node_loop, 2) = all_node_coords(local_node_id, 2); + if (nodal_density_flag) + { + nodal_density(node_loop) = all_node_densities(local_node_id, 0); + } + /* + if(myrank==1&&nodal_positions(node_loop,2)>10000000){ + std::cout << " LOCAL MATRIX DEBUG ON TASK " << myrank << std::endl; + std::cout << node_loop+1 <<" " << local_node_id <<" "<< nodes_in_elem(ielem, node_loop) << " "<< nodal_positions(node_loop,2) << std::endl; + std::fflush(stdout); + } + */ + // std::cout << local_node_id << " " << nodes_in_elem(ielem, node_loop) << " " << nodal_positions(node_loop,0) << " " << nodal_positions(node_loop,1) << " "<< nodal_positions(node_loop,2) <basis(basis_values, quad_coordinate); + + // compute all the necessary coordinates and derivatives at this point + + // compute shape function derivatives + elem->partial_xi_basis(basis_derivative_s1, quad_coordinate); + elem->partial_eta_basis(basis_derivative_s2, quad_coordinate); + elem->partial_mu_basis(basis_derivative_s3, quad_coordinate); + + // compute derivatives of x,y,z w.r.t the s,t,w isoparametric space needed by JT (Transpose of the Jacobian) + // derivative of x,y,z w.r.t s + JT_row1(0) = 0; + JT_row1(1) = 0; + JT_row1(2) = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row1(0) += nodal_positions(node_loop, 0) * basis_derivative_s1(node_loop); + JT_row1(1) += nodal_positions(node_loop, 1) * basis_derivative_s1(node_loop); + JT_row1(2) += nodal_positions(node_loop, 2) * basis_derivative_s1(node_loop); + } + + // derivative of x,y,z w.r.t t + JT_row2(0) = 0; + JT_row2(1) = 0; + JT_row2(2) = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row2(0) += nodal_positions(node_loop, 0) * basis_derivative_s2(node_loop); + JT_row2(1) += nodal_positions(node_loop, 1) * basis_derivative_s2(node_loop); + JT_row2(2) += nodal_positions(node_loop, 2) * basis_derivative_s2(node_loop); + } + + // derivative of x,y,z w.r.t w + JT_row3(0) = 0; + JT_row3(1) = 0; + JT_row3(2) = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row3(0) += nodal_positions(node_loop, 0) * basis_derivative_s3(node_loop); + JT_row3(1) += nodal_positions(node_loop, 1) * basis_derivative_s3(node_loop); + JT_row3(2) += nodal_positions(node_loop, 2) * basis_derivative_s3(node_loop); + // debug print + /*if(myrank==1&&nodal_positions(node_loop,2)*basis_derivative_s3(node_loop)<-10000000){ + std::cout << " ELEMENT VOLUME JACOBIAN DEBUG ON TASK " << myrank << std::endl; + std::cout << node_loop+1 << " " << JT_row3(2) << " "<< nodal_positions(node_loop,2) <<" "<< basis_derivative_s3(node_loop) << std::endl; + std::fflush(stdout); + }*/ + } + + // compute the determinant of the Jacobian + Jacobian = JT_row1(0) * (JT_row2(1) * JT_row3(2) - JT_row3(1) * JT_row2(2)) - + JT_row1(1) * (JT_row2(0) * JT_row3(2) - JT_row3(0) * JT_row2(2)) + + JT_row1(2) * (JT_row2(0) * JT_row3(1) - JT_row3(0) * JT_row2(1)); + if (Jacobian < 0) + { + Jacobian = -Jacobian; + } + + // assign contribution to every local node this element has + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + if (map->isNodeGlobalElement(nodes_in_elem(ielem, node_loop))) + { + local_node_id = map->getLocalElement(nodes_in_elem(ielem, node_loop)); + design_gradients(local_node_id, 0) += weight_multiply * basis_values(node_loop) * Jacobian; + } + } + } + } } /* ------------------------------------------------------------------------------------------------------------------------ Compute the moment of each element for a specified component; estimated with quadrature --------------------------------------------------------------------------------------------------------------------------- */ -void FEA_Module_Inertial::compute_element_moments(const_host_vec_array design_densities, bool max_flag, int moment_component, bool use_initial_coords){ - //local number of uniquely assigned elements - size_t nonoverlap_nelements = element_map->getLocalNumElements(); - //initialize memory for volume storage - host_vec_array Element_Masses = Global_Element_Masses->getLocalView(Tpetra::Access::ReadWrite); - host_vec_array Element_Moments; - - if(moment_component==0) Element_Moments = Global_Element_Moments_x->getLocalView(Tpetra::Access::ReadWrite); - if(moment_component==1) Element_Moments = Global_Element_Moments_y->getLocalView(Tpetra::Access::ReadWrite); - if(moment_component==2) Element_Moments = Global_Element_Moments_z->getLocalView(Tpetra::Access::ReadWrite); - - const_host_vec_array Element_Volumes = Global_Element_Volumes->getLocalView(Tpetra::Access::ReadOnly); - //local variable for host view in the dual view - const_host_vec_array all_node_coords = all_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_host_vec_array all_initial_node_coords; - if(use_initial_coords) - all_initial_node_coords = all_initial_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_host_vec_array all_design_densities; - if(nodal_density_flag) - all_design_densities = all_node_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_host_elem_conn_array nodes_in_elem = global_nodes_in_elem_distributed->getLocalView (Tpetra::Access::ReadOnly); - int nodes_per_elem = elem->num_basis(); - int z_quad,y_quad,x_quad, direct_product_count; - size_t local_node_id; - LO ielem; - GO global_element_index; - - Solver::node_ordering_convention active_node_ordering_convention = Solver_Pointer_->active_node_ordering_convention; - CArrayKokkos convert_node_order(max_nodes_per_element); - if((active_node_ordering_convention == Solver::ENSIGHT && num_dim==3)||(active_node_ordering_convention == Solver::IJK && num_dim==2)){ - convert_node_order(0) = 0; - convert_node_order(1) = 1; - convert_node_order(2) = 3; - convert_node_order(3) = 2; - if(num_dim == 3){ - convert_node_order(4) = 4; - convert_node_order(5) = 5; - convert_node_order(6) = 7; - convert_node_order(7) = 6; +void FEA_Module_Inertial::compute_element_moments(const_host_vec_array design_densities, bool max_flag, int moment_component, bool use_initial_coords) +{ + // local number of uniquely assigned elements + size_t nonoverlap_nelements = element_map->getLocalNumElements(); + // initialize memory for volume storage + host_vec_array Element_Masses = Global_Element_Masses->getLocalView(Tpetra::Access::ReadWrite); + host_vec_array Element_Moments; + + if (moment_component == 0) + { + Element_Moments = Global_Element_Moments_x->getLocalView(Tpetra::Access::ReadWrite); } - } - else{ - convert_node_order(0) = 0; - convert_node_order(1) = 1; - convert_node_order(2) = 2; - convert_node_order(3) = 3; - if(num_dim==3){ - convert_node_order(4) = 4; - convert_node_order(5) = 5; - convert_node_order(6) = 6; - convert_node_order(7) = 7; + if (moment_component == 1) + { + Element_Moments = Global_Element_Moments_y->getLocalView(Tpetra::Access::ReadWrite); } - } - - real_t Jacobian, current_density, weight_multiply; - //CArrayKokkos legendre_nodes_1D(num_gauss_points); - //CArrayKokkos legendre_weights_1D(num_gauss_points); - CArray legendre_nodes_1D(num_gauss_points); - CArray legendre_weights_1D(num_gauss_points); - real_t pointer_quad_coordinate[num_dim]; - real_t pointer_quad_coordinate_weight[num_dim]; - real_t pointer_interpolated_point[num_dim]; - real_t pointer_JT_row1[num_dim]; - real_t pointer_JT_row2[num_dim]; - real_t pointer_JT_row3[num_dim]; - ViewCArray quad_coordinate(pointer_quad_coordinate,num_dim); - ViewCArray quad_coordinate_weight(pointer_quad_coordinate_weight,num_dim); - ViewCArray interpolated_point(pointer_interpolated_point,num_dim); - ViewCArray JT_row1(pointer_JT_row1,num_dim); - ViewCArray JT_row2(pointer_JT_row2,num_dim); - ViewCArray JT_row3(pointer_JT_row3,num_dim); - - real_t pointer_basis_values[elem->num_basis()]; - real_t pointer_basis_derivative_s1[elem->num_basis()]; - real_t pointer_basis_derivative_s2[elem->num_basis()]; - real_t pointer_basis_derivative_s3[elem->num_basis()]; - ViewCArray basis_values(pointer_basis_values,elem->num_basis()); - ViewCArray basis_derivative_s1(pointer_basis_derivative_s1,elem->num_basis()); - ViewCArray basis_derivative_s2(pointer_basis_derivative_s2,elem->num_basis()); - ViewCArray basis_derivative_s3(pointer_basis_derivative_s3,elem->num_basis()); - CArrayKokkos nodal_positions(elem->num_basis(),num_dim); - CArrayKokkos initial_nodal_positions(elem->num_basis(),num_dim); - CArrayKokkos nodal_density(elem->num_basis()); - CArrayKokkos current_position(num_dim); - - //initialize weights - elements::legendre_nodes_1D(legendre_nodes_1D,num_gauss_points); - elements::legendre_weights_1D(legendre_weights_1D,num_gauss_points); - - //loop over elements and use quadrature rule to compute volume from Jacobian determinant - for(int nonoverlapping_ielem = 0; nonoverlapping_ielem < nonoverlap_nelements; nonoverlapping_ielem++){ - global_element_index = element_map->getGlobalElement(nonoverlapping_ielem); - ielem = all_element_map->getLocalElement(global_element_index); - //acquire set of nodes for this local element - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - local_node_id = all_node_map->getLocalElement(nodes_in_elem(ielem, convert_node_order(node_loop))); - nodal_positions(node_loop,0) = all_node_coords(local_node_id,0); - nodal_positions(node_loop,1) = all_node_coords(local_node_id,1); - nodal_positions(node_loop,2) = all_node_coords(local_node_id,2); - if(use_initial_coords){ - initial_nodal_positions(node_loop,0) = all_initial_node_coords(local_node_id,0); - initial_nodal_positions(node_loop,1) = all_initial_node_coords(local_node_id,1); - initial_nodal_positions(node_loop,2) = all_initial_node_coords(local_node_id,2); - } - - - if(nodal_density_flag) nodal_density(node_loop) = all_design_densities(local_node_id,0); - /* - if(myrank==1&&nodal_positions(node_loop,2)>10000000){ - std::cout << " LOCAL MATRIX DEBUG ON TASK " << myrank << std::endl; - std::cout << node_loop+1 <<" " << local_node_id <<" "<< nodes_in_elem(ielem, node_loop) << " "<< nodal_positions(node_loop,2) << std::endl; - std::fflush(stdout); - } - */ - //std::cout << local_node_id << " " << nodes_in_elem(ielem, node_loop) << " " - //<< nodal_positions(node_loop,0) << " " << nodal_positions(node_loop,1) << " "<< nodal_positions(node_loop,2) << " " << nodal_density(node_loop) <getLocalView(Tpetra::Access::ReadWrite); } - - //debug print of index - //std::cout << "nonoverlap element id on TASK " << myrank << " is " << nonoverlapping_ielem << std::endl; - //std::fflush(stdout); - - //initialize element mass - Element_Moments(nonoverlapping_ielem,0) = 0; - - if(Element_Types(ielem)==elements::elem_types::Hex8){ - direct_product_count = std::pow(num_gauss_points,num_dim); + + const_host_vec_array Element_Volumes = Global_Element_Volumes->getLocalView(Tpetra::Access::ReadOnly); + // local variable for host view in the dual view + const_host_vec_array all_node_coords = all_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_host_vec_array all_initial_node_coords; + if (use_initial_coords) + { + all_initial_node_coords = all_initial_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + } + const_host_vec_array all_design_densities; + if (nodal_density_flag) + { + all_design_densities = all_node_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); } - - //loop over quadrature points - for(int iquad=0; iquad < direct_product_count; iquad++){ - - //set current quadrature point - if(num_dim==3) z_quad = iquad/(num_gauss_points*num_gauss_points); - y_quad = (iquad % (num_gauss_points*num_gauss_points))/num_gauss_points; - x_quad = iquad % num_gauss_points; - quad_coordinate(0) = legendre_nodes_1D(x_quad); - quad_coordinate(1) = legendre_nodes_1D(y_quad); - if(num_dim==3) - quad_coordinate(2) = legendre_nodes_1D(z_quad); - - //set current quadrature weight - quad_coordinate_weight(0) = legendre_weights_1D(x_quad); - quad_coordinate_weight(1) = legendre_weights_1D(y_quad); - if(num_dim==3) - quad_coordinate_weight(2) = legendre_weights_1D(z_quad); - else - quad_coordinate_weight(2) = 1; - weight_multiply = quad_coordinate_weight(0)*quad_coordinate_weight(1)*quad_coordinate_weight(2); - - //compute shape functions at this point for the element type - elem->basis(basis_values,quad_coordinate); - - //compute all the necessary coordinates and derivatives at this point - - //compute shape function derivatives - elem->partial_xi_basis(basis_derivative_s1,quad_coordinate); - elem->partial_eta_basis(basis_derivative_s2,quad_coordinate); - elem->partial_mu_basis(basis_derivative_s3,quad_coordinate); - - //compute derivatives of x,y,z w.r.t the s,t,w isoparametric space needed by JT (Transpose of the Jacobian) - //derivative of x,y,z w.r.t s - JT_row1(0) = 0; - JT_row1(1) = 0; - JT_row1(2) = 0; - if(use_initial_coords){ - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - JT_row1(0) += initial_nodal_positions(node_loop,0)*basis_derivative_s1(node_loop); - JT_row1(1) += initial_nodal_positions(node_loop,1)*basis_derivative_s1(node_loop); - JT_row1(2) += initial_nodal_positions(node_loop,2)*basis_derivative_s1(node_loop); + const_host_elem_conn_array nodes_in_elem = global_nodes_in_elem_distributed->getLocalView(Tpetra::Access::ReadOnly); + int nodes_per_elem = elem->num_basis(); + int z_quad, y_quad, x_quad, direct_product_count; + size_t local_node_id; + LO ielem; + GO global_element_index; + + Solver::node_ordering_convention active_node_ordering_convention = Solver_Pointer_->active_node_ordering_convention; + CArrayKokkos convert_node_order(max_nodes_per_element); + if ((active_node_ordering_convention == Solver::ENSIGHT && num_dim == 3) || (active_node_ordering_convention == Solver::IJK && num_dim == 2)) + { + convert_node_order(0) = 0; + convert_node_order(1) = 1; + convert_node_order(2) = 3; + convert_node_order(3) = 2; + if (num_dim == 3) + { + convert_node_order(4) = 4; + convert_node_order(5) = 5; + convert_node_order(6) = 7; + convert_node_order(7) = 6; } - - //derivative of x,y,z w.r.t t - JT_row2(0) = 0; - JT_row2(1) = 0; - JT_row2(2) = 0; - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - JT_row2(0) += initial_nodal_positions(node_loop,0)*basis_derivative_s2(node_loop); - JT_row2(1) += initial_nodal_positions(node_loop,1)*basis_derivative_s2(node_loop); - JT_row2(2) += initial_nodal_positions(node_loop,2)*basis_derivative_s2(node_loop); + } + else + { + convert_node_order(0) = 0; + convert_node_order(1) = 1; + convert_node_order(2) = 2; + convert_node_order(3) = 3; + if (num_dim == 3) + { + convert_node_order(4) = 4; + convert_node_order(5) = 5; + convert_node_order(6) = 6; + convert_node_order(7) = 7; } + } - //derivative of x,y,z w.r.t w - JT_row3(0) = 0; - JT_row3(1) = 0; - JT_row3(2) = 0; - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - JT_row3(0) += initial_nodal_positions(node_loop,0)*basis_derivative_s3(node_loop); - JT_row3(1) += initial_nodal_positions(node_loop,1)*basis_derivative_s3(node_loop); - JT_row3(2) += initial_nodal_positions(node_loop,2)*basis_derivative_s3(node_loop); - //debug print - /*if(myrank==1&&nodal_positions(node_loop,2)*basis_derivative_s3(node_loop)<-10000000){ - std::cout << " ELEMENT VOLUME JACOBIAN DEBUG ON TASK " << myrank << std::endl; - std::cout << node_loop+1 << " " << JT_row3(2) << " "<< nodal_positions(node_loop,2) <<" "<< basis_derivative_s3(node_loop) << std::endl; - std::fflush(stdout); - }*/ - } + real_t Jacobian, current_density, weight_multiply; + // CArrayKokkos legendre_nodes_1D(num_gauss_points); + // CArrayKokkos legendre_weights_1D(num_gauss_points); + CArray legendre_nodes_1D(num_gauss_points); + CArray legendre_weights_1D(num_gauss_points); + real_t pointer_quad_coordinate[num_dim]; + real_t pointer_quad_coordinate_weight[num_dim]; + real_t pointer_interpolated_point[num_dim]; + real_t pointer_JT_row1[num_dim]; + real_t pointer_JT_row2[num_dim]; + real_t pointer_JT_row3[num_dim]; + ViewCArray quad_coordinate(pointer_quad_coordinate, num_dim); + ViewCArray quad_coordinate_weight(pointer_quad_coordinate_weight, num_dim); + ViewCArray interpolated_point(pointer_interpolated_point, num_dim); + ViewCArray JT_row1(pointer_JT_row1, num_dim); + ViewCArray JT_row2(pointer_JT_row2, num_dim); + ViewCArray JT_row3(pointer_JT_row3, num_dim); + + real_t pointer_basis_values[elem->num_basis()]; + real_t pointer_basis_derivative_s1[elem->num_basis()]; + real_t pointer_basis_derivative_s2[elem->num_basis()]; + real_t pointer_basis_derivative_s3[elem->num_basis()]; + ViewCArray basis_values(pointer_basis_values, elem->num_basis()); + ViewCArray basis_derivative_s1(pointer_basis_derivative_s1, elem->num_basis()); + ViewCArray basis_derivative_s2(pointer_basis_derivative_s2, elem->num_basis()); + ViewCArray basis_derivative_s3(pointer_basis_derivative_s3, elem->num_basis()); + CArrayKokkos nodal_positions(elem->num_basis(), num_dim); + CArrayKokkos initial_nodal_positions(elem->num_basis(), num_dim); + CArrayKokkos nodal_density(elem->num_basis()); + CArrayKokkos current_position(num_dim); + + // initialize weights + elements::legendre_nodes_1D(legendre_nodes_1D, num_gauss_points); + elements::legendre_weights_1D(legendre_weights_1D, num_gauss_points); + + // loop over elements and use quadrature rule to compute volume from Jacobian determinant + for (int nonoverlapping_ielem = 0; nonoverlapping_ielem < nonoverlap_nelements; nonoverlapping_ielem++) + { + global_element_index = element_map->getGlobalElement(nonoverlapping_ielem); + ielem = all_element_map->getLocalElement(global_element_index); + // acquire set of nodes for this local element + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + local_node_id = all_node_map->getLocalElement(nodes_in_elem(ielem, convert_node_order(node_loop))); + nodal_positions(node_loop, 0) = all_node_coords(local_node_id, 0); + nodal_positions(node_loop, 1) = all_node_coords(local_node_id, 1); + nodal_positions(node_loop, 2) = all_node_coords(local_node_id, 2); + if (use_initial_coords) + { + initial_nodal_positions(node_loop, 0) = all_initial_node_coords(local_node_id, 0); + initial_nodal_positions(node_loop, 1) = all_initial_node_coords(local_node_id, 1); + initial_nodal_positions(node_loop, 2) = all_initial_node_coords(local_node_id, 2); + } - } - else{ - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - JT_row1(0) += nodal_positions(node_loop,0)*basis_derivative_s1(node_loop); - JT_row1(1) += nodal_positions(node_loop,1)*basis_derivative_s1(node_loop); - JT_row1(2) += nodal_positions(node_loop,2)*basis_derivative_s1(node_loop); + if (nodal_density_flag) + { + nodal_density(node_loop) = all_design_densities(local_node_id, 0); + } + /* + if(myrank==1&&nodal_positions(node_loop,2)>10000000){ + std::cout << " LOCAL MATRIX DEBUG ON TASK " << myrank << std::endl; + std::cout << node_loop+1 <<" " << local_node_id <<" "<< nodes_in_elem(ielem, node_loop) << " "<< nodal_positions(node_loop,2) << std::endl; + std::fflush(stdout); + } + */ + // std::cout << local_node_id << " " << nodes_in_elem(ielem, node_loop) << " " + // << nodal_positions(node_loop,0) << " " << nodal_positions(node_loop,1) << " "<< nodal_positions(node_loop,2) << " " << nodal_density(node_loop) <num_basis(); node_loop++){ - JT_row2(0) += nodal_positions(node_loop,0)*basis_derivative_s2(node_loop); - JT_row2(1) += nodal_positions(node_loop,1)*basis_derivative_s2(node_loop); - JT_row2(2) += nodal_positions(node_loop,2)*basis_derivative_s2(node_loop); - } + // debug print of index + // std::cout << "nonoverlap element id on TASK " << myrank << " is " << nonoverlapping_ielem << std::endl; + // std::fflush(stdout); - //derivative of x,y,z w.r.t w - JT_row3(0) = 0; - JT_row3(1) = 0; - JT_row3(2) = 0; - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - JT_row3(0) += nodal_positions(node_loop,0)*basis_derivative_s3(node_loop); - JT_row3(1) += nodal_positions(node_loop,1)*basis_derivative_s3(node_loop); - JT_row3(2) += nodal_positions(node_loop,2)*basis_derivative_s3(node_loop); - //debug print - /*if(myrank==1&&nodal_positions(node_loop,2)*basis_derivative_s3(node_loop)<-10000000){ - std::cout << " ELEMENT VOLUME JACOBIAN DEBUG ON TASK " << myrank << std::endl; - std::cout << node_loop+1 << " " << JT_row3(2) << " "<< nodal_positions(node_loop,2) <<" "<< basis_derivative_s3(node_loop) << std::endl; - std::fflush(stdout); - }*/ - } - } - - - //compute the determinant of the Jacobian - Jacobian = JT_row1(0)*(JT_row2(1)*JT_row3(2)-JT_row3(1)*JT_row2(2))- - JT_row1(1)*(JT_row2(0)*JT_row3(2)-JT_row3(0)*JT_row2(2))+ - JT_row1(2)*(JT_row2(0)*JT_row3(1)-JT_row3(0)*JT_row2(1)); - if(Jacobian<0) Jacobian = -Jacobian; - - //compute density - if(max_flag){ - current_density = 1; - } - else{ - if(nodal_density_flag){ - current_density = 0; - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - current_density += nodal_density(node_loop)*basis_values(node_loop); - } - }// if - else{ - current_density = design_densities(nonoverlapping_ielem,0); + // initialize element mass + Element_Moments(nonoverlapping_ielem, 0) = 0; + + if (Element_Types(ielem) == elements::elem_types::Hex8) + { + direct_product_count = std::pow(num_gauss_points, num_dim); } - } - //compute current position - current_position(0) = current_position(1) = current_position(2) = 0; - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - current_position(moment_component) += nodal_positions(node_loop,moment_component)*basis_values(node_loop); - } + // loop over quadrature points + for (int iquad = 0; iquad < direct_product_count; iquad++) + { + // set current quadrature point + if (num_dim == 3) + { + z_quad = iquad / (num_gauss_points * num_gauss_points); + } + y_quad = (iquad % (num_gauss_points * num_gauss_points)) / num_gauss_points; + x_quad = iquad % num_gauss_points; + quad_coordinate(0) = legendre_nodes_1D(x_quad); + quad_coordinate(1) = legendre_nodes_1D(y_quad); + if (num_dim == 3) + { + quad_coordinate(2) = legendre_nodes_1D(z_quad); + } - Element_Moments(nonoverlapping_ielem,0) += current_density*current_position(moment_component)*weight_multiply*Jacobian; + // set current quadrature weight + quad_coordinate_weight(0) = legendre_weights_1D(x_quad); + quad_coordinate_weight(1) = legendre_weights_1D(y_quad); + if (num_dim == 3) + { + quad_coordinate_weight(2) = legendre_weights_1D(z_quad); + } + else + { + quad_coordinate_weight(2) = 1; + } + weight_multiply = quad_coordinate_weight(0) * quad_coordinate_weight(1) * quad_coordinate_weight(2); + + // compute shape functions at this point for the element type + elem->basis(basis_values, quad_coordinate); + + // compute all the necessary coordinates and derivatives at this point + + // compute shape function derivatives + elem->partial_xi_basis(basis_derivative_s1, quad_coordinate); + elem->partial_eta_basis(basis_derivative_s2, quad_coordinate); + elem->partial_mu_basis(basis_derivative_s3, quad_coordinate); + + // compute derivatives of x,y,z w.r.t the s,t,w isoparametric space needed by JT (Transpose of the Jacobian) + // derivative of x,y,z w.r.t s + JT_row1(0) = 0; + JT_row1(1) = 0; + JT_row1(2) = 0; + if (use_initial_coords) + { + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row1(0) += initial_nodal_positions(node_loop, 0) * basis_derivative_s1(node_loop); + JT_row1(1) += initial_nodal_positions(node_loop, 1) * basis_derivative_s1(node_loop); + JT_row1(2) += initial_nodal_positions(node_loop, 2) * basis_derivative_s1(node_loop); + } + + // derivative of x,y,z w.r.t t + JT_row2(0) = 0; + JT_row2(1) = 0; + JT_row2(2) = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row2(0) += initial_nodal_positions(node_loop, 0) * basis_derivative_s2(node_loop); + JT_row2(1) += initial_nodal_positions(node_loop, 1) * basis_derivative_s2(node_loop); + JT_row2(2) += initial_nodal_positions(node_loop, 2) * basis_derivative_s2(node_loop); + } + + // derivative of x,y,z w.r.t w + JT_row3(0) = 0; + JT_row3(1) = 0; + JT_row3(2) = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row3(0) += initial_nodal_positions(node_loop, 0) * basis_derivative_s3(node_loop); + JT_row3(1) += initial_nodal_positions(node_loop, 1) * basis_derivative_s3(node_loop); + JT_row3(2) += initial_nodal_positions(node_loop, 2) * basis_derivative_s3(node_loop); + // debug print + /*if(myrank==1&&nodal_positions(node_loop,2)*basis_derivative_s3(node_loop)<-10000000){ + std::cout << " ELEMENT VOLUME JACOBIAN DEBUG ON TASK " << myrank << std::endl; + std::cout << node_loop+1 << " " << JT_row3(2) << " "<< nodal_positions(node_loop,2) <<" "<< basis_derivative_s3(node_loop) << std::endl; + std::fflush(stdout); + }*/ + } + } + else + { + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row1(0) += nodal_positions(node_loop, 0) * basis_derivative_s1(node_loop); + JT_row1(1) += nodal_positions(node_loop, 1) * basis_derivative_s1(node_loop); + JT_row1(2) += nodal_positions(node_loop, 2) * basis_derivative_s1(node_loop); + } + + // derivative of x,y,z w.r.t t + JT_row2(0) = 0; + JT_row2(1) = 0; + JT_row2(2) = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row2(0) += nodal_positions(node_loop, 0) * basis_derivative_s2(node_loop); + JT_row2(1) += nodal_positions(node_loop, 1) * basis_derivative_s2(node_loop); + JT_row2(2) += nodal_positions(node_loop, 2) * basis_derivative_s2(node_loop); + } + + // derivative of x,y,z w.r.t w + JT_row3(0) = 0; + JT_row3(1) = 0; + JT_row3(2) = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row3(0) += nodal_positions(node_loop, 0) * basis_derivative_s3(node_loop); + JT_row3(1) += nodal_positions(node_loop, 1) * basis_derivative_s3(node_loop); + JT_row3(2) += nodal_positions(node_loop, 2) * basis_derivative_s3(node_loop); + // debug print + /*if(myrank==1&&nodal_positions(node_loop,2)*basis_derivative_s3(node_loop)<-10000000){ + std::cout << " ELEMENT VOLUME JACOBIAN DEBUG ON TASK " << myrank << std::endl; + std::cout << node_loop+1 << " " << JT_row3(2) << " "<< nodal_positions(node_loop,2) <<" "<< basis_derivative_s3(node_loop) << std::endl; + std::fflush(stdout); + }*/ + } + } + + // compute the determinant of the Jacobian + Jacobian = JT_row1(0) * (JT_row2(1) * JT_row3(2) - JT_row3(1) * JT_row2(2)) - + JT_row1(1) * (JT_row2(0) * JT_row3(2) - JT_row3(0) * JT_row2(2)) + + JT_row1(2) * (JT_row2(0) * JT_row3(1) - JT_row3(0) * JT_row2(1)); + if (Jacobian < 0) + { + Jacobian = -Jacobian; + } + + // compute density + if (max_flag) + { + current_density = 1; + } + else + { + if (nodal_density_flag) + { + current_density = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + current_density += nodal_density(node_loop) * basis_values(node_loop); + } + } // if + else + { + current_density = design_densities(nonoverlapping_ielem, 0); + } + } + + // compute current position + current_position(0) = current_position(1) = current_position(2) = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + current_position(moment_component) += nodal_positions(node_loop, moment_component) * basis_values(node_loop); + } + + Element_Moments(nonoverlapping_ielem, 0) += current_density * current_position(moment_component) * weight_multiply * Jacobian; + } } - } - - //std::ostream &out = std::cout; - //Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - //if(myrank==0) - //*fos << "Global Element Masses:" << std::endl; - //Global_Element_Masses->describe(*fos,Teuchos::VERB_EXTREME); - //*fos << std::endl; + + // std::ostream &out = std::cout; + // Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + // if(myrank==0) + // *fos << "Global Element Masses:" << std::endl; + // Global_Element_Masses->describe(*fos,Teuchos::VERB_EXTREME); + // *fos << std::endl; } /* --------------------------------------------------------------------------------------------------- Compute the gradients of the specified moment component with respect to design densities ------------------------------------------------------------------------------------------------------ */ -void FEA_Module_Inertial::compute_moment_gradients(const_host_vec_array design_variables, host_vec_array design_gradients, int moment_component, bool use_initial_coords){ - //local number of uniquely assigned elements - size_t nonoverlap_nelements = element_map->getLocalNumElements(); - //local variable for host view in the dual view - const_host_vec_array all_node_coords = all_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_host_vec_array all_initial_node_coords; - if(use_initial_coords) - all_initial_node_coords = all_initial_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_host_elem_conn_array nodes_in_elem = global_nodes_in_elem_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_host_vec_array all_node_densities; - if(nodal_density_flag) - all_node_densities = all_node_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); - int nodes_per_elem = elem->num_basis(); - int z_quad,y_quad,x_quad, direct_product_count; - size_t local_node_id; - LO ielem; - GO global_element_index; - - Solver::node_ordering_convention active_node_ordering_convention = Solver_Pointer_->active_node_ordering_convention; - CArrayKokkos convert_node_order(max_nodes_per_element); - if((active_node_ordering_convention == Solver::ENSIGHT && num_dim==3)||(active_node_ordering_convention == Solver::IJK && num_dim==2)){ - convert_node_order(0) = 0; - convert_node_order(1) = 1; - convert_node_order(2) = 3; - convert_node_order(3) = 2; - if(num_dim == 3){ - convert_node_order(4) = 4; - convert_node_order(5) = 5; - convert_node_order(6) = 7; - convert_node_order(7) = 6; +void FEA_Module_Inertial::compute_moment_gradients(const_host_vec_array design_variables, host_vec_array design_gradients, int moment_component, bool use_initial_coords) +{ + // local number of uniquely assigned elements + size_t nonoverlap_nelements = element_map->getLocalNumElements(); + // local variable for host view in the dual view + const_host_vec_array all_node_coords = all_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_host_vec_array all_initial_node_coords; + if (use_initial_coords) + { + all_initial_node_coords = all_initial_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); } - } - else{ - convert_node_order(0) = 0; - convert_node_order(1) = 1; - convert_node_order(2) = 2; - convert_node_order(3) = 3; - if(num_dim==3){ - convert_node_order(4) = 4; - convert_node_order(5) = 5; - convert_node_order(6) = 6; - convert_node_order(7) = 7; + const_host_elem_conn_array nodes_in_elem = global_nodes_in_elem_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_host_vec_array all_node_densities; + if (nodal_density_flag) + { + all_node_densities = all_node_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); } - } - - - real_t Jacobian, weight_multiply; - //CArrayKokkos legendre_nodes_1D(num_gauss_points); - //CArrayKokkos legendre_weights_1D(num_gauss_points); - CArray legendre_nodes_1D(num_gauss_points); - CArray legendre_weights_1D(num_gauss_points); - real_t pointer_quad_coordinate[num_dim]; - real_t pointer_quad_coordinate_weight[num_dim]; - real_t pointer_interpolated_point[num_dim]; - real_t pointer_JT_row1[num_dim]; - real_t pointer_JT_row2[num_dim]; - real_t pointer_JT_row3[num_dim]; - ViewCArray quad_coordinate(pointer_quad_coordinate,num_dim); - ViewCArray quad_coordinate_weight(pointer_quad_coordinate_weight,num_dim); - ViewCArray interpolated_point(pointer_interpolated_point,num_dim); - ViewCArray JT_row1(pointer_JT_row1,num_dim); - ViewCArray JT_row2(pointer_JT_row2,num_dim); - ViewCArray JT_row3(pointer_JT_row3,num_dim); - - real_t pointer_basis_values[elem->num_basis()]; - real_t pointer_basis_derivative_s1[elem->num_basis()]; - real_t pointer_basis_derivative_s2[elem->num_basis()]; - real_t pointer_basis_derivative_s3[elem->num_basis()]; - ViewCArray basis_values(pointer_basis_values,elem->num_basis()); - ViewCArray basis_derivative_s1(pointer_basis_derivative_s1,elem->num_basis()); - ViewCArray basis_derivative_s2(pointer_basis_derivative_s2,elem->num_basis()); - ViewCArray basis_derivative_s3(pointer_basis_derivative_s3,elem->num_basis()); - CArrayKokkos nodal_positions(elem->num_basis(),num_dim); - CArrayKokkos initial_nodal_positions(elem->num_basis(),num_dim); - CArrayKokkos nodal_density(elem->num_basis()); - CArrayKokkos current_position(num_dim); - - //initialize weights - elements::legendre_nodes_1D(legendre_nodes_1D,num_gauss_points); - elements::legendre_weights_1D(legendre_weights_1D,num_gauss_points); - - //initialize design gradients to 0 - for(int init = 0; init < nlocal_nodes; init++) - design_gradients(init,0) = 0; - - //loop over elements and use quadrature rule to compute volume from Jacobian determinant - for(int ielem = 0; ielem < rnum_elem; ielem++){ - //acquire set of nodes for this local element - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - local_node_id = all_node_map->getLocalElement(nodes_in_elem(ielem, convert_node_order(node_loop))); - nodal_positions(node_loop,0) = all_node_coords(local_node_id,0); - nodal_positions(node_loop,1) = all_node_coords(local_node_id,1); - nodal_positions(node_loop,2) = all_node_coords(local_node_id,2); - if(use_initial_coords){ - initial_nodal_positions(node_loop,0) = all_initial_node_coords(local_node_id,0); - initial_nodal_positions(node_loop,1) = all_initial_node_coords(local_node_id,1); - initial_nodal_positions(node_loop,2) = all_initial_node_coords(local_node_id,2); - } - if(nodal_density_flag) nodal_density(node_loop) = all_node_densities(local_node_id,0); - /* - if(myrank==1&&nodal_positions(node_loop,2)>10000000){ - std::cout << " LOCAL MATRIX DEBUG ON TASK " << myrank << std::endl; - std::cout << node_loop+1 <<" " << local_node_id <<" "<< nodes_in_elem(ielem, node_loop) << " "<< nodal_positions(node_loop,2) << std::endl; - std::fflush(stdout); - } - */ - //std::cout << local_node_id << " " << nodes_in_elem(ielem, node_loop) << " " << nodal_positions(node_loop,0) << " " << nodal_positions(node_loop,1) << " "<< nodal_positions(node_loop,2) <num_basis(); + int z_quad, y_quad, x_quad, direct_product_count; + size_t local_node_id; + LO ielem; + GO global_element_index; + + Solver::node_ordering_convention active_node_ordering_convention = Solver_Pointer_->active_node_ordering_convention; + CArrayKokkos convert_node_order(max_nodes_per_element); + if ((active_node_ordering_convention == Solver::ENSIGHT && num_dim == 3) || (active_node_ordering_convention == Solver::IJK && num_dim == 2)) + { + convert_node_order(0) = 0; + convert_node_order(1) = 1; + convert_node_order(2) = 3; + convert_node_order(3) = 2; + if (num_dim == 3) + { + convert_node_order(4) = 4; + convert_node_order(5) = 5; + convert_node_order(6) = 7; + convert_node_order(7) = 6; + } } - - //loop over quadrature points - for(int iquad=0; iquad < direct_product_count; iquad++){ - - //set current quadrature point - if(num_dim==3) z_quad = iquad/(num_gauss_points*num_gauss_points); - y_quad = (iquad % (num_gauss_points*num_gauss_points))/num_gauss_points; - x_quad = iquad % num_gauss_points; - quad_coordinate(0) = legendre_nodes_1D(x_quad); - quad_coordinate(1) = legendre_nodes_1D(y_quad); - if(num_dim==3) - quad_coordinate(2) = legendre_nodes_1D(z_quad); - - //set current quadrature weight - quad_coordinate_weight(0) = legendre_weights_1D(x_quad); - quad_coordinate_weight(1) = legendre_weights_1D(y_quad); - if(num_dim==3) - quad_coordinate_weight(2) = legendre_weights_1D(z_quad); - else - quad_coordinate_weight(2) = 1; - weight_multiply = quad_coordinate_weight(0)*quad_coordinate_weight(1)*quad_coordinate_weight(2); - - //compute shape functions at this point for the element type - elem->basis(basis_values,quad_coordinate); - - //compute all the necessary coordinates and derivatives at this point - - //compute shape function derivatives - elem->partial_xi_basis(basis_derivative_s1,quad_coordinate); - elem->partial_eta_basis(basis_derivative_s2,quad_coordinate); - elem->partial_mu_basis(basis_derivative_s3,quad_coordinate); - - //compute derivatives of x,y,z w.r.t the s,t,w isoparametric space needed by JT (Transpose of the Jacobian) - //derivative of x,y,z w.r.t s - JT_row1(0) = 0; - JT_row1(1) = 0; - JT_row1(2) = 0; - if(use_initial_coords){ - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - JT_row1(0) += initial_nodal_positions(node_loop,0)*basis_derivative_s1(node_loop); - JT_row1(1) += initial_nodal_positions(node_loop,1)*basis_derivative_s1(node_loop); - JT_row1(2) += initial_nodal_positions(node_loop,2)*basis_derivative_s1(node_loop); + else + { + convert_node_order(0) = 0; + convert_node_order(1) = 1; + convert_node_order(2) = 2; + convert_node_order(3) = 3; + if (num_dim == 3) + { + convert_node_order(4) = 4; + convert_node_order(5) = 5; + convert_node_order(6) = 6; + convert_node_order(7) = 7; } + } - //derivative of x,y,z w.r.t t - JT_row2(0) = 0; - JT_row2(1) = 0; - JT_row2(2) = 0; - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - JT_row2(0) += initial_nodal_positions(node_loop,0)*basis_derivative_s2(node_loop); - JT_row2(1) += initial_nodal_positions(node_loop,1)*basis_derivative_s2(node_loop); - JT_row2(2) += initial_nodal_positions(node_loop,2)*basis_derivative_s2(node_loop); - } + real_t Jacobian, weight_multiply; + // CArrayKokkos legendre_nodes_1D(num_gauss_points); + // CArrayKokkos legendre_weights_1D(num_gauss_points); + CArray legendre_nodes_1D(num_gauss_points); + CArray legendre_weights_1D(num_gauss_points); + real_t pointer_quad_coordinate[num_dim]; + real_t pointer_quad_coordinate_weight[num_dim]; + real_t pointer_interpolated_point[num_dim]; + real_t pointer_JT_row1[num_dim]; + real_t pointer_JT_row2[num_dim]; + real_t pointer_JT_row3[num_dim]; + ViewCArray quad_coordinate(pointer_quad_coordinate, num_dim); + ViewCArray quad_coordinate_weight(pointer_quad_coordinate_weight, num_dim); + ViewCArray interpolated_point(pointer_interpolated_point, num_dim); + ViewCArray JT_row1(pointer_JT_row1, num_dim); + ViewCArray JT_row2(pointer_JT_row2, num_dim); + ViewCArray JT_row3(pointer_JT_row3, num_dim); + + real_t pointer_basis_values[elem->num_basis()]; + real_t pointer_basis_derivative_s1[elem->num_basis()]; + real_t pointer_basis_derivative_s2[elem->num_basis()]; + real_t pointer_basis_derivative_s3[elem->num_basis()]; + ViewCArray basis_values(pointer_basis_values, elem->num_basis()); + ViewCArray basis_derivative_s1(pointer_basis_derivative_s1, elem->num_basis()); + ViewCArray basis_derivative_s2(pointer_basis_derivative_s2, elem->num_basis()); + ViewCArray basis_derivative_s3(pointer_basis_derivative_s3, elem->num_basis()); + CArrayKokkos nodal_positions(elem->num_basis(), num_dim); + CArrayKokkos initial_nodal_positions(elem->num_basis(), num_dim); + CArrayKokkos nodal_density(elem->num_basis()); + CArrayKokkos current_position(num_dim); + + // initialize weights + elements::legendre_nodes_1D(legendre_nodes_1D, num_gauss_points); + elements::legendre_weights_1D(legendre_weights_1D, num_gauss_points); + + // initialize design gradients to 0 + for (int init = 0; init < nlocal_nodes; init++) + { + design_gradients(init, 0) = 0; + } - //derivative of x,y,z w.r.t w - JT_row3(0) = 0; - JT_row3(1) = 0; - JT_row3(2) = 0; - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - JT_row3(0) += initial_nodal_positions(node_loop,0)*basis_derivative_s3(node_loop); - JT_row3(1) += initial_nodal_positions(node_loop,1)*basis_derivative_s3(node_loop); - JT_row3(2) += initial_nodal_positions(node_loop,2)*basis_derivative_s3(node_loop); + // loop over elements and use quadrature rule to compute volume from Jacobian determinant + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + // acquire set of nodes for this local element + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + local_node_id = all_node_map->getLocalElement(nodes_in_elem(ielem, convert_node_order(node_loop))); + nodal_positions(node_loop, 0) = all_node_coords(local_node_id, 0); + nodal_positions(node_loop, 1) = all_node_coords(local_node_id, 1); + nodal_positions(node_loop, 2) = all_node_coords(local_node_id, 2); + if (use_initial_coords) + { + initial_nodal_positions(node_loop, 0) = all_initial_node_coords(local_node_id, 0); + initial_nodal_positions(node_loop, 1) = all_initial_node_coords(local_node_id, 1); + initial_nodal_positions(node_loop, 2) = all_initial_node_coords(local_node_id, 2); + } + if (nodal_density_flag) + { + nodal_density(node_loop) = all_node_densities(local_node_id, 0); + } + /* + if(myrank==1&&nodal_positions(node_loop,2)>10000000){ + std::cout << " LOCAL MATRIX DEBUG ON TASK " << myrank << std::endl; + std::cout << node_loop+1 <<" " << local_node_id <<" "<< nodes_in_elem(ielem, node_loop) << " "<< nodal_positions(node_loop,2) << std::endl; + std::fflush(stdout); + } + */ + // std::cout << local_node_id << " " << nodes_in_elem(ielem, node_loop) << " " << nodal_positions(node_loop,0) << " " << nodal_positions(node_loop,1) << " "<< nodal_positions(node_loop,2) <num_basis(); node_loop++){ - JT_row1(0) += nodal_positions(node_loop,0)*basis_derivative_s1(node_loop); - JT_row1(1) += nodal_positions(node_loop,1)*basis_derivative_s1(node_loop); - JT_row1(2) += nodal_positions(node_loop,2)*basis_derivative_s1(node_loop); + if (Element_Types(ielem) == elements::elem_types::Hex8) + { + direct_product_count = std::pow(num_gauss_points, num_dim); } - //derivative of x,y,z w.r.t t - JT_row2(0) = 0; - JT_row2(1) = 0; - JT_row2(2) = 0; - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - JT_row2(0) += nodal_positions(node_loop,0)*basis_derivative_s2(node_loop); - JT_row2(1) += nodal_positions(node_loop,1)*basis_derivative_s2(node_loop); - JT_row2(2) += nodal_positions(node_loop,2)*basis_derivative_s2(node_loop); - } + // loop over quadrature points + for (int iquad = 0; iquad < direct_product_count; iquad++) + { + // set current quadrature point + if (num_dim == 3) + { + z_quad = iquad / (num_gauss_points * num_gauss_points); + } + y_quad = (iquad % (num_gauss_points * num_gauss_points)) / num_gauss_points; + x_quad = iquad % num_gauss_points; + quad_coordinate(0) = legendre_nodes_1D(x_quad); + quad_coordinate(1) = legendre_nodes_1D(y_quad); + if (num_dim == 3) + { + quad_coordinate(2) = legendre_nodes_1D(z_quad); + } - //derivative of x,y,z w.r.t w - JT_row3(0) = 0; - JT_row3(1) = 0; - JT_row3(2) = 0; - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - JT_row3(0) += nodal_positions(node_loop,0)*basis_derivative_s3(node_loop); - JT_row3(1) += nodal_positions(node_loop,1)*basis_derivative_s3(node_loop); - JT_row3(2) += nodal_positions(node_loop,2)*basis_derivative_s3(node_loop); - } - } - - - //compute the determinant of the Jacobian - Jacobian = JT_row1(0)*(JT_row2(1)*JT_row3(2)-JT_row3(1)*JT_row2(2))- - JT_row1(1)*(JT_row2(0)*JT_row3(2)-JT_row3(0)*JT_row2(2))+ - JT_row1(2)*(JT_row2(0)*JT_row3(1)-JT_row3(0)*JT_row2(1)); - if(Jacobian<0) Jacobian = -Jacobian; - - //compute current position - current_position(0) = current_position(1) = current_position(2) = 0; - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - current_position(moment_component) += nodal_positions(node_loop,moment_component)*basis_values(node_loop); - } - - //assign contribution to every local node this element has - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - if(map->isNodeGlobalElement(nodes_in_elem(ielem, node_loop))){ - local_node_id = map->getLocalElement(nodes_in_elem(ielem, node_loop)); - design_gradients(local_node_id,0)+=weight_multiply*basis_values(node_loop)*current_position(moment_component)*Jacobian; + // set current quadrature weight + quad_coordinate_weight(0) = legendre_weights_1D(x_quad); + quad_coordinate_weight(1) = legendre_weights_1D(y_quad); + if (num_dim == 3) + { + quad_coordinate_weight(2) = legendre_weights_1D(z_quad); + } + else + { + quad_coordinate_weight(2) = 1; + } + weight_multiply = quad_coordinate_weight(0) * quad_coordinate_weight(1) * quad_coordinate_weight(2); + + // compute shape functions at this point for the element type + elem->basis(basis_values, quad_coordinate); + + // compute all the necessary coordinates and derivatives at this point + + // compute shape function derivatives + elem->partial_xi_basis(basis_derivative_s1, quad_coordinate); + elem->partial_eta_basis(basis_derivative_s2, quad_coordinate); + elem->partial_mu_basis(basis_derivative_s3, quad_coordinate); + + // compute derivatives of x,y,z w.r.t the s,t,w isoparametric space needed by JT (Transpose of the Jacobian) + // derivative of x,y,z w.r.t s + JT_row1(0) = 0; + JT_row1(1) = 0; + JT_row1(2) = 0; + if (use_initial_coords) + { + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row1(0) += initial_nodal_positions(node_loop, 0) * basis_derivative_s1(node_loop); + JT_row1(1) += initial_nodal_positions(node_loop, 1) * basis_derivative_s1(node_loop); + JT_row1(2) += initial_nodal_positions(node_loop, 2) * basis_derivative_s1(node_loop); + } + + // derivative of x,y,z w.r.t t + JT_row2(0) = 0; + JT_row2(1) = 0; + JT_row2(2) = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row2(0) += initial_nodal_positions(node_loop, 0) * basis_derivative_s2(node_loop); + JT_row2(1) += initial_nodal_positions(node_loop, 1) * basis_derivative_s2(node_loop); + JT_row2(2) += initial_nodal_positions(node_loop, 2) * basis_derivative_s2(node_loop); + } + + // derivative of x,y,z w.r.t w + JT_row3(0) = 0; + JT_row3(1) = 0; + JT_row3(2) = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row3(0) += initial_nodal_positions(node_loop, 0) * basis_derivative_s3(node_loop); + JT_row3(1) += initial_nodal_positions(node_loop, 1) * basis_derivative_s3(node_loop); + JT_row3(2) += initial_nodal_positions(node_loop, 2) * basis_derivative_s3(node_loop); + } + } + else + { + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row1(0) += nodal_positions(node_loop, 0) * basis_derivative_s1(node_loop); + JT_row1(1) += nodal_positions(node_loop, 1) * basis_derivative_s1(node_loop); + JT_row1(2) += nodal_positions(node_loop, 2) * basis_derivative_s1(node_loop); + } + + // derivative of x,y,z w.r.t t + JT_row2(0) = 0; + JT_row2(1) = 0; + JT_row2(2) = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row2(0) += nodal_positions(node_loop, 0) * basis_derivative_s2(node_loop); + JT_row2(1) += nodal_positions(node_loop, 1) * basis_derivative_s2(node_loop); + JT_row2(2) += nodal_positions(node_loop, 2) * basis_derivative_s2(node_loop); + } + + // derivative of x,y,z w.r.t w + JT_row3(0) = 0; + JT_row3(1) = 0; + JT_row3(2) = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row3(0) += nodal_positions(node_loop, 0) * basis_derivative_s3(node_loop); + JT_row3(1) += nodal_positions(node_loop, 1) * basis_derivative_s3(node_loop); + JT_row3(2) += nodal_positions(node_loop, 2) * basis_derivative_s3(node_loop); + } + } + + // compute the determinant of the Jacobian + Jacobian = JT_row1(0) * (JT_row2(1) * JT_row3(2) - JT_row3(1) * JT_row2(2)) - + JT_row1(1) * (JT_row2(0) * JT_row3(2) - JT_row3(0) * JT_row2(2)) + + JT_row1(2) * (JT_row2(0) * JT_row3(1) - JT_row3(0) * JT_row2(1)); + if (Jacobian < 0) + { + Jacobian = -Jacobian; + } + + // compute current position + current_position(0) = current_position(1) = current_position(2) = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + current_position(moment_component) += nodal_positions(node_loop, moment_component) * basis_values(node_loop); + } + + // assign contribution to every local node this element has + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + if (map->isNodeGlobalElement(nodes_in_elem(ielem, node_loop))) + { + local_node_id = map->getLocalElement(nodes_in_elem(ielem, node_loop)); + design_gradients(local_node_id, 0) += weight_multiply * basis_values(node_loop) * current_position(moment_component) * Jacobian; + } + } } - } } - - } - } - /* ------------------------------------------------------------------------------------------------------------------------ Compute the moment of inertia of each element for a specified component of the inertia tensor; estimated with quadrature --------------------------------------------------------------------------------------------------------------------------- */ -void FEA_Module_Inertial::compute_element_moments_of_inertia(const_host_vec_array design_densities, bool max_flag, int inertia_component, bool use_initial_coords){ - //local number of uniquely assigned elements - size_t nonoverlap_nelements = element_map->getLocalNumElements(); - //initialize memory for volume storage - host_vec_array Element_Masses = Global_Element_Masses->getLocalView(Tpetra::Access::ReadWrite); - host_vec_array Element_Moments_of_Inertia; - - if(inertia_component==0) Element_Moments_of_Inertia = Global_Element_Moments_of_Inertia_xx->getLocalView(Tpetra::Access::ReadWrite); - if(inertia_component==1) Element_Moments_of_Inertia = Global_Element_Moments_of_Inertia_yy->getLocalView(Tpetra::Access::ReadWrite); - if(inertia_component==2) Element_Moments_of_Inertia = Global_Element_Moments_of_Inertia_zz->getLocalView(Tpetra::Access::ReadWrite); - if(inertia_component==3) Element_Moments_of_Inertia = Global_Element_Moments_of_Inertia_xy->getLocalView(Tpetra::Access::ReadWrite); - if(inertia_component==4) Element_Moments_of_Inertia = Global_Element_Moments_of_Inertia_xz->getLocalView(Tpetra::Access::ReadWrite); - if(inertia_component==5) Element_Moments_of_Inertia = Global_Element_Moments_of_Inertia_yz->getLocalView(Tpetra::Access::ReadWrite); - - const_host_vec_array Element_Volumes = Global_Element_Volumes->getLocalView(Tpetra::Access::ReadOnly); - //local variable for host view in the dual view - const_host_vec_array all_node_coords = all_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_host_vec_array all_initial_node_coords; - if(use_initial_coords) - all_initial_node_coords = all_initial_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_host_vec_array all_design_densities; - if(nodal_density_flag) - all_design_densities = all_node_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_host_elem_conn_array nodes_in_elem = global_nodes_in_elem_distributed->getLocalView (Tpetra::Access::ReadOnly); - double inertia_center[3]; - if(module_params->enable_inertia_center[0]){ - inertia_center[0] = module_params->moment_of_inertia_center[0]; - } - else{ - inertia_center[0] = center_of_mass[0]; - } - if(module_params->enable_inertia_center[1]){ - inertia_center[1] = module_params->moment_of_inertia_center[1]; - } - else{ - inertia_center[1] = center_of_mass[1]; - } - if(num_dim==3){ - if(module_params->enable_inertia_center[2]){ - inertia_center[2] = module_params->moment_of_inertia_center[2]; +void FEA_Module_Inertial::compute_element_moments_of_inertia(const_host_vec_array design_densities, bool max_flag, int inertia_component, bool use_initial_coords) +{ + // local number of uniquely assigned elements + size_t nonoverlap_nelements = element_map->getLocalNumElements(); + // initialize memory for volume storage + host_vec_array Element_Masses = Global_Element_Masses->getLocalView(Tpetra::Access::ReadWrite); + host_vec_array Element_Moments_of_Inertia; + + if (inertia_component == 0) + { + Element_Moments_of_Inertia = Global_Element_Moments_of_Inertia_xx->getLocalView(Tpetra::Access::ReadWrite); } - else{ - inertia_center[2] = center_of_mass[2]; + if (inertia_component == 1) + { + Element_Moments_of_Inertia = Global_Element_Moments_of_Inertia_yy->getLocalView(Tpetra::Access::ReadWrite); } - } - int nodes_per_elem = elem->num_basis(); - int z_quad,y_quad,x_quad, direct_product_count; - size_t local_node_id; - LO ielem; - GO global_element_index; - real_t delx1, delx2; - - real_t Jacobian, current_density, weight_multiply; - //CArrayKokkos legendre_nodes_1D(num_gauss_points); - //CArrayKokkos legendre_weights_1D(num_gauss_points); - CArray legendre_nodes_1D(num_gauss_points); - CArray legendre_weights_1D(num_gauss_points); - real_t pointer_quad_coordinate[num_dim]; - real_t pointer_quad_coordinate_weight[num_dim]; - real_t pointer_interpolated_point[num_dim]; - real_t pointer_JT_row1[num_dim]; - real_t pointer_JT_row2[num_dim]; - real_t pointer_JT_row3[num_dim]; - ViewCArray quad_coordinate(pointer_quad_coordinate,num_dim); - ViewCArray quad_coordinate_weight(pointer_quad_coordinate_weight,num_dim); - ViewCArray interpolated_point(pointer_interpolated_point,num_dim); - ViewCArray JT_row1(pointer_JT_row1,num_dim); - ViewCArray JT_row2(pointer_JT_row2,num_dim); - ViewCArray JT_row3(pointer_JT_row3,num_dim); - - real_t pointer_basis_values[elem->num_basis()]; - real_t pointer_basis_derivative_s1[elem->num_basis()]; - real_t pointer_basis_derivative_s2[elem->num_basis()]; - real_t pointer_basis_derivative_s3[elem->num_basis()]; - ViewCArray basis_values(pointer_basis_values,elem->num_basis()); - ViewCArray basis_derivative_s1(pointer_basis_derivative_s1,elem->num_basis()); - ViewCArray basis_derivative_s2(pointer_basis_derivative_s2,elem->num_basis()); - ViewCArray basis_derivative_s3(pointer_basis_derivative_s3,elem->num_basis()); - CArrayKokkos nodal_positions(elem->num_basis(),num_dim); - CArrayKokkos initial_nodal_positions(elem->num_basis(),num_dim); - CArrayKokkos nodal_density(elem->num_basis()); - CArrayKokkos current_position(num_dim); - - //initialize weights - elements::legendre_nodes_1D(legendre_nodes_1D,num_gauss_points); - elements::legendre_weights_1D(legendre_weights_1D,num_gauss_points); - - Solver::node_ordering_convention active_node_ordering_convention = Solver_Pointer_->active_node_ordering_convention; - CArrayKokkos convert_node_order(max_nodes_per_element); - if((active_node_ordering_convention == Solver::ENSIGHT && num_dim==3)||(active_node_ordering_convention == Solver::IJK && num_dim==2)){ - convert_node_order(0) = 0; - convert_node_order(1) = 1; - convert_node_order(2) = 3; - convert_node_order(3) = 2; - if(num_dim == 3){ - convert_node_order(4) = 4; - convert_node_order(5) = 5; - convert_node_order(6) = 7; - convert_node_order(7) = 6; + if (inertia_component == 2) + { + Element_Moments_of_Inertia = Global_Element_Moments_of_Inertia_zz->getLocalView(Tpetra::Access::ReadWrite); } - } - else{ - convert_node_order(0) = 0; - convert_node_order(1) = 1; - convert_node_order(2) = 2; - convert_node_order(3) = 3; - if(num_dim==3){ - convert_node_order(4) = 4; - convert_node_order(5) = 5; - convert_node_order(6) = 6; - convert_node_order(7) = 7; + if (inertia_component == 3) + { + Element_Moments_of_Inertia = Global_Element_Moments_of_Inertia_xy->getLocalView(Tpetra::Access::ReadWrite); } - } - - //loop over elements and use quadrature rule to compute volume from Jacobian determinant - for(int nonoverlapping_ielem = 0; nonoverlapping_ielem < nonoverlap_nelements; nonoverlapping_ielem++){ - global_element_index = element_map->getGlobalElement(nonoverlapping_ielem); - ielem = all_element_map->getLocalElement(global_element_index); - //acquire set of nodes for this local element - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - local_node_id = all_node_map->getLocalElement(nodes_in_elem(ielem, convert_node_order(node_loop))); - nodal_positions(node_loop,0) = all_node_coords(local_node_id,0); - nodal_positions(node_loop,1) = all_node_coords(local_node_id,1); - nodal_positions(node_loop,2) = all_node_coords(local_node_id,2); - if(use_initial_coords){ - initial_nodal_positions(node_loop,0) = all_initial_node_coords(local_node_id,0); - initial_nodal_positions(node_loop,1) = all_initial_node_coords(local_node_id,1); - initial_nodal_positions(node_loop,2) = all_initial_node_coords(local_node_id,2); - } - if(nodal_density_flag) nodal_density(node_loop) = all_design_densities(local_node_id,0); - /* - if(myrank==1&&nodal_positions(node_loop,2)>10000000){ - std::cout << " LOCAL MATRIX DEBUG ON TASK " << myrank << std::endl; - std::cout << node_loop+1 <<" " << local_node_id <<" "<< nodes_in_elem(ielem, node_loop) << " "<< nodal_positions(node_loop,2) << std::endl; - std::fflush(stdout); - } - */ - //std::cout << local_node_id << " " << nodes_in_elem(ielem, node_loop) << " " - //<< nodal_positions(node_loop,0) << " " << nodal_positions(node_loop,1) << " "<< nodal_positions(node_loop,2) << " " << nodal_density(node_loop) <getLocalView(Tpetra::Access::ReadWrite); } - - //debug print of index - //std::cout << "nonoverlap element id on TASK " << myrank << " is " << nonoverlapping_ielem << std::endl; - //std::fflush(stdout); - - //initialize element mass - Element_Moments_of_Inertia(nonoverlapping_ielem,0) = 0; - - if(Element_Types(ielem)==elements::elem_types::Hex8){ - direct_product_count = std::pow(num_gauss_points,num_dim); + if (inertia_component == 5) + { + Element_Moments_of_Inertia = Global_Element_Moments_of_Inertia_yz->getLocalView(Tpetra::Access::ReadWrite); } - - //loop over quadrature points - for(int iquad=0; iquad < direct_product_count; iquad++){ - - //set current quadrature point - if(num_dim==3) z_quad = iquad/(num_gauss_points*num_gauss_points); - y_quad = (iquad % (num_gauss_points*num_gauss_points))/num_gauss_points; - x_quad = iquad % num_gauss_points; - quad_coordinate(0) = legendre_nodes_1D(x_quad); - quad_coordinate(1) = legendre_nodes_1D(y_quad); - if(num_dim==3) - quad_coordinate(2) = legendre_nodes_1D(z_quad); - - //set current quadrature weight - quad_coordinate_weight(0) = legendre_weights_1D(x_quad); - quad_coordinate_weight(1) = legendre_weights_1D(y_quad); - if(num_dim==3) - quad_coordinate_weight(2) = legendre_weights_1D(z_quad); - else - quad_coordinate_weight(2) = 1; - weight_multiply = quad_coordinate_weight(0)*quad_coordinate_weight(1)*quad_coordinate_weight(2); - - //compute shape functions at this point for the element type - elem->basis(basis_values,quad_coordinate); - - //compute all the necessary coordinates and derivatives at this point - - //compute shape function derivatives - elem->partial_xi_basis(basis_derivative_s1,quad_coordinate); - elem->partial_eta_basis(basis_derivative_s2,quad_coordinate); - elem->partial_mu_basis(basis_derivative_s3,quad_coordinate); - - //compute derivatives of x,y,z w.r.t the s,t,w isoparametric space needed by JT (Transpose of the Jacobian) - //derivative of x,y,z w.r.t s - JT_row1(0) = 0; - JT_row1(1) = 0; - JT_row1(2) = 0; - if(use_initial_coords){ - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - JT_row1(0) += initial_nodal_positions(node_loop,0)*basis_derivative_s1(node_loop); - JT_row1(1) += initial_nodal_positions(node_loop,1)*basis_derivative_s1(node_loop); - JT_row1(2) += initial_nodal_positions(node_loop,2)*basis_derivative_s1(node_loop); - } - //derivative of x,y,z w.r.t t - JT_row2(0) = 0; - JT_row2(1) = 0; - JT_row2(2) = 0; - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - JT_row2(0) += initial_nodal_positions(node_loop,0)*basis_derivative_s2(node_loop); - JT_row2(1) += initial_nodal_positions(node_loop,1)*basis_derivative_s2(node_loop); - JT_row2(2) += initial_nodal_positions(node_loop,2)*basis_derivative_s2(node_loop); + const_host_vec_array Element_Volumes = Global_Element_Volumes->getLocalView(Tpetra::Access::ReadOnly); + // local variable for host view in the dual view + const_host_vec_array all_node_coords = all_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_host_vec_array all_initial_node_coords; + if (use_initial_coords) + { + all_initial_node_coords = all_initial_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + } + const_host_vec_array all_design_densities; + if (nodal_density_flag) + { + all_design_densities = all_node_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); + } + const_host_elem_conn_array nodes_in_elem = global_nodes_in_elem_distributed->getLocalView(Tpetra::Access::ReadOnly); + double inertia_center[3]; + if (module_params->enable_inertia_center[0]) + { + inertia_center[0] = module_params->moment_of_inertia_center[0]; + } + else + { + inertia_center[0] = center_of_mass[0]; + } + if (module_params->enable_inertia_center[1]) + { + inertia_center[1] = module_params->moment_of_inertia_center[1]; + } + else + { + inertia_center[1] = center_of_mass[1]; + } + if (num_dim == 3) + { + if (module_params->enable_inertia_center[2]) + { + inertia_center[2] = module_params->moment_of_inertia_center[2]; } - - //derivative of x,y,z w.r.t w - JT_row3(0) = 0; - JT_row3(1) = 0; - JT_row3(2) = 0; - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - JT_row3(0) += initial_nodal_positions(node_loop,0)*basis_derivative_s3(node_loop); - JT_row3(1) += initial_nodal_positions(node_loop,1)*basis_derivative_s3(node_loop); - JT_row3(2) += initial_nodal_positions(node_loop,2)*basis_derivative_s3(node_loop); + else + { + inertia_center[2] = center_of_mass[2]; } - - } - else{ - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - JT_row1(0) += nodal_positions(node_loop,0)*basis_derivative_s1(node_loop); - JT_row1(1) += nodal_positions(node_loop,1)*basis_derivative_s1(node_loop); - JT_row1(2) += nodal_positions(node_loop,2)*basis_derivative_s1(node_loop); + } + int nodes_per_elem = elem->num_basis(); + int z_quad, y_quad, x_quad, direct_product_count; + size_t local_node_id; + LO ielem; + GO global_element_index; + real_t delx1, delx2; + + real_t Jacobian, current_density, weight_multiply; + // CArrayKokkos legendre_nodes_1D(num_gauss_points); + // CArrayKokkos legendre_weights_1D(num_gauss_points); + CArray legendre_nodes_1D(num_gauss_points); + CArray legendre_weights_1D(num_gauss_points); + real_t pointer_quad_coordinate[num_dim]; + real_t pointer_quad_coordinate_weight[num_dim]; + real_t pointer_interpolated_point[num_dim]; + real_t pointer_JT_row1[num_dim]; + real_t pointer_JT_row2[num_dim]; + real_t pointer_JT_row3[num_dim]; + ViewCArray quad_coordinate(pointer_quad_coordinate, num_dim); + ViewCArray quad_coordinate_weight(pointer_quad_coordinate_weight, num_dim); + ViewCArray interpolated_point(pointer_interpolated_point, num_dim); + ViewCArray JT_row1(pointer_JT_row1, num_dim); + ViewCArray JT_row2(pointer_JT_row2, num_dim); + ViewCArray JT_row3(pointer_JT_row3, num_dim); + + real_t pointer_basis_values[elem->num_basis()]; + real_t pointer_basis_derivative_s1[elem->num_basis()]; + real_t pointer_basis_derivative_s2[elem->num_basis()]; + real_t pointer_basis_derivative_s3[elem->num_basis()]; + ViewCArray basis_values(pointer_basis_values, elem->num_basis()); + ViewCArray basis_derivative_s1(pointer_basis_derivative_s1, elem->num_basis()); + ViewCArray basis_derivative_s2(pointer_basis_derivative_s2, elem->num_basis()); + ViewCArray basis_derivative_s3(pointer_basis_derivative_s3, elem->num_basis()); + CArrayKokkos nodal_positions(elem->num_basis(), num_dim); + CArrayKokkos initial_nodal_positions(elem->num_basis(), num_dim); + CArrayKokkos nodal_density(elem->num_basis()); + CArrayKokkos current_position(num_dim); + + // initialize weights + elements::legendre_nodes_1D(legendre_nodes_1D, num_gauss_points); + elements::legendre_weights_1D(legendre_weights_1D, num_gauss_points); + + Solver::node_ordering_convention active_node_ordering_convention = Solver_Pointer_->active_node_ordering_convention; + CArrayKokkos convert_node_order(max_nodes_per_element); + if ((active_node_ordering_convention == Solver::ENSIGHT && num_dim == 3) || (active_node_ordering_convention == Solver::IJK && num_dim == 2)) + { + convert_node_order(0) = 0; + convert_node_order(1) = 1; + convert_node_order(2) = 3; + convert_node_order(3) = 2; + if (num_dim == 3) + { + convert_node_order(4) = 4; + convert_node_order(5) = 5; + convert_node_order(6) = 7; + convert_node_order(7) = 6; + } + } + else + { + convert_node_order(0) = 0; + convert_node_order(1) = 1; + convert_node_order(2) = 2; + convert_node_order(3) = 3; + if (num_dim == 3) + { + convert_node_order(4) = 4; + convert_node_order(5) = 5; + convert_node_order(6) = 6; + convert_node_order(7) = 7; } + } - //derivative of x,y,z w.r.t t - JT_row2(0) = 0; - JT_row2(1) = 0; - JT_row2(2) = 0; - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - JT_row2(0) += nodal_positions(node_loop,0)*basis_derivative_s2(node_loop); - JT_row2(1) += nodal_positions(node_loop,1)*basis_derivative_s2(node_loop); - JT_row2(2) += nodal_positions(node_loop,2)*basis_derivative_s2(node_loop); + // loop over elements and use quadrature rule to compute volume from Jacobian determinant + for (int nonoverlapping_ielem = 0; nonoverlapping_ielem < nonoverlap_nelements; nonoverlapping_ielem++) + { + global_element_index = element_map->getGlobalElement(nonoverlapping_ielem); + ielem = all_element_map->getLocalElement(global_element_index); + // acquire set of nodes for this local element + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + local_node_id = all_node_map->getLocalElement(nodes_in_elem(ielem, convert_node_order(node_loop))); + nodal_positions(node_loop, 0) = all_node_coords(local_node_id, 0); + nodal_positions(node_loop, 1) = all_node_coords(local_node_id, 1); + nodal_positions(node_loop, 2) = all_node_coords(local_node_id, 2); + if (use_initial_coords) + { + initial_nodal_positions(node_loop, 0) = all_initial_node_coords(local_node_id, 0); + initial_nodal_positions(node_loop, 1) = all_initial_node_coords(local_node_id, 1); + initial_nodal_positions(node_loop, 2) = all_initial_node_coords(local_node_id, 2); + } + if (nodal_density_flag) + { + nodal_density(node_loop) = all_design_densities(local_node_id, 0); + } + /* + if(myrank==1&&nodal_positions(node_loop,2)>10000000){ + std::cout << " LOCAL MATRIX DEBUG ON TASK " << myrank << std::endl; + std::cout << node_loop+1 <<" " << local_node_id <<" "<< nodes_in_elem(ielem, node_loop) << " "<< nodal_positions(node_loop,2) << std::endl; + std::fflush(stdout); + } + */ + // std::cout << local_node_id << " " << nodes_in_elem(ielem, node_loop) << " " + // << nodal_positions(node_loop,0) << " " << nodal_positions(node_loop,1) << " "<< nodal_positions(node_loop,2) << " " << nodal_density(node_loop) <num_basis(); node_loop++){ - JT_row3(0) += nodal_positions(node_loop,0)*basis_derivative_s3(node_loop); - JT_row3(1) += nodal_positions(node_loop,1)*basis_derivative_s3(node_loop); - JT_row3(2) += nodal_positions(node_loop,2)*basis_derivative_s3(node_loop); + // debug print of index + // std::cout << "nonoverlap element id on TASK " << myrank << " is " << nonoverlapping_ielem << std::endl; + // std::fflush(stdout); + + // initialize element mass + Element_Moments_of_Inertia(nonoverlapping_ielem, 0) = 0; + + if (Element_Types(ielem) == elements::elem_types::Hex8) + { + direct_product_count = std::pow(num_gauss_points, num_dim); } - } - - - //compute the determinant of the Jacobian - Jacobian = JT_row1(0)*(JT_row2(1)*JT_row3(2)-JT_row3(1)*JT_row2(2))- - JT_row1(1)*(JT_row2(0)*JT_row3(2)-JT_row3(0)*JT_row2(2))+ - JT_row1(2)*(JT_row2(0)*JT_row3(1)-JT_row3(0)*JT_row2(1)); - if(Jacobian<0) Jacobian = -Jacobian; - - //compute density - if(max_flag){ - current_density = 1; - } - else{ - if(nodal_density_flag){ - current_density = 0; - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - current_density += nodal_density(node_loop)*basis_values(node_loop); - } - }// if - else{ - current_density = design_densities(nonoverlapping_ielem,0); + + // loop over quadrature points + for (int iquad = 0; iquad < direct_product_count; iquad++) + { + // set current quadrature point + if (num_dim == 3) + { + z_quad = iquad / (num_gauss_points * num_gauss_points); + } + y_quad = (iquad % (num_gauss_points * num_gauss_points)) / num_gauss_points; + x_quad = iquad % num_gauss_points; + quad_coordinate(0) = legendre_nodes_1D(x_quad); + quad_coordinate(1) = legendre_nodes_1D(y_quad); + if (num_dim == 3) + { + quad_coordinate(2) = legendre_nodes_1D(z_quad); + } + + // set current quadrature weight + quad_coordinate_weight(0) = legendre_weights_1D(x_quad); + quad_coordinate_weight(1) = legendre_weights_1D(y_quad); + if (num_dim == 3) + { + quad_coordinate_weight(2) = legendre_weights_1D(z_quad); + } + else + { + quad_coordinate_weight(2) = 1; + } + weight_multiply = quad_coordinate_weight(0) * quad_coordinate_weight(1) * quad_coordinate_weight(2); + + // compute shape functions at this point for the element type + elem->basis(basis_values, quad_coordinate); + + // compute all the necessary coordinates and derivatives at this point + + // compute shape function derivatives + elem->partial_xi_basis(basis_derivative_s1, quad_coordinate); + elem->partial_eta_basis(basis_derivative_s2, quad_coordinate); + elem->partial_mu_basis(basis_derivative_s3, quad_coordinate); + + // compute derivatives of x,y,z w.r.t the s,t,w isoparametric space needed by JT (Transpose of the Jacobian) + // derivative of x,y,z w.r.t s + JT_row1(0) = 0; + JT_row1(1) = 0; + JT_row1(2) = 0; + if (use_initial_coords) + { + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row1(0) += initial_nodal_positions(node_loop, 0) * basis_derivative_s1(node_loop); + JT_row1(1) += initial_nodal_positions(node_loop, 1) * basis_derivative_s1(node_loop); + JT_row1(2) += initial_nodal_positions(node_loop, 2) * basis_derivative_s1(node_loop); + } + + // derivative of x,y,z w.r.t t + JT_row2(0) = 0; + JT_row2(1) = 0; + JT_row2(2) = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row2(0) += initial_nodal_positions(node_loop, 0) * basis_derivative_s2(node_loop); + JT_row2(1) += initial_nodal_positions(node_loop, 1) * basis_derivative_s2(node_loop); + JT_row2(2) += initial_nodal_positions(node_loop, 2) * basis_derivative_s2(node_loop); + } + + // derivative of x,y,z w.r.t w + JT_row3(0) = 0; + JT_row3(1) = 0; + JT_row3(2) = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row3(0) += initial_nodal_positions(node_loop, 0) * basis_derivative_s3(node_loop); + JT_row3(1) += initial_nodal_positions(node_loop, 1) * basis_derivative_s3(node_loop); + JT_row3(2) += initial_nodal_positions(node_loop, 2) * basis_derivative_s3(node_loop); + } + } + else + { + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row1(0) += nodal_positions(node_loop, 0) * basis_derivative_s1(node_loop); + JT_row1(1) += nodal_positions(node_loop, 1) * basis_derivative_s1(node_loop); + JT_row1(2) += nodal_positions(node_loop, 2) * basis_derivative_s1(node_loop); + } + + // derivative of x,y,z w.r.t t + JT_row2(0) = 0; + JT_row2(1) = 0; + JT_row2(2) = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row2(0) += nodal_positions(node_loop, 0) * basis_derivative_s2(node_loop); + JT_row2(1) += nodal_positions(node_loop, 1) * basis_derivative_s2(node_loop); + JT_row2(2) += nodal_positions(node_loop, 2) * basis_derivative_s2(node_loop); + } + + // derivative of x,y,z w.r.t w + JT_row3(0) = 0; + JT_row3(1) = 0; + JT_row3(2) = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row3(0) += nodal_positions(node_loop, 0) * basis_derivative_s3(node_loop); + JT_row3(1) += nodal_positions(node_loop, 1) * basis_derivative_s3(node_loop); + JT_row3(2) += nodal_positions(node_loop, 2) * basis_derivative_s3(node_loop); + } + } + + // compute the determinant of the Jacobian + Jacobian = JT_row1(0) * (JT_row2(1) * JT_row3(2) - JT_row3(1) * JT_row2(2)) - + JT_row1(1) * (JT_row2(0) * JT_row3(2) - JT_row3(0) * JT_row2(2)) + + JT_row1(2) * (JT_row2(0) * JT_row3(1) - JT_row3(0) * JT_row2(1)); + if (Jacobian < 0) + { + Jacobian = -Jacobian; + } + + // compute density + if (max_flag) + { + current_density = 1; + } + else + { + if (nodal_density_flag) + { + current_density = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + current_density += nodal_density(node_loop) * basis_values(node_loop); + } + } // if + else + { + current_density = design_densities(nonoverlapping_ielem, 0); + } + } + + // compute current position + current_position(0) = current_position(1) = current_position(2) = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + current_position(0) += nodal_positions(node_loop, 0) * basis_values(node_loop); + current_position(1) += nodal_positions(node_loop, 1) * basis_values(node_loop); + current_position(2) += nodal_positions(node_loop, 2) * basis_values(node_loop); + } + + if (inertia_component == 0) + { + delx1 = current_position(1) - inertia_center[1]; + delx2 = current_position(2) - inertia_center[2]; + Element_Moments_of_Inertia(nonoverlapping_ielem, 0) += current_density * (delx1 * delx1 + delx2 * delx2) * weight_multiply * Jacobian; + } + if (inertia_component == 1) + { + delx1 = current_position(0) - inertia_center[0]; + delx2 = current_position(2) - inertia_center[2]; + Element_Moments_of_Inertia(nonoverlapping_ielem, 0) += current_density * (delx1 * delx1 + delx2 * delx2) * weight_multiply * Jacobian; + } + if (inertia_component == 2) + { + delx1 = current_position(0) - inertia_center[0]; + delx2 = current_position(1) - inertia_center[1]; + Element_Moments_of_Inertia(nonoverlapping_ielem, 0) += current_density * (delx1 * delx1 + delx2 * delx2) * weight_multiply * Jacobian; + } + if (inertia_component == 3) + { + delx1 = current_position(0) - inertia_center[0]; + delx2 = current_position(1) - inertia_center[1]; + Element_Moments_of_Inertia(nonoverlapping_ielem, 0) -= current_density * (delx1 * delx2) * weight_multiply * Jacobian; + } + if (inertia_component == 4) + { + delx1 = current_position(0) - inertia_center[0]; + delx2 = current_position(2) - inertia_center[2]; + Element_Moments_of_Inertia(nonoverlapping_ielem, 0) -= current_density * (delx1 * delx2) * weight_multiply * Jacobian; + } + if (inertia_component == 5) + { + delx1 = current_position(1) - inertia_center[1]; + delx2 = current_position(2) - inertia_center[2]; + Element_Moments_of_Inertia(nonoverlapping_ielem, 0) -= current_density * (delx1 * delx2) * weight_multiply * Jacobian; + } } - } - - //compute current position - current_position(0) = current_position(1) = current_position(2) = 0; - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - current_position(0) += nodal_positions(node_loop,0)*basis_values(node_loop); - current_position(1) += nodal_positions(node_loop,1)*basis_values(node_loop); - current_position(2) += nodal_positions(node_loop,2)*basis_values(node_loop); - } - - if(inertia_component==0){ - delx1 = current_position(1) - inertia_center[1]; - delx2 = current_position(2) - inertia_center[2]; - Element_Moments_of_Inertia(nonoverlapping_ielem,0) += current_density*(delx1*delx1 + delx2*delx2)*weight_multiply*Jacobian; - } - if(inertia_component==1){ - delx1 = current_position(0) - inertia_center[0]; - delx2 = current_position(2) - inertia_center[2]; - Element_Moments_of_Inertia(nonoverlapping_ielem,0) += current_density*(delx1*delx1 + delx2*delx2)*weight_multiply*Jacobian; - } - if(inertia_component==2){ - delx1 = current_position(0) - inertia_center[0]; - delx2 = current_position(1) - inertia_center[1]; - Element_Moments_of_Inertia(nonoverlapping_ielem,0) += current_density*(delx1*delx1 + delx2*delx2)*weight_multiply*Jacobian; - } - if(inertia_component==3){ - delx1 = current_position(0) - inertia_center[0]; - delx2 = current_position(1) - inertia_center[1]; - Element_Moments_of_Inertia(nonoverlapping_ielem,0) -= current_density*(delx1*delx2)*weight_multiply*Jacobian; - } - if(inertia_component==4){ - delx1 = current_position(0) - inertia_center[0]; - delx2 = current_position(2) - inertia_center[2]; - Element_Moments_of_Inertia(nonoverlapping_ielem,0) -= current_density*(delx1*delx2)*weight_multiply*Jacobian; - } - if(inertia_component==5){ - delx1 = current_position(1) - inertia_center[1]; - delx2 = current_position(2) - inertia_center[2]; - Element_Moments_of_Inertia(nonoverlapping_ielem,0) -= current_density*(delx1*delx2)*weight_multiply*Jacobian; - } } - } - - //std::ostream &out = std::cout; - //Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - //if(myrank==0) - //*fos << "Global Element Masses:" << std::endl; - //Global_Element_Masses->describe(*fos,Teuchos::VERB_EXTREME); - //*fos << std::endl; + + // std::ostream &out = std::cout; + // Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + // if(myrank==0) + // *fos << "Global Element Masses:" << std::endl; + // Global_Element_Masses->describe(*fos,Teuchos::VERB_EXTREME); + // *fos << std::endl; } /* --------------------------------------------------------------------------------------------------- Compute the gradients of the specified moment of inertia component with respect to design densities ------------------------------------------------------------------------------------------------------ */ -void FEA_Module_Inertial::compute_moment_of_inertia_gradients(const_host_vec_array design_variables, host_vec_array design_gradients, int inertia_component, bool use_initial_coords){ - //local number of uniquely assigned elements - size_t nonoverlap_nelements = element_map->getLocalNumElements(); - //local variable for host view in the dual view - const_host_vec_array all_node_coords = all_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_host_vec_array all_initial_node_coords; - if(use_initial_coords) - all_initial_node_coords = all_initial_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_host_elem_conn_array nodes_in_elem = global_nodes_in_elem_distributed->getLocalView (Tpetra::Access::ReadOnly); - double inertia_center[3]; - - Solver::node_ordering_convention active_node_ordering_convention = Solver_Pointer_->active_node_ordering_convention; - CArrayKokkos convert_node_order(max_nodes_per_element); - if((active_node_ordering_convention == Solver::ENSIGHT && num_dim==3)||(active_node_ordering_convention == Solver::IJK && num_dim==2)){ - convert_node_order(0) = 0; - convert_node_order(1) = 1; - convert_node_order(2) = 3; - convert_node_order(3) = 2; - if(num_dim == 3){ - convert_node_order(4) = 4; - convert_node_order(5) = 5; - convert_node_order(6) = 7; - convert_node_order(7) = 6; +void FEA_Module_Inertial::compute_moment_of_inertia_gradients(const_host_vec_array design_variables, host_vec_array design_gradients, int inertia_component, bool use_initial_coords) +{ + // local number of uniquely assigned elements + size_t nonoverlap_nelements = element_map->getLocalNumElements(); + // local variable for host view in the dual view + const_host_vec_array all_node_coords = all_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_host_vec_array all_initial_node_coords; + if (use_initial_coords) + { + all_initial_node_coords = all_initial_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); } - } - else{ - convert_node_order(0) = 0; - convert_node_order(1) = 1; - convert_node_order(2) = 2; - convert_node_order(3) = 3; - if(num_dim==3){ - convert_node_order(4) = 4; - convert_node_order(5) = 5; - convert_node_order(6) = 6; - convert_node_order(7) = 7; + const_host_elem_conn_array nodes_in_elem = global_nodes_in_elem_distributed->getLocalView(Tpetra::Access::ReadOnly); + double inertia_center[3]; + + Solver::node_ordering_convention active_node_ordering_convention = Solver_Pointer_->active_node_ordering_convention; + CArrayKokkos convert_node_order(max_nodes_per_element); + if ((active_node_ordering_convention == Solver::ENSIGHT && num_dim == 3) || (active_node_ordering_convention == Solver::IJK && num_dim == 2)) + { + convert_node_order(0) = 0; + convert_node_order(1) = 1; + convert_node_order(2) = 3; + convert_node_order(3) = 2; + if (num_dim == 3) + { + convert_node_order(4) = 4; + convert_node_order(5) = 5; + convert_node_order(6) = 7; + convert_node_order(7) = 6; + } } - } - - if(module_params->enable_inertia_center[0]){ - inertia_center[0] = module_params->moment_of_inertia_center[0]; - } - else{ - inertia_center[0] = center_of_mass[0]; - } - if(module_params->enable_inertia_center[1]){ - inertia_center[1] = module_params->moment_of_inertia_center[1]; - } - else{ - inertia_center[1] = center_of_mass[1]; - } - if(num_dim==3){ - if(module_params->enable_inertia_center[2]){ - inertia_center[2] = module_params->moment_of_inertia_center[2]; + else + { + convert_node_order(0) = 0; + convert_node_order(1) = 1; + convert_node_order(2) = 2; + convert_node_order(3) = 3; + if (num_dim == 3) + { + convert_node_order(4) = 4; + convert_node_order(5) = 5; + convert_node_order(6) = 6; + convert_node_order(7) = 7; + } } - else{ - inertia_center[2] = center_of_mass[2]; + + if (module_params->enable_inertia_center[0]) + { + inertia_center[0] = module_params->moment_of_inertia_center[0]; } - } - const_host_vec_array all_node_densities; - if(nodal_density_flag) - all_node_densities = all_node_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); - int nodes_per_elem = elem->num_basis(); - int z_quad,y_quad,x_quad, direct_product_count; - size_t local_node_id; - LO ielem; - GO global_element_index; - real_t delx1, delx2; - - real_t Jacobian, weight_multiply; - //CArrayKokkos legendre_nodes_1D(num_gauss_points); - //CArrayKokkos legendre_weights_1D(num_gauss_points); - CArray legendre_nodes_1D(num_gauss_points); - CArray legendre_weights_1D(num_gauss_points); - real_t pointer_quad_coordinate[num_dim]; - real_t pointer_quad_coordinate_weight[num_dim]; - real_t pointer_interpolated_point[num_dim]; - real_t pointer_JT_row1[num_dim]; - real_t pointer_JT_row2[num_dim]; - real_t pointer_JT_row3[num_dim]; - ViewCArray quad_coordinate(pointer_quad_coordinate,num_dim); - ViewCArray quad_coordinate_weight(pointer_quad_coordinate_weight,num_dim); - ViewCArray interpolated_point(pointer_interpolated_point,num_dim); - ViewCArray JT_row1(pointer_JT_row1,num_dim); - ViewCArray JT_row2(pointer_JT_row2,num_dim); - ViewCArray JT_row3(pointer_JT_row3,num_dim); - - real_t pointer_basis_values[elem->num_basis()]; - real_t pointer_basis_derivative_s1[elem->num_basis()]; - real_t pointer_basis_derivative_s2[elem->num_basis()]; - real_t pointer_basis_derivative_s3[elem->num_basis()]; - ViewCArray basis_values(pointer_basis_values,elem->num_basis()); - ViewCArray basis_derivative_s1(pointer_basis_derivative_s1,elem->num_basis()); - ViewCArray basis_derivative_s2(pointer_basis_derivative_s2,elem->num_basis()); - ViewCArray basis_derivative_s3(pointer_basis_derivative_s3,elem->num_basis()); - CArrayKokkos nodal_positions(elem->num_basis(),num_dim); - CArrayKokkos initial_nodal_positions(elem->num_basis(),num_dim); - CArrayKokkos nodal_density(elem->num_basis()); - CArrayKokkos current_position(num_dim); - - //initialize weights - elements::legendre_nodes_1D(legendre_nodes_1D,num_gauss_points); - elements::legendre_weights_1D(legendre_weights_1D,num_gauss_points); - - //initialize design gradients to 0 - for(int init = 0; init < nlocal_nodes; init++) - design_gradients(init,0) = 0; - - //loop over elements and use quadrature rule to compute volume from Jacobian determinant - for(int ielem = 0; ielem < rnum_elem; ielem++){ - //acquire set of nodes for this local element - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - local_node_id = all_node_map->getLocalElement(nodes_in_elem(ielem, convert_node_order(node_loop))); - nodal_positions(node_loop,0) = all_node_coords(local_node_id,0); - nodal_positions(node_loop,1) = all_node_coords(local_node_id,1); - nodal_positions(node_loop,2) = all_node_coords(local_node_id,2); - if(use_initial_coords){ - initial_nodal_positions(node_loop,0) = all_initial_node_coords(local_node_id,0); - initial_nodal_positions(node_loop,1) = all_initial_node_coords(local_node_id,1); - initial_nodal_positions(node_loop,2) = all_initial_node_coords(local_node_id,2); - } - if(nodal_density_flag) nodal_density(node_loop) = all_node_densities(local_node_id,0); - /* - if(myrank==1&&nodal_positions(node_loop,2)>10000000){ - std::cout << " LOCAL MATRIX DEBUG ON TASK " << myrank << std::endl; - std::cout << node_loop+1 <<" " << local_node_id <<" "<< nodes_in_elem(ielem, node_loop) << " "<< nodal_positions(node_loop,2) << std::endl; - std::fflush(stdout); - } - */ - //std::cout << local_node_id << " " << nodes_in_elem(ielem, node_loop) << " " << nodal_positions(node_loop,0) << " " << nodal_positions(node_loop,1) << " "<< nodal_positions(node_loop,2) <enable_inertia_center[1]) + { + inertia_center[1] = module_params->moment_of_inertia_center[1]; } - - //loop over quadrature points - for(int iquad=0; iquad < direct_product_count; iquad++){ - - //set current quadrature point - if(num_dim==3) z_quad = iquad/(num_gauss_points*num_gauss_points); - y_quad = (iquad % (num_gauss_points*num_gauss_points))/num_gauss_points; - x_quad = iquad % num_gauss_points; - quad_coordinate(0) = legendre_nodes_1D(x_quad); - quad_coordinate(1) = legendre_nodes_1D(y_quad); - if(num_dim==3) - quad_coordinate(2) = legendre_nodes_1D(z_quad); - - //set current quadrature weight - quad_coordinate_weight(0) = legendre_weights_1D(x_quad); - quad_coordinate_weight(1) = legendre_weights_1D(y_quad); - if(num_dim==3) - quad_coordinate_weight(2) = legendre_weights_1D(z_quad); - else - quad_coordinate_weight(2) = 1; - weight_multiply = quad_coordinate_weight(0)*quad_coordinate_weight(1)*quad_coordinate_weight(2); - - //compute shape functions at this point for the element type - elem->basis(basis_values,quad_coordinate); - - //compute all the necessary coordinates and derivatives at this point - - //compute shape function derivatives - elem->partial_xi_basis(basis_derivative_s1,quad_coordinate); - elem->partial_eta_basis(basis_derivative_s2,quad_coordinate); - elem->partial_mu_basis(basis_derivative_s3,quad_coordinate); - - //compute derivatives of x,y,z w.r.t the s,t,w isoparametric space needed by JT (Transpose of the Jacobian) - //derivative of x,y,z w.r.t s - JT_row1(0) = 0; - JT_row1(1) = 0; - JT_row1(2) = 0; - if(use_initial_coords){ - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - JT_row1(0) += initial_nodal_positions(node_loop,0)*basis_derivative_s1(node_loop); - JT_row1(1) += initial_nodal_positions(node_loop,1)*basis_derivative_s1(node_loop); - JT_row1(2) += initial_nodal_positions(node_loop,2)*basis_derivative_s1(node_loop); + else + { + inertia_center[1] = center_of_mass[1]; + } + if (num_dim == 3) + { + if (module_params->enable_inertia_center[2]) + { + inertia_center[2] = module_params->moment_of_inertia_center[2]; } - - //derivative of x,y,z w.r.t t - JT_row2(0) = 0; - JT_row2(1) = 0; - JT_row2(2) = 0; - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - JT_row2(0) += initial_nodal_positions(node_loop,0)*basis_derivative_s2(node_loop); - JT_row2(1) += initial_nodal_positions(node_loop,1)*basis_derivative_s2(node_loop); - JT_row2(2) += initial_nodal_positions(node_loop,2)*basis_derivative_s2(node_loop); + else + { + inertia_center[2] = center_of_mass[2]; } + } + const_host_vec_array all_node_densities; + if (nodal_density_flag) + { + all_node_densities = all_node_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); + } + int nodes_per_elem = elem->num_basis(); + int z_quad, y_quad, x_quad, direct_product_count; + size_t local_node_id; + LO ielem; + GO global_element_index; + real_t delx1, delx2; + + real_t Jacobian, weight_multiply; + // CArrayKokkos legendre_nodes_1D(num_gauss_points); + // CArrayKokkos legendre_weights_1D(num_gauss_points); + CArray legendre_nodes_1D(num_gauss_points); + CArray legendre_weights_1D(num_gauss_points); + real_t pointer_quad_coordinate[num_dim]; + real_t pointer_quad_coordinate_weight[num_dim]; + real_t pointer_interpolated_point[num_dim]; + real_t pointer_JT_row1[num_dim]; + real_t pointer_JT_row2[num_dim]; + real_t pointer_JT_row3[num_dim]; + ViewCArray quad_coordinate(pointer_quad_coordinate, num_dim); + ViewCArray quad_coordinate_weight(pointer_quad_coordinate_weight, num_dim); + ViewCArray interpolated_point(pointer_interpolated_point, num_dim); + ViewCArray JT_row1(pointer_JT_row1, num_dim); + ViewCArray JT_row2(pointer_JT_row2, num_dim); + ViewCArray JT_row3(pointer_JT_row3, num_dim); + + real_t pointer_basis_values[elem->num_basis()]; + real_t pointer_basis_derivative_s1[elem->num_basis()]; + real_t pointer_basis_derivative_s2[elem->num_basis()]; + real_t pointer_basis_derivative_s3[elem->num_basis()]; + ViewCArray basis_values(pointer_basis_values, elem->num_basis()); + ViewCArray basis_derivative_s1(pointer_basis_derivative_s1, elem->num_basis()); + ViewCArray basis_derivative_s2(pointer_basis_derivative_s2, elem->num_basis()); + ViewCArray basis_derivative_s3(pointer_basis_derivative_s3, elem->num_basis()); + CArrayKokkos nodal_positions(elem->num_basis(), num_dim); + CArrayKokkos initial_nodal_positions(elem->num_basis(), num_dim); + CArrayKokkos nodal_density(elem->num_basis()); + CArrayKokkos current_position(num_dim); + + // initialize weights + elements::legendre_nodes_1D(legendre_nodes_1D, num_gauss_points); + elements::legendre_weights_1D(legendre_weights_1D, num_gauss_points); + + // initialize design gradients to 0 + for (int init = 0; init < nlocal_nodes; init++) + { + design_gradients(init, 0) = 0; + } - //derivative of x,y,z w.r.t w - JT_row3(0) = 0; - JT_row3(1) = 0; - JT_row3(2) = 0; - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - JT_row3(0) += initial_nodal_positions(node_loop,0)*basis_derivative_s3(node_loop); - JT_row3(1) += initial_nodal_positions(node_loop,1)*basis_derivative_s3(node_loop); - JT_row3(2) += initial_nodal_positions(node_loop,2)*basis_derivative_s3(node_loop); + // loop over elements and use quadrature rule to compute volume from Jacobian determinant + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + // acquire set of nodes for this local element + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + local_node_id = all_node_map->getLocalElement(nodes_in_elem(ielem, convert_node_order(node_loop))); + nodal_positions(node_loop, 0) = all_node_coords(local_node_id, 0); + nodal_positions(node_loop, 1) = all_node_coords(local_node_id, 1); + nodal_positions(node_loop, 2) = all_node_coords(local_node_id, 2); + if (use_initial_coords) + { + initial_nodal_positions(node_loop, 0) = all_initial_node_coords(local_node_id, 0); + initial_nodal_positions(node_loop, 1) = all_initial_node_coords(local_node_id, 1); + initial_nodal_positions(node_loop, 2) = all_initial_node_coords(local_node_id, 2); + } + if (nodal_density_flag) + { + nodal_density(node_loop) = all_node_densities(local_node_id, 0); + } + /* + if(myrank==1&&nodal_positions(node_loop,2)>10000000){ + std::cout << " LOCAL MATRIX DEBUG ON TASK " << myrank << std::endl; + std::cout << node_loop+1 <<" " << local_node_id <<" "<< nodes_in_elem(ielem, node_loop) << " "<< nodal_positions(node_loop,2) << std::endl; + std::fflush(stdout); + } + */ + // std::cout << local_node_id << " " << nodes_in_elem(ielem, node_loop) << " " << nodal_positions(node_loop,0) << " " << nodal_positions(node_loop,1) << " "<< nodal_positions(node_loop,2) <num_basis(); node_loop++){ - JT_row1(0) += nodal_positions(node_loop,0)*basis_derivative_s1(node_loop); - JT_row1(1) += nodal_positions(node_loop,1)*basis_derivative_s1(node_loop); - JT_row1(2) += nodal_positions(node_loop,2)*basis_derivative_s1(node_loop); + if (Element_Types(ielem) == elements::elem_types::Hex8) + { + direct_product_count = std::pow(num_gauss_points, num_dim); } - //derivative of x,y,z w.r.t t - JT_row2(0) = 0; - JT_row2(1) = 0; - JT_row2(2) = 0; - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - JT_row2(0) += nodal_positions(node_loop,0)*basis_derivative_s2(node_loop); - JT_row2(1) += nodal_positions(node_loop,1)*basis_derivative_s2(node_loop); - JT_row2(2) += nodal_positions(node_loop,2)*basis_derivative_s2(node_loop); - } + // loop over quadrature points + for (int iquad = 0; iquad < direct_product_count; iquad++) + { + // set current quadrature point + if (num_dim == 3) + { + z_quad = iquad / (num_gauss_points * num_gauss_points); + } + y_quad = (iquad % (num_gauss_points * num_gauss_points)) / num_gauss_points; + x_quad = iquad % num_gauss_points; + quad_coordinate(0) = legendre_nodes_1D(x_quad); + quad_coordinate(1) = legendre_nodes_1D(y_quad); + if (num_dim == 3) + { + quad_coordinate(2) = legendre_nodes_1D(z_quad); + } - //derivative of x,y,z w.r.t w - JT_row3(0) = 0; - JT_row3(1) = 0; - JT_row3(2) = 0; - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - JT_row3(0) += nodal_positions(node_loop,0)*basis_derivative_s3(node_loop); - JT_row3(1) += nodal_positions(node_loop,1)*basis_derivative_s3(node_loop); - JT_row3(2) += nodal_positions(node_loop,2)*basis_derivative_s3(node_loop); - } - } - - - //compute the determinant of the Jacobian - Jacobian = JT_row1(0)*(JT_row2(1)*JT_row3(2)-JT_row3(1)*JT_row2(2))- - JT_row1(1)*(JT_row2(0)*JT_row3(2)-JT_row3(0)*JT_row2(2))+ - JT_row1(2)*(JT_row2(0)*JT_row3(1)-JT_row3(0)*JT_row2(1)); - if(Jacobian<0) Jacobian = -Jacobian; - - //compute current position - current_position(0) = current_position(1) = current_position(2) = 0; - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - current_position(0) += nodal_positions(node_loop,0)*basis_values(node_loop); - current_position(1) += nodal_positions(node_loop,1)*basis_values(node_loop); - current_position(2) += nodal_positions(node_loop,2)*basis_values(node_loop); - } - - //assign contribution to every local node this element has - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - if(map->isNodeGlobalElement(nodes_in_elem(ielem, node_loop))){ - local_node_id = map->getLocalElement(nodes_in_elem(ielem, node_loop)); - if(inertia_component==0){ - delx1 = current_position(1) - inertia_center[1]; - delx2 = current_position(2) - inertia_center[2]; - design_gradients(local_node_id,0)+=weight_multiply*basis_values(node_loop)*(delx1*delx1 + delx2*delx2)*Jacobian; - } - if(inertia_component==1){ - delx1 = current_position(0) - inertia_center[0]; - delx2 = current_position(2) - inertia_center[2]; - design_gradients(local_node_id,0)+=weight_multiply*basis_values(node_loop)*(delx1*delx1 + delx2*delx2)*Jacobian; - } - if(inertia_component==2){ - delx1 = current_position(0) - inertia_center[0]; - delx2 = current_position(1) - inertia_center[1]; - design_gradients(local_node_id,0)+=weight_multiply*basis_values(node_loop)*(delx1*delx1 + delx2*delx2)*Jacobian; - } - if(inertia_component==3){ - delx1 = current_position(0) - inertia_center[0]; - delx2 = current_position(1) - inertia_center[1]; - design_gradients(local_node_id,0)-=weight_multiply*basis_values(node_loop)*(delx1*delx2)*Jacobian; - } - if(inertia_component==4){ - delx1 = current_position(0) - inertia_center[0]; - delx2 = current_position(2) - inertia_center[2]; - design_gradients(local_node_id,0)-=weight_multiply*basis_values(node_loop)*(delx1*delx2)*Jacobian; - } - if(inertia_component==5){ - delx1 = current_position(1) - inertia_center[1]; - delx2 = current_position(2) - inertia_center[2]; - design_gradients(local_node_id,0)-=weight_multiply*basis_values(node_loop)*(delx1*delx2)*Jacobian; + // set current quadrature weight + quad_coordinate_weight(0) = legendre_weights_1D(x_quad); + quad_coordinate_weight(1) = legendre_weights_1D(y_quad); + if (num_dim == 3) + { + quad_coordinate_weight(2) = legendre_weights_1D(z_quad); + } + else + { + quad_coordinate_weight(2) = 1; + } + weight_multiply = quad_coordinate_weight(0) * quad_coordinate_weight(1) * quad_coordinate_weight(2); + + // compute shape functions at this point for the element type + elem->basis(basis_values, quad_coordinate); + + // compute all the necessary coordinates and derivatives at this point + + // compute shape function derivatives + elem->partial_xi_basis(basis_derivative_s1, quad_coordinate); + elem->partial_eta_basis(basis_derivative_s2, quad_coordinate); + elem->partial_mu_basis(basis_derivative_s3, quad_coordinate); + + // compute derivatives of x,y,z w.r.t the s,t,w isoparametric space needed by JT (Transpose of the Jacobian) + // derivative of x,y,z w.r.t s + JT_row1(0) = 0; + JT_row1(1) = 0; + JT_row1(2) = 0; + if (use_initial_coords) + { + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row1(0) += initial_nodal_positions(node_loop, 0) * basis_derivative_s1(node_loop); + JT_row1(1) += initial_nodal_positions(node_loop, 1) * basis_derivative_s1(node_loop); + JT_row1(2) += initial_nodal_positions(node_loop, 2) * basis_derivative_s1(node_loop); + } + + // derivative of x,y,z w.r.t t + JT_row2(0) = 0; + JT_row2(1) = 0; + JT_row2(2) = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row2(0) += initial_nodal_positions(node_loop, 0) * basis_derivative_s2(node_loop); + JT_row2(1) += initial_nodal_positions(node_loop, 1) * basis_derivative_s2(node_loop); + JT_row2(2) += initial_nodal_positions(node_loop, 2) * basis_derivative_s2(node_loop); + } + + // derivative of x,y,z w.r.t w + JT_row3(0) = 0; + JT_row3(1) = 0; + JT_row3(2) = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row3(0) += initial_nodal_positions(node_loop, 0) * basis_derivative_s3(node_loop); + JT_row3(1) += initial_nodal_positions(node_loop, 1) * basis_derivative_s3(node_loop); + JT_row3(2) += initial_nodal_positions(node_loop, 2) * basis_derivative_s3(node_loop); + } + } + else + { + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row1(0) += nodal_positions(node_loop, 0) * basis_derivative_s1(node_loop); + JT_row1(1) += nodal_positions(node_loop, 1) * basis_derivative_s1(node_loop); + JT_row1(2) += nodal_positions(node_loop, 2) * basis_derivative_s1(node_loop); + } + + // derivative of x,y,z w.r.t t + JT_row2(0) = 0; + JT_row2(1) = 0; + JT_row2(2) = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row2(0) += nodal_positions(node_loop, 0) * basis_derivative_s2(node_loop); + JT_row2(1) += nodal_positions(node_loop, 1) * basis_derivative_s2(node_loop); + JT_row2(2) += nodal_positions(node_loop, 2) * basis_derivative_s2(node_loop); + } + + // derivative of x,y,z w.r.t w + JT_row3(0) = 0; + JT_row3(1) = 0; + JT_row3(2) = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row3(0) += nodal_positions(node_loop, 0) * basis_derivative_s3(node_loop); + JT_row3(1) += nodal_positions(node_loop, 1) * basis_derivative_s3(node_loop); + JT_row3(2) += nodal_positions(node_loop, 2) * basis_derivative_s3(node_loop); + } + } + + // compute the determinant of the Jacobian + Jacobian = JT_row1(0) * (JT_row2(1) * JT_row3(2) - JT_row3(1) * JT_row2(2)) - + JT_row1(1) * (JT_row2(0) * JT_row3(2) - JT_row3(0) * JT_row2(2)) + + JT_row1(2) * (JT_row2(0) * JT_row3(1) - JT_row3(0) * JT_row2(1)); + if (Jacobian < 0) + { + Jacobian = -Jacobian; + } + + // compute current position + current_position(0) = current_position(1) = current_position(2) = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + current_position(0) += nodal_positions(node_loop, 0) * basis_values(node_loop); + current_position(1) += nodal_positions(node_loop, 1) * basis_values(node_loop); + current_position(2) += nodal_positions(node_loop, 2) * basis_values(node_loop); + } + + // assign contribution to every local node this element has + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + if (map->isNodeGlobalElement(nodes_in_elem(ielem, node_loop))) + { + local_node_id = map->getLocalElement(nodes_in_elem(ielem, node_loop)); + if (inertia_component == 0) + { + delx1 = current_position(1) - inertia_center[1]; + delx2 = current_position(2) - inertia_center[2]; + design_gradients(local_node_id, 0) += weight_multiply * basis_values(node_loop) * (delx1 * delx1 + delx2 * delx2) * Jacobian; + } + if (inertia_component == 1) + { + delx1 = current_position(0) - inertia_center[0]; + delx2 = current_position(2) - inertia_center[2]; + design_gradients(local_node_id, 0) += weight_multiply * basis_values(node_loop) * (delx1 * delx1 + delx2 * delx2) * Jacobian; + } + if (inertia_component == 2) + { + delx1 = current_position(0) - inertia_center[0]; + delx2 = current_position(1) - inertia_center[1]; + design_gradients(local_node_id, 0) += weight_multiply * basis_values(node_loop) * (delx1 * delx1 + delx2 * delx2) * Jacobian; + } + if (inertia_component == 3) + { + delx1 = current_position(0) - inertia_center[0]; + delx2 = current_position(1) - inertia_center[1]; + design_gradients(local_node_id, 0) -= weight_multiply * basis_values(node_loop) * (delx1 * delx2) * Jacobian; + } + if (inertia_component == 4) + { + delx1 = current_position(0) - inertia_center[0]; + delx2 = current_position(2) - inertia_center[2]; + design_gradients(local_node_id, 0) -= weight_multiply * basis_values(node_loop) * (delx1 * delx2) * Jacobian; + } + if (inertia_component == 5) + { + delx1 = current_position(1) - inertia_center[1]; + delx2 = current_position(2) - inertia_center[2]; + design_gradients(local_node_id, 0) -= weight_multiply * basis_values(node_loop) * (delx1 * delx2) * Jacobian; + } + } } } - } } - - } - } /* ---------------------------------------------------------------------- Compute the volume of each element; estimated with quadrature ------------------------------------------------------------------------- */ -void FEA_Module_Inertial::compute_element_volumes(){ - //local number of uniquely assigned elements - size_t nonoverlap_nelements = element_map->getLocalNumElements(); - //local variable for host view in the dual view - const_host_vec_array all_node_coords = all_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_host_elem_conn_array nodes_in_elem = global_nodes_in_elem_distributed->getLocalView (Tpetra::Access::ReadOnly); - host_vec_array Element_Volumes = Global_Element_Volumes->getLocalView(Tpetra::Access::ReadWrite); - int nodes_per_elem = elem->num_basis(); - int z_quad,y_quad,x_quad, direct_product_count; - size_t local_node_id; - LO ielem; - GO global_element_index; - - real_t Jacobian, weight_multiply; - //CArrayKokkos legendre_nodes_1D(num_gauss_points); - //CArrayKokkos legendre_weights_1D(num_gauss_points); - CArray legendre_nodes_1D(num_gauss_points); - CArray legendre_weights_1D(num_gauss_points); - real_t pointer_quad_coordinate[num_dim]; - real_t pointer_quad_coordinate_weight[num_dim]; - real_t pointer_interpolated_point[num_dim]; - real_t pointer_JT_row1[num_dim]; - real_t pointer_JT_row2[num_dim]; - real_t pointer_JT_row3[num_dim]; - ViewCArray quad_coordinate(pointer_quad_coordinate,num_dim); - ViewCArray quad_coordinate_weight(pointer_quad_coordinate_weight,num_dim); - ViewCArray interpolated_point(pointer_interpolated_point,num_dim); - ViewCArray JT_row1(pointer_JT_row1,num_dim); - ViewCArray JT_row2(pointer_JT_row2,num_dim); - ViewCArray JT_row3(pointer_JT_row3,num_dim); - - real_t pointer_basis_values[elem->num_basis()]; - real_t pointer_basis_derivative_s1[elem->num_basis()]; - real_t pointer_basis_derivative_s2[elem->num_basis()]; - real_t pointer_basis_derivative_s3[elem->num_basis()]; - ViewCArray basis_values(pointer_basis_values,elem->num_basis()); - ViewCArray basis_derivative_s1(pointer_basis_derivative_s1,elem->num_basis()); - ViewCArray basis_derivative_s2(pointer_basis_derivative_s2,elem->num_basis()); - ViewCArray basis_derivative_s3(pointer_basis_derivative_s3,elem->num_basis()); - CArrayKokkos nodal_positions(elem->num_basis(),num_dim); - - //initialize weights - elements::legendre_nodes_1D(legendre_nodes_1D,num_gauss_points); - elements::legendre_weights_1D(legendre_weights_1D,num_gauss_points); - - Solver::node_ordering_convention active_node_ordering_convention = Solver_Pointer_->active_node_ordering_convention; - CArrayKokkos convert_node_order(max_nodes_per_element); - if((active_node_ordering_convention == Solver::ENSIGHT && num_dim==3)||(active_node_ordering_convention == Solver::IJK && num_dim==2)){ - convert_node_order(0) = 0; - convert_node_order(1) = 1; - convert_node_order(2) = 3; - convert_node_order(3) = 2; - if(num_dim == 3){ - convert_node_order(4) = 4; - convert_node_order(5) = 5; - convert_node_order(6) = 7; - convert_node_order(7) = 6; - } - } - else{ - convert_node_order(0) = 0; - convert_node_order(1) = 1; - convert_node_order(2) = 2; - convert_node_order(3) = 3; - if(num_dim==3){ - convert_node_order(4) = 4; - convert_node_order(5) = 5; - convert_node_order(6) = 6; - convert_node_order(7) = 7; - } - } - - - //loop over elements and use quadrature rule to compute volume from Jacobian determinant - for(int nonoverlapping_ielem = 0; nonoverlapping_ielem < nonoverlap_nelements; nonoverlapping_ielem++){ - global_element_index = element_map->getGlobalElement(nonoverlapping_ielem); - ielem = all_element_map->getLocalElement(global_element_index); - //debug print - //std::cout << "ELEMENT INDEX IS: " << ielem << " " <num_basis(); node_loop++){ - local_node_id = all_node_map->getLocalElement(nodes_in_elem(ielem, convert_node_order(node_loop))); - nodal_positions(node_loop,0) = all_node_coords(local_node_id,0); - nodal_positions(node_loop,1) = all_node_coords(local_node_id,1); - nodal_positions(node_loop,2) = all_node_coords(local_node_id,2); - /* - if(myrank==1&&nodal_positions(node_loop,2)>10000000){ - std::cout << " LOCAL MATRIX DEBUG ON TASK " << myrank << std::endl; - std::cout << node_loop+1 <<" " << local_node_id <<" "<< host_elem_conn_(ielem, node_loop) << " "<< nodal_positions(node_loop,2) << std::endl; - std::fflush(stdout); - } - */ - //std::cout << local_node_id << " " << nodes_in_elem(ielem, node_loop) << " " << nodal_positions(node_loop,0) << " " << nodal_positions(node_loop,1) << " "<< nodal_positions(node_loop,2) <getLocalNumElements(); + // local variable for host view in the dual view + const_host_vec_array all_node_coords = all_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_host_elem_conn_array nodes_in_elem = global_nodes_in_elem_distributed->getLocalView(Tpetra::Access::ReadOnly); + host_vec_array Element_Volumes = Global_Element_Volumes->getLocalView(Tpetra::Access::ReadWrite); + int nodes_per_elem = elem->num_basis(); + int z_quad, y_quad, x_quad, direct_product_count; + size_t local_node_id; + LO ielem; + GO global_element_index; + + real_t Jacobian, weight_multiply; + // CArrayKokkos legendre_nodes_1D(num_gauss_points); + // CArrayKokkos legendre_weights_1D(num_gauss_points); + CArray legendre_nodes_1D(num_gauss_points); + CArray legendre_weights_1D(num_gauss_points); + real_t pointer_quad_coordinate[num_dim]; + real_t pointer_quad_coordinate_weight[num_dim]; + real_t pointer_interpolated_point[num_dim]; + real_t pointer_JT_row1[num_dim]; + real_t pointer_JT_row2[num_dim]; + real_t pointer_JT_row3[num_dim]; + ViewCArray quad_coordinate(pointer_quad_coordinate, num_dim); + ViewCArray quad_coordinate_weight(pointer_quad_coordinate_weight, num_dim); + ViewCArray interpolated_point(pointer_interpolated_point, num_dim); + ViewCArray JT_row1(pointer_JT_row1, num_dim); + ViewCArray JT_row2(pointer_JT_row2, num_dim); + ViewCArray JT_row3(pointer_JT_row3, num_dim); + + real_t pointer_basis_values[elem->num_basis()]; + real_t pointer_basis_derivative_s1[elem->num_basis()]; + real_t pointer_basis_derivative_s2[elem->num_basis()]; + real_t pointer_basis_derivative_s3[elem->num_basis()]; + ViewCArray basis_values(pointer_basis_values, elem->num_basis()); + ViewCArray basis_derivative_s1(pointer_basis_derivative_s1, elem->num_basis()); + ViewCArray basis_derivative_s2(pointer_basis_derivative_s2, elem->num_basis()); + ViewCArray basis_derivative_s3(pointer_basis_derivative_s3, elem->num_basis()); + CArrayKokkos nodal_positions(elem->num_basis(), num_dim); + + // initialize weights + elements::legendre_nodes_1D(legendre_nodes_1D, num_gauss_points); + elements::legendre_weights_1D(legendre_weights_1D, num_gauss_points); + + Solver::node_ordering_convention active_node_ordering_convention = Solver_Pointer_->active_node_ordering_convention; + CArrayKokkos convert_node_order(max_nodes_per_element); + if ((active_node_ordering_convention == Solver::ENSIGHT && num_dim == 3) || (active_node_ordering_convention == Solver::IJK && num_dim == 2)) + { + convert_node_order(0) = 0; + convert_node_order(1) = 1; + convert_node_order(2) = 3; + convert_node_order(3) = 2; + if (num_dim == 3) + { + convert_node_order(4) = 4; + convert_node_order(5) = 5; + convert_node_order(6) = 7; + convert_node_order(7) = 6; + } } - - //initialize element volume - Element_Volumes(nonoverlapping_ielem,0) = 0; - - if(Element_Types(ielem)==elements::elem_types::Hex8){ - direct_product_count = std::pow(num_gauss_points,num_dim); + else + { + convert_node_order(0) = 0; + convert_node_order(1) = 1; + convert_node_order(2) = 2; + convert_node_order(3) = 3; + if (num_dim == 3) + { + convert_node_order(4) = 4; + convert_node_order(5) = 5; + convert_node_order(6) = 6; + convert_node_order(7) = 7; + } } - - //loop over quadrature points - for(int iquad=0; iquad < direct_product_count; iquad++){ - - //set current quadrature point - if(num_dim==3) z_quad = iquad/(num_gauss_points*num_gauss_points); - y_quad = (iquad % (num_gauss_points*num_gauss_points))/num_gauss_points; - x_quad = iquad % num_gauss_points; - quad_coordinate(0) = legendre_nodes_1D(x_quad); - quad_coordinate(1) = legendre_nodes_1D(y_quad); - if(num_dim==3) - quad_coordinate(2) = legendre_nodes_1D(z_quad); - - //set current quadrature weight - quad_coordinate_weight(0) = legendre_weights_1D(x_quad); - quad_coordinate_weight(1) = legendre_weights_1D(y_quad); - if(num_dim==3) - quad_coordinate_weight(2) = legendre_weights_1D(z_quad); - else - quad_coordinate_weight(2) = 1; - weight_multiply = quad_coordinate_weight(0)*quad_coordinate_weight(1)*quad_coordinate_weight(2); - - //compute shape functions at this point for the element type - elem->basis(basis_values,quad_coordinate); - - //compute all the necessary coordinates and derivatives at this point - - //compute shape function derivatives - elem->partial_xi_basis(basis_derivative_s1,quad_coordinate); - elem->partial_eta_basis(basis_derivative_s2,quad_coordinate); - elem->partial_mu_basis(basis_derivative_s3,quad_coordinate); - - //compute derivatives of x,y,z w.r.t the s,t,w isoparametric space needed by JT (Transpose of the Jacobian) - //derivative of x,y,z w.r.t s - JT_row1(0) = 0; - JT_row1(1) = 0; - JT_row1(2) = 0; - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - JT_row1(0) += nodal_positions(node_loop,0)*basis_derivative_s1(node_loop); - JT_row1(1) += nodal_positions(node_loop,1)*basis_derivative_s1(node_loop); - JT_row1(2) += nodal_positions(node_loop,2)*basis_derivative_s1(node_loop); - } - - //derivative of x,y,z w.r.t t - JT_row2(0) = 0; - JT_row2(1) = 0; - JT_row2(2) = 0; - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - JT_row2(0) += nodal_positions(node_loop,0)*basis_derivative_s2(node_loop); - JT_row2(1) += nodal_positions(node_loop,1)*basis_derivative_s2(node_loop); - JT_row2(2) += nodal_positions(node_loop,2)*basis_derivative_s2(node_loop); - } - - //derivative of x,y,z w.r.t w - JT_row3(0) = 0; - JT_row3(1) = 0; - JT_row3(2) = 0; - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - JT_row3(0) += nodal_positions(node_loop,0)*basis_derivative_s3(node_loop); - JT_row3(1) += nodal_positions(node_loop,1)*basis_derivative_s3(node_loop); - JT_row3(2) += nodal_positions(node_loop,2)*basis_derivative_s3(node_loop); - //debug print - /*if(myrank==1&&nodal_positions(node_loop,2)*basis_derivative_s3(node_loop)<-10000000){ - std::cout << " ELEMENT VOLUME JACOBIAN DEBUG ON TASK " << myrank << std::endl; - std::cout << node_loop+1 << " " << JT_row3(2) << " "<< nodal_positions(node_loop,2) <<" "<< basis_derivative_s3(node_loop) << std::endl; - std::fflush(stdout); - }*/ - } - - - //compute the determinant of the Jacobian - Jacobian = JT_row1(0)*(JT_row2(1)*JT_row3(2)-JT_row3(1)*JT_row2(2))- - JT_row1(1)*(JT_row2(0)*JT_row3(2)-JT_row3(0)*JT_row2(2))+ - JT_row1(2)*(JT_row2(0)*JT_row3(1)-JT_row3(0)*JT_row2(1)); - if(Jacobian<0) Jacobian = -Jacobian; - - Element_Volumes(nonoverlapping_ielem,0) += weight_multiply*Jacobian; + + // loop over elements and use quadrature rule to compute volume from Jacobian determinant + for (int nonoverlapping_ielem = 0; nonoverlapping_ielem < nonoverlap_nelements; nonoverlapping_ielem++) + { + global_element_index = element_map->getGlobalElement(nonoverlapping_ielem); + ielem = all_element_map->getLocalElement(global_element_index); + // debug print + // std::cout << "ELEMENT INDEX IS: " << ielem << " " <num_basis(); node_loop++) + { + local_node_id = all_node_map->getLocalElement(nodes_in_elem(ielem, convert_node_order(node_loop))); + nodal_positions(node_loop, 0) = all_node_coords(local_node_id, 0); + nodal_positions(node_loop, 1) = all_node_coords(local_node_id, 1); + nodal_positions(node_loop, 2) = all_node_coords(local_node_id, 2); + /* + if(myrank==1&&nodal_positions(node_loop,2)>10000000){ + std::cout << " LOCAL MATRIX DEBUG ON TASK " << myrank << std::endl; + std::cout << node_loop+1 <<" " << local_node_id <<" "<< host_elem_conn_(ielem, node_loop) << " "<< nodal_positions(node_loop,2) << std::endl; + std::fflush(stdout); + } + */ + // std::cout << local_node_id << " " << nodes_in_elem(ielem, node_loop) << " " << nodal_positions(node_loop,0) << " " << nodal_positions(node_loop,1) << " "<< nodal_positions(node_loop,2) <basis(basis_values, quad_coordinate); + + // compute all the necessary coordinates and derivatives at this point + + // compute shape function derivatives + elem->partial_xi_basis(basis_derivative_s1, quad_coordinate); + elem->partial_eta_basis(basis_derivative_s2, quad_coordinate); + elem->partial_mu_basis(basis_derivative_s3, quad_coordinate); + + // compute derivatives of x,y,z w.r.t the s,t,w isoparametric space needed by JT (Transpose of the Jacobian) + // derivative of x,y,z w.r.t s + JT_row1(0) = 0; + JT_row1(1) = 0; + JT_row1(2) = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row1(0) += nodal_positions(node_loop, 0) * basis_derivative_s1(node_loop); + JT_row1(1) += nodal_positions(node_loop, 1) * basis_derivative_s1(node_loop); + JT_row1(2) += nodal_positions(node_loop, 2) * basis_derivative_s1(node_loop); + } + + // derivative of x,y,z w.r.t t + JT_row2(0) = 0; + JT_row2(1) = 0; + JT_row2(2) = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row2(0) += nodal_positions(node_loop, 0) * basis_derivative_s2(node_loop); + JT_row2(1) += nodal_positions(node_loop, 1) * basis_derivative_s2(node_loop); + JT_row2(2) += nodal_positions(node_loop, 2) * basis_derivative_s2(node_loop); + } + + // derivative of x,y,z w.r.t w + JT_row3(0) = 0; + JT_row3(1) = 0; + JT_row3(2) = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row3(0) += nodal_positions(node_loop, 0) * basis_derivative_s3(node_loop); + JT_row3(1) += nodal_positions(node_loop, 1) * basis_derivative_s3(node_loop); + JT_row3(2) += nodal_positions(node_loop, 2) * basis_derivative_s3(node_loop); + // debug print + /*if(myrank==1&&nodal_positions(node_loop,2)*basis_derivative_s3(node_loop)<-10000000){ + std::cout << " ELEMENT VOLUME JACOBIAN DEBUG ON TASK " << myrank << std::endl; + std::cout << node_loop+1 << " " << JT_row3(2) << " "<< nodal_positions(node_loop,2) <<" "<< basis_derivative_s3(node_loop) << std::endl; + std::fflush(stdout); + }*/ + } + + // compute the determinant of the Jacobian + Jacobian = JT_row1(0) * (JT_row2(1) * JT_row3(2) - JT_row3(1) * JT_row2(2)) - + JT_row1(1) * (JT_row2(0) * JT_row3(2) - JT_row3(0) * JT_row2(2)) + + JT_row1(2) * (JT_row2(0) * JT_row3(1) - JT_row3(0) * JT_row2(1)); + if (Jacobian < 0) + { + Jacobian = -Jacobian; + } + + Element_Volumes(nonoverlapping_ielem, 0) += weight_multiply * Jacobian; + } } - } - - std::ostream &out = std::cout; - Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - //if(myrank==0) - //*fos << "Global Element Volumes:" << std::endl; - //Global_Element_Volumes->describe(*fos,Teuchos::VERB_EXTREME); - //*fos << std::endl; + + std::ostream& out = std::cout; + Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + // if(myrank==0) + // *fos << "Global Element Volumes:" << std::endl; + // Global_Element_Volumes->describe(*fos,Teuchos::VERB_EXTREME); + // *fos << std::endl; } /* ------------------------------------------------------------------------------------------- Communicate ghosts using the current optimization design data ---------------------------------------------------------------------------------------------- */ -void FEA_Module_Inertial::comm_variables(Teuchos::RCP zp){ - comm_densities(zp); +void FEA_Module_Inertial::comm_variables(Teuchos::RCP zp) +{ + comm_densities(zp); } diff --git a/src/Parallel-Solvers/FEA_Module_Inertial.h b/src/Parallel-Solvers/FEA_Module_Inertial.h index 395ef7c33..265adf32b 100644 --- a/src/Parallel-Solvers/FEA_Module_Inertial.h +++ b/src/Parallel-Solvers/FEA_Module_Inertial.h @@ -11,14 +11,14 @@ This program is open source under the BSD-3 License. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - + 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - + 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - + 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. @@ -42,58 +42,57 @@ class Inertial_Parameters; -class FEA_Module_Inertial: public FEA_Module{ - +class FEA_Module_Inertial : public FEA_Module +{ public: - FEA_Module_Inertial(Inertial_Parameters& params, Solver *Solver_Pointer, const int my_fea_module_index = 0); - ~FEA_Module_Inertial(); + FEA_Module_Inertial(Inertial_Parameters& params, Solver* Solver_Pointer, const int my_fea_module_index = 0); + ~FEA_Module_Inertial(); - void comm_variables(Teuchos::RCP zp); + void comm_variables(Teuchos::RCP zp); - void compute_element_volumes(); + void compute_element_volumes(); - void compute_element_masses(const_host_vec_array design_densities, bool max_flag, bool use_initial_coords = false); + void compute_element_masses(const_host_vec_array design_densities, bool max_flag, bool use_initial_coords = false); - void compute_element_moments(const_host_vec_array design_densities, bool max_flag, int moment_component, bool use_initial_coords = false); + void compute_element_moments(const_host_vec_array design_densities, bool max_flag, int moment_component, bool use_initial_coords = false); - void compute_element_moments_of_inertia(const_host_vec_array design_densities, bool max_flag, int inertia_component, bool use_initial_coords = false); + void compute_element_moments_of_inertia(const_host_vec_array design_densities, bool max_flag, int inertia_component, bool use_initial_coords = false); - void compute_nodal_gradients(const_host_vec_array design_densities, host_vec_array gradients, bool use_initial_coords = false); + void compute_nodal_gradients(const_host_vec_array design_densities, host_vec_array gradients, bool use_initial_coords = false); - void compute_moment_gradients(const_host_vec_array design_densities, host_vec_array gradients, int moment_component, bool use_initial_coords = false); + void compute_moment_gradients(const_host_vec_array design_densities, host_vec_array gradients, int moment_component, bool use_initial_coords = false); - void compute_moment_of_inertia_gradients(const_host_vec_array design_densities, host_vec_array gradients, int intertia_component, bool use_initial_coords = false); - - //forward declare - Inertial_Parameters *module_params; + void compute_moment_of_inertia_gradients(const_host_vec_array design_densities, host_vec_array gradients, int intertia_component, bool use_initial_coords = false); - //Global FEA data - Teuchos::RCP mass_gradients_distributed; - Teuchos::RCP center_of_mass_gradients_distributed; - Teuchos::RCP Global_Element_Volumes; - Teuchos::RCP Global_Element_Masses; - Teuchos::RCP Global_Element_Moments_x; - Teuchos::RCP Global_Element_Moments_y; - Teuchos::RCP Global_Element_Moments_z; - Teuchos::RCP Global_Element_Moments_of_Inertia_xx; - Teuchos::RCP Global_Element_Moments_of_Inertia_yy; - Teuchos::RCP Global_Element_Moments_of_Inertia_zz; - Teuchos::RCP Global_Element_Moments_of_Inertia_xy; - Teuchos::RCP Global_Element_Moments_of_Inertia_xz; - Teuchos::RCP Global_Element_Moments_of_Inertia_yz; + // forward declare + Inertial_Parameters* module_params; - //inertial properties - real_t mass, center_of_mass[3], moments_of_inertia[6]; + // Global FEA data + Teuchos::RCP mass_gradients_distributed; + Teuchos::RCP center_of_mass_gradients_distributed; + Teuchos::RCP Global_Element_Volumes; + Teuchos::RCP Global_Element_Masses; + Teuchos::RCP Global_Element_Moments_x; + Teuchos::RCP Global_Element_Moments_y; + Teuchos::RCP Global_Element_Moments_z; + Teuchos::RCP Global_Element_Moments_of_Inertia_xx; + Teuchos::RCP Global_Element_Moments_of_Inertia_yy; + Teuchos::RCP Global_Element_Moments_of_Inertia_zz; + Teuchos::RCP Global_Element_Moments_of_Inertia_xy; + Teuchos::RCP Global_Element_Moments_of_Inertia_xz; + Teuchos::RCP Global_Element_Moments_of_Inertia_yz; - bool use_initial_density; //if density variable is from initial configuration then jacobian is not needed + // inertial properties + real_t mass, center_of_mass[3], moments_of_inertia[6]; - //runtime flags - bool mass_init, com_init[3]; + bool use_initial_density; // if density variable is from initial configuration then jacobian is not needed - //update counters (first attempt at reducing redundant calls through ROL for Moments of Inertia and Center of Mass) - int mass_update, com_update[3]; - int mass_gradient_update, com_gradient_update[3]; + // runtime flags + bool mass_init, com_init[3]; + // update counters (first attempt at reducing redundant calls through ROL for Moments of Inertia and Center of Mass) + int mass_update, com_update[3]; + int mass_gradient_update, com_gradient_update[3]; }; #endif // end HEADER_H diff --git a/src/Parallel-Solvers/Parallel-Explicit/CMakeLists.txt b/src/Parallel-Solvers/Parallel-Explicit/CMakeLists.txt index c0dff6bb9..cbc46a920 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/CMakeLists.txt +++ b/src/Parallel-Solvers/Parallel-Explicit/CMakeLists.txt @@ -4,7 +4,7 @@ set(CMAKE_CXX_FLAGS "${Trilinos_CXX_COMPILER_FLAGS} ${CMAKE_CXX_FLAGS}") set(CMAKE_C_FLAGS "${Trilinos_C_COMPILER_FLAGS} ${CMAKE_C_FLAGS}") set(CMAKE_Fortran_FLAGS "${Trilinos_Fortran_COMPILER_FLAGS} ${CMAKE_Fortran_FLAGS}") -include_directories(SGH_Solver) +include_directories(SGH_Solver/include) include_directories(Dynamic_Elastic_Solver) #include_directories(Eulerian_Solver) include_directories(Topology_Optimization) diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.cpp b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.cpp index e09c67741..652d7e797 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.cpp @@ -11,14 +11,14 @@ This program is open source under the BSD-3 License. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - + 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - + 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - + 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. @@ -40,7 +40,7 @@ #include #include #include -#include +#include #include // fmin, fmax, abs note: fminl is long #include #include @@ -67,7 +67,7 @@ #include "Simulation_Parameters/FEA_Module/Dynamic_Elasticity_Parameters.h" #include "Simulation_Parameters/Simulation_Parameters_Explicit.h" -//optimization +// optimization #include "ROL_Solver.hpp" #include "Kinetic_Energy_Minimize.h" @@ -79,141 +79,143 @@ using namespace utils; - FEA_Module_Dynamic_Elasticity::FEA_Module_Dynamic_Elasticity( - Dynamic_Elasticity_Parameters& params, Solver *Solver_Pointer, - std::shared_ptr mesh_in, const int my_fea_module_index) - : FEA_Module(Solver_Pointer) { - - //assign interfacing index - my_fea_module_index_ = my_fea_module_index; - Module_Type = FEA_MODULE_TYPE::Dynamic_Elasticity; - - //recast solver pointer for non-base class access - Explicit_Solver_Pointer_ = dynamic_cast(Solver_Pointer); - module_params = ¶ms; - simparam = &(Explicit_Solver_Pointer_->simparam); - - mesh = mesh_in; - - //boundary condition data - max_boundary_sets = 0; - Local_Index_Boundary_Patches = Explicit_Solver_Pointer_->Local_Index_Boundary_Patches; - - //set Tpetra vector pointers - initial_node_velocities_distributed = Explicit_Solver_Pointer_->initial_node_velocities_distributed; - initial_node_coords_distributed = Explicit_Solver_Pointer_->initial_node_coords_distributed; - all_initial_node_coords_distributed = Explicit_Solver_Pointer_->all_initial_node_coords_distributed; - node_coords_distributed = Explicit_Solver_Pointer_->node_coords_distributed; - node_velocities_distributed = Explicit_Solver_Pointer_->node_velocities_distributed; - all_node_velocities_distributed = Explicit_Solver_Pointer_->all_node_velocities_distributed; - - if(simparam->topology_optimization_on||simparam->shape_optimization_on){ - all_cached_node_velocities_distributed = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - force_gradient_velocity = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - force_gradient_position = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - force_gradient_design = Teuchos::rcp(new MV(all_node_map, 1)); - corner_value_storage = Solver_Pointer->corner_value_storage; - corner_vector_storage = Solver_Pointer->corner_vector_storage; - relative_element_densities = DCArrayKokkos(rnum_elem, "relative_element_densities"); - } - - if(simparam->topology_optimization_on||simparam->shape_optimization_on||simparam->num_dims==2){ - node_masses_distributed = Teuchos::rcp(new MV(map, 1)); - ghost_node_masses_distributed = Teuchos::rcp(new MV(ghost_node_map, 1)); - adjoint_vector_distributed = Teuchos::rcp(new MV(map, simparam->num_dims)); - phi_adjoint_vector_distributed = Teuchos::rcp(new MV(map, simparam->num_dims)); - } - - //setup output - noutput = 0; - init_output(); - - //optimization flags - kinetic_energy_objective = false; - - - //set parameters - Dynamic_Options dynamic_options = simparam->dynamic_options; - time_value = dynamic_options.time_value; - time_final = dynamic_options.time_final; - dt_max = dynamic_options.dt_max; - dt_min = dynamic_options.dt_min; - dt_cfl = dynamic_options.dt_cfl; - graphics_time = simparam->output_options.graphics_time; - graphics_dt_ival = simparam->output_options.graphics_dt_ival; - graphics_cyc_ival = simparam->output_options.graphics_cyc_ival; - cycle_stop = dynamic_options.cycle_stop; - rk_num_stages = dynamic_options.rk_num_stages; - dt = dynamic_options.dt; - fuzz = dynamic_options.fuzz; - tiny = dynamic_options.tiny; - small = dynamic_options.small; - graphics_times = simparam->output_options.graphics_times; - graphics_id = simparam->output_options.graphics_id; - rk_num_bins = simparam->dynamic_options.rk_num_bins; - - if(simparam->topology_optimization_on){ - max_time_steps = BUFFER_GROW; - forward_solve_velocity_data = Teuchos::rcp(new std::vector>(max_time_steps+1)); - time_data.resize(max_time_steps+1); - forward_solve_coordinate_data = Teuchos::rcp(new std::vector>(max_time_steps+1)); - adjoint_vector_data = Teuchos::rcp(new std::vector>(max_time_steps+1)); - phi_adjoint_vector_data = Teuchos::rcp(new std::vector>(max_time_steps+1)); - //assign a multivector of corresponding size to each new timestep in the buffer - for(int istep = 0; istep < max_time_steps+1; istep++){ - (*forward_solve_velocity_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - (*forward_solve_coordinate_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - (*adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - (*phi_adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + Dynamic_Elasticity_Parameters& params, Solver* Solver_Pointer, + std::shared_ptr mesh_in, const int my_fea_module_index) + : FEA_Module(Solver_Pointer) +{ + // assign interfacing index + my_fea_module_index_ = my_fea_module_index; + Module_Type = FEA_MODULE_TYPE::Dynamic_Elasticity; + + // recast solver pointer for non-base class access + Explicit_Solver_Pointer_ = dynamic_cast(Solver_Pointer); + module_params = ¶ms; + simparam = &(Explicit_Solver_Pointer_->simparam); + + mesh = mesh_in; + + // boundary condition data + max_boundary_sets = 0; + Local_Index_Boundary_Patches = Explicit_Solver_Pointer_->Local_Index_Boundary_Patches; + + // set Tpetra vector pointers + initial_node_velocities_distributed = Explicit_Solver_Pointer_->initial_node_velocities_distributed; + initial_node_coords_distributed = Explicit_Solver_Pointer_->initial_node_coords_distributed; + all_initial_node_coords_distributed = Explicit_Solver_Pointer_->all_initial_node_coords_distributed; + node_coords_distributed = Explicit_Solver_Pointer_->node_coords_distributed; + node_velocities_distributed = Explicit_Solver_Pointer_->node_velocities_distributed; + all_node_velocities_distributed = Explicit_Solver_Pointer_->all_node_velocities_distributed; + + if (simparam->topology_optimization_on || simparam->shape_optimization_on) + { + all_cached_node_velocities_distributed = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + force_gradient_velocity = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + force_gradient_position = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + force_gradient_design = Teuchos::rcp(new MV(all_node_map, 1)); + corner_value_storage = Solver_Pointer->corner_value_storage; + corner_vector_storage = Solver_Pointer->corner_vector_storage; + relative_element_densities = DCArrayKokkos(rnum_elem, "relative_element_densities"); + } + + if (simparam->topology_optimization_on || simparam->shape_optimization_on || simparam->num_dims == 2) + { + node_masses_distributed = Teuchos::rcp(new MV(map, 1)); + ghost_node_masses_distributed = Teuchos::rcp(new MV(ghost_node_map, 1)); + adjoint_vector_distributed = Teuchos::rcp(new MV(map, simparam->num_dims)); + phi_adjoint_vector_distributed = Teuchos::rcp(new MV(map, simparam->num_dims)); + } + + // setup output + noutput = 0; + init_output(); + + // optimization flags + kinetic_energy_objective = false; + + // set parameters + Dynamic_Options dynamic_options = simparam->dynamic_options; + time_value = dynamic_options.time_value; + time_final = dynamic_options.time_final; + dt_max = dynamic_options.dt_max; + dt_min = dynamic_options.dt_min; + dt_cfl = dynamic_options.dt_cfl; + graphics_time = simparam->output_options.graphics_time; + graphics_dt_ival = simparam->output_options.graphics_dt_ival; + graphics_cyc_ival = simparam->output_options.graphics_cyc_ival; + cycle_stop = dynamic_options.cycle_stop; + rk_num_stages = dynamic_options.rk_num_stages; + dt = dynamic_options.dt; + fuzz = dynamic_options.fuzz; + tiny = dynamic_options.tiny; + small = dynamic_options.small; + graphics_times = simparam->output_options.graphics_times; + graphics_id = simparam->output_options.graphics_id; + rk_num_bins = simparam->dynamic_options.rk_num_bins; + + if (simparam->topology_optimization_on) + { + max_time_steps = BUFFER_GROW; + forward_solve_velocity_data = Teuchos::rcp(new std::vector>(max_time_steps + 1)); + time_data.resize(max_time_steps + 1); + forward_solve_coordinate_data = Teuchos::rcp(new std::vector>(max_time_steps + 1)); + adjoint_vector_data = Teuchos::rcp(new std::vector>(max_time_steps + 1)); + phi_adjoint_vector_data = Teuchos::rcp(new std::vector>(max_time_steps + 1)); + // assign a multivector of corresponding size to each new timestep in the buffer + for (int istep = 0; istep < max_time_steps + 1; istep++) + { + (*forward_solve_velocity_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*forward_solve_coordinate_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*phi_adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + } } - - } } -FEA_Module_Dynamic_Elasticity::~FEA_Module_Dynamic_Elasticity(){ - //delete simparam; +FEA_Module_Dynamic_Elasticity::~FEA_Module_Dynamic_Elasticity() +{ + // delete simparam; } /* ---------------------------------------------------------------------- Read ANSYS dat format mesh file ------------------------------------------------------------------------- */ -void FEA_Module_Dynamic_Elasticity::read_conditions_ansys_dat(std::ifstream *in, std::streampos before_condition_header){ - - char ch; - int num_dim = simparam->num_dims; - int buffer_lines = 1000; - int max_word = 30; - auto input_options = simparam->input_options.value(); - int p_order = input_options.p_order; - real_t unit_scaling = input_options.unit_scaling; - int local_node_index, current_column_index; - size_t strain_count; - std::string skip_line, read_line, substring, token; - std::stringstream line_parse, line_parse2; - CArrayKokkos read_buffer; - CArrayKokkos read_buffer_indices; - int buffer_loop, buffer_iteration, buffer_iterations, scan_loop, nodes_per_element, words_per_line; - size_t read_index_start, node_rid, elem_gid; - LO local_dof_id; - GO node_gid; - real_t dof_value; - host_vec_array node_densities; - +void FEA_Module_Dynamic_Elasticity::read_conditions_ansys_dat(std::ifstream* in, std::streampos before_condition_header) +{ + char ch; + int num_dim = simparam->num_dims; + int buffer_lines = 1000; + int max_word = 30; + auto input_options = simparam->input_options.value(); + int p_order = input_options.p_order; + real_t unit_scaling = input_options.unit_scaling; + int local_node_index, current_column_index; + size_t strain_count; + std::string skip_line, read_line, substring, token; + std::stringstream line_parse, line_parse2; + CArrayKokkos read_buffer; + CArrayKokkos read_buffer_indices; + int buffer_loop, buffer_iteration, buffer_iterations, scan_loop, nodes_per_element, words_per_line; + size_t read_index_start, node_rid, elem_gid; + LO local_dof_id; + GO node_gid; + real_t dof_value; + host_vec_array node_densities; } // end read_conditions_ansys_dat // ----------------------------------------------------------------------------- // Interfaces read in data with the elastic solver data; currently a hack to streamline -//------------------------------------------------------------------------------ -void FEA_Module_Dynamic_Elasticity::elastic_interface_setup(node_t &node, - elem_t &elem, - corner_t &corner){ - - const size_t num_dim = simparam->num_dims; +// ------------------------------------------------------------------------------ +void FEA_Module_Dynamic_Elasticity::elastic_interface_setup(node_t& node, + elem_t& elem, + corner_t& corner) +{ + const size_t num_dim = simparam->num_dims; const size_t rk_num_bins = simparam->dynamic_options.rk_num_bins; num_nodes_in_elem = 1; - for (int dim=0; dimnall_nodes; int myrank = Explicit_Solver_Pointer_->myrank; int nranks = Explicit_Solver_Pointer_->nranks; - //printf("Num nodes assigned to MPI rank %lu is %lu\n" , myrank, nall_nodes); + // printf("Num nodes assigned to MPI rank %lu is %lu\n" , myrank, nall_nodes); // intialize node variables mesh->initialize_nodes(nall_nodes); mesh->initialize_local_nodes(Explicit_Solver_Pointer_->nlocal_nodes); node.initialize(rk_num_bins, nall_nodes, num_dim); - //std::cout << "Bin counts " << rk_num_bins << " Node counts " << nall_nodes << " Num dim " << num_dim << std::endl; + // std::cout << "Bin counts " << rk_num_bins << " Node counts " << nall_nodes << " Num dim " << num_dim << std::endl; - //view scope + // view scope { - host_vec_array interface_node_coords = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); - //save node data to node.coords - //std::cout << "NODE DATA ON RANK " << myrank << std::endl; - if(num_dim==2){ - for(int inode = 0; inode < nall_nodes; inode++){ - //std::cout << "Node index " << inode+1 << " "; - node.coords.host(0,inode,0) = interface_node_coords(inode,0); - //std::cout << host_node_coords_state(0,inode,0)+1<< " "; - node.coords.host(0,inode,1) = interface_node_coords(inode,1); - //std::cout << host_node_coords_state(0,inode,1)+1<< " "; + host_vec_array interface_node_coords = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + // save node data to node.coords + // std::cout << "NODE DATA ON RANK " << myrank << std::endl; + if (num_dim == 2) + { + for (int inode = 0; inode < nall_nodes; inode++) + { + // std::cout << "Node index " << inode+1 << " "; + node.coords.host(0, inode, 0) = interface_node_coords(inode, 0); + // std::cout << host_node_coords_state(0,inode,0)+1<< " "; + node.coords.host(0, inode, 1) = interface_node_coords(inode, 1); + // std::cout << host_node_coords_state(0,inode,1)+1<< " "; + } } - } - else if(num_dim==3){ - for(int inode = 0; inode < nall_nodes; inode++){ - //std::cout << "Node index " << inode+1 << " "; - node.coords.host(0,inode,0) = interface_node_coords(inode,0); - //std::cout << host_node_coords_state(0,inode,0)+1<< " "; - node.coords.host(0,inode,1) = interface_node_coords(inode,1); - //std::cout << host_node_coords_state(0,inode,1)+1<< " "; - - node.coords.host(0,inode,2) = interface_node_coords(inode,2); - //std::cout << host_node_coords_state(0,inode,2)+1<< std::endl; + else if (num_dim == 3) + { + for (int inode = 0; inode < nall_nodes; inode++) + { + // std::cout << "Node index " << inode+1 << " "; + node.coords.host(0, inode, 0) = interface_node_coords(inode, 0); + // std::cout << host_node_coords_state(0,inode,0)+1<< " "; + node.coords.host(0, inode, 1) = interface_node_coords(inode, 1); + // std::cout << host_node_coords_state(0,inode,1)+1<< " "; + + node.coords.host(0, inode, 2) = interface_node_coords(inode, 2); + // std::cout << host_node_coords_state(0,inode,2)+1<< std::endl; + } } - } - } //end view scope - // --- read in the elements in the mesh --- - + } // end view scope + // --- read in the elements in the mesh --- + rnum_elem = Explicit_Solver_Pointer_->rnum_elem; - //printf("Num elems assigned to MPI rank %lu is %lu\n" , myrank, rnum_elem); + // printf("Num elems assigned to MPI rank %lu is %lu\n" , myrank, rnum_elem); // intialize elem variables mesh->initialize_elems(rnum_elem, num_dim); elem.initialize(rk_num_bins, nall_nodes, 3); // always 3D here, even for 2D nodes_in_elem = mesh->nodes_in_elem; - //save data to nodes_in_elem.host - //CArrayKokkos host_mesh_nodes_in_elem(rnum_elem, num_nodes_in_elem); - //view scope + // save data to nodes_in_elem.host + // CArrayKokkos host_mesh_nodes_in_elem(rnum_elem, num_nodes_in_elem); + // view scope { - host_elem_conn_array interface_nodes_in_elem = Explicit_Solver_Pointer_->global_nodes_in_elem_distributed->getLocalView (Tpetra::Access::ReadWrite); - //save node data to node.coords - //std::cout << "ELEMENT CONNECTIVITY ON RANK " << myrank << std::endl; - for(int ielem = 0; ielem < rnum_elem; ielem++){ - //std::cout << "Element index " << ielem+1 << " "; - for(int inode = 0; inode < num_nodes_in_elem; inode++){ - nodes_in_elem.host(ielem,inode) = Explicit_Solver_Pointer_->all_node_map->getLocalElement(interface_nodes_in_elem(ielem,inode)); - //debug print - //std::cout << nodes_in_elem.get_kokkos_dual_view().h_view(ielem*num_nodes_in_elem + inode)+1<< " "; + host_elem_conn_array interface_nodes_in_elem = Explicit_Solver_Pointer_->global_nodes_in_elem_distributed->getLocalView(Tpetra::Access::ReadWrite); + // save node data to node.coords + // std::cout << "ELEMENT CONNECTIVITY ON RANK " << myrank << std::endl; + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + // std::cout << "Element index " << ielem+1 << " "; + for (int inode = 0; inode < num_nodes_in_elem; inode++) + { + nodes_in_elem.host(ielem, inode) = Explicit_Solver_Pointer_->all_node_map->getLocalElement(interface_nodes_in_elem(ielem, inode)); + // debug print + // std::cout << nodes_in_elem.get_kokkos_dual_view().h_view(ielem*num_nodes_in_elem + inode)+1<< " "; + } + // std::cout << std::endl; } - //std::cout << std::endl; - } } // update device side nodes_in_elem.update_device(); - //debug print - - //CArrayKokkos device_mesh_nodes_in_elem(rnum_elem, num_nodes_in_elem); - //device_mesh_nodes_in_elem.get_kokkos_view() = nodes_in_elem.get_kokkos_dual_view().d_view; - //host_mesh_nodes_in_elem.get_kokkos_view() = nodes_in_elem.get_kokkos_dual_view().view_host(); + // debug print + + // CArrayKokkos device_mesh_nodes_in_elem(rnum_elem, num_nodes_in_elem); + // device_mesh_nodes_in_elem.get_kokkos_view() = nodes_in_elem.get_kokkos_dual_view().d_view; + // host_mesh_nodes_in_elem.get_kokkos_view() = nodes_in_elem.get_kokkos_dual_view().view_host(); /* if(myrank==1){ std::cout << "ELEMENT CONNECTIVITY ON RANK 1 in LOCAL INDICES" << myrank << std::endl; @@ -335,7 +343,7 @@ void FEA_Module_Dynamic_Elasticity::elastic_interface_setup(node_t &node, //host_node_coords_state = CArrayKokkos(rk_num_bins, nall_nodes, num_dim); //host_all_node_coords_state.get_kokkos_view() = Kokkos::View("debug", rk_num_bins*nall_nodes*num_dim); //save node data to node.coords - + //std::cout << "ALL NODE DATA ON RANK " << myrank << std::endl; for(int inode = 0; inode < nall_nodes; inode++){ //std::cout << "Node index " << inode+1 << " "; @@ -349,38 +357,38 @@ void FEA_Module_Dynamic_Elasticity::elastic_interface_setup(node_t &node, */ // save the node coords to the current RK value - for (size_t node_gid=0; node_gidinitialize_corners(num_corners); corner.initialize(num_corners, num_dim); - + /* for(int inode = 0; inode < nall_nodes; inode++){ std::cout << "Node index " << inode+1 << " "; @@ -393,538 +401,578 @@ void FEA_Module_Dynamic_Elasticity::elastic_interface_setup(node_t &node, } */ // Close mesh input file - //fclose(in); + // fclose(in); return; - } /* ---------------------------------------------------------------------------- Initialize sets of element boundary surfaces and arrays for input conditions ------------------------------------------------------------------------------- */ -void FEA_Module_Dynamic_Elasticity::init_boundaries() { - max_boundary_sets = module_params->boundary_conditions.size(); - int num_dim = simparam->num_dims; - - // set the number of boundary sets - if(myrank == 0) - std::cout << "building boundary sets " << std::endl; - - //initialize to 1 since there must be at least 1 boundary set anyway; read in may occure later - if(max_boundary_sets==0) max_boundary_sets = 1; - //std::cout << "NUM BOUNDARY CONDITIONS ON RANK " << myrank << " FOR INIT " << num_boundary_conditions <(nall_nodes*num_dim, "Node_DOF_Boundary_Condition_Type"); - - //initialize - for(int init=0; init < nall_nodes*num_dim; init++) - Node_DOF_Boundary_Condition_Type(init) = NONE; - - Number_DOF_BCS = 0; +void FEA_Module_Dynamic_Elasticity::init_boundaries() +{ + max_boundary_sets = module_params->boundary_conditions.size(); + int num_dim = simparam->num_dims; + + // set the number of boundary sets + if (myrank == 0) + { + std::cout << "building boundary sets " << std::endl; + } + + // initialize to 1 since there must be at least 1 boundary set anyway; read in may occure later + if (max_boundary_sets == 0) + { + max_boundary_sets = 1; + } + // std::cout << "NUM BOUNDARY CONDITIONS ON RANK " << myrank << " FOR INIT " << num_boundary_conditions <(nall_nodes * num_dim, "Node_DOF_Boundary_Condition_Type"); + + // initialize + for (int init = 0; init < nall_nodes * num_dim; init++) + { + Node_DOF_Boundary_Condition_Type(init) = NONE; + } + + Number_DOF_BCS = 0; } /* ---------------------------------------------------------------------- initialize storage for element boundary surfaces corresponding to user BCs ------------------------------------------------------------------------- */ -void FEA_Module_Dynamic_Elasticity::init_boundary_sets (int num_sets){ +void FEA_Module_Dynamic_Elasticity::init_boundary_sets(int num_sets) +{ + if (num_sets == 0) + { + std::cout << " Warning: number of boundary conditions = 0"; + return; + } + // initialize maximum + max_boundary_sets = num_sets; + // std::cout << " DEBUG PRINT "<(num_sets, "Boundary_Condition_Type_List"); + NBoundary_Condition_Patches = CArrayKokkos(num_sets, "NBoundary_Condition_Patches"); + // std::cout << "NBOUNDARY PATCHES ON RANK " << myrank << " FOR INIT IS " << nboundary_patches <(num_sets, nboundary_patches, "Boundary_Condition_Patches"); + + // initialize data + for (int iset = 0; iset < num_sets; iset++) + { + NBoundary_Condition_Patches(iset) = 0; + } - if(num_sets == 0){ - std::cout << " Warning: number of boundary conditions = 0"; - return; - } - //initialize maximum - max_boundary_sets = num_sets; - //std::cout << " DEBUG PRINT "<(num_sets, "Boundary_Condition_Type_List"); - NBoundary_Condition_Patches = CArrayKokkos(num_sets, "NBoundary_Condition_Patches"); - //std::cout << "NBOUNDARY PATCHES ON RANK " << myrank << " FOR INIT IS " << nboundary_patches <(num_sets, nboundary_patches, "Boundary_Condition_Patches"); - - //initialize data - for(int iset = 0; iset < num_sets; iset++) NBoundary_Condition_Patches(iset) = 0; - - //initialize - for(int ibdy=0; ibdy < num_sets; ibdy++) Boundary_Condition_Type_List(ibdy) = NONE; + // initialize + for (int ibdy = 0; ibdy < num_sets; ibdy++) + { + Boundary_Condition_Type_List(ibdy) = NONE; + } } /* ---------------------------------------------------------------------------- Grow boundary conditions sets of element boundary surfaces ------------------------------------------------------------------------------- */ -void FEA_Module_Dynamic_Elasticity::grow_boundary_sets(int num_sets){ - int num_dim = simparam->num_dims; +void FEA_Module_Dynamic_Elasticity::grow_boundary_sets(int num_sets) +{ + int num_dim = simparam->num_dims; - if(num_sets == 0){ - std::cout << " Warning: number of boundary conditions being set to 0"; - return; - } - - //std::cout << " DEBUG PRINT "<max_boundary_sets){ - //temporary storage for previous data - CArrayKokkos Temp_Boundary_Condition_Type_List = Boundary_Condition_Type_List; - CArrayKokkos Temp_NBoundary_Condition_Patches = NBoundary_Condition_Patches; - CArrayKokkos Temp_Boundary_Condition_Patches = Boundary_Condition_Patches; - - max_boundary_sets = num_sets + 5; //5 is an arbitrary buffer - Boundary_Condition_Type_List = CArrayKokkos(max_boundary_sets, "Boundary_Condition_Type_List"); - NBoundary_Condition_Patches = CArrayKokkos(max_boundary_sets, "NBoundary_Condition_Patches"); - //std::cout << "NBOUNDARY PATCHES ON RANK " << myrank << " FOR GROW " << nboundary_patches <(max_boundary_sets, nboundary_patches, "Boundary_Condition_Patches"); - - //copy previous data back over - //std::cout << "NUM BOUNDARY CONDITIONS ON RANK " << myrank << " FOR COPY " << max_boundary_sets < max_boundary_sets) + { + // temporary storage for previous data + CArrayKokkos Temp_Boundary_Condition_Type_List = Boundary_Condition_Type_List; + CArrayKokkos Temp_NBoundary_Condition_Patches = NBoundary_Condition_Patches; + CArrayKokkos Temp_Boundary_Condition_Patches = Boundary_Condition_Patches; + + max_boundary_sets = num_sets + 5; // 5 is an arbitrary buffer + Boundary_Condition_Type_List = CArrayKokkos(max_boundary_sets, "Boundary_Condition_Type_List"); + NBoundary_Condition_Patches = CArrayKokkos(max_boundary_sets, "NBoundary_Condition_Patches"); + // std::cout << "NBOUNDARY PATCHES ON RANK " << myrank << " FOR GROW " << nboundary_patches <(max_boundary_sets, nboundary_patches, "Boundary_Condition_Patches"); + + // copy previous data back over + // std::cout << "NUM BOUNDARY CONDITIONS ON RANK " << myrank << " FOR COPY " << max_boundary_sets <output(FIELD::velocity); - bool output_strain_flag = simparam->output(FIELD::strain); - bool output_stress_flag = simparam->output(FIELD::stress); - int num_dim = simparam->num_dims; - int Brows; - if(num_dim==3) Brows = 6; - else Brows = 3; - - if(output_velocity_flag){ - //displacement_index is accessed by writers at the solver level for deformed output - output_velocity_index = noutput; - noutput += 1; - module_outputs.resize(noutput); - - vector_style.resize(noutput); - vector_style[noutput-1] = DOF; - - output_vector_sizes.resize(noutput); - output_vector_sizes[noutput-1] = num_dim; - - output_dof_names.resize(noutput); - output_dof_names[noutput-1].resize(num_dim); - output_dof_names[noutput-1][0] = "vx"; - output_dof_names[noutput-1][1] = "vy"; - if(num_dim==3) - output_dof_names[noutput-1][2] = "vz"; - } - if(output_strain_flag){ - output_strain_index = noutput; - noutput += 1; - module_outputs.resize(noutput); - - vector_style.resize(noutput); - vector_style[noutput-1] = NODAL; - - output_vector_sizes.resize(noutput); - output_vector_sizes[noutput-1] = Brows; - - output_dof_names.resize(noutput); - output_dof_names[noutput-1].resize(Brows); - if(num_dim==2){ - output_dof_names[noutput-1][0] = "strain_xx"; - output_dof_names[noutput-1][1] = "strain_yy"; - output_dof_names[noutput-1][2] = "strain_xy"; +void FEA_Module_Dynamic_Elasticity::init_output() +{ + // check user parameters for output + bool output_velocity_flag = simparam->output(FIELD::velocity); + bool output_strain_flag = simparam->output(FIELD::strain); + bool output_stress_flag = simparam->output(FIELD::stress); + int num_dim = simparam->num_dims; + int Brows; + if (num_dim == 3) + { + Brows = 6; + } + else + { + Brows = 3; } - if(num_dim==3){ - output_dof_names[noutput-1][0] = "strain_xx"; - output_dof_names[noutput-1][1] = "strain_yy"; - output_dof_names[noutput-1][2] = "strain_zz"; - output_dof_names[noutput-1][3] = "strain_xy"; - output_dof_names[noutput-1][4] = "strain_xz"; - output_dof_names[noutput-1][5] = "strain_yz"; + + if (output_velocity_flag) + { + // displacement_index is accessed by writers at the solver level for deformed output + output_velocity_index = noutput; + noutput += 1; + module_outputs.resize(noutput); + + vector_style.resize(noutput); + vector_style[noutput - 1] = DOF; + + output_vector_sizes.resize(noutput); + output_vector_sizes[noutput - 1] = num_dim; + + output_dof_names.resize(noutput); + output_dof_names[noutput - 1].resize(num_dim); + output_dof_names[noutput - 1][0] = "vx"; + output_dof_names[noutput - 1][1] = "vy"; + if (num_dim == 3) + { + output_dof_names[noutput - 1][2] = "vz"; + } } - } - if(output_stress_flag){ - output_stress_index = noutput; - noutput += 1; - module_outputs.resize(noutput); - - vector_style.resize(noutput); - vector_style[noutput-1] = NODAL; - - output_vector_sizes.resize(noutput); - output_vector_sizes[noutput-1] = Brows; - - output_dof_names.resize(noutput); - output_dof_names[noutput-1].resize(Brows); - if(num_dim==2){ - output_dof_names[noutput-1][0] = "stress_xx"; - output_dof_names[noutput-1][1] = "stress_yy"; - output_dof_names[noutput-1][3] = "stress_xy"; + if (output_strain_flag) + { + output_strain_index = noutput; + noutput += 1; + module_outputs.resize(noutput); + + vector_style.resize(noutput); + vector_style[noutput - 1] = NODAL; + + output_vector_sizes.resize(noutput); + output_vector_sizes[noutput - 1] = Brows; + + output_dof_names.resize(noutput); + output_dof_names[noutput - 1].resize(Brows); + if (num_dim == 2) + { + output_dof_names[noutput - 1][0] = "strain_xx"; + output_dof_names[noutput - 1][1] = "strain_yy"; + output_dof_names[noutput - 1][2] = "strain_xy"; + } + if (num_dim == 3) + { + output_dof_names[noutput - 1][0] = "strain_xx"; + output_dof_names[noutput - 1][1] = "strain_yy"; + output_dof_names[noutput - 1][2] = "strain_zz"; + output_dof_names[noutput - 1][3] = "strain_xy"; + output_dof_names[noutput - 1][4] = "strain_xz"; + output_dof_names[noutput - 1][5] = "strain_yz"; + } } - if(num_dim==3){ - output_dof_names[noutput-1][0] = "stress_xx"; - output_dof_names[noutput-1][1] = "stress_yy"; - output_dof_names[noutput-1][2] = "stress_zz"; - output_dof_names[noutput-1][3] = "stress_xy"; - output_dof_names[noutput-1][4] = "stress_xz"; - output_dof_names[noutput-1][5] = "stress_yz"; + if (output_stress_flag) + { + output_stress_index = noutput; + noutput += 1; + module_outputs.resize(noutput); + + vector_style.resize(noutput); + vector_style[noutput - 1] = NODAL; + + output_vector_sizes.resize(noutput); + output_vector_sizes[noutput - 1] = Brows; + + output_dof_names.resize(noutput); + output_dof_names[noutput - 1].resize(Brows); + if (num_dim == 2) + { + output_dof_names[noutput - 1][0] = "stress_xx"; + output_dof_names[noutput - 1][1] = "stress_yy"; + output_dof_names[noutput - 1][3] = "stress_xy"; + } + if (num_dim == 3) + { + output_dof_names[noutput - 1][0] = "stress_xx"; + output_dof_names[noutput - 1][1] = "stress_yy"; + output_dof_names[noutput - 1][2] = "stress_zz"; + output_dof_names[noutput - 1][3] = "stress_xy"; + output_dof_names[noutput - 1][4] = "stress_xz"; + output_dof_names[noutput - 1][5] = "stress_yz"; + } } - } } /* ------------------------------------------------------------------------------------------- Prompts sorting for elastic response output data. For now, nodal strains. ---------------------------------------------------------------------------------------------- */ -void FEA_Module_Dynamic_Elasticity::sort_output(Teuchos::RCP > sorted_map){ - - +void FEA_Module_Dynamic_Elasticity::sort_output(Teuchos::RCP> sorted_map) +{ } /* ------------------------------------------------------------------------------------------- populate requests this module makes for output data ---------------------------------------------------------------------------------------------- */ -void FEA_Module_Dynamic_Elasticity::write_data(std::map &point_data_scalars_double, - std::map &point_data_vectors_double, - std::map &cell_data_scalars_double, - std::map &cell_data_scalars_int, - std::map > &cell_data_fields_double){ - - - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; +void FEA_Module_Dynamic_Elasticity::write_data(std::map& point_data_scalars_double, + std::map& point_data_vectors_double, + std::map& cell_data_scalars_double, + std::map& cell_data_scalars_int, + std::map>& cell_data_fields_double) +{ + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - for (const FIELD& field_name : simparam->output_options.output_fields) { - switch (field_name) + for (const FIELD& field_name : simparam->output_options.output_fields) { - - case FIELD::velocity: - // node "velocity" - node_vel.update_host(); - point_data_vectors_double["velocity"] = &node_vel.host(rk_level,0,0); - break; - - case FIELD::element_density: - // element "density" - elem_den.update_host(); - cell_data_scalars_double["element_density"] = elem_den.host_pointer(); - break; - - case FIELD::pressure: - // element "pressure" - elem_pres.update_host(); - cell_data_scalars_double["pressure"] = elem_pres.host_pointer(); - break; - - case FIELD::volume: - // element "volume" - elem_vol.update_host(); - cell_data_scalars_double["volume"] = elem_vol.host_pointer(); - break; - - case FIELD::mass: - // element "mass" - elem_mass.update_host(); - cell_data_scalars_double["mass"] = elem_mass.host_pointer(); - break; - - case FIELD::material_id: - // element "material_id" - elem_mat_id.update_host(); - cell_data_scalars_int["material_id"] = reinterpret_cast(elem_mat_id.host_pointer()); - break; - - case FIELD::user_vars: - // element "user_vars" - elem_user_output_vars.update_host(); - cell_data_fields_double["user_vars"] = std::make_pair(elem_user_output_vars.host_pointer(), + switch (field_name) + { + case FIELD::velocity: + // node "velocity" + node_vel.update_host(); + point_data_vectors_double["velocity"] = &node_vel.host(rk_level, 0, 0); + break; + + case FIELD::element_density: + // element "density" + elem_den.update_host(); + cell_data_scalars_double["element_density"] = elem_den.host_pointer(); + break; + + case FIELD::pressure: + // element "pressure" + elem_pres.update_host(); + cell_data_scalars_double["pressure"] = elem_pres.host_pointer(); + break; + + case FIELD::volume: + // element "volume" + elem_vol.update_host(); + cell_data_scalars_double["volume"] = elem_vol.host_pointer(); + break; + + case FIELD::mass: + // element "mass" + elem_mass.update_host(); + cell_data_scalars_double["mass"] = elem_mass.host_pointer(); + break; + + case FIELD::material_id: + // element "material_id" + elem_mat_id.update_host(); + cell_data_scalars_int["material_id"] = reinterpret_cast(elem_mat_id.host_pointer()); + break; + + case FIELD::user_vars: + // element "user_vars" + elem_user_output_vars.update_host(); + cell_data_fields_double["user_vars"] = std::make_pair(elem_user_output_vars.host_pointer(), elem_user_output_vars.dims(1)); - case FIELD::stress: - // element "stress" - elem_stress.update_host(); - cell_data_fields_double["stress"] = std::make_pair(&elem_stress.host(rk_level,0,0,0), 9); - break; - - default: - break; - - } // end switch - } // end if - - // element "mat_id" //uncomment if needed (works fine) - //sgh_module->elem_mat_id.update_host(); - //cell_data_scalars_int["mat_id"] = reinterpret_cast(&sgh_module->elem_mat_id.host(0)); - - // element "user_output_vars" //uncomment if needed (works fine) - //sgh_module->elem_user_output_vars.update_host(); - //cell_data_fields_double["user_output_vars"] = std::make_pair(&sgh_module->elem_user_output_vars.host_pointer(), - // sgh_module->elem_user_output_vars.dims(1)); - - // element "stress" //uncomment if needed (works fine) - //sgh_module->elem_stress.update_host(); - //cell_data_fields_double["stress"] = std::make_pair(&sgh_module->elem_stress.host(rk_level,0,0,0), 9); + case FIELD::stress: + // element "stress" + elem_stress.update_host(); + cell_data_fields_double["stress"] = std::make_pair(&elem_stress.host(rk_level, 0, 0, 0), 9); + break; + + default: + break; + } // end switch + } // end if + + // element "mat_id" //uncomment if needed (works fine) + // sgh_module->elem_mat_id.update_host(); + // cell_data_scalars_int["mat_id"] = reinterpret_cast(&sgh_module->elem_mat_id.host(0)); + + // element "user_output_vars" //uncomment if needed (works fine) + // sgh_module->elem_user_output_vars.update_host(); + // cell_data_fields_double["user_output_vars"] = std::make_pair(&sgh_module->elem_user_output_vars.host_pointer(), + // sgh_module->elem_user_output_vars.dims(1)); + + // element "stress" //uncomment if needed (works fine) + // sgh_module->elem_stress.update_host(); + // cell_data_fields_double["stress"] = std::make_pair(&sgh_module->elem_stress.host(rk_level,0,0,0), 9); } /* ------------------------------------------------------------------------------------------- Prompts sorting for elastic response output data. For now, nodal strains. ---------------------------------------------------------------------------------------------- */ -void FEA_Module_Dynamic_Elasticity::sort_element_output(Teuchos::RCP > sorted_map){ - //interface element density data - { - host_vec_array Element_Densities = Global_Element_Densities->getLocalView (Tpetra::Access::ReadWrite); - elem_den.update_host(); - for(int ielem = 0; ielem < rnum_elem; ielem++){ - Element_Densities(ielem,0) = elem_den.host(ielem); - } - } - +void FEA_Module_Dynamic_Elasticity::sort_element_output(Teuchos::RCP> sorted_map) +{ + // interface element density data + { + host_vec_array Element_Densities = Global_Element_Densities->getLocalView(Tpetra::Access::ReadWrite); + elem_den.update_host(); + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + Element_Densities(ielem, 0) = elem_den.host(ielem); + } + } } /* ------------------------------------------------------------------------------------------- Prompts computation of elastic response output data. For now, nodal strains. ---------------------------------------------------------------------------------------------- */ -void FEA_Module_Dynamic_Elasticity::collect_output(Teuchos::RCP > global_reduce_map){ - +void FEA_Module_Dynamic_Elasticity::collect_output(Teuchos::RCP> global_reduce_map) +{ } /* ------------------------------------------------------------------------------------------- Prompts computation of elastic response output data. For now, nodal strains. ---------------------------------------------------------------------------------------------- */ -void FEA_Module_Dynamic_Elasticity::compute_output(){ - +void FEA_Module_Dynamic_Elasticity::compute_output() +{ } /* ---------------------------------------------------------------------- Communicate updated nodal velocities to ghost nodes ------------------------------------------------------------------------- */ -void FEA_Module_Dynamic_Elasticity::comm_node_masses(){ - - //debug print of design vector - //std::ostream &out = std::cout; - //Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - //if(myrank==0) - //*fos << "Density data :" << std::endl; - //node_densities_distributed->describe(*fos,Teuchos::VERB_EXTREME); - //*fos << std::endl; - //std::fflush(stdout); - - //communicate design densities - //create import object using local node indices map and all indices map - //Tpetra::Import importer(map, ghost_node_map); - - //comms to get ghosts - ghost_node_masses_distributed->doImport(*node_masses_distributed, *ghost_importer, Tpetra::INSERT); - //all_node_map->describe(*fos,Teuchos::VERB_EXTREME); - //all_node_velocities_distributed->describe(*fos,Teuchos::VERB_EXTREME); - - //update_count++; - //if(update_count==1){ - //MPI_Barrier(world); - //MPI_Abort(world,4); - //} +void FEA_Module_Dynamic_Elasticity::comm_node_masses() +{ + // debug print of design vector + // std::ostream &out = std::cout; + // Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + // if(myrank==0) + // *fos << "Density data :" << std::endl; + // node_densities_distributed->describe(*fos,Teuchos::VERB_EXTREME); + // *fos << std::endl; + // std::fflush(stdout); + + // communicate design densities + // create import object using local node indices map and all indices map + // Tpetra::Import importer(map, ghost_node_map); + + // comms to get ghosts + ghost_node_masses_distributed->doImport(*node_masses_distributed, *ghost_importer, Tpetra::INSERT); + // all_node_map->describe(*fos,Teuchos::VERB_EXTREME); + // all_node_velocities_distributed->describe(*fos,Teuchos::VERB_EXTREME); + + // update_count++; + // if(update_count==1){ + // MPI_Barrier(world); + // MPI_Abort(world,4); + // } } /* ------------------------------------------------------------------------------------------- Communicate ghosts using the current optimization design data ---------------------------------------------------------------------------------------------- */ -void FEA_Module_Dynamic_Elasticity::comm_variables(Teuchos::RCP zp){ - - if(simparam->topology_optimization_on){ - //set density vector to the current value chosen by the optimizer - test_node_densities_distributed = zp; - - //debug print of design vector - //std::ostream &out = std::cout; - //Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - //if(myrank==0) - //*fos << "Density data :" << std::endl; - //node_densities_distributed->describe(*fos,Teuchos::VERB_EXTREME); - //*fos << std::endl; - //std::fflush(stdout); - - //communicate design densities - //create import object using local node indices map and all indices map - //Tpetra::Import importer(map, all_node_map); - - //comms to get ghosts - all_node_densities_distributed->doImport(*test_node_densities_distributed, *importer, Tpetra::INSERT); - } - else if(simparam->shape_optimization_on){ - //clause to communicate boundary node data if the boundary nodes are ghosts on this rank - } +void FEA_Module_Dynamic_Elasticity::comm_variables(Teuchos::RCP zp) +{ + if (simparam->topology_optimization_on) + { + // set density vector to the current value chosen by the optimizer + test_node_densities_distributed = zp; + + // debug print of design vector + // std::ostream &out = std::cout; + // Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + // if(myrank==0) + // *fos << "Density data :" << std::endl; + // node_densities_distributed->describe(*fos,Teuchos::VERB_EXTREME); + // *fos << std::endl; + // std::fflush(stdout); + + // communicate design densities + // create import object using local node indices map and all indices map + // Tpetra::Import importer(map, all_node_map); + + // comms to get ghosts + all_node_densities_distributed->doImport(*test_node_densities_distributed, *importer, Tpetra::INSERT); + } + else if (simparam->shape_optimization_on) + { + // clause to communicate boundary node data if the boundary nodes are ghosts on this rank + } } - /* ------------------------------------------------------------------------------------------- enforce constraints on nodes due to BCS ---------------------------------------------------------------------------------------------- */ -void FEA_Module_Dynamic_Elasticity::node_density_constraints(host_vec_array &node_densities_lower_bound){ - - const size_t num_dim = mesh->num_dims; - const_vec_array all_initial_node_coords = all_initial_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - const size_t num_lcs = module_params->loading_conditions.size(); - - const DCArrayKokkos mat_fill = simparam->mat_fill; - const DCArrayKokkos loading = module_params->loading; - - //debug check - //std::cout << "NUMBER OF LOADING CONDITIONS: " << num_lcs << std::endl; - - // walk over the nodes to update the velocity - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - double current_node_coords[3]; - double radius; - for (size_t dim = 0; dim < num_dim; dim++){ - current_node_coords[dim] = all_initial_node_coords(node_gid, dim); - } // end for dim - radius = sqrt(current_node_coords[0]*current_node_coords[0]+current_node_coords[1]*current_node_coords[1]+current_node_coords[2]*current_node_coords[2]); - for(size_t ilc=0; ilc < num_lcs; ilc++){ - //debug check - //std::cout << "LOADING CONDITION VOLUME TYPE: " << to_string(loading(ilc).volume) << std::endl; - - bool fill_this = loading(ilc).volume.contains(current_node_coords); - if(fill_this){ - node_densities_lower_bound(node_gid,0) = 1; +void FEA_Module_Dynamic_Elasticity::node_density_constraints(host_vec_array& node_densities_lower_bound) +{ + const size_t num_dim = mesh->num_dims; + const_vec_array all_initial_node_coords = all_initial_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + const size_t num_lcs = module_params->loading_conditions.size(); + + const DCArrayKokkos mat_fill = simparam->mat_fill; + const DCArrayKokkos loading = module_params->loading; + + // debug check + // std::cout << "NUMBER OF LOADING CONDITIONS: " << num_lcs << std::endl; + + // walk over the nodes to update the velocity + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + double current_node_coords[3]; + double radius; + for (size_t dim = 0; dim < num_dim; dim++) + { + current_node_coords[dim] = all_initial_node_coords(node_gid, dim); + } // end for dim + radius = sqrt(current_node_coords[0] * current_node_coords[0] + current_node_coords[1] * current_node_coords[1] + current_node_coords[2] * current_node_coords[2]); + for (size_t ilc = 0; ilc < num_lcs; ilc++) + { + // debug check + // std::cout << "LOADING CONDITION VOLUME TYPE: " << to_string(loading(ilc).volume) << std::endl; + + bool fill_this = loading(ilc).volume.contains(current_node_coords); + if (fill_this) + { + node_densities_lower_bound(node_gid, 0) = 1; + } } - } }); // end for parallel for over nodes - } /* ---------------------------------------------------------------------------- Setup elastic solver data ------------------------------------------------------------------------------- */ -void FEA_Module_Dynamic_Elasticity::setup(){ +void FEA_Module_Dynamic_Elasticity::setup() +{ Dynamic_Options dynamic_options = simparam->dynamic_options; - const size_t rk_level = dynamic_options.rk_num_bins - 1; - const size_t num_fills = simparam->regions.size(); - const size_t rk_num_bins = dynamic_options.rk_num_bins; - const size_t num_bcs = module_params->boundary_conditions.size(); - const size_t num_materials = simparam->materials.size(); - const int num_dim = simparam->num_dims; + const size_t rk_level = dynamic_options.rk_num_bins - 1; + const size_t num_fills = simparam->regions.size(); + const size_t rk_num_bins = dynamic_options.rk_num_bins; + const size_t num_bcs = module_params->boundary_conditions.size(); + const size_t num_materials = simparam->materials.size(); + const int num_dim = simparam->num_dims; // --------------------------------------------------------------------- // obtain mesh data - // --------------------------------------------------------------------- + // --------------------------------------------------------------------- elastic_interface_setup(node_interface, elem_interface, corner_interface); mesh->build_corner_connectivity(); - //debug print of corner ids - /* - if(myrank==1){ - for(int i = 0; i < mesh.num_nodes; i++){ - - // loop over all corners around the node and calculate the nodal force - for (size_t corner_lid=0; corner_lidgetGlobalElement(i) << " " << i << " " << all_node_map->getLocalElement(all_node_map->getGlobalElement(i)) << " " << corner_gid << " " << std::endl; - - } // end for corner_lid - //std::cout << explicit_solver_pointer->all_node_map->getGlobalElement(i) << " " << node_force[0] << " " << node_force[1] << " " << node_force[2] << std::endl; - //std::cout << explicit_solver_pointer->all_node_map->getGlobalElement(i) << " " << node_mass(i) << std::endl; - } - } - */ - /* - if(myrank==1){ - for(int i = 0; i < mesh.num_elems; i++){ - - // loop over all corners around the node and calculate the nodal force - for (size_t corner_lid=0; corner_lidgetGlobalElement(mesh.nodes_in_elem(i, corner_lid)) <<" " << corner_gid << " " << std::endl; - - } // end for corner_lid - //std::cout << explicit_solver_pointer->all_node_map->getGlobalElement(i) << " " << node_force[0] << " " << node_force[1] << " " << node_force[2] << std::endl; - //std::cout << explicit_solver_pointer->all_node_map->getGlobalElement(i) << " " << node_mass(i) << std::endl; - } - } - */ + // debug print of corner ids + /* + if(myrank==1){ + for(int i = 0; i < mesh.num_nodes; i++){ + + // loop over all corners around the node and calculate the nodal force + for (size_t corner_lid=0; corner_lidgetGlobalElement(i) << " " << i << " " << all_node_map->getLocalElement(all_node_map->getGlobalElement(i)) << " " << corner_gid << " " << std::endl; + + } // end for corner_lid + //std::cout << explicit_solver_pointer->all_node_map->getGlobalElement(i) << " " << node_force[0] << " " << node_force[1] << " " << node_force[2] << std::endl; + //std::cout << explicit_solver_pointer->all_node_map->getGlobalElement(i) << " " << node_mass(i) << std::endl; + } + } + */ + /* + if(myrank==1){ + for(int i = 0; i < mesh.num_elems; i++){ + + // loop over all corners around the node and calculate the nodal force + for (size_t corner_lid=0; corner_lidgetGlobalElement(mesh.nodes_in_elem(i, corner_lid)) <<" " << corner_gid << " " << std::endl; + + } // end for corner_lid + //std::cout << explicit_solver_pointer->all_node_map->getGlobalElement(i) << " " << node_force[0] << " " << node_force[1] << " " << node_force[2] << std::endl; + //std::cout << explicit_solver_pointer->all_node_map->getGlobalElement(i) << " " << node_mass(i) << std::endl; + } + } + */ mesh->build_elem_elem_connectivity(); mesh->num_bdy_patches = nboundary_patches; - if(num_dim==2){ - mesh->build_patch_connectivity(); - mesh->build_node_node_connectivity(); + if (num_dim == 2) + { + mesh->build_patch_connectivity(); + mesh->build_node_node_connectivity(); } - - // --------------------------------------------------------------------- - // allocate memory - // --------------------------------------------------------------------- - - // shorthand names - const size_t num_nodes = mesh->num_nodes; - const size_t num_elems = mesh->num_elems; + + // --------------------------------------------------------------------- + // allocate memory + // --------------------------------------------------------------------- + + // shorthand names + const size_t num_nodes = mesh->num_nodes; + const size_t num_elems = mesh->num_elems; const size_t num_corners = mesh->num_corners; - - // --- make dual views of data on CPU and GPU --- - // Notes: - // Instead of using a struct of dual types like the mesh type, - // individual dual views will be made for all the state - // variables. The motivation is to reduce memory movement - // when passing state into a function. Passing a struct by - // reference will copy the meta data and pointers for the - // variables held inside the struct. Since all the mesh - // variables are typically used by most functions, a single - // mesh struct or passing the arrays will be roughly equivalent - // for memory movement. - - + // --- make dual views of data on CPU and GPU --- + // Notes: + // Instead of using a struct of dual types like the mesh type, + // individual dual views will be made for all the state + // variables. The motivation is to reduce memory movement + // when passing state into a function. Passing a struct by + // reference will copy the meta data and pointers for the + // variables held inside the struct. Since all the mesh + // variables are typically used by most functions, a single + // mesh struct or passing the arrays will be roughly equivalent + // for memory movement. + // create Dual Views of the individual node struct variables - node_coords = DViewCArrayKokkos(node_interface.coords.get_kokkos_dual_view().view_host().data(),rk_num_bins,num_nodes,num_dim); + node_coords = DViewCArrayKokkos(node_interface.coords.get_kokkos_dual_view().view_host().data(), rk_num_bins, num_nodes, num_dim); - node_vel = DViewCArrayKokkos(node_interface.vel.get_kokkos_dual_view().view_host().data(),rk_num_bins,num_nodes,num_dim); + node_vel = DViewCArrayKokkos(node_interface.vel.get_kokkos_dual_view().view_host().data(), rk_num_bins, num_nodes, num_dim); + + node_mass = DViewCArrayKokkos(node_interface.mass.get_kokkos_dual_view().view_host().data(), num_nodes); - node_mass = DViewCArrayKokkos(node_interface.mass.get_kokkos_dual_view().view_host().data(),num_nodes); - - // create Dual Views of the individual elem struct variables - elem_den= DViewCArrayKokkos(&elem_interface.den(0), + elem_den = DViewCArrayKokkos(&elem_interface.den(0), num_elems); elem_pres = DViewCArrayKokkos(&elem_interface.pres(0), num_elems); - elem_stress = DViewCArrayKokkos(&elem_interface.stress(0,0,0,0), + elem_stress = DViewCArrayKokkos(&elem_interface.stress(0, 0, 0, 0), rk_num_bins, num_elems, 3, @@ -933,38 +981,37 @@ void FEA_Module_Dynamic_Elasticity::setup(){ elem_sspd = DViewCArrayKokkos(&elem_interface.sspd(0), num_elems); - elem_sie = DViewCArrayKokkos(&elem_interface.sie(0,0), + elem_sie = DViewCArrayKokkos(&elem_interface.sie(0, 0), rk_num_bins, num_elems); elem_vol = DViewCArrayKokkos(&elem_interface.vol(0), num_elems); - + elem_div = DViewCArrayKokkos(&elem_interface.div(0), num_elems); - elem_mass = DViewCArrayKokkos(&elem_interface.mass(0), num_elems); elem_mat_id = DViewCArrayKokkos(&elem_interface.mat_id(0), num_elems); - + // create Dual Views of the corner struct variables - corner_force = DViewCArrayKokkos (&corner_interface.force(0,0), - num_corners, + corner_force = DViewCArrayKokkos(&corner_interface.force(0, 0), + num_corners, num_dim); - corner_mass = DViewCArrayKokkos (&corner_interface.mass(0), + corner_mass = DViewCArrayKokkos(&corner_interface.mass(0), num_corners); - + // allocate elem_vel_grad - elem_vel_grad = DCArrayKokkos (num_elems,3,3); + elem_vel_grad = DCArrayKokkos(num_elems, 3, 3); // allocate material models - elem_eos = DCArrayKokkos (num_elems); - elem_strength = DCArrayKokkos (num_elems); - + elem_eos = DCArrayKokkos(num_elems); + elem_strength = DCArrayKokkos(num_elems); + // --------------------------------------------------------------------- // calculate geometry // --------------------------------------------------------------------- @@ -973,76 +1020,79 @@ void FEA_Module_Dynamic_Elasticity::setup(){ get_vol(); - //FEA_Module bc variable + // FEA_Module bc variable num_boundary_conditions = num_bcs; - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos mat_fill = simparam->mat_fill; - const DCArrayKokkos material = simparam->material; + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos mat_fill = simparam->mat_fill; + const DCArrayKokkos material = simparam->material; global_vars = simparam->global_vars; - state_vars = DCArrayKokkos (rnum_elem, simparam->max_num_state_vars); - elem_user_output_vars = DCArrayKokkos (rnum_elem, simparam->output_options.max_num_user_output_vars); - - //--- calculate bdy sets ---// - mesh->num_nodes_in_patch = 2*(num_dim-1); // 2 (2D) or 4 (3D) - mesh->num_patches_in_elem = 2*num_dim; // 4 (2D) or 6 (3D) + state_vars = DCArrayKokkos(rnum_elem, simparam->max_num_state_vars); + elem_user_output_vars = DCArrayKokkos(rnum_elem, simparam->output_options.max_num_user_output_vars); + + // --- calculate bdy sets ---// + mesh->num_nodes_in_patch = 2 * (num_dim - 1); // 2 (2D) or 4 (3D) + mesh->num_patches_in_elem = 2 * num_dim; // 4 (2D) or 6 (3D) mesh->init_bdy_sets(num_bcs); num_bdy_sets = mesh->num_bdy_sets; printf("Num BC's = %lu\n", num_bcs); // patch ids in bdy set bdy_patches_in_set = mesh->bdy_patches_in_set; - if(num_dim==2) - bdy_nodes = mesh->bdy_nodes; + if (num_dim == 2) + { + bdy_nodes = mesh->bdy_nodes; + } // tag boundary patches in the set tag_bdys(boundary, *mesh, node_coords); build_boundry_node_sets(*mesh); - + // node ids in bdy_patch set - bdy_nodes_in_set = mesh->bdy_nodes_in_set; + bdy_nodes_in_set = mesh->bdy_nodes_in_set; num_bdy_nodes_in_set = mesh->num_bdy_nodes_in_set; - - //assign mesh views needed by the FEA module + + // assign mesh views needed by the FEA module // elem ids in elem - elems_in_elem = mesh->elems_in_elem; + elems_in_elem = mesh->elems_in_elem; num_elems_in_elem = mesh->num_elems_in_elem; - //corners + // corners num_corners_in_node = mesh->num_corners_in_node; - corners_in_node = mesh->corners_in_node; - corners_in_elem = mesh->corners_in_elem; + corners_in_node = mesh->corners_in_node; + corners_in_elem = mesh->corners_in_elem; - //elem-node conn & node-node conn + // elem-node conn & node-node conn elems_in_node = mesh->elems_in_node; - if(num_dim==2){ - nodes_in_node = mesh->nodes_in_node; - num_nodes_in_node = mesh->num_nodes_in_node; - //patch conn - - patches_in_elem = mesh->patches_in_elem; - nodes_in_patch = mesh->nodes_in_patch; - elems_in_patch = mesh->elems_in_patch; + if (num_dim == 2) + { + nodes_in_node = mesh->nodes_in_node; + num_nodes_in_node = mesh->num_nodes_in_node; + // patch conn + + patches_in_elem = mesh->patches_in_elem; + nodes_in_patch = mesh->nodes_in_patch; + elems_in_patch = mesh->elems_in_patch; } // loop over BCs - for (size_t this_bdy = 0; this_bdy < num_bcs; this_bdy++){ - + for (size_t this_bdy = 0; this_bdy < num_bcs; this_bdy++) + { RUN_CLASS({ printf("Boundary Condition number %lu \n", this_bdy); printf(" Num bdy patches in this set = %lu \n", bdy_patches_in_set.stride(this_bdy)); printf(" Num bdy nodes in this set = %lu \n", bdy_nodes_in_set.stride(this_bdy)); }); Kokkos::fence(); - - }// end for + } // end for // elem_mat_id needs to be initialized before initialization of material models - for (int f_id = 0; f_id < num_fills; f_id++){ - FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { - elem_mat_id(elem_gid) = mat_fill(f_id).material_id; + for (int f_id = 0; f_id < num_fills; f_id++) + { + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { + elem_mat_id(elem_gid) = mat_fill(f_id).material_id; }); } elem_mat_id.update_host(); @@ -1054,7 +1104,7 @@ void FEA_Module_Dynamic_Elasticity::setup(){ global_vars, elem_user_output_vars, rnum_elem); - + // initialize strength model init_strength_model(elem_strength, material, @@ -1072,23 +1122,24 @@ void FEA_Module_Dynamic_Elasticity::setup(){ global_vars, elem_user_output_vars, rnum_elem); - - //--- apply the fill instructions over each of the Elements---// - - //initialize if topology optimization is used - if(simparam->topology_optimization_on){ - for(int elem_id = 0; elem_id < rnum_elem; elem_id++){ - relative_element_densities.host(elem_id) = 1; - }//for - relative_element_densities.update_device(); + + // --- apply the fill instructions over each of the Elements---// + + // initialize if topology optimization is used + if (simparam->topology_optimization_on) + { + for (int elem_id = 0; elem_id < rnum_elem; elem_id++) + { + relative_element_densities.host(elem_id) = 1; + } // for + relative_element_densities.update_device(); } - + // loop over the fill instructures - for (int f_id = 0; f_id < num_fills; f_id++){ - + for (int f_id = 0; f_id < num_fills; f_id++) + { // parallel loop over elements in mesh FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { - // calculate the coordinates and radius of the element double elem_coords[3]; // note:initialization with a list won't work elem_coords[0] = 0.0; @@ -1096,47 +1147,52 @@ void FEA_Module_Dynamic_Elasticity::setup(){ elem_coords[2] = 0.0; // get the coordinates of the element center - for (int node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ + for (int node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { elem_coords[0] += node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), 0); elem_coords[1] += node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), 1); - if (num_dim == 3){ + if (num_dim == 3) + { elem_coords[2] += node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), 2); - } else + } + else { elem_coords[2] = 0.0; } } // end loop over nodes in element - elem_coords[0] = elem_coords[0]/num_nodes_in_elem; - elem_coords[1] = elem_coords[1]/num_nodes_in_elem; - elem_coords[2] = elem_coords[2]/num_nodes_in_elem; + elem_coords[0] = elem_coords[0] / num_nodes_in_elem; + elem_coords[1] = elem_coords[1] / num_nodes_in_elem; + elem_coords[2] = elem_coords[2] / num_nodes_in_elem; // default is not to fill the element bool fill_this = mat_fill(f_id).volume.contains(elem_coords); // paint the material state on the element - if (fill_this){ - + if (fill_this) + { // density elem_den(elem_gid) = mat_fill(f_id).den; - + // mass - elem_mass(elem_gid) = elem_den(elem_gid)*elem_vol(elem_gid); - + elem_mass(elem_gid) = elem_den(elem_gid) * elem_vol(elem_gid); + // specific internal energy elem_sie(rk_level, elem_gid) = mat_fill(f_id).sie; - + size_t mat_id = elem_mat_id(elem_gid); // short name - + // --- stress tensor --- // always 3D even for 2D-RZ - for (size_t i=0; i<3; i++){ - for (size_t j=0; j<3; j++){ - elem_stress(rk_level,elem_gid,i,j) = 0.0; - } + for (size_t i = 0; i < 3; i++) + { + for (size_t j = 0; j < 3; j++) + { + elem_stress(rk_level, elem_gid, i, j) = 0.0; + } } // end for - + // short form for clean code - EOSParent * eos_model = elem_eos(elem_gid).model; + EOSParent* eos_model = elem_eos(elem_gid).model; // --- Pressure --- eos_model->calc_pressure(elem_pres, @@ -1148,7 +1204,7 @@ void FEA_Module_Dynamic_Elasticity::setup(){ elem_user_output_vars, elem_sspd, elem_den(elem_gid), - elem_sie(rk_level,elem_gid)); + elem_sie(rk_level, elem_gid)); // --- Sound speed --- eos_model->calc_sound_speed(elem_pres, @@ -1160,226 +1216,224 @@ void FEA_Module_Dynamic_Elasticity::setup(){ elem_user_output_vars, elem_sspd, elem_den(elem_gid), - elem_sie(rk_level,elem_gid)); - - // loop over the nodes of this element and apply velocity - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ + elem_sie(rk_level, elem_gid)); + // loop over the nodes of this element and apply velocity + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { // get the mesh node index size_t node_gid = nodes_in_elem(elem_gid, node_lid); - // --- Velocity --- - switch(mat_fill(f_id).velocity) + switch (mat_fill(f_id).velocity) { case VELOCITY_TYPE::cartesian: - { - - node_vel(rk_level, node_gid, 0) = mat_fill(f_id).u; - node_vel(rk_level, node_gid, 1) = mat_fill(f_id).v; - if (num_dim == 3) node_vel(rk_level, node_gid, 2) = mat_fill(f_id).w; - - - break; - } - case VELOCITY_TYPE::radial: - { - // Setting up cylindrical - double dir[2]; - dir[0] = 0.0; - dir[1] = 0.0; - double radius_val = 0.0; - - for(int dim=0; dim<2; dim++){ - dir[dim] = node_coords(rk_level, node_gid, dim); - radius_val += node_coords(rk_level, node_gid, dim)*node_coords(rk_level, node_gid, dim); - } // end for - radius_val = sqrt(radius_val); - - for(int dim=0; dim<2; dim++){ - if (radius_val > 1.0e-14){ - dir[dim] /= (radius_val); + { + node_vel(rk_level, node_gid, 0) = mat_fill(f_id).u; + node_vel(rk_level, node_gid, 1) = mat_fill(f_id).v; + if (num_dim == 3) + { + node_vel(rk_level, node_gid, 2) = mat_fill(f_id).w; } - else{ - dir[dim] = 0.0; + + break; + } + case VELOCITY_TYPE::radial: + { + // Setting up cylindrical + double dir[2]; + dir[0] = 0.0; + dir[1] = 0.0; + double radius_val = 0.0; + + for (int dim = 0; dim < 2; dim++) + { + dir[dim] = node_coords(rk_level, node_gid, dim); + radius_val += node_coords(rk_level, node_gid, dim) * node_coords(rk_level, node_gid, dim); + } // end for + radius_val = sqrt(radius_val); + + for (int dim = 0; dim < 2; dim++) + { + if (radius_val > 1.0e-14) + { + dir[dim] /= (radius_val); + } + else + { + dir[dim] = 0.0; + } + } // end for + + node_vel(rk_level, node_gid, 0) = mat_fill(f_id).speed * dir[0]; + node_vel(rk_level, node_gid, 1) = mat_fill(f_id).speed * dir[1]; + if (num_dim == 3) + { + node_vel(rk_level, node_gid, 2) = 0.0; } - } // end for - - - node_vel(rk_level, node_gid, 0) = mat_fill(f_id).speed*dir[0]; - node_vel(rk_level, node_gid, 1) = mat_fill(f_id).speed*dir[1]; - if (num_dim == 3) node_vel(rk_level, node_gid, 2) = 0.0; - - break; - } + + break; + } case VELOCITY_TYPE::spherical: - { - - // Setting up spherical - double dir[3]; - dir[0] = 0.0; - dir[1] = 0.0; - dir[2] = 0.0; - double radius_val = 0.0; - - for(int dim=0; dim<3; dim++){ - dir[dim] = node_coords(rk_level, node_gid, dim); - radius_val += node_coords(rk_level, node_gid, dim)*node_coords(rk_level, node_gid, dim); - } // end for - radius_val = sqrt(radius_val); - - for(int dim=0; dim<3; dim++){ - if (radius_val > 1.0e-14){ - dir[dim] /= (radius_val); + { + // Setting up spherical + double dir[3]; + dir[0] = 0.0; + dir[1] = 0.0; + dir[2] = 0.0; + double radius_val = 0.0; + + for (int dim = 0; dim < 3; dim++) + { + dir[dim] = node_coords(rk_level, node_gid, dim); + radius_val += node_coords(rk_level, node_gid, dim) * node_coords(rk_level, node_gid, dim); + } // end for + radius_val = sqrt(radius_val); + + for (int dim = 0; dim < 3; dim++) + { + if (radius_val > 1.0e-14) + { + dir[dim] /= (radius_val); + } + else + { + dir[dim] = 0.0; + } + } // end for + + node_vel(rk_level, node_gid, 0) = mat_fill(f_id).speed * dir[0]; + node_vel(rk_level, node_gid, 1) = mat_fill(f_id).speed * dir[1]; + if (num_dim == 3) + { + node_vel(rk_level, node_gid, 2) = mat_fill(f_id).speed * dir[2]; } - else{ - dir[dim] = 0.0; - } - } // end for - - node_vel(rk_level, node_gid, 0) = mat_fill(f_id).speed*dir[0]; - node_vel(rk_level, node_gid, 1) = mat_fill(f_id).speed*dir[1]; - if (num_dim == 3) node_vel(rk_level, node_gid, 2) = mat_fill(f_id).speed*dir[2]; - - break; - } + + break; + } case VELOCITY_TYPE::radial_linear: - { - - break; - } + { + break; + } case VELOCITY_TYPE::spherical_linear: - { - - break; - } + { + break; + } case VELOCITY_TYPE::tg_vortex: - { - - node_vel(rk_level, node_gid, 0) = sin(PI * node_coords(rk_level,node_gid, 0)) * cos(PI * node_coords(rk_level,node_gid, 1)); - node_vel(rk_level, node_gid, 1) = -1.0*cos(PI * node_coords(rk_level,node_gid, 0)) * sin(PI * node_coords(rk_level,node_gid, 1)); - if (num_dim == 3) node_vel(rk_level, node_gid, 2) = 0.0; + { + node_vel(rk_level, node_gid, 0) = sin(PI * node_coords(rk_level, node_gid, 0)) * cos(PI * node_coords(rk_level, node_gid, 1)); + node_vel(rk_level, node_gid, 1) = -1.0 * cos(PI * node_coords(rk_level, node_gid, 0)) * sin(PI * node_coords(rk_level, node_gid, 1)); + if (num_dim == 3) + { + node_vel(rk_level, node_gid, 2) = 0.0; + } - break; - } + break; + } } // end of switch + } // end loop over nodes of element - }// end loop over nodes of element - - - if(mat_fill(f_id).velocity == VELOCITY_TYPE::tg_vortex) + if (mat_fill(f_id).velocity == VELOCITY_TYPE::tg_vortex) { - elem_pres(elem_gid) = 0.25*( cos(2.0*PI*elem_coords[0]) + cos(2.0*PI*elem_coords[1]) ) + 1.0; - + elem_pres(elem_gid) = 0.25 * (cos(2.0 * PI * elem_coords[0]) + cos(2.0 * PI * elem_coords[1]) ) + 1.0; + // p = rho*ie*(gamma - 1) size_t mat_id = f_id; - double gamma = global_vars(mat_id,0); // gamma value + double gamma = global_vars(mat_id, 0); // gamma value elem_sie(rk_level, elem_gid) = - elem_pres(elem_gid)/(mat_fill(f_id).den*(gamma - 1.0)); + elem_pres(elem_gid) / (mat_fill(f_id).den * (gamma - 1.0)); } // end if - } // end if fill - }); // end FOR_ALL_CLASS element loop Kokkos::fence(); - - } // end for loop over fills - // apply BC's to velocity FEA_Module_Dynamic_Elasticity::boundary_velocity(*mesh, boundary, node_vel); - - + // calculate the corner massess if 2D - if(num_dim==2){ - + if (num_dim == 2) + { FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { - // facial area of the corners double corner_areas_array[4]; - - ViewCArrayKokkos corner_areas(&corner_areas_array[0],4); - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 4); - + + ViewCArrayKokkos corner_areas(&corner_areas_array[0], 4); + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 4); + get_area_weights2D(corner_areas, elem_gid, node_coords, elem_node_gids, rk_level); - + // loop over the corners of the element and calculate the mass - for (size_t corner_lid=0; corner_lid<4; corner_lid++){ - - size_t corner_gid = corners_in_elem(elem_gid, corner_lid); - corner_mass(corner_gid) = corner_areas(corner_lid)*elem_den(elem_gid); // node radius is added later - + for (size_t corner_lid = 0; corner_lid < 4; corner_lid++) + { + size_t corner_gid = corners_in_elem(elem_gid, corner_lid); + corner_mass(corner_gid) = corner_areas(corner_lid) * elem_den(elem_gid); // node radius is added later } // end for over corners }); - } // end of - - + // calculate the nodal mass FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - node_mass(node_gid) = 0.0; - - if(num_dim==3){ - - for(size_t elem_lid=0; elem_lidtopology_optimization_on||simparam->shape_optimization_on||simparam->num_dims==2){ - { - vec_array node_mass_interface = node_masses_distributed->getLocalView (Tpetra::Access::ReadWrite); + // current interface has differing mass arrays; this equates them until we unify memory + // view scope + if (simparam->topology_optimization_on || simparam->shape_optimization_on || simparam->num_dims == 2) + { + { + vec_array node_mass_interface = node_masses_distributed->getLocalView(Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - node_mass_interface(node_gid,0) = node_mass(node_gid); + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + node_mass_interface(node_gid, 0) = node_mass(node_gid); }); // end parallel for - } //end view scope - Kokkos::fence(); - //communicate ghost densities - comm_node_masses(); - - //this is forcing a copy to the device - //view scope - { - vec_array ghost_node_mass_interface = ghost_node_masses_distributed->getLocalView (Tpetra::Access::ReadWrite); + } // end view scope + Kokkos::fence(); + // communicate ghost densities + comm_node_masses(); + // this is forcing a copy to the device + // view scope + { + vec_array ghost_node_mass_interface = ghost_node_masses_distributed->getLocalView(Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, nlocal_nodes, nall_nodes, { - node_mass(node_gid) = ghost_node_mass_interface(node_gid-nlocal_nodes,0); + FOR_ALL_CLASS(node_gid, nlocal_nodes, nall_nodes, { + node_mass(node_gid) = ghost_node_mass_interface(node_gid - nlocal_nodes, 0); }); // end parallel for - } //end view scope - Kokkos::fence(); - } //endif - - //initialize if topology optimization is used - if(simparam->topology_optimization_on || simparam->shape_optimization_on){ - init_assembly(); - assemble_matrix(); + } // end view scope + Kokkos::fence(); + } // endif + + // initialize if topology optimization is used + if (simparam->topology_optimization_on || simparam->shape_optimization_on) + { + init_assembly(); + assemble_matrix(); } // update host copies of arrays modified in this function @@ -1388,27 +1442,27 @@ void FEA_Module_Dynamic_Elasticity::setup(){ elem_sie.update_host(); elem_stress.update_host(); elem_pres.update_host(); - elem_sspd.update_host(); + elem_sspd.update_host(); return; - } // end of setup /* ---------------------------------------------------------------------------- solve function called by solver ------------------------------------------------------------------------------- */ -void FEA_Module_Dynamic_Elasticity::module_cleanup(){ - cleanup_material_models(); +void FEA_Module_Dynamic_Elasticity::module_cleanup() +{ + cleanup_material_models(); } /* ---------------------------------------------------------------------------- Deallocate memory used for material models ------------------------------------------------------------------------------- */ -void FEA_Module_Dynamic_Elasticity::cleanup_material_models() { - - const DCArrayKokkos material = simparam->material; +void FEA_Module_Dynamic_Elasticity::cleanup_material_models() +{ + const DCArrayKokkos material = simparam->material; // destroy strength model destroy_strength_model(elem_strength, @@ -1428,50 +1482,46 @@ void FEA_Module_Dynamic_Elasticity::cleanup_material_models() { elem_user_output_vars, rnum_elem); return; - } // end cleanup_user_strength_model; - /** * Determines which of the boundary patches are associated with which boundary. - * + * * Modifies: bdy_patches_in_set */ -void FEA_Module_Dynamic_Elasticity::tag_bdys(const DCArrayKokkos &boundary, - mesh_t &mesh, - const DViewCArrayKokkos &node_coords){ - +void FEA_Module_Dynamic_Elasticity::tag_bdys(const DCArrayKokkos& boundary, + mesh_t& mesh, + const DViewCArrayKokkos& node_coords) +{ const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - size_t num_dim = simparam->num_dims; - int nboundary_patches = Explicit_Solver_Pointer_->nboundary_patches; - int num_nodes_in_patch = mesh.num_nodes_in_patch; - - //if (bdy_set == mesh.num_bdy_sets){ + size_t num_dim = simparam->num_dims; + int nboundary_patches = Explicit_Solver_Pointer_->nboundary_patches; + int num_nodes_in_patch = mesh.num_nodes_in_patch; + + // if (bdy_set == mesh.num_bdy_sets){ // printf(" ERROR: number of boundary sets must be increased by %zu", // bdy_set-mesh.num_bdy_sets+1); // exit(0); - //} // end if - - //error and debug flag - //DCArrayKokkos print_flag(1, "print_flag"); - //print_flag.host(0) = false; - //print_flag.update_device(); - - + // } // end if + + // error and debug flag + // DCArrayKokkos print_flag(1, "print_flag"); + // print_flag.host(0) = false; + // print_flag.update_device(); + FOR_ALL_CLASS(bdy_set, 0, num_bdy_sets, { - //for (size_t bdy_set = 0; bdy_set < num_bdy_sets; bdy_set++) { - + // for (size_t bdy_set = 0; bdy_set < num_bdy_sets; bdy_set++) { + // tag boundaries BOUNDARY_TYPE bc_type = boundary(bdy_set).surface.type; double val = boundary(bdy_set).surface.plane_position; - + // save the boundary patches to this set that are on the plane, spheres, etc. - for (size_t bdy_patch_lid=0; bdy_patch_lid < nboundary_patches; bdy_patch_lid++){ - + for (size_t bdy_patch_lid = 0; bdy_patch_lid < nboundary_patches; bdy_patch_lid++) + { // save the patch index size_t bdy_patch_gid = bdy_patch_lid; - - + // check to see if this patch is on the specified plane bool is_on_bdy = check_bdy(bdy_patch_gid, num_dim, @@ -1480,8 +1530,8 @@ void FEA_Module_Dynamic_Elasticity::tag_bdys(const DCArrayKokkos &b val, node_coords, rk_level); // no=0, yes=1 - - //debug check + + // debug check /* for (size_t patch_node_lid=0; patch_node_lid &b } */ - if (is_on_bdy){ - + if (is_on_bdy) + { size_t index = bdy_patches_in_set.stride(bdy_set); - + // increment the number of boundary patches saved - bdy_patches_in_set.stride(bdy_set) ++; - - + bdy_patches_in_set.stride(bdy_set)++; + bdy_patches_in_set(bdy_set, index) = bdy_patch_gid; } // end if - - } // end for bdy_patch - //} + // } }); // end FOR_ALL_CLASS bdy_sets - - //debug check - //print_flag.update_host(); - //if(print_flag.host(0)) std::cout << "found boundary node with id 549412" << std::endl; + + // debug check + // print_flag.update_host(); + // if(print_flag.host(0)) std::cout << "found boundary node with id 549412" << std::endl; return; } // end tag - KOKKOS_INLINE_FUNCTION -bool FEA_Module_Dynamic_Elasticity::check_bdy(const size_t patch_gid, - const int num_dim, - const int num_nodes_in_patch, - const BOUNDARY_TYPE bc_type, - const double val, - const DViewCArrayKokkos &node_coords, - const size_t rk_level) const { - +bool FEA_Module_Dynamic_Elasticity::check_bdy(const size_t patch_gid, + const int num_dim, + const int num_nodes_in_patch, + const BOUNDARY_TYPE bc_type, + const double val, + const DViewCArrayKokkos& node_coords, + const size_t rk_level) const +{ // default bool is not on the boundary size_t is_on_bdy = 0; - + // the patch coordinates double these_patch_coords[3]; // Note: cannot allocated array with num_dim - + // loop over the nodes on the patch - for (size_t patch_node_lid=0; patch_node_lid < num_nodes_in_patch; patch_node_lid++) { + for (size_t patch_node_lid = 0; patch_node_lid < num_nodes_in_patch; patch_node_lid++) + { // get the nodal_gid for this node in the patch - //size_t node_gid = mesh.nodes_in_patch(patch_gid, patch_node_lid); + // size_t node_gid = mesh.nodes_in_patch(patch_gid, patch_node_lid); size_t node_gid = Local_Index_Boundary_Patches(patch_gid, patch_node_lid); - for (size_t dim = 0; dim < num_dim; dim++){ + for (size_t dim = 0; dim < num_dim; dim++) + { these_patch_coords[dim] = node_coords(rk_level, node_gid, dim); // (rk, node_gid, dim) } - - if (bc_type == BOUNDARY_TYPE::x_plane) { - if ( fabs(these_patch_coords[0] - val) <= 1.0e-7 ) is_on_bdy += 1; + + if (bc_type == BOUNDARY_TYPE::x_plane) + { + if (fabs(these_patch_coords[0] - val) <= 1.0e-7) + { + is_on_bdy += 1; + } } - else if (bc_type == BOUNDARY_TYPE::y_plane) { - if ( fabs(these_patch_coords[1] - val) <= 1.0e-7 ) is_on_bdy += 1; + else if (bc_type == BOUNDARY_TYPE::y_plane) + { + if (fabs(these_patch_coords[1] - val) <= 1.0e-7) + { + is_on_bdy += 1; + } } - else if (bc_type == BOUNDARY_TYPE::z_plane) { - if ( fabs(these_patch_coords[2] - val) <= 1.0e-7 ) is_on_bdy += 1; + else if (bc_type == BOUNDARY_TYPE::z_plane) + { + if (fabs(these_patch_coords[2] - val) <= 1.0e-7) + { + is_on_bdy += 1; + } } - else if (bc_type == BOUNDARY_TYPE::cylinder){ - real_t R = sqrt(these_patch_coords[0]*these_patch_coords[0] + - these_patch_coords[1]*these_patch_coords[1]); - - if ( fabs(R - val) <= 1.0e-7 ) is_on_bdy += 1; + else if (bc_type == BOUNDARY_TYPE::cylinder) + { + real_t R = sqrt(these_patch_coords[0] * these_patch_coords[0] + + these_patch_coords[1] * these_patch_coords[1]); + + if (fabs(R - val) <= 1.0e-7) + { + is_on_bdy += 1; + } } - else if (bc_type == BOUNDARY_TYPE::sphere) { - real_t R = sqrt(these_patch_coords[0]*these_patch_coords[0] + - these_patch_coords[1]*these_patch_coords[1] + - these_patch_coords[2]*these_patch_coords[2]); - - if ( fabs(R - val) <= 1.0e-7 ) is_on_bdy += 1; + else if (bc_type == BOUNDARY_TYPE::sphere) + { + real_t R = sqrt(these_patch_coords[0] * these_patch_coords[0] + + these_patch_coords[1] * these_patch_coords[1] + + these_patch_coords[2] * these_patch_coords[2]); + + if (fabs(R - val) <= 1.0e-7) + { + is_on_bdy += 1; + } } } - + // if all nodes in the patch are on the surface return is_on_bdy == num_nodes_in_patch; } // end method to check bdy @@ -1570,94 +1638,82 @@ bool FEA_Module_Dynamic_Elasticity::check_bdy(const size_t patch_gid, Build set of nodes assigned to each boundary condition ------------------------------------------------------------------------------- */ -void FEA_Module_Dynamic_Elasticity::build_boundry_node_sets(mesh_t &mesh){ - +void FEA_Module_Dynamic_Elasticity::build_boundry_node_sets(mesh_t& mesh) +{ // build boundary nodes in each boundary set - int nboundary_patches = Explicit_Solver_Pointer_->nboundary_patches; + int nboundary_patches = Explicit_Solver_Pointer_->nboundary_patches; int num_nodes_in_patch = mesh.num_nodes_in_patch; - num_bdy_nodes_in_set = mesh.num_bdy_nodes_in_set = DCArrayKokkos (num_bdy_sets, "num_bdy_nodes_in_set"); - CArrayKokkos temp_count_num_bdy_nodes_in_set(num_bdy_sets, nall_nodes, "temp_count_num_bdy_nodes_in_set"); - - DynamicRaggedRightArrayKokkos temp_nodes_in_set (mesh.num_bdy_sets, nboundary_patches*mesh.num_nodes_in_patch, "temp_nodes_in_set"); - + num_bdy_nodes_in_set = mesh.num_bdy_nodes_in_set = DCArrayKokkos(num_bdy_sets, "num_bdy_nodes_in_set"); + CArrayKokkos temp_count_num_bdy_nodes_in_set(num_bdy_sets, nall_nodes, "temp_count_num_bdy_nodes_in_set"); + + DynamicRaggedRightArrayKokkos temp_nodes_in_set(mesh.num_bdy_sets, nboundary_patches * mesh.num_nodes_in_patch, "temp_nodes_in_set"); + // Parallel loop over boundary sets on device FOR_ALL_CLASS(bdy_set, 0, num_bdy_sets, { - // finde the number of patches_in_set size_t num_bdy_patches_in_set = bdy_patches_in_set.stride(bdy_set); num_bdy_nodes_in_set(bdy_set) = 0; - + // Loop over boundary patches in boundary set - for (size_t bdy_patch_gid = 0; bdy_patch_gid (mesh.num_bdy_nodes_in_set, "bdy_nodes_in_set"); + bdy_nodes_in_set = mesh.bdy_nodes_in_set = RaggedRightArrayKokkos(mesh.num_bdy_nodes_in_set, "bdy_nodes_in_set"); - FOR_ALL_CLASS (bdy_set, 0, num_bdy_sets, { - + FOR_ALL_CLASS(bdy_set, 0, num_bdy_sets, { // Loop over boundary patches in boundary set - for (size_t bdy_node_lid=0; bdy_node_liddynamic_options; - - const size_t rk_level = dynamic_options.rk_num_bins - 1; - time_value = dynamic_options.time_initial; - time_final = dynamic_options.time_final; - dt_max = dynamic_options.dt_max; - dt_min = dynamic_options.dt_min; - dt_cfl = dynamic_options.dt_cfl; - graphics_time = simparam->output_options.graphics_step; + + const size_t rk_level = dynamic_options.rk_num_bins - 1; + time_value = dynamic_options.time_initial; + time_final = dynamic_options.time_final; + dt_max = dynamic_options.dt_max; + dt_min = dynamic_options.dt_min; + dt_cfl = dynamic_options.dt_cfl; + graphics_time = simparam->output_options.graphics_step; graphics_dt_ival = simparam->output_options.graphics_step; - cycle_stop = dynamic_options.cycle_stop; - rk_num_stages = dynamic_options.rk_num_stages; - dt = dynamic_options.dt; - fuzz = dynamic_options.fuzz; - tiny = dynamic_options.tiny; + cycle_stop = dynamic_options.cycle_stop; + rk_num_stages = dynamic_options.rk_num_stages; + dt = dynamic_options.dt; + fuzz = dynamic_options.fuzz; + tiny = dynamic_options.tiny; small = dynamic_options.small; graphics_times = simparam->output_options.graphics_times; - graphics_id = simparam->output_options.graphics_id; - size_t num_bdy_nodes = mesh->num_bdy_nodes; - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos material = simparam->material; - int nTO_modules; - int old_max_forward_buffer; - unsigned long cycle; - const int num_dim = simparam->num_dims; - real_t objective_accumulation, global_objective_accumulation; - std::vector> FEA_Module_My_TO_Modules = simparam->FEA_Module_My_TO_Modules; - problem = Explicit_Solver_Pointer_->problem; //Pointer to ROL optimization problem object + graphics_id = simparam->output_options.graphics_id; + size_t num_bdy_nodes = mesh->num_bdy_nodes; + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos material = simparam->material; + int nTO_modules; + int old_max_forward_buffer; + unsigned long cycle; + const int num_dim = simparam->num_dims; + real_t objective_accumulation, global_objective_accumulation; + std::vector> FEA_Module_My_TO_Modules = simparam->FEA_Module_My_TO_Modules; + problem = Explicit_Solver_Pointer_->problem; // Pointer to ROL optimization problem object ROL::Ptr> obj_pointer; - //reset time accumulating objective and constraints + // reset time accumulating objective and constraints /* for(int imodule = 0 ; imodule < FEA_Module_My_TO_Modules[my_fea_module_index_].size(); imodule++){ current_module_index = FEA_Module_My_TO_Modules[my_fea_module_index_][imodule]; //test if module needs reset if(){ - + } } */ - //simple setup to just request KE for now; above loop to be expanded and used later for scanning modules - if(simparam->topology_optimization_on){ - obj_pointer = problem->getObjective(); - KineticEnergyMinimize_TopOpt& kinetic_energy_minimize_function = dynamic_cast(*obj_pointer); - kinetic_energy_minimize_function.objective_accumulation = 0; - global_objective_accumulation = objective_accumulation = 0; - kinetic_energy_objective = true; - if(max_time_steps +1 > forward_solve_velocity_data->size()){ - old_max_forward_buffer = forward_solve_velocity_data->size(); - time_data.resize(max_time_steps+1); - forward_solve_velocity_data->resize(max_time_steps+1); - forward_solve_coordinate_data->resize(max_time_steps+1); - adjoint_vector_data->resize(max_time_steps+1); - phi_adjoint_vector_data->resize(max_time_steps+1); - //assign a multivector of corresponding size to each new timestep in the buffer - for(int istep = old_max_forward_buffer; istep < max_time_steps+1; istep++){ - (*forward_solve_velocity_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - (*forward_solve_coordinate_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - (*adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - (*phi_adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + // simple setup to just request KE for now; above loop to be expanded and used later for scanning modules + if (simparam->topology_optimization_on) + { + obj_pointer = problem->getObjective(); + KineticEnergyMinimize_TopOpt& kinetic_energy_minimize_function = dynamic_cast(*obj_pointer); + kinetic_energy_minimize_function.objective_accumulation = 0; + global_objective_accumulation = objective_accumulation = 0; + kinetic_energy_objective = true; + if (max_time_steps + 1 > forward_solve_velocity_data->size()) + { + old_max_forward_buffer = forward_solve_velocity_data->size(); + time_data.resize(max_time_steps + 1); + forward_solve_velocity_data->resize(max_time_steps + 1); + forward_solve_coordinate_data->resize(max_time_steps + 1); + adjoint_vector_data->resize(max_time_steps + 1); + phi_adjoint_vector_data->resize(max_time_steps + 1); + // assign a multivector of corresponding size to each new timestep in the buffer + for (int istep = old_max_forward_buffer; istep < max_time_steps + 1; istep++) + { + (*forward_solve_velocity_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*forward_solve_coordinate_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*phi_adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + } } - } } - if(simparam->topology_optimization_on) - nTO_modules = simparam->TO_Module_List.size(); + if (simparam->topology_optimization_on) + { + nTO_modules = simparam->TO_Module_List.size(); + } int myrank = Explicit_Solver_Pointer_->myrank; - if(simparam->output_options.output_file_format==OUTPUT_FORMAT::vtk&&simparam->output_options.write_initial) + if (simparam->output_options.output_file_format == OUTPUT_FORMAT::vtk && simparam->output_options.write_initial) { - if(myrank==0) - printf("Writing outputs to file at %f \n", time_value); + if (myrank == 0) + { + printf("Writing outputs to file at %f \n", time_value); + } - double comm_time1 = Explicit_Solver_Pointer_->CPU_Time(); - Explicit_Solver_Pointer_->write_outputs(); - double comm_time2 = Explicit_Solver_Pointer_->CPU_Time(); - Explicit_Solver_Pointer_->output_time += comm_time2 - comm_time1; + double comm_time1 = Explicit_Solver_Pointer_->CPU_Time(); + Explicit_Solver_Pointer_->write_outputs(); + double comm_time2 = Explicit_Solver_Pointer_->CPU_Time(); + Explicit_Solver_Pointer_->output_time += comm_time2 - comm_time1; } - - CArrayKokkos node_extensive_mass(nall_nodes, "node_extensive_mass"); - + + CArrayKokkos node_extensive_mass(nall_nodes, "node_extensive_mass"); + // extensive energy tallies over the mesh elements local to this MPI rank double IE_t0 = 0.0; double KE_t0 = 0.0; double TE_t0 = 0.0; - + double IE_sum = 0.0; double KE_sum = 0.0; - + double IE_loc_sum = 0.0; double KE_loc_sum = 0.0; @@ -1783,138 +1848,139 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve(){ double global_TE_tend = 0.0; int nlocal_elem_non_overlapping = Explicit_Solver_Pointer_->nlocal_elem_non_overlapping; - + // extensive IE REDUCE_SUM_CLASS(elem_gid, 0, nlocal_elem_non_overlapping, IE_loc_sum, { - IE_loc_sum += elem_mass(elem_gid)*elem_sie(rk_level,elem_gid); - + IE_loc_sum += elem_mass(elem_gid) * elem_sie(rk_level, elem_gid); }, IE_sum); IE_t0 = IE_sum; - MPI_Allreduce(&IE_t0,&global_IE_t0,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD); - + MPI_Allreduce(&IE_t0, &global_IE_t0, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); + // extensive KE REDUCE_SUM_CLASS(node_gid, 0, nlocal_nodes, KE_loc_sum, { - double ke = 0; - for (size_t dim=0; dimtopology_optimization_on||simparam->shape_optimization_on){ - time_data[0] = 0; - //assign current velocity data to multivector - //view scope + // save initial data + if (simparam->topology_optimization_on || simparam->shape_optimization_on) { - vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView (Tpetra::Access::ReadWrite); - vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - node_velocities_interface(node_gid,idim) = node_vel(rk_level,node_gid,idim); - node_coords_interface(node_gid,idim) = node_coords(rk_level,node_gid,idim); - } + time_data[0] = 0; + // assign current velocity data to multivector + // view scope + { + vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); + vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + node_velocities_interface(node_gid, idim) = node_vel(rk_level, node_gid, idim); + node_coords_interface(node_gid, idim) = node_coords(rk_level, node_gid, idim); + } }); - } //end view scope - Kokkos::fence(); - - //communicate ghosts - double comm_time1 = Explicit_Solver_Pointer_->CPU_Time(); - - //active view scope; triggers host comms from updated data on device - { - const_host_vec_array node_velocities_host = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_host_vec_array node_coords_host = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - } - double comm_time2 = Explicit_Solver_Pointer_->CPU_Time(); - Explicit_Solver_Pointer_->dev2host_time += comm_time2-comm_time1; + } // end view scope + Kokkos::fence(); - //communicate ghost velocities - Explicit_Solver_Pointer_->comm_velocities(); - Explicit_Solver_Pointer_->comm_coordinates(); - - - double comm_time3 = Explicit_Solver_Pointer_->CPU_Time(); + // communicate ghosts + double comm_time1 = Explicit_Solver_Pointer_->CPU_Time(); - //view scope - { - const_vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array ghost_node_velocities_interface = Explicit_Solver_Pointer_->ghost_node_velocities_distributed->getLocalView (Tpetra::Access::ReadOnly); - vec_array all_node_velocities_interface = Explicit_Solver_Pointer_->all_node_velocities_distributed->getLocalView (Tpetra::Access::ReadWrite); - const_vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array ghost_node_coords_interface = Explicit_Solver_Pointer_->ghost_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - all_node_velocities_interface(node_gid,idim) = node_velocities_interface(node_gid,idim); - all_node_coords_interface(node_gid,idim) = node_coords_interface(node_gid,idim); + // active view scope; triggers host comms from updated data on device + { + const_host_vec_array node_velocities_host = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_host_vec_array node_coords_host = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); } + double comm_time2 = Explicit_Solver_Pointer_->CPU_Time(); + Explicit_Solver_Pointer_->dev2host_time += comm_time2 - comm_time1; + + // communicate ghost velocities + Explicit_Solver_Pointer_->comm_velocities(); + Explicit_Solver_Pointer_->comm_coordinates(); + + double comm_time3 = Explicit_Solver_Pointer_->CPU_Time(); + + // view scope + { + const_vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array ghost_node_velocities_interface = Explicit_Solver_Pointer_->ghost_node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); + vec_array all_node_velocities_interface = Explicit_Solver_Pointer_->all_node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); + const_vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array ghost_node_coords_interface = Explicit_Solver_Pointer_->ghost_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + all_node_velocities_interface(node_gid, idim) = node_velocities_interface(node_gid, idim); + all_node_coords_interface(node_gid, idim) = node_coords_interface(node_gid, idim); + } }); // end parallel for - Kokkos::fence(); + Kokkos::fence(); - FOR_ALL_CLASS(node_gid, nlocal_nodes, nlocal_nodes+nghost_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - all_node_velocities_interface(node_gid,idim) = ghost_node_velocities_interface(node_gid-nlocal_nodes,idim); - all_node_coords_interface(node_gid,idim) = ghost_node_coords_interface(node_gid-nlocal_nodes,idim); - } + FOR_ALL_CLASS(node_gid, nlocal_nodes, nlocal_nodes + nghost_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + all_node_velocities_interface(node_gid, idim) = ghost_node_velocities_interface(node_gid - nlocal_nodes, idim); + all_node_coords_interface(node_gid, idim) = ghost_node_coords_interface(node_gid - nlocal_nodes, idim); + } }); // end parallel for - Kokkos::fence(); - } //end view scope - - - (*forward_solve_velocity_data)[0]->assign(*Explicit_Solver_Pointer_->all_node_velocities_distributed); - (*forward_solve_coordinate_data)[0]->assign(*Explicit_Solver_Pointer_->all_node_coords_distributed); - } - - // loop over the max number of time integration cycles - for (cycle = 0; cycle < cycle_stop; cycle++) { - - // get the step - if(num_dim==2){ + Kokkos::fence(); + } // end view scope + + (*forward_solve_velocity_data)[0]->assign(*Explicit_Solver_Pointer_->all_node_velocities_distributed); + (*forward_solve_coordinate_data)[0]->assign(*Explicit_Solver_Pointer_->all_node_coords_distributed); + } + + // loop over the max number of time integration cycles + for (cycle = 0; cycle < cycle_stop; cycle++) + { + // get the step + if (num_dim == 2) + { get_timestep2D(*mesh, node_coords, node_vel, elem_sspd, elem_vol); } - else { + else + { get_timestep(*mesh, node_coords, node_vel, @@ -1923,29 +1989,35 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve(){ } // end if 2D double global_dt; - MPI_Allreduce(&dt,&global_dt,1,MPI_DOUBLE,MPI_MIN,MPI_COMM_WORLD); + MPI_Allreduce(&dt, &global_dt, 1, MPI_DOUBLE, MPI_MIN, MPI_COMM_WORLD); dt = global_dt; - // stop calculation if flag - //if (stop_calc == 1) break; - - - if(simparam->dynamic_options.output_time_sequence_level>=TIME_OUTPUT_LEVEL::high){ - if (cycle==0){ - if(myrank==0) - printf("cycle = %lu, time = %12.5e, time step = %12.5e \n", cycle, time_value, dt); + // stop calculation if flag + // if (stop_calc == 1) break; + + if (simparam->dynamic_options.output_time_sequence_level >= TIME_OUTPUT_LEVEL::high) + { + if (cycle == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %12.5e, time step = %12.5e \n", cycle, time_value, dt); + } + } + // print time step every 10 cycles + else if (cycle % 20 == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %12.5e, time step = %12.5e \n", cycle, time_value, dt); + } + } // end if } - // print time step every 10 cycles - else if (cycle%20==0){ - if(myrank==0) - printf("cycle = %lu, time = %12.5e, time step = %12.5e \n", cycle, time_value, dt); - } // end if - } - + // --------------------------------------------------------------------- // integrate the solution forward to t(n+1) via Runge Kutta (RK) method // --------------------------------------------------------------------- - + // save the values at t_n rk_init(node_coords, node_vel, @@ -1953,18 +2025,15 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve(){ elem_stress, rnum_elem, nall_nodes); - - - - // integrate solution forward in time - for (size_t rk_stage = 0; rk_stage < rk_num_stages; rk_stage++){ - + // integrate solution forward in time + for (size_t rk_stage = 0; rk_stage < rk_num_stages; rk_stage++) + { // ---- RK coefficient ---- - double rk_alpha = 1.0/((double)rk_num_stages - (double)rk_stage); - + double rk_alpha = 1.0 / ((double)rk_num_stages - (double)rk_stage); + // ---- Calculate velocity diveregence for the element ---- - get_divergence(elem_div, + get_divergence(elem_div, *mesh, node_coords, node_vel, @@ -1979,10 +2048,10 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve(){ for (size_t dim = 0; dim < num_dim; dim++){ node_force[dim] = 0.0; } // end for dim - + // loop over all corners around the node and calculate the nodal force for (size_t corner_lid=0; corner_lidall_node_map->getGlobalElement(i) << " " << corner_gid << " " << corner_force(corner_gid, 0) << " " << corner_force(corner_gid, 1) << " " << corner_force(corner_gid, 2) << std::endl; @@ -1990,7 +2059,7 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve(){ for (size_t dim = 0; dim < num_dim; dim++){ node_force[dim] += corner_force(corner_gid, dim); } // end for dim - + } // end for corner_lid //std::cout << Explicit_Solver_Pointer_->all_node_map->getGlobalElement(i) << " " << node_force[0] << " " << node_force[1] << " " << node_force[2] << std::endl; //std::cout << Explicit_Solver_Pointer_->all_node_map->getGlobalElement(i) << " " << node_mass(i) << std::endl; @@ -2018,7 +2087,7 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve(){ corner_force, rk_alpha, cycle); - applied_forces(material, + applied_forces(material, *mesh, node_coords, node_vel, @@ -2030,73 +2099,69 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve(){ corner_force, rk_alpha, cycle); - - + // ---- apply force boundary conditions to the boundary patches---- boundary_velocity(*mesh, boundary, node_vel); - //current interface has differing velocity arrays; this equates them until we unify memory - //first comm time interval point + // current interface has differing velocity arrays; this equates them until we unify memory + // first comm time interval point double comm_time1 = Explicit_Solver_Pointer_->CPU_Time(); - //view scope + // view scope { - vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - node_velocities_interface(node_gid,idim) = node_vel(rk_level,node_gid,idim); - } + vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + node_velocities_interface(node_gid, idim) = node_vel(rk_level, node_gid, idim); + } }); // end parallel for - } //end view scope + } // end view scope Kokkos::fence(); - - //active view scope + + // active view scope { - const_host_vec_array node_velocities_host = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView (Tpetra::Access::ReadOnly); + const_host_vec_array node_velocities_host = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); } double comm_time2 = Explicit_Solver_Pointer_->CPU_Time(); - Explicit_Solver_Pointer_->dev2host_time += comm_time2-comm_time1; - //communicate ghost velocities + Explicit_Solver_Pointer_->dev2host_time += comm_time2 - comm_time1; + // communicate ghost velocities Explicit_Solver_Pointer_->comm_velocities(); - + double comm_time3 = Explicit_Solver_Pointer_->CPU_Time(); - //this is forcing a copy to the device - //view scope + // this is forcing a copy to the device + // view scope { - vec_array ghost_node_velocities_interface = Explicit_Solver_Pointer_->ghost_node_velocities_distributed->getLocalView (Tpetra::Access::ReadWrite); + vec_array ghost_node_velocities_interface = Explicit_Solver_Pointer_->ghost_node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, nlocal_nodes, nall_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - node_vel(rk_level,node_gid,idim) = ghost_node_velocities_interface(node_gid-nlocal_nodes,idim); - } - + FOR_ALL_CLASS(node_gid, nlocal_nodes, nall_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + node_vel(rk_level, node_gid, idim) = ghost_node_velocities_interface(node_gid - nlocal_nodes, idim); + } }); // end parallel for - } //end view scope + } // end view scope Kokkos::fence(); - + double comm_time4 = Explicit_Solver_Pointer_->CPU_Time(); - Explicit_Solver_Pointer_->host2dev_time += comm_time4-comm_time3; - Explicit_Solver_Pointer_->communication_time += comm_time4-comm_time1; - //debug print vector values on a rank + Explicit_Solver_Pointer_->host2dev_time += comm_time4 - comm_time3; + Explicit_Solver_Pointer_->communication_time += comm_time4 - comm_time1; + // debug print vector values on a rank /* if(myrank==0) for(int i = 0; i < nall_nodes; i++){ std::cout << Explicit_Solver_Pointer_->all_node_map->getGlobalElement(i) << " " << node_vel(rk_level,i,0) << " " << node_vel(rk_level,i,1) << " " << node_vel(rk_level,i,2) << std::endl; } - */ - - + */ + // ---- Update nodal positions ---- update_position_elastic(rk_alpha, nall_nodes, node_coords, node_vel); - - + // ---- Calculate cell volume for next time step ---- get_vol(); - - - + // ---- Calculate elem state (den, pres, sound speed, stress) for next time step ---- /* if(num_dim==2){ @@ -2138,48 +2203,45 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve(){ // 2) hypo-elastic strength models are called in get_force // 3) strength models must be added by the user in user_mat.cpp - // calculate the new corner masses if 2D - if(num_dim==2){ - + if (num_dim == 2) + { // calculate the nodal areal mass FOR_ALL_CLASS(node_gid, 0, nall_nodes, { - node_mass(node_gid) = 0.0; - - if (node_coords(rk_level,node_gid,1) > tiny){ - node_mass(node_gid) = node_extensive_mass(node_gid)/node_coords(rk_level,node_gid,1); - } - //if(cycle==0&&node_gid==1&&myrank==0) - //std::cout << "index " << node_gid << " on rank " << myrank << " node vel " << node_vel(rk_level,node_gid,0) << " " << node_mass(node_gid) << std::endl << std::flush; + if (node_coords(rk_level, node_gid, 1) > tiny) + { + node_mass(node_gid) = node_extensive_mass(node_gid) / node_coords(rk_level, node_gid, 1); + } + // if(cycle==0&&node_gid==1&&myrank==0) + // std::cout << "index " << node_gid << " on rank " << myrank << " node vel " << node_vel(rk_level,node_gid,0) << " " << node_mass(node_gid) << std::endl << std::flush; }); // end parallel for over node_gid Kokkos::fence(); - //current interface has differing density arrays; this equates them until we unify memory - //view scope + // current interface has differing density arrays; this equates them until we unify memory + // view scope { - vec_array node_mass_interface = node_masses_distributed->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - node_mass_interface(node_gid,0) = node_mass(node_gid); + vec_array node_mass_interface = node_masses_distributed->getLocalView(Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + node_mass_interface(node_gid, 0) = node_mass(node_gid); }); // end parallel for - } //end view scope + } // end view scope Kokkos::fence(); - //communicate ghost densities + // communicate ghost densities comm_node_masses(); - //this is forcing a copy to the device - //view scope + // this is forcing a copy to the device + // view scope { - vec_array ghost_node_mass_interface = ghost_node_masses_distributed->getLocalView (Tpetra::Access::ReadWrite); + vec_array ghost_node_mass_interface = ghost_node_masses_distributed->getLocalView(Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, nlocal_nodes, nall_nodes, { - node_mass(node_gid) = ghost_node_mass_interface(node_gid-nlocal_nodes,0); + FOR_ALL_CLASS(node_gid, nlocal_nodes, nall_nodes, { + node_mass(node_gid) = ghost_node_mass_interface(node_gid - nlocal_nodes, 0); }); // end parallel for - } //end view scope + } // end view scope Kokkos::fence(); - - + // ----------------------------------------------- // Calcualte the areal mass for nodes on the axis // ----------------------------------------------- @@ -2192,318 +2254,334 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve(){ // 0---1 /* FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { - + // loop over the corners of the element and calculate the mass for (size_t node_lid=0; node_lid<4; node_lid++){ - + size_t node_gid = nodes_in_elem(elem_gid, node_lid); size_t node_minus_gid; size_t node_plus_gid; - - + + if (node_coords(rk_level,node_gid,1) < tiny){ // node is on the axis - + // minus node if (node_lid==0){ node_minus_gid = nodes_in_elem(elem_gid, 3); } else { node_minus_gid = nodes_in_elem(elem_gid, node_lid-1); } - + // plus node if (node_lid==3){ node_plus_gid = nodes_in_elem(elem_gid, 0); } else { node_plus_gid = nodes_in_elem(elem_gid, node_lid+1); } - + node_mass(node_gid) = fmax(node_mass(node_plus_gid), node_mass(node_minus_gid))/2.0; - + } // end if - + } // end for over corners - + }); // end parallel for over elem_gid Kokkos::fence(); */ - + FOR_ALL_CLASS(node_bdy_gid, 0, num_bdy_nodes, { - //FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + // FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { size_t node_gid = bdy_nodes(node_bdy_gid); - - if (node_coords(rk_level,node_gid,1) < tiny){ + + if (node_coords(rk_level, node_gid, 1) < tiny) + { // node is on the axis - - for(size_t node_lid=0; node_lid < num_nodes_in_node(node_gid); node_lid++){ - + + for (size_t node_lid = 0; node_lid < num_nodes_in_node(node_gid); node_lid++) + { size_t node_neighbor_gid = nodes_in_node(node_gid, node_lid); - + // if the node is off the axis, use it's areal mass on the boundary - if (node_coords(rk_level,node_neighbor_gid,1) > tiny){ - node_mass(node_gid) = fmax(node_mass(node_gid), node_mass(node_neighbor_gid)/2.0); + if (node_coords(rk_level, node_neighbor_gid, 1) > tiny) + { + node_mass(node_gid) = fmax(node_mass(node_gid), node_mass(node_neighbor_gid) / 2.0); } - } // end for over neighboring nodes - } // end if - }); // end parallel for over elem_gid - } // end of if 2D-RZ + } // end of RK loop + // increment the time + Explicit_Solver_Pointer_->time_value = simparam->dynamic_options.time_value = time_value += dt; + + if (simparam->topology_optimization_on || simparam->shape_optimization_on) + { + if (cycle >= max_time_steps) + { + max_time_steps = cycle + 1; + } - } // end of RK loop + if (max_time_steps + 1 > forward_solve_velocity_data->size()) + { + old_max_forward_buffer = forward_solve_velocity_data->size(); + time_data.resize(max_time_steps + BUFFER_GROW + 1); + forward_solve_velocity_data->resize(max_time_steps + BUFFER_GROW + 1); + forward_solve_coordinate_data->resize(max_time_steps + BUFFER_GROW + 1); + adjoint_vector_data->resize(max_time_steps + BUFFER_GROW + 1); + phi_adjoint_vector_data->resize(max_time_steps + BUFFER_GROW + 1); + // assign a multivector of corresponding size to each new timestep in the buffer + for (int istep = old_max_forward_buffer; istep < max_time_steps + BUFFER_GROW + 1; istep++) + { + (*forward_solve_velocity_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*forward_solve_coordinate_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*phi_adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + } + } - // increment the time - Explicit_Solver_Pointer_->time_value = simparam->dynamic_options.time_value = time_value+=dt; + time_data[cycle + 1] = dt + time_data[cycle]; - if(simparam->topology_optimization_on||simparam->shape_optimization_on){ - if(cycle >= max_time_steps) - max_time_steps = cycle + 1; + // assign current velocity data to multivector + // view scope + { + vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); + vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + node_velocities_interface(node_gid, idim) = node_vel(rk_level, node_gid, idim); + node_coords_interface(node_gid, idim) = node_coords(rk_level, node_gid, idim); + } + }); + } // end view scope + Kokkos::fence(); - if(max_time_steps + 1 > forward_solve_velocity_data->size()){ - old_max_forward_buffer = forward_solve_velocity_data->size(); - time_data.resize(max_time_steps + BUFFER_GROW +1); - forward_solve_velocity_data->resize(max_time_steps + BUFFER_GROW +1); - forward_solve_coordinate_data->resize(max_time_steps + BUFFER_GROW +1); - adjoint_vector_data->resize(max_time_steps + BUFFER_GROW +1); - phi_adjoint_vector_data->resize(max_time_steps + BUFFER_GROW +1); - //assign a multivector of corresponding size to each new timestep in the buffer - for(int istep = old_max_forward_buffer; istep < max_time_steps + BUFFER_GROW +1; istep++){ - (*forward_solve_velocity_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - (*forward_solve_coordinate_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - (*adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - (*phi_adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - } - } + // communicate ghosts + double comm_time1 = Explicit_Solver_Pointer_->CPU_Time(); - - time_data[cycle+1] = dt + time_data[cycle]; - - - //assign current velocity data to multivector - //view scope - { - vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView (Tpetra::Access::ReadWrite); - vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - node_velocities_interface(node_gid,idim) = node_vel(rk_level,node_gid,idim); - node_coords_interface(node_gid,idim) = node_coords(rk_level,node_gid,idim); + // active view scope; triggers host comms from updated data on device + { + const_host_vec_array node_velocities_host = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_host_vec_array node_coords_host = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); } - }); - } //end view scope - Kokkos::fence(); + double comm_time2 = Explicit_Solver_Pointer_->CPU_Time(); + Explicit_Solver_Pointer_->dev2host_time += comm_time2 - comm_time1; - //communicate ghosts - double comm_time1 = Explicit_Solver_Pointer_->CPU_Time(); - - //active view scope; triggers host comms from updated data on device - { - const_host_vec_array node_velocities_host = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_host_vec_array node_coords_host = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - } - double comm_time2 = Explicit_Solver_Pointer_->CPU_Time(); - Explicit_Solver_Pointer_->dev2host_time += comm_time2-comm_time1; + // communicate ghost velocities + Explicit_Solver_Pointer_->comm_velocities(); + Explicit_Solver_Pointer_->comm_coordinates(); - //communicate ghost velocities - Explicit_Solver_Pointer_->comm_velocities(); - Explicit_Solver_Pointer_->comm_coordinates(); - - - double comm_time3 = Explicit_Solver_Pointer_->CPU_Time(); + double comm_time3 = Explicit_Solver_Pointer_->CPU_Time(); - //view scope - { - const_vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array ghost_node_velocities_interface = Explicit_Solver_Pointer_->ghost_node_velocities_distributed->getLocalView (Tpetra::Access::ReadOnly); - vec_array all_node_velocities_interface = Explicit_Solver_Pointer_->all_node_velocities_distributed->getLocalView (Tpetra::Access::ReadWrite); - const_vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array ghost_node_coords_interface = Explicit_Solver_Pointer_->ghost_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - all_node_velocities_interface(node_gid,idim) = node_velocities_interface(node_gid,idim); - all_node_coords_interface(node_gid,idim) = node_coords_interface(node_gid,idim); - } + // view scope + { + const_vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array ghost_node_velocities_interface = Explicit_Solver_Pointer_->ghost_node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); + vec_array all_node_velocities_interface = Explicit_Solver_Pointer_->all_node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); + const_vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array ghost_node_coords_interface = Explicit_Solver_Pointer_->ghost_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + all_node_velocities_interface(node_gid, idim) = node_velocities_interface(node_gid, idim); + all_node_coords_interface(node_gid, idim) = node_coords_interface(node_gid, idim); + } }); // end parallel for - Kokkos::fence(); + Kokkos::fence(); - FOR_ALL_CLASS(node_gid, nlocal_nodes, nlocal_nodes+nghost_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - all_node_velocities_interface(node_gid,idim) = ghost_node_velocities_interface(node_gid-nlocal_nodes,idim); - all_node_coords_interface(node_gid,idim) = ghost_node_coords_interface(node_gid-nlocal_nodes,idim); - } + FOR_ALL_CLASS(node_gid, nlocal_nodes, nlocal_nodes + nghost_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + all_node_velocities_interface(node_gid, idim) = ghost_node_velocities_interface(node_gid - nlocal_nodes, idim); + all_node_coords_interface(node_gid, idim) = ghost_node_coords_interface(node_gid - nlocal_nodes, idim); + } }); // end parallel for - Kokkos::fence(); - } //end view scope - - double comm_time4 = Explicit_Solver_Pointer_->CPU_Time(); - Explicit_Solver_Pointer_->host2dev_time += comm_time4-comm_time3; - Explicit_Solver_Pointer_->communication_time += comm_time4-comm_time1; - - (*forward_solve_velocity_data)[cycle+1]->assign(*Explicit_Solver_Pointer_->all_node_velocities_distributed); - (*forward_solve_coordinate_data)[cycle+1]->assign(*Explicit_Solver_Pointer_->all_node_coords_distributed); - - //kinetic energy accumulation - if(kinetic_energy_objective){ - const_vec_array node_velocities_interface = (*forward_solve_velocity_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array previous_node_velocities_interface = (*forward_solve_velocity_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - KE_loc_sum = 0.0; - KE_sum = 0.0; - // extensive KE - REDUCE_SUM_CLASS(node_gid, 0, nlocal_nodes, KE_loc_sum, { - - double ke = 0; - for (size_t dim=0; dimCPU_Time(); + Explicit_Solver_Pointer_->host2dev_time += comm_time4 - comm_time3; + Explicit_Solver_Pointer_->communication_time += comm_time4 - comm_time1; + + (*forward_solve_velocity_data)[cycle + 1]->assign(*Explicit_Solver_Pointer_->all_node_velocities_distributed); + (*forward_solve_coordinate_data)[cycle + 1]->assign(*Explicit_Solver_Pointer_->all_node_coords_distributed); + + // kinetic energy accumulation + if (kinetic_energy_objective) + { + const_vec_array node_velocities_interface = (*forward_solve_velocity_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array previous_node_velocities_interface = (*forward_solve_velocity_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + KE_loc_sum = 0.0; + KE_sum = 0.0; + // extensive KE + REDUCE_SUM_CLASS(node_gid, 0, nlocal_nodes, KE_loc_sum, { + double ke = 0; + for (size_t dim = 0; dim < num_dim; dim++) + { + // midpoint integration approximation + ke += (node_velocities_interface(node_gid, dim) + previous_node_velocities_interface(node_gid, dim)) * (node_velocities_interface(node_gid, + dim) + previous_node_velocities_interface(node_gid, dim)) / 4; // 1/2 at end + } // end for + + if (num_dim == 2) + { + KE_loc_sum += node_mass(node_gid) * node_coords(rk_level, node_gid, 1) * ke; + } + else + { + KE_loc_sum += node_mass(node_gid) * ke; + } }, KE_sum); - Kokkos::fence(); - KE_sum = 0.5*KE_sum; - objective_accumulation += KE_sum*dt; + Kokkos::fence(); + KE_sum = 0.5 * KE_sum; + objective_accumulation += KE_sum * dt; + } + } + + size_t write = 0; + if ((cycle + 1) % graphics_cyc_ival == 0 && cycle > 0) + { + write = 1; } - } - - size_t write = 0; - if ((cycle+1)%graphics_cyc_ival == 0 && cycle>0){ - write = 1; - } - else if (cycle == cycle_stop) { - write = 1; - } - else if (time_value >= time_final&&simparam->output_options.write_final){ - write = 1; - } - else if (time_value >= graphics_time){ - write = 1; - } - + else if (cycle == cycle_stop) + { + write = 1; + } + else if (time_value >= time_final && simparam->output_options.write_final) + { + write = 1; + } + else if (time_value >= graphics_time) + { + write = 1; + } + // write outputs - if (write == 1){ - //interface nodal coordinate data (note: this is not needed if using write_outputs()) - //view scope + if (write == 1) + { + // interface nodal coordinate data (note: this is not needed if using write_outputs()) + // view scope { - vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - node_coords_interface(node_gid,idim) = node_coords(rk_level,node_gid,idim); - } + vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + node_coords_interface(node_gid, idim) = node_coords(rk_level, node_gid, idim); + } }); // end parallel for - } //end view scope - if(simparam->output_options.output_file_format==OUTPUT_FORMAT::vtk){ - if(myrank==0){ - printf("Writing outputs to file at %f \n", graphics_time); - } + } // end view scope + if (simparam->output_options.output_file_format == OUTPUT_FORMAT::vtk) + { + if (myrank == 0) + { + printf("Writing outputs to file at %f \n", graphics_time); + } - double comm_time1 = Explicit_Solver_Pointer_->CPU_Time(); - Explicit_Solver_Pointer_->write_outputs(); + double comm_time1 = Explicit_Solver_Pointer_->CPU_Time(); + Explicit_Solver_Pointer_->write_outputs(); - double comm_time2 = Explicit_Solver_Pointer_->CPU_Time(); - Explicit_Solver_Pointer_->output_time += comm_time2 - comm_time1; + double comm_time2 = Explicit_Solver_Pointer_->CPU_Time(); + Explicit_Solver_Pointer_->output_time += comm_time2 - comm_time1; } graphics_time = time_value + graphics_dt_ival; - } // end if - - - // end of calculation - if (time_value>=time_final) break; + } // end if - + // end of calculation + if (time_value >= time_final) + { + break; + } } // end for cycle loop last_time_step = cycle; - //simple setup to just calculate KE minimize objective for now - if(simparam->topology_optimization_on){ - KineticEnergyMinimize_TopOpt& kinetic_energy_minimize_function = dynamic_cast(*obj_pointer); + // simple setup to just calculate KE minimize objective for now + if (simparam->topology_optimization_on) + { + KineticEnergyMinimize_TopOpt& kinetic_energy_minimize_function = dynamic_cast(*obj_pointer); - //collect local objective values - MPI_Allreduce(&objective_accumulation,&global_objective_accumulation,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD); - kinetic_energy_minimize_function.objective_accumulation = global_objective_accumulation; + // collect local objective values + MPI_Allreduce(&objective_accumulation, &global_objective_accumulation, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); + kinetic_energy_minimize_function.objective_accumulation = global_objective_accumulation; - if(myrank==0) - std::cout << "CURRENT TIME INTEGRAL OF KINETIC ENERGY " << global_objective_accumulation << std::endl; + if (myrank == 0) + { + std::cout << "CURRENT TIME INTEGRAL OF KINETIC ENERGY " << global_objective_accumulation << std::endl; + } } - - + auto time_2 = std::chrono::high_resolution_clock::now(); auto time_difference = time_2 - time_1; - //double calc_time = std::chrono::duration_cast(diff).count(); + // double calc_time = std::chrono::duration_cast(diff).count(); double calc_time = std::chrono::duration_cast(time_difference).count(); - if(myrank==0) - printf("\nCalculation time in seconds: %f \n", calc_time*1e-09); - + if (myrank == 0) + { + printf("\nCalculation time in seconds: %f \n", calc_time * 1e-09); + } + IE_loc_sum = 0.0; KE_loc_sum = 0.0; - IE_sum = 0.0; - KE_sum = 0.0; - + IE_sum = 0.0; + KE_sum = 0.0; + // extensive IE REDUCE_SUM_CLASS(elem_gid, 0, nlocal_elem_non_overlapping, IE_loc_sum, { - - IE_loc_sum += elem_mass(elem_gid)*elem_sie(rk_level,elem_gid); - + IE_loc_sum += elem_mass(elem_gid) * elem_sie(rk_level, elem_gid); }, IE_sum); IE_tend = IE_sum; - //reduce over MPI ranks - MPI_Allreduce(&IE_tend,&global_IE_tend,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD); + // reduce over MPI ranks + MPI_Allreduce(&IE_tend, &global_IE_tend, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); // extensive KE REDUCE_SUM_CLASS(node_gid, 0, nlocal_nodes, KE_loc_sum, { - double ke = 0; - for (size_t dim=0; dim mesh_in, const int my_fea_module_index = 0); - ~FEA_Module_Dynamic_Elasticity(); - - //initialize data for boundaries of the model and storage for boundary conditions and applied loads - void elastic_interface_setup(node_t &node, elem_t &elem, corner_t &corner); - - void setup(); - - void cleanup_material_models(); - - int solve(); - - void module_cleanup(); - - void elastic_solve(); - - void get_force_vgradient_elastic(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &elem_mat_id, - const double rk_alpha, - const size_t cycle); - - void get_force_ugradient_elastic(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &elem_mat_id, - const double rk_alpha, - const size_t cycle); - - void get_force_dgradient_elastic(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &elem_mat_id, - const double rk_alpha, - const size_t cycle); - - void force_design_gradient_term(const_vec_array design_variables, vec_array design_gradients); - - void update_position_elastic(double rk_alpha, - const size_t num_nodes, - DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel); - - void get_vol(); - - void init_assembly(); - - KOKKOS_INLINE_FUNCTION - void get_vol_hex(const DViewCArrayKokkos &elem_vol, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const; - - - KOKKOS_INLINE_FUNCTION - void get_vol_quad(const DViewCArrayKokkos &elem_vol, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const; - - - KOKKOS_FUNCTION - double get_area_quad(const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const; - - - KOKKOS_FUNCTION - void get_bmatrix(const ViewCArrayKokkos &B_matrix, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const; - - - KOKKOS_FUNCTION - void get_bmatrix2D(const ViewCArrayKokkos &B_matrix, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const; - - KOKKOS_FUNCTION - void get_area_weights2D(const ViewCArrayKokkos &corner_areas, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const; - - - KOKKOS_INLINE_FUNCTION - double heron(const double x1, - const double y1, - const double x2, - const double y2, - const double x3, - const double y3) const; - - double average_element_density(const int nodes_per_elem, const CArray current_element_densities) const; - - void get_divergence(DViewCArrayKokkos &elem_div, - const mesh_t mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_vol); - - - void get_divergence2D(DViewCArrayKokkos &elem_div, - const mesh_t mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_vol); - - - KOKKOS_FUNCTION - void get_velgrad(ViewCArrayKokkos &vel_grad, - const ViewCArrayKokkos &elem_node_gids, - const DViewCArrayKokkos &node_vel, - const ViewCArrayKokkos &b_matrix, - const double elem_vol, - const size_t elem_gid, - const size_t rk_level) const; - - - KOKKOS_FUNCTION - void get_velgrad2D(ViewCArrayKokkos &vel_grad, - const ViewCArrayKokkos &elem_node_gids, - const DViewCArrayKokkos &node_vel, - const ViewCArrayKokkos &b_matrix, - const double elem_vol, - const double elem_area, - const size_t elem_gid, - const size_t rk_level) const; - - KOKKOS_INLINE_FUNCTION - void decompose_vel_grad(ViewCArrayKokkos &D_tensor, - ViewCArrayKokkos &W_tensor, - const ViewCArrayKokkos &vel_grad, - const ViewCArrayKokkos &elem_node_gids, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const double vol) const; - - void boundary_velocity(const mesh_t &mesh, - const DCArrayKokkos &boundary, - DViewCArrayKokkos &node_vel); - - void boundary_adjoint(const mesh_t &mesh, - const DCArrayKokkos &boundary, - vec_array &node_adjoint, - vec_array &node_phi_adjoint); - - void tag_bdys(const DCArrayKokkos &boundary, - mesh_t &mesh, - const DViewCArrayKokkos &node_coords); - - KOKKOS_INLINE_FUNCTION - bool check_bdy(const size_t patch_gid, - const int num_dim, - const int num_nodes_in_patch, - const BOUNDARY_TYPE this_bc_tag, - const double val, - const DViewCArrayKokkos &node_coords, - const size_t rk_level) const; - - void rk_init(DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &elem_sie, - DViewCArrayKokkos &elem_stress, - const size_t num_elems, - const size_t num_nodes); - - - void get_timestep(mesh_t &mesh, - DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &elem_sspd, - DViewCArrayKokkos &elem_vol); - - - void get_timestep2D(mesh_t &mesh, - DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &elem_sspd, - DViewCArrayKokkos &elem_vol); - - void update_state(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &elem_den, - DViewCArrayKokkos &elem_pres, - DViewCArrayKokkos &elem_stress, - DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_mass, - const DViewCArrayKokkos &elem_mat_id, - const double rk_alpha, - const size_t cycle); - - - void update_state2D(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &elem_den, - DViewCArrayKokkos &elem_pres, - DViewCArrayKokkos &elem_stress, - DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_mass, - const DViewCArrayKokkos &elem_mat_id, - const double rk_alpha, - const size_t cycle); - - void build_boundry_node_sets(mesh_t &mesh); - - void init_boundaries(); - - //initializes memory for arrays used in the global stiffness matrix assembly - void init_boundary_sets(int num_boundary_sets); - - void grow_boundary_sets(int num_boundary_sets); - - virtual void update_forward_solve(Teuchos::RCP zp); - - void comm_node_masses(); - - void comm_adjoint_vectors(int cycle); - - void comm_variables(Teuchos::RCP zp); - - void read_conditions_ansys_dat(std::ifstream *in, std::streampos before_condition_header); - - //interfaces between user input and creating data structures for bcs - void generate_bcs(); - - void Displacement_Boundary_Conditions(); - - void init_output(); - - void compute_output(); - - void sort_output(Teuchos::RCP > sorted_map); - - void sort_element_output(Teuchos::RCP > sorted_map); - - void collect_output(Teuchos::RCP > global_reduce_map); - - void write_data(std::map &point_data_scalars_double, - std::map &point_data_vectors_double, - std::map &cell_data_scalars_double, - std::map &cell_data_scalars_int, - std::map > &cell_data_fields_double); - - void write_outputs (const mesh_t &mesh, - DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &node_mass, - DViewCArrayKokkos &elem_den, - DViewCArrayKokkos &elem_pres, - DViewCArrayKokkos &elem_stress, - DViewCArrayKokkos &elem_sspd, - DViewCArrayKokkos &elem_sie, - DViewCArrayKokkos &elem_vol, - DViewCArrayKokkos &elem_mass, - DViewCArrayKokkos &elem_mat_id); - - - void ensight(const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &node_mass, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_mass, - const DViewCArrayKokkos &elem_mat_id); - - - void state_file(const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &node_mass, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_mass, - const DViewCArrayKokkos &elem_mat_id); - - void node_density_constraints(host_vec_array &node_densities_lower_bound); - - void compute_topology_optimization_adjoint(); //Force does not depend on node coords and velocity - - void compute_topology_optimization_adjoint_full(); //Force depends on node coords and velocity - - void compute_topology_optimization_gradient(const_vec_array design_densities, vec_array gradients); - - void compute_topology_optimization_gradient_full(Teuchos::RCP design_densities_distributed, Teuchos::RCP design_gradients_distributed); - - //elastic TO stuff - void get_force_elastic(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &node_mass, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &elem_mat_id, - DViewCArrayKokkos &corner_force, - const double rk_alpha, - const size_t cycle); - - void applied_forces(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &node_mass, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &elem_mat_id, - DViewCArrayKokkos &corner_force, - const double rk_alpha, - const size_t cycle); - - void Element_Material_Properties(size_t ielem, real_t &Element_Modulus, real_t &Poisson_Ratio, real_t density); - - void compute_stiffness_gradients(const_host_vec_array &design_densities, host_vec_array &gradients); - - void Gradient_Element_Material_Properties(size_t ielem, real_t &Element_Modulus, real_t &Poisson_Ratio, real_t density); - - void local_matrix_multiply(int ielem, CArrayKokkos &Local_Matrix); - - void assemble_matrix(); - - bool nodal_density_flag; - real_t penalty_power; - Teuchos::RCP Global_Stiffness_Matrix; - RaggedRightArrayKokkos Stiffness_Matrix; - DCArrayKokkos Stiffness_Matrix_Strides; - DCArrayKokkos Global_Stiffness_Matrix_Assembly_Map; - //end elastic TO data - - Dynamic_Elasticity_Parameters *module_params; - Simulation_Parameters_Explicit *simparam; - Explicit_Solver *Explicit_Solver_Pointer_; - - elements::ref_element *ref_elem; - - std::shared_ptr mesh; - //shallow copies of mesh class views - size_t num_nodes_in_elem; - // corner ids in node - RaggedRightArrayKokkos corners_in_node; - CArrayKokkos num_corners_in_node; - - // elem ids in node - RaggedRightArrayKokkos elems_in_node; - - // node ids in node - RaggedRightArrayKokkos nodes_in_node; - CArrayKokkos num_nodes_in_node; - - // node ids in elem - DCArrayKokkos nodes_in_elem; - - // corner ids in elem - CArrayKokkos corners_in_elem; - - // elem ids in elem - RaggedRightArrayKokkos elems_in_elem; - CArrayKokkos num_elems_in_elem; - - // patch ids in elem - CArrayKokkos patches_in_elem; - - // node ids in a patch - CArrayKokkos nodes_in_patch; - - // element ids in a patch - CArrayKokkos elems_in_patch; - - // bdy nodes - CArrayKokkos bdy_nodes; - - //Topology optimization filter variable - DCArrayKokkos relative_element_densities; - - //Local FEA data - host_elem_conn_array interface_nodes_in_elem; //host view of element connectivity to nodes - - //Global FEA data - Teuchos::RCP node_velocities_distributed; - Teuchos::RCP initial_node_coords_distributed; - Teuchos::RCP all_initial_node_coords_distributed; - Teuchos::RCP initial_node_velocities_distributed; - Teuchos::RCP all_node_velocities_distributed; - Teuchos::RCP all_cached_node_velocities_distributed; - Teuchos::RCP node_masses_distributed; - Teuchos::RCP ghost_node_masses_distributed; - Teuchos::RCP adjoint_vector_distributed; - Teuchos::RCP phi_adjoint_vector_distributed; - Teuchos::RCP>> forward_solve_velocity_data; - Teuchos::RCP>> forward_solve_coordinate_data; - Teuchos::RCP>> adjoint_vector_data; - Teuchos::RCP>> phi_adjoint_vector_data; - Teuchos::RCP force_gradient_design; - Teuchos::RCP force_gradient_position; - Teuchos::RCP force_gradient_velocity; - - //Local FEA data - DCArrayKokkos Global_Gradient_Matrix_Assembly_Map; - RaggedRightArrayKokkos Graph_Matrix; //stores global indices - RaggedRightArrayKokkos DOF_Graph_Matrix; //stores global indices - RaggedRightArrayKokkos Force_Gradient_Positions; - RaggedRightArrayKokkos Force_Gradient_Velocities; - DCArrayKokkos Gradient_Matrix_Strides; - DCArrayKokkos Graph_Matrix_Strides; - RaggedRightArrayKokkos Original_Gradient_Entries; - RaggedRightArrayKokkos Original_Gradient_Entry_Indices; - DCArrayKokkos Original_Gradient_Entries_Strides; - - //distributed matrices - Teuchos::RCP distributed_force_gradient_positions; - Teuchos::RCP distributed_force_gradient_velocities; - - std::vector time_data; - unsigned long max_time_steps, last_time_step; - - // --------------------------------------------------------------------- - // state data type declarations (must stay in scope for output after run) - // --------------------------------------------------------------------- - node_t node_interface; - elem_t elem_interface; - corner_t corner_interface; - - //Dual View wrappers - // Dual Views of the individual node struct variables - DViewCArrayKokkos node_coords; - DViewCArrayKokkos node_vel; - DViewCArrayKokkos node_mass; - - // Dual Views of the individual elem struct variables - DViewCArrayKokkos elem_den; - DViewCArrayKokkos elem_pres; - DViewCArrayKokkos elem_stress; // always 3D even in 2D-RZ - DViewCArrayKokkos elem_sspd; - DViewCArrayKokkos elem_sie; - DViewCArrayKokkos elem_vol; - DViewCArrayKokkos elem_div; - DViewCArrayKokkos elem_mass; - DViewCArrayKokkos elem_mat_id; - - // Element velocity gradient - DCArrayKokkos elem_vel_grad; - - // for storing global variables used in user material model - DCArrayKokkos global_vars; - - // for storing state variables used in user material model - DCArrayKokkos state_vars; - - //elem_user_output_vars allow users to output variables of interest per element - DCArrayKokkos elem_user_output_vars; - - //material models - DCArrayKokkos elem_eos; - DCArrayKokkos elem_strength; - - // Dual Views of the corner struct variables - DViewCArrayKokkos corner_force; - DViewCArrayKokkos corner_mass; - - //Boundary Conditions Data - DCArrayKokkos Local_Index_Boundary_Patches; - //CArray Patch_Nodes; - enum bc_type {NONE, POINT_LOADING_CONDITION, LINE_LOADING_CONDITION, SURFACE_LOADING_CONDITION}; - - //Boundary Conditions Data - int max_boundary_sets; - - //output dof data - //Global arrays with collected data used to print - int output_velocity_index, output_strain_index, output_stress_index; - - //parameters - double time_value, time_final, dt, dt_max, dt_min, dt_cfl, graphics_time, graphics_dt_ival; - size_t graphics_cyc_ival, cycle_stop, rk_num_stages, graphics_id; - double fuzz, tiny, small; - CArray graphics_times; - int rk_num_bins; - - //optimization flags - bool kinetic_energy_objective; + + FEA_Module_Dynamic_Elasticity(Dynamic_Elasticity_Parameters& params, Solver* Solver_Pointer, std::shared_ptr mesh_in, const int my_fea_module_index = 0); + ~FEA_Module_Dynamic_Elasticity(); + + // initialize data for boundaries of the model and storage for boundary conditions and applied loads + void elastic_interface_setup(node_t& node, elem_t& elem, corner_t& corner); + + void setup(); + + void cleanup_material_models(); + + int solve(); + + void module_cleanup(); + + void elastic_solve(); + + void get_force_vgradient_elastic(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_pres, + const DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + const double rk_alpha, + const size_t cycle); + + void get_force_ugradient_elastic(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_pres, + const DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + const double rk_alpha, + const size_t cycle); + + void get_force_dgradient_elastic(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_pres, + const DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + const double rk_alpha, + const size_t cycle); + + void force_design_gradient_term(const_vec_array design_variables, vec_array design_gradients); + + void update_position_elastic(double rk_alpha, + const size_t num_nodes, + DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel); + + void get_vol(); + + void init_assembly(); + + KOKKOS_INLINE_FUNCTION + void get_vol_hex(const DViewCArrayKokkos& elem_vol, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const; + + KOKKOS_INLINE_FUNCTION + void get_vol_quad(const DViewCArrayKokkos& elem_vol, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const; + + KOKKOS_FUNCTION + double get_area_quad(const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const; + + KOKKOS_FUNCTION + void get_bmatrix(const ViewCArrayKokkos& B_matrix, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const; + + KOKKOS_FUNCTION + void get_bmatrix2D(const ViewCArrayKokkos& B_matrix, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const; + + KOKKOS_FUNCTION + void get_area_weights2D(const ViewCArrayKokkos& corner_areas, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const; + + KOKKOS_INLINE_FUNCTION + double heron(const double x1, + const double y1, + const double x2, + const double y2, + const double x3, + const double y3) const; + + double average_element_density(const int nodes_per_elem, const CArray current_element_densities) const; + + void get_divergence(DViewCArrayKokkos& elem_div, + const mesh_t mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_vol); + + void get_divergence2D(DViewCArrayKokkos& elem_div, + const mesh_t mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_vol); + + KOKKOS_FUNCTION + void get_velgrad(ViewCArrayKokkos& vel_grad, + const ViewCArrayKokkos& elem_node_gids, + const DViewCArrayKokkos& node_vel, + const ViewCArrayKokkos& b_matrix, + const double elem_vol, + const size_t elem_gid, + const size_t rk_level) const; + + KOKKOS_FUNCTION + void get_velgrad2D(ViewCArrayKokkos& vel_grad, + const ViewCArrayKokkos& elem_node_gids, + const DViewCArrayKokkos& node_vel, + const ViewCArrayKokkos& b_matrix, + const double elem_vol, + const double elem_area, + const size_t elem_gid, + const size_t rk_level) const; + + KOKKOS_INLINE_FUNCTION + void decompose_vel_grad(ViewCArrayKokkos& D_tensor, + ViewCArrayKokkos& W_tensor, + const ViewCArrayKokkos& vel_grad, + const ViewCArrayKokkos& elem_node_gids, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const double vol) const; + + void boundary_velocity(const mesh_t& mesh, + const DCArrayKokkos& boundary, + DViewCArrayKokkos& node_vel); + + void boundary_adjoint(const mesh_t& mesh, + const DCArrayKokkos& boundary, + vec_array& node_adjoint, + vec_array& node_phi_adjoint); + + void tag_bdys(const DCArrayKokkos& boundary, + mesh_t& mesh, + const DViewCArrayKokkos& node_coords); + + KOKKOS_INLINE_FUNCTION + bool check_bdy(const size_t patch_gid, + const int num_dim, + const int num_nodes_in_patch, + const BOUNDARY_TYPE this_bc_tag, + const double val, + const DViewCArrayKokkos& node_coords, + const size_t rk_level) const; + + void rk_init(DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_sie, + DViewCArrayKokkos& elem_stress, + const size_t num_elems, + const size_t num_nodes); + + void get_timestep(mesh_t& mesh, + DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_sspd, + DViewCArrayKokkos& elem_vol); + + void get_timestep2D(mesh_t& mesh, + DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_sspd, + DViewCArrayKokkos& elem_vol); + + void update_state(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_den, + DViewCArrayKokkos& elem_pres, + DViewCArrayKokkos& elem_stress, + DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& elem_mat_id, + const double rk_alpha, + const size_t cycle); + + void update_state2D(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_den, + DViewCArrayKokkos& elem_pres, + DViewCArrayKokkos& elem_stress, + DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& elem_mat_id, + const double rk_alpha, + const size_t cycle); + + void build_boundry_node_sets(mesh_t& mesh); + + void init_boundaries(); + + // initializes memory for arrays used in the global stiffness matrix assembly + void init_boundary_sets(int num_boundary_sets); + + void grow_boundary_sets(int num_boundary_sets); + + virtual void update_forward_solve(Teuchos::RCP zp); + + void comm_node_masses(); + + void comm_adjoint_vectors(int cycle); + + void comm_variables(Teuchos::RCP zp); + + void read_conditions_ansys_dat(std::ifstream* in, std::streampos before_condition_header); + + // interfaces between user input and creating data structures for bcs + void generate_bcs(); + + void Displacement_Boundary_Conditions(); + + void init_output(); + + void compute_output(); + + void sort_output(Teuchos::RCP> sorted_map); + + void sort_element_output(Teuchos::RCP> sorted_map); + + void collect_output(Teuchos::RCP> global_reduce_map); + + void write_data(std::map& point_data_scalars_double, + std::map& point_data_vectors_double, + std::map& cell_data_scalars_double, + std::map& cell_data_scalars_int, + std::map>& cell_data_fields_double); + + void write_outputs(const mesh_t& mesh, + DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& node_mass, + DViewCArrayKokkos& elem_den, + DViewCArrayKokkos& elem_pres, + DViewCArrayKokkos& elem_stress, + DViewCArrayKokkos& elem_sspd, + DViewCArrayKokkos& elem_sie, + DViewCArrayKokkos& elem_vol, + DViewCArrayKokkos& elem_mass, + DViewCArrayKokkos& elem_mat_id); + + void ensight(const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_mass, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_pres, + const DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& elem_mat_id); + + void state_file(const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_mass, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_pres, + const DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& elem_mat_id); + + void node_density_constraints(host_vec_array& node_densities_lower_bound); + + void compute_topology_optimization_adjoint(); // Force does not depend on node coords and velocity + + void compute_topology_optimization_adjoint_full(); // Force depends on node coords and velocity + + void compute_topology_optimization_gradient(const_vec_array design_densities, vec_array gradients); + + void compute_topology_optimization_gradient_full(Teuchos::RCP design_densities_distributed, Teuchos::RCP design_gradients_distributed); + + // elastic TO stuff + void get_force_elastic(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_mass, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + DViewCArrayKokkos& corner_force, + const double rk_alpha, + const size_t cycle); + + void applied_forces(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_mass, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + DViewCArrayKokkos& corner_force, + const double rk_alpha, + const size_t cycle); + + void Element_Material_Properties(size_t ielem, real_t& Element_Modulus, real_t& Poisson_Ratio, real_t density); + + void compute_stiffness_gradients(const_host_vec_array& design_densities, host_vec_array& gradients); + + void Gradient_Element_Material_Properties(size_t ielem, real_t& Element_Modulus, real_t& Poisson_Ratio, real_t density); + + void local_matrix_multiply(int ielem, CArrayKokkos& Local_Matrix); + + void assemble_matrix(); + + bool nodal_density_flag; + real_t penalty_power; + Teuchos::RCP Global_Stiffness_Matrix; + RaggedRightArrayKokkos Stiffness_Matrix; + DCArrayKokkos Stiffness_Matrix_Strides; + DCArrayKokkos Global_Stiffness_Matrix_Assembly_Map; + // end elastic TO data + + Dynamic_Elasticity_Parameters* module_params; + Simulation_Parameters_Explicit* simparam; + Explicit_Solver* Explicit_Solver_Pointer_; + + elements::ref_element* ref_elem; + + std::shared_ptr mesh; + // shallow copies of mesh class views + size_t num_nodes_in_elem; + // corner ids in node + RaggedRightArrayKokkos corners_in_node; + CArrayKokkos num_corners_in_node; + + // elem ids in node + RaggedRightArrayKokkos elems_in_node; + + // node ids in node + RaggedRightArrayKokkos nodes_in_node; + CArrayKokkos num_nodes_in_node; + + // node ids in elem + DCArrayKokkos nodes_in_elem; + + // corner ids in elem + CArrayKokkos corners_in_elem; + + // elem ids in elem + RaggedRightArrayKokkos elems_in_elem; + CArrayKokkos num_elems_in_elem; + + // patch ids in elem + CArrayKokkos patches_in_elem; + + // node ids in a patch + CArrayKokkos nodes_in_patch; + + // element ids in a patch + CArrayKokkos elems_in_patch; + + // bdy nodes + CArrayKokkos bdy_nodes; + + // Topology optimization filter variable + DCArrayKokkos relative_element_densities; + + // Local FEA data + host_elem_conn_array interface_nodes_in_elem; // host view of element connectivity to nodes + + // Global FEA data + Teuchos::RCP node_velocities_distributed; + Teuchos::RCP initial_node_coords_distributed; + Teuchos::RCP all_initial_node_coords_distributed; + Teuchos::RCP initial_node_velocities_distributed; + Teuchos::RCP all_node_velocities_distributed; + Teuchos::RCP all_cached_node_velocities_distributed; + Teuchos::RCP node_masses_distributed; + Teuchos::RCP ghost_node_masses_distributed; + Teuchos::RCP adjoint_vector_distributed; + Teuchos::RCP phi_adjoint_vector_distributed; + Teuchos::RCP>> forward_solve_velocity_data; + Teuchos::RCP>> forward_solve_coordinate_data; + Teuchos::RCP>> adjoint_vector_data; + Teuchos::RCP>> phi_adjoint_vector_data; + Teuchos::RCP force_gradient_design; + Teuchos::RCP force_gradient_position; + Teuchos::RCP force_gradient_velocity; + + // Local FEA data + DCArrayKokkos Global_Gradient_Matrix_Assembly_Map; + RaggedRightArrayKokkos Graph_Matrix; // stores global indices + RaggedRightArrayKokkos DOF_Graph_Matrix; // stores global indices + RaggedRightArrayKokkos Force_Gradient_Positions; + RaggedRightArrayKokkos Force_Gradient_Velocities; + DCArrayKokkos Gradient_Matrix_Strides; + DCArrayKokkos Graph_Matrix_Strides; + RaggedRightArrayKokkos Original_Gradient_Entries; + RaggedRightArrayKokkos Original_Gradient_Entry_Indices; + DCArrayKokkos Original_Gradient_Entries_Strides; + + // distributed matrices + Teuchos::RCP distributed_force_gradient_positions; + Teuchos::RCP distributed_force_gradient_velocities; + + std::vector time_data; + unsigned long max_time_steps, last_time_step; + + // --------------------------------------------------------------------- + // state data type declarations (must stay in scope for output after run) + // --------------------------------------------------------------------- + node_t node_interface; + elem_t elem_interface; + corner_t corner_interface; + + // Dual View wrappers + // Dual Views of the individual node struct variables + DViewCArrayKokkos node_coords; + DViewCArrayKokkos node_vel; + DViewCArrayKokkos node_mass; + + // Dual Views of the individual elem struct variables + DViewCArrayKokkos elem_den; + DViewCArrayKokkos elem_pres; + DViewCArrayKokkos elem_stress; // always 3D even in 2D-RZ + DViewCArrayKokkos elem_sspd; + DViewCArrayKokkos elem_sie; + DViewCArrayKokkos elem_vol; + DViewCArrayKokkos elem_div; + DViewCArrayKokkos elem_mass; + DViewCArrayKokkos elem_mat_id; + + // Element velocity gradient + DCArrayKokkos elem_vel_grad; + + // for storing global variables used in user material model + DCArrayKokkos global_vars; + + // for storing state variables used in user material model + DCArrayKokkos state_vars; + + // elem_user_output_vars allow users to output variables of interest per element + DCArrayKokkos elem_user_output_vars; + + // material models + DCArrayKokkos elem_eos; + DCArrayKokkos elem_strength; + + // Dual Views of the corner struct variables + DViewCArrayKokkos corner_force; + DViewCArrayKokkos corner_mass; + + // Boundary Conditions Data + DCArrayKokkos Local_Index_Boundary_Patches; + // CArray Patch_Nodes; + enum bc_type { NONE, POINT_LOADING_CONDITION, LINE_LOADING_CONDITION, SURFACE_LOADING_CONDITION }; + + // Boundary Conditions Data + int max_boundary_sets; + + // output dof data + // Global arrays with collected data used to print + int output_velocity_index, output_strain_index, output_stress_index; + + // parameters + double time_value, time_final, dt, dt_max, dt_min, dt_cfl, graphics_time, graphics_dt_ival; + size_t graphics_cyc_ival, cycle_stop, rk_num_stages, graphics_id; + double fuzz, tiny, small; + CArray graphics_times; + int rk_num_bins; + + // optimization flags + bool kinetic_energy_objective; }; #endif // end HEADER_H diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/boundary.cpp b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/boundary.cpp index 06a017b8a..9f3a5c7a0 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/boundary.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/boundary.cpp @@ -1,76 +1,76 @@ // ------------------------------------------------------- // This function applys the boundary condition // to points on a list of patches created at setup -//-------------------------------------------------------- +// -------------------------------------------------------- #include "mesh.h" #include "state.h" #include "FEA_Module_Dynamic_Elasticity.h" #include "Simulation_Parameters/FEA_Module/Boundary_Conditions.h" -void FEA_Module_Dynamic_Elasticity::boundary_velocity(const mesh_t &mesh, - const DCArrayKokkos &boundary, - DViewCArrayKokkos &node_vel){ +void FEA_Module_Dynamic_Elasticity::boundary_velocity(const mesh_t& mesh, + const DCArrayKokkos& boundary, + DViewCArrayKokkos& node_vel) +{ + // error and debug flag + // DCArrayKokkos print_flag(1, "print_flag"); + // print_flag.host(0) = false; + // print_flag.update_device(); - //error and debug flag - //DCArrayKokkos print_flag(1, "print_flag"); - //print_flag.host(0) = false; - //print_flag.update_device(); - - const size_t rk_level = rk_num_bins - 1; - int num_dims = num_dim; + const size_t rk_level = rk_num_bins - 1; + int num_dims = num_dim; // Loop over boundary sets - for (size_t bdy_set=0; bdy_set #include #include -#include +#include #include // fmin, fmax, abs note: fminl is long #include #include @@ -34,7 +34,7 @@ #include "FEA_Module_Dynamic_Elasticity.h" #include "Explicit_Solver.h" -//optimization +// optimization #include "ROL_Solver.hpp" #include "Kinetic_Energy_Minimize.h" @@ -42,213 +42,228 @@ Compute new system response due to the design variable update ------------------------------------------------------------------------- */ -void FEA_Module_Dynamic_Elasticity::update_forward_solve(Teuchos::RCP zp){ - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - //local variable for host view in the dual view - int num_dim = simparam->num_dims; - int nodes_per_elem = max_nodes_per_element; - int local_node_index, current_row, current_column; - int max_stride = 0; - int current_module_index; - size_t access_index, row_access_index, row_counter; - GO global_index, global_dof_index; - LO local_dof_index; - const size_t num_fills = simparam->regions.size(); - const size_t rk_num_bins = simparam->dynamic_options.rk_num_bins; - const size_t num_bcs = module_params->boundary_conditions.size(); - const size_t num_materials = simparam->materials.size(); - real_t objective_accumulation; - - // --- Read in the nodes in the mesh --- - int myrank = Explicit_Solver_Pointer_->myrank; - int nranks = Explicit_Solver_Pointer_->nranks; - - const DCArrayKokkos mat_fill = simparam->mat_fill; - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos material = simparam->material; - CArray current_element_nodal_densities = CArray(num_nodes_in_elem); - - std::vector> FEA_Module_My_TO_Modules = simparam->FEA_Module_My_TO_Modules; - problem = Explicit_Solver_Pointer_->problem; //Pointer to ROL optimization problem object - ROL::Ptr> obj_pointer; - - //compute element averaged density ratios corresponding to nodal density design variables - {//view scope - const_host_vec_array all_node_densities = all_node_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); - //debug print - //std::cout << "NODE DENSITY TEST " << all_node_densities(0,0) << std::endl; - for(int elem_id = 0; elem_id < rnum_elem; elem_id++){ - for(int inode = 0; inode < num_nodes_in_elem; inode++){ - current_element_nodal_densities(inode) = all_node_densities(nodes_in_elem(elem_id,inode),0); - } - relative_element_densities.host(elem_id) = average_element_density(num_nodes_in_elem, current_element_nodal_densities); - }//for - } //view scope - //debug print - //std::cout << "ELEMENT RELATIVE DENSITY TEST " << relative_element_densities.host(0) << std::endl; - relative_element_densities.update_device(); - - //set density vector to the current value chosen by the optimizer - test_node_densities_distributed = zp; - - //reset nodal coordinates to initial values - node_coords_distributed->assign(*initial_node_coords_distributed); - - //comms for ghosts - Explicit_Solver_Pointer_->comm_coordinates(); - - //view scope - { - const_vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array ghost_node_coords_interface = Explicit_Solver_Pointer_->ghost_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - all_node_coords_interface(node_gid,idim) = node_coords_interface(node_gid,idim); - } +void FEA_Module_Dynamic_Elasticity::update_forward_solve(Teuchos::RCP zp) +{ + // local variable for host view in the dual view + int num_dim = simparam->num_dims; + int nodes_per_elem = max_nodes_per_element; + int local_node_index, current_row, current_column; + int max_stride = 0; + int current_module_index; + size_t access_index, row_access_index, row_counter; + GO global_index, global_dof_index; + LO local_dof_index; + + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + const size_t num_fills = simparam->regions.size(); + const size_t rk_num_bins = simparam->dynamic_options.rk_num_bins; + const size_t num_bcs = module_params->boundary_conditions.size(); + const size_t num_materials = simparam->materials.size(); + real_t objective_accumulation; + + // --- Read in the nodes in the mesh --- + int myrank = Explicit_Solver_Pointer_->myrank; + int nranks = Explicit_Solver_Pointer_->nranks; + + const DCArrayKokkos mat_fill = simparam->mat_fill; + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos material = simparam->material; + CArray current_element_nodal_densities = CArray(num_nodes_in_elem); + + std::vector> FEA_Module_My_TO_Modules = simparam->FEA_Module_My_TO_Modules; + problem = Explicit_Solver_Pointer_->problem; // Pointer to ROL optimization problem object + ROL::Ptr> obj_pointer; + + // compute element averaged density ratios corresponding to nodal density design variables + { // view scope + const_host_vec_array all_node_densities = all_node_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); + // debug print + // std::cout << "NODE DENSITY TEST " << all_node_densities(0,0) << std::endl; + for (int elem_id = 0; elem_id < rnum_elem; elem_id++) + { + for (int inode = 0; inode < num_nodes_in_elem; inode++) + { + current_element_nodal_densities(inode) = all_node_densities(nodes_in_elem(elem_id, inode), 0); + } + relative_element_densities.host(elem_id) = average_element_density(num_nodes_in_elem, current_element_nodal_densities); + } // for + } // view scope + // debug print + // std::cout << "ELEMENT RELATIVE DENSITY TEST " << relative_element_densities.host(0) << std::endl; + relative_element_densities.update_device(); + + // set density vector to the current value chosen by the optimizer + test_node_densities_distributed = zp; + + // reset nodal coordinates to initial values + node_coords_distributed->assign(*initial_node_coords_distributed); + + // comms for ghosts + Explicit_Solver_Pointer_->comm_coordinates(); + + // view scope + { + const_vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array ghost_node_coords_interface = Explicit_Solver_Pointer_->ghost_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + all_node_coords_interface(node_gid, idim) = node_coords_interface(node_gid, idim); + } }); // end parallel for - Kokkos::fence(); + Kokkos::fence(); - FOR_ALL_CLASS(node_gid, nlocal_nodes, nlocal_nodes+nghost_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - all_node_coords_interface(node_gid,idim) = ghost_node_coords_interface(node_gid-nlocal_nodes,idim); - } + FOR_ALL_CLASS(node_gid, nlocal_nodes, nlocal_nodes + nghost_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + all_node_coords_interface(node_gid, idim) = ghost_node_coords_interface(node_gid - nlocal_nodes, idim); + } }); // end parallel for - Kokkos::fence(); - } //end view scope - - //reset velocities to initial conditions - node_velocities_distributed->assign(*initial_node_velocities_distributed); - - //reset time accumulating objective and constraints - /* - for(int imodule = 0 ; imodule < FEA_Module_My_TO_Modules[my_fea_module_index_].size(); imodule++){ - current_module_index = FEA_Module_My_TO_Modules[my_fea_module_index_][imodule]; - //test if module needs reset - if(){ - - } - } - */ - //simple setup to just request KE for now; above loop to be expanded and used later for scanning modules - obj_pointer = problem->getObjective(); - KineticEnergyMinimize_TopOpt& kinetic_energy_minimize_function = dynamic_cast(*obj_pointer); - kinetic_energy_minimize_function.objective_accumulation = 0; - - //interface trial density vector - - //interfacing of vectors(should be removed later once made compatible) - //view scope - { - host_vec_array interface_node_coords = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); - for(size_t ibin = 0; ibin < rk_num_bins; ibin++){ - //save node data to node.coords - //std::cout << "NODE DATA ON RANK " << myrank << std::endl; - if(num_dim==2){ - for(int inode = 0; inode < nall_nodes; inode++){ - //std::cout << "Node index " << inode+1 << " "; - node_coords.host(ibin,inode,0) = interface_node_coords(inode,0); - //std::cout << host_node_coords_state(0,inode,0)+1<< " "; - node_coords.host(ibin,inode,1) = interface_node_coords(inode,1); - //std::cout << host_node_coords_state(0,inode,1)+1<< " "; - } - } - else if(num_dim==3){ - for(int inode = 0; inode < nall_nodes; inode++){ - //std::cout << "Node index " << inode+1 << " "; - node_coords.host(ibin,inode,0) = interface_node_coords(inode,0); - //std::cout << host_node_coords_state(0,inode,0)+1<< " "; - node_coords.host(ibin,inode,1) = interface_node_coords(inode,1); - //std::cout << host_node_coords_state(0,inode,1)+1<< " "; - - node_coords.host(ibin,inode,2) = interface_node_coords(inode,2); - //std::cout << host_node_coords_state(0,inode,2)+1<< std::endl; - } + Kokkos::fence(); + } // end view scope + + // reset velocities to initial conditions + node_velocities_distributed->assign(*initial_node_velocities_distributed); + + // reset time accumulating objective and constraints + /* + for(int imodule = 0 ; imodule < FEA_Module_My_TO_Modules[my_fea_module_index_].size(); imodule++){ + current_module_index = FEA_Module_My_TO_Modules[my_fea_module_index_][imodule]; + //test if module needs reset + if(){ + } } - } //end view scope + */ + // simple setup to just request KE for now; above loop to be expanded and used later for scanning modules + obj_pointer = problem->getObjective(); + KineticEnergyMinimize_TopOpt& kinetic_energy_minimize_function = dynamic_cast(*obj_pointer); + kinetic_energy_minimize_function.objective_accumulation = 0; + + // interface trial density vector + + // interfacing of vectors(should be removed later once made compatible) + // view scope + { + host_vec_array interface_node_coords = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + for (size_t ibin = 0; ibin < rk_num_bins; ibin++) + { + // save node data to node.coords + // std::cout << "NODE DATA ON RANK " << myrank << std::endl; + if (num_dim == 2) + { + for (int inode = 0; inode < nall_nodes; inode++) + { + // std::cout << "Node index " << inode+1 << " "; + node_coords.host(ibin, inode, 0) = interface_node_coords(inode, 0); + // std::cout << host_node_coords_state(0,inode,0)+1<< " "; + node_coords.host(ibin, inode, 1) = interface_node_coords(inode, 1); + // std::cout << host_node_coords_state(0,inode,1)+1<< " "; + } + } + else if (num_dim == 3) + { + for (int inode = 0; inode < nall_nodes; inode++) + { + // std::cout << "Node index " << inode+1 << " "; + node_coords.host(ibin, inode, 0) = interface_node_coords(inode, 0); + // std::cout << host_node_coords_state(0,inode,0)+1<< " "; + node_coords.host(ibin, inode, 1) = interface_node_coords(inode, 1); + // std::cout << host_node_coords_state(0,inode,1)+1<< " "; + + node_coords.host(ibin, inode, 2) = interface_node_coords(inode, 2); + // std::cout << host_node_coords_state(0,inode,2)+1<< std::endl; + } + } + } + } // end view scope // save the node coords to the current RK value - for (size_t node_gid=0; node_gid < nall_nodes; node_gid++){ - - for(int rk=1; rkcalc_pressure(elem_pres, + elem_coords[0] += node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), 0); + elem_coords[1] += node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), 1); + if (num_dim == 3) + { + elem_coords[2] += node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), 2); + } + else + { + elem_coords[2] = 0.0; + } + } // end loop over nodes in element + elem_coords[0] = elem_coords[0] / num_nodes_in_elem; + elem_coords[1] = elem_coords[1] / num_nodes_in_elem; + elem_coords[2] = elem_coords[2] / num_nodes_in_elem; + + // default is not to fill the element + bool fill_this = mat_fill(f_id).volume.contains(elem_coords); + + // paint the material state on the element + if (fill_this) + { + // density + elem_den(elem_gid) = mat_fill(f_id).den; + + // compute element average density from initial nodal density variables used as TO design variables + elem_den(elem_gid) = elem_den(elem_gid) * relative_element_densities(elem_gid); + + // mass + elem_mass(elem_gid) = elem_den(elem_gid) * elem_vol(elem_gid); + + // specific internal energy + elem_sie(rk_level, elem_gid) = mat_fill(f_id).sie; + + elem_mat_id(elem_gid) = mat_fill(f_id).material_id; + size_t mat_id = elem_mat_id(elem_gid); // short name + + // --- stress tensor --- + // always 3D even for 2D-RZ + for (size_t i = 0; i < 3; i++) + { + for (size_t j = 0; j < 3; j++) + { + elem_stress(rk_level, elem_gid, i, j) = 0.0; + } + } // end for + + // short form for clean code + EOSParent* eos_model = elem_eos(elem_gid).model; + + // --- Pressure --- + eos_model->calc_pressure(elem_pres, elem_stress, elem_gid, elem_mat_id(elem_gid), @@ -257,10 +272,10 @@ void FEA_Module_Dynamic_Elasticity::update_forward_solve(Teuchos::RCP elem_user_output_vars, elem_sspd, elem_den(elem_gid), - elem_sie(rk_level,elem_gid)); + elem_sie(rk_level, elem_gid)); - // --- Sound speed --- - eos_model->calc_sound_speed(elem_pres, + // --- Sound speed --- + eos_model->calc_sound_speed(elem_pres, elem_stress, elem_gid, elem_mat_id(elem_gid), @@ -269,229 +284,224 @@ void FEA_Module_Dynamic_Elasticity::update_forward_solve(Teuchos::RCP elem_user_output_vars, elem_sspd, elem_den(elem_gid), - elem_sie(rk_level,elem_gid)); - - // loop over the nodes of this element and apply velocity - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - - // get the mesh node index - size_t node_gid = nodes_in_elem(elem_gid, node_lid); + elem_sie(rk_level, elem_gid)); - - // --- Velocity --- - switch(mat_fill(f_id).velocity) + // loop over the nodes of this element and apply velocity + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { - case VELOCITY_TYPE::cartesian: - { - - node_vel(rk_level, node_gid, 0) = mat_fill(f_id).u; - node_vel(rk_level, node_gid, 1) = mat_fill(f_id).v; - if (num_dim == 3) node_vel(rk_level, node_gid, 2) = mat_fill(f_id).w; - - - break; - } - case VELOCITY_TYPE::radial: + // get the mesh node index + size_t node_gid = nodes_in_elem(elem_gid, node_lid); + + // --- Velocity --- + switch (mat_fill(f_id).velocity) { - // Setting up cylindrical - double dir[2]; - dir[0] = 0.0; - dir[1] = 0.0; - double radius_val = 0.0; - - for(int dim=0; dim<2; dim++){ - dir[dim] = node_coords(rk_level, node_gid, dim); - radius_val += node_coords(rk_level, node_gid, dim)*node_coords(rk_level, node_gid, dim); - } // end for - radius_val = sqrt(radius_val); - - for(int dim=0; dim<2; dim++){ - if (radius_val > 1.0e-14){ - dir[dim] /= (radius_val); + case VELOCITY_TYPE::cartesian: + { + node_vel(rk_level, node_gid, 0) = mat_fill(f_id).u; + node_vel(rk_level, node_gid, 1) = mat_fill(f_id).v; + if (num_dim == 3) + { + node_vel(rk_level, node_gid, 2) = mat_fill(f_id).w; + } + + break; } - else{ - dir[dim] = 0.0; + case VELOCITY_TYPE::radial: + { + // Setting up cylindrical + double dir[2]; + dir[0] = 0.0; + dir[1] = 0.0; + double radius_val = 0.0; + + for (int dim = 0; dim < 2; dim++) + { + dir[dim] = node_coords(rk_level, node_gid, dim); + radius_val += node_coords(rk_level, node_gid, dim) * node_coords(rk_level, node_gid, dim); + } // end for + radius_val = sqrt(radius_val); + + for (int dim = 0; dim < 2; dim++) + { + if (radius_val > 1.0e-14) + { + dir[dim] /= (radius_val); + } + else + { + dir[dim] = 0.0; + } + } // end for + + node_vel(rk_level, node_gid, 0) = mat_fill(f_id).speed * dir[0]; + node_vel(rk_level, node_gid, 1) = mat_fill(f_id).speed * dir[1]; + if (num_dim == 3) + { + node_vel(rk_level, node_gid, 2) = 0.0; + } + + break; } - } // end for - - - node_vel(rk_level, node_gid, 0) = mat_fill(f_id).speed*dir[0]; - node_vel(rk_level, node_gid, 1) = mat_fill(f_id).speed*dir[1]; - if (num_dim == 3) node_vel(rk_level, node_gid, 2) = 0.0; - - break; - } - case VELOCITY_TYPE::spherical: - { - - // Setting up spherical - double dir[3]; - dir[0] = 0.0; - dir[1] = 0.0; - dir[2] = 0.0; - double radius_val = 0.0; - - for(int dim=0; dim<3; dim++){ - dir[dim] = node_coords(rk_level, node_gid, dim); - radius_val += node_coords(rk_level, node_gid, dim)*node_coords(rk_level, node_gid, dim); - } // end for - radius_val = sqrt(radius_val); - - for(int dim=0; dim<3; dim++){ - if (radius_val > 1.0e-14){ - dir[dim] /= (radius_val); + case VELOCITY_TYPE::spherical: + { + // Setting up spherical + double dir[3]; + dir[0] = 0.0; + dir[1] = 0.0; + dir[2] = 0.0; + double radius_val = 0.0; + + for (int dim = 0; dim < 3; dim++) + { + dir[dim] = node_coords(rk_level, node_gid, dim); + radius_val += node_coords(rk_level, node_gid, dim) * node_coords(rk_level, node_gid, dim); + } // end for + radius_val = sqrt(radius_val); + + for (int dim = 0; dim < 3; dim++) + { + if (radius_val > 1.0e-14) + { + dir[dim] /= (radius_val); + } + else + { + dir[dim] = 0.0; + } + } // end for + + node_vel(rk_level, node_gid, 0) = mat_fill(f_id).speed * dir[0]; + node_vel(rk_level, node_gid, 1) = mat_fill(f_id).speed * dir[1]; + if (num_dim == 3) + { + node_vel(rk_level, node_gid, 2) = mat_fill(f_id).speed * dir[2]; + } + + break; } - else{ - dir[dim] = 0.0; + case VELOCITY_TYPE::radial_linear: + { + break; } - } // end for - - node_vel(rk_level, node_gid, 0) = mat_fill(f_id).speed*dir[0]; - node_vel(rk_level, node_gid, 1) = mat_fill(f_id).speed*dir[1]; - if (num_dim == 3) node_vel(rk_level, node_gid, 2) = mat_fill(f_id).speed*dir[2]; - - break; - } - case VELOCITY_TYPE::radial_linear: - { - - break; - } - case VELOCITY_TYPE::spherical_linear: - { - - break; - } - case VELOCITY_TYPE::tg_vortex: - { - - node_vel(rk_level, node_gid, 0) = sin(PI * node_coords(rk_level,node_gid, 0)) * cos(PI * node_coords(rk_level,node_gid, 1)); - node_vel(rk_level, node_gid, 1) = -1.0*cos(PI * node_coords(rk_level,node_gid, 0)) * sin(PI * node_coords(rk_level,node_gid, 1)); - if (num_dim == 3) node_vel(rk_level, node_gid, 2) = 0.0; - - break; - } - } // end of switch - - }// end loop over nodes of element - - - if(mat_fill(f_id).velocity == VELOCITY_TYPE::tg_vortex) - { - elem_pres(elem_gid) = 0.25*( cos(2.0*PI*elem_coords[0]) + cos(2.0*PI*elem_coords[1]) ) + 1.0; - - // p = rho*ie*(gamma - 1) - size_t mat_id = f_id; - double gamma = global_vars(mat_id,0); // gamma value - elem_sie(rk_level, elem_gid) = - elem_pres(elem_gid)/(mat_fill(f_id).den*(gamma - 1.0)); - } // end if + case VELOCITY_TYPE::spherical_linear: + { + break; + } + case VELOCITY_TYPE::tg_vortex: + { + node_vel(rk_level, node_gid, 0) = sin(PI * node_coords(rk_level, node_gid, 0)) * cos(PI * node_coords(rk_level, node_gid, 1)); + node_vel(rk_level, node_gid, 1) = -1.0 * cos(PI * node_coords(rk_level, node_gid, 0)) * sin(PI * node_coords(rk_level, node_gid, 1)); + if (num_dim == 3) + { + node_vel(rk_level, node_gid, 2) = 0.0; + } + + break; + } + } // end of switch + } // end loop over nodes of element - } // end if fill - + if (mat_fill(f_id).velocity == VELOCITY_TYPE::tg_vortex) + { + elem_pres(elem_gid) = 0.25 * (cos(2.0 * PI * elem_coords[0]) + cos(2.0 * PI * elem_coords[1]) ) + 1.0; + + // p = rho*ie*(gamma - 1) + size_t mat_id = f_id; + double gamma = global_vars(mat_id, 0); // gamma value + elem_sie(rk_level, elem_gid) = + elem_pres(elem_gid) / (mat_fill(f_id).den * (gamma - 1.0)); + } // end if + } // end if fill }); // end FOR_ALL_CLASS element loop - Kokkos::fence(); - - - } // end for loop over fills - }//end view scope - - - + Kokkos::fence(); + } // end for loop over fills + } // end view scope + // apply BC's to velocity FEA_Module_Dynamic_Elasticity::boundary_velocity(*mesh, boundary, node_vel); - - + // calculate the corner massess if 2D - if(num_dim==2){ - + if (num_dim == 2) + { FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { - // facial area of the corners double corner_areas_array[4]; - - ViewCArrayKokkos corner_areas(&corner_areas_array[0],4); - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 4); - + + ViewCArrayKokkos corner_areas(&corner_areas_array[0], 4); + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 4); + get_area_weights2D(corner_areas, elem_gid, node_coords, elem_node_gids, rk_level); - + // loop over the corners of the element and calculate the mass - for (size_t corner_lid=0; corner_lid<4; corner_lid++){ - - size_t corner_gid = corners_in_elem(elem_gid, corner_lid); - corner_mass(corner_gid) = corner_areas(corner_lid)*elem_den(elem_gid); // node radius is added later - + for (size_t corner_lid = 0; corner_lid < 4; corner_lid++) + { + size_t corner_gid = corners_in_elem(elem_gid, corner_lid); + corner_mass(corner_gid) = corner_areas(corner_lid) * elem_den(elem_gid); // node radius is added later } // end for over corners }); - } // end of - - + // calculate the nodal mass FOR_ALL_CLASS(node_gid, 0, nall_nodes, { - node_mass(node_gid) = 0.0; - - if(num_dim==3){ - - for(size_t elem_lid=0; elem_lid < num_corners_in_node(node_gid); elem_lid++){ - size_t elem_gid = elems_in_node(node_gid,elem_lid); - node_mass(node_gid) += 1.0/8.0*elem_mass(elem_gid); + + if (num_dim == 3) + { + for (size_t elem_lid = 0; elem_lid < num_corners_in_node(node_gid); elem_lid++) + { + size_t elem_gid = elems_in_node(node_gid, elem_lid); + node_mass(node_gid) += 1.0 / 8.0 * elem_mass(elem_gid); } // end for elem_lid - - }// end if dims=3 - else { - + } // end if dims=3 + else + { // 2D-RZ - for(size_t corner_lid=0; corner_lid < num_corners_in_node(node_gid); corner_lid++){ - - size_t corner_gid = corners_in_node(node_gid, corner_lid); + for (size_t corner_lid = 0; corner_lid < num_corners_in_node(node_gid); corner_lid++) + { + size_t corner_gid = corners_in_node(node_gid, corner_lid); node_mass(node_gid) += corner_mass(corner_gid); // sans the radius so it is areal node mass - - corner_mass(corner_gid) *= node_coords(rk_level,node_gid,1); // true corner mass now + + corner_mass(corner_gid) *= node_coords(rk_level, node_gid, 1); // true corner mass now } // end for elem_lid - } // end else - }); // end FOR_ALL_CLASS Kokkos::fence(); - - //current interface has differing mass arrays; this equates them until we unify memory - //view scope + // current interface has differing mass arrays; this equates them until we unify memory + // view scope { - vec_array node_mass_interface = node_masses_distributed->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - node_mass_interface(node_gid,0) = node_mass(node_gid); + vec_array node_mass_interface = node_masses_distributed->getLocalView(Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + node_mass_interface(node_gid, 0) = node_mass(node_gid); }); // end parallel for - } //end view scope + } // end view scope Kokkos::fence(); - //communicate ghost densities + // communicate ghost densities comm_node_masses(); - //this is forcing a copy to the device - //view scope + // this is forcing a copy to the device + // view scope { - vec_array ghost_node_mass_interface = ghost_node_masses_distributed->getLocalView (Tpetra::Access::ReadWrite); + vec_array ghost_node_mass_interface = ghost_node_masses_distributed->getLocalView(Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, nlocal_nodes, nall_nodes, { - node_mass(node_gid) = ghost_node_mass_interface(node_gid-nlocal_nodes,0); + FOR_ALL_CLASS(node_gid, nlocal_nodes, nall_nodes, { + node_mass(node_gid) = ghost_node_mass_interface(node_gid - nlocal_nodes, 0); }); // end parallel for - } //end view scope + } // end view scope Kokkos::fence(); - //update stiffness matrix - if(simparam->topology_optimization_on || simparam->shape_optimization_on){ - assemble_matrix(); + // update stiffness matrix + if (simparam->topology_optimization_on || simparam->shape_optimization_on) + { + assemble_matrix(); } - - //execute solve - elastic_solve(); + // execute solve + elastic_solve(); } /* ------------------------------------------------------------------------------------------- @@ -500,141 +510,163 @@ void FEA_Module_Dynamic_Elasticity::update_forward_solve(Teuchos::RCP double FEA_Module_Dynamic_Elasticity::average_element_density(const int nodes_per_elem, const CArray current_element_densities) const { - double result = 0; - for(int i=0; i < nodes_per_elem; i++){ - result += current_element_densities(i)/nodes_per_elem; - } + double result = 0; + for (int i = 0; i < nodes_per_elem; i++) + { + result += current_element_densities(i) / nodes_per_elem; + } - return result; + return result; } /* --------------------------------------------------------------------------------------------------------------- - Simpler adjoint vector solve for the kinetic energy minimization problem + Simpler adjoint vector solve for the kinetic energy minimization problem when force does not depend on u and v. ------------------------------------------------------------------------------------------------------------------ */ -void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_adjoint(){ - - size_t num_bdy_nodes = mesh->num_bdy_nodes; - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos material = simparam->material; - const int num_dim = simparam->num_dims; - real_t global_dt; - size_t current_data_index, next_data_index; - Teuchos::RCP previous_adjoint_vector_distributed, current_adjoint_vector_distributed, previous_velocity_vector_distributed, current_velocity_vector_distributed; - //initialize first adjoint vector at last_time_step to 0 as the terminal value - (*adjoint_vector_data)[last_time_step+1]->putScalar(0); - - //solve terminal value problem, proceeds in time backward. For simplicity, we use the same timestep data from the forward solve. - //A linear interpolant is assumed between velocity data points; velocity midpoint is used to update the adjoint. - if(myrank==0) - std::cout << "Computing adjoint vector " << time_data.size() << std::endl; - - for (int cycle = last_time_step; cycle >= 0; cycle--) { - //compute timestep from time data - global_dt = time_data[cycle+1] - time_data[cycle]; - - //print - if(simparam->dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ - if (cycle==last_time_step){ - if(myrank==0) - printf("cycle = %d, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } - // print time step every 10 cycles - else if (cycle%20==0){ - if(myrank==0) - printf("cycle = %d, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } // end if +void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_adjoint() +{ + size_t num_bdy_nodes = mesh->num_bdy_nodes; + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos material = simparam->material; + const int num_dim = simparam->num_dims; + real_t global_dt; + size_t current_data_index, next_data_index; + Teuchos::RCP previous_adjoint_vector_distributed, current_adjoint_vector_distributed, previous_velocity_vector_distributed, current_velocity_vector_distributed; + // initialize first adjoint vector at last_time_step to 0 as the terminal value + (*adjoint_vector_data)[last_time_step + 1]->putScalar(0); + + // solve terminal value problem, proceeds in time backward. For simplicity, we use the same timestep data from the forward solve. + // A linear interpolant is assumed between velocity data points; velocity midpoint is used to update the adjoint. + if (myrank == 0) + { + std::cout << "Computing adjoint vector " << time_data.size() << std::endl; } - //else if (cycle==1){ - //if(myrank==0) - //printf("cycle = %lu, time = %f, time step = %f \n", cycle-1, time_data[cycle-1], global_dt); - //} // end if - //compute adjoint vector for this data point; use velocity midpoint - //view scope + for (int cycle = last_time_step; cycle >= 0; cycle--) { - const_vec_array previous_velocity_vector = (*forward_solve_velocity_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - - const_vec_array previous_adjoint_vector = (*adjoint_vector_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView (Tpetra::Access::ReadWrite); - - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes + nghost_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - //cancellation of half from midpoint and 2 from adjoint equation already done - current_adjoint_vector(node_gid,idim) = -0.5*(current_velocity_vector(node_gid,idim)+previous_velocity_vector(node_gid,idim))*global_dt + previous_adjoint_vector(node_gid,idim); - } + // compute timestep from time data + global_dt = time_data[cycle + 1] - time_data[cycle]; + + // print + if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) + { + if (cycle == last_time_step) + { + if (myrank == 0) + { + printf("cycle = %d, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } + // print time step every 10 cycles + else if (cycle % 20 == 0) + { + if (myrank == 0) + { + printf("cycle = %d, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } // end if + } + // else if (cycle==1){ + // if(myrank==0) + // printf("cycle = %lu, time = %f, time step = %f \n", cycle-1, time_data[cycle-1], global_dt); + // } // end if + + // compute adjoint vector for this data point; use velocity midpoint + // view scope + { + const_vec_array previous_velocity_vector = (*forward_solve_velocity_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + + const_vec_array previous_adjoint_vector = (*adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadWrite); + + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes + nghost_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + // cancellation of half from midpoint and 2 from adjoint equation already done + current_adjoint_vector(node_gid, idim) = -0.5 * (current_velocity_vector(node_gid, idim) + previous_velocity_vector(node_gid, idim)) * global_dt + previous_adjoint_vector(node_gid, + idim); + } }); // end parallel for - Kokkos::fence(); - } //end view scope - - } + Kokkos::fence(); + } // end view scope + } } - /* ------------------------------------------------------------------------------ Coupled adjoint problem for the kinetic energy minimization problem --------------------------------------------------------------------------------- */ -void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_adjoint_full(){ - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - size_t num_bdy_nodes = mesh->num_bdy_nodes; - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos material = simparam->material; - const int num_dim = simparam->num_dims; - const real_t damping_constant = module_params->damping_constant; - real_t global_dt; - size_t current_data_index, next_data_index; - Teuchos::RCP previous_adjoint_vector_distributed, current_adjoint_vector_distributed, previous_velocity_vector_distributed, current_velocity_vector_distributed; - Teuchos::RCP previous_phi_adjoint_vector_distributed, current_phi_adjoint_vector_distributed; - //initialize first adjoint vector at last_time_step to 0 as the terminal value - (*adjoint_vector_data)[last_time_step+1]->putScalar(0); - (*phi_adjoint_vector_data)[last_time_step+1]->putScalar(0); - - //solve terminal value problem, proceeds in time backward. For simplicity, we use the same timestep data from the forward solve. - //A linear interpolant is assumed between velocity data points; velocity midpoint is used to update the adjoint. - if(myrank==0) - std::cout << "Computing adjoint vector " << time_data.size() << std::endl; - - for (int cycle = last_time_step; cycle >= 0; cycle--) { - //compute timestep from time data - global_dt = time_data[cycle+1] - time_data[cycle]; - - //print - if(simparam->dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ - if (cycle==last_time_step){ - if(myrank==0) - printf("cycle = %d, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } - // print time step every 10 cycles - else if (cycle%20==0){ - if(myrank==0) - printf("cycle = %d, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } // end if +void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_adjoint_full() +{ + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + size_t num_bdy_nodes = mesh->num_bdy_nodes; + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos material = simparam->material; + const int num_dim = simparam->num_dims; + const real_t damping_constant = module_params->damping_constant; + real_t global_dt; + size_t current_data_index, next_data_index; + Teuchos::RCP previous_adjoint_vector_distributed, current_adjoint_vector_distributed, previous_velocity_vector_distributed, current_velocity_vector_distributed; + Teuchos::RCP previous_phi_adjoint_vector_distributed, current_phi_adjoint_vector_distributed; + // initialize first adjoint vector at last_time_step to 0 as the terminal value + (*adjoint_vector_data)[last_time_step + 1]->putScalar(0); + (*phi_adjoint_vector_data)[last_time_step + 1]->putScalar(0); + + // solve terminal value problem, proceeds in time backward. For simplicity, we use the same timestep data from the forward solve. + // A linear interpolant is assumed between velocity data points; velocity midpoint is used to update the adjoint. + if (myrank == 0) + { + std::cout << "Computing adjoint vector " << time_data.size() << std::endl; } - - //view scope + + for (int cycle = last_time_step; cycle >= 0; cycle--) { - - //set velocity and position for this timestep - const_vec_array previous_velocity_vector = (*forward_solve_velocity_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - - const_vec_array previous_coordinate_vector = (*forward_solve_coordinate_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_coordinate_vector = (*forward_solve_coordinate_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - - //interface of arrays for current implementation of force calculation - - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes+nghost_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - node_vel(rk_level,node_gid,idim) = previous_velocity_vector(node_gid,idim); - node_coords(rk_level,node_gid,idim) = previous_coordinate_vector(node_gid,idim); + // compute timestep from time data + global_dt = time_data[cycle + 1] - time_data[cycle]; + + // print + if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) + { + if (cycle == last_time_step) + { + if (myrank == 0) + { + printf("cycle = %d, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } + // print time step every 10 cycles + else if (cycle % 20 == 0) + { + if (myrank == 0) + { + printf("cycle = %d, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } // end if } + + // view scope + { + // set velocity and position for this timestep + const_vec_array previous_velocity_vector = (*forward_solve_velocity_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + + const_vec_array previous_coordinate_vector = (*forward_solve_coordinate_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array current_coordinate_vector = (*forward_solve_coordinate_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + + // interface of arrays for current implementation of force calculation + + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes + nghost_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + node_vel(rk_level, node_gid, idim) = previous_velocity_vector(node_gid, idim); + node_coords(rk_level, node_gid, idim) = previous_coordinate_vector(node_gid, idim); + } }); - Kokkos::fence(); + Kokkos::fence(); - get_force_vgradient_elastic(material, + get_force_vgradient_elastic(material, *mesh, node_coords, node_vel, @@ -649,965 +681,1050 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_adjoint_full() 1, cycle); - //force_gradient_velocity->describe(*fos,Teuchos::VERB_EXTREME); - const_vec_array previous_force_gradient_position = force_gradient_position->getLocalView (Tpetra::Access::ReadOnly); - //const_vec_array current_force_gradient_position = force_gradient_position->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array previous_force_gradient_velocity = force_gradient_velocity->getLocalView (Tpetra::Access::ReadOnly); - //const_vec_array current_force_gradient_velocity = force_gradient_velocity->getLocalView (Tpetra::Access::ReadOnly); - //compute gradient of force with respect to velocity - - const_vec_array previous_adjoint_vector = (*adjoint_vector_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array phi_previous_adjoint_vector = (*phi_adjoint_vector_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - vec_array midpoint_adjoint_vector = adjoint_vector_distributed->getLocalView (Tpetra::Access::ReadWrite); - vec_array phi_midpoint_adjoint_vector = phi_adjoint_vector_distributed->getLocalView (Tpetra::Access::ReadWrite); - - //half step update for RK2 scheme - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - real_t rate_of_change; - real_t matrix_contribution; - size_t dof_id; - for (int idim = 0; idim < num_dim; idim++){ - matrix_contribution = 0; - //compute resulting row of force velocity gradient matrix transpose right multiplied by adjoint vector - /* - for(int idof = 0; idof < Gradient_Matrix_Strides(node_gid*num_dim+idim); idof++){ - dof_id = DOF_Graph_Matrix(node_gid*num_dim+idim,idof); - matrix_contribution += previous_adjoint_vector(dof_id/num_dim,dof_id%num_dim)*Force_Gradient_Velocities(node_gid*num_dim+idim,idof); - } - */ - matrix_contribution = -damping_constant*previous_adjoint_vector(node_gid,idim); - rate_of_change = previous_velocity_vector(node_gid,idim)- - matrix_contribution/node_mass(node_gid)- - phi_previous_adjoint_vector(node_gid,idim)/node_mass(node_gid); - midpoint_adjoint_vector(node_gid,idim) = -rate_of_change*global_dt/2 + previous_adjoint_vector(node_gid,idim); - matrix_contribution = 0; - //compute resulting row of force displacement gradient matrix transpose right multiplied by adjoint vector - for(int idof = 0; idof < Gradient_Matrix_Strides(node_gid*num_dim+idim); idof++){ - dof_id = DOF_Graph_Matrix(node_gid*num_dim+idim,idof); - matrix_contribution += -previous_adjoint_vector(dof_id/num_dim,dof_id%num_dim)*Force_Gradient_Positions(node_gid*num_dim+idim,idof); - } - rate_of_change = -matrix_contribution; - //rate_of_change = -0.0000001*previous_adjoint_vector(node_gid,idim); - phi_midpoint_adjoint_vector(node_gid,idim) = -rate_of_change*global_dt/2 + phi_previous_adjoint_vector(node_gid,idim); - } + // force_gradient_velocity->describe(*fos,Teuchos::VERB_EXTREME); + const_vec_array previous_force_gradient_position = force_gradient_position->getLocalView(Tpetra::Access::ReadOnly); + // const_vec_array current_force_gradient_position = force_gradient_position->getLocalView (Tpetra::Access::ReadOnly); + const_vec_array previous_force_gradient_velocity = force_gradient_velocity->getLocalView(Tpetra::Access::ReadOnly); + // const_vec_array current_force_gradient_velocity = force_gradient_velocity->getLocalView (Tpetra::Access::ReadOnly); + // compute gradient of force with respect to velocity + + const_vec_array previous_adjoint_vector = (*adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array phi_previous_adjoint_vector = (*phi_adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + vec_array midpoint_adjoint_vector = adjoint_vector_distributed->getLocalView(Tpetra::Access::ReadWrite); + vec_array phi_midpoint_adjoint_vector = phi_adjoint_vector_distributed->getLocalView(Tpetra::Access::ReadWrite); + + // half step update for RK2 scheme + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + real_t rate_of_change; + real_t matrix_contribution; + size_t dof_id; + for (int idim = 0; idim < num_dim; idim++) + { + matrix_contribution = 0; + // compute resulting row of force velocity gradient matrix transpose right multiplied by adjoint vector + /* + for(int idof = 0; idof < Gradient_Matrix_Strides(node_gid*num_dim+idim); idof++){ + dof_id = DOF_Graph_Matrix(node_gid*num_dim+idim,idof); + matrix_contribution += previous_adjoint_vector(dof_id/num_dim,dof_id%num_dim)*Force_Gradient_Velocities(node_gid*num_dim+idim,idof); + } + */ + matrix_contribution = -damping_constant * previous_adjoint_vector(node_gid, idim); + rate_of_change = previous_velocity_vector(node_gid, idim) - + matrix_contribution / node_mass(node_gid) - + phi_previous_adjoint_vector(node_gid, idim) / node_mass(node_gid); + midpoint_adjoint_vector(node_gid, idim) = -rate_of_change * global_dt / 2 + previous_adjoint_vector(node_gid, idim); + matrix_contribution = 0; + // compute resulting row of force displacement gradient matrix transpose right multiplied by adjoint vector + for (int idof = 0; idof < Gradient_Matrix_Strides(node_gid * num_dim + idim); idof++) + { + dof_id = DOF_Graph_Matrix(node_gid * num_dim + idim, idof); + matrix_contribution += -previous_adjoint_vector(dof_id / num_dim, dof_id % num_dim) * Force_Gradient_Positions(node_gid * num_dim + idim, idof); + } + rate_of_change = -matrix_contribution; + // rate_of_change = -0.0000001*previous_adjoint_vector(node_gid,idim); + phi_midpoint_adjoint_vector(node_gid, idim) = -rate_of_change * global_dt / 2 + phi_previous_adjoint_vector(node_gid, idim); + } }); // end parallel for - Kokkos::fence(); - - boundary_adjoint(*mesh, boundary,midpoint_adjoint_vector, phi_midpoint_adjoint_vector); - comm_adjoint_vectors(cycle); - //swap names to get ghost nodes for the midpoint vectors - vec_array current_adjoint_vector = adjoint_vector_distributed->getLocalView (Tpetra::Access::ReadWrite); - vec_array phi_current_adjoint_vector = phi_adjoint_vector_distributed->getLocalView (Tpetra::Access::ReadWrite); - midpoint_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView (Tpetra::Access::ReadWrite); - phi_midpoint_adjoint_vector = (*phi_adjoint_vector_data)[cycle]->getLocalView (Tpetra::Access::ReadWrite); - - //full step update with midpoint gradient for RK2 scheme - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - real_t rate_of_change; - real_t matrix_contribution; - size_t dof_id; - for (int idim = 0; idim < num_dim; idim++){ - matrix_contribution = 0; - //compute resulting row of force velocity gradient matrix transpose right multiplied by adjoint vector - /* - for(int idof = 0; idof < Gradient_Matrix_Strides(node_gid*num_dim+idim); idof++){ - dof_id = DOF_Graph_Matrix(node_gid*num_dim+idim,idof); - matrix_contribution += midpoint_adjoint_vector(dof_id/num_dim,dof_id%num_dim)*Force_Gradient_Velocities(node_gid*num_dim+idim,idof); - } - */ - matrix_contribution = -damping_constant*midpoint_adjoint_vector(node_gid,idim); - rate_of_change = (previous_velocity_vector(node_gid,idim) + current_velocity_vector(node_gid,idim))/2- - matrix_contribution/node_mass(node_gid)- - phi_midpoint_adjoint_vector(node_gid,idim)/node_mass(node_gid); - current_adjoint_vector(node_gid,idim) = -rate_of_change*global_dt + previous_adjoint_vector(node_gid,idim); - matrix_contribution = 0; - //compute resulting row of force displacement gradient matrix transpose right multiplied by adjoint vector - for(int idof = 0; idof < Gradient_Matrix_Strides(node_gid*num_dim+idim); idof++){ - dof_id = DOF_Graph_Matrix(node_gid*num_dim+idim,idof); - matrix_contribution += -midpoint_adjoint_vector(dof_id/num_dim,dof_id%num_dim)*Force_Gradient_Positions(node_gid*num_dim+idim,idof); - } - rate_of_change = -matrix_contribution; - //rate_of_change = -0.0000001*midpoint_adjoint_vector(node_gid,idim); - phi_current_adjoint_vector(node_gid,idim) = -rate_of_change*global_dt + phi_previous_adjoint_vector(node_gid,idim); - } + Kokkos::fence(); + + boundary_adjoint(*mesh, boundary, midpoint_adjoint_vector, phi_midpoint_adjoint_vector); + comm_adjoint_vectors(cycle); + // swap names to get ghost nodes for the midpoint vectors + vec_array current_adjoint_vector = adjoint_vector_distributed->getLocalView(Tpetra::Access::ReadWrite); + vec_array phi_current_adjoint_vector = phi_adjoint_vector_distributed->getLocalView(Tpetra::Access::ReadWrite); + midpoint_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadWrite); + phi_midpoint_adjoint_vector = (*phi_adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadWrite); + + // full step update with midpoint gradient for RK2 scheme + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + real_t rate_of_change; + real_t matrix_contribution; + size_t dof_id; + for (int idim = 0; idim < num_dim; idim++) + { + matrix_contribution = 0; + // compute resulting row of force velocity gradient matrix transpose right multiplied by adjoint vector + /* + for(int idof = 0; idof < Gradient_Matrix_Strides(node_gid*num_dim+idim); idof++){ + dof_id = DOF_Graph_Matrix(node_gid*num_dim+idim,idof); + matrix_contribution += midpoint_adjoint_vector(dof_id/num_dim,dof_id%num_dim)*Force_Gradient_Velocities(node_gid*num_dim+idim,idof); + } + */ + matrix_contribution = -damping_constant * midpoint_adjoint_vector(node_gid, idim); + rate_of_change = (previous_velocity_vector(node_gid, idim) + current_velocity_vector(node_gid, idim)) / 2 - + matrix_contribution / node_mass(node_gid) - + phi_midpoint_adjoint_vector(node_gid, idim) / node_mass(node_gid); + current_adjoint_vector(node_gid, idim) = -rate_of_change * global_dt + previous_adjoint_vector(node_gid, idim); + matrix_contribution = 0; + // compute resulting row of force displacement gradient matrix transpose right multiplied by adjoint vector + for (int idof = 0; idof < Gradient_Matrix_Strides(node_gid * num_dim + idim); idof++) + { + dof_id = DOF_Graph_Matrix(node_gid * num_dim + idim, idof); + matrix_contribution += -midpoint_adjoint_vector(dof_id / num_dim, dof_id % num_dim) * Force_Gradient_Positions(node_gid * num_dim + idim, idof); + } + rate_of_change = -matrix_contribution; + // rate_of_change = -0.0000001*midpoint_adjoint_vector(node_gid,idim); + phi_current_adjoint_vector(node_gid, idim) = -rate_of_change * global_dt + phi_previous_adjoint_vector(node_gid, idim); + } }); // end parallel for - Kokkos::fence(); + Kokkos::fence(); - - boundary_adjoint(*mesh, boundary,current_adjoint_vector, phi_current_adjoint_vector); + boundary_adjoint(*mesh, boundary, current_adjoint_vector, phi_current_adjoint_vector); + } // end view scope - } //end view scope - - comm_adjoint_vectors(cycle); - //phi_adjoint_vector_distributed->describe(*fos,Teuchos::VERB_EXTREME); - } + comm_adjoint_vectors(cycle); + // phi_adjoint_vector_distributed->describe(*fos,Teuchos::VERB_EXTREME); + } } - /* ---------------------------------------------------------------------------- Gradient calculation for the kinetic energy minimization problem ------------------------------------------------------------------------------- */ -void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient(const_vec_array design_variables, vec_array design_gradients){ - - size_t num_bdy_nodes = mesh->num_bdy_nodes; - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos material = simparam->material; - const int num_dim = simparam->num_dims; - int num_corners = rnum_elem*num_nodes_in_elem; - real_t global_dt; - size_t current_data_index, next_data_index; - CArrayKokkos current_element_velocities = CArrayKokkos(num_nodes_in_elem,num_dim); - CArrayKokkos current_element_adjoint = CArrayKokkos(num_nodes_in_elem,num_dim); - - if(myrank==0) - std::cout << "Computing accumulated kinetic energy gradient" << std::endl; +void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient(const_vec_array design_variables, vec_array design_gradients) +{ + size_t num_bdy_nodes = mesh->num_bdy_nodes; + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos material = simparam->material; + const int num_dim = simparam->num_dims; + int num_corners = rnum_elem * num_nodes_in_elem; + real_t global_dt; + size_t current_data_index, next_data_index; + CArrayKokkos current_element_velocities = CArrayKokkos(num_nodes_in_elem, num_dim); + CArrayKokkos current_element_adjoint = CArrayKokkos(num_nodes_in_elem, num_dim); + + if (myrank == 0) + { + std::cout << "Computing accumulated kinetic energy gradient" << std::endl; + } - compute_topology_optimization_adjoint(); + compute_topology_optimization_adjoint(); - //compute design gradients - FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { - design_gradients(node_id,0) = 0; + // compute design gradients + FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { + design_gradients(node_id, 0) = 0; }); // end parallel for - Kokkos::fence(); - - //gradient contribution from kinetic energy vMv product. - for (unsigned long cycle = 0; cycle < last_time_step+1; cycle++) { - //compute timestep from time data - global_dt = time_data[cycle+1] - time_data[cycle]; - - //print - if(simparam->dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ - if (cycle==0){ - if(myrank==0) - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } - // print time step every 10 cycles - else if (cycle%20==0){ - if(myrank==0) - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } // end if - } + Kokkos::fence(); - //compute adjoint vector for this data point; use velocity midpoint - //view scope - { - const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array next_velocity_vector = (*forward_solve_velocity_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array next_adjoint_vector = (*adjoint_vector_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - - FOR_ALL_CLASS(elem_id, 0, rnum_elem, { - size_t node_id; - size_t corner_id; - real_t inner_product; - //std::cout << elem_mass(elem_id) <dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) + { + if (cycle == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } } - } - - for (int inode = 0; inode < num_nodes_in_elem; inode++){ - //compute gradient of local element contribution to v^t*M*v product - corner_id = elem_id*num_nodes_in_elem + inode; - corner_value_storage(corner_id) = inner_product*global_dt; - } - + // print time step every 10 cycles + else if (cycle % 20 == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } // end if + } + + // compute adjoint vector for this data point; use velocity midpoint + // view scope + { + const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_velocity_vector = (*forward_solve_velocity_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_adjoint_vector = (*adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + + FOR_ALL_CLASS(elem_id, 0, rnum_elem, { + size_t node_id; + size_t corner_id; + real_t inner_product; + // std::cout << elem_mass(elem_id) <dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ - if (cycle==0){ - if(myrank==0) - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } - // print time step every 10 cycles - else if (cycle%20==0){ - if(myrank==0) - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } // end if - } + Kokkos::fence(); - //compute adjoint vector for this data point; use velocity midpoint - //view scope - { - //const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array next_adjoint_vector = (*adjoint_vector_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - //const_vec_array current_coord_vector = forward_solve_coordinate_data[cycle]->getLocalView (Tpetra::Access::ReadOnly); - //const_vec_array final_coordinates = forward_solve_coordinate_data[last_time_step+1]->getLocalView (Tpetra::Access::ReadOnly); - - FOR_ALL_CLASS(elem_id, 0, rnum_elem, { - size_t node_id; - size_t corner_id; - real_t inner_product; - //std::cout << elem_mass(elem_id) <dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) + { + if (cycle == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } + // print time step every 10 cycles + else if (cycle % 20 == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } // end if + } -void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient_full(Teuchos::RCP design_densities_distributed, Teuchos::RCP design_gradients_distributed){ + // compute adjoint vector for this data point; use velocity midpoint + // view scope + { + // const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); + const_vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_adjoint_vector = (*adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + // const_vec_array current_coord_vector = forward_solve_coordinate_data[cycle]->getLocalView (Tpetra::Access::ReadOnly); + // const_vec_array final_coordinates = forward_solve_coordinate_data[last_time_step+1]->getLocalView (Tpetra::Access::ReadOnly); + + FOR_ALL_CLASS(elem_id, 0, rnum_elem, { + size_t node_id; + size_t corner_id; + real_t inner_product; + // std::cout << elem_mass(elem_id) <num_bdy_nodes; - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos material = simparam->material; - const int num_dim = simparam->num_dims; - int num_corners = rnum_elem*num_nodes_in_elem; - real_t global_dt; - bool element_constant_density = true; - size_t current_data_index, next_data_index; - CArrayKokkos current_element_velocities = CArrayKokkos(num_nodes_in_elem,num_dim); - CArrayKokkos current_element_adjoint = CArrayKokkos(num_nodes_in_elem,num_dim); + inner_product = 0; + for (int ifill = 0; ifill < num_nodes_in_elem; ifill++) + { + node_id = nodes_in_elem(elem_id, ifill); + for (int idim = 0; idim < num_dim; idim++) + { + inner_product += 0.00001 * current_element_adjoint(ifill, idim); + // inner_product += 0.0001; + } + } - if(myrank==0) - std::cout << "Computing accumulated kinetic energy gradient" << std::endl; + for (int inode = 0; inode < num_nodes_in_elem; inode++) + { + // compute gradient of local element contribution to v^t*M*v product + corner_id = elem_id * num_nodes_in_elem + inode; + corner_value_storage(corner_id) = -inner_product * global_dt / (double)num_nodes_in_elem; + } + }); // end parallel for + Kokkos::fence(); - compute_topology_optimization_adjoint_full(); + // accumulate node values from corner storage + // multiply + FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { + size_t corner_id; + for (int icorner = 0; icorner < num_corners_in_node(node_id); icorner++) + { + corner_id = corners_in_node(node_id, icorner); + design_gradients(node_id, 0) += corner_value_storage(corner_id); + } + }); // end parallel for + Kokkos::fence(); + } // end view scope + } +} - { //view scope - vec_array design_gradients = design_gradients_distributed->getLocalView (Tpetra::Access::ReadWrite); - const_vec_array design_densities = design_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); - //compute design gradients - FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { - design_gradients(node_id,0) = 0; - }); // end parallel for - Kokkos::fence(); +/* ---------------------------------------------------------------------------- + Gradient for the (unsimplified) kinetic energy minimization problem +------------------------------------------------------------------------------- */ - //gradient contribution from kinetic energy v(dM/drho)v product. - if(simparam->dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ - if(myrank==0){ - std::cout << "v*dM/drho*v term" << std::endl; - } +void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient_full(Teuchos::RCP design_densities_distributed, Teuchos::RCP design_gradients_distributed) +{ + size_t num_bdy_nodes = mesh->num_bdy_nodes; + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos material = simparam->material; + const int num_dim = simparam->num_dims; + int num_corners = rnum_elem * num_nodes_in_elem; + real_t global_dt; + bool element_constant_density = true; + size_t current_data_index, next_data_index; + CArrayKokkos current_element_velocities = CArrayKokkos(num_nodes_in_elem, num_dim); + CArrayKokkos current_element_adjoint = CArrayKokkos(num_nodes_in_elem, num_dim); + + if (myrank == 0) + { + std::cout << "Computing accumulated kinetic energy gradient" << std::endl; } - for (unsigned long cycle = 0; cycle < last_time_step+1; cycle++) { - //compute timestep from time data - global_dt = time_data[cycle+1] - time_data[cycle]; - - //print - if(simparam->dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ + compute_topology_optimization_adjoint_full(); - if (cycle==0){ - if(myrank==0) - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } - // print time step every 10 cycles - else if (cycle%20==0){ - if(myrank==0) - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } // end if - } + { // view scope + vec_array design_gradients = design_gradients_distributed->getLocalView(Tpetra::Access::ReadWrite); + const_vec_array design_densities = design_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); + // compute design gradients + FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { + design_gradients(node_id, 0) = 0; + }); // end parallel for + Kokkos::fence(); - //compute adjoint vector for this data point; use velocity midpoint - //view scope + // gradient contribution from kinetic energy v(dM/drho)v product. + if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) { - const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array next_velocity_vector = (*forward_solve_velocity_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array next_adjoint_vector = (*adjoint_vector_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - - FOR_ALL_CLASS(elem_id, 0, rnum_elem, { - size_t node_id; - size_t corner_id; - real_t inner_product; - //std::cout << elem_mass(elem_id) <dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) + { + if (cycle == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } + // print time step every 10 cycles + else if (cycle % 20 == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } // end if } - }); // end parallel for - Kokkos::fence(); - } //end view scope + // compute adjoint vector for this data point; use velocity midpoint + // view scope + { + const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_velocity_vector = (*forward_solve_velocity_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_adjoint_vector = (*adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + + FOR_ALL_CLASS(elem_id, 0, rnum_elem, { + size_t node_id; + size_t corner_id; + real_t inner_product; + // std::cout << elem_mass(elem_id) <dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ - if(myrank==0){ - std::cout << "gradient term involving adjoint derivative" << std::endl; + // accumulate node values from corner storage + // multiply + FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { + size_t corner_id; + for (int icorner = 0; icorner < num_corners_in_node(node_id); icorner++) + { + corner_id = corners_in_node(node_id, icorner); + design_gradients(node_id, 0) += corner_value_storage(corner_id); + } + }); // end parallel for + Kokkos::fence(); + } // end view scope } - } - for (unsigned long cycle = 0; cycle < last_time_step+1; cycle++) { - //compute timestep from time data - global_dt = time_data[cycle+1] - time_data[cycle]; - //print - if(simparam->dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ + // multiply by Hex8 constants (the diagonlization here only works for Hex8 anyway) + FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { + design_gradients(node_id, 0) *= 0.5 / (double)num_nodes_in_elem / (double)num_nodes_in_elem; + // design_gradients(node_id,0) =0.00001; + }); // end parallel for + Kokkos::fence(); - if (cycle==0){ - if(myrank==0) - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + // gradient contribution from time derivative of adjoint \dot{lambda}(dM/drho)v product. + if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) + { + if (myrank == 0) + { + std::cout << "gradient term involving adjoint derivative" << std::endl; + } } - // print time step every 10 cycles - else if (cycle%20==0){ - if(myrank==0) - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } // end if - } - //compute adjoint vector for this data point; use velocity midpoint - //view scope + for (unsigned long cycle = 0; cycle < last_time_step + 1; cycle++) { - const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_phi_adjoint_vector = (*phi_adjoint_vector_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array next_velocity_vector = (*forward_solve_velocity_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array next_adjoint_vector = (*adjoint_vector_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array next_phi_adjoint_vector = (*phi_adjoint_vector_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - - const real_t damping_constant = module_params->damping_constant; - FOR_ALL_CLASS(elem_id, 0, rnum_elem, { - real_t lambda_dot_current; - real_t lambda_dot_next; - size_t node_id; - size_t corner_id; - real_t inner_product; - //std::cout << elem_mass(elem_id) <dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) + { + if (cycle == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } + // print time step every 10 cycles + else if (cycle % 20 == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } // end if } - inner_product = 0; - for(int ifill=0; ifill < num_nodes_in_elem; ifill++){ - node_id = nodes_in_elem(elem_id, ifill); - for(int idim=0; idim < num_dim; idim++){ - //lambda_dot_current = lambda_dot_next = (next_adjoint_vector(node_id,idim)-current_adjoint_vector(node_id,idim))/global_dt; - lambda_dot_current = current_velocity_vector(node_id,idim) + damping_constant*current_adjoint_vector(node_id,idim)/node_mass(node_id) - current_phi_adjoint_vector(node_id,idim)/node_mass(node_id); - lambda_dot_next = next_velocity_vector(node_id,idim) + damping_constant*next_adjoint_vector(node_id,idim)/node_mass(node_id) - next_phi_adjoint_vector(node_id,idim)/node_mass(node_id); - inner_product += elem_mass(elem_id)*(lambda_dot_current+lambda_dot_next)*current_element_velocities(ifill,idim)/2; - } - } + // compute adjoint vector for this data point; use velocity midpoint + // view scope + { + const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array current_phi_adjoint_vector = (*phi_adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_velocity_vector = (*forward_solve_velocity_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_adjoint_vector = (*adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_phi_adjoint_vector = (*phi_adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + + const real_t damping_constant = module_params->damping_constant; + FOR_ALL_CLASS(elem_id, 0, rnum_elem, { + real_t lambda_dot_current; + real_t lambda_dot_next; + size_t node_id; + size_t corner_id; + real_t inner_product; + // std::cout << elem_mass(elem_id) <getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_adjoint_vector = (*adjoint_vector_data)[0]->getLocalView (Tpetra::Access::ReadOnly); - - FOR_ALL_CLASS(elem_id, 0, rnum_elem, { - real_t lambda_dot; - size_t node_id; - size_t corner_id; - real_t inner_product; - //std::cout << elem_mass(elem_id) <getLocalView(Tpetra::Access::ReadOnly); + const_vec_array current_adjoint_vector = (*adjoint_vector_data)[0]->getLocalView(Tpetra::Access::ReadOnly); + + FOR_ALL_CLASS(elem_id, 0, rnum_elem, { + real_t lambda_dot; + size_t node_id; + size_t corner_id; + real_t inner_product; + // std::cout << elem_mass(elem_id) <getLocalView (Tpetra::Access::ReadWrite); - const_host_vec_array host_design_variables = design_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); - compute_stiffness_gradients(host_design_variables, host_design_gradients); - }//end view scope + Kokkos::fence(); + } // end view scope + } // end view scope + // force_design_gradient_term(design_variables, design_gradients); + // view scope + { + host_vec_array host_design_gradients = design_gradients_distributed->getLocalView(Tpetra::Access::ReadWrite); + const_host_vec_array host_design_variables = design_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); + compute_stiffness_gradients(host_design_variables, host_design_gradients); + } // end view scope } /* ---------------------------------------------------------------------- Initialize global vectors and array maps needed for matrix assembly ------------------------------------------------------------------------- */ -void FEA_Module_Dynamic_Elasticity::init_assembly(){ - int num_dim = simparam->num_dims; - //const_host_elem_conn_array nodes_in_elem = global_nodes_in_elem_distributed->getLocalView (Tpetra::Access::ReadOnly); - Gradient_Matrix_Strides = DCArrayKokkos (nlocal_nodes*num_dim, "Gradient_Matrix_Strides"); - CArrayKokkos Graph_Fill(nall_nodes, "nall_nodes"); - CArrayKokkos current_row_nodes_scanned; - int local_node_index, current_column_index; - size_t max_stride = 0; - size_t nodes_per_element; - nodal_density_flag = simparam->nodal_density_flag; - penalty_power = simparam->optimization_options.simp_penalty_power; - - //allocate stride arrays - CArrayKokkos Graph_Matrix_Strides_initial(nlocal_nodes, "Graph_Matrix_Strides_initial"); - DCArrayKokkos Dual_Graph_Matrix_Strides_initial(nlocal_nodes, "Host_Graph_Matrix_Strides_initial"); - Graph_Matrix_Strides = DCArrayKokkos(nlocal_nodes, "Graph_Matrix_Strides"); - - //allocate storage for the sparse stiffness matrix map used in the assembly process - Global_Stiffness_Matrix_Assembly_Map = Global_Gradient_Matrix_Assembly_Map = DCArrayKokkos(rnum_elem, - max_nodes_per_element,max_nodes_per_element, "Global_Gradient_Matrix_Assembly_Map"); - - //allocate array used to determine global node repeats in the sparse graph later - DCArrayKokkos node_indices_used(nall_nodes, "node_indices_used"); - - /*allocate array that stores which column the node index occured on for the current row - when removing repeats*/ - DCArrayKokkos column_index(nall_nodes, "column_index"); - - //initialize nlocal arrays - FOR_ALL_CLASS(inode, 0, nlocal_nodes, { - Graph_Matrix_Strides_initial(inode) = 0; - Graph_Matrix_Strides(inode) = 0; - Graph_Fill(inode) = 0; +void FEA_Module_Dynamic_Elasticity::init_assembly() +{ + int num_dim = simparam->num_dims; + // const_host_elem_conn_array nodes_in_elem = global_nodes_in_elem_distributed->getLocalView (Tpetra::Access::ReadOnly); + Gradient_Matrix_Strides = DCArrayKokkos(nlocal_nodes * num_dim, "Gradient_Matrix_Strides"); + CArrayKokkos Graph_Fill(nall_nodes, "nall_nodes"); + CArrayKokkos current_row_nodes_scanned; + int local_node_index, current_column_index; + size_t max_stride = 0; + size_t nodes_per_element; + nodal_density_flag = simparam->nodal_density_flag; + penalty_power = simparam->optimization_options.simp_penalty_power; + + // allocate stride arrays + CArrayKokkos Graph_Matrix_Strides_initial(nlocal_nodes, "Graph_Matrix_Strides_initial"); + DCArrayKokkos Dual_Graph_Matrix_Strides_initial(nlocal_nodes, "Host_Graph_Matrix_Strides_initial"); + Graph_Matrix_Strides = DCArrayKokkos(nlocal_nodes, "Graph_Matrix_Strides"); + + // allocate storage for the sparse stiffness matrix map used in the assembly process + Global_Stiffness_Matrix_Assembly_Map = Global_Gradient_Matrix_Assembly_Map = DCArrayKokkos(rnum_elem, + max_nodes_per_element, max_nodes_per_element, "Global_Gradient_Matrix_Assembly_Map"); + + // allocate array used to determine global node repeats in the sparse graph later + DCArrayKokkos node_indices_used(nall_nodes, "node_indices_used"); + + /*allocate array that stores which column the node index occured on for the current row + when removing repeats*/ + DCArrayKokkos column_index(nall_nodes, "column_index"); + + // initialize nlocal arrays + FOR_ALL_CLASS(inode, 0, nlocal_nodes, { + Graph_Matrix_Strides_initial(inode) = 0; + Graph_Matrix_Strides(inode) = 0; + Graph_Fill(inode) = 0; }); // end parallel for - Kokkos::fence(); + Kokkos::fence(); - //initialize nall arrays - //initialize nlocal arrays - FOR_ALL_CLASS(inode, 0, nall_nodes, { - node_indices_used(inode) = 0; - column_index(inode) = 0; + // initialize nall arrays + // initialize nlocal arrays + FOR_ALL_CLASS(inode, 0, nall_nodes, { + node_indices_used(inode) = 0; + column_index(inode) = 0; }); // end parallel for - Kokkos::fence(); - - //count upper bound of strides for Sparse Pattern Graph by allowing repeats due to connectivity - if(num_dim == 2) - for (int ielem = 0; ielem < rnum_elem; ielem++){ - element_select->choose_2Delem_type(Element_Types(ielem), elem2D); - nodes_per_element = elem2D->num_nodes(); - for (int lnode = 0; lnode < nodes_per_element; lnode++){ - local_node_index = nodes_in_elem(ielem, lnode); - if(local_node_index < nlocal_nodes){ - Dual_Graph_Matrix_Strides_initial.host(local_node_index) += nodes_per_element; + Kokkos::fence(); + + // count upper bound of strides for Sparse Pattern Graph by allowing repeats due to connectivity + if (num_dim == 2) + { + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + element_select->choose_2Delem_type(Element_Types(ielem), elem2D); + nodes_per_element = elem2D->num_nodes(); + for (int lnode = 0; lnode < nodes_per_element; lnode++) + { + local_node_index = nodes_in_elem(ielem, lnode); + if (local_node_index < nlocal_nodes) + { + Dual_Graph_Matrix_Strides_initial.host(local_node_index) += nodes_per_element; + } + } } - } } - if(num_dim == 3) - for (int ielem = 0; ielem < rnum_elem; ielem++){ - element_select->choose_3Delem_type(Element_Types(ielem), elem); - nodes_per_element = elem->num_nodes(); - for (int lnode = 0; lnode < nodes_per_element; lnode++){ - local_node_index = nodes_in_elem(ielem, lnode); - if(local_node_index < nlocal_nodes){ - Dual_Graph_Matrix_Strides_initial.host(local_node_index) += nodes_per_element; + if (num_dim == 3) + { + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + element_select->choose_3Delem_type(Element_Types(ielem), elem); + nodes_per_element = elem->num_nodes(); + for (int lnode = 0; lnode < nodes_per_element; lnode++) + { + local_node_index = nodes_in_elem(ielem, lnode); + if (local_node_index < nlocal_nodes) + { + Dual_Graph_Matrix_Strides_initial.host(local_node_index) += nodes_per_element; + } + } } - } } - - Dual_Graph_Matrix_Strides_initial.update_device(); - //equate strides for later - FOR_ALL_CLASS(inode, 0, nlocal_nodes, { - Graph_Matrix_Strides(inode) = Graph_Matrix_Strides_initial(inode) = Dual_Graph_Matrix_Strides_initial(inode); + Dual_Graph_Matrix_Strides_initial.update_device(); + + // equate strides for later + FOR_ALL_CLASS(inode, 0, nlocal_nodes, { + Graph_Matrix_Strides(inode) = Graph_Matrix_Strides_initial(inode) = Dual_Graph_Matrix_Strides_initial(inode); }); // end parallel for - - //for (int inode = 0; inode < nlocal_nodes; inode++) - //std::cout << Graph_Matrix_Strides_initial(inode) << std::endl; - - //compute maximum stride - size_t update = 0; - REDUCE_MAX_CLASS(inode, 0, nlocal_nodes, update, { - if(update < Graph_Matrix_Strides_initial(inode)) - update = Graph_Matrix_Strides_initial(inode); + + // for (int inode = 0; inode < nlocal_nodes; inode++) + // std::cout << Graph_Matrix_Strides_initial(inode) << std::endl; + + // compute maximum stride + size_t update = 0; + REDUCE_MAX_CLASS(inode, 0, nlocal_nodes, update, { + if (update < Graph_Matrix_Strides_initial(inode)) + { + update = Graph_Matrix_Strides_initial(inode); + } }, max_stride); - - //std::cout << "THE MAX STRIDE" << max_stride << std::endl; - //allocate array used in the repeat removal process - current_row_nodes_scanned = CArrayKokkos(max_stride, "current_row_nodes_scanned"); - - //allocate sparse graph with node repeats - RaggedRightArrayKokkos Repeat_Graph_Matrix(Graph_Matrix_Strides_initial); - RaggedRightArrayofVectorsKokkos Element_local_indices(Graph_Matrix_Strides_initial,num_dim); - - //Fill the initial Graph with repeats - if(num_dim == 2){ - for (int ielem = 0; ielem < rnum_elem; ielem++){ - element_select->choose_2Delem_type(Element_Types(ielem), elem2D); - nodes_per_element = elem2D->num_nodes(); - for (int lnode = 0; lnode < nodes_per_element; lnode++){ - local_node_index = nodes_in_elem(ielem, lnode); - if(local_node_index < nlocal_nodes){ - for (int jnode = 0; jnode < nodes_per_element; jnode++){ - current_column_index = Graph_Fill(local_node_index)+jnode; - Repeat_Graph_Matrix(local_node_index, current_column_index) = nodes_in_elem(ielem,jnode); - - //fill inverse map - Element_local_indices(local_node_index,current_column_index,0) = ielem; - Element_local_indices(local_node_index,current_column_index,1) = lnode; - Element_local_indices(local_node_index,current_column_index,2) = jnode; - - //fill forward map - Global_Gradient_Matrix_Assembly_Map(ielem,lnode,jnode) = current_column_index; - } - Graph_Fill(local_node_index) += nodes_per_element; + + // std::cout << "THE MAX STRIDE" << max_stride << std::endl; + // allocate array used in the repeat removal process + current_row_nodes_scanned = CArrayKokkos(max_stride, "current_row_nodes_scanned"); + + // allocate sparse graph with node repeats + RaggedRightArrayKokkos Repeat_Graph_Matrix(Graph_Matrix_Strides_initial); + RaggedRightArrayofVectorsKokkos Element_local_indices(Graph_Matrix_Strides_initial, num_dim); + + // Fill the initial Graph with repeats + if (num_dim == 2) + { + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + element_select->choose_2Delem_type(Element_Types(ielem), elem2D); + nodes_per_element = elem2D->num_nodes(); + for (int lnode = 0; lnode < nodes_per_element; lnode++) + { + local_node_index = nodes_in_elem(ielem, lnode); + if (local_node_index < nlocal_nodes) + { + for (int jnode = 0; jnode < nodes_per_element; jnode++) + { + current_column_index = Graph_Fill(local_node_index) + jnode; + Repeat_Graph_Matrix(local_node_index, current_column_index) = nodes_in_elem(ielem, jnode); + + // fill inverse map + Element_local_indices(local_node_index, current_column_index, 0) = ielem; + Element_local_indices(local_node_index, current_column_index, 1) = lnode; + Element_local_indices(local_node_index, current_column_index, 2) = jnode; + + // fill forward map + Global_Gradient_Matrix_Assembly_Map(ielem, lnode, jnode) = current_column_index; + } + Graph_Fill(local_node_index) += nodes_per_element; + } + } } - } } - } - - if(num_dim == 3){ - for (int ielem = 0; ielem < rnum_elem; ielem++){ - element_select->choose_3Delem_type(Element_Types(ielem), elem); - nodes_per_element = elem->num_nodes(); - for (int lnode = 0; lnode < nodes_per_element; lnode++){ - local_node_index = nodes_in_elem(ielem, lnode); - if(local_node_index < nlocal_nodes){ - for (int jnode = 0; jnode < nodes_per_element; jnode++){ - current_column_index = Graph_Fill(local_node_index)+jnode; - Repeat_Graph_Matrix(local_node_index, current_column_index) = nodes_in_elem(ielem,jnode); - - //fill inverse map - Element_local_indices(local_node_index,current_column_index,0) = ielem; - Element_local_indices(local_node_index,current_column_index,1) = lnode; - Element_local_indices(local_node_index,current_column_index,2) = jnode; - - //fill forward map - Global_Gradient_Matrix_Assembly_Map(ielem,lnode,jnode) = current_column_index; - } - Graph_Fill(local_node_index) += nodes_per_element; + + if (num_dim == 3) + { + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + element_select->choose_3Delem_type(Element_Types(ielem), elem); + nodes_per_element = elem->num_nodes(); + for (int lnode = 0; lnode < nodes_per_element; lnode++) + { + local_node_index = nodes_in_elem(ielem, lnode); + if (local_node_index < nlocal_nodes) + { + for (int jnode = 0; jnode < nodes_per_element; jnode++) + { + current_column_index = Graph_Fill(local_node_index) + jnode; + Repeat_Graph_Matrix(local_node_index, current_column_index) = nodes_in_elem(ielem, jnode); + + // fill inverse map + Element_local_indices(local_node_index, current_column_index, 0) = ielem; + Element_local_indices(local_node_index, current_column_index, 1) = lnode; + Element_local_indices(local_node_index, current_column_index, 2) = jnode; + + // fill forward map + Global_Gradient_Matrix_Assembly_Map(ielem, lnode, jnode) = current_column_index; + } + Graph_Fill(local_node_index) += nodes_per_element; + } + } } - } } - } - - //debug statement - //std::cout << "started run" << std::endl; - //std::cout << "Graph Matrix Strides Repeat on task " << myrank << std::endl; - //for (int inode = 0; inode < nlocal_nodes; inode++) - //std::cout << Graph_Matrix_Strides(inode) << std::endl; - RUN_CLASS({ - //remove repeats from the inital graph setup - int current_node; - //remove repeats from the inital graph setup - int current_element_index; - int element_row_index; - int element_column_index; - int current_stride; - int current_row_n_nodes_scanned; - for (int inode = 0; inode < nlocal_nodes; inode++){ - current_row_n_nodes_scanned = 0; - for (int istride = 0; istride < Graph_Matrix_Strides(inode); istride++){ - //convert global index in graph to its local index for the flagging array - current_node = Repeat_Graph_Matrix(inode,istride); - //debug - //if(current_node==-1) - //std::cout << "Graph Matrix node access on task " << myrank << std::endl; - //std::cout << Repeat_Graph_Matrix(inode,istride) << std::endl; - if(node_indices_used(current_node)){ - //set global assembly map index to the location in the graph matrix where this global node was first found - current_element_index = Element_local_indices(inode,istride,0); - element_row_index = Element_local_indices(inode,istride,1); - element_column_index = Element_local_indices(inode,istride,2); - Global_Gradient_Matrix_Assembly_Map(current_element_index,element_row_index, element_column_index) - = column_index(current_node); - - - //swap current node with the end of the current row and shorten the stride of the row - //first swap information about the inverse and forward maps - - current_stride = Graph_Matrix_Strides(inode); - if(istride!=current_stride-1){ - Element_local_indices(inode,istride,0) = Element_local_indices(inode,current_stride-1,0); - Element_local_indices(inode,istride,1) = Element_local_indices(inode,current_stride-1,1); - Element_local_indices(inode,istride,2) = Element_local_indices(inode,current_stride-1,2); - current_element_index = Element_local_indices(inode,istride,0); - element_row_index = Element_local_indices(inode,istride,1); - element_column_index = Element_local_indices(inode,istride,2); - - Global_Gradient_Matrix_Assembly_Map(current_element_index,element_row_index, element_column_index) - = istride; - - //now that the element map information has been copied, copy the global node index and delete the last index - - Repeat_Graph_Matrix(inode,istride) = Repeat_Graph_Matrix(inode,current_stride-1); - } - istride--; - Graph_Matrix_Strides(inode)--; + + // debug statement + // std::cout << "started run" << std::endl; + // std::cout << "Graph Matrix Strides Repeat on task " << myrank << std::endl; + // for (int inode = 0; inode < nlocal_nodes; inode++) + // std::cout << Graph_Matrix_Strides(inode) << std::endl; + RUN_CLASS({ + // remove repeats from the inital graph setup + int current_node; + // remove repeats from the inital graph setup + int current_element_index; + int element_row_index; + int element_column_index; + int current_stride; + int current_row_n_nodes_scanned; + for (int inode = 0; inode < nlocal_nodes; inode++) + { + current_row_n_nodes_scanned = 0; + for (int istride = 0; istride < Graph_Matrix_Strides(inode); istride++) + { + // convert global index in graph to its local index for the flagging array + current_node = Repeat_Graph_Matrix(inode, istride); + // debug + // if(current_node==-1) + // std::cout << "Graph Matrix node access on task " << myrank << std::endl; + // std::cout << Repeat_Graph_Matrix(inode,istride) << std::endl; + if (node_indices_used(current_node)) + { + // set global assembly map index to the location in the graph matrix where this global node was first found + current_element_index = Element_local_indices(inode, istride, 0); + element_row_index = Element_local_indices(inode, istride, 1); + element_column_index = Element_local_indices(inode, istride, 2); + Global_Gradient_Matrix_Assembly_Map(current_element_index, element_row_index, element_column_index) + = column_index(current_node); + + // swap current node with the end of the current row and shorten the stride of the row + // first swap information about the inverse and forward maps + + current_stride = Graph_Matrix_Strides(inode); + if (istride != current_stride - 1) + { + Element_local_indices(inode, istride, 0) = Element_local_indices(inode, current_stride - 1, 0); + Element_local_indices(inode, istride, 1) = Element_local_indices(inode, current_stride - 1, 1); + Element_local_indices(inode, istride, 2) = Element_local_indices(inode, current_stride - 1, 2); + current_element_index = Element_local_indices(inode, istride, 0); + element_row_index = Element_local_indices(inode, istride, 1); + element_column_index = Element_local_indices(inode, istride, 2); + + Global_Gradient_Matrix_Assembly_Map(current_element_index, element_row_index, element_column_index) + = istride; + + // now that the element map information has been copied, copy the global node index and delete the last index + + Repeat_Graph_Matrix(inode, istride) = Repeat_Graph_Matrix(inode, current_stride - 1); + } + istride--; + Graph_Matrix_Strides(inode)--; + } + else + { + /*this node hasn't shown up in the row before; add it to the list of nodes + that have been scanned uniquely. Use this list to reset the flag array + afterwards without having to loop over all the nodes in the system*/ + node_indices_used(current_node) = 1; + column_index(current_node) = istride; + current_row_nodes_scanned(current_row_n_nodes_scanned) = current_node; + current_row_n_nodes_scanned++; + } + } + // reset nodes used list for the next row of the sparse list + for (int node_reset = 0; node_reset < current_row_n_nodes_scanned; node_reset++) + { + node_indices_used(current_row_nodes_scanned(node_reset)) = 0; + } } - else{ - /*this node hasn't shown up in the row before; add it to the list of nodes - that have been scanned uniquely. Use this list to reset the flag array - afterwards without having to loop over all the nodes in the system*/ - node_indices_used(current_node) = 1; - column_index(current_node) = istride; - current_row_nodes_scanned(current_row_n_nodes_scanned) = current_node; - current_row_n_nodes_scanned++; + }); + Kokkos::fence(); + + Graph_Matrix_Strides.update_host(); + // copy reduced content to non_repeat storage + Graph_Matrix = RaggedRightArrayKokkos(Graph_Matrix_Strides); + + FOR_ALL_CLASS(inode, 0, nlocal_nodes, { + for (int istride = 0; istride < Graph_Matrix_Strides(inode); istride++) + { + Graph_Matrix(inode, istride) = Repeat_Graph_Matrix(inode, istride); } - } - //reset nodes used list for the next row of the sparse list - for(int node_reset = 0; node_reset < current_row_n_nodes_scanned; node_reset++) - node_indices_used(current_row_nodes_scanned(node_reset)) = 0; + }); // end parallel for - } - }); - Kokkos::fence(); - - Graph_Matrix_Strides.update_host(); - //copy reduced content to non_repeat storage - Graph_Matrix = RaggedRightArrayKokkos(Graph_Matrix_Strides); - - FOR_ALL_CLASS(inode, 0, nlocal_nodes, { - for(int istride = 0; istride < Graph_Matrix_Strides(inode); istride++){ - Graph_Matrix(inode,istride) = Repeat_Graph_Matrix(inode,istride); - } + // deallocate repeat matrix + + /*At this stage the sparse graph should have unique global indices on each row. + The constructed Assembly map (to the global sparse matrix) + is used to loop over each element's local stiffness matrix in the assembly process.*/ + + // expand strides for stiffness matrix by multipling by dim + FOR_ALL_CLASS(idof, 0, num_dim * nlocal_nodes, { + Gradient_Matrix_Strides(idof) = num_dim * Graph_Matrix_Strides(idof / num_dim); }); // end parallel for - //deallocate repeat matrix - - /*At this stage the sparse graph should have unique global indices on each row. - The constructed Assembly map (to the global sparse matrix) - is used to loop over each element's local stiffness matrix in the assembly process.*/ - - //expand strides for stiffness matrix by multipling by dim - FOR_ALL_CLASS(idof, 0, num_dim*nlocal_nodes, { - Gradient_Matrix_Strides(idof) = num_dim*Graph_Matrix_Strides(idof/num_dim); + Stiffness_Matrix = Force_Gradient_Positions = RaggedRightArrayKokkos(Gradient_Matrix_Strides); + Force_Gradient_Velocities = RaggedRightArrayKokkos(Gradient_Matrix_Strides); + DOF_Graph_Matrix = RaggedRightArrayKokkos(Gradient_Matrix_Strides); + + // set stiffness Matrix Graph + // debug print + // std::cout << "DOF GRAPH MATRIX ENTRIES ON TASK " << myrank << std::endl; + FOR_ALL_CLASS(idof, 0, num_dim * nlocal_nodes, { + for (int istride = 0; istride < Gradient_Matrix_Strides(idof); istride++) + { + DOF_Graph_Matrix(idof, istride) = Graph_Matrix(idof / num_dim, istride / num_dim) * num_dim + istride % num_dim; + } }); // end parallel for - Stiffness_Matrix = Force_Gradient_Positions = RaggedRightArrayKokkos(Gradient_Matrix_Strides); - Force_Gradient_Velocities = RaggedRightArrayKokkos(Gradient_Matrix_Strides); - DOF_Graph_Matrix = RaggedRightArrayKokkos (Gradient_Matrix_Strides); + Stiffness_Matrix_Strides = Gradient_Matrix_Strides; + + /* + //construct distributed gradient matrix from local kokkos data + //build column map for the global gradient matrix + Teuchos::RCP > colmap; + const Teuchos::RCP > dommap = local_dof_map; + + Tpetra::Details::makeColMap(colmap,dommap,DOF_Graph_Matrix.get_kokkos_view(), nullptr); + + size_t nnz = DOF_Graph_Matrix.size(); + + //debug print + //std::cout << "DOF GRAPH SIZE ON RANK " << myrank << " IS " << nnz << std::endl; - //set stiffness Matrix Graph - //debug print - //std::cout << "DOF GRAPH MATRIX ENTRIES ON TASK " << myrank << std::endl; - FOR_ALL_CLASS(idof, 0, num_dim*nlocal_nodes, { - for (int istride = 0; istride < Gradient_Matrix_Strides(idof); istride++){ - DOF_Graph_Matrix(idof,istride) = Graph_Matrix(idof/num_dim,istride/num_dim)*num_dim + istride%num_dim; + //local indices in the graph using the constructed column map + CArrayKokkos gradient_local_indices(nnz, "gradient_local_indices"); + + //row offsets with compatible template arguments + Kokkos::View row_offsets = DOF_Graph_Matrix.start_index_; + row_pointers row_offsets_pass("row_offsets", nlocal_nodes*num_dim+1); + for(int ipass = 0; ipass < nlocal_nodes*num_dim + 1; ipass++){ + row_offsets_pass(ipass) = row_offsets(ipass); } - }); // end parallel for - Stiffness_Matrix_Strides = Gradient_Matrix_Strides; - - /* - //construct distributed gradient matrix from local kokkos data - //build column map for the global gradient matrix - Teuchos::RCP > colmap; - const Teuchos::RCP > dommap = local_dof_map; - - Tpetra::Details::makeColMap(colmap,dommap,DOF_Graph_Matrix.get_kokkos_view(), nullptr); - - size_t nnz = DOF_Graph_Matrix.size(); - - //debug print - //std::cout << "DOF GRAPH SIZE ON RANK " << myrank << " IS " << nnz << std::endl; - - //local indices in the graph using the constructed column map - CArrayKokkos gradient_local_indices(nnz, "gradient_local_indices"); - - //row offsets with compatible template arguments - Kokkos::View row_offsets = DOF_Graph_Matrix.start_index_; - row_pointers row_offsets_pass("row_offsets", nlocal_nodes*num_dim+1); - for(int ipass = 0; ipass < nlocal_nodes*num_dim + 1; ipass++){ - row_offsets_pass(ipass) = row_offsets(ipass); - } - - size_t entrycount = 0; - for(int irow = 0; irow < nlocal_nodes*num_dim; irow++){ - for(int istride = 0; istride < Gradient_Matrix_Strides(irow); istride++){ - gradient_local_indices(entrycount) = colmap->getLocalElement(DOF_Graph_Matrix(irow,istride)); - entrycount++; + size_t entrycount = 0; + for(int irow = 0; irow < nlocal_nodes*num_dim; irow++){ + for(int istride = 0; istride < Gradient_Matrix_Strides(irow); istride++){ + gradient_local_indices(entrycount) = colmap->getLocalElement(DOF_Graph_Matrix(irow,istride)); + entrycount++; + } } - } - - - //sort values and indices - Tpetra::Import_Util::sortCrsEntries(row_offsets_pass, gradient_local_indices.get_kokkos_view(), Force_Gradient_Positions.get_kokkos_view()); - Tpetra::Import_Util::sortCrsEntries(row_offsets_pass, gradient_local_indices.get_kokkos_view(), Force_Gradient_Velocities.get_kokkos_view()); - - //Teuchos::RCP crs_matrix_params = Teuchos::rcp(new Teuchos::ParameterList("crsmatrix")); - //crs_matrix_params->set("sorted", false); - distributed_force_gradient_positions = Teuchos::rcp(new MAT(local_dof_map, colmap, row_offsets_pass, gradient_local_indices.get_kokkos_view(), Force_Gradient_Positions.get_kokkos_view())); - distributed_force_gradient_positions->fillComplete(); - distributed_force_gradient_velocities = Teuchos::rcp(new MAT(local_dof_map, colmap, row_offsets_pass, gradient_local_indices.get_kokkos_view(), Force_Gradient_Velocities.get_kokkos_view())); - distributed_force_gradient_velocities->fillComplete(); - */ - //distributed_force_gradient_positions->describe(*fos,Teuchos::VERB_EXTREME); - //distributed_force_gradient_velocities->describe(*fos,Teuchos::VERB_EXTREME); + + + //sort values and indices + Tpetra::Import_Util::sortCrsEntries(row_offsets_pass, gradient_local_indices.get_kokkos_view(), Force_Gradient_Positions.get_kokkos_view()); + Tpetra::Import_Util::sortCrsEntries(row_offsets_pass, gradient_local_indices.get_kokkos_view(), Force_Gradient_Velocities.get_kokkos_view()); + + //Teuchos::RCP crs_matrix_params = Teuchos::rcp(new Teuchos::ParameterList("crsmatrix")); + //crs_matrix_params->set("sorted", false); + distributed_force_gradient_positions = Teuchos::rcp(new MAT(local_dof_map, colmap, row_offsets_pass, gradient_local_indices.get_kokkos_view(), Force_Gradient_Positions.get_kokkos_view())); + distributed_force_gradient_positions->fillComplete(); + distributed_force_gradient_velocities = Teuchos::rcp(new MAT(local_dof_map, colmap, row_offsets_pass, gradient_local_indices.get_kokkos_view(), Force_Gradient_Velocities.get_kokkos_view())); + distributed_force_gradient_velocities->fillComplete(); + */ + // distributed_force_gradient_positions->describe(*fos,Teuchos::VERB_EXTREME); + // distributed_force_gradient_velocities->describe(*fos,Teuchos::VERB_EXTREME); } /* ---------------------------------------------------------------------- Enforce boundary conditions on the adjoint vectors ------------------------------------------------------------------------- */ -void FEA_Module_Dynamic_Elasticity::boundary_adjoint(const mesh_t &mesh, - const DCArrayKokkos &boundary, - vec_array &node_adjoint, - vec_array &node_phi_adjoint){ - - //error and debug flag - //DCArrayKokkos print_flag(1, "print_flag"); - //print_flag.host(0) = false; - //print_flag.update_device(); - - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - int num_dims = simparam->num_dims; +void FEA_Module_Dynamic_Elasticity::boundary_adjoint(const mesh_t& mesh, + const DCArrayKokkos& boundary, + vec_array& node_adjoint, + vec_array& node_phi_adjoint) +{ + // error and debug flag + // DCArrayKokkos print_flag(1, "print_flag"); + // print_flag.host(0) = false; + // print_flag.update_device(); + + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + int num_dims = simparam->num_dims; // Loop over boundary sets - for (size_t bdy_set=0; bdy_set fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - //if(myrank==0) - //*fos << "Density data :" << std::endl; - //node_densities_distributed->describe(*fos,Teuchos::VERB_EXTREME); - //*fos << std::endl; - //std::fflush(stdout); - - //communicate design densities - //create import object using local node indices map and all indices map - //Tpetra::Import importer(map, all_node_map); - - //comms to get ghosts - (*adjoint_vector_data)[cycle]->doImport(*adjoint_vector_distributed, *importer, Tpetra::INSERT); - (*phi_adjoint_vector_data)[cycle]->doImport(*phi_adjoint_vector_distributed, *importer, Tpetra::INSERT); - //all_node_map->describe(*fos,Teuchos::VERB_EXTREME); - //all_node_velocities_distributed->describe(*fos,Teuchos::VERB_EXTREME); - - //update_count++; - //if(update_count==1){ - //MPI_Barrier(world); - //MPI_Abort(world,4); - //} +void FEA_Module_Dynamic_Elasticity::comm_adjoint_vectors(int cycle) +{ + // debug print of design vector + // std::ostream &out = std::cout; + // Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + // if(myrank==0) + // *fos << "Density data :" << std::endl; + // node_densities_distributed->describe(*fos,Teuchos::VERB_EXTREME); + // *fos << std::endl; + // std::fflush(stdout); + + // communicate design densities + // create import object using local node indices map and all indices map + // Tpetra::Import importer(map, all_node_map); + + // comms to get ghosts + (*adjoint_vector_data)[cycle]->doImport(*adjoint_vector_distributed, *importer, Tpetra::INSERT); + (*phi_adjoint_vector_data)[cycle]->doImport(*phi_adjoint_vector_distributed, *importer, Tpetra::INSERT); + // all_node_map->describe(*fos,Teuchos::VERB_EXTREME); + // all_node_velocities_distributed->describe(*fos,Teuchos::VERB_EXTREME); + + // update_count++; + // if(update_count==1){ + // MPI_Barrier(world); + // MPI_Abort(world,4); + // } } diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/force_elastic.cpp b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/force_elastic.cpp index 20012dd28..4ef144028 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/force_elastic.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/force_elastic.cpp @@ -9,690 +9,796 @@ // ----------------------------------------------------------------------------- // This function calculates the corner forces and the evolves stress (hypo) -//------------------------------------------------------------------------------ -void FEA_Module_Dynamic_Elasticity::get_force_elastic(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &node_mass, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &elem_mat_id, - DViewCArrayKokkos &corner_force, - const double rk_alpha, - const size_t cycle - ){ - - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - const size_t num_dim = mesh.num_dims; - const real_t damping_constant = module_params->damping_constant; - const_vec_array all_initial_node_coords = all_initial_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); +// ------------------------------------------------------------------------------ +void FEA_Module_Dynamic_Elasticity::get_force_elastic(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_mass, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + DViewCArrayKokkos& corner_force, + const double rk_alpha, + const size_t cycle + ) +{ + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + const size_t num_dim = mesh.num_dims; + const real_t damping_constant = module_params->damping_constant; + const_vec_array all_initial_node_coords = all_initial_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); // walk over the nodes to update the velocity FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - size_t dof_id; double node_force[3]; - for (size_t dim = 0; dim < num_dim; dim++){ + for (size_t dim = 0; dim < num_dim; dim++) + { node_force[dim] = 0.0; } // end for dim - + // loop over all corners around the node and calculate the nodal force - for (size_t corner_lid=0; corner_lid &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &node_mass, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &elem_mat_id, - DViewCArrayKokkos &corner_force, - const double rk_alpha, - const size_t cycle - ){ - - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - const size_t num_dim = mesh.num_dims; - const_vec_array all_initial_node_coords = all_initial_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - const size_t num_lcs = module_params->loading.size(); - - const DCArrayKokkos mat_fill = simparam->mat_fill; - const DCArrayKokkos loading = module_params->loading; - - //debug check - //std::cout << "NUMBER OF LOADING CONDITIONS: " << num_lcs << std::endl; +// ------------------------------------------------------------------------------ +void FEA_Module_Dynamic_Elasticity::applied_forces(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_mass, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + DViewCArrayKokkos& corner_force, + const double rk_alpha, + const size_t cycle + ) +{ + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + const size_t num_dim = mesh.num_dims; + const_vec_array all_initial_node_coords = all_initial_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + const size_t num_lcs = module_params->loading.size(); + + const DCArrayKokkos mat_fill = simparam->mat_fill; + const DCArrayKokkos loading = module_params->loading; + + // debug check + // std::cout << "NUMBER OF LOADING CONDITIONS: " << num_lcs << std::endl; // walk over the nodes to update the velocity - //FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (size_t node_gid = 0; node_gid <= nlocal_nodes; node_gid++) { + // FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + for (size_t node_gid = 0; node_gid <= nlocal_nodes; node_gid++) + { double current_node_coords[3]; size_t dof_id; double node_force[3]; double applied_force[3]; double radius; - for (size_t dim = 0; dim < num_dim; dim++){ + for (size_t dim = 0; dim < num_dim; dim++) + { node_force[dim] = 0.0; current_node_coords[dim] = all_initial_node_coords(node_gid, dim); } // end for dim - radius = sqrt(current_node_coords[0]*current_node_coords[0]+current_node_coords[1]*current_node_coords[1]+current_node_coords[2]*current_node_coords[2]); - for(size_t ilc=0; ilc < num_lcs; ilc++){ - //debug check - //std::cout << "LOADING CONDITION VOLUME TYPE: " << to_string(loading(ilc).volume) << std::endl; - - bool fill_this = loading(ilc).contains(current_node_coords); - if(fill_this){ - // loop over all corners around the node and calculate the nodal force - for (size_t corner_lid=0; corner_lidnum_dims; - int nodes_per_elem; - int current_row_n_nodes_scanned; - int local_dof_index, local_node_index, current_row, current_column; - int max_stride = 0; - - CArrayKokkos Local_Stiffness_Matrix(num_dim*max_nodes_per_element,num_dim*max_nodes_per_element); - - //initialize stiffness Matrix entries to 0 - //debug print - //std::cout << "DOF GRAPH MATRIX ENTRIES ON TASK " << myrank << std::endl; - for (int idof = 0; idof < num_dim*nlocal_nodes; idof++){ - for (int istride = 0; istride < Stiffness_Matrix_Strides(idof); istride++){ - Stiffness_Matrix(idof,istride) = 0; - //debug print - //std::cout << "{" <num_dims; + int nodes_per_elem; + int current_row_n_nodes_scanned; + int local_dof_index, local_node_index, current_row, current_column; + int max_stride = 0; + + CArrayKokkos Local_Stiffness_Matrix(num_dim * max_nodes_per_element, num_dim * max_nodes_per_element); + + // initialize stiffness Matrix entries to 0 + // debug print + // std::cout << "DOF GRAPH MATRIX ENTRIES ON TASK " << myrank << std::endl; + for (int idof = 0; idof < num_dim * nlocal_nodes; idof++) + { + for (int istride = 0; istride < Stiffness_Matrix_Strides(idof); istride++) + { + Stiffness_Matrix(idof, istride) = 0; + // debug print + // std::cout << "{" <choose_2Delem_type(Element_Types(ielem), elem2D); - nodes_per_elem = elem2D->num_nodes(); - //construct local stiffness matrix for this element - local_matrix_multiply(ielem, Local_Stiffness_Matrix); - //assign entries of this local matrix to the sparse global matrix storage; - for (int inode = 0; inode < nodes_per_elem; inode++){ - //see if this node is local - local_node_index = nodes_in_elem.host(ielem,inode); - if(!map->isNodeLocalElement(local_node_index)) continue; - //set dof row start index - current_row = num_dim*local_node_index; - for(int jnode = 0; jnode < nodes_per_elem; jnode++){ - - current_column = num_dim*Global_Stiffness_Matrix_Assembly_Map(ielem,inode,jnode); - for (int idim = 0; idim < num_dim; idim++){ - for (int jdim = 0; jdim < num_dim; jdim++){ - - //debug print - //if(current_row + idim==15&¤t_column + jdim==4) - //std::cout << " Local stiffness matrix contribution for row " << current_row + idim +1 << " and column " << current_column + jdim + 1 << " : " << - //Local_Stiffness_Matrix(num_dim*inode + idim,num_dim*jnode + jdim) << " from " << ielem +1 << " i: " << num_dim*inode+idim+1 << " j: " << num_dim*jnode + jdim +1 << std::endl << std::endl; - //end debug - - Stiffness_Matrix(current_row + idim, current_column + jdim) += Local_Stiffness_Matrix(num_dim*inode + idim,num_dim*jnode + jdim); - } + + // reset unsorted DOF Graph corresponding to assembly mapped values + // debug print + for (int idof = 0; idof < num_dim * nlocal_nodes; idof++) + { + for (int istride = 0; istride < Stiffness_Matrix_Strides(idof); istride++) + { + DOF_Graph_Matrix(idof, istride) = Graph_Matrix(idof / num_dim, istride / num_dim) * num_dim + istride % num_dim; + // debug print + // std::cout << "{" <choose_3Delem_type(Element_Types(ielem), elem); - nodes_per_elem = elem->num_nodes(); - //construct local stiffness matrix for this element - local_matrix_multiply(ielem, Local_Stiffness_Matrix); - //assign entries of this local matrix to the sparse global matrix storage; - for (int inode = 0; inode < nodes_per_elem; inode++){ - //see if this node is local - local_node_index = nodes_in_elem.host(ielem,inode); - if(!map->isNodeLocalElement(local_node_index)) continue; - //set dof row start index - current_row = num_dim*local_node_index; - for(int jnode = 0; jnode < nodes_per_elem; jnode++){ - - current_column = num_dim*Global_Stiffness_Matrix_Assembly_Map(ielem,inode,jnode); - for (int idim = 0; idim < num_dim; idim++){ - for (int jdim = 0; jdim < num_dim; jdim++){ - - //debug print - //if(current_row + idim==15&¤t_column + jdim==4) - //std::cout << " Local stiffness matrix contribution for row " << current_row + idim +1 << " and column " << current_column + jdim + 1 << " : " << - //Local_Stiffness_Matrix(num_dim*inode + idim,num_dim*jnode + jdim) << " from " << ielem +1 << " i: " << num_dim*inode+idim+1 << " j: " << num_dim*jnode + jdim +1 << std::endl << std::endl; - //end debug - - Stiffness_Matrix(current_row + idim, current_column + jdim) += Local_Stiffness_Matrix(num_dim*inode + idim,num_dim*jnode + jdim); - } + + // assemble the global stiffness matrix + if (num_dim == 2) + { + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + element_select->choose_2Delem_type(Element_Types(ielem), elem2D); + nodes_per_elem = elem2D->num_nodes(); + // construct local stiffness matrix for this element + local_matrix_multiply(ielem, Local_Stiffness_Matrix); + // assign entries of this local matrix to the sparse global matrix storage; + for (int inode = 0; inode < nodes_per_elem; inode++) + { + // see if this node is local + local_node_index = nodes_in_elem.host(ielem, inode); + if (!map->isNodeLocalElement(local_node_index)) + { + continue; + } + // set dof row start index + current_row = num_dim * local_node_index; + for (int jnode = 0; jnode < nodes_per_elem; jnode++) + { + current_column = num_dim * Global_Stiffness_Matrix_Assembly_Map(ielem, inode, jnode); + for (int idim = 0; idim < num_dim; idim++) + { + for (int jdim = 0; jdim < num_dim; jdim++) + { + // debug print + // if(current_row + idim==15&¤t_column + jdim==4) + // std::cout << " Local stiffness matrix contribution for row " << current_row + idim +1 << " and column " << current_column + jdim + 1 << " : " << + // Local_Stiffness_Matrix(num_dim*inode + idim,num_dim*jnode + jdim) << " from " << ielem +1 << " i: " << num_dim*inode+idim+1 << " j: " << num_dim*jnode + jdim +1 << std::endl << std::endl; + // end debug + + Stiffness_Matrix(current_row + idim, current_column + jdim) += Local_Stiffness_Matrix(num_dim * inode + idim, num_dim * jnode + jdim); + } + } + } + } } - } } - } + if (num_dim == 3) + { + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + element_select->choose_3Delem_type(Element_Types(ielem), elem); + nodes_per_elem = elem->num_nodes(); + // construct local stiffness matrix for this element + local_matrix_multiply(ielem, Local_Stiffness_Matrix); + // assign entries of this local matrix to the sparse global matrix storage; + for (int inode = 0; inode < nodes_per_elem; inode++) + { + // see if this node is local + local_node_index = nodes_in_elem.host(ielem, inode); + if (!map->isNodeLocalElement(local_node_index)) + { + continue; + } + // set dof row start index + current_row = num_dim * local_node_index; + for (int jnode = 0; jnode < nodes_per_elem; jnode++) + { + current_column = num_dim * Global_Stiffness_Matrix_Assembly_Map(ielem, inode, jnode); + for (int idim = 0; idim < num_dim; idim++) + { + for (int jdim = 0; jdim < num_dim; jdim++) + { + // debug print + // if(current_row + idim==15&¤t_column + jdim==4) + // std::cout << " Local stiffness matrix contribution for row " << current_row + idim +1 << " and column " << current_column + jdim + 1 << " : " << + // Local_Stiffness_Matrix(num_dim*inode + idim,num_dim*jnode + jdim) << " from " << ielem +1 << " i: " << num_dim*inode+idim+1 << " j: " << num_dim*jnode + jdim +1 << std::endl << std::endl; + // end debug + + Stiffness_Matrix(current_row + idim, current_column + jdim) += Local_Stiffness_Matrix(num_dim * inode + idim, num_dim * jnode + jdim); + } + } + } + } + } + } - //debug print of A matrix - //*fos << "Global Stiffness Matrix :" << std::endl; - //Global_Stiffness_Matrix->describe(*fos,Teuchos::VERB_EXTREME); - //*fos << std::endl; + // debug print of A matrix + // *fos << "Global Stiffness Matrix :" << std::endl; + // Global_Stiffness_Matrix->describe(*fos,Teuchos::VERB_EXTREME); + // *fos << std::endl; - //filter small negative numbers (that should have been 0 from cancellation) from floating point error - /* - for (int idof = 0; idof < num_dim*nlocal_nodes; idof++){ - for (int istride = 0; istride < Stiffness_Matrix_Strides(idof); istride++){ - if(Stiffness_Matrix(idof,istride)<0.000000001*simparam->Elastic_Modulus*density_epsilon||Stiffness_Matrix(idof,istride)>-0.000000001*simparam->Elastic_Modulus*density_epsilon) - Stiffness_Matrix(idof,istride) = 0; + // filter small negative numbers (that should have been 0 from cancellation) from floating point error + /* + for (int idof = 0; idof < num_dim*nlocal_nodes; idof++){ + for (int istride = 0; istride < Stiffness_Matrix_Strides(idof); istride++){ + if(Stiffness_Matrix(idof,istride)<0.000000001*simparam->Elastic_Modulus*density_epsilon||Stiffness_Matrix(idof,istride)>-0.000000001*simparam->Elastic_Modulus*density_epsilon) + Stiffness_Matrix(idof,istride) = 0; + //debug print + //std::cout << "{" <get_unit_scaling(); - real_t penalty_product = 1; - real_t density_epsilon = simparam->optimization_options.density_epsilon; - if(density < 0) density = 0; - for(int i = 0; i < penalty_power; i++) - penalty_product *= density; - //relationship between density and stiffness - Element_Modulus = (density_epsilon + (1 - density_epsilon)*penalty_product)*module_params->material.elastic_modulus/unit_scaling/unit_scaling; - //Element_Modulus = density*simparam->Elastic_Modulus/unit_scaling/unit_scaling; - Poisson_Ratio = module_params->material.poisson_ratio; +void FEA_Module_Dynamic_Elasticity::Element_Material_Properties(size_t ielem, real_t& Element_Modulus, real_t& Poisson_Ratio, real_t density) +{ + real_t unit_scaling = simparam->get_unit_scaling(); + real_t penalty_product = 1; + real_t density_epsilon = simparam->optimization_options.density_epsilon; + if (density < 0) + { + density = 0; + } + for (int i = 0; i < penalty_power; i++) + { + penalty_product *= density; + } + // relationship between density and stiffness + Element_Modulus = (density_epsilon + (1 - density_epsilon) * penalty_product) * module_params->material.elastic_modulus / unit_scaling / unit_scaling; + // Element_Modulus = density*simparam->Elastic_Modulus/unit_scaling/unit_scaling; + Poisson_Ratio = module_params->material.poisson_ratio; } /* ---------------------------------------------------------------------- Retrieve derivative of material properties with respect to local density ------------------------------------------------------------------------- */ -void FEA_Module_Dynamic_Elasticity::Gradient_Element_Material_Properties(size_t ielem, real_t &Element_Modulus_Derivative, real_t &Poisson_Ratio, real_t density){ - real_t unit_scaling = simparam->get_unit_scaling(); - real_t penalty_product = 1; - real_t density_epsilon = simparam->optimization_options.density_epsilon; - Element_Modulus_Derivative = 0; - if(density < 0) density = 0; - for(int i = 0; i < penalty_power - 1; i++) - penalty_product *= density; - //relationship between density and stiffness - Element_Modulus_Derivative = penalty_power*(1 - density_epsilon)*penalty_product*module_params->material.elastic_modulus/unit_scaling/unit_scaling; - //Element_Modulus_Derivative = simparam->Elastic_Modulus/unit_scaling/unit_scaling; - Poisson_Ratio = module_params->material.poisson_ratio; +void FEA_Module_Dynamic_Elasticity::Gradient_Element_Material_Properties(size_t ielem, real_t& Element_Modulus_Derivative, real_t& Poisson_Ratio, real_t density) +{ + real_t unit_scaling = simparam->get_unit_scaling(); + real_t penalty_product = 1; + real_t density_epsilon = simparam->optimization_options.density_epsilon; + Element_Modulus_Derivative = 0; + if (density < 0) + { + density = 0; + } + for (int i = 0; i < penalty_power - 1; i++) + { + penalty_product *= density; + } + // relationship between density and stiffness + Element_Modulus_Derivative = penalty_power * (1 - density_epsilon) * penalty_product * module_params->material.elastic_modulus / unit_scaling / unit_scaling; + // Element_Modulus_Derivative = simparam->Elastic_Modulus/unit_scaling/unit_scaling; + Poisson_Ratio = module_params->material.poisson_ratio; } /* ---------------------------------------------------------------------- Construct the local stiffness matrix ------------------------------------------------------------------------- */ -void FEA_Module_Dynamic_Elasticity::local_matrix_multiply(int ielem, CArrayKokkos &Local_Matrix){ - //local variable for host view in the dual view - const_host_vec_array all_initial_node_coords = all_initial_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_host_vec_array Element_Densities; - //local variable for host view of densities from the dual view - - const_host_vec_array all_node_densities; - if(nodal_density_flag){ - if(simparam->optimization_options.density_filter == DENSITY_FILTER::helmholtz_filter) - all_node_densities = all_filtered_node_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); - else - all_node_densities = all_node_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); - } - else{ - Element_Densities = Global_Element_Densities->getLocalView(Tpetra::Access::ReadOnly); - } - int num_dim = simparam->num_dims; - int nodes_per_elem = elem->num_basis(); - int num_gauss_points = simparam->num_gauss_points; - int z_quad,y_quad,x_quad, direct_product_count; - size_t local_node_id; - - direct_product_count = std::pow(num_gauss_points,num_dim); - real_t Elastic_Constant, Shear_Term, Pressure_Term, matrix_term; - real_t matrix_subterm1, matrix_subterm2, matrix_subterm3, invJacobian, Jacobian, weight_multiply; - real_t Element_Modulus, Poisson_Ratio; - //CArrayKokkos legendre_nodes_1D(num_gauss_points); - //CArrayKokkos legendre_weights_1D(num_gauss_points); - CArray legendre_nodes_1D(num_gauss_points); - CArray legendre_weights_1D(num_gauss_points); - real_t pointer_quad_coordinate[num_dim]; - real_t pointer_quad_coordinate_weight[num_dim]; - real_t pointer_interpolated_point[num_dim]; - real_t pointer_JT_row1[num_dim]; - real_t pointer_JT_row2[num_dim]; - real_t pointer_JT_row3[num_dim]; - ViewCArray quad_coordinate(pointer_quad_coordinate,num_dim); - ViewCArray quad_coordinate_weight(pointer_quad_coordinate_weight,num_dim); - ViewCArray interpolated_point(pointer_interpolated_point,num_dim); - ViewCArray JT_row1(pointer_JT_row1,num_dim); - ViewCArray JT_row2(pointer_JT_row2,num_dim); - ViewCArray JT_row3(pointer_JT_row3,num_dim); - - real_t pointer_basis_values[elem->num_basis()]; - real_t pointer_basis_derivative_s1[elem->num_basis()]; - real_t pointer_basis_derivative_s2[elem->num_basis()]; - real_t pointer_basis_derivative_s3[elem->num_basis()]; - ViewCArray basis_values(pointer_basis_values,elem->num_basis()); - ViewCArray basis_derivative_s1(pointer_basis_derivative_s1,elem->num_basis()); - ViewCArray basis_derivative_s2(pointer_basis_derivative_s2,elem->num_basis()); - ViewCArray basis_derivative_s3(pointer_basis_derivative_s3,elem->num_basis()); - CArrayKokkos nodal_positions(elem->num_basis(),num_dim); - CArrayKokkos nodal_density(elem->num_basis()); - - size_t Brows; - if(num_dim==2) Brows = 3; - if(num_dim==3) Brows = 6; - FArrayKokkos B_matrix_contribution(Brows,num_dim*elem->num_basis()); - CArrayKokkos B_matrix(Brows,num_dim*elem->num_basis()); - FArrayKokkos CB_matrix_contribution(Brows,num_dim*elem->num_basis()); - CArrayKokkos CB_matrix(Brows,num_dim*elem->num_basis()); - CArrayKokkos C_matrix(Brows,Brows); - - //initialize weights - elements::legendre_nodes_1D(legendre_nodes_1D,num_gauss_points); - elements::legendre_weights_1D(legendre_weights_1D,num_gauss_points); - Solver::node_ordering_convention active_node_ordering_convention = Explicit_Solver_Pointer_->active_node_ordering_convention; - - real_t current_density = 1; - CArrayKokkos convert_node_order(max_nodes_per_element); - if((active_node_ordering_convention == Solver::ENSIGHT && num_dim==3)||(active_node_ordering_convention == Solver::IJK && num_dim==2)){ - convert_node_order(0) = 0; - convert_node_order(1) = 1; - convert_node_order(2) = 3; - convert_node_order(3) = 2; - if(num_dim == 3){ - convert_node_order(4) = 4; - convert_node_order(5) = 5; - convert_node_order(6) = 7; - convert_node_order(7) = 6; - } - } - else if((active_node_ordering_convention == Solver::IJK && num_dim==3)||(active_node_ordering_convention == Solver::ENSIGHT && num_dim==2)){ - convert_node_order(0) = 0; - convert_node_order(1) = 1; - convert_node_order(2) = 2; - convert_node_order(3) = 3; - if(num_dim==3){ - convert_node_order(4) = 4; - convert_node_order(5) = 5; - convert_node_order(6) = 6; - convert_node_order(7) = 7; - } - } - - //acquire set of nodes for this local element - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - local_node_id = nodes_in_elem.host(ielem, convert_node_order(node_loop)); - nodal_positions(node_loop,0) = all_initial_node_coords(local_node_id,0); - nodal_positions(node_loop,1) = all_initial_node_coords(local_node_id,1); - nodal_positions(node_loop,2) = all_initial_node_coords(local_node_id,2); - if(nodal_density_flag) nodal_density(node_loop) = all_node_densities(local_node_id,0); - /* - if(myrank==1&&nodal_positions(node_loop,2)>10000000){ - std::cout << " LOCAL MATRIX DEBUG ON TASK " << myrank << std::endl; - std::cout << node_loop+1 <<" " << local_node_id <<" "<< nodes_in_elem(ielem, node_loop) << " "<< nodal_positions(node_loop,2) << std::endl; - std::fflush(stdout); - } - */ - //std::cout << local_node_id << " " << nodes_in_elem(ielem, node_loop) << " " << nodal_positions(node_loop,0) << " " << nodal_positions(node_loop,1) << " "<< nodal_positions(node_loop,2) <& Local_Matrix) +{ + // local variable for host view in the dual view + const_host_vec_array all_initial_node_coords = all_initial_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_host_vec_array Element_Densities; + // local variable for host view of densities from the dual view - //loop over quadrature points - for(int iquad=0; iquad < direct_product_count; iquad++){ - - //set current quadrature point - if(num_dim==3) z_quad = iquad/(num_gauss_points*num_gauss_points); - y_quad = (iquad % (num_gauss_points*num_gauss_points))/num_gauss_points; - x_quad = iquad % num_gauss_points; - quad_coordinate(0) = legendre_nodes_1D(x_quad); - quad_coordinate(1) = legendre_nodes_1D(y_quad); - if(num_dim==3) - quad_coordinate(2) = legendre_nodes_1D(z_quad); - - //set current quadrature weight - quad_coordinate_weight(0) = legendre_weights_1D(x_quad); - quad_coordinate_weight(1) = legendre_weights_1D(y_quad); - if(num_dim==3) - quad_coordinate_weight(2) = legendre_weights_1D(z_quad); + const_host_vec_array all_node_densities; + if (nodal_density_flag) + { + if (simparam->optimization_options.density_filter == DENSITY_FILTER::helmholtz_filter) + { + all_node_densities = all_filtered_node_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); + } + else + { + all_node_densities = all_node_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); + } + } else - quad_coordinate_weight(2) = 1; - weight_multiply = quad_coordinate_weight(0)*quad_coordinate_weight(1)*quad_coordinate_weight(2); - - //compute shape functions at this point for the element type - elem->basis(basis_values,quad_coordinate); - - //compute density - current_density = relative_element_densities.host(ielem); - - //debug print - //std::cout << "Current Density " << current_density << std::endl; - - //look up element material properties at this point as a function of density - Element_Material_Properties((size_t) ielem,Element_Modulus,Poisson_Ratio, current_density); - Elastic_Constant = Element_Modulus/((1 + Poisson_Ratio)*(1 - 2*Poisson_Ratio)); - Shear_Term = 0.5-Poisson_Ratio; - Pressure_Term = 1 - Poisson_Ratio; - - //debug print - //std::cout << "Element Material Params " << Elastic_Constant << std::endl; - - //compute Elastic (C) matrix - if(num_dim==2){ - C_matrix(0,0) = Pressure_Term; - C_matrix(1,1) = Pressure_Term; - C_matrix(0,1) = Poisson_Ratio; - C_matrix(1,0) = Poisson_Ratio; - C_matrix(2,2) = Shear_Term; + { + Element_Densities = Global_Element_Densities->getLocalView(Tpetra::Access::ReadOnly); } - if(num_dim==3){ - C_matrix(0,0) = Pressure_Term; - C_matrix(1,1) = Pressure_Term; - C_matrix(2,2) = Pressure_Term; - C_matrix(0,1) = Poisson_Ratio; - C_matrix(0,2) = Poisson_Ratio; - C_matrix(1,0) = Poisson_Ratio; - C_matrix(1,2) = Poisson_Ratio; - C_matrix(2,0) = Poisson_Ratio; - C_matrix(2,1) = Poisson_Ratio; - C_matrix(3,3) = Shear_Term; - C_matrix(4,4) = Shear_Term; - C_matrix(5,5) = Shear_Term; + int num_dim = simparam->num_dims; + int nodes_per_elem = elem->num_basis(); + int num_gauss_points = simparam->num_gauss_points; + int z_quad, y_quad, x_quad, direct_product_count; + size_t local_node_id; + + direct_product_count = std::pow(num_gauss_points, num_dim); + real_t Elastic_Constant, Shear_Term, Pressure_Term, matrix_term; + real_t matrix_subterm1, matrix_subterm2, matrix_subterm3, invJacobian, Jacobian, weight_multiply; + real_t Element_Modulus, Poisson_Ratio; + // CArrayKokkos legendre_nodes_1D(num_gauss_points); + // CArrayKokkos legendre_weights_1D(num_gauss_points); + CArray legendre_nodes_1D(num_gauss_points); + CArray legendre_weights_1D(num_gauss_points); + real_t pointer_quad_coordinate[num_dim]; + real_t pointer_quad_coordinate_weight[num_dim]; + real_t pointer_interpolated_point[num_dim]; + real_t pointer_JT_row1[num_dim]; + real_t pointer_JT_row2[num_dim]; + real_t pointer_JT_row3[num_dim]; + ViewCArray quad_coordinate(pointer_quad_coordinate, num_dim); + ViewCArray quad_coordinate_weight(pointer_quad_coordinate_weight, num_dim); + ViewCArray interpolated_point(pointer_interpolated_point, num_dim); + ViewCArray JT_row1(pointer_JT_row1, num_dim); + ViewCArray JT_row2(pointer_JT_row2, num_dim); + ViewCArray JT_row3(pointer_JT_row3, num_dim); + + real_t pointer_basis_values[elem->num_basis()]; + real_t pointer_basis_derivative_s1[elem->num_basis()]; + real_t pointer_basis_derivative_s2[elem->num_basis()]; + real_t pointer_basis_derivative_s3[elem->num_basis()]; + ViewCArray basis_values(pointer_basis_values, elem->num_basis()); + ViewCArray basis_derivative_s1(pointer_basis_derivative_s1, elem->num_basis()); + ViewCArray basis_derivative_s2(pointer_basis_derivative_s2, elem->num_basis()); + ViewCArray basis_derivative_s3(pointer_basis_derivative_s3, elem->num_basis()); + CArrayKokkos nodal_positions(elem->num_basis(), num_dim); + CArrayKokkos nodal_density(elem->num_basis()); + + size_t Brows; + if (num_dim == 2) + { + Brows = 3; } - - /* - //debug print of elasticity matrix - std::cout << " ------------ELASTICITY MATRIX "<< ielem + 1 <<"--------------"<partial_xi_basis(basis_derivative_s1,quad_coordinate); - elem->partial_eta_basis(basis_derivative_s2,quad_coordinate); - elem->partial_mu_basis(basis_derivative_s3,quad_coordinate); - - //compute derivatives of x,y,z w.r.t the s,t,w isoparametric space needed by JT (Transpose of the Jacobian) - //derivative of x,y,z w.r.t s - JT_row1(0) = 0; - JT_row1(1) = 0; - JT_row1(2) = 0; - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - JT_row1(0) += nodal_positions(node_loop,0)*basis_derivative_s1(node_loop); - JT_row1(1) += nodal_positions(node_loop,1)*basis_derivative_s1(node_loop); - JT_row1(2) += nodal_positions(node_loop,2)*basis_derivative_s1(node_loop); + FArrayKokkos B_matrix_contribution(Brows, num_dim * elem->num_basis()); + CArrayKokkos B_matrix(Brows, num_dim * elem->num_basis()); + FArrayKokkos CB_matrix_contribution(Brows, num_dim * elem->num_basis()); + CArrayKokkos CB_matrix(Brows, num_dim * elem->num_basis()); + CArrayKokkos C_matrix(Brows, Brows); + + // initialize weights + elements::legendre_nodes_1D(legendre_nodes_1D, num_gauss_points); + elements::legendre_weights_1D(legendre_weights_1D, num_gauss_points); + Solver::node_ordering_convention active_node_ordering_convention = Explicit_Solver_Pointer_->active_node_ordering_convention; + + real_t current_density = 1; + CArrayKokkos convert_node_order(max_nodes_per_element); + if ((active_node_ordering_convention == Solver::ENSIGHT && num_dim == 3) || (active_node_ordering_convention == Solver::IJK && num_dim == 2)) + { + convert_node_order(0) = 0; + convert_node_order(1) = 1; + convert_node_order(2) = 3; + convert_node_order(3) = 2; + if (num_dim == 3) + { + convert_node_order(4) = 4; + convert_node_order(5) = 5; + convert_node_order(6) = 7; + convert_node_order(7) = 6; + } } - - //derivative of x,y,z w.r.t t - JT_row2(0) = 0; - JT_row2(1) = 0; - JT_row2(2) = 0; - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - JT_row2(0) += nodal_positions(node_loop,0)*basis_derivative_s2(node_loop); - JT_row2(1) += nodal_positions(node_loop,1)*basis_derivative_s2(node_loop); - JT_row2(2) += nodal_positions(node_loop,2)*basis_derivative_s2(node_loop); + else if ((active_node_ordering_convention == Solver::IJK && num_dim == 3) || (active_node_ordering_convention == Solver::ENSIGHT && num_dim == 2)) + { + convert_node_order(0) = 0; + convert_node_order(1) = 1; + convert_node_order(2) = 2; + convert_node_order(3) = 3; + if (num_dim == 3) + { + convert_node_order(4) = 4; + convert_node_order(5) = 5; + convert_node_order(6) = 6; + convert_node_order(7) = 7; + } } - //derivative of x,y,z w.r.t w - JT_row3(0) = 0; - JT_row3(1) = 0; - JT_row3(2) = 0; - for(int node_loop=0; node_loop < elem->num_basis(); node_loop++){ - JT_row3(0) += nodal_positions(node_loop,0)*basis_derivative_s3(node_loop); - JT_row3(1) += nodal_positions(node_loop,1)*basis_derivative_s3(node_loop); - JT_row3(2) += nodal_positions(node_loop,2)*basis_derivative_s3(node_loop); - //debug print - /*if(myrank==1&&nodal_positions(node_loop,2)*basis_derivative_s3(node_loop)<-10000000){ - std::cout << " LOCAL MATRIX DEBUG ON TASK " << myrank << std::endl; - std::cout << node_loop+1 << " " << JT_row3(2) << " "<< nodal_positions(node_loop,2) <<" "<< basis_derivative_s3(node_loop) << std::endl; - std::fflush(stdout); - }*/ - } - - - //compute the determinant of the Jacobian - Jacobian = JT_row1(0)*(JT_row2(1)*JT_row3(2)-JT_row3(1)*JT_row2(2))- - JT_row1(1)*(JT_row2(0)*JT_row3(2)-JT_row3(0)*JT_row2(2))+ - JT_row1(2)*(JT_row2(0)*JT_row3(1)-JT_row3(0)*JT_row2(1)); - if(Jacobian<0) Jacobian = -Jacobian; - invJacobian = 1/Jacobian; - //compute the contributions of this quadrature point to the B matrix - if(num_dim==2) - for(int ishape=0; ishape < nodes_per_elem; ishape++){ - B_matrix_contribution(0,ishape*num_dim) = (basis_derivative_s1(ishape)*(JT_row2(1)*JT_row3(2)-JT_row3(1)*JT_row2(2))- - basis_derivative_s2(ishape)*(JT_row1(1)*JT_row3(2)-JT_row3(1)*JT_row1(2))+ - basis_derivative_s3(ishape)*(JT_row1(1)*JT_row2(2)-JT_row2(1)*JT_row1(2))); - B_matrix_contribution(1,ishape*num_dim) = 0; - B_matrix_contribution(2,ishape*num_dim) = 0; - B_matrix_contribution(3,ishape*num_dim) = (-basis_derivative_s1(ishape)*(JT_row2(0)*JT_row3(2)-JT_row3(0)*JT_row2(2))+ - basis_derivative_s2(ishape)*(JT_row1(0)*JT_row3(2)-JT_row3(0)*JT_row1(2))- - basis_derivative_s3(ishape)*(JT_row1(0)*JT_row2(2)-JT_row2(0)*JT_row1(2))); - B_matrix_contribution(4,ishape*num_dim) = (basis_derivative_s1(ishape)*(JT_row2(0)*JT_row3(1)-JT_row3(0)*JT_row2(1))- - basis_derivative_s2(ishape)*(JT_row1(0)*JT_row3(1)-JT_row3(0)*JT_row1(1))+ - basis_derivative_s3(ishape)*(JT_row1(0)*JT_row2(1)-JT_row2(0)*JT_row1(1))); - B_matrix_contribution(5,ishape*num_dim) = 0; - B_matrix_contribution(0,ishape*num_dim+1) = 0; - B_matrix_contribution(1,ishape*num_dim+1) = (-basis_derivative_s1(ishape)*(JT_row2(0)*JT_row3(2)-JT_row3(0)*JT_row2(2))+ - basis_derivative_s2(ishape)*(JT_row1(0)*JT_row3(2)-JT_row3(0)*JT_row1(2))- - basis_derivative_s3(ishape)*(JT_row1(0)*JT_row2(2)-JT_row2(0)*JT_row1(2))); - B_matrix_contribution(2,ishape*num_dim+1) = 0; - B_matrix_contribution(3,ishape*num_dim+1) = (basis_derivative_s1(ishape)*(JT_row2(1)*JT_row3(2)-JT_row3(1)*JT_row2(2))- - basis_derivative_s2(ishape)*(JT_row1(1)*JT_row3(2)-JT_row3(1)*JT_row1(2))+ - basis_derivative_s3(ishape)*(JT_row1(1)*JT_row2(2)-JT_row2(1)*JT_row1(2))); - B_matrix_contribution(4,ishape*num_dim+1) = 0; - B_matrix_contribution(5,ishape*num_dim+1) = (basis_derivative_s1(ishape)*(JT_row2(0)*JT_row3(1)-JT_row3(0)*JT_row2(1))- - basis_derivative_s2(ishape)*(JT_row1(0)*JT_row3(1)-JT_row3(0)*JT_row1(1))+ - basis_derivative_s3(ishape)*(JT_row1(0)*JT_row2(1)-JT_row2(0)*JT_row1(1))); - B_matrix_contribution(0,ishape*num_dim+2) = 0; - B_matrix_contribution(1,ishape*num_dim+2) = 0; - B_matrix_contribution(2,ishape*num_dim+2) = (basis_derivative_s1(ishape)*(JT_row2(0)*JT_row3(1)-JT_row3(0)*JT_row2(1))- - basis_derivative_s2(ishape)*(JT_row1(0)*JT_row3(1)-JT_row3(0)*JT_row1(1))+ - basis_derivative_s3(ishape)*(JT_row1(0)*JT_row2(1)-JT_row2(0)*JT_row1(1))); - B_matrix_contribution(3,ishape*num_dim+2) = 0; - B_matrix_contribution(4,ishape*num_dim+2) = (basis_derivative_s1(ishape)*(JT_row2(1)*JT_row3(2)-JT_row3(1)*JT_row2(2))- - basis_derivative_s2(ishape)*(JT_row1(1)*JT_row3(2)-JT_row3(1)*JT_row1(2))+ - basis_derivative_s3(ishape)*(JT_row1(1)*JT_row2(2)-JT_row2(1)*JT_row1(2))); - B_matrix_contribution(5,ishape*num_dim+2) = (-basis_derivative_s1(ishape)*(JT_row2(0)*JT_row3(2)-JT_row3(0)*JT_row2(2))+ - basis_derivative_s2(ishape)*(JT_row1(0)*JT_row3(2)-JT_row3(0)*JT_row1(2))- - basis_derivative_s3(ishape)*(JT_row1(0)*JT_row2(2)-JT_row2(0)*JT_row1(2))); + // acquire set of nodes for this local element + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + local_node_id = nodes_in_elem.host(ielem, convert_node_order(node_loop)); + nodal_positions(node_loop, 0) = all_initial_node_coords(local_node_id, 0); + nodal_positions(node_loop, 1) = all_initial_node_coords(local_node_id, 1); + nodal_positions(node_loop, 2) = all_initial_node_coords(local_node_id, 2); + if (nodal_density_flag) + { + nodal_density(node_loop) = all_node_densities(local_node_id, 0); + } + /* + if(myrank==1&&nodal_positions(node_loop,2)>10000000){ + std::cout << " LOCAL MATRIX DEBUG ON TASK " << myrank << std::endl; + std::cout << node_loop+1 <<" " << local_node_id <<" "<< nodes_in_elem(ielem, node_loop) << " "<< nodal_positions(node_loop,2) << std::endl; + std::fflush(stdout); + } + */ + // std::cout << local_node_id << " " << nodes_in_elem(ielem, node_loop) << " " << nodal_positions(node_loop,0) << " " << nodal_positions(node_loop,1) << " "<< nodal_positions(node_loop,2) <basis(basis_values, quad_coordinate); + + // compute density + current_density = relative_element_densities.host(ielem); + + // debug print + // std::cout << "Current Density " << current_density << std::endl; + + // look up element material properties at this point as a function of density + Element_Material_Properties((size_t) ielem, Element_Modulus, Poisson_Ratio, current_density); + Elastic_Constant = Element_Modulus / ((1 + Poisson_Ratio) * (1 - 2 * Poisson_Ratio)); + Shear_Term = 0.5 - Poisson_Ratio; + Pressure_Term = 1 - Poisson_Ratio; + + // debug print + // std::cout << "Element Material Params " << Elastic_Constant << std::endl; + + // compute Elastic (C) matrix + if (num_dim == 2) + { + C_matrix(0, 0) = Pressure_Term; + C_matrix(1, 1) = Pressure_Term; + C_matrix(0, 1) = Poisson_Ratio; + C_matrix(1, 0) = Poisson_Ratio; + C_matrix(2, 2) = Shear_Term; + } + if (num_dim == 3) + { + C_matrix(0, 0) = Pressure_Term; + C_matrix(1, 1) = Pressure_Term; + C_matrix(2, 2) = Pressure_Term; + C_matrix(0, 1) = Poisson_Ratio; + C_matrix(0, 2) = Poisson_Ratio; + C_matrix(1, 0) = Poisson_Ratio; + C_matrix(1, 2) = Poisson_Ratio; + C_matrix(2, 0) = Poisson_Ratio; + C_matrix(2, 1) = Poisson_Ratio; + C_matrix(3, 3) = Shear_Term; + C_matrix(4, 4) = Shear_Term; + C_matrix(5, 5) = Shear_Term; + } + + /* + //debug print of elasticity matrix + std::cout << " ------------ELASTICITY MATRIX "<< ielem + 1 <<"--------------"<partial_xi_basis(basis_derivative_s1, quad_coordinate); + elem->partial_eta_basis(basis_derivative_s2, quad_coordinate); + elem->partial_mu_basis(basis_derivative_s3, quad_coordinate); + + // compute derivatives of x,y,z w.r.t the s,t,w isoparametric space needed by JT (Transpose of the Jacobian) + // derivative of x,y,z w.r.t s + JT_row1(0) = 0; + JT_row1(1) = 0; + JT_row1(2) = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row1(0) += nodal_positions(node_loop, 0) * basis_derivative_s1(node_loop); + JT_row1(1) += nodal_positions(node_loop, 1) * basis_derivative_s1(node_loop); + JT_row1(2) += nodal_positions(node_loop, 2) * basis_derivative_s1(node_loop); + } + + // derivative of x,y,z w.r.t t + JT_row2(0) = 0; + JT_row2(1) = 0; + JT_row2(2) = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row2(0) += nodal_positions(node_loop, 0) * basis_derivative_s2(node_loop); + JT_row2(1) += nodal_positions(node_loop, 1) * basis_derivative_s2(node_loop); + JT_row2(2) += nodal_positions(node_loop, 2) * basis_derivative_s2(node_loop); + } + + // derivative of x,y,z w.r.t w + JT_row3(0) = 0; + JT_row3(1) = 0; + JT_row3(2) = 0; + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) + { + JT_row3(0) += nodal_positions(node_loop, 0) * basis_derivative_s3(node_loop); + JT_row3(1) += nodal_positions(node_loop, 1) * basis_derivative_s3(node_loop); + JT_row3(2) += nodal_positions(node_loop, 2) * basis_derivative_s3(node_loop); + // debug print + /*if(myrank==1&&nodal_positions(node_loop,2)*basis_derivative_s3(node_loop)<-10000000){ + std::cout << " LOCAL MATRIX DEBUG ON TASK " << myrank << std::endl; + std::cout << node_loop+1 << " " << JT_row3(2) << " "<< nodal_positions(node_loop,2) <<" "<< basis_derivative_s3(node_loop) << std::endl; + std::fflush(stdout); + }*/ + } + + // compute the determinant of the Jacobian + Jacobian = JT_row1(0) * (JT_row2(1) * JT_row3(2) - JT_row3(1) * JT_row2(2)) - + JT_row1(1) * (JT_row2(0) * JT_row3(2) - JT_row3(0) * JT_row2(2)) + + JT_row1(2) * (JT_row2(0) * JT_row3(1) - JT_row3(0) * JT_row2(1)); + if (Jacobian < 0) + { + Jacobian = -Jacobian; + } + invJacobian = 1 / Jacobian; + // compute the contributions of this quadrature point to the B matrix + if (num_dim == 2) + { + for (int ishape = 0; ishape < nodes_per_elem; ishape++) + { + B_matrix_contribution(0, ishape * num_dim) = (basis_derivative_s1(ishape) * (JT_row2(1) * JT_row3(2) - JT_row3(1) * JT_row2(2)) - + basis_derivative_s2(ishape) * (JT_row1(1) * JT_row3(2) - JT_row3(1) * JT_row1(2)) + + basis_derivative_s3(ishape) * (JT_row1(1) * JT_row2(2) - JT_row2(1) * JT_row1(2))); + B_matrix_contribution(1, ishape * num_dim) = 0; + B_matrix_contribution(2, ishape * num_dim) = 0; + B_matrix_contribution(3, ishape * num_dim) = (-basis_derivative_s1(ishape) * (JT_row2(0) * JT_row3(2) - JT_row3(0) * JT_row2(2)) + + basis_derivative_s2(ishape) * (JT_row1(0) * JT_row3(2) - JT_row3(0) * JT_row1(2)) - + basis_derivative_s3(ishape) * (JT_row1(0) * JT_row2(2) - JT_row2(0) * JT_row1(2))); + B_matrix_contribution(4, ishape * num_dim) = (basis_derivative_s1(ishape) * (JT_row2(0) * JT_row3(1) - JT_row3(0) * JT_row2(1)) - + basis_derivative_s2(ishape) * (JT_row1(0) * JT_row3(1) - JT_row3(0) * JT_row1(1)) + + basis_derivative_s3(ishape) * (JT_row1(0) * JT_row2(1) - JT_row2(0) * JT_row1(1))); + B_matrix_contribution(5, ishape * num_dim) = 0; + B_matrix_contribution(0, ishape * num_dim + 1) = 0; + B_matrix_contribution(1, ishape * num_dim + 1) = (-basis_derivative_s1(ishape) * (JT_row2(0) * JT_row3(2) - JT_row3(0) * JT_row2(2)) + + basis_derivative_s2(ishape) * (JT_row1(0) * JT_row3(2) - JT_row3(0) * JT_row1(2)) - + basis_derivative_s3(ishape) * (JT_row1(0) * JT_row2(2) - JT_row2(0) * JT_row1(2))); + B_matrix_contribution(2, ishape * num_dim + 1) = 0; + B_matrix_contribution(3, ishape * num_dim + 1) = (basis_derivative_s1(ishape) * (JT_row2(1) * JT_row3(2) - JT_row3(1) * JT_row2(2)) - + basis_derivative_s2(ishape) * (JT_row1(1) * JT_row3(2) - JT_row3(1) * JT_row1(2)) + + basis_derivative_s3(ishape) * (JT_row1(1) * JT_row2(2) - JT_row2(1) * JT_row1(2))); + B_matrix_contribution(4, ishape * num_dim + 1) = 0; + B_matrix_contribution(5, ishape * num_dim + 1) = (basis_derivative_s1(ishape) * (JT_row2(0) * JT_row3(1) - JT_row3(0) * JT_row2(1)) - + basis_derivative_s2(ishape) * (JT_row1(0) * JT_row3(1) - JT_row3(0) * JT_row1(1)) + + basis_derivative_s3(ishape) * (JT_row1(0) * JT_row2(1) - JT_row2(0) * JT_row1(1))); + B_matrix_contribution(0, ishape * num_dim + 2) = 0; + B_matrix_contribution(1, ishape * num_dim + 2) = 0; + B_matrix_contribution(2, ishape * num_dim + 2) = (basis_derivative_s1(ishape) * (JT_row2(0) * JT_row3(1) - JT_row3(0) * JT_row2(1)) - + basis_derivative_s2(ishape) * (JT_row1(0) * JT_row3(1) - JT_row3(0) * JT_row1(1)) + + basis_derivative_s3(ishape) * (JT_row1(0) * JT_row2(1) - JT_row2(0) * JT_row1(1))); + B_matrix_contribution(3, ishape * num_dim + 2) = 0; + B_matrix_contribution(4, ishape * num_dim + 2) = (basis_derivative_s1(ishape) * (JT_row2(1) * JT_row3(2) - JT_row3(1) * JT_row2(2)) - + basis_derivative_s2(ishape) * (JT_row1(1) * JT_row3(2) - JT_row3(1) * JT_row1(2)) + + basis_derivative_s3(ishape) * (JT_row1(1) * JT_row2(2) - JT_row2(1) * JT_row1(2))); + B_matrix_contribution(5, ishape * num_dim + 2) = (-basis_derivative_s1(ishape) * (JT_row2(0) * JT_row3(2) - JT_row3(0) * JT_row2(2)) + + basis_derivative_s2(ishape) * (JT_row1(0) * JT_row3(2) - JT_row3(0) * JT_row1(2)) - + basis_derivative_s3(ishape) * (JT_row1(0) * JT_row2(2) - JT_row2(0) * JT_row1(2))); + } + } + if (num_dim == 3) + { + for (int ishape = 0; ishape < nodes_per_elem; ishape++) + { + B_matrix_contribution(0, ishape * num_dim) = (basis_derivative_s1(ishape) * (JT_row2(1) * JT_row3(2) - JT_row3(1) * JT_row2(2)) - + basis_derivative_s2(ishape) * (JT_row1(1) * JT_row3(2) - JT_row3(1) * JT_row1(2)) + + basis_derivative_s3(ishape) * (JT_row1(1) * JT_row2(2) - JT_row2(1) * JT_row1(2))); + B_matrix_contribution(1, ishape * num_dim) = 0; + B_matrix_contribution(2, ishape * num_dim) = 0; + B_matrix_contribution(3, ishape * num_dim) = (-basis_derivative_s1(ishape) * (JT_row2(0) * JT_row3(2) - JT_row3(0) * JT_row2(2)) + + basis_derivative_s2(ishape) * (JT_row1(0) * JT_row3(2) - JT_row3(0) * JT_row1(2)) - + basis_derivative_s3(ishape) * (JT_row1(0) * JT_row2(2) - JT_row2(0) * JT_row1(2))); + B_matrix_contribution(4, ishape * num_dim) = (basis_derivative_s1(ishape) * (JT_row2(0) * JT_row3(1) - JT_row3(0) * JT_row2(1)) - + basis_derivative_s2(ishape) * (JT_row1(0) * JT_row3(1) - JT_row3(0) * JT_row1(1)) + + basis_derivative_s3(ishape) * (JT_row1(0) * JT_row2(1) - JT_row2(0) * JT_row1(1))); + B_matrix_contribution(5, ishape * num_dim) = 0; + B_matrix_contribution(0, ishape * num_dim + 1) = 0; + B_matrix_contribution(1, ishape * num_dim + 1) = (-basis_derivative_s1(ishape) * (JT_row2(0) * JT_row3(2) - JT_row3(0) * JT_row2(2)) + + basis_derivative_s2(ishape) * (JT_row1(0) * JT_row3(2) - JT_row3(0) * JT_row1(2)) - + basis_derivative_s3(ishape) * (JT_row1(0) * JT_row2(2) - JT_row2(0) * JT_row1(2))); + B_matrix_contribution(2, ishape * num_dim + 1) = 0; + B_matrix_contribution(3, ishape * num_dim + 1) = (basis_derivative_s1(ishape) * (JT_row2(1) * JT_row3(2) - JT_row3(1) * JT_row2(2)) - + basis_derivative_s2(ishape) * (JT_row1(1) * JT_row3(2) - JT_row3(1) * JT_row1(2)) + + basis_derivative_s3(ishape) * (JT_row1(1) * JT_row2(2) - JT_row2(1) * JT_row1(2))); + B_matrix_contribution(4, ishape * num_dim + 1) = 0; + B_matrix_contribution(5, ishape * num_dim + 1) = (basis_derivative_s1(ishape) * (JT_row2(0) * JT_row3(1) - JT_row3(0) * JT_row2(1)) - + basis_derivative_s2(ishape) * (JT_row1(0) * JT_row3(1) - JT_row3(0) * JT_row1(1)) + + basis_derivative_s3(ishape) * (JT_row1(0) * JT_row2(1) - JT_row2(0) * JT_row1(1))); + B_matrix_contribution(0, ishape * num_dim + 2) = 0; + B_matrix_contribution(1, ishape * num_dim + 2) = 0; + B_matrix_contribution(2, ishape * num_dim + 2) = (basis_derivative_s1(ishape) * (JT_row2(0) * JT_row3(1) - JT_row3(0) * JT_row2(1)) - + basis_derivative_s2(ishape) * (JT_row1(0) * JT_row3(1) - JT_row3(0) * JT_row1(1)) + + basis_derivative_s3(ishape) * (JT_row1(0) * JT_row2(1) - JT_row2(0) * JT_row1(1))); + B_matrix_contribution(3, ishape * num_dim + 2) = 0; + B_matrix_contribution(4, ishape * num_dim + 2) = (basis_derivative_s1(ishape) * (JT_row2(1) * JT_row3(2) - JT_row3(1) * JT_row2(2)) - + basis_derivative_s2(ishape) * (JT_row1(1) * JT_row3(2) - JT_row3(1) * JT_row1(2)) + + basis_derivative_s3(ishape) * (JT_row1(1) * JT_row2(2) - JT_row2(1) * JT_row1(2))); + B_matrix_contribution(5, ishape * num_dim + 2) = (-basis_derivative_s1(ishape) * (JT_row2(0) * JT_row3(2) - JT_row3(0) * JT_row2(2)) + + basis_derivative_s2(ishape) * (JT_row1(0) * JT_row3(2) - JT_row3(0) * JT_row1(2)) - + basis_derivative_s3(ishape) * (JT_row1(0) * JT_row2(2) - JT_row2(0) * JT_row1(2))); + } + } + /* + //debug print of B matrix per quadrature point + std::cout << " ------------B MATRIX QUADRATURE CONTRIBUTION"<< ielem + 1 <<"--------------"<getLocalView (Tpetra::Access::ReadOnly); - - const_host_vec_array all_initial_node_coords = all_initial_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_host_vec_array Element_Densities; - //local variable for host view of densities from the dual view - const_host_vec_array all_node_densities; - if(nodal_density_flag) - all_node_densities = all_node_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); - else - Element_Densities = Global_Element_Densities->getLocalView(Tpetra::Access::ReadOnly); - int num_dim = simparam->num_dims; - int nodes_per_elem = elem->num_basis(); - int num_gauss_points = simparam->num_gauss_points; - int z_quad,y_quad,x_quad, direct_product_count; - size_t local_node_id, local_dof_idx, local_dof_idy, local_dof_idz; - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - GO current_global_index; - - real_t global_dt; - size_t current_data_index, next_data_index; - CArrayKokkos current_element_adjoint = CArrayKokkos(num_nodes_in_elem*num_dim); - - direct_product_count = std::pow(num_gauss_points,num_dim); - real_t Element_Modulus_Gradient, Poisson_Ratio, gradient_force_density[3]; - real_t Elastic_Constant, Shear_Term, Pressure_Term; - real_t inner_product, matrix_term, Jacobian, invJacobian, weight_multiply; - //CArrayKokkos legendre_nodes_1D(num_gauss_points); - //CArrayKokkos legendre_weights_1D(num_gauss_points); - CArray legendre_nodes_1D(num_gauss_points); - CArray legendre_weights_1D(num_gauss_points); - real_t pointer_quad_coordinate[num_dim]; - real_t pointer_quad_coordinate_weight[num_dim]; - real_t pointer_interpolated_point[num_dim]; - real_t pointer_JT_row1[num_dim]; - real_t pointer_JT_row2[num_dim]; - real_t pointer_JT_row3[num_dim]; - ViewCArray quad_coordinate(pointer_quad_coordinate,num_dim); - ViewCArray quad_coordinate_weight(pointer_quad_coordinate_weight,num_dim); - ViewCArray interpolated_point(pointer_interpolated_point,num_dim); - ViewCArray JT_row1(pointer_JT_row1,num_dim); - ViewCArray JT_row2(pointer_JT_row2,num_dim); - ViewCArray JT_row3(pointer_JT_row3,num_dim); - - real_t pointer_basis_values[elem->num_basis()]; - real_t pointer_basis_derivative_s1[elem->num_basis()]; - real_t pointer_basis_derivative_s2[elem->num_basis()]; - real_t pointer_basis_derivative_s3[elem->num_basis()]; - ViewCArray basis_values(pointer_basis_values,elem->num_basis()); - ViewCArray basis_derivative_s1(pointer_basis_derivative_s1,elem->num_basis()); - ViewCArray basis_derivative_s2(pointer_basis_derivative_s2,elem->num_basis()); - ViewCArray basis_derivative_s3(pointer_basis_derivative_s3,elem->num_basis()); - CArrayKokkos nodal_positions(elem->num_basis(),num_dim); - CArrayKokkos current_nodal_displacements(elem->num_basis()*num_dim); - CArrayKokkos nodal_density(elem->num_basis()); - - size_t Brows; - if(num_dim==2) Brows = 3; - if(num_dim==3) Brows = 6; - FArrayKokkos B_matrix_contribution(Brows,num_dim*elem->num_basis()); - CArrayKokkos B_matrix(Brows,num_dim*elem->num_basis()); - FArrayKokkos CB_matrix_contribution(Brows,num_dim*elem->num_basis()); - CArrayKokkos CB_matrix(Brows,num_dim*elem->num_basis()); - CArrayKokkos C_matrix(Brows,Brows); - CArrayKokkos Local_Matrix_Contribution(num_dim*nodes_per_elem,num_dim*nodes_per_elem); - - //initialize weights - elements::legendre_nodes_1D(legendre_nodes_1D,num_gauss_points); - elements::legendre_weights_1D(legendre_weights_1D,num_gauss_points); - Solver::node_ordering_convention active_node_ordering_convention = Explicit_Solver_Pointer_->active_node_ordering_convention; - - real_t current_density = 1; - CArrayKokkos convert_node_order(max_nodes_per_element); - if((active_node_ordering_convention == Solver::ENSIGHT && num_dim==3)||(active_node_ordering_convention == Solver::IJK && num_dim==2)){ - convert_node_order(0) = 0; - convert_node_order(1) = 1; - convert_node_order(2) = 3; - convert_node_order(3) = 2; - if(num_dim == 3){ - convert_node_order(4) = 4; - convert_node_order(5) = 5; - convert_node_order(6) = 7; - convert_node_order(7) = 6; +void FEA_Module_Dynamic_Elasticity::compute_stiffness_gradients(const_host_vec_array& design_variables, host_vec_array& design_gradients) +{ + // local variable for host view in the dual view + const_host_vec_array all_node_coords = all_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + + const_host_vec_array all_initial_node_coords = all_initial_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_host_vec_array Element_Densities; + // local variable for host view of densities from the dual view + const_host_vec_array all_node_densities; + if (nodal_density_flag) + { + all_node_densities = all_node_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); } - } - else if((active_node_ordering_convention == Solver::IJK && num_dim==3)||(active_node_ordering_convention == Solver::ENSIGHT && num_dim==2)){ - convert_node_order(0) = 0; - convert_node_order(1) = 1; - convert_node_order(2) = 2; - convert_node_order(3) = 3; - if(num_dim==3){ - convert_node_order(4) = 4; - convert_node_order(5) = 5; - convert_node_order(6) = 6; - convert_node_order(7) = 7; + else + { + Element_Densities = Global_Element_Densities->getLocalView(Tpetra::Access::ReadOnly); + } + int num_dim = simparam->num_dims; + int nodes_per_elem = elem->num_basis(); + int num_gauss_points = simparam->num_gauss_points; + int z_quad, y_quad, x_quad, direct_product_count; + size_t local_node_id, local_dof_idx, local_dof_idy, local_dof_idz; + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + GO current_global_index; + + real_t global_dt; + size_t current_data_index, next_data_index; + CArrayKokkos current_element_adjoint = CArrayKokkos(num_nodes_in_elem * num_dim); + + direct_product_count = std::pow(num_gauss_points, num_dim); + real_t Element_Modulus_Gradient, Poisson_Ratio, gradient_force_density[3]; + real_t Elastic_Constant, Shear_Term, Pressure_Term; + real_t inner_product, matrix_term, Jacobian, invJacobian, weight_multiply; + // CArrayKokkos legendre_nodes_1D(num_gauss_points); + // CArrayKokkos legendre_weights_1D(num_gauss_points); + CArray legendre_nodes_1D(num_gauss_points); + CArray legendre_weights_1D(num_gauss_points); + real_t pointer_quad_coordinate[num_dim]; + real_t pointer_quad_coordinate_weight[num_dim]; + real_t pointer_interpolated_point[num_dim]; + real_t pointer_JT_row1[num_dim]; + real_t pointer_JT_row2[num_dim]; + real_t pointer_JT_row3[num_dim]; + ViewCArray quad_coordinate(pointer_quad_coordinate, num_dim); + ViewCArray quad_coordinate_weight(pointer_quad_coordinate_weight, num_dim); + ViewCArray interpolated_point(pointer_interpolated_point, num_dim); + ViewCArray JT_row1(pointer_JT_row1, num_dim); + ViewCArray JT_row2(pointer_JT_row2, num_dim); + ViewCArray JT_row3(pointer_JT_row3, num_dim); + + real_t pointer_basis_values[elem->num_basis()]; + real_t pointer_basis_derivative_s1[elem->num_basis()]; + real_t pointer_basis_derivative_s2[elem->num_basis()]; + real_t pointer_basis_derivative_s3[elem->num_basis()]; + ViewCArray basis_values(pointer_basis_values, elem->num_basis()); + ViewCArray basis_derivative_s1(pointer_basis_derivative_s1, elem->num_basis()); + ViewCArray basis_derivative_s2(pointer_basis_derivative_s2, elem->num_basis()); + ViewCArray basis_derivative_s3(pointer_basis_derivative_s3, elem->num_basis()); + CArrayKokkos nodal_positions(elem->num_basis(), num_dim); + CArrayKokkos current_nodal_displacements(elem->num_basis() * num_dim); + CArrayKokkos nodal_density(elem->num_basis()); + + size_t Brows; + if (num_dim == 2) + { + Brows = 3; + } + if (num_dim == 3) + { + Brows = 6; } - } - - //loop through each element and assign the contribution to compliance gradient for each of its local nodes - if(simparam->dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ - if(myrank==0){ - std::cout << "gradient term derivative of force" << std::endl; + FArrayKokkos B_matrix_contribution(Brows, num_dim * elem->num_basis()); + CArrayKokkos B_matrix(Brows, num_dim * elem->num_basis()); + FArrayKokkos CB_matrix_contribution(Brows, num_dim * elem->num_basis()); + CArrayKokkos CB_matrix(Brows, num_dim * elem->num_basis()); + CArrayKokkos C_matrix(Brows, Brows); + CArrayKokkos Local_Matrix_Contribution(num_dim * nodes_per_elem, num_dim * nodes_per_elem); + + // initialize weights + elements::legendre_nodes_1D(legendre_nodes_1D, num_gauss_points); + elements::legendre_weights_1D(legendre_weights_1D, num_gauss_points); + Solver::node_ordering_convention active_node_ordering_convention = Explicit_Solver_Pointer_->active_node_ordering_convention; + + real_t current_density = 1; + CArrayKokkos convert_node_order(max_nodes_per_element); + if ((active_node_ordering_convention == Solver::ENSIGHT && num_dim == 3) || (active_node_ordering_convention == Solver::IJK && num_dim == 2)) + { + convert_node_order(0) = 0; + convert_node_order(1) = 1; + convert_node_order(2) = 3; + convert_node_order(3) = 2; + if (num_dim == 3) + { + convert_node_order(4) = 4; + convert_node_order(5) = 5; + convert_node_order(6) = 7; + convert_node_order(7) = 6; } - } - - for (unsigned long cycle = 0; cycle < last_time_step+1; cycle++) { - //compute timestep from time data - global_dt = time_data[cycle+1] - time_data[cycle]; - - //print - if(simparam->dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ - if (cycle==0){ - if(myrank==0) - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } - // print time step every 10 cycles - else if (cycle%20==0){ - if(myrank==0) - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } // end if } - - //view scope + else if ((active_node_ordering_convention == Solver::IJK && num_dim == 3) || (active_node_ordering_convention == Solver::ENSIGHT && num_dim == 2)) { - //const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_host_vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_host_vec_array next_adjoint_vector = (*adjoint_vector_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - - const_host_vec_array next_coordinate_vector = (*forward_solve_coordinate_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_host_vec_array current_coordinate_vector = (*forward_solve_coordinate_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - - //interface of arrays for current implementation of force calculation - for(size_t ielem = 0; ielem < rnum_elem; ielem++){ - nodes_per_elem = elem->num_basis(); - - //initialize C matrix - for(int irow = 0; irow < Brows; irow++) - for(int icol = 0; icol < Brows; icol++) - C_matrix(irow,icol) = 0; - - //B matrix initialization - for(int irow=0; irow < Brows; irow++) - for(int icol=0; icol < num_dim*nodes_per_elem; icol++){ - CB_matrix(irow,icol) = 0; - } - - //acquire set of nodes and nodal displacements for this local element - for(int node_loop=0; node_loop < nodes_per_elem; node_loop++){ - local_node_id = nodes_in_elem.host(ielem, convert_node_order(node_loop)); - local_dof_idx = nodes_in_elem.host(ielem, convert_node_order(node_loop))*num_dim; - local_dof_idy = local_dof_idx + 1; - local_dof_idz = local_dof_idx + 2; - - nodal_positions(node_loop,0) = all_initial_node_coords(local_node_id,0); - nodal_positions(node_loop,1) = all_initial_node_coords(local_node_id,1); - current_nodal_displacements(node_loop*num_dim) = 0.5*(next_coordinate_vector(local_node_id, 0)-all_initial_node_coords(local_node_id,0)+current_coordinate_vector(local_node_id, 0)-all_initial_node_coords(local_node_id,0)); - current_nodal_displacements(node_loop*num_dim+1) = 0.5*(next_coordinate_vector(local_node_id, 1)-all_initial_node_coords(local_node_id,1)+current_coordinate_vector(local_node_id, 1)-all_initial_node_coords(local_node_id,1)); - current_element_adjoint(node_loop*num_dim) = 0.5*(current_adjoint_vector(local_node_id,0)+next_adjoint_vector(local_node_id,0)); - current_element_adjoint(node_loop*num_dim+1) = 0.5*(current_adjoint_vector(local_node_id,1)+next_adjoint_vector(local_node_id,1)); - - if(num_dim==3){ - nodal_positions(node_loop,2) = all_initial_node_coords(local_node_id,2); - current_nodal_displacements(node_loop*num_dim+2) = 0.5*(next_coordinate_vector(local_node_id, 2)-all_initial_node_coords(local_node_id,2)+current_coordinate_vector(local_node_id, 2)-all_initial_node_coords(local_node_id,2)); - current_element_adjoint(node_loop*num_dim+2) = 0.5*(current_adjoint_vector(local_node_id,2)+next_adjoint_vector(local_node_id,2)); - } - - if(nodal_density_flag) nodal_density(node_loop) = all_node_densities(local_node_id,0); - //debug print - /* - std::cout << "node index access x "<< local_node_id << std::endl; - std::cout << "local index access x "<< local_dof_idx << " displacement x " << current_nodal_displacements(node_loop*num_dim) <dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) + { + if (myrank == 0) + { + std::cout << "gradient term derivative of force" << std::endl; } - std::cout << " }"<< std::endl; - */ - - //loop over quadrature points - for(int iquad=0; iquad < direct_product_count; iquad++){ - - //set current quadrature point - if(num_dim==3) z_quad = iquad/(num_gauss_points*num_gauss_points); - y_quad = (iquad % (num_gauss_points*num_gauss_points))/num_gauss_points; - x_quad = iquad % num_gauss_points; - quad_coordinate(0) = legendre_nodes_1D(x_quad); - quad_coordinate(1) = legendre_nodes_1D(y_quad); - if(num_dim==3) - quad_coordinate(2) = legendre_nodes_1D(z_quad); - - //set current quadrature weight - quad_coordinate_weight(0) = legendre_weights_1D(x_quad); - quad_coordinate_weight(1) = legendre_weights_1D(y_quad); - if(num_dim==3) - quad_coordinate_weight(2) = legendre_weights_1D(z_quad); - else - quad_coordinate_weight(2) = 1; - weight_multiply = quad_coordinate_weight(0)*quad_coordinate_weight(1)*quad_coordinate_weight(2); - - //compute shape functions at this point for the element type - elem->basis(basis_values,quad_coordinate); - - //compute all the necessary coordinates and derivatives at this point - //compute shape function derivatives - elem->partial_xi_basis(basis_derivative_s1,quad_coordinate); - elem->partial_eta_basis(basis_derivative_s2,quad_coordinate); - elem->partial_mu_basis(basis_derivative_s3,quad_coordinate); - - //compute derivatives of x,y,z w.r.t the s,t,w isoparametric space needed by JT (Transpose of the Jacobian) - //derivative of x,y,z w.r.t s - JT_row1(0) = 0; - JT_row1(1) = 0; - JT_row1(2) = 0; - for(int node_loop=0; node_loop < nodes_per_elem; node_loop++){ - JT_row1(0) += nodal_positions(node_loop,0)*basis_derivative_s1(node_loop); - JT_row1(1) += nodal_positions(node_loop,1)*basis_derivative_s1(node_loop); - JT_row1(2) += nodal_positions(node_loop,2)*basis_derivative_s1(node_loop); - } - - //derivative of x,y,z w.r.t t - JT_row2(0) = 0; - JT_row2(1) = 0; - JT_row2(2) = 0; - for(int node_loop=0; node_loop < nodes_per_elem; node_loop++){ - JT_row2(0) += nodal_positions(node_loop,0)*basis_derivative_s2(node_loop); - JT_row2(1) += nodal_positions(node_loop,1)*basis_derivative_s2(node_loop); - JT_row2(2) += nodal_positions(node_loop,2)*basis_derivative_s2(node_loop); - } - - //derivative of x,y,z w.r.t w - JT_row3(0) = 0; - JT_row3(1) = 0; - JT_row3(2) = 0; - for(int node_loop=0; node_loop < nodes_per_elem; node_loop++){ - JT_row3(0) += nodal_positions(node_loop,0)*basis_derivative_s3(node_loop); - JT_row3(1) += nodal_positions(node_loop,1)*basis_derivative_s3(node_loop); - JT_row3(2) += nodal_positions(node_loop,2)*basis_derivative_s3(node_loop); - } - - //compute the determinant of the Jacobian - Jacobian = JT_row1(0)*(JT_row2(1)*JT_row3(2)-JT_row3(1)*JT_row2(2))- - JT_row1(1)*(JT_row2(0)*JT_row3(2)-JT_row3(0)*JT_row2(2))+ - JT_row1(2)*(JT_row2(0)*JT_row3(1)-JT_row3(0)*JT_row2(1)); - if(Jacobian<0) Jacobian = -Jacobian; - invJacobian = 1/Jacobian; - - //compute density - current_density = relative_element_densities.host(ielem); - - //debug print - //std::cout << "Current Density " << current_density << std::endl; - - //compute the contributions of this quadrature point to the B matrix - if(num_dim==2) - for(int ishape=0; ishape < nodes_per_elem; ishape++){ - B_matrix_contribution(0,ishape*num_dim) = (basis_derivative_s1(ishape)*(JT_row2(1)*JT_row3(2)-JT_row3(1)*JT_row2(2))- - basis_derivative_s2(ishape)*(JT_row1(1)*JT_row3(2)-JT_row3(1)*JT_row1(2))+ - basis_derivative_s3(ishape)*(JT_row1(1)*JT_row2(2)-JT_row2(1)*JT_row1(2))); - B_matrix_contribution(1,ishape*num_dim) = 0; - B_matrix_contribution(2,ishape*num_dim) = 0; - B_matrix_contribution(3,ishape*num_dim) = (-basis_derivative_s1(ishape)*(JT_row2(0)*JT_row3(2)-JT_row3(0)*JT_row2(2))+ - basis_derivative_s2(ishape)*(JT_row1(0)*JT_row3(2)-JT_row3(0)*JT_row1(2))- - basis_derivative_s3(ishape)*(JT_row1(0)*JT_row2(2)-JT_row2(0)*JT_row1(2))); - B_matrix_contribution(4,ishape*num_dim) = (basis_derivative_s1(ishape)*(JT_row2(0)*JT_row3(1)-JT_row3(0)*JT_row2(1))- - basis_derivative_s2(ishape)*(JT_row1(0)*JT_row3(1)-JT_row3(0)*JT_row1(1))+ - basis_derivative_s3(ishape)*(JT_row1(0)*JT_row2(1)-JT_row2(0)*JT_row1(1))); - B_matrix_contribution(5,ishape*num_dim) = 0; - B_matrix_contribution(0,ishape*num_dim+1) = 0; - B_matrix_contribution(1,ishape*num_dim+1) = (-basis_derivative_s1(ishape)*(JT_row2(0)*JT_row3(2)-JT_row3(0)*JT_row2(2))+ - basis_derivative_s2(ishape)*(JT_row1(0)*JT_row3(2)-JT_row3(0)*JT_row1(2))- - basis_derivative_s3(ishape)*(JT_row1(0)*JT_row2(2)-JT_row2(0)*JT_row1(2))); - B_matrix_contribution(2,ishape*num_dim+1) = 0; - B_matrix_contribution(3,ishape*num_dim+1) = (basis_derivative_s1(ishape)*(JT_row2(1)*JT_row3(2)-JT_row3(1)*JT_row2(2))- - basis_derivative_s2(ishape)*(JT_row1(1)*JT_row3(2)-JT_row3(1)*JT_row1(2))+ - basis_derivative_s3(ishape)*(JT_row1(1)*JT_row2(2)-JT_row2(1)*JT_row1(2))); - B_matrix_contribution(4,ishape*num_dim+1) = 0; - B_matrix_contribution(5,ishape*num_dim+1) = (basis_derivative_s1(ishape)*(JT_row2(0)*JT_row3(1)-JT_row3(0)*JT_row2(1))- - basis_derivative_s2(ishape)*(JT_row1(0)*JT_row3(1)-JT_row3(0)*JT_row1(1))+ - basis_derivative_s3(ishape)*(JT_row1(0)*JT_row2(1)-JT_row2(0)*JT_row1(1))); - B_matrix_contribution(0,ishape*num_dim+2) = 0; - B_matrix_contribution(1,ishape*num_dim+2) = 0; - B_matrix_contribution(2,ishape*num_dim+2) = (basis_derivative_s1(ishape)*(JT_row2(0)*JT_row3(1)-JT_row3(0)*JT_row2(1))- - basis_derivative_s2(ishape)*(JT_row1(0)*JT_row3(1)-JT_row3(0)*JT_row1(1))+ - basis_derivative_s3(ishape)*(JT_row1(0)*JT_row2(1)-JT_row2(0)*JT_row1(1))); - B_matrix_contribution(3,ishape*num_dim+2) = 0; - B_matrix_contribution(4,ishape*num_dim+2) = (basis_derivative_s1(ishape)*(JT_row2(1)*JT_row3(2)-JT_row3(1)*JT_row2(2))- - basis_derivative_s2(ishape)*(JT_row1(1)*JT_row3(2)-JT_row3(1)*JT_row1(2))+ - basis_derivative_s3(ishape)*(JT_row1(1)*JT_row2(2)-JT_row2(1)*JT_row1(2))); - B_matrix_contribution(5,ishape*num_dim+2) = (-basis_derivative_s1(ishape)*(JT_row2(0)*JT_row3(2)-JT_row3(0)*JT_row2(2))+ - basis_derivative_s2(ishape)*(JT_row1(0)*JT_row3(2)-JT_row3(0)*JT_row1(2))- - basis_derivative_s3(ishape)*(JT_row1(0)*JT_row2(2)-JT_row2(0)*JT_row1(2))); - } - if(num_dim==3) - for(int ishape=0; ishape < nodes_per_elem; ishape++){ - B_matrix_contribution(0,ishape*num_dim) = (basis_derivative_s1(ishape)*(JT_row2(1)*JT_row3(2)-JT_row3(1)*JT_row2(2))- - basis_derivative_s2(ishape)*(JT_row1(1)*JT_row3(2)-JT_row3(1)*JT_row1(2))+ - basis_derivative_s3(ishape)*(JT_row1(1)*JT_row2(2)-JT_row2(1)*JT_row1(2))); - B_matrix_contribution(1,ishape*num_dim) = 0; - B_matrix_contribution(2,ishape*num_dim) = 0; - B_matrix_contribution(3,ishape*num_dim) = (-basis_derivative_s1(ishape)*(JT_row2(0)*JT_row3(2)-JT_row3(0)*JT_row2(2))+ - basis_derivative_s2(ishape)*(JT_row1(0)*JT_row3(2)-JT_row3(0)*JT_row1(2))- - basis_derivative_s3(ishape)*(JT_row1(0)*JT_row2(2)-JT_row2(0)*JT_row1(2))); - B_matrix_contribution(4,ishape*num_dim) = (basis_derivative_s1(ishape)*(JT_row2(0)*JT_row3(1)-JT_row3(0)*JT_row2(1))- - basis_derivative_s2(ishape)*(JT_row1(0)*JT_row3(1)-JT_row3(0)*JT_row1(1))+ - basis_derivative_s3(ishape)*(JT_row1(0)*JT_row2(1)-JT_row2(0)*JT_row1(1))); - B_matrix_contribution(5,ishape*num_dim) = 0; - B_matrix_contribution(0,ishape*num_dim+1) = 0; - B_matrix_contribution(1,ishape*num_dim+1) = (-basis_derivative_s1(ishape)*(JT_row2(0)*JT_row3(2)-JT_row3(0)*JT_row2(2))+ - basis_derivative_s2(ishape)*(JT_row1(0)*JT_row3(2)-JT_row3(0)*JT_row1(2))- - basis_derivative_s3(ishape)*(JT_row1(0)*JT_row2(2)-JT_row2(0)*JT_row1(2))); - B_matrix_contribution(2,ishape*num_dim+1) = 0; - B_matrix_contribution(3,ishape*num_dim+1) = (basis_derivative_s1(ishape)*(JT_row2(1)*JT_row3(2)-JT_row3(1)*JT_row2(2))- - basis_derivative_s2(ishape)*(JT_row1(1)*JT_row3(2)-JT_row3(1)*JT_row1(2))+ - basis_derivative_s3(ishape)*(JT_row1(1)*JT_row2(2)-JT_row2(1)*JT_row1(2))); - B_matrix_contribution(4,ishape*num_dim+1) = 0; - B_matrix_contribution(5,ishape*num_dim+1) = (basis_derivative_s1(ishape)*(JT_row2(0)*JT_row3(1)-JT_row3(0)*JT_row2(1))- - basis_derivative_s2(ishape)*(JT_row1(0)*JT_row3(1)-JT_row3(0)*JT_row1(1))+ - basis_derivative_s3(ishape)*(JT_row1(0)*JT_row2(1)-JT_row2(0)*JT_row1(1))); - B_matrix_contribution(0,ishape*num_dim+2) = 0; - B_matrix_contribution(1,ishape*num_dim+2) = 0; - B_matrix_contribution(2,ishape*num_dim+2) = (basis_derivative_s1(ishape)*(JT_row2(0)*JT_row3(1)-JT_row3(0)*JT_row2(1))- - basis_derivative_s2(ishape)*(JT_row1(0)*JT_row3(1)-JT_row3(0)*JT_row1(1))+ - basis_derivative_s3(ishape)*(JT_row1(0)*JT_row2(1)-JT_row2(0)*JT_row1(1))); - B_matrix_contribution(3,ishape*num_dim+2) = 0; - B_matrix_contribution(4,ishape*num_dim+2) = (basis_derivative_s1(ishape)*(JT_row2(1)*JT_row3(2)-JT_row3(1)*JT_row2(2))- - basis_derivative_s2(ishape)*(JT_row1(1)*JT_row3(2)-JT_row3(1)*JT_row1(2))+ - basis_derivative_s3(ishape)*(JT_row1(1)*JT_row2(2)-JT_row2(1)*JT_row1(2))); - B_matrix_contribution(5,ishape*num_dim+2) = (-basis_derivative_s1(ishape)*(JT_row2(0)*JT_row3(2)-JT_row3(0)*JT_row2(2))+ - basis_derivative_s2(ishape)*(JT_row1(0)*JT_row3(2)-JT_row3(0)*JT_row1(2))- - basis_derivative_s3(ishape)*(JT_row1(0)*JT_row2(2)-JT_row2(0)*JT_row1(2))); - } - - //look up element material properties at this point as a function of density - Gradient_Element_Material_Properties(ielem, Element_Modulus_Gradient, Poisson_Ratio, current_density); - Elastic_Constant = Element_Modulus_Gradient/((1 + Poisson_Ratio)*(1 - 2*Poisson_Ratio)); - Shear_Term = 0.5 - Poisson_Ratio; - Pressure_Term = 1 - Poisson_Ratio; - - //debug print - //std::cout << "Element Material Params " << Elastic_Constant << std::endl; - - //compute Elastic (C) matrix - if(num_dim==2){ - C_matrix(0,0) = Pressure_Term; - C_matrix(1,1) = Pressure_Term; - C_matrix(0,1) = Poisson_Ratio; - C_matrix(1,0) = Poisson_Ratio; - C_matrix(2,2) = Shear_Term; - } - if(num_dim==3){ - C_matrix(0,0) = Pressure_Term; - C_matrix(1,1) = Pressure_Term; - C_matrix(2,2) = Pressure_Term; - C_matrix(0,1) = Poisson_Ratio; - C_matrix(0,2) = Poisson_Ratio; - C_matrix(1,0) = Poisson_Ratio; - C_matrix(1,2) = Poisson_Ratio; - C_matrix(2,0) = Poisson_Ratio; - C_matrix(2,1) = Poisson_Ratio; - C_matrix(3,3) = Shear_Term; - C_matrix(4,4) = Shear_Term; - C_matrix(5,5) = Shear_Term; - } + } - //compute the previous multiplied by the Elastic (C) Matrix - for(int irow=0; irow < Brows; irow++){ - for(int icol=0; icol < num_dim*nodes_per_elem; icol++){ - CB_matrix_contribution(irow,icol) = 0; - for(int span=0; span < Brows; span++){ - CB_matrix_contribution(irow,icol) += C_matrix(irow,span)*B_matrix_contribution(span,icol); - } - } - } - - //compute the contributions of this quadrature point to all the local stiffness matrix elements - for(int ifill=0; ifill < num_dim*nodes_per_elem; ifill++){ - for(int jfill=ifill; jfill < num_dim*nodes_per_elem; jfill++){ - matrix_term = 0; - for(int span = 0; span < Brows; span++){ - matrix_term += B_matrix_contribution(span,ifill)*CB_matrix_contribution(span,jfill); - } - Local_Matrix_Contribution(ifill,jfill) = matrix_term; - if(ifill!=jfill) - Local_Matrix_Contribution(jfill,ifill) = Local_Matrix_Contribution(ifill,jfill); + for (unsigned long cycle = 0; cycle < last_time_step + 1; cycle++) + { + // compute timestep from time data + global_dt = time_data[cycle + 1] - time_data[cycle]; + + // print + if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) + { + if (cycle == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } } - } + // print time step every 10 cycles + else if (cycle % 20 == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } // end if + } - //compute inner product for this quadrature point contribution - inner_product = 0; - for(int ifill=0; ifill < num_dim*nodes_per_elem; ifill++){ - for(int jfill=0; jfill < num_dim*nodes_per_elem; jfill++){ - inner_product += Local_Matrix_Contribution(ifill, jfill)*current_element_adjoint(ifill)*current_nodal_displacements(jfill); - //debug - //if(Local_Matrix_Contribution(ifill, jfill)<0) Local_Matrix_Contribution(ifill, jfill) = - Local_Matrix_Contribution(ifill, jfill); - //inner_product += Local_Matrix_Contribution(ifill, jfill); + // view scope + { + // const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); + const_host_vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_host_vec_array next_adjoint_vector = (*adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + + const_host_vec_array next_coordinate_vector = (*forward_solve_coordinate_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_host_vec_array current_coordinate_vector = (*forward_solve_coordinate_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + + // interface of arrays for current implementation of force calculation + for (size_t ielem = 0; ielem < rnum_elem; ielem++) + { + nodes_per_elem = elem->num_basis(); + + // initialize C matrix + for (int irow = 0; irow < Brows; irow++) + { + for (int icol = 0; icol < Brows; icol++) + { + C_matrix(irow, icol) = 0; + } + } + + // B matrix initialization + for (int irow = 0; irow < Brows; irow++) + { + for (int icol = 0; icol < num_dim * nodes_per_elem; icol++) + { + CB_matrix(irow, icol) = 0; + } + } + + // acquire set of nodes and nodal displacements for this local element + for (int node_loop = 0; node_loop < nodes_per_elem; node_loop++) + { + local_node_id = nodes_in_elem.host(ielem, convert_node_order(node_loop)); + local_dof_idx = nodes_in_elem.host(ielem, convert_node_order(node_loop)) * num_dim; + local_dof_idy = local_dof_idx + 1; + local_dof_idz = local_dof_idx + 2; + + nodal_positions(node_loop, 0) = all_initial_node_coords(local_node_id, 0); + nodal_positions(node_loop, 1) = all_initial_node_coords(local_node_id, 1); + current_nodal_displacements(node_loop * num_dim) = 0.5 * (next_coordinate_vector(local_node_id, 0) - all_initial_node_coords(local_node_id, + 0) + current_coordinate_vector(local_node_id, 0) - all_initial_node_coords(local_node_id, 0)); + current_nodal_displacements(node_loop * num_dim + 1) = 0.5 * (next_coordinate_vector(local_node_id, 1) - all_initial_node_coords(local_node_id, + 1) + current_coordinate_vector(local_node_id, 1) - all_initial_node_coords(local_node_id, 1)); + current_element_adjoint(node_loop * num_dim) = 0.5 * (current_adjoint_vector(local_node_id, 0) + next_adjoint_vector(local_node_id, 0)); + current_element_adjoint(node_loop * num_dim + 1) = 0.5 * (current_adjoint_vector(local_node_id, 1) + next_adjoint_vector(local_node_id, 1)); + + if (num_dim == 3) + { + nodal_positions(node_loop, 2) = all_initial_node_coords(local_node_id, 2); + current_nodal_displacements(node_loop * num_dim + 2) = 0.5 * (next_coordinate_vector(local_node_id, 2) - all_initial_node_coords(local_node_id, + 2) + current_coordinate_vector(local_node_id, 2) - all_initial_node_coords(local_node_id, 2)); + current_element_adjoint(node_loop * num_dim + 2) = 0.5 * (current_adjoint_vector(local_node_id, 2) + next_adjoint_vector(local_node_id, 2)); + } + + if (nodal_density_flag) + { + nodal_density(node_loop) = all_node_densities(local_node_id, 0); + } + // debug print + /* + std::cout << "node index access x "<< local_node_id << std::endl; + std::cout << "local index access x "<< local_dof_idx << " displacement x " << current_nodal_displacements(node_loop*num_dim) <basis(basis_values, quad_coordinate); + + // compute all the necessary coordinates and derivatives at this point + // compute shape function derivatives + elem->partial_xi_basis(basis_derivative_s1, quad_coordinate); + elem->partial_eta_basis(basis_derivative_s2, quad_coordinate); + elem->partial_mu_basis(basis_derivative_s3, quad_coordinate); + + // compute derivatives of x,y,z w.r.t the s,t,w isoparametric space needed by JT (Transpose of the Jacobian) + // derivative of x,y,z w.r.t s + JT_row1(0) = 0; + JT_row1(1) = 0; + JT_row1(2) = 0; + for (int node_loop = 0; node_loop < nodes_per_elem; node_loop++) + { + JT_row1(0) += nodal_positions(node_loop, 0) * basis_derivative_s1(node_loop); + JT_row1(1) += nodal_positions(node_loop, 1) * basis_derivative_s1(node_loop); + JT_row1(2) += nodal_positions(node_loop, 2) * basis_derivative_s1(node_loop); + } + + // derivative of x,y,z w.r.t t + JT_row2(0) = 0; + JT_row2(1) = 0; + JT_row2(2) = 0; + for (int node_loop = 0; node_loop < nodes_per_elem; node_loop++) + { + JT_row2(0) += nodal_positions(node_loop, 0) * basis_derivative_s2(node_loop); + JT_row2(1) += nodal_positions(node_loop, 1) * basis_derivative_s2(node_loop); + JT_row2(2) += nodal_positions(node_loop, 2) * basis_derivative_s2(node_loop); + } + + // derivative of x,y,z w.r.t w + JT_row3(0) = 0; + JT_row3(1) = 0; + JT_row3(2) = 0; + for (int node_loop = 0; node_loop < nodes_per_elem; node_loop++) + { + JT_row3(0) += nodal_positions(node_loop, 0) * basis_derivative_s3(node_loop); + JT_row3(1) += nodal_positions(node_loop, 1) * basis_derivative_s3(node_loop); + JT_row3(2) += nodal_positions(node_loop, 2) * basis_derivative_s3(node_loop); + } + + // compute the determinant of the Jacobian + Jacobian = JT_row1(0) * (JT_row2(1) * JT_row3(2) - JT_row3(1) * JT_row2(2)) - + JT_row1(1) * (JT_row2(0) * JT_row3(2) - JT_row3(0) * JT_row2(2)) + + JT_row1(2) * (JT_row2(0) * JT_row3(1) - JT_row3(0) * JT_row2(1)); + if (Jacobian < 0) + { + Jacobian = -Jacobian; + } + invJacobian = 1 / Jacobian; + + // compute density + current_density = relative_element_densities.host(ielem); + + // debug print + // std::cout << "Current Density " << current_density << std::endl; + + // compute the contributions of this quadrature point to the B matrix + if (num_dim == 2) + { + for (int ishape = 0; ishape < nodes_per_elem; ishape++) + { + B_matrix_contribution(0, ishape * num_dim) = (basis_derivative_s1(ishape) * (JT_row2(1) * JT_row3(2) - JT_row3(1) * JT_row2(2)) - + basis_derivative_s2(ishape) * (JT_row1(1) * JT_row3(2) - JT_row3(1) * JT_row1(2)) + + basis_derivative_s3(ishape) * (JT_row1(1) * JT_row2(2) - JT_row2(1) * JT_row1(2))); + B_matrix_contribution(1, ishape * num_dim) = 0; + B_matrix_contribution(2, ishape * num_dim) = 0; + B_matrix_contribution(3, ishape * num_dim) = (-basis_derivative_s1(ishape) * (JT_row2(0) * JT_row3(2) - JT_row3(0) * JT_row2(2)) + + basis_derivative_s2(ishape) * (JT_row1(0) * JT_row3(2) - JT_row3(0) * JT_row1(2)) - + basis_derivative_s3(ishape) * (JT_row1(0) * JT_row2(2) - JT_row2(0) * JT_row1(2))); + B_matrix_contribution(4, ishape * num_dim) = (basis_derivative_s1(ishape) * (JT_row2(0) * JT_row3(1) - JT_row3(0) * JT_row2(1)) - + basis_derivative_s2(ishape) * (JT_row1(0) * JT_row3(1) - JT_row3(0) * JT_row1(1)) + + basis_derivative_s3(ishape) * (JT_row1(0) * JT_row2(1) - JT_row2(0) * JT_row1(1))); + B_matrix_contribution(5, ishape * num_dim) = 0; + B_matrix_contribution(0, ishape * num_dim + 1) = 0; + B_matrix_contribution(1, ishape * num_dim + 1) = (-basis_derivative_s1(ishape) * (JT_row2(0) * JT_row3(2) - JT_row3(0) * JT_row2(2)) + + basis_derivative_s2(ishape) * (JT_row1(0) * JT_row3(2) - JT_row3(0) * JT_row1(2)) - + basis_derivative_s3(ishape) * (JT_row1(0) * JT_row2(2) - JT_row2(0) * JT_row1(2))); + B_matrix_contribution(2, ishape * num_dim + 1) = 0; + B_matrix_contribution(3, ishape * num_dim + 1) = (basis_derivative_s1(ishape) * (JT_row2(1) * JT_row3(2) - JT_row3(1) * JT_row2(2)) - + basis_derivative_s2(ishape) * (JT_row1(1) * JT_row3(2) - JT_row3(1) * JT_row1(2)) + + basis_derivative_s3(ishape) * (JT_row1(1) * JT_row2(2) - JT_row2(1) * JT_row1(2))); + B_matrix_contribution(4, ishape * num_dim + 1) = 0; + B_matrix_contribution(5, ishape * num_dim + 1) = (basis_derivative_s1(ishape) * (JT_row2(0) * JT_row3(1) - JT_row3(0) * JT_row2(1)) - + basis_derivative_s2(ishape) * (JT_row1(0) * JT_row3(1) - JT_row3(0) * JT_row1(1)) + + basis_derivative_s3(ishape) * (JT_row1(0) * JT_row2(1) - JT_row2(0) * JT_row1(1))); + B_matrix_contribution(0, ishape * num_dim + 2) = 0; + B_matrix_contribution(1, ishape * num_dim + 2) = 0; + B_matrix_contribution(2, ishape * num_dim + 2) = (basis_derivative_s1(ishape) * (JT_row2(0) * JT_row3(1) - JT_row3(0) * JT_row2(1)) - + basis_derivative_s2(ishape) * (JT_row1(0) * JT_row3(1) - JT_row3(0) * JT_row1(1)) + + basis_derivative_s3(ishape) * (JT_row1(0) * JT_row2(1) - JT_row2(0) * JT_row1(1))); + B_matrix_contribution(3, ishape * num_dim + 2) = 0; + B_matrix_contribution(4, ishape * num_dim + 2) = (basis_derivative_s1(ishape) * (JT_row2(1) * JT_row3(2) - JT_row3(1) * JT_row2(2)) - + basis_derivative_s2(ishape) * (JT_row1(1) * JT_row3(2) - JT_row3(1) * JT_row1(2)) + + basis_derivative_s3(ishape) * (JT_row1(1) * JT_row2(2) - JT_row2(1) * JT_row1(2))); + B_matrix_contribution(5, ishape * num_dim + 2) = (-basis_derivative_s1(ishape) * (JT_row2(0) * JT_row3(2) - JT_row3(0) * JT_row2(2)) + + basis_derivative_s2(ishape) * (JT_row1(0) * JT_row3(2) - JT_row3(0) * JT_row1(2)) - + basis_derivative_s3(ishape) * (JT_row1(0) * JT_row2(2) - JT_row2(0) * JT_row1(2))); + } + } + if (num_dim == 3) + { + for (int ishape = 0; ishape < nodes_per_elem; ishape++) + { + B_matrix_contribution(0, ishape * num_dim) = (basis_derivative_s1(ishape) * (JT_row2(1) * JT_row3(2) - JT_row3(1) * JT_row2(2)) - + basis_derivative_s2(ishape) * (JT_row1(1) * JT_row3(2) - JT_row3(1) * JT_row1(2)) + + basis_derivative_s3(ishape) * (JT_row1(1) * JT_row2(2) - JT_row2(1) * JT_row1(2))); + B_matrix_contribution(1, ishape * num_dim) = 0; + B_matrix_contribution(2, ishape * num_dim) = 0; + B_matrix_contribution(3, ishape * num_dim) = (-basis_derivative_s1(ishape) * (JT_row2(0) * JT_row3(2) - JT_row3(0) * JT_row2(2)) + + basis_derivative_s2(ishape) * (JT_row1(0) * JT_row3(2) - JT_row3(0) * JT_row1(2)) - + basis_derivative_s3(ishape) * (JT_row1(0) * JT_row2(2) - JT_row2(0) * JT_row1(2))); + B_matrix_contribution(4, ishape * num_dim) = (basis_derivative_s1(ishape) * (JT_row2(0) * JT_row3(1) - JT_row3(0) * JT_row2(1)) - + basis_derivative_s2(ishape) * (JT_row1(0) * JT_row3(1) - JT_row3(0) * JT_row1(1)) + + basis_derivative_s3(ishape) * (JT_row1(0) * JT_row2(1) - JT_row2(0) * JT_row1(1))); + B_matrix_contribution(5, ishape * num_dim) = 0; + B_matrix_contribution(0, ishape * num_dim + 1) = 0; + B_matrix_contribution(1, ishape * num_dim + 1) = (-basis_derivative_s1(ishape) * (JT_row2(0) * JT_row3(2) - JT_row3(0) * JT_row2(2)) + + basis_derivative_s2(ishape) * (JT_row1(0) * JT_row3(2) - JT_row3(0) * JT_row1(2)) - + basis_derivative_s3(ishape) * (JT_row1(0) * JT_row2(2) - JT_row2(0) * JT_row1(2))); + B_matrix_contribution(2, ishape * num_dim + 1) = 0; + B_matrix_contribution(3, ishape * num_dim + 1) = (basis_derivative_s1(ishape) * (JT_row2(1) * JT_row3(2) - JT_row3(1) * JT_row2(2)) - + basis_derivative_s2(ishape) * (JT_row1(1) * JT_row3(2) - JT_row3(1) * JT_row1(2)) + + basis_derivative_s3(ishape) * (JT_row1(1) * JT_row2(2) - JT_row2(1) * JT_row1(2))); + B_matrix_contribution(4, ishape * num_dim + 1) = 0; + B_matrix_contribution(5, ishape * num_dim + 1) = (basis_derivative_s1(ishape) * (JT_row2(0) * JT_row3(1) - JT_row3(0) * JT_row2(1)) - + basis_derivative_s2(ishape) * (JT_row1(0) * JT_row3(1) - JT_row3(0) * JT_row1(1)) + + basis_derivative_s3(ishape) * (JT_row1(0) * JT_row2(1) - JT_row2(0) * JT_row1(1))); + B_matrix_contribution(0, ishape * num_dim + 2) = 0; + B_matrix_contribution(1, ishape * num_dim + 2) = 0; + B_matrix_contribution(2, ishape * num_dim + 2) = (basis_derivative_s1(ishape) * (JT_row2(0) * JT_row3(1) - JT_row3(0) * JT_row2(1)) - + basis_derivative_s2(ishape) * (JT_row1(0) * JT_row3(1) - JT_row3(0) * JT_row1(1)) + + basis_derivative_s3(ishape) * (JT_row1(0) * JT_row2(1) - JT_row2(0) * JT_row1(1))); + B_matrix_contribution(3, ishape * num_dim + 2) = 0; + B_matrix_contribution(4, ishape * num_dim + 2) = (basis_derivative_s1(ishape) * (JT_row2(1) * JT_row3(2) - JT_row3(1) * JT_row2(2)) - + basis_derivative_s2(ishape) * (JT_row1(1) * JT_row3(2) - JT_row3(1) * JT_row1(2)) + + basis_derivative_s3(ishape) * (JT_row1(1) * JT_row2(2) - JT_row2(1) * JT_row1(2))); + B_matrix_contribution(5, ishape * num_dim + 2) = (-basis_derivative_s1(ishape) * (JT_row2(0) * JT_row3(2) - JT_row3(0) * JT_row2(2)) + + basis_derivative_s2(ishape) * (JT_row1(0) * JT_row3(2) - JT_row3(0) * JT_row1(2)) - + basis_derivative_s3(ishape) * (JT_row1(0) * JT_row2(2) - JT_row2(0) * JT_row1(2))); + } + } + + // look up element material properties at this point as a function of density + Gradient_Element_Material_Properties(ielem, Element_Modulus_Gradient, Poisson_Ratio, current_density); + Elastic_Constant = Element_Modulus_Gradient / ((1 + Poisson_Ratio) * (1 - 2 * Poisson_Ratio)); + Shear_Term = 0.5 - Poisson_Ratio; + Pressure_Term = 1 - Poisson_Ratio; + + // debug print + // std::cout << "Element Material Params " << Elastic_Constant << std::endl; + + // compute Elastic (C) matrix + if (num_dim == 2) + { + C_matrix(0, 0) = Pressure_Term; + C_matrix(1, 1) = Pressure_Term; + C_matrix(0, 1) = Poisson_Ratio; + C_matrix(1, 0) = Poisson_Ratio; + C_matrix(2, 2) = Shear_Term; + } + if (num_dim == 3) + { + C_matrix(0, 0) = Pressure_Term; + C_matrix(1, 1) = Pressure_Term; + C_matrix(2, 2) = Pressure_Term; + C_matrix(0, 1) = Poisson_Ratio; + C_matrix(0, 2) = Poisson_Ratio; + C_matrix(1, 0) = Poisson_Ratio; + C_matrix(1, 2) = Poisson_Ratio; + C_matrix(2, 0) = Poisson_Ratio; + C_matrix(2, 1) = Poisson_Ratio; + C_matrix(3, 3) = Shear_Term; + C_matrix(4, 4) = Shear_Term; + C_matrix(5, 5) = Shear_Term; + } + + // compute the previous multiplied by the Elastic (C) Matrix + for (int irow = 0; irow < Brows; irow++) + { + for (int icol = 0; icol < num_dim * nodes_per_elem; icol++) + { + CB_matrix_contribution(irow, icol) = 0; + for (int span = 0; span < Brows; span++) + { + CB_matrix_contribution(irow, icol) += C_matrix(irow, span) * B_matrix_contribution(span, icol); + } + } + } + + // compute the contributions of this quadrature point to all the local stiffness matrix elements + for (int ifill = 0; ifill < num_dim * nodes_per_elem; ifill++) + { + for (int jfill = ifill; jfill < num_dim * nodes_per_elem; jfill++) + { + matrix_term = 0; + for (int span = 0; span < Brows; span++) + { + matrix_term += B_matrix_contribution(span, ifill) * CB_matrix_contribution(span, jfill); + } + Local_Matrix_Contribution(ifill, jfill) = matrix_term; + if (ifill != jfill) + { + Local_Matrix_Contribution(jfill, ifill) = Local_Matrix_Contribution(ifill, jfill); + } + } + } + + // compute inner product for this quadrature point contribution + inner_product = 0; + for (int ifill = 0; ifill < num_dim * nodes_per_elem; ifill++) + { + for (int jfill = 0; jfill < num_dim * nodes_per_elem; jfill++) + { + inner_product += Local_Matrix_Contribution(ifill, jfill) * current_element_adjoint(ifill) * current_nodal_displacements(jfill); + // debug + // if(Local_Matrix_Contribution(ifill, jfill)<0) Local_Matrix_Contribution(ifill, jfill) = - Local_Matrix_Contribution(ifill, jfill); + // inner_product += Local_Matrix_Contribution(ifill, jfill); + } + } + + // evaluate local stiffness matrix gradient with respect to igradient + for (int igradient = 0; igradient < nodes_per_elem; igradient++) + { + if (!map->isNodeLocalElement(nodes_in_elem.host(ielem, igradient))) + { + continue; + } + local_node_id = nodes_in_elem.host(ielem, igradient); + + // debug print + // std::cout << "contribution for " << igradient + 1 << " is " << inner_product << std::endl; + design_gradients(local_node_id, 0) -= -inner_product * Elastic_Constant * weight_multiply * invJacobian * global_dt / ((double)num_nodes_in_elem); + } + } } - } - - //evaluate local stiffness matrix gradient with respect to igradient - for(int igradient=0; igradient < nodes_per_elem; igradient++){ - if(!map->isNodeLocalElement(nodes_in_elem.host(ielem, igradient))) continue; - local_node_id = nodes_in_elem.host(ielem, igradient); - - //debug print - //std::cout << "contribution for " << igradient + 1 << " is " << inner_product << std::endl; - design_gradients(local_node_id,0) -= -inner_product*Elastic_Constant*weight_multiply*invJacobian*global_dt/((double)num_nodes_in_elem); - } } - } + // end view scope } - //end view scope - } } diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/force_gradients_elastic.cpp b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/force_gradients_elastic.cpp index e22e888ae..5e8999635 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/force_gradients_elastic.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/force_gradients_elastic.cpp @@ -5,7 +5,7 @@ #include #include #include -#include +#include #include // fmin, fmax, abs note: fminl is long #include #include @@ -22,88 +22,80 @@ #include "Simulation_Parameters/Simulation_Parameters_Explicit.h" #include "FEA_Module_Dynamic_Elasticity.h" - // ----------------------------------------------------------------------------- // This function calculates the corner forces and the evolves stress (hypo) -//------------------------------------------------------------------------------ -void FEA_Module_Dynamic_Elasticity::get_force_vgradient_elastic(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &elem_mat_id, - const double rk_alpha, - const size_t cycle - ){ - +// ------------------------------------------------------------------------------ +void FEA_Module_Dynamic_Elasticity::get_force_vgradient_elastic(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_pres, + const DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + const double rk_alpha, + const size_t cycle + ) +{ const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; const size_t num_dims = simparam->num_dims; // --- calculate the forces acting on the nodes from the element --- - FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { - - + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { const size_t num_nodes_in_elem = 8; - + // total Cauchy stress double tau_array[9]; - + // corner area normals double area_normal_array[24]; - + // estimate of shock direction double shock_dir_array[3]; - + // the sums in the Riemann solver double sum_array[4]; - + // corner shock impeadance x |corner area normal dot shock_dir| double muc_array[8]; - + // Riemann velocity double vel_star_array[3]; - + // velocity gradient double vel_grad_array[9]; - + // --- Create views of arrays to aid the force calculation --- - - ViewCArrayKokkos tau(tau_array, num_dims, num_dims); - ViewCArrayKokkos area_normal(area_normal_array, num_nodes_in_elem, num_dims); - ViewCArrayKokkos shock_dir(shock_dir_array, num_dims); - ViewCArrayKokkos sum(sum_array, 4); - ViewCArrayKokkos muc(muc_array, num_nodes_in_elem); - ViewCArrayKokkos vel_star(vel_star_array, num_dims); - ViewCArrayKokkos vel_grad(vel_grad_array, num_dims, num_dims); - - + + ViewCArrayKokkos tau(tau_array, num_dims, num_dims); + ViewCArrayKokkos area_normal(area_normal_array, num_nodes_in_elem, num_dims); + ViewCArrayKokkos shock_dir(shock_dir_array, num_dims); + ViewCArrayKokkos sum(sum_array, 4); + ViewCArrayKokkos muc(muc_array, num_nodes_in_elem); + ViewCArrayKokkos vel_star(vel_star_array, num_dims); + ViewCArrayKokkos vel_grad(vel_grad_array, num_dims, num_dims); + // --- abviatations of variables --- - + // element volume double vol = elem_vol(elem_gid); - + // create a view of the stress_matrix - ViewCArrayKokkos stress(&elem_stress(rk_level, elem_gid, 0,0), 3, 3); - - + ViewCArrayKokkos stress(&elem_stress(rk_level, elem_gid, 0, 0), 3, 3); + // cut out the node_gids for this element - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 8); - - - + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 8); + // get the B matrix which are the OUTWARD corner area normals get_bmatrix(area_normal, elem_gid, node_coords, elem_node_gids, rk_level); - - + // --- Calculate the velocity gradient --- get_velgrad(vel_grad, elem_node_gids, @@ -112,76 +104,74 @@ void FEA_Module_Dynamic_Elasticity::get_force_vgradient_elastic(const DCArrayKok vol, elem_gid, rk_level); - - + // the -1 is for the inward surface area normal, - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - for (size_t dim = 0; dim < num_dims; dim++){ - area_normal(node_lid, dim) = (-1.0)*area_normal(node_lid,dim); + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + for (size_t dim = 0; dim < num_dims; dim++) + { + area_normal(node_lid, dim) = (-1.0) * area_normal(node_lid, dim); } // end for } // end for - - - + double div = elem_div(elem_gid); - - + // vel = [u,v,w] // [du/dx, du/dy, du/dz] // vel_grad = [dv/dx, dv/dy, dv/dz] // [dw/dx, dw/dy, dw/dz] double curl[3]; - curl[0] = vel_grad(2,1) - vel_grad(1,2); // dw/dy - dv/dz - curl[1] = vel_grad(0,2) - vel_grad(2,0); // du/dz - dw/dx - curl[2] = vel_grad(1,0) - vel_grad(0,1); // dv/dx - du/dy - - double mag_curl = sqrt(curl[0]*curl[0] + curl[1]*curl[1] + curl[2]*curl[2]); - - + curl[0] = vel_grad(2, 1) - vel_grad(1, 2); // dw/dy - dv/dz + curl[1] = vel_grad(0, 2) - vel_grad(2, 0); // du/dz - dw/dx + curl[2] = vel_grad(1, 0) - vel_grad(0, 1); // dv/dx - du/dy + + double mag_curl = sqrt(curl[0] * curl[0] + curl[1] * curl[1] + curl[2] * curl[2]); + // --- Calculate the Cauchy stress --- - for (size_t i = 0; i < 3; i++){ - for (size_t j = 0; j < 3; j++){ - tau(i, j) = stress(i,j); + for (size_t i = 0; i < 3; i++) + { + for (size_t j = 0; j < 3; j++) + { + tau(i, j) = stress(i, j); // artificial viscosity can be added here to tau } // end for - } //end for + } // end for // add the pressure - for (int i = 0; i < num_dims; i++){ + for (int i = 0; i < num_dims; i++) + { tau(i, i) -= elem_pres(elem_gid); } // end for - - - // ---- Multidirectional Approximate Riemann solver (MARS) ---- // find the average velocity of the elem, it is an // estimate of the Riemann velocity - + // initialize to Riemann velocity to zero - for (size_t dim = 0; dim < num_dims; dim++){ + for (size_t dim = 0; dim < num_dims; dim++) + { vel_star(dim) = 0.0; } // loop over nodes and calculate an average velocity, which is // an estimate of Riemann velocity - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { // Get node gloabl index and create view of nodal velocity int node_gid = nodes_in_elem(elem_gid, node_lid); - - ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); - - vel_star(0) += 0.125*vel(0); - vel_star(1) += 0.125*vel(1); - vel_star(2) += 0.125*vel(2); - + + ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); + + vel_star(0) += 0.125 * vel(0); + vel_star(1) += 0.125 * vel(1); + vel_star(2) += 0.125 * vel(2); } // end for loop over nodes // find shock direction and shock impedance associated with each node - + // initialize sum term in MARS to zero - for (int i = 0; i < 4; i++){ + for (int i = 0; i < 4; i++) + { sum(i) = 0.0; } @@ -189,103 +179,102 @@ void FEA_Module_Dynamic_Elasticity::get_force_vgradient_elastic(const DCArrayKok double mag_vel; // magnitude of velocity // loop over the nodes of the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { - + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { // Get global node id size_t node_gid = nodes_in_elem(elem_gid, node_lid); // Create view of nodal velocity - ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); + ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); // Get an estimate of the shock direction. - mag_vel = sqrt( (vel(0) - vel_star(0) )*(vel(0) - vel_star(0) ) - + (vel(1) - vel_star(1) )*(vel(1) - vel_star(1) ) - + (vel(2) - vel_star(2) )*(vel(2) - vel_star(2) ) ); + mag_vel = sqrt( (vel(0) - vel_star(0) ) * (vel(0) - vel_star(0) ) + + (vel(1) - vel_star(1) ) * (vel(1) - vel_star(1) ) + + (vel(2) - vel_star(2) ) * (vel(2) - vel_star(2) ) ); - - if (mag_vel > small) { - + if (mag_vel > small) + { // estimate of the shock direction, a unit normal - for (int dim = 0; dim < num_dims; dim++){ + for (int dim = 0; dim < num_dims; dim++) + { shock_dir(dim) = (vel(dim) - vel_star(dim)) / mag_vel; } } - - else { - + else + { // if there is no velocity change, then use the surface area // normal as the shock direction - mag = sqrt( area_normal(node_lid, 0)*area_normal(node_lid, 0) - + area_normal(node_lid, 1)*area_normal(node_lid, 1) - + area_normal(node_lid, 2)*area_normal(node_lid, 2) ); - + mag = sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) + + area_normal(node_lid, 1) * area_normal(node_lid, 1) + + area_normal(node_lid, 2) * area_normal(node_lid, 2) ); + // estimate of the shock direction - for (int dim = 0; dim < num_dims; dim++){ - shock_dir(dim) = area_normal(node_lid, dim)/mag; + for (int dim = 0; dim < num_dims; dim++) + { + shock_dir(dim) = area_normal(node_lid, dim) / mag; } - } // end if mag_vel - // cell divergence indicates compression or expansions size_t mat_id = elem_mat_id(elem_gid); - if (div < 0){ // element in compression + if (div < 0) // element in compression + { muc(node_lid) = elem_den(elem_gid) * - (material(mat_id).q1*elem_sspd(elem_gid) + material(mat_id).q2*mag_vel); + (material(mat_id).q1 * elem_sspd(elem_gid) + material(mat_id).q2 * mag_vel); } - else { // element in expansion + else // element in expansion + { muc(node_lid) = elem_den(elem_gid) * - (material(mat_id).q1ex*elem_sspd(elem_gid) + material(mat_id).q2ex*mag_vel); + (material(mat_id).q1ex * elem_sspd(elem_gid) + material(mat_id).q2ex * mag_vel); } // end if on divergence sign - size_t use_shock_dir = 0; double mu_term; - + // Coding to use shock direction - if (use_shock_dir == 1){ + if (use_shock_dir == 1) + { // this is denominator of the Riamann solver and the multiplier // on velocity in the numerator. It filters on the shock // direction - mu_term = muc(node_lid)* - fabs( shock_dir(0)*area_normal(node_lid,0) - + shock_dir(1)*area_normal(node_lid,1) - + shock_dir(2)*area_normal(node_lid,2) ); + mu_term = muc(node_lid) * + fabs(shock_dir(0) * area_normal(node_lid, 0) + + shock_dir(1) * area_normal(node_lid, 1) + + shock_dir(2) * area_normal(node_lid, 2) ); } - else { - // Using a full tensoral Riemann jump relation - mu_term = muc(node_lid) - * sqrt( area_normal(node_lid, 0)*area_normal(node_lid, 0) - + area_normal(node_lid, 1)*area_normal(node_lid, 1) - + area_normal(node_lid, 2)*area_normal(node_lid, 2) ); + else + { + // Using a full tensoral Riemann jump relation + mu_term = muc(node_lid) + * sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) + + area_normal(node_lid, 1) * area_normal(node_lid, 1) + + area_normal(node_lid, 2) * area_normal(node_lid, 2) ); } - - sum(0) += mu_term*vel(0); - sum(1) += mu_term*vel(1); - sum(2) += mu_term*vel(2); + + sum(0) += mu_term * vel(0); + sum(1) += mu_term * vel(1); + sum(2) += mu_term * vel(2); sum(3) += mu_term; muc(node_lid) = mu_term; // the impeadance time surface area is stored here - } // end for node_lid loop over nodes of the elem - - - // The Riemann velocity, called vel_star - if (sum(3) > fuzz) { - for (size_t i = 0; i < num_dims; i++) { - vel_star(i) = sum(i)/sum(3); + if (sum(3) > fuzz) + { + for (size_t i = 0; i < num_dims; i++) + { + vel_star(i) = sum(i) / sum(3); } } - else { - for (int i = 0; i < num_dims; i++){ + else + { + for (int i = 0; i < num_dims; i++) + { vel_star(i) = 0.0; } } // end if - - // ---- Calculate the shock detector for the Riemann-solver ---- // // The dissipation from the Riemann problem is limited by phi @@ -298,7 +287,7 @@ void FEA_Module_Dynamic_Elasticity::get_force_vgradient_elastic(const DCArrayKok // phi = 0 highest-order solution // phi = 1 first order solution // - + double phi = 0.0; // the shock detector double r_face = 1.0; // the ratio on the face double r_min = 1.0; // the min ratio for the cell @@ -307,92 +296,83 @@ void FEA_Module_Dynamic_Elasticity::get_force_vgradient_elastic(const DCArrayKok double n_coef = 1.0; // the power on the limiting coefficient // (1=nominal, and n_coeff > 1 oscillatory) - // loop over the nieghboring cells - for (size_t elem_lid = 0; elem_lid < num_elems_in_elem(elem_gid); elem_lid++){ - + for (size_t elem_lid = 0; elem_lid < num_elems_in_elem(elem_gid); elem_lid++) + { // Get global index for neighboring cell size_t neighbor_gid = elems_in_elem(elem_gid, elem_lid); - + // calculate the velocity divergence in neighbor double div_neighbor = elem_div(neighbor_gid); - r_face = r_coef*(div_neighbor + small)/(div + small); + r_face = r_coef * (div_neighbor + small) / (div + small); // store the smallest face ratio r_min = fmin(r_face, r_min); - } // end for elem_lid - // calculate standard shock detector phi = 1.0 - fmax(0.0, r_min); phi = pow(phi, n_coef); // Mach number shock detector - double omega = 20.0;//20.0; // weighting factor on Mach number - double third = 1.0/3.0; + double omega = 20.0; // 20.0; // weighting factor on Mach number + double third = 1.0 / 3.0; double c_length = pow(vol, third); // characteristic length - double alpha = fmin(1.0, omega * (c_length * fabs(div))/(elem_sspd(elem_gid) + fuzz) ); - + double alpha = fmin(1.0, omega * (c_length * fabs(div)) / (elem_sspd(elem_gid) + fuzz) ); + // use Mach based detector with standard shock detector // turn off dissipation in expansion - //alpha = fmax(-fabs(div0)/div0 * alpha, 0.0); // this should be if(div0<0) alpha=alpha else alpha=0 - - phi = alpha*phi; - + // alpha = fmax(-fabs(div0)/div0 * alpha, 0.0); // this should be if(div0<0) alpha=alpha else alpha=0 + + phi = alpha * phi; + // curl limiter on Q - double phi_curl = fmin(1.0, 1.0*fabs(div)/(mag_curl + fuzz)); // disable Q when vorticity is high - //phi = phi_curl*phi; + double phi_curl = fmin(1.0, 1.0 * fabs(div) / (mag_curl + fuzz)); // disable Q when vorticity is high + // phi = phi_curl*phi; phi = 0; // ---- Calculate the Riemann force on each node ---- // loop over the each node in the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { - + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { size_t corner_lid = node_lid; // Get corner gid size_t corner_gid = corners_in_elem(elem_gid, corner_lid); - + // Get node gid size_t node_gid = nodes_in_elem(elem_gid, node_lid); - - // loop over dimension - for (int dim = 0; dim < num_dims; dim++){ + // loop over dimension + for (int dim = 0; dim < num_dims; dim++) + { corner_vector_storage(corner_gid, dim) = -0; - //corner_vector_storage(corner_gid, dim) = 0; - + // corner_vector_storage(corner_gid, dim) = 0; } // end loop over dimension - } // end for loop over nodes in elem - - - + // --- Update Stress --- // calculate the new stress at the next rk level, if it is a hypo model - - size_t mat_id = elem_mat_id(elem_gid); - - + size_t mat_id = elem_mat_id(elem_gid); }); // end parallel for loop over elements - //accumulate node values from corner storage - //force_gradient_velocity->putScalar(0); - //force_gradient_position->putScalar(0); - //set back to zero - FOR_ALL_CLASS(idof, 0, nlocal_nodes*num_dims, { - for(int jdof = 0; jdof < Gradient_Matrix_Strides(idof); jdof++){ - Force_Gradient_Velocities(idof,jdof) = 0; + // accumulate node values from corner storage + // force_gradient_velocity->putScalar(0); + // force_gradient_position->putScalar(0); + // set back to zero + FOR_ALL_CLASS(idof, 0, nlocal_nodes * num_dims, { + for (int jdof = 0; jdof < Gradient_Matrix_Strides(idof); jdof++) + { + Force_Gradient_Velocities(idof, jdof) = 0; } }); // end parallel for Kokkos::fence(); - - //vec_array force_gradient_velocity_view = force_gradient_velocity->getLocalView (Tpetra::Access::ReadWrite); + + // vec_array force_gradient_velocity_view = force_gradient_velocity->getLocalView (Tpetra::Access::ReadWrite); /* FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { size_t corner_id; @@ -405,101 +385,92 @@ void FEA_Module_Dynamic_Elasticity::get_force_vgradient_elastic(const DCArrayKok }); // end parallel for Kokkos::fence(); */ - + const real_t damping_constant = module_params->damping_constant; FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { - - Force_Gradient_Velocities(node_id*num_dims,0) = -damping_constant; - Force_Gradient_Velocities(node_id*num_dims+1,0) = -damping_constant; - Force_Gradient_Velocities(node_id*num_dims+2,0) = -damping_constant; + Force_Gradient_Velocities(node_id * num_dims, 0) = -damping_constant; + Force_Gradient_Velocities(node_id * num_dims + 1, 0) = -damping_constant; + Force_Gradient_Velocities(node_id * num_dims + 2, 0) = -damping_constant; }); // end parallel for Kokkos::fence(); - + return; - } // end of routine // ----------------------------------------------------------------------------- // This function calculates the corner forces and the evolves stress (hypo) -//------------------------------------------------------------------------------ -void FEA_Module_Dynamic_Elasticity::get_force_ugradient_elastic(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &elem_mat_id, - const double rk_alpha, - const size_t cycle - ){ - +// ------------------------------------------------------------------------------ +void FEA_Module_Dynamic_Elasticity::get_force_ugradient_elastic(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_pres, + const DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + const double rk_alpha, + const size_t cycle + ) +{ const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; const size_t num_dims = simparam->num_dims; // --- calculate the forces acting on the nodes from the element --- - FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { - - + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { const size_t num_nodes_in_elem = 8; - + // total Cauchy stress double tau_array[9]; - + // corner area normals double area_normal_array[24]; - + // estimate of shock direction double shock_dir_array[3]; - + // the sums in the Riemann solver double sum_array[4]; - + // corner shock impeadance x |corner area normal dot shock_dir| double muc_array[8]; - + // Riemann velocity double vel_star_array[3]; - + // velocity gradient double vel_grad_array[9]; - + // --- Create views of arrays to aid the force calculation --- - - ViewCArrayKokkos tau(tau_array, num_dims, num_dims); - ViewCArrayKokkos area_normal(area_normal_array, num_nodes_in_elem, num_dims); - ViewCArrayKokkos shock_dir(shock_dir_array, num_dims); - ViewCArrayKokkos sum(sum_array, 4); - ViewCArrayKokkos muc(muc_array, num_nodes_in_elem); - ViewCArrayKokkos vel_star(vel_star_array, num_dims); - ViewCArrayKokkos vel_grad(vel_grad_array, num_dims, num_dims); - - + + ViewCArrayKokkos tau(tau_array, num_dims, num_dims); + ViewCArrayKokkos area_normal(area_normal_array, num_nodes_in_elem, num_dims); + ViewCArrayKokkos shock_dir(shock_dir_array, num_dims); + ViewCArrayKokkos sum(sum_array, 4); + ViewCArrayKokkos muc(muc_array, num_nodes_in_elem); + ViewCArrayKokkos vel_star(vel_star_array, num_dims); + ViewCArrayKokkos vel_grad(vel_grad_array, num_dims, num_dims); + // --- abviatations of variables --- - + // element volume double vol = elem_vol(elem_gid); - + // create a view of the stress_matrix - ViewCArrayKokkos stress(&elem_stress(rk_level, elem_gid, 0,0), 3, 3); - - + ViewCArrayKokkos stress(&elem_stress(rk_level, elem_gid, 0, 0), 3, 3); + // cut out the node_gids for this element - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 8); - - - + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 8); + // get the B matrix which are the OUTWARD corner area normals get_bmatrix(area_normal, elem_gid, node_coords, elem_node_gids, rk_level); - - + // --- Calculate the velocity gradient --- get_velgrad(vel_grad, elem_node_gids, @@ -508,77 +479,74 @@ void FEA_Module_Dynamic_Elasticity::get_force_ugradient_elastic(const DCArrayKok vol, elem_gid, rk_level); - - + // the -1 is for the inward surface area normal, - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - for (size_t dim = 0; dim < num_dims; dim++){ - area_normal(node_lid, dim) = (-1.0)*area_normal(node_lid,dim); + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + for (size_t dim = 0; dim < num_dims; dim++) + { + area_normal(node_lid, dim) = (-1.0) * area_normal(node_lid, dim); } // end for } // end for - - - + double div = elem_div(elem_gid); - - + // vel = [u,v,w] // [du/dx, du/dy, du/dz] // vel_grad = [dv/dx, dv/dy, dv/dz] // [dw/dx, dw/dy, dw/dz] double curl[3]; - curl[0] = vel_grad(2,1) - vel_grad(1,2); // dw/dy - dv/dz - curl[1] = vel_grad(0,2) - vel_grad(2,0); // du/dz - dw/dx - curl[2] = vel_grad(1,0) - vel_grad(0,1); // dv/dx - du/dy - - double mag_curl = sqrt(curl[0]*curl[0] + curl[1]*curl[1] + curl[2]*curl[2]); - - + curl[0] = vel_grad(2, 1) - vel_grad(1, 2); // dw/dy - dv/dz + curl[1] = vel_grad(0, 2) - vel_grad(2, 0); // du/dz - dw/dx + curl[2] = vel_grad(1, 0) - vel_grad(0, 1); // dv/dx - du/dy + + double mag_curl = sqrt(curl[0] * curl[0] + curl[1] * curl[1] + curl[2] * curl[2]); + // --- Calculate the Cauchy stress --- - for (size_t i = 0; i < 3; i++){ - for (size_t j = 0; j < 3; j++){ - tau(i, j) = stress(i,j); + for (size_t i = 0; i < 3; i++) + { + for (size_t j = 0; j < 3; j++) + { + tau(i, j) = stress(i, j); // artificial viscosity can be added here to tau } // end for - } //end for + } // end for // add the pressure - for (int i = 0; i < num_dims; i++){ + for (int i = 0; i < num_dims; i++) + { tau(i, i) -= elem_pres(elem_gid); } // end for - - - // ---- Multidirectional Approximate Riemann solver (MARS) ---- // find the average velocity of the elem, it is an // estimate of the Riemann velocity - + // initialize to Riemann velocity to zero - for (size_t dim = 0; dim < num_dims; dim++){ + for (size_t dim = 0; dim < num_dims; dim++) + { vel_star(dim) = 0.0; } // loop over nodes and calculate an average velocity, which is // an estimate of Riemann velocity - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { // Get node gloabl index and create view of nodal velocity int node_gid = nodes_in_elem(elem_gid, node_lid); - - ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); - - vel_star(0) += 0.125*vel(0); - vel_star(1) += 0.125*vel(1); - vel_star(2) += 0.125*vel(2); - + ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); + + vel_star(0) += 0.125 * vel(0); + vel_star(1) += 0.125 * vel(1); + vel_star(2) += 0.125 * vel(2); } // end for loop over nodes // find shock direction and shock impedance associated with each node - + // initialize sum term in MARS to zero - for (int i = 0; i < 4; i++){ + for (int i = 0; i < 4; i++) + { sum(i) = 0.0; } @@ -586,103 +554,102 @@ void FEA_Module_Dynamic_Elasticity::get_force_ugradient_elastic(const DCArrayKok double mag_vel; // magnitude of velocity // loop over the nodes of the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { - + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { // Get global node id size_t node_gid = nodes_in_elem(elem_gid, node_lid); // Create view of nodal velocity - ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); + ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); // Get an estimate of the shock direction. - mag_vel = sqrt( (vel(0) - vel_star(0) )*(vel(0) - vel_star(0) ) - + (vel(1) - vel_star(1) )*(vel(1) - vel_star(1) ) - + (vel(2) - vel_star(2) )*(vel(2) - vel_star(2) ) ); + mag_vel = sqrt( (vel(0) - vel_star(0) ) * (vel(0) - vel_star(0) ) + + (vel(1) - vel_star(1) ) * (vel(1) - vel_star(1) ) + + (vel(2) - vel_star(2) ) * (vel(2) - vel_star(2) ) ); - - if (mag_vel > small) { - + if (mag_vel > small) + { // estimate of the shock direction, a unit normal - for (int dim = 0; dim < num_dims; dim++){ + for (int dim = 0; dim < num_dims; dim++) + { shock_dir(dim) = (vel(dim) - vel_star(dim)) / mag_vel; } } - - else { - + else + { // if there is no velocity change, then use the surface area // normal as the shock direction - mag = sqrt( area_normal(node_lid, 0)*area_normal(node_lid, 0) - + area_normal(node_lid, 1)*area_normal(node_lid, 1) - + area_normal(node_lid, 2)*area_normal(node_lid, 2) ); - + mag = sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) + + area_normal(node_lid, 1) * area_normal(node_lid, 1) + + area_normal(node_lid, 2) * area_normal(node_lid, 2) ); + // estimate of the shock direction - for (int dim = 0; dim < num_dims; dim++){ - shock_dir(dim) = area_normal(node_lid, dim)/mag; + for (int dim = 0; dim < num_dims; dim++) + { + shock_dir(dim) = area_normal(node_lid, dim) / mag; } - } // end if mag_vel - // cell divergence indicates compression or expansions size_t mat_id = elem_mat_id(elem_gid); - if (div < 0){ // element in compression + if (div < 0) // element in compression + { muc(node_lid) = elem_den(elem_gid) * - (material(mat_id).q1*elem_sspd(elem_gid) + material(mat_id).q2*mag_vel); + (material(mat_id).q1 * elem_sspd(elem_gid) + material(mat_id).q2 * mag_vel); } - else { // element in expansion + else // element in expansion + { muc(node_lid) = elem_den(elem_gid) * - (material(mat_id).q1ex*elem_sspd(elem_gid) + material(mat_id).q2ex*mag_vel); + (material(mat_id).q1ex * elem_sspd(elem_gid) + material(mat_id).q2ex * mag_vel); } // end if on divergence sign - size_t use_shock_dir = 0; double mu_term; - + // Coding to use shock direction - if (use_shock_dir == 1){ + if (use_shock_dir == 1) + { // this is denominator of the Riamann solver and the multiplier // on velocity in the numerator. It filters on the shock // direction - mu_term = muc(node_lid)* - fabs( shock_dir(0)*area_normal(node_lid,0) - + shock_dir(1)*area_normal(node_lid,1) - + shock_dir(2)*area_normal(node_lid,2) ); + mu_term = muc(node_lid) * + fabs(shock_dir(0) * area_normal(node_lid, 0) + + shock_dir(1) * area_normal(node_lid, 1) + + shock_dir(2) * area_normal(node_lid, 2) ); } - else { - // Using a full tensoral Riemann jump relation - mu_term = muc(node_lid) - * sqrt( area_normal(node_lid, 0)*area_normal(node_lid, 0) - + area_normal(node_lid, 1)*area_normal(node_lid, 1) - + area_normal(node_lid, 2)*area_normal(node_lid, 2) ); + else + { + // Using a full tensoral Riemann jump relation + mu_term = muc(node_lid) + * sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) + + area_normal(node_lid, 1) * area_normal(node_lid, 1) + + area_normal(node_lid, 2) * area_normal(node_lid, 2) ); } - - sum(0) += mu_term*vel(0); - sum(1) += mu_term*vel(1); - sum(2) += mu_term*vel(2); + + sum(0) += mu_term * vel(0); + sum(1) += mu_term * vel(1); + sum(2) += mu_term * vel(2); sum(3) += mu_term; muc(node_lid) = mu_term; // the impeadance time surface area is stored here - } // end for node_lid loop over nodes of the elem - - - // The Riemann velocity, called vel_star - if (sum(3) > fuzz) { - for (size_t i = 0; i < num_dims; i++) { - vel_star(i) = sum(i)/sum(3); + if (sum(3) > fuzz) + { + for (size_t i = 0; i < num_dims; i++) + { + vel_star(i) = sum(i) / sum(3); } } - else { - for (int i = 0; i < num_dims; i++){ + else + { + for (int i = 0; i < num_dims; i++) + { vel_star(i) = 0.0; } } // end if - - // ---- Calculate the shock detector for the Riemann-solver ---- // // The dissipation from the Riemann problem is limited by phi @@ -695,7 +662,7 @@ void FEA_Module_Dynamic_Elasticity::get_force_ugradient_elastic(const DCArrayKok // phi = 0 highest-order solution // phi = 1 first order solution // - + double phi = 0.0; // the shock detector double r_face = 1.0; // the ratio on the face double r_min = 1.0; // the min ratio for the cell @@ -704,299 +671,299 @@ void FEA_Module_Dynamic_Elasticity::get_force_ugradient_elastic(const DCArrayKok double n_coef = 1.0; // the power on the limiting coefficient // (1=nominal, and n_coeff > 1 oscillatory) - // loop over the nieghboring cells - for (size_t elem_lid = 0; elem_lid < num_elems_in_elem(elem_gid); elem_lid++){ - + for (size_t elem_lid = 0; elem_lid < num_elems_in_elem(elem_gid); elem_lid++) + { // Get global index for neighboring cell size_t neighbor_gid = elems_in_elem(elem_gid, elem_lid); - + // calculate the velocity divergence in neighbor double div_neighbor = elem_div(neighbor_gid); - r_face = r_coef*(div_neighbor + small)/(div + small); + r_face = r_coef * (div_neighbor + small) / (div + small); // store the smallest face ratio r_min = fmin(r_face, r_min); - } // end for elem_lid - // calculate standard shock detector phi = 1.0 - fmax(0.0, r_min); phi = pow(phi, n_coef); // Mach number shock detector - double omega = 20.0;//20.0; // weighting factor on Mach number - double third = 1.0/3.0; + double omega = 20.0; // 20.0; // weighting factor on Mach number + double third = 1.0 / 3.0; double c_length = pow(vol, third); // characteristic length - double alpha = fmin(1.0, omega * (c_length * fabs(div))/(elem_sspd(elem_gid) + fuzz) ); - + double alpha = fmin(1.0, omega * (c_length * fabs(div)) / (elem_sspd(elem_gid) + fuzz) ); + // use Mach based detector with standard shock detector // turn off dissipation in expansion - //alpha = fmax(-fabs(div0)/div0 * alpha, 0.0); // this should be if(div0<0) alpha=alpha else alpha=0 - - phi = alpha*phi; - + // alpha = fmax(-fabs(div0)/div0 * alpha, 0.0); // this should be if(div0<0) alpha=alpha else alpha=0 + + phi = alpha * phi; + // curl limiter on Q - double phi_curl = fmin(1.0, 1.0*fabs(div)/(mag_curl + fuzz)); // disable Q when vorticity is high - //phi = phi_curl*phi; + double phi_curl = fmin(1.0, 1.0 * fabs(div) / (mag_curl + fuzz)); // disable Q when vorticity is high + // phi = phi_curl*phi; phi = 0; // ---- Calculate the Riemann force on each node ---- // loop over the each node in the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { - + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { size_t corner_lid = node_lid; // Get corner gid size_t corner_gid = corners_in_elem(elem_gid, corner_lid); - + // Get node gid size_t node_gid = nodes_in_elem(elem_gid, node_lid); - - // loop over dimension - for (int dim = 0; dim < num_dims; dim++){ + // loop over dimension + for (int dim = 0; dim < num_dims; dim++) + { corner_vector_storage(corner_gid, dim) = -0.0001; - //corner_vector_storage(corner_gid, dim) = 0; - + // corner_vector_storage(corner_gid, dim) = 0; } // end loop over dimension - } // end for loop over nodes in elem - - - + // --- Update Stress --- // calculate the new stress at the next rk level, if it is a hypo model - - size_t mat_id = elem_mat_id(elem_gid); - - + size_t mat_id = elem_mat_id(elem_gid); }); // end parallel for loop over elements - //accumulate node values from corner storage - //force_gradient_position->putScalar(0); - //set back to zero - FOR_ALL_CLASS(idof, 0, nlocal_nodes*num_dims, { - for(int jdof = 0; jdof < Gradient_Matrix_Strides(idof); jdof++){ - Force_Gradient_Positions(idof,jdof) = 0; + // accumulate node values from corner storage + // force_gradient_position->putScalar(0); + // set back to zero + FOR_ALL_CLASS(idof, 0, nlocal_nodes * num_dims, { + for (int jdof = 0; jdof < Gradient_Matrix_Strides(idof); jdof++) + { + Force_Gradient_Positions(idof, jdof) = 0; } }); // end parallel for Kokkos::fence(); - - //vec_array force_gradient_position_view = force_gradient_position->getLocalView (Tpetra::Access::ReadWrite); + + // vec_array force_gradient_position_view = force_gradient_position->getLocalView (Tpetra::Access::ReadWrite); FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { size_t corner_id; - for(int icorner=0; icorner < num_corners_in_node(node_id); icorner++){ - corner_id = corners_in_node(node_id,icorner); - Force_Gradient_Positions(node_id*num_dims,0) += -0.001; - Force_Gradient_Positions(node_id*num_dims+1,0) += -0.001; - Force_Gradient_Positions(node_id*num_dims+2,0) += -0.001; + for (int icorner = 0; icorner < num_corners_in_node(node_id); icorner++) + { + corner_id = corners_in_node(node_id, icorner); + Force_Gradient_Positions(node_id * num_dims, 0) += -0.001; + Force_Gradient_Positions(node_id * num_dims + 1, 0) += -0.001; + Force_Gradient_Positions(node_id * num_dims + 2, 0) += -0.001; } }); // end parallel for Kokkos::fence(); - return; - } // end of routine // -------------------------------------------------------------------------------------- // Computes corner contribution of gradient of force with respect to the design variable -//--------------------------------------------------------------------------------------- - -void FEA_Module_Dynamic_Elasticity::force_design_gradient_term(const_vec_array design_variables, vec_array design_gradients){ - - size_t num_bdy_nodes = mesh->num_bdy_nodes; - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos material = simparam->material; - const int num_dim = simparam->num_dims; - int num_corners = rnum_elem*num_nodes_in_elem; - real_t global_dt; - bool element_constant_density = true; - size_t current_data_index, next_data_index; - CArrayKokkos current_element_adjoint = CArrayKokkos(num_nodes_in_elem,num_dim); - - //gradient contribution from gradient of Force vector with respect to design variable. - for (unsigned long cycle = 0; cycle < last_time_step+1; cycle++) { - //compute timestep from time data - global_dt = time_data[cycle+1] - time_data[cycle]; - - //print - if (cycle==0){ - if(myrank==0) - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } +// --------------------------------------------------------------------------------------- + +void FEA_Module_Dynamic_Elasticity::force_design_gradient_term(const_vec_array design_variables, vec_array design_gradients) +{ + size_t num_bdy_nodes = mesh->num_bdy_nodes; + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos material = simparam->material; + const int num_dim = simparam->num_dims; + int num_corners = rnum_elem * num_nodes_in_elem; + real_t global_dt; + bool element_constant_density = true; + size_t current_data_index, next_data_index; + CArrayKokkos current_element_adjoint = CArrayKokkos(num_nodes_in_elem, num_dim); + + // gradient contribution from gradient of Force vector with respect to design variable. + for (unsigned long cycle = 0; cycle < last_time_step + 1; cycle++) + { + // compute timestep from time data + global_dt = time_data[cycle + 1] - time_data[cycle]; + + // print + if (cycle == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } // print time step every 10 cycles - else if (cycle%20==0){ - if(myrank==0) - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } // end if - - //compute adjoint vector for this data point; use velocity midpoint - //view scope - { - //const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array next_adjoint_vector = (*adjoint_vector_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - //const_vec_array current_coord_vector = forward_solve_coordinate_data[cycle]->getLocalView (Tpetra::Access::ReadOnly); - //const_vec_array final_coordinates = forward_solve_coordinate_data[last_time_step+1]->getLocalView (Tpetra::Access::ReadOnly); - CArrayKokkos inner_products(num_nodes_in_elem); - FOR_ALL_CLASS(elem_id, 0, rnum_elem, { - size_t node_id; - size_t corner_id; - real_t inner_product; - //std::cout << elem_mass(elem_id) <getLocalView (Tpetra::Access::ReadOnly); + const_vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_adjoint_vector = (*adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + // const_vec_array current_coord_vector = forward_solve_coordinate_data[cycle]->getLocalView (Tpetra::Access::ReadOnly); + // const_vec_array final_coordinates = forward_solve_coordinate_data[last_time_step+1]->getLocalView (Tpetra::Access::ReadOnly); + CArrayKokkos inner_products(num_nodes_in_elem); + FOR_ALL_CLASS(elem_id, 0, rnum_elem, { + size_t node_id; + size_t corner_id; + real_t inner_product; + // std::cout << elem_mass(elem_id) < &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &elem_mat_id, - const double rk_alpha, - const size_t cycle - ) { - +// ------------------------------------------------------------------------------ +void FEA_Module_Dynamic_Elasticity::get_force_dgradient_elastic(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_pres, + const DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + const double rk_alpha, + const size_t cycle + ) +{ const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; const size_t num_dims = simparam->num_dims; // --- calculate the forces acting on the nodes from the element --- - FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { - + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { const size_t num_nodes_in_elem = 8; - + // total Cauchy stress double tau_array[9]; - + // corner area normals double area_normal_array[24]; - + // estimate of shock direction double shock_dir_array[3]; - + // the sums in the Riemann solver double sum_array[4]; - + // corner shock impeadance x |corner area normal dot shock_dir| double muc_array[8]; - + // Riemann velocity double vel_star_array[3]; - + // velocity gradient double vel_grad_array[9]; - + // --- Create views of arrays to aid the force calculation --- - - ViewCArrayKokkos tau(tau_array, num_dims, num_dims); - ViewCArrayKokkos area_normal(area_normal_array, num_nodes_in_elem, num_dims); - ViewCArrayKokkos shock_dir(shock_dir_array, num_dims); - ViewCArrayKokkos sum(sum_array, 4); - ViewCArrayKokkos muc(muc_array, num_nodes_in_elem); - ViewCArrayKokkos vel_star(vel_star_array, num_dims); - ViewCArrayKokkos vel_grad(vel_grad_array, num_dims, num_dims); - - + + ViewCArrayKokkos tau(tau_array, num_dims, num_dims); + ViewCArrayKokkos area_normal(area_normal_array, num_nodes_in_elem, num_dims); + ViewCArrayKokkos shock_dir(shock_dir_array, num_dims); + ViewCArrayKokkos sum(sum_array, 4); + ViewCArrayKokkos muc(muc_array, num_nodes_in_elem); + ViewCArrayKokkos vel_star(vel_star_array, num_dims); + ViewCArrayKokkos vel_grad(vel_grad_array, num_dims, num_dims); + // --- abviatations of variables --- - + // element volume double vol = elem_vol(elem_gid); - + // create a view of the stress_matrix - ViewCArrayKokkos stress(&elem_stress(rk_level, elem_gid, 0,0), 3, 3); - - + ViewCArrayKokkos stress(&elem_stress(rk_level, elem_gid, 0, 0), 3, 3); + // cut out the node_gids for this element - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 8); - - - + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 8); + // get the B matrix which are the OUTWARD corner area normals get_bmatrix(area_normal, elem_gid, node_coords, elem_node_gids, rk_level); - - + // --- Calculate the velocity gradient --- get_velgrad(vel_grad, elem_node_gids, @@ -1005,77 +972,74 @@ void FEA_Module_Dynamic_Elasticity::get_force_dgradient_elastic(const DCArrayKok vol, elem_gid, rk_level); - - + // the -1 is for the inward surface area normal, - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - for (size_t dim = 0; dim < num_dims; dim++){ - area_normal(node_lid, dim) = (-1.0)*area_normal(node_lid,dim); + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + for (size_t dim = 0; dim < num_dims; dim++) + { + area_normal(node_lid, dim) = (-1.0) * area_normal(node_lid, dim); } // end for } // end for - - - + double div = elem_div(elem_gid); - - + // vel = [u,v,w] // [du/dx, du/dy, du/dz] // vel_grad = [dv/dx, dv/dy, dv/dz] // [dw/dx, dw/dy, dw/dz] double curl[3]; - curl[0] = vel_grad(2,1) - vel_grad(1,2); // dw/dy - dv/dz - curl[1] = vel_grad(0,2) - vel_grad(2,0); // du/dz - dw/dx - curl[2] = vel_grad(1,0) - vel_grad(0,1); // dv/dx - du/dy - - double mag_curl = sqrt(curl[0]*curl[0] + curl[1]*curl[1] + curl[2]*curl[2]); - - + curl[0] = vel_grad(2, 1) - vel_grad(1, 2); // dw/dy - dv/dz + curl[1] = vel_grad(0, 2) - vel_grad(2, 0); // du/dz - dw/dx + curl[2] = vel_grad(1, 0) - vel_grad(0, 1); // dv/dx - du/dy + + double mag_curl = sqrt(curl[0] * curl[0] + curl[1] * curl[1] + curl[2] * curl[2]); + // --- Calculate the Cauchy stress --- - for (size_t i = 0; i < 3; i++){ - for (size_t j = 0; j < 3; j++){ - tau(i, j) = stress(i,j); + for (size_t i = 0; i < 3; i++) + { + for (size_t j = 0; j < 3; j++) + { + tau(i, j) = stress(i, j); // artificial viscosity can be added here to tau } // end for - } //end for + } // end for // add the pressure - for (int i = 0; i < num_dims; i++){ + for (int i = 0; i < num_dims; i++) + { tau(i, i) -= elem_pres(elem_gid); } // end for - - - // ---- Multidirectional Approximate Riemann solver (MARS) ---- // find the average velocity of the elem, it is an // estimate of the Riemann velocity - + // initialize to Riemann velocity to zero - for (size_t dim = 0; dim < num_dims; dim++){ + for (size_t dim = 0; dim < num_dims; dim++) + { vel_star(dim) = 0.0; } // loop over nodes and calculate an average velocity, which is // an estimate of Riemann velocity - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { // Get node gloabl index and create view of nodal velocity int node_gid = nodes_in_elem(elem_gid, node_lid); - - ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); - - vel_star(0) += 0.125*vel(0); - vel_star(1) += 0.125*vel(1); - vel_star(2) += 0.125*vel(2); - + ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); + + vel_star(0) += 0.125 * vel(0); + vel_star(1) += 0.125 * vel(1); + vel_star(2) += 0.125 * vel(2); } // end for loop over nodes // find shock direction and shock impedance associated with each node - + // initialize sum term in MARS to zero - for (int i = 0; i < 4; i++){ + for (int i = 0; i < 4; i++) + { sum(i) = 0.0; } @@ -1083,103 +1047,102 @@ void FEA_Module_Dynamic_Elasticity::get_force_dgradient_elastic(const DCArrayKok double mag_vel; // magnitude of velocity // loop over the nodes of the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { - + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { // Get global node id size_t node_gid = nodes_in_elem(elem_gid, node_lid); // Create view of nodal velocity - ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); + ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); // Get an estimate of the shock direction. - mag_vel = sqrt( (vel(0) - vel_star(0) )*(vel(0) - vel_star(0) ) - + (vel(1) - vel_star(1) )*(vel(1) - vel_star(1) ) - + (vel(2) - vel_star(2) )*(vel(2) - vel_star(2) ) ); + mag_vel = sqrt( (vel(0) - vel_star(0) ) * (vel(0) - vel_star(0) ) + + (vel(1) - vel_star(1) ) * (vel(1) - vel_star(1) ) + + (vel(2) - vel_star(2) ) * (vel(2) - vel_star(2) ) ); - - if (mag_vel > small) { - + if (mag_vel > small) + { // estimate of the shock direction, a unit normal - for (int dim = 0; dim < num_dims; dim++){ + for (int dim = 0; dim < num_dims; dim++) + { shock_dir(dim) = (vel(dim) - vel_star(dim)) / mag_vel; } } - - else { - + else + { // if there is no velocity change, then use the surface area // normal as the shock direction - mag = sqrt( area_normal(node_lid, 0)*area_normal(node_lid, 0) - + area_normal(node_lid, 1)*area_normal(node_lid, 1) - + area_normal(node_lid, 2)*area_normal(node_lid, 2) ); - + mag = sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) + + area_normal(node_lid, 1) * area_normal(node_lid, 1) + + area_normal(node_lid, 2) * area_normal(node_lid, 2) ); + // estimate of the shock direction - for (int dim = 0; dim < num_dims; dim++){ - shock_dir(dim) = area_normal(node_lid, dim)/mag; + for (int dim = 0; dim < num_dims; dim++) + { + shock_dir(dim) = area_normal(node_lid, dim) / mag; } - } // end if mag_vel - // cell divergence indicates compression or expansions size_t mat_id = elem_mat_id(elem_gid); - if (div < 0){ // element in compression + if (div < 0) // element in compression + { muc(node_lid) = elem_den(elem_gid) * - (material(mat_id).q1*elem_sspd(elem_gid) + material(mat_id).q2*mag_vel); + (material(mat_id).q1 * elem_sspd(elem_gid) + material(mat_id).q2 * mag_vel); } - else { // element in expansion + else // element in expansion + { muc(node_lid) = elem_den(elem_gid) * - (material(mat_id).q1ex*elem_sspd(elem_gid) + material(mat_id).q2ex*mag_vel); + (material(mat_id).q1ex * elem_sspd(elem_gid) + material(mat_id).q2ex * mag_vel); } // end if on divergence sign - size_t use_shock_dir = 0; double mu_term; - + // Coding to use shock direction - if (use_shock_dir == 1){ + if (use_shock_dir == 1) + { // this is denominator of the Riamann solver and the multiplier // on velocity in the numerator. It filters on the shock // direction - mu_term = muc(node_lid)* - fabs( shock_dir(0)*area_normal(node_lid,0) - + shock_dir(1)*area_normal(node_lid,1) - + shock_dir(2)*area_normal(node_lid,2) ); + mu_term = muc(node_lid) * + fabs(shock_dir(0) * area_normal(node_lid, 0) + + shock_dir(1) * area_normal(node_lid, 1) + + shock_dir(2) * area_normal(node_lid, 2) ); } - else { - // Using a full tensoral Riemann jump relation - mu_term = muc(node_lid) - * sqrt( area_normal(node_lid, 0)*area_normal(node_lid, 0) - + area_normal(node_lid, 1)*area_normal(node_lid, 1) - + area_normal(node_lid, 2)*area_normal(node_lid, 2) ); + else + { + // Using a full tensoral Riemann jump relation + mu_term = muc(node_lid) + * sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) + + area_normal(node_lid, 1) * area_normal(node_lid, 1) + + area_normal(node_lid, 2) * area_normal(node_lid, 2) ); } - - sum(0) += mu_term*vel(0); - sum(1) += mu_term*vel(1); - sum(2) += mu_term*vel(2); + + sum(0) += mu_term * vel(0); + sum(1) += mu_term * vel(1); + sum(2) += mu_term * vel(2); sum(3) += mu_term; muc(node_lid) = mu_term; // the impeadance time surface area is stored here - } // end for node_lid loop over nodes of the elem - - - // The Riemann velocity, called vel_star - if (sum(3) > fuzz) { - for (size_t i = 0; i < num_dims; i++) { - vel_star(i) = sum(i)/sum(3); + if (sum(3) > fuzz) + { + for (size_t i = 0; i < num_dims; i++) + { + vel_star(i) = sum(i) / sum(3); } } - else { - for (int i = 0; i < num_dims; i++){ + else + { + for (int i = 0; i < num_dims; i++) + { vel_star(i) = 0.0; } } // end if - - // ---- Calculate the shock detector for the Riemann-solver ---- // // The dissipation from the Riemann problem is limited by phi @@ -1192,7 +1155,7 @@ void FEA_Module_Dynamic_Elasticity::get_force_dgradient_elastic(const DCArrayKok // phi = 0 highest-order solution // phi = 1 first order solution // - + double phi = 0.0; // the shock detector double r_face = 1.0; // the ratio on the face double r_min = 1.0; // the min ratio for the cell @@ -1201,95 +1164,84 @@ void FEA_Module_Dynamic_Elasticity::get_force_dgradient_elastic(const DCArrayKok double n_coef = 1.0; // the power on the limiting coefficient // (1=nominal, and n_coeff > 1 oscillatory) - // loop over the nieghboring cells - for (size_t elem_lid = 0; elem_lid < num_elems_in_elem(elem_gid); elem_lid++){ - + for (size_t elem_lid = 0; elem_lid < num_elems_in_elem(elem_gid); elem_lid++) + { // Get global index for neighboring cell size_t neighbor_gid = elems_in_elem(elem_gid, elem_lid); - + // calculate the velocity divergence in neighbor double div_neighbor = elem_div(neighbor_gid); - r_face = r_coef*(div_neighbor + small)/(div + small); + r_face = r_coef * (div_neighbor + small) / (div + small); // store the smallest face ratio r_min = fmin(r_face, r_min); - } // end for elem_lid - // calculate standard shock detector phi = 1.0 - fmax(0.0, r_min); phi = pow(phi, n_coef); // Mach number shock detector - double omega = 20.0;//20.0; // weighting factor on Mach number - double third = 1.0/3.0; + double omega = 20.0; // 20.0; // weighting factor on Mach number + double third = 1.0 / 3.0; double c_length = pow(vol, third); // characteristic length - double alpha = fmin(1.0, omega * (c_length * fabs(div))/(elem_sspd(elem_gid) + fuzz) ); - + double alpha = fmin(1.0, omega * (c_length * fabs(div)) / (elem_sspd(elem_gid) + fuzz) ); + // use Mach based detector with standard shock detector // turn off dissipation in expansion - //alpha = fmax(-fabs(div0)/div0 * alpha, 0.0); // this should be if(div0<0) alpha=alpha else alpha=0 - - phi = alpha*phi; - + // alpha = fmax(-fabs(div0)/div0 * alpha, 0.0); // this should be if(div0<0) alpha=alpha else alpha=0 + + phi = alpha * phi; + // curl limiter on Q - double phi_curl = fmin(1.0, 1.0*fabs(div)/(mag_curl + fuzz)); // disable Q when vorticity is high - //phi = phi_curl*phi; + double phi_curl = fmin(1.0, 1.0 * fabs(div) / (mag_curl + fuzz)); // disable Q when vorticity is high + // phi = phi_curl*phi; phi = 0; // ---- Calculate the Riemann force on each node ---- // loop over the each node in the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { - + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { size_t corner_lid = node_lid; // Get corner gid size_t corner_gid = corners_in_elem(elem_gid, corner_lid); - + // Get node gid size_t node_gid = nodes_in_elem(elem_gid, node_lid); - - // loop over dimension - for (int dim = 0; dim < num_dims; dim++){ + // loop over dimension + for (int dim = 0; dim < num_dims; dim++) + { corner_vector_storage(corner_gid, dim) = 0.0001; - } // end loop over dimension - } // end for loop over nodes in elem - - - + // --- Update Stress --- // calculate the new stress at the next rk level, if it is a hypo model - - size_t mat_id = elem_mat_id(elem_gid); - - + size_t mat_id = elem_mat_id(elem_gid); }); // end parallel for loop over elements - //accumulate node values from corner storage + // accumulate node values from corner storage force_gradient_design->putScalar(0); - - vec_array force_gradient_design_view = force_gradient_design->getLocalView (Tpetra::Access::ReadWrite); + + vec_array force_gradient_design_view = force_gradient_design->getLocalView(Tpetra::Access::ReadWrite); FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { size_t corner_id; - for(int icorner=0; icorner < num_corners_in_node(node_id); icorner++){ - corner_id = corners_in_node(node_id,icorner); - force_gradient_design_view(node_id,0) += corner_vector_storage(corner_id, 0); - force_gradient_design_view(node_id,1) += corner_vector_storage(corner_id, 1); - force_gradient_design_view(node_id,2) += corner_vector_storage(corner_id, 2); + for (int icorner = 0; icorner < num_corners_in_node(node_id); icorner++) + { + corner_id = corners_in_node(node_id, icorner); + force_gradient_design_view(node_id, 0) += corner_vector_storage(corner_id, 0); + force_gradient_design_view(node_id, 1) += corner_vector_storage(corner_id, 1); + force_gradient_design_view(node_id, 2) += corner_vector_storage(corner_id, 2); } }); // end parallel for Kokkos::fence(); - return; - } // end of routine diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/geometry.cpp b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/geometry.cpp index 51aabc00e..7e4034a8a 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/geometry.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/geometry.cpp @@ -1,31 +1,28 @@ // ----------------------------------------------------------------------------- // This code handles the geometric information for the mesh for the SHG solver -//------------------------------------------------------------------------------ +// ------------------------------------------------------------------------------ #include "matar.h" #include "state.h" #include "FEA_Module_Dynamic_Elasticity.h" -void FEA_Module_Dynamic_Elasticity::update_position_elastic(double rk_alpha, - const size_t num_nodes, - DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel){ - +void FEA_Module_Dynamic_Elasticity::update_position_elastic(double rk_alpha, + const size_t num_nodes, + DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel) +{ const size_t rk_level = rk_num_bins - 1; - int num_dims = num_dims; + int num_dims = num_dims; // loop over all the nodes in the mesh FOR_ALL_CLASS(node_gid, 0, num_nodes, { - - for (int dim = 0; dim < num_dims; dim++){ - double half_vel = (node_vel(rk_level, node_gid, dim) + node_vel(0, node_gid, dim))*0.5; - node_coords(rk_level, node_gid, dim) = node_coords(0, node_gid, dim) + rk_alpha*dt*half_vel; + for (int dim = 0; dim < num_dims; dim++) + { + double half_vel = (node_vel(rk_level, node_gid, dim) + node_vel(0, node_gid, dim)) * 0.5; + node_coords(rk_level, node_gid, dim) = node_coords(0, node_gid, dim) + rk_alpha * dt * half_vel; } - }); // end parallel for over nodes - } // end subroutine - // ----------------------------------------------------------------------------- // This function claculates // B_p = J^{-T} \cdot (\nabla_{xi} \phi_p w @@ -36,303 +33,299 @@ void FEA_Module_Dynamic_Elasticity::update_position_elastic(double rk_alpha, // \nabla_{xi} is the gradient opperator in the reference coordinates // // B_p is the OUTWARD corner area normal at node p -//------------------------------------------------------------------------------ +// ------------------------------------------------------------------------------ KOKKOS_FUNCTION -void FEA_Module_Dynamic_Elasticity::get_bmatrix(const ViewCArrayKokkos &B_matrix, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const { - +void FEA_Module_Dynamic_Elasticity::get_bmatrix(const ViewCArrayKokkos& B_matrix, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const +{ const size_t num_nodes = 8; double x_array[8]; double y_array[8]; double z_array[8]; - + // x, y, z coordinates of elem vertices - auto x = ViewCArrayKokkos (x_array, num_nodes); - auto y = ViewCArrayKokkos (y_array, num_nodes); - auto z = ViewCArrayKokkos (z_array, num_nodes); + auto x = ViewCArrayKokkos(x_array, num_nodes); + auto y = ViewCArrayKokkos(y_array, num_nodes); + auto z = ViewCArrayKokkos(z_array, num_nodes); // get the coordinates of the nodes(rk,elem,node) in this element - for (int node_lid = 0; node_lid < num_nodes; node_lid++){ + for (int node_lid = 0; node_lid < num_nodes; node_lid++) + { x(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 0); y(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 1); z(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 2); } // end for - double twelth = 1./12.; - - B_matrix(0,0) = ( +y(1)*( -z(2) -z(3) +z(4) +z(5) ) - +y(2)*( +z(1) -z(3) ) - +y(3)*( +z(1) +z(2) -z(4) -z(7) ) - +y(4)*( -z(1) +z(3) -z(5) +z(7) ) - +y(5)*( -z(1) +z(4) ) - +y(7)*( +z(3) -z(4) ) )*twelth; - - B_matrix(1,0) = ( +y(0)*( +z(2) +z(3) -z(4) -z(5) ) - +y(2)*( -z(0) -z(3) +z(5) +z(6) ) - +y(3)*( -z(0) +z(2) ) - +y(4)*( +z(0) -z(5) ) - +y(5)*( +z(0) -z(2) +z(4) -z(6) ) - +y(6)*( -z(2) +z(5) ) )*twelth; - - B_matrix(2,0) = ( +y(0)*( -z(1) +z(3) ) - +y(1)*( +z(0) +z(3) -z(5) -z(6) ) - +y(3)*( -z(0) -z(1) +z(6) +z(7) ) - +y(5)*( +z(1) -z(6) ) - +y(6)*( +z(1) -z(3) +z(5) -z(7) ) - +y(7)*( -z(3) +z(6) ) )*twelth; - - B_matrix(3,0) = ( +y(0)*( -z(1) -z(2) +z(4) +z(7) ) - +y(1)*( +z(0) -z(2) ) - +y(2)*( +z(0) +z(1) -z(6) -z(7) ) - +y(4)*( -z(0) +z(7) ) - +y(6)*( +z(2) -z(7) ) - +y(7)*( -z(0) +z(2) -z(4) +z(6) ) )*twelth; - - B_matrix(4,0) = ( +y(0)*( +z(1) -z(3) +z(5) -z(7) ) - +y(1)*( -z(0) +z(5) ) - +y(3)*( +z(0) -z(7) ) - +y(5)*( -z(0) -z(1) +z(6) +z(7) ) - +y(6)*( -z(5) +z(7) ) - +y(7)*( +z(0) +z(3) -z(5) -z(6) ) )*twelth; - - B_matrix(5,0) = ( +y(0)*( +z(1) -z(4) ) - +y(1)*( -z(0) +z(2) -z(4) +z(6) ) - +y(2)*( -z(1) +z(6) ) - +y(4)*( +z(0) +z(1) -z(6) -z(7) ) - +y(6)*( -z(1) -z(2) +z(4) +z(7) ) - +y(7)*( +z(4) -z(6) ) )*twelth; - - B_matrix(6,0) = ( +y(1)*( +z(2) -z(5) ) - +y(2)*( -z(1) +z(3) -z(5) +z(7) ) - +y(3)*( -z(2) +z(7) ) - +y(4)*( +z(5) -z(7) ) - +y(5)*( +z(1) +z(2) -z(4) -z(7) ) - +y(7)*( -z(2) -z(3) +z(4) +z(5) ) )*twelth; - - B_matrix(7,0) = ( +y(0)*( -z(3) +z(4) ) - +y(2)*( +z(3) -z(6) ) - +y(3)*( +z(0) -z(2) +z(4) -z(6) ) - +y(4)*( -z(0) -z(3) +z(5) +z(6) ) - +y(5)*( -z(4) +z(6) ) - +y(6)*( +z(2) +z(3) -z(4) -z(5) ) )*twelth; - - B_matrix(0,1) = ( +z(1)*( -x(2) -x(3) +x(4) +x(5) ) - +z(2)*( +x(1) -x(3) ) - +z(3)*( +x(1) +x(2) -x(4) -x(7) ) - +z(4)*( -x(1) +x(3) -x(5) +x(7) ) - +z(5)*( -x(1) +x(4) ) - +z(7)*( +x(3) -x(4) ) )*twelth; - - B_matrix(1,1) = ( +z(0)*( +x(2) +x(3) -x(4) -x(5) ) - +z(2)*( -x(0) -x(3) +x(5) +x(6) ) - +z(3)*( -x(0) +x(2) ) - +z(4)*( +x(0) -x(5) ) - +z(5)*( +x(0) -x(2) +x(4) -x(6) ) - +z(6)*( -x(2) +x(5) ) )*twelth; - - B_matrix(2,1) = ( +z(0)*( -x(1) +x(3) ) - +z(1)*( +x(0) +x(3) -x(5) -x(6) ) - +z(3)*( -x(0) -x(1) +x(6) +x(7) ) - +z(5)*( +x(1) -x(6) ) - +z(6)*( +x(1) -x(3) +x(5) -x(7) ) - +z(7)*( -x(3) +x(6) ) )*twelth; - - B_matrix(3,1) = ( +z(0)*( -x(1) -x(2) +x(4) +x(7) ) - +z(1)*( +x(0) -x(2) ) - +z(2)*( +x(0) +x(1) -x(6) -x(7) ) - +z(4)*( -x(0) +x(7) ) - +z(6)*( +x(2) -x(7) ) - +z(7)*( -x(0) +x(2) -x(4) +x(6) ) )*twelth; - - B_matrix(4,1) = ( +z(0)*( +x(1) -x(3) +x(5) -x(7) ) - +z(1)*( -x(0) +x(5) ) - +z(3)*( +x(0) -x(7) ) - +z(5)*( -x(0) -x(1) +x(6) +x(7) ) - +z(6)*( -x(5) +x(7) ) - +z(7)*( +x(0) +x(3) -x(5) -x(6) ) )*twelth; - - B_matrix(5,1) = ( +z(0)*( +x(1) -x(4) ) - +z(1)*( -x(0) +x(2) -x(4) +x(6) ) - +z(2)*( -x(1) +x(6) ) - +z(4)*( +x(0) +x(1) -x(6) -x(7) ) - +z(6)*( -x(1) -x(2) +x(4) +x(7) ) - +z(7)*( +x(4) -x(6) ) )*twelth; - - B_matrix(6,1) = ( +z(1)*( +x(2) -x(5) ) - +z(2)*( -x(1) +x(3) -x(5) +x(7) ) - +z(3)*( -x(2) +x(7) ) - +z(4)*( +x(5) -x(7) ) - +z(5)*( +x(1) +x(2) -x(4) -x(7) ) - +z(7)*( -x(2) -x(3) +x(4) +x(5) ) )*twelth; - - B_matrix(7,1) = ( +z(0)*( -x(3) +x(4) ) - +z(2)*( +x(3) -x(6) ) - +z(3)*( +x(0) -x(2) +x(4) -x(6) ) - +z(4)*( -x(0) -x(3) +x(5) +x(6) ) - +z(5)*( -x(4) +x(6) ) - +z(6)*( +x(2) +x(3) -x(4) -x(5) ) )*twelth; - - B_matrix(0,2) = ( +x(1)*( -y(2) -y(3) +y(4) +y(5) ) - +x(2)*( +y(1) -y(3) ) - +x(3)*( +y(1) +y(2) -y(4) -y(7) ) - +x(4)*( -y(1) +y(3) -y(5) +y(7) ) - +x(5)*( -y(1) +y(4) ) - +x(7)*( +y(3) -y(4) ) )*twelth; - - B_matrix(1,2) = ( +x(0)*( +y(2) +y(3) -y(4) -y(5) ) - +x(2)*( -y(0) -y(3) +y(5) +y(6) ) - +x(3)*( -y(0) +y(2) ) - +x(4)*( +y(0) -y(5) ) - +x(5)*( +y(0) -y(2) +y(4) -y(6) ) - +x(6)*( -y(2) +y(5) ) )*twelth; - - B_matrix(2,2) = ( +x(0)*( -y(1) +y(3) ) - +x(1)*( +y(0) +y(3) -y(5) -y(6) ) - +x(3)*( -y(0) -y(1) +y(6) +y(7) ) - +x(5)*( +y(1) -y(6) ) - +x(6)*( +y(1) -y(3) +y(5) -y(7) ) - +x(7)*( -y(3) +y(6) ) )*twelth; - - B_matrix(3,2) = ( +x(0)*( -y(1) -y(2) +y(4) +y(7) ) - +x(1)*( +y(0) -y(2) ) - +x(2)*( +y(0) +y(1) -y(6) -y(7) ) - +x(4)*( -y(0) +y(7) ) - +x(6)*( +y(2) -y(7) ) - +x(7)*( -y(0) +y(2) -y(4) +y(6) ) )*twelth; - - B_matrix(4,2) = ( +x(0)*( +y(1) -y(3) +y(5) -y(7) ) - +x(1)*( -y(0) +y(5) ) - +x(3)*( +y(0) -y(7) ) - +x(5)*( -y(0) -y(1) +y(6) +y(7) ) - +x(6)*( -y(5) +y(7) ) - +x(7)*( +y(0) +y(3) -y(5) -y(6) ) )*twelth; - - B_matrix(5,2) = ( +x(0)*( +y(1) -y(4) ) - +x(1)*( -y(0) +y(2) -y(4) +y(6) ) - +x(2)*( -y(1) +y(6) ) - +x(4)*( +y(0) +y(1) -y(6) -y(7) ) - +x(6)*( -y(1) -y(2) +y(4) +y(7) ) - +x(7)*( +y(4) -y(6) ) )*twelth; - - B_matrix(6,2) = ( +x(1)*( +y(2) -y(5) ) - +x(2)*( -y(1) +y(3) -y(5) +y(7) ) - +x(3)*( -y(2) +y(7) ) - +x(4)*( +y(5) -y(7) ) - +x(5)*( +y(1) +y(2) -y(4) -y(7) ) - +x(7)*( -y(2) -y(3) +y(4) +y(5) ) )*twelth; - - B_matrix(7,2) = ( +x(0)*( -y(3) +y(4) ) - +x(2)*( +y(3) -y(6) ) - +x(3)*( +y(0) -y(2) +y(4) -y(6) ) - +x(4)*( -y(0) -y(3) +y(5) +y(6) ) - +x(5)*( -y(4) +y(6) ) - +x(6)*( +y(2) +y(3) -y(4) -y(5) ) )*twelth; - + double twelth = 1. / 12.; + + B_matrix(0, 0) = (+y(1) * (-z(2) - z(3) + z(4) + z(5) ) + + y(2) * (+z(1) - z(3) ) + + y(3) * (+z(1) + z(2) - z(4) - z(7) ) + + y(4) * (-z(1) + z(3) - z(5) + z(7) ) + + y(5) * (-z(1) + z(4) ) + + y(7) * (+z(3) - z(4) ) ) * twelth; + + B_matrix(1, 0) = (+y(0) * (+z(2) + z(3) - z(4) - z(5) ) + + y(2) * (-z(0) - z(3) + z(5) + z(6) ) + + y(3) * (-z(0) + z(2) ) + + y(4) * (+z(0) - z(5) ) + + y(5) * (+z(0) - z(2) + z(4) - z(6) ) + + y(6) * (-z(2) + z(5) ) ) * twelth; + + B_matrix(2, 0) = (+y(0) * (-z(1) + z(3) ) + + y(1) * (+z(0) + z(3) - z(5) - z(6) ) + + y(3) * (-z(0) - z(1) + z(6) + z(7) ) + + y(5) * (+z(1) - z(6) ) + + y(6) * (+z(1) - z(3) + z(5) - z(7) ) + + y(7) * (-z(3) + z(6) ) ) * twelth; + + B_matrix(3, 0) = (+y(0) * (-z(1) - z(2) + z(4) + z(7) ) + + y(1) * (+z(0) - z(2) ) + + y(2) * (+z(0) + z(1) - z(6) - z(7) ) + + y(4) * (-z(0) + z(7) ) + + y(6) * (+z(2) - z(7) ) + + y(7) * (-z(0) + z(2) - z(4) + z(6) ) ) * twelth; + + B_matrix(4, 0) = (+y(0) * (+z(1) - z(3) + z(5) - z(7) ) + + y(1) * (-z(0) + z(5) ) + + y(3) * (+z(0) - z(7) ) + + y(5) * (-z(0) - z(1) + z(6) + z(7) ) + + y(6) * (-z(5) + z(7) ) + + y(7) * (+z(0) + z(3) - z(5) - z(6) ) ) * twelth; + + B_matrix(5, 0) = (+y(0) * (+z(1) - z(4) ) + + y(1) * (-z(0) + z(2) - z(4) + z(6) ) + + y(2) * (-z(1) + z(6) ) + + y(4) * (+z(0) + z(1) - z(6) - z(7) ) + + y(6) * (-z(1) - z(2) + z(4) + z(7) ) + + y(7) * (+z(4) - z(6) ) ) * twelth; + + B_matrix(6, 0) = (+y(1) * (+z(2) - z(5) ) + + y(2) * (-z(1) + z(3) - z(5) + z(7) ) + + y(3) * (-z(2) + z(7) ) + + y(4) * (+z(5) - z(7) ) + + y(5) * (+z(1) + z(2) - z(4) - z(7) ) + + y(7) * (-z(2) - z(3) + z(4) + z(5) ) ) * twelth; + + B_matrix(7, 0) = (+y(0) * (-z(3) + z(4) ) + + y(2) * (+z(3) - z(6) ) + + y(3) * (+z(0) - z(2) + z(4) - z(6) ) + + y(4) * (-z(0) - z(3) + z(5) + z(6) ) + + y(5) * (-z(4) + z(6) ) + + y(6) * (+z(2) + z(3) - z(4) - z(5) ) ) * twelth; + + B_matrix(0, 1) = (+z(1) * (-x(2) - x(3) + x(4) + x(5) ) + + z(2) * (+x(1) - x(3) ) + + z(3) * (+x(1) + x(2) - x(4) - x(7) ) + + z(4) * (-x(1) + x(3) - x(5) + x(7) ) + + z(5) * (-x(1) + x(4) ) + + z(7) * (+x(3) - x(4) ) ) * twelth; + + B_matrix(1, 1) = (+z(0) * (+x(2) + x(3) - x(4) - x(5) ) + + z(2) * (-x(0) - x(3) + x(5) + x(6) ) + + z(3) * (-x(0) + x(2) ) + + z(4) * (+x(0) - x(5) ) + + z(5) * (+x(0) - x(2) + x(4) - x(6) ) + + z(6) * (-x(2) + x(5) ) ) * twelth; + + B_matrix(2, 1) = (+z(0) * (-x(1) + x(3) ) + + z(1) * (+x(0) + x(3) - x(5) - x(6) ) + + z(3) * (-x(0) - x(1) + x(6) + x(7) ) + + z(5) * (+x(1) - x(6) ) + + z(6) * (+x(1) - x(3) + x(5) - x(7) ) + + z(7) * (-x(3) + x(6) ) ) * twelth; + + B_matrix(3, 1) = (+z(0) * (-x(1) - x(2) + x(4) + x(7) ) + + z(1) * (+x(0) - x(2) ) + + z(2) * (+x(0) + x(1) - x(6) - x(7) ) + + z(4) * (-x(0) + x(7) ) + + z(6) * (+x(2) - x(7) ) + + z(7) * (-x(0) + x(2) - x(4) + x(6) ) ) * twelth; + + B_matrix(4, 1) = (+z(0) * (+x(1) - x(3) + x(5) - x(7) ) + + z(1) * (-x(0) + x(5) ) + + z(3) * (+x(0) - x(7) ) + + z(5) * (-x(0) - x(1) + x(6) + x(7) ) + + z(6) * (-x(5) + x(7) ) + + z(7) * (+x(0) + x(3) - x(5) - x(6) ) ) * twelth; + + B_matrix(5, 1) = (+z(0) * (+x(1) - x(4) ) + + z(1) * (-x(0) + x(2) - x(4) + x(6) ) + + z(2) * (-x(1) + x(6) ) + + z(4) * (+x(0) + x(1) - x(6) - x(7) ) + + z(6) * (-x(1) - x(2) + x(4) + x(7) ) + + z(7) * (+x(4) - x(6) ) ) * twelth; + + B_matrix(6, 1) = (+z(1) * (+x(2) - x(5) ) + + z(2) * (-x(1) + x(3) - x(5) + x(7) ) + + z(3) * (-x(2) + x(7) ) + + z(4) * (+x(5) - x(7) ) + + z(5) * (+x(1) + x(2) - x(4) - x(7) ) + + z(7) * (-x(2) - x(3) + x(4) + x(5) ) ) * twelth; + + B_matrix(7, 1) = (+z(0) * (-x(3) + x(4) ) + + z(2) * (+x(3) - x(6) ) + + z(3) * (+x(0) - x(2) + x(4) - x(6) ) + + z(4) * (-x(0) - x(3) + x(5) + x(6) ) + + z(5) * (-x(4) + x(6) ) + + z(6) * (+x(2) + x(3) - x(4) - x(5) ) ) * twelth; + + B_matrix(0, 2) = (+x(1) * (-y(2) - y(3) + y(4) + y(5) ) + + x(2) * (+y(1) - y(3) ) + + x(3) * (+y(1) + y(2) - y(4) - y(7) ) + + x(4) * (-y(1) + y(3) - y(5) + y(7) ) + + x(5) * (-y(1) + y(4) ) + + x(7) * (+y(3) - y(4) ) ) * twelth; + + B_matrix(1, 2) = (+x(0) * (+y(2) + y(3) - y(4) - y(5) ) + + x(2) * (-y(0) - y(3) + y(5) + y(6) ) + + x(3) * (-y(0) + y(2) ) + + x(4) * (+y(0) - y(5) ) + + x(5) * (+y(0) - y(2) + y(4) - y(6) ) + + x(6) * (-y(2) + y(5) ) ) * twelth; + + B_matrix(2, 2) = (+x(0) * (-y(1) + y(3) ) + + x(1) * (+y(0) + y(3) - y(5) - y(6) ) + + x(3) * (-y(0) - y(1) + y(6) + y(7) ) + + x(5) * (+y(1) - y(6) ) + + x(6) * (+y(1) - y(3) + y(5) - y(7) ) + + x(7) * (-y(3) + y(6) ) ) * twelth; + + B_matrix(3, 2) = (+x(0) * (-y(1) - y(2) + y(4) + y(7) ) + + x(1) * (+y(0) - y(2) ) + + x(2) * (+y(0) + y(1) - y(6) - y(7) ) + + x(4) * (-y(0) + y(7) ) + + x(6) * (+y(2) - y(7) ) + + x(7) * (-y(0) + y(2) - y(4) + y(6) ) ) * twelth; + + B_matrix(4, 2) = (+x(0) * (+y(1) - y(3) + y(5) - y(7) ) + + x(1) * (-y(0) + y(5) ) + + x(3) * (+y(0) - y(7) ) + + x(5) * (-y(0) - y(1) + y(6) + y(7) ) + + x(6) * (-y(5) + y(7) ) + + x(7) * (+y(0) + y(3) - y(5) - y(6) ) ) * twelth; + + B_matrix(5, 2) = (+x(0) * (+y(1) - y(4) ) + + x(1) * (-y(0) + y(2) - y(4) + y(6) ) + + x(2) * (-y(1) + y(6) ) + + x(4) * (+y(0) + y(1) - y(6) - y(7) ) + + x(6) * (-y(1) - y(2) + y(4) + y(7) ) + + x(7) * (+y(4) - y(6) ) ) * twelth; + + B_matrix(6, 2) = (+x(1) * (+y(2) - y(5) ) + + x(2) * (-y(1) + y(3) - y(5) + y(7) ) + + x(3) * (-y(2) + y(7) ) + + x(4) * (+y(5) - y(7) ) + + x(5) * (+y(1) + y(2) - y(4) - y(7) ) + + x(7) * (-y(2) - y(3) + y(4) + y(5) ) ) * twelth; + + B_matrix(7, 2) = (+x(0) * (-y(3) + y(4) ) + + x(2) * (+y(3) - y(6) ) + + x(3) * (+y(0) - y(2) + y(4) - y(6) ) + + x(4) * (-y(0) - y(3) + y(5) + y(6) ) + + x(5) * (-y(4) + y(6) ) + + x(6) * (+y(2) + y(3) - y(4) - y(5) ) ) * twelth; } // end subroutine - /* ---------------------------------------------------------------------------- Compute Volume of each finite element ------------------------------------------------------------------------------- */ -void FEA_Module_Dynamic_Elasticity::get_vol(){ - +void FEA_Module_Dynamic_Elasticity::get_vol() +{ const size_t rk_level = rk_num_bins - 1; const size_t num_dims = num_dim; - if (num_dims == 2){ + if (num_dims == 2) + { FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { - // cut out the node_gids for this element - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 4); + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 4); get_vol_quad(elem_vol, elem_gid, node_coords, elem_node_gids, rk_level); - }); Kokkos::fence(); } - else { + else + { FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { - // cut out the node_gids for this element - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 8); + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 8); get_vol_hex(elem_vol, elem_gid, node_coords, elem_node_gids, rk_level); - }); Kokkos::fence(); } // end if - + return; - } // end subroutine - // Exact volume for a hex element KOKKOS_INLINE_FUNCTION -void FEA_Module_Dynamic_Elasticity::get_vol_hex(const DViewCArrayKokkos &elem_vol, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const { - +void FEA_Module_Dynamic_Elasticity::get_vol_hex(const DViewCArrayKokkos& elem_vol, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const +{ const size_t num_nodes = 8; double x_array[8]; double y_array[8]; double z_array[8]; - + // x, y, z coordinates of elem vertices - auto x = ViewCArrayKokkos (x_array, num_nodes); - auto y = ViewCArrayKokkos (y_array, num_nodes); - auto z = ViewCArrayKokkos (z_array, num_nodes); - + auto x = ViewCArrayKokkos(x_array, num_nodes); + auto y = ViewCArrayKokkos(y_array, num_nodes); + auto z = ViewCArrayKokkos(z_array, num_nodes); + // get the coordinates of the nodes(rk,elem,node) in this element - for (int node_lid = 0; node_lid < num_nodes; node_lid++){ + for (int node_lid = 0; node_lid < num_nodes; node_lid++) + { x(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 0); y(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 1); z(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 2); } // end for - double twelth = 1./12.; - + double twelth = 1. / 12.; + // element volume elem_vol(elem_gid) = - (x(1)*(y(3)*(-z(0) + z(2)) + y(4)*( z(0) - z(5)) + y(0)*( z(2) + z(3) - z(4) - z(5)) + y(6)*(-z(2) + z(5)) + y(5)*(z(0) - z(2) + z(4) - z(6)) + y(2)*(-z(0) - z(3) + z(5) + z(6))) + - x(7)*(y(0)*(-z(3) + z(4)) + y(6)*( z(2) + z(3) - z(4) - z(5)) + y(2)*(z(3) - z(6)) + y(3)*(z(0) - z(2) + z(4) - z(6)) + y(5)*(-z(4) + z(6)) + y(4)*(-z(0) - z(3) + z(5) + z(6))) + - x(3)*(y(1)*( z(0) - z(2)) + y(7)*(-z(0) + z(2) - z(4) + z(6)) + y(6)*(z(2) - z(7)) + y(2)*(z(0) + z(1) - z(6) - z(7)) + y(4)*(-z(0) + z(7)) + y(0)*(-z(1) - z(2) + z(4) + z(7))) + - x(5)*(y(0)*( z(1) - z(4)) + y(7)*( z(4) - z(6)) + y(2)*(-z(1) + z(6)) + y(1)*(-z(0) + z(2) - z(4) + z(6)) + y(4)*(z(0) + z(1) - z(6) - z(7)) + y(6)*(-z(1) - z(2) + z(4) + z(7))) + - x(6)*(y(1)*( z(2) - z(5)) + y(7)*(-z(2) - z(3) + z(4) + z(5)) + y(5)*(z(1) + z(2) - z(4) - z(7)) + y(4)*(z(5) - z(7)) + y(3)*(-z(2) + z(7)) + y(2)*(-z(1) + z(3) - z(5) + z(7))) + - x(0)*(y(2)*( z(1) - z(3)) + y(7)*( z(3) - z(4)) + y(5)*(-z(1) + z(4)) + y(1)*(-z(2) - z(3) + z(4) + z(5)) + y(3)*(z(1) + z(2) - z(4) - z(7)) + y(4)*(-z(1) + z(3) - z(5) + z(7))) + - x(2)*(y(0)*(-z(1) + z(3)) + y(5)*( z(1) - z(6)) + y(1)*( z(0) + z(3) - z(5) - z(6)) + y(7)*(-z(3) + z(6)) + y(6)*(z(1) - z(3) + z(5) - z(7)) + y(3)*(-z(0) - z(1) + z(6) + z(7))) + - x(4)*(y(1)*(-z(0) + z(5)) + y(7)*( z(0) + z(3) - z(5) - z(6)) + y(3)*(z(0) - z(7)) + y(0)*(z(1) - z(3) + z(5) - z(7)) + y(6)*(-z(5) + z(7)) + y(5)*(-z(0) - z(1) + z(6) + z(7))))*twelth; + (x(1) * (y(3) * (-z(0) + z(2)) + y(4) * (z(0) - z(5)) + y(0) * (z(2) + z(3) - z(4) - z(5)) + y(6) * (-z(2) + z(5)) + y(5) * (z(0) - z(2) + z(4) - z(6)) + y(2) * (-z(0) - z(3) + z(5) + z(6))) + + x(7) * (y(0) * (-z(3) + z(4)) + y(6) * (z(2) + z(3) - z(4) - z(5)) + y(2) * (z(3) - z(6)) + y(3) * (z(0) - z(2) + z(4) - z(6)) + y(5) * (-z(4) + z(6)) + y(4) * (-z(0) - z(3) + z(5) + z(6))) + + x(3) * (y(1) * (z(0) - z(2)) + y(7) * (-z(0) + z(2) - z(4) + z(6)) + y(6) * (z(2) - z(7)) + y(2) * (z(0) + z(1) - z(6) - z(7)) + y(4) * (-z(0) + z(7)) + y(0) * (-z(1) - z(2) + z(4) + z(7))) + + x(5) * (y(0) * (z(1) - z(4)) + y(7) * (z(4) - z(6)) + y(2) * (-z(1) + z(6)) + y(1) * (-z(0) + z(2) - z(4) + z(6)) + y(4) * (z(0) + z(1) - z(6) - z(7)) + y(6) * (-z(1) - z(2) + z(4) + z(7))) + + x(6) * (y(1) * (z(2) - z(5)) + y(7) * (-z(2) - z(3) + z(4) + z(5)) + y(5) * (z(1) + z(2) - z(4) - z(7)) + y(4) * (z(5) - z(7)) + y(3) * (-z(2) + z(7)) + y(2) * (-z(1) + z(3) - z(5) + z(7))) + + x(0) * (y(2) * (z(1) - z(3)) + y(7) * (z(3) - z(4)) + y(5) * (-z(1) + z(4)) + y(1) * (-z(2) - z(3) + z(4) + z(5)) + y(3) * (z(1) + z(2) - z(4) - z(7)) + y(4) * (-z(1) + z(3) - z(5) + z(7))) + + x(2) * (y(0) * (-z(1) + z(3)) + y(5) * (z(1) - z(6)) + y(1) * (z(0) + z(3) - z(5) - z(6)) + y(7) * (-z(3) + z(6)) + y(6) * (z(1) - z(3) + z(5) - z(7)) + y(3) * (-z(0) - z(1) + z(6) + z(7))) + + x(4) * + (y(1) * (-z(0) + z(5)) + y(7) * (z(0) + z(3) - z(5) - z(6)) + y(3) * (z(0) - z(7)) + y(0) * (z(1) - z(3) + z(5) - z(7)) + y(6) * (-z(5) + z(7)) + y(5) * (-z(0) - z(1) + z(6) + z(7)))) * + twelth; return; - } // end subroutine - - KOKKOS_FUNCTION -void FEA_Module_Dynamic_Elasticity::get_bmatrix2D(const ViewCArrayKokkos &B_matrix, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const { - +void FEA_Module_Dynamic_Elasticity::get_bmatrix2D(const ViewCArrayKokkos& B_matrix, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const +{ const size_t num_nodes = 4; double x_array[4]; double y_array[4]; - + // x, y coordinates of elem vertices - auto x = ViewCArrayKokkos (x_array, num_nodes); - auto y = ViewCArrayKokkos (y_array, num_nodes); + auto x = ViewCArrayKokkos(x_array, num_nodes); + auto y = ViewCArrayKokkos(y_array, num_nodes); // get the coordinates of the nodes(rk,elem,node) in this element - for (int node_lid = 0; node_lid < num_nodes; node_lid++){ + for (int node_lid = 0; node_lid < num_nodes; node_lid++) + { x(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 0); y(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 1); } // end for @@ -340,26 +333,23 @@ void FEA_Module_Dynamic_Elasticity::get_bmatrix2D(const ViewCArrayKokkos &elem_vol, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const { - +void FEA_Module_Dynamic_Elasticity::get_vol_quad(const DViewCArrayKokkos& elem_vol, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const +{ // --- testing here --- /* double test_vol = 0.0; // getting the corner facial area double corner_areas_array[4]; ViewCArrayKokkos corner_areas(&corner_areas_array[0],4); - + get_area_weights2D(corner_areas, elem_gid, node_coords, elem_node_gids, rk_level); - + for(size_t node_lid=0; node_lid<4; node_lid++){ double y = node_coords(rk_level, elem_node_gids(node_lid), 1); // node radius test_vol += corner_areas(node_lid)*y; } // end for - + test_vol matches the Barlow volume formula */ // ------------------- - - + elem_vol(elem_gid) = 0.0; - + const size_t num_nodes = 4; double x_array[4]; double y_array[4]; - + // x, y coordinates of elem vertices - auto x = ViewCArrayKokkos (x_array, num_nodes); - auto y = ViewCArrayKokkos (y_array, num_nodes); - + auto x = ViewCArrayKokkos(x_array, num_nodes); + auto y = ViewCArrayKokkos(y_array, num_nodes); + // get the coordinates of the nodes(rk,elem,node) in this element - for (int node_lid = 0; node_lid < num_nodes; node_lid++){ + for (int node_lid = 0; node_lid < num_nodes; node_lid++) + { x(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 0); y(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 1); } // end for @@ -436,34 +424,33 @@ void FEA_Module_Dynamic_Elasticity::get_vol_quad(const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const { +double FEA_Module_Dynamic_Elasticity::get_area_quad(const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const +{ + double elem_area = 0.0; - double elem_area=0.0; - const size_t num_nodes = 4; double x_array[4]; double y_array[4]; - + // x, y coordinates of elem vertices - auto x = ViewCArrayKokkos (x_array, num_nodes); - auto y = ViewCArrayKokkos (y_array, num_nodes); - + auto x = ViewCArrayKokkos(x_array, num_nodes); + auto y = ViewCArrayKokkos(y_array, num_nodes); + // get the coordinates of the nodes(rk,elem,node) in this element - for (int node_lid = 0; node_lid < num_nodes; node_lid++){ + for (int node_lid = 0; node_lid < num_nodes; node_lid++) + { x(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 0); y(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 1); } // end for @@ -471,86 +458,79 @@ double FEA_Module_Dynamic_Elasticity::get_area_quad(const size_t elem_gid, /* ensight node order 0 1 2 3 Flanaghan node order 3 4 1 2 */ - + // element facial area - elem_area = 0.5*((x(0)-x(2))*(y(1)-y(3))+(x(3)-x(1))*(y(0)-y(2))); + elem_area = 0.5 * ((x(0) - x(2)) * (y(1) - y(3)) + (x(3) - x(1)) * (y(0) - y(2))); return elem_area; - } // end subroutine - KOKKOS_INLINE_FUNCTION double FEA_Module_Dynamic_Elasticity::heron(const double x1, - const double y1, - const double x2, - const double y2, - const double x3, - const double y3) const + const double y1, + const double x2, + const double y2, + const double x3, + const double y3) const { - double S,a,b,c,area; + double S, a, b, c, area; - S=0.0; - a=sqrt((x2-x1)*(x2-x1)+(y2-y1)*(y2-y1)); - S+=a; - b=sqrt((x3-x2)*(x3-x2)+(y3-y2)*(y3-y2)); - S+=b; - c=sqrt((x3-x1)*(x3-x1)+(y3-y1)*(y3-y1)); - S+=c; + S = 0.0; + a = sqrt((x2 - x1) * (x2 - x1) + (y2 - y1) * (y2 - y1)); + S += a; + b = sqrt((x3 - x2) * (x3 - x2) + (y3 - y2) * (y3 - y2)); + S += b; + c = sqrt((x3 - x1) * (x3 - x1) + (y3 - y1) * (y3 - y1)); + S += c; - S*=0.5; - area=sqrt(S*(S-a)*(S-b)*(S-c)); + S *= 0.5; + area = sqrt(S * (S - a) * (S - b) * (S - c)); - return area; + return area; } - - KOKKOS_FUNCTION -void FEA_Module_Dynamic_Elasticity::get_area_weights2D(const ViewCArrayKokkos &corner_areas, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const { - +void FEA_Module_Dynamic_Elasticity::get_area_weights2D(const ViewCArrayKokkos& corner_areas, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const +{ const size_t num_nodes = 4; double x_array[4]; double y_array[4]; - double rc,zc; - double A12,A23,A34,A41; - + double rc, zc; + double A12, A23, A34, A41; + // x, y coordinates of elem vertices - ViewCArrayKokkos x(x_array, num_nodes); - ViewCArrayKokkos y(y_array, num_nodes); + ViewCArrayKokkos x(x_array, num_nodes); + ViewCArrayKokkos y(y_array, num_nodes); // get the coordinates of the nodes(rk,elem,node) in this element - rc=zc=0.0; - for (int node_lid = 0; node_lid < num_nodes; node_lid++){ + rc = zc = 0.0; + for (int node_lid = 0; node_lid < num_nodes; node_lid++) + { x(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 0); y(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 1); - rc+=0.25*y(node_lid); - zc+=0.25*x(node_lid); + rc += 0.25 * y(node_lid); + zc += 0.25 * x(node_lid); } // end for /* ensight node order 0 1 2 3 Barlow node order 1 2 3 4 */ - - A12 = heron(x(0),y(0),zc,rc,x(1),y(1)); - A23 = heron(x(1),y(1),zc,rc,x(2),y(2)); - A34 = heron(x(2),y(2),zc,rc,x(3),y(3)); - A41 = heron(x(3),y(3),zc,rc,x(0),y(0)); - - corner_areas(0)=(5.*A41+5.*A12+A23+A34)/12.; - corner_areas(1)=(A41+5.*A12+5.*A23+A34)/12.; - corner_areas(2)=(A41+A12+5.*A23+5.*A34)/12.; - corner_areas(3)=(5.*A41+A12+A23+5.*A34)/12.; + A12 = heron(x(0), y(0), zc, rc, x(1), y(1)); + A23 = heron(x(1), y(1), zc, rc, x(2), y(2)); + A34 = heron(x(2), y(2), zc, rc, x(3), y(3)); + A41 = heron(x(3), y(3), zc, rc, x(0), y(0)); + corner_areas(0) = (5. * A41 + 5. * A12 + A23 + A34) / 12.; + corner_areas(1) = (A41 + 5. * A12 + 5. * A23 + A34) / 12.; + corner_areas(2) = (A41 + A12 + 5. * A23 + 5. * A34) / 12.; + corner_areas(3) = (5. * A41 + A12 + A23 + 5. * A34) / 12.; return; - } // end subroutine - diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/mesh.h b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/mesh.h index 25111f2ed..2a4d852c8 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/mesh.h +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/mesh.h @@ -1,7 +1,6 @@ #ifndef MESH_H #define MESH_H - #include "matar.h" #include "state.h" @@ -48,195 +47,181 @@ patch 6: [4,5,6,7] zeta-plus dir // sort in ascending order using bubble sort KOKKOS_INLINE_FUNCTION -void bubble_sort(size_t arr[], const size_t num){ - - for (size_t i=0; i<(num-1); i++){ - for (size_t j=0; j<(num-i-1); j++){ - - if (arr[j]>arr[j+1]){ +void bubble_sort(size_t arr[], const size_t num) +{ + for (size_t i = 0; i < (num - 1); i++) + { + for (size_t j = 0; j < (num - i - 1); j++) + { + if (arr[j] > arr[j + 1]) + { size_t temp = arr[j]; - arr[j] = arr[j+1]; - arr[j+1] = temp; + arr[j] = arr[j + 1]; + arr[j + 1] = temp; } // end if - } // end for j } // end for i } // end function - // mesh sizes and connectivity data structures -struct mesh_t { - +struct mesh_t +{ size_t num_dims; - + size_t num_nodes, num_local_nodes; - + size_t num_elems; size_t num_nodes_in_elem; size_t num_patches_in_elem; size_t num_corners; - + size_t num_patches; - + size_t num_bdy_patches; size_t num_bdy_nodes; size_t num_bdy_sets; size_t num_nodes_in_patch; - // ---- nodes ---- - + // corner ids in node - RaggedRightArrayKokkos corners_in_node; - CArrayKokkos num_corners_in_node; - + RaggedRightArrayKokkos corners_in_node; + CArrayKokkos num_corners_in_node; + // elem ids in node - RaggedRightArrayKokkos elems_in_node; - + RaggedRightArrayKokkos elems_in_node; + // node ids in node - RaggedRightArrayKokkos nodes_in_node; - CArrayKokkos num_nodes_in_node; - - + RaggedRightArrayKokkos nodes_in_node; + CArrayKokkos num_nodes_in_node; + // ---- elems ---- - + // node ids in elem - DCArrayKokkos nodes_in_elem; - + DCArrayKokkos nodes_in_elem; + // corner ids in elem - CArrayKokkos corners_in_elem; - + CArrayKokkos corners_in_elem; + // elem ids in elem - RaggedRightArrayKokkos elems_in_elem; - CArrayKokkos num_elems_in_elem; - + RaggedRightArrayKokkos elems_in_elem; + CArrayKokkos num_elems_in_elem; + // patch ids in elem - CArrayKokkos patches_in_elem; - - + CArrayKokkos patches_in_elem; + // ---- patches ---- - + // node ids in a patch - CArrayKokkos nodes_in_patch; - + CArrayKokkos nodes_in_patch; + // element ids in a patch - CArrayKokkos elems_in_patch; - - + CArrayKokkos elems_in_patch; + // ---- bdy ---- - + // bdy_patches - CArrayKokkos bdy_patches; - + CArrayKokkos bdy_patches; + // bdy nodes - CArrayKokkos bdy_nodes; - + CArrayKokkos bdy_nodes; + // patch ids in bdy set - DynamicRaggedRightArrayKokkos bdy_patches_in_set; - + DynamicRaggedRightArrayKokkos bdy_patches_in_set; + // node ids in bdy_patch set - RaggedRightArrayKokkos bdy_nodes_in_set; - DCArrayKokkos num_bdy_nodes_in_set; - + RaggedRightArrayKokkos bdy_nodes_in_set; + DCArrayKokkos num_bdy_nodes_in_set; + // initialization methods void initialize_nodes(const size_t num_nodes_inp) { num_nodes = num_nodes_inp; - + return; - }; // end method // initialization methods void initialize_local_nodes(const size_t num_nodes_inp) { num_local_nodes = num_nodes_inp; - + return; - }; // end method - - + // initialization methods void initialize_elems(const size_t num_elems_inp, const size_t num_dims_inp) { num_dims = num_dims_inp; num_nodes_in_elem = 1; - for (int dim=0; dim (num_elems, num_nodes_in_elem, "nodes_in_elem"); - corners_in_elem = CArrayKokkos (num_elems, num_nodes_in_elem, "corners_in_elem"); - + num_elems = num_elems_inp; + nodes_in_elem = DCArrayKokkos(num_elems, num_nodes_in_elem, "nodes_in_elem"); + corners_in_elem = CArrayKokkos(num_elems, num_nodes_in_elem, "corners_in_elem"); + return; - }; // end method - - + // initialization methods void initialize_corners(const size_t num_corners_inp) { num_corners = num_corners_inp; - + return; - }; // end method - - + // build the corner mesh connectivity arrays - void build_corner_connectivity(){ - - num_corners_in_node = CArrayKokkos (num_nodes); // stride sizes - + void build_corner_connectivity() + { + num_corners_in_node = CArrayKokkos(num_nodes); // stride sizes + // initializing the number of corners (node-cell pair) to be zero FOR_ALL_CLASS(node_gid, 0, num_nodes, { num_corners_in_node(node_gid) = 0; }); - - - for (size_t elem_gid = 0; elem_gid < num_elems; elem_gid++){ - FOR_ALL_CLASS(node_lid, 0, num_nodes_in_elem,{ - + + for (size_t elem_gid = 0; elem_gid < num_elems; elem_gid++) + { + FOR_ALL_CLASS(node_lid, 0, num_nodes_in_elem, { // get the global_id of the node size_t node_gid = nodes_in_elem(elem_gid, node_lid); // increment the number of corners attached to this point num_corners_in_node(node_gid) = num_corners_in_node(node_gid) + 1; - }); // end FOR_ALL over nodes in element } // end for elem_gid - - + // the stride sizes are the num_corners_in_node at the node - corners_in_node = RaggedRightArrayKokkos (num_corners_in_node, "corners_in_node"); + corners_in_node = RaggedRightArrayKokkos(num_corners_in_node, "corners_in_node"); - CArrayKokkos count_saved_corners_in_node(num_nodes, "count_saved_corners_in_node"); + CArrayKokkos count_saved_corners_in_node(num_nodes, "count_saved_corners_in_node"); // reset num_corners to zero FOR_ALL_CLASS(node_gid, 0, num_nodes, { count_saved_corners_in_node(node_gid) = 0; }); - - + // he elems_in_elem data type - elems_in_node = RaggedRightArrayKokkos (num_corners_in_node, "elems_in_node"); - + elems_in_node = RaggedRightArrayKokkos(num_corners_in_node, "elems_in_node"); + // populate the elems connected to a node list and corners in a node - for (size_t elem_gid = 0; elem_gid < num_elems; elem_gid++){ + for (size_t elem_gid = 0; elem_gid < num_elems; elem_gid++) + { FOR_ALL_CLASS(node_lid, 0, num_nodes_in_elem, { - // get the global_id of the node size_t node_gid = nodes_in_elem(elem_gid, node_lid); - + // the column index is the num corners saved size_t j = count_saved_corners_in_node(node_gid); // Save corner index to this node_gid - size_t corner_gid = node_lid + elem_gid*num_nodes_in_elem; + size_t corner_gid = node_lid + elem_gid * num_nodes_in_elem; corners_in_node(node_gid, j) = corner_gid; - + elems_in_node(node_gid, j) = elem_gid; // save the elem_gid // Save corner index to element @@ -245,133 +230,127 @@ struct mesh_t { // increment the number of corners saved to this node_gid count_saved_corners_in_node(node_gid) = count_saved_corners_in_node(node_gid) + 1; - }); // end FOR_ALL over nodes in element } // end for elem_gid - + return; - } // end of build_corner_connectivity - - + // build elem connectivity arrays - void build_elem_elem_connectivity(){ - + void build_elem_elem_connectivity() + { // find the max number of elems around a node size_t max_num_elems_in_node; size_t max_num_lcl; REDUCE_MAX_CLASS(node_gid, 0, num_nodes, max_num_lcl, { - // num_corners_in_node = num_elems_in_node size_t max_num = num_corners_in_node(node_gid); - - if (max_num > max_num_lcl) max_num_lcl = max_num; - + + if (max_num > max_num_lcl) + { + max_num_lcl = max_num; + } }, max_num_elems_in_node); // end parallel reduction on max Kokkos::fence(); - + // a temporary ragged array to save the elems around an elem - DynamicRaggedRightArrayKokkos temp_elems_in_elem(num_nodes, num_nodes_in_elem*max_num_elems_in_node, "temp_elems_in_elem"); - - num_elems_in_elem = CArrayKokkos (num_elems, "num_elems_in_elem"); + DynamicRaggedRightArrayKokkos temp_elems_in_elem(num_nodes, num_nodes_in_elem * max_num_elems_in_node, "temp_elems_in_elem"); + + num_elems_in_elem = CArrayKokkos(num_elems, "num_elems_in_elem"); FOR_ALL_CLASS(elem_gid, 0, num_elems, { num_elems_in_elem(elem_gid) = 0; }); Kokkos::fence(); - + // find and save neighboring elem_gids of an elem FOR_ALL_CLASS(elem_gid, 0, num_elems, { - for (int node_lid=0; node_lid (num_elems_in_elem, "elems_in_elem"); - + elems_in_elem = RaggedRightArrayKokkos(num_elems_in_elem, "elems_in_elem"); + FOR_ALL_CLASS(elem_gid, 0, num_elems, { - for (size_t i=0; i node_ordering_in_elem; // node lids in a patch - - num_nodes_in_patch = 2*(num_dims-1); // 2 (2D) or 4 (3D) - num_patches_in_elem = 2*num_dims; // 4 (2D) or 6 (3D) - + DViewCArrayKokkos node_ordering_in_elem; // node lids in a patch + + num_nodes_in_patch = 2 * (num_dims - 1); // 2 (2D) or 4 (3D) + num_patches_in_elem = 2 * num_dims; // 4 (2D) or 6 (3D) + size_t node_lids_in_patch_in_elem[24]; - - if(num_dims == 3) { - size_t temp_node_lids[24] = {0,4,7,3, - 1,2,6,5, - 0,1,5,4, - 2,3,7,6, - 0,3,2,1, - 4,5,6,7}; - - for (size_t i=0; i<24; i++){ + + if (num_dims == 3) + { + size_t temp_node_lids[24] = { 0, 4, 7, 3, + 1, 2, 6, 5, + 0, 1, 5, 4, + 2, 3, 7, 6, + 0, 3, 2, 1, + 4, 5, 6, 7 }; + + for (size_t i = 0; i < 24; i++) + { node_lids_in_patch_in_elem[i] = temp_node_lids[i]; } // end for i - } - else { + else + { // J // | // 3---2 @@ -379,447 +358,409 @@ struct mesh_t { // 0---1 // size_t temp_node_lids[8] = - {0,3, - 1,2, - 0,1, - 3,2}; - - for (size_t i=0; i<8; i++){ + { 0, 3, + 1, 2, + 0, 1, + 3, 2 }; + + for (size_t i = 0; i < 8; i++) + { node_lids_in_patch_in_elem[i] = temp_node_lids[i]; } // end for i - } // end if on dims - - node_ordering_in_elem = DViewCArrayKokkos (&node_lids_in_patch_in_elem[0],num_patches_in_elem,num_nodes_in_patch); - - + + node_ordering_in_elem = DViewCArrayKokkos(&node_lids_in_patch_in_elem[0], num_patches_in_elem, num_nodes_in_patch); + // for saviong the hash keys of the patches and then the nighboring elem_gid - CArrayKokkos hash_keys_in_elem (num_elems, num_patches_in_elem, "hash_keys_in_elem"); - + CArrayKokkos hash_keys_in_elem(num_elems, num_patches_in_elem, "hash_keys_in_elem"); + // for saving the adjacient patch_lid, which is the slide_lid - //CArrayKokkos neighboring_side_lids (num_elems, num_patches_in_elem); - + // CArrayKokkos neighboring_side_lids (num_elems, num_patches_in_elem); + // allocate memory for the patches in the elem - patches_in_elem = CArrayKokkos (num_elems, num_patches_in_elem, "patches_in_elem"); - + patches_in_elem = CArrayKokkos(num_elems, num_patches_in_elem, "patches_in_elem"); + // a temporary storaage for the patch_gids that are on the mesh boundary - CArrayKokkos temp_bdy_patches(num_elems*num_patches_in_elem, "temp_bdy_patches"); - + CArrayKokkos temp_bdy_patches(num_elems * num_patches_in_elem, "temp_bdy_patches"); + // step 1) calculate the hash values for each patch in the element FOR_ALL_CLASS(elem_gid, 0, num_elems, { - - for (size_t patch_lid = 0; patch_lid num_values(2, "num_values"); - + + DCArrayKokkos num_values(2, "num_values"); + // step 2: walk around the elements and save the elem pairs that have the same hash_key RUN_CLASS({ // serial execution on GPU - - size_t patch_gid = 0; + + size_t patch_gid = 0; size_t bdy_patch_gid = 0; - for (size_t elem_gid = 0; elem_gid (num_patches, 2, "elems_in_patch"); - nodes_in_patch = CArrayKokkos (num_patches, num_nodes_in_patch, "nodes_in_patch"); - + + elems_in_patch = CArrayKokkos(num_patches, 2, "elems_in_patch"); + nodes_in_patch = CArrayKokkos(num_patches, num_nodes_in_patch, "nodes_in_patch"); + // a temporary variable to help populate patch structures - CArrayKokkos num_elems_in_patch_saved (num_patches, "num_elems_in_patch_saved"); - + CArrayKokkos num_elems_in_patch_saved(num_patches, "num_elems_in_patch_saved"); + // initialize the number of elems in a patch saved to zero FOR_ALL_CLASS(patch_gid, 0, num_patches, { num_elems_in_patch_saved(patch_gid) = 0; }); - - for(size_t elem_gid=0; elem_gid (num_bdy_patches, "bdy_patches"); - + bdy_patches = CArrayKokkos(num_bdy_patches, "bdy_patches"); + FOR_ALL_CLASS(bdy_patch_gid, 0, num_bdy_patches, { bdy_patches(bdy_patch_gid) = temp_bdy_patches(bdy_patch_gid); }); // end FOR_ALL bdy_patch_gid - - - - + // find and store the boundary nodes - CArrayKokkos temp_bdy_nodes(num_nodes, "temp_bdy_nodes"); - CArrayKokkos hash_bdy_nodes(num_nodes, "hash_bdy_nodes"); - - FOR_ALL_CLASS (node_gid, 0, num_nodes, { + CArrayKokkos temp_bdy_nodes(num_nodes, "temp_bdy_nodes"); + CArrayKokkos hash_bdy_nodes(num_nodes, "hash_bdy_nodes"); + + FOR_ALL_CLASS(node_gid, 0, num_nodes, { hash_bdy_nodes(node_gid) = -1; }); // end for node_gid - + // Parallel loop over boundary patches - DCArrayKokkos num_bdy_nodes_saved(1); - + DCArrayKokkos num_bdy_nodes_saved(1); + RUN_CLASS({ num_bdy_nodes_saved(0) = 0; - for (size_t bdy_patch_gid=0; bdy_patch_gid (num_bdy_nodes, "bdy_nodes"); - - FOR_ALL_CLASS (node_gid, 0, num_bdy_nodes, { + + bdy_nodes = CArrayKokkos(num_bdy_nodes, "bdy_nodes"); + + FOR_ALL_CLASS(node_gid, 0, num_bdy_nodes, { bdy_nodes(node_gid) = temp_bdy_nodes(node_gid); }); // end for boundary node_gid - + printf("Num boundary nodes = %lu \n", num_bdy_nodes); - + return; - } // end patch connectivity method - - - + // build the patches - void build_node_node_connectivity(){ - - + void build_node_node_connectivity() + { // find the max number of elems around a node size_t max_num_elems_in_node; size_t max_num_lcl; REDUCE_MAX_CLASS(node_gid, 0, num_nodes, max_num_lcl, { - // num_corners_in_node = num_elems_in_node size_t max_num = num_corners_in_node(node_gid); - - if (max_num > max_num_lcl) max_num_lcl = max_num; - + + if (max_num > max_num_lcl) + { + max_num_lcl = max_num; + } }, max_num_elems_in_node); // end parallel reduction on max Kokkos::fence(); - + // each elem corner will contribute 3 edges to the node. Those edges will likely be the same // ones from an adjacent element so it is a safe estimate to multiply by 3 - DynamicRaggedRightArrayKokkos temp_nodes_in_nodes(num_nodes, max_num_elems_in_node*3, "temp_nodes_in_nodes"); - - num_nodes_in_node = CArrayKokkos (num_nodes, "num_nodes_in_node"); - + DynamicRaggedRightArrayKokkos temp_nodes_in_nodes(num_nodes, max_num_elems_in_node * 3, "temp_nodes_in_nodes"); + + num_nodes_in_node = CArrayKokkos(num_nodes, "num_nodes_in_node"); + // walk over the patches and save the node node connectivity RUN_CLASS({ - if (num_dims==3){ - - for (size_t patch_gid=0; patch_gid (num_nodes_in_node, "nodes_in_node"); - + + nodes_in_node = RaggedRightArrayKokkos(num_nodes_in_node, "nodes_in_node"); + // save the connectivity FOR_ALL_CLASS(node_gid, 0, num_nodes, { - size_t num_saved = 0; - for (size_t node_lid=0; node_lid (num_bcs, num_bdy_patches, "bdy_patches_in_set"); - + num_bdy_sets = num_bcs; + bdy_patches_in_set = DynamicRaggedRightArrayKokkos(num_bcs, num_bdy_patches, "bdy_patches_in_set"); + return; - } // end of init_bdy_sets method - - - }; // end mesh_t -#endif \ No newline at end of file +#endif \ No newline at end of file diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/momentum.cpp b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/momentum.cpp index 9fc15a8b4..98e83b455 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/momentum.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/momentum.cpp @@ -5,7 +5,7 @@ // ----------------------------------------------------------------------------- // This function evolves the velocity at the nodes of the mesh -//------------------------------------------------------------------------------ +// ------------------------------------------------------------------------------ /* void FEA_Module_Dynamic_Elasticity::update_velocity_elastic(double rk_alpha, const mesh_t &mesh, @@ -14,9 +14,9 @@ void FEA_Module_Dynamic_Elasticity::update_velocity_elastic(double rk_alpha, const DViewCArrayKokkos &corner_force ){ - const size_t rk_level = simparam->rk_num_bins - 1; + const size_t rk_level = simparam->rk_num_bins - 1; const size_t num_dims = mesh.num_dims; - + // walk over the nodes to update the velocity FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { @@ -24,399 +24,374 @@ void FEA_Module_Dynamic_Elasticity::update_velocity_elastic(double rk_alpha, for (size_t dim = 0; dim < num_dims; dim++){ node_force[dim] = 0.0; } // end for dim - + // loop over all corners around the node and calculate the nodal force for (size_t corner_lid=0; corner_lid &vel_grad, - const ViewCArrayKokkos &elem_node_gids, - const DViewCArrayKokkos &node_vel, - const ViewCArrayKokkos &b_matrix, - const double elem_vol, - const size_t elem_gid, - const size_t rk_level - ) const { - +void FEA_Module_Dynamic_Elasticity::get_velgrad(ViewCArrayKokkos& vel_grad, + const ViewCArrayKokkos& elem_node_gids, + const DViewCArrayKokkos& node_vel, + const ViewCArrayKokkos& b_matrix, + const double elem_vol, + const size_t elem_gid, + const size_t rk_level + ) const +{ const size_t num_nodes_in_elem = 8; - - double u_array[num_nodes_in_elem]; - double v_array[num_nodes_in_elem]; - double w_array[num_nodes_in_elem]; - ViewCArrayKokkos u(u_array, num_nodes_in_elem); // x-dir vel component - ViewCArrayKokkos v(v_array, num_nodes_in_elem); // y-dir vel component - ViewCArrayKokkos w(w_array, num_nodes_in_elem); // z-dir vel component - + + double u_array[num_nodes_in_elem]; + double v_array[num_nodes_in_elem]; + double w_array[num_nodes_in_elem]; + ViewCArrayKokkos u(u_array, num_nodes_in_elem); // x-dir vel component + ViewCArrayKokkos v(v_array, num_nodes_in_elem); // y-dir vel component + ViewCArrayKokkos w(w_array, num_nodes_in_elem); // z-dir vel component + // get the vertex velocities for the cell - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { // Get node gid size_t node_gid = elem_node_gids(node_lid); u(node_lid) = node_vel(rk_level, node_gid, 0); v(node_lid) = node_vel(rk_level, node_gid, 1); w(node_lid) = node_vel(rk_level, node_gid, 2); - } // end for - // --- calculate the velocity gradient terms --- - double inverse_vol = 1.0/elem_vol; - + double inverse_vol = 1.0 / elem_vol; + // x-dir - vel_grad(0,0) = (u(0)*b_matrix(0,0) + u(1)*b_matrix(1,0) - + u(2)*b_matrix(2,0) + u(3)*b_matrix(3,0) - + u(4)*b_matrix(4,0) + u(5)*b_matrix(5,0) - + u(6)*b_matrix(6,0) + u(7)*b_matrix(7,0))*inverse_vol; - - vel_grad(0,1) = (u(0)*b_matrix(0,1) + u(1)*b_matrix(1,1) - + u(2)*b_matrix(2,1) + u(3)*b_matrix(3,1) - + u(4)*b_matrix(4,1) + u(5)*b_matrix(5,1) - + u(6)*b_matrix(6,1) + u(7)*b_matrix(7,1))*inverse_vol; - - vel_grad(0,2) = (u(0)*b_matrix(0,2) + u(1)*b_matrix(1,2) - + u(2)*b_matrix(2,2) + u(3)*b_matrix(3,2) - + u(4)*b_matrix(4,2) + u(5)*b_matrix(5,2) - + u(6)*b_matrix(6,2) + u(7)*b_matrix(7,2))*inverse_vol; - + vel_grad(0, 0) = (u(0) * b_matrix(0, 0) + u(1) * b_matrix(1, 0) + + u(2) * b_matrix(2, 0) + u(3) * b_matrix(3, 0) + + u(4) * b_matrix(4, 0) + u(5) * b_matrix(5, 0) + + u(6) * b_matrix(6, 0) + u(7) * b_matrix(7, 0)) * inverse_vol; + + vel_grad(0, 1) = (u(0) * b_matrix(0, 1) + u(1) * b_matrix(1, 1) + + u(2) * b_matrix(2, 1) + u(3) * b_matrix(3, 1) + + u(4) * b_matrix(4, 1) + u(5) * b_matrix(5, 1) + + u(6) * b_matrix(6, 1) + u(7) * b_matrix(7, 1)) * inverse_vol; + + vel_grad(0, 2) = (u(0) * b_matrix(0, 2) + u(1) * b_matrix(1, 2) + + u(2) * b_matrix(2, 2) + u(3) * b_matrix(3, 2) + + u(4) * b_matrix(4, 2) + u(5) * b_matrix(5, 2) + + u(6) * b_matrix(6, 2) + u(7) * b_matrix(7, 2)) * inverse_vol; + // y-dir - vel_grad(1,0) = (v(0)*b_matrix(0,0) + v(1)*b_matrix(1,0) - + v(2)*b_matrix(2,0) + v(3)*b_matrix(3,0) - + v(4)*b_matrix(4,0) + v(5)*b_matrix(5,0) - + v(6)*b_matrix(6,0) + v(7)*b_matrix(7,0))*inverse_vol; - - vel_grad(1,1) = (v(0)*b_matrix(0,1) + v(1)*b_matrix(1,1) - + v(2)*b_matrix(2,1) + v(3)*b_matrix(3,1) - + v(4)*b_matrix(4,1) + v(5)*b_matrix(5,1) - + v(6)*b_matrix(6,1) + v(7)*b_matrix(7,1))*inverse_vol; - vel_grad(1,2) = (v(0)*b_matrix(0,2) + v(1)*b_matrix(1,2) - + v(2)*b_matrix(2,2) + v(3)*b_matrix(3,2) - + v(4)*b_matrix(4,2) + v(5)*b_matrix(5,2) - + v(6)*b_matrix(6,2) + v(7)*b_matrix(7,2))*inverse_vol; - + vel_grad(1, 0) = (v(0) * b_matrix(0, 0) + v(1) * b_matrix(1, 0) + + v(2) * b_matrix(2, 0) + v(3) * b_matrix(3, 0) + + v(4) * b_matrix(4, 0) + v(5) * b_matrix(5, 0) + + v(6) * b_matrix(6, 0) + v(7) * b_matrix(7, 0)) * inverse_vol; + + vel_grad(1, 1) = (v(0) * b_matrix(0, 1) + v(1) * b_matrix(1, 1) + + v(2) * b_matrix(2, 1) + v(3) * b_matrix(3, 1) + + v(4) * b_matrix(4, 1) + v(5) * b_matrix(5, 1) + + v(6) * b_matrix(6, 1) + v(7) * b_matrix(7, 1)) * inverse_vol; + vel_grad(1, 2) = (v(0) * b_matrix(0, 2) + v(1) * b_matrix(1, 2) + + v(2) * b_matrix(2, 2) + v(3) * b_matrix(3, 2) + + v(4) * b_matrix(4, 2) + v(5) * b_matrix(5, 2) + + v(6) * b_matrix(6, 2) + v(7) * b_matrix(7, 2)) * inverse_vol; + // z-dir - vel_grad(2,0) = (w(0)*b_matrix(0,0) + w(1)*b_matrix(1,0) - + w(2)*b_matrix(2,0) + w(3)*b_matrix(3,0) - + w(4)*b_matrix(4,0) + w(5)*b_matrix(5,0) - + w(6)*b_matrix(6,0) + w(7)*b_matrix(7,0))*inverse_vol; - - vel_grad(2,1) = (w(0)*b_matrix(0,1) + w(1)*b_matrix(1,1) - + w(2)*b_matrix(2,1) + w(3)*b_matrix(3,1) - + w(4)*b_matrix(4,1) + w(5)*b_matrix(5,1) - + w(6)*b_matrix(6,1) + w(7)*b_matrix(7,1))*inverse_vol; - - vel_grad(2,2) = (w(0)*b_matrix(0,2) + w(1)*b_matrix(1,2) - + w(2)*b_matrix(2,2) + w(3)*b_matrix(3,2) - + w(4)*b_matrix(4,2) + w(5)*b_matrix(5,2) - + w(6)*b_matrix(6,2) + w(7)*b_matrix(7,2))*inverse_vol; - - + vel_grad(2, 0) = (w(0) * b_matrix(0, 0) + w(1) * b_matrix(1, 0) + + w(2) * b_matrix(2, 0) + w(3) * b_matrix(3, 0) + + w(4) * b_matrix(4, 0) + w(5) * b_matrix(5, 0) + + w(6) * b_matrix(6, 0) + w(7) * b_matrix(7, 0)) * inverse_vol; + + vel_grad(2, 1) = (w(0) * b_matrix(0, 1) + w(1) * b_matrix(1, 1) + + w(2) * b_matrix(2, 1) + w(3) * b_matrix(3, 1) + + w(4) * b_matrix(4, 1) + w(5) * b_matrix(5, 1) + + w(6) * b_matrix(6, 1) + w(7) * b_matrix(7, 1)) * inverse_vol; + + vel_grad(2, 2) = (w(0) * b_matrix(0, 2) + w(1) * b_matrix(1, 2) + + w(2) * b_matrix(2, 2) + w(3) * b_matrix(3, 2) + + w(4) * b_matrix(4, 2) + w(5) * b_matrix(5, 2) + + w(6) * b_matrix(6, 2) + w(7) * b_matrix(7, 2)) * inverse_vol; + return; - } // end function - // ----------------------------------------------------------------------------- // This function calculates the velocity gradient -//------------------------------------------------------------------------------ +// ------------------------------------------------------------------------------ KOKKOS_FUNCTION -void FEA_Module_Dynamic_Elasticity::get_velgrad2D(ViewCArrayKokkos &vel_grad, - const ViewCArrayKokkos &elem_node_gids, - const DViewCArrayKokkos &node_vel, - const ViewCArrayKokkos &b_matrix, - const double elem_vol, - const double elem_area, - const size_t elem_gid, - const size_t rk_level - ) const { - +void FEA_Module_Dynamic_Elasticity::get_velgrad2D(ViewCArrayKokkos& vel_grad, + const ViewCArrayKokkos& elem_node_gids, + const DViewCArrayKokkos& node_vel, + const ViewCArrayKokkos& b_matrix, + const double elem_vol, + const double elem_area, + const size_t elem_gid, + const size_t rk_level + ) const +{ const size_t num_nodes_in_elem = 4; - - double u_array[num_nodes_in_elem]; - double v_array[num_nodes_in_elem]; - ViewCArrayKokkos u(u_array, num_nodes_in_elem); // x-dir vel component - ViewCArrayKokkos v(v_array, num_nodes_in_elem); // y-dir vel component - + + double u_array[num_nodes_in_elem]; + double v_array[num_nodes_in_elem]; + ViewCArrayKokkos u(u_array, num_nodes_in_elem); // x-dir vel component + ViewCArrayKokkos v(v_array, num_nodes_in_elem); // y-dir vel component + // get the vertex velocities for the cell - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { // Get node gid size_t node_gid = elem_node_gids(node_lid); u(node_lid) = node_vel(rk_level, node_gid, 0); // x-comp v(node_lid) = node_vel(rk_level, node_gid, 1); // y-comp - } // end for - + // initialize to zero - for (size_t i=0; i<3; i++){ - for (size_t j=0; j<3; j++){ - vel_grad(i,j) = 0.0; + for (size_t i = 0; i < 3; i++) + { + for (size_t j = 0; j < 3; j++) + { + vel_grad(i, j) = 0.0; } } - - double mean_radius = elem_vol/elem_area; - double elem_vel_r = 0.25*(v(0) + v(1) + v(2) + v(3)); - - + double mean_radius = elem_vol / elem_area; + double elem_vel_r = 0.25 * (v(0) + v(1) + v(2) + v(3)); + // --- calculate the velocity gradient terms --- - double inverse_area = 1.0/elem_area; - + double inverse_area = 1.0 / elem_area; + // x-dir - vel_grad(0,0) = (u(0)*b_matrix(0,0) + u(1)*b_matrix(1,0) - + u(2)*b_matrix(2,0) + u(3)*b_matrix(3,0))*inverse_area; - - vel_grad(0,1) = (u(0)*b_matrix(0,1) + u(1)*b_matrix(1,1) - + u(2)*b_matrix(2,1) + u(3)*b_matrix(3,1))*inverse_area; - - + vel_grad(0, 0) = (u(0) * b_matrix(0, 0) + u(1) * b_matrix(1, 0) + + u(2) * b_matrix(2, 0) + u(3) * b_matrix(3, 0)) * inverse_area; + + vel_grad(0, 1) = (u(0) * b_matrix(0, 1) + u(1) * b_matrix(1, 1) + + u(2) * b_matrix(2, 1) + u(3) * b_matrix(3, 1)) * inverse_area; + // y-dir - vel_grad(1,0) = (v(0)*b_matrix(0,0) + v(1)*b_matrix(1,0) - + v(2)*b_matrix(2,0) + v(3)*b_matrix(3,0))*inverse_area; - - vel_grad(1,1) = (v(0)*b_matrix(0,1) + v(1)*b_matrix(1,1) - + v(2)*b_matrix(2,1) + v(3)*b_matrix(3,1))*inverse_area; - - vel_grad(2,2) = elem_vel_r/mean_radius; // + avg(vel_R)/R - - return; - -} // end function + vel_grad(1, 0) = (v(0) * b_matrix(0, 0) + v(1) * b_matrix(1, 0) + + v(2) * b_matrix(2, 0) + v(3) * b_matrix(3, 0)) * inverse_area; + vel_grad(1, 1) = (v(0) * b_matrix(0, 1) + v(1) * b_matrix(1, 1) + + v(2) * b_matrix(2, 1) + v(3) * b_matrix(3, 1)) * inverse_area; + vel_grad(2, 2) = elem_vel_r / mean_radius; // + avg(vel_R)/R + return; +} // end function // ----------------------------------------------------------------------------- // This subroutine to calculate the velocity divergence in all elements -//------------------------------------------------------------------------------ -void FEA_Module_Dynamic_Elasticity::get_divergence(DViewCArrayKokkos &elem_div, - const mesh_t mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_vol - ){ - +// ------------------------------------------------------------------------------ +void FEA_Module_Dynamic_Elasticity::get_divergence(DViewCArrayKokkos& elem_div, + const mesh_t mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_vol + ) +{ const size_t rk_level = rk_num_bins - 1; // --- calculate the forces acting on the nodes from the element --- - FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { - + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { const size_t num_nodes_in_elem = 8; const size_t num_dims = 3; - + double u_array[num_nodes_in_elem]; double v_array[num_nodes_in_elem]; double w_array[num_nodes_in_elem]; - ViewCArrayKokkos u(u_array, num_nodes_in_elem); // x-dir vel component - ViewCArrayKokkos v(v_array, num_nodes_in_elem); // y-dir vel component - ViewCArrayKokkos w(w_array, num_nodes_in_elem); // z-dir vel component - + ViewCArrayKokkos u(u_array, num_nodes_in_elem); // x-dir vel component + ViewCArrayKokkos v(v_array, num_nodes_in_elem); // y-dir vel component + ViewCArrayKokkos w(w_array, num_nodes_in_elem); // z-dir vel component + // cut out the node_gids for this element - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 8); - + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 8); + // The b_matrix are the outward corner area normals double b_matrix_array[24]; - ViewCArrayKokkos b_matrix(b_matrix_array, num_nodes_in_elem, num_dims); + ViewCArrayKokkos b_matrix(b_matrix_array, num_nodes_in_elem, num_dims); get_bmatrix(b_matrix, elem_gid, node_coords, elem_node_gids, rk_level); - + // get the vertex velocities for the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { // Get node gid size_t node_gid = elem_node_gids(node_lid); - + u(node_lid) = node_vel(rk_level, node_gid, 0); v(node_lid) = node_vel(rk_level, node_gid, 1); w(node_lid) = node_vel(rk_level, node_gid, 2); - } // end for - - + // --- calculate the velocity divergence terms --- - double inverse_vol = 1.0/elem_vol(elem_gid); - + double inverse_vol = 1.0 / elem_vol(elem_gid); + elem_div(elem_gid) = 0.0; - + // x-dir - elem_div(elem_gid) += (u(0)*b_matrix(0,0) + u(1)*b_matrix(1,0) - + u(2)*b_matrix(2,0) + u(3)*b_matrix(3,0) - + u(4)*b_matrix(4,0) + u(5)*b_matrix(5,0) - + u(6)*b_matrix(6,0) + u(7)*b_matrix(7,0))*inverse_vol; - + elem_div(elem_gid) += (u(0) * b_matrix(0, 0) + u(1) * b_matrix(1, 0) + + u(2) * b_matrix(2, 0) + u(3) * b_matrix(3, 0) + + u(4) * b_matrix(4, 0) + u(5) * b_matrix(5, 0) + + u(6) * b_matrix(6, 0) + u(7) * b_matrix(7, 0)) * inverse_vol; + // y-dir - elem_div(elem_gid) += (v(0)*b_matrix(0,1) + v(1)*b_matrix(1,1) - + v(2)*b_matrix(2,1) + v(3)*b_matrix(3,1) - + v(4)*b_matrix(4,1) + v(5)*b_matrix(5,1) - + v(6)*b_matrix(6,1) + v(7)*b_matrix(7,1))*inverse_vol; - + elem_div(elem_gid) += (v(0) * b_matrix(0, 1) + v(1) * b_matrix(1, 1) + + v(2) * b_matrix(2, 1) + v(3) * b_matrix(3, 1) + + v(4) * b_matrix(4, 1) + v(5) * b_matrix(5, 1) + + v(6) * b_matrix(6, 1) + v(7) * b_matrix(7, 1)) * inverse_vol; + // z-dir - elem_div(elem_gid) += (w(0)*b_matrix(0,2) + w(1)*b_matrix(1,2) - + w(2)*b_matrix(2,2) + w(3)*b_matrix(3,2) - + w(4)*b_matrix(4,2) + w(5)*b_matrix(5,2) - + w(6)*b_matrix(6,2) + w(7)*b_matrix(7,2))*inverse_vol; - + elem_div(elem_gid) += (w(0) * b_matrix(0, 2) + w(1) * b_matrix(1, 2) + + w(2) * b_matrix(2, 2) + w(3) * b_matrix(3, 2) + + w(4) * b_matrix(4, 2) + w(5) * b_matrix(5, 2) + + w(6) * b_matrix(6, 2) + w(7) * b_matrix(7, 2)) * inverse_vol; }); // end parallel for over elem_gid - + return; - } // end subroutine - // ----------------------------------------------------------------------------- // This subroutine to calculate the velocity divergence in all elements -//------------------------------------------------------------------------------ -void FEA_Module_Dynamic_Elasticity::get_divergence2D(DViewCArrayKokkos &elem_div, - const mesh_t mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_vol - ){ - +// ------------------------------------------------------------------------------ +void FEA_Module_Dynamic_Elasticity::get_divergence2D(DViewCArrayKokkos& elem_div, + const mesh_t mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_vol + ) +{ const size_t rk_level = rk_num_bins - 1; // --- calculate the forces acting on the nodes from the element --- - FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { - + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { const size_t num_nodes_in_elem = 4; const size_t num_dims = 2; - + double u_array[num_nodes_in_elem]; double v_array[num_nodes_in_elem]; - ViewCArrayKokkos u(u_array, num_nodes_in_elem); // x-dir vel component - ViewCArrayKokkos v(v_array, num_nodes_in_elem); // y-dir vel component - - + ViewCArrayKokkos u(u_array, num_nodes_in_elem); // x-dir vel component + ViewCArrayKokkos v(v_array, num_nodes_in_elem); // y-dir vel component + // true volume RZ - //double r_array[num_nodes_in_elem]; - //ViewCArrayKokkos r(r_array, num_nodes_in_elem); // r-dir coordinate - - + // double r_array[num_nodes_in_elem]; + // ViewCArrayKokkos r(r_array, num_nodes_in_elem); // r-dir coordinate + // cut out the node_gids for this element - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 4); - + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 4); + // The b_matrix are the outward corner area normals double b_matrix_array[24]; - ViewCArrayKokkos b_matrix(b_matrix_array, num_nodes_in_elem, num_dims); + ViewCArrayKokkos b_matrix(b_matrix_array, num_nodes_in_elem, num_dims); get_bmatrix2D(b_matrix, elem_gid, node_coords, elem_node_gids, rk_level); - + // calculate the area of the quad double elem_area = get_area_quad(elem_gid, node_coords, elem_node_gids, rk_level); // true volume uses the elem_vol - - + // get the vertex velocities and node coordinate for the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { // Get node gid size_t node_gid = elem_node_gids(node_lid); - + u(node_lid) = node_vel(rk_level, node_gid, 0); v(node_lid) = node_vel(rk_level, node_gid, 1); - - //r(node_lid) = node_coords(rk_level, node_gid, 1); // true volume RZ - + + // r(node_lid) = node_coords(rk_level, node_gid, 1); // true volume RZ } // end for - - + // --- calculate the velocity divergence terms --- - double inverse_area = 1.0/elem_area; - - double mean_radius = elem_vol(elem_gid)/elem_area; - double elem_vel_r = 0.25*(v(0) + v(1) + v(2) + v(3)); - - + double inverse_area = 1.0 / elem_area; + + double mean_radius = elem_vol(elem_gid) / elem_area; + double elem_vel_r = 0.25 * (v(0) + v(1) + v(2) + v(3)); + elem_div(elem_gid) = 0.0; - + // x-dir - elem_div(elem_gid) += (u(0)*b_matrix(0,0) - + u(1)*b_matrix(1,0) - + u(2)*b_matrix(2,0) - + u(3)*b_matrix(3,0))*inverse_area; - - // y-dir (i.e., r direction) - elem_div(elem_gid) += (v(0)*b_matrix(0,1) - + v(1)*b_matrix(1,1) - + v(2)*b_matrix(2,1) - + v(3)*b_matrix(3,1))*inverse_area - + elem_vel_r/mean_radius; // + avg(u_R)/R + elem_div(elem_gid) += (u(0) * b_matrix(0, 0) + + u(1) * b_matrix(1, 0) + + u(2) * b_matrix(2, 0) + + u(3) * b_matrix(3, 0)) * inverse_area; + // y-dir (i.e., r direction) + elem_div(elem_gid) += (v(0) * b_matrix(0, 1) + + v(1) * b_matrix(1, 1) + + v(2) * b_matrix(2, 1) + + v(3) * b_matrix(3, 1)) * inverse_area + + elem_vel_r / mean_radius; // + avg(u_R)/R }); // end parallel for over elem_gid - + return; - } // end subroutine - // The velocity gradient can be decomposed into symmetric and antisymmetric tensors // L = vel_grad // D = sym(L) // W = antisym(L) KOKKOS_INLINE_FUNCTION -void FEA_Module_Dynamic_Elasticity::decompose_vel_grad(ViewCArrayKokkos &D_tensor, - ViewCArrayKokkos &W_tensor, - const ViewCArrayKokkos &vel_grad, - const ViewCArrayKokkos &elem_node_gids, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const double vol - ) const { - - +void FEA_Module_Dynamic_Elasticity::decompose_vel_grad(ViewCArrayKokkos& D_tensor, + ViewCArrayKokkos& W_tensor, + const ViewCArrayKokkos& vel_grad, + const ViewCArrayKokkos& elem_node_gids, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const double vol + ) const +{ // --- Calculate the velocity gradient --- - + const size_t num_dims = 3; - + // initialize to zero - for(size_t i=0; i &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &elem_den, - DViewCArrayKokkos &elem_pres, - DViewCArrayKokkos &elem_stress, - DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_mass, - const DViewCArrayKokkos &elem_mat_id, - const double rk_alpha, - const size_t cycle - ){ - +void FEA_Module_Dynamic_Elasticity::update_state(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_den, + DViewCArrayKokkos& elem_pres, + DViewCArrayKokkos& elem_stress, + DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& elem_mat_id, + const double rk_alpha, + const size_t cycle + ) +{ const size_t rk_level = rk_num_bins - 1; - int num_dims = num_dim; - + int num_dims = num_dim; + // loop over all the elements in the mesh - FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { - + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { const size_t num_dims = num_dims; const size_t num_nodes_in_elem = num_nodes_in_elem; // cut out the node_gids for this element - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), num_nodes_in_elem); - + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), num_nodes_in_elem); + // --- Density --- - elem_den(elem_gid) = elem_mass(elem_gid)/elem_vol(elem_gid); - + elem_den(elem_gid) = elem_mass(elem_gid) / elem_vol(elem_gid); + size_t mat_id = elem_mat_id(elem_gid); - //initialize elem pressure + // initialize elem pressure elem_pres(elem_gid) = 0; - + // --- Stress --- // hyper elastic plastic model - if(material(mat_id).strength_type == STRENGTH_TYPE::hyper){ - + if (material(mat_id).strength_type == STRENGTH_TYPE::hyper) + { // cut out the node_gids for this element - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), num_nodes_in_elem); - + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), num_nodes_in_elem); + // --- Density --- - elem_den(elem_gid) = elem_mass(elem_gid)/elem_vol(elem_gid); - + elem_den(elem_gid) = elem_mass(elem_gid) / elem_vol(elem_gid); + // corner area normals double area_array[24]; - ViewCArrayKokkos area(area_array, num_nodes_in_elem, num_dims); - + ViewCArrayKokkos area(area_array, num_nodes_in_elem, num_dims); + // velocity gradient double vel_grad_array[9]; - ViewCArrayKokkos vel_grad(vel_grad_array, num_dims, num_dims); - + ViewCArrayKokkos vel_grad(vel_grad_array, num_dims, num_dims); + // get the B matrix which are the OUTWARD corner area normals get_bmatrix(area, elem_gid, node_coords, elem_node_gids, rk_level); - - + // --- Calculate the velocity gradient --- get_velgrad(vel_grad, elem_node_gids, @@ -76,16 +74,18 @@ void FEA_Module_Dynamic_Elasticity::update_state(const DCArrayKokkos calc_stress(elem_pres, elem_stress, @@ -106,7 +106,6 @@ void FEA_Module_Dynamic_Elasticity::update_state(const DCArrayKokkos calc_sound_speed(elem_pres, @@ -133,79 +132,73 @@ void FEA_Module_Dynamic_Elasticity::update_state(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &elem_den, - DViewCArrayKokkos &elem_pres, - DViewCArrayKokkos &elem_stress, - DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_mass, - const DViewCArrayKokkos &elem_mat_id, - const double rk_alpha, - const size_t cycle - ){ - +void FEA_Module_Dynamic_Elasticity::update_state2D(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_den, + DViewCArrayKokkos& elem_pres, + DViewCArrayKokkos& elem_stress, + DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& elem_mat_id, + const double rk_alpha, + const size_t cycle + ) +{ const size_t rk_level = rk_num_bins - 1; - int num_dims = num_dim; - + int num_dims = num_dim; + // loop over all the elements in the mesh - FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { - + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { const size_t num_dims = num_dims; const size_t num_nodes_in_elem = num_nodes_in_elem; // cut out the node_gids for this element - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), num_nodes_in_elem); - + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), num_nodes_in_elem); + // --- Density --- - elem_den(elem_gid) = elem_mass(elem_gid)/elem_vol(elem_gid); - + elem_den(elem_gid) = elem_mass(elem_gid) / elem_vol(elem_gid); + size_t mat_id = elem_mat_id(elem_gid); - //initialize elem pressure + // initialize elem pressure elem_pres(elem_gid) = 0; - + // --- Stress --- // hyper elastic plastic model - if(material(mat_id).strength_type == STRENGTH_TYPE::hyper){ - + if (material(mat_id).strength_type == STRENGTH_TYPE::hyper) + { // cut out the node_gids for this element - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), num_nodes_in_elem); - + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), num_nodes_in_elem); + // --- Density --- - elem_den(elem_gid) = elem_mass(elem_gid)/elem_vol(elem_gid); - + elem_den(elem_gid) = elem_mass(elem_gid) / elem_vol(elem_gid); + // corner area normals double area_array[8]; - ViewCArrayKokkos area(area_array, num_nodes_in_elem, num_dims); - + ViewCArrayKokkos area(area_array, num_nodes_in_elem, num_dims); + // velocity gradient double vel_grad_array[4]; - ViewCArrayKokkos vel_grad(vel_grad_array, num_dims, num_dims); - + ViewCArrayKokkos vel_grad(vel_grad_array, num_dims, num_dims); + // get the B matrix which are the OUTWARD corner area normals get_bmatrix(area, elem_gid, node_coords, elem_node_gids, rk_level); - - + // --- Calculate the velocity gradient --- get_velgrad(vel_grad, elem_node_gids, @@ -214,7 +207,7 @@ void FEA_Module_Dynamic_Elasticity::update_state2D(const DCArrayKokkos calc_sound_speed(elem_pres, @@ -264,14 +256,9 @@ void FEA_Module_Dynamic_Elasticity::update_state2D(const DCArrayKokkos coords; + DCArrayKokkos coords; // velocity - DCArrayKokkos vel; + DCArrayKokkos vel; // mass at nodes - DCArrayKokkos mass; + DCArrayKokkos mass; // Includes Ghost Positions - //DCArrayKokkos all_coords; + // DCArrayKokkos all_coords; // Includes Ghost velocities - //DCArrayKokkos all_vel; + // DCArrayKokkos all_vel; // Includes Ghost masses - //DCArrayKokkos all_mass; + // DCArrayKokkos all_mass; - // initialization method (num_rk_storage_bins, num_nodes, num_dims) void initialize(size_t num_rk, size_t num_nodes, size_t num_dims) { - this->coords = DCArrayKokkos (num_rk, num_nodes, num_dims); - this->vel = DCArrayKokkos (num_rk, num_nodes, num_dims); - this->mass = DCArrayKokkos (num_nodes); + this->coords = DCArrayKokkos(num_rk, num_nodes, num_dims); + this->vel = DCArrayKokkos(num_rk, num_nodes, num_dims); + this->mass = DCArrayKokkos(num_nodes); }; // end method - }; // end node_t - // elem_state -struct elem_t { - +struct elem_t +{ // den - CArray den; - + CArray den; + // pres - CArray pres; - + CArray pres; + // stress - CArray stress; - + CArray stress; + // sspd - CArray sspd; - + CArray sspd; + // sie - CArray sie; + CArray sie; // vol - CArray vol; - + CArray vol; + // divergence of velocity - CArray div; - + CArray div; + // mass of elem - CArray mass; - + CArray mass; + // mat ids - CArray mat_id; - + CArray mat_id; + // initialization method (num_rk_storage_bins, num_cells, num_dims) void initialize(size_t num_rk, size_t num_elems, size_t num_dims) { - this->den = CArray (num_elems); - this->pres = CArray (num_elems); - this->stress = CArray (num_rk, num_elems, num_dims, num_dims); - this->sspd = CArray (num_elems); - this->sie = CArray (num_rk, num_elems); - this->vol = CArray (num_elems); - this->div = CArray (num_elems); - this->mass = CArray (num_elems); - this->mat_id = CArray (num_elems); + this->den = CArray(num_elems); + this->pres = CArray(num_elems); + this->stress = CArray(num_rk, num_elems, num_dims, num_dims); + this->sspd = CArray(num_elems); + this->sie = CArray(num_rk, num_elems); + this->vol = CArray(num_elems); + this->div = CArray(num_elems); + this->mass = CArray(num_elems); + this->mat_id = CArray(num_elems); }; // end method - }; // end elem_t - // corner_state -struct corner_t { - +struct corner_t +{ // force - CArray force; - + CArray force; + // mass of corner - CArray mass; + CArray mass; - // initialization method (num_corners, num_dims) void initialize(size_t num_corners, size_t num_dims) { - this->force = CArray (num_corners, num_dims); - this->mass = CArray (num_corners); + this->force = CArray(num_corners, num_dims); + this->mass = CArray(num_corners); }; // end method - }; // end corner_t -#endif +#endif diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/time_integration.cpp b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/time_integration.cpp index 5f79d7b6b..76d9eda73 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/time_integration.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/time_integration.cpp @@ -5,83 +5,75 @@ // ----------------------------------------------------------------------------- // This function saves the variables at rk_stage = 0, which is t_n -//------------------------------------------------------------------------------ -void FEA_Module_Dynamic_Elasticity::rk_init(DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &elem_sie, - DViewCArrayKokkos &elem_stress, - const size_t num_elems, - const size_t num_nodes){ - +// ------------------------------------------------------------------------------ +void FEA_Module_Dynamic_Elasticity::rk_init(DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_sie, + DViewCArrayKokkos& elem_stress, + const size_t num_elems, + const size_t num_nodes) +{ const size_t rk_level = rk_num_bins - 1; - int num_dims = num_dim; + int num_dims = num_dim; // save elem quantities FOR_ALL_CLASS(elem_gid, 0, num_elems, { - // stress is always 3D even with 2D-RZ - for(size_t i=0; i<3; i++){ - for(size_t j=0; j<3; j++){ - elem_stress(0,elem_gid,i,j) = elem_stress(rk_level,elem_gid,i,j); + for (size_t i = 0; i < 3; i++) + { + for (size_t j = 0; j < 3; j++) + { + elem_stress(0, elem_gid, i, j) = elem_stress(rk_level, elem_gid, i, j); } } // end for - elem_sie(0,elem_gid) = elem_sie(rk_level,elem_gid); - + elem_sie(0, elem_gid) = elem_sie(rk_level, elem_gid); }); // end parallel for - - + // save nodal quantities FOR_ALL_CLASS(node_gid, 0, num_nodes, { - - for(size_t i=0; i &node_coords, - DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &elem_sspd, - DViewCArrayKokkos &elem_vol){ - - +void FEA_Module_Dynamic_Elasticity::get_timestep(mesh_t& mesh, + DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_sspd, + DViewCArrayKokkos& elem_vol) +{ const size_t rk_level = rk_num_bins - 1; // increase dt by 10%, that is the largest dt value - dt = dt*1.1; - int num_dims = num_dim; + dt = dt * 1.1; + int num_dims = num_dim; double dt_lcl; double min_dt_calc; REDUCE_MIN_CLASS(elem_gid, 0, rnum_elem, dt_lcl, { - double coords0[24]; // element coords - ViewCArrayKokkos coords(coords0, 8, 3); + ViewCArrayKokkos coords(coords0, 8, 3); - double distance0[28]; // array for holding distances between each node - ViewCArrayKokkos dist(distance0, 28); - - // Getting the coordinates of the element - for(size_t node_lid = 0; node_lid < 8; node_lid++){ + ViewCArrayKokkos dist(distance0, 28); - for (size_t dim = 0; dim < num_dims; dim++){ - coords(node_lid, dim) = node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), dim); + // Getting the coordinates of the element + for (size_t node_lid = 0; node_lid < 8; node_lid++) + { + for (size_t dim = 0; dim < num_dims; dim++) + { + coords(node_lid, dim) = node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), dim); } // end for dim - } // end for loop over node_lid // loop conditions needed for distance calculation @@ -90,148 +82,150 @@ void FEA_Module_Dynamic_Elasticity::get_timestep(mesh_t &mesh, size_t a; size_t b; size_t loop = 0; - + // Only works for 3D // Solving for the magnitude of distance between each node - for (size_t i = 0; i < 28; i++){ - + for (size_t i = 0; i < 28; i++) + { a = countA; b = countB; - + // returns magnitude of distance between each node, 28 total options - dist(i) = fabs(sqrt(( pow((coords(b, 0) - coords(a, 0)), 2.0) - + pow((coords(b, 1) - coords(a, 1)), 2.0) - + pow((coords(b, 2) - coords(a, 2)), 2.0)))); + dist(i) = fabs(sqrt((pow((coords(b, 0) - coords(a, 0)), 2.0) + + pow((coords(b, 1) - coords(a, 1)), 2.0) + + pow((coords(b, 2) - coords(a, 2)), 2.0)))); countB++; countA++; - + // tricky indexing - if (countB > 7) { + if (countB > 7) + { loop++; countB = 1 + loop; countA = 0; } - } // endo for i - double dist_min = dist(0); - - for(int i = 0; i < 28; ++i){ + + for (int i = 0; i < 28; ++i) + { dist_min = fmin(dist(i), dist_min); } - + // local dt calc based on CFL - double dt_lcl_ = dt_cfl*dist_min/(elem_sspd(elem_gid) + fuzz); - + double dt_lcl_ = dt_cfl * dist_min / (elem_sspd(elem_gid) + fuzz); + // make dt be in bounds dt_lcl_ = fmin(dt_lcl_, dt_max); // make dt small than dt_max dt_lcl_ = fmax(dt_lcl_, dt_min); // make dt larger than dt_min - - - if (dt_lcl_ < dt_lcl) dt_lcl = dt_lcl_; - + + if (dt_lcl_ < dt_lcl) + { + dt_lcl = dt_lcl_; + } }, min_dt_calc); // end parallel reduction Kokkos::fence(); - + // save the min dt - if(min_dt_calc < dt) dt = min_dt_calc; - + if (min_dt_calc < dt) + { + dt = min_dt_calc; + } + // ensure time step hits the graphics time intervals - dt = fmin(dt, (graphics_time - time_value)+fuzz); - + dt = fmin(dt, (graphics_time - time_value) + fuzz); + // make dt be exact for final time - dt = fmin(dt, time_final-time_value); - + dt = fmin(dt, time_final - time_value); + return; - } // end get_timestep // ----------------------------------------------------------------------------- // This function calculates the time step by finding the shortest distance // between any two nodes in the mesh -//------------------------------------------------------------------------------ +// ------------------------------------------------------------------------------ // WARNING WARNING : Only works for 3D, 8 node elements -void FEA_Module_Dynamic_Elasticity::get_timestep2D(mesh_t &mesh, - DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &elem_sspd, - DViewCArrayKokkos &elem_vol){ - +void FEA_Module_Dynamic_Elasticity::get_timestep2D(mesh_t& mesh, + DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_sspd, + DViewCArrayKokkos& elem_vol) +{ const size_t rk_level = rk_num_bins - 1; // increase dt by 10%, that is the largest dt value - dt = dt*1.1; - int num_dims = num_dim; + dt = dt * 1.1; + int num_dims = num_dim; double dt_lcl; double min_dt_calc; REDUCE_MIN_CLASS(elem_gid, 0, rnum_elem, dt_lcl, { - double coords0[8]; // element coords - ViewCArrayKokkos coords(coords0, 4, 2); + ViewCArrayKokkos coords(coords0, 4, 2); - double distance0[6]; // array for holding distances between each node - ViewCArrayKokkos dist(distance0, 6); - - // Getting the coordinates of the nodes of the element - for(size_t node_lid = 0; node_lid < 4; node_lid++){ + ViewCArrayKokkos dist(distance0, 6); - for (size_t dim = 0; dim < num_dims; dim++){ - coords(node_lid, dim) = node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), dim); + // Getting the coordinates of the nodes of the element + for (size_t node_lid = 0; node_lid < 4; node_lid++) + { + for (size_t dim = 0; dim < num_dims; dim++) + { + coords(node_lid, dim) = node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), dim); } // end for dim - } // end for loop over node_lid - // Only works for 2D // Solving for the magnitude of distance between each node size_t count = 0; - for (size_t i = 0; i < 3; i++){ - for (size_t j=i+1; j<=3; j++){ - + for (size_t i = 0; i < 3; i++) + { + for (size_t j = i + 1; j <= 3; j++) + { // returns magnitude of distance between each node, 6 total options dist(count) = fabs( - sqrt( pow((coords(i, 0) - coords(j, 0)), 2.0) - + pow((coords(i, 1) - coords(j, 1)), 2.0) ) - ); - count ++; + sqrt(pow((coords(i, 0) - coords(j, 0)), 2.0) + + pow((coords(i, 1) - coords(j, 1)), 2.0) ) + ); + count++; } // end for j } // end for i - double dist_min = dist(0); - - for(int i = 0; i < 6; ++i){ + + for (int i = 0; i < 6; ++i) + { dist_min = fmin(dist(i), dist_min); } - + // local dt calc based on CFL - double dt_lcl_ = dt_cfl*dist_min/(elem_sspd(elem_gid) + fuzz); - + double dt_lcl_ = dt_cfl * dist_min / (elem_sspd(elem_gid) + fuzz); + // make dt be in bounds dt_lcl_ = fmin(dt_lcl_, dt_max); // make dt small than dt_max dt_lcl_ = fmax(dt_lcl_, dt_min); // make dt larger than dt_min - - - if (dt_lcl_ < dt_lcl) dt_lcl = dt_lcl_; - + + if (dt_lcl_ < dt_lcl) + { + dt_lcl = dt_lcl_; + } }, min_dt_calc); // end parallel reduction Kokkos::fence(); - + // save the min dt - if(min_dt_calc < dt) dt = min_dt_calc; - + if (min_dt_calc < dt) + { + dt = min_dt_calc; + } + // ensure time step hits the graphics time intervals - dt = fmin(dt, (graphics_time - time_value)+fuzz); - + dt = fmin(dt, (graphics_time - time_value) + fuzz); + // make dt be exact for final time - dt = fmin(dt, time_final-time_value); - + dt = fmin(dt, time_final - time_value); + return; - } // end get_timestep2D - - diff --git a/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/FEA_Module_Eulerian.cpp b/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/FEA_Module_Eulerian.cpp index 6602f6c77..d6319aa08 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/FEA_Module_Eulerian.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/FEA_Module_Eulerian.cpp @@ -11,14 +11,14 @@ This program is open source under the BSD-3 License. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - + 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - + 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - + 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. @@ -40,7 +40,7 @@ #include #include #include -#include +#include #include // fmin, fmax, abs note: fminl is long #include #include @@ -76,7 +76,7 @@ #include "FEA_Module_Eulerian.h" #include "Explicit_Solver_Eulerian.h" -//optimization +// optimization #include "ROL_Algorithm.hpp" #include "ROL_Solver.hpp" #include "ROL_LineSearchStep.hpp" @@ -100,617 +100,668 @@ using namespace utils; +FEA_Module_Eulerian::FEA_Module_Eulerian(Solver* Solver_Pointer, mesh_t& mesh, const int my_fea_module_index) : FEA_Module(Solver_Pointer), mesh(mesh) +{ + // assign interfacing index + my_fea_module_index_ = my_fea_module_index; + + // recast solver pointer for non-base class access + Explicit_Solver_Pointer_ = dynamic_cast(Solver_Pointer); + + // create parameter object + simparam = &Explicit_Solver_Pointer_->simparam; + // ---- Read input file, define state and boundary conditions ---- // + // simparam->input(); + + // TO parameters + simparam_dynamic_opt = Explicit_Solver_Pointer_->simparam_dynamic_opt; + + // create ref element object + // ref_elem = new elements::ref_element(); + // create mesh objects + // init_mesh = new swage::mesh_t(simparam); + // mesh = new swage::mesh_t(simparam); + + // boundary condition data + max_boundary_sets = 0; + Local_Index_Boundary_Patches = Explicit_Solver_Pointer_->Local_Index_Boundary_Patches; + + // set Tpetra vector pointers + initial_node_velocities_distributed = Explicit_Solver_Pointer_->initial_node_velocities_distributed; + initial_node_coords_distributed = Explicit_Solver_Pointer_->initial_node_coords_distributed; + all_initial_node_coords_distributed = Explicit_Solver_Pointer_->all_initial_node_coords_distributed; + node_coords_distributed = Explicit_Solver_Pointer_->node_coords_distributed; + node_velocities_distributed = Explicit_Solver_Pointer_->node_velocities_distributed; + all_node_velocities_distributed = Explicit_Solver_Pointer_->all_node_velocities_distributed; + if (simparam_dynamic_opt.topology_optimization_on || simparam_dynamic_opt.shape_optimization_on) + { + all_cached_node_velocities_distributed = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + force_gradient_velocity = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + force_gradient_position = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + force_gradient_design = Teuchos::rcp(new MV(all_node_map, 1)); + corner_value_storage = Solver_Pointer->corner_value_storage; + corner_vector_storage = Solver_Pointer->corner_vector_storage; + relative_element_densities = DCArrayKokkos(rnum_elem, "relative_element_densities"); + } + + if (simparam_dynamic_opt.topology_optimization_on || simparam_dynamic_opt.shape_optimization_on || simparam->num_dims == 2) + { + node_masses_distributed = Teuchos::rcp(new MV(map, 1)); + ghost_node_masses_distributed = Teuchos::rcp(new MV(ghost_node_map, 1)); + adjoint_vector_distributed = Teuchos::rcp(new MV(map, simparam->num_dims)); + phi_adjoint_vector_distributed = Teuchos::rcp(new MV(map, simparam->num_dims)); + } + + // setup output + noutput = 0; + init_output(); -FEA_Module_Eulerian::FEA_Module_Eulerian(Solver *Solver_Pointer, mesh_t& mesh, const int my_fea_module_index) :FEA_Module(Solver_Pointer), mesh(mesh){ - - //assign interfacing index - my_fea_module_index_ = my_fea_module_index; - - //recast solver pointer for non-base class access - Explicit_Solver_Pointer_ = dynamic_cast(Solver_Pointer); - - //create parameter object - simparam = &Explicit_Solver_Pointer_->simparam; - // ---- Read input file, define state and boundary conditions ---- // - //simparam->input(); - - //TO parameters - simparam_dynamic_opt = Explicit_Solver_Pointer_->simparam_dynamic_opt; - - //create ref element object - //ref_elem = new elements::ref_element(); - //create mesh objects - //init_mesh = new swage::mesh_t(simparam); - //mesh = new swage::mesh_t(simparam); - - //boundary condition data - max_boundary_sets = 0; - Local_Index_Boundary_Patches = Explicit_Solver_Pointer_->Local_Index_Boundary_Patches; - - //set Tpetra vector pointers - initial_node_velocities_distributed = Explicit_Solver_Pointer_->initial_node_velocities_distributed; - initial_node_coords_distributed = Explicit_Solver_Pointer_->initial_node_coords_distributed; - all_initial_node_coords_distributed = Explicit_Solver_Pointer_->all_initial_node_coords_distributed; - node_coords_distributed = Explicit_Solver_Pointer_->node_coords_distributed; - node_velocities_distributed = Explicit_Solver_Pointer_->node_velocities_distributed; - all_node_velocities_distributed = Explicit_Solver_Pointer_->all_node_velocities_distributed; - if(simparam_dynamic_opt.topology_optimization_on||simparam_dynamic_opt.shape_optimization_on){ - all_cached_node_velocities_distributed = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - force_gradient_velocity = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - force_gradient_position = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - force_gradient_design = Teuchos::rcp(new MV(all_node_map, 1)); - corner_value_storage = Solver_Pointer->corner_value_storage; - corner_vector_storage = Solver_Pointer->corner_vector_storage; - relative_element_densities = DCArrayKokkos(rnum_elem, "relative_element_densities"); - } - - if(simparam_dynamic_opt.topology_optimization_on||simparam_dynamic_opt.shape_optimization_on||simparam->num_dims==2){ - node_masses_distributed = Teuchos::rcp(new MV(map, 1)); - ghost_node_masses_distributed = Teuchos::rcp(new MV(ghost_node_map, 1)); - adjoint_vector_distributed = Teuchos::rcp(new MV(map, simparam->num_dims)); - phi_adjoint_vector_distributed = Teuchos::rcp(new MV(map, simparam->num_dims)); - } - - //setup output - noutput = 0; - init_output(); - - //optimization flags - kinetic_energy_objective = false; - - - //set parameters - Time_Variables tv = simparam->time_variables; - time_value = simparam->time_value; - time_final = tv.time_final; - dt_max = tv.dt_max; - dt_min = tv.dt_min; - dt_cfl = tv.dt_cfl; - graphics_time = simparam->graphics_options.graphics_time; - graphics_cyc_ival = simparam->graphics_options.graphics_cyc_ival; - graphics_dt_ival = simparam->graphics_options.graphics_dt_ival; - cycle_stop = tv.cycle_stop; - rk_num_stages = simparam->rk_num_stages; - dt = tv.dt; - fuzz = tv.fuzz; - tiny = tv.tiny; - small = tv.small; - graphics_times = simparam->graphics_options.graphics_times; - graphics_id = simparam->graphics_options.graphics_id; - - if(simparam_dynamic_opt.topology_optimization_on){ - max_time_steps = BUFFER_GROW; - forward_solve_velocity_data = Teuchos::rcp(new std::vector>(max_time_steps+1)); - time_data.resize(max_time_steps+1); - forward_solve_coordinate_data = Teuchos::rcp(new std::vector>(max_time_steps+1)); - adjoint_vector_data = Teuchos::rcp(new std::vector>(max_time_steps+1)); - phi_adjoint_vector_data = Teuchos::rcp(new std::vector>(max_time_steps+1)); - //assign a multivector of corresponding size to each new timestep in the buffer - for(int istep = 0; istep < max_time_steps+1; istep++){ - (*forward_solve_velocity_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - (*forward_solve_coordinate_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - (*adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - (*phi_adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + // optimization flags + kinetic_energy_objective = false; + + // set parameters + Time_Variables tv = simparam->time_variables; + time_value = simparam->time_value; + time_final = tv.time_final; + dt_max = tv.dt_max; + dt_min = tv.dt_min; + dt_cfl = tv.dt_cfl; + graphics_time = simparam->graphics_options.graphics_time; + graphics_cyc_ival = simparam->graphics_options.graphics_cyc_ival; + graphics_dt_ival = simparam->graphics_options.graphics_dt_ival; + cycle_stop = tv.cycle_stop; + rk_num_stages = simparam->rk_num_stages; + dt = tv.dt; + fuzz = tv.fuzz; + tiny = tv.tiny; + small = tv.small; + graphics_times = simparam->graphics_options.graphics_times; + graphics_id = simparam->graphics_options.graphics_id; + + if (simparam_dynamic_opt.topology_optimization_on) + { + max_time_steps = BUFFER_GROW; + forward_solve_velocity_data = Teuchos::rcp(new std::vector>(max_time_steps + 1)); + time_data.resize(max_time_steps + 1); + forward_solve_coordinate_data = Teuchos::rcp(new std::vector>(max_time_steps + 1)); + adjoint_vector_data = Teuchos::rcp(new std::vector>(max_time_steps + 1)); + phi_adjoint_vector_data = Teuchos::rcp(new std::vector>(max_time_steps + 1)); + // assign a multivector of corresponding size to each new timestep in the buffer + for (int istep = 0; istep < max_time_steps + 1; istep++) + { + (*forward_solve_velocity_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*forward_solve_coordinate_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*phi_adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + } } - - } } -FEA_Module_Eulerian::~FEA_Module_Eulerian(){ - //delete simparam; +FEA_Module_Eulerian::~FEA_Module_Eulerian() +{ + // delete simparam; } /* ---------------------------------------------------------------------- Read ANSYS dat format mesh file ------------------------------------------------------------------------- */ -void FEA_Module_Eulerian::read_conditions_ansys_dat(std::ifstream *in, std::streampos before_condition_header){ - - char ch; - int num_dim = simparam->num_dims; - int buffer_lines = 1000; - int max_word = 30; - int p_order = simparam->p_order; - real_t unit_scaling = simparam->get_unit_scaling(); - int local_node_index, current_column_index; - size_t strain_count; - std::string skip_line, read_line, substring, token; - std::stringstream line_parse, line_parse2; - CArrayKokkos read_buffer; - CArrayKokkos read_buffer_indices; - int buffer_loop, buffer_iteration, buffer_iterations, scan_loop, nodes_per_element, words_per_line; - size_t read_index_start, node_rid, elem_gid; - LO local_dof_id; - GO node_gid; - real_t dof_value; - host_vec_array node_densities; - +void FEA_Module_Eulerian::read_conditions_ansys_dat(std::ifstream* in, std::streampos before_condition_header) +{ + char ch; + int num_dim = simparam->num_dims; + int buffer_lines = 1000; + int max_word = 30; + int p_order = simparam->p_order; + real_t unit_scaling = simparam->get_unit_scaling(); + int local_node_index, current_column_index; + size_t strain_count; + std::string skip_line, read_line, substring, token; + std::stringstream line_parse, line_parse2; + CArrayKokkos read_buffer; + CArrayKokkos read_buffer_indices; + int buffer_loop, buffer_iteration, buffer_iterations, scan_loop, nodes_per_element, words_per_line; + size_t read_index_start, node_rid, elem_gid; + LO local_dof_id; + GO node_gid; + real_t dof_value; + host_vec_array node_densities; } // end read_conditions_ansys_dat - /* ---------------------------------------------------------------------------- Initialize sets of element boundary surfaces and arrays for input conditions ------------------------------------------------------------------------------- */ -void FEA_Module_Eulerian::init_boundaries(){ - max_boundary_sets = simparam->NB; - int num_dim = simparam->num_dims; - - // set the number of boundary sets - if(myrank == 0) - std::cout << "building boundary sets " << std::endl; - - //initialize to 1 since there must be at least 1 boundary set anyway; read in may occure later - if(max_boundary_sets==0) max_boundary_sets = 1; - //std::cout << "NUM BOUNDARY CONDITIONS ON RANK " << myrank << " FOR INIT " << num_boundary_conditions <(nall_nodes*num_dim, "Node_DOF_Boundary_Condition_Type"); - - //initialize - for(int init=0; init < nall_nodes*num_dim; init++) - Node_DOF_Boundary_Condition_Type(init) = NONE; - - Number_DOF_BCS = 0; +void FEA_Module_Eulerian::init_boundaries() +{ + max_boundary_sets = simparam->NB; + int num_dim = simparam->num_dims; + + // set the number of boundary sets + if (myrank == 0) + { + std::cout << "building boundary sets " << std::endl; + } + + // initialize to 1 since there must be at least 1 boundary set anyway; read in may occure later + if (max_boundary_sets == 0) + { + max_boundary_sets = 1; + } + // std::cout << "NUM BOUNDARY CONDITIONS ON RANK " << myrank << " FOR INIT " << num_boundary_conditions <(nall_nodes * num_dim, "Node_DOF_Boundary_Condition_Type"); + + // initialize + for (int init = 0; init < nall_nodes * num_dim; init++) + { + Node_DOF_Boundary_Condition_Type(init) = NONE; + } + + Number_DOF_BCS = 0; } /* ---------------------------------------------------------------------- initialize storage for element boundary surfaces corresponding to user BCs ------------------------------------------------------------------------- */ -void FEA_Module_Eulerian::init_boundary_sets (int num_sets){ +void FEA_Module_Eulerian::init_boundary_sets(int num_sets) +{ + if (num_sets == 0) + { + std::cout << " Warning: number of boundary conditions = 0"; + return; + } + // initialize maximum + max_boundary_sets = num_sets; + // std::cout << " DEBUG PRINT "<(num_sets, "Boundary_Condition_Type_List"); + NBoundary_Condition_Patches = CArrayKokkos(num_sets, "NBoundary_Condition_Patches"); + // std::cout << "NBOUNDARY PATCHES ON RANK " << myrank << " FOR INIT IS " << nboundary_patches <(num_sets, nboundary_patches, "Boundary_Condition_Patches"); + + // initialize data + for (int iset = 0; iset < num_sets; iset++) + { + NBoundary_Condition_Patches(iset) = 0; + } - if(num_sets == 0){ - std::cout << " Warning: number of boundary conditions = 0"; - return; - } - //initialize maximum - max_boundary_sets = num_sets; - //std::cout << " DEBUG PRINT "<(num_sets, "Boundary_Condition_Type_List"); - NBoundary_Condition_Patches = CArrayKokkos(num_sets, "NBoundary_Condition_Patches"); - //std::cout << "NBOUNDARY PATCHES ON RANK " << myrank << " FOR INIT IS " << nboundary_patches <(num_sets, nboundary_patches, "Boundary_Condition_Patches"); - - //initialize data - for(int iset = 0; iset < num_sets; iset++) NBoundary_Condition_Patches(iset) = 0; - - //initialize - for(int ibdy=0; ibdy < num_sets; ibdy++) Boundary_Condition_Type_List(ibdy) = NONE; + // initialize + for (int ibdy = 0; ibdy < num_sets; ibdy++) + { + Boundary_Condition_Type_List(ibdy) = NONE; + } } /* ---------------------------------------------------------------------------- Grow boundary conditions sets of element boundary surfaces ------------------------------------------------------------------------------- */ -void FEA_Module_Eulerian::grow_boundary_sets(int num_sets){ - int num_dim = simparam->num_dims; +void FEA_Module_Eulerian::grow_boundary_sets(int num_sets) +{ + int num_dim = simparam->num_dims; - if(num_sets == 0){ - std::cout << " Warning: number of boundary conditions being set to 0"; - return; - } - - //std::cout << " DEBUG PRINT "<max_boundary_sets){ - //temporary storage for previous data - CArrayKokkos Temp_Boundary_Condition_Type_List = Boundary_Condition_Type_List; - CArrayKokkos Temp_NBoundary_Condition_Patches = NBoundary_Condition_Patches; - CArrayKokkos Temp_Boundary_Condition_Patches = Boundary_Condition_Patches; - - max_boundary_sets = num_sets + 5; //5 is an arbitrary buffer - Boundary_Condition_Type_List = CArrayKokkos(max_boundary_sets, "Boundary_Condition_Type_List"); - NBoundary_Condition_Patches = CArrayKokkos(max_boundary_sets, "NBoundary_Condition_Patches"); - //std::cout << "NBOUNDARY PATCHES ON RANK " << myrank << " FOR GROW " << nboundary_patches <(max_boundary_sets, nboundary_patches, "Boundary_Condition_Patches"); - - //copy previous data back over - //std::cout << "NUM BOUNDARY CONDITIONS ON RANK " << myrank << " FOR COPY " << max_boundary_sets < max_boundary_sets) + { + // temporary storage for previous data + CArrayKokkos Temp_Boundary_Condition_Type_List = Boundary_Condition_Type_List; + CArrayKokkos Temp_NBoundary_Condition_Patches = NBoundary_Condition_Patches; + CArrayKokkos Temp_Boundary_Condition_Patches = Boundary_Condition_Patches; + + max_boundary_sets = num_sets + 5; // 5 is an arbitrary buffer + Boundary_Condition_Type_List = CArrayKokkos(max_boundary_sets, "Boundary_Condition_Type_List"); + NBoundary_Condition_Patches = CArrayKokkos(max_boundary_sets, "NBoundary_Condition_Patches"); + // std::cout << "NBOUNDARY PATCHES ON RANK " << myrank << " FOR GROW " << nboundary_patches <(max_boundary_sets, nboundary_patches, "Boundary_Condition_Patches"); + + // copy previous data back over + // std::cout << "NUM BOUNDARY CONDITIONS ON RANK " << myrank << " FOR COPY " << max_boundary_sets <output_options.output_velocity; - bool output_strain_flag = simparam->output_options.output_strain; - bool output_stress_flag = simparam->output_options.output_stress; - int num_dim = simparam->num_dims; - int Brows; - if(num_dim==3) Brows = 6; - else Brows = 3; - - if(output_velocity_flag){ - //displacement_index is accessed by writers at the solver level for deformed output - output_velocity_index = noutput; - noutput += 1; - module_outputs.resize(noutput); - - vector_style.resize(noutput); - vector_style[noutput-1] = DOF; - - output_vector_sizes.resize(noutput); - output_vector_sizes[noutput-1] = num_dim; - - output_dof_names.resize(noutput); - output_dof_names[noutput-1].resize(num_dim); - output_dof_names[noutput-1][0] = "vx"; - output_dof_names[noutput-1][1] = "vy"; - if(num_dim==3) - output_dof_names[noutput-1][2] = "vz"; - } - if(output_strain_flag){ - output_strain_index = noutput; - noutput += 1; - module_outputs.resize(noutput); - - vector_style.resize(noutput); - vector_style[noutput-1] = NODAL; - - output_vector_sizes.resize(noutput); - output_vector_sizes[noutput-1] = Brows; - - output_dof_names.resize(noutput); - output_dof_names[noutput-1].resize(Brows); - if(num_dim==2){ - output_dof_names[noutput-1][0] = "strain_xx"; - output_dof_names[noutput-1][1] = "strain_yy"; - output_dof_names[noutput-1][2] = "strain_xy"; +void FEA_Module_Eulerian::init_output() +{ + // check user parameters for output + bool output_velocity_flag = simparam->output_options.output_velocity; + bool output_strain_flag = simparam->output_options.output_strain; + bool output_stress_flag = simparam->output_options.output_stress; + int num_dim = simparam->num_dims; + int Brows; + if (num_dim == 3) + { + Brows = 6; } - if(num_dim==3){ - output_dof_names[noutput-1][0] = "strain_xx"; - output_dof_names[noutput-1][1] = "strain_yy"; - output_dof_names[noutput-1][2] = "strain_zz"; - output_dof_names[noutput-1][3] = "strain_xy"; - output_dof_names[noutput-1][4] = "strain_xz"; - output_dof_names[noutput-1][5] = "strain_yz"; + else + { + Brows = 3; } - } - if(output_stress_flag){ - output_stress_index = noutput; - noutput += 1; - module_outputs.resize(noutput); - - vector_style.resize(noutput); - vector_style[noutput-1] = NODAL; - - output_vector_sizes.resize(noutput); - output_vector_sizes[noutput-1] = Brows; - - output_dof_names.resize(noutput); - output_dof_names[noutput-1].resize(Brows); - if(num_dim==2){ - output_dof_names[noutput-1][0] = "stress_xx"; - output_dof_names[noutput-1][1] = "stress_yy"; - output_dof_names[noutput-1][3] = "stress_xy"; + + if (output_velocity_flag) + { + // displacement_index is accessed by writers at the solver level for deformed output + output_velocity_index = noutput; + noutput += 1; + module_outputs.resize(noutput); + + vector_style.resize(noutput); + vector_style[noutput - 1] = DOF; + + output_vector_sizes.resize(noutput); + output_vector_sizes[noutput - 1] = num_dim; + + output_dof_names.resize(noutput); + output_dof_names[noutput - 1].resize(num_dim); + output_dof_names[noutput - 1][0] = "vx"; + output_dof_names[noutput - 1][1] = "vy"; + if (num_dim == 3) + { + output_dof_names[noutput - 1][2] = "vz"; + } } - if(num_dim==3){ - output_dof_names[noutput-1][0] = "stress_xx"; - output_dof_names[noutput-1][1] = "stress_yy"; - output_dof_names[noutput-1][2] = "stress_zz"; - output_dof_names[noutput-1][3] = "stress_xy"; - output_dof_names[noutput-1][4] = "stress_xz"; - output_dof_names[noutput-1][5] = "stress_yz"; + if (output_strain_flag) + { + output_strain_index = noutput; + noutput += 1; + module_outputs.resize(noutput); + + vector_style.resize(noutput); + vector_style[noutput - 1] = NODAL; + + output_vector_sizes.resize(noutput); + output_vector_sizes[noutput - 1] = Brows; + + output_dof_names.resize(noutput); + output_dof_names[noutput - 1].resize(Brows); + if (num_dim == 2) + { + output_dof_names[noutput - 1][0] = "strain_xx"; + output_dof_names[noutput - 1][1] = "strain_yy"; + output_dof_names[noutput - 1][2] = "strain_xy"; + } + if (num_dim == 3) + { + output_dof_names[noutput - 1][0] = "strain_xx"; + output_dof_names[noutput - 1][1] = "strain_yy"; + output_dof_names[noutput - 1][2] = "strain_zz"; + output_dof_names[noutput - 1][3] = "strain_xy"; + output_dof_names[noutput - 1][4] = "strain_xz"; + output_dof_names[noutput - 1][5] = "strain_yz"; + } + } + if (output_stress_flag) + { + output_stress_index = noutput; + noutput += 1; + module_outputs.resize(noutput); + + vector_style.resize(noutput); + vector_style[noutput - 1] = NODAL; + + output_vector_sizes.resize(noutput); + output_vector_sizes[noutput - 1] = Brows; + + output_dof_names.resize(noutput); + output_dof_names[noutput - 1].resize(Brows); + if (num_dim == 2) + { + output_dof_names[noutput - 1][0] = "stress_xx"; + output_dof_names[noutput - 1][1] = "stress_yy"; + output_dof_names[noutput - 1][3] = "stress_xy"; + } + if (num_dim == 3) + { + output_dof_names[noutput - 1][0] = "stress_xx"; + output_dof_names[noutput - 1][1] = "stress_yy"; + output_dof_names[noutput - 1][2] = "stress_zz"; + output_dof_names[noutput - 1][3] = "stress_xy"; + output_dof_names[noutput - 1][4] = "stress_xz"; + output_dof_names[noutput - 1][5] = "stress_yz"; + } } - } } /* ------------------------------------------------------------------------------------------- Prompts sorting for elastic response output data. For now, nodal strains. ---------------------------------------------------------------------------------------------- */ -void FEA_Module_Eulerian::sort_output(Teuchos::RCP > sorted_map){ - - +void FEA_Module_Eulerian::sort_output(Teuchos::RCP> sorted_map) +{ } /* ------------------------------------------------------------------------------------------- Prompts computation of elastic response output data. For now, nodal strains. ---------------------------------------------------------------------------------------------- */ -void FEA_Module_Eulerian::collect_output(Teuchos::RCP > global_reduce_map){ - +void FEA_Module_Eulerian::collect_output(Teuchos::RCP> global_reduce_map) +{ } /* ------------------------------------------------------------------------------------------- Prompts computation of elastic response output data. For now, nodal strains. ---------------------------------------------------------------------------------------------- */ -void FEA_Module_Eulerian::compute_output(){ - +void FEA_Module_Eulerian::compute_output() +{ } /* ---------------------------------------------------------------------- Communicate updated nodal velocities to ghost nodes ------------------------------------------------------------------------- */ -void FEA_Module_Eulerian::comm_node_masses(){ - - //debug print of design vector - //std::ostream &out = std::cout; - //Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - //if(myrank==0) - //*fos << "Density data :" << std::endl; - //node_densities_distributed->describe(*fos,Teuchos::VERB_EXTREME); - //*fos << std::endl; - //std::fflush(stdout); - - //communicate design densities - //create import object using local node indices map and all indices map - //Tpetra::Import importer(map, ghost_node_map); - - //comms to get ghosts - ghost_node_masses_distributed->doImport(*node_masses_distributed, *ghost_importer, Tpetra::INSERT); - //all_node_map->describe(*fos,Teuchos::VERB_EXTREME); - //all_node_velocities_distributed->describe(*fos,Teuchos::VERB_EXTREME); - - //update_count++; - //if(update_count==1){ - //MPI_Barrier(world); - //MPI_Abort(world,4); - //} +void FEA_Module_Eulerian::comm_node_masses() +{ + // debug print of design vector + // std::ostream &out = std::cout; + // Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + // if(myrank==0) + // *fos << "Density data :" << std::endl; + // node_densities_distributed->describe(*fos,Teuchos::VERB_EXTREME); + // *fos << std::endl; + // std::fflush(stdout); + + // communicate design densities + // create import object using local node indices map and all indices map + // Tpetra::Import importer(map, ghost_node_map); + + // comms to get ghosts + ghost_node_masses_distributed->doImport(*node_masses_distributed, *ghost_importer, Tpetra::INSERT); + // all_node_map->describe(*fos,Teuchos::VERB_EXTREME); + // all_node_velocities_distributed->describe(*fos,Teuchos::VERB_EXTREME); + + // update_count++; + // if(update_count==1){ + // MPI_Barrier(world); + // MPI_Abort(world,4); + // } } /* ---------------------------------------------------------------------- Communicate updated nodal adjoint vectors to ghost nodes ------------------------------------------------------------------------- */ -void FEA_Module_Eulerian::comm_adjoint_vectors(int cycle){ - - //debug print of design vector - //std::ostream &out = std::cout; - //Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - //if(myrank==0) - //*fos << "Density data :" << std::endl; - //node_densities_distributed->describe(*fos,Teuchos::VERB_EXTREME); - //*fos << std::endl; - //std::fflush(stdout); - - //communicate design densities - //create import object using local node indices map and all indices map - //Tpetra::Import importer(map, all_node_map); - - //comms to get ghosts - (*adjoint_vector_data)[cycle]->doImport(*adjoint_vector_distributed, *importer, Tpetra::INSERT); - (*phi_adjoint_vector_data)[cycle]->doImport(*phi_adjoint_vector_distributed, *importer, Tpetra::INSERT); - //all_node_map->describe(*fos,Teuchos::VERB_EXTREME); - //all_node_velocities_distributed->describe(*fos,Teuchos::VERB_EXTREME); - - //update_count++; - //if(update_count==1){ - //MPI_Barrier(world); - //MPI_Abort(world,4); - //} +void FEA_Module_Eulerian::comm_adjoint_vectors(int cycle) +{ + // debug print of design vector + // std::ostream &out = std::cout; + // Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + // if(myrank==0) + // *fos << "Density data :" << std::endl; + // node_densities_distributed->describe(*fos,Teuchos::VERB_EXTREME); + // *fos << std::endl; + // std::fflush(stdout); + + // communicate design densities + // create import object using local node indices map and all indices map + // Tpetra::Import importer(map, all_node_map); + + // comms to get ghosts + (*adjoint_vector_data)[cycle]->doImport(*adjoint_vector_distributed, *importer, Tpetra::INSERT); + (*phi_adjoint_vector_data)[cycle]->doImport(*phi_adjoint_vector_distributed, *importer, Tpetra::INSERT); + // all_node_map->describe(*fos,Teuchos::VERB_EXTREME); + // all_node_velocities_distributed->describe(*fos,Teuchos::VERB_EXTREME); + + // update_count++; + // if(update_count==1){ + // MPI_Barrier(world); + // MPI_Abort(world,4); + // } } /* ------------------------------------------------------------------------------------------- Communicate ghosts using the current optimization design data ---------------------------------------------------------------------------------------------- */ -void FEA_Module_Eulerian::comm_variables(Teuchos::RCP zp){ - - if(simparam_dynamic_opt.topology_optimization_on){ - //set density vector to the current value chosen by the optimizer - test_node_densities_distributed = zp; - - //debug print of design vector - //std::ostream &out = std::cout; - //Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - //if(myrank==0) - //*fos << "Density data :" << std::endl; - //node_densities_distributed->describe(*fos,Teuchos::VERB_EXTREME); - //*fos << std::endl; - //std::fflush(stdout); - - //communicate design densities - //create import object using local node indices map and all indices map - //Tpetra::Import importer(map, all_node_map); - - //comms to get ghosts - all_node_densities_distributed->doImport(*test_node_densities_distributed, *importer, Tpetra::INSERT); - } - else if(simparam_dynamic_opt.shape_optimization_on){ - //clause to communicate boundary node data if the boundary nodes are ghosts on this rank - } +void FEA_Module_Eulerian::comm_variables(Teuchos::RCP zp) +{ + if (simparam_dynamic_opt.topology_optimization_on) + { + // set density vector to the current value chosen by the optimizer + test_node_densities_distributed = zp; + + // debug print of design vector + // std::ostream &out = std::cout; + // Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + // if(myrank==0) + // *fos << "Density data :" << std::endl; + // node_densities_distributed->describe(*fos,Teuchos::VERB_EXTREME); + // *fos << std::endl; + // std::fflush(stdout); + + // communicate design densities + // create import object using local node indices map and all indices map + // Tpetra::Import importer(map, all_node_map); + + // comms to get ghosts + all_node_densities_distributed->doImport(*test_node_densities_distributed, *importer, Tpetra::INSERT); + } + else if (simparam_dynamic_opt.shape_optimization_on) + { + // clause to communicate boundary node data if the boundary nodes are ghosts on this rank + } } - /* ------------------------------------------------------------------------------------------- enforce constraints on nodes due to BCS ---------------------------------------------------------------------------------------------- */ -void FEA_Module_Eulerian::node_density_constraints(host_vec_array node_densities_lower_bound){ - +void FEA_Module_Eulerian::node_density_constraints(host_vec_array node_densities_lower_bound) +{ } /* ---------------------------------------------------------------------------- Setup Eulerian solver data ------------------------------------------------------------------------------- */ -void FEA_Module_Eulerian::setup(){ - - const size_t rk_level = simparam->rk_num_bins - 1; - const size_t num_fills = simparam->region_options.size(); - const size_t rk_num_bins = simparam->rk_num_bins; - const size_t num_bcs = simparam->boundary_conditions.size(); +void FEA_Module_Eulerian::setup() +{ + const size_t rk_level = simparam->rk_num_bins - 1; + const size_t num_fills = simparam->region_options.size(); + const size_t rk_num_bins = simparam->rk_num_bins; + const size_t num_bcs = simparam->boundary_conditions.size(); const size_t num_materials = simparam->material_options.size(); - const int num_dim = simparam->num_dims; + const int num_dim = simparam->num_dims; - const DCArrayKokkos mat_fill = simparam->mat_fill; - const DCArrayKokkos boundary = simparam->boundary; - const DCArrayKokkos material = simparam->material; + const DCArrayKokkos mat_fill = simparam->mat_fill; + const DCArrayKokkos boundary = simparam->boundary; + const DCArrayKokkos material = simparam->material; global_vars = simparam->global_vars; - state_vars = DCArrayKokkos (rnum_elem, simparam->max_num_state_vars); - - //--- calculate bdy sets ---// - mesh.num_nodes_in_patch = 2*(num_dim-1); // 2 (2D) or 4 (3D) - mesh.num_patches_in_elem = 2*num_dim; // 4 (2D) or 6 (3D) + state_vars = DCArrayKokkos(rnum_elem, simparam->max_num_state_vars); + + // --- calculate bdy sets ---// + mesh.num_nodes_in_patch = 2 * (num_dim - 1); // 2 (2D) or 4 (3D) + mesh.num_patches_in_elem = 2 * num_dim; // 4 (2D) or 6 (3D) mesh.init_bdy_sets(num_bcs); num_bdy_sets = mesh.num_bdy_sets; printf("Num BC's = %lu\n", num_bcs); // patch ids in bdy set bdy_patches_in_set = mesh.bdy_patches_in_set; - if(num_dim==2) - bdy_nodes = mesh.bdy_nodes; - + if (num_dim == 2) + { + bdy_nodes = mesh.bdy_nodes; + } + // node ids in bdy_patch set - bdy_nodes_in_set = mesh.bdy_nodes_in_set; + bdy_nodes_in_set = mesh.bdy_nodes_in_set; num_bdy_nodes_in_set = mesh.num_bdy_nodes_in_set; - - //assign mesh views needed by the FEA module + + // assign mesh views needed by the FEA module // elem ids in elem - elems_in_elem = mesh.elems_in_elem; + elems_in_elem = mesh.elems_in_elem; num_elems_in_elem = mesh.num_elems_in_elem; - //corners + // corners num_corners_in_node = mesh.num_corners_in_node; - corners_in_node = mesh.corners_in_node; - corners_in_elem = mesh.corners_in_elem; + corners_in_node = mesh.corners_in_node; + corners_in_elem = mesh.corners_in_elem; - //elem-node conn & node-node conn + // elem-node conn & node-node conn elems_in_node = mesh.elems_in_node; - if(num_dim==2){ - nodes_in_node = mesh.nodes_in_node; - num_nodes_in_node = mesh.num_nodes_in_node; - //patch conn - - patches_in_elem = mesh.patches_in_elem; - nodes_in_patch = mesh.nodes_in_patch; - elems_in_patch = mesh.elems_in_patch; + if (num_dim == 2) + { + nodes_in_node = mesh.nodes_in_node; + num_nodes_in_node = mesh.num_nodes_in_node; + // patch conn + + patches_in_elem = mesh.patches_in_elem; + nodes_in_patch = mesh.nodes_in_patch; + elems_in_patch = mesh.elems_in_patch; } - - //initialize if topology optimization is used - if(simparam_dynamic_opt.topology_optimization_on||simparam_dynamic_opt.shape_optimization_on){ - //create parameter object - simparam_elasticity = Simulation_Parameters_Elasticity(); - init_assembly(); - assemble_matrix(); + + // initialize if topology optimization is used + if (simparam_dynamic_opt.topology_optimization_on || simparam_dynamic_opt.shape_optimization_on) + { + // create parameter object + simparam_elasticity = Simulation_Parameters_Elasticity(); + init_assembly(); + assemble_matrix(); } return; - } // end of setup /* ---------------------------------------------------------------------------- Eulerian solver loop ------------------------------------------------------------------------------- */ -void FEA_Module_Eulerian::euler_solve(){ - +void FEA_Module_Eulerian::euler_solve() +{ Time_Variables tv = simparam->time_variables; - - const size_t rk_level = simparam->rk_num_bins - 1; - time_value = simparam->time_value; - time_final = tv.time_final; - dt_max = tv.dt_max; - dt_min = tv.dt_min; - dt_cfl = tv.dt_cfl; - graphics_time = simparam->graphics_options.graphics_time; + + const size_t rk_level = simparam->rk_num_bins - 1; + time_value = simparam->time_value; + time_final = tv.time_final; + dt_max = tv.dt_max; + dt_min = tv.dt_min; + dt_cfl = tv.dt_cfl; + graphics_time = simparam->graphics_options.graphics_time; graphics_cyc_ival = simparam->graphics_options.graphics_cyc_ival; - graphics_dt_ival = simparam->graphics_options.graphics_dt_ival; - cycle_stop = tv.cycle_stop; - rk_num_stages = simparam->rk_num_stages; - dt = tv.dt; - fuzz = tv.fuzz; - tiny = tv.tiny; + graphics_dt_ival = simparam->graphics_options.graphics_dt_ival; + cycle_stop = tv.cycle_stop; + rk_num_stages = simparam->rk_num_stages; + dt = tv.dt; + fuzz = tv.fuzz; + tiny = tv.tiny; small = tv.small; graphics_times = simparam->graphics_options.graphics_times; - graphics_id = simparam->graphics_options.graphics_id; - size_t num_bdy_nodes = mesh.num_bdy_nodes; - const DCArrayKokkos boundary = simparam->boundary; - const DCArrayKokkos material = simparam->material; - int nTO_modules; - int old_max_forward_buffer; - size_t cycle; - const int num_dim = simparam->num_dims; - real_t objective_accumulation, global_objective_accumulation; - std::vector> FEA_Module_My_TO_Modules = simparam_dynamic_opt.FEA_Module_My_TO_Modules; - problem = Explicit_Solver_Pointer_->problem; //Pointer to ROL optimization problem object + graphics_id = simparam->graphics_options.graphics_id; + size_t num_bdy_nodes = mesh.num_bdy_nodes; + const DCArrayKokkos boundary = simparam->boundary; + const DCArrayKokkos material = simparam->material; + int nTO_modules; + int old_max_forward_buffer; + size_t cycle; + const int num_dim = simparam->num_dims; + real_t objective_accumulation, global_objective_accumulation; + std::vector> FEA_Module_My_TO_Modules = simparam_dynamic_opt.FEA_Module_My_TO_Modules; + problem = Explicit_Solver_Pointer_->problem; // Pointer to ROL optimization problem object ROL::Ptr> obj_pointer; - //reset time accumulating objective and constraints + // reset time accumulating objective and constraints /* for(int imodule = 0 ; imodule < FEA_Module_My_TO_Modules[my_fea_module_index_].size(); imodule++){ current_module_index = FEA_Module_My_TO_Modules[my_fea_module_index_][imodule]; //test if module needs reset if(){ - + } } */ - //simple setup to just request KE for now; above loop to be expanded and used later for scanning modules - if(simparam_dynamic_opt.topology_optimization_on){ - obj_pointer = problem->getObjective(); - KineticEnergyMinimize_TopOpt& kinetic_energy_minimize_function = dynamic_cast(*obj_pointer); - kinetic_energy_minimize_function.objective_accumulation = 0; - global_objective_accumulation = objective_accumulation = 0; - kinetic_energy_objective = true; - if(max_time_steps +1 > forward_solve_velocity_data->size()){ - old_max_forward_buffer = forward_solve_velocity_data->size(); - time_data.resize(max_time_steps+1); - forward_solve_velocity_data->resize(max_time_steps+1); - forward_solve_coordinate_data->resize(max_time_steps+1); - adjoint_vector_data->resize(max_time_steps+1); - phi_adjoint_vector_data->resize(max_time_steps+1); - //assign a multivector of corresponding size to each new timestep in the buffer - for(int istep = old_max_forward_buffer; istep < max_time_steps+1; istep++){ - (*forward_solve_velocity_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - (*forward_solve_coordinate_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - (*adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - (*phi_adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + // simple setup to just request KE for now; above loop to be expanded and used later for scanning modules + if (simparam_dynamic_opt.topology_optimization_on) + { + obj_pointer = problem->getObjective(); + KineticEnergyMinimize_TopOpt& kinetic_energy_minimize_function = dynamic_cast(*obj_pointer); + kinetic_energy_minimize_function.objective_accumulation = 0; + global_objective_accumulation = objective_accumulation = 0; + kinetic_energy_objective = true; + if (max_time_steps + 1 > forward_solve_velocity_data->size()) + { + old_max_forward_buffer = forward_solve_velocity_data->size(); + time_data.resize(max_time_steps + 1); + forward_solve_velocity_data->resize(max_time_steps + 1); + forward_solve_coordinate_data->resize(max_time_steps + 1); + adjoint_vector_data->resize(max_time_steps + 1); + phi_adjoint_vector_data->resize(max_time_steps + 1); + // assign a multivector of corresponding size to each new timestep in the buffer + for (int istep = old_max_forward_buffer; istep < max_time_steps + 1; istep++) + { + (*forward_solve_velocity_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*forward_solve_coordinate_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*phi_adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + } } - } } - if(simparam_dynamic_opt.topology_optimization_on) - nTO_modules = simparam_dynamic_opt.TO_Module_List.size(); + if (simparam_dynamic_opt.topology_optimization_on) + { + nTO_modules = simparam_dynamic_opt.TO_Module_List.size(); + } int myrank = Explicit_Solver_Pointer_->myrank; - if(simparam->output_options.output_file_format==OUTPUT_FORMAT::vtk) + if (simparam->output_options.output_file_format == OUTPUT_FORMAT::vtk) { - if(myrank==0) - printf("Writing outputs to file at %f \n", time_value); + if (myrank == 0) + { + printf("Writing outputs to file at %f \n", time_value); + } - double comm_time1 = Explicit_Solver_Pointer_->CPU_Time(); - //Explicit_Solver_Pointer_->write_outputs_new(); - double comm_time2 = Explicit_Solver_Pointer_->CPU_Time(); - Explicit_Solver_Pointer_->output_time += comm_time2 - comm_time1; + double comm_time1 = Explicit_Solver_Pointer_->CPU_Time(); + // Explicit_Solver_Pointer_->write_outputs_new(); + double comm_time2 = Explicit_Solver_Pointer_->CPU_Time(); + Explicit_Solver_Pointer_->output_time += comm_time2 - comm_time1; } /* write_outputs(mesh, @@ -730,18 +781,17 @@ void FEA_Module_Eulerian::euler_solve(){ graphics_id, time_value); */ - - - CArrayKokkos node_extensive_mass(nall_nodes, "node_extensive_mass"); - + + CArrayKokkos node_extensive_mass(nall_nodes, "node_extensive_mass"); + // extensive energy tallies over the mesh elements local to this MPI rank double IE_t0 = 0.0; double KE_t0 = 0.0; double TE_t0 = 0.0; - + double IE_sum = 0.0; double KE_sum = 0.0; - + double IE_loc_sum = 0.0; double KE_loc_sum = 0.0; @@ -760,138 +810,139 @@ void FEA_Module_Eulerian::euler_solve(){ double global_TE_tend = 0.0; int nlocal_elem_non_overlapping = Explicit_Solver_Pointer_->nlocal_elem_non_overlapping; - + // extensive IE REDUCE_SUM_CLASS(elem_gid, 0, nlocal_elem_non_overlapping, IE_loc_sum, { - IE_loc_sum += elem_mass(elem_gid)*elem_sie(rk_level,elem_gid); - + IE_loc_sum += elem_mass(elem_gid) * elem_sie(rk_level, elem_gid); }, IE_sum); IE_t0 = IE_sum; - MPI_Allreduce(&IE_t0,&global_IE_t0,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD); - + MPI_Allreduce(&IE_t0, &global_IE_t0, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); + // extensive KE REDUCE_SUM_CLASS(node_gid, 0, nlocal_nodes, KE_loc_sum, { - double ke = 0; - for (size_t dim=0; dimnode_velocities_distributed->getLocalView (Tpetra::Access::ReadWrite); - vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - node_velocities_interface(node_gid,idim) = node_vel(rk_level,node_gid,idim); - node_coords_interface(node_gid,idim) = node_coords(rk_level,node_gid,idim); - } + time_data[0] = 0; + // assign current velocity data to multivector + // view scope + { + vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); + vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + node_velocities_interface(node_gid, idim) = node_vel(rk_level, node_gid, idim); + node_coords_interface(node_gid, idim) = node_coords(rk_level, node_gid, idim); + } }); - } //end view scope - Kokkos::fence(); - - //communicate ghosts - double comm_time1 = Explicit_Solver_Pointer_->CPU_Time(); - - //active view scope; triggers host comms from updated data on device - { - const_host_vec_array node_velocities_host = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_host_vec_array node_coords_host = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - } - double comm_time2 = Explicit_Solver_Pointer_->CPU_Time(); - Explicit_Solver_Pointer_->dev2host_time += comm_time2-comm_time1; + } // end view scope + Kokkos::fence(); - //communicate ghost velocities - Explicit_Solver_Pointer_->comm_velocities(); - Explicit_Solver_Pointer_->comm_coordinates(); - - - double comm_time3 = Explicit_Solver_Pointer_->CPU_Time(); + // communicate ghosts + double comm_time1 = Explicit_Solver_Pointer_->CPU_Time(); - //view scope - { - const_vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array ghost_node_velocities_interface = Explicit_Solver_Pointer_->ghost_node_velocities_distributed->getLocalView (Tpetra::Access::ReadOnly); - vec_array all_node_velocities_interface = Explicit_Solver_Pointer_->all_node_velocities_distributed->getLocalView (Tpetra::Access::ReadWrite); - const_vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array ghost_node_coords_interface = Explicit_Solver_Pointer_->ghost_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - all_node_velocities_interface(node_gid,idim) = node_velocities_interface(node_gid,idim); - all_node_coords_interface(node_gid,idim) = node_coords_interface(node_gid,idim); + // active view scope; triggers host comms from updated data on device + { + const_host_vec_array node_velocities_host = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_host_vec_array node_coords_host = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); } + double comm_time2 = Explicit_Solver_Pointer_->CPU_Time(); + Explicit_Solver_Pointer_->dev2host_time += comm_time2 - comm_time1; + + // communicate ghost velocities + Explicit_Solver_Pointer_->comm_velocities(); + Explicit_Solver_Pointer_->comm_coordinates(); + + double comm_time3 = Explicit_Solver_Pointer_->CPU_Time(); + + // view scope + { + const_vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array ghost_node_velocities_interface = Explicit_Solver_Pointer_->ghost_node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); + vec_array all_node_velocities_interface = Explicit_Solver_Pointer_->all_node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); + const_vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array ghost_node_coords_interface = Explicit_Solver_Pointer_->ghost_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + all_node_velocities_interface(node_gid, idim) = node_velocities_interface(node_gid, idim); + all_node_coords_interface(node_gid, idim) = node_coords_interface(node_gid, idim); + } }); // end parallel for - Kokkos::fence(); + Kokkos::fence(); - FOR_ALL_CLASS(node_gid, nlocal_nodes, nlocal_nodes+nghost_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - all_node_velocities_interface(node_gid,idim) = ghost_node_velocities_interface(node_gid-nlocal_nodes,idim); - all_node_coords_interface(node_gid,idim) = ghost_node_coords_interface(node_gid-nlocal_nodes,idim); - } + FOR_ALL_CLASS(node_gid, nlocal_nodes, nlocal_nodes + nghost_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + all_node_velocities_interface(node_gid, idim) = ghost_node_velocities_interface(node_gid - nlocal_nodes, idim); + all_node_coords_interface(node_gid, idim) = ghost_node_coords_interface(node_gid - nlocal_nodes, idim); + } }); // end parallel for - Kokkos::fence(); - } //end view scope - - - (*forward_solve_velocity_data)[0]->assign(*Explicit_Solver_Pointer_->all_node_velocities_distributed); - (*forward_solve_coordinate_data)[0]->assign(*Explicit_Solver_Pointer_->all_node_coords_distributed); - } - - // loop over the max number of time integration cycles - for (cycle = 0; cycle < cycle_stop; cycle++) { - - // get the step - if(num_dim==2){ + Kokkos::fence(); + } // end view scope + + (*forward_solve_velocity_data)[0]->assign(*Explicit_Solver_Pointer_->all_node_velocities_distributed); + (*forward_solve_coordinate_data)[0]->assign(*Explicit_Solver_Pointer_->all_node_coords_distributed); + } + + // loop over the max number of time integration cycles + for (cycle = 0; cycle < cycle_stop; cycle++) + { + // get the step + if (num_dim == 2) + { get_timestep2D(mesh, node_coords, node_vel, elem_sspd, elem_vol); } - else { + else + { get_timestep(mesh, node_coords, node_vel, @@ -900,29 +951,32 @@ void FEA_Module_Eulerian::euler_solve(){ } // end if 2D double global_dt; - MPI_Allreduce(&dt,&global_dt,1,MPI_DOUBLE,MPI_MIN,MPI_COMM_WORLD); + MPI_Allreduce(&dt, &global_dt, 1, MPI_DOUBLE, MPI_MIN, MPI_COMM_WORLD); dt = global_dt; - // stop calculation if flag - //if (stop_calc == 1) break; - - + // stop calculation if flag + // if (stop_calc == 1) break; - if (cycle==0){ - if(myrank==0) - printf("cycle = %lu, time = %12.5e, time step = %12.5e \n", cycle, time_value, dt); + if (cycle == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %12.5e, time step = %12.5e \n", cycle, time_value, dt); + } } // print time step every 10 cycles - else if (cycle%20==0){ - if(myrank==0) - printf("cycle = %lu, time = %12.5e, time step = %12.5e \n", cycle, time_value, dt); + else if (cycle % 20 == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %12.5e, time step = %12.5e \n", cycle, time_value, dt); + } } // end if - - + // --------------------------------------------------------------------- // integrate the solution forward to t(n+1) via Runge Kutta (RK) method // --------------------------------------------------------------------- - + // save the values at t_n rk_init(node_coords, node_vel, @@ -930,107 +984,102 @@ void FEA_Module_Eulerian::euler_solve(){ elem_stress, rnum_elem, nall_nodes); - - - - // integrate solution forward in time - for (size_t rk_stage = 0; rk_stage < rk_num_stages; rk_stage++){ - + // integrate solution forward in time + for (size_t rk_stage = 0; rk_stage < rk_num_stages; rk_stage++) + { // ---- RK coefficient ---- - double rk_alpha = 1.0/((double)rk_num_stages - (double)rk_stage); + double rk_alpha = 1.0 / ((double)rk_num_stages - (double)rk_stage); - //current interface has differing velocity arrays; this equates them until we unify memory - //first comm time interval point + // current interface has differing velocity arrays; this equates them until we unify memory + // first comm time interval point double comm_time1 = Explicit_Solver_Pointer_->CPU_Time(); - //view scope + // view scope { - vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - node_velocities_interface(node_gid,idim) = node_vel(rk_level,node_gid,idim); - } + vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + node_velocities_interface(node_gid, idim) = node_vel(rk_level, node_gid, idim); + } }); // end parallel for - } //end view scope + } // end view scope Kokkos::fence(); - - //active view scope + + // active view scope { - const_host_vec_array node_velocities_host = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView (Tpetra::Access::ReadOnly); + const_host_vec_array node_velocities_host = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); } double comm_time2 = Explicit_Solver_Pointer_->CPU_Time(); - Explicit_Solver_Pointer_->dev2host_time += comm_time2-comm_time1; - //communicate ghost velocities + Explicit_Solver_Pointer_->dev2host_time += comm_time2 - comm_time1; + // communicate ghost velocities Explicit_Solver_Pointer_->comm_velocities(); - + double comm_time3 = Explicit_Solver_Pointer_->CPU_Time(); - //this is forcing a copy to the device - //view scope + // this is forcing a copy to the device + // view scope { - vec_array ghost_node_velocities_interface = Explicit_Solver_Pointer_->ghost_node_velocities_distributed->getLocalView (Tpetra::Access::ReadWrite); + vec_array ghost_node_velocities_interface = Explicit_Solver_Pointer_->ghost_node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, nlocal_nodes, nall_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - node_vel(rk_level,node_gid,idim) = ghost_node_velocities_interface(node_gid-nlocal_nodes,idim); - } - + FOR_ALL_CLASS(node_gid, nlocal_nodes, nall_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + node_vel(rk_level, node_gid, idim) = ghost_node_velocities_interface(node_gid - nlocal_nodes, idim); + } }); // end parallel for - } //end view scope + } // end view scope Kokkos::fence(); - + double comm_time4 = Explicit_Solver_Pointer_->CPU_Time(); - Explicit_Solver_Pointer_->host2dev_time += comm_time4-comm_time3; - Explicit_Solver_Pointer_->communication_time += comm_time4-comm_time1; - //debug print vector values on a rank + Explicit_Solver_Pointer_->host2dev_time += comm_time4 - comm_time3; + Explicit_Solver_Pointer_->communication_time += comm_time4 - comm_time1; + // debug print vector values on a rank /* if(myrank==0) for(int i = 0; i < nall_nodes; i++){ std::cout << Explicit_Solver_Pointer_->all_node_map->getGlobalElement(i) << " " << node_vel(rk_level,i,0) << " " << node_vel(rk_level,i,1) << " " << node_vel(rk_level,i,2) << std::endl; } - */ + */ - // calculate the new corner masses if 2D - if(num_dim==2){ - + if (num_dim == 2) + { // calculate the nodal areal mass FOR_ALL_CLASS(node_gid, 0, nall_nodes, { - node_mass(node_gid) = 0.0; - - if (node_coords(rk_level,node_gid,1) > tiny){ - node_mass(node_gid) = node_extensive_mass(node_gid)/node_coords(rk_level,node_gid,1); - } - //if(cycle==0&&node_gid==1&&myrank==0) - //std::cout << "index " << node_gid << " on rank " << myrank << " node vel " << node_vel(rk_level,node_gid,0) << " " << node_mass(node_gid) << std::endl << std::flush; + if (node_coords(rk_level, node_gid, 1) > tiny) + { + node_mass(node_gid) = node_extensive_mass(node_gid) / node_coords(rk_level, node_gid, 1); + } + // if(cycle==0&&node_gid==1&&myrank==0) + // std::cout << "index " << node_gid << " on rank " << myrank << " node vel " << node_vel(rk_level,node_gid,0) << " " << node_mass(node_gid) << std::endl << std::flush; }); // end parallel for over node_gid Kokkos::fence(); - //current interface has differing density arrays; this equates them until we unify memory - //view scope + // current interface has differing density arrays; this equates them until we unify memory + // view scope { - vec_array node_mass_interface = node_masses_distributed->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - node_mass_interface(node_gid,0) = node_mass(node_gid); + vec_array node_mass_interface = node_masses_distributed->getLocalView(Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + node_mass_interface(node_gid, 0) = node_mass(node_gid); }); // end parallel for - } //end view scope + } // end view scope Kokkos::fence(); - //communicate ghost densities + // communicate ghost densities comm_node_masses(); - //this is forcing a copy to the device - //view scope + // this is forcing a copy to the device + // view scope { - vec_array ghost_node_mass_interface = ghost_node_masses_distributed->getLocalView (Tpetra::Access::ReadWrite); + vec_array ghost_node_mass_interface = ghost_node_masses_distributed->getLocalView(Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, nlocal_nodes, nall_nodes, { - node_mass(node_gid) = ghost_node_mass_interface(node_gid-nlocal_nodes,0); + FOR_ALL_CLASS(node_gid, nlocal_nodes, nall_nodes, { + node_mass(node_gid) = ghost_node_mass_interface(node_gid - nlocal_nodes, 0); }); // end parallel for - } //end view scope + } // end view scope Kokkos::fence(); - - + // ----------------------------------------------- // Calcualte the areal mass for nodes on the axis // ----------------------------------------------- @@ -1043,339 +1092,355 @@ void FEA_Module_Eulerian::euler_solve(){ // 0---1 /* FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { - + // loop over the corners of the element and calculate the mass for (size_t node_lid=0; node_lid<4; node_lid++){ - + size_t node_gid = nodes_in_elem(elem_gid, node_lid); size_t node_minus_gid; size_t node_plus_gid; - - + + if (node_coords(rk_level,node_gid,1) < tiny){ // node is on the axis - + // minus node if (node_lid==0){ node_minus_gid = nodes_in_elem(elem_gid, 3); } else { node_minus_gid = nodes_in_elem(elem_gid, node_lid-1); } - + // plus node if (node_lid==3){ node_plus_gid = nodes_in_elem(elem_gid, 0); } else { node_plus_gid = nodes_in_elem(elem_gid, node_lid+1); } - + node_mass(node_gid) = fmax(node_mass(node_plus_gid), node_mass(node_minus_gid))/2.0; - + } // end if - + } // end for over corners - + }); // end parallel for over elem_gid Kokkos::fence(); */ - + FOR_ALL_CLASS(node_bdy_gid, 0, num_bdy_nodes, { - //FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + // FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { size_t node_gid = bdy_nodes(node_bdy_gid); - - if (node_coords(rk_level,node_gid,1) < tiny){ + + if (node_coords(rk_level, node_gid, 1) < tiny) + { // node is on the axis - - for(size_t node_lid=0; node_lid < num_nodes_in_node(node_gid); node_lid++){ - + + for (size_t node_lid = 0; node_lid < num_nodes_in_node(node_gid); node_lid++) + { size_t node_neighbor_gid = nodes_in_node(node_gid, node_lid); - + // if the node is off the axis, use it's areal mass on the boundary - if (node_coords(rk_level,node_neighbor_gid,1) > tiny){ - node_mass(node_gid) = fmax(node_mass(node_gid), node_mass(node_neighbor_gid)/2.0); + if (node_coords(rk_level, node_neighbor_gid, 1) > tiny) + { + node_mass(node_gid) = fmax(node_mass(node_gid), node_mass(node_neighbor_gid) / 2.0); } - } // end for over neighboring nodes - } // end if - }); // end parallel for over elem_gid - } // end of if 2D-RZ + } // end of RK loop + // increment the time + time_value += dt; + simparam->time_value = time_value; - } // end of RK loop + if (simparam_dynamic_opt.topology_optimization_on || simparam_dynamic_opt.shape_optimization_on) + { + if (cycle >= max_time_steps) + { + max_time_steps = cycle + 1; + } - // increment the time - time_value+=dt; - simparam->time_value = time_value; + if (max_time_steps + 1 > forward_solve_velocity_data->size()) + { + old_max_forward_buffer = forward_solve_velocity_data->size(); + time_data.resize(max_time_steps + BUFFER_GROW + 1); + forward_solve_velocity_data->resize(max_time_steps + BUFFER_GROW + 1); + forward_solve_coordinate_data->resize(max_time_steps + BUFFER_GROW + 1); + adjoint_vector_data->resize(max_time_steps + BUFFER_GROW + 1); + phi_adjoint_vector_data->resize(max_time_steps + BUFFER_GROW + 1); + // assign a multivector of corresponding size to each new timestep in the buffer + for (int istep = old_max_forward_buffer; istep < max_time_steps + BUFFER_GROW + 1; istep++) + { + (*forward_solve_velocity_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*forward_solve_coordinate_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*phi_adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + } + } - if(simparam_dynamic_opt.topology_optimization_on||simparam_dynamic_opt.shape_optimization_on){ - if(cycle >= max_time_steps) - max_time_steps = cycle + 1; + time_data[cycle + 1] = dt + time_data[cycle]; - if(max_time_steps + 1 > forward_solve_velocity_data->size()){ - old_max_forward_buffer = forward_solve_velocity_data->size(); - time_data.resize(max_time_steps + BUFFER_GROW +1); - forward_solve_velocity_data->resize(max_time_steps + BUFFER_GROW +1); - forward_solve_coordinate_data->resize(max_time_steps + BUFFER_GROW +1); - adjoint_vector_data->resize(max_time_steps + BUFFER_GROW +1); - phi_adjoint_vector_data->resize(max_time_steps + BUFFER_GROW +1); - //assign a multivector of corresponding size to each new timestep in the buffer - for(int istep = old_max_forward_buffer; istep < max_time_steps + BUFFER_GROW +1; istep++){ - (*forward_solve_velocity_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - (*forward_solve_coordinate_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - (*adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - (*phi_adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - } - } + // assign current velocity data to multivector + // view scope + { + vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); + vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + node_velocities_interface(node_gid, idim) = node_vel(rk_level, node_gid, idim); + node_coords_interface(node_gid, idim) = node_coords(rk_level, node_gid, idim); + } + }); + } // end view scope + Kokkos::fence(); - - time_data[cycle+1] = dt + time_data[cycle]; - - - //assign current velocity data to multivector - //view scope - { - vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView (Tpetra::Access::ReadWrite); - vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - node_velocities_interface(node_gid,idim) = node_vel(rk_level,node_gid,idim); - node_coords_interface(node_gid,idim) = node_coords(rk_level,node_gid,idim); + // communicate ghosts + double comm_time1 = Explicit_Solver_Pointer_->CPU_Time(); + + // active view scope; triggers host comms from updated data on device + { + const_host_vec_array node_velocities_host = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_host_vec_array node_coords_host = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); } - }); - } //end view scope - Kokkos::fence(); + double comm_time2 = Explicit_Solver_Pointer_->CPU_Time(); + Explicit_Solver_Pointer_->dev2host_time += comm_time2 - comm_time1; - //communicate ghosts - double comm_time1 = Explicit_Solver_Pointer_->CPU_Time(); - - //active view scope; triggers host comms from updated data on device - { - const_host_vec_array node_velocities_host = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_host_vec_array node_coords_host = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - } - double comm_time2 = Explicit_Solver_Pointer_->CPU_Time(); - Explicit_Solver_Pointer_->dev2host_time += comm_time2-comm_time1; + // communicate ghost velocities + Explicit_Solver_Pointer_->comm_velocities(); + Explicit_Solver_Pointer_->comm_coordinates(); - //communicate ghost velocities - Explicit_Solver_Pointer_->comm_velocities(); - Explicit_Solver_Pointer_->comm_coordinates(); - - - double comm_time3 = Explicit_Solver_Pointer_->CPU_Time(); + double comm_time3 = Explicit_Solver_Pointer_->CPU_Time(); - //view scope - { - const_vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array ghost_node_velocities_interface = Explicit_Solver_Pointer_->ghost_node_velocities_distributed->getLocalView (Tpetra::Access::ReadOnly); - vec_array all_node_velocities_interface = Explicit_Solver_Pointer_->all_node_velocities_distributed->getLocalView (Tpetra::Access::ReadWrite); - const_vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array ghost_node_coords_interface = Explicit_Solver_Pointer_->ghost_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - all_node_velocities_interface(node_gid,idim) = node_velocities_interface(node_gid,idim); - all_node_coords_interface(node_gid,idim) = node_coords_interface(node_gid,idim); - } + // view scope + { + const_vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array ghost_node_velocities_interface = Explicit_Solver_Pointer_->ghost_node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); + vec_array all_node_velocities_interface = Explicit_Solver_Pointer_->all_node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); + const_vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array ghost_node_coords_interface = Explicit_Solver_Pointer_->ghost_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + all_node_velocities_interface(node_gid, idim) = node_velocities_interface(node_gid, idim); + all_node_coords_interface(node_gid, idim) = node_coords_interface(node_gid, idim); + } }); // end parallel for - Kokkos::fence(); + Kokkos::fence(); - FOR_ALL_CLASS(node_gid, nlocal_nodes, nlocal_nodes+nghost_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - all_node_velocities_interface(node_gid,idim) = ghost_node_velocities_interface(node_gid-nlocal_nodes,idim); - all_node_coords_interface(node_gid,idim) = ghost_node_coords_interface(node_gid-nlocal_nodes,idim); - } + FOR_ALL_CLASS(node_gid, nlocal_nodes, nlocal_nodes + nghost_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + all_node_velocities_interface(node_gid, idim) = ghost_node_velocities_interface(node_gid - nlocal_nodes, idim); + all_node_coords_interface(node_gid, idim) = ghost_node_coords_interface(node_gid - nlocal_nodes, idim); + } }); // end parallel for - Kokkos::fence(); - } //end view scope - - double comm_time4 = Explicit_Solver_Pointer_->CPU_Time(); - Explicit_Solver_Pointer_->host2dev_time += comm_time4-comm_time3; - Explicit_Solver_Pointer_->communication_time += comm_time4-comm_time1; - - (*forward_solve_velocity_data)[cycle+1]->assign(*Explicit_Solver_Pointer_->all_node_velocities_distributed); - (*forward_solve_coordinate_data)[cycle+1]->assign(*Explicit_Solver_Pointer_->all_node_coords_distributed); - - //kinetic energy accumulation - if(kinetic_energy_objective){ - const_vec_array node_velocities_interface = (*forward_solve_velocity_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array previous_node_velocities_interface = (*forward_solve_velocity_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - KE_loc_sum = 0.0; - KE_sum = 0.0; - // extensive KE - REDUCE_SUM_CLASS(node_gid, 0, nlocal_nodes, KE_loc_sum, { - - double ke = 0; - for (size_t dim=0; dimCPU_Time(); + Explicit_Solver_Pointer_->host2dev_time += comm_time4 - comm_time3; + Explicit_Solver_Pointer_->communication_time += comm_time4 - comm_time1; + + (*forward_solve_velocity_data)[cycle + 1]->assign(*Explicit_Solver_Pointer_->all_node_velocities_distributed); + (*forward_solve_coordinate_data)[cycle + 1]->assign(*Explicit_Solver_Pointer_->all_node_coords_distributed); + + // kinetic energy accumulation + if (kinetic_energy_objective) + { + const_vec_array node_velocities_interface = (*forward_solve_velocity_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array previous_node_velocities_interface = (*forward_solve_velocity_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + KE_loc_sum = 0.0; + KE_sum = 0.0; + // extensive KE + REDUCE_SUM_CLASS(node_gid, 0, nlocal_nodes, KE_loc_sum, { + double ke = 0; + for (size_t dim = 0; dim < num_dim; dim++) + { + // midpoint integration approximation + ke += (node_velocities_interface(node_gid, dim) + node_velocities_interface(node_gid, dim)) * (node_velocities_interface(node_gid, dim) + node_velocities_interface(node_gid, + dim)) / 4; // 1/2 at end + } // end for + + if (num_dim == 2) + { + KE_loc_sum += node_mass(node_gid) * node_coords(rk_level, node_gid, 1) * ke; + } + else + { + KE_loc_sum += node_mass(node_gid) * ke; + } }, KE_sum); - Kokkos::fence(); - KE_sum = 0.5*KE_sum; - objective_accumulation += KE_sum*dt; + Kokkos::fence(); + KE_sum = 0.5 * KE_sum; + objective_accumulation += KE_sum * dt; + } + } + + size_t write = 0; + if ((cycle + 1) % graphics_cyc_ival == 0 && cycle > 0) + { + write = 1; + } + else if (cycle == cycle_stop) + { + write = 1; + } + else if (time_value >= time_final) + { + write = 1; + } + else if (time_value >= graphics_time) + { + write = 1; } - } - - size_t write = 0; - if ((cycle+1)%graphics_cyc_ival == 0 && cycle>0){ - write = 1; - } - else if (cycle == cycle_stop) { - write = 1; - } - else if (time_value >= time_final){ - write = 1; - } - else if (time_value >= graphics_time){ - write = 1; - } - + // write outputs - if (write == 1){ - //interface nodal coordinate data (note: this is not needed if using write_outputs_new()) - //view scope + if (write == 1) + { + // interface nodal coordinate data (note: this is not needed if using write_outputs_new()) + // view scope { - vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - node_coords_interface(node_gid,idim) = node_coords(rk_level,node_gid,idim); - } + vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + node_coords_interface(node_gid, idim) = node_coords(rk_level, node_gid, idim); + } }); // end parallel for - } //end view scope - if(simparam->output_options.output_file_format==OUTPUT_FORMAT::vtk){ - if(myrank==0){ - printf("Writing outputs to file at %f \n", graphics_time); - } + } // end view scope + if (simparam->output_options.output_file_format == OUTPUT_FORMAT::vtk) + { + if (myrank == 0) + { + printf("Writing outputs to file at %f \n", graphics_time); + } - double comm_time1 = Explicit_Solver_Pointer_->CPU_Time(); - //Explicit_Solver_Pointer_->write_outputs_new(); + double comm_time1 = Explicit_Solver_Pointer_->CPU_Time(); + // Explicit_Solver_Pointer_->write_outputs_new(); - double comm_time2 = Explicit_Solver_Pointer_->CPU_Time(); - Explicit_Solver_Pointer_->output_time += comm_time2 - comm_time1; + double comm_time2 = Explicit_Solver_Pointer_->CPU_Time(); + Explicit_Solver_Pointer_->output_time += comm_time2 - comm_time1; } - //Explicit_Solver_Pointer_->parallel_vtk_writer(); - //Explicit_Solver_Pointer_->parallel_vtk_writer_new(); - //Explicit_Solver_Pointer_->parallel_tecplot_writer(); - /* - write_outputs(mesh, - Explicit_Solver_Pointer_, - node_coords, - node_vel, - node_mass, - elem_den, - elem_pres, - elem_stress, - elem_sspd, - elem_sie, - elem_vol, - elem_mass, - elem_mat_id, - graphics_times, - graphics_id, - time_value); - */ + // Explicit_Solver_Pointer_->parallel_vtk_writer(); + // Explicit_Solver_Pointer_->parallel_vtk_writer_new(); + // Explicit_Solver_Pointer_->parallel_tecplot_writer(); + /* + write_outputs(mesh, + Explicit_Solver_Pointer_, + node_coords, + node_vel, + node_mass, + elem_den, + elem_pres, + elem_stress, + elem_sspd, + elem_sie, + elem_vol, + elem_mass, + elem_mat_id, + graphics_times, + graphics_id, + time_value); + */ graphics_time = time_value + graphics_dt_ival; - } // end if - - - // end of calculation - if (time_value>=time_final) break; + } // end if - + // end of calculation + if (time_value >= time_final) + { + break; + } } // end for cycle loop last_time_step = cycle; - //simple setup to just calculate KE minimize objective for now - if(simparam_dynamic_opt.topology_optimization_on){ - KineticEnergyMinimize_TopOpt& kinetic_energy_minimize_function = dynamic_cast(*obj_pointer); + // simple setup to just calculate KE minimize objective for now + if (simparam_dynamic_opt.topology_optimization_on) + { + KineticEnergyMinimize_TopOpt& kinetic_energy_minimize_function = dynamic_cast(*obj_pointer); - //collect local objective values - MPI_Allreduce(&objective_accumulation,&global_objective_accumulation,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD); - kinetic_energy_minimize_function.objective_accumulation = global_objective_accumulation; + // collect local objective values + MPI_Allreduce(&objective_accumulation, &global_objective_accumulation, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); + kinetic_energy_minimize_function.objective_accumulation = global_objective_accumulation; - if(myrank==0) - std::cout << "CURRENT TIME INTEGRAL OF KINETIC ENERGY " << global_objective_accumulation << std::endl; + if (myrank == 0) + { + std::cout << "CURRENT TIME INTEGRAL OF KINETIC ENERGY " << global_objective_accumulation << std::endl; + } } - - + auto time_2 = std::chrono::high_resolution_clock::now(); auto time_difference = time_2 - time_1; - //double calc_time = std::chrono::duration_cast(diff).count(); + // double calc_time = std::chrono::duration_cast(diff).count(); double calc_time = std::chrono::duration_cast(time_difference).count(); - if(myrank==0) - printf("\nCalculation time in seconds: %f \n", calc_time*1e-09); - + if (myrank == 0) + { + printf("\nCalculation time in seconds: %f \n", calc_time * 1e-09); + } + IE_loc_sum = 0.0; KE_loc_sum = 0.0; - IE_sum = 0.0; - KE_sum = 0.0; - + IE_sum = 0.0; + KE_sum = 0.0; + // extensive IE REDUCE_SUM_CLASS(elem_gid, 0, nlocal_elem_non_overlapping, IE_loc_sum, { - - IE_loc_sum += elem_mass(elem_gid)*elem_sie(rk_level,elem_gid); - + IE_loc_sum += elem_mass(elem_gid) * elem_sie(rk_level, elem_gid); }, IE_sum); IE_tend = IE_sum; - //reduce over MPI ranks - MPI_Allreduce(&IE_tend,&global_IE_tend,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD); + // reduce over MPI ranks + MPI_Allreduce(&IE_tend, &global_IE_tend, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); // extensive KE REDUCE_SUM_CLASS(node_gid, 0, nlocal_nodes, KE_loc_sum, { - double ke = 0; - for (size_t dim=0; dim zp); - - void read_conditions_ansys_dat(std::ifstream *in, std::streampos before_condition_header); - - //interfaces between user input and creating data structures for bcs - void generate_bcs(); - - void Displacement_Boundary_Conditions(); - - void init_output(); - - void compute_output(); - - void sort_output(Teuchos::RCP > sorted_map); - - void collect_output(Teuchos::RCP > global_reduce_map); - - void init_boundaries(); - - //initializes memory for arrays used in the global stiffness matrix assembly - void init_boundary_sets(int num_boundary_sets); - - void grow_boundary_sets(int num_boundary_sets); - - void rk_init(DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &elem_sie, - DViewCArrayKokkos &elem_stress, - const size_t num_elems, - const size_t num_nodes); - - void get_timestep(mesh_t &mesh, - DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &elem_sspd, - DViewCArrayKokkos &elem_vol); - - - void get_timestep2D(mesh_t &mesh, - DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &elem_sspd, - DViewCArrayKokkos &elem_vol); - - void write_outputs (const mesh_t &mesh, - DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &node_mass, - DViewCArrayKokkos &elem_den, - DViewCArrayKokkos &elem_pres, - DViewCArrayKokkos &elem_stress, - DViewCArrayKokkos &elem_sspd, - DViewCArrayKokkos &elem_sie, - DViewCArrayKokkos &elem_vol, - DViewCArrayKokkos &elem_mass, - DViewCArrayKokkos &elem_mat_id); - - void node_density_constraints(host_vec_array node_densities_lower_bound); - - void example_function(double rk_alpha, - const size_t num_nodes, - DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel); - - KOKKOS_FUNCTION - void example_device_function(const ViewCArrayKokkos &B_matrix, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const; - - - bool nodal_density_flag; - real_t penalty_power; - Teuchos::RCP Global_Stiffness_Matrix; - RaggedRightArrayKokkos Stiffness_Matrix; - DCArrayKokkos Stiffness_Matrix_Strides; - DCArrayKokkos Global_Stiffness_Matrix_Assembly_Map; - //end elastic TO data - - Simulation_Parameters_Eulerian *simparam; - Simulation_Parameters_Elasticity simparam_elasticity; - Simulation_Parameters_Dynamic_Optimization simparam_dynamic_opt; - Explicit_Solver_Eulerian *Explicit_Solver_Pointer_; - - elements::ref_element *ref_elem; - - mesh_t& mesh; - //shallow copies of mesh class views - size_t num_nodes_in_elem; - // corner ids in node - RaggedRightArrayKokkos corners_in_node; - CArrayKokkos num_corners_in_node; - - // elem ids in node - RaggedRightArrayKokkos elems_in_node; - - // node ids in node - RaggedRightArrayKokkos nodes_in_node; - CArrayKokkos num_nodes_in_node; - - // node ids in elem - DCArrayKokkos nodes_in_elem; - - // corner ids in elem - CArrayKokkos corners_in_elem; - - // elem ids in elem - RaggedRightArrayKokkos elems_in_elem; - CArrayKokkos num_elems_in_elem; - - // patch ids in elem - CArrayKokkos patches_in_elem; - - // node ids in a patch - CArrayKokkos nodes_in_patch; - - // element ids in a patch - CArrayKokkos elems_in_patch; - - // patch ids in bdy set - size_t num_bdy_sets; - DynamicRaggedRightArrayKokkos bdy_patches_in_set; - - // bdy nodes - CArrayKokkos bdy_nodes; - - // node ids in bdy_patch set - RaggedRightArrayKokkos bdy_nodes_in_set; - DCArrayKokkos num_bdy_nodes_in_set; - - //Topology optimization filter variable - DCArrayKokkos relative_element_densities; - - //Local FEA data - host_elem_conn_array interface_nodes_in_elem; //host view of element connectivity to nodes - - //Global FEA data - Teuchos::RCP node_velocities_distributed; - Teuchos::RCP initial_node_coords_distributed; - Teuchos::RCP all_initial_node_coords_distributed; - Teuchos::RCP initial_node_velocities_distributed; - Teuchos::RCP all_node_velocities_distributed; - Teuchos::RCP all_cached_node_velocities_distributed; - Teuchos::RCP node_masses_distributed; - Teuchos::RCP ghost_node_masses_distributed; - Teuchos::RCP adjoint_vector_distributed; - Teuchos::RCP phi_adjoint_vector_distributed; - Teuchos::RCP>> forward_solve_velocity_data; - Teuchos::RCP>> forward_solve_coordinate_data; - Teuchos::RCP>> adjoint_vector_data; - Teuchos::RCP>> phi_adjoint_vector_data; - Teuchos::RCP force_gradient_design; - Teuchos::RCP force_gradient_position; - Teuchos::RCP force_gradient_velocity; - - //Local FEA data - DCArrayKokkos Global_Gradient_Matrix_Assembly_Map; - RaggedRightArrayKokkos Graph_Matrix; //stores global indices - RaggedRightArrayKokkos DOF_Graph_Matrix; //stores global indices - RaggedRightArrayKokkos Force_Gradient_Positions; - RaggedRightArrayKokkos Force_Gradient_Velocities; - DCArrayKokkos Gradient_Matrix_Strides; - DCArrayKokkos Graph_Matrix_Strides; - RaggedRightArrayKokkos Original_Gradient_Entries; - RaggedRightArrayKokkos Original_Gradient_Entry_Indices; - DCArrayKokkos Original_Gradient_Entries_Strides; - - //distributed matrices - Teuchos::RCP distributed_force_gradient_positions; - Teuchos::RCP distributed_force_gradient_velocities; - - std::vector time_data; - int max_time_steps, last_time_step; - - //Dual View wrappers - // Dual Views of the individual node struct variables - DViewCArrayKokkos node_coords; - DViewCArrayKokkos node_vel; - DViewCArrayKokkos node_mass; - - // Dual Views of the individual elem struct variables - DViewCArrayKokkos elem_den; - DViewCArrayKokkos elem_pres; - DViewCArrayKokkos elem_stress; // always 3D even in 2D-RZ - DViewCArrayKokkos elem_sspd; - DViewCArrayKokkos elem_sie; - DViewCArrayKokkos elem_vol; - DViewCArrayKokkos elem_div; - DViewCArrayKokkos elem_mass; - DViewCArrayKokkos elem_mat_id; - - // Element velocity gradient - DCArrayKokkos elem_vel_grad; - - // for storing global variables used in user material model - DCArrayKokkos global_vars; - - // for storing global variables used in user material model - DCArrayKokkos state_vars; - - // Dual Views of the corner struct variables - DViewCArrayKokkos corner_force; - DViewCArrayKokkos corner_mass; - - //Boundary Conditions Data - DCArrayKokkos Local_Index_Boundary_Patches; - //CArray Patch_Nodes; - enum bc_type {NONE, POINT_LOADING_CONDITION, LINE_LOADING_CONDITION, SURFACE_LOADING_CONDITION}; - - //Boundary Conditions Data - int max_boundary_sets; - - //output dof data - //Global arrays with collected data used to print - int output_velocity_index, output_strain_index, output_stress_index; - - //file parameters - DCArrayKokkos read_from_file; - - //parameters - double time_value, time_final, dt, dt_max, dt_min, dt_cfl, graphics_time, graphics_dt_ival; - size_t graphics_cyc_ival, cycle_stop, rk_num_stages, graphics_id; - double fuzz, tiny, small; - CArray graphics_times; - - //optimization flags - bool kinetic_energy_objective; + + FEA_Module_Eulerian(Solver* Solver_Pointer, mesh_t& mesh, const int my_fea_module_index = 0); + ~FEA_Module_Eulerian(); + + void setup(); + + void euler_solve(); + + void comm_node_masses(); + + void comm_adjoint_vectors(int cycle); + + void comm_variables(Teuchos::RCP zp); + + void read_conditions_ansys_dat(std::ifstream* in, std::streampos before_condition_header); + + // interfaces between user input and creating data structures for bcs + void generate_bcs(); + + void Displacement_Boundary_Conditions(); + + void init_output(); + + void compute_output(); + + void sort_output(Teuchos::RCP> sorted_map); + + void collect_output(Teuchos::RCP> global_reduce_map); + + void init_boundaries(); + + // initializes memory for arrays used in the global stiffness matrix assembly + void init_boundary_sets(int num_boundary_sets); + + void grow_boundary_sets(int num_boundary_sets); + + void rk_init(DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_sie, + DViewCArrayKokkos& elem_stress, + const size_t num_elems, + const size_t num_nodes); + + void get_timestep(mesh_t& mesh, + DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_sspd, + DViewCArrayKokkos& elem_vol); + + void get_timestep2D(mesh_t& mesh, + DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_sspd, + DViewCArrayKokkos& elem_vol); + + void write_outputs(const mesh_t& mesh, + DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& node_mass, + DViewCArrayKokkos& elem_den, + DViewCArrayKokkos& elem_pres, + DViewCArrayKokkos& elem_stress, + DViewCArrayKokkos& elem_sspd, + DViewCArrayKokkos& elem_sie, + DViewCArrayKokkos& elem_vol, + DViewCArrayKokkos& elem_mass, + DViewCArrayKokkos& elem_mat_id); + + void node_density_constraints(host_vec_array node_densities_lower_bound); + + void example_function(double rk_alpha, + const size_t num_nodes, + DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel); + + KOKKOS_FUNCTION + void example_device_function(const ViewCArrayKokkos& B_matrix, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const; + + bool nodal_density_flag; + real_t penalty_power; + Teuchos::RCP Global_Stiffness_Matrix; + RaggedRightArrayKokkos Stiffness_Matrix; + DCArrayKokkos Stiffness_Matrix_Strides; + DCArrayKokkos Global_Stiffness_Matrix_Assembly_Map; + // end elastic TO data + + Simulation_Parameters_Eulerian* simparam; + Simulation_Parameters_Elasticity simparam_elasticity; + Simulation_Parameters_Dynamic_Optimization simparam_dynamic_opt; + Explicit_Solver_Eulerian* Explicit_Solver_Pointer_; + + elements::ref_element* ref_elem; + + mesh_t& mesh; + // shallow copies of mesh class views + size_t num_nodes_in_elem; + // corner ids in node + RaggedRightArrayKokkos corners_in_node; + CArrayKokkos num_corners_in_node; + + // elem ids in node + RaggedRightArrayKokkos elems_in_node; + + // node ids in node + RaggedRightArrayKokkos nodes_in_node; + CArrayKokkos num_nodes_in_node; + + // node ids in elem + DCArrayKokkos nodes_in_elem; + + // corner ids in elem + CArrayKokkos corners_in_elem; + + // elem ids in elem + RaggedRightArrayKokkos elems_in_elem; + CArrayKokkos num_elems_in_elem; + + // patch ids in elem + CArrayKokkos patches_in_elem; + + // node ids in a patch + CArrayKokkos nodes_in_patch; + + // element ids in a patch + CArrayKokkos elems_in_patch; + + // patch ids in bdy set + size_t num_bdy_sets; + DynamicRaggedRightArrayKokkos bdy_patches_in_set; + + // bdy nodes + CArrayKokkos bdy_nodes; + + // node ids in bdy_patch set + RaggedRightArrayKokkos bdy_nodes_in_set; + DCArrayKokkos num_bdy_nodes_in_set; + + // Topology optimization filter variable + DCArrayKokkos relative_element_densities; + + // Local FEA data + host_elem_conn_array interface_nodes_in_elem; // host view of element connectivity to nodes + + // Global FEA data + Teuchos::RCP node_velocities_distributed; + Teuchos::RCP initial_node_coords_distributed; + Teuchos::RCP all_initial_node_coords_distributed; + Teuchos::RCP initial_node_velocities_distributed; + Teuchos::RCP all_node_velocities_distributed; + Teuchos::RCP all_cached_node_velocities_distributed; + Teuchos::RCP node_masses_distributed; + Teuchos::RCP ghost_node_masses_distributed; + Teuchos::RCP adjoint_vector_distributed; + Teuchos::RCP phi_adjoint_vector_distributed; + Teuchos::RCP>> forward_solve_velocity_data; + Teuchos::RCP>> forward_solve_coordinate_data; + Teuchos::RCP>> adjoint_vector_data; + Teuchos::RCP>> phi_adjoint_vector_data; + Teuchos::RCP force_gradient_design; + Teuchos::RCP force_gradient_position; + Teuchos::RCP force_gradient_velocity; + + // Local FEA data + DCArrayKokkos Global_Gradient_Matrix_Assembly_Map; + RaggedRightArrayKokkos Graph_Matrix; // stores global indices + RaggedRightArrayKokkos DOF_Graph_Matrix; // stores global indices + RaggedRightArrayKokkos Force_Gradient_Positions; + RaggedRightArrayKokkos Force_Gradient_Velocities; + DCArrayKokkos Gradient_Matrix_Strides; + DCArrayKokkos Graph_Matrix_Strides; + RaggedRightArrayKokkos Original_Gradient_Entries; + RaggedRightArrayKokkos Original_Gradient_Entry_Indices; + DCArrayKokkos Original_Gradient_Entries_Strides; + + // distributed matrices + Teuchos::RCP distributed_force_gradient_positions; + Teuchos::RCP distributed_force_gradient_velocities; + + std::vector time_data; + int max_time_steps, last_time_step; + + // Dual View wrappers + // Dual Views of the individual node struct variables + DViewCArrayKokkos node_coords; + DViewCArrayKokkos node_vel; + DViewCArrayKokkos node_mass; + + // Dual Views of the individual elem struct variables + DViewCArrayKokkos elem_den; + DViewCArrayKokkos elem_pres; + DViewCArrayKokkos elem_stress; // always 3D even in 2D-RZ + DViewCArrayKokkos elem_sspd; + DViewCArrayKokkos elem_sie; + DViewCArrayKokkos elem_vol; + DViewCArrayKokkos elem_div; + DViewCArrayKokkos elem_mass; + DViewCArrayKokkos elem_mat_id; + + // Element velocity gradient + DCArrayKokkos elem_vel_grad; + + // for storing global variables used in user material model + DCArrayKokkos global_vars; + + // for storing global variables used in user material model + DCArrayKokkos state_vars; + + // Dual Views of the corner struct variables + DViewCArrayKokkos corner_force; + DViewCArrayKokkos corner_mass; + + // Boundary Conditions Data + DCArrayKokkos Local_Index_Boundary_Patches; + // CArray Patch_Nodes; + enum bc_type { NONE, POINT_LOADING_CONDITION, LINE_LOADING_CONDITION, SURFACE_LOADING_CONDITION }; + + // Boundary Conditions Data + int max_boundary_sets; + + // output dof data + // Global arrays with collected data used to print + int output_velocity_index, output_strain_index, output_stress_index; + + // file parameters + DCArrayKokkos read_from_file; + + // parameters + double time_value, time_final, dt, dt_max, dt_min, dt_cfl, graphics_time, graphics_dt_ival; + size_t graphics_cyc_ival, cycle_stop, rk_num_stages, graphics_id; + double fuzz, tiny, small; + CArray graphics_times; + + // optimization flags + bool kinetic_energy_objective; }; #endif // end HEADER_H diff --git a/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/Simulation_Parameters_Eulerian.h b/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/Simulation_Parameters_Eulerian.h index ace620341..87017b8fa 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/Simulation_Parameters_Eulerian.h +++ b/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/Simulation_Parameters_Eulerian.h @@ -11,14 +11,14 @@ This program is open source under the BSD-3 License. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - + 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - + 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - + 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. @@ -52,80 +52,94 @@ using namespace mtr; -struct Simulation_Parameters_Eulerian : Simulation_Parameters { - Time_Variables time_variables; - std::vector region_options; - std::vector material_options; - std::vector boundary_conditions; - Graphics_Options graphics_options; - - bool gravity_flag = false; - bool report_runtime = true; - - size_t rk_num_stages = 2; - int NB = 6; // number of boundaries - int NBSF = 4; //number of surface density force conditions - int NBV = 2; //number of surface sets used to specify a fixed displacement on nodes belonging to respective surfaces - - //Non-serialized fields - int num_gauss_points = 2; - size_t max_num_global_vars; - size_t rk_num_bins; - double time_value = 0.0; - DCArrayKokkos global_vars; - - DCArrayKokkos mat_fill; - DCArrayKokkos material; - DCArrayKokkos boundary; - std::vector gravity_vector {9.81, 0., 0.}; - - void init_material_variable_arrays(size_t nglobal_vars) { - global_vars = DCArrayKokkos (material_options.size(), nglobal_vars); - - for (size_t i = 0; i < material_options.size(); i++) { - auto mat = material_options[i]; - - for (size_t j = 0; j < mat.global_vars.size(); j++) - global_vars.host(i, j) = mat.global_vars[j]; +struct Simulation_Parameters_Eulerian : Simulation_Parameters +{ + Time_Variables time_variables; + std::vector region_options; + std::vector material_options; + std::vector boundary_conditions; + Graphics_Options graphics_options; + + bool gravity_flag = false; + bool report_runtime = true; + + size_t rk_num_stages = 2; + int NB = 6; // number of boundaries + int NBSF = 4; // number of surface density force conditions + int NBV = 2; // number of surface sets used to specify a fixed displacement on nodes belonging to respective surfaces + + // Non-serialized fields + int num_gauss_points = 2; + size_t max_num_global_vars; + size_t rk_num_bins; + double time_value = 0.0; + DCArrayKokkos global_vars; + + DCArrayKokkos mat_fill; + DCArrayKokkos material; + DCArrayKokkos boundary; + std::vector gravity_vector { 9.81, 0., 0. }; + + void init_material_variable_arrays(size_t nglobal_vars) + { + global_vars = DCArrayKokkos(material_options.size(), nglobal_vars); + + for (size_t i = 0; i < material_options.size(); i++) + { + auto mat = material_options[i]; + + for (size_t j = 0; j < mat.global_vars.size(); j++) + { + global_vars.host(i, j) = mat.global_vars[j]; + } + } + } + + template void from_vector(DCArrayKokkos& array, const std::vector& vec) + { + array = DCArrayKokkos(vec.size()); + for (size_t i = 0; i < vec.size(); i++) + { + array.host(i) = *(T*)&vec[i]; + } + } + + void derive_kokkos_arrays() + { + max_num_global_vars = 0; + for (auto mo : material_options) + { + max_num_global_vars = std::max(max_num_global_vars, mo.global_vars.size()); + } + + init_material_variable_arrays(max_num_global_vars); + + from_vector(mat_fill, region_options); + from_vector(material, material_options); + from_vector(boundary, boundary_conditions); + + // Send to device. + mat_fill.update_device(); + boundary.update_device(); + material.update_device(); + global_vars.update_device(); + } + + void derive() + { + derive_kokkos_arrays(); + rk_num_bins = rk_num_stages; + } + + void validate() + { + validate_module_is_specified(FEA_MODULE_TYPE::Eulerian); } - } - - template void from_vector(DCArrayKokkos& array, const std::vector& vec) { - array = DCArrayKokkos(vec.size()); - for (size_t i = 0; i < vec.size(); i++) - array.host(i) = *(T*)&vec[i]; - } - void derive_kokkos_arrays() { - max_num_global_vars = 0; - for (auto mo : material_options) - max_num_global_vars = std::max(max_num_global_vars, mo.global_vars.size()); - - init_material_variable_arrays(max_num_global_vars); - - from_vector(mat_fill, region_options); - from_vector(material, material_options); - from_vector(boundary, boundary_conditions); - - // Send to device. - mat_fill.update_device(); - boundary.update_device(); - material.update_device(); - global_vars.update_device(); - - } - - void derive() { - derive_kokkos_arrays(); - rk_num_bins = rk_num_stages; - } - void validate() { - validate_module_is_specified(FEA_MODULE_TYPE::Eulerian); - } }; -IMPL_YAML_SERIALIZABLE_WITH_BASE(Simulation_Parameters_Eulerian, Simulation_Parameters, - time_variables, material_options, region_options, +IMPL_YAML_SERIALIZABLE_WITH_BASE(Simulation_Parameters_Eulerian, Simulation_Parameters, + time_variables, material_options, region_options, boundary_conditions, gravity_flag, report_runtime, rk_num_stages, NB, NBSF, NBV, graphics_options -) + ) #endif // end HEADER_H diff --git a/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/eulerian_geometry.cpp b/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/eulerian_geometry.cpp index d07b69392..174ac6b14 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/eulerian_geometry.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/eulerian_geometry.cpp @@ -1,32 +1,29 @@ // ----------------------------------------------------------------------------- // This code handles the geometric information for the mesh for the SHG solver -//------------------------------------------------------------------------------ +// ------------------------------------------------------------------------------ #include "matar.h" #include "mesh.h" #include "state.h" #include "FEA_Module_Eulerian.h" -void FEA_Module_Eulerian::example_function(double rk_alpha, - const size_t num_nodes, - DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel){ - +void FEA_Module_Eulerian::example_function(double rk_alpha, + const size_t num_nodes, + DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel) +{ const size_t rk_level = rk_num_bins - 1; - int num_dims = num_dim; + int num_dims = num_dim; // example device loop FOR_ALL_CLASS(node_gid, 0, num_nodes, { - - for (int dim = 0; dim < num_dims; dim++){ - double half_vel = (node_vel(rk_level, node_gid, dim) + node_vel(0, node_gid, dim))*0.5; - node_coords(rk_level, node_gid, dim) = node_coords(0, node_gid, dim) + rk_alpha*dt*half_vel; + for (int dim = 0; dim < num_dims; dim++) + { + double half_vel = (node_vel(rk_level, node_gid, dim) + node_vel(0, node_gid, dim)) * 0.5; + node_coords(rk_level, node_gid, dim) = node_coords(0, node_gid, dim) + rk_alpha * dt * half_vel; } - }); // end parallel for over nodes - } // end subroutine - // ----------------------------------------------------------------------------- // This function claculates // B_p = J^{-T} \cdot (\nabla_{xi} \phi_p w @@ -37,15 +34,12 @@ void FEA_Module_Eulerian::example_function(double rk_alpha, // \nabla_{xi} is the gradient opperator in the reference coordinates // // B_p is the OUTWARD corner area normal at node p -//------------------------------------------------------------------------------ +// ------------------------------------------------------------------------------ KOKKOS_FUNCTION -void FEA_Module_Eulerian::example_device_function(const ViewCArrayKokkos &B_matrix, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const { - - +void FEA_Module_Eulerian::example_device_function(const ViewCArrayKokkos& B_matrix, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const +{ } // end subroutine - - diff --git a/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/eulerian_time_integration.cpp b/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/eulerian_time_integration.cpp index fd95ab16e..2e99a4718 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/eulerian_time_integration.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/eulerian_time_integration.cpp @@ -5,83 +5,75 @@ // ----------------------------------------------------------------------------- // This function saves the variables at rk_stage = 0, which is t_n -//------------------------------------------------------------------------------ -void FEA_Module_Eulerian::rk_init(DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &elem_sie, - DViewCArrayKokkos &elem_stress, - const size_t num_elems, - const size_t num_nodes){ - +// ------------------------------------------------------------------------------ +void FEA_Module_Eulerian::rk_init(DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_sie, + DViewCArrayKokkos& elem_stress, + const size_t num_elems, + const size_t num_nodes) +{ const size_t rk_level = rk_num_bins - 1; - int num_dims = num_dim; + int num_dims = num_dim; // save elem quantities FOR_ALL_CLASS(elem_gid, 0, num_elems, { - // stress is always 3D even with 2D-RZ - for(size_t i=0; i<3; i++){ - for(size_t j=0; j<3; j++){ - elem_stress(0,elem_gid,i,j) = elem_stress(rk_level,elem_gid,i,j); + for (size_t i = 0; i < 3; i++) + { + for (size_t j = 0; j < 3; j++) + { + elem_stress(0, elem_gid, i, j) = elem_stress(rk_level, elem_gid, i, j); } } // end for - elem_sie(0,elem_gid) = elem_sie(rk_level,elem_gid); - + elem_sie(0, elem_gid) = elem_sie(rk_level, elem_gid); }); // end parallel for - - + // save nodal quantities FOR_ALL_CLASS(node_gid, 0, num_nodes, { - - for(size_t i=0; i &node_coords, - DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &elem_sspd, - DViewCArrayKokkos &elem_vol){ - - --> const size_t rk_level = rk_num_bins - 1; +void FEA_Module_Eulerian::get_timestep(mesh_t& mesh, + DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_sspd, + DViewCArrayKokkos& elem_vol) +{ + ->const size_t rk_level = rk_num_bins - 1; // increase dt by 10%, that is the largest dt value - dt = dt*1.1; - int num_dims = num_dim; + dt = dt * 1.1; + int num_dims = num_dim; double dt_lcl; double min_dt_calc; REDUCE_MIN_CLASS(elem_gid, 0, rnum_elem, dt_lcl, { - double coords0[24]; // element coords - ViewCArrayKokkos coords(coords0, 8, 3); + ViewCArrayKokkos coords(coords0, 8, 3); - double distance0[28]; // array for holding distances between each node - ViewCArrayKokkos dist(distance0, 28); - - // Getting the coordinates of the element - for(size_t node_lid = 0; node_lid < 8; node_lid++){ + ViewCArrayKokkos dist(distance0, 28); - for (size_t dim = 0; dim < num_dims; dim++){ - coords(node_lid, dim) = node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), dim); + // Getting the coordinates of the element + for (size_t node_lid = 0; node_lid < 8; node_lid++) + { + for (size_t dim = 0; dim < num_dims; dim++) + { + coords(node_lid, dim) = node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), dim); } // end for dim - } // end for loop over node_lid // loop conditions needed for distance calculation @@ -90,148 +82,150 @@ void FEA_Module_Eulerian::get_timestep(mesh_t &mesh, size_t a; size_t b; size_t loop = 0; - + // Only works for 3D // Solving for the magnitude of distance between each node - for (size_t i = 0; i < 28; i++){ - + for (size_t i = 0; i < 28; i++) + { a = countA; b = countB; - + // returns magnitude of distance between each node, 28 total options - dist(i) = fabs(sqrt(( pow((coords(b, 0) - coords(a, 0)), 2.0) - + pow((coords(b, 1) - coords(a, 1)), 2.0) - + pow((coords(b, 2) - coords(a, 2)), 2.0)))); + dist(i) = fabs(sqrt((pow((coords(b, 0) - coords(a, 0)), 2.0) + + pow((coords(b, 1) - coords(a, 1)), 2.0) + + pow((coords(b, 2) - coords(a, 2)), 2.0)))); countB++; countA++; - + // tricky indexing - if (countB > 7) { + if (countB > 7) + { loop++; countB = 1 + loop; countA = 0; } - } // endo for i - double dist_min = dist(0); - - for(int i = 0; i < 28; ++i){ + + for (int i = 0; i < 28; ++i) + { dist_min = fmin(dist(i), dist_min); } - + // local dt calc based on CFL - double dt_lcl_ = dt_cfl*dist_min/(elem_sspd(elem_gid) + fuzz); - + double dt_lcl_ = dt_cfl * dist_min / (elem_sspd(elem_gid) + fuzz); + // make dt be in bounds dt_lcl_ = fmin(dt_lcl_, dt_max); // make dt small than dt_max dt_lcl_ = fmax(dt_lcl_, dt_min); // make dt larger than dt_min - - - if (dt_lcl_ < dt_lcl) dt_lcl = dt_lcl_; - + + if (dt_lcl_ < dt_lcl) + { + dt_lcl = dt_lcl_; + } }, min_dt_calc); // end parallel reduction Kokkos::fence(); - + // save the min dt - if(min_dt_calc < dt) dt = min_dt_calc; - + if (min_dt_calc < dt) + { + dt = min_dt_calc; + } + // ensure time step hits the graphics time intervals - dt = fmin(dt, (graphics_time - time_value)+fuzz); - + dt = fmin(dt, (graphics_time - time_value) + fuzz); + // make dt be exact for final time - dt = fmin(dt, time_final-time_value); - + dt = fmin(dt, time_final - time_value); + return; - } // end get_timestep // ----------------------------------------------------------------------------- // This function calculates the time step by finding the shortest distance // between any two nodes in the mesh -//------------------------------------------------------------------------------ +// ------------------------------------------------------------------------------ // WARNING WARNING : Only works for 3D, 8 node elements -void FEA_Module_Eulerian::get_timestep2D(mesh_t &mesh, - DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &elem_sspd, - DViewCArrayKokkos &elem_vol){ - --> const size_t rk_level = rk_num_bins - 1; +void FEA_Module_Eulerian::get_timestep2D(mesh_t& mesh, + DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_sspd, + DViewCArrayKokkos& elem_vol) +{ + ->const size_t rk_level = rk_num_bins - 1; // increase dt by 10%, that is the largest dt value - dt = dt*1.1; - int num_dims = num_dim; + dt = dt * 1.1; + int num_dims = num_dim; double dt_lcl; double min_dt_calc; REDUCE_MIN_CLASS(elem_gid, 0, rnum_elem, dt_lcl, { - double coords0[8]; // element coords - ViewCArrayKokkos coords(coords0, 4, 2); + ViewCArrayKokkos coords(coords0, 4, 2); - double distance0[6]; // array for holding distances between each node - ViewCArrayKokkos dist(distance0, 6); - - // Getting the coordinates of the nodes of the element - for(size_t node_lid = 0; node_lid < 4; node_lid++){ + ViewCArrayKokkos dist(distance0, 6); - for (size_t dim = 0; dim < num_dims; dim++){ - coords(node_lid, dim) = node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), dim); + // Getting the coordinates of the nodes of the element + for (size_t node_lid = 0; node_lid < 4; node_lid++) + { + for (size_t dim = 0; dim < num_dims; dim++) + { + coords(node_lid, dim) = node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), dim); } // end for dim - } // end for loop over node_lid - // Only works for 2D // Solving for the magnitude of distance between each node size_t count = 0; - for (size_t i = 0; i < 3; i++){ - for (size_t j=i+1; j<=3; j++){ - + for (size_t i = 0; i < 3; i++) + { + for (size_t j = i + 1; j <= 3; j++) + { // returns magnitude of distance between each node, 6 total options dist(count) = fabs( - sqrt( pow((coords(i, 0) - coords(j, 0)), 2.0) - + pow((coords(i, 1) - coords(j, 1)), 2.0) ) - ); - count ++; + sqrt(pow((coords(i, 0) - coords(j, 0)), 2.0) + + pow((coords(i, 1) - coords(j, 1)), 2.0) ) + ); + count++; } // end for j } // end for i - double dist_min = dist(0); - - for(int i = 0; i < 6; ++i){ + + for (int i = 0; i < 6; ++i) + { dist_min = fmin(dist(i), dist_min); } - + // local dt calc based on CFL - double dt_lcl_ = dt_cfl*dist_min/(elem_sspd(elem_gid) + fuzz); - + double dt_lcl_ = dt_cfl * dist_min / (elem_sspd(elem_gid) + fuzz); + // make dt be in bounds dt_lcl_ = fmin(dt_lcl_, dt_max); // make dt small than dt_max dt_lcl_ = fmax(dt_lcl_, dt_min); // make dt larger than dt_min - - - if (dt_lcl_ < dt_lcl) dt_lcl = dt_lcl_; - + + if (dt_lcl_ < dt_lcl) + { + dt_lcl = dt_lcl_; + } }, min_dt_calc); // end parallel reduction Kokkos::fence(); - + // save the min dt - if(min_dt_calc < dt) dt = min_dt_calc; - + if (min_dt_calc < dt) + { + dt = min_dt_calc; + } + // ensure time step hits the graphics time intervals - dt = fmin(dt, (graphics_time - time_value)+fuzz); - + dt = fmin(dt, (graphics_time - time_value) + fuzz); + // make dt be exact for final time - dt = fmin(dt, time_final-time_value); - + dt = fmin(dt, time_final - time_value); + return; - } // end get_timestep2D - - diff --git a/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp b/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp index 056c13c34..13870d5ae 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp @@ -11,14 +11,14 @@ This program is open source under the BSD-3 License. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - + 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - + 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - + 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. @@ -33,14 +33,14 @@ WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - **********************************************************************************************/ - +**********************************************************************************************/ + #include #include #include #include #include -#include +#include #include // fmin, fmax, abs note: fminl is long #include #include @@ -63,14 +63,14 @@ #include "Simulation_Parameters/Simulation_Parameters_Explicit.h" #include "Simulation_Parameters/FEA_Module/FEA_Module_Headers.h" #include "FEA_Module_SGH.h" -//#include "FEA_Module_Eulerian.h" +// #include "FEA_Module_Eulerian.h" #include "FEA_Module_Dynamic_Elasticity.h" #include "FEA_Module_Inertial.h" #include "Explicit_Solver.h" #include "mesh.h" #include "state.h" -//Optimization Package +// Optimization Package #include "ROL_Algorithm.hpp" #include "ROL_Solver.hpp" #include "ROL_LineSearchStep.hpp" @@ -85,8 +85,8 @@ #include "ROL_ParameterList.hpp" #include -//Objective Functions and Constraint Functions -//#include "Topology_Optimization_Function_Headers.h" +// Objective Functions and Constraint Functions +// #include "Topology_Optimization_Function_Headers.h" #include "Mass_Constraint.h" #include "Moment_of_Inertia_Constraint.h" #include "Kinetic_Energy_Minimize.h" @@ -104,1435 +104,1581 @@ using namespace mtr; /* -Swage is a reference to a swage block used in blacksmithing. -Its a large metal block that has multiple shaps carved into -each surface to use for hammering metal into to form it. +Swage is a reference to a swage block used in blacksmithing. +Its a large metal block that has multiple shaps carved into +each surface to use for hammering metal into to form it. */ -Explicit_Solver::Explicit_Solver(Simulation_Parameters_Explicit params) : Solver(params) { - //create parameter objects - simparam = params; - //create ref element object - ref_elem = std::make_shared(); - //create mesh objects - //init_mesh = new swage::mesh_t(simparaA); - mesh = std::make_shared(); - - element_select = std::make_shared(); - num_nodes = 0; - - //boundary condition data - current_bdy_id = 0; - - //Trilinos output stream - std::ostream &out = std::cout; - fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - (*fos).setOutputToRootOnly(0); - - //file readin parameter - active_node_ordering_convention = ENSIGHT; - //default simulation parameters - time_value = 0; +Explicit_Solver::Explicit_Solver(Simulation_Parameters_Explicit params) : Solver(params) +{ + // create parameter objects + simparam = params; + // create ref element object + ref_elem = std::make_shared(); + // create mesh objects + // init_mesh = new swage::mesh_t(simparaA); + mesh = std::make_shared(); + + element_select = std::make_shared(); + num_nodes = 0; + + // boundary condition data + current_bdy_id = 0; + + // Trilinos output stream + std::ostream& out = std::cout; + fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + (*fos).setOutputToRootOnly(0); + + // file readin parameter + active_node_ordering_convention = ENSIGHT; + // default simulation parameters + time_value = 0; } -Explicit_Solver::~Explicit_Solver(){ - if (myrank == 0 && in != NULL) - delete in; +Explicit_Solver::~Explicit_Solver() +{ + if (myrank == 0 && in != NULL) + { + delete in; + } } -//============================================================================== +// ============================================================================== // Primary simulation runtime routine -//============================================================================== +// ============================================================================== + +void Explicit_Solver::run() +{ + // MPI info + world = MPI_COMM_WORLD; // used for convenience to represent all the ranks in the job + MPI_Comm_rank(world, &myrank); + MPI_Comm_size(world, &nranks); + + if (myrank == 0) + { + std::cout << "Starting Lagrangian SGH code" << std::endl; + } + // initialize Trilinos communicator class + comm = Tpetra::getDefaultComm(); + + if (simparam.input_options.has_value()) + { + const Input_Options& input_options = simparam.input_options.value(); + const char* mesh_file_name = input_options.mesh_file_name.c_str(); + switch (input_options.mesh_file_format) + { + case MESH_FORMAT::tecplot: + read_mesh_tecplot(mesh_file_name); + break; + case MESH_FORMAT::vtk: + read_mesh_vtk(mesh_file_name); + break; + case MESH_FORMAT::ansys_dat: + read_mesh_ansys_dat(mesh_file_name); + break; + case MESH_FORMAT::ensight: + read_mesh_ensight(mesh_file_name); + break; + } + } + else + { + generate_mesh(simparam.mesh_generation_options.value()); + } -void Explicit_Solver::run() { - - //MPI info - world = MPI_COMM_WORLD; //used for convenience to represent all the ranks in the job - MPI_Comm_rank(world,&myrank); - MPI_Comm_size(world,&nranks); - - if(myrank == 0){ - std::cout << "Starting Lagrangian SGH code" << std::endl; - } + // debug + // return; + init_maps(); - //initialize Trilinos communicator class - comm = Tpetra::getDefaultComm(); - - if (simparam.input_options.has_value()) { - const Input_Options& input_options = simparam.input_options.value(); - const char* mesh_file_name = input_options.mesh_file_name.c_str(); - switch (input_options.mesh_file_format) { - case MESH_FORMAT::tecplot: - read_mesh_tecplot(mesh_file_name); - break; - case MESH_FORMAT::vtk: - read_mesh_vtk(mesh_file_name); - break; - case MESH_FORMAT::ansys_dat: - read_mesh_ansys_dat(mesh_file_name); - break; - case MESH_FORMAT::ensight: - read_mesh_ensight(mesh_file_name); - break; - } - } else { - generate_mesh(simparam.mesh_generation_options.value()); - } - - //debug - //return; - init_maps(); - - init_state_vectors(); - - //set initial saved coordinates - //initial_node_coords_distributed->assign(*node_coords_distributed); - all_initial_node_coords_distributed->assign(*all_node_coords_distributed); - initial_node_coords_distributed = Teuchos::rcp(new MV(*all_initial_node_coords_distributed, map)); - - std::cout << "Num elements on process " << myrank << " = " << rnum_elem << std::endl; - - //initialize timing - init_clock(); - - //initialize runtime counters and timers - int hessvec_count = 0; - int update_count = 0; - file_index = 0; - real_t linear_solve_time = 0; - real_t hessvec_time = 0; - real_t hessvec_linear_time = 0; - - // ---- Find Boundaries on mesh ---- // - init_boundaries(); - - //set boundary conditions - //generate_tcs(); - //initialize TO design variable storage - if(simparam.topology_optimization_on || simparam.shape_optimization_on) - init_design(); - //process process list of requested FEA modules to construct list of objects - FEA_module_setup(); - - //Have modules read in boundary/loading conditions if file format provides it - /* - for(int imodule = 0; imodule < nfea_modules; imodule++){ - if(fea_module_must_read[imodule]){ - fea_modules[imodule]->read_conditions_ansys_dat(in, before_condition_header); - } - else{ - fea_modules[imodule]->init_boundaries(); + init_state_vectors(); + + // set initial saved coordinates + // initial_node_coords_distributed->assign(*node_coords_distributed); + all_initial_node_coords_distributed->assign(*all_node_coords_distributed); + initial_node_coords_distributed = Teuchos::rcp(new MV(*all_initial_node_coords_distributed, map)); + + std::cout << "Num elements on process " << myrank << " = " << rnum_elem << std::endl; - //set boundary conditions for FEA modules - fea_modules[imodule]->generate_bcs(); + // initialize timing + init_clock(); - //set applied loading conditions for FEA modules - fea_modules[imodule]->generate_applied_loads(); + // initialize runtime counters and timers + int hessvec_count = 0; + int update_count = 0; + file_index = 0; + real_t linear_solve_time = 0; + real_t hessvec_time = 0; + real_t hessvec_linear_time = 0; + + // ---- Find Boundaries on mesh ---- // + init_boundaries(); + + // set boundary conditions + // generate_tcs(); + // initialize TO design variable storage + if (simparam.topology_optimization_on || simparam.shape_optimization_on) + { + init_design(); } - } - - //std::cout << "FEA MODULES " << nfea_modules << " " << simparam->nfea_modules << std::endl; - //call boundary routines on fea modules - - if(myrank == 0) - std::cout << "Starting init assembly" << std::endl <init_assembly(); - - //assemble the global solution (stiffness matrix etc. and nodal forces) - for(int imodule = 0; imodule < nfea_modules; imodule++) - fea_modules[imodule]->assemble_matrix(); + // process process list of requested FEA modules to construct list of objects + FEA_module_setup(); - if(myrank == 0) - std::cout << "Finished matrix assembly" << std::endl <assemble_vector(); + // Have modules read in boundary/loading conditions if file format provides it + /* + for(int imodule = 0; imodule < nfea_modules; imodule++){ + if(fea_module_must_read[imodule]){ + fea_modules[imodule]->read_conditions_ansys_dat(in, before_condition_header); + } + else{ + fea_modules[imodule]->init_boundaries(); - for(int imodule = 0; imodule < nfea_modules; imodule++) - fea_modules[imodule]->linear_solver_parameters(); - - if(myrank == 0) - std::cout << "Starting First Solve" << std::endl <solve(); - if(solver_exit != EXIT_SUCCESS){ - std::cout << "Linear Explicit_Solver Error" << std::endl < design_gradients_distributed = Teuchos::rcp(new MV(map, 1)); - const_host_vec_array node_densities = design_node_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); - host_vec_array design_gradients = design_gradients_distributed->getLocalView (Tpetra::Access::ReadWrite); - compute_adjoint_gradients(node_densities, design_gradients); - std::ostream &out = std::cout; - Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - if(myrank==0) - *fos << "Strain Energy gradients :" << std::endl; - design_gradients_distributed->describe(*fos,Teuchos::VERB_EXTREME); - *fos << std::endl; - std::fflush(stdout); - */ - //return; - - //solver_exit = solve(); - //if(solver_exit == EXIT_SUCCESS){ - //std::cout << "Linear Explicit_Solver Error" << std::endl <setup(); - - for(int imodule = 0; imodule < nfea_modules; imodule++){ - if(myrank == 0) - std::cout << "Starting setup for FEA module " << imodule <setup(); - } - - //set initial saved velocities - initial_node_velocities_distributed->assign(*node_velocities_distributed); - - if(simparam.topology_optimization_on || simparam.shape_optimization_on){ - //design_node_densities_distributed->randomize(1,1); - setup_optimization_problem(); - //problem = ROL::makePtr>(obj,x); - } - else{ - // --------------------------------------------------------------------- - // Calculate the SGH solution - // --------------------------------------------------------------------- - //sgh_module->sgh_solve(); - for(int imodule = 0; imodule < nfea_modules; imodule++){ - if(myrank == 0) - std::cout << "Starting solve for FEA module " << imodule <solve(); + //set boundary conditions for FEA modules + fea_modules[imodule]->generate_bcs(); + + //set applied loading conditions for FEA modules + fea_modules[imodule]->generate_applied_loads(); } - } + } + + //std::cout << "FEA MODULES " << nfea_modules << " " << simparam->nfea_modules << std::endl; + //call boundary routines on fea modules - // clean up all material models - //sgh_module->cleanup_material_models(); - for(int imodule = 0; imodule < nfea_modules; imodule++){ if(myrank == 0) - std::cout << "Starting solve for FEA module " << imodule <module_cleanup(); - } + for(int imodule = 0; imodule < nfea_modules; imodule++) + fea_modules[imodule]->init_assembly(); - //printf("Finished\n"); - - //benchmark simulation run time end - double current_cpu = CPU_Time(); - /* - for(int imodule = 0; imodule < nfea_modules; imodule++){ - linear_solve_time += fea_modules[imodule]->linear_solve_time; - hessvec_linear_time += fea_modules[imodule]->hessvec_linear_time; - } - */ - - std::cout << " RUNTIME OF CODE ON TASK " << myrank << " is "<< current_cpu-initial_CPU_time << " comms time " - << communication_time << " host to dev time " << host2dev_time << " dev to host time " << dev2host_time << std::endl; - - if(simparam.output_options.timer_output_level == TIMER_VERBOSITY::thorough){ - std::cout << " OUTPUT TIME OF CODE ON TASK " << myrank << " is "<< output_time << std::endl; - } + //assemble the global solution (stiffness matrix etc. and nodal forces) + for(int imodule = 0; imodule < nfea_modules; imodule++) + fea_modules[imodule]->assemble_matrix(); - //parallel_vtk_writer(); - - //test forward solve call - int ntests = 0; - if(simparam.topology_optimization_on){ - for(int itest = 0; itest < ntests; itest++){ - design_node_densities_distributed->randomize(1,1); - //test_node_densities_distributed = Teuchos::rcp(new MV(*design_node_densities_distributed)); - Teuchos::RCP test_gradients_distributed = Teuchos::rcp(new MV(map, 1)); - const_vec_array test_node_densities = design_node_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); - vec_array test_gradients = test_gradients_distributed->getLocalView (Tpetra::Access::ReadWrite); - - fea_modules[0]->comm_variables(design_node_densities_distributed); - fea_modules[0]->update_forward_solve(design_node_densities_distributed); - fea_modules[0]->compute_topology_optimization_adjoint(); - fea_modules[0]->compute_topology_optimization_gradient(test_node_densities, test_gradients); - // Data writers - //parallel_vtk_writer(); - } - } - - // vtk_writer(); - /* - if(myrank==0){ - std::cout << "Total number of solves and assembly " << fea_modules[0]->update_count <hessvec_count <assemble_vector(); + + for(int imodule = 0; imodule < nfea_modules; imodule++) + fea_modules[imodule]->linear_solver_parameters(); + + if(myrank == 0) + std::cout << "Starting First Solve" << std::endl <solve(); + if(solver_exit != EXIT_SUCCESS){ + std::cout << "Linear Explicit_Solver Error" << std::endl < design_gradients_distributed = Teuchos::rcp(new MV(map, 1)); + const_host_vec_array node_densities = design_node_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); + host_vec_array design_gradients = design_gradients_distributed->getLocalView (Tpetra::Access::ReadWrite); + compute_adjoint_gradients(node_densities, design_gradients); + std::ostream &out = std::cout; + Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + if(myrank==0) + *fos << "Strain Energy gradients :" << std::endl; + design_gradients_distributed->describe(*fos,Teuchos::VERB_EXTREME); + *fos << std::endl; + std::fflush(stdout); + */ + // return; + + // solver_exit = solve(); + // if(solver_exit == EXIT_SUCCESS){ + // std::cout << "Linear Explicit_Solver Error" << std::endl <setup(); + + for (int imodule = 0; imodule < nfea_modules; imodule++) + { + if (myrank == 0) + { + std::cout << "Starting setup for FEA module " << imodule << std::endl << std::flush; + } + // allocate and fill sparse structures needed for global solution in each FEA module + fea_modules[imodule]->setup(); + } + + // set initial saved velocities + initial_node_velocities_distributed->assign(*node_velocities_distributed); + + if (simparam.topology_optimization_on || simparam.shape_optimization_on) + { + // design_node_densities_distributed->randomize(1,1); + setup_optimization_problem(); + // problem = ROL::makePtr>(obj,x); + } + else + { + // --------------------------------------------------------------------- + // Calculate the SGH solution + // --------------------------------------------------------------------- + // sgh_module->sgh_solve(); + for (int imodule = 0; imodule < nfea_modules; imodule++) + { + if (myrank == 0) + { + std::cout << "Starting solve for FEA module " << imodule << std::endl << std::flush; + } + // allocate and fill sparse structures needed for global solution in each FEA module + fea_modules[imodule]->solve(); + } + } + + // clean up all material models + // sgh_module->cleanup_material_models(); + for (int imodule = 0; imodule < nfea_modules; imodule++) + { + if (myrank == 0) + { + std::cout << "Starting solve for FEA module " << imodule << std::endl << std::flush; + } + // allocate and fill sparse structures needed for global solution in each FEA module + fea_modules[imodule]->module_cleanup(); + } + + // printf("Finished\n"); + + // benchmark simulation run time end + double current_cpu = CPU_Time(); + /* + for(int imodule = 0; imodule < nfea_modules; imodule++){ + linear_solve_time += fea_modules[imodule]->linear_solve_time; + hessvec_linear_time += fea_modules[imodule]->hessvec_linear_time; + } + */ + + std::cout << " RUNTIME OF CODE ON TASK " << myrank << " is " << current_cpu - initial_CPU_time << " comms time " + << communication_time << " host to dev time " << host2dev_time << " dev to host time " << dev2host_time << std::endl; + + if (simparam.output_options.timer_output_level == TIMER_VERBOSITY::thorough) + { + std::cout << " OUTPUT TIME OF CODE ON TASK " << myrank << " is " << output_time << std::endl; + } + + // parallel_vtk_writer(); + + // test forward solve call + int ntests = 0; + if (simparam.topology_optimization_on) + { + for (int itest = 0; itest < ntests; itest++) + { + design_node_densities_distributed->randomize(1, 1); + // test_node_densities_distributed = Teuchos::rcp(new MV(*design_node_densities_distributed)); + Teuchos::RCP test_gradients_distributed = Teuchos::rcp(new MV(map, 1)); + const_vec_array test_node_densities = design_node_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); + vec_array test_gradients = test_gradients_distributed->getLocalView(Tpetra::Access::ReadWrite); + + fea_modules[0]->comm_variables(design_node_densities_distributed); + fea_modules[0]->update_forward_solve(design_node_densities_distributed); + fea_modules[0]->compute_topology_optimization_adjoint(); + fea_modules[0]->compute_topology_optimization_gradient(test_node_densities, test_gradients); + // Data writers + // parallel_vtk_writer(); + } + } + + // vtk_writer(); + /* + if(myrank==0){ + std::cout << "Total number of solves and assembly " << fea_modules[0]->update_count <hessvec_count < read_buffer; - int buffer_loop, buffer_iteration, buffer_iterations, dof_limit, scan_loop, nodes_per_element; - size_t read_index_start, node_rid, elem_gid; - GO node_gid; - real_t dof_value; - host_vec_array node_densities; - bool zero_index_base = input_options.zero_index_base; - int negative_index_found = 0; - int global_negative_index_found = 0; - //Nodes_Per_Element_Type = elements::elem_types::Nodes_Per_Element_Type; - - //read the mesh - //PLACEHOLDER: ensight_format(MESH); - // abaqus_format(MESH); - // vtk_format(MESH) - - //task 0 reads file - if(myrank==0){ - in = new std::ifstream(); - in->open(MESH); - if (!(*in)) throw std::runtime_error(std::string("Can't open ") + MESH); - } - - //ANSYS dat file doesn't specify total number of nodes, which is needed for the node map. - //First pass reads in node section to determine the maximum number of nodes, second pass distributes node data - //The elements section header does specify element count - num_nodes = 0; - if(myrank==0){ - bool searching_for_nodes = true; - //skip lines at the top with nonessential info; stop skipping when "Nodes for the whole assembly" string is reached - while (searching_for_nodes&&in->good()) { - getline(*in, skip_line); - //std::cout << skip_line << std::endl; - line_parse.clear(); - line_parse.str(skip_line); - //stop when the NODES= string is reached - while (!line_parse.eof()){ - line_parse >> substring; - //std::cout << substring << std::endl; - if(!substring.compare("Nodes")){ - searching_for_nodes = false; - break; - } - } //while - - } - if(searching_for_nodes){ - std::cout << "FILE FORMAT ERROR" << std::endl; - } - //skip 2 lines - for (int j = 0; j < 2; j++) { - getline(*in, skip_line); - std::cout << skip_line << std::endl; - } //for - - //tally node count (bug in dat files seems to print wrong node count so read is done in two passes) - //stop when apparent "-1" zone delimiter is reacher - searching_for_nodes = true; - int node_tally = 0; - while (searching_for_nodes) { - getline(*in, read_line); - //std::cout << read_line << std::endl; - line_parse.clear(); - line_parse.str(read_line); - line_parse >> substring; - - //std::cout << substring << std::endl; - if(substring == "-1"){ - searching_for_nodes = false; - break; - } - else{ - node_tally++; +void Explicit_Solver::read_mesh_ansys_dat(const char* MESH) +{ + char ch; + int num_dim = simparam.num_dims; + Input_Options input_options = simparam.input_options.value(); + int p_order = input_options.p_order; + real_t unit_scaling = input_options.unit_scaling; + bool restart_file = simparam.restart_file; + int local_node_index, current_column_index; + size_t strain_count; + std::string skip_line, read_line, substring, token; + std::stringstream line_parse; + CArrayKokkos read_buffer; + int buffer_loop, buffer_iteration, buffer_iterations, dof_limit, scan_loop, nodes_per_element; + size_t read_index_start, node_rid, elem_gid; + GO node_gid; + real_t dof_value; + host_vec_array node_densities; + bool zero_index_base = input_options.zero_index_base; + int negative_index_found = 0; + int global_negative_index_found = 0; + // Nodes_Per_Element_Type = elements::elem_types::Nodes_Per_Element_Type; + + // read the mesh + // PLACEHOLDER: ensight_format(MESH); + // abaqus_format(MESH); + // vtk_format(MESH) + + // task 0 reads file + if (myrank == 0) + { + in = new std::ifstream(); + in->open(MESH); + if (!(*in)) + { + throw std::runtime_error(std::string("Can't open ") + MESH); + } + } + + // ANSYS dat file doesn't specify total number of nodes, which is needed for the node map. + // First pass reads in node section to determine the maximum number of nodes, second pass distributes node data + // The elements section header does specify element count + num_nodes = 0; + if (myrank == 0) + { + bool searching_for_nodes = true; + // skip lines at the top with nonessential info; stop skipping when "Nodes for the whole assembly" string is reached + while (searching_for_nodes && in->good()) { + getline(*in, skip_line); + // std::cout << skip_line << std::endl; + line_parse.clear(); + line_parse.str(skip_line); + // stop when the NODES= string is reached + while (!line_parse.eof()) { + line_parse >> substring; + // std::cout << substring << std::endl; + if (!substring.compare("Nodes")) + { + searching_for_nodes = false; + break; + } + } // while + } + if (searching_for_nodes) + { + std::cout << "FILE FORMAT ERROR" << std::endl; + } + // skip 2 lines + for (int j = 0; j < 2; j++) + { + getline(*in, skip_line); + std::cout << skip_line << std::endl; + } // for + + // tally node count (bug in dat files seems to print wrong node count so read is done in two passes) + // stop when apparent "-1" zone delimiter is reacher + searching_for_nodes = true; + int node_tally = 0; + while (searching_for_nodes) { + getline(*in, read_line); + // std::cout << read_line << std::endl; + line_parse.clear(); + line_parse.str(read_line); + line_parse >> substring; + + // std::cout << substring << std::endl; + if (substring == "-1") + { + searching_for_nodes = false; + break; + } + else + { + node_tally++; + } + } + num_nodes = node_tally; + std::cout << "declared node count: " << num_nodes << std::endl; + } + + // broadcast number of nodes + MPI_Bcast(&num_nodes, 1, MPI_LONG_LONG_INT, 0, world); + + // construct contiguous parallel row map now that we know the number of nodes + map = Teuchos::rcp(new Tpetra::Map(num_nodes, 0, comm)); + + // close and reopen file for second pass now that global node count is known + if (myrank == 0) + { + in->close(); + // in = new std::ifstream(); + in->open(MESH); + } + + // set the vertices in the mesh read in + nlocal_nodes = map->getLocalNumElements(); + // populate local row offset data from global data + global_size_t min_gid = map->getMinGlobalIndex(); + global_size_t max_gid = map->getMaxGlobalIndex(); + global_size_t index_base = map->getIndexBase(); + // debug print + // std::cout << "local node count on task: " << " " << nlocal_nodes << std::endl; + + // allocate node storage with dual view + // dual_node_coords = dual_vec_array("dual_node_coords", nlocal_nodes,num_dim); + // if(restart_file) + // dual_node_densities = dual_vec_array("dual_node_densities", nlocal_nodes,1); + + // local variable for host view in the dual view + node_coords_distributed = Teuchos::rcp(new MV(map, num_dim)); + // active view scope + { + host_vec_array node_coords = node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + // host_vec_array node_coords = dual_node_coords.view_host(); + if (restart_file) + { + design_node_densities_distributed = Teuchos::rcp(new MV(map, 1)); + node_densities = design_node_densities_distributed->getLocalView(Tpetra::Access::ReadWrite); + } + // notify that the host view is going to be modified in the file readin + // dual_node_coords.modify_host(); + // if(restart_file) + // dual_node_densities.modify_host(); + + // old swage method + // mesh->init_nodes(local_nrows); // add 1 for index starting at 1 + + std::cout << "Num nodes assigned to task " << myrank << " = " << nlocal_nodes << std::endl; + + // read the initial mesh coordinates + // x-coords + /*only task 0 reads in nodes and elements from the input file + stores node data in a buffer and communicates once the buffer cap is reached + or the data ends*/ + + words_per_line = input_options.words_per_line; + // if(restart_file) words_per_line++; + elem_words_per_line = input_options.elem_words_per_line; + + // allocate read buffer + read_buffer = CArrayKokkos(BUFFER_LINES, words_per_line, MAX_WORD); + + dof_limit = num_nodes; + buffer_iterations = dof_limit / BUFFER_LINES; + if (dof_limit % BUFFER_LINES != 0) + { + buffer_iterations++; + } + + // second pass to now read node coords with global node map defines + if (myrank == 0) + { + bool searching_for_nodes = true; + // skip lines at the top with nonessential info; stop skipping when "Nodes for the whole assembly" string is reached + while (searching_for_nodes && in->good()) { + getline(*in, skip_line); + // std::cout << skip_line << std::endl; + line_parse.clear(); + line_parse.str(skip_line); + // stop when the NODES= string is reached + while (!line_parse.eof()) { + line_parse >> substring; + // std::cout << substring << std::endl; + if (!substring.compare("Nodes")) + { + searching_for_nodes = false; + break; + } + } // while + } + if (searching_for_nodes) + { + std::cout << "FILE FORMAT ERROR" << std::endl; + } + // skip 2 lines + for (int j = 0; j < 2; j++) + { + getline(*in, skip_line); + std::cout << skip_line << std::endl; + } // for + } + + // read coords, also density if restarting + read_index_start = 0; + for (buffer_iteration = 0; buffer_iteration < buffer_iterations; buffer_iteration++) + { + // pack buffer on rank 0 + if (myrank == 0 && buffer_iteration < buffer_iterations - 1) + { + for (buffer_loop = 0; buffer_loop < BUFFER_LINES; buffer_loop++) + { + getline(*in, read_line); + line_parse.clear(); + line_parse.str(read_line); + + for (int iword = 0; iword < words_per_line; iword++) + { + // read portions of the line into the substring variable + line_parse >> substring; + // debug print + // std::cout<<" "<< substring <> substring; + // assign the substring variable as a word of the read buffer + strcpy(&read_buffer(buffer_loop, iword, 0), substring.c_str()); + } + buffer_loop++; + } + } + + // broadcast buffer to all ranks; each rank will determine which nodes in the buffer belong + MPI_Bcast(read_buffer.pointer(), BUFFER_LINES * words_per_line * MAX_WORD, MPI_CHAR, 0, world); + // broadcast how many nodes were read into this buffer iteration + MPI_Bcast(&buffer_loop, 1, MPI_INT, 0, world); + + // debug_print + // std::cout << "NODE BUFFER LOOP IS: " << buffer_loop << std::endl; + // for(int iprint=0; iprint < buffer_loop; iprint++) + // std::cout<<"buffer packing: " << std::string(&read_buffer(iprint,0,0)) << std::endl; + // return; + + // determine which data to store in the swage mesh members (the local node data) + // loop through read buffer + for (scan_loop = 0; scan_loop < buffer_loop; scan_loop++) + { + // set global node id (ensight specific order) + node_gid = read_index_start + scan_loop; + // let map decide if this node id belongs locally; if yes store data + if (map->isNodeGlobalElement(node_gid)) + { + // set local node index in this mpi rank + node_rid = map->getLocalElement(node_gid); + // extract nodal position from the read buffer + // for tecplot format this is the three coords in the same line + dof_value = atof(&read_buffer(scan_loop, 1, 0)); + node_coords(node_rid, 0) = dof_value * unit_scaling; + dof_value = atof(&read_buffer(scan_loop, 2, 0)); + node_coords(node_rid, 1) = dof_value * unit_scaling; + if (num_dim == 3) + { + dof_value = atof(&read_buffer(scan_loop, 3, 0)); + node_coords(node_rid, 2) = dof_value * unit_scaling; + } + // extract density if restarting + } + } + read_index_start += BUFFER_LINES; + } + } // end view scope + // repartition node distribution + repartition_nodes(); + + // synchronize device data + // dual_node_coords.sync_device(); + // dual_node_coords.modify_device(); + // if(restart_file){ + // dual_node_densities.sync_device(); + // dual_node_densities.modify_device(); + // } + + // debug print of nodal data + + // debug print nodal positions and indices + /* + std::cout << " ------------NODAL POSITIONS ON TASK " << myrank << " --------------"<getGlobalElement(inode) + 1 << " { "; + for (int istride = 0; istride < num_dim; istride++){ + std::cout << node_coords(inode,istride) << " , "; } - + //std::cout << node_densities(inode,0); + std::cout << " }"<< std::endl; } - num_nodes = node_tally; - std::cout << "declared node count: " << num_nodes << std::endl; - } + */ - //broadcast number of nodes - MPI_Bcast(&num_nodes,1,MPI_LONG_LONG_INT,0,world); - - //construct contiguous parallel row map now that we know the number of nodes - map = Teuchos::rcp( new Tpetra::Map(num_nodes,0,comm)); - - //close and reopen file for second pass now that global node count is known - if(myrank==0){ - in->close(); - //in = new std::ifstream(); - in->open(MESH); - } - - // set the vertices in the mesh read in - nlocal_nodes = map->getLocalNumElements(); - //populate local row offset data from global data - global_size_t min_gid = map->getMinGlobalIndex(); - global_size_t max_gid = map->getMaxGlobalIndex(); - global_size_t index_base = map->getIndexBase(); - //debug print - //std::cout << "local node count on task: " << " " << nlocal_nodes << std::endl; - - //allocate node storage with dual view - //dual_node_coords = dual_vec_array("dual_node_coords", nlocal_nodes,num_dim); - //if(restart_file) - //dual_node_densities = dual_vec_array("dual_node_densities", nlocal_nodes,1); - - //local variable for host view in the dual view - node_coords_distributed = Teuchos::rcp(new MV(map, num_dim)); - //active view scope - { - host_vec_array node_coords = node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); - //host_vec_array node_coords = dual_node_coords.view_host(); - if(restart_file){ - design_node_densities_distributed = Teuchos::rcp(new MV(map, 1)); - node_densities = design_node_densities_distributed->getLocalView (Tpetra::Access::ReadWrite); - } - //notify that the host view is going to be modified in the file readin - //dual_node_coords.modify_host(); - //if(restart_file) - //dual_node_densities.modify_host(); - - //old swage method - //mesh->init_nodes(local_nrows); // add 1 for index starting at 1 - - std::cout << "Num nodes assigned to task " << myrank << " = " << nlocal_nodes << std::endl; - - // read the initial mesh coordinates - // x-coords - /*only task 0 reads in nodes and elements from the input file - stores node data in a buffer and communicates once the buffer cap is reached - or the data ends*/ - - words_per_line = input_options.words_per_line; - //if(restart_file) words_per_line++; - elem_words_per_line = input_options.elem_words_per_line; - - //allocate read buffer - read_buffer = CArrayKokkos(BUFFER_LINES,words_per_line,MAX_WORD); - - dof_limit = num_nodes; - buffer_iterations = dof_limit/BUFFER_LINES; - if(dof_limit%BUFFER_LINES!=0) buffer_iterations++; - - //second pass to now read node coords with global node map defines - if(myrank==0){ - bool searching_for_nodes = true; - //skip lines at the top with nonessential info; stop skipping when "Nodes for the whole assembly" string is reached - while (searching_for_nodes&&in->good()) { - getline(*in, skip_line); - //std::cout << skip_line << std::endl; - line_parse.clear(); - line_parse.str(skip_line); - //stop when the NODES= string is reached - while (!line_parse.eof()){ - line_parse >> substring; - //std::cout << substring << std::endl; - if(!substring.compare("Nodes")){ - searching_for_nodes = false; - break; - } - } //while - - } - if(searching_for_nodes){ - std::cout << "FILE FORMAT ERROR" << std::endl; - } - //skip 2 lines - for (int j = 0; j < 2; j++) { - getline(*in, skip_line); - std::cout << skip_line << std::endl; - } //for - } - - //read coords, also density if restarting - read_index_start = 0; - for(buffer_iteration = 0; buffer_iteration < buffer_iterations; buffer_iteration++){ - //pack buffer on rank 0 - if(myrank==0&&buffer_iterationgood()) { + getline(*in, skip_line); + // std::cout << skip_line << std::endl; + line_parse.clear(); + line_parse.str(skip_line); + // stop when the NODES= string is reached + while (!line_parse.eof()) { + line_parse >> substring; + // std::cout << substring << std::endl; + if (!substring.compare("Elements")) + { + searching_for_elements = false; + break; + } + } // while + } + if (searching_for_elements) + { + std::cout << "FILE FORMAT ERROR" << std::endl; + } + + // read in element type from following line + getline(*in, read_line); + std::cout << read_line << std::endl; line_parse.clear(); line_parse.str(read_line); - - for(int iword = 0; iword < words_per_line; iword++){ - //read portions of the line into the substring variable line_parse >> substring; - //debug print - //std::cout<<" "<< substring <> substring; - //assign the substring variable as a word of the read buffer - strcpy(&read_buffer(buffer_loop,iword,0),substring.c_str()); + // parse element line out of jumble of comma delimited entries + line_parse.clear(); + line_parse.str(substring); + while (line_parse.good()) { + getline(line_parse, token, ','); + } + // element count should be the last token read in + num_elem = std::stoi(token); + + // skip line + for (int j = 0; j < 1; j++) + { + getline(*in, skip_line); + std::cout << skip_line << std::endl; } - buffer_loop++; - } - - } - - //broadcast buffer to all ranks; each rank will determine which nodes in the buffer belong - MPI_Bcast(read_buffer.pointer(),BUFFER_LINES*words_per_line*MAX_WORD,MPI_CHAR,0,world); - //broadcast how many nodes were read into this buffer iteration - MPI_Bcast(&buffer_loop,1,MPI_INT,0,world); - - //debug_print - //std::cout << "NODE BUFFER LOOP IS: " << buffer_loop << std::endl; - //for(int iprint=0; iprint < buffer_loop; iprint++) - //std::cout<<"buffer packing: " << std::string(&read_buffer(iprint,0,0)) << std::endl; - //return; - - //determine which data to store in the swage mesh members (the local node data) - //loop through read buffer - for(scan_loop = 0; scan_loop < buffer_loop; scan_loop++){ - //set global node id (ensight specific order) - node_gid = read_index_start + scan_loop; - //let map decide if this node id belongs locally; if yes store data - if(map->isNodeGlobalElement(node_gid)){ - //set local node index in this mpi rank - node_rid = map->getLocalElement(node_gid); - //extract nodal position from the read buffer - //for tecplot format this is the three coords in the same line - dof_value = atof(&read_buffer(scan_loop,1,0)); - node_coords(node_rid, 0) = dof_value * unit_scaling; - dof_value = atof(&read_buffer(scan_loop,2,0)); - node_coords(node_rid, 1) = dof_value * unit_scaling; - if(num_dim==3){ - dof_value = atof(&read_buffer(scan_loop,3,0)); - node_coords(node_rid, 2) = dof_value * unit_scaling; - } - //extract density if restarting - } } - read_index_start+=BUFFER_LINES; - } - } //end view scope - //repartition node distribution - repartition_nodes(); - - //synchronize device data - //dual_node_coords.sync_device(); - //dual_node_coords.modify_device(); - //if(restart_file){ - //dual_node_densities.sync_device(); - //dual_node_densities.modify_device(); - //} - - //debug print of nodal data - - //debug print nodal positions and indices - /* - std::cout << " ------------NODAL POSITIONS ON TASK " << myrank << " --------------"<getGlobalElement(inode) + 1 << " { "; - for (int istride = 0; istride < num_dim; istride++){ - std::cout << node_coords(inode,istride) << " , "; + + // broadcast element type + MPI_Bcast(&etype_index, 1, MPI_INT, 0, world); + + elements::elem_types::elem_type mesh_element_type; + int elem_words_per_line_no_nodes = elem_words_per_line; + + if (etype_index == 1) + { + mesh_element_type = elements::elem_types::Hex8; + nodes_per_element = 8; + elem_words_per_line += 8; + } + else if (etype_index == 2) + { + mesh_element_type = elements::elem_types::Hex20; + nodes_per_element = 20; + elem_words_per_line += 20; + } + else if (etype_index == 3) + { + mesh_element_type = elements::elem_types::Hex32; + nodes_per_element = 32; + elem_words_per_line += 32; + } + else + { + *fos << "ERROR: ANSYS ELEMENT TYPE NOT FOUND OR RECOGNIZED" << std::endl; + exit_solver(0); } - //std::cout << node_densities(inode,0); - std::cout << " }"<< std::endl; - } - */ - //check that local assignments match global total + // broadcast number of elements + MPI_Bcast(&num_elem, 1, MPI_LONG_LONG_INT, 0, world); - - //read in element info - //seek element connectivity zone - int etype_index = 0; - if(myrank==0){ - bool searching_for_elements = true; - //skip lines at the top with nonessential info; stop skipping when "Nodes for the whole assembly" string is reached - while (searching_for_elements&&in->good()) { - getline(*in, skip_line); - //std::cout << skip_line << std::endl; - line_parse.clear(); - line_parse.str(skip_line); - //stop when the NODES= string is reached - while (!line_parse.eof()){ - line_parse >> substring; - //std::cout << substring << std::endl; - if(!substring.compare("Elements")){ - searching_for_elements = false; - break; + *fos << "declared element count: " << num_elem << std::endl; + // std::cout<<"before initial mesh initialization"<(BUFFER_LINES, elem_words_per_line, MAX_WORD); + CArrayKokkos node_store(nodes_per_element); + + // calculate buffer iterations to read number of lines + buffer_iterations = num_elem / BUFFER_LINES; + int assign_flag; + + // dynamic buffer used to store elements before we know how many this rank needs + std::vector element_temp(BUFFER_LINES * elem_words_per_line); + std::vector global_indices_temp(BUFFER_LINES); + size_t buffer_max = BUFFER_LINES * elem_words_per_line; + size_t indices_buffer_max = BUFFER_LINES; + + if (num_elem % BUFFER_LINES != 0) + { + buffer_iterations++; + } + read_index_start = 0; + // std::cout << "ELEMENT BUFFER ITERATIONS: " << buffer_iterations << std::endl; + rnum_elem = 0; + // std::cout << "BUFFER ITERATIONS IS: " << buffer_iterations << std::endl; + for (buffer_iteration = 0; buffer_iteration < buffer_iterations; buffer_iteration++) + { + // pack buffer on rank 0 + if (myrank == 0 && buffer_iteration < buffer_iterations - 1) + { + for (buffer_loop = 0; buffer_loop < BUFFER_LINES; buffer_loop++) + { + getline(*in, read_line); + line_parse.clear(); + line_parse.str(read_line); + for (int iword = 0; iword < elem_words_per_line; iword++) + { + // read portions of the line into the substring variable + line_parse >> substring; + // assign the substring variable as a word of the read buffer + strcpy(&read_buffer(buffer_loop, iword, 0), substring.c_str()); + } + } + } + else if (myrank == 0) + { + buffer_loop = 0; + while (buffer_iteration * BUFFER_LINES + buffer_loop < num_elem) { + getline(*in, read_line); + line_parse.clear(); + line_parse.str(read_line); + for (int iword = 0; iword < elem_words_per_line; iword++) + { + // read portions of the line into the substring variable + line_parse >> substring; + // assign the substring variable as a word of the read buffer + strcpy(&read_buffer(buffer_loop, iword, 0), substring.c_str()); + } + buffer_loop++; + // std::cout<<" "<< node_coords(node_gid, 0)<isNodeGlobalElement(node_gid) && !assign_flag) + { + assign_flag = 1; + rnum_elem++; + } + } + else + { + if (map->isNodeGlobalElement(node_gid - 1) && !assign_flag) + { + assign_flag = 1; + rnum_elem++; + } + } + } + + if (assign_flag) + { + for (int inode = 0; inode < nodes_per_element; inode++) + { + if ((rnum_elem - 1) * nodes_per_element + inode >= buffer_max) + { + element_temp.resize((rnum_elem - 1) * nodes_per_element + inode + BUFFER_LINES * nodes_per_element); + buffer_max = (rnum_elem - 1) * nodes_per_element + inode + BUFFER_LINES * nodes_per_element; + } + element_temp[(rnum_elem - 1) * nodes_per_element + inode] = node_store(inode); + // std::cout << "VECTOR STORAGE FOR ELEM " << rnum_elem << " ON TASK " << myrank << " NODE " << inode+1 << " IS " << node_store(inode) + 1 << std::endl; + } + // assign global element id to temporary list + if (rnum_elem - 1 >= indices_buffer_max) + { + global_indices_temp.resize(rnum_elem - 1 + BUFFER_LINES); + indices_buffer_max = rnum_elem - 1 + BUFFER_LINES; + } + global_indices_temp[rnum_elem - 1] = elem_gid; + } } - } //while - + read_index_start += BUFFER_LINES; } - if(searching_for_elements){ - std::cout << "FILE FORMAT ERROR" << std::endl; + + // check if ANSYS file has boundary and loading condition zones + bool No_Conditions = true; + if (myrank == 0) + { + if (in->good()) + { + before_condition_header = in->tellg(); + } + bool searching_for_conditions = true; + // skip lines at the top with nonessential info; stop skipping when "Fixed Supports or Pressure" string is reached + while (searching_for_conditions && in->good()) { + getline(*in, skip_line); + // std::cout << skip_line << std::endl; + line_parse.clear(); + line_parse.str(skip_line); + // stop when the NODES= string is reached + while (!line_parse.eof()) { + line_parse >> substring; + // std::cout << substring << std::endl; + if (!substring.compare("Supports") || !substring.compare("Pressure")) + { + No_Conditions = searching_for_conditions = false; + break; + } + } // while + } // while } - //read in element type from following line - getline(*in, read_line); - std::cout << read_line << std::endl; - line_parse.clear(); - line_parse.str(read_line); - line_parse >> substring; - //std::cout << substring << std::endl; - if(!substring.compare("et,1,185")){ - //Hex8 type - etype_index = 1; + // broadcast search condition + MPI_Bcast(&No_Conditions, 1, MPI_CXX_BOOL, 0, world); + // flag elasticity fea module for boundary/loading conditions readin that remains + if (!No_Conditions) + { + // check that the input file has configured some kind of acceptable module + simparam.validate_module_is_specified(FEA_MODULE_TYPE::Elasticity); + simparam.fea_module_must_read.insert(FEA_MODULE_TYPE::Elasticity); } - else if(!substring.compare("et,1,186")){ - //Hex20 type - etype_index = 2; + + // Close mesh input file if no further readin is done by FEA modules for conditions + if (myrank == 0 && No_Conditions) + { + in->close(); } - else{ - etype_index = 0; - } - //for - - //seek element count line - //read in element count from the following line - getline(*in, read_line); - std::cout << read_line << std::endl; - line_parse.clear(); - line_parse.str(read_line); - line_parse >> substring; - //parse element line out of jumble of comma delimited entries - line_parse.clear(); - line_parse.str(substring); - while(line_parse.good()){ - getline(line_parse, token, ','); - } - //element count should be the last token read in - num_elem = std::stoi(token); - - //skip line - for (int j = 0; j < 1; j++) { - getline(*in, skip_line); - std::cout << skip_line << std::endl; - } - } - - //broadcast element type - MPI_Bcast(&etype_index,1,MPI_INT,0,world); - - elements::elem_types::elem_type mesh_element_type; - int elem_words_per_line_no_nodes = elem_words_per_line; - - if(etype_index==1){ - mesh_element_type = elements::elem_types::Hex8; - nodes_per_element = 8; - elem_words_per_line += 8; - } - else if(etype_index==2){ - mesh_element_type = elements::elem_types::Hex20; - nodes_per_element = 20; - elem_words_per_line += 20; - } - else if(etype_index==3){ - mesh_element_type = elements::elem_types::Hex32; - nodes_per_element = 32; - elem_words_per_line += 32; - } - else{ - *fos << "ERROR: ANSYS ELEMENT TYPE NOT FOUND OR RECOGNIZED" << std::endl; - exit_solver(0); - } - - //broadcast number of elements - MPI_Bcast(&num_elem,1,MPI_LONG_LONG_INT,0,world); - - *fos << "declared element count: " << num_elem << std::endl; - //std::cout<<"before initial mesh initialization"<(BUFFER_LINES,elem_words_per_line,MAX_WORD); - CArrayKokkos node_store(nodes_per_element); - - //calculate buffer iterations to read number of lines - buffer_iterations = num_elem/BUFFER_LINES; - int assign_flag; - - //dynamic buffer used to store elements before we know how many this rank needs - std::vector element_temp(BUFFER_LINES*elem_words_per_line); - std::vector global_indices_temp(BUFFER_LINES); - size_t buffer_max = BUFFER_LINES*elem_words_per_line; - size_t indices_buffer_max = BUFFER_LINES; - - if(num_elem%BUFFER_LINES!=0) buffer_iterations++; - read_index_start = 0; - //std::cout << "ELEMENT BUFFER ITERATIONS: " << buffer_iterations << std::endl; - rnum_elem = 0; - //std::cout << "BUFFER ITERATIONS IS: " << buffer_iterations << std::endl; - for(buffer_iteration = 0; buffer_iteration < buffer_iterations; buffer_iteration++){ - //pack buffer on rank 0 - if(myrank==0&&buffer_iteration> substring; - //assign the substring variable as a word of the read buffer - strcpy(&read_buffer(buffer_loop,iword,0),substring.c_str()); - } - } + + std::cout << "RNUM ELEMENTS IS: " << rnum_elem << std::endl; + // copy temporary element storage to multivector storage + Element_Types = CArrayKokkos(rnum_elem); + + // set element object pointer + if (simparam.num_dims == 2) + { + element_select->choose_2Delem_type(mesh_element_type, elem2D); + max_nodes_per_element = elem2D->num_nodes(); } - else if(myrank==0){ - buffer_loop=0; - while(buffer_iteration*BUFFER_LINES+buffer_loop < num_elem) { - getline(*in,read_line); - line_parse.clear(); - line_parse.str(read_line); - for(int iword = 0; iword < elem_words_per_line; iword++){ - //read portions of the line into the substring variable - line_parse >> substring; - //assign the substring variable as a word of the read buffer - strcpy(&read_buffer(buffer_loop,iword,0),substring.c_str()); - } - buffer_loop++; - //std::cout<<" "<< node_coords(node_gid, 0)<choose_3Delem_type(mesh_element_type, elem); + max_nodes_per_element = elem->num_nodes(); } - //broadcast buffer to all ranks; each rank will determine which nodes in the buffer belong - MPI_Bcast(read_buffer.pointer(),BUFFER_LINES*elem_words_per_line*MAX_WORD,MPI_CHAR,0,world); - //broadcast how many nodes were read into this buffer iteration - MPI_Bcast(&buffer_loop,1,MPI_INT,0,world); - - //store element connectivity that belongs to this rank - //loop through read buffer - for(scan_loop = 0; scan_loop < buffer_loop; scan_loop++){ - //set global node id (ensight specific order) - elem_gid = read_index_start + scan_loop; - //add this element to the local list if any of its nodes belong to this rank according to the map - //get list of nodes for each element line and check if they belong to the map - assign_flag = 0; - for(int inode = elem_words_per_line_no_nodes; inode < elem_words_per_line; inode++){ - //as we loop through the nodes belonging to this element we store them - //if any of these nodes belongs to this rank this list is used to store the element locally - node_gid = atoi(&read_buffer(scan_loop,inode,0)); - if(zero_index_base) - node_store(inode-elem_words_per_line_no_nodes) = node_gid; //subtract 1 since file index start is 1 but code expects 0 - else - node_store(inode-elem_words_per_line_no_nodes) = node_gid - 1; //subtract 1 since file index start is 1 but code expects 0 - if(node_store(inode-elem_words_per_line_no_nodes) < 0){ - negative_index_found = 1; - } - //first we add the elements to a dynamically allocated list - if(zero_index_base){ - if(map->isNodeGlobalElement(node_gid)&&!assign_flag){ - assign_flag = 1; - rnum_elem++; - } - } - else{ - if(map->isNodeGlobalElement(node_gid-1)&&!assign_flag){ - assign_flag = 1; - rnum_elem++; - } + // 1 type per mesh for now + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + Element_Types(ielem) = mesh_element_type; + } + + dual_nodes_in_elem = dual_elem_conn_array("dual_nodes_in_elem", rnum_elem, max_nodes_per_element); + host_elem_conn_array nodes_in_elem = dual_nodes_in_elem.view_host(); + dual_nodes_in_elem.modify_host(); + + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + for (int inode = 0; inode < nodes_per_element; inode++) + { + nodes_in_elem(ielem, inode) = element_temp[ielem * nodes_per_element + inode]; } - } + } - if(assign_flag){ - for(int inode = 0; inode < nodes_per_element; inode++){ - if((rnum_elem-1)*nodes_per_element + inode>=buffer_max){ - element_temp.resize((rnum_elem-1)*nodes_per_element + inode + BUFFER_LINES*nodes_per_element); - buffer_max = (rnum_elem-1)*nodes_per_element + inode + BUFFER_LINES*nodes_per_element; - } - element_temp[(rnum_elem-1)*nodes_per_element + inode] = node_store(inode); - //std::cout << "VECTOR STORAGE FOR ELEM " << rnum_elem << " ON TASK " << myrank << " NODE " << inode+1 << " IS " << node_store(inode) + 1 << std::endl; + // view storage for all local elements connected to local nodes on this rank + Kokkos::DualView All_Element_Global_Indices("All_Element_Global_Indices", rnum_elem); + // copy temporary global indices storage to view storage + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + All_Element_Global_Indices.h_view(ielem) = global_indices_temp[ielem]; + if (global_indices_temp[ielem] < 0) + { + negative_index_found = 1; } - //assign global element id to temporary list - if(rnum_elem-1>=indices_buffer_max){ - global_indices_temp.resize(rnum_elem-1 + BUFFER_LINES); - indices_buffer_max = rnum_elem-1 + BUFFER_LINES; + } + + MPI_Allreduce(&negative_index_found, &global_negative_index_found, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD); + if (global_negative_index_found) + { + if (myrank == 0) + { + std::cout << "Node index less than or equal to zero detected; set \"zero_index_base: true\" under \"input_options\" in your yaml file if indices start at 0" << std::endl; } - global_indices_temp[rnum_elem-1] = elem_gid; - } + exit_solver(0); } - read_index_start+=BUFFER_LINES; - } - - //check if ANSYS file has boundary and loading condition zones - bool No_Conditions = true; - if(myrank==0){ - if(in->good()) - before_condition_header = in->tellg(); - bool searching_for_conditions = true; - //skip lines at the top with nonessential info; stop skipping when "Fixed Supports or Pressure" string is reached - while (searching_for_conditions&&in->good()) { - getline(*in, skip_line); - //std::cout << skip_line << std::endl; - line_parse.clear(); - line_parse.str(skip_line); - //stop when the NODES= string is reached - while (!line_parse.eof()){ - line_parse >> substring; - //std::cout << substring << std::endl; - if(!substring.compare("Supports")||!substring.compare("Pressure")){ - No_Conditions = searching_for_conditions = false; - break; - } - } //while - - } //while - } - - //broadcast search condition - MPI_Bcast(&No_Conditions,1,MPI_CXX_BOOL,0,world); - - //flag elasticity fea module for boundary/loading conditions readin that remains - if(!No_Conditions){ - // check that the input file has configured some kind of acceptable module - simparam.validate_module_is_specified(FEA_MODULE_TYPE::Elasticity); - simparam.fea_module_must_read.insert(FEA_MODULE_TYPE::Elasticity); - } - - // Close mesh input file if no further readin is done by FEA modules for conditions - if(myrank==0&&No_Conditions){ - in->close(); - } - - std::cout << "RNUM ELEMENTS IS: " << rnum_elem << std::endl; - //copy temporary element storage to multivector storage - Element_Types = CArrayKokkos(rnum_elem); - - //set element object pointer - if(simparam.num_dims==2){ - element_select->choose_2Delem_type(mesh_element_type, elem2D); - max_nodes_per_element = elem2D->num_nodes(); - } - else if(simparam.num_dims==3){ - element_select->choose_3Delem_type(mesh_element_type, elem); - max_nodes_per_element = elem->num_nodes(); - } - - //1 type per mesh for now - for(int ielem = 0; ielem < rnum_elem; ielem++) - Element_Types(ielem) = mesh_element_type; - - dual_nodes_in_elem = dual_elem_conn_array("dual_nodes_in_elem", rnum_elem, max_nodes_per_element); - host_elem_conn_array nodes_in_elem = dual_nodes_in_elem.view_host(); - dual_nodes_in_elem.modify_host(); - - for(int ielem = 0; ielem < rnum_elem; ielem++) - for(int inode = 0; inode < nodes_per_element; inode++){ - nodes_in_elem(ielem, inode) = element_temp[ielem*nodes_per_element + inode]; - } - - //view storage for all local elements connected to local nodes on this rank - Kokkos::DualView All_Element_Global_Indices("All_Element_Global_Indices",rnum_elem); - //copy temporary global indices storage to view storage - for(int ielem = 0; ielem < rnum_elem; ielem++){ - All_Element_Global_Indices.h_view(ielem) = global_indices_temp[ielem]; - if(global_indices_temp[ielem]<0){ - negative_index_found = 1; - } - } - - MPI_Allreduce(&negative_index_found,&global_negative_index_found,1,MPI_INT,MPI_MAX,MPI_COMM_WORLD); - if(global_negative_index_found){ - if(myrank==0){ - std::cout << "Node index less than or equal to zero detected; set \"zero_index_base: true\" under \"input_options\" in your yaml file if indices start at 0" << std::endl; + + // delete temporary element connectivity and index storage + std::vector().swap(element_temp); + std::vector().swap(global_indices_temp); + + // construct overlapping element map (since different ranks can own the same elements due to the local node map) + All_Element_Global_Indices.modify_host(); + All_Element_Global_Indices.sync_device(); + + all_element_map = Teuchos::rcp(new Tpetra::Map(Teuchos::OrdinalTraits::invalid(), All_Element_Global_Indices.d_view, 0, comm)); + + // element type selection (subject to change) + // ---- Set Element Type ---- // + // allocate element type memory + // elements::elem_type_t* elem_choice; + + int NE = 1; // number of element types in problem + + // Convert ijk index system to the finite element numbering convention + // for vertices in cell + CArrayKokkos convert_ensight_to_ijk(max_nodes_per_element); + CArrayKokkos tmp_ijk_indx(max_nodes_per_element); + convert_ensight_to_ijk(0) = 0; + convert_ensight_to_ijk(1) = 1; + convert_ensight_to_ijk(2) = 3; + convert_ensight_to_ijk(3) = 2; + convert_ensight_to_ijk(4) = 4; + convert_ensight_to_ijk(5) = 5; + convert_ensight_to_ijk(6) = 7; + convert_ensight_to_ijk(7) = 6; + + if (num_dim == 2) + { + for (int cell_rid = 0; cell_rid < rnum_elem; cell_rid++) + { + // set nodes per element + element_select->choose_2Delem_type(Element_Types(cell_rid), elem2D); + nodes_per_element = elem2D->num_nodes(); + for (int node_lid = 0; node_lid < nodes_per_element; node_lid++) + { + tmp_ijk_indx(node_lid) = nodes_in_elem(cell_rid, convert_ensight_to_ijk(node_lid)); + } + + for (int node_lid = 0; node_lid < nodes_per_element; node_lid++) + { + nodes_in_elem(cell_rid, node_lid) = tmp_ijk_indx(node_lid); + } + } } - exit_solver(0); - } - //delete temporary element connectivity and index storage - std::vector().swap(element_temp); - std::vector().swap(global_indices_temp); - - //construct overlapping element map (since different ranks can own the same elements due to the local node map) - All_Element_Global_Indices.modify_host(); - All_Element_Global_Indices.sync_device(); - - all_element_map = Teuchos::rcp( new Tpetra::Map(Teuchos::OrdinalTraits::invalid(),All_Element_Global_Indices.d_view,0,comm)); - - - //element type selection (subject to change) - // ---- Set Element Type ---- // - // allocate element type memory - //elements::elem_type_t* elem_choice; - - int NE = 1; // number of element types in problem - - // Convert ijk index system to the finite element numbering convention - // for vertices in cell - CArrayKokkos convert_ensight_to_ijk(max_nodes_per_element); - CArrayKokkos tmp_ijk_indx(max_nodes_per_element); - convert_ensight_to_ijk(0) = 0; - convert_ensight_to_ijk(1) = 1; - convert_ensight_to_ijk(2) = 3; - convert_ensight_to_ijk(3) = 2; - convert_ensight_to_ijk(4) = 4; - convert_ensight_to_ijk(5) = 5; - convert_ensight_to_ijk(6) = 7; - convert_ensight_to_ijk(7) = 6; - - if(num_dim==2) - for (int cell_rid = 0; cell_rid < rnum_elem; cell_rid++) { - //set nodes per element - element_select->choose_2Delem_type(Element_Types(cell_rid), elem2D); - nodes_per_element = elem2D->num_nodes(); - for (int node_lid = 0; node_lid < nodes_per_element; node_lid++){ - tmp_ijk_indx(node_lid) = nodes_in_elem(cell_rid, convert_ensight_to_ijk(node_lid)); - } - - for (int node_lid = 0; node_lid < nodes_per_element; node_lid++){ - nodes_in_elem(cell_rid, node_lid) = tmp_ijk_indx(node_lid); - } - } - - if(num_dim==3) - for (int cell_rid = 0; cell_rid < rnum_elem; cell_rid++) { - //set nodes per element - element_select->choose_3Delem_type(Element_Types(cell_rid), elem); - nodes_per_element = elem->num_nodes(); - for (int node_lid = 0; node_lid < nodes_per_element; node_lid++){ - tmp_ijk_indx(node_lid) = nodes_in_elem(cell_rid, convert_ensight_to_ijk(node_lid)); - } - - for (int node_lid = 0; node_lid < nodes_per_element; node_lid++){ - nodes_in_elem(cell_rid, node_lid) = tmp_ijk_indx(node_lid); - } - } - + if (num_dim == 3) + { + for (int cell_rid = 0; cell_rid < rnum_elem; cell_rid++) + { + // set nodes per element + element_select->choose_3Delem_type(Element_Types(cell_rid), elem); + nodes_per_element = elem->num_nodes(); + for (int node_lid = 0; node_lid < nodes_per_element; node_lid++) + { + tmp_ijk_indx(node_lid) = nodes_in_elem(cell_rid, convert_ensight_to_ijk(node_lid)); + } + + for (int node_lid = 0; node_lid < nodes_per_element; node_lid++) + { + nodes_in_elem(cell_rid, node_lid) = tmp_ijk_indx(node_lid); + } + } + } } // end read_mesh /* ---------------------------------------------------------------------- Initialize Distributed State Vectors Pertinent to this Solver ------------------------------------------------------------------------- */ -void Explicit_Solver::init_state_vectors(){ - int num_dim = simparam.num_dims; - //allocate node_velocities - //node_velocities_distributed = Teuchos::rcp(new MV(map, num_dim)); - initial_node_coords_distributed = Teuchos::rcp(new MV(map, num_dim)); - all_initial_node_coords_distributed = Teuchos::rcp(new MV(all_node_map, num_dim)); - initial_node_velocities_distributed = Teuchos::rcp(new MV(map, num_dim)); - all_node_velocities_distributed = Teuchos::rcp(new MV(all_node_map, num_dim)); - node_velocities_distributed = Teuchos::rcp(new MV(*all_node_velocities_distributed, map)); - //ghost_node_velocities_distributed = Teuchos::rcp(new MV(ghost_node_map, num_dim)); - ghost_node_velocities_distributed = Teuchos::rcp(new MV(*all_node_velocities_distributed, ghost_node_map, nlocal_nodes)); - if(simparam.topology_optimization_on){ - test_node_densities_distributed = Teuchos::rcp(new MV(map, 1)); - } - if(simparam.topology_optimization_on || simparam.shape_optimization_on){ - corner_value_storage = CArrayKokkos(rnum_elem*max_nodes_per_element); - corner_vector_storage = CArrayKokkos(rnum_elem*max_nodes_per_element,num_dim); - corner_gradient_storage = CArrayKokkos(rnum_elem*max_nodes_per_element, num_dim, max_nodes_per_element, num_dim); - } - all_node_densities_distributed = Teuchos::rcp(new MV(all_node_map, 1)); - Global_Element_Densities = Teuchos::rcp(new MV(all_element_map, 1)); +void Explicit_Solver::init_state_vectors() +{ + int num_dim = simparam.num_dims; + // allocate node_velocities + // node_velocities_distributed = Teuchos::rcp(new MV(map, num_dim)); + initial_node_coords_distributed = Teuchos::rcp(new MV(map, num_dim)); + all_initial_node_coords_distributed = Teuchos::rcp(new MV(all_node_map, num_dim)); + initial_node_velocities_distributed = Teuchos::rcp(new MV(map, num_dim)); + all_node_velocities_distributed = Teuchos::rcp(new MV(all_node_map, num_dim)); + node_velocities_distributed = Teuchos::rcp(new MV(*all_node_velocities_distributed, map)); + // ghost_node_velocities_distributed = Teuchos::rcp(new MV(ghost_node_map, num_dim)); + ghost_node_velocities_distributed = Teuchos::rcp(new MV(*all_node_velocities_distributed, ghost_node_map, nlocal_nodes)); + if (simparam.topology_optimization_on) + { + test_node_densities_distributed = Teuchos::rcp(new MV(map, 1)); + } + if (simparam.topology_optimization_on || simparam.shape_optimization_on) + { + corner_value_storage = CArrayKokkos(rnum_elem * max_nodes_per_element); + corner_vector_storage = CArrayKokkos(rnum_elem * max_nodes_per_element, num_dim); + corner_gradient_storage = CArrayKokkos(rnum_elem * max_nodes_per_element, num_dim, max_nodes_per_element, num_dim); + } + all_node_densities_distributed = Teuchos::rcp(new MV(all_node_map, 1)); + Global_Element_Densities = Teuchos::rcp(new MV(all_element_map, 1)); } /* ---------------------------------------------------------------------- - Construct list of objects for FEA modules + Construct list of objects for FEA modules ------------------------------------------------------------------------- */ -void Explicit_Solver::FEA_module_setup(){ - nfea_modules = simparam.fea_module_parameters.size(); - fea_module_must_read = simparam.fea_module_must_read; - - //allocate lists to size - fea_module_types = std::vector(); - fea_modules = std::vector(); - - for (auto& param : simparam.fea_module_parameters) { - fea_module_types.push_back(param->type); - param->apply( - [&](SGH_Parameters& param) { - sgh_module = new FEA_Module_SGH(param, this, mesh); - fea_modules.push_back(sgh_module); +void Explicit_Solver::FEA_module_setup() +{ + nfea_modules = simparam.fea_module_parameters.size(); + fea_module_must_read = simparam.fea_module_must_read; + + // allocate lists to size + fea_module_types = std::vector(); + fea_modules = std::vector(); + + for (auto& param : simparam.fea_module_parameters) + { + fea_module_types.push_back(param->type); + param->apply( + [&](SGH_Parameters& param) { + sgh_module = new FEA_Module_SGH(param, this, mesh); + fea_modules.push_back(sgh_module); }, - [&](Dynamic_Elasticity_Parameters& param) { - fea_modules.push_back(new FEA_Module_Dynamic_Elasticity(param, this, mesh)); + [&](Dynamic_Elasticity_Parameters& param) { + fea_modules.push_back(new FEA_Module_Dynamic_Elasticity(param, this, mesh)); }, - [&](Inertial_Parameters& param) { - fea_modules.push_back(new FEA_Module_Inertial(param, this)); + [&](Inertial_Parameters& param) { + fea_modules.push_back(new FEA_Module_Inertial(param, this)); }, - [&](const FEA_Module_Parameters& param) { - *fos << "PROGRAM IS ENDING DUE TO ERROR; UNDEFINED FEA MODULE REQUESTED WITH NAME \"" - << param.type <<"\"" << std::endl; - exit_solver(0); + [&](const FEA_Module_Parameters& param) { + *fos << "PROGRAM IS ENDING DUE TO ERROR; UNDEFINED FEA MODULE REQUESTED WITH NAME \"" + << param.type << "\"" << std::endl; + exit_solver(0); } - ); + ); - *fos << " " << fea_module_types.back() << " MODULE ALLOCATED AS " << fea_module_types.size() - 1 << std::endl; - } + *fos << " " << fea_module_types.back() << " MODULE ALLOCATED AS " << fea_module_types.size() - 1 << std::endl; + } } /* ---------------------------------------------------------------------- Setup Optimization Problem Object, Relevant Objective, and Constraints ------------------------------------------------------------------------- */ -void Explicit_Solver::setup_optimization_problem(){ - int num_dim = simparam.num_dims; - bool nodal_density_flag = simparam.nodal_density_flag; - int nTO_modules = simparam.TO_Module_List.size(); - //int nmulti_objective_modules = simparam->nmulti_objective_modules; - std::vector TO_Module_List = simparam.TO_Module_List; - std::vector TO_Module_My_FEA_Module = simparam.TO_Module_My_FEA_Module; - //std::vector Multi_Objective_Modules = simparam->Multi_Objective_Modules; - //std::vector Multi_Objective_Weights = simparam->Multi_Objective_Weights; - std::vector> Function_Arguments = simparam.Function_Arguments; - std::vector TO_Function_Type = simparam.TO_Function_Type; - std::vector>> Multi_Objective_Terms; - - std::string constraint_base, constraint_name; - std::stringstream number_union; - CArray Surface_Nodes; - GO current_node_index, current_element_index; - LO local_node_index, local_element_id; - int num_bdy_patches_in_set; - size_t node_id, patch_id, module_id; - int num_boundary_sets; - int local_surface_id; - const_host_vec_array design_densities; - typedef ROL::TpetraMultiVector ROL_MV; - const_host_elem_conn_array nodes_in_elem = global_nodes_in_elem_distributed->getLocalView (Tpetra::Access::ReadOnly); - - // fill parameter list with desired algorithmic options or leave as default - // Read optimization input parameter list. - std::string filename = "optimization_parameters.xml"; - auto parlist = ROL::getParametersFromXmlFile( filename ); - //ROL::ParameterList parlist; - - //Design variables to optimize - ROL::Ptr> x; - if(nodal_density_flag){ - x = ROL::makePtr>(design_node_densities_distributed); - } - else - x = ROL::makePtr>(Global_Element_Densities); - - //Instantiate (the one) objective function for the problem - ROL::Ptr> obj; - bool objective_declared = false; - for(int imodule = 0; imodule < nTO_modules; imodule++){ - if(TO_Function_Type[imodule] == FUNCTION_TYPE::OBJECTIVE){ - //check if previous module already defined an objective, there must be one objective module - if(objective_declared){ - // TODO: Put this validation earlier. - *fos << "PROGRAM IS ENDING DUE TO ERROR; ANOTHER OBJECTIVE FUNCTION WITH NAME \"" - << TO_Module_List[imodule] <<"\" ATTEMPTED TO REPLACE A PREVIOUS OBJECTIVE; THERE MUST BE ONE OBJECTIVE." << std::endl; - exit_solver(0); - } - if(TO_Module_List[imodule] == TO_MODULE_TYPE::Kinetic_Energy_Minimize){ - //debug print - *fos << " KINETIC ENERGY OBJECTIVE EXPECTS FEA MODULE INDEX " <(this, nodal_density_flag); - } - /* - else if(TO_Module_List[imodule] == "Heat_Capacity_Potential_Minimize"){ - //debug print - *fos << " HEAT CAPACITY POTENTIAL OBJECTIVE EXPECTS FEA MODULE INDEX " <(fea_modules[TO_Module_My_FEA_Module[imodule]], nodal_density_flag); - } - */ - //Multi-Objective case - /* - else if(TO_Module_List[imodule] == "Multi_Objective"){ - //allocate vector of Objective Functions to pass - Multi_Objective_Terms = std::vector>>(nmulti_objective_modules); - for(int imulti = 0; imulti < nmulti_objective_modules; imulti++){ - //get module index for objective term - module_id = Multi_Objective_Modules[imulti]; - if(TO_Module_List[module_id] == "Strain_Energy_Minimize"){ - //debug print - *fos << " STRAIN ENERGY OBJECTIVE EXPECTS FEA MODULE INDEX " <(fea_modules[TO_Module_My_FEA_Module[module_id]], nodal_density_flag); - } - else if(TO_Module_List[module_id] == "Heat_Capacity_Potential_Minimize"){ - //debug print - *fos << " HEAT CAPACITY POTENTIAL OBJECTIVE EXPECTS FEA MODULE INDEX " <(fea_modules[TO_Module_My_FEA_Module[module_id]], nodal_density_flag); - } - } - //allocate multi objective function - obj = ROL::makePtr(Multi_Objective_Terms, Multi_Objective_Weights); - } - */ - else{ - // TODO: Put validation earlier - *fos << "PROGRAM IS ENDING DUE TO ERROR; UNDEFINED OBJECTIVE FUNCTION REQUESTED WITH NAME \"" - << TO_Module_List[imodule] << "\"" << std::endl; - exit_solver(0); - } - objective_declared = true; +void Explicit_Solver::setup_optimization_problem() +{ + int num_dim = simparam.num_dims; + bool nodal_density_flag = simparam.nodal_density_flag; + int nTO_modules = simparam.TO_Module_List.size(); + // int nmulti_objective_modules = simparam->nmulti_objective_modules; + std::vector TO_Module_List = simparam.TO_Module_List; + std::vector TO_Module_My_FEA_Module = simparam.TO_Module_My_FEA_Module; + // std::vector Multi_Objective_Modules = simparam->Multi_Objective_Modules; + // std::vector Multi_Objective_Weights = simparam->Multi_Objective_Weights; + std::vector> Function_Arguments = simparam.Function_Arguments; + std::vector TO_Function_Type = simparam.TO_Function_Type; + std::vector>> Multi_Objective_Terms; + + std::string constraint_base, constraint_name; + std::stringstream number_union; + CArray Surface_Nodes; + GO current_node_index, current_element_index; + LO local_node_index, local_element_id; + int num_bdy_patches_in_set; + size_t node_id, patch_id, module_id; + int num_boundary_sets; + int local_surface_id; + const_host_vec_array design_densities; + typedef ROL::TpetraMultiVector ROL_MV; + const_host_elem_conn_array nodes_in_elem = global_nodes_in_elem_distributed->getLocalView(Tpetra::Access::ReadOnly); + + // fill parameter list with desired algorithmic options or leave as default + // Read optimization input parameter list. + std::string filename = "optimization_parameters.xml"; + auto parlist = ROL::getParametersFromXmlFile(filename); + // ROL::ParameterList parlist; + + // Design variables to optimize + ROL::Ptr> x; + if (nodal_density_flag) + { + x = ROL::makePtr>(design_node_densities_distributed); } - } - - //optimization problem interface that can have constraints added to it before passing to solver object - problem = ROL::makePtr>(obj,x); - - //ROL::Ptr> lin_icon = ROL::makePtr>(); - //ROL::Ptr> lin_imul = ROL::makePtr>(); - //ROL::Ptr> lin_ibnd = ROL::makePtr>(); - //problem.addLinearConstraint("Linear Inequality Constraint",lin_icon,lin_imul,lin_ibnd); - - // TypeG (generally constrained) specification - //ROL::Ptr> econ = ROL::makePtr>(); - //ROL::Ptr> emul = ROL::makePtr>(); - //problem.addConstraint("Equality Constraint",econ,emul); - - //ROL::Ptr> ineq_constraint = ROL::makePtr(this, nodal_density_flag); - //ROL::Ptr> lin_econ = ROL::makePtr>(); - //ROL::Ptr lin_emul = ROL::makePtr>(); - //problem.addLinearConstraint("Linear Equality Constraint",lin_econ,lin_mul); - - //ROL::Ptr> ineq_constraint = ROL::makePtr(fea_elasticity, nodal_density_flag); - //problem->addConstraint("Inequality Constraint",ineq_constraint,constraint_mul,constraint_bnd); - //problem->addConstraint("equality Constraint 2",eq_constraint2,constraint_mul2); - //problem->addConstraint("equality Constraint 3",eq_constraint3,constraint_mul3); - //problem->addLinearConstraint("Equality Constraint",eq_constraint,constraint_mul); - - for(int imodule = 0; imodule < nTO_modules; imodule++){ - number_union.str(constraint_base); - number_union << imodule + 1; - constraint_name = number_union.str(); - ROL::Ptr > li_ptr = ROL::makePtr>(1,0.0); - ROL::Ptr > constraint_mul = ROL::makePtr>(li_ptr); - if(TO_Function_Type[imodule] == FUNCTION_TYPE::EQUALITY_CONSTRAINT){ - //pointers are reference counting - ROL::Ptr> eq_constraint; - if(TO_Module_List[imodule]==TO_MODULE_TYPE::Mass_Constraint){ - - *fos << " MASS CONSTRAINT EXPECTS FEA MODULE INDEX " <(fea_modules[TO_Module_My_FEA_Module[imodule]], nodal_density_flag, Function_Arguments[imodule][0], false, true); - } - else if(TO_Module_List[imodule]==TO_MODULE_TYPE::Moment_of_Inertia_Constraint){ - *fos << " MOMENT OF INERTIA CONSTRAINT EXPECTS FEA MODULE INDEX " <(fea_modules[TO_Module_My_FEA_Module[imodule]], nodal_density_flag, Function_Arguments[imodule][1], Function_Arguments[imodule][0], false, true); - } - else{ - // TODO: Put validation earlier - *fos << "PROGRAM IS ENDING DUE TO ERROR; UNDEFINED EQUALITY CONSTRAINT FUNCTION REQUESTED WITH NAME \"" - << TO_Module_List[imodule] <<"\"" << std::endl; - exit_solver(0); - } - *fos << " ADDING CONSTRAINT " << constraint_name << std::endl; - problem->addConstraint(constraint_name, eq_constraint, constraint_mul); - } - - if(TO_Function_Type[imodule] == FUNCTION_TYPE::INEQUALITY_CONSTRAINT){ - //pointers are reference counting - ROL::Ptr> ineq_constraint; - ROL::Ptr > ll_ptr = ROL::makePtr>(1,Function_Arguments[imodule][0]); - ROL::Ptr > lu_ptr = ROL::makePtr>(1,Function_Arguments[imodule][1]); - ROL::Ptr > ll = ROL::makePtr>(ll_ptr); - ROL::Ptr > lu = ROL::makePtr>(lu_ptr); - ROL::Ptr> constraint_bnd = ROL::makePtr>(ll,lu); - if(TO_Module_List[imodule]==TO_MODULE_TYPE::Mass_Constraint){ - *fos << " MASS CONSTRAINT EXPECTS FEA MODULE INDEX " <(fea_modules[TO_Module_My_FEA_Module[imodule]], nodal_density_flag, true, true); - } - else if(TO_Module_List[imodule]==TO_MODULE_TYPE::Moment_of_Inertia_Constraint){ - *fos << " MOMENT OF INERTIA CONSTRAINT EXPECTS FEA MODULE INDEX " <(fea_modules[TO_Module_My_FEA_Module[imodule]], nodal_density_flag, Function_Arguments[imodule][1], Function_Arguments[imodule][0], true, true); - } - else{ - // TODO: Put this validation earlier - *fos << "PROGRAM IS ENDING DUE TO ERROR; UNDEFINED INEQUALITY CONSTRAINT FUNCTION REQUESTED WITH NAME \"" - << TO_Module_List[imodule] << "\"" << std::endl; - exit_solver(0); - } - *fos << " ADDING CONSTRAINT " << constraint_name << std::endl; - problem->addConstraint(constraint_name, ineq_constraint, constraint_mul, constraint_bnd); + else + { + x = ROL::makePtr>(Global_Element_Densities); } - number_union.clear(); - } - //set bounds on design variables - if(nodal_density_flag){ - dual_vec_array dual_node_densities_upper_bound = dual_vec_array("dual_node_densities_upper_bound", nlocal_nodes, 1); - dual_vec_array dual_node_densities_lower_bound = dual_vec_array("dual_node_densities_lower_bound", nlocal_nodes, 1); - host_vec_array node_densities_upper_bound = dual_node_densities_upper_bound.view_host(); - host_vec_array node_densities_lower_bound = dual_node_densities_lower_bound.view_host(); - //notify that the host view is going to be modified in the file readin - dual_node_densities_upper_bound.modify_host(); - dual_node_densities_lower_bound.modify_host(); + // Instantiate (the one) objective function for the problem + ROL::Ptr> obj; + bool objective_declared = false; + for (int imodule = 0; imodule < nTO_modules; imodule++) + { + if (TO_Function_Type[imodule] == FUNCTION_TYPE::OBJECTIVE) + { + // check if previous module already defined an objective, there must be one objective module + if (objective_declared) + { + // TODO: Put this validation earlier. + *fos << "PROGRAM IS ENDING DUE TO ERROR; ANOTHER OBJECTIVE FUNCTION WITH NAME \"" + << TO_Module_List[imodule] << "\" ATTEMPTED TO REPLACE A PREVIOUS OBJECTIVE; THERE MUST BE ONE OBJECTIVE." << std::endl; + exit_solver(0); + } + if (TO_Module_List[imodule] == TO_MODULE_TYPE::Kinetic_Energy_Minimize) + { + // debug print + *fos << " KINETIC ENERGY OBJECTIVE EXPECTS FEA MODULE INDEX " << TO_Module_My_FEA_Module[imodule] << std::endl; + obj = ROL::makePtr(this, nodal_density_flag); + } + /* + else if(TO_Module_List[imodule] == "Heat_Capacity_Potential_Minimize"){ + //debug print + *fos << " HEAT CAPACITY POTENTIAL OBJECTIVE EXPECTS FEA MODULE INDEX " <(fea_modules[TO_Module_My_FEA_Module[imodule]], nodal_density_flag); + } + */ + // Multi-Objective case + /* + else if(TO_Module_List[imodule] == "Multi_Objective"){ + //allocate vector of Objective Functions to pass + Multi_Objective_Terms = std::vector>>(nmulti_objective_modules); + for(int imulti = 0; imulti < nmulti_objective_modules; imulti++){ + //get module index for objective term + module_id = Multi_Objective_Modules[imulti]; + if(TO_Module_List[module_id] == "Strain_Energy_Minimize"){ + //debug print + *fos << " STRAIN ENERGY OBJECTIVE EXPECTS FEA MODULE INDEX " <(fea_modules[TO_Module_My_FEA_Module[module_id]], nodal_density_flag); + } + else if(TO_Module_List[module_id] == "Heat_Capacity_Potential_Minimize"){ + //debug print + *fos << " HEAT CAPACITY POTENTIAL OBJECTIVE EXPECTS FEA MODULE INDEX " <(fea_modules[TO_Module_My_FEA_Module[module_id]], nodal_density_flag); + } + } + //allocate multi objective function + obj = ROL::makePtr(Multi_Objective_Terms, Multi_Objective_Weights); + } + */ + else + { + // TODO: Put validation earlier + *fos << "PROGRAM IS ENDING DUE TO ERROR; UNDEFINED OBJECTIVE FUNCTION REQUESTED WITH NAME \"" + << TO_Module_List[imodule] << "\"" << std::endl; + exit_solver(0); + } + objective_declared = true; + } + } - //initialize densities to 1 for now; in the future there might be an option to read in an initial condition for each node - for(int inode = 0; inode < nlocal_nodes; inode++){ - node_densities_upper_bound(inode,0) = 1; - node_densities_lower_bound(inode,0) = simparam.optimization_options.density_epsilon; + // optimization problem interface that can have constraints added to it before passing to solver object + problem = ROL::makePtr>(obj, x); + + // ROL::Ptr> lin_icon = ROL::makePtr>(); + // ROL::Ptr> lin_imul = ROL::makePtr>(); + // ROL::Ptr> lin_ibnd = ROL::makePtr>(); + // problem.addLinearConstraint("Linear Inequality Constraint",lin_icon,lin_imul,lin_ibnd); + + // TypeG (generally constrained) specification + // ROL::Ptr> econ = ROL::makePtr>(); + // ROL::Ptr> emul = ROL::makePtr>(); + // problem.addConstraint("Equality Constraint",econ,emul); + + //debug checks + ROL::Ptr> rol_x = + ROL::makePtr>(design_node_densities_distributed); + //construct direction vector for check + Teuchos::RCP directions_distributed = Teuchos::rcp(new MV(map, 1)); + directions_distributed->putScalar(-0.1); + directions_distributed->randomize(-0.8,1); + Kokkos::View direction_norm("gradient norm",1); + directions_distributed->norm2(direction_norm); + directions_distributed->scale(1/direction_norm(0)); + //set all but first component to 0 for debug + host_vec_array directions = directions_distributed->getLocalView (Tpetra::Access::ReadWrite); + //for(int init = 1; init < nlocal_nodes; init++) + //directions(4,0) = -0.3; + ROL::Ptr> rol_d = + ROL::makePtr>(directions_distributed); + obj->checkGradient(*rol_x, *rol_d); + //obj->checkHessVec(*rol_x, *rol_d); + //directions_distributed->putScalar(-0.000001); + //obj->checkGradient(*rol_x, *rol_d); + //directions_distributed->putScalar(-0.0000001); + //obj->checkGradient(*rol_x, *rol_d); + + + + // ROL::Ptr> ineq_constraint = ROL::makePtr(fea_elasticity, nodal_density_flag); + // problem->addConstraint("Inequality Constraint",ineq_constraint,constraint_mul,constraint_bnd); + // problem->addConstraint("equality Constraint 2",eq_constraint2,constraint_mul2); + // problem->addConstraint("equality Constraint 3",eq_constraint3,constraint_mul3); + // problem->addLinearConstraint("Equality Constraint",eq_constraint,constraint_mul); + + for (int imodule = 0; imodule < nTO_modules; imodule++) + { + number_union.str(constraint_base); + number_union << imodule + 1; + constraint_name = number_union.str(); + ROL::Ptr> li_ptr = ROL::makePtr>(1, 0.0); + ROL::Ptr> constraint_mul = ROL::makePtr>(li_ptr); + if (TO_Function_Type[imodule] == FUNCTION_TYPE::EQUALITY_CONSTRAINT) + { + // pointers are reference counting + ROL::Ptr> eq_constraint; + if (TO_Module_List[imodule] == TO_MODULE_TYPE::Mass_Constraint) + { + *fos << " MASS CONSTRAINT EXPECTS FEA MODULE INDEX " << TO_Module_My_FEA_Module[imodule] << std::endl; + eq_constraint = ROL::makePtr(fea_modules[TO_Module_My_FEA_Module[imodule]], nodal_density_flag, Function_Arguments[imodule][0], false, true); + } + else if (TO_Module_List[imodule] == TO_MODULE_TYPE::Moment_of_Inertia_Constraint) + { + *fos << " MOMENT OF INERTIA CONSTRAINT EXPECTS FEA MODULE INDEX " << TO_Module_My_FEA_Module[imodule] << std::endl; + eq_constraint = ROL::makePtr(fea_modules[TO_Module_My_FEA_Module[imodule]], nodal_density_flag, Function_Arguments[imodule][1], + Function_Arguments[imodule][0], false, true); + } + else + { + // TODO: Put validation earlier + *fos << "PROGRAM IS ENDING DUE TO ERROR; UNDEFINED EQUALITY CONSTRAINT FUNCTION REQUESTED WITH NAME \"" + << TO_Module_List[imodule] << "\"" << std::endl; + exit_solver(0); + } + *fos << " ADDING CONSTRAINT " << constraint_name << std::endl; + problem->addConstraint(constraint_name, eq_constraint, constraint_mul); + } + if (TO_Function_Type[imodule] == FUNCTION_TYPE::INEQUALITY_CONSTRAINT) + { + // pointers are reference counting + ROL::Ptr> ineq_constraint; + ROL::Ptr> ll_ptr = ROL::makePtr>(1, Function_Arguments[imodule][0]); + ROL::Ptr> lu_ptr = ROL::makePtr>(1, Function_Arguments[imodule][1]); + ROL::Ptr> ll = ROL::makePtr>(ll_ptr); + ROL::Ptr> lu = ROL::makePtr>(lu_ptr); + ROL::Ptr> constraint_bnd = ROL::makePtr>(ll, lu); + if (TO_Module_List[imodule] == TO_MODULE_TYPE::Mass_Constraint) + { + *fos << " MASS CONSTRAINT EXPECTS FEA MODULE INDEX " << TO_Module_My_FEA_Module[imodule] << std::endl; + ineq_constraint = ROL::makePtr(fea_modules[TO_Module_My_FEA_Module[imodule]], nodal_density_flag, true, true); + } + else if (TO_Module_List[imodule] == TO_MODULE_TYPE::Moment_of_Inertia_Constraint) + { + *fos << " MOMENT OF INERTIA CONSTRAINT EXPECTS FEA MODULE INDEX " << TO_Module_My_FEA_Module[imodule] << std::endl; + ineq_constraint = ROL::makePtr(fea_modules[TO_Module_My_FEA_Module[imodule]], nodal_density_flag, Function_Arguments[imodule][1], + Function_Arguments[imodule][0], true, true); + } + else + { + // TODO: Put this validation earlier + *fos << "PROGRAM IS ENDING DUE TO ERROR; UNDEFINED INEQUALITY CONSTRAINT FUNCTION REQUESTED WITH NAME \"" + << TO_Module_List[imodule] << "\"" << std::endl; + exit_solver(0); + } + *fos << " ADDING CONSTRAINT " << constraint_name << std::endl; + problem->addConstraint(constraint_name, ineq_constraint, constraint_mul, constraint_bnd); + } + number_union.clear(); } - //set lower bounds for nodes on surfaces with boundary and loading conditions - for(int imodule = 0; imodule < nfea_modules; imodule++){ - num_boundary_sets = fea_modules[imodule]->num_boundary_conditions; - for(int iboundary = 0; iboundary < num_boundary_sets; iboundary++){ - - num_bdy_patches_in_set = fea_modules[imodule]->bdy_patches_in_set.stride(iboundary); - - //loop over boundary patches for this boundary set - if(simparam.optimization_options.thick_condition_boundary){ - for (int bdy_patch_gid = 0; bdy_patch_gid < num_bdy_patches_in_set; bdy_patch_gid++){ - - // get the global id for this boundary patch - patch_id = fea_modules[imodule]->bdy_patches_in_set(iboundary, bdy_patch_gid); - Surface_Nodes = Boundary_Patches(patch_id).node_set; - current_element_index = Boundary_Patches(patch_id).element_id; - //debug print of local surface ids - //std::cout << " LOCAL SURFACE IDS " << std::endl; - //std::cout << local_surface_id << std::endl; - //acquire set of nodes for this face - for(int node_loop=0; node_loop < max_nodes_per_element; node_loop++){ - current_node_index = nodes_in_elem(current_element_index,node_loop); - if(map->isNodeGlobalElement(current_node_index)){ - local_node_index = map->getLocalElement(current_node_index); - node_densities_lower_bound(local_node_index,0) = 1; + + // set bounds on design variables + if (nodal_density_flag) + { + dual_vec_array dual_node_densities_upper_bound = dual_vec_array("dual_node_densities_upper_bound", nlocal_nodes, 1); + dual_vec_array dual_node_densities_lower_bound = dual_vec_array("dual_node_densities_lower_bound", nlocal_nodes, 1); + host_vec_array node_densities_upper_bound = dual_node_densities_upper_bound.view_host(); + host_vec_array node_densities_lower_bound = dual_node_densities_lower_bound.view_host(); + // notify that the host view is going to be modified in the file readin + dual_node_densities_upper_bound.modify_host(); + dual_node_densities_lower_bound.modify_host(); + + // initialize densities to 1 for now; in the future there might be an option to read in an initial condition for each node + for (int inode = 0; inode < nlocal_nodes; inode++) + { + node_densities_upper_bound(inode, 0) = 1; + node_densities_lower_bound(inode, 0) = simparam.optimization_options.density_epsilon; + } + // set lower bounds for nodes on surfaces with boundary and loading conditions + for (int imodule = 0; imodule < nfea_modules; imodule++) + { + num_boundary_sets = fea_modules[imodule]->num_boundary_conditions; + for (int iboundary = 0; iboundary < num_boundary_sets; iboundary++) + { + num_bdy_patches_in_set = fea_modules[imodule]->bdy_patches_in_set.stride(iboundary); + + // loop over boundary patches for this boundary set + if (simparam.optimization_options.thick_condition_boundary) + { + for (int bdy_patch_gid = 0; bdy_patch_gid < num_bdy_patches_in_set; bdy_patch_gid++) + { + // get the global id for this boundary patch + patch_id = fea_modules[imodule]->bdy_patches_in_set(iboundary, bdy_patch_gid); + Surface_Nodes = Boundary_Patches(patch_id).node_set; + current_element_index = Boundary_Patches(patch_id).element_id; + // debug print of local surface ids + // std::cout << " LOCAL SURFACE IDS " << std::endl; + // std::cout << local_surface_id << std::endl; + // acquire set of nodes for this face + for (int node_loop = 0; node_loop < max_nodes_per_element; node_loop++) + { + current_node_index = nodes_in_elem(current_element_index, node_loop); + if (map->isNodeGlobalElement(current_node_index)) + { + local_node_index = map->getLocalElement(current_node_index); + node_densities_lower_bound(local_node_index, 0) = 1; + } + } // node loop for + + // if + /* + else{ + Surface_Nodes = Boundary_Patches(patch_id).node_set; + local_surface_id = Boundary_Patches(patch_id).local_patch_id; + //debug print of local surface ids + //std::cout << " LOCAL SURFACE IDS " << std::endl; + //std::cout << local_surface_id << std::endl; + //acquire set of nodes for this face + for(int node_loop=0; node_loop < Surface_Nodes.size(); node_loop++){ + current_node_index = Surface_Nodes(node_loop); + if(map->isNodeGlobalElement(current_node_index)){ + local_node_index = map->getLocalElement(current_node_index); + node_densities_lower_bound(local_node_index,0) = 1; + } + }// node loop for + }//if + */ + } // boundary patch for } - }// node loop for - - //if - /* - else{ - Surface_Nodes = Boundary_Patches(patch_id).node_set; - local_surface_id = Boundary_Patches(patch_id).local_patch_id; - //debug print of local surface ids - //std::cout << " LOCAL SURFACE IDS " << std::endl; - //std::cout << local_surface_id << std::endl; - //acquire set of nodes for this face - for(int node_loop=0; node_loop < Surface_Nodes.size(); node_loop++){ - current_node_index = Surface_Nodes(node_loop); - if(map->isNodeGlobalElement(current_node_index)){ - local_node_index = map->getLocalElement(current_node_index); - node_densities_lower_bound(local_node_index,0) = 1; + else + { + for (int node_loop = 0; node_loop < fea_modules[imodule]->num_bdy_nodes_in_set(iboundary); node_loop++) + { + local_node_index = fea_modules[imodule]->bdy_nodes_in_set(iboundary, node_loop); + node_densities_lower_bound(local_node_index, 0) = 1; + } } - }// node loop for - }//if - */ - }//boundary patch for - } - else{ - for(int node_loop=0; node_loop < fea_modules[imodule]->num_bdy_nodes_in_set(iboundary); node_loop++){ - local_node_index = fea_modules[imodule]->bdy_nodes_in_set(iboundary, node_loop); - node_densities_lower_bound(local_node_index,0) = 1; - } + } // boundary set for + + // set node conditions due to point BCS that might not show up in boundary sets + // possible to have overlap in which nodes are set with the previous loop + fea_modules[imodule]->node_density_constraints(node_densities_lower_bound); + } // module for + + // sync device view + dual_node_densities_upper_bound.sync_device(); + dual_node_densities_lower_bound.sync_device(); + + // allocate global vector information + upper_bound_node_densities_distributed = Teuchos::rcp(new MV(map, dual_node_densities_upper_bound)); + lower_bound_node_densities_distributed = Teuchos::rcp(new MV(map, dual_node_densities_lower_bound)); + } + else + { + // initialize memory for volume storage + vec_array Element_Densities_Upper_Bound("Element Densities_Upper_Bound", rnum_elem, 1); + vec_array Element_Densities_Lower_Bound("Element Densities_Lower_Bound", rnum_elem, 1); + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + Element_Densities_Upper_Bound(ielem, 0) = 1; + Element_Densities_Lower_Bound(ielem, 0) = simparam.optimization_options.density_epsilon; } - }//boundary set for - //set node conditions due to point BCS that might not show up in boundary sets - //possible to have overlap in which nodes are set with the previous loop - fea_modules[imodule]->node_density_constraints(node_densities_lower_bound); - }//module for - - //sync device view - dual_node_densities_upper_bound.sync_device(); - dual_node_densities_lower_bound.sync_device(); - - //allocate global vector information - upper_bound_node_densities_distributed = Teuchos::rcp(new MV(map, dual_node_densities_upper_bound)); - lower_bound_node_densities_distributed = Teuchos::rcp(new MV(map, dual_node_densities_lower_bound)); - - } - else{ - //initialize memory for volume storage - vec_array Element_Densities_Upper_Bound("Element Densities_Upper_Bound", rnum_elem, 1); - vec_array Element_Densities_Lower_Bound("Element Densities_Lower_Bound", rnum_elem, 1); - for(int ielem = 0; ielem < rnum_elem; ielem++){ - Element_Densities_Upper_Bound(ielem,0) = 1; - Element_Densities_Lower_Bound(ielem,0) = simparam.optimization_options.density_epsilon; - } - - //create global vector - Global_Element_Densities_Upper_Bound = Teuchos::rcp(new MV(element_map, Element_Densities_Upper_Bound)); - Global_Element_Densities_Lower_Bound = Teuchos::rcp(new MV(element_map, Element_Densities_Lower_Bound)); - } - - // Bound constraint defining the possible range of design density variables - ROL::Ptr > lower_bounds; - ROL::Ptr > upper_bounds; - if(nodal_density_flag){ - lower_bounds = ROL::makePtr>(lower_bound_node_densities_distributed); - upper_bounds = ROL::makePtr>(upper_bound_node_densities_distributed); - } - else{ - lower_bounds = ROL::makePtr>(Global_Element_Densities_Lower_Bound); - upper_bounds = ROL::makePtr>(Global_Element_Densities_Upper_Bound); - } - ROL::Ptr > bnd = ROL::makePtr>(lower_bounds, upper_bounds); - problem->addBoundConstraint(bnd); - - //compute initial constraint satisfaction - //ROL::Ptr ROL_Element_Masses = ROL::makePtr(fea_elasticity->Global_Element_Masses); - ROL::Elementwise::ReductionSum sumreduc; - if(nodal_density_flag) - design_densities = design_node_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); - else - design_densities = Global_Element_Densities->getLocalView (Tpetra::Access::ReadOnly); - //fea_elasticity->compute_element_masses(design_densities,true); - - //real_t initial_mass = ROL_Element_Masses->reduce(sumreduc); - - problem->setProjectionAlgorithm(*parlist); - //finalize problem - problem->finalize(false,true,*fos); - //problem->check(true,std::cout); - - // //check area normal gradients - // ROL::Ptr> rol_debug_coords = - // ROL::makePtr>(all_node_coords_distributed); - // Teuchos::RCP debug_directions_distributed = Teuchos::rcp(new MV(all_node_map, num_dim)); - // ROL::Ptr> debug_rol_d = - // ROL::makePtr>(debug_directions_distributed); - // debug_directions_distributed->randomize(-0.00375,0.00375); - // if(myrank==0){ - // for(int inode = 0; inode < 8; inode++){ - // for(int idim = 0; idim < num_dim; idim++){ - // std::cout<< "AREA NORMALS GRADIENT FOR NODE " << inode << " AND DIM " << idim << std::endl; - // ROL::Ptr> debug_obj = ROL::makePtr(this, inode, idim, nodal_density_flag); - // //debug_directions_distributed->putScalar(-0.1); - // //debug_directions_distributed->norm2(debug_direction_norm); - // //debug_directions_distributed->scale(1/debug_direction_norm(0)); - // //set all but first component to 0 for debug - // host_vec_array debug_directions = debug_directions_distributed->getLocalView (Tpetra::Access::ReadWrite); - // //for(int init = 1; init < nlocal_nodes; init++) - // //directions(4,0) = -0.3; - // debug_obj->checkGradient(*rol_debug_coords, *debug_rol_d); - // } - // } - // } - - //debug checks - ROL::Ptr> rol_x = - ROL::makePtr>(design_node_densities_distributed); - //construct direction vector for check - Teuchos::RCP directions_distributed = Teuchos::rcp(new MV(map, 1)); - directions_distributed->putScalar(-0.1); - directions_distributed->randomize(-0.8,1); - Kokkos::View direction_norm("gradient norm",1); - directions_distributed->norm2(direction_norm); - directions_distributed->scale(1/direction_norm(0)); - //set all but first component to 0 for debug - host_vec_array directions = directions_distributed->getLocalView (Tpetra::Access::ReadWrite); - //for(int init = 1; init < nlocal_nodes; init++) - //directions(4,0) = -0.3; - ROL::Ptr> rol_d = - ROL::makePtr>(directions_distributed); - obj->checkGradient(*rol_x, *rol_d); - //obj->checkHessVec(*rol_x, *rol_d); - //directions_distributed->putScalar(-0.000001); - //obj->checkGradient(*rol_x, *rol_d); - //directions_distributed->putScalar(-0.0000001); - //obj->checkGradient(*rol_x, *rol_d); - + // create global vector + Global_Element_Densities_Upper_Bound = Teuchos::rcp(new MV(element_map, Element_Densities_Upper_Bound)); + Global_Element_Densities_Lower_Bound = Teuchos::rcp(new MV(element_map, Element_Densities_Lower_Bound)); + } - // Instantiate Solver. - ROL::Solver solver(problem,*parlist); - - // Solve optimization problem. - //std::ostream outStream; - //solver.solve(*fos); - - //print final constraint satisfaction - //fea_elasticity->compute_element_masses(design_densities,false); - //real_t final_mass = ROL_Element_Masses->reduce(sumreduc); - //if(myrank==0) - //std::cout << "Final Mass Constraint is " << final_mass/initial_mass << std::endl; + // Bound constraint defining the possible range of design density variables + ROL::Ptr> lower_bounds; + ROL::Ptr> upper_bounds; + if (nodal_density_flag) + { + lower_bounds = ROL::makePtr>(lower_bound_node_densities_distributed); + upper_bounds = ROL::makePtr>(upper_bound_node_densities_distributed); + } + else + { + lower_bounds = ROL::makePtr>(Global_Element_Densities_Lower_Bound); + upper_bounds = ROL::makePtr>(Global_Element_Densities_Upper_Bound); + } + ROL::Ptr> bnd = ROL::makePtr>(lower_bounds, upper_bounds); + problem->addBoundConstraint(bnd); + + // compute initial constraint satisfaction + // ROL::Ptr ROL_Element_Masses = ROL::makePtr(fea_elasticity->Global_Element_Masses); + ROL::Elementwise::ReductionSum sumreduc; + if (nodal_density_flag) + { + design_densities = design_node_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); + } + else + { + design_densities = Global_Element_Densities->getLocalView(Tpetra::Access::ReadOnly); + } + // fea_elasticity->compute_element_masses(design_densities,true); + + // real_t initial_mass = ROL_Element_Masses->reduce(sumreduc); + + problem->setProjectionAlgorithm(*parlist); + // finalize problem + problem->finalize(false, true, *fos); + // problem->check(true,std::cout); + + // debug checks + ROL::Ptr> rol_x = + ROL::makePtr>(design_node_densities_distributed); + // construct direction vector for check + Teuchos::RCP directions_distributed = Teuchos::rcp(new MV(map, 1)); + directions_distributed->putScalar(-0.1); + directions_distributed->randomize(-0.8, 1); + Kokkos::View direction_norm("gradient norm", 1); + directions_distributed->norm2(direction_norm); + directions_distributed->scale(1 / direction_norm(0)); + // set all but first component to 0 for debug + host_vec_array directions = directions_distributed->getLocalView(Tpetra::Access::ReadWrite); + // for(int init = 1; init < nlocal_nodes; init++) + // directions(4,0) = -0.3; + ROL::Ptr> rol_d = + ROL::makePtr>(directions_distributed); + obj->checkGradient(*rol_x, *rol_d); + // obj->checkHessVec(*rol_x, *rol_d); + // directions_distributed->putScalar(-0.000001); + // obj->checkGradient(*rol_x, *rol_d); + // directions_distributed->putScalar(-0.0000001); + // obj->checkGradient(*rol_x, *rol_d); + + // Instantiate Solver. + ROL::Solver solver(problem, *parlist); + + // Solve optimization problem. + // std::ostream outStream; + // solver.solve(*fos); + + // print final constraint satisfaction + // fea_elasticity->compute_element_masses(design_densities,false); + // real_t final_mass = ROL_Element_Masses->reduce(sumreduc); + // if(myrank==0) + // std::cout << "Final Mass Constraint is " << final_mass/initial_mass << std::endl; } - /* ---------------------------------------------------------------------------- Initialize sets of element boundary surfaces and arrays for input conditions ------------------------------------------------------------------------------- */ -void Explicit_Solver::init_boundaries(){ - //int num_boundary_sets = simparam.NB; - int num_dim = simparam.num_dims; - size_t num_nodes_in_patch; - // build boundary mesh patches - if(myrank == 0) - std::cout << "Starting boundary patch setup" << std::endl <(nboundary_patches, num_nodes_in_patch, "Local_Index_Boundary_Patches"); - for(int iboundary = 0; iboundary < nboundary_patches; iboundary++){ - num_nodes_in_patch = Boundary_Patches(iboundary).node_set.size(); - for(int inode = 0; inode < num_nodes_in_patch; inode++){ - Local_Index_Boundary_Patches.host(iboundary,inode) = all_node_map->getLocalElement(Boundary_Patches(iboundary).node_set(inode)); - } - } - Local_Index_Boundary_Patches.update_device(); - - //std::cout << "Done with boundary patch setup" << std::endl <(nboundary_patches, num_nodes_in_patch, "Local_Index_Boundary_Patches"); + for (int iboundary = 0; iboundary < nboundary_patches; iboundary++) + { + num_nodes_in_patch = Boundary_Patches(iboundary).node_set.size(); + for (int inode = 0; inode < num_nodes_in_patch; inode++) + { + Local_Index_Boundary_Patches.host(iboundary, inode) = all_node_map->getLocalElement(Boundary_Patches(iboundary).node_set(inode)); + } + } + Local_Index_Boundary_Patches.update_device(); + + // std::cout << "Done with boundary patch setup" << std::endl <(num_sets, "NBoundary_Condition_Patches"); - Topology_Condition_Patches = CArrayKokkos(num_sets, nboundary_patches, "Boundary_Condition_Patches"); - - //initialize data - for(int iset = 0; iset < num_sets; iset++) NTopology_Condition_Patches(iset) = 0; +void Explicit_Solver::init_topology_conditions(int num_sets) +{ + // surface conditions + num_boundary_conditions = num_sets; + if (num_sets == 0) + { + std::cout << " Warning: number of boundary conditions = 0"; + return; + } + NTopology_Condition_Patches = CArrayKokkos(num_sets, "NBoundary_Condition_Patches"); + Topology_Condition_Patches = CArrayKokkos(num_sets, nboundary_patches, "Boundary_Condition_Patches"); + + // initialize data + for (int iset = 0; iset < num_sets; iset++) + { + NTopology_Condition_Patches(iset) = 0; + } } /* ---------------------------------------------------------------------- @@ -1541,43 +1687,53 @@ void Explicit_Solver::init_topology_conditions (int num_sets){ val = plane value, cylinder radius, shell radius ------------------------------------------------------------------------- */ -void Explicit_Solver::tag_boundaries(int bc_tag, real_t val, int bdy_set, real_t *patch_limits){ - - int num_dim = simparam.num_dims; - int is_on_set; - /* - if (bdy_set == num_bdy_sets_){ - std::cout << " ERROR: number of boundary sets must be increased by " - << bdy_set-num_bdy_sets_+1 << std::endl; - exit(0); - } - */ - - //test patch limits for feasibility - if(patch_limits != NULL){ - //test for upper bounds being greater than lower bounds - if(patch_limits[1] <= patch_limits[0]) std::cout << " Warning: patch limits for boundary condition are infeasible"; - if(num_dim==3) - if(patch_limits[2] <= patch_limits[3]) std::cout << " Warning: patch limits for boundary condition are infeasible"; - } - - // save the boundary vertices to this set that are on the plane - int counter = 0; - for (int iboundary_patch = 0; iboundary_patch < nboundary_patches; iboundary_patch++) { - - // check to see if this patch is on the specified plane - is_on_set = check_boundary(Boundary_Patches(iboundary_patch), bc_tag, val, patch_limits); // no=0, yes=1 - - if (is_on_set == 1){ - Topology_Condition_Patches(bdy_set,counter) = iboundary_patch; - counter ++; - } - } // end for bdy_patch - - // save the number of bdy patches in the set - NTopology_Condition_Patches(bdy_set) = counter; - - *fos << " tagged boundary patches " << std::endl; +void Explicit_Solver::tag_boundaries(int bc_tag, real_t val, int bdy_set, real_t* patch_limits) +{ + int num_dim = simparam.num_dims; + int is_on_set; + /* + if (bdy_set == num_bdy_sets_){ + std::cout << " ERROR: number of boundary sets must be increased by " + << bdy_set-num_bdy_sets_+1 << std::endl; + exit(0); + } + */ + + // test patch limits for feasibility + if (patch_limits != NULL) + { + // test for upper bounds being greater than lower bounds + if (patch_limits[1] <= patch_limits[0]) + { + std::cout << " Warning: patch limits for boundary condition are infeasible"; + } + if (num_dim == 3) + { + if (patch_limits[2] <= patch_limits[3]) + { + std::cout << " Warning: patch limits for boundary condition are infeasible"; + } + } + } + + // save the boundary vertices to this set that are on the plane + int counter = 0; + for (int iboundary_patch = 0; iboundary_patch < nboundary_patches; iboundary_patch++) + { + // check to see if this patch is on the specified plane + is_on_set = check_boundary(Boundary_Patches(iboundary_patch), bc_tag, val, patch_limits); // no=0, yes=1 + + if (is_on_set == 1) + { + Topology_Condition_Patches(bdy_set, counter) = iboundary_patch; + counter++; + } + } // end for bdy_patch + + // save the number of bdy patches in the set + NTopology_Condition_Patches(bdy_set) = counter; + + *fos << " tagged boundary patches " << std::endl; } /* ---------------------------------------------------------------------- @@ -1586,94 +1742,121 @@ void Explicit_Solver::tag_boundaries(int bc_tag, real_t val, int bdy_set, real_t val = plane value, radius, radius ------------------------------------------------------------------------- */ +int Explicit_Solver::check_boundary(Node_Combination& Patch_Nodes, int bc_tag, real_t val, real_t* patch_limits) +{ + int is_on_set = 1; + const_host_vec_array all_node_coords = all_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + + // Nodes on the Patch + auto node_list = Patch_Nodes.node_set; + int num_dim = simparam.num_dims; + size_t nnodes = node_list.size(); + size_t node_rid; + real_t node_coord[num_dim]; + int dim_other1, dim_other2; + CArrayKokkos node_on_flags(nnodes, "node_on_flags"); + + // initialize + for (int inode = 0; inode < nnodes; inode++) + { + node_on_flags(inode) = 0; + } -int Explicit_Solver::check_boundary(Node_Combination &Patch_Nodes, int bc_tag, real_t val, real_t *patch_limits){ - - int is_on_set = 1; - const_host_vec_array all_node_coords = all_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - - //Nodes on the Patch - auto node_list = Patch_Nodes.node_set; - int num_dim = simparam.num_dims; - size_t nnodes = node_list.size(); - size_t node_rid; - real_t node_coord[num_dim]; - int dim_other1, dim_other2; - CArrayKokkos node_on_flags(nnodes, "node_on_flags"); - - //initialize - for(int inode = 0; inode < nnodes; inode++) node_on_flags(inode) = 0; - - if(bc_tag==0){ - dim_other1 = 1; - dim_other2 = 2; - } - else if(bc_tag==1){ - dim_other1 = 0; - dim_other2 = 2; - } - else if(bc_tag==2){ - dim_other1 = 0; - dim_other2 = 1; - } - - - //test for planes - if(bc_tag < 3) - for(int inode = 0; inode < nnodes; inode++){ - - node_rid = all_node_map->getLocalElement(node_list(inode)); - for(int init=0; init < num_dim; init++){ - node_coord[init] = all_node_coords(node_rid,init); + if (bc_tag == 0) + { + dim_other1 = 1; + dim_other2 = 2; + } + else if (bc_tag == 1) + { + dim_other1 = 0; + dim_other2 = 2; + } + else if (bc_tag == 2) + { + dim_other1 = 0; + dim_other2 = 1; } - if ( fabs(node_coord[bc_tag] - val) <= BC_EPSILON){ node_on_flags(inode) = 1; - //test if within patch segment if user specified - if(patch_limits!=NULL){ - if (node_coord[dim_other1] - patch_limits[0] <= -BC_EPSILON) node_on_flags(inode) = 0; - if (node_coord[dim_other1] - patch_limits[1] >= BC_EPSILON) node_on_flags(inode) = 0; - if(num_dim==3){ - if (node_coord[dim_other2] - patch_limits[2] <= -BC_EPSILON) node_on_flags(inode) = 0; - if (node_coord[dim_other2] - patch_limits[3] >= BC_EPSILON) node_on_flags(inode) = 0; + // test for planes + if (bc_tag < 3) + { + for (int inode = 0; inode < nnodes; inode++) + { + node_rid = all_node_map->getLocalElement(node_list(inode)); + for (int init = 0; init < num_dim; init++) + { + node_coord[init] = all_node_coords(node_rid, init); + } + if (fabs(node_coord[bc_tag] - val) <= BC_EPSILON) + { + node_on_flags(inode) = 1; + + // test if within patch segment if user specified + if (patch_limits != NULL) + { + if (node_coord[dim_other1] - patch_limits[0] <= -BC_EPSILON) + { + node_on_flags(inode) = 0; + } + if (node_coord[dim_other1] - patch_limits[1] >= BC_EPSILON) + { + node_on_flags(inode) = 0; + } + if (num_dim == 3) + { + if (node_coord[dim_other2] - patch_limits[2] <= -BC_EPSILON) + { + node_on_flags(inode) = 0; + } + if (node_coord[dim_other2] - patch_limits[3] >= BC_EPSILON) + { + node_on_flags(inode) = 0; + } + } + } + } + // debug print of node id and node coord + // std::cout << "node coords on task " << myrank << " for node " << node_rid << std::endl; + // std::cout << "coord " <(num_nodes,0,comm)); - - //importer from local node distribution to sorted distribution - //Tpetra::Import node_sorting_importer(map, sorted_map); - - sorted_node_coords_distributed = Teuchos::rcp(new MV(sorted_map, num_dim)); - sorted_node_velocities_distributed = Teuchos::rcp(new MV(sorted_map, num_dim)); - - //comms to sort - sorted_node_coords_distributed->doImport(*node_coords_distributed, *node_sorting_importer, Tpetra::INSERT); - sorted_node_velocities_distributed->doImport(*node_velocities_distributed, *node_sorting_importer, Tpetra::INSERT); - - //comms to sort FEA module related vector data - /* - for (int imodule = 0; imodule < nfea_modules; imodule++){ - fea_modules[imodule]->collect_output(global_reduce_map); - //collected_node_displacements_distributed->doImport(*(fea_elasticity->node_displacements_distributed), dof_collection_importer, Tpetra::INSERT); - } - */ - - //sorted nodal density information - if(simparam.topology_optimization_on){ - sorted_node_densities_distributed = Teuchos::rcp(new MV(sorted_map, 1)); - sorted_node_densities_distributed->doImport(*design_node_densities_distributed, *node_sorting_importer, Tpetra::INSERT); - } - //comms to sort - //collected_node_densities_distributed->doImport(*design_node_densities_distributed, node_collection_importer, Tpetra::INSERT); - - //comms to collect FEA module related vector data - for (int imodule = 0; imodule < nfea_modules; imodule++){ - fea_modules[imodule]->sort_output(sorted_map); - //collected_node_displacements_distributed->doImport(*(fea_elasticity->node_displacements_distributed), dof_collection_importer, Tpetra::INSERT); - } - - //Global_Element_Densities->describe(*fos,Teuchos::VERB_EXTREME); - - sorted_element_densities_distributed = Teuchos::rcp(new MV(sorted_element_map, 1)); +void Explicit_Solver::sort_information() +{ + int num_dim = simparam.num_dims; + // sorted_map = Teuchos::rcp( new Tpetra::Map(num_nodes,0,comm)); - for (int imodule = 0; imodule < nfea_modules; imodule++){ - fea_modules[imodule]->sort_element_output(sorted_element_map); - //collected_node_displacements_distributed->doImport(*(fea_elasticity->node_displacements_distributed), dof_collection_importer, Tpetra::INSERT); - } - - sorted_nodes_in_elem_distributed = Teuchos::rcp(new MCONN(sorted_element_map, max_nodes_per_element)); + // importer from local node distribution to sorted distribution + // Tpetra::Import node_sorting_importer(map, sorted_map); - //comms - sorted_nodes_in_elem_distributed->doImport(*global_nodes_in_elem_distributed, *element_sorting_importer, Tpetra::INSERT); - sorted_element_densities_distributed->doImport(*Global_Element_Densities, *element_sorting_importer, Tpetra::INSERT); - + sorted_node_coords_distributed = Teuchos::rcp(new MV(sorted_map, num_dim)); + sorted_node_velocities_distributed = Teuchos::rcp(new MV(sorted_map, num_dim)); + + // comms to sort + sorted_node_coords_distributed->doImport(*node_coords_distributed, *node_sorting_importer, Tpetra::INSERT); + sorted_node_velocities_distributed->doImport(*node_velocities_distributed, *node_sorting_importer, Tpetra::INSERT); + + // comms to sort FEA module related vector data + /* + for (int imodule = 0; imodule < nfea_modules; imodule++){ + fea_modules[imodule]->collect_output(global_reduce_map); + //collected_node_displacements_distributed->doImport(*(fea_elasticity->node_displacements_distributed), dof_collection_importer, Tpetra::INSERT); + } + */ + + // sorted nodal density information + if (simparam.topology_optimization_on) + { + sorted_node_densities_distributed = Teuchos::rcp(new MV(sorted_map, 1)); + sorted_node_densities_distributed->doImport(*design_node_densities_distributed, *node_sorting_importer, Tpetra::INSERT); + } + // comms to sort + // collected_node_densities_distributed->doImport(*design_node_densities_distributed, node_collection_importer, Tpetra::INSERT); + + // comms to collect FEA module related vector data + for (int imodule = 0; imodule < nfea_modules; imodule++) + { + fea_modules[imodule]->sort_output(sorted_map); + // collected_node_displacements_distributed->doImport(*(fea_elasticity->node_displacements_distributed), dof_collection_importer, Tpetra::INSERT); + } + + // Global_Element_Densities->describe(*fos,Teuchos::VERB_EXTREME); + + sorted_element_densities_distributed = Teuchos::rcp(new MV(sorted_element_map, 1)); + + for (int imodule = 0; imodule < nfea_modules; imodule++) + { + fea_modules[imodule]->sort_element_output(sorted_element_map); + // collected_node_displacements_distributed->doImport(*(fea_elasticity->node_displacements_distributed), dof_collection_importer, Tpetra::INSERT); + } + + sorted_nodes_in_elem_distributed = Teuchos::rcp(new MCONN(sorted_element_map, max_nodes_per_element)); + + // comms + sorted_nodes_in_elem_distributed->doImport(*global_nodes_in_elem_distributed, *element_sorting_importer, Tpetra::INSERT); + sorted_element_densities_distributed->doImport(*Global_Element_Densities, *element_sorting_importer, Tpetra::INSERT); } /* ---------------------------------------------------------------------- Collect Nodal Information on Rank 0 ------------------------------------------------------------------------- */ -void Explicit_Solver::collect_information(){ - GO nreduce_nodes = 0; - GO nreduce_elem = 0; - int num_dim = simparam.num_dims; +void Explicit_Solver::collect_information() +{ + GO nreduce_nodes = 0; + GO nreduce_elem = 0; + int num_dim = simparam.num_dims; - //collect nodal coordinate information - if(myrank==0) nreduce_nodes = num_nodes; - Teuchos::RCP > global_reduce_map = - Teuchos::rcp(new Tpetra::Map(Teuchos::OrdinalTraits::invalid(),nreduce_nodes,0,comm)); - - //importer from local node distribution to collected distribution - Tpetra::Import node_collection_importer(map, global_reduce_map); + // collect nodal coordinate information + if (myrank == 0) + { + nreduce_nodes = num_nodes; + } + Teuchos::RCP> global_reduce_map = + Teuchos::rcp(new Tpetra::Map(Teuchos::OrdinalTraits::invalid(), nreduce_nodes, 0, comm)); + + // importer from local node distribution to collected distribution + Tpetra::Import node_collection_importer(map, global_reduce_map); - collected_node_coords_distributed = Teuchos::rcp(new MV(global_reduce_map, num_dim)); - collected_node_velocities_distributed = Teuchos::rcp(new MV(global_reduce_map, num_dim)); + collected_node_coords_distributed = Teuchos::rcp(new MV(global_reduce_map, num_dim)); + collected_node_velocities_distributed = Teuchos::rcp(new MV(global_reduce_map, num_dim)); - //comms to collect - collected_node_coords_distributed->doImport(*node_coords_distributed, node_collection_importer, Tpetra::INSERT); - collected_node_velocities_distributed->doImport(*node_velocities_distributed, node_collection_importer, Tpetra::INSERT); + // comms to collect + collected_node_coords_distributed->doImport(*node_coords_distributed, node_collection_importer, Tpetra::INSERT); + collected_node_velocities_distributed->doImport(*node_velocities_distributed, node_collection_importer, Tpetra::INSERT); - //comms to collect FEA module related vector data - /* - for (int imodule = 0; imodule < nfea_modules; imodule++){ - fea_modules[imodule]->collect_output(global_reduce_map); - //collected_node_displacements_distributed->doImport(*(fea_elasticity->node_displacements_distributed), dof_collection_importer, Tpetra::INSERT); - } - */ + // comms to collect FEA module related vector data + /* + for (int imodule = 0; imodule < nfea_modules; imodule++){ + fea_modules[imodule]->collect_output(global_reduce_map); + //collected_node_displacements_distributed->doImport(*(fea_elasticity->node_displacements_distributed), dof_collection_importer, Tpetra::INSERT); + } + */ - //collected nodal density information - collected_node_densities_distributed = Teuchos::rcp(new MV(global_reduce_map, 1)); + // collected nodal density information + collected_node_densities_distributed = Teuchos::rcp(new MV(global_reduce_map, 1)); - //comms to collect - //collected_node_densities_distributed->doImport(*design_node_densities_distributed, node_collection_importer, Tpetra::INSERT); + // comms to collect + // collected_node_densities_distributed->doImport(*design_node_densities_distributed, node_collection_importer, Tpetra::INSERT); - //collect element connectivity data - if(myrank==0) nreduce_elem = num_elem; - Teuchos::RCP > global_reduce_element_map = - Teuchos::rcp(new Tpetra::Map(Teuchos::OrdinalTraits::invalid(),nreduce_elem,0,comm)); + // collect element connectivity data + if (myrank == 0) + { + nreduce_elem = num_elem; + } + Teuchos::RCP> global_reduce_element_map = + Teuchos::rcp(new Tpetra::Map(Teuchos::OrdinalTraits::invalid(), nreduce_elem, 0, comm)); - //importer from all element map to collected distribution - Tpetra::Import element_collection_importer(all_element_map, global_reduce_element_map); - - collected_nodes_in_elem_distributed = Teuchos::rcp(new MCONN(global_reduce_element_map, max_nodes_per_element)); + // importer from all element map to collected distribution + Tpetra::Import element_collection_importer(all_element_map, global_reduce_element_map); - //comms to collect - collected_nodes_in_elem_distributed->doImport(*global_nodes_in_elem_distributed, element_collection_importer, Tpetra::INSERT); + collected_nodes_in_elem_distributed = Teuchos::rcp(new MCONN(global_reduce_element_map, max_nodes_per_element)); - //collect element type data + // comms to collect + collected_nodes_in_elem_distributed->doImport(*global_nodes_in_elem_distributed, element_collection_importer, Tpetra::INSERT); + // collect element type data } /* ---------------------------------------------------------------------- Communicate updated nodal velocities to ghost nodes ------------------------------------------------------------------------- */ -void Explicit_Solver::comm_velocities(){ - - //debug print of design vector - //std::ostream &out = std::cout; - //Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - //if(myrank==0) - //*fos << "Density data :" << std::endl; - //node_densities_distributed->describe(*fos,Teuchos::VERB_EXTREME); - //*fos << std::endl; - //std::fflush(stdout); - - //communicate design densities - //create import object using local node indices map and all indices map - //Tpetra::Import importer(map, ghost_node_map); - - //comms to get ghosts - ghost_node_velocities_distributed->doImport(*node_velocities_distributed, *ghost_importer, Tpetra::INSERT); - //all_node_map->describe(*fos,Teuchos::VERB_EXTREME); - //all_node_velocities_distributed->describe(*fos,Teuchos::VERB_EXTREME); - - //update_count++; - //if(update_count==1){ - //MPI_Barrier(world); - //MPI_Abort(world,4); - //} +void Explicit_Solver::comm_velocities() +{ + // debug print of design vector + // std::ostream &out = std::cout; + // Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + // if(myrank==0) + // *fos << "Density data :" << std::endl; + // node_densities_distributed->describe(*fos,Teuchos::VERB_EXTREME); + // *fos << std::endl; + // std::fflush(stdout); + + // communicate design densities + // create import object using local node indices map and all indices map + // Tpetra::Import importer(map, ghost_node_map); + + // comms to get ghosts + ghost_node_velocities_distributed->doImport(*node_velocities_distributed, *ghost_importer, Tpetra::INSERT); + // all_node_map->describe(*fos,Teuchos::VERB_EXTREME); + // all_node_velocities_distributed->describe(*fos,Teuchos::VERB_EXTREME); + + // update_count++; + // if(update_count==1){ + // MPI_Barrier(world); + // MPI_Abort(world,4); + // } } /* ---------------------------------------------------------------------- Communicate nodal masses/densities to ghost nodes ------------------------------------------------------------------------- */ -void Explicit_Solver::comm_densities(){ - - //debug print of design vector - //std::ostream &out = std::cout; - //Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - //if(myrank==0) - //*fos << "Density data :" << std::endl; - //node_densities_distributed->describe(*fos,Teuchos::VERB_EXTREME); - //*fos << std::endl; - //std::fflush(stdout); - - //communicate design densities - //create import object using local node indices map and all indices map - //Tpetra::Import importer(map, all_node_map); - - //active view scope - { - const_host_vec_array node_densities_host = design_node_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); - } - //comms to get ghosts - all_node_densities_distributed->doImport(*design_node_densities_distributed, *importer, Tpetra::INSERT); - //all_node_map->describe(*fos,Teuchos::VERB_EXTREME); - //all_node_velocities_distributed->describe(*fos,Teuchos::VERB_EXTREME); - - //update_count++; - //if(update_count==1){ - //MPI_Barrier(world); - //MPI_Abort(world,4); - //} +void Explicit_Solver::comm_densities() +{ + // debug print of design vector + // std::ostream &out = std::cout; + // Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + // if(myrank==0) + // *fos << "Density data :" << std::endl; + // node_densities_distributed->describe(*fos,Teuchos::VERB_EXTREME); + // *fos << std::endl; + // std::fflush(stdout); + + // communicate design densities + // create import object using local node indices map and all indices map + // Tpetra::Import importer(map, all_node_map); + + // active view scope + { + const_host_vec_array node_densities_host = design_node_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); + } + // comms to get ghosts + all_node_densities_distributed->doImport(*design_node_densities_distributed, *importer, Tpetra::INSERT); + // all_node_map->describe(*fos,Teuchos::VERB_EXTREME); + // all_node_velocities_distributed->describe(*fos,Teuchos::VERB_EXTREME); + + // update_count++; + // if(update_count==1){ + // MPI_Barrier(world); + // MPI_Abort(world,4); + // } } /* ---------------------------------------------------------------------- Output Model Information in tecplot format ------------------------------------------------------------------------- */ -void Explicit_Solver::parallel_tecplot_writer(){ - int num_dim = simparam.num_dims; - std::string current_file_name; - std::string base_file_name= "TecplotTO"; - std::string base_file_name_undeformed= "TecplotTO_undeformed"; - std::stringstream current_line_stream; - std::string current_line; - std::string file_extension= ".dat"; - std::string file_count; - std::stringstream count_temp; - int time_step = 0; - int temp_convert; - int noutput, nvector; - bool displace_geometry = false; - // Convert ijk index system to the finite element numbering convention - // for vertices in cell - - //comm to vectors with contigously sorted global indices from unsorted zoltan2 repartition map - sort_information(); - //set host views of the communicated data to print out from - const_host_vec_array sorted_node_coords = sorted_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_host_vec_array sorted_node_velocities = sorted_node_velocities_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_host_vec_array sorted_node_densities; - if(simparam.topology_optimization_on){ - sorted_node_densities = sorted_node_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); - } - const_host_elem_conn_array sorted_nodes_in_elem = sorted_nodes_in_elem_distributed->getLocalView (Tpetra::Access::ReadOnly); - - CArrayKokkos convert_ijk_to_ensight(max_nodes_per_element); - CArrayKokkos tmp_ijk_indx(max_nodes_per_element); - convert_ijk_to_ensight(0) = 0; - convert_ijk_to_ensight(1) = 1; - convert_ijk_to_ensight(2) = 3; - convert_ijk_to_ensight(3) = 2; - if(num_dim==3){ - convert_ijk_to_ensight(4) = 4; - convert_ijk_to_ensight(5) = 5; - convert_ijk_to_ensight(6) = 7; - convert_ijk_to_ensight(7) = 6; - } - - MPI_File myfile_parallel; - MPI_Offset header_stream_offset = 0; - //initial undeformed geometry - count_temp.str(""); - count_temp << file_index; - file_index++; - file_count = count_temp.str(); - if(displace_geometry&&displacement_module>=0) - current_file_name = base_file_name_undeformed + file_count + file_extension; - else - current_file_name = base_file_name + file_count + file_extension; - MPI_File_open(MPI_COMM_WORLD, current_file_name.c_str(), - MPI_MODE_CREATE|MPI_MODE_WRONLY, +void Explicit_Solver::parallel_tecplot_writer() +{ + int num_dim = simparam.num_dims; + std::string current_file_name; + std::string base_file_name = "TecplotTO"; + std::string base_file_name_undeformed = "TecplotTO_undeformed"; + std::stringstream current_line_stream; + std::string current_line; + std::string file_extension = ".dat"; + std::string file_count; + std::stringstream count_temp; + int time_step = 0; + int temp_convert; + int noutput, nvector; + bool displace_geometry = false; + // Convert ijk index system to the finite element numbering convention + // for vertices in cell + + // comm to vectors with contigously sorted global indices from unsorted zoltan2 repartition map + sort_information(); + // set host views of the communicated data to print out from + const_host_vec_array sorted_node_coords = sorted_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_host_vec_array sorted_node_velocities = sorted_node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_host_vec_array sorted_node_densities; + if (simparam.topology_optimization_on) + { + sorted_node_densities = sorted_node_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); + } + const_host_elem_conn_array sorted_nodes_in_elem = sorted_nodes_in_elem_distributed->getLocalView(Tpetra::Access::ReadOnly); + + CArrayKokkos convert_ijk_to_ensight(max_nodes_per_element); + CArrayKokkos tmp_ijk_indx(max_nodes_per_element); + convert_ijk_to_ensight(0) = 0; + convert_ijk_to_ensight(1) = 1; + convert_ijk_to_ensight(2) = 3; + convert_ijk_to_ensight(3) = 2; + if (num_dim == 3) + { + convert_ijk_to_ensight(4) = 4; + convert_ijk_to_ensight(5) = 5; + convert_ijk_to_ensight(6) = 7; + convert_ijk_to_ensight(7) = 6; + } + + MPI_File myfile_parallel; + MPI_Offset header_stream_offset = 0; + // initial undeformed geometry + count_temp.str(""); + count_temp << file_index; + file_index++; + file_count = count_temp.str(); + if (displace_geometry && displacement_module >= 0) + { + current_file_name = base_file_name_undeformed + file_count + file_extension; + } + else + { + current_file_name = base_file_name + file_count + file_extension; + } + MPI_File_open(MPI_COMM_WORLD, current_file_name.c_str(), + MPI_MODE_CREATE | MPI_MODE_WRONLY, MPI_INFO_NULL, &myfile_parallel); - - int err = MPI_File_open(MPI_COMM_WORLD, current_file_name.c_str(), MPI_MODE_CREATE|MPI_MODE_EXCL|MPI_MODE_WRONLY, MPI_INFO_NULL, &myfile_parallel); - //allows overwriting the file if it already existed in the directory - if (err != MPI_SUCCESS) { - if (myrank == 0){ - MPI_File_delete(current_file_name.c_str(),MPI_INFO_NULL); - } - MPI_File_open(MPI_COMM_WORLD, current_file_name.c_str(), MPI_MODE_CREATE|MPI_MODE_EXCL|MPI_MODE_WRONLY, MPI_INFO_NULL, &myfile_parallel); + + int err = MPI_File_open(MPI_COMM_WORLD, current_file_name.c_str(), MPI_MODE_CREATE | MPI_MODE_EXCL | MPI_MODE_WRONLY, MPI_INFO_NULL, &myfile_parallel); + // allows overwriting the file if it already existed in the directory + if (err != MPI_SUCCESS) + { + if (myrank == 0) + { + MPI_File_delete(current_file_name.c_str(), MPI_INFO_NULL); + } + MPI_File_open(MPI_COMM_WORLD, current_file_name.c_str(), MPI_MODE_CREATE | MPI_MODE_EXCL | MPI_MODE_WRONLY, MPI_INFO_NULL, &myfile_parallel); } - //output header of the tecplot file - - //std::cout << current_file_name << std::endl; - current_line_stream << "TITLE=\"results for FEA simulation\"" "\n"; - current_line = current_line_stream.str(); - if(myrank == 0) - MPI_File_write(myfile_parallel,current_line.c_str(),current_line.length(), MPI_CHAR, MPI_STATUS_IGNORE); - header_stream_offset += current_line.length(); - //myfile << "VARIABLES = \"x\", \"y\", \"z\", \"density\", \"sigmaxx\", \"sigmayy\", \"sigmazz\", \"sigmaxy\", \"sigmaxz\", \"sigmayz\"" "\n"; - //else - current_line_stream.str(""); - if(num_dim==3) - current_line_stream << "VARIABLES = \"x\", \"y\", \"z\", \"vx\", \"vy\", \"vz\""; - else if(num_dim==2) - current_line_stream << "VARIABLES = \"x\", \"y\", \"vx\", \"vy\""; - current_line = current_line_stream.str(); - if(myrank == 0) - MPI_File_write(myfile_parallel,current_line.c_str(),current_line.length(), MPI_CHAR, MPI_STATUS_IGNORE); - header_stream_offset += current_line.length(); + // output header of the tecplot file + + // std::cout << current_file_name << std::endl; + current_line_stream << "TITLE=\"results for FEA simulation\"" "\n"; + current_line = current_line_stream.str(); + if (myrank == 0) + { + MPI_File_write(myfile_parallel, current_line.c_str(), current_line.length(), MPI_CHAR, MPI_STATUS_IGNORE); + } + header_stream_offset += current_line.length(); + // myfile << "VARIABLES = \"x\", \"y\", \"z\", \"density\", \"sigmaxx\", \"sigmayy\", \"sigmazz\", \"sigmaxy\", \"sigmaxz\", \"sigmayz\"" "\n"; + // else + current_line_stream.str(""); + if (num_dim == 3) + { + current_line_stream << "VARIABLES = \"x\", \"y\", \"z\", \"vx\", \"vy\", \"vz\""; + } + else if (num_dim == 2) + { + current_line_stream << "VARIABLES = \"x\", \"y\", \"vx\", \"vy\""; + } + current_line = current_line_stream.str(); + if (myrank == 0) + { + MPI_File_write(myfile_parallel, current_line.c_str(), current_line.length(), MPI_CHAR, MPI_STATUS_IGNORE); + } + header_stream_offset += current_line.length(); /* for (int imodule = 0; imodule < nfea_modules; imodule++){ for(int ioutput = 0; ioutput < fea_modules[imodule]->noutput; ioutput++){ @@ -1954,53 +2163,64 @@ void Explicit_Solver::parallel_tecplot_writer(){ } } */ - current_line_stream.str(""); - current_line_stream << "\n"; - current_line = current_line_stream.str(); - if(myrank == 0) - MPI_File_write(myfile_parallel,current_line.c_str(),current_line.length(), MPI_CHAR, MPI_STATUS_IGNORE); - header_stream_offset += current_line.length(); - - current_line_stream.str(""); - if(num_dim==2){ - current_line_stream << "ZONE T=\"load step " << time_step << "\", NODES= " << num_nodes - << ", ELEMENTS= " << num_elem << ", DATAPACKING=POINT, ZONETYPE=FEQUADRILATERAL" "\n"; - } - else if(num_dim==3){ - current_line_stream << "ZONE T=\"load step " << time_step << "\", NODES= " << num_nodes - << ", ELEMENTS= " << num_elem << ", DATAPACKING=POINT, ZONETYPE=FEBRICK" "\n"; - } - current_line = current_line_stream.str(); - if(myrank == 0) - MPI_File_write(myfile_parallel,current_line.c_str(),current_line.length(), MPI_CHAR, MPI_STATUS_IGNORE); - header_stream_offset += current_line.length(); - - //output nodal data - //compute buffer output size and file stream offset for this MPI rank - int default_vector_count = 2; - int buffer_size_per_node_line = 26*default_vector_count*num_dim + 1; //25 width per number + 1 space times 6 entries plus line terminator - int nlocal_sorted_nodes = sorted_map->getLocalNumElements(); - GO first_node_global_id = sorted_map->getGlobalElement(0); - CArrayKokkos print_buffer(buffer_size_per_node_line*nlocal_sorted_nodes); - MPI_Offset file_stream_offset = buffer_size_per_node_line*first_node_global_id; - - //populate buffer - long long current_buffer_position = 0; - current_line_stream << std::fixed << std::setprecision(8); - for (int nodeline = 0; nodeline < nlocal_sorted_nodes; nodeline++) { current_line_stream.str(""); - current_line_stream << std::setw(25) << sorted_node_coords(nodeline,0) << " "; - current_line_stream << std::setw(25) << sorted_node_coords(nodeline,1) << " "; - if(num_dim==3) - current_line_stream << std::setw(25) << sorted_node_coords(nodeline,2) << " "; - - //velocity print - current_line_stream << std::setw(25) << sorted_node_velocities(nodeline,0) << " "; - current_line_stream << std::setw(25) << sorted_node_velocities(nodeline,1) << " "; - if(num_dim==3) - current_line_stream << std::setw(25) << sorted_node_velocities(nodeline,2) << " "; - - //myfile << std::setw(25) << collected_node_densities(nodeline,0) << " "; + current_line_stream << "\n"; + current_line = current_line_stream.str(); + if (myrank == 0) + { + MPI_File_write(myfile_parallel, current_line.c_str(), current_line.length(), MPI_CHAR, MPI_STATUS_IGNORE); + } + header_stream_offset += current_line.length(); + + current_line_stream.str(""); + if (num_dim == 2) + { + current_line_stream << "ZONE T=\"load step " << time_step << "\", NODES= " << num_nodes + << ", ELEMENTS= " << num_elem << ", DATAPACKING=POINT, ZONETYPE=FEQUADRILATERAL" "\n"; + } + else if (num_dim == 3) + { + current_line_stream << "ZONE T=\"load step " << time_step << "\", NODES= " << num_nodes + << ", ELEMENTS= " << num_elem << ", DATAPACKING=POINT, ZONETYPE=FEBRICK" "\n"; + } + current_line = current_line_stream.str(); + if (myrank == 0) + { + MPI_File_write(myfile_parallel, current_line.c_str(), current_line.length(), MPI_CHAR, MPI_STATUS_IGNORE); + } + header_stream_offset += current_line.length(); + + // output nodal data + // compute buffer output size and file stream offset for this MPI rank + int default_vector_count = 2; + int buffer_size_per_node_line = 26 * default_vector_count * num_dim + 1; // 25 width per number + 1 space times 6 entries plus line terminator + int nlocal_sorted_nodes = sorted_map->getLocalNumElements(); + GO first_node_global_id = sorted_map->getGlobalElement(0); + CArrayKokkos print_buffer(buffer_size_per_node_line * nlocal_sorted_nodes); + MPI_Offset file_stream_offset = buffer_size_per_node_line * first_node_global_id; + + // populate buffer + long long current_buffer_position = 0; + current_line_stream << std::fixed << std::setprecision(8); + for (int nodeline = 0; nodeline < nlocal_sorted_nodes; nodeline++) + { + current_line_stream.str(""); + current_line_stream << std::setw(25) << sorted_node_coords(nodeline, 0) << " "; + current_line_stream << std::setw(25) << sorted_node_coords(nodeline, 1) << " "; + if (num_dim == 3) + { + current_line_stream << std::setw(25) << sorted_node_coords(nodeline, 2) << " "; + } + + // velocity print + current_line_stream << std::setw(25) << sorted_node_velocities(nodeline, 0) << " "; + current_line_stream << std::setw(25) << sorted_node_velocities(nodeline, 1) << " "; + if (num_dim == 3) + { + current_line_stream << std::setw(25) << sorted_node_velocities(nodeline, 2) << " "; + } + + // myfile << std::setw(25) << collected_node_densities(nodeline,0) << " "; /* for (int imodule = 0; imodule < nfea_modules; imodule++){ noutput = fea_modules[imodule]->noutput; @@ -2021,165 +2241,186 @@ void Explicit_Solver::parallel_tecplot_writer(){ } } */ - current_line_stream << std::endl; - - current_line = current_line_stream.str(); + current_line_stream << std::endl; - //copy current line over to C style string buffer (wrapped by matar) - strcpy(&print_buffer(current_buffer_position),current_line.c_str()); + current_line = current_line_stream.str(); - current_buffer_position += current_line.length(); - } + // copy current line over to C style string buffer (wrapped by matar) + strcpy(&print_buffer(current_buffer_position), current_line.c_str()); - //print buffers at offsets with collective MPI write - //MPI_Offset current_stream_position = MPI_File_get_position(myfile_parallel,0); - MPI_Barrier(world); - MPI_File_write_at_all(myfile_parallel, file_stream_offset + header_stream_offset, print_buffer.get_kokkos_view().data(), buffer_size_per_node_line*nlocal_sorted_nodes, MPI_CHAR, MPI_STATUS_IGNORE); - //MPI_File_close(&myfile_parallel); - - //write element connectivity; reopen to reset offset baseline. - //err = MPI_File_open(MPI_COMM_WORLD, current_file_name.c_str(), MPI_MODE_APPEND|MPI_MODE_WRONLY, MPI_INFO_NULL, &myfile_parallel); - - MPI_Offset current_stream_position; - MPI_Barrier(world); - MPI_File_sync(myfile_parallel); - MPI_File_seek_shared(myfile_parallel, 0, MPI_SEEK_END); - MPI_File_sync(myfile_parallel); - MPI_File_get_position_shared(myfile_parallel, ¤t_stream_position); - - //debug check - //std::cout << "offset on rank " << myrank << " is " << file_stream_offset + header_stream_offset + current_buffer_position << std::endl; - //std::cout << "get position on rank " << myrank << " is " << current_stream_position << std::endl; - - //expand print buffer if needed - int buffer_size_per_element_line = 11*max_nodes_per_element + 1; //25 width per number plus 6 spaces plus line terminator - int nlocal_elements = sorted_element_map->getLocalNumElements(); - GO first_element_global_id = sorted_element_map->getGlobalElement(0); - if(buffer_size_per_element_line*nlocal_elements > print_buffer.size()) - print_buffer = CArrayKokkos(buffer_size_per_element_line*nlocal_elements); - file_stream_offset = buffer_size_per_element_line*first_element_global_id + current_stream_position; - - current_buffer_position = 0; - for (int elementline = 0; elementline < nlocal_elements; elementline++) { - current_line_stream.str(""); - //convert node ordering - for (int ii = 0; ii < max_nodes_per_element; ii++) { - if(active_node_ordering_convention == IJK) - temp_convert = convert_ijk_to_ensight(ii); - else - temp_convert = ii; - current_line_stream << std::setw(10) << sorted_nodes_in_elem(elementline, temp_convert) + 1 << " "; - } - current_line_stream << std::endl; - current_line = current_line_stream.str(); + current_buffer_position += current_line.length(); + } - //copy current line over to C style string buffer (wrapped by matar) - strcpy(&print_buffer(current_buffer_position),current_line.c_str()); + // print buffers at offsets with collective MPI write + // MPI_Offset current_stream_position = MPI_File_get_position(myfile_parallel,0); + MPI_Barrier(world); + MPI_File_write_at_all(myfile_parallel, file_stream_offset + header_stream_offset, print_buffer.get_kokkos_view().data(), buffer_size_per_node_line * nlocal_sorted_nodes, MPI_CHAR, + MPI_STATUS_IGNORE); + // MPI_File_close(&myfile_parallel); + + // write element connectivity; reopen to reset offset baseline. + // err = MPI_File_open(MPI_COMM_WORLD, current_file_name.c_str(), MPI_MODE_APPEND|MPI_MODE_WRONLY, MPI_INFO_NULL, &myfile_parallel); + + MPI_Offset current_stream_position; + MPI_Barrier(world); + MPI_File_sync(myfile_parallel); + MPI_File_seek_shared(myfile_parallel, 0, MPI_SEEK_END); + MPI_File_sync(myfile_parallel); + MPI_File_get_position_shared(myfile_parallel, ¤t_stream_position); + + // debug check + // std::cout << "offset on rank " << myrank << " is " << file_stream_offset + header_stream_offset + current_buffer_position << std::endl; + // std::cout << "get position on rank " << myrank << " is " << current_stream_position << std::endl; + + // expand print buffer if needed + int buffer_size_per_element_line = 11 * max_nodes_per_element + 1; // 25 width per number plus 6 spaces plus line terminator + int nlocal_elements = sorted_element_map->getLocalNumElements(); + GO first_element_global_id = sorted_element_map->getGlobalElement(0); + if (buffer_size_per_element_line * nlocal_elements > print_buffer.size()) + { + print_buffer = CArrayKokkos(buffer_size_per_element_line * nlocal_elements); + } + file_stream_offset = buffer_size_per_element_line * first_element_global_id + current_stream_position; + + current_buffer_position = 0; + for (int elementline = 0; elementline < nlocal_elements; elementline++) + { + current_line_stream.str(""); + // convert node ordering + for (int ii = 0; ii < max_nodes_per_element; ii++) + { + if (active_node_ordering_convention == IJK) + { + temp_convert = convert_ijk_to_ensight(ii); + } + else + { + temp_convert = ii; + } + current_line_stream << std::setw(10) << sorted_nodes_in_elem(elementline, temp_convert) + 1 << " "; + } + current_line_stream << std::endl; + current_line = current_line_stream.str(); - current_buffer_position += current_line.length(); - } - - MPI_Barrier(world); - MPI_File_write_at_all(myfile_parallel, file_stream_offset, print_buffer.get_kokkos_view().data(), buffer_size_per_element_line*nlocal_elements, MPI_CHAR, MPI_STATUS_IGNORE); - - MPI_File_close(&myfile_parallel); + // copy current line over to C style string buffer (wrapped by matar) + strcpy(&print_buffer(current_buffer_position), current_line.c_str()); + current_buffer_position += current_line.length(); + } + + MPI_Barrier(world); + MPI_File_write_at_all(myfile_parallel, file_stream_offset, print_buffer.get_kokkos_view().data(), buffer_size_per_element_line * nlocal_elements, MPI_CHAR, MPI_STATUS_IGNORE); + + MPI_File_close(&myfile_parallel); } /* ---------------------------------------------------------------------- Output Model Information in tecplot format ------------------------------------------------------------------------- */ -void Explicit_Solver::parallel_vtk_writer(){ - int num_dim = simparam.num_dims; - std::string current_file_name; - std::string base_file_name= "VTK"; - std::string base_file_name_undeformed= "VTK_undeformed"; - std::stringstream current_line_stream; - std::string current_line; - std::string file_extension= ".vtk"; - std::string file_count; - std::stringstream count_temp; - int time_step = 0; - int temp_convert; - int noutput, nvector; - bool displace_geometry = false; - MPI_Offset current_stream_position, header_stream_offset, file_stream_offset; - MPI_File myfile_parallel; - int buffer_size_per_element_line, nlocal_elements; - GO first_element_global_id; - int default_vector_count; - int buffer_size_per_node_line; - int nlocal_sorted_nodes; - GO first_node_global_id; - // Convert ijk index system to the finite element numbering convention - // for vertices in cell - - //comm to vectors with contigously sorted global indices from unsorted zoltan2 repartition map - sort_information(); - //set host views of the communicated data to print out from - const_host_vec_array sorted_node_coords = sorted_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_host_vec_array sorted_node_velocities = sorted_node_velocities_distributed->getLocalView (Tpetra::Access::ReadOnly); - //const_host_vec_array sorted_node_densities = sorted_node_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_host_elem_conn_array sorted_nodes_in_elem = sorted_nodes_in_elem_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_host_vec_array sorted_element_densities = sorted_element_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_host_vec_array sorted_node_densities; - if(simparam.topology_optimization_on){ - sorted_node_densities = sorted_node_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); - } - - CArrayKokkos convert_ijk_to_ensight(max_nodes_per_element); - CArrayKokkos tmp_ijk_indx(max_nodes_per_element); - convert_ijk_to_ensight(0) = 0; - convert_ijk_to_ensight(1) = 1; - convert_ijk_to_ensight(2) = 3; - convert_ijk_to_ensight(3) = 2; - if(num_dim==3){ - convert_ijk_to_ensight(4) = 4; - convert_ijk_to_ensight(5) = 5; - convert_ijk_to_ensight(6) = 7; - convert_ijk_to_ensight(7) = 6; - } - - header_stream_offset = 0; - //initial undeformed geometry - count_temp.str(""); - count_temp << file_index; - file_index++; - file_count = count_temp.str(); - if(displace_geometry&&displacement_module>=0) - current_file_name = base_file_name_undeformed + file_count + file_extension; - else - current_file_name = base_file_name + file_count + file_extension; - MPI_File_open(MPI_COMM_WORLD, current_file_name.c_str(), - MPI_MODE_CREATE|MPI_MODE_WRONLY, +void Explicit_Solver::parallel_vtk_writer() +{ + int num_dim = simparam.num_dims; + std::string current_file_name; + std::string base_file_name = "VTK"; + std::string base_file_name_undeformed = "VTK_undeformed"; + std::stringstream current_line_stream; + std::string current_line; + std::string file_extension = ".vtk"; + std::string file_count; + std::stringstream count_temp; + int time_step = 0; + int temp_convert; + int noutput, nvector; + bool displace_geometry = false; + MPI_Offset current_stream_position, header_stream_offset, file_stream_offset; + MPI_File myfile_parallel; + int buffer_size_per_element_line, nlocal_elements; + GO first_element_global_id; + int default_vector_count; + int buffer_size_per_node_line; + int nlocal_sorted_nodes; + GO first_node_global_id; + // Convert ijk index system to the finite element numbering convention + // for vertices in cell + + // comm to vectors with contigously sorted global indices from unsorted zoltan2 repartition map + sort_information(); + // set host views of the communicated data to print out from + const_host_vec_array sorted_node_coords = sorted_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_host_vec_array sorted_node_velocities = sorted_node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); + // const_host_vec_array sorted_node_densities = sorted_node_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); + const_host_elem_conn_array sorted_nodes_in_elem = sorted_nodes_in_elem_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_host_vec_array sorted_element_densities = sorted_element_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_host_vec_array sorted_node_densities; + if (simparam.topology_optimization_on) + { + sorted_node_densities = sorted_node_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); + } + + CArrayKokkos convert_ijk_to_ensight(max_nodes_per_element); + CArrayKokkos tmp_ijk_indx(max_nodes_per_element); + convert_ijk_to_ensight(0) = 0; + convert_ijk_to_ensight(1) = 1; + convert_ijk_to_ensight(2) = 3; + convert_ijk_to_ensight(3) = 2; + if (num_dim == 3) + { + convert_ijk_to_ensight(4) = 4; + convert_ijk_to_ensight(5) = 5; + convert_ijk_to_ensight(6) = 7; + convert_ijk_to_ensight(7) = 6; + } + + header_stream_offset = 0; + // initial undeformed geometry + count_temp.str(""); + count_temp << file_index; + file_index++; + file_count = count_temp.str(); + if (displace_geometry && displacement_module >= 0) + { + current_file_name = base_file_name_undeformed + file_count + file_extension; + } + else + { + current_file_name = base_file_name + file_count + file_extension; + } + MPI_File_open(MPI_COMM_WORLD, current_file_name.c_str(), + MPI_MODE_CREATE | MPI_MODE_WRONLY, MPI_INFO_NULL, &myfile_parallel); - - int err = MPI_File_open(MPI_COMM_WORLD, current_file_name.c_str(), MPI_MODE_CREATE|MPI_MODE_EXCL|MPI_MODE_WRONLY, MPI_INFO_NULL, &myfile_parallel); - //allows overwriting the file if it already existed in the directory - if (err != MPI_SUCCESS) { - if (myrank == 0){ - MPI_File_delete(current_file_name.c_str(),MPI_INFO_NULL); - } - MPI_File_open(MPI_COMM_WORLD, current_file_name.c_str(), MPI_MODE_CREATE|MPI_MODE_EXCL|MPI_MODE_WRONLY, MPI_INFO_NULL, &myfile_parallel); + + int err = MPI_File_open(MPI_COMM_WORLD, current_file_name.c_str(), MPI_MODE_CREATE | MPI_MODE_EXCL | MPI_MODE_WRONLY, MPI_INFO_NULL, &myfile_parallel); + // allows overwriting the file if it already existed in the directory + if (err != MPI_SUCCESS) + { + if (myrank == 0) + { + MPI_File_delete(current_file_name.c_str(), MPI_INFO_NULL); + } + MPI_File_open(MPI_COMM_WORLD, current_file_name.c_str(), MPI_MODE_CREATE | MPI_MODE_EXCL | MPI_MODE_WRONLY, MPI_INFO_NULL, &myfile_parallel); } - //output header of the tecplot file - - //std::cout << current_file_name << std::endl; - current_line_stream << "# vtk DataFile Version 2.0\n"; - current_line = current_line_stream.str(); - if(myrank == 0) - MPI_File_write(myfile_parallel,current_line.c_str(),current_line.length(), MPI_CHAR, MPI_STATUS_IGNORE); - header_stream_offset += current_line.length(); - //myfile << "VARIABLES = \"x\", \"y\", \"z\", \"density\", \"sigmaxx\", \"sigmayy\", \"sigmazz\", \"sigmaxy\", \"sigmaxz\", \"sigmayz\"" "\n"; - //else - current_line_stream.str(""); - current_line_stream << "Mesh for Fierro\n"; - current_line = current_line_stream.str(); - if(myrank == 0) - MPI_File_write(myfile_parallel,current_line.c_str(),current_line.length(), MPI_CHAR, MPI_STATUS_IGNORE); - header_stream_offset += current_line.length(); + // output header of the tecplot file + + // std::cout << current_file_name << std::endl; + current_line_stream << "# vtk DataFile Version 2.0\n"; + current_line = current_line_stream.str(); + if (myrank == 0) + { + MPI_File_write(myfile_parallel, current_line.c_str(), current_line.length(), MPI_CHAR, MPI_STATUS_IGNORE); + } + header_stream_offset += current_line.length(); + // myfile << "VARIABLES = \"x\", \"y\", \"z\", \"density\", \"sigmaxx\", \"sigmayy\", \"sigmazz\", \"sigmaxy\", \"sigmaxz\", \"sigmayz\"" "\n"; + // else + current_line_stream.str(""); + current_line_stream << "Mesh for Fierro\n"; + current_line = current_line_stream.str(); + if (myrank == 0) + { + MPI_File_write(myfile_parallel, current_line.c_str(), current_line.length(), MPI_CHAR, MPI_STATUS_IGNORE); + } + header_stream_offset += current_line.length(); /* for (int imodule = 0; imodule < nfea_modules; imodule++){ for(int ioutput = 0; ioutput < fea_modules[imodule]->noutput; ioutput++){ @@ -2191,48 +2432,57 @@ void Explicit_Solver::parallel_vtk_writer(){ } */ - current_line_stream.str(""); - current_line_stream << "ASCII\n"; - current_line = current_line_stream.str(); - if(myrank == 0) - MPI_File_write(myfile_parallel,current_line.c_str(),current_line.length(), MPI_CHAR, MPI_STATUS_IGNORE); - header_stream_offset += current_line.length(); - - current_line_stream.str(""); - current_line_stream << "DATASET UNSTRUCTURED_GRID\n\n"; - current_line = current_line_stream.str(); - if(myrank == 0) - MPI_File_write(myfile_parallel,current_line.c_str(),current_line.length(), MPI_CHAR, MPI_STATUS_IGNORE); - header_stream_offset += current_line.length(); - - current_line_stream.str(""); - current_line_stream << "POINTS " << num_nodes - << " float\n"; - current_line = current_line_stream.str(); - if(myrank == 0) - MPI_File_write(myfile_parallel,current_line.c_str(),current_line.length(), MPI_CHAR, MPI_STATUS_IGNORE); - header_stream_offset += current_line.length(); - - //output nodal data - //compute buffer output size and file stream offset for this MPI rank - default_vector_count = 1; - buffer_size_per_node_line = 26*default_vector_count*num_dim + 1; //25 width per number + 1 space times 6 entries plus line terminator - nlocal_sorted_nodes = sorted_map->getLocalNumElements(); - first_node_global_id = sorted_map->getGlobalElement(0); - CArrayKokkos print_buffer(buffer_size_per_node_line*nlocal_sorted_nodes); - file_stream_offset = buffer_size_per_node_line*first_node_global_id; - - //populate buffer - long long current_buffer_position = 0; - current_line_stream << std::fixed << std::setprecision(8); - for (int nodeline = 0; nodeline < nlocal_sorted_nodes; nodeline++) { current_line_stream.str(""); - current_line_stream << std::left << std::setw(25) << sorted_node_coords(nodeline,0) << " "; - current_line_stream << std::left << std::setw(25) << sorted_node_coords(nodeline,1) << " "; - if(num_dim==3) - current_line_stream << std::left << std::setw(25) << sorted_node_coords(nodeline,2) << " "; - - //myfile << std::setw(25) << collected_node_densities(nodeline,0) << " "; + current_line_stream << "ASCII\n"; + current_line = current_line_stream.str(); + if (myrank == 0) + { + MPI_File_write(myfile_parallel, current_line.c_str(), current_line.length(), MPI_CHAR, MPI_STATUS_IGNORE); + } + header_stream_offset += current_line.length(); + + current_line_stream.str(""); + current_line_stream << "DATASET UNSTRUCTURED_GRID\n\n"; + current_line = current_line_stream.str(); + if (myrank == 0) + { + MPI_File_write(myfile_parallel, current_line.c_str(), current_line.length(), MPI_CHAR, MPI_STATUS_IGNORE); + } + header_stream_offset += current_line.length(); + + current_line_stream.str(""); + current_line_stream << "POINTS " << num_nodes + << " float\n"; + current_line = current_line_stream.str(); + if (myrank == 0) + { + MPI_File_write(myfile_parallel, current_line.c_str(), current_line.length(), MPI_CHAR, MPI_STATUS_IGNORE); + } + header_stream_offset += current_line.length(); + + // output nodal data + // compute buffer output size and file stream offset for this MPI rank + default_vector_count = 1; + buffer_size_per_node_line = 26 * default_vector_count * num_dim + 1; // 25 width per number + 1 space times 6 entries plus line terminator + nlocal_sorted_nodes = sorted_map->getLocalNumElements(); + first_node_global_id = sorted_map->getGlobalElement(0); + CArrayKokkos print_buffer(buffer_size_per_node_line * nlocal_sorted_nodes); + file_stream_offset = buffer_size_per_node_line * first_node_global_id; + + // populate buffer + long long current_buffer_position = 0; + current_line_stream << std::fixed << std::setprecision(8); + for (int nodeline = 0; nodeline < nlocal_sorted_nodes; nodeline++) + { + current_line_stream.str(""); + current_line_stream << std::left << std::setw(25) << sorted_node_coords(nodeline, 0) << " "; + current_line_stream << std::left << std::setw(25) << sorted_node_coords(nodeline, 1) << " "; + if (num_dim == 3) + { + current_line_stream << std::left << std::setw(25) << sorted_node_coords(nodeline, 2) << " "; + } + + // myfile << std::setw(25) << collected_node_densities(nodeline,0) << " "; /* for (int imodule = 0; imodule < nfea_modules; imodule++){ noutput = fea_modules[imodule]->noutput; @@ -2253,496 +2503,570 @@ void Explicit_Solver::parallel_vtk_writer(){ } } */ - current_line_stream << std::endl; - - current_line = current_line_stream.str(); + current_line_stream << std::endl; - //copy current line over to C style string buffer (wrapped by matar) - strcpy(&print_buffer(current_buffer_position),current_line.c_str()); + current_line = current_line_stream.str(); - current_buffer_position += current_line.length(); - } + // copy current line over to C style string buffer (wrapped by matar) + strcpy(&print_buffer(current_buffer_position), current_line.c_str()); + + current_buffer_position += current_line.length(); + } + + // print buffers at offsets with collective MPI write + // MPI_Offset current_stream_position = MPI_File_get_position(myfile_parallel,0); + MPI_Barrier(world); + MPI_File_write_at_all(myfile_parallel, file_stream_offset + header_stream_offset, print_buffer.get_kokkos_view().data(), buffer_size_per_node_line * nlocal_sorted_nodes, MPI_CHAR, + MPI_STATUS_IGNORE); + // MPI_File_close(&myfile_parallel); + + // write element connectivity; reopen to reset offset baseline. + // err = MPI_File_open(MPI_COMM_WORLD, current_file_name.c_str(), MPI_MODE_APPEND|MPI_MODE_WRONLY, MPI_INFO_NULL, &myfile_parallel); + + header_stream_offset = 0; + MPI_Barrier(world); + MPI_File_sync(myfile_parallel); + MPI_File_seek_shared(myfile_parallel, 0, MPI_SEEK_END); + MPI_File_sync(myfile_parallel); + MPI_File_get_position_shared(myfile_parallel, ¤t_stream_position); + + // debug check + // std::cout << "offset on rank " << myrank << " is " << file_stream_offset + header_stream_offset + current_buffer_position << std::endl; + // std::cout << "get position on rank " << myrank << " is " << current_stream_position << std::endl; - //print buffers at offsets with collective MPI write - //MPI_Offset current_stream_position = MPI_File_get_position(myfile_parallel,0); - MPI_Barrier(world); - MPI_File_write_at_all(myfile_parallel, file_stream_offset + header_stream_offset, print_buffer.get_kokkos_view().data(), buffer_size_per_node_line*nlocal_sorted_nodes, MPI_CHAR, MPI_STATUS_IGNORE); - //MPI_File_close(&myfile_parallel); - - //write element connectivity; reopen to reset offset baseline. - //err = MPI_File_open(MPI_COMM_WORLD, current_file_name.c_str(), MPI_MODE_APPEND|MPI_MODE_WRONLY, MPI_INFO_NULL, &myfile_parallel); - - header_stream_offset = 0; - MPI_Barrier(world); - MPI_File_sync(myfile_parallel); - MPI_File_seek_shared(myfile_parallel, 0, MPI_SEEK_END); - MPI_File_sync(myfile_parallel); - MPI_File_get_position_shared(myfile_parallel, ¤t_stream_position); - - //debug check - //std::cout << "offset on rank " << myrank << " is " << file_stream_offset + header_stream_offset + current_buffer_position << std::endl; - //std::cout << "get position on rank " << myrank << " is " << current_stream_position << std::endl; - - current_line_stream.str(""); - current_line_stream << std::endl << "CELLS " << num_elem << " " << num_elem*(max_nodes_per_element+1) << std::endl; - current_line = current_line_stream.str(); - //std::cout << current_line; - file_stream_offset = current_stream_position; - if(myrank == 0) - MPI_File_write_at(myfile_parallel, file_stream_offset, current_line.c_str(),current_line.length(), MPI_CHAR, MPI_STATUS_IGNORE); - header_stream_offset += current_line.length(); - - //expand print buffer if needed - buffer_size_per_element_line = 11*(max_nodes_per_element+1) + 1; //25 width per number plus 6 spaces plus line terminator - nlocal_elements = sorted_element_map->getLocalNumElements(); - first_element_global_id = sorted_element_map->getGlobalElement(0); - if(buffer_size_per_element_line*nlocal_elements > print_buffer.size()) - print_buffer = CArrayKokkos(buffer_size_per_element_line*nlocal_elements); - file_stream_offset = buffer_size_per_element_line*first_element_global_id + current_stream_position + header_stream_offset; - - current_buffer_position = 0; - for (int elementline = 0; elementline < nlocal_elements; elementline++) { current_line_stream.str(""); - //convert node ordering - current_line_stream << std::left << std::setw(10) << max_nodes_per_element << " "; - for (int ii = 0; ii < max_nodes_per_element; ii++) { - if(active_node_ordering_convention == IJK) - temp_convert = convert_ijk_to_ensight(ii); - else - temp_convert = ii; - current_line_stream << std::left << std::setw(10) << sorted_nodes_in_elem(elementline, temp_convert)<< " "; - } - current_line_stream << std::endl; + current_line_stream << std::endl << "CELLS " << num_elem << " " << num_elem * (max_nodes_per_element + 1) << std::endl; current_line = current_line_stream.str(); + // std::cout << current_line; + file_stream_offset = current_stream_position; + if (myrank == 0) + { + MPI_File_write_at(myfile_parallel, file_stream_offset, current_line.c_str(), current_line.length(), MPI_CHAR, MPI_STATUS_IGNORE); + } + header_stream_offset += current_line.length(); + + // expand print buffer if needed + buffer_size_per_element_line = 11 * (max_nodes_per_element + 1) + 1; // 25 width per number plus 6 spaces plus line terminator + nlocal_elements = sorted_element_map->getLocalNumElements(); + first_element_global_id = sorted_element_map->getGlobalElement(0); + if (buffer_size_per_element_line * nlocal_elements > print_buffer.size()) + { + print_buffer = CArrayKokkos(buffer_size_per_element_line * nlocal_elements); + } + file_stream_offset = buffer_size_per_element_line * first_element_global_id + current_stream_position + header_stream_offset; + + current_buffer_position = 0; + for (int elementline = 0; elementline < nlocal_elements; elementline++) + { + current_line_stream.str(""); + // convert node ordering + current_line_stream << std::left << std::setw(10) << max_nodes_per_element << " "; + for (int ii = 0; ii < max_nodes_per_element; ii++) + { + if (active_node_ordering_convention == IJK) + { + temp_convert = convert_ijk_to_ensight(ii); + } + else + { + temp_convert = ii; + } + current_line_stream << std::left << std::setw(10) << sorted_nodes_in_elem(elementline, temp_convert) << " "; + } + current_line_stream << std::endl; + current_line = current_line_stream.str(); - //copy current line over to C style string buffer (wrapped by matar) - strcpy(&print_buffer(current_buffer_position),current_line.c_str()); + // copy current line over to C style string buffer (wrapped by matar) + strcpy(&print_buffer(current_buffer_position), current_line.c_str()); - current_buffer_position += current_line.length(); - } - - MPI_Barrier(world); - MPI_File_write_at_all(myfile_parallel, file_stream_offset, print_buffer.get_kokkos_view().data(), buffer_size_per_element_line*nlocal_elements, MPI_CHAR, MPI_STATUS_IGNORE); + current_buffer_position += current_line.length(); + } - //print Element Types + MPI_Barrier(world); + MPI_File_write_at_all(myfile_parallel, file_stream_offset, print_buffer.get_kokkos_view().data(), buffer_size_per_element_line * nlocal_elements, MPI_CHAR, MPI_STATUS_IGNORE); + + // print Element Types + + header_stream_offset = 0; + MPI_Barrier(world); + MPI_File_sync(myfile_parallel); + MPI_File_seek_shared(myfile_parallel, 0, MPI_SEEK_END); + MPI_File_sync(myfile_parallel); + MPI_File_get_position_shared(myfile_parallel, ¤t_stream_position); + + // debug check + // std::cout << "offset on rank " << myrank << " is " << file_stream_offset + header_stream_offset + current_buffer_position << std::endl; + // std::cout << "get position on rank " << myrank << " is " << current_stream_position << std::endl; - header_stream_offset = 0; - MPI_Barrier(world); - MPI_File_sync(myfile_parallel); - MPI_File_seek_shared(myfile_parallel, 0, MPI_SEEK_END); - MPI_File_sync(myfile_parallel); - MPI_File_get_position_shared(myfile_parallel, ¤t_stream_position); - - //debug check - //std::cout << "offset on rank " << myrank << " is " << file_stream_offset + header_stream_offset + current_buffer_position << std::endl; - //std::cout << "get position on rank " << myrank << " is " << current_stream_position << std::endl; - - current_line_stream.str(""); - current_line_stream << std::endl << "CELL_TYPES " << num_elem << std::endl; - current_line = current_line_stream.str(); - //std::cout << current_line; - file_stream_offset = current_stream_position; - if(myrank == 0) - MPI_File_write_at(myfile_parallel, file_stream_offset, current_line.c_str(),current_line.length(), MPI_CHAR, MPI_STATUS_IGNORE); - header_stream_offset += current_line.length(); - - //expand print buffer if needed - buffer_size_per_element_line = 11; //10 width per number plus line terminator - nlocal_elements = sorted_element_map->getLocalNumElements(); - first_element_global_id = sorted_element_map->getGlobalElement(0); - if(buffer_size_per_element_line*nlocal_elements > print_buffer.size()) - print_buffer = CArrayKokkos(buffer_size_per_element_line*nlocal_elements); - file_stream_offset = buffer_size_per_element_line*first_element_global_id + current_stream_position + header_stream_offset; - - current_buffer_position = 0; - for (int elementline = 0; elementline < nlocal_elements; elementline++) { current_line_stream.str(""); - //convert node ordering - current_line_stream << std::left << std::setw(10) << 12 << std::endl; + current_line_stream << std::endl << "CELL_TYPES " << num_elem << std::endl; current_line = current_line_stream.str(); + // std::cout << current_line; + file_stream_offset = current_stream_position; + if (myrank == 0) + { + MPI_File_write_at(myfile_parallel, file_stream_offset, current_line.c_str(), current_line.length(), MPI_CHAR, MPI_STATUS_IGNORE); + } + header_stream_offset += current_line.length(); + + // expand print buffer if needed + buffer_size_per_element_line = 11; // 10 width per number plus line terminator + nlocal_elements = sorted_element_map->getLocalNumElements(); + first_element_global_id = sorted_element_map->getGlobalElement(0); + if (buffer_size_per_element_line * nlocal_elements > print_buffer.size()) + { + print_buffer = CArrayKokkos(buffer_size_per_element_line * nlocal_elements); + } + file_stream_offset = buffer_size_per_element_line * first_element_global_id + current_stream_position + header_stream_offset; - //copy current line over to C style string buffer (wrapped by matar) - strcpy(&print_buffer(current_buffer_position),current_line.c_str()); + current_buffer_position = 0; + for (int elementline = 0; elementline < nlocal_elements; elementline++) + { + current_line_stream.str(""); + // convert node ordering + current_line_stream << std::left << std::setw(10) << 12 << std::endl; + current_line = current_line_stream.str(); - current_buffer_position += current_line.length(); - } - - MPI_Barrier(world); - MPI_File_write_at_all(myfile_parallel, file_stream_offset, print_buffer.get_kokkos_view().data(), buffer_size_per_element_line*nlocal_elements, MPI_CHAR, MPI_STATUS_IGNORE); + // copy current line over to C style string buffer (wrapped by matar) + strcpy(&print_buffer(current_buffer_position), current_line.c_str()); - //Print Node scalars + current_buffer_position += current_line.length(); + } + + MPI_Barrier(world); + MPI_File_write_at_all(myfile_parallel, file_stream_offset, print_buffer.get_kokkos_view().data(), buffer_size_per_element_line * nlocal_elements, MPI_CHAR, MPI_STATUS_IGNORE); + + // Print Node scalars + + header_stream_offset = 0; + MPI_Barrier(world); + MPI_File_sync(myfile_parallel); + MPI_File_seek_shared(myfile_parallel, 0, MPI_SEEK_END); + MPI_File_sync(myfile_parallel); + MPI_File_get_position_shared(myfile_parallel, ¤t_stream_position); + + // debug check + // std::cout << "offset on rank " << myrank << " is " << file_stream_offset + header_stream_offset + current_buffer_position << std::endl; + // std::cout << "get position on rank " << myrank << " is " << current_stream_position << std::endl; - header_stream_offset = 0; - MPI_Barrier(world); - MPI_File_sync(myfile_parallel); - MPI_File_seek_shared(myfile_parallel, 0, MPI_SEEK_END); - MPI_File_sync(myfile_parallel); - MPI_File_get_position_shared(myfile_parallel, ¤t_stream_position); - - //debug check - //std::cout << "offset on rank " << myrank << " is " << file_stream_offset + header_stream_offset + current_buffer_position << std::endl; - //std::cout << "get position on rank " << myrank << " is " << current_stream_position << std::endl; - - current_line_stream.str(""); - current_line_stream << std::endl << "POINT_DATA " << num_nodes << std::endl; - current_line_stream << "SCALARS design_density float 1" << std::endl; - current_line_stream << "LOOKUP_TABLE default" << std::endl; - current_line = current_line_stream.str(); - //std::cout << current_line; - file_stream_offset = current_stream_position; - if(myrank == 0) - MPI_File_write_at(myfile_parallel, file_stream_offset, current_line.c_str(),current_line.length(), MPI_CHAR, MPI_STATUS_IGNORE); - header_stream_offset += current_line.length(); - - //expand print buffer if needed - default_vector_count = 1; - buffer_size_per_node_line = 26; //25 width per number + 1 space times 6 entries plus line terminator - nlocal_sorted_nodes = sorted_map->getLocalNumElements(); - first_node_global_id = sorted_map->getGlobalElement(0); - if(buffer_size_per_node_line*nlocal_sorted_nodes > print_buffer.size()) - print_buffer = CArrayKokkos(buffer_size_per_node_line*nlocal_sorted_nodes); - file_stream_offset = buffer_size_per_node_line*first_node_global_id + current_stream_position + header_stream_offset; - - current_buffer_position = 0; - current_line_stream << std::fixed << std::setprecision(8); - for (int nodeline = 0; nodeline < nlocal_sorted_nodes; nodeline++) { current_line_stream.str(""); - //convert node ordering - if(simparam.topology_optimization_on) - current_line_stream << std::left << std::setw(25) << sorted_node_densities(nodeline,0) << std::endl; - else - current_line_stream << std::left << std::setw(25) << 1 << std::endl; + current_line_stream << std::endl << "POINT_DATA " << num_nodes << std::endl; + current_line_stream << "SCALARS design_density float 1" << std::endl; + current_line_stream << "LOOKUP_TABLE default" << std::endl; current_line = current_line_stream.str(); + // std::cout << current_line; + file_stream_offset = current_stream_position; + if (myrank == 0) + { + MPI_File_write_at(myfile_parallel, file_stream_offset, current_line.c_str(), current_line.length(), MPI_CHAR, MPI_STATUS_IGNORE); + } + header_stream_offset += current_line.length(); + + // expand print buffer if needed + default_vector_count = 1; + buffer_size_per_node_line = 26; // 25 width per number + 1 space times 6 entries plus line terminator + nlocal_sorted_nodes = sorted_map->getLocalNumElements(); + first_node_global_id = sorted_map->getGlobalElement(0); + if (buffer_size_per_node_line * nlocal_sorted_nodes > print_buffer.size()) + { + print_buffer = CArrayKokkos(buffer_size_per_node_line * nlocal_sorted_nodes); + } + file_stream_offset = buffer_size_per_node_line * first_node_global_id + current_stream_position + header_stream_offset; + + current_buffer_position = 0; + current_line_stream << std::fixed << std::setprecision(8); + for (int nodeline = 0; nodeline < nlocal_sorted_nodes; nodeline++) + { + current_line_stream.str(""); + // convert node ordering + if (simparam.topology_optimization_on) + { + current_line_stream << std::left << std::setw(25) << sorted_node_densities(nodeline, 0) << std::endl; + } + else + { + current_line_stream << std::left << std::setw(25) << 1 << std::endl; + } + current_line = current_line_stream.str(); - //copy current line over to C style string buffer (wrapped by matar) - strcpy(&print_buffer(current_buffer_position),current_line.c_str()); + // copy current line over to C style string buffer (wrapped by matar) + strcpy(&print_buffer(current_buffer_position), current_line.c_str()); - current_buffer_position += current_line.length(); - } - - MPI_Barrier(world); - MPI_File_write_at_all(myfile_parallel, file_stream_offset, print_buffer.get_kokkos_view().data(), buffer_size_per_node_line*nlocal_sorted_nodes, MPI_CHAR, MPI_STATUS_IGNORE); + current_buffer_position += current_line.length(); + } - //Print Node vectors + MPI_Barrier(world); + MPI_File_write_at_all(myfile_parallel, file_stream_offset, print_buffer.get_kokkos_view().data(), buffer_size_per_node_line * nlocal_sorted_nodes, MPI_CHAR, MPI_STATUS_IGNORE); + + // Print Node vectors + + header_stream_offset = 0; + MPI_Barrier(world); + MPI_File_sync(myfile_parallel); + MPI_File_seek_shared(myfile_parallel, 0, MPI_SEEK_END); + MPI_File_sync(myfile_parallel); + MPI_File_get_position_shared(myfile_parallel, ¤t_stream_position); + + // debug check + // std::cout << "offset on rank " << myrank << " is " << file_stream_offset + header_stream_offset + current_buffer_position << std::endl; + // std::cout << "get position on rank " << myrank << " is " << current_stream_position << std::endl; - header_stream_offset = 0; - MPI_Barrier(world); - MPI_File_sync(myfile_parallel); - MPI_File_seek_shared(myfile_parallel, 0, MPI_SEEK_END); - MPI_File_sync(myfile_parallel); - MPI_File_get_position_shared(myfile_parallel, ¤t_stream_position); - - //debug check - //std::cout << "offset on rank " << myrank << " is " << file_stream_offset + header_stream_offset + current_buffer_position << std::endl; - //std::cout << "get position on rank " << myrank << " is " << current_stream_position << std::endl; - - current_line_stream.str(""); - current_line_stream << std::endl << "VECTORS velocity float" << std::endl; - current_line = current_line_stream.str(); - //std::cout << current_line; - file_stream_offset = current_stream_position; - if(myrank == 0) - MPI_File_write_at(myfile_parallel, file_stream_offset, current_line.c_str(),current_line.length(), MPI_CHAR, MPI_STATUS_IGNORE); - header_stream_offset += current_line.length(); - - //expand print buffer if needed - default_vector_count = 1; - buffer_size_per_node_line = num_dim*26 + 1; //25 width per number + 1 space times 6 entries plus line terminator - nlocal_sorted_nodes = sorted_map->getLocalNumElements(); - first_node_global_id = sorted_map->getGlobalElement(0); - if(buffer_size_per_node_line*nlocal_sorted_nodes > print_buffer.size()) - print_buffer = CArrayKokkos(buffer_size_per_node_line*nlocal_sorted_nodes); - file_stream_offset = buffer_size_per_node_line*first_node_global_id + current_stream_position + header_stream_offset; - - current_buffer_position = 0; - current_line_stream << std::fixed << std::setprecision(8); - for (int nodeline = 0; nodeline < nlocal_sorted_nodes; nodeline++) { current_line_stream.str(""); - //convert node ordering - //velocity print - current_line_stream << std::left << std::setw(25) << sorted_node_velocities(nodeline,0) << " "; - current_line_stream << std::left << std::setw(25) << sorted_node_velocities(nodeline,1) << " "; - if(num_dim==3) - current_line_stream << std::left << std::setw(25) << sorted_node_velocities(nodeline,2) << " "; - - current_line_stream << std::endl; + current_line_stream << std::endl << "VECTORS velocity float" << std::endl; current_line = current_line_stream.str(); + // std::cout << current_line; + file_stream_offset = current_stream_position; + if (myrank == 0) + { + MPI_File_write_at(myfile_parallel, file_stream_offset, current_line.c_str(), current_line.length(), MPI_CHAR, MPI_STATUS_IGNORE); + } + header_stream_offset += current_line.length(); + + // expand print buffer if needed + default_vector_count = 1; + buffer_size_per_node_line = num_dim * 26 + 1; // 25 width per number + 1 space times 6 entries plus line terminator + nlocal_sorted_nodes = sorted_map->getLocalNumElements(); + first_node_global_id = sorted_map->getGlobalElement(0); + if (buffer_size_per_node_line * nlocal_sorted_nodes > print_buffer.size()) + { + print_buffer = CArrayKokkos(buffer_size_per_node_line * nlocal_sorted_nodes); + } + file_stream_offset = buffer_size_per_node_line * first_node_global_id + current_stream_position + header_stream_offset; + + current_buffer_position = 0; + current_line_stream << std::fixed << std::setprecision(8); + for (int nodeline = 0; nodeline < nlocal_sorted_nodes; nodeline++) + { + current_line_stream.str(""); + // convert node ordering + // velocity print + current_line_stream << std::left << std::setw(25) << sorted_node_velocities(nodeline, 0) << " "; + current_line_stream << std::left << std::setw(25) << sorted_node_velocities(nodeline, 1) << " "; + if (num_dim == 3) + { + current_line_stream << std::left << std::setw(25) << sorted_node_velocities(nodeline, 2) << " "; + } - //copy current line over to C style string buffer (wrapped by matar) - strcpy(&print_buffer(current_buffer_position),current_line.c_str()); + current_line_stream << std::endl; + current_line = current_line_stream.str(); - current_buffer_position += current_line.length(); - } - - MPI_Barrier(world); - MPI_File_write_at_all(myfile_parallel, file_stream_offset, print_buffer.get_kokkos_view().data(), buffer_size_per_node_line*nlocal_sorted_nodes, MPI_CHAR, MPI_STATUS_IGNORE); + // copy current line over to C style string buffer (wrapped by matar) + strcpy(&print_buffer(current_buffer_position), current_line.c_str()); - //print Element Fields + current_buffer_position += current_line.length(); + } + + MPI_Barrier(world); + MPI_File_write_at_all(myfile_parallel, file_stream_offset, print_buffer.get_kokkos_view().data(), buffer_size_per_node_line * nlocal_sorted_nodes, MPI_CHAR, MPI_STATUS_IGNORE); + + // print Element Fields + + header_stream_offset = 0; + MPI_Barrier(world); + MPI_File_sync(myfile_parallel); + MPI_File_seek_shared(myfile_parallel, 0, MPI_SEEK_END); + MPI_File_sync(myfile_parallel); + MPI_File_get_position_shared(myfile_parallel, ¤t_stream_position); + + // debug check + // std::cout << "offset on rank " << myrank << " is " << file_stream_offset + header_stream_offset + current_buffer_position << std::endl; + // std::cout << "get position on rank " << myrank << " is " << current_stream_position << std::endl; - header_stream_offset = 0; - MPI_Barrier(world); - MPI_File_sync(myfile_parallel); - MPI_File_seek_shared(myfile_parallel, 0, MPI_SEEK_END); - MPI_File_sync(myfile_parallel); - MPI_File_get_position_shared(myfile_parallel, ¤t_stream_position); - - //debug check - //std::cout << "offset on rank " << myrank << " is " << file_stream_offset + header_stream_offset + current_buffer_position << std::endl; - //std::cout << "get position on rank " << myrank << " is " << current_stream_position << std::endl; - - current_line_stream.str(""); - current_line_stream << std::endl << "CELL_DATA " << num_elem << std::endl; - current_line_stream << "SCALARS element_density float 1" << std::endl; - current_line_stream << "LOOKUP_TABLE default" << std::endl; - current_line = current_line_stream.str(); - //std::cout << current_line; - file_stream_offset = current_stream_position; - if(myrank == 0) - MPI_File_write_at(myfile_parallel, file_stream_offset, current_line.c_str(),current_line.length(), MPI_CHAR, MPI_STATUS_IGNORE); - header_stream_offset += current_line.length(); - - //expand print buffer if needed - buffer_size_per_element_line = 26; //25 width per number plus line terminator - nlocal_elements = sorted_element_map->getLocalNumElements(); - first_element_global_id = sorted_element_map->getGlobalElement(0); - if(buffer_size_per_element_line*nlocal_elements > print_buffer.size()) - print_buffer = CArrayKokkos(buffer_size_per_element_line*nlocal_elements); - file_stream_offset = buffer_size_per_element_line*first_element_global_id + current_stream_position + header_stream_offset; - - current_buffer_position = 0; - current_line_stream << std::fixed << std::setprecision(8); - for (int elementline = 0; elementline < nlocal_elements; elementline++) { current_line_stream.str(""); - //convert node ordering - current_line_stream << std::left << std::setw(25) << sorted_element_densities(elementline,0) << std::endl; + current_line_stream << std::endl << "CELL_DATA " << num_elem << std::endl; + current_line_stream << "SCALARS element_density float 1" << std::endl; + current_line_stream << "LOOKUP_TABLE default" << std::endl; current_line = current_line_stream.str(); + // std::cout << current_line; + file_stream_offset = current_stream_position; + if (myrank == 0) + { + MPI_File_write_at(myfile_parallel, file_stream_offset, current_line.c_str(), current_line.length(), MPI_CHAR, MPI_STATUS_IGNORE); + } + header_stream_offset += current_line.length(); + + // expand print buffer if needed + buffer_size_per_element_line = 26; // 25 width per number plus line terminator + nlocal_elements = sorted_element_map->getLocalNumElements(); + first_element_global_id = sorted_element_map->getGlobalElement(0); + if (buffer_size_per_element_line * nlocal_elements > print_buffer.size()) + { + print_buffer = CArrayKokkos(buffer_size_per_element_line * nlocal_elements); + } + file_stream_offset = buffer_size_per_element_line * first_element_global_id + current_stream_position + header_stream_offset; - //copy current line over to C style string buffer (wrapped by matar) - strcpy(&print_buffer(current_buffer_position),current_line.c_str()); + current_buffer_position = 0; + current_line_stream << std::fixed << std::setprecision(8); + for (int elementline = 0; elementline < nlocal_elements; elementline++) + { + current_line_stream.str(""); + // convert node ordering + current_line_stream << std::left << std::setw(25) << sorted_element_densities(elementline, 0) << std::endl; + current_line = current_line_stream.str(); - current_buffer_position += current_line.length(); - } - - MPI_Barrier(world); - MPI_File_write_at_all(myfile_parallel, file_stream_offset, print_buffer.get_kokkos_view().data(), buffer_size_per_element_line*nlocal_elements, MPI_CHAR, MPI_STATUS_IGNORE); - - MPI_Barrier(world); - MPI_File_sync(myfile_parallel); - MPI_File_close(&myfile_parallel); + // copy current line over to C style string buffer (wrapped by matar) + strcpy(&print_buffer(current_buffer_position), current_line.c_str()); + current_buffer_position += current_line.length(); + } + + MPI_Barrier(world); + MPI_File_write_at_all(myfile_parallel, file_stream_offset, print_buffer.get_kokkos_view().data(), buffer_size_per_element_line * nlocal_elements, MPI_CHAR, MPI_STATUS_IGNORE); + + MPI_Barrier(world); + MPI_File_sync(myfile_parallel); + MPI_File_close(&myfile_parallel); } /* ---------------------------------------------------------------------- Output Model Information in tecplot format ------------------------------------------------------------------------- */ -void Explicit_Solver::tecplot_writer(){ - - int num_dim = simparam.num_dims; - std::string current_file_name; - std::string base_file_name= "TecplotTO"; - std::string base_file_name_undeformed= "TecplotTO_undeformed"; - std::stringstream ss; - std::string file_extension= ".dat"; - std::string file_count; - std::stringstream count_temp; - int time_step = 0; - int temp_convert; - int noutput, nvector; - bool displace_geometry = false; - const_host_vec_array current_collected_output; - int displacement_index; - collect_information(); - //set host views of the collected data to print out from - const_host_vec_array collected_node_coords = collected_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_host_vec_array collected_node_velocities = collected_node_velocities_distributed->getLocalView (Tpetra::Access::ReadOnly); - //const_host_vec_array collected_node_densities = collected_node_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_host_elem_conn_array collected_nodes_in_elem = collected_nodes_in_elem_distributed->getLocalView (Tpetra::Access::ReadOnly); - - // Convert ijk index system to the finite element numbering convention - // for vertices in cell - CArrayKokkos convert_ijk_to_ensight(max_nodes_per_element); - CArrayKokkos tmp_ijk_indx(max_nodes_per_element); - convert_ijk_to_ensight(0) = 0; - convert_ijk_to_ensight(1) = 1; - convert_ijk_to_ensight(2) = 3; - convert_ijk_to_ensight(3) = 2; - if(num_dim==3){ - convert_ijk_to_ensight(4) = 4; - convert_ijk_to_ensight(5) = 5; - convert_ijk_to_ensight(6) = 7; - convert_ijk_to_ensight(7) = 6; - } - - //compared to primitive unit cell, assumes orthogonal primitive unit cell - if(myrank==0){ - //initial undeformed geometry - count_temp.str(""); - count_temp << file_index; - file_index++; - file_count = count_temp.str(); - if(displace_geometry&&displacement_module>=0) - current_file_name = base_file_name_undeformed + file_count + file_extension; - else - current_file_name = base_file_name + file_count + file_extension; - std::ofstream myfile (current_file_name.c_str()); //output filestream object for file output - //read in position data - myfile << std::fixed << std::setprecision(8); - - //output header of the tecplot file - - myfile << "TITLE=\"results for TO code\"" "\n"; - //myfile << "VARIABLES = \"x\", \"y\", \"z\", \"density\", \"sigmaxx\", \"sigmayy\", \"sigmazz\", \"sigmaxy\", \"sigmaxz\", \"sigmayz\"" "\n"; - //else - myfile << "VARIABLES = \"x\", \"y\", \"z\", \"vx\", \"vy\", \"vz\""; - /* - for (int imodule = 0; imodule < nfea_modules; imodule++){ - for(int ioutput = 0; ioutput < fea_modules[imodule]->noutput; ioutput++){ - nvector = fea_modules[imodule]->output_vector_sizes[ioutput]; - for(int ivector = 0; ivector < nvector; ivector++){ - myfile << ", \"" << fea_modules[imodule]->output_dof_names[ioutput][ivector] << "\""; - } - } - } - */ - myfile << "\n"; - - if(num_dim==2){ - myfile << "ZONE T=\"load step " << time_step << "\", NODES= " << num_nodes - << ", ELEMENTS= " << num_elem << ", DATAPACKING=POINT, ZONETYPE=FEQUADRILATERAL" "\n"; - } - else if(num_dim==3){ - myfile << "ZONE T=\"load step " << time_step << "\", NODES= " << num_nodes - << ", ELEMENTS= " << num_elem << ", DATAPACKING=POINT, ZONETYPE=FEBRICK" "\n"; - } +void Explicit_Solver::tecplot_writer() +{ + int num_dim = simparam.num_dims; + std::string current_file_name; + std::string base_file_name = "TecplotTO"; + std::string base_file_name_undeformed = "TecplotTO_undeformed"; + std::stringstream ss; + std::string file_extension = ".dat"; + std::string file_count; + std::stringstream count_temp; + int time_step = 0; + int temp_convert; + int noutput, nvector; + bool displace_geometry = false; + const_host_vec_array current_collected_output; + int displacement_index; + collect_information(); + // set host views of the collected data to print out from + const_host_vec_array collected_node_coords = collected_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_host_vec_array collected_node_velocities = collected_node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); + // const_host_vec_array collected_node_densities = collected_node_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); + const_host_elem_conn_array collected_nodes_in_elem = collected_nodes_in_elem_distributed->getLocalView(Tpetra::Access::ReadOnly); + + // Convert ijk index system to the finite element numbering convention + // for vertices in cell + CArrayKokkos convert_ijk_to_ensight(max_nodes_per_element); + CArrayKokkos tmp_ijk_indx(max_nodes_per_element); + convert_ijk_to_ensight(0) = 0; + convert_ijk_to_ensight(1) = 1; + convert_ijk_to_ensight(2) = 3; + convert_ijk_to_ensight(3) = 2; + if (num_dim == 3) + { + convert_ijk_to_ensight(4) = 4; + convert_ijk_to_ensight(5) = 5; + convert_ijk_to_ensight(6) = 7; + convert_ijk_to_ensight(7) = 6; + } - for (int nodeline = 0; nodeline < num_nodes; nodeline++) { - myfile << std::setw(25) << collected_node_coords(nodeline,0) << " "; - myfile << std::setw(25) << collected_node_coords(nodeline,1) << " "; - if(num_dim==3) - myfile << std::setw(25) << collected_node_coords(nodeline,2) << " "; + // compared to primitive unit cell, assumes orthogonal primitive unit cell + if (myrank == 0) + { + // initial undeformed geometry + count_temp.str(""); + count_temp << file_index; + file_index++; + file_count = count_temp.str(); + if (displace_geometry && displacement_module >= 0) + { + current_file_name = base_file_name_undeformed + file_count + file_extension; + } + else + { + current_file_name = base_file_name + file_count + file_extension; + } + std::ofstream myfile(current_file_name.c_str()); // output filestream object for file output + // read in position data + myfile << std::fixed << std::setprecision(8); - //velocity print - myfile << std::setw(25) << collected_node_velocities(nodeline,0) << " "; - myfile << std::setw(25) << collected_node_velocities(nodeline,1) << " "; - if(num_dim==3) - myfile << std::setw(25) << collected_node_velocities(nodeline,2) << " "; + // output header of the tecplot file - //myfile << std::setw(25) << collected_node_densities(nodeline,0) << " "; + myfile << "TITLE=\"results for TO code\"" "\n"; + // myfile << "VARIABLES = \"x\", \"y\", \"z\", \"density\", \"sigmaxx\", \"sigmayy\", \"sigmazz\", \"sigmaxy\", \"sigmaxz\", \"sigmayz\"" "\n"; + // else + myfile << "VARIABLES = \"x\", \"y\", \"z\", \"vx\", \"vy\", \"vz\""; /* for (int imodule = 0; imodule < nfea_modules; imodule++){ - noutput = fea_modules[imodule]->noutput; - for(int ioutput = 0; ioutput < noutput; ioutput++){ - current_collected_output = fea_modules[imodule]->module_outputs[ioutput]; - if(fea_modules[imodule]->vector_style[ioutput] == FEA_Module::DOF){ - nvector = fea_modules[imodule]->output_vector_sizes[ioutput]; - for(int ivector = 0; ivector < nvector; ivector++){ - myfile << std::setw(25) << current_collected_output(nodeline*nvector + ivector,0) << " "; - } - } - if(fea_modules[imodule]->vector_style[ioutput] == FEA_Module::NODAL){ - nvector = fea_modules[imodule]->output_vector_sizes[ioutput]; - for(int ivector = 0; ivector < nvector; ivector++){ - myfile << std::setw(25) << current_collected_output(nodeline,ivector) << " "; - } + for(int ioutput = 0; ioutput < fea_modules[imodule]->noutput; ioutput++){ + nvector = fea_modules[imodule]->output_vector_sizes[ioutput]; + for(int ivector = 0; ivector < nvector; ivector++){ + myfile << ", \"" << fea_modules[imodule]->output_dof_names[ioutput][ivector] << "\""; } } } */ - myfile << std::endl; - } - for (int elementline = 0; elementline < num_elem; elementline++) { - //convert node ordering - for (int ii = 0; ii < max_nodes_per_element; ii++) { - if(active_node_ordering_convention == IJK) - temp_convert = convert_ijk_to_ensight(ii); - else - temp_convert = ii; - myfile << std::setw(10) << collected_nodes_in_elem(elementline, temp_convert) + 1 << " "; - } - myfile << " \n"; - } - myfile.close(); - } - if(myrank==0&&(displacement_module>=0&&displace_geometry)){ - //deformed geometry - count_temp.str(""); - count_temp << file_index; - file_index++; - file_count = count_temp.str(); - - current_file_name = base_file_name + file_count + file_extension; - std::ofstream myfile (current_file_name.c_str()); //output filestream object for file output - //read in position data - myfile << std::fixed << std::setprecision(8); - - //output header of the tecplot file - - myfile << "TITLE=\"results for TO code\" \n"; - myfile << "VARIABLES = \"x\", \"y\", \"z\", \"vx\", \"vy\", \"vz\""; - /* - for (int imodule = 0; imodule < nfea_modules; imodule++){ - for(int ioutput = 0; ioutput < fea_modules[imodule]->noutput; ioutput++){ - nvector = fea_modules[imodule]->output_vector_sizes[ioutput]; - for(int ivector = 0; ivector < nvector; ivector++){ - myfile << ", \"" << fea_modules[imodule]->output_dof_names[ioutput][ivector] << "\""; - } + myfile << "\n"; + + if (num_dim == 2) + { + myfile << "ZONE T=\"load step " << time_step << "\", NODES= " << num_nodes + << ", ELEMENTS= " << num_elem << ", DATAPACKING=POINT, ZONETYPE=FEQUADRILATERAL" "\n"; } - } - */ - myfile << "\n"; - - myfile << "ZONE T=\"load step " << time_step + 1 << "\", NODES= " << num_nodes - << ", ELEMENTS= " << num_elem << ", DATAPACKING=POINT, ZONETYPE=FEBRICK" "\n"; - - for (int nodeline = 0; nodeline < num_nodes; nodeline++) { - current_collected_output = fea_modules[displacement_module]->module_outputs[displacement_index]; - myfile << std::setw(25) << collected_node_coords(nodeline,0) + current_collected_output(nodeline*num_dim,0) << " "; - myfile << std::setw(25) << collected_node_coords(nodeline,1) + current_collected_output(nodeline*num_dim + 1,0) << " "; - if(num_dim==3) - myfile << std::setw(25) << collected_node_coords(nodeline,2) + current_collected_output(nodeline*num_dim + 2,0) << " "; - - //velocity print - myfile << std::setw(25) << collected_node_velocities(nodeline,0) << " "; - myfile << std::setw(25) << collected_node_velocities(nodeline,1) << " "; - if(num_dim==3) - myfile << std::setw(25) << collected_node_velocities(nodeline,2) << " "; - - //myfile << std::setw(25) << collected_node_densities(nodeline,0) << " "; - for (int imodule = 0; imodule < nfea_modules; imodule++){ - noutput = fea_modules[imodule]->noutput; - for(int ioutput = 0; ioutput < noutput; ioutput++){ - current_collected_output = fea_modules[imodule]->module_outputs[ioutput]; - if(fea_modules[imodule]->vector_style[ioutput] == FEA_Module::DOF){ - nvector = fea_modules[imodule]->output_vector_sizes[ioutput]; - for(int ivector = 0; ivector < nvector; ivector++){ - myfile << std::setw(25) << current_collected_output(nodeline*nvector + ivector,0) << " "; - } + else if (num_dim == 3) + { + myfile << "ZONE T=\"load step " << time_step << "\", NODES= " << num_nodes + << ", ELEMENTS= " << num_elem << ", DATAPACKING=POINT, ZONETYPE=FEBRICK" "\n"; + } + + for (int nodeline = 0; nodeline < num_nodes; nodeline++) + { + myfile << std::setw(25) << collected_node_coords(nodeline, 0) << " "; + myfile << std::setw(25) << collected_node_coords(nodeline, 1) << " "; + if (num_dim == 3) + { + myfile << std::setw(25) << collected_node_coords(nodeline, 2) << " "; } - if(fea_modules[imodule]->vector_style[ioutput] == FEA_Module::NODAL){ - nvector = fea_modules[imodule]->output_vector_sizes[ioutput]; - for(int ivector = 0; ivector < nvector; ivector++){ - myfile << std::setw(25) << current_collected_output(nodeline,ivector) << " "; + + // velocity print + myfile << std::setw(25) << collected_node_velocities(nodeline, 0) << " "; + myfile << std::setw(25) << collected_node_velocities(nodeline, 1) << " "; + if (num_dim == 3) + { + myfile << std::setw(25) << collected_node_velocities(nodeline, 2) << " "; + } + + // myfile << std::setw(25) << collected_node_densities(nodeline,0) << " "; + /* + for (int imodule = 0; imodule < nfea_modules; imodule++){ + noutput = fea_modules[imodule]->noutput; + for(int ioutput = 0; ioutput < noutput; ioutput++){ + current_collected_output = fea_modules[imodule]->module_outputs[ioutput]; + if(fea_modules[imodule]->vector_style[ioutput] == FEA_Module::DOF){ + nvector = fea_modules[imodule]->output_vector_sizes[ioutput]; + for(int ivector = 0; ivector < nvector; ivector++){ + myfile << std::setw(25) << current_collected_output(nodeline*nvector + ivector,0) << " "; + } + } + if(fea_modules[imodule]->vector_style[ioutput] == FEA_Module::NODAL){ + nvector = fea_modules[imodule]->output_vector_sizes[ioutput]; + for(int ivector = 0; ivector < nvector; ivector++){ + myfile << std::setw(25) << current_collected_output(nodeline,ivector) << " "; + } + } } } + */ + myfile << std::endl; + } + for (int elementline = 0; elementline < num_elem; elementline++) + { + // convert node ordering + for (int ii = 0; ii < max_nodes_per_element; ii++) + { + if (active_node_ordering_convention == IJK) + { + temp_convert = convert_ijk_to_ensight(ii); + } + else + { + temp_convert = ii; + } + myfile << std::setw(10) << collected_nodes_in_elem(elementline, temp_convert) + 1 << " "; + } + myfile << " \n"; + } + myfile.close(); + } + if (myrank == 0 && (displacement_module >= 0 && displace_geometry)) + { + // deformed geometry + count_temp.str(""); + count_temp << file_index; + file_index++; + file_count = count_temp.str(); + + current_file_name = base_file_name + file_count + file_extension; + std::ofstream myfile(current_file_name.c_str()); // output filestream object for file output + // read in position data + myfile << std::fixed << std::setprecision(8); + + // output header of the tecplot file + + myfile << "TITLE=\"results for TO code\" \n"; + myfile << "VARIABLES = \"x\", \"y\", \"z\", \"vx\", \"vy\", \"vz\""; + /* + for (int imodule = 0; imodule < nfea_modules; imodule++){ + for(int ioutput = 0; ioutput < fea_modules[imodule]->noutput; ioutput++){ + nvector = fea_modules[imodule]->output_vector_sizes[ioutput]; + for(int ivector = 0; ivector < nvector; ivector++){ + myfile << ", \"" << fea_modules[imodule]->output_dof_names[ioutput][ivector] << "\""; + } } } - myfile << std::endl; - } - for (int elementline = 0; elementline < num_elem; elementline++) { - //convert node ordering - for (int ii = 0; ii < max_nodes_per_element; ii++) { - if(active_node_ordering_convention == IJK) - temp_convert = convert_ijk_to_ensight(ii); - else - temp_convert = ii; - myfile << std::setw(10) << collected_nodes_in_elem(elementline, temp_convert) + 1 << " "; - } - myfile << " \n"; - } - myfile.close(); + */ + myfile << "\n"; + + myfile << "ZONE T=\"load step " << time_step + 1 << "\", NODES= " << num_nodes + << ", ELEMENTS= " << num_elem << ", DATAPACKING=POINT, ZONETYPE=FEBRICK" "\n"; + + for (int nodeline = 0; nodeline < num_nodes; nodeline++) + { + current_collected_output = fea_modules[displacement_module]->module_outputs[displacement_index]; + myfile << std::setw(25) << collected_node_coords(nodeline, 0) + current_collected_output(nodeline * num_dim, 0) << " "; + myfile << std::setw(25) << collected_node_coords(nodeline, 1) + current_collected_output(nodeline * num_dim + 1, 0) << " "; + if (num_dim == 3) + { + myfile << std::setw(25) << collected_node_coords(nodeline, 2) + current_collected_output(nodeline * num_dim + 2, 0) << " "; + } + + // velocity print + myfile << std::setw(25) << collected_node_velocities(nodeline, 0) << " "; + myfile << std::setw(25) << collected_node_velocities(nodeline, 1) << " "; + if (num_dim == 3) + { + myfile << std::setw(25) << collected_node_velocities(nodeline, 2) << " "; + } + + // myfile << std::setw(25) << collected_node_densities(nodeline,0) << " "; + for (int imodule = 0; imodule < nfea_modules; imodule++) + { + noutput = fea_modules[imodule]->noutput; + for (int ioutput = 0; ioutput < noutput; ioutput++) + { + current_collected_output = fea_modules[imodule]->module_outputs[ioutput]; + if (fea_modules[imodule]->vector_style[ioutput] == FEA_Module::DOF) + { + nvector = fea_modules[imodule]->output_vector_sizes[ioutput]; + for (int ivector = 0; ivector < nvector; ivector++) + { + myfile << std::setw(25) << current_collected_output(nodeline * nvector + ivector, 0) << " "; + } + } + if (fea_modules[imodule]->vector_style[ioutput] == FEA_Module::NODAL) + { + nvector = fea_modules[imodule]->output_vector_sizes[ioutput]; + for (int ivector = 0; ivector < nvector; ivector++) + { + myfile << std::setw(25) << current_collected_output(nodeline, ivector) << " "; + } + } + } + } + myfile << std::endl; + } + for (int elementline = 0; elementline < num_elem; elementline++) + { + // convert node ordering + for (int ii = 0; ii < max_nodes_per_element; ii++) + { + if (active_node_ordering_convention == IJK) + { + temp_convert = convert_ijk_to_ensight(ii); + } + else + { + temp_convert = ii; + } + myfile << std::setw(10) << collected_nodes_in_elem(elementline, temp_convert) + 1 << " "; + } + myfile << " \n"; + } + myfile.close(); } } + /* ---------------------------------------------------------------------- Output Model Information in vtk format ------------------------------------------------------------------------- */ @@ -2773,15 +3097,15 @@ void Explicit_Solver::vtk_writer(){ // Setup of file and directoring for exporting // --------------------------------------------------------------------------- - + FILE *out[20]; // the output files that are written to char filename[128]; - + struct stat st; - + if(stat("vtk",&st) != 0) system("mkdir vtk"); - + if(stat("vtk/data",&st) != 0) system("mkdir vtk/data"); @@ -2789,14 +3113,14 @@ void Explicit_Solver::vtk_writer(){ // --------------------------------------------------------------------------- // Write the Geometry file // --------------------------------------------------------------------------- - - + + sprintf(filename, "vtk/data/%s_%05d_%i.vtu", name, graphics_id, 0); // filename has the full string - + out[0] = fopen(filename, "w"); - + int num_nodes = mesh->num_nodes(); int num_cells = mesh->num_cells(); @@ -2806,7 +3130,7 @@ void Explicit_Solver::vtk_writer(){ fprintf(out[0],"\n"); fprintf(out[0],"\n", num_nodes, num_cells); - + // --------------------------------------------------------------------------- // Write point data @@ -2828,11 +3152,11 @@ void Explicit_Solver::vtk_writer(){ fprintf(out[0]," \n"); for(int cell_var = 0; cell_var < num_cell_vars; cell_var++){ - + fprintf(out[0],"\n", cell_var_names[cell_var]); - + for (int cell_rid = 0; cell_rid < num_cells; cell_rid++){ - + fprintf(out[0],"%f\n",(float) cell_rid); } // end for k over cells @@ -2850,16 +3174,16 @@ void Explicit_Solver::vtk_writer(){ real_t max_coord = 2.0; fprintf(out[0]," \n"); - + fprintf(out[0],"\n", num_dim); - + for (int node_gid = 0; node_gid < num_nodes; node_gid++){ - + fprintf(out[0],"%f %f %f \n",node_coords(node_gid, 0), node_coords(node_gid, 1), node_coords(node_gid, 2)); - } + } fprintf(out[0]," \n"); fprintf(out[0]," \n"); @@ -2880,7 +3204,7 @@ void Explicit_Solver::vtk_writer(){ // write nodes in a cell for (int cell_rid = 0; cell_rid < num_cells; cell_rid++){ - + for(int node = 0; node < 8; node++){ fprintf(out[0],"%i ", nodes_in_elem(cell_rid, node)); } @@ -2909,7 +3233,7 @@ void Explicit_Solver::vtk_writer(){ fprintf(out[0],"\n"); - + for (int cell_rid = 0; cell_rid < num_cells; cell_rid++){ fprintf(out[0],"%i \n", 6); @@ -2934,7 +3258,7 @@ void Explicit_Solver::vtk_writer(){ for (int cell_rid = 0; cell_rid < num_cells; cell_rid++){ fprintf(out[0],"%i \n", faceoffsets*(cell_rid+1)); } // end for k over cells - + fprintf(out[0]," \n"); fprintf(out[0],"\n"); @@ -3001,7 +3325,7 @@ void Explicit_Solver::ensight_writer(){ for (int cell_rid=0; cell_ridfield(cell_rid); + cell_fields(cell_rid, 0) = mat_pt->field(cell_rid); } // end for k over cells int num_elem = rnum_elem; @@ -3011,7 +3335,7 @@ void Explicit_Solver::ensight_writer(){ if(mesh->elem_order() == 0){ num_sub_1d = 1; } - + else{ num_sub_1d = mesh->elem_order()*2; } @@ -3054,7 +3378,7 @@ void Explicit_Solver::ensight_writer(){ // Use average temp from each node as cell temp for (int cell_rid = 0; cell_rid < num_cells; cell_rid++){ - cell_fields(cell_rid, 3) = mat_pt->field(cell_rid); + cell_fields(cell_rid, 3) = mat_pt->field(cell_rid); } // end for k over cells @@ -3063,8 +3387,8 @@ void Explicit_Solver::ensight_writer(){ auto vec_fields = CArray (num_nodes, num_vec_vars, 3); for (int node_gid = 0; node_gid < num_nodes; node_gid++){ - - vec_fields(node_gid, 0, 0) = node_coords(node_gid, 0); + + vec_fields(node_gid, 0, 0) = node_coords(node_gid, 0); vec_fields(node_gid, 0, 1) = node_coords(node_gid, 1); vec_fields(node_gid, 0, 2) = node_coords(node_gid, 2); @@ -3074,15 +3398,15 @@ void Explicit_Solver::ensight_writer(){ // Setup of file and directoring for exporting // --------------------------------------------------------------------------- - + FILE *out[20]; // the output files that are written to char filename[128]; - + struct stat st; - + if(stat("ensight",&st) != 0) system("mkdir ensight"); - + if(stat("ensight/data",&st) != 0) system("mkdir ensight/data"); @@ -3090,47 +3414,47 @@ void Explicit_Solver::ensight_writer(){ // --------------------------------------------------------------------------- // Write the Geometry file // --------------------------------------------------------------------------- - - + + sprintf(filename, "ensight/data/%s.%05d.geo", name, graphics_id); // filename has the full string - + out[0] = fopen(filename, "w"); - - + + fprintf(out[0],"A graphics dump by Cercion \n"); fprintf(out[0],"%s","EnSight Gold geometry\n"); fprintf(out[0],"%s","node id assign\n"); fprintf(out[0],"%s","element id assign\n"); - + fprintf(out[0],"part\n"); fprintf(out[0],"%10d\n",1); fprintf(out[0],"Mesh\n"); - - + + // --- vertices --- fprintf(out[0],"coordinates\n"); fprintf(out[0],"%10d\n",mesh->num_nodes()); - + // write all components of the point coordinates for (int node_gid=0; node_gidgetLocalView (Tpetra::Access::ReadWrite); - //notify that the host view is going to be modified in the file readin - //dual_node_densities.modify_host(); - - //debug Tecplot file readin of initial densities (only works on runs with 1 MPI rank this way) - std::string skip_line, read_line, substring; - std::stringstream line_parse; - real_t read_density; - //in = new std::ifstream(); - //in->open("TecplotDensity.dat"); - //skip 3 lines - //for (int j = 1; j <= 3; j++) { - //getline(*in, skip_line); - // std::cout << skip_line << std::endl; - //} - - - for(int inode = 0; inode < nlocal_nodes; inode++){ - //getline(*in,read_line); - //line_parse.clear(); - //line_parse.str(read_line); - //for(int iword = 0; iword < 10; iword++){ - //read portions of the line into the substring variable - - //if(iword==3){ line_parse >> read_density;} - //else {line_parse >> substring;} - //} - //initialize densities to 1 for now; in the future there might be an option to read in an initial condition for each node - //if(read_density < 0.3) read_density = 0.1; - //node_densities(inode,0) = read_density; - - node_densities(inode,0) = 1; - } - - //sync device view - //dual_node_densities.sync_device(); - } - //allocate global vector information - all_node_densities_distributed = Teuchos::rcp(new MV(all_node_map, 1)); - - //communicate ghost information to the all vector - //create import object using local node indices map and all indices map - Tpetra::Import importer(map, all_node_map); +void Explicit_Solver::init_design() +{ + int num_dim = simparam.num_dims; + bool nodal_density_flag = simparam.nodal_density_flag; + + // set densities + if (nodal_density_flag) + { + if (!simparam.restart_file) + { + design_node_densities_distributed = Teuchos::rcp(new MV(map, 1)); + host_vec_array node_densities = design_node_densities_distributed->getLocalView(Tpetra::Access::ReadWrite); + // notify that the host view is going to be modified in the file readin + // dual_node_densities.modify_host(); + + // debug Tecplot file readin of initial densities (only works on runs with 1 MPI rank this way) + std::string skip_line, read_line, substring; + std::stringstream line_parse; + real_t read_density; + // in = new std::ifstream(); + // in->open("TecplotDensity.dat"); + // skip 3 lines + // for (int j = 1; j <= 3; j++) { + // getline(*in, skip_line); + // std::cout << skip_line << std::endl; + // } + + for (int inode = 0; inode < nlocal_nodes; inode++) + { + // getline(*in,read_line); + // line_parse.clear(); + // line_parse.str(read_line); + // for(int iword = 0; iword < 10; iword++){ + // read portions of the line into the substring variable + + // if(iword==3){ line_parse >> read_density;} + // else {line_parse >> substring;} + // } + // initialize densities to 1 for now; in the future there might be an option to read in an initial condition for each node + // if(read_density < 0.3) read_density = 0.1; + // node_densities(inode,0) = read_density; + + node_densities(inode, 0) = 1; + } - //comms to get ghosts - all_node_densities_distributed->doImport(*design_node_densities_distributed, importer, Tpetra::INSERT); + // sync device view + // dual_node_densities.sync_device(); + } + // allocate global vector information + all_node_densities_distributed = Teuchos::rcp(new MV(all_node_map, 1)); + + // communicate ghost information to the all vector + // create import object using local node indices map and all indices map + Tpetra::Import importer(map, all_node_map); + + // comms to get ghosts + all_node_densities_distributed->doImport(*design_node_densities_distributed, importer, Tpetra::INSERT); + + // debug print + // std::ostream &out = std::cout; + // Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + // if(myrank==0) + // *fos << "Node Densities with Ghosts :" << std::endl; + // all_node_densities_distributed->describe(*fos,Teuchos::VERB_EXTREME); + // *fos << std::endl; + // std::fflush(stdout); + } + else + { + // initialize memory for volume storage + vec_array Element_Densities("Element Densities", rnum_elem, 1); + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + Element_Densities(ielem, 0) = 1; + } - //debug print - //std::ostream &out = std::cout; - //Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - //if(myrank==0) - //*fos << "Node Densities with Ghosts :" << std::endl; - //all_node_densities_distributed->describe(*fos,Teuchos::VERB_EXTREME); - //*fos << std::endl; - //std::fflush(stdout); - - } - else{ - //initialize memory for volume storage - vec_array Element_Densities("Element Densities", rnum_elem, 1); - for(int ielem = 0; ielem < rnum_elem; ielem++) - Element_Densities(ielem,0) = 1; - - //create global vector - Global_Element_Densities = Teuchos::rcp(new MV(all_element_map, Element_Densities)); - - //if(myrank==0) - //*fos << "Global Element Densities:" << std::endl; - //Global_Element_Densities->describe(*fos,Teuchos::VERB_EXTREME); - //*fos << std::endl; - } + // create global vector + Global_Element_Densities = Teuchos::rcp(new MV(all_element_map, Element_Densities)); + // if(myrank==0) + // *fos << "Global Element Densities:" << std::endl; + // Global_Element_Densities->describe(*fos,Teuchos::VERB_EXTREME); + // *fos << std::endl; + } } diff --git a/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.h b/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.h index 6a78e7a2e..8f89fe46a 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.h +++ b/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.h @@ -11,14 +11,14 @@ This program is open source under the BSD-3 License. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - + 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - + 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - + 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. @@ -34,10 +34,9 @@ OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. **********************************************************************************************/ - -#ifndef ELEMENTS_EXPLICIT_SOLVER_H -#define ELEMENTS_EXPLICIT_SOLVER_H +#ifndef ELEMENTS_EXPLICIT_SOLVER_H +#define ELEMENTS_EXPLICIT_SOLVER_H #include "Solver.h" #include "utilities.h" @@ -60,124 +59,124 @@ using namespace mtr; -class Explicit_Solver: public Solver { - +class Explicit_Solver : public Solver +{ public: - Explicit_Solver(Simulation_Parameters_Explicit); - ~Explicit_Solver(); + Explicit_Solver(Simulation_Parameters_Explicit); + ~Explicit_Solver(); + + virtual void run(); + + // void read_mesh_ensight(char *MESH); + + void read_mesh_ansys_dat(const char* MESH); + + void init_state_vectors(); + + // void repartition_nodes(); + + void comm_velocities(); + + void comm_densities(); - virtual void run(); + void init_design(); - //void read_mesh_ensight(char *MESH); + void collect_information(); - void read_mesh_ansys_dat(const char *MESH); + void sort_information(); - void init_state_vectors(); + // process input to decide TO problem and FEA modules + void FEA_module_setup(); - //void repartition_nodes(); + void setup_optimization_problem(); - void comm_velocities(); + // initialize data for boundaries of the model and storage for boundary conditions and applied loads + void init_boundaries(); - void comm_densities(); + // interfaces between user input and creating data structures for bcs + void topology_conditions(); - void init_design(); + // void vtk_writer(); - void collect_information(); + // void ensight_writer(); - void sort_information(); + // interfaces between user input and creating data structures for topology conditions + void generate_tcs(); - //process input to decide TO problem and FEA modules - void FEA_module_setup(); + void init_topology_conditions(int num_sets); - void setup_optimization_problem(); - - //initialize data for boundaries of the model and storage for boundary conditions and applied loads - void init_boundaries(); - - //interfaces between user input and creating data structures for bcs - void topology_conditions(); + void tecplot_writer(); - //void vtk_writer(); + void parallel_tecplot_writer(); - //void ensight_writer(); + void parallel_vtk_writer(); - //interfaces between user input and creating data structures for topology conditions - void generate_tcs(); + void write_outputs(); + void parallel_vtk_writer_new(); + void parallel_vtu_writer_new(); // not yet added + // maps for variable_name:pointer + std::map point_data_scalars_double; + std::map point_data_vectors_double; + std::map cell_data_scalars_double; + std::map cell_data_scalars_int; + std::map> cell_data_fields_double; - void init_topology_conditions (int num_sets); + // void init_boundary_sets(int num_boundary_sets); - void tecplot_writer(); + void tag_boundaries(int this_bc_tag, real_t val, int bdy_set, real_t* patch_limits = NULL); - void parallel_tecplot_writer(); + int check_boundary(Node_Combination& Patch_Nodes, int this_bc_tag, real_t val, real_t* patch_limits); - void parallel_vtk_writer(); + mesh_t* init_mesh; + std::shared_ptr mesh; - void write_outputs(); - void parallel_vtk_writer_new(); - void parallel_vtu_writer_new(); // not yet added - // maps for variable_name:pointer - std::map point_data_scalars_double; - std::map point_data_vectors_double; - std::map cell_data_scalars_double; - std::map cell_data_scalars_int; - std::map > cell_data_fields_double; - - //void init_boundary_sets(int num_boundary_sets); + Simulation_Parameters_Explicit simparam; - void tag_boundaries(int this_bc_tag, real_t val, int bdy_set, real_t *patch_limits = NULL); + // FEA simulations + class FEA_Module_SGH* sgh_module; - int check_boundary(Node_Combination &Patch_Nodes, int this_bc_tag, real_t val, real_t *patch_limits); - - mesh_t *init_mesh; - std::shared_ptr mesh; - - Simulation_Parameters_Explicit simparam; + // Global FEA data + Teuchos::RCP node_velocities_distributed; + Teuchos::RCP initial_node_velocities_distributed; + Teuchos::RCP all_node_velocities_distributed; + Teuchos::RCP ghost_node_velocities_distributed; + Teuchos::RCP all_cached_node_velocities_distributed; - //FEA simulations - class FEA_Module_SGH *sgh_module; + // Distributions of data used to print + Teuchos::RCP collected_node_velocities_distributed; + Teuchos::RCP sorted_node_velocities_distributed; - //Global FEA data - Teuchos::RCP node_velocities_distributed; - Teuchos::RCP initial_node_velocities_distributed; - Teuchos::RCP all_node_velocities_distributed; - Teuchos::RCP ghost_node_velocities_distributed; - Teuchos::RCP all_cached_node_velocities_distributed; + // Boundary Conditions Data + DCArrayKokkos Local_Index_Boundary_Patches; + CArrayKokkos Topology_Condition_Patches; // set of patches corresponding to each boundary condition + CArrayKokkos NTopology_Condition_Patches; - //Distributions of data used to print - Teuchos::RCP collected_node_velocities_distributed; - Teuchos::RCP sorted_node_velocities_distributed; - - //Boundary Conditions Data - DCArrayKokkos Local_Index_Boundary_Patches; - CArrayKokkos Topology_Condition_Patches; //set of patches corresponding to each boundary condition - CArrayKokkos NTopology_Condition_Patches; + // types of boundary conditions + enum tc_type { NONE, TO_SURFACE_CONSTRAINT, TO_BODY_CONSTRAINT }; - //types of boundary conditions - enum tc_type {NONE, TO_SURFACE_CONSTRAINT, TO_BODY_CONSTRAINT}; + // lists what kind of boundary condition each boundary set is assigned to + CArrayKokkos Boundary_Condition_Type_List; - //lists what kind of boundary condition each boundary set is assigned to - CArrayKokkos Boundary_Condition_Type_List; + // time data + real_t time_value; - //time data - real_t time_value; - - //number of displacement boundary conditions acting on nodes; used to size the reduced global stiffness map - size_t Number_DOF_BCS; + // number of displacement boundary conditions acting on nodes; used to size the reduced global stiffness map + size_t Number_DOF_BCS; - //! mapping used to get local ghost index from the global ID. - //typedef ::Tpetra::Details::FixedHashTable - //global_to_local_table_host_type; + // ! mapping used to get local ghost index from the global ID. + // typedef ::Tpetra::Details::FixedHashTable + // global_to_local_table_host_type; - //global_to_local_table_host_type global2local_map; - //CArrayKokkos active_ranks; + // global_to_local_table_host_type global2local_map; + // CArrayKokkos active_ranks; - //allocation flags to avoid repeat MV and global matrix construction - int Matrix_alloc; + // allocation flags to avoid repeat MV and global matrix construction + int Matrix_alloc; - //debug flags - int gradient_print_sync; + // debug flags + int gradient_print_sync; - Teuchos::RCP initial_node_coords_distributed; + Teuchos::RCP initial_node_coords_distributed; }; #endif // end Header Guard diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/CMakeLists.txt b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/CMakeLists.txt index 9f2cdc62b..23092b53c 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/CMakeLists.txt +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/CMakeLists.txt @@ -3,9 +3,23 @@ cmake_minimum_required(VERSION 3.1.3) set(CMAKE_CXX_FLAGS "${Trilinos_CXX_COMPILER_FLAGS} ${CMAKE_CXX_FLAGS}") set(CMAKE_C_FLAGS "${Trilinos_C_COMPILER_FLAGS} ${CMAKE_C_FLAGS}") set(CMAKE_Fortran_FLAGS "${Trilinos_Fortran_COMPILER_FLAGS} ${CMAKE_Fortran_FLAGS}") -set(SRC_Files geometry.cpp boundary.cpp time_integration.cpp momentum.cpp force_sgh.cpp -sgh_optimization.cpp force_gradients_sgh.cpp power_gradients_sgh.cpp energy_sgh.cpp properties.cpp) -set(FEA_Module_SRC FEA_Module_SGH.cpp ) + +include_directories(include) + +set(SRC_Files +src/geometry.cpp +src/boundary.cpp +src/time_integration.cpp +src/momentum.cpp +src/force_sgh.cpp +src/sgh_optimization.cpp +src/force_gradients_sgh.cpp +src/power_gradients_sgh.cpp +src/energy_sgh.cpp +src/properties.cpp +src/setup_sgh.cpp) + +set(FEA_Module_SRC src/FEA_Module_SGH.cpp ) INCLUDE_DIRECTORIES (${Trilinos_INCLUDE_DIRS} ${Trilinos_TPL_INCLUDE_DIRS}) LINK_DIRECTORIES (${Trilinos_LIBRARY_DIRS} ${Trilinos_TPL_LIBRARY_DIRS}) diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/FEA_Module_SGH.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/FEA_Module_SGH.cpp deleted file mode 100644 index 648cfb3d1..000000000 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/FEA_Module_SGH.cpp +++ /dev/null @@ -1,2615 +0,0 @@ -/********************************************************************************************** - © 2020. Triad National Security, LLC. All rights reserved. - This program was produced under U.S. Government contract 89233218CNA000001 for Los Alamos - National Laboratory (LANL), which is operated by Triad National Security, LLC for the U.S. - Department of Energy/National Nuclear Security Administration. All rights in the program are - reserved by Triad National Security, LLC, and the U.S. Department of Energy/National Nuclear - Security Administration. The Government is granted for itself and others acting on its behalf a - nonexclusive, paid-up, irrevocable worldwide license in this material to reproduce, prepare - derivative works, distribute copies to the public, perform publicly and display publicly, and - to permit others to do so. - This program is open source under the BSD-3 License. - Redistribution and use in source and binary forms, with or without modification, are permitted - provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright notice, this list of - conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright notice, this list of - conditions and the following disclaimer in the documentation and/or other materials - provided with the distribution. - - 3. Neither the name of the copyright holder nor the names of its contributors may be used - to endorse or promote products derived from this software without specific prior - written permission. - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS - IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR - CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR - OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - **********************************************************************************************/ - -#include -#include -#include -#include -#include -#include -#include // fmin, fmax, abs note: fminl is long -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include "Tpetra_Import.hpp" -#include "Tpetra_Import_Util2.hpp" - -#include "elements.h" -#include "swage.h" -#include "matar.h" -#include "utilities.h" -#include "node_combination.h" -#include "Simulation_Parameters/Simulation_Parameters_Explicit.h" -#include "Simulation_Parameters/FEA_Module/SGH_Parameters.h" -#include "FEA_Module_SGH.h" -#include "Explicit_Solver.h" - -//optimization -#include "ROL_Solver.hpp" -#include "Kinetic_Energy_Minimize.h" - -#define MAX_ELEM_NODES 8 -#define STRAIN_EPSILON 0.000000001 -#define DENSITY_EPSILON 0.0001 -#define BC_EPSILON 1.0e-6 -#define BUFFER_GROW 100 - -using namespace utils; - - -FEA_Module_SGH::FEA_Module_SGH( - SGH_Parameters& params, Solver *Solver_Pointer, - std::shared_ptr mesh_in, const int my_fea_module_index) - : FEA_Module(Solver_Pointer) { - //assign interfacing index - my_fea_module_index_ = my_fea_module_index; - Module_Type = FEA_MODULE_TYPE::SGH; - - //recast solver pointer for non-base class access - Explicit_Solver_Pointer_ = dynamic_cast(Solver_Pointer); - simparam = &(Explicit_Solver_Pointer_->simparam); - module_params = ¶ms; - - //create ref element object - //ref_elem = new elements::ref_element(); - //create mesh objects - //init_mesh = new swage::mesh_t(simparam); - //mesh = new swage::mesh_t(simparam); - - mesh = mesh_in; - - //boundary condition data - max_boundary_sets = 0; - Local_Index_Boundary_Patches = Explicit_Solver_Pointer_->Local_Index_Boundary_Patches; - - //set Tpetra vector pointers - initial_node_velocities_distributed = Explicit_Solver_Pointer_->initial_node_velocities_distributed; - initial_node_coords_distributed = Explicit_Solver_Pointer_->initial_node_coords_distributed; - all_initial_node_coords_distributed = Explicit_Solver_Pointer_->all_initial_node_coords_distributed; - node_coords_distributed = Explicit_Solver_Pointer_->node_coords_distributed; - node_velocities_distributed = Explicit_Solver_Pointer_->node_velocities_distributed; - all_node_velocities_distributed = Explicit_Solver_Pointer_->all_node_velocities_distributed; - if(simparam->topology_optimization_on||simparam->shape_optimization_on){ - all_cached_node_velocities_distributed = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - force_gradient_velocity = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - force_gradient_position = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - force_gradient_design = Teuchos::rcp(new MV(all_node_map, 1)); - corner_value_storage = Solver_Pointer->corner_value_storage; - corner_vector_storage = Solver_Pointer->corner_vector_storage; - corner_gradient_storage = Solver_Pointer->corner_gradient_storage; - relative_element_densities = DCArrayKokkos(rnum_elem, "relative_element_densities"); - adjoint_vector_distributed = Teuchos::rcp(new MV(map, simparam->num_dims)); - phi_adjoint_vector_distributed = Teuchos::rcp(new MV(map, simparam->num_dims)); - psi_adjoint_vector_distributed = Teuchos::rcp(new MV(all_element_map, 1)); - } - - if(simparam->topology_optimization_on||simparam->shape_optimization_on||simparam->num_dims==2){ - node_masses_distributed = Teuchos::rcp(new MV(map, 1)); - ghost_node_masses_distributed = Teuchos::rcp(new MV(ghost_node_map, 1)); - } - - //setup output - noutput = 0; - init_output(); - - //optimization flags - kinetic_energy_objective = false; - - - //set parameters - Dynamic_Options dynamic_options = simparam->dynamic_options; - time_value = dynamic_options.time_value; - time_final = dynamic_options.time_final; - dt_max = dynamic_options.dt_max; - dt_min = dynamic_options.dt_min; - dt_cfl = dynamic_options.dt_cfl; - graphics_time = simparam->output_options.graphics_time; - graphics_dt_ival = simparam->output_options.graphics_dt_ival; - graphics_cyc_ival = simparam->output_options.graphics_cyc_ival; - cycle_stop = dynamic_options.cycle_stop; - rk_num_stages = dynamic_options.rk_num_stages; - dt = dynamic_options.dt; - fuzz = dynamic_options.fuzz; - tiny = dynamic_options.tiny; - small = dynamic_options.small; - graphics_times = simparam->output_options.graphics_times; - graphics_id = simparam->output_options.graphics_id; - rk_num_bins = simparam->dynamic_options.rk_num_bins; - - if(simparam->topology_optimization_on){ - max_time_steps = BUFFER_GROW; - time_data.resize(max_time_steps+1); - element_internal_energy_distributed = Teuchos::rcp(new MV(all_element_map, 1)); - forward_solve_velocity_data = Teuchos::rcp(new std::vector>(max_time_steps+1)); - forward_solve_coordinate_data = Teuchos::rcp(new std::vector>(max_time_steps+1)); - forward_solve_internal_energy_data = Teuchos::rcp(new std::vector>(max_time_steps+1)); - adjoint_vector_data = Teuchos::rcp(new std::vector>(max_time_steps+1)); - phi_adjoint_vector_data = Teuchos::rcp(new std::vector>(max_time_steps+1)); - psi_adjoint_vector_data = Teuchos::rcp(new std::vector>(max_time_steps+1)); - //assign a multivector of corresponding size to each new timestep in the buffer - for(int istep = 0; istep < max_time_steps+1; istep++){ - (*forward_solve_velocity_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - (*forward_solve_coordinate_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - (*forward_solve_internal_energy_data)[istep] = Teuchos::rcp(new MV(all_element_map, 1)); - (*adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - (*phi_adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - (*psi_adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_element_map, 1)); - } - - } - - have_loading_conditions = false; - -} - -FEA_Module_SGH::~FEA_Module_SGH(){ - //delete simparam; -} - -/* ---------------------------------------------------------------------- - Read ANSYS dat format mesh file -------------------------------------------------------------------------- */ -void FEA_Module_SGH::read_conditions_ansys_dat(std::ifstream *in, std::streampos before_condition_header){ - - char ch; - int num_dim = simparam->num_dims; - int buffer_lines = 1000; - int max_word = 30; - auto input_options = simparam->input_options.value(); - int p_order = input_options.p_order; - real_t unit_scaling = input_options.unit_scaling; - int local_node_index, current_column_index; - size_t strain_count; - std::string skip_line, read_line, substring, token; - std::stringstream line_parse, line_parse2; - CArrayKokkos read_buffer; - CArrayKokkos read_buffer_indices; - int buffer_loop, buffer_iteration, buffer_iterations, scan_loop, nodes_per_element, words_per_line; - size_t read_index_start, node_rid, elem_gid; - LO local_dof_id; - GO node_gid; - real_t dof_value; - host_vec_array node_densities; - -} // end read_conditions_ansys_dat - -// ----------------------------------------------------------------------------- -// Interfaces read in data with the SGH solver data; currently a hack to streamline -//------------------------------------------------------------------------------ -void FEA_Module_SGH::sgh_interface_setup(node_t &node, - elem_t &elem, - corner_t &corner){ - - const size_t num_dim = simparam->num_dims; - const size_t rk_num_bins = simparam->dynamic_options.rk_num_bins; - - num_nodes_in_elem = 1; - for (int dim=0; dimnall_nodes; - int myrank = Explicit_Solver_Pointer_->myrank; - int nranks = Explicit_Solver_Pointer_->nranks; - //printf("Num nodes assigned to MPI rank %lu is %lu\n" , myrank, nall_nodes); - - // intialize node variables - mesh->initialize_nodes(nall_nodes); - mesh->initialize_local_nodes(Explicit_Solver_Pointer_->nlocal_nodes); - node.initialize(rk_num_bins, nall_nodes, num_dim); - //std::cout << "Bin counts " << rk_num_bins << " Node counts " << nall_nodes << " Num dim " << num_dim << std::endl; - - //view scope - { - host_vec_array interface_node_coords = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); - //save node data to node.coords - //std::cout << "NODE DATA ON RANK " << myrank << std::endl; - if(num_dim==2){ - for(int inode = 0; inode < nall_nodes; inode++){ - //std::cout << "Node index " << inode+1 << " "; - node.coords.host(0,inode,0) = interface_node_coords(inode,0); - //std::cout << host_node_coords_state(0,inode,0)+1<< " "; - node.coords.host(0,inode,1) = interface_node_coords(inode,1); - //std::cout << host_node_coords_state(0,inode,1)+1<< " "; - } - } - else if(num_dim==3){ - for(int inode = 0; inode < nall_nodes; inode++){ - //std::cout << "Node index " << inode+1 << " "; - node.coords.host(0,inode,0) = interface_node_coords(inode,0); - //std::cout << host_node_coords_state(0,inode,0)+1<< " "; - node.coords.host(0,inode,1) = interface_node_coords(inode,1); - //std::cout << host_node_coords_state(0,inode,1)+1<< " "; - - node.coords.host(0,inode,2) = interface_node_coords(inode,2); - //std::cout << host_node_coords_state(0,inode,2)+1<< std::endl; - } - } - } //end view scope - // --- read in the elements in the mesh --- - - rnum_elem = Explicit_Solver_Pointer_->rnum_elem; - //printf("Num elems assigned to MPI rank %lu is %lu\n" , myrank, rnum_elem); - - // intialize elem variables - mesh->initialize_elems(rnum_elem, num_dim); - elem.initialize(rk_num_bins, nall_nodes, 3); // always 3D here, even for 2D - nodes_in_elem = mesh->nodes_in_elem; - //save data to nodes_in_elem.host - //CArrayKokkos host_mesh_nodes_in_elem(rnum_elem, num_nodes_in_elem); - //view scope - { - host_elem_conn_array interface_nodes_in_elem = Explicit_Solver_Pointer_->global_nodes_in_elem_distributed->getLocalView (Tpetra::Access::ReadWrite); - //save node data to node.coords - //std::cout << "ELEMENT CONNECTIVITY ON RANK " << myrank << std::endl; - for(int ielem = 0; ielem < rnum_elem; ielem++){ - //std::cout << "Element index " << ielem+1 << " "; - for(int inode = 0; inode < num_nodes_in_elem; inode++){ - nodes_in_elem.host(ielem,inode) = Explicit_Solver_Pointer_->all_node_map->getLocalElement(interface_nodes_in_elem(ielem,inode)); - //debug print - //std::cout << nodes_in_elem.get_kokkos_dual_view().h_view(ielem*num_nodes_in_elem + inode)+1<< " "; - } - //std::cout << std::endl; - } - } - // update device side - nodes_in_elem.update_device(); - - //debug print - - //CArrayKokkos device_mesh_nodes_in_elem(rnum_elem, num_nodes_in_elem); - //device_mesh_nodes_in_elem.get_kokkos_view() = nodes_in_elem.get_kokkos_dual_view().d_view; - //host_mesh_nodes_in_elem.get_kokkos_view() = nodes_in_elem.get_kokkos_dual_view().view_host(); - /* - if(myrank==1){ - std::cout << "ELEMENT CONNECTIVITY ON RANK 1 in LOCAL INDICES" << myrank << std::endl; - for(int ielem = 0; ielem < rnum_elem; ielem++){ - std::cout << "Element index " << ielem+1 << " "; - for(int inode = 0; inode < num_nodes_in_elem; inode++){ - //debug print - //device_mesh_nodes_in_elem(ielem,inode) = Explicit_Solver_Pointer_->all_node_map->getLocalElement(interface_nodes_in_elem(ielem,inode)); - std::cout << nodes_in_elem(ielem, inode)+1<< " "; - } - std::cout << std::endl; - } - } - */ - /* - std::cout.flush(); - if(myrank==1){ - std::cout << "ELEMENT CONNECTIVITY ON RANK 1 in GLOBAL INDICES" << myrank << std::endl; - std::cout << "local node index of global index 275 on rank 1 " << Explicit_Solver_Pointer_->all_node_map->getLocalElement(275) << std::endl; - for(int ielem = 0; ielem < rnum_elem; ielem++){ - std::cout << ielem << " "; - for(int inode = 0; inode < num_nodes_in_elem; inode++){ - //debug print - //device_mesh_nodes_in_elem(ielem,inode) = Explicit_Solver_Pointer_->all_node_map->getLocalElement(interface_nodes_in_elem(ielem,inode)); - std::cout << Explicit_Solver_Pointer_->all_node_map->getGlobalElement(nodes_in_elem(ielem, inode))<< " "; - } - std::cout << std::endl; - } - } - std::cout.flush(); - */ - /* - size_t nall_nodes = Explicit_Solver_Pointer_->nall_nodes; - node.all_coords = DCArrayKokkos (rk_num_bins, nall_nodes, num_dim); - node.all_vel = DCArrayKokkos (rk_num_bins, nall_nodes, num_dim); - node.all_mass = DCArrayKokkos (nall_nodes); - - //save all data (nlocal +nghost) - CArrayKokkos host_all_node_coords_state(rk_num_bins, nall_nodes, num_dim); - host_vec_array interface_all_node_coords = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); - host_all_node_coords_state.get_kokkos_view() = node.all_coords.get_kokkos_dual_view().view_host(); - //host_node_coords_state = CArrayKokkos(rk_num_bins, nall_nodes, num_dim); - //host_all_node_coords_state.get_kokkos_view() = Kokkos::View("debug", rk_num_bins*nall_nodes*num_dim); - //save node data to node.coords - - //std::cout << "ALL NODE DATA ON RANK " << myrank << std::endl; - for(int inode = 0; inode < nall_nodes; inode++){ - //std::cout << "Node index " << inode+1 << " "; - node.all_coords.host(0,inode,0) = interface_all_node_coords(inode,0); - //std::cout << host_all_node_coords_state(0,inode,0)+1<< " "; - node.all_coords.host(0,inode,1) = interface_all_node_coords(inode,1); - //std::cout << host_all_node_coords_state(0,inode,1)+1<< " "; - node.all_coords.host(0,inode,2) = interface_all_node_coords(inode,2); - //std::cout << host_all_node_coords_state(0,inode,2)+1<< std::endl; - } - */ - - // save the node coords to the current RK value - for (size_t node_gid=0; node_gidinitialize_corners(num_corners); - corner.initialize(num_corners, num_dim); - - /* - for(int inode = 0; inode < nall_nodes; inode++){ - std::cout << "Node index " << inode+1 << " "; - for(int rk=0; rkboundary_conditions.size(); - int num_dim = simparam->num_dims; - - // set the number of boundary sets - if(myrank == 0) - std::cout << "building boundary sets " << std::endl; - - //initialize to 1 since there must be at least 1 boundary set anyway; read in may occure later - if(max_boundary_sets==0) max_boundary_sets = 1; - //std::cout << "NUM BOUNDARY CONDITIONS ON RANK " << myrank << " FOR INIT " << num_boundary_conditions <(nall_nodes*num_dim, "Node_DOF_Boundary_Condition_Type"); - - //initialize - for(int init=0; init < nall_nodes*num_dim; init++) - Node_DOF_Boundary_Condition_Type(init) = NONE; - - Number_DOF_BCS = 0; -} - -/* ---------------------------------------------------------------------- - initialize storage for element boundary surfaces corresponding to user BCs -------------------------------------------------------------------------- */ - -void FEA_Module_SGH::init_boundary_sets (int num_sets){ - - if(num_sets == 0){ - std::cout << " Warning: number of boundary conditions = 0"; - return; - } - //initialize maximum - max_boundary_sets = num_sets; - //std::cout << " DEBUG PRINT "<(num_sets, "Boundary_Condition_Type_List"); - NBoundary_Condition_Patches = CArrayKokkos(num_sets, "NBoundary_Condition_Patches"); - //std::cout << "NBOUNDARY PATCHES ON RANK " << myrank << " FOR INIT IS " << nboundary_patches <(num_sets, nboundary_patches, "Boundary_Condition_Patches"); - - //initialize data - for(int iset = 0; iset < num_sets; iset++) NBoundary_Condition_Patches(iset) = 0; - - //initialize - for(int ibdy=0; ibdy < num_sets; ibdy++) Boundary_Condition_Type_List(ibdy) = NONE; -} - -/* ---------------------------------------------------------------------------- - Grow boundary conditions sets of element boundary surfaces -------------------------------------------------------------------------------- */ - -void FEA_Module_SGH::grow_boundary_sets(int num_sets){ - int num_dim = simparam->num_dims; - - if(num_sets == 0){ - std::cout << " Warning: number of boundary conditions being set to 0"; - return; - } - - //std::cout << " DEBUG PRINT "<max_boundary_sets){ - //temporary storage for previous data - CArrayKokkos Temp_Boundary_Condition_Type_List = Boundary_Condition_Type_List; - CArrayKokkos Temp_NBoundary_Condition_Patches = NBoundary_Condition_Patches; - CArrayKokkos Temp_Boundary_Condition_Patches = Boundary_Condition_Patches; - - max_boundary_sets = num_sets + 5; //5 is an arbitrary buffer - Boundary_Condition_Type_List = CArrayKokkos(max_boundary_sets, "Boundary_Condition_Type_List"); - NBoundary_Condition_Patches = CArrayKokkos(max_boundary_sets, "NBoundary_Condition_Patches"); - //std::cout << "NBOUNDARY PATCHES ON RANK " << myrank << " FOR GROW " << nboundary_patches <(max_boundary_sets, nboundary_patches, "Boundary_Condition_Patches"); - - //copy previous data back over - //std::cout << "NUM BOUNDARY CONDITIONS ON RANK " << myrank << " FOR COPY " << max_boundary_sets <output(FIELD::velocity); - bool output_strain_flag = simparam->output(FIELD::strain); - bool output_stress_flag = simparam->output(FIELD::stress); - int num_dim = simparam->num_dims; - int Brows; - if(num_dim==3) Brows = 6; - else Brows = 3; - - //Implicit compliant code - if(output_velocity_flag){ - //displacement_index is accessed by writers at the solver level for deformed output - output_velocity_index = noutput; - noutput += 1; - module_outputs.resize(noutput); - - vector_style.resize(noutput); - vector_style[noutput-1] = DOF; - - output_vector_sizes.resize(noutput); - output_vector_sizes[noutput-1] = num_dim; - - output_dof_names.resize(noutput); - output_dof_names[noutput-1].resize(num_dim); - output_dof_names[noutput-1][0] = "vx"; - output_dof_names[noutput-1][1] = "vy"; - if(num_dim==3) - output_dof_names[noutput-1][2] = "vz"; - } - if(output_strain_flag){ - output_strain_index = noutput; - noutput += 1; - module_outputs.resize(noutput); - - vector_style.resize(noutput); - vector_style[noutput-1] = NODAL; - - output_vector_sizes.resize(noutput); - output_vector_sizes[noutput-1] = Brows; - - output_dof_names.resize(noutput); - output_dof_names[noutput-1].resize(Brows); - if(num_dim==2){ - output_dof_names[noutput-1][0] = "strain_xx"; - output_dof_names[noutput-1][1] = "strain_yy"; - output_dof_names[noutput-1][2] = "strain_xy"; - } - if(num_dim==3){ - output_dof_names[noutput-1][0] = "strain_xx"; - output_dof_names[noutput-1][1] = "strain_yy"; - output_dof_names[noutput-1][2] = "strain_zz"; - output_dof_names[noutput-1][3] = "strain_xy"; - output_dof_names[noutput-1][4] = "strain_xz"; - output_dof_names[noutput-1][5] = "strain_yz"; - } - } - if(output_stress_flag){ - output_stress_index = noutput; - noutput += 1; - module_outputs.resize(noutput); - - vector_style.resize(noutput); - vector_style[noutput-1] = NODAL; - - output_vector_sizes.resize(noutput); - output_vector_sizes[noutput-1] = Brows; - - output_dof_names.resize(noutput); - output_dof_names[noutput-1].resize(Brows); - if(num_dim==2){ - output_dof_names[noutput-1][0] = "stress_xx"; - output_dof_names[noutput-1][1] = "stress_yy"; - output_dof_names[noutput-1][3] = "stress_xy"; - } - if(num_dim==3){ - output_dof_names[noutput-1][0] = "stress_xx"; - output_dof_names[noutput-1][1] = "stress_yy"; - output_dof_names[noutput-1][2] = "stress_zz"; - output_dof_names[noutput-1][3] = "stress_xy"; - output_dof_names[noutput-1][4] = "stress_xz"; - output_dof_names[noutput-1][5] = "stress_yz"; - } - } - -} - -/* ------------------------------------------------------------------------------------------- - Prompts sorting for elastic response output data. For now, nodal strains. ----------------------------------------------------------------------------------------------- */ - -void FEA_Module_SGH::sort_output(Teuchos::RCP > sorted_map){ - - -} - -/* ------------------------------------------------------------------------------------------- - populate requests this module makes for output data ----------------------------------------------------------------------------------------------- */ - -void FEA_Module_SGH::write_data(std::map &point_data_scalars_double, - std::map &point_data_vectors_double, - std::map &cell_data_scalars_double, - std::map &cell_data_scalars_int, - std::map > &cell_data_fields_double){ - - - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - - for (const auto& field_name : simparam->output_options.output_fields) { - switch (field_name) - { - - case FIELD::velocity: - // node "velocity" - node_vel.update_host(); - point_data_vectors_double["velocity"] = &node_vel.host(rk_level,0,0); - break; - - case FIELD::element_density: - // element "density" - elem_den.update_host(); - cell_data_scalars_double["element_density"] = elem_den.host_pointer(); - break; - - case FIELD::pressure: - // element "pressure" - elem_pres.update_host(); - cell_data_scalars_double["pressure"] = elem_pres.host_pointer(); - break; - - case FIELD::SIE: - // element "SIE" - elem_sie.update_host(); - cell_data_scalars_double["SIE"] = &elem_sie.host(rk_level,0); - break; - - case FIELD::volume: - // element "volume" - elem_vol.update_host(); - cell_data_scalars_double["volume"] = elem_vol.host_pointer(); - break; - - case FIELD::mass: - // element "mass" - elem_mass.update_host(); - cell_data_scalars_double["mass"] = elem_mass.host_pointer(); - break; - - case FIELD::sound_speed: - // element "sspd" - elem_sspd.update_host(); - cell_data_scalars_double["sound_speed"] = elem_sspd.host_pointer(); - break; - - case FIELD::material_id: - // element "material_id" - elem_mat_id.update_host(); - cell_data_scalars_int["material_id"] = reinterpret_cast(elem_mat_id.host_pointer()); - break; - - case FIELD::user_vars: - // element "user_vars" - elem_user_output_vars.update_host(); - cell_data_fields_double["user_vars"] = std::make_pair(elem_user_output_vars.host_pointer(), - elem_user_output_vars.dims(1)); - case FIELD::stress: - // element "stress" - elem_stress.update_host(); - cell_data_fields_double["stress"] = std::make_pair(&elem_stress.host(rk_level,0,0,0), 9); - break; - - default: - break; - - } // end switch - } // end if - - // element "mat_id" //uncomment if needed (works fine) - //sgh_module->elem_mat_id.update_host(); - //cell_data_scalars_int["mat_id"] = reinterpret_cast(&sgh_module->elem_mat_id.host(0)); - - // element "user_output_vars" //uncomment if needed (works fine) - //sgh_module->elem_user_output_vars.update_host(); - //cell_data_fields_double["user_output_vars"] = std::make_pair(&sgh_module->elem_user_output_vars.host_pointer(), - // sgh_module->elem_user_output_vars.dims(1)); - - // element "stress" //uncomment if needed (works fine) - //sgh_module->elem_stress.update_host(); - //cell_data_fields_double["stress"] = std::make_pair(&sgh_module->elem_stress.host(rk_level,0,0,0), 9); -} - -/* ------------------------------------------------------------------------------------------- - Prompts sorting for elastic response output data. For now, nodal strains. ----------------------------------------------------------------------------------------------- */ - -void FEA_Module_SGH::sort_element_output(Teuchos::RCP > sorted_map){ - //interface element density data - { - host_vec_array Element_Densities = Global_Element_Densities->getLocalView (Tpetra::Access::ReadWrite); - elem_den.update_host(); - for(int ielem = 0; ielem < rnum_elem; ielem++){ - Element_Densities(ielem,0) = elem_den.host(ielem); - } - } - -} - -/* ------------------------------------------------------------------------------------------- - Prompts computation of elastic response output data. For now, nodal strains. ----------------------------------------------------------------------------------------------- */ - -void FEA_Module_SGH::collect_output(Teuchos::RCP > global_reduce_map){ - -} - -/* ------------------------------------------------------------------------------------------- - Prompts computation of elastic response output data. For now, nodal strains. ----------------------------------------------------------------------------------------------- */ - -void FEA_Module_SGH::compute_output(){ - -} - -/* ---------------------------------------------------------------------- - Communicate updated nodal velocities to ghost nodes -------------------------------------------------------------------------- */ - -void FEA_Module_SGH::comm_node_masses(){ - - //debug print of design vector - //std::ostream &out = std::cout; - //Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - //if(myrank==0) - //*fos << "Density data :" << std::endl; - //node_densities_distributed->describe(*fos,Teuchos::VERB_EXTREME); - //*fos << std::endl; - //std::fflush(stdout); - - //communicate design densities - //create import object using local node indices map and all indices map - //Tpetra::Import importer(map, ghost_node_map); - - //comms to get ghosts - ghost_node_masses_distributed->doImport(*node_masses_distributed, *ghost_importer, Tpetra::INSERT); - //all_node_map->describe(*fos,Teuchos::VERB_EXTREME); - //all_node_velocities_distributed->describe(*fos,Teuchos::VERB_EXTREME); - - //update_count++; - //if(update_count==1){ - //MPI_Barrier(world); - //MPI_Abort(world,4); - //} -} - -/* ------------------------------------------------------------------------------------------- - Communicate ghosts using the current optimization design data ----------------------------------------------------------------------------------------------- */ - -void FEA_Module_SGH::comm_variables(Teuchos::RCP zp){ - - if(simparam->topology_optimization_on){ - //set density vector to the current value chosen by the optimizer - test_node_densities_distributed = zp; - - //debug print of design vector - //std::ostream &out = std::cout; - //Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - //if(myrank==0) - //*fos << "Density data :" << std::endl; - //node_densities_distributed->describe(*fos,Teuchos::VERB_EXTREME); - //*fos << std::endl; - //std::fflush(stdout); - - //communicate design densities - //create import object using local node indices map and all indices map - //Tpetra::Import importer(map, all_node_map); - - //comms to get ghosts - all_node_densities_distributed->doImport(*test_node_densities_distributed, *importer, Tpetra::INSERT); - } - else if(simparam->shape_optimization_on){ - //clause to communicate boundary node data if the boundary nodes are ghosts on this rank - } -} - - -/* ------------------------------------------------------------------------------------------- - enforce constraints on nodes due to BCS ----------------------------------------------------------------------------------------------- */ - -void FEA_Module_SGH::node_density_constraints(host_vec_array node_densities_lower_bound){ - - const size_t num_dim = mesh->num_dims; - const_vec_array all_initial_node_coords = all_initial_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - const size_t num_lcs = module_params->loading_conditions.size(); - - const DCArrayKokkos mat_fill = simparam->mat_fill; - const DCArrayKokkos loading = module_params->loading; - - //debug check - //std::cout << "NUMBER OF LOADING CONDITIONS: " << num_lcs << std::endl; - - // walk over the nodes to update the velocity - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - double current_node_coords[3]; - double radius; - for (size_t dim = 0; dim < num_dim; dim++){ - current_node_coords[dim] = all_initial_node_coords(node_gid, dim); - } // end for dim - radius = sqrt(current_node_coords[0]*current_node_coords[0]+current_node_coords[1]*current_node_coords[1]+current_node_coords[2]*current_node_coords[2]); - for(size_t ilc=0; ilc < num_lcs; ilc++){ - //debug check - //std::cout << "LOADING CONDITION VOLUME TYPE: " << to_string(loading(ilc).volume) << std::endl; - - bool fill_this = loading(ilc).volume.contains(current_node_coords); - if(fill_this){ - node_densities_lower_bound(node_gid,0) = 1; - } - } - }); // end for parallel for over nodes - -} - -/* ---------------------------------------------------------------------------- - Setup SGH solver data -------------------------------------------------------------------------------- */ - -void FEA_Module_SGH::setup(){ - - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - const size_t num_fills = simparam->regions.size(); - const size_t rk_num_bins = simparam->dynamic_options.rk_num_bins; - const size_t num_bcs = module_params->boundary_conditions.size(); - const size_t num_materials = simparam->materials.size(); - const int num_dim = simparam->num_dims; - const size_t num_lcs = module_params->loading.size(); - if(num_lcs) - have_loading_conditions = true; - - - // --------------------------------------------------------------------- - // obtain mesh data - // --------------------------------------------------------------------- - sgh_interface_setup(node_interface, elem_interface, corner_interface); - mesh->build_corner_connectivity(); - //debug print of corner ids - /* - if(myrank==1){ - for(int i = 0; i < mesh.num_nodes; i++){ - - // loop over all corners around the node and calculate the nodal force - for (size_t corner_lid=0; corner_lidgetGlobalElement(i) << " " << i << " " << all_node_map->getLocalElement(all_node_map->getGlobalElement(i)) << " " << corner_gid << " " << std::endl; - - } // end for corner_lid - //std::cout << explicit_solver_pointer->all_node_map->getGlobalElement(i) << " " << node_force[0] << " " << node_force[1] << " " << node_force[2] << std::endl; - //std::cout << explicit_solver_pointer->all_node_map->getGlobalElement(i) << " " << node_mass(i) << std::endl; - } - } - */ - /* - if(myrank==1){ - for(int i = 0; i < mesh.num_elems; i++){ - - // loop over all corners around the node and calculate the nodal force - for (size_t corner_lid=0; corner_lidgetGlobalElement(mesh.nodes_in_elem(i, corner_lid)) <<" " << corner_gid << " " << std::endl; - - } // end for corner_lid - //std::cout << explicit_solver_pointer->all_node_map->getGlobalElement(i) << " " << node_force[0] << " " << node_force[1] << " " << node_force[2] << std::endl; - //std::cout << explicit_solver_pointer->all_node_map->getGlobalElement(i) << " " << node_mass(i) << std::endl; - } - } - */ - mesh->build_elem_elem_connectivity(); - mesh->num_bdy_patches = nboundary_patches; - if(num_dim==2){ - mesh->build_patch_connectivity(); - mesh->build_node_node_connectivity(); - } - - // --------------------------------------------------------------------- - // allocate memory - // --------------------------------------------------------------------- - - // shorthand names - const size_t num_nodes = mesh->num_nodes; - const size_t num_elems = mesh->num_elems; - const size_t num_corners = mesh->num_corners; - - - // --- make dual views of data on CPU and GPU --- - // Notes: - // Instead of using a struct of dual types like the mesh type, - // individual dual views will be made for all the state - // variables. The motivation is to reduce memory movement - // when passing state into a function. Passing a struct by - // reference will copy the meta data and pointers for the - // variables held inside the struct. Since all the mesh - // variables are typically used by most functions, a single - // mesh struct or passing the arrays will be roughly equivalent - // for memory movement. - - - // create Dual Views of the individual node struct variables - node_coords = DViewCArrayKokkos(node_interface.coords.get_kokkos_dual_view().view_host().data(),rk_num_bins,num_nodes,num_dim); - - node_vel = DViewCArrayKokkos(node_interface.vel.get_kokkos_dual_view().view_host().data(),rk_num_bins,num_nodes,num_dim); - - node_mass = DViewCArrayKokkos(node_interface.mass.get_kokkos_dual_view().view_host().data(),num_nodes); - - - // create Dual Views of the individual elem struct variables - elem_den= DViewCArrayKokkos(&elem_interface.den(0), - num_elems); - - elem_pres = DViewCArrayKokkos(&elem_interface.pres(0), - num_elems); - - elem_stress = DViewCArrayKokkos(&elem_interface.stress(0,0,0,0), - rk_num_bins, - num_elems, - 3, - 3); // always 3D even in 2D-RZ - - elem_sspd = DViewCArrayKokkos(&elem_interface.sspd(0), - num_elems); - - elem_sie = DViewCArrayKokkos(&elem_interface.sie(0,0), - rk_num_bins, - num_elems); - - elem_vol = DViewCArrayKokkos(&elem_interface.vol(0), - num_elems); - - elem_div = DViewCArrayKokkos(&elem_interface.div(0), - num_elems); - - - elem_mass = DViewCArrayKokkos(&elem_interface.mass(0), - num_elems); - - elem_mat_id = DViewCArrayKokkos(&elem_interface.mat_id(0), - num_elems); - - // create Dual Views of the corner struct variables - corner_force = DViewCArrayKokkos (&corner_interface.force(0,0), - num_corners, - num_dim); - - corner_mass = DViewCArrayKokkos (&corner_interface.mass(0), - num_corners); - - // allocate elem_vel_grad - elem_vel_grad = DCArrayKokkos (num_elems,3,3); - - // allocate material models - elem_eos = DCArrayKokkos (num_elems); - elem_strength = DCArrayKokkos (num_elems); - - // --------------------------------------------------------------------- - // calculate geometry - // --------------------------------------------------------------------- - node_coords.update_device(); - Kokkos::fence(); - - get_vol(); - - //FEA_Module bc variable - num_boundary_conditions = num_bcs; - - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos mat_fill = simparam->mat_fill; - const DCArrayKokkos material = simparam->material; - global_vars = simparam->global_vars; - state_vars = DCArrayKokkos (rnum_elem, simparam->max_num_state_vars); - elem_user_output_vars = DCArrayKokkos (rnum_elem, simparam->output_options.max_num_user_output_vars); - - //--- calculate bdy sets ---// - mesh->num_nodes_in_patch = 2*(num_dim-1); // 2 (2D) or 4 (3D) - mesh->num_patches_in_elem = 2*num_dim; // 4 (2D) or 6 (3D) - mesh->init_bdy_sets(num_bcs); - num_bdy_sets = mesh->num_bdy_sets; - printf("Num BC's = %lu\n", num_bcs); - - // patch ids in bdy set - bdy_patches_in_set = mesh->bdy_patches_in_set; - if(num_dim==2) - bdy_nodes = mesh->bdy_nodes; - - // tag boundary patches in the set - tag_bdys(boundary, *mesh, node_coords); - - build_boundry_node_sets(*mesh); - - // node ids in bdy_patch set - bdy_nodes_in_set = mesh->bdy_nodes_in_set; - num_bdy_nodes_in_set = mesh->num_bdy_nodes_in_set; - - //assign mesh views needed by the FEA module - - // elem ids in elem - elems_in_elem = mesh->elems_in_elem; - num_elems_in_elem = mesh->num_elems_in_elem; - - //corners - num_corners_in_node = mesh->num_corners_in_node; - corners_in_node = mesh->corners_in_node; - corners_in_elem = mesh->corners_in_elem; - - //elem-node conn & node-node conn - elems_in_node = mesh->elems_in_node; - if(num_dim==2){ - nodes_in_node = mesh->nodes_in_node; - num_nodes_in_node = mesh->num_nodes_in_node; - //patch conn - - patches_in_elem = mesh->patches_in_elem; - nodes_in_patch = mesh->nodes_in_patch; - elems_in_patch = mesh->elems_in_patch; - } - - // loop over BCs - for (size_t this_bdy = 0; this_bdy < num_bcs; this_bdy++){ - - RUN_CLASS({ - printf("Boundary Condition number %lu \n", this_bdy); - printf(" Num bdy patches in this set = %lu \n", bdy_patches_in_set.stride(this_bdy)); - printf(" Num bdy nodes in this set = %lu \n", bdy_nodes_in_set.stride(this_bdy)); - }); - Kokkos::fence(); - - }// end for - - // elem_mat_id needs to be initialized before initialization of material models - for (int f_id = 0; f_id < num_fills; f_id++){ - FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { - elem_mat_id(elem_gid) = mat_fill(f_id).material_id; - }); - } - elem_mat_id.update_host(); - - // function for initializing state_vars - init_state_vars(material, - elem_mat_id, - state_vars, - global_vars, - elem_user_output_vars, - rnum_elem); - - // initialize strength model - init_strength_model(elem_strength, - material, - elem_mat_id, - state_vars, - global_vars, - elem_user_output_vars, - rnum_elem); - - // initialize eos model - init_eos_model(elem_eos, - material, - elem_mat_id, - state_vars, - global_vars, - elem_user_output_vars, - rnum_elem); - - //--- apply the fill instructions over each of the Elements---// - - //initialize if topology optimization is used - if(simparam->topology_optimization_on){ - for(int elem_id = 0; elem_id < rnum_elem; elem_id++){ - relative_element_densities.host(elem_id) = 1; - }//for - relative_element_densities.update_device(); - } - - // loop over the fill instructures - for (int f_id = 0; f_id < num_fills; f_id++){ - - // parallel loop over elements in mesh - //for (size_t elem_gid = 0; elem_gid <= rnum_elem; elem_gid++) { - FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { - - // calculate the coordinates and radius of the element - double elem_coords[3]; // note:initialization with a list won't work - elem_coords[0] = 0.0; - elem_coords[1] = 0.0; - elem_coords[2] = 0.0; - - // get the coordinates of the element center - for (int node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - elem_coords[0] += node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), 0); - elem_coords[1] += node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), 1); - if (num_dim == 3){ - elem_coords[2] += node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), 2); - } else - { - elem_coords[2] = 0.0; - } - } // end loop over nodes in element - elem_coords[0] = elem_coords[0]/num_nodes_in_elem; - elem_coords[1] = elem_coords[1]/num_nodes_in_elem; - elem_coords[2] = elem_coords[2]/num_nodes_in_elem; - - // default is not to fill the element - bool fill_this = mat_fill(f_id).volume.contains(elem_coords); - - // paint the material state on the element - if (fill_this){ - - // density - elem_den(elem_gid) = mat_fill(f_id).den; - - // mass - elem_mass(elem_gid) = elem_den(elem_gid)*elem_vol(elem_gid); - - // specific internal energy - elem_sie(rk_level, elem_gid) = mat_fill(f_id).sie; - - size_t mat_id = elem_mat_id(elem_gid); // short name - - // --- stress tensor --- - // always 3D even for 2D-RZ - for (size_t i=0; i<3; i++){ - for (size_t j=0; j<3; j++){ - elem_stress(rk_level,elem_gid,i,j) = 0.0; - } - } // end for - - // short form for clean code - EOSParent * eos_model = elem_eos(elem_gid).model; - // --- Pressure --- - eos_model->calc_pressure(elem_pres, - elem_stress, - elem_gid, - elem_mat_id(elem_gid), - state_vars, - global_vars, - elem_user_output_vars, - elem_sspd, - elem_den(elem_gid), - elem_sie(rk_level,elem_gid)); - - // --- Sound speed --- - eos_model->calc_sound_speed(elem_pres, - elem_stress, - elem_gid, - elem_mat_id(elem_gid), - state_vars, - global_vars, - elem_user_output_vars, - elem_sspd, - elem_den(elem_gid), - elem_sie(rk_level,elem_gid)); - - // loop over the nodes of this element and apply velocity - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - - // get the mesh node index - size_t node_gid = nodes_in_elem(elem_gid, node_lid); - - - // --- Velocity --- - switch(mat_fill(f_id).velocity) - { - case VELOCITY_TYPE::cartesian: - { - - node_vel(rk_level, node_gid, 0) = mat_fill(f_id).u; - node_vel(rk_level, node_gid, 1) = mat_fill(f_id).v; - if (num_dim == 3) node_vel(rk_level, node_gid, 2) = mat_fill(f_id).w; - - - break; - } - case VELOCITY_TYPE::radial: - { - // Setting up cylindrical - double dir[2]; - dir[0] = 0.0; - dir[1] = 0.0; - double radius_val = 0.0; - - for(int dim=0; dim<2; dim++){ - dir[dim] = node_coords(rk_level, node_gid, dim); - radius_val += node_coords(rk_level, node_gid, dim)*node_coords(rk_level, node_gid, dim); - } // end for - radius_val = sqrt(radius_val); - - for(int dim=0; dim<2; dim++){ - if (radius_val > 1.0e-14){ - dir[dim] /= (radius_val); - } - else{ - dir[dim] = 0.0; - } - } // end for - - - node_vel(rk_level, node_gid, 0) = mat_fill(f_id).speed*dir[0]; - node_vel(rk_level, node_gid, 1) = mat_fill(f_id).speed*dir[1]; - if (num_dim == 3) node_vel(rk_level, node_gid, 2) = 0.0; - - break; - } - case VELOCITY_TYPE::spherical: - { - - // Setting up spherical - double dir[3]; - dir[0] = 0.0; - dir[1] = 0.0; - dir[2] = 0.0; - double radius_val = 0.0; - - for(int dim=0; dim<3; dim++){ - dir[dim] = node_coords(rk_level, node_gid, dim); - radius_val += node_coords(rk_level, node_gid, dim)*node_coords(rk_level, node_gid, dim); - } // end for - radius_val = sqrt(radius_val); - - for(int dim=0; dim<3; dim++){ - if (radius_val > 1.0e-14){ - dir[dim] /= (radius_val); - } - else{ - dir[dim] = 0.0; - } - } // end for - - node_vel(rk_level, node_gid, 0) = mat_fill(f_id).speed*dir[0]; - node_vel(rk_level, node_gid, 1) = mat_fill(f_id).speed*dir[1]; - if (num_dim == 3) node_vel(rk_level, node_gid, 2) = mat_fill(f_id).speed*dir[2]; - - break; - } - case VELOCITY_TYPE::radial_linear: - { - - break; - } - case VELOCITY_TYPE::spherical_linear: - { - - break; - } - case VELOCITY_TYPE::tg_vortex: - { - - node_vel(rk_level, node_gid, 0) = sin(PI * node_coords(rk_level,node_gid, 0)) * cos(PI * node_coords(rk_level,node_gid, 1)); - node_vel(rk_level, node_gid, 1) = -1.0*cos(PI * node_coords(rk_level,node_gid, 0)) * sin(PI * node_coords(rk_level,node_gid, 1)); - if (num_dim == 3) node_vel(rk_level, node_gid, 2) = 0.0; - - break; - } - } // end of switch - - }// end loop over nodes of element - - - if(mat_fill(f_id).velocity == VELOCITY_TYPE::tg_vortex) - { - elem_pres(elem_gid) = 0.25*( cos(2.0*PI*elem_coords[0]) + cos(2.0*PI*elem_coords[1]) ) + 1.0; - - // p = rho*ie*(gamma - 1) - size_t mat_id = f_id; - double gamma = global_vars(mat_id,0); // gamma value - elem_sie(rk_level, elem_gid) = - elem_pres(elem_gid)/(mat_fill(f_id).den*(gamma - 1.0)); - } // end if - - } // end if fill - - }); // end FOR_ALL_CLASS element loop - Kokkos::fence(); - - - } // end for loop over fills - - - // apply BC's to velocity - FEA_Module_SGH::boundary_velocity(*mesh, boundary, node_vel); - - - // calculate the corner massess if 2D - if(num_dim==2){ - - FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { - - // facial area of the corners - double corner_areas_array[4]; - - ViewCArrayKokkos corner_areas(&corner_areas_array[0],4); - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 4); - - get_area_weights2D(corner_areas, - elem_gid, - node_coords, - elem_node_gids, - rk_level); - - // loop over the corners of the element and calculate the mass - for (size_t corner_lid=0; corner_lid<4; corner_lid++){ - - size_t corner_gid = corners_in_elem(elem_gid, corner_lid); - corner_mass(corner_gid) = corner_areas(corner_lid)*elem_den(elem_gid); // node radius is added later - - } // end for over corners - }); - - } // end of - - - // calculate the nodal mass - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - - node_mass(node_gid) = 0.0; - - if(num_dim==3){ - - for(size_t elem_lid=0; elem_lidtopology_optimization_on||simparam->shape_optimization_on||simparam->num_dims==2){ - { - vec_array node_mass_interface = node_masses_distributed->getLocalView (Tpetra::Access::ReadWrite); - - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - node_mass_interface(node_gid,0) = node_mass(node_gid); - }); // end parallel for - } //end view scope - Kokkos::fence(); - //communicate ghost densities - comm_node_masses(); - - //this is forcing a copy to the device - //view scope - { - vec_array ghost_node_mass_interface = ghost_node_masses_distributed->getLocalView (Tpetra::Access::ReadWrite); - - - FOR_ALL_CLASS(node_gid, nlocal_nodes, nall_nodes, { - node_mass(node_gid) = ghost_node_mass_interface(node_gid-nlocal_nodes,0); - }); // end parallel for - } //end view scope - Kokkos::fence(); - } //endif - - //initialize if topology optimization is used - if(simparam->topology_optimization_on || simparam->shape_optimization_on){ - init_assembly(); - //assemble_matrix(); - } - - // update host copies of arrays modified in this function - elem_den.update_host(); - elem_mass.update_host(); - elem_sie.update_host(); - elem_stress.update_host(); - elem_pres.update_host(); - elem_sspd.update_host(); - - return; - -} // end of setup - -/* ---------------------------------------------------------------------------- - solve function called by solver -------------------------------------------------------------------------------- */ - -void FEA_Module_SGH::module_cleanup(){ - cleanup_material_models(); -} - -/* ---------------------------------------------------------------------------- - Deallocate memory used for material models -------------------------------------------------------------------------------- */ - -void FEA_Module_SGH::cleanup_material_models() { - - const DCArrayKokkos material = simparam->material; - - // destroy strength model - destroy_strength_model(elem_strength, - material, - elem_mat_id, - state_vars, - global_vars, - elem_user_output_vars, - rnum_elem); - - // destroy eos model - destroy_eos_model(elem_eos, - material, - elem_mat_id, - state_vars, - global_vars, - elem_user_output_vars, - rnum_elem); - return; - -} // end cleanup_user_strength_model; - - -/** - * Determines which of the boundary patches are associated with which boundary. - * - * Modifies: bdy_patches_in_set -*/ -void FEA_Module_SGH::tag_bdys(const DCArrayKokkos &boundary, - mesh_t &mesh, - const DViewCArrayKokkos &node_coords){ - - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - size_t num_dim = simparam->num_dims; - int nboundary_patches = Explicit_Solver_Pointer_->nboundary_patches; - int num_nodes_in_patch = mesh.num_nodes_in_patch; - - //if (bdy_set == mesh.num_bdy_sets){ - // printf(" ERROR: number of boundary sets must be increased by %zu", - // bdy_set-mesh.num_bdy_sets+1); - // exit(0); - //} // end if - - //error and debug flag - //DCArrayKokkos print_flag(1, "print_flag"); - //print_flag.host(0) = false; - //print_flag.update_device(); - - - FOR_ALL_CLASS(bdy_set, 0, num_bdy_sets, { - - // tag boundaries - BOUNDARY_TYPE bc_type = boundary(bdy_set).surface.type; - double val = boundary(bdy_set).surface.plane_position; - - // save the boundary patches to this set that are on the plane, spheres, etc. - for (size_t bdy_patch_lid=0; bdy_patch_lid < nboundary_patches; bdy_patch_lid++){ - - // save the patch index - size_t bdy_patch_gid = bdy_patch_lid; - - - // check to see if this patch is on the specified plane - bool is_on_bdy = check_bdy(bdy_patch_gid, - num_dim, - num_nodes_in_patch, - bc_type, - val, - node_coords, - rk_level); // no=0, yes=1 - - //debug check - /* - for (size_t patch_node_lid=0; patch_node_lid &node_coords, - const size_t rk_level) const { - - // default bool is not on the boundary - size_t is_on_bdy = 0; - - // the patch coordinates - double these_patch_coords[3]; // Note: cannot allocated array with num_dim - - // loop over the nodes on the patch - for (size_t patch_node_lid=0; patch_node_lid < num_nodes_in_patch; patch_node_lid++) { - // get the nodal_gid for this node in the patch - //size_t node_gid = mesh.nodes_in_patch(patch_gid, patch_node_lid); - size_t node_gid = Local_Index_Boundary_Patches(patch_gid, patch_node_lid); - - for (size_t dim = 0; dim < num_dim; dim++){ - these_patch_coords[dim] = node_coords(rk_level, node_gid, dim); // (rk, node_gid, dim) - } - - if (bc_type == BOUNDARY_TYPE::x_plane) { - if ( fabs(these_patch_coords[0] - val) <= 1.0e-7 ) is_on_bdy += 1; - } - else if (bc_type == BOUNDARY_TYPE::y_plane) { - if ( fabs(these_patch_coords[1] - val) <= 1.0e-7 ) is_on_bdy += 1; - } - else if (bc_type == BOUNDARY_TYPE::z_plane) { - if ( fabs(these_patch_coords[2] - val) <= 1.0e-7 ) is_on_bdy += 1; - } - else if (bc_type == BOUNDARY_TYPE::cylinder){ - real_t R = sqrt(these_patch_coords[0]*these_patch_coords[0] + - these_patch_coords[1]*these_patch_coords[1]); - - if ( fabs(R - val) <= 1.0e-7 ) is_on_bdy += 1; - } - else if (bc_type == BOUNDARY_TYPE::sphere) { - real_t R = sqrt(these_patch_coords[0]*these_patch_coords[0] + - these_patch_coords[1]*these_patch_coords[1] + - these_patch_coords[2]*these_patch_coords[2]); - - if ( fabs(R - val) <= 1.0e-7 ) is_on_bdy += 1; - } - } - - // if all nodes in the patch are on the surface - return is_on_bdy == num_nodes_in_patch; -} // end method to check bdy - -/* ---------------------------------------------------------------------------- - Build set of nodes assigned to each boundary condition -------------------------------------------------------------------------------- */ - -void FEA_Module_SGH::build_boundry_node_sets(mesh_t &mesh){ - - // build boundary nodes in each boundary set - int nboundary_patches = Explicit_Solver_Pointer_->nboundary_patches; - int num_nodes_in_patch = mesh.num_nodes_in_patch; - num_bdy_nodes_in_set = mesh.num_bdy_nodes_in_set = DCArrayKokkos (num_bdy_sets, "num_bdy_nodes_in_set"); - CArrayKokkos temp_count_num_bdy_nodes_in_set(num_bdy_sets, nall_nodes, "temp_count_num_bdy_nodes_in_set"); - - DynamicRaggedRightArrayKokkos temp_nodes_in_set (mesh.num_bdy_sets, nboundary_patches*mesh.num_nodes_in_patch, "temp_nodes_in_set"); - - // Parallel loop over boundary sets on device - FOR_ALL_CLASS(bdy_set, 0, num_bdy_sets, { - - // finde the number of patches_in_set - size_t num_bdy_patches_in_set = bdy_patches_in_set.stride(bdy_set); - - num_bdy_nodes_in_set(bdy_set) = 0; - - // Loop over boundary patches in boundary set - for (size_t bdy_patch_gid = 0; bdy_patch_gid (mesh.num_bdy_nodes_in_set, "bdy_nodes_in_set"); - - FOR_ALL_CLASS (bdy_set, 0, num_bdy_sets, { - - // Loop over boundary patches in boundary set - for (size_t bdy_node_lid=0; bdy_node_liddynamic_options; - - const size_t rk_level = dynamic_options.rk_num_bins - 1; - time_value = dynamic_options.time_initial; - time_final = dynamic_options.time_final; - dt_max = dynamic_options.dt_max; - dt_min = dynamic_options.dt_min; - dt_cfl = dynamic_options.dt_cfl; - graphics_time = simparam->output_options.graphics_step; - graphics_dt_ival = simparam->output_options.graphics_step; - cycle_stop = dynamic_options.cycle_stop; - rk_num_stages = dynamic_options.rk_num_stages; - dt = dynamic_options.dt; - fuzz = dynamic_options.fuzz; - tiny = dynamic_options.tiny; - small = dynamic_options.small; - graphics_times = simparam->output_options.graphics_times; - graphics_id = simparam->output_options.graphics_id; - size_t num_bdy_nodes = mesh->num_bdy_nodes; - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos material = simparam->material; - int nTO_modules; - int old_max_forward_buffer; - size_t cycle; - const int num_dim = simparam->num_dims; - real_t objective_accumulation, global_objective_accumulation; - std::vector> FEA_Module_My_TO_Modules = simparam->FEA_Module_My_TO_Modules; - problem = Explicit_Solver_Pointer_->problem; //Pointer to ROL optimization problem object - ROL::Ptr> obj_pointer; - - //reset time accumulating objective and constraints - /* - for(int imodule = 0 ; imodule < FEA_Module_My_TO_Modules[my_fea_module_index_].size(); imodule++){ - current_module_index = FEA_Module_My_TO_Modules[my_fea_module_index_][imodule]; - //test if module needs reset - if(){ - - } - } - */ - //simple setup to just request KE for now; above loop to be expanded and used later for scanning modules - if(simparam->topology_optimization_on){ - obj_pointer = problem->getObjective(); - KineticEnergyMinimize_TopOpt& kinetic_energy_minimize_function = dynamic_cast(*obj_pointer); - kinetic_energy_minimize_function.objective_accumulation = 0; - global_objective_accumulation = objective_accumulation = 0; - kinetic_energy_objective = true; - if(max_time_steps +1 > forward_solve_velocity_data->size()){ - old_max_forward_buffer = forward_solve_velocity_data->size(); - time_data.resize(max_time_steps+1); - forward_solve_velocity_data->resize(max_time_steps+1); - forward_solve_coordinate_data->resize(max_time_steps+1); - forward_solve_internal_energy_data->resize(max_time_steps+1); - adjoint_vector_data->resize(max_time_steps+1); - phi_adjoint_vector_data->resize(max_time_steps+1); - psi_adjoint_vector_data->resize(max_time_steps+1); - //assign a multivector of corresponding size to each new timestep in the buffer - for(int istep = old_max_forward_buffer; istep < max_time_steps+1; istep++){ - (*forward_solve_velocity_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - (*forward_solve_coordinate_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - (*forward_solve_internal_energy_data)[istep] = Teuchos::rcp(new MV(all_element_map, 1)); - (*adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - (*phi_adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - (*psi_adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_element_map, 1)); - } - } - } - - if(simparam->topology_optimization_on) - nTO_modules = simparam->TO_Module_List.size(); - - int myrank = Explicit_Solver_Pointer_->myrank; - if(simparam->output_options.output_file_format==OUTPUT_FORMAT::vtk&&simparam->output_options.write_initial) - { - if(myrank==0) - printf("Writing outputs to file at %f \n", time_value); - - double comm_time1 = Explicit_Solver_Pointer_->CPU_Time(); - Explicit_Solver_Pointer_->write_outputs(); - double comm_time2 = Explicit_Solver_Pointer_->CPU_Time(); - Explicit_Solver_Pointer_->output_time += comm_time2 - comm_time1; - } - - CArrayKokkos node_extensive_mass(nall_nodes, "node_extensive_mass"); - - // extensive energy tallies over the mesh elements local to this MPI rank - double IE_t0 = 0.0; - double KE_t0 = 0.0; - double TE_t0 = 0.0; - - double IE_sum = 0.0; - double KE_sum = 0.0; - - double IE_loc_sum = 0.0; - double KE_loc_sum = 0.0; - - // extensive energy tallies over the entire mesh - double global_IE_t0 = 0.0; - double global_KE_t0 = 0.0; - double global_TE_t0 = 0.0; - - // ---- Calculate energy tallies ---- - double IE_tend = 0.0; - double KE_tend = 0.0; - double TE_tend = 0.0; - - double global_IE_tend = 0.0; - double global_KE_tend = 0.0; - double global_TE_tend = 0.0; - - int nlocal_elem_non_overlapping = Explicit_Solver_Pointer_->nlocal_elem_non_overlapping; - - // extensive IE - REDUCE_SUM_CLASS(elem_gid, 0, nlocal_elem_non_overlapping, IE_loc_sum, { - IE_loc_sum += elem_mass(elem_gid)*elem_sie(rk_level,elem_gid); - - }, IE_sum); - IE_t0 = IE_sum; - - MPI_Allreduce(&IE_t0,&global_IE_t0,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD); - - // extensive KE - REDUCE_SUM_CLASS(node_gid, 0, nlocal_nodes, KE_loc_sum, { - - double ke = 0; - for (size_t dim=0; dimtopology_optimization_on||simparam->shape_optimization_on){ - time_data[0] = 0; - //assign current velocity data to multivector - //view scope - { - vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView (Tpetra::Access::ReadWrite); - vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - node_velocities_interface(node_gid,idim) = node_vel(rk_level,node_gid,idim); - node_coords_interface(node_gid,idim) = node_coords(rk_level,node_gid,idim); - } - }); - } //end view scope - Kokkos::fence(); - - //communicate ghosts - double comm_time1 = Explicit_Solver_Pointer_->CPU_Time(); - - //active view scope; triggers host comms from updated data on device - { - const_host_vec_array node_velocities_host = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_host_vec_array node_coords_host = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - } - double comm_time2 = Explicit_Solver_Pointer_->CPU_Time(); - Explicit_Solver_Pointer_->dev2host_time += comm_time2-comm_time1; - - //communicate ghost velocities - Explicit_Solver_Pointer_->comm_velocities(); - Explicit_Solver_Pointer_->comm_coordinates(); - - - double comm_time3 = Explicit_Solver_Pointer_->CPU_Time(); - - //view scope - { - const_vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array ghost_node_velocities_interface = Explicit_Solver_Pointer_->ghost_node_velocities_distributed->getLocalView (Tpetra::Access::ReadOnly); - vec_array all_node_velocities_interface = Explicit_Solver_Pointer_->all_node_velocities_distributed->getLocalView (Tpetra::Access::ReadWrite); - vec_array element_internal_energy = element_internal_energy_distributed->getLocalView (Tpetra::Access::ReadWrite); - const_vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array ghost_node_coords_interface = Explicit_Solver_Pointer_->ghost_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - all_node_velocities_interface(node_gid,idim) = node_velocities_interface(node_gid,idim); - all_node_coords_interface(node_gid,idim) = node_coords_interface(node_gid,idim); - } - }); // end parallel for - Kokkos::fence(); - - FOR_ALL_CLASS(node_gid, nlocal_nodes, nlocal_nodes+nghost_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - all_node_velocities_interface(node_gid,idim) = ghost_node_velocities_interface(node_gid-nlocal_nodes,idim); - all_node_coords_interface(node_gid,idim) = ghost_node_coords_interface(node_gid-nlocal_nodes,idim); - } - }); // end parallel for - Kokkos::fence(); - - - //interface for element internal energies - FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { - element_internal_energy(elem_gid,0) = elem_sie(rk_level,elem_gid); - }); // end parallel for - Kokkos::fence(); - } //end view scope - - (*forward_solve_internal_energy_data)[0]->assign(*element_internal_energy_distributed); - (*forward_solve_velocity_data)[0]->assign(*Explicit_Solver_Pointer_->all_node_velocities_distributed); - (*forward_solve_coordinate_data)[0]->assign(*Explicit_Solver_Pointer_->all_node_coords_distributed); - } - - // loop over the max number of time integration cycles - for (cycle = 0; cycle < cycle_stop; cycle++) { - - // get the step - if(num_dim==2){ - get_timestep2D(*mesh, - node_coords, - node_vel, - elem_sspd, - elem_vol); - } - else { - get_timestep(*mesh, - node_coords, - node_vel, - elem_sspd, - elem_vol); - } // end if 2D - - double global_dt; - MPI_Allreduce(&dt,&global_dt,1,MPI_DOUBLE,MPI_MIN,MPI_COMM_WORLD); - dt = global_dt; - - // stop calculation if flag - //if (stop_calc == 1) break; - - - if(simparam->dynamic_options.output_time_sequence_level>=TIME_OUTPUT_LEVEL::high){ - if (cycle==0){ - if(myrank==0) - printf("cycle = %lu, time = %12.5e, time step = %12.5e \n", cycle, time_value, dt); - } - // print time step every 10 cycles - else if (cycle%20==0){ - if(myrank==0) - printf("cycle = %lu, time = %12.5e, time step = %12.5e \n", cycle, time_value, dt); - } // end if - } - - // --------------------------------------------------------------------- - // integrate the solution forward to t(n+1) via Runge Kutta (RK) method - // --------------------------------------------------------------------- - - // save the values at t_n - rk_init(node_coords, - node_vel, - elem_sie, - elem_stress, - rnum_elem, - nall_nodes); - - - - // integrate solution forward in time - for (size_t rk_stage = 0; rk_stage < rk_num_stages; rk_stage++){ - - - // ---- RK coefficient ---- - double rk_alpha = 1.0/((double)rk_num_stages - (double)rk_stage); - - // ---- Calculate velocity diveregence for the element ---- - if(num_dim==2){ - get_divergence2D(elem_div, - node_coords, - node_vel, - elem_vol); - } - else { - get_divergence(elem_div, - node_coords, - node_vel, - elem_vol); - } // end if 2D - - // ---- calculate the forces on the vertices and evolve stress (hypo model) ---- - if(num_dim==2){ - get_force_sgh2D(material, - *mesh, - node_coords, - node_vel, - elem_den, - elem_sie, - elem_pres, - elem_stress, - elem_sspd, - elem_vol, - elem_div, - elem_mat_id, - corner_force, - rk_alpha, - cycle); - } - else { - get_force_sgh(material, - *mesh, - node_coords, - node_vel, - elem_den, - elem_sie, - elem_pres, - elem_stress, - elem_sspd, - elem_vol, - elem_div, - elem_mat_id, - corner_force, - rk_alpha, - cycle); - } - - /* - debug block - if(myrank==1){ - std::cout << rk_alpha << " " << dt << std::endl; - for(int i = 0; i < nall_nodes; i++){ - double node_force[3]; - for (size_t dim = 0; dim < num_dim; dim++){ - node_force[dim] = 0.0; - } // end for dim - - // loop over all corners around the node and calculate the nodal force - for (size_t corner_lid=0; corner_lidall_node_map->getGlobalElement(i) << " " << corner_gid << " " << corner_force(corner_gid, 0) << " " << corner_force(corner_gid, 1) << " " << corner_force(corner_gid, 2) << std::endl; - // loop over dimension - for (size_t dim = 0; dim < num_dim; dim++){ - node_force[dim] += corner_force(corner_gid, dim); - } // end for dim - - } // end for corner_lid - //std::cout << Explicit_Solver_Pointer_->all_node_map->getGlobalElement(i) << " " << node_force[0] << " " << node_force[1] << " " << node_force[2] << std::endl; - //std::cout << Explicit_Solver_Pointer_->all_node_map->getGlobalElement(i) << " " << node_mass(i) << std::endl; - } - } - /* - //debug print vector values on a rank - /* - if(myrank==0) - for(int i = 0; i < nall_nodes; i++){ - std::cout << Explicit_Solver_Pointer_->all_node_map->getGlobalElement(i) << " " << node_vel(rk_level,i,0) << " " << node_vel(rk_level,i,1) << " " << node_vel(rk_level,i,2) << std::endl; - } - */ - - // ---- Update nodal velocities ---- // - update_velocity_sgh(rk_alpha, - node_vel, - node_mass, - corner_force); - - if(have_loading_conditions){ - applied_forces(material, - *mesh, - node_coords, - node_vel, - node_mass, - elem_den, - elem_vol, - elem_div, - elem_mat_id, - corner_force, - rk_alpha, - cycle); - } - - // ---- apply force boundary conditions to the boundary patches---- - boundary_velocity(*mesh, boundary, node_vel); - - //current interface has differing velocity arrays; this equates them until we unify memory - //first comm time interval point - double comm_time1 = Explicit_Solver_Pointer_->CPU_Time(); - //view scope - { - vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - node_velocities_interface(node_gid,idim) = node_vel(rk_level,node_gid,idim); - } - }); // end parallel for - } //end view scope - Kokkos::fence(); - - //active view scope - { - const_host_vec_array node_velocities_host = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView (Tpetra::Access::ReadOnly); - } - double comm_time2 = Explicit_Solver_Pointer_->CPU_Time(); - Explicit_Solver_Pointer_->dev2host_time += comm_time2-comm_time1; - //communicate ghost velocities - Explicit_Solver_Pointer_->comm_velocities(); - - double comm_time3 = Explicit_Solver_Pointer_->CPU_Time(); - //this is forcing a copy to the device - //view scope - { - vec_array ghost_node_velocities_interface = Explicit_Solver_Pointer_->ghost_node_velocities_distributed->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, nlocal_nodes, nall_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - node_vel(rk_level,node_gid,idim) = ghost_node_velocities_interface(node_gid-nlocal_nodes,idim); - } - - }); // end parallel for - } //end view scope - Kokkos::fence(); - - double comm_time4 = Explicit_Solver_Pointer_->CPU_Time(); - Explicit_Solver_Pointer_->host2dev_time += comm_time4-comm_time3; - Explicit_Solver_Pointer_->communication_time += comm_time4-comm_time1; - //debug print vector values on a rank - /* - if(myrank==0) - for(int i = 0; i < nall_nodes; i++){ - std::cout << Explicit_Solver_Pointer_->all_node_map->getGlobalElement(i) << " " << node_vel(rk_level,i,0) << " " << node_vel(rk_level,i,1) << " " << node_vel(rk_level,i,2) << std::endl; - } - */ - // ---- Update specific internal energy in the elements ---- - update_energy_sgh(rk_alpha, - *mesh, - node_vel, - node_coords, - elem_sie, - elem_mass, - corner_force); - - - // ---- Update nodal positions ---- - update_position_sgh(rk_alpha, - nall_nodes, - node_coords, - node_vel); - - - // ---- Calculate cell volume for next time step ---- - get_vol(); - - - - // ---- Calculate elem state (den, pres, sound speed, stress) for next time step ---- - if(num_dim==2){ - update_state2D(material, - *mesh, - node_coords, - node_vel, - elem_den, - elem_pres, - elem_stress, - elem_sspd, - elem_sie, - elem_vol, - elem_mass, - elem_mat_id, - rk_alpha, - cycle); - } - else{ - update_state(material, - *mesh, - node_coords, - node_vel, - elem_den, - elem_pres, - elem_stress, - elem_sspd, - elem_sie, - elem_vol, - elem_mass, - elem_mat_id, - rk_alpha, - cycle); - } - // ---- - // Notes on strength: - // 1) hyper-elastic strength models are called in update_state - // 2) hypo-elastic strength models are called in get_force - // 3) strength models must be added by the user in user_mat.cpp - - - // calculate the new corner masses if 2D - if(num_dim==2){ - - // calculate the nodal areal mass - FOR_ALL_CLASS(node_gid, 0, nall_nodes, { - - node_mass(node_gid) = 0.0; - - if (node_coords(rk_level,node_gid,1) > tiny){ - node_mass(node_gid) = node_extensive_mass(node_gid)/node_coords(rk_level,node_gid,1); - } - //if(cycle==0&&node_gid==1&&myrank==0) - //std::cout << "index " << node_gid << " on rank " << myrank << " node vel " << node_vel(rk_level,node_gid,0) << " " << node_mass(node_gid) << std::endl << std::flush; - - }); // end parallel for over node_gid - Kokkos::fence(); - - //current interface has differing density arrays; this equates them until we unify memory - //view scope - { - vec_array node_mass_interface = node_masses_distributed->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - node_mass_interface(node_gid,0) = node_mass(node_gid); - }); // end parallel for - } //end view scope - Kokkos::fence(); - //communicate ghost densities - comm_node_masses(); - - //this is forcing a copy to the device - //view scope - { - vec_array ghost_node_mass_interface = ghost_node_masses_distributed->getLocalView (Tpetra::Access::ReadWrite); - - FOR_ALL_CLASS(node_gid, nlocal_nodes, nall_nodes, { - node_mass(node_gid) = ghost_node_mass_interface(node_gid-nlocal_nodes,0); - }); // end parallel for - } //end view scope - Kokkos::fence(); - - - // ----------------------------------------------- - // Calcualte the areal mass for nodes on the axis - // ----------------------------------------------- - // The node order of the 2D element is - // - // J - // | - // 3---2 - // | | -- I - // 0---1 - /* - FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { - - // loop over the corners of the element and calculate the mass - for (size_t node_lid=0; node_lid<4; node_lid++){ - - size_t node_gid = nodes_in_elem(elem_gid, node_lid); - size_t node_minus_gid; - size_t node_plus_gid; - - - if (node_coords(rk_level,node_gid,1) < tiny){ - // node is on the axis - - // minus node - if (node_lid==0){ - node_minus_gid = nodes_in_elem(elem_gid, 3); - } else { - node_minus_gid = nodes_in_elem(elem_gid, node_lid-1); - } - - // plus node - if (node_lid==3){ - node_plus_gid = nodes_in_elem(elem_gid, 0); - } else { - node_plus_gid = nodes_in_elem(elem_gid, node_lid+1); - } - - node_mass(node_gid) = fmax(node_mass(node_plus_gid), node_mass(node_minus_gid))/2.0; - - } // end if - - } // end for over corners - - }); // end parallel for over elem_gid - Kokkos::fence(); - */ - - FOR_ALL_CLASS(node_bdy_gid, 0, num_bdy_nodes, { - //FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - size_t node_gid = bdy_nodes(node_bdy_gid); - - if (node_coords(rk_level,node_gid,1) < tiny){ - // node is on the axis - - for(size_t node_lid=0; node_lid < num_nodes_in_node(node_gid); node_lid++){ - - size_t node_neighbor_gid = nodes_in_node(node_gid, node_lid); - - // if the node is off the axis, use it's areal mass on the boundary - if (node_coords(rk_level,node_neighbor_gid,1) > tiny){ - node_mass(node_gid) = fmax(node_mass(node_gid), node_mass(node_neighbor_gid)/2.0); - } - - } // end for over neighboring nodes - - } // end if - - }); // end parallel for over elem_gid - - } // end of if 2D-RZ - - - } // end of RK loop - - // increment the time - Explicit_Solver_Pointer_->time_value = simparam->dynamic_options.time_value = time_value+=dt; - - if(simparam->topology_optimization_on||simparam->shape_optimization_on){ - if(cycle >= max_time_steps) - max_time_steps = cycle + 1; - - if(max_time_steps + 1 > forward_solve_velocity_data->size()){ - old_max_forward_buffer = forward_solve_velocity_data->size(); - time_data.resize(max_time_steps + BUFFER_GROW +1); - forward_solve_velocity_data->resize(max_time_steps + BUFFER_GROW +1); - forward_solve_coordinate_data->resize(max_time_steps + BUFFER_GROW +1); - forward_solve_internal_energy_data->resize(max_time_steps + BUFFER_GROW +1); - adjoint_vector_data->resize(max_time_steps + BUFFER_GROW +1); - phi_adjoint_vector_data->resize(max_time_steps + BUFFER_GROW +1); - psi_adjoint_vector_data->resize(max_time_steps + BUFFER_GROW +1); - //assign a multivector of corresponding size to each new timestep in the buffer - for(int istep = old_max_forward_buffer; istep < max_time_steps + BUFFER_GROW +1; istep++){ - (*forward_solve_velocity_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - (*forward_solve_coordinate_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - (*forward_solve_internal_energy_data)[istep] = Teuchos::rcp(new MV(all_element_map, 1)); - (*adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - (*phi_adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - (*psi_adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_element_map, 1)); - } - } - - - time_data[cycle+1] = dt + time_data[cycle]; - - - //assign current velocity data to multivector - //view scope - { - vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView (Tpetra::Access::ReadWrite); - vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - node_velocities_interface(node_gid,idim) = node_vel(rk_level,node_gid,idim); - node_coords_interface(node_gid,idim) = node_coords(rk_level,node_gid,idim); - } - }); - } //end view scope - Kokkos::fence(); - - //communicate ghosts - double comm_time1 = Explicit_Solver_Pointer_->CPU_Time(); - - //active view scope; triggers host comms from updated data on device - { - const_host_vec_array node_velocities_host = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_host_vec_array node_coords_host = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - } - double comm_time2 = Explicit_Solver_Pointer_->CPU_Time(); - Explicit_Solver_Pointer_->dev2host_time += comm_time2-comm_time1; - - //communicate ghost velocities - Explicit_Solver_Pointer_->comm_velocities(); - Explicit_Solver_Pointer_->comm_coordinates(); - - - double comm_time3 = Explicit_Solver_Pointer_->CPU_Time(); - - //view scope - { - const_vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array ghost_node_velocities_interface = Explicit_Solver_Pointer_->ghost_node_velocities_distributed->getLocalView (Tpetra::Access::ReadOnly); - vec_array all_node_velocities_interface = Explicit_Solver_Pointer_->all_node_velocities_distributed->getLocalView (Tpetra::Access::ReadWrite); - vec_array element_internal_energy = element_internal_energy_distributed->getLocalView (Tpetra::Access::ReadWrite); - const_vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array ghost_node_coords_interface = Explicit_Solver_Pointer_->ghost_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - all_node_velocities_interface(node_gid,idim) = node_velocities_interface(node_gid,idim); - all_node_coords_interface(node_gid,idim) = node_coords_interface(node_gid,idim); - } - }); // end parallel for - Kokkos::fence(); - - FOR_ALL_CLASS(node_gid, nlocal_nodes, nlocal_nodes+nghost_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - all_node_velocities_interface(node_gid,idim) = ghost_node_velocities_interface(node_gid-nlocal_nodes,idim); - all_node_coords_interface(node_gid,idim) = ghost_node_coords_interface(node_gid-nlocal_nodes,idim); - } - }); // end parallel for - Kokkos::fence(); - - //interface for element internal energies - FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { - element_internal_energy(elem_gid,0) = elem_sie(rk_level,elem_gid); - }); // end parallel for - Kokkos::fence(); - } //end view scope - - double comm_time4 = Explicit_Solver_Pointer_->CPU_Time(); - Explicit_Solver_Pointer_->host2dev_time += comm_time4-comm_time3; - Explicit_Solver_Pointer_->communication_time += comm_time4-comm_time1; - - - (*forward_solve_internal_energy_data)[cycle+1]->assign(*element_internal_energy_distributed); - (*forward_solve_velocity_data)[cycle+1]->assign(*Explicit_Solver_Pointer_->all_node_velocities_distributed); - (*forward_solve_coordinate_data)[cycle+1]->assign(*Explicit_Solver_Pointer_->all_node_coords_distributed); - - //kinetic energy accumulation - if(kinetic_energy_objective){ - const_vec_array node_velocities_interface = (*forward_solve_velocity_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array previous_node_velocities_interface = (*forward_solve_velocity_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - KE_loc_sum = 0.0; - KE_sum = 0.0; - // extensive KE - REDUCE_SUM_CLASS(node_gid, 0, nlocal_nodes, KE_loc_sum, { - - double ke = 0; - for (size_t dim=0; dim0){ - write = 1; - } - else if (cycle == cycle_stop) { - write = 1; - } - else if (time_value >= time_final&&simparam->output_options.write_final){ - write = 1; - } - else if (time_value >= graphics_time){ - write = 1; - } - - // write outputs - if (write == 1){ - //interface nodal coordinate data (note: this is not needed if using write_outputs()) - //view scope - { - vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - node_coords_interface(node_gid,idim) = node_coords(rk_level,node_gid,idim); - } - }); // end parallel for - } //end view scope - if(simparam->output_options.output_file_format==OUTPUT_FORMAT::vtk){ - if(myrank==0){ - printf("Writing outputs to file at %f \n", graphics_time); - } - - double comm_time1 = Explicit_Solver_Pointer_->CPU_Time(); - Explicit_Solver_Pointer_->write_outputs(); - - double comm_time2 = Explicit_Solver_Pointer_->CPU_Time(); - Explicit_Solver_Pointer_->output_time += comm_time2 - comm_time1; - } - - graphics_time = time_value + graphics_dt_ival; - } // end if - - - // end of calculation - if (time_value>=time_final) break; - - - } // end for cycle loop - - last_time_step = cycle; - - //simple setup to just calculate KE minimize objective for now - if(simparam->topology_optimization_on){ - KineticEnergyMinimize_TopOpt& kinetic_energy_minimize_function = dynamic_cast(*obj_pointer); - - //collect local objective values - MPI_Allreduce(&objective_accumulation,&global_objective_accumulation,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD); - kinetic_energy_minimize_function.objective_accumulation = global_objective_accumulation; - - if(myrank==0) - std::cout << "CURRENT TIME INTEGRAL OF KINETIC ENERGY " << global_objective_accumulation << std::endl; - } - - - auto time_2 = std::chrono::high_resolution_clock::now(); - auto time_difference = time_2 - time_1; - //double calc_time = std::chrono::duration_cast(diff).count(); - double calc_time = std::chrono::duration_cast(time_difference).count(); - if(myrank==0) - printf("\nCalculation time in seconds: %f \n", calc_time*1e-09); - - IE_loc_sum = 0.0; - KE_loc_sum = 0.0; - IE_sum = 0.0; - KE_sum = 0.0; - - // extensive IE - REDUCE_SUM_CLASS(elem_gid, 0, nlocal_elem_non_overlapping, IE_loc_sum, { - - IE_loc_sum += elem_mass(elem_gid)*elem_sie(rk_level,elem_gid); - - }, IE_sum); - IE_tend = IE_sum; - - //reduce over MPI ranks - MPI_Allreduce(&IE_tend,&global_IE_tend,1,MPI_DOUBLE,MPI_SUM,MPI_COMM_WORLD); - - // extensive KE - REDUCE_SUM_CLASS(node_gid, 0, nlocal_nodes, KE_loc_sum, { - - double ke = 0; - for (size_t dim=0; dim mesh_in, const int my_fea_module_index = 0); - ~FEA_Module_SGH(); - - //initialize data for boundaries of the model and storage for boundary conditions and applied loads - void sgh_interface_setup(node_t &node, elem_t &elem, corner_t &corner); - - void setup(); - - void cleanup_material_models(); - - int solve(); - - void module_cleanup(); - - void sgh_solve(); - - void get_force_sgh(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_pres, - DViewCArrayKokkos &elem_stress, - const DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &elem_mat_id, - DViewCArrayKokkos &corner_force, - const double rk_alpha, - const size_t cycle); - - void get_force_vgradient_sgh(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &elem_mat_id, - const double rk_alpha, - const size_t cycle); - - void get_force_ugradient_sgh(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &elem_mat_id, - const double rk_alpha, - const size_t cycle); - - void get_force_egradient_sgh(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &elem_mat_id, - const double rk_alpha, - const size_t cycle); - - void get_force_dgradient_sgh(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &elem_mat_id, - const double rk_alpha, - const size_t cycle); - - void force_design_gradient_term(const_vec_array design_variables, vec_array design_gradients); - - - void get_force_sgh2D(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &elem_mat_id, - DViewCArrayKokkos &corner_force, - const double rk_alpha, - const size_t cycle); - - void update_position_sgh(double rk_alpha, - const size_t num_nodes, - DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel); - - void get_vol(); - - void get_vol_ugradient(const size_t gradient_node_id, const size_t gradient_dim); - - void init_assembly(); - - KOKKOS_INLINE_FUNCTION - void get_vol_hex(const DViewCArrayKokkos &elem_vol, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const; - - KOKKOS_FUNCTION - void get_vol_hex_ugradient(const ViewCArrayKokkos &elem_vol_gradients, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const; - - - KOKKOS_INLINE_FUNCTION - void get_vol_quad(const DViewCArrayKokkos &elem_vol, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const; - - - KOKKOS_FUNCTION - double get_area_quad(const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const; - - - KOKKOS_FUNCTION - void get_bmatrix(const ViewCArrayKokkos &B_matrix, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const; - - KOKKOS_FUNCTION - void get_bmatrix_gradients(const ViewCArrayKokkos &B_matrix_gradients, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const; - - - KOKKOS_FUNCTION - void get_bmatrix2D(const ViewCArrayKokkos &B_matrix, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const; - - KOKKOS_FUNCTION - void get_area_weights2D(const ViewCArrayKokkos &corner_areas, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const; - - - KOKKOS_INLINE_FUNCTION - double heron(const double x1, - const double y1, - const double x2, - const double y2, - const double x3, - const double y3) const; - - double average_element_density(const int nodes_per_elem, const CArray current_element_densities) const; - - void get_divergence(DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_vol); - - - void get_divergence2D(DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_vol); - - - KOKKOS_FUNCTION - void get_velgrad(ViewCArrayKokkos &vel_grad, - const ViewCArrayKokkos &elem_node_gids, - const DViewCArrayKokkos &node_vel, - const ViewCArrayKokkos &b_matrix, - const double elem_vol, - const size_t elem_gid, - const size_t rk_level) const; - - - KOKKOS_FUNCTION - void get_velgrad2D(ViewCArrayKokkos &vel_grad, - const ViewCArrayKokkos &elem_node_gids, - const DViewCArrayKokkos &node_vel, - const ViewCArrayKokkos &b_matrix, - const double elem_vol, - const double elem_area, - const size_t elem_gid, - const size_t rk_level) const; - - KOKKOS_INLINE_FUNCTION - void decompose_vel_grad(ViewCArrayKokkos &D_tensor, - ViewCArrayKokkos &W_tensor, - const ViewCArrayKokkos &vel_grad, - const ViewCArrayKokkos &elem_node_gids, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const double vol) const; - - - void update_velocity_sgh(double rk_alpha, - DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &node_mass, - const DViewCArrayKokkos &corner_force); - - void tag_bdys(const DCArrayKokkos &boundary, - mesh_t &mesh, - const DViewCArrayKokkos &node_coords); - - void boundary_velocity(const mesh_t &mesh, - const DCArrayKokkos &boundary, - DViewCArrayKokkos &node_vel); - - KOKKOS_INLINE_FUNCTION - bool check_bdy(const size_t patch_gid, - const int num_dim, - const int num_nodes_in_patch, - const BOUNDARY_TYPE this_bc_tag, - const double val, - const DViewCArrayKokkos &node_coords, - const size_t rk_level) const; - - void rk_init(DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &elem_sie, - DViewCArrayKokkos &elem_stress, - const size_t num_elems, - const size_t num_nodes); - - - void get_timestep(mesh_t &mesh, - DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &elem_sspd, - DViewCArrayKokkos &elem_vol); - - - void get_timestep2D(mesh_t &mesh, - DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &elem_sspd, - DViewCArrayKokkos &elem_vol); - - void update_energy_sgh(double rk_alpha, - const mesh_t &mesh, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_mass, - const DViewCArrayKokkos &corner_force); - - - void power_design_gradient_term(const_vec_array design_variables, vec_array design_gradients); - - void get_power_dgradient_sgh(double rk_alpha, - const mesh_t &mesh, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_mass, - const DViewCArrayKokkos &corner_force, - DCArrayKokkos elem_power_dgradients); - - void get_power_ugradient_sgh(double rk_alpha, - const mesh_t &mesh, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_mass, - const DViewCArrayKokkos &corner_force); - - void get_power_vgradient_sgh(double rk_alpha, - const mesh_t &mesh, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_mass, - const DViewCArrayKokkos &corner_force); - - void get_power_egradient_sgh(double rk_alpha, - const mesh_t &mesh, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_mass, - const DViewCArrayKokkos &corner_force); - - void update_state(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &elem_den, - DViewCArrayKokkos &elem_pres, - DViewCArrayKokkos &elem_stress, - DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_mass, - const DViewCArrayKokkos &elem_mat_id, - const double rk_alpha, - const size_t cycle); - - - void update_state2D(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &elem_den, - DViewCArrayKokkos &elem_pres, - DViewCArrayKokkos &elem_stress, - DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_mass, - const DViewCArrayKokkos &elem_mat_id, - const double rk_alpha, - const size_t cycle); - - void build_boundry_node_sets(mesh_t &mesh); - - void init_boundaries(); - - //initializes memory for arrays used in the global stiffness matrix assembly - void init_boundary_sets(int num_boundary_sets); - - void grow_boundary_sets(int num_boundary_sets); - - virtual void update_forward_solve(Teuchos::RCP zp); - - void comm_node_masses(); - - void comm_adjoint_vector(int cycle); - - void comm_phi_adjoint_vector(int cycle); - - void comm_variables(Teuchos::RCP zp); - - void read_conditions_ansys_dat(std::ifstream *in, std::streampos before_condition_header); - - //interfaces between user input and creating data structures for bcs - void generate_bcs(); - - void Displacement_Boundary_Conditions(); - - void init_output(); - - void compute_output(); - - void output_control(); - - void sort_output(Teuchos::RCP > sorted_map); - - void sort_element_output(Teuchos::RCP > sorted_map); - - void collect_output(Teuchos::RCP > global_reduce_map); - - void write_data(std::map &point_data_scalars_double, - std::map &point_data_vectors_double, - std::map &cell_data_scalars_double, - std::map &cell_data_scalars_int, - std::map > &cell_data_fields_double); - - void write_outputs (const mesh_t &mesh, - DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &node_mass, - DViewCArrayKokkos &elem_den, - DViewCArrayKokkos &elem_pres, - DViewCArrayKokkos &elem_stress, - DViewCArrayKokkos &elem_sspd, - DViewCArrayKokkos &elem_sie, - DViewCArrayKokkos &elem_vol, - DViewCArrayKokkos &elem_mass, - DViewCArrayKokkos &elem_mat_id); - - - void ensight(const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &node_mass, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_mass, - const DViewCArrayKokkos &elem_mat_id); - - - void state_file(const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &node_mass, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_mass, - const DViewCArrayKokkos &elem_mat_id); - - void node_density_constraints(host_vec_array node_densities_lower_bound); - - void compute_topology_optimization_adjoint_full(); //Force depends on node coords and velocity - - void compute_topology_optimization_gradient_full(Teuchos::RCP design_densities_distributed, Teuchos::RCP design_gradients_distributed); - - void boundary_adjoint(const mesh_t &mesh, - const DCArrayKokkos &boundary, - vec_array &node_adjoint, - vec_array &node_phi_adjoint, - vec_array &node_psi_adjoint); - - void applied_forces(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &node_mass, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &elem_mat_id, - DViewCArrayKokkos &corner_force, - const double rk_alpha, - const size_t cycle); - - bool have_loading_conditions; - bool nodal_density_flag; - real_t penalty_power; - - Simulation_Parameters_Explicit *simparam; - SGH_Parameters *module_params; - Explicit_Solver *Explicit_Solver_Pointer_; - - elements::ref_element *ref_elem; - - std::shared_ptr mesh; - //shallow copies of mesh class views - size_t num_nodes_in_elem; - // corner ids in node - RaggedRightArrayKokkos corners_in_node; - CArrayKokkos num_corners_in_node; - - // elem ids in node - RaggedRightArrayKokkos elems_in_node; - - // node ids in node - RaggedRightArrayKokkos nodes_in_node; - CArrayKokkos num_nodes_in_node; - - // node ids in elem - DCArrayKokkos nodes_in_elem; - - // corner ids in elem - CArrayKokkos corners_in_elem; - - // elem ids in elem - RaggedRightArrayKokkos elems_in_elem; - CArrayKokkos num_elems_in_elem; - - // patch ids in elem - CArrayKokkos patches_in_elem; - - // node ids in a patch - CArrayKokkos nodes_in_patch; - - // element ids in a patch - CArrayKokkos elems_in_patch; - - // bdy nodes - CArrayKokkos bdy_nodes; - - //Topology optimization filter variable - DCArrayKokkos relative_element_densities; - - //Local FEA data - host_elem_conn_array interface_nodes_in_elem; //host view of element connectivity to nodes - - //Global FEA data - Teuchos::RCP node_velocities_distributed; - Teuchos::RCP initial_node_coords_distributed; - Teuchos::RCP all_initial_node_coords_distributed; - Teuchos::RCP initial_node_velocities_distributed; - Teuchos::RCP all_node_velocities_distributed; - Teuchos::RCP all_cached_node_velocities_distributed; - Teuchos::RCP node_masses_distributed; - Teuchos::RCP ghost_node_masses_distributed; - Teuchos::RCP adjoint_vector_distributed; - Teuchos::RCP phi_adjoint_vector_distributed; - Teuchos::RCP psi_adjoint_vector_distributed; - Teuchos::RCP element_internal_energy_distributed; - Teuchos::RCP>> forward_solve_velocity_data; - Teuchos::RCP>> forward_solve_coordinate_data; - Teuchos::RCP>> forward_solve_internal_energy_data; - Teuchos::RCP>> adjoint_vector_data; - Teuchos::RCP>> phi_adjoint_vector_data; - Teuchos::RCP>> psi_adjoint_vector_data; - Teuchos::RCP force_gradient_design; - Teuchos::RCP force_gradient_position; - Teuchos::RCP force_gradient_velocity; - - //Local FEA data - DCArrayKokkos Global_Gradient_Matrix_Assembly_Map; //Maps element local nodes to columns on ragged right node connectivity graph - DCArrayKokkos Element_Gradient_Matrix_Assembly_Map; //Maps element-node pair to columns on ragged right node to element connectivity - RaggedRightArrayKokkos Graph_Matrix; //stores local indices - RaggedRightArrayKokkos DOF_Graph_Matrix; //stores local indices - RaggedRightArrayKokkos Force_Gradient_Positions; - RaggedRightArrayKokkos Force_Gradient_Velocities; - CArrayKokkos Force_Gradient_Energies; //transposed such that elem ids correspond to rows - RaggedRightArrayKokkos Power_Gradient_Positions; //transposed such that node dofs correspond to rows - RaggedRightArrayKokkos Power_Gradient_Velocities; //transposed such that node dofs correspond to rows - CArrayKokkos Power_Gradient_Energies; - DCArrayKokkos Gradient_Matrix_Strides; - DCArrayKokkos DOF_to_Elem_Matrix_Strides; - DCArrayKokkos Elem_to_Elem_Matrix_Strides; - DCArrayKokkos Graph_Matrix_Strides; - RaggedRightArrayKokkos Original_Gradient_Entries; - RaggedRightArrayKokkos Original_Gradient_Entry_Indices; - DCArrayKokkos Original_Gradient_Entries_Strides; - - //distributed matrices - Teuchos::RCP distributed_force_gradient_positions; - Teuchos::RCP distributed_force_gradient_velocities; - - std::vector time_data; - int max_time_steps, last_time_step; - - // --------------------------------------------------------------------- - // state data type declarations (must stay in scope for output after run) - // --------------------------------------------------------------------- - node_t node_interface; - elem_t elem_interface; - corner_t corner_interface; - - //Dual View wrappers - // Dual Views of the individual node struct variables - DViewCArrayKokkos node_coords; - DViewCArrayKokkos node_vel; - DViewCArrayKokkos node_mass; - - // Dual Views of the individual elem struct variables - DViewCArrayKokkos elem_den; - DViewCArrayKokkos elem_pres; - DViewCArrayKokkos elem_stress; // always 3D even in 2D-RZ - DViewCArrayKokkos elem_sspd; - DViewCArrayKokkos elem_sie; - DViewCArrayKokkos elem_vol; - DViewCArrayKokkos elem_div; - DViewCArrayKokkos elem_mass; - DViewCArrayKokkos elem_mat_id; - - // Element velocity gradient - DCArrayKokkos elem_vel_grad; - - // for storing global variables used in user material model - DCArrayKokkos global_vars; - - // for storing state variables used in user material model - DCArrayKokkos state_vars; - - //elem_user_output_vars allow users to output variables of interest per element - DCArrayKokkos elem_user_output_vars; - - //material models - DCArrayKokkos elem_eos; - DCArrayKokkos elem_strength; - - // Dual Views of the corner struct variables - DViewCArrayKokkos corner_force; - DViewCArrayKokkos corner_mass; - - //Boundary Conditions Data - DCArrayKokkos Local_Index_Boundary_Patches; - //CArray Patch_Nodes; - enum bc_type {NONE, POINT_LOADING_CONDITION, LINE_LOADING_CONDITION, SURFACE_LOADING_CONDITION}; - - //Boundary Conditions Data - int max_boundary_sets; - - //output dof data - //Global arrays with collected data used to print - int output_velocity_index, output_strain_index, output_stress_index; - - //parameters - double time_value, time_final, dt, dt_max, dt_min, dt_cfl, graphics_time, graphics_dt_ival; - size_t graphics_cyc_ival, cycle_stop, rk_num_stages, graphics_id; - double fuzz, tiny, small; - CArray graphics_times; - int rk_num_bins; - - //optimization flags - bool kinetic_energy_objective; -}; - -#endif // end HEADER_H diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/energy_sgh.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/energy_sgh.cpp deleted file mode 100644 index cf62a2b5d..000000000 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/energy_sgh.cpp +++ /dev/null @@ -1,57 +0,0 @@ - -#include "mesh.h" -#include "state.h" -#include "FEA_Module_SGH.h" - -void FEA_Module_SGH::update_energy_sgh(double rk_alpha, - const mesh_t &mesh, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_mass, - const DViewCArrayKokkos &corner_force){ - - const size_t rk_level = rk_num_bins - 1; - int num_dims = num_dim; - - // loop over all the elements in the mesh - FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { - - double elem_power = 0.0; - - // --- tally the contribution from each corner to the element --- - - // Loop over the nodes in the element - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - - size_t corner_lid = node_lid; - - // Get node global id for the local node id - size_t node_gid = nodes_in_elem(elem_gid, node_lid); - - // Get the corner global id for the local corner id - size_t corner_gid = corners_in_elem(elem_gid, corner_lid); - - double node_radius = 1; - if(num_dims==2){ - node_radius = node_coords(rk_level,node_gid,1); - } - - // calculate the Power=F dot V for this corner - for (size_t dim=0; dim -#include -#include -#include -#include -#include // fmin, fmax, abs note: fminl is long -#include -#include - -#include -#include -#include - -#include "elements.h" -#include "swage.h" -#include "matar.h" -#include "utilities.h" -#include "FEA_Module_SGH.h" -#include "Simulation_Parameters/Simulation_Parameters_Explicit.h" -#include "Simulation_Parameters/FEA_Module/SGH_Parameters.h" - - -// ----------------------------------------------------------------------------- -// This function calculates the corner force gradients w.r.t velocity; the -// current implementation assumes material.q2 = 0 -//------------------------------------------------------------------------------ -void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &elem_mat_id, - const double rk_alpha, - const size_t cycle - ){ - - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - const size_t num_dims = simparam->num_dims; - size_t num_corners = rnum_elem*num_nodes_in_elem; - - //initialize gradient matrix - FOR_ALL_CLASS(dof_gid, 0, nlocal_nodes*num_dims, { - for(int idof = 0; idof < Gradient_Matrix_Strides(dof_gid); idof++){ - Force_Gradient_Velocities(dof_gid,idof) = 0; - } - }); // end parallel for loop over nodes - Kokkos::fence(); - - //initialize buffer storage; not all components are explicitly set for this routine - FOR_ALL_CLASS(corner_gid, 0, num_corners, { - for (int dim = 0; dim < num_dims; dim++){ - //assign gradient of corner contribution of force to relevant matrix entries with non-zero node velocity gradient - for(int igradient = 0; igradient < num_nodes_in_elem; igradient++){ - for(int jdim = 0; jdim < num_dims; jdim++){ - corner_gradient_storage(corner_gid,dim,igradient,jdim) = 0; - } - } - } - }); // end parallel for loop over nodes - Kokkos::fence(); - - // --- calculate the forces acting on the nodes from the element --- - for (size_t elem_gid = 0; elem_gid < rnum_elem; elem_gid++){ - //FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { - - const size_t num_nodes_in_elem = 8; - real_t gradient_result[num_dims]; - // total Cauchy stress - double tau_array[9]; - double tau_gradient_array[9]; - - // corner area normals - double area_normal_array[24]; - - // estimate of shock direction - double shock_dir_array[3]; - - // the sums in the Riemann solver - double sum_array[4]; - - // corner shock impeadance x |corner area normal dot shock_dir| - double muc_array[8]; - double muc_gradient_array[8]; - - // Riemann velocity - double vel_star_array[3]; - double vel_star_gradient_array[3*num_nodes_in_elem]; - - // velocity gradient - double vel_grad_array[9]; - - // --- Create views of arrays to aid the force calculation --- - - ViewCArrayKokkos tau(tau_array, num_dims, num_dims); - ViewCArrayKokkos tau_gradient(tau_gradient_array, num_dims, num_dims); - ViewCArrayKokkos area_normal(area_normal_array, num_nodes_in_elem, num_dims); - ViewCArrayKokkos shock_dir(shock_dir_array, num_dims); - ViewCArrayKokkos sum(sum_array, 4); - ViewCArrayKokkos muc(muc_array, num_nodes_in_elem); - ViewCArrayKokkos muc_gradient(muc_gradient_array, num_nodes_in_elem); - ViewCArrayKokkos vel_star(vel_star_array, num_dims); - ViewCArrayKokkos vel_star_gradient(vel_star_gradient_array, num_nodes_in_elem, num_dims); - ViewCArrayKokkos vel_grad(vel_grad_array, num_dims, num_dims); - - EOSParent* eos_model = elem_eos(elem_gid).model; - - // --- abviatations of variables --- - - // element volume - double vol = elem_vol(elem_gid); - - // create a view of the stress_matrix - ViewCArrayKokkos stress(&elem_stress(rk_level, elem_gid, 0,0), 3, 3); - - - // cut out the node_gids for this element - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 8); - - - - // get the B matrix which are the OUTWARD corner area normals - get_bmatrix(area_normal, - elem_gid, - node_coords, - elem_node_gids, - rk_level); - - - // --- Calculate the velocity gradient --- - get_velgrad(vel_grad, - elem_node_gids, - node_vel, - area_normal, - vol, - elem_gid, - rk_level); - - - // the -1 is for the inward surface area normal, - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - for (size_t dim = 0; dim < num_dims; dim++){ - area_normal(node_lid, dim) = (-1.0)*area_normal(node_lid,dim); - } // end for - } // end for - - - - double div = elem_div(elem_gid); - - - // vel = [u,v,w] - // [du/dx, du/dy, du/dz] - // vel_grad = [dv/dx, dv/dy, dv/dz] - // [dw/dx, dw/dy, dw/dz] - double curl[3]; - curl[0] = vel_grad(2,1) - vel_grad(1,2); // dw/dy - dv/dz - curl[1] = vel_grad(0,2) - vel_grad(2,0); // du/dz - dw/dx - curl[2] = vel_grad(1,0) - vel_grad(0,1); // dv/dx - du/dy - - double mag_curl = sqrt(curl[0]*curl[0] + curl[1]*curl[1] + curl[2]*curl[2]); - - - // --- Calculate the Cauchy stress --- - for (size_t i = 0; i < 3; i++){ - for (size_t j = 0; j < 3; j++){ - tau(i, j) = stress(i,j); - // artificial viscosity can be added here to tau - } // end for - } //end for - - // add the pressure - for (int i = 0; i < num_dims; i++){ - tau(i, i) -= elem_pres(elem_gid); - } // end for - - - - - // ---- Multidirectional Approximate Riemann solver (MARS) ---- - // find the average velocity of the elem, it is an - // estimate of the Riemann velocity - - // initialize to Riemann velocity to zero - for (size_t dim = 0; dim < num_dims; dim++){ - vel_star(dim) = 0.0; - } - - // loop over nodes and calculate an average velocity, which is - // an estimate of Riemann velocity - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - - // Get node gloabl index and create view of nodal velocity - int node_gid = nodes_in_elem(elem_gid, node_lid); - - - ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); - - vel_star(0) += 0.125*vel(0); - vel_star(1) += 0.125*vel(1); - vel_star(2) += 0.125*vel(2); - - } // end for loop over nodes - - // find shock direction and shock impedance associated with each node - - // initialize sum term in MARS to zero - for (int i = 0; i < 4; i++){ - sum(i) = 0.0; - } - - double mag; // magnitude of the area normal - double mag_vel; // magnitude of velocity - - // loop over the nodes of the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { - - // Get global node id - size_t node_gid = nodes_in_elem(elem_gid, node_lid); - - // Create view of nodal velocity - ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); - - // Get an estimate of the shock direction. - mag_vel = sqrt( (vel(0) - vel_star(0) )*(vel(0) - vel_star(0) ) - + (vel(1) - vel_star(1) )*(vel(1) - vel_star(1) ) - + (vel(2) - vel_star(2) )*(vel(2) - vel_star(2) ) ); - - - if (mag_vel > small) { - - // estimate of the shock direction, a unit normal - for (int dim = 0; dim < num_dims; dim++){ - shock_dir(dim) = (vel(dim) - vel_star(dim)) / mag_vel; - } - } - - else { - - // if there is no velocity change, then use the surface area - // normal as the shock direction - mag = sqrt( area_normal(node_lid, 0)*area_normal(node_lid, 0) - + area_normal(node_lid, 1)*area_normal(node_lid, 1) - + area_normal(node_lid, 2)*area_normal(node_lid, 2) ); - - // estimate of the shock direction - for (int dim = 0; dim < num_dims; dim++){ - shock_dir(dim) = area_normal(node_lid, dim)/mag; - } - - } // end if mag_vel - - - // cell divergence indicates compression or expansions - size_t mat_id = elem_mat_id(elem_gid); - if (div < 0){ // element in compression - muc(node_lid) = elem_den(elem_gid) * - (material(mat_id).q1*elem_sspd(elem_gid) + material(mat_id).q2*mag_vel); - } - else { // element in expansion - muc(node_lid) = elem_den(elem_gid) * - (material(mat_id).q1ex*elem_sspd(elem_gid) + material(mat_id).q2ex*mag_vel); - } // end if on divergence sign - - size_t use_shock_dir = 0; - double mu_term; - double mu_term_gradient; - - // Coding to use shock direction - if (use_shock_dir == 1){ - // this is denominator of the Riamann solver and the multiplier - // on velocity in the numerator. It filters on the shock - // direction - mu_term = muc(node_lid)* - fabs( shock_dir(0)*area_normal(node_lid,0) - + shock_dir(1)*area_normal(node_lid,1) - + shock_dir(2)*area_normal(node_lid,2) ); //code gradient for shock dir w.r.t velocity if using shock_dir - } - else { - // Using a full tensoral Riemann jump relation - mu_term = muc(node_lid) - * sqrt( area_normal(node_lid, 0)*area_normal(node_lid, 0) - + area_normal(node_lid, 1)*area_normal(node_lid, 1) - + area_normal(node_lid, 2)*area_normal(node_lid, 2) ); - } - - sum(0) += mu_term*vel(0); - sum(1) += mu_term*vel(1); - sum(2) += mu_term*vel(2); - sum(3) += mu_term; - - muc(node_lid) = mu_term; // the impeadance time surface area is stored here - - } // end for node_lid loop over nodes of the elem - - - - - // The Riemann velocity, called vel_star - if (sum(3) > fuzz) { - for (size_t i = 0; i < num_dims; i++) { - vel_star(i) = sum(i)/sum(3); - } - } - else { - for (int i = 0; i < num_dims; i++){ - vel_star(i) = 0.0; - } - } // end if - - - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { - // The Riemann velocity, called vel_star - if (sum(3) > fuzz) { - for (size_t i = 0; i < num_dims; i++) { - vel_star_gradient(node_lid,i) = muc(node_lid)/sum(3); - } - } - else { - for (int i = 0; i < num_dims; i++){ - vel_star_gradient(node_lid,i) = 0; - } - } // end if - } - - - - // ---- Calculate the shock detector for the Riemann-solver ---- - // - // The dissipation from the Riemann problem is limited by phi - // phi = (1. - max( 0., min( 1. , r_face ) ))^n - // where - // r_face = (C* div(u_+)/div(u_z)) - // The plus denotes the cell center divergence of a neighbor. - // The solution will be first order when phi=1 and have - // zero dissipation when phi=0. - // phi = 0 highest-order solution - // phi = 1 first order solution - // - - double phi = 0.0; // the shock detector - double r_face = 1.0; // the ratio on the face - double r_min = 1.0; // the min ratio for the cell - double r_coef = 0.9; // 0.9; the coefficient on the ratio - // (1=minmod and 2=superbee) - double n_coef = 1.0; // the power on the limiting coefficient - // (1=nominal, and n_coeff > 1 oscillatory) - - - // loop over the nieghboring cells - for (size_t elem_lid = 0; elem_lid < num_elems_in_elem(elem_gid); elem_lid++){ - - // Get global index for neighboring cell - size_t neighbor_gid = elems_in_elem(elem_gid, elem_lid); - - // calculate the velocity divergence in neighbor - double div_neighbor = elem_div(neighbor_gid); - - r_face = r_coef*(div_neighbor + small)/(div + small); - - // store the smallest face ratio - r_min = fmin(r_face, r_min); - - } // end for elem_lid - - - // calculate standard shock detector - phi = 1.0 - fmax(0.0, r_min); - phi = pow(phi, n_coef); - - // Mach number shock detector - double omega = 20.0;//20.0; // weighting factor on Mach number - double third = 1.0/3.0; - double c_length = pow(vol, third); // characteristic length - double alpha = fmin(1.0, omega * (c_length * fabs(div))/(elem_sspd(elem_gid) + fuzz) ); - - // use Mach based detector with standard shock detector - - // turn off dissipation in expansion - //alpha = fmax(-fabs(div0)/div0 * alpha, 0.0); // this should be if(div0<0) alpha=alpha else alpha=0 - - phi = alpha*phi; - - // curl limiter on Q - double phi_curl = fmin(1.0, 1.0*fabs(div)/(mag_curl + fuzz)); // disable Q when vorticity is high - //phi = phi_curl*phi; - phi = 1; - - // ---- Calculate the Riemann force on each node ---- - - // loop over the each node in the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { - - size_t corner_lid = node_lid; - size_t column_index; - - // Get corner gid - size_t corner_gid = corners_in_elem(elem_gid, corner_lid); - - // Get node gid - size_t node_gid = nodes_in_elem(elem_gid, node_lid); - - // loop over dimension - for (int dim = 0; dim < num_dims; dim++){ - //assign gradient of corner contribution of force to relevant matrix entries with non-zero node velocity gradient - for(int igradient = 0; igradient < num_nodes_in_elem; igradient++){ - size_t gradient_node_gid = nodes_in_elem(elem_gid, igradient); - //if(!map->isNodeLocalElement(gradient_node_gid)) continue; - column_index = num_dims*Global_Gradient_Matrix_Assembly_Map(elem_gid, igradient, node_lid); - if(node_lid==igradient){ - if(map->isNodeLocalElement(gradient_node_gid)){ - Force_Gradient_Velocities(gradient_node_gid*num_dims+dim, column_index+dim) += phi*muc(node_lid)*(vel_star_gradient(igradient,dim) - 1); - } - corner_gradient_storage(corner_gid,dim,igradient,dim) = phi*muc(node_lid)*(vel_star_gradient(igradient,dim) - 1); - } - else{ - if(map->isNodeLocalElement(gradient_node_gid)){ - Force_Gradient_Velocities(gradient_node_gid*num_dims+dim, column_index+dim) += phi*muc(node_lid)*(vel_star_gradient(igradient,dim)); - } - corner_gradient_storage(corner_gid,dim,igradient,dim) = phi*muc(node_lid)*(vel_star_gradient(igradient,dim)); - } - } - } // end loop over dimension - - } // end for loop over nodes in elem - - // --- Update Stress --- - // calculate the new stress at the next rk level, if it is a hypo model - size_t mat_id = elem_mat_id(elem_gid); - - - - //}); // end parallel for loop over elements - } - - /* - //accumulate node values from corner storage - force_gradient_design->putScalar(0); - - vec_array force_gradient_design_view = force_gradient_design->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { - size_t corner_id; - for(int icorner=0; icorner < num_corners_in_node(node_id); icorner++){ - corner_id = corners_in_node(node_id,icorner); - force_gradient_design_view(node_id,0) += corner_vector_storage(corner_id, 0); - force_gradient_design_view(node_id,1) += corner_vector_storage(corner_id, 1); - force_gradient_design_view(node_id,2) += corner_vector_storage(corner_id, 2); - } - }); // end parallel for - Kokkos::fence(); - */ - - return; - -} // end of routine - -// ----------------------------------------------------------------------------- -// This function calculates the corner forces and the evolves stress (hypo) -//------------------------------------------------------------------------------ -void FEA_Module_SGH::get_force_egradient_sgh(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &elem_mat_id, - const double rk_alpha, - const size_t cycle - ){ - - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - const size_t num_dims = simparam->num_dims; - - //initialize gradient matrix - FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { - //compute resulting row of force displacement gradient matrix transpose right multiplied by adjoint vector - for(int idof = 0; idof < num_nodes_in_elem*num_dims; idof++){ - Force_Gradient_Energies(elem_gid,idof) = 0; - } - }); // end parallel for - Kokkos::fence(); - - // --- calculate the forces acting on the nodes from the element --- - for (size_t elem_gid = 0; elem_gid < rnum_elem; elem_gid++){ - //FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { - - const size_t num_nodes_in_elem = 8; - real_t gradient_result[num_dims]; - // total Cauchy stress - double tau_array[9]; - double tau_gradient_array[9]; - - // corner area normals - double area_normal_array[24]; - - // estimate of shock direction - double shock_dir_array[3]; - - // the sums in the Riemann solver - double sum_array[4]; - double sum_gradient_array[4]; - - // corner shock impeadance x |corner area normal dot shock_dir| - double muc_array[8]; - double muc_gradient_array[8]; - - // Riemann velocity - double vel_star_array[3]; - double vel_star_gradient_array[3]; - - // velocity gradient - double vel_grad_array[9]; - - // --- Create views of arrays to aid the force calculation --- - - ViewCArrayKokkos tau(tau_array, num_dims, num_dims); - ViewCArrayKokkos tau_gradient(tau_gradient_array, num_dims, num_dims); - ViewCArrayKokkos area_normal(area_normal_array, num_nodes_in_elem, num_dims); - ViewCArrayKokkos shock_dir(shock_dir_array, num_dims); - ViewCArrayKokkos sum(sum_array, 4); - ViewCArrayKokkos sum_gradient(sum_gradient_array, 4); - ViewCArrayKokkos muc(muc_array, num_nodes_in_elem); - ViewCArrayKokkos muc_gradient(muc_gradient_array, num_nodes_in_elem); - ViewCArrayKokkos vel_star(vel_star_array, num_dims); - ViewCArrayKokkos vel_star_gradient(vel_star_gradient_array, num_dims); - ViewCArrayKokkos vel_grad(vel_grad_array, num_dims, num_dims); - - EOSParent* eos_model = elem_eos(elem_gid).model; - - // --- abviatations of variables --- - - // element volume - double vol = elem_vol(elem_gid); - - // create a view of the stress_matrix - ViewCArrayKokkos stress(&elem_stress(rk_level, elem_gid, 0,0), 3, 3); - - - // cut out the node_gids for this element - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 8); - - - - // get the B matrix which are the OUTWARD corner area normals - get_bmatrix(area_normal, - elem_gid, - node_coords, - elem_node_gids, - rk_level); - - - // --- Calculate the velocity gradient --- - get_velgrad(vel_grad, - elem_node_gids, - node_vel, - area_normal, - vol, - elem_gid, - rk_level); - - - // the -1 is for the inward surface area normal, - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - for (size_t dim = 0; dim < num_dims; dim++){ - area_normal(node_lid, dim) = (-1.0)*area_normal(node_lid,dim); - } // end for - } // end for - - - - double div = elem_div(elem_gid); - - - // vel = [u,v,w] - // [du/dx, du/dy, du/dz] - // vel_grad = [dv/dx, dv/dy, dv/dz] - // [dw/dx, dw/dy, dw/dz] - double curl[3]; - curl[0] = vel_grad(2,1) - vel_grad(1,2); // dw/dy - dv/dz - curl[1] = vel_grad(0,2) - vel_grad(2,0); // du/dz - dw/dx - curl[2] = vel_grad(1,0) - vel_grad(0,1); // dv/dx - du/dy - - double mag_curl = sqrt(curl[0]*curl[0] + curl[1]*curl[1] + curl[2]*curl[2]); - - - // --- Calculate the Cauchy stress --- - for (size_t i = 0; i < num_dims; i++){ - for (size_t j = 0; j < num_dims; j++){ - tau(i, j) = stress(i,j); - tau_gradient(i,j) = 0; - // artificial viscosity can be added here to tau - } // end for - } //end for - - // add the pressure - for (int i = 0; i < num_dims; i++){ - tau(i, i) -= elem_pres(elem_gid); - tau_gradient(i, i) = -eos_model->calc_pressure_gradient_internal_energy(elem_pres, - elem_stress, - elem_gid, - elem_mat_id(elem_gid), - state_vars, - global_vars, - elem_user_output_vars, - elem_sspd, - elem_den(elem_gid), - elem_sie(rk_level,elem_gid)); - } // end for - - - - - // ---- Multidirectional Approximate Riemann solver (MARS) ---- - // find the average velocity of the elem, it is an - // estimate of the Riemann velocity - - // initialize to Riemann velocity to zero - for (size_t dim = 0; dim < num_dims; dim++){ - vel_star(dim) = 0.0; - } - - // loop over nodes and calculate an average velocity, which is - // an estimate of Riemann velocity - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - - // Get node gloabl index and create view of nodal velocity - int node_gid = nodes_in_elem(elem_gid, node_lid); - - - ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); - - vel_star(0) += 0.125*vel(0); - vel_star(1) += 0.125*vel(1); - vel_star(2) += 0.125*vel(2); - - } // end for loop over nodes - - // find shock direction and shock impedance associated with each node - - // initialize sum term in MARS to zero - for (int i = 0; i < 4; i++){ - sum(i) = sum_gradient(i) = 0.0; - } - - double mag; // magnitude of the area normal - double mag_vel; // magnitude of velocity - - // --- Sound speed --- - real_t sound_speed_gradient_energy = - eos_model->calc_sound_speed_gradient_internal_energy(elem_pres, - elem_stress, - elem_gid, - elem_mat_id(elem_gid), - state_vars, - global_vars, - elem_user_output_vars, - elem_sspd, - elem_den(elem_gid), - elem_sie(rk_level,elem_gid)); - - // loop over the nodes of the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { - - // Get global node id - size_t node_gid = nodes_in_elem(elem_gid, node_lid); - - // Create view of nodal velocity - ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); - - // Get an estimate of the shock direction. - mag_vel = sqrt( (vel(0) - vel_star(0) )*(vel(0) - vel_star(0) ) - + (vel(1) - vel_star(1) )*(vel(1) - vel_star(1) ) - + (vel(2) - vel_star(2) )*(vel(2) - vel_star(2) ) ); - - - if (mag_vel > small) { - - // estimate of the shock direction, a unit normal - for (int dim = 0; dim < num_dims; dim++){ - shock_dir(dim) = (vel(dim) - vel_star(dim)) / mag_vel; - } - } - - else { - - // if there is no velocity change, then use the surface area - // normal as the shock direction - mag = sqrt( area_normal(node_lid, 0)*area_normal(node_lid, 0) - + area_normal(node_lid, 1)*area_normal(node_lid, 1) - + area_normal(node_lid, 2)*area_normal(node_lid, 2) ); - - // estimate of the shock direction - for (int dim = 0; dim < num_dims; dim++){ - shock_dir(dim) = area_normal(node_lid, dim)/mag; - } - - } // end if mag_vel - - - // cell divergence indicates compression or expansions - size_t mat_id = elem_mat_id(elem_gid); - if (div < 0){ // element in compression - muc(node_lid) = elem_den(elem_gid) * - (material(mat_id).q1*elem_sspd(elem_gid) + material(mat_id).q2*mag_vel); - muc_gradient(node_lid) = elem_den(elem_gid) * - (material(mat_id).q1*sound_speed_gradient_energy); - } - else { // element in expansion - muc(node_lid) = elem_den(elem_gid) * - (material(mat_id).q1ex*elem_sspd(elem_gid) + material(mat_id).q2ex*mag_vel); - muc_gradient(node_lid) = elem_den(elem_gid) * - (material(mat_id).q1ex*sound_speed_gradient_energy); - } // end if on divergence sign - - - size_t use_shock_dir = 0; - double mu_term; - double mu_term_gradient; - - // Coding to use shock direction - if (use_shock_dir == 1){ - // this is denominator of the Riamann solver and the multiplier - // on velocity in the numerator. It filters on the shock - // direction - mu_term = muc(node_lid)* - fabs( shock_dir(0)*area_normal(node_lid,0) - + shock_dir(1)*area_normal(node_lid,1) - + shock_dir(2)*area_normal(node_lid,2) ); - mu_term_gradient = muc_gradient(node_lid)* //amend if shock dir has dependence for gradient not captured here - fabs( shock_dir(0)*area_normal(node_lid,0) - + shock_dir(1)*area_normal(node_lid,1) - + shock_dir(2)*area_normal(node_lid,2) ); - } - else { - // Using a full tensoral Riemann jump relation - mu_term = muc(node_lid) - * sqrt( area_normal(node_lid, 0)*area_normal(node_lid, 0) - + area_normal(node_lid, 1)*area_normal(node_lid, 1) - + area_normal(node_lid, 2)*area_normal(node_lid, 2) ); - mu_term_gradient = muc_gradient(node_lid) - * sqrt( area_normal(node_lid, 0)*area_normal(node_lid, 0) - + area_normal(node_lid, 1)*area_normal(node_lid, 1) - + area_normal(node_lid, 2)*area_normal(node_lid, 2) ); - } - - sum(0) += mu_term*vel(0); - sum(1) += mu_term*vel(1); - sum(2) += mu_term*vel(2); - sum(3) += mu_term; - - sum_gradient(0) += mu_term_gradient*vel(0); - sum_gradient(1) += mu_term_gradient*vel(1); - sum_gradient(2) += mu_term_gradient*vel(2); - sum_gradient(3) += mu_term_gradient; - - muc(node_lid) = mu_term; // the impeadance time surface area is stored here - muc_gradient(node_lid) = mu_term_gradient; - - } // end for node_lid loop over nodes of the elem - - - - - // The Riemann velocity, called vel_star - if (sum(3) > fuzz) { - for (size_t i = 0; i < num_dims; i++) { - vel_star(i) = sum(i)/sum(3); - vel_star_gradient(i) = sum_gradient(i)/sum(3) - sum_gradient(3)*sum(i)/sum(3)/sum(3); - } - } - else { - for (int i = 0; i < num_dims; i++){ - vel_star(i) = 0.0; - vel_star_gradient(i) = 0; - } - } // end if - - - - // ---- Calculate the shock detector for the Riemann-solver ---- - // - // The dissipation from the Riemann problem is limited by phi - // phi = (1. - max( 0., min( 1. , r_face ) ))^n - // where - // r_face = (C* div(u_+)/div(u_z)) - // The plus denotes the cell center divergence of a neighbor. - // The solution will be first order when phi=1 and have - // zero dissipation when phi=0. - // phi = 0 highest-order solution - // phi = 1 first order solution - // - - double phi = 0.0; // the shock detector - double r_face = 1.0; // the ratio on the face - double r_min = 1.0; // the min ratio for the cell - double r_coef = 0.9; // 0.9; the coefficient on the ratio - // (1=minmod and 2=superbee) - double n_coef = 1.0; // the power on the limiting coefficient - // (1=nominal, and n_coeff > 1 oscillatory) - - - // loop over the nieghboring cells - for (size_t elem_lid = 0; elem_lid < num_elems_in_elem(elem_gid); elem_lid++){ - - // Get global index for neighboring cell - size_t neighbor_gid = elems_in_elem(elem_gid, elem_lid); - - // calculate the velocity divergence in neighbor - double div_neighbor = elem_div(neighbor_gid); - - r_face = r_coef*(div_neighbor + small)/(div + small); - - // store the smallest face ratio - r_min = fmin(r_face, r_min); - - } // end for elem_lid - - - // calculate standard shock detector - phi = 1.0 - fmax(0.0, r_min); - phi = pow(phi, n_coef); - - // Mach number shock detector - double omega = 20.0;//20.0; // weighting factor on Mach number - double third = 1.0/3.0; - double c_length = pow(vol, third); // characteristic length - double alpha = fmin(1.0, omega * (c_length * fabs(div))/(elem_sspd(elem_gid) + fuzz) ); - - // use Mach based detector with standard shock detector - - // turn off dissipation in expansion - //alpha = fmax(-fabs(div0)/div0 * alpha, 0.0); // this should be if(div0<0) alpha=alpha else alpha=0 - - phi = alpha*phi; - - // curl limiter on Q - double phi_curl = fmin(1.0, 1.0*fabs(div)/(mag_curl + fuzz)); // disable Q when vorticity is high - //phi = phi_curl*phi; - phi = 1; - - // ---- Calculate the Riemann force on each node ---- - - // loop over the each node in the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { - - size_t corner_lid = node_lid; - - // Get corner gid - size_t corner_gid = corners_in_elem(elem_gid, corner_lid); - - // Get node gid - size_t node_gid = nodes_in_elem(elem_gid, node_lid); - - // loop over dimension - for (int dim = 0; dim < num_dims; dim++){ - - Force_Gradient_Energies(elem_gid,node_lid*num_dims+dim) = corner_vector_storage(corner_gid, dim) = - area_normal(node_lid, 0)*tau_gradient(0, dim) - + area_normal(node_lid, 1)*tau_gradient(1, dim) - + area_normal(node_lid, 2)*tau_gradient(2, dim) - + phi*muc_gradient(node_lid)*(vel_star(dim) - node_vel(rk_level, node_gid, dim)) - + phi*muc(node_lid)*(vel_star_gradient(dim)); - - } // end loop over dimension - - } // end for loop over nodes in elem - - - - // --- Update Stress --- - // calculate the new stress at the next rk level, if it is a hypo model - - size_t mat_id = elem_mat_id(elem_gid); - - - } - //}); // end parallel for loop over elements - - /* - //accumulate node values from corner storage - force_gradient_design->putScalar(0); - - vec_array force_gradient_design_view = force_gradient_design->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { - size_t corner_id; - for(int icorner=0; icorner < num_corners_in_node(node_id); icorner++){ - corner_id = corners_in_node(node_id,icorner); - force_gradient_design_view(node_id,0) += corner_vector_storage(corner_id, 0); - force_gradient_design_view(node_id,1) += corner_vector_storage(corner_id, 1); - force_gradient_design_view(node_id,2) += corner_vector_storage(corner_id, 2); - } - }); // end parallel for - Kokkos::fence(); - */ - - return; - -} // end of routine - -// ----------------------------------------------------------------------------- -// This function calculates the corner forces and the evolves stress (hypo) -//------------------------------------------------------------------------------ -void FEA_Module_SGH::get_force_ugradient_sgh(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &elem_mat_id, - const double rk_alpha, - const size_t cycle - ){ - - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - const size_t num_dims = simparam->num_dims; - - //initialize gradient matrix - FOR_ALL_CLASS(dof_gid, 0, nlocal_nodes*num_dims, { - for(int idof = 0; idof < Gradient_Matrix_Strides(dof_gid); idof++){ - Force_Gradient_Positions(dof_gid,idof) = 0; - } - }); // end parallel for loop over nodes - Kokkos::fence(); - // --- calculate the forces acting on the nodes from the element --- - for (size_t elem_gid = 0; elem_gid < rnum_elem; elem_gid++){ - //FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { - - const size_t num_nodes_in_elem = 8; - real_t gradient_result[num_dims]; - // total Cauchy stress - double tau_array[9]; - double tau_gradient_array[9*max_nodes_per_element*num_dims]; - - // corner area normals - double area_normal_gradients_array[max_nodes_per_element*max_nodes_per_element*num_dims*num_dims]; - double area_normal_array[24]; - - //volume data - double volume; - double volume_gradients_array[max_nodes_per_element*num_dims]; - - // estimate of shock direction - double shock_dir_array[3]; - - // the sums in the Riemann solver - double sum_array[4]; - double sum_gradient_array[4*max_nodes_per_element*num_dims]; - - // corner shock impeadance x |corner area normal dot shock_dir| - double muc_array[8]; - double muc_gradient_array[8*max_nodes_per_element*num_dims]; - - // Riemann velocity - double vel_star_array[3]; - double vel_star_gradient_array[num_dims*max_nodes_per_element*num_dims]; - - // velocity gradient - double vel_grad_array[9]; - - // --- Create views of arrays to aid the force calculation --- - - ViewCArrayKokkos tau(tau_array, num_dims, num_dims); - ViewCArrayKokkos tau_gradient(tau_gradient_array, num_dims, num_dims, max_nodes_per_element, num_dims); - ViewCArrayKokkos volume_gradients(volume_gradients_array, num_nodes_in_elem, num_dims); - ViewCArrayKokkos area_normal(area_normal_array, num_nodes_in_elem, num_dims); - ViewCArrayKokkos area_normal_gradients(area_normal_gradients_array, num_nodes_in_elem, num_dims, num_nodes_in_elem, num_dims); - ViewCArrayKokkos shock_dir(shock_dir_array, num_dims); - ViewCArrayKokkos sum(sum_array, 4); - ViewCArrayKokkos sum_gradient(sum_gradient_array, 4, max_nodes_per_element, num_dims); - ViewCArrayKokkos muc(muc_array, num_nodes_in_elem); - ViewCArrayKokkos muc_gradient(muc_gradient_array, num_nodes_in_elem, num_nodes_in_elem, num_dims); - ViewCArrayKokkos vel_star(vel_star_array, num_dims); - ViewCArrayKokkos vel_star_gradient(vel_star_gradient_array, num_dims, num_nodes_in_elem, num_dims); - ViewCArrayKokkos vel_grad(vel_grad_array, num_dims, num_dims); - - EOSParent* eos_model = elem_eos(elem_gid).model; - - // --- abviatations of variables --- - - // element volume - double vol = elem_vol(elem_gid); - - // create a view of the stress_matrix - ViewCArrayKokkos stress(&elem_stress(rk_level, elem_gid, 0,0), 3, 3); - - // cut out the node_gids for this element - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 8); - - //gradients of the element volume - get_vol_hex_ugradient(volume_gradients, elem_gid, node_coords, elem_node_gids, rk_level); - - - // //debug - // for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - // for (size_t dim = 0; dim < num_dims; dim++){ - // if(volume_gradients(node_lid, dim)>1 || volume_gradients(node_lid, dim) < -1) - // std::cout << volume_gradients(node_lid, dim) << " "; - // } // end for - // } // end for - // std::cout << std::endl; - - // get the B matrix which are the OUTWARD corner area normals - get_bmatrix(area_normal, - elem_gid, - node_coords, - elem_node_gids, - rk_level); - - get_bmatrix_gradients(area_normal_gradients, - elem_gid, - node_coords, - elem_node_gids, - rk_level); - - - // --- Calculate the velocity gradient --- - get_velgrad(vel_grad, - elem_node_gids, - node_vel, - area_normal, - vol, - elem_gid, - rk_level); - - - // the -1 is for the inward surface area normal, - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - for (size_t dim = 0; dim < num_dims; dim++){ - area_normal(node_lid, dim) = (-1.0)*area_normal(node_lid,dim); - } // end for - } // end for - - // the -1 is for the inward surface area normal, - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - for (size_t dim = 0; dim < num_dims; dim++){ - for (size_t igradient = 0; igradient < num_nodes_in_elem; igradient++){ - for (size_t jdim = 0; jdim < num_dims; jdim++){ - area_normal_gradients(node_lid, dim, igradient, jdim) = (-1.0)*area_normal_gradients(node_lid,dim, igradient, jdim); - } - } - } // end for - } // end for - - - - double div = elem_div(elem_gid); - - - // vel = [u,v,w] - // [du/dx, du/dy, du/dz] - // vel_grad = [dv/dx, dv/dy, dv/dz] - // [dw/dx, dw/dy, dw/dz] - double curl[3]; - curl[0] = vel_grad(2,1) - vel_grad(1,2); // dw/dy - dv/dz - curl[1] = vel_grad(0,2) - vel_grad(2,0); // du/dz - dw/dx - curl[2] = vel_grad(1,0) - vel_grad(0,1); // dv/dx - du/dy - - double mag_curl = sqrt(curl[0]*curl[0] + curl[1]*curl[1] + curl[2]*curl[2]); - - - // --- Calculate the Cauchy stress --- - for (size_t i = 0; i < 3; i++){ - for (size_t j = 0; j < 3; j++){ - tau(i, j) = stress(i,j); - // artificial viscosity can be added here to tau - } // end for - } //end for - - //initialize gradient array - for (size_t i = 0; i < 3; i++){ - for (size_t j = 0; j < 3; j++){ - for(int igradient = 0; igradient < num_nodes_in_elem; igradient++){ - tau_gradient(i, j, igradient, 0) = 0; - tau_gradient(i, j, igradient, 1) = 0; - if(num_dims==3) - tau_gradient(i, j, igradient, 2) = 0; - } - } // end for - } //end for - - // add the pressure - for (int i = 0; i < num_dims; i++){ - tau(i, i) -= elem_pres(elem_gid); - //current expression only works for ideal gas! generalize. - for(int igradient = 0; igradient < num_nodes_in_elem; igradient++){ - tau_gradient(i, i,igradient,0) = elem_pres(elem_gid)*volume_gradients(igradient,0)/vol; - tau_gradient(i, i,igradient,1) = elem_pres(elem_gid)*volume_gradients(igradient,1)/vol; - tau_gradient(i, i,igradient,2) = elem_pres(elem_gid)*volume_gradients(igradient,2)/vol; - } - } // end for - - - - - // ---- Multidirectional Approximate Riemann solver (MARS) ---- - // find the average velocity of the elem, it is an - // estimate of the Riemann velocity - - // initialize to Riemann velocity to zero - for (size_t dim = 0; dim < num_dims; dim++){ - vel_star(dim) = 0.0; - } - - // loop over nodes and calculate an average velocity, which is - // an estimate of Riemann velocity - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - - // Get node gloabl index and create view of nodal velocity - int node_gid = nodes_in_elem(elem_gid, node_lid); - - - ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); - - vel_star(0) += 0.125*vel(0); - vel_star(1) += 0.125*vel(1); - vel_star(2) += 0.125*vel(2); - - } // end for loop over nodes - - // find shock direction and shock impedance associated with each node - - // initialize sum term in MARS to zero - for (int i = 0; i < 4; i++){ - sum(i) = 0.0; - } - for (int i = 0; i < 4; i++){ - for (size_t igradient = 0; igradient < num_nodes_in_elem; igradient++) { - for (size_t jdim = 0; jdim < num_dims; jdim++) { - sum_gradient(i, igradient, jdim) = 0.0; - } - } - } - - double mag; // magnitude of the area normal - double mag_vel; // magnitude of velocity - - // loop over the nodes of the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { - - // Get global node id - size_t node_gid = nodes_in_elem(elem_gid, node_lid); - - // Create view of nodal velocity - ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); - - // Get an estimate of the shock direction. - mag_vel = sqrt( (vel(0) - vel_star(0) )*(vel(0) - vel_star(0) ) - + (vel(1) - vel_star(1) )*(vel(1) - vel_star(1) ) - + (vel(2) - vel_star(2) )*(vel(2) - vel_star(2) ) ); - - - if (mag_vel > small) { - - // estimate of the shock direction, a unit normal - for (int dim = 0; dim < num_dims; dim++){ - shock_dir(dim) = (vel(dim) - vel_star(dim)) / mag_vel; - } - } - - else { - - // if there is no velocity change, then use the surface area - // normal as the shock direction - mag = sqrt( area_normal(node_lid, 0)*area_normal(node_lid, 0) - + area_normal(node_lid, 1)*area_normal(node_lid, 1) - + area_normal(node_lid, 2)*area_normal(node_lid, 2) ); - - // estimate of the shock direction - for (int dim = 0; dim < num_dims; dim++){ - shock_dir(dim) = area_normal(node_lid, dim)/mag; - } - - } // end if mag_vel - - - // cell divergence indicates compression or expansions - size_t mat_id = elem_mat_id(elem_gid); - if (div < 0){ // element in compression - muc(node_lid) = elem_den(elem_gid) * - (material(mat_id).q1*elem_sspd(elem_gid) + material(mat_id).q2*mag_vel); - - for(int igradient = 0; igradient < num_nodes_in_elem; igradient++){ - muc_gradient(node_lid, igradient, 0) = -elem_den(elem_gid)*volume_gradients(igradient,0)/vol* - (material(mat_id).q1*elem_sspd(elem_gid)); - muc_gradient(node_lid, igradient, 1) = -elem_den(elem_gid)*volume_gradients(igradient,1)/vol* - (material(mat_id).q1*elem_sspd(elem_gid)); - muc_gradient(node_lid, igradient, 2) = -elem_den(elem_gid)*volume_gradients(igradient,2)/vol* - (material(mat_id).q1*elem_sspd(elem_gid)); - } - } - else { // element in expansion - muc(node_lid) = elem_den(elem_gid) * - (material(mat_id).q1ex*elem_sspd(elem_gid) + material(mat_id).q2ex*mag_vel); - for(int igradient = 0; igradient < num_nodes_in_elem; igradient++){ - muc_gradient(node_lid, igradient, 0) = -elem_den(elem_gid)*volume_gradients(igradient,0)/vol* - (material(mat_id).q1ex*elem_sspd(elem_gid)); - muc_gradient(node_lid, igradient, 1) = -elem_den(elem_gid)*volume_gradients(igradient,1)/vol* - (material(mat_id).q1ex*elem_sspd(elem_gid)); - muc_gradient(node_lid, igradient, 2) = -elem_den(elem_gid)*volume_gradients(igradient,2)/vol* - (material(mat_id).q1ex*elem_sspd(elem_gid)); - } - } // end if on divergence sign - - - size_t use_shock_dir = 0; - double mu_term; - double mu_term_gradient; - - // Coding to use shock direction - if (use_shock_dir == 1){ - // this is denominator of the Riamann solver and the multiplier - // on velocity in the numerator. It filters on the shock - // direction - mu_term = muc(node_lid)* - fabs( shock_dir(0)*area_normal(node_lid,0) - + shock_dir(1)*area_normal(node_lid,1) - + shock_dir(2)*area_normal(node_lid,2) ); - // muc_gradient(node_lid) = muc_gradient(node_lid)* - // fabs( shock_dir(0)*area_normal(node_lid,0) - // + shock_dir(1)*area_normal(node_lid,1) - // + shock_dir(2)*area_normal(node_lid,2) ); - // mu_term_gradient = muc(node_lid)* //amend if shock dir has dependence for gradient not captured here - // fabs( shock_dir(0)*area_normal_gradients(node_lid,0) - // + shock_dir(1)*area_normal_gradients(node_lid,1) - // + shock_dir(2)*area_normal_gradients(node_lid,2) ); - } - else { - // Using a full tensoral Riemann jump relation - mu_term = muc(node_lid) - * sqrt( area_normal(node_lid, 0)*area_normal(node_lid, 0) - + area_normal(node_lid, 1)*area_normal(node_lid, 1) - + area_normal(node_lid, 2)*area_normal(node_lid, 2) ); - for(int igradient = 0; igradient < num_nodes_in_elem; igradient++){ - for( int jdim = 0; jdim < num_dims; jdim++){ - mu_term_gradient = muc_gradient(node_lid,igradient,jdim) - * sqrt( area_normal(node_lid, 0)*area_normal(node_lid, 0) - + area_normal(node_lid, 1)*area_normal(node_lid, 1) - + area_normal(node_lid, 2)*area_normal(node_lid, 2) ); - - mu_term_gradient += muc(node_lid)*(area_normal(node_lid, 0)*area_normal_gradients(node_lid, 0,igradient,jdim) - + area_normal(node_lid, 1)*area_normal_gradients(node_lid, 1,igradient,jdim) - + area_normal(node_lid, 2)*area_normal_gradients(node_lid, 2,igradient,jdim))/ - sqrt( area_normal(node_lid, 0)*area_normal(node_lid, 0) - + area_normal(node_lid, 1)*area_normal(node_lid, 1) - + area_normal(node_lid, 2)*area_normal(node_lid, 2) ); - - muc_gradient(node_lid, igradient,jdim) = mu_term_gradient; - } - } - } - - sum(0) += mu_term*vel(0); - sum(1) += mu_term*vel(1); - sum(2) += mu_term*vel(2); - sum(3) += mu_term; - - //sum gradients - - for(int igradient = 0; igradient < num_nodes_in_elem; igradient++){ - for( int jdim = 0; jdim < num_dims; jdim++){ - sum_gradient(0,igradient,jdim) += muc_gradient(node_lid, igradient,jdim)*vel(0); - sum_gradient(1,igradient,jdim) += muc_gradient(node_lid, igradient,jdim)*vel(1); - sum_gradient(2,igradient,jdim) += muc_gradient(node_lid, igradient,jdim)*vel(2); - sum_gradient(3,igradient,jdim) += muc_gradient(node_lid, igradient,jdim); - } - } - - muc(node_lid) = mu_term; // the impeadance time surface area is stored here - - } // end for node_lid loop over nodes of the elem - - - // The Riemann velocity, called vel_star - if (sum(3) > fuzz) { - for (size_t i = 0; i < num_dims; i++) { - vel_star(i) = sum(i)/sum(3); - } - } - else { - for (int i = 0; i < num_dims; i++){ - vel_star(i) = 0.0; - } - } // end if - - //vel star gradients - if (sum(3) > fuzz) { - for (size_t i = 0; i < num_dims; i++) { - for(int igradient = 0; igradient < num_nodes_in_elem; igradient++){ - for( int jdim = 0; jdim < num_dims; jdim++){ - vel_star_gradient(i,igradient,jdim) = sum_gradient(i,igradient,jdim)/sum(3)-sum(i)/(sum(3)*sum(3))*sum_gradient(3,igradient,jdim); - } - } - } - } - else { - for (int i = 0; i < num_dims; i++){ - for(int igradient = 0; igradient < num_nodes_in_elem; igradient++){ - for( int jdim = 0; jdim < num_dims; jdim++){ - vel_star_gradient(i,igradient,jdim) = 0; - } - } - } - } // end if - - - // ---- Calculate the shock detector for the Riemann-solver ---- - // - // The dissipation from the Riemann problem is limited by phi - // phi = (1. - max( 0., min( 1. , r_face ) ))^n - // where - // r_face = (C* div(u_+)/div(u_z)) - // The plus denotes the cell center divergence of a neighbor. - // The solution will be first order when phi=1 and have - // zero dissipation when phi=0. - // phi = 0 highest-order solution - // phi = 1 first order solution - // - - double phi = 0.0; // the shock detector - double r_face = 1.0; // the ratio on the face - double r_min = 1.0; // the min ratio for the cell - double r_coef = 0.9; // 0.9; the coefficient on the ratio - // (1=minmod and 2=superbee) - double n_coef = 1.0; // the power on the limiting coefficient - // (1=nominal, and n_coeff > 1 oscillatory) - - - // loop over the nieghboring cells - for (size_t elem_lid = 0; elem_lid < num_elems_in_elem(elem_gid); elem_lid++){ - - // Get global index for neighboring cell - size_t neighbor_gid = elems_in_elem(elem_gid, elem_lid); - - // calculate the velocity divergence in neighbor - double div_neighbor = elem_div(neighbor_gid); - - r_face = r_coef*(div_neighbor + small)/(div + small); - - // store the smallest face ratio - r_min = fmin(r_face, r_min); - - } // end for elem_lid - - - // calculate standard shock detector - phi = 1.0 - fmax(0.0, r_min); - phi = pow(phi, n_coef); - - // Mach number shock detector - double omega = 20.0;//20.0; // weighting factor on Mach number - double third = 1.0/3.0; - double c_length = pow(vol, third); // characteristic length - double alpha = fmin(1.0, omega * (c_length * fabs(div))/(elem_sspd(elem_gid) + fuzz) ); - - // use Mach based detector with standard shock detector - - // turn off dissipation in expansion - //alpha = fmax(-fabs(div0)/div0 * alpha, 0.0); // this should be if(div0<0) alpha=alpha else alpha=0 - - phi = alpha*phi; - - // curl limiter on Q - double phi_curl = fmin(1.0, 1.0*fabs(div)/(mag_curl + fuzz)); // disable Q when vorticity is high - //phi = phi_curl*phi; - phi = 1; - - // ---- Calculate the Riemann force on each node ---- - - // loop over the each node in the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { - - size_t corner_lid = node_lid; - size_t column_index; - - // Get corner gid - size_t corner_gid = corners_in_elem(elem_gid, corner_lid); - - // Get node gid - size_t node_gid = nodes_in_elem(elem_gid, node_lid); - - // loop over dimension - for (int dim = 0; dim < num_dims; dim++){ - //assign gradient of corner contribution of force to relevant matrix entries with non-zero node velocity gradient - for(int igradient = 0; igradient < num_nodes_in_elem; igradient++){ - for(int jdim = 0; jdim < num_dims; jdim++){ - size_t gradient_node_gid = nodes_in_elem(elem_gid, igradient); - //if(!map->isNodeLocalElement(gradient_node_gid)) continue; - column_index = num_dims*Global_Gradient_Matrix_Assembly_Map(elem_gid, igradient, node_lid); - if(map->isNodeLocalElement(gradient_node_gid)){ - Force_Gradient_Positions(gradient_node_gid*num_dims+jdim, column_index+dim) += area_normal(node_lid, 0)*tau_gradient(0, dim, igradient, jdim) - + area_normal(node_lid, 1)*tau_gradient(1, dim, igradient, jdim) - + area_normal(node_lid, 2)*tau_gradient(2, dim, igradient, jdim) - + area_normal_gradients(node_lid, 0, igradient, jdim)*tau(0, dim) - + area_normal_gradients(node_lid, 1, igradient, jdim)*tau(1, dim) - + area_normal_gradients(node_lid, 2, igradient, jdim)*tau(2, dim) - + phi*muc_gradient(node_lid, igradient, jdim)*(vel_star(dim) - node_vel(rk_level, node_gid, dim)) - + phi*muc(node_lid)*(vel_star_gradient(dim, igradient, jdim)); - } - corner_gradient_storage(corner_gid,dim,igradient,jdim) = area_normal(node_lid, 0)*tau_gradient(0, dim, igradient, jdim) - + area_normal(node_lid, 1)*tau_gradient(1, dim, igradient, jdim) - + area_normal(node_lid, 2)*tau_gradient(2, dim, igradient, jdim) - + area_normal_gradients(node_lid, 0, igradient, jdim)*tau(0, dim) - + area_normal_gradients(node_lid, 1, igradient, jdim)*tau(1, dim) - + area_normal_gradients(node_lid, 2, igradient, jdim)*tau(2, dim) - + phi*muc_gradient(node_lid, igradient, jdim)*(vel_star(dim) - node_vel(rk_level, node_gid, dim)) - + phi*muc(node_lid)*(vel_star_gradient(dim, igradient, jdim)); - // if(map->isNodeLocalElement(gradient_node_gid)){ - // Force_Gradient_Positions(gradient_node_gid*num_dims+jdim, column_index+dim) += - // + area_normal_gradients(node_lid, 0, igradient, jdim)*tau(0, dim) - // + area_normal_gradients(node_lid, 1, igradient, jdim)*tau(1, dim) - // + area_normal_gradients(node_lid, 2, igradient, jdim)*tau(2, dim); - // } - // corner_gradient_storage(corner_gid,dim,igradient,jdim) = - // + area_normal_gradients(node_lid, 0, igradient, jdim)*tau(0, dim) - // + area_normal_gradients(node_lid, 1, igradient, jdim)*tau(1, dim) - // + area_normal_gradients(node_lid, 2, igradient, jdim)*tau(2, dim); - } - } - } // end loop over dimension - - } // end for loop over nodes in elem - - - - // --- Update Stress --- - // calculate the new stress at the next rk level, if it is a hypo model - - size_t mat_id = elem_mat_id(elem_gid); - - - } - //}); // end parallel for loop over elements - - /* - //accumulate node values from corner storage - force_gradient_design->putScalar(0); - - vec_array force_gradient_design_view = force_gradient_design->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { - size_t corner_id; - for(int icorner=0; icorner < num_corners_in_node(node_id); icorner++){ - corner_id = corners_in_node(node_id,icorner); - force_gradient_design_view(node_id,0) += corner_vector_storage(corner_id, 0); - force_gradient_design_view(node_id,1) += corner_vector_storage(corner_id, 1); - force_gradient_design_view(node_id,2) += corner_vector_storage(corner_id, 2); - } - }); // end parallel for - Kokkos::fence(); - */ - - return; - -} // end of routine - -// -------------------------------------------------------------------------------------------------------- -// Computes term objective derivative term involving gradient of power with respect to the design variable -//--------------------------------------------------------------------------------------------------------- - -void FEA_Module_SGH::force_design_gradient_term(const_vec_array design_variables, vec_array design_gradients){ - - size_t num_bdy_nodes = mesh->num_bdy_nodes; - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos material = simparam->material; - const int num_dim = simparam->num_dims; - size_t num_corners = rnum_elem*num_nodes_in_elem; - real_t global_dt; - bool element_constant_density = true; - size_t current_data_index, next_data_index; - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - CArrayKokkos current_element_adjoint = CArrayKokkos(num_nodes_in_elem,num_dim); - - //gradient contribution from gradient of Force vector with respect to design variable. - if(simparam->dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ - if(myrank==0){ - std::cout << "gradient term involving adjoint derivative" << std::endl; - } - } - - for (unsigned long cycle = 0; cycle < last_time_step+1; cycle++) { - //compute timestep from time data - global_dt = time_data[cycle+1] - time_data[cycle]; - //print - if(simparam->dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ - if (cycle==0){ - if(myrank==0) - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } - // print time step every 10 cycles - else if (cycle%20==0){ - if(myrank==0) - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } // end if - } - - - //view scope - { - const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_element_internal_energy = (*forward_solve_internal_energy_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_coord_vector = (*forward_solve_coordinate_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array next_velocity_vector = (*forward_solve_velocity_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array next_element_internal_energy = (*forward_solve_internal_energy_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array next_coord_vector = (*forward_solve_coordinate_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array next_adjoint_vector = (*adjoint_vector_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - - //first half of integration step calculation - FOR_ALL_CLASS(node_gid, 0, nall_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - node_vel(rk_level,node_gid,idim) = current_velocity_vector(node_gid,idim); - node_coords(rk_level,node_gid,idim) = current_coord_vector(node_gid,idim); - } - - }); // end parallel for - Kokkos::fence(); - - FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { - elem_sie(rk_level,elem_gid) = current_element_internal_energy(elem_gid,0); - }); // end parallel for - Kokkos::fence(); - - get_vol(); - - // ---- Calculate velocity diveregence for the element ---- - if(num_dim==2){ - get_divergence2D(elem_div, - node_coords, - node_vel, - elem_vol); - } - else { - get_divergence(elem_div, - node_coords, - node_vel, - elem_vol); - } // end if 2D - - // ---- Calculate elem state (den, pres, sound speed, stress) for next time step ---- - if(num_dim==2){ - update_state2D(material, - *mesh, - node_coords, - node_vel, - elem_den, - elem_pres, - elem_stress, - elem_sspd, - elem_sie, - elem_vol, - elem_mass, - elem_mat_id, - 1.0, - cycle); - } - else{ - update_state(material, - *mesh, - node_coords, - node_vel, - elem_den, - elem_pres, - elem_stress, - elem_sspd, - elem_sie, - elem_vol, - elem_mass, - elem_mat_id, - 1.0, - cycle); - } - - get_force_dgradient_sgh(material, - *mesh, - node_coords, - node_vel, - elem_den, - elem_sie, - elem_pres, - elem_stress, - elem_sspd, - elem_vol, - elem_div, - elem_mat_id, - 1.0, - cycle); - - //derivatives of forces at corners stored in corner_vector_storage buffer by previous routine - FOR_ALL_CLASS(elem_id, 0, rnum_elem, { - size_t node_id; - size_t corner_id; - real_t inner_product; - - inner_product = 0; - for(int ifill=0; ifill < num_nodes_in_elem; ifill++){ - node_id = nodes_in_elem(elem_id, ifill); - corner_id = elem_id*num_nodes_in_elem + ifill; - for(int idim=0; idim < num_dim; idim++){ - inner_product += corner_vector_storage(corner_id,idim)*current_adjoint_vector(node_id,idim); - } - } - - for (int inode = 0; inode < num_nodes_in_elem; inode++){ - //compute gradient of local element contribution to v^t*M*v product - corner_id = elem_id*num_nodes_in_elem + inode; - corner_value_storage(corner_id) = inner_product; - } - - }); // end parallel for - Kokkos::fence(); - - //accumulate node values from corner storage - //multiply - FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { - size_t corner_id; - for(int icorner=0; icorner < num_corners_in_node(node_id); icorner++){ - corner_id = corners_in_node(node_id,icorner); - design_gradients(node_id,0) += -corner_value_storage(corner_id)*global_dt; - } - }); // end parallel for - Kokkos::fence(); - - } //end view scope - } - -} - -// ----------------------------------------------------------------------------- -// This function calculates the corner forces and the evolves stress (hypo) -//------------------------------------------------------------------------------ -void FEA_Module_SGH::get_force_dgradient_sgh(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &elem_mat_id, - const double rk_alpha, - const size_t cycle - ) { - - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - const size_t num_dims = simparam->num_dims; - // --- calculate the forces acting on the nodes from the element --- - for (size_t elem_gid = 0; elem_gid < rnum_elem; elem_gid++){ - //FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { - - const size_t num_nodes_in_elem = 8; - real_t gradient_result[num_dims]; - // total Cauchy stress - double tau_array[9]; - double tau_gradient_array[9]; - - // corner area normals - double area_normal_array[24]; - - // estimate of shock direction - double shock_dir_array[3]; - - // the sums in the Riemann solver - double sum_array[4]; - double sum_gradient_array[4]; - - // corner shock impeadance x |corner area normal dot shock_dir| - double muc_array[8]; - double muc_gradient_array[8]; - - // Riemann velocity - double vel_star_array[3]; - double vel_star_gradient_array[3]; - - // velocity gradient - double vel_grad_array[9]; - - // --- Create views of arrays to aid the force calculation --- - - ViewCArrayKokkos tau(tau_array, num_dims, num_dims); - ViewCArrayKokkos tau_gradient(tau_gradient_array, num_dims, num_dims); - ViewCArrayKokkos area_normal(area_normal_array, num_nodes_in_elem, num_dims); - ViewCArrayKokkos shock_dir(shock_dir_array, num_dims); - ViewCArrayKokkos sum(sum_array, 4); - ViewCArrayKokkos sum_gradient(sum_gradient_array, 4); - ViewCArrayKokkos muc(muc_array, num_nodes_in_elem); - ViewCArrayKokkos muc_gradient(muc_gradient_array, num_nodes_in_elem); - ViewCArrayKokkos vel_star(vel_star_array, num_dims); - ViewCArrayKokkos vel_star_gradient(vel_star_gradient_array, num_dims); - ViewCArrayKokkos vel_grad(vel_grad_array, num_dims, num_dims); - - EOSParent* eos_model = elem_eos(elem_gid).model; - - // --- abviatations of variables --- - - // element volume - double vol = elem_vol(elem_gid); - - // create a view of the stress_matrix - ViewCArrayKokkos stress(&elem_stress(rk_level, elem_gid, 0,0), 3, 3); - - - // cut out the node_gids for this element - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 8); - - - - // get the B matrix which are the OUTWARD corner area normals - get_bmatrix(area_normal, - elem_gid, - node_coords, - elem_node_gids, - rk_level); - - - // --- Calculate the velocity gradient --- - get_velgrad(vel_grad, - elem_node_gids, - node_vel, - area_normal, - vol, - elem_gid, - rk_level); - - - // the -1 is for the inward surface area normal, - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - for (size_t dim = 0; dim < num_dims; dim++){ - area_normal(node_lid, dim) = (-1.0)*area_normal(node_lid,dim); - } // end for - } // end for - - - - double div = elem_div(elem_gid); - - - // vel = [u,v,w] - // [du/dx, du/dy, du/dz] - // vel_grad = [dv/dx, dv/dy, dv/dz] - // [dw/dx, dw/dy, dw/dz] - double curl[3]; - curl[0] = vel_grad(2,1) - vel_grad(1,2); // dw/dy - dv/dz - curl[1] = vel_grad(0,2) - vel_grad(2,0); // du/dz - dw/dx - curl[2] = vel_grad(1,0) - vel_grad(0,1); // dv/dx - du/dy - - double mag_curl = sqrt(curl[0]*curl[0] + curl[1]*curl[1] + curl[2]*curl[2]); - - - // --- Calculate the Cauchy stress --- - for (size_t i = 0; i < 3; i++){ - for (size_t j = 0; j < 3; j++){ - tau(i, j) = stress(i,j); - tau_gradient(i,j) = 0; - } // end for - } //end for - - // add the pressure - for (int i = 0; i < num_dims; i++){ - tau(i, i) -= elem_pres(elem_gid); - tau_gradient(i, i) -= elem_pres(elem_gid)/num_nodes_in_elem/relative_element_densities(elem_gid); - } // end for - - - - - // ---- Multidirectional Approximate Riemann solver (MARS) ---- - // find the average velocity of the elem, it is an - // estimate of the Riemann velocity - - // initialize to Riemann velocity to zero - for (size_t dim = 0; dim < num_dims; dim++){ - vel_star(dim) = 0.0; - } - - // loop over nodes and calculate an average velocity, which is - // an estimate of Riemann velocity - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - - // Get node gloabl index and create view of nodal velocity - int node_gid = nodes_in_elem(elem_gid, node_lid); - - - ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); - - vel_star(0) += 0.125*vel(0); - vel_star(1) += 0.125*vel(1); - vel_star(2) += 0.125*vel(2); - - } // end for loop over nodes - - // find shock direction and shock impedance associated with each node - - // initialize sum term in MARS to zero - for (int i = 0; i < 4; i++){ - sum(i) = sum_gradient(i) = 0.0; - } - - double mag; // magnitude of the area normal - double mag_vel; // magnitude of velocity - - // loop over the nodes of the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { - - // Get global node id - size_t node_gid = nodes_in_elem(elem_gid, node_lid); - - // Create view of nodal velocity - ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); - - // Get an estimate of the shock direction. - mag_vel = sqrt( (vel(0) - vel_star(0) )*(vel(0) - vel_star(0) ) - + (vel(1) - vel_star(1) )*(vel(1) - vel_star(1) ) - + (vel(2) - vel_star(2) )*(vel(2) - vel_star(2) ) ); - - - if (mag_vel > small) { - - // estimate of the shock direction, a unit normal - for (int dim = 0; dim < num_dims; dim++){ - shock_dir(dim) = (vel(dim) - vel_star(dim)) / mag_vel; - } - } - - else { - - // if there is no velocity change, then use the surface area - // normal as the shock direction - mag = sqrt( area_normal(node_lid, 0)*area_normal(node_lid, 0) - + area_normal(node_lid, 1)*area_normal(node_lid, 1) - + area_normal(node_lid, 2)*area_normal(node_lid, 2) ); - - // estimate of the shock direction - for (int dim = 0; dim < num_dims; dim++){ - shock_dir(dim) = area_normal(node_lid, dim)/mag; - } - - } // end if mag_vel - - - // cell divergence indicates compression or expansions - size_t mat_id = elem_mat_id(elem_gid); - if (div < 0){ // element in compression - muc(node_lid) = elem_den(elem_gid) * - (material(mat_id).q1*elem_sspd(elem_gid) + material(mat_id).q2*mag_vel); - - muc_gradient(node_lid) = elem_den(elem_gid)/relative_element_densities(elem_gid)/num_nodes_in_elem * - (material(mat_id).q1*elem_sspd(elem_gid) + material(mat_id).q2*mag_vel); - } - else { // element in expansion - muc(node_lid) = elem_den(elem_gid) * - (material(mat_id).q1ex*elem_sspd(elem_gid) + material(mat_id).q2ex*mag_vel); - muc_gradient(node_lid) = elem_den(elem_gid)/relative_element_densities(elem_gid)/num_nodes_in_elem * - (material(mat_id).q1ex*elem_sspd(elem_gid) + material(mat_id).q2ex*mag_vel); - } // end if on divergence sign - - - size_t use_shock_dir = 0; - double mu_term; - double mu_term_gradient; - - // Coding to use shock direction - if (use_shock_dir == 1){ - // this is denominator of the Riamann solver and the multiplier - // on velocity in the numerator. It filters on the shock - // direction - mu_term = muc(node_lid)* - fabs( shock_dir(0)*area_normal(node_lid,0) - + shock_dir(1)*area_normal(node_lid,1) - + shock_dir(2)*area_normal(node_lid,2) ); - mu_term_gradient = muc_gradient(node_lid)* //amend if shock dir has dependence for gradient not captured here - fabs( shock_dir(0)*area_normal(node_lid,0) - + shock_dir(1)*area_normal(node_lid,1) - + shock_dir(2)*area_normal(node_lid,2) ); - } - else { - // Using a full tensoral Riemann jump relation - mu_term = muc(node_lid) - * sqrt( area_normal(node_lid, 0)*area_normal(node_lid, 0) - + area_normal(node_lid, 1)*area_normal(node_lid, 1) - + area_normal(node_lid, 2)*area_normal(node_lid, 2) ); - mu_term_gradient = muc_gradient(node_lid) - * sqrt( area_normal(node_lid, 0)*area_normal(node_lid, 0) - + area_normal(node_lid, 1)*area_normal(node_lid, 1) - + area_normal(node_lid, 2)*area_normal(node_lid, 2) ); - } - - sum(0) += mu_term*vel(0); - sum(1) += mu_term*vel(1); - sum(2) += mu_term*vel(2); - sum(3) += mu_term; - - sum_gradient(0) += mu_term_gradient*vel(0); - sum_gradient(1) += mu_term_gradient*vel(1); - sum_gradient(2) += mu_term_gradient*vel(2); - sum_gradient(3) += mu_term_gradient; - - muc(node_lid) = mu_term; // the impeadance time surface area is stored here - muc_gradient(node_lid) = mu_term_gradient; - - } // end for node_lid loop over nodes of the elem - - - - - // The Riemann velocity, called vel_star - if (sum(3) > fuzz) { - for (size_t i = 0; i < num_dims; i++) { - vel_star(i) = sum(i)/sum(3); - vel_star_gradient(i) = sum_gradient(i)/sum(3) - sum_gradient(3)*sum(i)/sum(3)/sum(3); - } - } - else { - for (int i = 0; i < num_dims; i++){ - vel_star(i) = 0.0; - vel_star_gradient(i) = 0.0; - } - } // end if - - - - // ---- Calculate the shock detector for the Riemann-solver ---- - // - // The dissipation from the Riemann problem is limited by phi - // phi = (1. - max( 0., min( 1. , r_face ) ))^n - // where - // r_face = (C* div(u_+)/div(u_z)) - // The plus denotes the cell center divergence of a neighbor. - // The solution will be first order when phi=1 and have - // zero dissipation when phi=0. - // phi = 0 highest-order solution - // phi = 1 first order solution - // - - double phi = 0.0; // the shock detector - double r_face = 1.0; // the ratio on the face - double r_min = 1.0; // the min ratio for the cell - double r_coef = 0.9; // 0.9; the coefficient on the ratio - // (1=minmod and 2=superbee) - double n_coef = 1.0; // the power on the limiting coefficient - // (1=nominal, and n_coeff > 1 oscillatory) - - - // loop over the nieghboring cells - for (size_t elem_lid = 0; elem_lid < num_elems_in_elem(elem_gid); elem_lid++){ - - // Get global index for neighboring cell - size_t neighbor_gid = elems_in_elem(elem_gid, elem_lid); - - // calculate the velocity divergence in neighbor - double div_neighbor = elem_div(neighbor_gid); - - r_face = r_coef*(div_neighbor + small)/(div + small); - - // store the smallest face ratio - r_min = fmin(r_face, r_min); - - } // end for elem_lid - - - // calculate standard shock detector - phi = 1.0 - fmax(0.0, r_min); - phi = pow(phi, n_coef); - - // Mach number shock detector - double omega = 20.0;//20.0; // weighting factor on Mach number - double third = 1.0/3.0; - double c_length = pow(vol, third); // characteristic length - double alpha = fmin(1.0, omega * (c_length * fabs(div))/(elem_sspd(elem_gid) + fuzz) ); - - // use Mach based detector with standard shock detector - - // turn off dissipation in expansion - //alpha = fmax(-fabs(div0)/div0 * alpha, 0.0); // this should be if(div0<0) alpha=alpha else alpha=0 - - phi = alpha*phi; - - // curl limiter on Q - double phi_curl = fmin(1.0, 1.0*fabs(div)/(mag_curl + fuzz)); // disable Q when vorticity is high - //phi = phi_curl*phi; - phi = 1; - - // ---- Calculate the Riemann force on each node ---- - - // loop over the each node in the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { - - size_t corner_lid = node_lid; - - // Get corner gid - size_t corner_gid = corners_in_elem(elem_gid, corner_lid); - - // Get node gid - size_t node_gid = nodes_in_elem(elem_gid, node_lid); - - // loop over dimension - for (int dim = 0; dim < num_dims; dim++){ - - corner_vector_storage(corner_gid, dim) = - area_normal(node_lid, 0)*tau_gradient(0, dim) - + area_normal(node_lid, 1)*tau_gradient(1, dim) - + area_normal(node_lid, 2)*tau_gradient(2, dim) - + phi*muc_gradient(node_lid)*(vel_star(dim) - node_vel(rk_level, node_gid, dim)) - + phi*muc(node_lid)*(vel_star_gradient(dim)); - - } // end loop over dimension - - } // end for loop over nodes in elem - - - - // --- Update Stress --- - // calculate the new stress at the next rk level, if it is a hypo model - - size_t mat_id = elem_mat_id(elem_gid); - - - } - //}); // end parallel for loop over elements - - /* - //accumulate node values from corner storage - force_gradient_design->putScalar(0); - - vec_array force_gradient_design_view = force_gradient_design->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { - size_t corner_id; - for(int icorner=0; icorner < num_corners_in_node(node_id); icorner++){ - corner_id = corners_in_node(node_id,icorner); - force_gradient_design_view(node_id,0) += corner_vector_storage(corner_id, 0); - force_gradient_design_view(node_id,1) += corner_vector_storage(corner_id, 1); - force_gradient_design_view(node_id,2) += corner_vector_storage(corner_id, 2); - } - }); // end parallel for - Kokkos::fence(); - */ - - return; - -} // end of routine diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/geometry.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/geometry.cpp deleted file mode 100644 index a741de817..000000000 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/geometry.cpp +++ /dev/null @@ -1,1631 +0,0 @@ -// ----------------------------------------------------------------------------- -// This code handles the geometric information for the mesh for the SHG solver -//------------------------------------------------------------------------------ -#include "matar.h" -#include "state.h" -#include "FEA_Module_SGH.h" - -void FEA_Module_SGH::update_position_sgh(double rk_alpha, - const size_t num_nodes, - DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel){ - - const size_t rk_level = rk_num_bins - 1; - int num_dims = num_dim; - - // loop over all the nodes in the mesh - FOR_ALL_CLASS(node_gid, 0, num_nodes, { - - for (int dim = 0; dim < num_dims; dim++){ - double half_vel = (node_vel(rk_level, node_gid, dim) + node_vel(0, node_gid, dim))*0.5; - node_coords(rk_level, node_gid, dim) = node_coords(0, node_gid, dim) + rk_alpha*dt*half_vel; - } - - }); // end parallel for over nodes - -} // end subroutine - - -// ----------------------------------------------------------------------------- -// This function claculates -// B_p = J^{-T} \cdot (\nabla_{xi} \phi_p w -// where -// \phi_p is the basis function for vertex p -// w is the 1 gauss point for the cell (everything is evaluted at this point) -// J^{-T} is the inverse transpose of the Jacobi matrix -// \nabla_{xi} is the gradient opperator in the reference coordinates -// -// B_p is the OUTWARD corner area normal at node p -//------------------------------------------------------------------------------ -KOKKOS_FUNCTION -void FEA_Module_SGH::get_bmatrix(const ViewCArrayKokkos &B_matrix, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const { - - const size_t num_nodes = 8; - - double x_array[8]; - double y_array[8]; - double z_array[8]; - - // x, y, z coordinates of elem vertices - auto x = ViewCArrayKokkos (x_array, num_nodes); - auto y = ViewCArrayKokkos (y_array, num_nodes); - auto z = ViewCArrayKokkos (z_array, num_nodes); - - // get the coordinates of the nodes(rk,elem,node) in this element - for (int node_lid = 0; node_lid < num_nodes; node_lid++){ - x(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 0); - y(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 1); - z(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 2); - } // end for - - double twelth = 1./12.; - - B_matrix(0,0) = ( +y(1)*( -z(2) -z(3) +z(4) +z(5) ) - +y(2)*( +z(1) -z(3) ) - +y(3)*( +z(1) +z(2) -z(4) -z(7) ) - +y(4)*( -z(1) +z(3) -z(5) +z(7) ) - +y(5)*( -z(1) +z(4) ) - +y(7)*( +z(3) -z(4) ) )*twelth; - - B_matrix(1,0) = ( +y(0)*( +z(2) +z(3) -z(4) -z(5) ) - +y(2)*( -z(0) -z(3) +z(5) +z(6) ) - +y(3)*( -z(0) +z(2) ) - +y(4)*( +z(0) -z(5) ) - +y(5)*( +z(0) -z(2) +z(4) -z(6) ) - +y(6)*( -z(2) +z(5) ) )*twelth; - - B_matrix(2,0) = ( +y(0)*( -z(1) +z(3) ) - +y(1)*( +z(0) +z(3) -z(5) -z(6) ) - +y(3)*( -z(0) -z(1) +z(6) +z(7) ) - +y(5)*( +z(1) -z(6) ) - +y(6)*( +z(1) -z(3) +z(5) -z(7) ) - +y(7)*( -z(3) +z(6) ) )*twelth; - - B_matrix(3,0) = ( +y(0)*( -z(1) -z(2) +z(4) +z(7) ) - +y(1)*( +z(0) -z(2) ) - +y(2)*( +z(0) +z(1) -z(6) -z(7) ) - +y(4)*( -z(0) +z(7) ) - +y(6)*( +z(2) -z(7) ) - +y(7)*( -z(0) +z(2) -z(4) +z(6) ) )*twelth; - - B_matrix(4,0) = ( +y(0)*( +z(1) -z(3) +z(5) -z(7) ) - +y(1)*( -z(0) +z(5) ) - +y(3)*( +z(0) -z(7) ) - +y(5)*( -z(0) -z(1) +z(6) +z(7) ) - +y(6)*( -z(5) +z(7) ) - +y(7)*( +z(0) +z(3) -z(5) -z(6) ) )*twelth; - - B_matrix(5,0) = ( +y(0)*( +z(1) -z(4) ) - +y(1)*( -z(0) +z(2) -z(4) +z(6) ) - +y(2)*( -z(1) +z(6) ) - +y(4)*( +z(0) +z(1) -z(6) -z(7) ) - +y(6)*( -z(1) -z(2) +z(4) +z(7) ) - +y(7)*( +z(4) -z(6) ) )*twelth; - - B_matrix(6,0) = ( +y(1)*( +z(2) -z(5) ) - +y(2)*( -z(1) +z(3) -z(5) +z(7) ) - +y(3)*( -z(2) +z(7) ) - +y(4)*( +z(5) -z(7) ) - +y(5)*( +z(1) +z(2) -z(4) -z(7) ) - +y(7)*( -z(2) -z(3) +z(4) +z(5) ) )*twelth; - - B_matrix(7,0) = ( +y(0)*( -z(3) +z(4) ) - +y(2)*( +z(3) -z(6) ) - +y(3)*( +z(0) -z(2) +z(4) -z(6) ) - +y(4)*( -z(0) -z(3) +z(5) +z(6) ) - +y(5)*( -z(4) +z(6) ) - +y(6)*( +z(2) +z(3) -z(4) -z(5) ) )*twelth; - - B_matrix(0,1) = ( +z(1)*( -x(2) -x(3) +x(4) +x(5) ) - +z(2)*( +x(1) -x(3) ) - +z(3)*( +x(1) +x(2) -x(4) -x(7) ) - +z(4)*( -x(1) +x(3) -x(5) +x(7) ) - +z(5)*( -x(1) +x(4) ) - +z(7)*( +x(3) -x(4) ) )*twelth; - - B_matrix(1,1) = ( +z(0)*( +x(2) +x(3) -x(4) -x(5) ) - +z(2)*( -x(0) -x(3) +x(5) +x(6) ) - +z(3)*( -x(0) +x(2) ) - +z(4)*( +x(0) -x(5) ) - +z(5)*( +x(0) -x(2) +x(4) -x(6) ) - +z(6)*( -x(2) +x(5) ) )*twelth; - - B_matrix(2,1) = ( +z(0)*( -x(1) +x(3) ) - +z(1)*( +x(0) +x(3) -x(5) -x(6) ) - +z(3)*( -x(0) -x(1) +x(6) +x(7) ) - +z(5)*( +x(1) -x(6) ) - +z(6)*( +x(1) -x(3) +x(5) -x(7) ) - +z(7)*( -x(3) +x(6) ) )*twelth; - - B_matrix(3,1) = ( +z(0)*( -x(1) -x(2) +x(4) +x(7) ) - +z(1)*( +x(0) -x(2) ) - +z(2)*( +x(0) +x(1) -x(6) -x(7) ) - +z(4)*( -x(0) +x(7) ) - +z(6)*( +x(2) -x(7) ) - +z(7)*( -x(0) +x(2) -x(4) +x(6) ) )*twelth; - - B_matrix(4,1) = ( +z(0)*( +x(1) -x(3) +x(5) -x(7) ) - +z(1)*( -x(0) +x(5) ) - +z(3)*( +x(0) -x(7) ) - +z(5)*( -x(0) -x(1) +x(6) +x(7) ) - +z(6)*( -x(5) +x(7) ) - +z(7)*( +x(0) +x(3) -x(5) -x(6) ) )*twelth; - - B_matrix(5,1) = ( +z(0)*( +x(1) -x(4) ) - +z(1)*( -x(0) +x(2) -x(4) +x(6) ) - +z(2)*( -x(1) +x(6) ) - +z(4)*( +x(0) +x(1) -x(6) -x(7) ) - +z(6)*( -x(1) -x(2) +x(4) +x(7) ) - +z(7)*( +x(4) -x(6) ) )*twelth; - - B_matrix(6,1) = ( +z(1)*( +x(2) -x(5) ) - +z(2)*( -x(1) +x(3) -x(5) +x(7) ) - +z(3)*( -x(2) +x(7) ) - +z(4)*( +x(5) -x(7) ) - +z(5)*( +x(1) +x(2) -x(4) -x(7) ) - +z(7)*( -x(2) -x(3) +x(4) +x(5) ) )*twelth; - - B_matrix(7,1) = ( +z(0)*( -x(3) +x(4) ) - +z(2)*( +x(3) -x(6) ) - +z(3)*( +x(0) -x(2) +x(4) -x(6) ) - +z(4)*( -x(0) -x(3) +x(5) +x(6) ) - +z(5)*( -x(4) +x(6) ) - +z(6)*( +x(2) +x(3) -x(4) -x(5) ) )*twelth; - - B_matrix(0,2) = ( +x(1)*( -y(2) -y(3) +y(4) +y(5) ) - +x(2)*( +y(1) -y(3) ) - +x(3)*( +y(1) +y(2) -y(4) -y(7) ) - +x(4)*( -y(1) +y(3) -y(5) +y(7) ) - +x(5)*( -y(1) +y(4) ) - +x(7)*( +y(3) -y(4) ) )*twelth; - - B_matrix(1,2) = ( +x(0)*( +y(2) +y(3) -y(4) -y(5) ) - +x(2)*( -y(0) -y(3) +y(5) +y(6) ) - +x(3)*( -y(0) +y(2) ) - +x(4)*( +y(0) -y(5) ) - +x(5)*( +y(0) -y(2) +y(4) -y(6) ) - +x(6)*( -y(2) +y(5) ) )*twelth; - - B_matrix(2,2) = ( +x(0)*( -y(1) +y(3) ) - +x(1)*( +y(0) +y(3) -y(5) -y(6) ) - +x(3)*( -y(0) -y(1) +y(6) +y(7) ) - +x(5)*( +y(1) -y(6) ) - +x(6)*( +y(1) -y(3) +y(5) -y(7) ) - +x(7)*( -y(3) +y(6) ) )*twelth; - - B_matrix(3,2) = ( +x(0)*( -y(1) -y(2) +y(4) +y(7) ) - +x(1)*( +y(0) -y(2) ) - +x(2)*( +y(0) +y(1) -y(6) -y(7) ) - +x(4)*( -y(0) +y(7) ) - +x(6)*( +y(2) -y(7) ) - +x(7)*( -y(0) +y(2) -y(4) +y(6) ) )*twelth; - - B_matrix(4,2) = ( +x(0)*( +y(1) -y(3) +y(5) -y(7) ) - +x(1)*( -y(0) +y(5) ) - +x(3)*( +y(0) -y(7) ) - +x(5)*( -y(0) -y(1) +y(6) +y(7) ) - +x(6)*( -y(5) +y(7) ) - +x(7)*( +y(0) +y(3) -y(5) -y(6) ) )*twelth; - - B_matrix(5,2) = ( +x(0)*( +y(1) -y(4) ) - +x(1)*( -y(0) +y(2) -y(4) +y(6) ) - +x(2)*( -y(1) +y(6) ) - +x(4)*( +y(0) +y(1) -y(6) -y(7) ) - +x(6)*( -y(1) -y(2) +y(4) +y(7) ) - +x(7)*( +y(4) -y(6) ) )*twelth; - - B_matrix(6,2) = ( +x(1)*( +y(2) -y(5) ) - +x(2)*( -y(1) +y(3) -y(5) +y(7) ) - +x(3)*( -y(2) +y(7) ) - +x(4)*( +y(5) -y(7) ) - +x(5)*( +y(1) +y(2) -y(4) -y(7) ) - +x(7)*( -y(2) -y(3) +y(4) +y(5) ) )*twelth; - - B_matrix(7,2) = ( +x(0)*( -y(3) +y(4) ) - +x(2)*( +y(3) -y(6) ) - +x(3)*( +y(0) -y(2) +y(4) -y(6) ) - +x(4)*( -y(0) -y(3) +y(5) +y(6) ) - +x(5)*( -y(4) +y(6) ) - +x(6)*( +y(2) +y(3) -y(4) -y(5) ) )*twelth; - -} // end subroutine - - -/* ---------------------------------------------------------------------------- - Compute Volume of each finite element -------------------------------------------------------------------------------- */ - -void FEA_Module_SGH::get_vol(){ - - const size_t rk_level = rk_num_bins - 1; - const size_t num_dims = num_dim; - - if (num_dims == 2){ - FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { - - // cut out the node_gids for this element - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 4); - get_vol_quad(elem_vol, elem_gid, node_coords, elem_node_gids, rk_level); - - }); - Kokkos::fence(); - } - else { - FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { - - // cut out the node_gids for this element - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 8); - get_vol_hex(elem_vol, elem_gid, node_coords, elem_node_gids, rk_level); - - }); - Kokkos::fence(); - } // end if - - return; - -} // end subroutine - - -// Exact volume for a hex element -KOKKOS_INLINE_FUNCTION -void FEA_Module_SGH::get_vol_hex(const DViewCArrayKokkos &elem_vol, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const { - - const size_t num_nodes = 8; - - double x_array[8]; - double y_array[8]; - double z_array[8]; - - // x, y, z coordinates of elem vertices - auto x = ViewCArrayKokkos (x_array, num_nodes); - auto y = ViewCArrayKokkos (y_array, num_nodes); - auto z = ViewCArrayKokkos (z_array, num_nodes); - - // get the coordinates of the nodes(rk,elem,node) in this element - for (int node_lid = 0; node_lid < num_nodes; node_lid++){ - x(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 0); - y(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 1); - z(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 2); - } // end for - - double twelth = 1./12.; - - // element volume - elem_vol(elem_gid) = - (x(1)*(y(3)*(-z(0) + z(2)) + y(4)*( z(0) - z(5)) + y(0)*(z(2) + z(3) - z(4) - z(5)) + y(6)*(-z(2) + z(5)) + y(5)*(z(0) - z(2) + z(4) - z(6)) + y(2)*(-z(0) - z(3) + z(5) + z(6))) + - x(7)*(y(0)*(-z(3) + z(4)) + y(6)*( z(2) + z(3) - z(4) - z(5)) + y(2)*(z(3) - z(6)) + y(3)*(z(0) - z(2) + z(4) - z(6)) + y(5)*(-z(4) + z(6)) + y(4)*(-z(0) - z(3) + z(5) + z(6))) + - x(3)*(y(1)*( z(0) - z(2)) + y(7)*(-z(0) + z(2) - z(4) + z(6)) + y(6)*(z(2) - z(7)) + y(2)*(z(0) + z(1) - z(6) - z(7)) + y(4)*(-z(0) + z(7)) + y(0)*(-z(1) - z(2) + z(4) + z(7))) + - x(5)*(y(0)*( z(1) - z(4)) + y(7)*( z(4) - z(6)) + y(2)*(-z(1) + z(6)) + y(1)*(-z(0) + z(2) - z(4) + z(6)) + y(4)*(z(0) + z(1) - z(6) - z(7)) + y(6)*(-z(1) - z(2) + z(4) + z(7))) + - x(6)*(y(1)*( z(2) - z(5)) + y(7)*(-z(2) - z(3) + z(4) + z(5)) + y(5)*(z(1) + z(2) - z(4) - z(7)) + y(4)*(z(5) - z(7)) + y(3)*(-z(2) + z(7)) + y(2)*(-z(1) + z(3) - z(5) + z(7))) + - x(0)*(y(2)*( z(1) - z(3)) + y(7)*( z(3) - z(4)) + y(5)*(-z(1) + z(4)) + y(1)*(-z(2) - z(3) + z(4) + z(5)) + y(3)*(z(1) + z(2) - z(4) - z(7)) + y(4)*(-z(1) + z(3) - z(5) + z(7))) + - x(2)*(y(0)*(-z(1) + z(3)) + y(5)*( z(1) - z(6)) + y(1)*(z(0) + z(3) - z(5) - z(6)) + y(7)*(-z(3) + z(6)) + y(6)*(z(1) - z(3) + z(5) - z(7)) + y(3)*(-z(0) - z(1) + z(6) + z(7))) + - x(4)*(y(1)*(-z(0) + z(5)) + y(7)*( z(0) + z(3) - z(5) - z(6)) + y(3)*(z(0) - z(7)) + y(0)*(z(1) - z(3) + z(5) - z(7)) + y(6)*(-z(5) + z(7)) + y(5)*(-z(0) - z(1) + z(6) + z(7))))*twelth; - - return; - -} // end subroutine - - - -KOKKOS_FUNCTION -void FEA_Module_SGH::get_bmatrix2D(const ViewCArrayKokkos &B_matrix, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const { - - const size_t num_nodes = 4; - - double x_array[4]; - double y_array[4]; - - // x, y coordinates of elem vertices - auto x = ViewCArrayKokkos (x_array, num_nodes); - auto y = ViewCArrayKokkos (y_array, num_nodes); - - // get the coordinates of the nodes(rk,elem,node) in this element - for (int node_lid = 0; node_lid < num_nodes; node_lid++){ - x(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 0); - y(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 1); - } // end for - - /* ensight node order 0 1 2 3 - Flanaghan node order 3 4 1 2 - */ - - - B_matrix(0,0) = -0.5*(y(3)-y(1)); - - B_matrix(1,0) = -0.5*(y(0)-y(2)); - - B_matrix(2,0) = -0.5*(y(1)-y(3)); - - B_matrix(3,0) = -0.5*(y(2)-y(0)); - - - B_matrix(0,1) = -0.5*(x(1)-x(3)); - - B_matrix(1,1) = -0.5*(x(2)-x(0)); - - B_matrix(2,1) = -0.5*(x(3)-x(1)); - - B_matrix(3,1) = -0.5*(x(0)-x(2)); - - - // - /* - The Flanagan and Belytschko paper has: - x y - node 1: 0.5*(y2 - y4) , 0.5*(x4 - x2) - node 2: 0.5*(y3 - y1) , 0.5*(x1 - x3) - node 3: 0.5*(y4 - y2) , 0.5*(x2 - x4) - node 4: 0.5*(y1 - y3) , 0.5*(x3 - x1) - - Ensight order would be - - node 2: 0.5*(y3 - y1) , 0.5*(x1 - x3) - node 3: 0.5*(y0 - y2) , 0.5*(x2 - x0) - node 0: 0.5*(y1 - y3) , 0.5*(x3 - x1) - node 1: 0.5*(y2 - y0) , 0.5*(x0 - x2) - - */ - - return; - -} // end subroutine - - -// true volume of a quad in RZ coords -KOKKOS_INLINE_FUNCTION -void FEA_Module_SGH::get_vol_quad(const DViewCArrayKokkos &elem_vol, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const { - - // --- testing here --- - /* - double test_vol = 0.0; - // getting the corner facial area - double corner_areas_array[4]; - ViewCArrayKokkos corner_areas(&corner_areas_array[0],4); - - get_area_weights2D(corner_areas, - elem_gid, - node_coords, - elem_node_gids, - rk_level); - - - for(size_t node_lid=0; node_lid<4; node_lid++){ - double y = node_coords(rk_level, elem_node_gids(node_lid), 1); // node radius - test_vol += corner_areas(node_lid)*y; - } // end for - - test_vol matches the Barlow volume formula - */ - // ------------------- - - - elem_vol(elem_gid) = 0.0; - - const size_t num_nodes = 4; - - double x_array[4]; - double y_array[4]; - - // x, y coordinates of elem vertices - auto x = ViewCArrayKokkos (x_array, num_nodes); - auto y = ViewCArrayKokkos (y_array, num_nodes); - - // get the coordinates of the nodes(rk,elem,node) in this element - for (int node_lid = 0; node_lid < num_nodes; node_lid++){ - x(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 0); - y(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 1); - } // end for - - /* ensight node order 0 1 2 3 - Flanaghan node order 3 4 1 2 - */ - elem_vol(elem_gid) = - ( (y(2)+y(3)+y(0))*((y(2)-y(3))*(x(0)-x(3))-(y(0)-y(3))*(x(2)-x(3)) ) - + (y(0)+y(1)+y(2))*((y(0)-y(1))*(x(2)-x(1))-(y(2)-y(1))*(x(0)-x(1))) )/6.0; - - return; - -} // end subroutine - - -// element facial area -KOKKOS_FUNCTION -double FEA_Module_SGH::get_area_quad(const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const { - - double elem_area=0.0; - - const size_t num_nodes = 4; - - double x_array[4]; - double y_array[4]; - - // x, y coordinates of elem vertices - auto x = ViewCArrayKokkos (x_array, num_nodes); - auto y = ViewCArrayKokkos (y_array, num_nodes); - - // get the coordinates of the nodes(rk,elem,node) in this element - for (int node_lid = 0; node_lid < num_nodes; node_lid++){ - x(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 0); - y(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 1); - } // end for - - /* ensight node order 0 1 2 3 - Flanaghan node order 3 4 1 2 - */ - - // element facial area - elem_area = 0.5*((x(0)-x(2))*(y(1)-y(3))+(x(3)-x(1))*(y(0)-y(2))); - - return elem_area; - -} // end subroutine - - -KOKKOS_INLINE_FUNCTION -double FEA_Module_SGH::heron(const double x1, - const double y1, - const double x2, - const double y2, - const double x3, - const double y3) const -{ - double S,a,b,c,area; - - S=0.0; - a=sqrt((x2-x1)*(x2-x1)+(y2-y1)*(y2-y1)); - S+=a; - b=sqrt((x3-x2)*(x3-x2)+(y3-y2)*(y3-y2)); - S+=b; - c=sqrt((x3-x1)*(x3-x1)+(y3-y1)*(y3-y1)); - S+=c; - - S*=0.5; - area=sqrt(S*(S-a)*(S-b)*(S-c)); - - return area; -} - - - -KOKKOS_FUNCTION -void FEA_Module_SGH::get_area_weights2D(const ViewCArrayKokkos &corner_areas, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const { - - const size_t num_nodes = 4; - - double x_array[4]; - double y_array[4]; - - double rc,zc; - double A12,A23,A34,A41; - - // x, y coordinates of elem vertices - ViewCArrayKokkos x(x_array, num_nodes); - ViewCArrayKokkos y(y_array, num_nodes); - - // get the coordinates of the nodes(rk,elem,node) in this element - rc=zc=0.0; - for (int node_lid = 0; node_lid < num_nodes; node_lid++){ - x(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 0); - y(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 1); - rc+=0.25*y(node_lid); - zc+=0.25*x(node_lid); - } // end for - - /* ensight node order 0 1 2 3 - Barlow node order 1 2 3 4 - */ - - A12 = heron(x(0),y(0),zc,rc,x(1),y(1)); - A23 = heron(x(1),y(1),zc,rc,x(2),y(2)); - A34 = heron(x(2),y(2),zc,rc,x(3),y(3)); - A41 = heron(x(3),y(3),zc,rc,x(0),y(0)); - - corner_areas(0)=(5.*A41+5.*A12+A23+A34)/12.; - corner_areas(1)=(A41+5.*A12+5.*A23+A34)/12.; - corner_areas(2)=(A41+A12+5.*A23+5.*A34)/12.; - corner_areas(3)=(5.*A41+A12+A23+5.*A34)/12.; - - - - return; - -} // end subroutine - -/* ----------------------------------------------------------------------------------- - Compute Gradient of the Volume of each finite element with respect to displacement --------------------------------------------------------------------------------------- */ - -void FEA_Module_SGH::get_vol_ugradient(const size_t gradient_node_id, const size_t gradient_dim){ - - const size_t rk_level = rk_num_bins - 1; - const size_t num_dims = num_dim; - - if (num_dims == 2){ - FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { - - // cut out the node_gids for this element - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 4); - get_vol_quad(elem_vol, elem_gid, node_coords, elem_node_gids, rk_level); - - }); - Kokkos::fence(); - } - else { - FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { - // cut out the node_gids for this element - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 8); - //get_vol_hex_ugradient(elem_vol, elem_gid, node_coords, elem_node_gids, rk_level); - - }); - Kokkos::fence(); - } // end if - - return; - -} // end subroutine - - -// Exact volume for a hex element -KOKKOS_FUNCTION -void FEA_Module_SGH::get_vol_hex_ugradient(const ViewCArrayKokkos &elem_vol_gradients, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const { - - const size_t num_nodes = 8; - const size_t num_dims = num_dim; - double x_array[8]; - double y_array[8]; - double z_array[8]; - double gradient_result; - - // x, y, z coordinates of elem vertices - auto x = ViewCArrayKokkos (x_array, num_nodes); - auto y = ViewCArrayKokkos (y_array, num_nodes); - auto z = ViewCArrayKokkos (z_array, num_nodes); - - // get the coordinates of the nodes(rk,elem,node) in this element - for (int node_lid = 0; node_lid < num_nodes; node_lid++){ - x(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 0); - y(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 1); - z(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 2); - } // end for - - double twelth = 1./12.; - - // element volume gradient - for(int inode = 0; inode < 8; inode++){ - for(int idim = 0; idim < num_dims; idim++){ - switch(num_dims*inode + idim){ - case 0: - gradient_result = - ((y(2)*( z(1) - z(3)) + y(7)*( z(3) - z(4)) + y(5)*(-z(1) + z(4)) + y(1)*(-z(2) - z(3) + z(4) + z(5)) + y(3)*(z(1) + z(2) - z(4) - z(7)) + y(4)*(-z(1) + z(3) - z(5) + z(7))))*twelth; - break; - case 3: - gradient_result = - ((y(3)*(-z(0) + z(2)) + y(4)*( z(0) - z(5)) + y(0)*(z(2) + z(3) - z(4) - z(5)) + y(6)*(-z(2) + z(5)) + y(5)*(z(0) - z(2) + z(4) - z(6)) + y(2)*(-z(0) - z(3) + z(5) + z(6))))*twelth; - break; - - case 6: - gradient_result = - ((y(0)*(-z(1) + z(3)) + y(5)*( z(1) - z(6)) + y(1)*(z(0) + z(3) - z(5) - z(6)) + y(7)*(-z(3) + z(6)) + y(6)*(z(1) - z(3) + z(5) - z(7)) + y(3)*(-z(0) - z(1) + z(6) + z(7))))*twelth; - break; - case 9: - gradient_result = - ((y(1)*( z(0) - z(2)) + y(7)*(-z(0) + z(2) - z(4) + z(6)) + y(6)*(z(2) - z(7)) + y(2)*(z(0) + z(1) - z(6) - z(7)) + y(4)*(-z(0) + z(7)) + y(0)*(-z(1) - z(2) + z(4) + z(7))))*twelth; - break; - case 12: - gradient_result = - ((y(1)*(-z(0) + z(5)) + y(7)*( z(0) + z(3) - z(5) - z(6)) + y(3)*(z(0) - z(7)) + y(0)*(z(1) - z(3) + z(5) - z(7)) + y(6)*(-z(5) + z(7)) + y(5)*(-z(0) - z(1) + z(6) + z(7))))*twelth; - break; - case 15: - gradient_result = - ((y(0)*( z(1) - z(4)) + y(7)*( z(4) - z(6)) + y(2)*(-z(1) + z(6)) + y(1)*(-z(0) + z(2) - z(4) + z(6)) + y(4)*(z(0) + z(1) - z(6) - z(7)) + y(6)*(-z(1) - z(2) + z(4) + z(7))))*twelth; - break; - case 18: - gradient_result = - ((y(1)*( z(2) - z(5)) + y(7)*(-z(2) - z(3) + z(4) + z(5)) + y(5)*(z(1) + z(2) - z(4) - z(7)) + y(4)*(z(5) - z(7)) + y(3)*(-z(2) + z(7)) + y(2)*(-z(1) + z(3) - z(5) + z(7))))*twelth; - break; - case 21: - gradient_result = - ((y(0)*(-z(3) + z(4)) + y(6)*( z(2) + z(3) - z(4) - z(5)) + y(2)*(z(3) - z(6)) + y(3)*(z(0) - z(2) + z(4) - z(6)) + y(5)*(-z(4) + z(6)) + y(4)*(-z(0) - z(3) + z(5) + z(6))))*twelth; - break; - case 1: - gradient_result = - (x(1)*((z(2) + z(3) - z(4) - z(5))) + - x(7)*((-z(3) + z(4))) + - x(3)*((-z(1) - z(2) + z(4) + z(7))) + - x(5)*(( z(1) - z(4))) + - x(2)*((-z(1) + z(3))) + - x(4)*((z(1) - z(3) + z(5) - z(7))))*twelth; - break; - case 4: - gradient_result = - (x(3)*(( z(0) - z(2))) + - x(5)*((-z(0) + z(2) - z(4) + z(6)))+ - x(6)*(( z(2) - z(5))) + - x(0)*((-z(2) - z(3) + z(4) + z(5))) + - x(2)*((z(0) + z(3) - z(5) - z(6))) + - x(4)*((-z(0) + z(5))))*twelth; - break; - case 7: - gradient_result = - (x(1)*((-z(0) - z(3) + z(5) + z(6))) + - x(7)*((z(3) - z(6))) + - x(3)*((z(0) + z(1) - z(6) - z(7))) + - x(5)*((-z(1) + z(6))) + - x(6)*((-z(1) + z(3) - z(5) + z(7))) + - x(0)*(( z(1) - z(3))))*twelth; - break; - case 10: - gradient_result = - (x(1)*((-z(0) + z(2))) + - x(7)*((z(0) - z(2) + z(4) - z(6))) + - x(6)*((-z(2) + z(7))) + - x(0)*((z(1) + z(2) - z(4) - z(7))) + - x(2)*((-z(0) - z(1) + z(6) + z(7))) + - x(4)*((z(0) - z(7))))*twelth; - break; - case 13: - gradient_result = - (x(1)*(( z(0) - z(5))) + - x(7)*((-z(0) - z(3) + z(5) + z(6))) + - x(3)*((-z(0) + z(7))) + - x(5)*((z(0) + z(1) - z(6) - z(7))) + - x(6)*((z(5) - z(7))) + - x(0)*((-z(1) + z(3) - z(5) + z(7))))*twelth; - break; - case 16: - gradient_result = - (x(1)*((z(0) - z(2) + z(4) - z(6))) + - x(7)*((-z(4) + z(6))) + - x(6)*((z(1) + z(2) - z(4) - z(7))) + - x(0)*((-z(1) + z(4))) + - x(2)*(( z(1) - z(6))) + - x(4)*((-z(0) - z(1) + z(6) + z(7))))*twelth; - break; - case 19: - gradient_result = - (x(1)*((-z(2) + z(5))) + - x(7)*(( z(2) + z(3) - z(4) - z(5))) + - x(3)*((z(2) - z(7))) + - x(5)*((-z(1) - z(2) + z(4) + z(7))) + - x(2)*((z(1) - z(3) + z(5) - z(7))) + - x(4)*((-z(5) + z(7))))*twelth; - break; - case 22: - gradient_result = - (x(3)*((-z(0) + z(2) - z(4) + z(6))) + - x(5)*(( z(4) - z(6))) + - x(6)*((-z(2) - z(3) + z(4) + z(5))) + - x(0)*(( z(3) - z(4))) + - x(2)*((-z(3) + z(6))) + - x(4)*(( z(0) + z(3) - z(5) - z(6))))*twelth; - break; - case 2: - gradient_result = - (x(1)*(-y(3) + y(4) + y(5) - y(2)) + - x(7)*(y(3) - y(4)) + - x(3)*(y(1) - y(7) + y(2) - y(4)) + - x(5)*(-y(1) + y(4)) + - x(2)*(y(1) - y(3)) + - x(4)*(-y(1) + y(7) + y(3) - y(5)))*twelth; - break; - case 5: - gradient_result = - (x(3)*(y(2) - y(0)) + - x(5)*(y(0) - y(2) + y(4) - y(6)) + - x(6)*(y(5) - y(2)) + - x(0)*(y(2) - y(5) + y(3) - y(4)) + - x(2)*(-y(0) + y(5) + y(6) - y(3)) + - x(4)*(y(0) - y(5)))*twelth; - break; - case 8: - gradient_result = - (x(1)*(y(3) + y(0) - y(6) - y(5)) + - x(7)*(y(6) - y(3)) + - x(3)*(-y(1)+ y(7) + y(6) - y(0)) + - x(5)*(y(1) - y(6)) + - x(6)*(y(1) - y(7) + y(5) - y(3)) + - x(0)*(-y(1) + y(3)))*twelth; - break; - case 11: - gradient_result = - (x(1)*(y(0) - y(2)) + - x(7)*(-y(0) + y(6) + y(2) - y(4)) + - x(6)*(-y(7) + y(2)) + - x(0)*(-y(2) + y(7) - y(1) + y(4)) + - x(2)*(y(0) + y(1) - y(7) - y(6)) + - x(4)*(y(7) - y(0)))*twelth; - break; - case 14: - gradient_result = - (x(1)*(-y(0) + y(5)) + - x(7)*(y(0) - y(6) + y(3) - y(5)) + - x(3)*(-y(7)+ y(0)) + - x(5)*(-y(0) + y(7) - y(1)+ y(6)) + - x(6)*(y(7) - y(5)) + - x(0)*(-y(7) + y(5) + y(1) - y(3)))*twelth; - break; - case 17: - gradient_result = - (x(1)*(-y(4) - y(0) + y(6) + y(2)) + - x(7)*(-y(6) + y(4)) + - x(6)*(-y(1) + y(7)+ y(4) - y(2)) + - x(0)*(y(1) - y(4)) + - x(2)*(-y(1) + y(6)) + - x(4)*(y(1) - y(7) + y(0) - y(6)))*twelth; - break; - case 20: - gradient_result = - (x(1)*(-y(5) + y(2)) + - x(7)*(-y(2) - y(3) + y(5) + y(4)) + - x(3)*(y(7) - y(2)) + - x(5)*(-y(7) + y(2) + y(1) - y(4)) + - x(2)*(-y(5) - y(1) + y(7) + y(3)) + - x(4)*(-y(7) + y(5)))*twelth; - break; - case 23: - gradient_result = - (x(3)*(-y(6) - y(2) + y(4) + y(0)) + - x(5)*(-y(4) + y(6)) + - x(6)*(-y(5) - y(4) + y(3) + y(2)) + - x(0)*(-y(3) + y(4)) + - x(2)*(-y(6) + y(3)) + - x(4)*(-y(3) - y(0) + y(6) + y(5)))*twelth; - break; - } - elem_vol_gradients(inode, idim) = gradient_result; - } - } - return; - -} // end subroutine - -KOKKOS_FUNCTION -void FEA_Module_SGH::get_bmatrix_gradients(const ViewCArrayKokkos &B_matrix_gradients, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const { - - const size_t num_nodes = 8; - - double x_array[8]; - double y_array[8]; - double z_array[8]; - double gradient_terms_array[2*8]; - - // x, y, z coordinates of elem vertices - auto x = ViewCArrayKokkos (x_array, num_nodes); - auto y = ViewCArrayKokkos (y_array, num_nodes); - auto z = ViewCArrayKokkos (z_array, num_nodes); - auto gradient_terms = ViewCArrayKokkos (gradient_terms_array, 2, num_nodes); - - // get the coordinates of the nodes(rk,elem,node) in this element - for (int node_lid = 0; node_lid < num_nodes; node_lid++){ - x(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 0); - y(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 1); - z(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 2); - } // end for - - double twelth = 1./12.; - - - // B_matrix(0,0) = ( +y(1)*( -z(2) -z(3) +z(4) +z(5) ) - // +y(2)*( +z(1) -z(3) ) - // +y(3)*( +z(1) +z(2) -z(4) -z(7) ) - // +y(4)*( -z(1) +z(3) -z(5) +z(7) ) - // +y(5)*( -z(1) +z(4) ) - // +y(7)*( +z(3) -z(4) ) )*twelth; - - //y derivative - gradient_terms(0,0) = 0; - gradient_terms(0,1) = -z(2) - z(3) + z(4) + z(5); - gradient_terms(0,2) = z(1) - z(3); - gradient_terms(0,3) = z(1) + z(2) - z(4) - z(7); - gradient_terms(0,4) = -z(1) + z(3) - z(5) + z(7); - gradient_terms(0,5) = -z(1) + z(4); - gradient_terms(0,6) = 0; - gradient_terms(0,7) = z(3) - z(4); - - //z derivative - gradient_terms(1,0) = 0; - gradient_terms(1,1) = y(2) + y(3) - y(4) - y(5); - gradient_terms(1,2) = -y(1) + y(3); - gradient_terms(1,3) = -y(1) - y(2) +y(4) +y(7); - gradient_terms(1,4) = y(1) - y(3) + y(5) - y(7); - gradient_terms(1,5) = y(1) - y(4); - gradient_terms(1,6) = 0; - gradient_terms(1,7) = -y(3) + y(4); - - for(int inode = 0; inode < 8; inode++){ - B_matrix_gradients(0,0,inode,0) = 0; - B_matrix_gradients(0,0,inode,1) = gradient_terms(0,inode)*twelth; - B_matrix_gradients(0,0,inode,2) = gradient_terms(1,inode)*twelth; - } - - // B_matrix(1,0) = ( +y(0)*( +z(2) +z(3) -z(4) -z(5) ) - // +y(2)*( -z(0) -z(3) +z(5) +z(6) ) - // +y(3)*( -z(0) +z(2) ) - // +y(4)*( +z(0) -z(5) ) - // +y(5)*( +z(0) -z(2) +z(4) -z(6) ) - // +y(6)*( -z(2) +z(5) ) )*twelth; - - //y derivative - gradient_terms(0,0) = z(2) + z(3) - z(4) - z(5); - gradient_terms(0,1) = 0; - gradient_terms(0,2) = -z(0) -z(3) + z(5) + z(6); - gradient_terms(0,3) = -z(0) + z(2); - gradient_terms(0,4) = z(0) -z(5); - gradient_terms(0,5) = z(0) -z(2) + z(4) - z(6); - gradient_terms(0,6) = -z(2) + z(5); - gradient_terms(0,7) = 0; - - //z derivative - gradient_terms(1,0) = -y(2) - y(3) + y(4) + y(5); - gradient_terms(1,1) = 0; - gradient_terms(1,2) = y(0) + y(3) - y(5) - y(6); - gradient_terms(1,3) = y(0) - y(2); - gradient_terms(1,4) = -y(0) + y(5); - gradient_terms(1,5) = -y(0) + y(2) - y(4) + y(6); - gradient_terms(1,6) = y(2) - y(5); - gradient_terms(1,7) = 0; - - for(int inode = 0; inode < 8; inode++){ - B_matrix_gradients(1,0,inode,0) = 0; - B_matrix_gradients(1,0,inode,1) = gradient_terms(0,inode)*twelth; - B_matrix_gradients(1,0,inode,2) = gradient_terms(1,inode)*twelth; - } - - // B_matrix(2,0) = ( +y(0)*( -z(1) +z(3) ) - // +y(1)*( +z(0) +z(3) -z(5) -z(6) ) - // +y(3)*( -z(0) -z(1) +z(6) +z(7) ) - // +y(5)*( +z(1) -z(6) ) - // +y(6)*( +z(1) -z(3) +z(5) -z(7) ) - // +y(7)*( -z(3) +z(6) ) )*twelth; - - //y derivative - gradient_terms(0,0) = -z(1) + z(3); - gradient_terms(0,1) = z(0) + z(3) - z(5) - z(6); - gradient_terms(0,2) = 0; - gradient_terms(0,3) = -z(0) - z(1) + z(6) + z(7); - gradient_terms(0,4) = 0; - gradient_terms(0,5) = z(1) - z(6); - gradient_terms(0,6) = z(1) - z(3) + z(5) - z(7); - gradient_terms(0,7) = -z(3) + z(6); - - //z derivative - gradient_terms(1,0) = y(1) - y(3); - gradient_terms(1,1) = -y(0) - y(3) + y(5) + y(6); - gradient_terms(1,2) = 0; - gradient_terms(1,3) = y(0) + y(1) - y(6) - y(7); - gradient_terms(1,4) = 0; - gradient_terms(1,5) = -y(1) + y(6); - gradient_terms(1,6) = -y(1) + y(3) - y(5) + y(7); - gradient_terms(1,7) = y(3) - y(6); - - for(int inode = 0; inode < 8; inode++){ - B_matrix_gradients(2,0,inode,0) = 0; - B_matrix_gradients(2,0,inode,1) = gradient_terms(0,inode)*twelth; - B_matrix_gradients(2,0,inode,2) = gradient_terms(1,inode)*twelth; - } - - // B_matrix(3,0) = ( +y(0)*( -z(1) -z(2) +z(4) +z(7) ) - // +y(1)*( +z(0) -z(2) ) - // +y(2)*( +z(0) +z(1) -z(6) -z(7) ) - // +y(4)*( -z(0) +z(7) ) - // +y(6)*( +z(2) -z(7) ) - // +y(7)*( -z(0) +z(2) -z(4) +z(6) ) )*twelth; - - //y derivative - gradient_terms(0,0) = -z(1) - z(2) + z(4) + z(7); - gradient_terms(0,1) = z(0) - z(2); - gradient_terms(0,2) = z(0) + z(1) - z(6) - z(7); - gradient_terms(0,3) = 0; - gradient_terms(0,4) = -z(0) + z(7); - gradient_terms(0,5) = 0; - gradient_terms(0,6) = z(2) - z(7); - gradient_terms(0,7) = -z(0) + z(2) - z(4) + z(6); - - //z derivative - gradient_terms(1,0) = y(1) + y(2) - y(4) - y(7); - gradient_terms(1,1) = -y(0) + y(2); - gradient_terms(1,2) = -y(0) - y(1) + y(6) + y(7); - gradient_terms(1,3) = 0; - gradient_terms(1,4) = y(0) - y(7); - gradient_terms(1,5) = 0; - gradient_terms(1,6) = -y(2) + y(7); - gradient_terms(1,7) = y(0) - y(2) + y(4) - y(6); - - for(int inode = 0; inode < 8; inode++){ - B_matrix_gradients(3,0,inode,0) = 0; - B_matrix_gradients(3,0,inode,1) = gradient_terms(0,inode)*twelth; - B_matrix_gradients(3,0,inode,2) = gradient_terms(1,inode)*twelth; - } - - // B_matrix(4,0) = ( +y(0)*( +z(1) -z(3) +z(5) -z(7) ) - // +y(1)*( -z(0) +z(5) ) - // +y(3)*( +z(0) -z(7) ) - // +y(5)*( -z(0) -z(1) +z(6) +z(7) ) - // +y(6)*( -z(5) +z(7) ) - // +y(7)*( +z(0) +z(3) -z(5) -z(6) ) )*twelth; - - //y derivative - gradient_terms(0,0) = z(1) - z(3) + z(5) - z(7); - gradient_terms(0,1) = -z(0) + z(5); - gradient_terms(0,2) = 0; - gradient_terms(0,3) = z(0) - z(7); - gradient_terms(0,4) = 0; - gradient_terms(0,5) = -z(0) - z(1) + z(6) + z(7); - gradient_terms(0,6) = -z(5) + z(7); - gradient_terms(0,7) = z(0) + z(3) - z(5) - z(6); - - //z derivative - gradient_terms(1,0) = -y(1) + y(3) - y(5) + y(7); - gradient_terms(1,1) = y(0) - y(5); - gradient_terms(1,2) = 0; - gradient_terms(1,3) = -y(0) + y(7); - gradient_terms(1,4) = 0; - gradient_terms(1,5) = y(0) + y(1) - y(6) - y(7); - gradient_terms(1,6) = y(5) - y(7); - gradient_terms(1,7) = -y(0) - y(3) + y(5) + y(6); - - for(int inode = 0; inode < 8; inode++){ - B_matrix_gradients(4,0,inode,0) = 0; - B_matrix_gradients(4,0,inode,1) = gradient_terms(0,inode)*twelth; - B_matrix_gradients(4,0,inode,2) = gradient_terms(1,inode)*twelth; - } - - // B_matrix(5,0) = ( +y(0)*( +z(1) -z(4) ) - // +y(1)*( -z(0) +z(2) -z(4) +z(6) ) - // +y(2)*( -z(1) +z(6) ) - // +y(4)*( +z(0) +z(1) -z(6) -z(7) ) - // +y(6)*( -z(1) -z(2) +z(4) +z(7) ) - // +y(7)*( +z(4) -z(6) ) )*twelth; - - //y derivative - gradient_terms(0,0) = z(1) -z(4); - gradient_terms(0,1) = -z(0) +z(2) -z(4) +z(6); - gradient_terms(0,2) = -z(1) +z(6); - gradient_terms(0,3) = 0; - gradient_terms(0,4) = z(0) +z(1) -z(6) -z(7); - gradient_terms(0,5) = 0; - gradient_terms(0,6) = -z(1) -z(2) +z(4) +z(7); - gradient_terms(0,7) = z(4) -z(6); - - //z derivative - gradient_terms(1,0) = -y(1) + y(4); - gradient_terms(1,1) = y(0) - y(2) + y(4) - y(6); - gradient_terms(1,2) = y(1) - y(6); - gradient_terms(1,3) = 0; - gradient_terms(1,4) = -y(0) - y(1) + y(6) +y(7); - gradient_terms(1,5) = 0; - gradient_terms(1,6) = y(1) + y(2) - y(4) - y(7); - gradient_terms(1,7) = -y(4) + y(6); - - for(int inode = 0; inode < 8; inode++){ - B_matrix_gradients(5,0,inode,0) = 0; - B_matrix_gradients(5,0,inode,1) = gradient_terms(0,inode)*twelth; - B_matrix_gradients(5,0,inode,2) = gradient_terms(1,inode)*twelth; - } - - // B_matrix(6,0) = ( +y(1)*( +z(2) -z(5) ) - // +y(2)*( -z(1) +z(3) -z(5) +z(7) ) - // +y(3)*( -z(2) +z(7) ) - // +y(4)*( +z(5) -z(7) ) - // +y(5)*( +z(1) +z(2) -z(4) -z(7) ) - // +y(7)*( -z(2) -z(3) +z(4) +z(5) ) )*twelth; - - //y derivative - gradient_terms(0,0) = 0; - gradient_terms(0,1) = z(2) - z(5); - gradient_terms(0,2) = -z(1) + z(3) - z(5) + z(7); - gradient_terms(0,3) = -z(2) + z(7); - gradient_terms(0,4) = z(5) - z(7); - gradient_terms(0,5) = z(1) + z(2) - z(4) - z(7); - gradient_terms(0,6) = 0; - gradient_terms(0,7) = -z(2) - z(3) + z(4) + z(5); - - //z derivative - gradient_terms(1,0) = 0; - gradient_terms(1,1) = -y(2) + y(5); - gradient_terms(1,2) = y(1) - y(3) + y(5) - y(7); - gradient_terms(1,3) = y(2) - y(7); - gradient_terms(1,4) = -y(5) + y(7); - gradient_terms(1,5) = -y(1) - y(2) + y(4) + y(7); - gradient_terms(1,6) = 0; - gradient_terms(1,7) = y(2) + y(3) - y(4) - y(5); - - for(int inode = 0; inode < 8; inode++){ - B_matrix_gradients(6,0,inode,0) = 0; - B_matrix_gradients(6,0,inode,1) = gradient_terms(0,inode)*twelth; - B_matrix_gradients(6,0,inode,2) = gradient_terms(1,inode)*twelth; - } - - // B_matrix(7,0) = ( +y(0)*( -z(3) +z(4) ) - // +y(2)*( +z(3) -z(6) ) - // +y(3)*( +z(0) -z(2) +z(4) -z(6) ) - // +y(4)*( -z(0) -z(3) +z(5) +z(6) ) - // +y(5)*( -z(4) +z(6) ) - // +y(6)*( +z(2) +z(3) -z(4) -z(5) ) )*twelth; - - //y derivative - gradient_terms(0,0) = -z(3) +z(4); - gradient_terms(0,1) = 0; - gradient_terms(0,2) = z(3) -z(6); - gradient_terms(0,3) = z(0) -z(2) +z(4) -z(6); - gradient_terms(0,4) = -z(0) -z(3) +z(5) +z(6); - gradient_terms(0,5) = -z(4) +z(6); - gradient_terms(0,6) = z(2) +z(3) -z(4) -z(5); - gradient_terms(0,7) = 0; - - //z derivative - gradient_terms(1,0) = y(3) - y(4); - gradient_terms(1,1) = 0; - gradient_terms(1,2) = -y(3) + y(6); - gradient_terms(1,3) = -y(0) + y(2) - y(4) + y(6); - gradient_terms(1,4) = y(0) + y(3) - y(5) - y(6); - gradient_terms(1,5) = y(4) - y(6); - gradient_terms(1,6) = -y(2) - y(3) + y(4) + y(5); - gradient_terms(1,7) = 0; - - for(int inode = 0; inode < 8; inode++){ - B_matrix_gradients(7,0,inode,0) = 0; - B_matrix_gradients(7,0,inode,1) = gradient_terms(0,inode)*twelth; - B_matrix_gradients(7,0,inode,2) = gradient_terms(1,inode)*twelth; - } - - // B_matrix(0,1) = ( +z(1)*( -x(2) -x(3) +x(4) +x(5) ) - // +z(2)*( +x(1) -x(3) ) - // +z(3)*( +x(1) +x(2) -x(4) -x(7) ) - // +z(4)*( -x(1) +x(3) -x(5) +x(7) ) - // +z(5)*( -x(1) +x(4) ) - // +z(7)*( +x(3) -x(4) ) )*twelth; - - //x derivative - gradient_terms(0,0) = 0; - gradient_terms(0,1) = z(2) + z(3) - z(4) - z(5); - gradient_terms(0,2) = -z(1) + z(3); - gradient_terms(0,3) = -z(1) - z(2) + z(4) + z(7); - gradient_terms(0,4) = z(1) - z(3) + z(5) - z(7); - gradient_terms(0,5) = z(1) - z(4); - gradient_terms(0,6) = 0; - gradient_terms(0,7) = -z(3) + z(4); - - //z derivative - gradient_terms(1,0) = 0; - gradient_terms(1,1) = -x(2) - x(3) + x(4) + x(5); - gradient_terms(1,2) = x(1) - x(3); - gradient_terms(1,3) = x(1) + x(2) - x(4) - x(7); - gradient_terms(1,4) = -x(1) + x(3) - x(5) + x(7); - gradient_terms(1,5) = -x(1) + x(4); - gradient_terms(1,6) = 0; - gradient_terms(1,7) = x(3) - x(4); - - for(int inode = 0; inode < 8; inode++){ - B_matrix_gradients(0,1,inode,0) = gradient_terms(0,inode)*twelth; - B_matrix_gradients(0,1,inode,1) = 0; - B_matrix_gradients(0,1,inode,2) = gradient_terms(1,inode)*twelth; - } - - // B_matrix(1,1) = ( +z(0)*( +x(2) +x(3) -x(4) -x(5) ) - // +z(2)*( -x(0) -x(3) +x(5) +x(6) ) - // +z(3)*( -x(0) +x(2) ) - // +z(4)*( +x(0) -x(5) ) - // +z(5)*( +x(0) -x(2) +x(4) -x(6) ) - // +z(6)*( -x(2) +x(5) ) )*twelth; - - //x derivative - gradient_terms(0,0) = -z(2) - z(3) +z(4) + z(5); - gradient_terms(0,1) = 0; - gradient_terms(0,2) = z(0) + z(3) - z(5) - z(6); - gradient_terms(0,3) = z(0) - z(2); - gradient_terms(0,4) = -z(0) + z(5); - gradient_terms(0,5) = -z(0) + z(2) - z(4) + z(6); - gradient_terms(0,6) = z(2) - z(5); - gradient_terms(0,7) = 0; - - //z derivative - gradient_terms(1,0) = x(2) + x(3) - x(4) - x(5); - gradient_terms(1,1) = 0; - gradient_terms(1,2) = -x(0) -x(3) + x(5) + x(6); - gradient_terms(1,3) = -x(0) + x(2); - gradient_terms(1,4) = x(0) - x(5); - gradient_terms(1,5) = x(0) - x(2) + x(4) - x(6); - gradient_terms(1,6) = -x(2) + x(5); - gradient_terms(1,7) = 0; - - for(int inode = 0; inode < 8; inode++){ - B_matrix_gradients(1,1,inode,0) = gradient_terms(0,inode)*twelth; - B_matrix_gradients(1,1,inode,1) = 0; - B_matrix_gradients(1,1,inode,2) = gradient_terms(1,inode)*twelth; - } - - // B_matrix(2,1) = ( +z(0)*( -x(1) +x(3) ) - // +z(1)*( +x(0) +x(3) -x(5) -x(6) ) - // +z(3)*( -x(0) -x(1) +x(6) +x(7) ) - // +z(5)*( +x(1) -x(6) ) - // +z(6)*( +x(1) -x(3) +x(5) -x(7) ) - // +z(7)*( -x(3) +x(6) ) )*twelth; - - //x derivative - gradient_terms(0,0) = z(1) - z(3); - gradient_terms(0,1) = -z(0) - z(3) +z(5) + z(6); - gradient_terms(0,2) = 0; - gradient_terms(0,3) = z(0) + z(1) - z(6) - z(7); - gradient_terms(0,4) = 0; - gradient_terms(0,5) = -z(1) + z(6); - gradient_terms(0,6) = -z(1) + z(3) - z(5) + z(7); - gradient_terms(0,7) = z(3) - z(6); - - //z derivative - gradient_terms(1,0) = -x(1) + x(3); - gradient_terms(1,1) = x(0) + x(3) - x(5) - x(6); - gradient_terms(1,2) = 0; - gradient_terms(1,3) = -x(0) - x(1) + x(6) + x(7); - gradient_terms(1,4) = 0; - gradient_terms(1,5) = x(1) - x(6); - gradient_terms(1,6) = x(1) - x(3) + x(5) - x(7); - gradient_terms(1,7) = -x(3) + x(6); - - for(int inode = 0; inode < 8; inode++){ - B_matrix_gradients(2,1,inode,0) = gradient_terms(0,inode)*twelth; - B_matrix_gradients(2,1,inode,1) = 0; - B_matrix_gradients(2,1,inode,2) = gradient_terms(1,inode)*twelth; - } - - // B_matrix(3,1) = ( +z(0)*( -x(1) -x(2) +x(4) +x(7) ) - // +z(1)*( +x(0) -x(2) ) - // +z(2)*( +x(0) +x(1) -x(6) -x(7) ) - // +z(4)*( -x(0) +x(7) ) - // +z(6)*( +x(2) -x(7) ) - // +z(7)*( -x(0) +x(2) -x(4) +x(6) ) )*twelth; - - //x derivative - gradient_terms(0,0) = z(1) + z(2) - z(4) - z(7); - gradient_terms(0,1) = -z(0) + z(2); - gradient_terms(0,2) = -z(0) - z(1) + z(6) + z(7); - gradient_terms(0,3) = 0; - gradient_terms(0,4) = z(0) - z(7); - gradient_terms(0,5) = 0; - gradient_terms(0,6) = -z(2) + z(7); - gradient_terms(0,7) = z(0) - z(2) + z(4) - z(6); - - //z derivative - gradient_terms(1,0) = -x(1) - x(2) + x(4) + x(7); - gradient_terms(1,1) = x(0) - x(2); - gradient_terms(1,2) = x(0) + x(1) - x(6) - x(7); - gradient_terms(1,3) = 0; - gradient_terms(1,4) = -x(0) + x(7); - gradient_terms(1,5) = 0; - gradient_terms(1,6) = x(2) - x(7); - gradient_terms(1,7) = -x(0) + x(2) - x(4) + x(6); - - for(int inode = 0; inode < 8; inode++){ - B_matrix_gradients(3,1,inode,0) = gradient_terms(0,inode)*twelth; - B_matrix_gradients(3,1,inode,1) = 0; - B_matrix_gradients(3,1,inode,2) = gradient_terms(1,inode)*twelth; - } - - // B_matrix(4,1) = ( +z(0)*( +x(1) -x(3) +x(5) -x(7) ) - // +z(1)*( -x(0) +x(5) ) - // +z(3)*( +x(0) -x(7) ) - // +z(5)*( -x(0) -x(1) +x(6) +x(7) ) - // +z(6)*( -x(5) +x(7) ) - // +z(7)*( +x(0) +x(3) -x(5) -x(6) ) )*twelth; - - //x derivative - gradient_terms(0,0) = -z(1) + z(3) - z(5) + z(7); - gradient_terms(0,1) = z(0) - z(5); - gradient_terms(0,2) = 0; - gradient_terms(0,3) = -z(0) + z(7); - gradient_terms(0,4) = 0; - gradient_terms(0,5) = z(0) + z(1) - z(6) - z(7); - gradient_terms(0,6) = z(5) - z(7); - gradient_terms(0,7) = -z(0) - z(3) + z(5) + z(6); - - //z derivative - gradient_terms(1,0) = x(1) - x(3) + x(5) - x(7); - gradient_terms(1,1) = -x(0) + x(5); - gradient_terms(1,2) = 0; - gradient_terms(1,3) = x(0) -x(7); - gradient_terms(1,4) = 0; - gradient_terms(1,5) = -x(0) - x(1) + x(6) + x(7); - gradient_terms(1,6) = -x(5) + x(7); - gradient_terms(1,7) = x(0) + x(3) - x(5) - x(6); - - for(int inode = 0; inode < 8; inode++){ - B_matrix_gradients(4,1,inode,0) = gradient_terms(0,inode)*twelth; - B_matrix_gradients(4,1,inode,1) = 0; - B_matrix_gradients(4,1,inode,2) = gradient_terms(1,inode)*twelth; - } - - // B_matrix(5,1) = ( +z(0)*( +x(1) -x(4) ) - // +z(1)*( -x(0) +x(2) -x(4) +x(6) ) - // +z(2)*( -x(1) +x(6) ) - // +z(4)*( +x(0) +x(1) -x(6) -x(7) ) - // +z(6)*( -x(1) -x(2) +x(4) +x(7) ) - // +z(7)*( +x(4) -x(6) ) )*twelth; - - //x derivative - gradient_terms(0,0) = -z(1) + z(4); - gradient_terms(0,1) = z(0) - z(2) + z(4) - z(6); - gradient_terms(0,2) = z(1) - z(6); - gradient_terms(0,3) = 0; - gradient_terms(0,4) = -z(0) - z(1) + z(6) + z(7); - gradient_terms(0,5) = 0; - gradient_terms(0,6) = z(1) + z(2) - z(4) - z(7); - gradient_terms(0,7) = -z(4) + z(6); - - //z derivative - gradient_terms(1,0) = x(1) - x(4); - gradient_terms(1,1) = -x(0) + x(2) -x(4) + x(6); - gradient_terms(1,2) = -x(1) + x(6); - gradient_terms(1,3) = 0; - gradient_terms(1,4) = x(0) + x(1) - x(6) - x(7); - gradient_terms(1,5) = 0; - gradient_terms(1,6) = -x(1) - x(2) + x(4) + x(7); - gradient_terms(1,7) = x(4) - x(6); - - for(int inode = 0; inode < 8; inode++){ - B_matrix_gradients(5,1,inode,0) = gradient_terms(0,inode)*twelth; - B_matrix_gradients(5,1,inode,1) = 0; - B_matrix_gradients(5,1,inode,2) = gradient_terms(1,inode)*twelth; - } - - // B_matrix(6,1) = ( +z(1)*( +x(2) -x(5) ) - // +z(2)*( -x(1) +x(3) -x(5) +x(7) ) - // +z(3)*( -x(2) +x(7) ) - // +z(4)*( +x(5) -x(7) ) - // +z(5)*( +x(1) +x(2) -x(4) -x(7) ) - // +z(7)*( -x(2) -x(3) +x(4) +x(5) ) )*twelth; - - //x derivative - gradient_terms(0,0) = 0; - gradient_terms(0,1) = -z(2) + z(5); - gradient_terms(0,2) = z(1) - z(3) + z(5) - z(7); - gradient_terms(0,3) = z(2) - z(7); - gradient_terms(0,4) = -z(5) + z(7); - gradient_terms(0,5) = -z(1) - z(2) +z(4) +z(7); - gradient_terms(0,6) = 0; - gradient_terms(0,7) = z(2) + z(3) - z(4) - z(5); - - //z derivative - gradient_terms(1,0) = 0; - gradient_terms(1,1) = x(2) - x(5); - gradient_terms(1,2) = -x(1) + x(3) - x(5) + x(7); - gradient_terms(1,3) = -x(2) + x(7); - gradient_terms(1,4) = x(5) - x(7); - gradient_terms(1,5) = x(1) + x(2) - x(4) - x(7); - gradient_terms(1,6) = 0; - gradient_terms(1,7) = -x(2) - x(3) + x(4) + x(5); - - for(int inode = 0; inode < 8; inode++){ - B_matrix_gradients(6,1,inode,0) = gradient_terms(0,inode)*twelth; - B_matrix_gradients(6,1,inode,1) = 0; - B_matrix_gradients(6,1,inode,2) = gradient_terms(1,inode)*twelth; - } - - // B_matrix(7,1) = ( +z(0)*( -x(3) +x(4) ) - // +z(2)*( +x(3) -x(6) ) - // +z(3)*( +x(0) -x(2) +x(4) -x(6) ) - // +z(4)*( -x(0) -x(3) +x(5) +x(6) ) - // +z(5)*( -x(4) +x(6) ) - // +z(6)*( +x(2) +x(3) -x(4) -x(5) ) )*twelth; - - //x derivative - gradient_terms(0,0) = z(3) - z(4); - gradient_terms(0,1) = 0; - gradient_terms(0,2) = -z(3) + z(6); - gradient_terms(0,3) = -z(0) + z(2) - z(4) + z(6); - gradient_terms(0,4) = z(0) + z(3) - z(5) - z(6); - gradient_terms(0,5) = z(4) - z(6); - gradient_terms(0,6) = -z(2) - z(3) + z(4) + z(5); - gradient_terms(0,7) = 0; - - //z derivative - gradient_terms(1,0) = -x(3) + x(4); - gradient_terms(1,1) = 0; - gradient_terms(1,2) = x(3) - x(6); - gradient_terms(1,3) = x(0) - x(2) + x(4) - x(6); - gradient_terms(1,4) = -x(0) - x(3) + x(5) + x(6); - gradient_terms(1,5) = -x(4) + x(6); - gradient_terms(1,6) = x(2) + x(3) -x(4) - x(5); - gradient_terms(1,7) = 0; - - for(int inode = 0; inode < 8; inode++){ - B_matrix_gradients(7,1,inode,0) = gradient_terms(0,inode)*twelth; - B_matrix_gradients(7,1,inode,1) = 0; - B_matrix_gradients(7,1,inode,2) = gradient_terms(1,inode)*twelth; - } - - // B_matrix(0,2) = ( +x(1)*( -y(2) -y(3) +y(4) +y(5) ) - // +x(2)*( +y(1) -y(3) ) - // +x(3)*( +y(1) +y(2) -y(4) -y(7) ) - // +x(4)*( -y(1) +y(3) -y(5) +y(7) ) - // +x(5)*( -y(1) +y(4) ) - // +x(7)*( +y(3) -y(4) ) )*twelth; - - //x derivative - gradient_terms(0,0) = 0; - gradient_terms(0,1) = -y(2) - y(3) + y(4) + y(5); - gradient_terms(0,2) = y(1) - y(3); - gradient_terms(0,3) = y(1) + y(2) - y(4) - y(7); - gradient_terms(0,4) = -y(1) + y(3) - y(5) + y(7); - gradient_terms(0,5) = -y(1) + y(4); - gradient_terms(0,6) = 0; - gradient_terms(0,7) = y(3) - y(4); - - //y derivative - gradient_terms(1,0) = 0; - gradient_terms(1,1) = x(2) + x(3) - x(4) - x(5); - gradient_terms(1,2) = -x(1) + x(3); - gradient_terms(1,3) = -x(1) - x(2) + x(4) + x(7); - gradient_terms(1,4) = x(1) - x(3) + x(5) - x(7); - gradient_terms(1,5) = x(1) - x(4); - gradient_terms(1,6) = 0; - gradient_terms(1,7) = -x(3) + x(4); - - for(int inode = 0; inode < 8; inode++){ - B_matrix_gradients(0,2,inode,0) = gradient_terms(0,inode)*twelth; - B_matrix_gradients(0,2,inode,1) = gradient_terms(1,inode)*twelth; - B_matrix_gradients(0,2,inode,2) = 0; - } - - // B_matrix(1,2) = ( +x(0)*( +y(2) +y(3) -y(4) -y(5) ) - // +x(2)*( -y(0) -y(3) +y(5) +y(6) ) - // +x(3)*( -y(0) +y(2) ) - // +x(4)*( +y(0) -y(5) ) - // +x(5)*( +y(0) -y(2) +y(4) -y(6) ) - // +x(6)*( -y(2) +y(5) ) )*twelth; - - //x derivative - gradient_terms(0,0) = y(2) + y(3) - y(4) - y(5); - gradient_terms(0,1) = 0; - gradient_terms(0,2) = -y(0) - y(3) + y(5) + y(6); - gradient_terms(0,3) = -y(0) + y(2); - gradient_terms(0,4) = y(0) - y(5); - gradient_terms(0,5) = y(0) - y(2) + y(4) - y(6); - gradient_terms(0,6) = -y(2) + y(5); - gradient_terms(0,7) = 0; - - //y derivative - gradient_terms(1,0) = -x(2) - x(3) + x(4) + x(5); - gradient_terms(1,1) = 0; - gradient_terms(1,2) = x(0) + x(3) - x(5) - x(6); - gradient_terms(1,3) = x(0) - x(2); - gradient_terms(1,4) = -x(0) + x(5); - gradient_terms(1,5) = -x(0) + x(2) - x(4) + x(6); - gradient_terms(1,6) = x(2) - x(5); - gradient_terms(1,7) = 0; - - for(int inode = 0; inode < 8; inode++){ - B_matrix_gradients(1,2,inode,0) = gradient_terms(0,inode)*twelth; - B_matrix_gradients(1,2,inode,1) = gradient_terms(1,inode)*twelth; - B_matrix_gradients(1,2,inode,2) = 0; - } - - // B_matrix(2,2) = ( +x(0)*( -y(1) +y(3) ) - // +x(1)*( +y(0) +y(3) -y(5) -y(6) ) - // +x(3)*( -y(0) -y(1) +y(6) +y(7) ) - // +x(5)*( +y(1) -y(6) ) - // +x(6)*( +y(1) -y(3) +y(5) -y(7) ) - // +x(7)*( -y(3) +y(6) ) )*twelth; - - //x derivative - gradient_terms(0,0) = -y(1) + y(3); - gradient_terms(0,1) = y(0) + y(3) - y(5) - y(6); - gradient_terms(0,2) = 0; - gradient_terms(0,3) = -y(0) - y(1) + y(6) + y(7); - gradient_terms(0,4) = 0; - gradient_terms(0,5) = y(1) - y(6); - gradient_terms(0,6) = y(1) - y(3) + y(5) - y(7); - gradient_terms(0,7) = -y(3) + y(6); - - //y derivative - gradient_terms(1,0) = x(1) - x(3); - gradient_terms(1,1) = -x(0) - x(3) + x(5) + x(6); - gradient_terms(1,2) = 0; - gradient_terms(1,3) = x(0) + x(1) -x(6) - x(7); - gradient_terms(1,4) = 0; - gradient_terms(1,5) = -x(1) + x(6); - gradient_terms(1,6) = -x(1) + x(3) - x(5) + x(7); - gradient_terms(1,7) = x(3) - x(6); - - for(int inode = 0; inode < 8; inode++){ - B_matrix_gradients(2,2,inode,0) = gradient_terms(0,inode)*twelth; - B_matrix_gradients(2,2,inode,1) = gradient_terms(1,inode)*twelth; - B_matrix_gradients(2,2,inode,2) = 0; - } - - // B_matrix(3,2) = ( +x(0)*( -y(1) -y(2) +y(4) +y(7) ) - // +x(1)*( +y(0) -y(2) ) - // +x(2)*( +y(0) +y(1) -y(6) -y(7) ) - // +x(4)*( -y(0) +y(7) ) - // +x(6)*( +y(2) -y(7) ) - // +x(7)*( -y(0) +y(2) -y(4) +y(6) ) )*twelth; - - //x derivative - gradient_terms(0,0) = -y(1) - y(2) + y(4) + y(7); - gradient_terms(0,1) = y(0) - y(2); - gradient_terms(0,2) = y(0) + y(1) - y(6) - y(7); - gradient_terms(0,3) = 0; - gradient_terms(0,4) = -y(0) + y(7); - gradient_terms(0,5) = 0; - gradient_terms(0,6) = y(2) - y(7); - gradient_terms(0,7) = -y(0) + y(2) - y(4) + y(6); - - //y derivative - gradient_terms(1,0) = x(1) + x(2) - x(4) - x(7); - gradient_terms(1,1) = -x(0) + x(2); - gradient_terms(1,2) = -x(0) - x(1) + x(6) + x(7); - gradient_terms(1,3) = 0; - gradient_terms(1,4) = x(0) - x(7); - gradient_terms(1,5) = 0; - gradient_terms(1,6) = -x(2) + x(7); - gradient_terms(1,7) = x(0) - x(2) + x(4) - x(6); - - for(int inode = 0; inode < 8; inode++){ - B_matrix_gradients(3,2,inode,0) = gradient_terms(0,inode)*twelth; - B_matrix_gradients(3,2,inode,1) = gradient_terms(1,inode)*twelth; - B_matrix_gradients(3,2,inode,2) = 0; - } - - // B_matrix(4,2) = ( +x(0)*( +y(1) -y(3) +y(5) -y(7) ) - // +x(1)*( -y(0) +y(5) ) - // +x(3)*( +y(0) -y(7) ) - // +x(5)*( -y(0) -y(1) +y(6) +y(7) ) - // +x(6)*( -y(5) +y(7) ) - // +x(7)*( +y(0) +y(3) -y(5) -y(6) ) )*twelth; - - //x derivative - gradient_terms(0,0) = y(1) - y(3) + y(5) - y(7); - gradient_terms(0,1) = -y(0) + y(5); - gradient_terms(0,2) = 0; - gradient_terms(0,3) = y(0) - y(7); - gradient_terms(0,4) = 0; - gradient_terms(0,5) = -y(0) - y(1) + y(6) + y(7); - gradient_terms(0,6) = -y(5) + y(7); - gradient_terms(0,7) = y(0) + y(3) - y(5) - y(6); - - //y derivative - gradient_terms(1,0) = -x(1) + x(3) - x(5) + x(7); - gradient_terms(1,1) = x(0) - x(5); - gradient_terms(1,2) = 0; - gradient_terms(1,3) = -x(0) + x(7); - gradient_terms(1,4) = 0; - gradient_terms(1,5) = x(0) + x(1) - x(6) - x(7); - gradient_terms(1,6) = x(5) - x(7); - gradient_terms(1,7) = -x(0) - x(3) + x(5) + x(6); - - for(int inode = 0; inode < 8; inode++){ - B_matrix_gradients(4,2,inode,0) = gradient_terms(0,inode)*twelth; - B_matrix_gradients(4,2,inode,1) = gradient_terms(1,inode)*twelth; - B_matrix_gradients(4,2,inode,2) = 0; - } - - // B_matrix(5,2) = ( +x(0)*( +y(1) -y(4) ) - // +x(1)*( -y(0) +y(2) -y(4) +y(6) ) - // +x(2)*( -y(1) +y(6) ) - // +x(4)*( +y(0) +y(1) -y(6) -y(7) ) - // +x(6)*( -y(1) -y(2) +y(4) +y(7) ) - // +x(7)*( +y(4) -y(6) ) )*twelth; - - //x derivative - gradient_terms(0,0) = y(1) - y(4); - gradient_terms(0,1) = -y(0) + y(2) - y(4) + y(6); - gradient_terms(0,2) = -y(1) + y(6); - gradient_terms(0,3) = 0; - gradient_terms(0,4) = y(0) + y(1) - y(6) - y(7); - gradient_terms(0,5) = 0; - gradient_terms(0,6) = -y(1) - y(2) + y(4) + y(7); - gradient_terms(0,7) = y(4) - y(6); - - //y derivative - gradient_terms(1,0) = -x(1) + x(4); - gradient_terms(1,1) = x(0) - x(2) + x(4) - x(6); - gradient_terms(1,2) = x(1) - x(6); - gradient_terms(1,3) = 0; - gradient_terms(1,4) = -x(0) - x(1) + x(6) + x(7); - gradient_terms(1,5) = 0; - gradient_terms(1,6) = x(1) + x(2) - x(4) - x(7); - gradient_terms(1,7) = -x(4) + x(6); - - for(int inode = 0; inode < 8; inode++){ - B_matrix_gradients(5,2,inode,0) = gradient_terms(0,inode)*twelth; - B_matrix_gradients(5,2,inode,1) = gradient_terms(1,inode)*twelth; - B_matrix_gradients(5,2,inode,2) = 0; - } - - // B_matrix(6,2) = ( +x(1)*( +y(2) -y(5) ) - // +x(2)*( -y(1) +y(3) -y(5) +y(7) ) - // +x(3)*( -y(2) +y(7) ) - // +x(4)*( +y(5) -y(7) ) - // +x(5)*( +y(1) +y(2) -y(4) -y(7) ) - // +x(7)*( -y(2) -y(3) +y(4) +y(5) ) )*twelth; - - //x derivative - gradient_terms(0,0) = 0; - gradient_terms(0,1) = y(2) - y(5); - gradient_terms(0,2) = -y(1) + y(3) -y(5) + y(7); - gradient_terms(0,3) = -y(2) + y(7); - gradient_terms(0,4) = y(5) - y(7); - gradient_terms(0,5) = y(1) + y(2) - y(4) - y(7); - gradient_terms(0,6) = 0; - gradient_terms(0,7) = -y(2) - y(3) + y(4) + y(5); - - //y derivative - gradient_terms(1,0) = 0; - gradient_terms(1,1) = -x(2) + x(5); - gradient_terms(1,2) = x(1) - x(3) + x(5) - x(7); - gradient_terms(1,3) = x(2) - x(7); - gradient_terms(1,4) = -x(5) + x(7); - gradient_terms(1,5) = -x(1) - x(2) + x(4) + x(7); - gradient_terms(1,6) = 0; - gradient_terms(1,7) = x(2) + x(3) - x(4) - x(5); - - for(int inode = 0; inode < 8; inode++){ - B_matrix_gradients(6,2,inode,0) = gradient_terms(0,inode)*twelth; - B_matrix_gradients(6,2,inode,1) = gradient_terms(1,inode)*twelth; - B_matrix_gradients(6,2,inode,2) = 0; - } - - // B_matrix(7,2) = ( +x(0)*( -y(3) +y(4) ) - // +x(2)*( +y(3) -y(6) ) - // +x(3)*( +y(0) -y(2) +y(4) -y(6) ) - // +x(4)*( -y(0) -y(3) +y(5) +y(6) ) - // +x(5)*( -y(4) +y(6) ) - // +x(6)*( +y(2) +y(3) -y(4) -y(5) ) )*twelth; - - //x derivative - gradient_terms(0,0) = -y(3) + y(4); - gradient_terms(0,1) = 0; - gradient_terms(0,2) = y(3) - y(6); - gradient_terms(0,3) = y(0) - y(2) + y(4) - y(6); - gradient_terms(0,4) = -y(0) - y(3) + y(5) + y(6); - gradient_terms(0,5) = -y(4) + y(6); - gradient_terms(0,6) = y(2) + y(3) - y(4) - y(5); - gradient_terms(0,7) = 0; - - //y derivative - gradient_terms(1,0) = x(3) - x(4); - gradient_terms(1,1) = 0; - gradient_terms(1,2) = -x(3) + x(6); - gradient_terms(1,3) = -x(0) +x(2) - x(4) + x(6); - gradient_terms(1,4) = x(0) + x(3) - x(5) - x(6); - gradient_terms(1,5) = x(4) - x(6); - gradient_terms(1,6) = -x(2) - x(3) + x(4) + x(5); - gradient_terms(1,7) = 0; - - for(int inode = 0; inode < 8; inode++){ - B_matrix_gradients(7,2,inode,0) = gradient_terms(0,inode)*twelth; - B_matrix_gradients(7,2,inode,1) = gradient_terms(1,inode)*twelth; - B_matrix_gradients(7,2,inode,2) = 0; - } - -} // end subroutine diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/FEA_Module_SGH.h b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/FEA_Module_SGH.h new file mode 100644 index 000000000..a9cc158bd --- /dev/null +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/FEA_Module_SGH.h @@ -0,0 +1,700 @@ +/********************************************************************************************** + © 2020. Triad National Security, LLC. All rights reserved. + This program was produced under U.S. Government contract 89233218CNA000001 for Los Alamos + National Laboratory (LANL), which is operated by Triad National Security, LLC for the U.S. + Department of Energy/National Nuclear Security Administration. All rights in the program are + reserved by Triad National Security, LLC, and the U.S. Department of Energy/National Nuclear + Security Administration. The Government is granted for itself and others acting on its behalf a + nonexclusive, paid-up, irrevocable worldwide license in this material to reproduce, prepare + derivative works, distribute copies to the public, perform publicly and display publicly, and + to permit others to do so. + This program is open source under the BSD-3 License. + Redistribution and use in source and binary forms, with or without modification, are permitted + provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, this list of + conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, this list of + conditions and the following disclaimer in the documentation and/or other materials + provided with the distribution. + + 3. Neither the name of the copyright holder nor the names of its contributors may be used + to endorse or promote products derived from this software without specific prior + written permission. + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS + IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **********************************************************************************************/ + +#ifndef FEA_MODULE_SGH_H +#define FEA_MODULE_SGH_H + +#include "mesh.h" +#include "state.h" +#include "matar.h" +#include "elements.h" +#include "node_combination.h" +#include "FEA_Module.h" +#include "material_models.h" + +// Forward declarations +class Explicit_Solver; +class Solver; +class Simulation_Parameters_Explicit; +class SGH_Parameters; + +struct material_t; +struct boundary_t; + +class FEA_Module_SGH : public FEA_Module +{ +public: + + FEA_Module_SGH(SGH_Parameters& params, Solver* Solver_Pointer, std::shared_ptr mesh_in, const int my_fea_module_index = 0); + ~FEA_Module_SGH(); + + // initialize data for boundaries of the model and storage for boundary conditions and applied loads + void sgh_interface_setup(node_t& node, elem_t& elem, corner_t& corner); + + void setup(); + + void cleanup_material_models(); + + int solve(); + + void module_cleanup(); + + void sgh_solve(); + + /** + * \brief Calculate forces + * + * This function calculates the corner forces and the evolves stress + * according to the chosen material constitutive model + * + * \param material Array of material types + * \param mesh Simulation mesh + * \param node_coords View into nodal position array + * \param node_vel View into the nodal velocity array + * \param elem_den View into the element density array + * \param elem_sie View into the element specific internal energy array + * \param elem_pres View into the element pressure array + * \param elem_stress View into the element stress array + * \param elem_sspd View into the element sound speed array + * \param elem_vol View into the element volume array + * \param elem_div View into the element divergence array + * \param elem_mat_id View into the element material id array + * \param corner_force View into the corner forces + * \param rk_alpha Runge Kutta alpha value + * \param cycle Simulation cycle + * + */ + void get_force_sgh(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_pres, + DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + DViewCArrayKokkos& corner_force, + const double rk_alpha, + const size_t cycle); + + void get_force_vgradient_sgh(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_pres, + const DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + const double rk_alpha, + const size_t cycle); + + void get_force_ugradient_sgh(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_pres, + const DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + const double rk_alpha, + const size_t cycle); + + void get_force_egradient_sgh(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_pres, + const DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + const double rk_alpha, + const size_t cycle); + + void get_force_dgradient_sgh(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_pres, + const DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + const double rk_alpha, + const size_t cycle); + + void force_design_gradient_term(const_vec_array design_variables, vec_array design_gradients); + + void get_force_sgh2D(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_pres, + const DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + DViewCArrayKokkos& corner_force, + const double rk_alpha, + const size_t cycle); + + void update_position_sgh(double rk_alpha, + const size_t num_nodes, + DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel); + + void get_vol(); + + void get_vol_ugradient(const size_t gradient_node_id, const size_t gradient_dim); + + void init_assembly(); + + KOKKOS_INLINE_FUNCTION + void get_vol_hex(const DViewCArrayKokkos& elem_vol, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const; + + KOKKOS_FUNCTION + void get_vol_hex_ugradient(const ViewCArrayKokkos& elem_vol_gradients, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const; + + KOKKOS_INLINE_FUNCTION + void get_vol_quad(const DViewCArrayKokkos& elem_vol, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const; + + KOKKOS_FUNCTION + double get_area_quad(const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const; + + KOKKOS_FUNCTION + void get_bmatrix(const ViewCArrayKokkos& B_matrix, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const; + + KOKKOS_FUNCTION + void get_bmatrix_gradients(const ViewCArrayKokkos& B_matrix_gradients, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const; + + KOKKOS_FUNCTION + void get_bmatrix2D(const ViewCArrayKokkos& B_matrix, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const; + + KOKKOS_FUNCTION + void get_area_weights2D(const ViewCArrayKokkos& corner_areas, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const; + + KOKKOS_INLINE_FUNCTION + double heron(const double x1, + const double y1, + const double x2, + const double y2, + const double x3, + const double y3) const; + + double average_element_density(const int nodes_per_elem, const CArray current_element_densities) const; + + void get_divergence(DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_vol); + + void get_divergence2D(DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_vol); + + KOKKOS_FUNCTION + void get_velgrad(ViewCArrayKokkos& vel_grad, + const ViewCArrayKokkos& elem_node_gids, + const DViewCArrayKokkos& node_vel, + const ViewCArrayKokkos& b_matrix, + const double elem_vol, + const size_t elem_gid, + const size_t rk_level) const; + + KOKKOS_FUNCTION + void get_velgrad2D(ViewCArrayKokkos& vel_grad, + const ViewCArrayKokkos& elem_node_gids, + const DViewCArrayKokkos& node_vel, + const ViewCArrayKokkos& b_matrix, + const double elem_vol, + const double elem_area, + const size_t elem_gid, + const size_t rk_level) const; + + KOKKOS_INLINE_FUNCTION + void decompose_vel_grad(ViewCArrayKokkos& D_tensor, + ViewCArrayKokkos& W_tensor, + const ViewCArrayKokkos& vel_grad, + const ViewCArrayKokkos& elem_node_gids, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const double vol) const; + + void update_velocity_sgh(double rk_alpha, + DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_mass, + const DViewCArrayKokkos& corner_force); + + void tag_bdys(const DCArrayKokkos& boundary, + mesh_t& mesh, + const DViewCArrayKokkos& node_coords); + + void boundary_velocity(const mesh_t& mesh, + const DCArrayKokkos& boundary, + DViewCArrayKokkos& node_vel); + + KOKKOS_INLINE_FUNCTION + bool check_bdy(const size_t patch_gid, + const int num_dim, + const int num_nodes_in_patch, + const BOUNDARY_TYPE this_bc_tag, + const double val, + const DViewCArrayKokkos& node_coords, + const size_t rk_level) const; + + void rk_init(DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_sie, + DViewCArrayKokkos& elem_stress, + const size_t num_elems, + const size_t num_nodes); + + void get_timestep(mesh_t& mesh, + DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_sspd, + DViewCArrayKokkos& elem_vol); + + void get_timestep2D(mesh_t& mesh, + DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_sspd, + DViewCArrayKokkos& elem_vol); + + void update_energy_sgh(double rk_alpha, + const mesh_t& mesh, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& corner_force); + + void power_design_gradient_term(const_vec_array design_variables, vec_array design_gradients); + + void get_power_dgradient_sgh(double rk_alpha, + const mesh_t& mesh, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& corner_force, + DCArrayKokkos elem_power_dgradients); + + void get_power_ugradient_sgh(double rk_alpha, + const mesh_t& mesh, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& corner_force); + + void get_power_vgradient_sgh(double rk_alpha, + const mesh_t& mesh, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& corner_force); + + void get_power_egradient_sgh(double rk_alpha, + const mesh_t& mesh, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& corner_force); + + void update_state(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_den, + DViewCArrayKokkos& elem_pres, + DViewCArrayKokkos& elem_stress, + DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& elem_mat_id, + const double rk_alpha, + const size_t cycle); + + void update_state2D(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_den, + DViewCArrayKokkos& elem_pres, + DViewCArrayKokkos& elem_stress, + DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& elem_mat_id, + const double rk_alpha, + const size_t cycle); + + void build_boundry_node_sets(mesh_t& mesh); + + void init_boundaries(); + + // initializes memory for arrays used in the global stiffness matrix assembly + void init_boundary_sets(int num_boundary_sets); + + void grow_boundary_sets(int num_boundary_sets); + + virtual void update_forward_solve(Teuchos::RCP zp); + + void comm_node_masses(); + + void comm_adjoint_vector(int cycle); + + void comm_phi_adjoint_vector(int cycle); + + void comm_variables(Teuchos::RCP zp); + + void read_conditions_ansys_dat(std::ifstream* in, std::streampos before_condition_header); + + // interfaces between user input and creating data structures for bcs + void generate_bcs(); + + void Displacement_Boundary_Conditions(); + + void init_output(); + + void compute_output(); + + void output_control(); + + void sort_output(Teuchos::RCP> sorted_map); + + void sort_element_output(Teuchos::RCP> sorted_map); + + void collect_output(Teuchos::RCP> global_reduce_map); + + void write_data(std::map& point_data_scalars_double, + std::map& point_data_vectors_double, + std::map& cell_data_scalars_double, + std::map& cell_data_scalars_int, + std::map>& cell_data_fields_double); + + void write_outputs(const mesh_t& mesh, + DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& node_mass, + DViewCArrayKokkos& elem_den, + DViewCArrayKokkos& elem_pres, + DViewCArrayKokkos& elem_stress, + DViewCArrayKokkos& elem_sspd, + DViewCArrayKokkos& elem_sie, + DViewCArrayKokkos& elem_vol, + DViewCArrayKokkos& elem_mass, + DViewCArrayKokkos& elem_mat_id); + + void ensight(const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_mass, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_pres, + const DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& elem_mat_id); + + void state_file(const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_mass, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_pres, + const DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& elem_mat_id); + + void node_density_constraints(host_vec_array node_densities_lower_bound); + + void compute_topology_optimization_adjoint_full(); // Force depends on node coords and velocity + + void compute_topology_optimization_gradient_full(Teuchos::RCP design_densities_distributed, Teuchos::RCP design_gradients_distributed); + + void boundary_adjoint(const mesh_t& mesh, + const DCArrayKokkos& boundary, + vec_array& node_adjoint, + vec_array& node_phi_adjoint, + vec_array& node_psi_adjoint); + + void applied_forces(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_mass, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + DViewCArrayKokkos& corner_force, + const double rk_alpha, + const size_t cycle); + + bool have_loading_conditions; + bool nodal_density_flag; + real_t penalty_power; + + Simulation_Parameters_Explicit* simparam; + SGH_Parameters* module_params; + Explicit_Solver* Explicit_Solver_Pointer_; + + elements::ref_element* ref_elem; + + std::shared_ptr mesh; + // shallow copies of mesh class views + size_t num_nodes_in_elem; + // corner ids in node + RaggedRightArrayKokkos corners_in_node; + CArrayKokkos num_corners_in_node; + + // elem ids in node + RaggedRightArrayKokkos elems_in_node; + + // node ids in node + RaggedRightArrayKokkos nodes_in_node; + CArrayKokkos num_nodes_in_node; + + // node ids in elem + DCArrayKokkos nodes_in_elem; + + // corner ids in elem + CArrayKokkos corners_in_elem; + + // elem ids in elem + RaggedRightArrayKokkos elems_in_elem; + CArrayKokkos num_elems_in_elem; + + // patch ids in elem + CArrayKokkos patches_in_elem; + + // node ids in a patch + CArrayKokkos nodes_in_patch; + + // element ids in a patch + CArrayKokkos elems_in_patch; + + // bdy nodes + CArrayKokkos bdy_nodes; + + // Topology optimization filter variable + DCArrayKokkos relative_element_densities; + + // Local FEA data + host_elem_conn_array interface_nodes_in_elem; // host view of element connectivity to nodes + + // Global FEA data + Teuchos::RCP node_velocities_distributed; + Teuchos::RCP initial_node_coords_distributed; + Teuchos::RCP all_initial_node_coords_distributed; + Teuchos::RCP initial_node_velocities_distributed; + Teuchos::RCP all_node_velocities_distributed; + Teuchos::RCP all_cached_node_velocities_distributed; + Teuchos::RCP node_masses_distributed; + Teuchos::RCP ghost_node_masses_distributed; + Teuchos::RCP adjoint_vector_distributed; + Teuchos::RCP phi_adjoint_vector_distributed; + Teuchos::RCP psi_adjoint_vector_distributed; + Teuchos::RCP element_internal_energy_distributed; + Teuchos::RCP>> forward_solve_velocity_data; + Teuchos::RCP>> forward_solve_coordinate_data; + Teuchos::RCP>> forward_solve_internal_energy_data; + Teuchos::RCP>> adjoint_vector_data; + Teuchos::RCP>> phi_adjoint_vector_data; + Teuchos::RCP>> psi_adjoint_vector_data; + Teuchos::RCP force_gradient_design; + Teuchos::RCP force_gradient_position; + Teuchos::RCP force_gradient_velocity; + + // Local FEA data + DCArrayKokkos Global_Gradient_Matrix_Assembly_Map; // Maps element local nodes to columns on ragged right node connectivity graph + DCArrayKokkos Element_Gradient_Matrix_Assembly_Map; // Maps element-node pair to columns on ragged right node to element connectivity + RaggedRightArrayKokkos Graph_Matrix; // stores global indices + RaggedRightArrayKokkos DOF_Graph_Matrix; // stores global indices + RaggedRightArrayKokkos Force_Gradient_Positions; + RaggedRightArrayKokkos Force_Gradient_Velocities; + CArrayKokkos Force_Gradient_Energies; // transposed such that elem ids correspond to rows + RaggedRightArrayKokkos Power_Gradient_Positions; // transposed such that node dofs correspond to rows + RaggedRightArrayKokkos Power_Gradient_Velocities; // transposed such that node dofs correspond to rows + CArrayKokkos Power_Gradient_Energies; + DCArrayKokkos Gradient_Matrix_Strides; + DCArrayKokkos DOF_to_Elem_Matrix_Strides; + DCArrayKokkos Elem_to_Elem_Matrix_Strides; + DCArrayKokkos Graph_Matrix_Strides; + RaggedRightArrayKokkos Original_Gradient_Entries; + RaggedRightArrayKokkos Original_Gradient_Entry_Indices; + DCArrayKokkos Original_Gradient_Entries_Strides; + + // distributed matrices + Teuchos::RCP distributed_force_gradient_positions; + Teuchos::RCP distributed_force_gradient_velocities; + + std::vector time_data; + int max_time_steps, last_time_step; + + // --------------------------------------------------------------------- + // state data type declarations (must stay in scope for output after run) + // --------------------------------------------------------------------- + node_t node_interface; + elem_t elem_interface; + corner_t corner_interface; + + // Dual View wrappers + // Dual Views of the individual node struct variables + DViewCArrayKokkos node_coords; + DViewCArrayKokkos node_vel; + DViewCArrayKokkos node_mass; + + // Dual Views of the individual elem struct variables + DViewCArrayKokkos elem_den; + DViewCArrayKokkos elem_pres; + DViewCArrayKokkos elem_stress; // always 3D even in 2D-RZ + DViewCArrayKokkos elem_sspd; + DViewCArrayKokkos elem_sie; + DViewCArrayKokkos elem_vol; + DViewCArrayKokkos elem_div; + DViewCArrayKokkos elem_mass; + DViewCArrayKokkos elem_mat_id; + + // Element velocity gradient + DCArrayKokkos elem_vel_grad; + + // for storing global variables used in user material model + DCArrayKokkos global_vars; + + // for storing state variables used in user material model + DCArrayKokkos state_vars; + + // elem_user_output_vars allow users to output variables of interest per element + DCArrayKokkos elem_user_output_vars; + + // material models + DCArrayKokkos elem_eos; + DCArrayKokkos elem_strength; + + // Dual Views of the corner struct variables + DViewCArrayKokkos corner_force; + DViewCArrayKokkos corner_mass; + + // Boundary Conditions Data + DCArrayKokkos Local_Index_Boundary_Patches; + // CArray Patch_Nodes; + enum bc_type { NONE, POINT_LOADING_CONDITION, LINE_LOADING_CONDITION, SURFACE_LOADING_CONDITION }; + + // Boundary Conditions Data + int max_boundary_sets; + + // output dof data + // Global arrays with collected data used to print + int output_velocity_index, output_strain_index, output_stress_index; + + // parameters + double time_value, time_final, dt, dt_max, dt_min, dt_cfl, graphics_time, graphics_dt_ival; + size_t graphics_cyc_ival, cycle_stop, rk_num_stages, graphics_id; + double fuzz, tiny, small; + CArray graphics_times; + int rk_num_bins; + + // optimization flags + bool kinetic_energy_objective; +}; + +#endif // end HEADER_H diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/mesh.h b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/mesh.h similarity index 60% rename from src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/mesh.h rename to src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/mesh.h index 25111f2ed..d33862889 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/mesh.h +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/mesh.h @@ -1,7 +1,6 @@ #ifndef MESH_H #define MESH_H - #include "matar.h" #include "state.h" @@ -48,195 +47,181 @@ patch 6: [4,5,6,7] zeta-plus dir // sort in ascending order using bubble sort KOKKOS_INLINE_FUNCTION -void bubble_sort(size_t arr[], const size_t num){ - - for (size_t i=0; i<(num-1); i++){ - for (size_t j=0; j<(num-i-1); j++){ - - if (arr[j]>arr[j+1]){ +void bubble_sort(size_t arr[], const size_t num) +{ + for (size_t i = 0; i < (num - 1); i++) + { + for (size_t j = 0; j < (num - i - 1); j++) + { + if (arr[j] > arr[j + 1]) + { size_t temp = arr[j]; - arr[j] = arr[j+1]; - arr[j+1] = temp; + arr[j] = arr[j + 1]; + arr[j + 1] = temp; } // end if - } // end for j } // end for i } // end function - // mesh sizes and connectivity data structures -struct mesh_t { - +struct mesh_t +{ size_t num_dims; - + size_t num_nodes, num_local_nodes; - + size_t num_elems; size_t num_nodes_in_elem; size_t num_patches_in_elem; size_t num_corners; - + size_t num_patches; - + size_t num_bdy_patches; size_t num_bdy_nodes; size_t num_bdy_sets; size_t num_nodes_in_patch; - // ---- nodes ---- - + // corner ids in node - RaggedRightArrayKokkos corners_in_node; - CArrayKokkos num_corners_in_node; - + RaggedRightArrayKokkos corners_in_node; + CArrayKokkos num_corners_in_node; + // elem ids in node - RaggedRightArrayKokkos elems_in_node; - + RaggedRightArrayKokkos elems_in_node; + // node ids in node - RaggedRightArrayKokkos nodes_in_node; - CArrayKokkos num_nodes_in_node; - - + RaggedRightArrayKokkos nodes_in_node; + CArrayKokkos num_nodes_in_node; + // ---- elems ---- - + // node ids in elem - DCArrayKokkos nodes_in_elem; - + DCArrayKokkos nodes_in_elem; + // corner ids in elem - CArrayKokkos corners_in_elem; - + CArrayKokkos corners_in_elem; + // elem ids in elem - RaggedRightArrayKokkos elems_in_elem; - CArrayKokkos num_elems_in_elem; - + RaggedRightArrayKokkos elems_in_elem; + CArrayKokkos num_elems_in_elem; + // patch ids in elem - CArrayKokkos patches_in_elem; - - + CArrayKokkos patches_in_elem; + // ---- patches ---- - + // node ids in a patch - CArrayKokkos nodes_in_patch; - + CArrayKokkos nodes_in_patch; + // element ids in a patch - CArrayKokkos elems_in_patch; - - + CArrayKokkos elems_in_patch; + // ---- bdy ---- - + // bdy_patches - CArrayKokkos bdy_patches; - + CArrayKokkos bdy_patches; + // bdy nodes - CArrayKokkos bdy_nodes; - + CArrayKokkos bdy_nodes; + // patch ids in bdy set - DynamicRaggedRightArrayKokkos bdy_patches_in_set; - + DynamicRaggedRightArrayKokkos bdy_patches_in_set; + // node ids in bdy_patch set - RaggedRightArrayKokkos bdy_nodes_in_set; - DCArrayKokkos num_bdy_nodes_in_set; - + RaggedRightArrayKokkos bdy_nodes_in_set; + DCArrayKokkos num_bdy_nodes_in_set; + // initialization methods void initialize_nodes(const size_t num_nodes_inp) { num_nodes = num_nodes_inp; - + return; - }; // end method // initialization methods void initialize_local_nodes(const size_t num_nodes_inp) { num_local_nodes = num_nodes_inp; - + return; - }; // end method - - + // initialization methods void initialize_elems(const size_t num_elems_inp, const size_t num_dims_inp) { num_dims = num_dims_inp; num_nodes_in_elem = 1; - for (int dim=0; dim (num_elems, num_nodes_in_elem, "nodes_in_elem"); - corners_in_elem = CArrayKokkos (num_elems, num_nodes_in_elem, "corners_in_elem"); - + num_elems = num_elems_inp; + nodes_in_elem = DCArrayKokkos(num_elems, num_nodes_in_elem, "nodes_in_elem"); + corners_in_elem = CArrayKokkos(num_elems, num_nodes_in_elem, "corners_in_elem"); + return; - }; // end method - - + // initialization methods void initialize_corners(const size_t num_corners_inp) { num_corners = num_corners_inp; - + return; - }; // end method - - + // build the corner mesh connectivity arrays - void build_corner_connectivity(){ - - num_corners_in_node = CArrayKokkos (num_nodes); // stride sizes - + void build_corner_connectivity() + { + num_corners_in_node = CArrayKokkos(num_nodes); // stride sizes + // initializing the number of corners (node-cell pair) to be zero FOR_ALL_CLASS(node_gid, 0, num_nodes, { num_corners_in_node(node_gid) = 0; }); - - - for (size_t elem_gid = 0; elem_gid < num_elems; elem_gid++){ - FOR_ALL_CLASS(node_lid, 0, num_nodes_in_elem,{ - + + for (size_t elem_gid = 0; elem_gid < num_elems; elem_gid++) + { + FOR_ALL_CLASS(node_lid, 0, num_nodes_in_elem, { // get the global_id of the node size_t node_gid = nodes_in_elem(elem_gid, node_lid); // increment the number of corners attached to this point num_corners_in_node(node_gid) = num_corners_in_node(node_gid) + 1; - }); // end FOR_ALL over nodes in element } // end for elem_gid - - + // the stride sizes are the num_corners_in_node at the node - corners_in_node = RaggedRightArrayKokkos (num_corners_in_node, "corners_in_node"); + corners_in_node = RaggedRightArrayKokkos(num_corners_in_node, "corners_in_node"); - CArrayKokkos count_saved_corners_in_node(num_nodes, "count_saved_corners_in_node"); + CArrayKokkos count_saved_corners_in_node(num_nodes, "count_saved_corners_in_node"); // reset num_corners to zero FOR_ALL_CLASS(node_gid, 0, num_nodes, { count_saved_corners_in_node(node_gid) = 0; }); - - + // he elems_in_elem data type - elems_in_node = RaggedRightArrayKokkos (num_corners_in_node, "elems_in_node"); - + elems_in_node = RaggedRightArrayKokkos(num_corners_in_node, "elems_in_node"); + // populate the elems connected to a node list and corners in a node - for (size_t elem_gid = 0; elem_gid < num_elems; elem_gid++){ + for (size_t elem_gid = 0; elem_gid < num_elems; elem_gid++) + { FOR_ALL_CLASS(node_lid, 0, num_nodes_in_elem, { - // get the global_id of the node size_t node_gid = nodes_in_elem(elem_gid, node_lid); - + // the column index is the num corners saved size_t j = count_saved_corners_in_node(node_gid); // Save corner index to this node_gid - size_t corner_gid = node_lid + elem_gid*num_nodes_in_elem; + size_t corner_gid = node_lid + elem_gid * num_nodes_in_elem; corners_in_node(node_gid, j) = corner_gid; - + elems_in_node(node_gid, j) = elem_gid; // save the elem_gid // Save corner index to element @@ -245,133 +230,127 @@ struct mesh_t { // increment the number of corners saved to this node_gid count_saved_corners_in_node(node_gid) = count_saved_corners_in_node(node_gid) + 1; - }); // end FOR_ALL over nodes in element } // end for elem_gid - + return; - } // end of build_corner_connectivity - - + // build elem connectivity arrays - void build_elem_elem_connectivity(){ - + void build_elem_elem_connectivity() + { // find the max number of elems around a node size_t max_num_elems_in_node; size_t max_num_lcl; REDUCE_MAX_CLASS(node_gid, 0, num_nodes, max_num_lcl, { - // num_corners_in_node = num_elems_in_node size_t max_num = num_corners_in_node(node_gid); - - if (max_num > max_num_lcl) max_num_lcl = max_num; - + + if (max_num > max_num_lcl) + { + max_num_lcl = max_num; + } }, max_num_elems_in_node); // end parallel reduction on max Kokkos::fence(); - + // a temporary ragged array to save the elems around an elem - DynamicRaggedRightArrayKokkos temp_elems_in_elem(num_nodes, num_nodes_in_elem*max_num_elems_in_node, "temp_elems_in_elem"); - - num_elems_in_elem = CArrayKokkos (num_elems, "num_elems_in_elem"); + DynamicRaggedRightArrayKokkos temp_elems_in_elem(num_nodes, num_nodes_in_elem * max_num_elems_in_node, "temp_elems_in_elem"); + + num_elems_in_elem = CArrayKokkos(num_elems, "num_elems_in_elem"); FOR_ALL_CLASS(elem_gid, 0, num_elems, { num_elems_in_elem(elem_gid) = 0; }); Kokkos::fence(); - + // find and save neighboring elem_gids of an elem FOR_ALL_CLASS(elem_gid, 0, num_elems, { - for (int node_lid=0; node_lid (num_elems_in_elem, "elems_in_elem"); - + elems_in_elem = RaggedRightArrayKokkos(num_elems_in_elem, "elems_in_elem"); + FOR_ALL_CLASS(elem_gid, 0, num_elems, { - for (size_t i=0; i node_ordering_in_elem; // node lids in a patch - - num_nodes_in_patch = 2*(num_dims-1); // 2 (2D) or 4 (3D) - num_patches_in_elem = 2*num_dims; // 4 (2D) or 6 (3D) - + DViewCArrayKokkos node_ordering_in_elem; // node lids in a patch + + num_nodes_in_patch = 2 * (num_dims - 1); // 2 (2D) or 4 (3D) + num_patches_in_elem = 2 * num_dims; // 4 (2D) or 6 (3D) + size_t node_lids_in_patch_in_elem[24]; - - if(num_dims == 3) { - size_t temp_node_lids[24] = {0,4,7,3, - 1,2,6,5, - 0,1,5,4, - 2,3,7,6, - 0,3,2,1, - 4,5,6,7}; - - for (size_t i=0; i<24; i++){ + + if (num_dims == 3) + { + size_t temp_node_lids[24] = { 0, 4, 7, 3, + 1, 2, 6, 5, + 0, 1, 5, 4, + 2, 3, 7, 6, + 0, 3, 2, 1, + 4, 5, 6, 7 }; + + for (size_t i = 0; i < 24; i++) + { node_lids_in_patch_in_elem[i] = temp_node_lids[i]; } // end for i - } - else { + else + { // J // | // 3---2 @@ -379,447 +358,409 @@ struct mesh_t { // 0---1 // size_t temp_node_lids[8] = - {0,3, - 1,2, - 0,1, - 3,2}; - - for (size_t i=0; i<8; i++){ + { 0, 3, + 1, 2, + 0, 1, + 3, 2 }; + + for (size_t i = 0; i < 8; i++) + { node_lids_in_patch_in_elem[i] = temp_node_lids[i]; } // end for i - } // end if on dims - - node_ordering_in_elem = DViewCArrayKokkos (&node_lids_in_patch_in_elem[0],num_patches_in_elem,num_nodes_in_patch); - - + + node_ordering_in_elem = DViewCArrayKokkos(&node_lids_in_patch_in_elem[0], num_patches_in_elem, num_nodes_in_patch); + // for saviong the hash keys of the patches and then the nighboring elem_gid - CArrayKokkos hash_keys_in_elem (num_elems, num_patches_in_elem, "hash_keys_in_elem"); - + CArrayKokkos hash_keys_in_elem(num_elems, num_patches_in_elem, "hash_keys_in_elem"); + // for saving the adjacient patch_lid, which is the slide_lid - //CArrayKokkos neighboring_side_lids (num_elems, num_patches_in_elem); - + // CArrayKokkos neighboring_side_lids (num_elems, num_patches_in_elem); + // allocate memory for the patches in the elem - patches_in_elem = CArrayKokkos (num_elems, num_patches_in_elem, "patches_in_elem"); - + patches_in_elem = CArrayKokkos(num_elems, num_patches_in_elem, "patches_in_elem"); + // a temporary storaage for the patch_gids that are on the mesh boundary - CArrayKokkos temp_bdy_patches(num_elems*num_patches_in_elem, "temp_bdy_patches"); - + CArrayKokkos temp_bdy_patches(num_elems * num_patches_in_elem, "temp_bdy_patches"); + // step 1) calculate the hash values for each patch in the element FOR_ALL_CLASS(elem_gid, 0, num_elems, { - - for (size_t patch_lid = 0; patch_lid num_values(2, "num_values"); - + + DCArrayKokkos num_values(2, "num_values"); + // step 2: walk around the elements and save the elem pairs that have the same hash_key RUN_CLASS({ // serial execution on GPU - - size_t patch_gid = 0; + + size_t patch_gid = 0; size_t bdy_patch_gid = 0; - for (size_t elem_gid = 0; elem_gid (num_patches, 2, "elems_in_patch"); - nodes_in_patch = CArrayKokkos (num_patches, num_nodes_in_patch, "nodes_in_patch"); - + + elems_in_patch = CArrayKokkos(num_patches, 2, "elems_in_patch"); + nodes_in_patch = CArrayKokkos(num_patches, num_nodes_in_patch, "nodes_in_patch"); + // a temporary variable to help populate patch structures - CArrayKokkos num_elems_in_patch_saved (num_patches, "num_elems_in_patch_saved"); - + CArrayKokkos num_elems_in_patch_saved(num_patches, "num_elems_in_patch_saved"); + // initialize the number of elems in a patch saved to zero FOR_ALL_CLASS(patch_gid, 0, num_patches, { num_elems_in_patch_saved(patch_gid) = 0; }); - - for(size_t elem_gid=0; elem_gid (num_bdy_patches, "bdy_patches"); - + bdy_patches = CArrayKokkos(num_bdy_patches, "bdy_patches"); + FOR_ALL_CLASS(bdy_patch_gid, 0, num_bdy_patches, { bdy_patches(bdy_patch_gid) = temp_bdy_patches(bdy_patch_gid); }); // end FOR_ALL bdy_patch_gid - - - - + // find and store the boundary nodes - CArrayKokkos temp_bdy_nodes(num_nodes, "temp_bdy_nodes"); - CArrayKokkos hash_bdy_nodes(num_nodes, "hash_bdy_nodes"); - - FOR_ALL_CLASS (node_gid, 0, num_nodes, { + CArrayKokkos temp_bdy_nodes(num_nodes, "temp_bdy_nodes"); + CArrayKokkos hash_bdy_nodes(num_nodes, "hash_bdy_nodes"); + + FOR_ALL_CLASS(node_gid, 0, num_nodes, { hash_bdy_nodes(node_gid) = -1; }); // end for node_gid - + // Parallel loop over boundary patches - DCArrayKokkos num_bdy_nodes_saved(1); - + DCArrayKokkos num_bdy_nodes_saved(1); + RUN_CLASS({ num_bdy_nodes_saved(0) = 0; - for (size_t bdy_patch_gid=0; bdy_patch_gid (num_bdy_nodes, "bdy_nodes"); - - FOR_ALL_CLASS (node_gid, 0, num_bdy_nodes, { + + bdy_nodes = CArrayKokkos(num_bdy_nodes, "bdy_nodes"); + + FOR_ALL_CLASS(node_gid, 0, num_bdy_nodes, { bdy_nodes(node_gid) = temp_bdy_nodes(node_gid); }); // end for boundary node_gid - + printf("Num boundary nodes = %lu \n", num_bdy_nodes); - + return; - } // end patch connectivity method - - - - // build the patches - void build_node_node_connectivity(){ - - + + // build the node-node connectivity data + void build_node_node_connectivity() + { // find the max number of elems around a node size_t max_num_elems_in_node; size_t max_num_lcl; REDUCE_MAX_CLASS(node_gid, 0, num_nodes, max_num_lcl, { - // num_corners_in_node = num_elems_in_node size_t max_num = num_corners_in_node(node_gid); - - if (max_num > max_num_lcl) max_num_lcl = max_num; - + + if (max_num > max_num_lcl) + { + max_num_lcl = max_num; + } }, max_num_elems_in_node); // end parallel reduction on max Kokkos::fence(); - + // each elem corner will contribute 3 edges to the node. Those edges will likely be the same // ones from an adjacent element so it is a safe estimate to multiply by 3 - DynamicRaggedRightArrayKokkos temp_nodes_in_nodes(num_nodes, max_num_elems_in_node*3, "temp_nodes_in_nodes"); - - num_nodes_in_node = CArrayKokkos (num_nodes, "num_nodes_in_node"); - + DynamicRaggedRightArrayKokkos temp_nodes_in_nodes(num_nodes, max_num_elems_in_node * 3, "temp_nodes_in_nodes"); + + num_nodes_in_node = CArrayKokkos(num_nodes, "num_nodes_in_node"); + // walk over the patches and save the node node connectivity RUN_CLASS({ - if (num_dims==3){ - - for (size_t patch_gid=0; patch_gid (num_nodes_in_node, "nodes_in_node"); - + + nodes_in_node = RaggedRightArrayKokkos(num_nodes_in_node, "nodes_in_node"); + // save the connectivity FOR_ALL_CLASS(node_gid, 0, num_nodes, { - size_t num_saved = 0; - for (size_t node_lid=0; node_lid (num_bcs, num_bdy_patches, "bdy_patches_in_set"); - + num_bdy_sets = num_bcs; + bdy_patches_in_set = DynamicRaggedRightArrayKokkos(num_bcs, num_bdy_patches, "bdy_patches_in_set"); + return; - } // end of init_bdy_sets method - - - }; // end mesh_t -#endif \ No newline at end of file +#endif \ No newline at end of file diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/state.h b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/state.h new file mode 100644 index 000000000..8200848b6 --- /dev/null +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/state.h @@ -0,0 +1,80 @@ +#ifndef STATE_H +#define STATE_H + +#include "matar.h" + +using namespace mtr; + +// node_state +struct node_t +{ + // Position + DCArrayKokkos coords; + + // velocity + DCArrayKokkos vel; + + // mass at nodes + DCArrayKokkos mass; + + // Includes Ghost Positions + // DCArrayKokkos all_coords; + + // Includes Ghost velocities + // DCArrayKokkos all_vel; + + // Includes Ghost masses + // DCArrayKokkos all_mass; + + // initialization method (num_rk_storage_bins, num_nodes, num_dims) + void initialize(size_t num_rk, size_t num_nodes, size_t num_dims) + { + this->coords = DCArrayKokkos(num_rk, num_nodes, num_dims); + this->vel = DCArrayKokkos(num_rk, num_nodes, num_dims); + this->mass = DCArrayKokkos(num_nodes); + }; // end method +}; // end node_t + +// elem_state +struct elem_t +{ + CArray den; ///< element density + CArray pres; ///< element pressure + CArray stress; ///< element stress + CArray sspd; ///< element sound speed + CArray sie; ///< specific internal energy + CArray vol; ///< element volume + CArray div; ///< divergence of velocity + CArray mass; ///< element mass + CArray mat_id; ///< element material id + + // initialization method (num_rk_storage_bins, num_cells, num_dims) + void initialize(size_t num_rk, size_t num_elems, size_t num_dims) + { + this->den = CArray(num_elems); + this->pres = CArray(num_elems); + this->stress = CArray(num_rk, num_elems, num_dims, num_dims); + this->sspd = CArray(num_elems); + this->sie = CArray(num_rk, num_elems); + this->vol = CArray(num_elems); + this->div = CArray(num_elems); + this->mass = CArray(num_elems); + this->mat_id = CArray(num_elems); + }; // end method +}; // end elem_t + +// corner_state +struct corner_t +{ + CArray force; ///< Force acting on a corner + CArray mass; ///< Partitioned mass of the corner + + // initialization method (num_corners, num_dims) + void initialize(size_t num_corners, size_t num_dims) + { + this->force = CArray(num_corners, num_dims); + this->mass = CArray(num_corners); + }; // end method +}; // end corner_t + +#endif diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/momentum.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/momentum.cpp deleted file mode 100644 index 4f4ecd9fe..000000000 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/momentum.cpp +++ /dev/null @@ -1,417 +0,0 @@ - -#include "state.h" -#include "FEA_Module_SGH.h" - -// ----------------------------------------------------------------------------- -// This function evolves the velocity at the nodes of the mesh -//------------------------------------------------------------------------------ -void FEA_Module_SGH::update_velocity_sgh(double rk_alpha, - DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &node_mass, - const DViewCArrayKokkos &corner_force - ){ - - const size_t rk_level = rk_num_bins - 1; - const size_t num_dims = num_dim; - - // walk over the nodes to update the velocity - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - - double node_force[3]; - for (size_t dim = 0; dim < num_dims; dim++){ - node_force[dim] = 0.0; - } // end for dim - - // loop over all corners around the node and calculate the nodal force - for (size_t corner_lid=0; corner_lid &vel_grad, - const ViewCArrayKokkos &elem_node_gids, - const DViewCArrayKokkos &node_vel, - const ViewCArrayKokkos &b_matrix, - const double elem_vol, - const size_t elem_gid, - const size_t rk_level - ) const { - - const size_t num_nodes_in_elem = 8; - - double u_array[num_nodes_in_elem]; - double v_array[num_nodes_in_elem]; - double w_array[num_nodes_in_elem]; - ViewCArrayKokkos u(u_array, num_nodes_in_elem); // x-dir vel component - ViewCArrayKokkos v(v_array, num_nodes_in_elem); // y-dir vel component - ViewCArrayKokkos w(w_array, num_nodes_in_elem); // z-dir vel component - - // get the vertex velocities for the cell - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - - // Get node gid - size_t node_gid = elem_node_gids(node_lid); - - u(node_lid) = node_vel(rk_level, node_gid, 0); - v(node_lid) = node_vel(rk_level, node_gid, 1); - w(node_lid) = node_vel(rk_level, node_gid, 2); - - } // end for - - - // --- calculate the velocity gradient terms --- - double inverse_vol = 1.0/elem_vol; - - // x-dir - vel_grad(0,0) = (u(0)*b_matrix(0,0) + u(1)*b_matrix(1,0) - + u(2)*b_matrix(2,0) + u(3)*b_matrix(3,0) - + u(4)*b_matrix(4,0) + u(5)*b_matrix(5,0) - + u(6)*b_matrix(6,0) + u(7)*b_matrix(7,0))*inverse_vol; - - vel_grad(0,1) = (u(0)*b_matrix(0,1) + u(1)*b_matrix(1,1) - + u(2)*b_matrix(2,1) + u(3)*b_matrix(3,1) - + u(4)*b_matrix(4,1) + u(5)*b_matrix(5,1) - + u(6)*b_matrix(6,1) + u(7)*b_matrix(7,1))*inverse_vol; - - vel_grad(0,2) = (u(0)*b_matrix(0,2) + u(1)*b_matrix(1,2) - + u(2)*b_matrix(2,2) + u(3)*b_matrix(3,2) - + u(4)*b_matrix(4,2) + u(5)*b_matrix(5,2) - + u(6)*b_matrix(6,2) + u(7)*b_matrix(7,2))*inverse_vol; - - // y-dir - vel_grad(1,0) = (v(0)*b_matrix(0,0) + v(1)*b_matrix(1,0) - + v(2)*b_matrix(2,0) + v(3)*b_matrix(3,0) - + v(4)*b_matrix(4,0) + v(5)*b_matrix(5,0) - + v(6)*b_matrix(6,0) + v(7)*b_matrix(7,0))*inverse_vol; - - vel_grad(1,1) = (v(0)*b_matrix(0,1) + v(1)*b_matrix(1,1) - + v(2)*b_matrix(2,1) + v(3)*b_matrix(3,1) - + v(4)*b_matrix(4,1) + v(5)*b_matrix(5,1) - + v(6)*b_matrix(6,1) + v(7)*b_matrix(7,1))*inverse_vol; - vel_grad(1,2) = (v(0)*b_matrix(0,2) + v(1)*b_matrix(1,2) - + v(2)*b_matrix(2,2) + v(3)*b_matrix(3,2) - + v(4)*b_matrix(4,2) + v(5)*b_matrix(5,2) - + v(6)*b_matrix(6,2) + v(7)*b_matrix(7,2))*inverse_vol; - - // z-dir - vel_grad(2,0) = (w(0)*b_matrix(0,0) + w(1)*b_matrix(1,0) - + w(2)*b_matrix(2,0) + w(3)*b_matrix(3,0) - + w(4)*b_matrix(4,0) + w(5)*b_matrix(5,0) - + w(6)*b_matrix(6,0) + w(7)*b_matrix(7,0))*inverse_vol; - - vel_grad(2,1) = (w(0)*b_matrix(0,1) + w(1)*b_matrix(1,1) - + w(2)*b_matrix(2,1) + w(3)*b_matrix(3,1) - + w(4)*b_matrix(4,1) + w(5)*b_matrix(5,1) - + w(6)*b_matrix(6,1) + w(7)*b_matrix(7,1))*inverse_vol; - - vel_grad(2,2) = (w(0)*b_matrix(0,2) + w(1)*b_matrix(1,2) - + w(2)*b_matrix(2,2) + w(3)*b_matrix(3,2) - + w(4)*b_matrix(4,2) + w(5)*b_matrix(5,2) - + w(6)*b_matrix(6,2) + w(7)*b_matrix(7,2))*inverse_vol; - - - return; - -} // end function - - -// ----------------------------------------------------------------------------- -// This function calculates the velocity gradient -//------------------------------------------------------------------------------ -KOKKOS_FUNCTION -void FEA_Module_SGH::get_velgrad2D(ViewCArrayKokkos &vel_grad, - const ViewCArrayKokkos &elem_node_gids, - const DViewCArrayKokkos &node_vel, - const ViewCArrayKokkos &b_matrix, - const double elem_vol, - const double elem_area, - const size_t elem_gid, - const size_t rk_level - ) const { - - const size_t num_nodes_in_elem = 4; - - double u_array[num_nodes_in_elem]; - double v_array[num_nodes_in_elem]; - ViewCArrayKokkos u(u_array, num_nodes_in_elem); // x-dir vel component - ViewCArrayKokkos v(v_array, num_nodes_in_elem); // y-dir vel component - - // get the vertex velocities for the cell - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - - // Get node gid - size_t node_gid = elem_node_gids(node_lid); - - u(node_lid) = node_vel(rk_level, node_gid, 0); // x-comp - v(node_lid) = node_vel(rk_level, node_gid, 1); // y-comp - - } // end for - - // initialize to zero - for (size_t i=0; i<3; i++){ - for (size_t j=0; j<3; j++){ - vel_grad(i,j) = 0.0; - } - } - - - double mean_radius = elem_vol/elem_area; - double elem_vel_r = 0.25*(v(0) + v(1) + v(2) + v(3)); - - - // --- calculate the velocity gradient terms --- - double inverse_area = 1.0/elem_area; - - // x-dir - vel_grad(0,0) = (u(0)*b_matrix(0,0) + u(1)*b_matrix(1,0) - + u(2)*b_matrix(2,0) + u(3)*b_matrix(3,0))*inverse_area; - - vel_grad(0,1) = (u(0)*b_matrix(0,1) + u(1)*b_matrix(1,1) - + u(2)*b_matrix(2,1) + u(3)*b_matrix(3,1))*inverse_area; - - - // y-dir - vel_grad(1,0) = (v(0)*b_matrix(0,0) + v(1)*b_matrix(1,0) - + v(2)*b_matrix(2,0) + v(3)*b_matrix(3,0))*inverse_area; - - vel_grad(1,1) = (v(0)*b_matrix(0,1) + v(1)*b_matrix(1,1) - + v(2)*b_matrix(2,1) + v(3)*b_matrix(3,1))*inverse_area; - - vel_grad(2,2) = elem_vel_r/mean_radius; // + avg(vel_R)/R - - return; - -} // end function - - - - -// ----------------------------------------------------------------------------- -// This subroutine to calculate the velocity divergence in all elements -//------------------------------------------------------------------------------ -void FEA_Module_SGH::get_divergence(DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_vol - ){ - - const size_t rk_level = rk_num_bins - 1; - - // --- calculate the forces acting on the nodes from the element --- - FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { - - const size_t num_nodes_in_elem = 8; - const size_t num_dims = 3; - - double u_array[num_nodes_in_elem]; - double v_array[num_nodes_in_elem]; - double w_array[num_nodes_in_elem]; - ViewCArrayKokkos u(u_array, num_nodes_in_elem); // x-dir vel component - ViewCArrayKokkos v(v_array, num_nodes_in_elem); // y-dir vel component - ViewCArrayKokkos w(w_array, num_nodes_in_elem); // z-dir vel component - - // cut out the node_gids for this element - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 8); - - // The b_matrix are the outward corner area normals - double b_matrix_array[24]; - ViewCArrayKokkos b_matrix(b_matrix_array, num_nodes_in_elem, num_dims); - get_bmatrix(b_matrix, - elem_gid, - node_coords, - elem_node_gids, - rk_level); - - // get the vertex velocities for the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - - // Get node gid - size_t node_gid = elem_node_gids(node_lid); - - u(node_lid) = node_vel(rk_level, node_gid, 0); - v(node_lid) = node_vel(rk_level, node_gid, 1); - w(node_lid) = node_vel(rk_level, node_gid, 2); - - } // end for - - - // --- calculate the velocity divergence terms --- - double inverse_vol = 1.0/elem_vol(elem_gid); - - elem_div(elem_gid) = 0.0; - - // x-dir - elem_div(elem_gid) += (u(0)*b_matrix(0,0) + u(1)*b_matrix(1,0) - + u(2)*b_matrix(2,0) + u(3)*b_matrix(3,0) - + u(4)*b_matrix(4,0) + u(5)*b_matrix(5,0) - + u(6)*b_matrix(6,0) + u(7)*b_matrix(7,0))*inverse_vol; - - // y-dir - elem_div(elem_gid) += (v(0)*b_matrix(0,1) + v(1)*b_matrix(1,1) - + v(2)*b_matrix(2,1) + v(3)*b_matrix(3,1) - + v(4)*b_matrix(4,1) + v(5)*b_matrix(5,1) - + v(6)*b_matrix(6,1) + v(7)*b_matrix(7,1))*inverse_vol; - - // z-dir - elem_div(elem_gid) += (w(0)*b_matrix(0,2) + w(1)*b_matrix(1,2) - + w(2)*b_matrix(2,2) + w(3)*b_matrix(3,2) - + w(4)*b_matrix(4,2) + w(5)*b_matrix(5,2) - + w(6)*b_matrix(6,2) + w(7)*b_matrix(7,2))*inverse_vol; - - }); // end parallel for over elem_gid - - return; - -} // end subroutine - - -// ----------------------------------------------------------------------------- -// This subroutine to calculate the velocity divergence in all elements -//------------------------------------------------------------------------------ -void FEA_Module_SGH::get_divergence2D(DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_vol - ){ - - const size_t rk_level = rk_num_bins - 1; - - // --- calculate the forces acting on the nodes from the element --- - FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { - - const size_t num_nodes_in_elem = 4; - const size_t num_dims = 2; - - double u_array[num_nodes_in_elem]; - double v_array[num_nodes_in_elem]; - ViewCArrayKokkos u(u_array, num_nodes_in_elem); // x-dir vel component - ViewCArrayKokkos v(v_array, num_nodes_in_elem); // y-dir vel component - - - // true volume RZ - //double r_array[num_nodes_in_elem]; - //ViewCArrayKokkos r(r_array, num_nodes_in_elem); // r-dir coordinate - - - // cut out the node_gids for this element - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 4); - - // The b_matrix are the outward corner area normals - double b_matrix_array[24]; - ViewCArrayKokkos b_matrix(b_matrix_array, num_nodes_in_elem, num_dims); - get_bmatrix2D(b_matrix, - elem_gid, - node_coords, - elem_node_gids, - rk_level); - - // calculate the area of the quad - double elem_area = get_area_quad(elem_gid, node_coords, elem_node_gids, rk_level); - // true volume uses the elem_vol - - - // get the vertex velocities and node coordinate for the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - - // Get node gid - size_t node_gid = elem_node_gids(node_lid); - - u(node_lid) = node_vel(rk_level, node_gid, 0); - v(node_lid) = node_vel(rk_level, node_gid, 1); - - //r(node_lid) = node_coords(rk_level, node_gid, 1); // true volume RZ - - } // end for - - - // --- calculate the velocity divergence terms --- - double inverse_area = 1.0/elem_area; - - double mean_radius = elem_vol(elem_gid)/elem_area; - double elem_vel_r = 0.25*(v(0) + v(1) + v(2) + v(3)); - - - elem_div(elem_gid) = 0.0; - - // x-dir - elem_div(elem_gid) += (u(0)*b_matrix(0,0) - + u(1)*b_matrix(1,0) - + u(2)*b_matrix(2,0) - + u(3)*b_matrix(3,0))*inverse_area; - - // y-dir (i.e., r direction) - elem_div(elem_gid) += (v(0)*b_matrix(0,1) - + v(1)*b_matrix(1,1) - + v(2)*b_matrix(2,1) - + v(3)*b_matrix(3,1))*inverse_area - + elem_vel_r/mean_radius; // + avg(u_R)/R - - }); // end parallel for over elem_gid - - return; - -} // end subroutine - - -// The velocity gradient can be decomposed into symmetric and antisymmetric tensors -// L = vel_grad -// D = sym(L) -// W = antisym(L) -KOKKOS_INLINE_FUNCTION -void FEA_Module_SGH::decompose_vel_grad(ViewCArrayKokkos &D_tensor, - ViewCArrayKokkos &W_tensor, - const ViewCArrayKokkos &vel_grad, - const ViewCArrayKokkos &elem_node_gids, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const double vol - ) const { - - - // --- Calculate the velocity gradient --- - - const size_t num_dims = 3; - - // initialize to zero - for(size_t i=0; inum_bdy_nodes; - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos material = simparam->material; - const int num_dim = simparam->num_dims; - int num_corners = rnum_elem*num_nodes_in_elem; - real_t global_dt; - bool element_constant_density = true; - size_t current_data_index, next_data_index; - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - CArrayKokkos current_element_adjoint = CArrayKokkos(num_nodes_in_elem,num_dim); - DCArrayKokkos elem_power_dgradients(rnum_elem); - //gradient contribution from gradient of Force vector with respect to design variable. - if(simparam->dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ - if(myrank==0){ - std::cout << "gradient term involving adjoint derivative" << std::endl; - } - } - - for (unsigned long cycle = 0; cycle < last_time_step+1; cycle++) { - //compute timestep from time data - global_dt = time_data[cycle+1] - time_data[cycle]; - //print - if(simparam->dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ - if (cycle==0){ - if(myrank==0) - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } - // print time step every 10 cycles - else if (cycle%20==0){ - if(myrank==0) - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } // end if - } - - - //view scope - { - const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_element_internal_energy = (*forward_solve_internal_energy_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_coord_vector = (*forward_solve_coordinate_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_psi_adjoint_vector = (*psi_adjoint_vector_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array next_velocity_vector = (*forward_solve_velocity_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array next_element_internal_energy = (*forward_solve_internal_energy_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array next_coord_vector = (*forward_solve_coordinate_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array next_psi_adjoint_vector = (*psi_adjoint_vector_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - - //first half of integration step calculation - FOR_ALL_CLASS(node_gid, 0, nall_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - node_vel(rk_level,node_gid,idim) = current_velocity_vector(node_gid,idim); - node_coords(rk_level,node_gid,idim) = current_coord_vector(node_gid,idim); - } - - }); // end parallel for - Kokkos::fence(); - - FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { - elem_sie(rk_level,elem_gid) = current_element_internal_energy(elem_gid,0); - }); // end parallel for - Kokkos::fence(); - - get_vol(); - - // ---- Calculate velocity diveregence for the element ---- - if(num_dim==2){ - get_divergence2D(elem_div, - node_coords, - node_vel, - elem_vol); - } - else { - get_divergence(elem_div, - node_coords, - node_vel, - elem_vol); - } // end if 2D - - // ---- Calculate elem state (den, pres, sound speed, stress) for next time step ---- - if(num_dim==2){ - update_state2D(material, - *mesh, - node_coords, - node_vel, - elem_den, - elem_pres, - elem_stress, - elem_sspd, - elem_sie, - elem_vol, - elem_mass, - elem_mat_id, - 1.0, - cycle); - } - else{ - update_state(material, - *mesh, - node_coords, - node_vel, - elem_den, - elem_pres, - elem_stress, - elem_sspd, - elem_sie, - elem_vol, - elem_mass, - elem_mat_id, - 1.0, - cycle); - } - - // ---- calculate the forces on the vertices and evolve stress (hypo model) ---- - if(num_dim==2){ - get_force_sgh2D(material, - *mesh, - node_coords, - node_vel, - elem_den, - elem_sie, - elem_pres, - elem_stress, - elem_sspd, - elem_vol, - elem_div, - elem_mat_id, - corner_force, - 1.0, - cycle); - } - else { - get_force_sgh(material, - *mesh, - node_coords, - node_vel, - elem_den, - elem_sie, - elem_pres, - elem_stress, - elem_sspd, - elem_vol, - elem_div, - elem_mat_id, - corner_force, - 1.0, - cycle); - } - - get_force_dgradient_sgh(material, - *mesh, - node_coords, - node_vel, - elem_den, - elem_sie, - elem_pres, - elem_stress, - elem_sspd, - elem_vol, - elem_div, - elem_mat_id, - 1.0, - cycle); - - get_power_dgradient_sgh(1.0, - *mesh, - node_vel, - node_coords, - elem_sie, - elem_mass, - corner_force, - elem_power_dgradients); - - //derivatives of forces at corners stored in corner_vector_storage buffer by previous routine - FOR_ALL_CLASS(elem_id, 0, rnum_elem, { - size_t node_id; - size_t corner_id; - real_t inner_product; - - inner_product = current_psi_adjoint_vector(elem_id,0)*elem_power_dgradients(elem_id); - - for (int inode = 0; inode < num_nodes_in_elem; inode++){ - //compute gradient of local element contribution to v^t*M*v product - corner_id = elem_id*num_nodes_in_elem + inode; - corner_value_storage(corner_id) = inner_product; - } - - }); // end parallel for - Kokkos::fence(); - - //accumulate node values from corner storage - //multiply - FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { - size_t corner_id; - for(int icorner=0; icorner < num_corners_in_node(node_id); icorner++){ - corner_id = corners_in_node(node_id,icorner); - design_gradients(node_id,0) += -corner_value_storage(corner_id)*global_dt; - } - }); // end parallel for - Kokkos::fence(); - - } //end view scope - } - -} - -// --------------------------------------------------------------------------------------- -// This function calculates the gradient for element power with respect to design variable -//---------------------------------------------------------------------------------------- - -void FEA_Module_SGH::get_power_dgradient_sgh(double rk_alpha, - const mesh_t &mesh, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_mass, - const DViewCArrayKokkos &corner_force, - DCArrayKokkos elem_power_dgradients){ - - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - int num_dims = simparam->num_dims; - - // loop over all the elements in the mesh - FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { - - double elem_power = 0.0; - elem_power_dgradients(elem_gid) = 0; - - // --- tally the contribution from each corner to the element --- - - // Loop over the nodes in the element - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - - size_t corner_lid = node_lid; - - // Get node global id for the local node id - size_t node_gid = nodes_in_elem(elem_gid, node_lid); - - // Get the corner global id for the local corner id - size_t corner_gid = corners_in_elem(elem_gid, corner_lid); - - double node_radius = 1; - if(num_dims==2){ - node_radius = node_coords(rk_level,node_gid,1); - } - - // calculate the Power=F dot V for this corner - for (size_t dim=0; dim &node_vel, - const DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_mass, - const DViewCArrayKokkos &corner_force){ - - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - int num_dims = simparam->num_dims; - - //initialize gradient matrix - FOR_ALL_CLASS(dof_gid, 0, nlocal_nodes*num_dims, { - for(int idof = 0; idof < DOF_to_Elem_Matrix_Strides(dof_gid); idof++){ - Power_Gradient_Positions(dof_gid,idof) = 0; - } - }); // end parallel for loop over nodes - Kokkos::fence(); - - // loop over all the elements in the mesh - for (size_t elem_gid = 0; elem_gid < rnum_elem; elem_gid++){ - //FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { - - double elem_power = 0.0; - - // --- tally the contribution from each corner to the element --- - - // Loop over the nodes in the element - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - - size_t corner_lid = node_lid; - size_t column_id; - size_t gradient_node_id; - // Get node global id for the local node id - size_t node_gid = nodes_in_elem(elem_gid, node_lid); - - // Get the corner global id for the local corner id - size_t corner_gid = corners_in_elem(elem_gid, corner_lid); - - double node_radius = 1; - if(num_dims==2){ - node_radius = node_coords(rk_level,node_gid,1); - } - - // calculate the Power=F dot V for this corner - for (size_t dim=0; dimisNodeLocalElement(gradient_node_id)) continue; - Power_Gradient_Positions(gradient_node_id*num_dims+jdim, column_id) -= corner_gradient_storage(corner_gid,dim,igradient,jdim)*node_vel(rk_level, node_gid, dim)*node_radius; - } - } - } // end for dim - - } // end for node_lid - - //}); // end parallel loop over the elements - } - - return; -} // end subroutine - -// ----------------------------------------------------------------------------- -// This function calculates the gradient for element power with respect to velocity -//------------------------------------------------------------------------------ - -void FEA_Module_SGH::get_power_vgradient_sgh(double rk_alpha, - const mesh_t &mesh, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_mass, - const DViewCArrayKokkos &corner_force){ - - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - int num_dims = simparam->num_dims; - - //initialize gradient matrix - FOR_ALL_CLASS(dof_gid, 0, nlocal_nodes*num_dims, { - for(int idof = 0; idof < DOF_to_Elem_Matrix_Strides(dof_gid); idof++){ - Power_Gradient_Velocities(dof_gid,idof) = 0; - } - }); // end parallel for loop over nodes - Kokkos::fence(); - - // loop over all the elements in the mesh - for (size_t elem_gid = 0; elem_gid < rnum_elem; elem_gid++){ - //FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { - - double elem_power = 0.0; - - // --- tally the contribution from each corner to the element --- - - // Loop over the nodes in the element - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - - size_t corner_lid = node_lid; - size_t column_id; - size_t gradient_node_id; - // Get node global id for the local node id - size_t node_gid = nodes_in_elem(elem_gid, node_lid); - - // Get the corner global id for the local corner id - size_t corner_gid = corners_in_elem(elem_gid, corner_lid); - - double node_radius = 1; - if(num_dims==2){ - node_radius = node_coords(rk_level,node_gid,1); - } - - // calculate the Power=F dot V for this corner - for (size_t dim=0; dimisNodeLocalElement(gradient_node_id)) continue; - if(node_lid==igradient){ - Power_Gradient_Velocities(gradient_node_id*num_dims+dim, column_id) -= corner_gradient_storage(corner_gid,dim,igradient,dim)*node_vel(rk_level, node_gid, dim)*node_radius+ - corner_force(corner_gid, dim)*node_radius; - } - else{ - Power_Gradient_Velocities(gradient_node_id*num_dims+dim, column_id) -= corner_gradient_storage(corner_gid,dim,igradient,dim)*node_vel(rk_level, node_gid, dim)*node_radius; - } - } - } // end for dim - - } // end for node_lid - - //}); // end parallel loop over the elements - } - - return; -} // end subroutine - -// ----------------------------------------------------------------------------- -// This function calculates the gradient for element power with respect to energy -//------------------------------------------------------------------------------ - -void FEA_Module_SGH::get_power_egradient_sgh(double rk_alpha, - const mesh_t &mesh, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_mass, - const DViewCArrayKokkos &corner_force){ - - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - int num_dims = simparam->num_dims; - - //initialize gradient storage - FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { - Power_Gradient_Energies(elem_gid) = 0; - }); // end parallel loop over the elements - - // loop over all the elements in the mesh - FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { - - double elem_power = 0.0; - - // --- tally the contribution from each corner to the element --- - - // Loop over the nodes in the element - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - - size_t corner_lid = node_lid; - - // Get node global id for the local node id - size_t node_gid = nodes_in_elem(elem_gid, node_lid); - - // Get the corner global id for the local corner id - size_t corner_gid = corners_in_elem(elem_gid, corner_lid); - - double node_radius = 1; - if(num_dims==2){ - node_radius = node_coords(rk_level,node_gid,1); - } - - // calculate the Power=F dot V for this corner - for (size_t dim=0; dim -#include -#include -#include -#include -#include // fmin, fmax, abs note: fminl is long -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include "Tpetra_Details_makeColMap.hpp" -#include "Tpetra_Details_DefaultTypes.hpp" -#include "Tpetra_Details_FixedHashTable.hpp" -#include "Tpetra_Import.hpp" -#include "Tpetra_Import_Util2.hpp" - -#include "elements.h" -#include "swage.h" -#include "matar.h" -#include "utilities.h" -#include "node_combination.h" -#include "FEA_Module_SGH.h" -#include "Explicit_Solver.h" -#include "Simulation_Parameters/Simulation_Parameters_Explicit.h" -#include "Simulation_Parameters/FEA_Module/SGH_Parameters.h" - -//optimization -#include "ROL_Solver.hpp" -#include "Kinetic_Energy_Minimize.h" - -/* ---------------------------------------------------------------------- - Compute new system response due to the design variable update -------------------------------------------------------------------------- */ - -void FEA_Module_SGH::update_forward_solve(Teuchos::RCP zp){ - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - //local variable for host view in the dual view - int num_dim = simparam->num_dims; - int nodes_per_elem = max_nodes_per_element; - int local_node_index, current_row, current_column; - int max_stride = 0; - int current_module_index; - size_t access_index, row_access_index, row_counter; - GO global_index, global_dof_index; - LO local_dof_index; - const size_t num_fills = simparam->regions.size(); - const size_t rk_num_bins = simparam->dynamic_options.rk_num_bins; - const size_t num_bcs = module_params->boundary_conditions.size(); - const size_t num_materials = simparam->materials.size(); - real_t objective_accumulation; - - // --- Read in the nodes in the mesh --- - int myrank = Explicit_Solver_Pointer_->myrank; - int nranks = Explicit_Solver_Pointer_->nranks; - - const DCArrayKokkos mat_fill = simparam->mat_fill; - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos material = simparam->material; - CArray current_element_nodal_densities = CArray(num_nodes_in_elem); - - std::vector> FEA_Module_My_TO_Modules = simparam->FEA_Module_My_TO_Modules; - problem = Explicit_Solver_Pointer_->problem; //Pointer to ROL optimization problem object - ROL::Ptr> obj_pointer; - - //compute element averaged density ratios corresponding to nodal density design variables - {//view scope - const_host_vec_array all_node_densities = all_node_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); - //debug print - //std::cout << "NODE DENSITY TEST " << all_node_densities(0,0) << std::endl; - for(int elem_id = 0; elem_id < rnum_elem; elem_id++){ - for(int inode = 0; inode < num_nodes_in_elem; inode++){ - current_element_nodal_densities(inode) = all_node_densities(nodes_in_elem(elem_id,inode),0); - } - relative_element_densities.host(elem_id) = average_element_density(num_nodes_in_elem, current_element_nodal_densities); - }//for - } //view scope - //debug print - //std::cout << "ELEMENT RELATIVE DENSITY TEST " << relative_element_densities.host(0) << std::endl; - relative_element_densities.update_device(); - - //set density vector to the current value chosen by the optimizer - test_node_densities_distributed = zp; - - //reset nodal coordinates to initial values - node_coords_distributed->assign(*initial_node_coords_distributed); - - //comms for ghosts - Explicit_Solver_Pointer_->comm_coordinates(); - - //view scope - { - const_vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array ghost_node_coords_interface = Explicit_Solver_Pointer_->ghost_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - all_node_coords_interface(node_gid,idim) = node_coords_interface(node_gid,idim); - } - }); // end parallel for - Kokkos::fence(); - - FOR_ALL_CLASS(node_gid, nlocal_nodes, nlocal_nodes+nghost_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - all_node_coords_interface(node_gid,idim) = ghost_node_coords_interface(node_gid-nlocal_nodes,idim); - } - }); // end parallel for - Kokkos::fence(); - } //end view scope - - //reset velocities to initial conditions - node_velocities_distributed->assign(*initial_node_velocities_distributed); - - //reset time accumulating objective and constraints - /* - for(int imodule = 0 ; imodule < FEA_Module_My_TO_Modules[my_fea_module_index_].size(); imodule++){ - current_module_index = FEA_Module_My_TO_Modules[my_fea_module_index_][imodule]; - //test if module needs reset - if(){ - - } - } - */ - //simple setup to just request KE for now; above loop to be expanded and used later for scanning modules - obj_pointer = problem->getObjective(); - KineticEnergyMinimize_TopOpt& kinetic_energy_minimize_function = dynamic_cast(*obj_pointer); - kinetic_energy_minimize_function.objective_accumulation = 0; - - //interface trial density vector - - //interfacing of vectors(should be removed later once made compatible) - //view scope - { - host_vec_array interface_node_coords = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); - for(size_t ibin = 0; ibin < rk_num_bins; ibin++){ - //save node data to node.coords - //std::cout << "NODE DATA ON RANK " << myrank << std::endl; - if(num_dim==2){ - for(int inode = 0; inode < nall_nodes; inode++){ - //std::cout << "Node index " << inode+1 << " "; - node_coords.host(ibin,inode,0) = interface_node_coords(inode,0); - //std::cout << host_node_coords_state(0,inode,0)+1<< " "; - node_coords.host(ibin,inode,1) = interface_node_coords(inode,1); - //std::cout << host_node_coords_state(0,inode,1)+1<< " "; - } - } - else if(num_dim==3){ - for(int inode = 0; inode < nall_nodes; inode++){ - //std::cout << "Node index " << inode+1 << " "; - node_coords.host(ibin,inode,0) = interface_node_coords(inode,0); - //std::cout << host_node_coords_state(0,inode,0)+1<< " "; - node_coords.host(ibin,inode,1) = interface_node_coords(inode,1); - //std::cout << host_node_coords_state(0,inode,1)+1<< " "; - - node_coords.host(ibin,inode,2) = interface_node_coords(inode,2); - //std::cout << host_node_coords_state(0,inode,2)+1<< std::endl; - } - } - } - } //end view scope - - // save the node coords to the current RK value - for (size_t node_gid=0; node_gid < nall_nodes; node_gid++){ - - for(int rk=1; rkcalc_pressure(elem_pres, - elem_stress, - elem_gid, - elem_mat_id(elem_gid), - state_vars, - global_vars, - elem_user_output_vars, - elem_sspd, - elem_den(elem_gid), - elem_sie(rk_level,elem_gid)); - - // --- Sound speed --- - eos_model->calc_sound_speed(elem_pres, - elem_stress, - elem_gid, - elem_mat_id(elem_gid), - state_vars, - global_vars, - elem_user_output_vars, - elem_sspd, - elem_den(elem_gid), - elem_sie(rk_level,elem_gid)); - - // loop over the nodes of this element and apply velocity - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - - // get the mesh node index - size_t node_gid = nodes_in_elem(elem_gid, node_lid); - - - // --- Velocity --- - switch(mat_fill(f_id).velocity) - { - case VELOCITY_TYPE::cartesian: - { - - node_vel(rk_level, node_gid, 0) = mat_fill(f_id).u; - node_vel(rk_level, node_gid, 1) = mat_fill(f_id).v; - if (num_dim == 3) node_vel(rk_level, node_gid, 2) = mat_fill(f_id).w; - - - break; - } - case VELOCITY_TYPE::radial: - { - // Setting up cylindrical - double dir[2]; - dir[0] = 0.0; - dir[1] = 0.0; - double radius_val = 0.0; - - for(int dim=0; dim<2; dim++){ - dir[dim] = node_coords(rk_level, node_gid, dim); - radius_val += node_coords(rk_level, node_gid, dim)*node_coords(rk_level, node_gid, dim); - } // end for - radius_val = sqrt(radius_val); - - for(int dim=0; dim<2; dim++){ - if (radius_val > 1.0e-14){ - dir[dim] /= (radius_val); - } - else{ - dir[dim] = 0.0; - } - } // end for - - - node_vel(rk_level, node_gid, 0) = mat_fill(f_id).speed*dir[0]; - node_vel(rk_level, node_gid, 1) = mat_fill(f_id).speed*dir[1]; - if (num_dim == 3) node_vel(rk_level, node_gid, 2) = 0.0; - - break; - } - case VELOCITY_TYPE::spherical: - { - - // Setting up spherical - double dir[3]; - dir[0] = 0.0; - dir[1] = 0.0; - dir[2] = 0.0; - double radius_val = 0.0; - - for(int dim=0; dim<3; dim++){ - dir[dim] = node_coords(rk_level, node_gid, dim); - radius_val += node_coords(rk_level, node_gid, dim)*node_coords(rk_level, node_gid, dim); - } // end for - radius_val = sqrt(radius_val); - - for(int dim=0; dim<3; dim++){ - if (radius_val > 1.0e-14){ - dir[dim] /= (radius_val); - } - else{ - dir[dim] = 0.0; - } - } // end for - - node_vel(rk_level, node_gid, 0) = mat_fill(f_id).speed*dir[0]; - node_vel(rk_level, node_gid, 1) = mat_fill(f_id).speed*dir[1]; - if (num_dim == 3) node_vel(rk_level, node_gid, 2) = mat_fill(f_id).speed*dir[2]; - - break; - } - case VELOCITY_TYPE::radial_linear: - { - - break; - } - case VELOCITY_TYPE::spherical_linear: - { - - break; - } - case VELOCITY_TYPE::tg_vortex: - { - - node_vel(rk_level, node_gid, 0) = sin(PI * node_coords(rk_level,node_gid, 0)) * cos(PI * node_coords(rk_level,node_gid, 1)); - node_vel(rk_level, node_gid, 1) = -1.0*cos(PI * node_coords(rk_level,node_gid, 0)) * sin(PI * node_coords(rk_level,node_gid, 1)); - if (num_dim == 3) node_vel(rk_level, node_gid, 2) = 0.0; - - break; - } - } // end of switch - - }// end loop over nodes of element - - - if(mat_fill(f_id).velocity == VELOCITY_TYPE::tg_vortex) - { - elem_pres(elem_gid) = 0.25*( cos(2.0*PI*elem_coords[0]) + cos(2.0*PI*elem_coords[1]) ) + 1.0; - - // p = rho*ie*(gamma - 1) - size_t mat_id = f_id; - double gamma = global_vars(mat_id,0); // gamma value - elem_sie(rk_level, elem_gid) = - elem_pres(elem_gid)/(mat_fill(f_id).den*(gamma - 1.0)); - } // end if - - } // end if fill - - }); // end FOR_ALL_CLASS element loop - Kokkos::fence(); - - - } // end for loop over fills - }//end view scope - - - - // apply BC's to velocity - FEA_Module_SGH::boundary_velocity(*mesh, boundary, node_vel); - - - // calculate the corner massess if 2D - if(num_dim==2){ - - FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { - - // facial area of the corners - double corner_areas_array[4]; - - ViewCArrayKokkos corner_areas(&corner_areas_array[0],4); - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 4); - - get_area_weights2D(corner_areas, - elem_gid, - node_coords, - elem_node_gids, - rk_level); - - // loop over the corners of the element and calculate the mass - for (size_t corner_lid=0; corner_lid<4; corner_lid++){ - - size_t corner_gid = corners_in_elem(elem_gid, corner_lid); - corner_mass(corner_gid) = corner_areas(corner_lid)*elem_den(elem_gid); // node radius is added later - - } // end for over corners - }); - - } // end of - - - // calculate the nodal mass - FOR_ALL_CLASS(node_gid, 0, nall_nodes, { - - node_mass(node_gid) = 0.0; - - if(num_dim==3){ - - for(size_t elem_lid=0; elem_lid < num_corners_in_node(node_gid); elem_lid++){ - size_t elem_gid = elems_in_node(node_gid,elem_lid); - node_mass(node_gid) += 1.0/8.0*elem_mass(elem_gid); - } // end for elem_lid - - }// end if dims=3 - else { - - // 2D-RZ - for(size_t corner_lid=0; corner_lid < num_corners_in_node(node_gid); corner_lid++){ - - size_t corner_gid = corners_in_node(node_gid, corner_lid); - node_mass(node_gid) += corner_mass(corner_gid); // sans the radius so it is areal node mass - - corner_mass(corner_gid) *= node_coords(rk_level,node_gid,1); // true corner mass now - } // end for elem_lid - - } // end else - - }); // end FOR_ALL_CLASS - Kokkos::fence(); - - - //current interface has differing mass arrays; this equates them until we unify memory - //view scope - { - vec_array node_mass_interface = node_masses_distributed->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - node_mass_interface(node_gid,0) = node_mass(node_gid); - }); // end parallel for - } //end view scope - Kokkos::fence(); - //communicate ghost densities - comm_node_masses(); - - //this is forcing a copy to the device - //view scope - { - vec_array ghost_node_mass_interface = ghost_node_masses_distributed->getLocalView (Tpetra::Access::ReadWrite); - - FOR_ALL_CLASS(node_gid, nlocal_nodes, nall_nodes, { - node_mass(node_gid) = ghost_node_mass_interface(node_gid-nlocal_nodes,0); - }); // end parallel for - } //end view scope - Kokkos::fence(); - - //update stiffness matrix - if(simparam->topology_optimization_on||simparam->shape_optimization_on){ - //assemble_matrix(); - } - - // update host copies of arrays modified in this function - elem_den.update_host(); - elem_mass.update_host(); - elem_sie.update_host(); - elem_stress.update_host(); - elem_pres.update_host(); - elem_sspd.update_host(); - - //execute solve - sgh_solve(); - -} - -/* ------------------------------------------------------------------------------------------- - Compute average density of an element from nodal densities ----------------------------------------------------------------------------------------------- */ - -double FEA_Module_SGH::average_element_density(const int nodes_per_elem, const CArray current_element_densities) const -{ - double result = 0; - for(int i=0; i < nodes_per_elem; i++){ - result += current_element_densities(i)/nodes_per_elem; - } - - return result; -} - - -/* ------------------------------------------------------------------------------ - Coupled adjoint problem for the kinetic energy minimization problem ---------------------------------------------------------------------------------- */ - -void FEA_Module_SGH::compute_topology_optimization_adjoint_full(){ - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - size_t num_bdy_nodes = mesh->num_bdy_nodes; - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos material = simparam->material; - const int num_dim = simparam->num_dims; - real_t global_dt; - size_t current_data_index, next_data_index; - Teuchos::RCP previous_adjoint_vector_distributed, current_adjoint_vector_distributed, previous_velocity_vector_distributed, current_velocity_vector_distributed; - Teuchos::RCP previous_phi_adjoint_vector_distributed, current_phi_adjoint_vector_distributed; - //initialize first adjoint vector at last_time_step to 0 as the terminal value - (*adjoint_vector_data)[last_time_step+1]->putScalar(0); - (*phi_adjoint_vector_data)[last_time_step+1]->putScalar(0); - (*psi_adjoint_vector_data)[last_time_step+1]->putScalar(0); - - //solve terminal value problem, proceeds in time backward. For simplicity, we use the same timestep data from the forward solve. - //A linear interpolant is assumed between velocity data points; velocity midpoint is used to update the adjoint. - if(myrank==0) - std::cout << "Computing adjoint vector " << time_data.size() << std::endl; - - for (int cycle = last_time_step; cycle >= 0; cycle--) { - //compute timestep from time data - global_dt = time_data[cycle+1] - time_data[cycle]; - //print - if(simparam->dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ - if (cycle==last_time_step){ - if(myrank==0) - printf("cycle = %d, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } - // print time step every 20 cycles - else if (cycle%20==0){ - if(myrank==0) - printf("cycle = %d, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } // end if - } - //else if (cycle==1){ - //if(myrank==0) - //printf("cycle = %lu, time = %f, time step = %f \n", cycle-1, time_data[cycle-1], global_dt); - //} // end if - - //compute adjoint vector for this data point; use velocity midpoint - //view scope - { - - //set velocity, internal energy, and position for this timestep - const_vec_array previous_velocity_vector = (*forward_solve_velocity_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - - const_vec_array previous_coordinate_vector = (*forward_solve_coordinate_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_coordinate_vector = (*forward_solve_coordinate_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - - const_vec_array previous_element_internal_energy = (*forward_solve_internal_energy_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_element_internal_energy = (*forward_solve_internal_energy_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - - //interface of arrays for current implementation of force calculation - - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes+nghost_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - node_vel(rk_level,node_gid,idim) = previous_velocity_vector(node_gid,idim); - node_coords(rk_level,node_gid,idim) = previous_coordinate_vector(node_gid,idim); - } - }); - Kokkos::fence(); - - FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { - elem_sie(rk_level,elem_gid) = previous_element_internal_energy(elem_gid,0); - }); - Kokkos::fence(); - - //set state according to phase data at this timestep - - get_vol(); - - // ---- Calculate velocity diveregence for the element ---- - if(num_dim==2){ - get_divergence2D(elem_div, - node_coords, - node_vel, - elem_vol); - } - else { - get_divergence(elem_div, - node_coords, - node_vel, - elem_vol); - } // end if 2D - - // ---- Calculate elem state (den, pres, sound speed, stress) for next time step ---- - if(num_dim==2){ - update_state2D(material, - *mesh, - node_coords, - node_vel, - elem_den, - elem_pres, - elem_stress, - elem_sspd, - elem_sie, - elem_vol, - elem_mass, - elem_mat_id, - 1.0, - cycle); - } - else{ - update_state(material, - *mesh, - node_coords, - node_vel, - elem_den, - elem_pres, - elem_stress, - elem_sspd, - elem_sie, - elem_vol, - elem_mass, - elem_mat_id, - 1.0, - cycle); - } - - if(num_dim==2){ - get_force_sgh2D(material, - *mesh, - node_coords, - node_vel, - elem_den, - elem_sie, - elem_pres, - elem_stress, - elem_sspd, - elem_vol, - elem_div, - elem_mat_id, - corner_force, - 1.0, - cycle); - } - else { - get_force_sgh(material, - *mesh, - node_coords, - node_vel, - elem_den, - elem_sie, - elem_pres, - elem_stress, - elem_sspd, - elem_vol, - elem_div, - elem_mat_id, - corner_force, - 1.0, - cycle); - } - - //compute gradient matrices - get_force_egradient_sgh(material, - *mesh, - node_coords, - node_vel, - elem_den, - elem_sie, - elem_pres, - elem_stress, - elem_sspd, - elem_vol, - elem_div, - elem_mat_id, - 1.0, - cycle); - - get_power_egradient_sgh(1.0, - *mesh, - node_vel, - node_coords, - elem_sie, - elem_mass, - corner_force); - - get_force_vgradient_sgh(material, - *mesh, - node_coords, - node_vel, - elem_den, - elem_sie, - elem_pres, - elem_stress, - elem_sspd, - elem_vol, - elem_div, - elem_mat_id, - 1.0, - cycle); - - get_power_vgradient_sgh(1.0, - *mesh, - node_vel, - node_coords, - elem_sie, - elem_mass, - corner_force); - - get_force_ugradient_sgh(material, - *mesh, - node_coords, - node_vel, - elem_den, - elem_sie, - elem_pres, - elem_stress, - elem_sspd, - elem_vol, - elem_div, - elem_mat_id, - 1.0, - cycle); - - get_power_ugradient_sgh(1.0, - *mesh, - node_vel, - node_coords, - elem_sie, - elem_mass, - corner_force); - - - //force_gradient_velocity->describe(*fos,Teuchos::VERB_EXTREME); - const_vec_array previous_force_gradient_position = force_gradient_position->getLocalView (Tpetra::Access::ReadOnly); - //const_vec_array current_force_gradient_position = force_gradient_position->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array previous_force_gradient_velocity = force_gradient_velocity->getLocalView (Tpetra::Access::ReadOnly); - //const_vec_array current_force_gradient_velocity = force_gradient_velocity->getLocalView (Tpetra::Access::ReadOnly); - //compute gradient of force with respect to velocity - - const_vec_array previous_adjoint_vector = (*adjoint_vector_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array phi_previous_adjoint_vector = (*phi_adjoint_vector_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array psi_previous_adjoint_vector = (*psi_adjoint_vector_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - vec_array midpoint_adjoint_vector = adjoint_vector_distributed->getLocalView (Tpetra::Access::ReadWrite); - vec_array phi_midpoint_adjoint_vector = phi_adjoint_vector_distributed->getLocalView (Tpetra::Access::ReadWrite); - vec_array psi_midpoint_adjoint_vector = psi_adjoint_vector_distributed->getLocalView (Tpetra::Access::ReadWrite); - - //half step update for RK2 scheme; EQUATION 1 - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - real_t rate_of_change; - real_t matrix_contribution; - size_t dof_id; - size_t elem_id; - for (int idim = 0; idim < num_dim; idim++){ - //EQUATION 1 - matrix_contribution = 0; - //compute resulting row of force velocity gradient matrix transpose right multiplied by adjoint vector - for(int idof = 0; idof < Gradient_Matrix_Strides(node_gid*num_dim+idim); idof++){ - dof_id = DOF_Graph_Matrix(node_gid*num_dim+idim,idof); - matrix_contribution += previous_adjoint_vector(dof_id/num_dim,dof_id%num_dim)*Force_Gradient_Velocities(node_gid*num_dim+idim,idof); - } - - //compute resulting row of transpose of power gradient w.r.t velocity matrix right multiplied by psi adjoint vector - for(int ielem = 0; ielem < DOF_to_Elem_Matrix_Strides(node_gid*num_dim+idim); ielem++){ - elem_id = elems_in_node(node_gid,ielem); - matrix_contribution += psi_previous_adjoint_vector(elem_id,0)*Power_Gradient_Velocities(node_gid*num_dim+idim,ielem); - } - rate_of_change = previous_velocity_vector(node_gid,idim)- - matrix_contribution/node_mass(node_gid)- - phi_previous_adjoint_vector(node_gid,idim)/node_mass(node_gid); - midpoint_adjoint_vector(node_gid,idim) = -rate_of_change*global_dt/2 + previous_adjoint_vector(node_gid,idim); - - } - }); // end parallel for - Kokkos::fence(); - - //half step update for RK2 scheme; EQUATION 2 - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - real_t rate_of_change; - real_t matrix_contribution; - size_t dof_id; - size_t elem_id; - for (int idim = 0; idim < num_dim; idim++){ - - //EQUATION 2 - matrix_contribution = 0; - //compute resulting row of force displacement gradient matrix transpose right multiplied by adjoint vector - for(int idof = 0; idof < Gradient_Matrix_Strides(node_gid*num_dim+idim); idof++){ - dof_id = DOF_Graph_Matrix(node_gid*num_dim+idim,idof); - matrix_contribution += previous_adjoint_vector(dof_id/num_dim,dof_id%num_dim)*Force_Gradient_Positions(node_gid*num_dim+idim,idof); - } - - //compute resulting row of transpose of power gradient w.r.t displacement matrix right multiplied by psi adjoint vector - for(int ielem = 0; ielem < DOF_to_Elem_Matrix_Strides(node_gid*num_dim+idim); ielem++){ - elem_id = elems_in_node(node_gid,ielem); - matrix_contribution += psi_previous_adjoint_vector(elem_id,0)*Power_Gradient_Positions(node_gid*num_dim+idim,ielem); - } - - rate_of_change = -matrix_contribution; - //rate_of_change = -0.0000001*previous_adjoint_vector(node_gid,idim); - phi_midpoint_adjoint_vector(node_gid,idim) = -rate_of_change*global_dt/2 + phi_previous_adjoint_vector(node_gid,idim); - - } - }); // end parallel for - Kokkos::fence(); - - //phi_adjoint_vector_distributed->describe(*fos,Teuchos::VERB_EXTREME); - - //half step update for RK2 scheme; EQUATION 3 - FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { - real_t rate_of_change; - real_t matrix_contribution; - size_t dof_id; - size_t elem_id; - //EQUATION 3 - matrix_contribution = 0; - //compute resulting row of force displacement gradient matrix transpose right multiplied by adjoint vector - for(int idof = 0; idof < num_nodes_in_elem*num_dim; idof++){ - dof_id = nodes_in_elem(elem_gid,idof/num_dim)*num_dim + idof%num_dim; - matrix_contribution += previous_adjoint_vector(dof_id/num_dim,dof_id%num_dim)*Force_Gradient_Energies(elem_gid,idof); - } - rate_of_change = -(matrix_contribution + psi_previous_adjoint_vector(elem_gid,0)*Power_Gradient_Energies(elem_gid))/elem_mass(elem_gid); - //rate_of_change = -0.0000001*previous_adjoint_vector(node_gid,idim); - psi_midpoint_adjoint_vector(elem_gid,0) = -rate_of_change*global_dt/2 + psi_previous_adjoint_vector(elem_gid,0); - }); // end parallel for - Kokkos::fence(); - - //apply BCs to adjoint vector, only matters for the momentum adjoint if using strictly velocity boundary conditions - boundary_adjoint(*mesh, boundary,midpoint_adjoint_vector, phi_midpoint_adjoint_vector, psi_midpoint_adjoint_vector); - comm_adjoint_vector(cycle); - comm_phi_adjoint_vector(cycle); - - //save for second half of RK - (*psi_adjoint_vector_data)[cycle]->assign(*psi_adjoint_vector_distributed); - - //swap names to get ghost nodes for the midpoint vectors - vec_array current_adjoint_vector = adjoint_vector_distributed->getLocalView (Tpetra::Access::ReadWrite); - vec_array phi_current_adjoint_vector = phi_adjoint_vector_distributed->getLocalView (Tpetra::Access::ReadWrite); - vec_array psi_current_adjoint_vector = psi_adjoint_vector_distributed->getLocalView (Tpetra::Access::ReadWrite); - midpoint_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView (Tpetra::Access::ReadWrite); - phi_midpoint_adjoint_vector = (*phi_adjoint_vector_data)[cycle]->getLocalView (Tpetra::Access::ReadWrite); - psi_midpoint_adjoint_vector = (*psi_adjoint_vector_data)[cycle]->getLocalView (Tpetra::Access::ReadWrite); - - //compute gradients at midpoint - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes+nghost_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - node_vel(rk_level,node_gid,idim) = 0.5*(previous_velocity_vector(node_gid,idim)+current_velocity_vector(node_gid,idim)); - node_coords(rk_level,node_gid,idim) = 0.5*(previous_coordinate_vector(node_gid,idim)+current_coordinate_vector(node_gid,idim)); - } - }); - Kokkos::fence(); - - FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { - elem_sie(rk_level,elem_gid) = 0.5*(previous_element_internal_energy(elem_gid,0)+current_element_internal_energy(elem_gid,0)); - }); - Kokkos::fence(); - - //set state according to phase data at this timestep - - get_vol(); - - // ---- Calculate velocity diveregence for the element ---- - if(num_dim==2){ - get_divergence2D(elem_div, - node_coords, - node_vel, - elem_vol); - } - else { - get_divergence(elem_div, - node_coords, - node_vel, - elem_vol); - } // end if 2D - - // ---- Calculate elem state (den, pres, sound speed, stress) for next time step ---- - if(num_dim==2){ - update_state2D(material, - *mesh, - node_coords, - node_vel, - elem_den, - elem_pres, - elem_stress, - elem_sspd, - elem_sie, - elem_vol, - elem_mass, - elem_mat_id, - 1.0, - cycle); - } - else{ - update_state(material, - *mesh, - node_coords, - node_vel, - elem_den, - elem_pres, - elem_stress, - elem_sspd, - elem_sie, - elem_vol, - elem_mass, - elem_mat_id, - 1.0, - cycle); - } - - if(num_dim==2){ - get_force_sgh2D(material, - *mesh, - node_coords, - node_vel, - elem_den, - elem_sie, - elem_pres, - elem_stress, - elem_sspd, - elem_vol, - elem_div, - elem_mat_id, - corner_force, - 1.0, - cycle); - } - else { - get_force_sgh(material, - *mesh, - node_coords, - node_vel, - elem_den, - elem_sie, - elem_pres, - elem_stress, - elem_sspd, - elem_vol, - elem_div, - elem_mat_id, - corner_force, - 1.0, - cycle); - } - - //compute gradient matrices - get_force_egradient_sgh(material, - *mesh, - node_coords, - node_vel, - elem_den, - elem_sie, - elem_pres, - elem_stress, - elem_sspd, - elem_vol, - elem_div, - elem_mat_id, - 1.0, - cycle); - - get_power_egradient_sgh(1.0, - *mesh, - node_vel, - node_coords, - elem_sie, - elem_mass, - corner_force); - - get_force_vgradient_sgh(material, - *mesh, - node_coords, - node_vel, - elem_den, - elem_sie, - elem_pres, - elem_stress, - elem_sspd, - elem_vol, - elem_div, - elem_mat_id, - 1.0, - cycle); - - get_power_vgradient_sgh(1.0, - *mesh, - node_vel, - node_coords, - elem_sie, - elem_mass, - corner_force); - - get_force_ugradient_sgh(material, - *mesh, - node_coords, - node_vel, - elem_den, - elem_sie, - elem_pres, - elem_stress, - elem_sspd, - elem_vol, - elem_div, - elem_mat_id, - 1.0, - cycle); - - get_power_ugradient_sgh(1.0, - *mesh, - node_vel, - node_coords, - elem_sie, - elem_mass, - corner_force); - - - //full step update with midpoint gradient for RK2 scheme; EQUATION 1 - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - real_t rate_of_change; - real_t matrix_contribution; - size_t dof_id; - size_t elem_id; - for (int idim = 0; idim < num_dim; idim++){ - //EQUATION 1 - matrix_contribution = 0; - //compute resulting row of force velocity gradient matrix transpose right multiplied by adjoint vector - - for(int idof = 0; idof < Gradient_Matrix_Strides(node_gid*num_dim+idim); idof++){ - dof_id = DOF_Graph_Matrix(node_gid*num_dim+idim,idof); - matrix_contribution += midpoint_adjoint_vector(dof_id/num_dim,dof_id%num_dim)*Force_Gradient_Velocities(node_gid*num_dim+idim,idof); - } - - //compute resulting row of transpose of power gradient w.r.t velocity matrix right multiplied by psi adjoint vector - for(int ielem = 0; ielem < DOF_to_Elem_Matrix_Strides(node_gid*num_dim+idim); ielem++){ - elem_id = elems_in_node(node_gid,ielem); - matrix_contribution += psi_midpoint_adjoint_vector(elem_id,0)*Power_Gradient_Velocities(node_gid*num_dim+idim,ielem); - } - - rate_of_change = (previous_velocity_vector(node_gid,idim) + current_velocity_vector(node_gid,idim))/2- - matrix_contribution/node_mass(node_gid)- - phi_midpoint_adjoint_vector(node_gid,idim)/node_mass(node_gid); - current_adjoint_vector(node_gid,idim) = -rate_of_change*global_dt + previous_adjoint_vector(node_gid,idim); - } - }); // end parallel for - Kokkos::fence(); - - //full step update with midpoint gradient for RK2 scheme; EQUATION 2 - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - real_t rate_of_change; - real_t matrix_contribution; - size_t dof_id; - size_t elem_id; - for (int idim = 0; idim < num_dim; idim++){ - - //EQUATION 2 - matrix_contribution = 0; - //compute resulting row of force displacement gradient matrix transpose right multiplied by adjoint vector - for(int idof = 0; idof < Gradient_Matrix_Strides(node_gid*num_dim+idim); idof++){ - dof_id = DOF_Graph_Matrix(node_gid*num_dim+idim,idof); - matrix_contribution += midpoint_adjoint_vector(dof_id/num_dim,dof_id%num_dim)*Force_Gradient_Positions(node_gid*num_dim+idim,idof); - } - - //compute resulting row of transpose of power gradient w.r.t displacement matrix right multiplied by psi adjoint vector - for(int ielem = 0; ielem < DOF_to_Elem_Matrix_Strides(node_gid*num_dim+idim); ielem++){ - elem_id = elems_in_node(node_gid,ielem); - matrix_contribution += psi_midpoint_adjoint_vector(elem_id,0)*Power_Gradient_Positions(node_gid*num_dim+idim,ielem); - } - - rate_of_change = -matrix_contribution; - //rate_of_change = -0.0000001*midpoint_adjoint_vector(node_gid,idim); - phi_current_adjoint_vector(node_gid,idim) = -rate_of_change*global_dt + phi_previous_adjoint_vector(node_gid,idim); - } - }); // end parallel for - Kokkos::fence(); - - //full step update for RK2 scheme; EQUATION 3 - FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { - real_t rate_of_change; - real_t matrix_contribution; - size_t dof_id; - size_t elem_id; - //EQUATION 3 - matrix_contribution = 0; - //compute resulting row of force displacement gradient matrix transpose right multiplied by adjoint vector - for(int idof = 0; idof < num_nodes_in_elem*num_dim; idof++){ - dof_id = nodes_in_elem(elem_gid,idof/num_dim)*num_dim + idof%num_dim; - matrix_contribution += midpoint_adjoint_vector(dof_id/num_dim,dof_id%num_dim)*Force_Gradient_Energies(elem_gid,idof); - } - rate_of_change = -(matrix_contribution + psi_midpoint_adjoint_vector(elem_gid,0)*Power_Gradient_Energies(elem_gid))/elem_mass(elem_gid); - //debug - //std::cout << "PSI RATE OF CHANGE " << rate_of_change << std::endl; - psi_current_adjoint_vector(elem_gid,0) = -rate_of_change*global_dt + psi_previous_adjoint_vector(elem_gid,0); - }); // end parallel for - Kokkos::fence(); - - boundary_adjoint(*mesh, boundary,current_adjoint_vector, phi_current_adjoint_vector, psi_current_adjoint_vector); - comm_adjoint_vector(cycle); - comm_phi_adjoint_vector(cycle); - //save data from time-step completion - (*psi_adjoint_vector_data)[cycle]->assign(*psi_adjoint_vector_distributed); - - } //end view scope - - //phi_adjoint_vector_distributed->describe(*fos,Teuchos::VERB_EXTREME); - - } -} - -/* ---------------------------------------------------------------------------- - Gradient for the (unsimplified) kinetic energy minimization problem -------------------------------------------------------------------------------- */ - -void FEA_Module_SGH::compute_topology_optimization_gradient_full(Teuchos::RCP design_densities_distributed, Teuchos::RCP design_gradients_distributed){ - - size_t num_bdy_nodes = mesh->num_bdy_nodes; - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos material = simparam->material; - const int num_dim = simparam->num_dims; - int num_corners = rnum_elem*num_nodes_in_elem; - real_t global_dt; - bool element_constant_density = true; - size_t current_data_index, next_data_index; - CArrayKokkos current_element_velocities = CArrayKokkos(num_nodes_in_elem,num_dim); - CArrayKokkos current_element_adjoint = CArrayKokkos(num_nodes_in_elem,num_dim); - - if(myrank==0) - std::cout << "Computing accumulated kinetic energy gradient" << std::endl; - - compute_topology_optimization_adjoint_full(); - - - - { //view scope - vec_array design_gradients = design_gradients_distributed->getLocalView (Tpetra::Access::ReadWrite); - const_vec_array design_densities = design_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); - //initialize design gradients - FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { - design_gradients(node_id,0) = 0; - }); // end parallel for - Kokkos::fence(); - - //gradient contribution from kinetic energy v(dM/drho)v product. - if(simparam->dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ - if(myrank==0){ - std::cout << "v*dM/drho*v term" << std::endl; - } - } - - for (unsigned long cycle = 0; cycle < last_time_step+1; cycle++) { - //compute timestep from time data - global_dt = time_data[cycle+1] - time_data[cycle]; - - //print - if(simparam->dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ - - if (cycle==0){ - if(myrank==0) - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } - // print time step every 10 cycles - else if (cycle%20==0){ - if(myrank==0) - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } // end if - } - //view scope - { - const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array next_velocity_vector = (*forward_solve_velocity_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - - FOR_ALL_CLASS(elem_id, 0, rnum_elem, { - size_t node_id; - size_t corner_id; - real_t inner_product; - //std::cout << elem_mass(elem_id) <dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ - if(myrank==0){ - std::cout << "gradient term involving adjoint derivative" << std::endl; - } - } - - for (unsigned long cycle = 0; cycle < last_time_step+1; cycle++) { - //compute timestep from time data - global_dt = time_data[cycle+1] - time_data[cycle]; - //print - if(simparam->dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ - - if (cycle==0){ - if(myrank==0) - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } - // print time step every 10 cycles - else if (cycle%20==0){ - if(myrank==0) - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } // end if - } - - //compute adjoint vector for this data point; use velocity midpoint - //view scope - { - const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array next_velocity_vector = (*forward_solve_velocity_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array next_adjoint_vector = (*adjoint_vector_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - - FOR_ALL_CLASS(elem_id, 0, rnum_elem, { - real_t lambda_dot_current; - real_t lambda_dot_next; - size_t node_id; - size_t corner_id; - real_t inner_product; - //std::cout << elem_mass(elem_id) <dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ - if(myrank==0){ - std::cout << "gradient term involving adjoint derivative" << std::endl; - } - } - - for (unsigned long cycle = 0; cycle < last_time_step+1; cycle++) { - //compute timestep from time data - global_dt = time_data[cycle+1] - time_data[cycle]; - //print - if(simparam->dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ - - if (cycle==0){ - if(myrank==0) - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } - // print time step every 10 cycles - else if (cycle%20==0){ - if(myrank==0) - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } // end if - } - - //compute adjoint vector for this data point; use velocity midpoint - //view scope - { - const_vec_array current_element_internal_energy = (*forward_solve_internal_energy_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_psi_adjoint_vector = (*psi_adjoint_vector_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array next_element_internal_energy = (*forward_solve_internal_energy_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array next_psi_adjoint_vector = (*psi_adjoint_vector_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - - FOR_ALL_CLASS(elem_id, 0, rnum_elem, { - real_t psi_dot_current; - real_t psi_dot_next; - size_t node_id; - size_t corner_id; - real_t inner_product; - - psi_dot_current = (next_psi_adjoint_vector(elem_id,0) - current_psi_adjoint_vector(elem_id,0))/global_dt; - inner_product = elem_mass(elem_id)*(psi_dot_current+psi_dot_current)*current_element_internal_energy(elem_id,0)/2; - - for (int inode = 0; inode < num_nodes_in_elem; inode++){ - //compute gradient of local element contribution to v^t*M*v product - corner_id = elem_id*num_nodes_in_elem + inode; - corner_value_storage(corner_id) = inner_product*global_dt/relative_element_densities(elem_id); - } - - }); // end parallel for - Kokkos::fence(); - - //accumulate node values from corner storage - //multiply - FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { - size_t corner_id; - for(int icorner=0; icorner < num_corners_in_node(node_id); icorner++){ - corner_id = corners_in_node(node_id,icorner); - design_gradients(node_id,0) += -corner_value_storage(corner_id)/(double)num_nodes_in_elem; - } - }); // end parallel for - Kokkos::fence(); - - } //end view scope - } - - //compute initial condition contribution from velocities - //view scope - { - const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[0]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_adjoint_vector = (*adjoint_vector_data)[0]->getLocalView (Tpetra::Access::ReadOnly); - - FOR_ALL_CLASS(elem_id, 0, rnum_elem, { - real_t lambda_dot; - size_t node_id; - size_t corner_id; - real_t inner_product; - //std::cout << elem_mass(elem_id) <getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_psi_adjoint_vector = (*psi_adjoint_vector_data)[0]->getLocalView (Tpetra::Access::ReadOnly); - - //(*psi_adjoint_vector_data)[100]->describe(*fos,Teuchos::VERB_EXTREME); - FOR_ALL_CLASS(elem_id, 0, rnum_elem, { - real_t lambda_dot; - size_t node_id; - size_t corner_id; - real_t inner_product; - //std::cout << elem_mass(elem_id) <getLocalView (Tpetra::Access::ReadWrite); - //const_host_vec_array host_design_variables = design_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); - vec_array design_gradients = design_gradients_distributed->getLocalView (Tpetra::Access::ReadWrite); - const_vec_array design_variables = design_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); - force_design_gradient_term(design_variables, design_gradients); - power_design_gradient_term(design_variables, design_gradients); - }//end view scope - -} - -/* ---------------------------------------------------------------------- - Initialize global vectors and array maps needed for matrix assembly -------------------------------------------------------------------------- */ -void FEA_Module_SGH::init_assembly(){ - int num_dim = simparam->num_dims; - //const_host_elem_conn_array nodes_in_elem = global_nodes_in_elem_distributed->getLocalView (Tpetra::Access::ReadOnly); - Gradient_Matrix_Strides = DCArrayKokkos (nlocal_nodes*num_dim, "Gradient_Matrix_Strides"); - DOF_to_Elem_Matrix_Strides = DCArrayKokkos (nlocal_nodes*num_dim, "Gradient_Matrix_Strides"); - Elem_to_Elem_Matrix_Strides = DCArrayKokkos (rnum_elem, "Gradient_Matrix_Strides"); - CArrayKokkos Graph_Fill(nall_nodes, "nall_nodes"); - CArrayKokkos current_row_nodes_scanned; - CArrayKokkos count_saved_corners_in_node(nall_nodes, "count_saved_corners_in_node"); - int local_node_index, current_column_index; - size_t max_stride = 0; - size_t nodes_per_element; - nodal_density_flag = simparam->nodal_density_flag; - penalty_power = simparam->optimization_options.simp_penalty_power; - - //allocate stride arrays - CArrayKokkos Graph_Matrix_Strides_initial(nlocal_nodes, "Graph_Matrix_Strides_initial"); - DCArrayKokkos Dual_Graph_Matrix_Strides_initial(nlocal_nodes, "Host_Graph_Matrix_Strides_initial"); - Graph_Matrix_Strides = DCArrayKokkos(nlocal_nodes, "Graph_Matrix_Strides"); - - //allocate storage for the sparse gradient matrix map for node to node connectivity - Global_Gradient_Matrix_Assembly_Map = DCArrayKokkos(rnum_elem, - max_nodes_per_element,max_nodes_per_element, "Global_Gradient_Matrix_Assembly_Map"); - - //allocate storage for the sparse gradient matrix map for node to element connectivity - Element_Gradient_Matrix_Assembly_Map = DCArrayKokkos(rnum_elem, - max_nodes_per_element, "Element_Gradient_Matrix_Assembly_Map"); - - //allocate array used to determine global node repeats in the sparse graph later - DCArrayKokkos node_indices_used(nall_nodes, "node_indices_used"); - - /*allocate array that stores which column the node index occured on for the current row - when removing repeats*/ - DCArrayKokkos column_index(nall_nodes, "column_index"); - - //initialize nlocal arrays - FOR_ALL_CLASS(inode, 0, nlocal_nodes, { - Graph_Matrix_Strides_initial(inode) = 0; - Graph_Matrix_Strides(inode) = 0; - Graph_Fill(inode) = 0; - }); // end parallel for - Kokkos::fence(); - - //initialize nall arrays - //initialize nlocal arrays - FOR_ALL_CLASS(inode, 0, nall_nodes, { - node_indices_used(inode) = 0; - column_index(inode) = 0; - count_saved_corners_in_node(inode) = 0; - }); // end parallel for - Kokkos::fence(); - - //count upper bound of strides for Sparse Pattern Graph by allowing repeats due to connectivity - if(num_dim == 2) - for (int ielem = 0; ielem < rnum_elem; ielem++){ - element_select->choose_2Delem_type(Element_Types(ielem), elem2D); - nodes_per_element = elem2D->num_nodes(); - for (int lnode = 0; lnode < nodes_per_element; lnode++){ - local_node_index = nodes_in_elem(ielem, lnode); - if(local_node_index < nlocal_nodes){ - Dual_Graph_Matrix_Strides_initial.host(local_node_index) += nodes_per_element; - } - } - } - - if(num_dim == 3) - for (int ielem = 0; ielem < rnum_elem; ielem++){ - element_select->choose_3Delem_type(Element_Types(ielem), elem); - nodes_per_element = elem->num_nodes(); - for (int lnode = 0; lnode < nodes_per_element; lnode++){ - local_node_index = nodes_in_elem(ielem, lnode); - if(local_node_index < nlocal_nodes){ - Dual_Graph_Matrix_Strides_initial.host(local_node_index) += nodes_per_element; - } - } - } - - Dual_Graph_Matrix_Strides_initial.update_device(); - - //equate strides for later - FOR_ALL_CLASS(inode, 0, nlocal_nodes, { - Graph_Matrix_Strides(inode) = Graph_Matrix_Strides_initial(inode) = Dual_Graph_Matrix_Strides_initial(inode); - }); // end parallel for - - //for (int inode = 0; inode < nlocal_nodes; inode++) - //std::cout << Graph_Matrix_Strides_initial(inode) << std::endl; - - //compute maximum stride - size_t update = 0; - REDUCE_MAX_CLASS(inode, 0, nlocal_nodes, update, { - if(update < Graph_Matrix_Strides_initial(inode)) - update = Graph_Matrix_Strides_initial(inode); - }, max_stride); - - //std::cout << "THE MAX STRIDE" << max_stride << std::endl; - //allocate array used in the repeat removal process - current_row_nodes_scanned = CArrayKokkos(max_stride, "current_row_nodes_scanned"); - - //allocate sparse graph with node repeats - RaggedRightArrayKokkos Repeat_Graph_Matrix(Graph_Matrix_Strides_initial); - RaggedRightArrayofVectorsKokkos Element_local_indices(Graph_Matrix_Strides_initial,3); - - //Fill the initial Graph with repeats - if(num_dim == 2){ - for (int ielem = 0; ielem < rnum_elem; ielem++){ - element_select->choose_2Delem_type(Element_Types(ielem), elem2D); - nodes_per_element = elem2D->num_nodes(); - for (int lnode = 0; lnode < nodes_per_element; lnode++){ - local_node_index = nodes_in_elem(ielem, lnode); - if(local_node_index < nlocal_nodes){ - for (int jnode = 0; jnode < nodes_per_element; jnode++){ - current_column_index = Graph_Fill(local_node_index)+jnode; - Repeat_Graph_Matrix(local_node_index, current_column_index) = nodes_in_elem(ielem,jnode); - - //fill inverse map - Element_local_indices(local_node_index,current_column_index,0) = ielem; - Element_local_indices(local_node_index,current_column_index,1) = lnode; - Element_local_indices(local_node_index,current_column_index,2) = jnode; - - //fill forward map - Global_Gradient_Matrix_Assembly_Map(ielem,lnode,jnode) = current_column_index; - } - Graph_Fill(local_node_index) += nodes_per_element; - } - } - } - } - - if(num_dim == 3){ - for (int ielem = 0; ielem < rnum_elem; ielem++){ - element_select->choose_3Delem_type(Element_Types(ielem), elem); - nodes_per_element = elem->num_nodes(); - for (int lnode = 0; lnode < nodes_per_element; lnode++){ - local_node_index = nodes_in_elem(ielem, lnode); - if(local_node_index < nlocal_nodes){ - for (int jnode = 0; jnode < nodes_per_element; jnode++){ - current_column_index = Graph_Fill(local_node_index)+jnode; - Repeat_Graph_Matrix(local_node_index, current_column_index) = nodes_in_elem(ielem,jnode); - - //fill inverse map - Element_local_indices(local_node_index,current_column_index,0) = ielem; - Element_local_indices(local_node_index,current_column_index,1) = lnode; - Element_local_indices(local_node_index,current_column_index,2) = jnode; - - //fill forward map - Global_Gradient_Matrix_Assembly_Map(ielem,lnode,jnode) = current_column_index; - } - Graph_Fill(local_node_index) += nodes_per_element; - } - } - } - } - - //debug statement - //std::cout << "started run" << std::endl; - //std::cout << "Graph Matrix Strides Repeat on task " << myrank << std::endl; - //for (int inode = 0; inode < nlocal_nodes; inode++) - //std::cout << Graph_Matrix_Strides(inode) << std::endl; - RUN_CLASS({ - //remove repeats from the inital graph setup - int current_node; - //remove repeats from the inital graph setup - int current_element_index; - int element_row_index; - int element_column_index; - int current_stride; - int current_row_n_nodes_scanned; - for (int inode = 0; inode < nlocal_nodes; inode++){ - current_row_n_nodes_scanned = 0; - for (int istride = 0; istride < Graph_Matrix_Strides(inode); istride++){ - //convert global index in graph to its local index for the flagging array - current_node = Repeat_Graph_Matrix(inode,istride); - //debug - //if(current_node==-1) - //std::cout << "Graph Matrix node access on task " << myrank << std::endl; - //std::cout << Repeat_Graph_Matrix(inode,istride) << std::endl; - if(node_indices_used(current_node)){ - //set global assembly map index to the location in the graph matrix where this global node was first found - current_element_index = Element_local_indices(inode,istride,0); - element_row_index = Element_local_indices(inode,istride,1); - element_column_index = Element_local_indices(inode,istride,2); - Global_Gradient_Matrix_Assembly_Map(current_element_index,element_row_index, element_column_index) - = column_index(current_node); - - - //swap current node with the end of the current row and shorten the stride of the row - //first swap information about the inverse and forward maps - - current_stride = Graph_Matrix_Strides(inode); - if(istride!=current_stride-1){ - Element_local_indices(inode,istride,0) = Element_local_indices(inode,current_stride-1,0); - Element_local_indices(inode,istride,1) = Element_local_indices(inode,current_stride-1,1); - Element_local_indices(inode,istride,2) = Element_local_indices(inode,current_stride-1,2); - current_element_index = Element_local_indices(inode,istride,0); - element_row_index = Element_local_indices(inode,istride,1); - element_column_index = Element_local_indices(inode,istride,2); - - Global_Gradient_Matrix_Assembly_Map(current_element_index,element_row_index, element_column_index) - = istride; - - //now that the element map information has been copied, copy the global node index and delete the last index - - Repeat_Graph_Matrix(inode,istride) = Repeat_Graph_Matrix(inode,current_stride-1); - } - istride--; - Graph_Matrix_Strides(inode)--; - } - else{ - /*this node hasn't shown up in the row before; add it to the list of nodes - that have been scanned uniquely. Use this list to reset the flag array - afterwards without having to loop over all the nodes in the system*/ - node_indices_used(current_node) = 1; - column_index(current_node) = istride; - current_row_nodes_scanned(current_row_n_nodes_scanned) = current_node; - current_row_n_nodes_scanned++; - } - } - //reset nodes used list for the next row of the sparse list - for(int node_reset = 0; node_reset < current_row_n_nodes_scanned; node_reset++) - node_indices_used(current_row_nodes_scanned(node_reset)) = 0; - - } - }); - Kokkos::fence(); - - Graph_Matrix_Strides.update_host(); - //copy reduced content to non_repeat storage - Graph_Matrix = RaggedRightArrayKokkos(Graph_Matrix_Strides); - - FOR_ALL_CLASS(inode, 0, nlocal_nodes, { - for(int istride = 0; istride < Graph_Matrix_Strides(inode); istride++){ - Graph_Matrix(inode,istride) = Repeat_Graph_Matrix(inode,istride); - } - }); // end parallel for - - //deallocate repeat matrix - - /*At this stage the sparse graph should have unique global indices on each row. - The constructed Assembly map (to the global sparse matrix) - is used to loop over each element's local stiffness matrix in the assembly process.*/ - - //expand strides for stiffness matrix by multipling by dim - FOR_ALL_CLASS(idof, 0, num_dim*nlocal_nodes, { - Gradient_Matrix_Strides(idof) = num_dim*Graph_Matrix_Strides(idof/num_dim); - }); // end parallel for - - Gradient_Matrix_Strides.update_host(); - - //build inverse map for element gradient assembly - for (size_t elem_gid = 0; elem_gid < rnum_elem; elem_gid++){ - FOR_ALL_CLASS(node_lid, 0, num_nodes_in_elem, { - - // get the global_id of the node - size_t node_gid = nodes_in_elem(elem_gid, node_lid); - - // the column index is the num corners saved - size_t j = count_saved_corners_in_node(node_gid); - Element_Gradient_Matrix_Assembly_Map(elem_gid, node_lid) = j; - - // increment the number of corners saved to this node_gid - count_saved_corners_in_node(node_gid)++; - - }); // end FOR_ALL over nodes in element - Kokkos::fence(); - } // end for elem_gid - - DOF_Graph_Matrix = RaggedRightArrayKokkos(Gradient_Matrix_Strides); - Force_Gradient_Positions = RaggedRightArrayKokkos(Gradient_Matrix_Strides); - Force_Gradient_Velocities = RaggedRightArrayKokkos(Gradient_Matrix_Strides); - //needs different graph of node to elem rather than node to node - //DOF_to_Elem_Matrix_Strides.get_kokkos_dual_view().d_view = elems_in_node.mystrides_; - Kokkos::View node_to_elem_strides = elems_in_node.mystrides_; - DOF_to_Elem_Matrix_Strides = DCArrayKokkos(nlocal_nodes*num_dim); - FOR_ALL_CLASS(inode, 0, nlocal_nodes, { - for(int idim = 0; idim < num_dim; idim++){ - DOF_to_Elem_Matrix_Strides(inode*num_dim + idim) = count_saved_corners_in_node(inode); - } - }); // end parallel for - DOF_to_Elem_Matrix_Strides.update_host(); - Force_Gradient_Energies = CArrayKokkos(rnum_elem, num_nodes_in_elem*num_dim); - Power_Gradient_Energies = CArrayKokkos(rnum_elem); - Power_Gradient_Positions = RaggedRightArrayKokkos(DOF_to_Elem_Matrix_Strides); - Power_Gradient_Velocities = RaggedRightArrayKokkos(DOF_to_Elem_Matrix_Strides); - - //set stiffness Matrix Graph - //debug print - //std::cout << "DOF GRAPH MATRIX ENTRIES ON TASK " << myrank << std::endl; - FOR_ALL_CLASS(idof, 0, num_dim*nlocal_nodes, { - for (int istride = 0; istride < Gradient_Matrix_Strides(idof); istride++){ - DOF_Graph_Matrix(idof,istride) = Graph_Matrix(idof/num_dim,istride/num_dim)*num_dim + istride%num_dim; - } - }); // end parallel for - - - /* - //construct distributed gradient matrix from local kokkos data - //build column map for the global gradient matrix - Teuchos::RCP > colmap; - const Teuchos::RCP > dommap = local_dof_map; - - Tpetra::Details::makeColMap(colmap,dommap,DOF_Graph_Matrix.get_kokkos_view(), nullptr); - - size_t nnz = DOF_Graph_Matrix.size(); - - //debug print - //std::cout << "DOF GRAPH SIZE ON RANK " << myrank << " IS " << nnz << std::endl; - - //local indices in the graph using the constructed column map - CArrayKokkos gradient_local_indices(nnz, "gradient_local_indices"); - - //row offsets with compatible template arguments - Kokkos::View row_offsets = DOF_Graph_Matrix.start_index_; - row_pointers row_offsets_pass("row_offsets", nlocal_nodes*num_dim+1); - for(int ipass = 0; ipass < nlocal_nodes*num_dim + 1; ipass++){ - row_offsets_pass(ipass) = row_offsets(ipass); - } - - size_t entrycount = 0; - for(int irow = 0; irow < nlocal_nodes*num_dim; irow++){ - for(int istride = 0; istride < Gradient_Matrix_Strides(irow); istride++){ - gradient_local_indices(entrycount) = colmap->getLocalElement(DOF_Graph_Matrix(irow,istride)); - entrycount++; - } - } - - - //sort values and indices - Tpetra::Import_Util::sortCrsEntries(row_offsets_pass, gradient_local_indices.get_kokkos_view(), Force_Gradient_Positions.get_kokkos_view()); - Tpetra::Import_Util::sortCrsEntries(row_offsets_pass, gradient_local_indices.get_kokkos_view(), Force_Gradient_Velocities.get_kokkos_view()); - - //Teuchos::RCP crs_matrix_params = Teuchos::rcp(new Teuchos::ParameterList("crsmatrix")); - //crs_matrix_params->set("sorted", false); - distributed_force_gradient_positions = Teuchos::rcp(new MAT(local_dof_map, colmap, row_offsets_pass, gradient_local_indices.get_kokkos_view(), Force_Gradient_Positions.get_kokkos_view())); - distributed_force_gradient_positions->fillComplete(); - distributed_force_gradient_velocities = Teuchos::rcp(new MAT(local_dof_map, colmap, row_offsets_pass, gradient_local_indices.get_kokkos_view(), Force_Gradient_Velocities.get_kokkos_view())); - distributed_force_gradient_velocities->fillComplete(); - */ - //distributed_force_gradient_positions->describe(*fos,Teuchos::VERB_EXTREME); - //distributed_force_gradient_velocities->describe(*fos,Teuchos::VERB_EXTREME); -} - -/* ---------------------------------------------------------------------- - Enforce boundary conditions on the adjoint vectors -------------------------------------------------------------------------- */ - -void FEA_Module_SGH::boundary_adjoint(const mesh_t &mesh, - const DCArrayKokkos &boundary, - vec_array &node_adjoint, - vec_array &node_phi_adjoint, - vec_array &node_psi_adjoint){ - - //error and debug flag - //DCArrayKokkos print_flag(1, "print_flag"); - //print_flag.host(0) = false; - //print_flag.update_device(); - - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - int num_dims = simparam->num_dims; - // Loop over boundary sets - for (size_t bdy_set=0; bdy_setdoImport(*adjoint_vector_distributed, *importer, Tpetra::INSERT); -} - -/* ---------------------------------------------------------------------- - Communicate updated nodal adjoint vectors to ghost nodes -------------------------------------------------------------------------- */ - -void FEA_Module_SGH::comm_phi_adjoint_vector(int cycle){ - //comms to get ghosts - (*phi_adjoint_vector_data)[cycle]->doImport(*phi_adjoint_vector_distributed, *importer, Tpetra::INSERT); -} diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/FEA_Module_SGH.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/FEA_Module_SGH.cpp new file mode 100644 index 000000000..ac3a309ed --- /dev/null +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/FEA_Module_SGH.cpp @@ -0,0 +1,1725 @@ +/********************************************************************************************** + © 2020. Triad National Security, LLC. All rights reserved. + This program was produced under U.S. Government contract 89233218CNA000001 for Los Alamos + National Laboratory (LANL), which is operated by Triad National Security, LLC for the U.S. + Department of Energy/National Nuclear Security Administration. All rights in the program are + reserved by Triad National Security, LLC, and the U.S. Department of Energy/National Nuclear + Security Administration. The Government is granted for itself and others acting on its behalf a + nonexclusive, paid-up, irrevocable worldwide license in this material to reproduce, prepare + derivative works, distribute copies to the public, perform publicly and display publicly, and + to permit others to do so. + This program is open source under the BSD-3 License. + Redistribution and use in source and binary forms, with or without modification, are permitted + provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, this list of + conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, this list of + conditions and the following disclaimer in the documentation and/or other materials + provided with the distribution. + + 3. Neither the name of the copyright holder nor the names of its contributors may be used + to endorse or promote products derived from this software without specific prior + written permission. + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS + IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **********************************************************************************************/ + +#include +#include +#include +#include +#include +#include +#include // fmin, fmax, abs note: fminl is long +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include "Tpetra_Import.hpp" +#include "Tpetra_Import_Util2.hpp" + +#include "elements.h" +#include "swage.h" +#include "matar.h" +#include "utilities.h" +#include "node_combination.h" +#include "Simulation_Parameters/Simulation_Parameters_Explicit.h" +#include "Simulation_Parameters/FEA_Module/SGH_Parameters.h" +#include "FEA_Module_SGH.h" +#include "Explicit_Solver.h" + +// optimization +#include "ROL_Solver.hpp" +#include "Kinetic_Energy_Minimize.h" + +#define MAX_ELEM_NODES 8 +#define STRAIN_EPSILON 0.000000001 +#define DENSITY_EPSILON 0.0001 +#define BC_EPSILON 1.0e-6 +#define BUFFER_GROW 100 + +using namespace utils; + +FEA_Module_SGH::FEA_Module_SGH( + SGH_Parameters& params, Solver* Solver_Pointer, + std::shared_ptr mesh_in, const int my_fea_module_index) + : FEA_Module(Solver_Pointer) +{ + // assign interfacing index + my_fea_module_index_ = my_fea_module_index; + Module_Type = FEA_MODULE_TYPE::SGH; + + // recast solver pointer for non-base class access + Explicit_Solver_Pointer_ = dynamic_cast(Solver_Pointer); + simparam = &(Explicit_Solver_Pointer_->simparam); + module_params = ¶ms; + + // create ref element object + // ref_elem = new elements::ref_element(); + // create mesh objects + // init_mesh = new swage::mesh_t(simparam); + // mesh = new swage::mesh_t(simparam); + + mesh = mesh_in; + + // boundary condition data + max_boundary_sets = 0; + Local_Index_Boundary_Patches = Explicit_Solver_Pointer_->Local_Index_Boundary_Patches; + + // set Tpetra vector pointers + initial_node_velocities_distributed = Explicit_Solver_Pointer_->initial_node_velocities_distributed; + initial_node_coords_distributed = Explicit_Solver_Pointer_->initial_node_coords_distributed; + all_initial_node_coords_distributed = Explicit_Solver_Pointer_->all_initial_node_coords_distributed; + node_coords_distributed = Explicit_Solver_Pointer_->node_coords_distributed; + node_velocities_distributed = Explicit_Solver_Pointer_->node_velocities_distributed; + all_node_velocities_distributed = Explicit_Solver_Pointer_->all_node_velocities_distributed; + + // Switch for optimization solver + if (simparam->topology_optimization_on || simparam->shape_optimization_on) + { + all_cached_node_velocities_distributed = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + force_gradient_velocity = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + force_gradient_position = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + force_gradient_design = Teuchos::rcp(new MV(all_node_map, 1)); + corner_value_storage = Solver_Pointer->corner_value_storage; + corner_vector_storage = Solver_Pointer->corner_vector_storage; + corner_gradient_storage = Solver_Pointer->corner_gradient_storage; + relative_element_densities = DCArrayKokkos(rnum_elem, "relative_element_densities"); + adjoint_vector_distributed = Teuchos::rcp(new MV(map, simparam->num_dims)); + phi_adjoint_vector_distributed = Teuchos::rcp(new MV(map, simparam->num_dims)); + psi_adjoint_vector_distributed = Teuchos::rcp(new MV(all_element_map, 1)); + } + + if (simparam->topology_optimization_on || simparam->shape_optimization_on || simparam->num_dims == 2) + { + node_masses_distributed = Teuchos::rcp(new MV(map, 1)); + ghost_node_masses_distributed = Teuchos::rcp(new MV(ghost_node_map, 1)); + } + + // setup output + noutput = 0; + init_output(); + + // optimization flags + kinetic_energy_objective = false; + + // set parameters + Dynamic_Options dynamic_options = simparam->dynamic_options; + time_value = dynamic_options.time_value; + time_final = dynamic_options.time_final; + dt_max = dynamic_options.dt_max; + dt_min = dynamic_options.dt_min; + dt_cfl = dynamic_options.dt_cfl; + graphics_time = simparam->output_options.graphics_time; + graphics_dt_ival = simparam->output_options.graphics_dt_ival; + graphics_cyc_ival = simparam->output_options.graphics_cyc_ival; + cycle_stop = dynamic_options.cycle_stop; + rk_num_stages = dynamic_options.rk_num_stages; + dt = dynamic_options.dt; + fuzz = dynamic_options.fuzz; + tiny = dynamic_options.tiny; + small = dynamic_options.small; + graphics_times = simparam->output_options.graphics_times; + graphics_id = simparam->output_options.graphics_id; + rk_num_bins = simparam->dynamic_options.rk_num_bins; + + if (simparam->topology_optimization_on) + { + max_time_steps = BUFFER_GROW; + time_data.resize(max_time_steps + 1); + element_internal_energy_distributed = Teuchos::rcp(new MV(all_element_map, 1)); + forward_solve_velocity_data = Teuchos::rcp(new std::vector>(max_time_steps + 1)); + forward_solve_coordinate_data = Teuchos::rcp(new std::vector>(max_time_steps + 1)); + forward_solve_internal_energy_data = Teuchos::rcp(new std::vector>(max_time_steps + 1)); + adjoint_vector_data = Teuchos::rcp(new std::vector>(max_time_steps + 1)); + phi_adjoint_vector_data = Teuchos::rcp(new std::vector>(max_time_steps + 1)); + psi_adjoint_vector_data = Teuchos::rcp(new std::vector>(max_time_steps + 1)); + + // assign a multivector of corresponding size to each new timestep in the buffer + for (int istep = 0; istep < max_time_steps + 1; istep++) + { + (*forward_solve_velocity_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*forward_solve_coordinate_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*forward_solve_internal_energy_data)[istep] = Teuchos::rcp(new MV(all_element_map, 1)); + (*adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*phi_adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*psi_adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_element_map, 1)); + } + } + + have_loading_conditions = false; +} + +FEA_Module_SGH::~FEA_Module_SGH() +{ + // delete simparam; +} + +/* ---------------------------------------------------------------------- + Read ANSYS dat format mesh file +------------------------------------------------------------------------- */ +void FEA_Module_SGH::read_conditions_ansys_dat(std::ifstream* in, std::streampos before_condition_header) +{ + auto input_options = simparam->input_options.value(); + + char ch; + std::string skip_line, read_line, substring, token; + std::stringstream line_parse, line_parse2; + + int num_dim = simparam->num_dims; + int buffer_lines = 1000; + int max_word = 30; + int local_node_index, current_column_index; + int p_order = input_options.p_order; + int buffer_loop, buffer_iteration, buffer_iterations, scan_loop, nodes_per_element, words_per_line; + + size_t read_index_start, node_rid, elem_gid; + size_t strain_count; + + real_t unit_scaling = input_options.unit_scaling; + real_t dof_value; + + CArrayKokkos read_buffer; + CArrayKokkos read_buffer_indices; + + LO local_dof_id; + GO node_gid; + + host_vec_array node_densities; +} // end read_conditions_ansys_dat + +/* ---------------------------------------------------------------------- + Assign sets of element boundary surfaces corresponding to user BCs +------------------------------------------------------------------------- */ + +void FEA_Module_SGH::generate_bcs() +{ +} // end generate_bcs + +/* ---------------------------------------------------------------------- + Loop through applied boundary conditions and tag node ids to remove + necessary rows and columns from the assembled linear system +------------------------------------------------------------------------- */ + +void FEA_Module_SGH::Displacement_Boundary_Conditions() +{ +} + +/* ---------------------------------------------------------------------------- + Output field settings and file settings +------------------------------------------------------------------------------- */ + +void FEA_Module_SGH::output_control() +{ +} + +/* ---------------------------------------------------------------------------- + Initialize output data structures +------------------------------------------------------------------------------- */ + +void FEA_Module_SGH::init_output() +{ + // check user parameters for output + bool output_velocity_flag = simparam->output(FIELD::velocity); + bool output_strain_flag = simparam->output(FIELD::strain); + bool output_stress_flag = simparam->output(FIELD::stress); + int num_dim = simparam->num_dims; + int Brows; + if (num_dim == 3) + { + Brows = 6; + } + else + { + Brows = 3; + } + + // Implicit compliant code + if (output_velocity_flag) + { + // displacement_index is accessed by writers at the solver level for deformed output + output_velocity_index = noutput; + noutput += 1; + module_outputs.resize(noutput); + + vector_style.resize(noutput); + vector_style[noutput - 1] = DOF; + + output_vector_sizes.resize(noutput); + output_vector_sizes[noutput - 1] = num_dim; + + output_dof_names.resize(noutput); + output_dof_names[noutput - 1].resize(num_dim); + output_dof_names[noutput - 1][0] = "vx"; + output_dof_names[noutput - 1][1] = "vy"; + if (num_dim == 3) + { + output_dof_names[noutput - 1][2] = "vz"; + } + } + if (output_strain_flag) + { + output_strain_index = noutput; + noutput += 1; + module_outputs.resize(noutput); + + vector_style.resize(noutput); + vector_style[noutput - 1] = NODAL; + + output_vector_sizes.resize(noutput); + output_vector_sizes[noutput - 1] = Brows; + + output_dof_names.resize(noutput); + output_dof_names[noutput - 1].resize(Brows); + if (num_dim == 2) + { + output_dof_names[noutput - 1][0] = "strain_xx"; + output_dof_names[noutput - 1][1] = "strain_yy"; + output_dof_names[noutput - 1][2] = "strain_xy"; + } + if (num_dim == 3) + { + output_dof_names[noutput - 1][0] = "strain_xx"; + output_dof_names[noutput - 1][1] = "strain_yy"; + output_dof_names[noutput - 1][2] = "strain_zz"; + output_dof_names[noutput - 1][3] = "strain_xy"; + output_dof_names[noutput - 1][4] = "strain_xz"; + output_dof_names[noutput - 1][5] = "strain_yz"; + } + } + if (output_stress_flag) + { + output_stress_index = noutput; + noutput += 1; + module_outputs.resize(noutput); + + vector_style.resize(noutput); + vector_style[noutput - 1] = NODAL; + + output_vector_sizes.resize(noutput); + output_vector_sizes[noutput - 1] = Brows; + + output_dof_names.resize(noutput); + output_dof_names[noutput - 1].resize(Brows); + if (num_dim == 2) + { + output_dof_names[noutput - 1][0] = "stress_xx"; + output_dof_names[noutput - 1][1] = "stress_yy"; + output_dof_names[noutput - 1][3] = "stress_xy"; + } + if (num_dim == 3) + { + output_dof_names[noutput - 1][0] = "stress_xx"; + output_dof_names[noutput - 1][1] = "stress_yy"; + output_dof_names[noutput - 1][2] = "stress_zz"; + output_dof_names[noutput - 1][3] = "stress_xy"; + output_dof_names[noutput - 1][4] = "stress_xz"; + output_dof_names[noutput - 1][5] = "stress_yz"; + } + } +} + +/* ------------------------------------------------------------------------------------------- + Prompts sorting for elastic response output data. For now, nodal strains. +---------------------------------------------------------------------------------------------- */ + +void FEA_Module_SGH::sort_output(Teuchos::RCP> sorted_map) +{ +} + +/* ------------------------------------------------------------------------------------------- + populate requests this module makes for output data +---------------------------------------------------------------------------------------------- */ + +void FEA_Module_SGH::write_data(std::map& point_data_scalars_double, + std::map& point_data_vectors_double, + std::map& cell_data_scalars_double, + std::map& cell_data_scalars_int, + std::map>& cell_data_fields_double) +{ + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + + for (const auto& field_name : simparam->output_options.output_fields) + { + switch (field_name) + { + case FIELD::velocity: + // node "velocity" + node_vel.update_host(); + point_data_vectors_double["velocity"] = &node_vel.host(rk_level, 0, 0); + break; + + case FIELD::element_density: + // element "density" + elem_den.update_host(); + cell_data_scalars_double["element_density"] = elem_den.host_pointer(); + break; + + case FIELD::pressure: + // element "pressure" + elem_pres.update_host(); + cell_data_scalars_double["pressure"] = elem_pres.host_pointer(); + break; + + case FIELD::SIE: + // element "SIE" + elem_sie.update_host(); + cell_data_scalars_double["SIE"] = &elem_sie.host(rk_level, 0); + break; + + case FIELD::volume: + // element "volume" + elem_vol.update_host(); + cell_data_scalars_double["volume"] = elem_vol.host_pointer(); + break; + + case FIELD::mass: + // element "mass" + elem_mass.update_host(); + cell_data_scalars_double["mass"] = elem_mass.host_pointer(); + break; + + case FIELD::sound_speed: + // element "sspd" + elem_sspd.update_host(); + cell_data_scalars_double["sound_speed"] = elem_sspd.host_pointer(); + break; + + case FIELD::material_id: + // element "material_id" + elem_mat_id.update_host(); + cell_data_scalars_int["material_id"] = reinterpret_cast(elem_mat_id.host_pointer()); + break; + + case FIELD::user_vars: + // element "user_vars" + elem_user_output_vars.update_host(); + cell_data_fields_double["user_vars"] = std::make_pair(elem_user_output_vars.host_pointer(), + elem_user_output_vars.dims(1)); + case FIELD::stress: + // element "stress" + elem_stress.update_host(); + cell_data_fields_double["stress"] = std::make_pair(&elem_stress.host(rk_level, 0, 0, 0), 9); + break; + + default: + break; + } // end switch + } // end if + + // element "mat_id" //uncomment if needed (works fine) + // sgh_module->elem_mat_id.update_host(); + // cell_data_scalars_int["mat_id"] = reinterpret_cast(&sgh_module->elem_mat_id.host(0)); + + // element "user_output_vars" //uncomment if needed (works fine) + // sgh_module->elem_user_output_vars.update_host(); + // cell_data_fields_double["user_output_vars"] = std::make_pair(&sgh_module->elem_user_output_vars.host_pointer(), + // sgh_module->elem_user_output_vars.dims(1)); + + // element "stress" //uncomment if needed (works fine) + // sgh_module->elem_stress.update_host(); + // cell_data_fields_double["stress"] = std::make_pair(&sgh_module->elem_stress.host(rk_level,0,0,0), 9); +} + +/* ------------------------------------------------------------------------------------------- + Prompts sorting for elastic response output data. For now, nodal strains. +---------------------------------------------------------------------------------------------- */ + +void FEA_Module_SGH::sort_element_output(Teuchos::RCP> sorted_map) +{ + // interface element density data + { + host_vec_array Element_Densities = Global_Element_Densities->getLocalView(Tpetra::Access::ReadWrite); + elem_den.update_host(); + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + Element_Densities(ielem, 0) = elem_den.host(ielem); + } + } +} + +/* ------------------------------------------------------------------------------------------- + Prompts computation of elastic response output data. For now, nodal strains. +---------------------------------------------------------------------------------------------- */ + +void FEA_Module_SGH::collect_output(Teuchos::RCP> global_reduce_map) +{ +} + +/* ------------------------------------------------------------------------------------------- + Prompts computation of elastic response output data. For now, nodal strains. +---------------------------------------------------------------------------------------------- */ + +void FEA_Module_SGH::compute_output() +{ +} + +/* ---------------------------------------------------------------------- + Communicate updated nodal velocities to ghost nodes +------------------------------------------------------------------------- */ + +void FEA_Module_SGH::comm_node_masses() +{ + // debug print of design vector + // std::ostream &out = std::cout; + // Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + // if(myrank==0) + // *fos << "Density data :" << std::endl; + // node_densities_distributed->describe(*fos,Teuchos::VERB_EXTREME); + // *fos << std::endl; + // std::fflush(stdout); + + // communicate design densities + // create import object using local node indices map and all indices map + // Tpetra::Import importer(map, ghost_node_map); + + // comms to get ghosts + ghost_node_masses_distributed->doImport(*node_masses_distributed, *ghost_importer, Tpetra::INSERT); + // all_node_map->describe(*fos,Teuchos::VERB_EXTREME); + // all_node_velocities_distributed->describe(*fos,Teuchos::VERB_EXTREME); + + // update_count++; + // if(update_count==1){ + // MPI_Barrier(world); + // MPI_Abort(world,4); + // } +} + +/* ------------------------------------------------------------------------------------------- + Communicate ghosts using the current optimization design data +---------------------------------------------------------------------------------------------- */ + +void FEA_Module_SGH::comm_variables(Teuchos::RCP zp) +{ + if (simparam->topology_optimization_on) + { + // set density vector to the current value chosen by the optimizer + test_node_densities_distributed = zp; + + // debug print of design vector + // std::ostream &out = std::cout; + // Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + // if(myrank==0) + // *fos << "Density data :" << std::endl; + // node_densities_distributed->describe(*fos,Teuchos::VERB_EXTREME); + // *fos << std::endl; + // std::fflush(stdout); + + // communicate design densities + // create import object using local node indices map and all indices map + // Tpetra::Import importer(map, all_node_map); + + // comms to get ghosts + all_node_densities_distributed->doImport(*test_node_densities_distributed, *importer, Tpetra::INSERT); + } + else if (simparam->shape_optimization_on) + { + // clause to communicate boundary node data if the boundary nodes are ghosts on this rank + } +} + +/* ------------------------------------------------------------------------------------------- + enforce density constraints on nodes due to BCS +---------------------------------------------------------------------------------------------- */ +void FEA_Module_SGH::node_density_constraints(host_vec_array node_densities_lower_bound) +{ + const size_t num_dim = mesh->num_dims; + const_vec_array all_initial_node_coords = all_initial_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + const size_t num_lcs = module_params->loading_conditions.size(); + + const DCArrayKokkos mat_fill = simparam->mat_fill; + const DCArrayKokkos loading = module_params->loading; + + // debug check + // std::cout << "NUMBER OF LOADING CONDITIONS: " << num_lcs << std::endl; + + // walk over the nodes to update the velocity + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + double current_node_coords[3]; + double radius; + for (size_t dim = 0; dim < num_dim; dim++) + { + current_node_coords[dim] = all_initial_node_coords(node_gid, dim); + } // end for dim + radius = sqrt(current_node_coords[0] * current_node_coords[0] + + current_node_coords[1] * current_node_coords[1] + + current_node_coords[2] * current_node_coords[2]); + for (size_t ilc = 0; ilc < num_lcs; ilc++) + { + // debug check + // std::cout << "LOADING CONDITION VOLUME TYPE: " << to_string(loading(ilc).volume) << std::endl; + + bool fill_this = loading(ilc).volume.contains(current_node_coords); + if (fill_this) + { + node_densities_lower_bound(node_gid, 0) = 1; + } + } + }); // end for parallel for over nodes +} + +/* ---------------------------------------------------------------------------- + solve function called by solver +------------------------------------------------------------------------------- */ + +void FEA_Module_SGH::module_cleanup() +{ + cleanup_material_models(); +} + +/* ---------------------------------------------------------------------------- + Deallocate memory used for material models +------------------------------------------------------------------------------- */ + +void FEA_Module_SGH::cleanup_material_models() +{ + const DCArrayKokkos material = simparam->material; + + // destroy strength model + destroy_strength_model(elem_strength, + material, + elem_mat_id, + state_vars, + global_vars, + elem_user_output_vars, + rnum_elem); + + // destroy eos model + destroy_eos_model(elem_eos, + material, + elem_mat_id, + state_vars, + global_vars, + elem_user_output_vars, + rnum_elem); + return; +} // end cleanup_user_strength_model; + +/** + * Determines which of the boundary patches are associated with which boundary. + * + * Modifies: bdy_patches_in_set +*/ +void FEA_Module_SGH::tag_bdys(const DCArrayKokkos& boundary, + mesh_t& mesh, + const DViewCArrayKokkos& node_coords) +{ + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + size_t num_dim = simparam->num_dims; + int nboundary_patches = Explicit_Solver_Pointer_->nboundary_patches; + int num_nodes_in_patch = mesh.num_nodes_in_patch; + + // if (bdy_set == mesh.num_bdy_sets){ + // printf(" ERROR: number of boundary sets must be increased by %zu", + // bdy_set-mesh.num_bdy_sets+1); + // exit(0); + // } // end if + + // error and debug flag + // DCArrayKokkos print_flag(1, "print_flag"); + // print_flag.host(0) = false; + // print_flag.update_device(); + + FOR_ALL_CLASS(bdy_set, 0, num_bdy_sets, { + // tag boundaries + BOUNDARY_TYPE bc_type = boundary(bdy_set).surface.type; + double val = boundary(bdy_set).surface.plane_position; + + // save the boundary patches to this set that are on the plane, spheres, etc. + for (size_t bdy_patch_lid = 0; bdy_patch_lid < nboundary_patches; bdy_patch_lid++) + { + // save the patch index + size_t bdy_patch_gid = bdy_patch_lid; + + // check to see if this patch is on the specified plane + bool is_on_bdy = check_bdy(bdy_patch_gid, + num_dim, + num_nodes_in_patch, + bc_type, + val, + node_coords, + rk_level); // no=0, yes=1 + + // debug check + /* + for (size_t patch_node_lid=0; patch_node_lid& node_coords, + const size_t rk_level) const +{ + // default bool is not on the boundary + size_t is_on_bdy = 0; + + // the patch coordinates + double these_patch_coords[3]; // Note: cannot allocated array with num_dim + + // loop over the nodes on the patch + for (size_t patch_node_lid = 0; patch_node_lid < num_nodes_in_patch; patch_node_lid++) + { + // get the nodal_gid for this node in the patch + // size_t node_gid = mesh.nodes_in_patch(patch_gid, patch_node_lid); + size_t node_gid = Local_Index_Boundary_Patches(patch_gid, patch_node_lid); + + for (size_t dim = 0; dim < num_dim; dim++) + { + these_patch_coords[dim] = node_coords(rk_level, node_gid, dim); // (rk, node_gid, dim) + } + + if (bc_type == BOUNDARY_TYPE::x_plane) + { + if (fabs(these_patch_coords[0] - val) <= 1.0e-7) + { + is_on_bdy += 1; + } + } + else if (bc_type == BOUNDARY_TYPE::y_plane) + { + if (fabs(these_patch_coords[1] - val) <= 1.0e-7) + { + is_on_bdy += 1; + } + } + else if (bc_type == BOUNDARY_TYPE::z_plane) + { + if (fabs(these_patch_coords[2] - val) <= 1.0e-7) + { + is_on_bdy += 1; + } + } + else if (bc_type == BOUNDARY_TYPE::cylinder) + { + real_t R = sqrt(these_patch_coords[0] * these_patch_coords[0] + + these_patch_coords[1] * these_patch_coords[1]); + + if (fabs(R - val) <= 1.0e-7) + { + is_on_bdy += 1; + } + } + else if (bc_type == BOUNDARY_TYPE::sphere) + { + real_t R = sqrt(these_patch_coords[0] * these_patch_coords[0] + + these_patch_coords[1] * these_patch_coords[1] + + these_patch_coords[2] * these_patch_coords[2]); + + if (fabs(R - val) <= 1.0e-7) + { + is_on_bdy += 1; + } + } + } + + // if all nodes in the patch are on the surface + return is_on_bdy == num_nodes_in_patch; +} // end method to check bdy + +/* ---------------------------------------------------------------------------- + solve function called by solver +------------------------------------------------------------------------------- */ + +int FEA_Module_SGH::solve() +{ + sgh_solve(); + + return 0; +} + +/* ---------------------------------------------------------------------------- + SGH solver loop +------------------------------------------------------------------------------- */ + +void FEA_Module_SGH::sgh_solve() +{ + Dynamic_Options dynamic_options = simparam->dynamic_options; + + const size_t rk_level = dynamic_options.rk_num_bins - 1; + time_value = dynamic_options.time_initial; + time_final = dynamic_options.time_final; + dt_max = dynamic_options.dt_max; + dt_min = dynamic_options.dt_min; + dt_cfl = dynamic_options.dt_cfl; + graphics_time = simparam->output_options.graphics_step; + graphics_dt_ival = simparam->output_options.graphics_step; + cycle_stop = dynamic_options.cycle_stop; + rk_num_stages = dynamic_options.rk_num_stages; + dt = dynamic_options.dt; + fuzz = dynamic_options.fuzz; + tiny = dynamic_options.tiny; + small = dynamic_options.small; + graphics_times = simparam->output_options.graphics_times; + graphics_id = simparam->output_options.graphics_id; + size_t num_bdy_nodes = mesh->num_bdy_nodes; + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos material = simparam->material; + int nTO_modules; + int old_max_forward_buffer; + size_t cycle; + const int num_dim = simparam->num_dims; + real_t objective_accumulation, global_objective_accumulation; + std::vector> FEA_Module_My_TO_Modules = simparam->FEA_Module_My_TO_Modules; + problem = Explicit_Solver_Pointer_->problem; // Pointer to ROL optimization problem object + ROL::Ptr> obj_pointer; + + // reset time accumulating objective and constraints + /* + for(int imodule = 0 ; imodule < FEA_Module_My_TO_Modules[my_fea_module_index_].size(); imodule++){ + current_module_index = FEA_Module_My_TO_Modules[my_fea_module_index_][imodule]; + //test if module needs reset + if(){ + + } + } + */ + // simple setup to just request KE for now; above loop to be expanded and used later for scanning modules + if (simparam->topology_optimization_on) + { + obj_pointer = problem->getObjective(); + KineticEnergyMinimize_TopOpt& kinetic_energy_minimize_function = dynamic_cast(*obj_pointer); + kinetic_energy_minimize_function.objective_accumulation = 0; + global_objective_accumulation = objective_accumulation = 0; + kinetic_energy_objective = true; + if (max_time_steps + 1 > forward_solve_velocity_data->size()) + { + old_max_forward_buffer = forward_solve_velocity_data->size(); + time_data.resize(max_time_steps + 1); + forward_solve_velocity_data->resize(max_time_steps + 1); + forward_solve_coordinate_data->resize(max_time_steps + 1); + forward_solve_internal_energy_data->resize(max_time_steps + 1); + adjoint_vector_data->resize(max_time_steps + 1); + phi_adjoint_vector_data->resize(max_time_steps + 1); + psi_adjoint_vector_data->resize(max_time_steps + 1); + // assign a multivector of corresponding size to each new timestep in the buffer + for (int istep = old_max_forward_buffer; istep < max_time_steps + 1; istep++) + { + (*forward_solve_velocity_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*forward_solve_coordinate_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*forward_solve_internal_energy_data)[istep] = Teuchos::rcp(new MV(all_element_map, 1)); + (*adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*phi_adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*psi_adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_element_map, 1)); + } + } + } + + if (simparam->topology_optimization_on) + { + nTO_modules = simparam->TO_Module_List.size(); + } + + int myrank = Explicit_Solver_Pointer_->myrank; + if (simparam->output_options.output_file_format == OUTPUT_FORMAT::vtk && simparam->output_options.write_initial) + { + if (myrank == 0) + { + printf("Writing outputs to file at %f \n", time_value); + } + + double comm_time1 = Explicit_Solver_Pointer_->CPU_Time(); + Explicit_Solver_Pointer_->write_outputs(); + double comm_time2 = Explicit_Solver_Pointer_->CPU_Time(); + Explicit_Solver_Pointer_->output_time += comm_time2 - comm_time1; + } + + CArrayKokkos node_extensive_mass(nall_nodes, "node_extensive_mass"); + + // extensive energy tallies over the mesh elements local to this MPI rank + double IE_t0 = 0.0; + double KE_t0 = 0.0; + double TE_t0 = 0.0; + + double IE_sum = 0.0; + double KE_sum = 0.0; + + double IE_loc_sum = 0.0; + double KE_loc_sum = 0.0; + + // extensive energy tallies over the entire mesh + double global_IE_t0 = 0.0; + double global_KE_t0 = 0.0; + double global_TE_t0 = 0.0; + + // ---- Calculate energy tallies ---- + double IE_tend = 0.0; + double KE_tend = 0.0; + double TE_tend = 0.0; + + double global_IE_tend = 0.0; + double global_KE_tend = 0.0; + double global_TE_tend = 0.0; + + int nlocal_elem_non_overlapping = Explicit_Solver_Pointer_->nlocal_elem_non_overlapping; + + // extensive IE + REDUCE_SUM_CLASS(elem_gid, 0, nlocal_elem_non_overlapping, IE_loc_sum, { + IE_loc_sum += elem_mass(elem_gid) * elem_sie(rk_level, elem_gid); + }, IE_sum); + IE_t0 = IE_sum; + + MPI_Allreduce(&IE_t0, &global_IE_t0, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); + + // extensive KE + REDUCE_SUM_CLASS(node_gid, 0, nlocal_nodes, KE_loc_sum, { + double ke = 0; + for (size_t dim = 0; dim < num_dim; dim++) + { + ke += node_vel(rk_level, node_gid, dim) * node_vel(rk_level, node_gid, dim); // 1/2 at end + } // end for + + if (num_dim == 2) + { + KE_loc_sum += node_mass(node_gid) * node_coords(rk_level, node_gid, 1) * ke; + } + else + { + KE_loc_sum += node_mass(node_gid) * ke; + } + }, KE_sum); + Kokkos::fence(); + KE_t0 = 0.5 * KE_sum; + + MPI_Allreduce(&KE_t0, &global_KE_t0, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); + + // extensive TE + global_TE_t0 = global_IE_t0 + global_KE_t0; + TE_t0 = global_TE_t0; + KE_t0 = global_KE_t0; + IE_t0 = global_IE_t0; + + // save the nodal mass + FOR_ALL_CLASS(node_gid, 0, nall_nodes, { + double radius = 1.0; + if (num_dim == 2) + { + radius = node_coords(rk_level, node_gid, 1); + } + node_extensive_mass(node_gid) = node_mass(node_gid) * radius; + }); // end parallel for + + // a flag to exit the calculation + size_t stop_calc = 0; + + auto time_1 = std::chrono::high_resolution_clock::now(); + + // save initial data + if (simparam->topology_optimization_on || simparam->shape_optimization_on) + { + time_data[0] = 0; + // assign current velocity data to multivector + // view scope + { + vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); + vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + node_velocities_interface(node_gid, idim) = node_vel(rk_level, node_gid, idim); + node_coords_interface(node_gid, idim) = node_coords(rk_level, node_gid, idim); + } + }); + } // end view scope + Kokkos::fence(); + + // communicate ghosts + double comm_time1 = Explicit_Solver_Pointer_->CPU_Time(); + + // active view scope; triggers host comms from updated data on device + { + const_host_vec_array node_velocities_host = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_host_vec_array node_coords_host = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + } + double comm_time2 = Explicit_Solver_Pointer_->CPU_Time(); + Explicit_Solver_Pointer_->dev2host_time += comm_time2 - comm_time1; + + // communicate ghost velocities + Explicit_Solver_Pointer_->comm_velocities(); + Explicit_Solver_Pointer_->comm_coordinates(); + + double comm_time3 = Explicit_Solver_Pointer_->CPU_Time(); + + // view scope + { + const_vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array ghost_node_velocities_interface = Explicit_Solver_Pointer_->ghost_node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); + vec_array all_node_velocities_interface = Explicit_Solver_Pointer_->all_node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); + vec_array element_internal_energy = element_internal_energy_distributed->getLocalView(Tpetra::Access::ReadWrite); + const_vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array ghost_node_coords_interface = Explicit_Solver_Pointer_->ghost_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + all_node_velocities_interface(node_gid, idim) = node_velocities_interface(node_gid, idim); + all_node_coords_interface(node_gid, idim) = node_coords_interface(node_gid, idim); + } + }); // end parallel for + Kokkos::fence(); + + FOR_ALL_CLASS(node_gid, nlocal_nodes, nlocal_nodes + nghost_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + all_node_velocities_interface(node_gid, idim) = ghost_node_velocities_interface(node_gid - nlocal_nodes, idim); + all_node_coords_interface(node_gid, idim) = ghost_node_coords_interface(node_gid - nlocal_nodes, idim); + } + }); // end parallel for + Kokkos::fence(); + + // interface for element internal energies + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { + element_internal_energy(elem_gid, 0) = elem_sie(rk_level, elem_gid); + }); // end parallel for + Kokkos::fence(); + } // end view scope + + (*forward_solve_internal_energy_data)[0]->assign(*element_internal_energy_distributed); + (*forward_solve_velocity_data)[0]->assign(*Explicit_Solver_Pointer_->all_node_velocities_distributed); + (*forward_solve_coordinate_data)[0]->assign(*Explicit_Solver_Pointer_->all_node_coords_distributed); + } + + // loop over the max number of time integration cycles + for (cycle = 0; cycle < cycle_stop; cycle++) + { + // get the step + if (num_dim == 2) + { + get_timestep2D(*mesh, + node_coords, + node_vel, + elem_sspd, + elem_vol); + } + else + { + get_timestep(*mesh, + node_coords, + node_vel, + elem_sspd, + elem_vol); + } // end if 2D + + double global_dt; + MPI_Allreduce(&dt, &global_dt, 1, MPI_DOUBLE, MPI_MIN, MPI_COMM_WORLD); + dt = global_dt; + + // stop calculation if flag + // if (stop_calc == 1) break; + + if (simparam->dynamic_options.output_time_sequence_level >= TIME_OUTPUT_LEVEL::high) + { + if (cycle == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %12.5e, time step = %12.5e \n", cycle, time_value, dt); + } + } + // print time step every 10 cycles + else if (cycle % 20 == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %12.5e, time step = %12.5e \n", cycle, time_value, dt); + } + } // end if + } + + // --------------------------------------------------------------------- + // integrate the solution forward to t(n+1) via Runge Kutta (RK) method + // --------------------------------------------------------------------- + + // save the values at t_n + rk_init(node_coords, + node_vel, + elem_sie, + elem_stress, + rnum_elem, + nall_nodes); + + // integrate solution forward in time + for (size_t rk_stage = 0; rk_stage < rk_num_stages; rk_stage++) + { + // ---- RK coefficient ---- + double rk_alpha = 1.0 / ((double)rk_num_stages - (double)rk_stage); + + // ---- Calculate velocity diveregence for the element ---- + if (num_dim == 2) + { + get_divergence2D(elem_div, + node_coords, + node_vel, + elem_vol); + } + else + { + get_divergence(elem_div, + node_coords, + node_vel, + elem_vol); + } // end if 2D + + // ---- calculate the forces on the vertices and evolve stress (hypo model) ---- + if (num_dim == 2) + { + get_force_sgh2D(material, + *mesh, + node_coords, + node_vel, + elem_den, + elem_sie, + elem_pres, + elem_stress, + elem_sspd, + elem_vol, + elem_div, + elem_mat_id, + corner_force, + rk_alpha, + cycle); + } + else + { + get_force_sgh(material, + *mesh, + node_coords, + node_vel, + elem_den, + elem_sie, + elem_pres, + elem_stress, + elem_sspd, + elem_vol, + elem_div, + elem_mat_id, + corner_force, + rk_alpha, + cycle); + } + + /* + debug block + if(myrank==1){ + std::cout << rk_alpha << " " << dt << std::endl; + for(int i = 0; i < nall_nodes; i++){ + double node_force[3]; + for (size_t dim = 0; dim < num_dim; dim++){ + node_force[dim] = 0.0; + } // end for dim + + // loop over all corners around the node and calculate the nodal force + for (size_t corner_lid=0; corner_lidall_node_map->getGlobalElement(i) << " " << corner_gid << " " << corner_force(corner_gid, 0) << " " << corner_force(corner_gid, 1) << " " << corner_force(corner_gid, 2) << std::endl; + // loop over dimension + for (size_t dim = 0; dim < num_dim; dim++){ + node_force[dim] += corner_force(corner_gid, dim); + } // end for dim + + } // end for corner_lid + //std::cout << Explicit_Solver_Pointer_->all_node_map->getGlobalElement(i) << " " << node_force[0] << " " << node_force[1] << " " << node_force[2] << std::endl; + //std::cout << Explicit_Solver_Pointer_->all_node_map->getGlobalElement(i) << " " << node_mass(i) << std::endl; + } + } + /* + //debug print vector values on a rank + /* + if(myrank==0) + for(int i = 0; i < nall_nodes; i++){ + std::cout << Explicit_Solver_Pointer_->all_node_map->getGlobalElement(i) << " " << node_vel(rk_level,i,0) << " " << node_vel(rk_level,i,1) << " " << node_vel(rk_level,i,2) << std::endl; + } + */ + + // ---- Update nodal velocities ---- // + update_velocity_sgh(rk_alpha, + node_vel, + node_mass, + corner_force); + + if (have_loading_conditions) + { + applied_forces(material, + *mesh, + node_coords, + node_vel, + node_mass, + elem_den, + elem_vol, + elem_div, + elem_mat_id, + corner_force, + rk_alpha, + cycle); + } + + // ---- apply force boundary conditions to the boundary patches---- + boundary_velocity(*mesh, boundary, node_vel); + + // current interface has differing velocity arrays; this equates them until we unify memory + // first comm time interval point + double comm_time1 = Explicit_Solver_Pointer_->CPU_Time(); + // view scope + { + vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + node_velocities_interface(node_gid, idim) = node_vel(rk_level, node_gid, idim); + } + }); // end parallel for + } // end view scope + Kokkos::fence(); + + // active view scope + { + const_host_vec_array node_velocities_host = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); + } + double comm_time2 = Explicit_Solver_Pointer_->CPU_Time(); + Explicit_Solver_Pointer_->dev2host_time += comm_time2 - comm_time1; + // communicate ghost velocities + Explicit_Solver_Pointer_->comm_velocities(); + + double comm_time3 = Explicit_Solver_Pointer_->CPU_Time(); + // this is forcing a copy to the device + // view scope + { + vec_array ghost_node_velocities_interface = Explicit_Solver_Pointer_->ghost_node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_gid, nlocal_nodes, nall_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + node_vel(rk_level, node_gid, idim) = ghost_node_velocities_interface(node_gid - nlocal_nodes, idim); + } + }); // end parallel for + } // end view scope + Kokkos::fence(); + + double comm_time4 = Explicit_Solver_Pointer_->CPU_Time(); + Explicit_Solver_Pointer_->host2dev_time += comm_time4 - comm_time3; + Explicit_Solver_Pointer_->communication_time += comm_time4 - comm_time1; + // debug print vector values on a rank + /* + if(myrank==0) + for(int i = 0; i < nall_nodes; i++){ + std::cout << Explicit_Solver_Pointer_->all_node_map->getGlobalElement(i) << " " << node_vel(rk_level,i,0) << " " << node_vel(rk_level,i,1) << " " << node_vel(rk_level,i,2) << std::endl; + } + */ + // ---- Update specific internal energy in the elements ---- + update_energy_sgh(rk_alpha, + *mesh, + node_vel, + node_coords, + elem_sie, + elem_mass, + corner_force); + + // ---- Update nodal positions ---- + update_position_sgh(rk_alpha, + nall_nodes, + node_coords, + node_vel); + + // ---- Calculate cell volume for next time step ---- + get_vol(); + + // ---- Calculate elem state (den, pres, sound speed, stress) for next time step ---- + if (num_dim == 2) + { + update_state2D(material, + *mesh, + node_coords, + node_vel, + elem_den, + elem_pres, + elem_stress, + elem_sspd, + elem_sie, + elem_vol, + elem_mass, + elem_mat_id, + rk_alpha, + cycle); + } + else + { + update_state(material, + *mesh, + node_coords, + node_vel, + elem_den, + elem_pres, + elem_stress, + elem_sspd, + elem_sie, + elem_vol, + elem_mass, + elem_mat_id, + rk_alpha, + cycle); + } + // ---- + // Notes on strength: + // 1) hyper-elastic strength models are called in update_state + // 2) hypo-elastic strength models are called in get_force + // 3) strength models must be added by the user in user_mat.cpp + + // calculate the new corner masses if 2D + if (num_dim == 2) + { + // calculate the nodal areal mass + FOR_ALL_CLASS(node_gid, 0, nall_nodes, { + node_mass(node_gid) = 0.0; + + if (node_coords(rk_level, node_gid, 1) > tiny) + { + node_mass(node_gid) = node_extensive_mass(node_gid) / node_coords(rk_level, node_gid, 1); + } + // if(cycle==0&&node_gid==1&&myrank==0) + // std::cout << "index " << node_gid << " on rank " << myrank << " node vel " << node_vel(rk_level,node_gid,0) << " " << node_mass(node_gid) << std::endl << std::flush; + }); // end parallel for over node_gid + Kokkos::fence(); + + // current interface has differing density arrays; this equates them until we unify memory + // view scope + { + vec_array node_mass_interface = node_masses_distributed->getLocalView(Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + node_mass_interface(node_gid, 0) = node_mass(node_gid); + }); // end parallel for + } // end view scope + Kokkos::fence(); + // communicate ghost densities + comm_node_masses(); + + // this is forcing a copy to the device + // view scope + { + vec_array ghost_node_mass_interface = ghost_node_masses_distributed->getLocalView(Tpetra::Access::ReadWrite); + + FOR_ALL_CLASS(node_gid, nlocal_nodes, nall_nodes, { + node_mass(node_gid) = ghost_node_mass_interface(node_gid - nlocal_nodes, 0); + }); // end parallel for + } // end view scope + Kokkos::fence(); + + // ----------------------------------------------- + // Calcualte the areal mass for nodes on the axis + // ----------------------------------------------- + // The node order of the 2D element is + // + // J + // | + // 3---2 + // | | -- I + // 0---1 + /* + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { + + // loop over the corners of the element and calculate the mass + for (size_t node_lid=0; node_lid<4; node_lid++){ + + size_t node_gid = nodes_in_elem(elem_gid, node_lid); + size_t node_minus_gid; + size_t node_plus_gid; + + + if (node_coords(rk_level,node_gid,1) < tiny){ + // node is on the axis + + // minus node + if (node_lid==0){ + node_minus_gid = nodes_in_elem(elem_gid, 3); + } else { + node_minus_gid = nodes_in_elem(elem_gid, node_lid-1); + } + + // plus node + if (node_lid==3){ + node_plus_gid = nodes_in_elem(elem_gid, 0); + } else { + node_plus_gid = nodes_in_elem(elem_gid, node_lid+1); + } + + node_mass(node_gid) = fmax(node_mass(node_plus_gid), node_mass(node_minus_gid))/2.0; + + } // end if + + } // end for over corners + + }); // end parallel for over elem_gid + Kokkos::fence(); + */ + + FOR_ALL_CLASS(node_bdy_gid, 0, num_bdy_nodes, { + // FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + size_t node_gid = bdy_nodes(node_bdy_gid); + + if (node_coords(rk_level, node_gid, 1) < tiny) + { + // node is on the axis + + for (size_t node_lid = 0; node_lid < num_nodes_in_node(node_gid); node_lid++) + { + size_t node_neighbor_gid = nodes_in_node(node_gid, node_lid); + + // if the node is off the axis, use it's areal mass on the boundary + if (node_coords(rk_level, node_neighbor_gid, 1) > tiny) + { + node_mass(node_gid) = fmax(node_mass(node_gid), node_mass(node_neighbor_gid) / 2.0); + } + } // end for over neighboring nodes + } // end if + }); // end parallel for over elem_gid + } // end of if 2D-RZ + } // end of RK loop + + // increment the time + Explicit_Solver_Pointer_->time_value = simparam->dynamic_options.time_value = time_value += dt; + + if (simparam->topology_optimization_on || simparam->shape_optimization_on) + { + if (cycle >= max_time_steps) + { + max_time_steps = cycle + 1; + } + + if (max_time_steps + 1 > forward_solve_velocity_data->size()) + { + old_max_forward_buffer = forward_solve_velocity_data->size(); + time_data.resize(max_time_steps + BUFFER_GROW + 1); + forward_solve_velocity_data->resize(max_time_steps + BUFFER_GROW + 1); + forward_solve_coordinate_data->resize(max_time_steps + BUFFER_GROW + 1); + forward_solve_internal_energy_data->resize(max_time_steps + BUFFER_GROW + 1); + adjoint_vector_data->resize(max_time_steps + BUFFER_GROW + 1); + phi_adjoint_vector_data->resize(max_time_steps + BUFFER_GROW + 1); + psi_adjoint_vector_data->resize(max_time_steps + BUFFER_GROW + 1); + // assign a multivector of corresponding size to each new timestep in the buffer + for (int istep = old_max_forward_buffer; istep < max_time_steps + BUFFER_GROW + 1; istep++) + { + (*forward_solve_velocity_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*forward_solve_coordinate_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*forward_solve_internal_energy_data)[istep] = Teuchos::rcp(new MV(all_element_map, 1)); + (*adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*phi_adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*psi_adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_element_map, 1)); + } + } + + time_data[cycle + 1] = dt + time_data[cycle]; + + // assign current velocity data to multivector + // view scope + { + vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); + vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + node_velocities_interface(node_gid, idim) = node_vel(rk_level, node_gid, idim); + node_coords_interface(node_gid, idim) = node_coords(rk_level, node_gid, idim); + } + }); + } // end view scope + Kokkos::fence(); + + // communicate ghosts + double comm_time1 = Explicit_Solver_Pointer_->CPU_Time(); + + // active view scope; triggers host comms from updated data on device + { + const_host_vec_array node_velocities_host = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_host_vec_array node_coords_host = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + } + double comm_time2 = Explicit_Solver_Pointer_->CPU_Time(); + Explicit_Solver_Pointer_->dev2host_time += comm_time2 - comm_time1; + + // communicate ghost velocities + Explicit_Solver_Pointer_->comm_velocities(); + Explicit_Solver_Pointer_->comm_coordinates(); + + double comm_time3 = Explicit_Solver_Pointer_->CPU_Time(); + + // view scope + { + const_vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array ghost_node_velocities_interface = Explicit_Solver_Pointer_->ghost_node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); + vec_array all_node_velocities_interface = Explicit_Solver_Pointer_->all_node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); + vec_array element_internal_energy = element_internal_energy_distributed->getLocalView(Tpetra::Access::ReadWrite); + const_vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array ghost_node_coords_interface = Explicit_Solver_Pointer_->ghost_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + all_node_velocities_interface(node_gid, idim) = node_velocities_interface(node_gid, idim); + all_node_coords_interface(node_gid, idim) = node_coords_interface(node_gid, idim); + } + }); // end parallel for + Kokkos::fence(); + + FOR_ALL_CLASS(node_gid, nlocal_nodes, nlocal_nodes + nghost_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + all_node_velocities_interface(node_gid, idim) = ghost_node_velocities_interface(node_gid - nlocal_nodes, idim); + all_node_coords_interface(node_gid, idim) = ghost_node_coords_interface(node_gid - nlocal_nodes, idim); + } + }); // end parallel for + Kokkos::fence(); + + // interface for element internal energies + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { + element_internal_energy(elem_gid, 0) = elem_sie(rk_level, elem_gid); + }); // end parallel for + Kokkos::fence(); + } // end view scope + + double comm_time4 = Explicit_Solver_Pointer_->CPU_Time(); + Explicit_Solver_Pointer_->host2dev_time += comm_time4 - comm_time3; + Explicit_Solver_Pointer_->communication_time += comm_time4 - comm_time1; + + (*forward_solve_internal_energy_data)[cycle + 1]->assign(*element_internal_energy_distributed); + (*forward_solve_velocity_data)[cycle + 1]->assign(*Explicit_Solver_Pointer_->all_node_velocities_distributed); + (*forward_solve_coordinate_data)[cycle + 1]->assign(*Explicit_Solver_Pointer_->all_node_coords_distributed); + + // kinetic energy accumulation + if (kinetic_energy_objective) + { + const_vec_array node_velocities_interface = (*forward_solve_velocity_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array previous_node_velocities_interface = (*forward_solve_velocity_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + KE_loc_sum = 0.0; + KE_sum = 0.0; + // extensive KE + REDUCE_SUM_CLASS(node_gid, 0, nlocal_nodes, KE_loc_sum, { + double ke = 0; + for (size_t dim = 0; dim < num_dim; dim++) + { + // midpoint integration approximation + ke += (node_velocities_interface(node_gid, dim) + previous_node_velocities_interface(node_gid, dim)) * (node_velocities_interface(node_gid, + dim) + previous_node_velocities_interface(node_gid, dim)) / 4; // 1/2 at end + } // end for + + if (num_dim == 2) + { + KE_loc_sum += node_mass(node_gid) * node_coords(rk_level, node_gid, 1) * ke; + } + else + { + KE_loc_sum += node_mass(node_gid) * ke; + } + }, KE_sum); + Kokkos::fence(); + KE_sum = 0.5 * KE_sum; + objective_accumulation += KE_sum * dt; + } + } + + size_t write = 0; + if ((cycle + 1) % graphics_cyc_ival == 0 && cycle > 0) + { + write = 1; + } + else if (cycle == cycle_stop) + { + write = 1; + } + else if (time_value >= time_final && simparam->output_options.write_final) + { + write = 1; + } + else if (time_value >= graphics_time) + { + write = 1; + } + + // write outputs + if (write == 1) + { + // interface nodal coordinate data (note: this is not needed if using write_outputs()) + // view scope + { + vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + node_coords_interface(node_gid, idim) = node_coords(rk_level, node_gid, idim); + } + }); // end parallel for + } // end view scope + if (simparam->output_options.output_file_format == OUTPUT_FORMAT::vtk) + { + if (myrank == 0) + { + printf("Writing outputs to file at %f \n", graphics_time); + } + + double comm_time1 = Explicit_Solver_Pointer_->CPU_Time(); + Explicit_Solver_Pointer_->write_outputs(); + + double comm_time2 = Explicit_Solver_Pointer_->CPU_Time(); + Explicit_Solver_Pointer_->output_time += comm_time2 - comm_time1; + } + + graphics_time = time_value + graphics_dt_ival; + } // end if + + // end of calculation + if (time_value >= time_final) + { + break; + } + } // end for cycle loop + + last_time_step = cycle; + + // simple setup to just calculate KE minimize objective for now + if (simparam->topology_optimization_on) + { + KineticEnergyMinimize_TopOpt& kinetic_energy_minimize_function = dynamic_cast(*obj_pointer); + + // collect local objective values + MPI_Allreduce(&objective_accumulation, &global_objective_accumulation, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); + kinetic_energy_minimize_function.objective_accumulation = global_objective_accumulation; + + if (myrank == 0) + { + std::cout << "CURRENT TIME INTEGRAL OF KINETIC ENERGY " << global_objective_accumulation << std::endl; + } + } + + auto time_2 = std::chrono::high_resolution_clock::now(); + auto time_difference = time_2 - time_1; + // double calc_time = std::chrono::duration_cast(diff).count(); + double calc_time = std::chrono::duration_cast(time_difference).count(); + if (myrank == 0) + { + printf("\nCalculation time in seconds: %f \n", calc_time * 1e-09); + } + + IE_loc_sum = 0.0; + KE_loc_sum = 0.0; + IE_sum = 0.0; + KE_sum = 0.0; + + // extensive IE + REDUCE_SUM_CLASS(elem_gid, 0, nlocal_elem_non_overlapping, IE_loc_sum, { + IE_loc_sum += elem_mass(elem_gid) * elem_sie(rk_level, elem_gid); + }, IE_sum); + IE_tend = IE_sum; + + // reduce over MPI ranks + MPI_Allreduce(&IE_tend, &global_IE_tend, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); + + // extensive KE + REDUCE_SUM_CLASS(node_gid, 0, nlocal_nodes, KE_loc_sum, { + double ke = 0; + for (size_t dim = 0; dim < num_dim; dim++) + { + ke += node_vel(rk_level, node_gid, dim) * node_vel(rk_level, node_gid, dim); // 1/2 at end + } // end for + + if (num_dim == 2) + { + KE_loc_sum += node_mass(node_gid) * node_coords(rk_level, node_gid, 1) * ke; + } + else + { + KE_loc_sum += node_mass(node_gid) * ke; + } + }, KE_sum); + Kokkos::fence(); + KE_tend = 0.5 * KE_sum; + + // reduce over MPI ranks + MPI_Allreduce(&KE_tend, &global_KE_tend, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); + + // extensive TE + TE_tend = IE_tend + KE_tend; + KE_tend = global_KE_tend; + IE_tend = global_IE_tend; + + // extensive TE + TE_tend = IE_tend + KE_tend; + + // reduce over MPI ranks + + if (myrank == 0) + { + printf("Time=0: KE = %20.15f, IE = %20.15f, TE = %20.15f \n", KE_t0, IE_t0, TE_t0); + } + if (myrank == 0) + { + printf("Time=End: KE = %20.15f, IE = %20.15f, TE = %20.15f \n", KE_tend, IE_tend, TE_tend); + } + if (myrank == 0) + { + printf("total energy conservation error = %e \n\n", 100 * (TE_tend - TE_t0) / TE_t0); + } + + return; +} // end of SGH solve diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/boundary.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/boundary.cpp similarity index 50% rename from src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/boundary.cpp rename to src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/boundary.cpp index c94a8a961..5996ca246 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/boundary.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/boundary.cpp @@ -1,77 +1,67 @@ -// ------------------------------------------------------- -// This function applys the boundary condition -// to points on a list of patches created at setup -//-------------------------------------------------------- - #include "mesh.h" #include "state.h" #include "FEA_Module_SGH.h" #include "Simulation_Parameters/FEA_Module/Boundary_Conditions.h" -void FEA_Module_SGH::boundary_velocity(const mesh_t &mesh, - const DCArrayKokkos &boundary, - DViewCArrayKokkos &node_vel){ +void FEA_Module_SGH::boundary_velocity(const mesh_t& mesh, + const DCArrayKokkos& boundary, + DViewCArrayKokkos& node_vel) +{ + // error and debug flag + // DCArrayKokkos print_flag(1, "print_flag"); + // print_flag.host(0) = false; + // print_flag.update_device(); - //error and debug flag - //DCArrayKokkos print_flag(1, "print_flag"); - //print_flag.host(0) = false; - //print_flag.update_device(); - - const size_t rk_level = rk_num_bins - 1; - int num_dims = num_dim; + const size_t rk_level = rk_num_bins - 1; + int num_dims = num_dim; // Loop over boundary sets - for (size_t bdy_set=0; bdy_set& node_vel, + const DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& corner_force) +{ + const size_t rk_level = rk_num_bins - 1; + int num_dims = num_dim; + + // loop over all the elements in the mesh + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { + double elem_power = 0.0; + + // --- tally the contribution from each corner to the element --- + + // Loop over the nodes in the element + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + size_t corner_lid = node_lid; + + // Get node global id for the local node id + size_t node_gid = nodes_in_elem(elem_gid, node_lid); + + // Get the corner global id for the local corner id + size_t corner_gid = corners_in_elem(elem_gid, corner_lid); + + double node_radius = 1; + if (num_dims == 2) + { + node_radius = node_coords(rk_level, node_gid, 1); + } + + // calculate the Power=F dot V for this corner + for (size_t dim = 0; dim < num_dims; dim++) + { + double half_vel = (node_vel(rk_level, node_gid, dim) + node_vel(0, node_gid, dim)) * 0.5; + elem_power += corner_force(corner_gid, dim) * node_radius * half_vel; + } // end for dim + } // end for node_lid + + // update the specific energy + elem_sie(rk_level, elem_gid) = elem_sie(0, elem_gid) - + rk_alpha * dt / elem_mass(elem_gid) * elem_power; + }); // end parallel loop over the elements + + return; +} // end subroutine diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp new file mode 100644 index 000000000..9ef23e8c1 --- /dev/null +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp @@ -0,0 +1,2063 @@ + +#include "mesh.h" +#include "state.h" +#include +#include +#include +#include +#include +#include // fmin, fmax, abs note: fminl is long +#include +#include + +#include +#include +#include + +#include "elements.h" +#include "swage.h" +#include "matar.h" +#include "utilities.h" +#include "FEA_Module_SGH.h" +#include "Simulation_Parameters/Simulation_Parameters_Explicit.h" +#include "Simulation_Parameters/FEA_Module/SGH_Parameters.h" + +// ----------------------------------------------------------------------------- +// This function calculates the corner force gradients w.r.t velocity; the +// current implementation assumes material.q2 = 0 +// ------------------------------------------------------------------------------ +void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_pres, + const DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + const double rk_alpha, + const size_t cycle + ) +{ + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + const size_t num_dims = simparam->num_dims; + size_t num_corners = rnum_elem * num_nodes_in_elem; + + // initialize gradient matrix + FOR_ALL_CLASS(dof_gid, 0, nlocal_nodes * num_dims, { + for (int idof = 0; idof < Gradient_Matrix_Strides(dof_gid); idof++) + { + Force_Gradient_Velocities(dof_gid, idof) = 0; + } + }); // end parallel for loop over nodes + Kokkos::fence(); + + // initialize buffer storage; not all components are explicitly set for this routine + FOR_ALL_CLASS(corner_gid, 0, num_corners, { + for (int dim = 0; dim < num_dims; dim++) + { + // assign gradient of corner contribution of force to relevant matrix entries with non-zero node velocity gradient + for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) + { + for (int jdim = 0; jdim < num_dims; jdim++) + { + corner_gradient_storage(corner_gid, dim, igradient, jdim) = 0; + } + } + } + }); // end parallel for loop over nodes + Kokkos::fence(); + + // --- calculate the forces acting on the nodes from the element --- + for (size_t elem_gid = 0; elem_gid < rnum_elem; elem_gid++) + { + // FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { + + const size_t num_nodes_in_elem = 8; + real_t gradient_result[num_dims]; + /// total Cauchy stress + double tau_array[9]; + double tau_gradient_array[9]; + + /// corner area normals + double area_normal_array[24]; + + /// estimate of shock direction + double shock_dir_array[3]; + + /// the sums in the Riemann solver + double sum_array[4]; + + /// corner shock impeadance x |corner area normal dot shock_dir| + double muc_array[8]; + double muc_gradient_array[8]; + + /// Riemann velocity + double vel_star_array[3]; + double vel_star_gradient_array[3 * num_nodes_in_elem]; + + /// velocity gradient + double vel_grad_array[9]; + + // --- Create views of arrays to aid the force calculation --- + + ViewCArrayKokkos tau(tau_array, num_dims, num_dims); + ViewCArrayKokkos tau_gradient(tau_gradient_array, num_dims, num_dims); + ViewCArrayKokkos area_normal(area_normal_array, num_nodes_in_elem, num_dims); + ViewCArrayKokkos shock_dir(shock_dir_array, num_dims); + ViewCArrayKokkos sum(sum_array, 4); + ViewCArrayKokkos muc(muc_array, num_nodes_in_elem); + ViewCArrayKokkos muc_gradient(muc_gradient_array, num_nodes_in_elem); + ViewCArrayKokkos vel_star(vel_star_array, num_dims); + ViewCArrayKokkos vel_star_gradient(vel_star_gradient_array, num_nodes_in_elem, num_dims); + ViewCArrayKokkos vel_grad(vel_grad_array, num_dims, num_dims); + + EOSParent* eos_model = elem_eos(elem_gid).model; + + // --- abviatations of variables --- + + // element volume + double vol = elem_vol(elem_gid); + + // create a view of the stress_matrix + ViewCArrayKokkos stress(&elem_stress(rk_level, elem_gid, 0, 0), 3, 3); + + // cut out the node_gids for this element + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 8); + + // get the B matrix which are the OUTWARD corner area normals + get_bmatrix(area_normal, + elem_gid, + node_coords, + elem_node_gids, + rk_level); + + // --- Calculate the velocity gradient --- + get_velgrad(vel_grad, + elem_node_gids, + node_vel, + area_normal, + vol, + elem_gid, + rk_level); + + // the -1 is for the inward surface area normal, + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + for (size_t dim = 0; dim < num_dims; dim++) + { + area_normal(node_lid, dim) = (-1.0) * area_normal(node_lid, dim); + } // end for + } // end for + + double div = elem_div(elem_gid); + + // vel = [u,v,w] + // [du/dx, du/dy, du/dz] + // vel_grad = [dv/dx, dv/dy, dv/dz] + // [dw/dx, dw/dy, dw/dz] + double curl[3]; + curl[0] = vel_grad(2, 1) - vel_grad(1, 2); // dw/dy - dv/dz + curl[1] = vel_grad(0, 2) - vel_grad(2, 0); // du/dz - dw/dx + curl[2] = vel_grad(1, 0) - vel_grad(0, 1); // dv/dx - du/dy + + double mag_curl = sqrt(curl[0] * curl[0] + curl[1] * curl[1] + curl[2] * curl[2]); + + // --- Calculate the Cauchy stress --- + for (size_t i = 0; i < 3; i++) + { + for (size_t j = 0; j < 3; j++) + { + tau(i, j) = stress(i, j); + // artificial viscosity can be added here to tau + } // end for + } // end for + + // add the pressure + for (int i = 0; i < num_dims; i++) + { + tau(i, i) -= elem_pres(elem_gid); + } // end for + + // ---- Multidirectional Approximate Riemann solver (MARS) ---- + // find the average velocity of the elem, it is an + // estimate of the Riemann velocity + + // initialize to Riemann velocity to zero + for (size_t dim = 0; dim < num_dims; dim++) + { + vel_star(dim) = 0.0; + } + + // loop over nodes and calculate an average velocity, which is + // an estimate of Riemann velocity + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + // Get node gloabl index and create view of nodal velocity + int node_gid = nodes_in_elem(elem_gid, node_lid); + + ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); + + vel_star(0) += 0.125 * vel(0); + vel_star(1) += 0.125 * vel(1); + vel_star(2) += 0.125 * vel(2); + } // end for loop over nodes + + // find shock direction and shock impedance associated with each node + + // initialize sum term in MARS to zero + for (int i = 0; i < 4; i++) + { + sum(i) = 0.0; + } + + double mag; // magnitude of the area normal + double mag_vel; // magnitude of velocity + + // loop over the nodes of the elem + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + // Get global node id + size_t node_gid = nodes_in_elem(elem_gid, node_lid); + + // Create view of nodal velocity + ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); + + // Get an estimate of the shock direction. + mag_vel = sqrt( (vel(0) - vel_star(0) ) * (vel(0) - vel_star(0) ) + + (vel(1) - vel_star(1) ) * (vel(1) - vel_star(1) ) + + (vel(2) - vel_star(2) ) * (vel(2) - vel_star(2) ) ); + + if (mag_vel > small) + { + // estimate of the shock direction, a unit normal + for (int dim = 0; dim < num_dims; dim++) + { + shock_dir(dim) = (vel(dim) - vel_star(dim)) / mag_vel; + } + } + else + { + // if there is no velocity change, then use the surface area + // normal as the shock direction + mag = sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) + + area_normal(node_lid, 1) * area_normal(node_lid, 1) + + area_normal(node_lid, 2) * area_normal(node_lid, 2) ); + + // estimate of the shock direction + for (int dim = 0; dim < num_dims; dim++) + { + shock_dir(dim) = area_normal(node_lid, dim) / mag; + } + } // end if mag_vel + + // cell divergence indicates compression or expansions + size_t mat_id = elem_mat_id(elem_gid); + if (div < 0) // element in compression + { + muc(node_lid) = elem_den(elem_gid) * + (material(mat_id).q1 * elem_sspd(elem_gid) + material(mat_id).q2 * mag_vel); + } + else // element in expansion + { + muc(node_lid) = elem_den(elem_gid) * + (material(mat_id).q1ex * elem_sspd(elem_gid) + material(mat_id).q2ex * mag_vel); + } // end if on divergence sign + + size_t use_shock_dir = 0; + double mu_term; + double mu_term_gradient; + + // Coding to use shock direction + if (use_shock_dir == 1) + { + // this is denominator of the Riamann solver and the multiplier + // on velocity in the numerator. It filters on the shock + // direction + mu_term = muc(node_lid) * + fabs(shock_dir(0) * area_normal(node_lid, 0) + + shock_dir(1) * area_normal(node_lid, 1) + + shock_dir(2) * area_normal(node_lid, 2) ); // code gradient for shock dir w.r.t velocity if using shock_dir + } + else + { + // Using a full tensoral Riemann jump relation + mu_term = muc(node_lid) + * sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) + + area_normal(node_lid, 1) * area_normal(node_lid, 1) + + area_normal(node_lid, 2) * area_normal(node_lid, 2) ); + } + + sum(0) += mu_term * vel(0); + sum(1) += mu_term * vel(1); + sum(2) += mu_term * vel(2); + sum(3) += mu_term; + + muc(node_lid) = mu_term; // the impeadance time surface area is stored here + } // end for node_lid loop over nodes of the elem + + // The Riemann velocity, called vel_star + if (sum(3) > fuzz) + { + for (size_t i = 0; i < num_dims; i++) + { + vel_star(i) = sum(i) / sum(3); + } + } + else + { + for (int i = 0; i < num_dims; i++) + { + vel_star(i) = 0.0; + } + } // end if + + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + // The Riemann velocity, called vel_star + if (sum(3) > fuzz) + { + for (size_t i = 0; i < num_dims; i++) + { + vel_star_gradient(node_lid, i) = muc(node_lid) / sum(3); + } + } + else + { + for (int i = 0; i < num_dims; i++) + { + vel_star_gradient(node_lid, i) = 0; + } + } // end if + } + + // ---- Calculate the shock detector for the Riemann-solver ---- + // + // The dissipation from the Riemann problem is limited by phi + // phi = (1. - max( 0., min( 1. , r_face ) ))^n + // where + // r_face = (C* div(u_+)/div(u_z)) + // The plus denotes the cell center divergence of a neighbor. + // The solution will be first order when phi=1 and have + // zero dissipation when phi=0. + // phi = 0 highest-order solution + // phi = 1 first order solution + // + + double phi = 0.0; // the shock detector + double r_face = 1.0; // the ratio on the face + double r_min = 1.0; // the min ratio for the cell + double r_coef = 0.9; // 0.9; the coefficient on the ratio + // (1=minmod and 2=superbee) + double n_coef = 1.0; // the power on the limiting coefficient + // (1=nominal, and n_coeff > 1 oscillatory) + + // loop over the nieghboring cells + for (size_t elem_lid = 0; elem_lid < num_elems_in_elem(elem_gid); elem_lid++) + { + // Get global index for neighboring cell + size_t neighbor_gid = elems_in_elem(elem_gid, elem_lid); + + // calculate the velocity divergence in neighbor + double div_neighbor = elem_div(neighbor_gid); + + r_face = r_coef * (div_neighbor + small) / (div + small); + + // store the smallest face ratio + r_min = fmin(r_face, r_min); + } // end for elem_lid + + // calculate standard shock detector + phi = 1.0 - fmax(0.0, r_min); + phi = pow(phi, n_coef); + + // Mach number shock detector + double omega = 20.0; // 20.0; // weighting factor on Mach number + double third = 1.0 / 3.0; + double c_length = pow(vol, third); // characteristic length + double alpha = fmin(1.0, omega * (c_length * fabs(div)) / (elem_sspd(elem_gid) + fuzz) ); + + // use Mach based detector with standard shock detector + + // turn off dissipation in expansion + // alpha = fmax(-fabs(div0)/div0 * alpha, 0.0); // this should be if(div0<0) alpha=alpha else alpha=0 + + phi = alpha * phi; + + // curl limiter on Q + double phi_curl = fmin(1.0, 1.0 * fabs(div) / (mag_curl + fuzz)); // disable Q when vorticity is high + // phi = phi_curl*phi; + phi = 1; + + // ---- Calculate the Riemann force on each node ---- + + // loop over the each node in the elem + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + size_t corner_lid = node_lid; + size_t column_index; + + // Get corner gid + size_t corner_gid = corners_in_elem(elem_gid, corner_lid); + + // Get node gid + size_t node_gid = nodes_in_elem(elem_gid, node_lid); + + // loop over dimension + for (int dim = 0; dim < num_dims; dim++) + { + // assign gradient of corner contribution of force to relevant matrix entries with non-zero node velocity gradient + for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) + { + size_t gradient_node_gid = nodes_in_elem(elem_gid, igradient); + // if(!map->isNodeLocalElement(gradient_node_gid)) continue; + column_index = num_dims * Global_Gradient_Matrix_Assembly_Map(elem_gid, igradient, node_lid); + if (node_lid == igradient) + { + if (map->isNodeLocalElement(gradient_node_gid)) + { + Force_Gradient_Velocities(gradient_node_gid * num_dims + dim, column_index + dim) += phi * muc(node_lid) * (vel_star_gradient(igradient, dim) - 1); + } + corner_gradient_storage(corner_gid, dim, igradient, dim) = phi * muc(node_lid) * (vel_star_gradient(igradient, dim) - 1); + } + else + { + if (map->isNodeLocalElement(gradient_node_gid)) + { + Force_Gradient_Velocities(gradient_node_gid * num_dims + dim, column_index + dim) += phi * muc(node_lid) * (vel_star_gradient(igradient, dim)); + } + corner_gradient_storage(corner_gid, dim, igradient, dim) = phi * muc(node_lid) * (vel_star_gradient(igradient, dim)); + } + } + } // end loop over dimension + } // end for loop over nodes in elem + + // --- Update Stress --- + // calculate the new stress at the next rk level, if it is a hypo model + size_t mat_id = elem_mat_id(elem_gid); + + // }); // end parallel for loop over elements + } + + /* + //accumulate node values from corner storage + force_gradient_design->putScalar(0); + + vec_array force_gradient_design_view = force_gradient_design->getLocalView (Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { + size_t corner_id; + for(int icorner=0; icorner < num_corners_in_node(node_id); icorner++){ + corner_id = corners_in_node(node_id,icorner); + force_gradient_design_view(node_id,0) += corner_vector_storage(corner_id, 0); + force_gradient_design_view(node_id,1) += corner_vector_storage(corner_id, 1); + force_gradient_design_view(node_id,2) += corner_vector_storage(corner_id, 2); + } + }); // end parallel for + Kokkos::fence(); + */ + + return; +} // end of routine + +// ----------------------------------------------------------------------------- +// This function calculates the corner forces and the evolves stress (hypo) +// ------------------------------------------------------------------------------ +void FEA_Module_SGH::get_force_egradient_sgh(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_pres, + const DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + const double rk_alpha, + const size_t cycle + ) +{ + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + const size_t num_dims = simparam->num_dims; + + // initialize gradient matrix + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { + // compute resulting row of force displacement gradient matrix transpose right multiplied by adjoint vector + for (int idof = 0; idof < num_nodes_in_elem * num_dims; idof++) + { + Force_Gradient_Energies(elem_gid, idof) = 0; + } + }); // end parallel for + Kokkos::fence(); + + // --- calculate the forces acting on the nodes from the element --- + for (size_t elem_gid = 0; elem_gid < rnum_elem; elem_gid++) + { + // FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { + + const size_t num_nodes_in_elem = 8; + real_t gradient_result[num_dims]; + // total Cauchy stress + double tau_array[9]; + double tau_gradient_array[9]; + + // corner area normals + double area_normal_array[24]; + + // estimate of shock direction + double shock_dir_array[3]; + + // the sums in the Riemann solver + double sum_array[4]; + double sum_gradient_array[4]; + + // corner shock impeadance x |corner area normal dot shock_dir| + double muc_array[8]; + double muc_gradient_array[8]; + + // Riemann velocity + double vel_star_array[3]; + double vel_star_gradient_array[3]; + + // velocity gradient + double vel_grad_array[9]; + + // --- Create views of arrays to aid the force calculation --- + + ViewCArrayKokkos tau(tau_array, num_dims, num_dims); + ViewCArrayKokkos tau_gradient(tau_gradient_array, num_dims, num_dims); + ViewCArrayKokkos area_normal(area_normal_array, num_nodes_in_elem, num_dims); + ViewCArrayKokkos shock_dir(shock_dir_array, num_dims); + ViewCArrayKokkos sum(sum_array, 4); + ViewCArrayKokkos sum_gradient(sum_gradient_array, 4); + ViewCArrayKokkos muc(muc_array, num_nodes_in_elem); + ViewCArrayKokkos muc_gradient(muc_gradient_array, num_nodes_in_elem); + ViewCArrayKokkos vel_star(vel_star_array, num_dims); + ViewCArrayKokkos vel_star_gradient(vel_star_gradient_array, num_dims); + ViewCArrayKokkos vel_grad(vel_grad_array, num_dims, num_dims); + + EOSParent* eos_model = elem_eos(elem_gid).model; + + // --- abviatations of variables --- + + // element volume + double vol = elem_vol(elem_gid); + + // create a view of the stress_matrix + ViewCArrayKokkos stress(&elem_stress(rk_level, elem_gid, 0, 0), 3, 3); + + // cut out the node_gids for this element + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 8); + + // get the B matrix which are the OUTWARD corner area normals + get_bmatrix(area_normal, + elem_gid, + node_coords, + elem_node_gids, + rk_level); + + // --- Calculate the velocity gradient --- + get_velgrad(vel_grad, + elem_node_gids, + node_vel, + area_normal, + vol, + elem_gid, + rk_level); + + // the -1 is for the inward surface area normal, + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + for (size_t dim = 0; dim < num_dims; dim++) + { + area_normal(node_lid, dim) = (-1.0) * area_normal(node_lid, dim); + } // end for + } // end for + + double div = elem_div(elem_gid); + + // vel = [u,v,w] + // [du/dx, du/dy, du/dz] + // vel_grad = [dv/dx, dv/dy, dv/dz] + // [dw/dx, dw/dy, dw/dz] + double curl[3]; + curl[0] = vel_grad(2, 1) - vel_grad(1, 2); // dw/dy - dv/dz + curl[1] = vel_grad(0, 2) - vel_grad(2, 0); // du/dz - dw/dx + curl[2] = vel_grad(1, 0) - vel_grad(0, 1); // dv/dx - du/dy + + double mag_curl = sqrt(curl[0] * curl[0] + curl[1] * curl[1] + curl[2] * curl[2]); + + // --- Calculate the Cauchy stress --- + for (size_t i = 0; i < num_dims; i++) + { + for (size_t j = 0; j < num_dims; j++) + { + tau(i, j) = stress(i, j); + tau_gradient(i, j) = 0; + // artificial viscosity can be added here to tau + } // end for + } // end for + + // add the pressure + for (int i = 0; i < num_dims; i++) + { + tau(i, i) -= elem_pres(elem_gid); + tau_gradient(i, i) = -eos_model->calc_pressure_gradient_internal_energy(elem_pres, + elem_stress, + elem_gid, + elem_mat_id(elem_gid), + state_vars, + global_vars, + elem_user_output_vars, + elem_sspd, + elem_den(elem_gid), + elem_sie(rk_level, elem_gid)); + } // end for + + // ---- Multidirectional Approximate Riemann solver (MARS) ---- + // find the average velocity of the elem, it is an + // estimate of the Riemann velocity + + // initialize to Riemann velocity to zero + for (size_t dim = 0; dim < num_dims; dim++) + { + vel_star(dim) = 0.0; + } + + // loop over nodes and calculate an average velocity, which is + // an estimate of Riemann velocity + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + // Get node gloabl index and create view of nodal velocity + int node_gid = nodes_in_elem(elem_gid, node_lid); + + ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); + + vel_star(0) += 0.125 * vel(0); + vel_star(1) += 0.125 * vel(1); + vel_star(2) += 0.125 * vel(2); + } // end for loop over nodes + + // find shock direction and shock impedance associated with each node + + // initialize sum term in MARS to zero + for (int i = 0; i < 4; i++) + { + sum(i) = sum_gradient(i) = 0.0; + } + + double mag; // magnitude of the area normal + double mag_vel; // magnitude of velocity + + // --- Sound speed --- + real_t sound_speed_gradient_energy = + eos_model->calc_sound_speed_gradient_internal_energy(elem_pres, + elem_stress, + elem_gid, + elem_mat_id(elem_gid), + state_vars, + global_vars, + elem_user_output_vars, + elem_sspd, + elem_den(elem_gid), + elem_sie(rk_level, elem_gid)); + + // loop over the nodes of the elem + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + // Get global node id + size_t node_gid = nodes_in_elem(elem_gid, node_lid); + + // Create view of nodal velocity + ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); + + // Get an estimate of the shock direction. + mag_vel = sqrt( (vel(0) - vel_star(0) ) * (vel(0) - vel_star(0) ) + + (vel(1) - vel_star(1) ) * (vel(1) - vel_star(1) ) + + (vel(2) - vel_star(2) ) * (vel(2) - vel_star(2) ) ); + + if (mag_vel > small) + { + // estimate of the shock direction, a unit normal + for (int dim = 0; dim < num_dims; dim++) + { + shock_dir(dim) = (vel(dim) - vel_star(dim)) / mag_vel; + } + } + else + { + // if there is no velocity change, then use the surface area + // normal as the shock direction + mag = sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) + + area_normal(node_lid, 1) * area_normal(node_lid, 1) + + area_normal(node_lid, 2) * area_normal(node_lid, 2) ); + + // estimate of the shock direction + for (int dim = 0; dim < num_dims; dim++) + { + shock_dir(dim) = area_normal(node_lid, dim) / mag; + } + } // end if mag_vel + + // cell divergence indicates compression or expansions + size_t mat_id = elem_mat_id(elem_gid); + if (div < 0) // element in compression + { + muc(node_lid) = elem_den(elem_gid) * + (material(mat_id).q1 * elem_sspd(elem_gid) + material(mat_id).q2 * mag_vel); + muc_gradient(node_lid) = elem_den(elem_gid) * + (material(mat_id).q1 * sound_speed_gradient_energy); + } + else // element in expansion + { + muc(node_lid) = elem_den(elem_gid) * + (material(mat_id).q1ex * elem_sspd(elem_gid) + material(mat_id).q2ex * mag_vel); + muc_gradient(node_lid) = elem_den(elem_gid) * + (material(mat_id).q1ex * sound_speed_gradient_energy); + } // end if on divergence sign + + size_t use_shock_dir = 0; + double mu_term; + double mu_term_gradient; + + // Coding to use shock direction + if (use_shock_dir == 1) + { + // this is denominator of the Riamann solver and the multiplier + // on velocity in the numerator. It filters on the shock + // direction + mu_term = muc(node_lid) * + fabs(shock_dir(0) * area_normal(node_lid, 0) + + shock_dir(1) * area_normal(node_lid, 1) + + shock_dir(2) * area_normal(node_lid, 2) ); + mu_term_gradient = muc_gradient(node_lid) * // amend if shock dir has dependence for gradient not captured here + fabs(shock_dir(0) * area_normal(node_lid, 0) + + shock_dir(1) * area_normal(node_lid, 1) + + shock_dir(2) * area_normal(node_lid, 2) ); + } + else + { + // Using a full tensoral Riemann jump relation + mu_term = muc(node_lid) + * sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) + + area_normal(node_lid, 1) * area_normal(node_lid, 1) + + area_normal(node_lid, 2) * area_normal(node_lid, 2) ); + mu_term_gradient = muc_gradient(node_lid) + * sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) + + area_normal(node_lid, 1) * area_normal(node_lid, 1) + + area_normal(node_lid, 2) * area_normal(node_lid, 2) ); + } + + sum(0) += mu_term * vel(0); + sum(1) += mu_term * vel(1); + sum(2) += mu_term * vel(2); + sum(3) += mu_term; + + sum_gradient(0) += mu_term_gradient * vel(0); + sum_gradient(1) += mu_term_gradient * vel(1); + sum_gradient(2) += mu_term_gradient * vel(2); + sum_gradient(3) += mu_term_gradient; + + muc(node_lid) = mu_term; // the impeadance time surface area is stored here + muc_gradient(node_lid) = mu_term_gradient; + } // end for node_lid loop over nodes of the elem + + // The Riemann velocity, called vel_star + if (sum(3) > fuzz) + { + for (size_t i = 0; i < num_dims; i++) + { + vel_star(i) = sum(i) / sum(3); + vel_star_gradient(i) = sum_gradient(i) / sum(3) - sum_gradient(3) * sum(i) / sum(3) / sum(3); + } + } + else + { + for (int i = 0; i < num_dims; i++) + { + vel_star(i) = 0.0; + vel_star_gradient(i) = 0; + } + } // end if + + // ---- Calculate the shock detector for the Riemann-solver ---- + // + // The dissipation from the Riemann problem is limited by phi + // phi = (1. - max( 0., min( 1. , r_face ) ))^n + // where + // r_face = (C* div(u_+)/div(u_z)) + // The plus denotes the cell center divergence of a neighbor. + // The solution will be first order when phi=1 and have + // zero dissipation when phi=0. + // phi = 0 highest-order solution + // phi = 1 first order solution + // + + double phi = 0.0; // the shock detector + double r_face = 1.0; // the ratio on the face + double r_min = 1.0; // the min ratio for the cell + double r_coef = 0.9; // 0.9; the coefficient on the ratio + // (1=minmod and 2=superbee) + double n_coef = 1.0; // the power on the limiting coefficient + // (1=nominal, and n_coeff > 1 oscillatory) + + // loop over the nieghboring cells + for (size_t elem_lid = 0; elem_lid < num_elems_in_elem(elem_gid); elem_lid++) + { + // Get global index for neighboring cell + size_t neighbor_gid = elems_in_elem(elem_gid, elem_lid); + + // calculate the velocity divergence in neighbor + double div_neighbor = elem_div(neighbor_gid); + + r_face = r_coef * (div_neighbor + small) / (div + small); + + // store the smallest face ratio + r_min = fmin(r_face, r_min); + } // end for elem_lid + + // calculate standard shock detector + phi = 1.0 - fmax(0.0, r_min); + phi = pow(phi, n_coef); + + // Mach number shock detector + double omega = 20.0; // 20.0; // weighting factor on Mach number + double third = 1.0 / 3.0; + double c_length = pow(vol, third); // characteristic length + double alpha = fmin(1.0, omega * (c_length * fabs(div)) / (elem_sspd(elem_gid) + fuzz) ); + + // use Mach based detector with standard shock detector + + // turn off dissipation in expansion + // alpha = fmax(-fabs(div0)/div0 * alpha, 0.0); // this should be if(div0<0) alpha=alpha else alpha=0 + + phi = alpha * phi; + + // curl limiter on Q + double phi_curl = fmin(1.0, 1.0 * fabs(div) / (mag_curl + fuzz)); // disable Q when vorticity is high + // phi = phi_curl*phi; + phi = 1; + + // ---- Calculate the Riemann force on each node ---- + + // loop over the each node in the elem + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + size_t corner_lid = node_lid; + + // Get corner gid + size_t corner_gid = corners_in_elem(elem_gid, corner_lid); + + // Get node gid + size_t node_gid = nodes_in_elem(elem_gid, node_lid); + + // loop over dimension + for (int dim = 0; dim < num_dims; dim++) + { + Force_Gradient_Energies(elem_gid, node_lid * num_dims + dim) = corner_vector_storage(corner_gid, dim) = + area_normal(node_lid, 0) * tau_gradient(0, dim) + + area_normal(node_lid, 1) * tau_gradient(1, dim) + + area_normal(node_lid, 2) * tau_gradient(2, dim) + + phi * muc_gradient(node_lid) * (vel_star(dim) - node_vel(rk_level, node_gid, dim)) + + phi * muc(node_lid) * (vel_star_gradient(dim)); + } // end loop over dimension + } // end for loop over nodes in elem + + // --- Update Stress --- + // calculate the new stress at the next rk level, if it is a hypo model + + size_t mat_id = elem_mat_id(elem_gid); + } + // }); // end parallel for loop over elements + + /* + //accumulate node values from corner storage + force_gradient_design->putScalar(0); + + vec_array force_gradient_design_view = force_gradient_design->getLocalView (Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { + size_t corner_id; + for(int icorner=0; icorner < num_corners_in_node(node_id); icorner++){ + corner_id = corners_in_node(node_id,icorner); + force_gradient_design_view(node_id,0) += corner_vector_storage(corner_id, 0); + force_gradient_design_view(node_id,1) += corner_vector_storage(corner_id, 1); + force_gradient_design_view(node_id,2) += corner_vector_storage(corner_id, 2); + } + }); // end parallel for + Kokkos::fence(); + */ + + return; +} // end of routine + +// ----------------------------------------------------------------------------- +// This function calculates the corner forces and the evolves stress (hypo) +// ------------------------------------------------------------------------------ +void FEA_Module_SGH::get_force_ugradient_sgh(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_pres, + const DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + const double rk_alpha, + const size_t cycle + ) +{ + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + const size_t num_dims = simparam->num_dims; + + // initialize gradient matrix + FOR_ALL_CLASS(dof_gid, 0, nlocal_nodes * num_dims, { + for (int idof = 0; idof < Gradient_Matrix_Strides(dof_gid); idof++) + { + Force_Gradient_Positions(dof_gid, idof) = 0; + } + }); // end parallel for loop over nodes + Kokkos::fence(); + + // --- calculate the forces acting on the nodes from the element --- + for (size_t elem_gid = 0; elem_gid < rnum_elem; elem_gid++) + { + // FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { + + const size_t num_nodes_in_elem = 8; + real_t gradient_result[num_dims]; + // total Cauchy stress + double tau_array[9]; + double tau_gradient_array[9 * max_nodes_per_element * num_dims]; + + // corner area normals + double area_normal_gradients_array[max_nodes_per_element * max_nodes_per_element * num_dims * num_dims]; + double area_normal_array[24]; + + // volume data + double volume; + double volume_gradients_array[max_nodes_per_element * num_dims]; + + // estimate of shock direction + double shock_dir_array[3]; + + // the sums in the Riemann solver + double sum_array[4]; + double sum_gradient_array[4 * max_nodes_per_element * num_dim]; + + // corner shock impeadance x |corner area normal dot shock_dir| + double muc_array[8]; + double muc_gradient_array[8 * max_nodes_per_element * num_dims]; + + // Riemann velocity + double vel_star_array[3]; + double vel_star_gradient_array[num_dims * max_nodes_per_element * num_dims]; + + // velocity gradient + double vel_grad_array[9]; + + // --- Create views of arrays to aid the force calculation --- + + ViewCArrayKokkos tau(tau_array, num_dims, num_dims); + ViewCArrayKokkos tau_gradient(tau_gradient_array, num_dims, num_dims, max_nodes_per_element, num_dims); + ViewCArrayKokkos volume_gradients(volume_gradients_array, num_nodes_in_elem, num_dims); + ViewCArrayKokkos area_normal(area_normal_array, num_nodes_in_elem, num_dims); + ViewCArrayKokkos area_normal_gradients(area_normal_gradients_array, num_nodes_in_elem, num_dims, num_nodes_in_elem, num_dims); + ViewCArrayKokkos shock_dir(shock_dir_array, num_dims); + ViewCArrayKokkos sum(sum_array, 4); + ViewCArrayKokkos sum_gradient(sum_gradient_array, 4, max_nodes_per_element, num_dims); + ViewCArrayKokkos muc(muc_array, num_nodes_in_elem); + ViewCArrayKokkos muc_gradient(muc_gradient_array, num_nodes_in_elem, num_nodes_in_elem, num_dims); + ViewCArrayKokkos vel_star(vel_star_array, num_dims); + ViewCArrayKokkos vel_star_gradient(vel_star_gradient_array, num_dims, num_nodes_in_elem, num_dims); + ViewCArrayKokkos vel_grad(vel_grad_array, num_dims, num_dims); + + EOSParent* eos_model = elem_eos(elem_gid).model; + + // --- abviatations of variables --- + + // element volume + double vol = elem_vol(elem_gid); + + // create a view of the stress_matrix + ViewCArrayKokkos stress(&elem_stress(rk_level, elem_gid, 0, 0), 3, 3); + + // cut out the node_gids for this element + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 8); + + // gradients of the element volume + get_vol_hex_ugradient(volume_gradients, elem_gid, node_coords, elem_node_gids, rk_level); + + // //debug + // for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ + // for (size_t dim = 0; dim < num_dims; dim++){ + // if(volume_gradients(node_lid, dim)>1 || volume_gradients(node_lid, dim) < -1) + // std::cout << volume_gradients(node_lid, dim) << " "; + // } // end for + // } // end for + // std::cout << std::endl; + + // get the B matrix which are the OUTWARD corner area normals + get_bmatrix(area_normal, + elem_gid, + node_coords, + elem_node_gids, + rk_level); + + get_bmatrix_gradients(area_normal_gradients, + elem_gid, + node_coords, + elem_node_gids, + rk_level); + + // --- Calculate the velocity gradient --- + get_velgrad(vel_grad, + elem_node_gids, + node_vel, + area_normal, + vol, + elem_gid, + rk_level); + + // the -1 is for the inward surface area normal, + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + for (size_t dim = 0; dim < num_dims; dim++) + { + area_normal(node_lid, dim) = (-1.0) * area_normal(node_lid, dim); + } // end for + } // end for + + // the -1 is for the inward surface area normal, + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + for (size_t dim = 0; dim < num_dims; dim++) + { + for (size_t igradient = 0; igradient < num_nodes_in_elem; igradient++) + { + for (size_t jdim = 0; jdim < num_dims; jdim++) + { + area_normal_gradients(node_lid, dim, igradient, jdim) = (-1.0) * area_normal_gradients(node_lid, dim, igradient, jdim); + } + } + } // end for + } // end for + + double div = elem_div(elem_gid); + + // vel = [u,v,w] + // [du/dx, du/dy, du/dz] + // vel_grad = [dv/dx, dv/dy, dv/dz] + // [dw/dx, dw/dy, dw/dz] + double curl[3]; + curl[0] = vel_grad(2, 1) - vel_grad(1, 2); // dw/dy - dv/dz + curl[1] = vel_grad(0, 2) - vel_grad(2, 0); // du/dz - dw/dx + curl[2] = vel_grad(1, 0) - vel_grad(0, 1); // dv/dx - du/dy + + double mag_curl = sqrt(curl[0] * curl[0] + curl[1] * curl[1] + curl[2] * curl[2]); + + // --- Calculate the Cauchy stress --- + for (size_t i = 0; i < 3; i++) + { + for (size_t j = 0; j < 3; j++) + { + tau(i, j) = stress(i, j); + // artificial viscosity can be added here to tau + } // end for + } // end for + + // initialize gradient array + for (size_t i = 0; i < 3; i++) + { + for (size_t j = 0; j < 3; j++) + { + for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) + { + tau_gradient(i, j, igradient, 0) = 0; + tau_gradient(i, j, igradient, 1) = 0; + if (num_dims == 3) + { + tau_gradient(i, j, igradient, 2) = 0; + } + } + } // end for + } // end for + + // add the pressure + for (int i = 0; i < num_dims; i++) + { + tau(i, i) -= elem_pres(elem_gid); + // current expression only works for ideal gas! generalize. + for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) + { + tau_gradient(i, i, igradient, 0) = elem_pres(elem_gid) * volume_gradients(igradient, 0) / vol; + tau_gradient(i, i, igradient, 1) = elem_pres(elem_gid) * volume_gradients(igradient, 1) / vol; + tau_gradient(i, i, igradient, 2) = elem_pres(elem_gid) * volume_gradients(igradient, 2) / vol; + } + } // end for + + // ---- Multidirectional Approximate Riemann solver (MARS) ---- + // find the average velocity of the elem, it is an + // estimate of the Riemann velocity + + // initialize to Riemann velocity to zero + for (size_t dim = 0; dim < num_dims; dim++) + { + vel_star(dim) = 0.0; + } + + // loop over nodes and calculate an average velocity, which is + // an estimate of Riemann velocity + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + // Get node gloabl index and create view of nodal velocity + int node_gid = nodes_in_elem(elem_gid, node_lid); + + ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); + + vel_star(0) += 0.125 * vel(0); + vel_star(1) += 0.125 * vel(1); + vel_star(2) += 0.125 * vel(2); + } // end for loop over nodes + + // find shock direction and shock impedance associated with each node + + // initialize sum term in MARS to zero + for (int i = 0; i < 4; i++) + { + sum(i) = 0.0; + } + for (int i = 0; i < 4; i++) + { + for (size_t igradient = 0; igradient < num_nodes_in_elem; igradient++) + { + for (size_t jdim = 0; jdim < num_dims; jdim++) + { + sum_gradient(i, igradient, jdim) = 0.0; + } + } + } + + double mag; // magnitude of the area normal + double mag_vel; // magnitude of velocity + + // loop over the nodes of the elem + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + // Get global node id + size_t node_gid = nodes_in_elem(elem_gid, node_lid); + + // Create view of nodal velocity + ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); + + // Get an estimate of the shock direction. + mag_vel = sqrt( (vel(0) - vel_star(0) ) * (vel(0) - vel_star(0) ) + + (vel(1) - vel_star(1) ) * (vel(1) - vel_star(1) ) + + (vel(2) - vel_star(2) ) * (vel(2) - vel_star(2) ) ); + + if (mag_vel > small) + { + // estimate of the shock direction, a unit normal + for (int dim = 0; dim < num_dims; dim++) + { + shock_dir(dim) = (vel(dim) - vel_star(dim)) / mag_vel; + } + } + else + { + // if there is no velocity change, then use the surface area + // normal as the shock direction + mag = sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) + + area_normal(node_lid, 1) * area_normal(node_lid, 1) + + area_normal(node_lid, 2) * area_normal(node_lid, 2) ); + + // estimate of the shock direction + for (int dim = 0; dim < num_dims; dim++) + { + shock_dir(dim) = area_normal(node_lid, dim) / mag; + } + } // end if mag_vel + + // cell divergence indicates compression or expansions + size_t mat_id = elem_mat_id(elem_gid); + if (div < 0) // element in compression + { + muc(node_lid) = elem_den(elem_gid) * + (material(mat_id).q1 * elem_sspd(elem_gid) + material(mat_id).q2 * mag_vel); + + for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) + { + muc_gradient(node_lid, igradient, 0) = -elem_den(elem_gid) * volume_gradients(igradient, 0) / vol * + (material(mat_id).q1 * elem_sspd(elem_gid)); + muc_gradient(node_lid, igradient, 1) = -elem_den(elem_gid) * volume_gradients(igradient, 1) / vol * + (material(mat_id).q1 * elem_sspd(elem_gid)); + muc_gradient(node_lid, igradient, 2) = -elem_den(elem_gid) * volume_gradients(igradient, 2) / vol * + (material(mat_id).q1 * elem_sspd(elem_gid)); + } + } + else // element in expansion + { + muc(node_lid) = elem_den(elem_gid) * + (material(mat_id).q1ex * elem_sspd(elem_gid) + material(mat_id).q2ex * mag_vel); + for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) + { + muc_gradient(node_lid, igradient, 0) = -elem_den(elem_gid) * volume_gradients(igradient, 0) / vol * + (material(mat_id).q1ex * elem_sspd(elem_gid)); + muc_gradient(node_lid, igradient, 1) = -elem_den(elem_gid) * volume_gradients(igradient, 1) / vol * + (material(mat_id).q1ex * elem_sspd(elem_gid)); + muc_gradient(node_lid, igradient, 2) = -elem_den(elem_gid) * volume_gradients(igradient, 2) / vol * + (material(mat_id).q1ex * elem_sspd(elem_gid)); + } + } // end if on divergence sign + + size_t use_shock_dir = 0; + double mu_term; + double mu_term_gradient; + + // Coding to use shock direction + if (use_shock_dir == 1) + { + // this is denominator of the Riamann solver and the multiplier + // on velocity in the numerator. It filters on the shock + // direction + mu_term = muc(node_lid) * + fabs(shock_dir(0) * area_normal(node_lid, 0) + + shock_dir(1) * area_normal(node_lid, 1) + + shock_dir(2) * area_normal(node_lid, 2) ); + // muc_gradient(node_lid) = muc_gradient(node_lid)* + // fabs( shock_dir(0)*area_normal(node_lid,0) + // + shock_dir(1)*area_normal(node_lid,1) + // + shock_dir(2)*area_normal(node_lid,2) ); + // mu_term_gradient = muc(node_lid)* //amend if shock dir has dependence for gradient not captured here + // fabs( shock_dir(0)*area_normal_gradients(node_lid,0) + // + shock_dir(1)*area_normal_gradients(node_lid,1) + // + shock_dir(2)*area_normal_gradients(node_lid,2) ); + } + else + { + // Using a full tensoral Riemann jump relation + mu_term = muc(node_lid) + * sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) + + area_normal(node_lid, 1) * area_normal(node_lid, 1) + + area_normal(node_lid, 2) * area_normal(node_lid, 2) ); + for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) + { + for (int jdim = 0; jdim < num_dims; jdim++) + { + mu_term_gradient = muc_gradient(node_lid, igradient, jdim) + * sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) + + area_normal(node_lid, 1) * area_normal(node_lid, 1) + + area_normal(node_lid, 2) * area_normal(node_lid, 2) ); + + mu_term_gradient += muc(node_lid) * (area_normal(node_lid, 0) * area_normal_gradients(node_lid, 0, igradient, jdim) + + area_normal(node_lid, 1) * area_normal_gradients(node_lid, 1, igradient, jdim) + + area_normal(node_lid, 2) * area_normal_gradients(node_lid, 2, igradient, jdim)) / + sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) + + area_normal(node_lid, 1) * area_normal(node_lid, 1) + + area_normal(node_lid, 2) * area_normal(node_lid, 2) ); + + muc_gradient(node_lid, igradient, jdim) = mu_term_gradient; + } + } + } + + sum(0) += mu_term * vel(0); + sum(1) += mu_term * vel(1); + sum(2) += mu_term * vel(2); + sum(3) += mu_term; + + // sum gradients + + for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) + { + for (int jdim = 0; jdim < num_dims; jdim++) + { + sum_gradient(0, igradient, jdim) += muc_gradient(node_lid, igradient, jdim) * vel(0); + sum_gradient(1, igradient, jdim) += muc_gradient(node_lid, igradient, jdim) * vel(1); + sum_gradient(2, igradient, jdim) += muc_gradient(node_lid, igradient, jdim) * vel(2); + sum_gradient(3, igradient, jdim) += muc_gradient(node_lid, igradient, jdim); + } + } + + muc(node_lid) = mu_term; // the impeadance time surface area is stored here + } // end for node_lid loop over nodes of the elem + + // The Riemann velocity, called vel_star + if (sum(3) > fuzz) + { + for (size_t i = 0; i < num_dims; i++) + { + vel_star(i) = sum(i) / sum(3); + } + } + else + { + for (int i = 0; i < num_dims; i++) + { + vel_star(i) = 0.0; + } + } // end if + + // vel star gradients + if (sum(3) > fuzz) + { + for (size_t i = 0; i < num_dims; i++) + { + for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) + { + for (int jdim = 0; jdim < num_dims; jdim++) + { + vel_star_gradient(i, igradient, jdim) = sum_gradient(i, igradient, jdim) / sum(3) - sum(i) / (sum(3) * sum(3)) * sum_gradient(3, igradient, jdim); + } + } + } + } + else + { + for (int i = 0; i < num_dims; i++) + { + for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) + { + for (int jdim = 0; jdim < num_dims; jdim++) + { + vel_star_gradient(i, igradient, jdim) = 0; + } + } + } + } // end if + + // ---- Calculate the shock detector for the Riemann-solver ---- + // + // The dissipation from the Riemann problem is limited by phi + // phi = (1. - max( 0., min( 1. , r_face ) ))^n + // where + // r_face = (C* div(u_+)/div(u_z)) + // The plus denotes the cell center divergence of a neighbor. + // The solution will be first order when phi=1 and have + // zero dissipation when phi=0. + // phi = 0 highest-order solution + // phi = 1 first order solution + // + + double phi = 0.0; // the shock detector + double r_face = 1.0; // the ratio on the face + double r_min = 1.0; // the min ratio for the cell + double r_coef = 0.9; // 0.9; the coefficient on the ratio + // (1=minmod and 2=superbee) + double n_coef = 1.0; // the power on the limiting coefficient + // (1=nominal, and n_coeff > 1 oscillatory) + + // loop over the nieghboring cells + for (size_t elem_lid = 0; elem_lid < num_elems_in_elem(elem_gid); elem_lid++) + { + // Get global index for neighboring cell + size_t neighbor_gid = elems_in_elem(elem_gid, elem_lid); + + // calculate the velocity divergence in neighbor + double div_neighbor = elem_div(neighbor_gid); + + r_face = r_coef * (div_neighbor + small) / (div + small); + + // store the smallest face ratio + r_min = fmin(r_face, r_min); + } // end for elem_lid + + // calculate standard shock detector + phi = 1.0 - fmax(0.0, r_min); + phi = pow(phi, n_coef); + + // Mach number shock detector + double omega = 20.0; // 20.0; // weighting factor on Mach number + double third = 1.0 / 3.0; + double c_length = pow(vol, third); // characteristic length + double alpha = fmin(1.0, omega * (c_length * fabs(div)) / (elem_sspd(elem_gid) + fuzz) ); + + // use Mach based detector with standard shock detector + + // turn off dissipation in expansion + // alpha = fmax(-fabs(div0)/div0 * alpha, 0.0); // this should be if(div0<0) alpha=alpha else alpha=0 + + phi = alpha * phi; + + // curl limiter on Q + double phi_curl = fmin(1.0, 1.0 * fabs(div) / (mag_curl + fuzz)); // disable Q when vorticity is high + // phi = phi_curl*phi; + phi = 1; + + // ---- Calculate the Riemann force on each node ---- + + // loop over the each node in the elem + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + size_t corner_lid = node_lid; + size_t column_index; + + // Get corner gid + size_t corner_gid = corners_in_elem(elem_gid, corner_lid); + + // Get node gid + size_t node_gid = nodes_in_elem(elem_gid, node_lid); + + // loop over dimension + for (int dim = 0; dim < num_dims; dim++) + { + // assign gradient of corner contribution of force to relevant matrix entries with non-zero node velocity gradient + for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) + { + for (int jdim = 0; jdim < num_dims; jdim++) + { + size_t gradient_node_gid = nodes_in_elem(elem_gid, igradient); + // if(!map->isNodeLocalElement(gradient_node_gid)) continue; + column_index = num_dims * Global_Gradient_Matrix_Assembly_Map(elem_gid, igradient, node_lid); + if (map->isNodeLocalElement(gradient_node_gid)) + { + Force_Gradient_Positions(gradient_node_gid * num_dims + jdim, column_index + dim) += area_normal(node_lid, 0) * tau_gradient(0, dim, igradient, jdim) + + area_normal(node_lid, 1) * tau_gradient(1, dim, igradient, jdim) + + area_normal(node_lid, 2) * tau_gradient(2, dim, igradient, jdim) + + area_normal_gradients(node_lid, 0, igradient, jdim) * tau(0, dim) + + area_normal_gradients(node_lid, 1, igradient, jdim) * tau(1, dim) + + area_normal_gradients(node_lid, 2, igradient, jdim) * tau(2, dim) + + phi * muc_gradient(node_lid, igradient, + jdim) * (vel_star(dim) - node_vel(rk_level, node_gid, dim)) + + phi * muc(node_lid) * (vel_star_gradient(dim, igradient, jdim)); + } + corner_gradient_storage(corner_gid, dim, igradient, jdim) = area_normal(node_lid, 0) * tau_gradient(0, dim, igradient, jdim) + + area_normal(node_lid, 1) * tau_gradient(1, dim, igradient, jdim) + + area_normal(node_lid, 2) * tau_gradient(2, dim, igradient, jdim) + + area_normal_gradients(node_lid, 0, igradient, jdim) * tau(0, dim) + + area_normal_gradients(node_lid, 1, igradient, jdim) * tau(1, dim) + + area_normal_gradients(node_lid, 2, igradient, jdim) * tau(2, dim) + + phi * muc_gradient(node_lid, igradient, jdim) * (vel_star(dim) - node_vel(rk_level, node_gid, dim)) + + phi * muc(node_lid) * (vel_star_gradient(dim, igradient, jdim)); + // if(map->isNodeLocalElement(gradient_node_gid)){ + // Force_Gradient_Positions(gradient_node_gid*num_dims+jdim, column_index+dim) += + // + area_normal_gradients(node_lid, 0, igradient, jdim)*tau(0, dim) + // + area_normal_gradients(node_lid, 1, igradient, jdim)*tau(1, dim) + // + area_normal_gradients(node_lid, 2, igradient, jdim)*tau(2, dim); + // } + // corner_gradient_storage(corner_gid,dim,igradient,jdim) = + // + area_normal_gradients(node_lid, 0, igradient, jdim)*tau(0, dim) + // + area_normal_gradients(node_lid, 1, igradient, jdim)*tau(1, dim) + // + area_normal_gradients(node_lid, 2, igradient, jdim)*tau(2, dim); + } + } + } // end loop over dimension + } // end for loop over nodes in elem + + // --- Update Stress --- + // calculate the new stress at the next rk level, if it is a hypo model + + size_t mat_id = elem_mat_id(elem_gid); + } + // }); // end parallel for loop over elements + + /* + //accumulate node values from corner storage + force_gradient_design->putScalar(0); + + vec_array force_gradient_design_view = force_gradient_design->getLocalView (Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { + size_t corner_id; + for(int icorner=0; icorner < num_corners_in_node(node_id); icorner++){ + corner_id = corners_in_node(node_id,icorner); + force_gradient_design_view(node_id,0) += corner_vector_storage(corner_id, 0); + force_gradient_design_view(node_id,1) += corner_vector_storage(corner_id, 1); + force_gradient_design_view(node_id,2) += corner_vector_storage(corner_id, 2); + } + }); // end parallel for + Kokkos::fence(); + */ + + return; +} // end of routine + +// -------------------------------------------------------------------------------------------------------- +// Computes term objective derivative term involving gradient of power with respect to the design variable +// --------------------------------------------------------------------------------------------------------- + +void FEA_Module_SGH::force_design_gradient_term(const_vec_array design_variables, vec_array design_gradients) +{ + size_t num_bdy_nodes = mesh->num_bdy_nodes; + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos material = simparam->material; + const int num_dim = simparam->num_dims; + size_t num_corners = rnum_elem * num_nodes_in_elem; + real_t global_dt; + bool element_constant_density = true; + size_t current_data_index, next_data_index; + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + CArrayKokkos current_element_adjoint = CArrayKokkos(num_nodes_in_elem, num_dim); + + // gradient contribution from gradient of Force vector with respect to design variable. + if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) + { + if (myrank == 0) + { + std::cout << "gradient term involving adjoint derivative" << std::endl; + } + } + + for (unsigned long cycle = 0; cycle < last_time_step + 1; cycle++) + { + // compute timestep from time data + global_dt = time_data[cycle + 1] - time_data[cycle]; + // print + if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) + { + if (cycle == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } + // print time step every 10 cycles + else if (cycle % 20 == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } // end if + } + + // view scope + { + const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array current_element_internal_energy = (*forward_solve_internal_energy_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array current_coord_vector = (*forward_solve_coordinate_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_velocity_vector = (*forward_solve_velocity_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_element_internal_energy = (*forward_solve_internal_energy_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_coord_vector = (*forward_solve_coordinate_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_adjoint_vector = (*adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + + // first half of integration step calculation + FOR_ALL_CLASS(node_gid, 0, nall_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + node_vel(rk_level, node_gid, idim) = current_velocity_vector(node_gid, idim); + node_coords(rk_level, node_gid, idim) = current_coord_vector(node_gid, idim); + } + }); // end parallel for + Kokkos::fence(); + + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { + elem_sie(rk_level, elem_gid) = current_element_internal_energy(elem_gid, 0); + }); // end parallel for + Kokkos::fence(); + + get_vol(); + + // ---- Calculate velocity diveregence for the element ---- + if (num_dim == 2) + { + get_divergence2D(elem_div, + node_coords, + node_vel, + elem_vol); + } + else + { + get_divergence(elem_div, + node_coords, + node_vel, + elem_vol); + } // end if 2D + + // ---- Calculate elem state (den, pres, sound speed, stress) for next time step ---- + if (num_dim == 2) + { + update_state2D(material, + *mesh, + node_coords, + node_vel, + elem_den, + elem_pres, + elem_stress, + elem_sspd, + elem_sie, + elem_vol, + elem_mass, + elem_mat_id, + 1.0, + cycle); + } + else + { + update_state(material, + *mesh, + node_coords, + node_vel, + elem_den, + elem_pres, + elem_stress, + elem_sspd, + elem_sie, + elem_vol, + elem_mass, + elem_mat_id, + 1.0, + cycle); + } + + get_force_dgradient_sgh(material, + *mesh, + node_coords, + node_vel, + elem_den, + elem_sie, + elem_pres, + elem_stress, + elem_sspd, + elem_vol, + elem_div, + elem_mat_id, + 1.0, + cycle); + + // derivatives of forces at corners stored in corner_vector_storage buffer by previous routine + FOR_ALL_CLASS(elem_id, 0, rnum_elem, { + size_t node_id; + size_t corner_id; + real_t inner_product; + + inner_product = 0; + for (int ifill = 0; ifill < num_nodes_in_elem; ifill++) + { + node_id = nodes_in_elem(elem_id, ifill); + corner_id = elem_id * num_nodes_in_elem + ifill; + for (int idim = 0; idim < num_dim; idim++) + { + inner_product += corner_vector_storage(corner_id, idim) * current_adjoint_vector(node_id, idim); + } + } + + for (int inode = 0; inode < num_nodes_in_elem; inode++) + { + // compute gradient of local element contribution to v^t*M*v product + corner_id = elem_id * num_nodes_in_elem + inode; + corner_value_storage(corner_id) = inner_product; + } + }); // end parallel for + Kokkos::fence(); + + // accumulate node values from corner storage + // multiply + FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { + size_t corner_id; + for (int icorner = 0; icorner < num_corners_in_node(node_id); icorner++) + { + corner_id = corners_in_node(node_id, icorner); + design_gradients(node_id, 0) += -corner_value_storage(corner_id) * global_dt; + } + }); // end parallel for + Kokkos::fence(); + } // end view scope + } +} + +// ----------------------------------------------------------------------------- +// This function calculates the corner forces and the evolves stress (hypo) +// ------------------------------------------------------------------------------ +void FEA_Module_SGH::get_force_dgradient_sgh(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_pres, + const DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + const double rk_alpha, + const size_t cycle + ) +{ + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + const size_t num_dims = simparam->num_dims; + // --- calculate the forces acting on the nodes from the element --- + for (size_t elem_gid = 0; elem_gid < rnum_elem; elem_gid++) + { + // FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { + + const size_t num_nodes_in_elem = 8; + real_t gradient_result[num_dims]; + // total Cauchy stress + double tau_array[9]; + double tau_gradient_array[9]; + + // corner area normals + double area_normal_array[24]; + + // estimate of shock direction + double shock_dir_array[3]; + + // the sums in the Riemann solver + double sum_array[4]; + double sum_gradient_array[4]; + + // corner shock impeadance x |corner area normal dot shock_dir| + double muc_array[8]; + double muc_gradient_array[8]; + + // Riemann velocity + double vel_star_array[3]; + double vel_star_gradient_array[3]; + + // velocity gradient + double vel_grad_array[9]; + + // --- Create views of arrays to aid the force calculation --- + + ViewCArrayKokkos tau(tau_array, num_dims, num_dims); + ViewCArrayKokkos tau_gradient(tau_gradient_array, num_dims, num_dims); + ViewCArrayKokkos area_normal(area_normal_array, num_nodes_in_elem, num_dims); + ViewCArrayKokkos shock_dir(shock_dir_array, num_dims); + ViewCArrayKokkos sum(sum_array, 4); + ViewCArrayKokkos sum_gradient(sum_gradient_array, 4); + ViewCArrayKokkos muc(muc_array, num_nodes_in_elem); + ViewCArrayKokkos muc_gradient(muc_gradient_array, num_nodes_in_elem); + ViewCArrayKokkos vel_star(vel_star_array, num_dims); + ViewCArrayKokkos vel_star_gradient(vel_star_gradient_array, num_dims); + ViewCArrayKokkos vel_grad(vel_grad_array, num_dims, num_dims); + + EOSParent* eos_model = elem_eos(elem_gid).model; + + // --- abviatations of variables --- + + // element volume + double vol = elem_vol(elem_gid); + + // create a view of the stress_matrix + ViewCArrayKokkos stress(&elem_stress(rk_level, elem_gid, 0, 0), 3, 3); + + // cut out the node_gids for this element + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 8); + + // get the B matrix which are the OUTWARD corner area normals + get_bmatrix(area_normal, + elem_gid, + node_coords, + elem_node_gids, + rk_level); + + // --- Calculate the velocity gradient --- + get_velgrad(vel_grad, + elem_node_gids, + node_vel, + area_normal, + vol, + elem_gid, + rk_level); + + // the -1 is for the inward surface area normal, + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + for (size_t dim = 0; dim < num_dims; dim++) + { + area_normal(node_lid, dim) = (-1.0) * area_normal(node_lid, dim); + } // end for + } // end for + + double div = elem_div(elem_gid); + + // vel = [u,v,w] + // [du/dx, du/dy, du/dz] + // vel_grad = [dv/dx, dv/dy, dv/dz] + // [dw/dx, dw/dy, dw/dz] + double curl[3]; + curl[0] = vel_grad(2, 1) - vel_grad(1, 2); // dw/dy - dv/dz + curl[1] = vel_grad(0, 2) - vel_grad(2, 0); // du/dz - dw/dx + curl[2] = vel_grad(1, 0) - vel_grad(0, 1); // dv/dx - du/dy + + double mag_curl = sqrt(curl[0] * curl[0] + curl[1] * curl[1] + curl[2] * curl[2]); + + // --- Calculate the Cauchy stress --- + for (size_t i = 0; i < 3; i++) + { + for (size_t j = 0; j < 3; j++) + { + tau(i, j) = stress(i, j); + tau_gradient(i, j) = 0; + } // end for + } // end for + + // add the pressure + for (int i = 0; i < num_dims; i++) + { + tau(i, i) -= elem_pres(elem_gid); + tau_gradient(i, i) -= elem_pres(elem_gid) / num_nodes_in_elem / relative_element_densities(elem_gid); + } // end for + + // ---- Multidirectional Approximate Riemann solver (MARS) ---- + // find the average velocity of the elem, it is an + // estimate of the Riemann velocity + + // initialize to Riemann velocity to zero + for (size_t dim = 0; dim < num_dims; dim++) + { + vel_star(dim) = 0.0; + } + + // loop over nodes and calculate an average velocity, which is + // an estimate of Riemann velocity + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + // Get node gloabl index and create view of nodal velocity + int node_gid = nodes_in_elem(elem_gid, node_lid); + + ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); + + vel_star(0) += 0.125 * vel(0); + vel_star(1) += 0.125 * vel(1); + vel_star(2) += 0.125 * vel(2); + } // end for loop over nodes + + // find shock direction and shock impedance associated with each node + + // initialize sum term in MARS to zero + for (int i = 0; i < 4; i++) + { + sum(i) = sum_gradient(i) = 0.0; + } + + double mag; // magnitude of the area normal + double mag_vel; // magnitude of velocity + + // loop over the nodes of the elem + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + // Get global node id + size_t node_gid = nodes_in_elem(elem_gid, node_lid); + + // Create view of nodal velocity + ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); + + // Get an estimate of the shock direction. + mag_vel = sqrt( (vel(0) - vel_star(0) ) * (vel(0) - vel_star(0) ) + + (vel(1) - vel_star(1) ) * (vel(1) - vel_star(1) ) + + (vel(2) - vel_star(2) ) * (vel(2) - vel_star(2) ) ); + + if (mag_vel > small) + { + // estimate of the shock direction, a unit normal + for (int dim = 0; dim < num_dims; dim++) + { + shock_dir(dim) = (vel(dim) - vel_star(dim)) / mag_vel; + } + } + else + { + // if there is no velocity change, then use the surface area + // normal as the shock direction + mag = sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) + + area_normal(node_lid, 1) * area_normal(node_lid, 1) + + area_normal(node_lid, 2) * area_normal(node_lid, 2) ); + + // estimate of the shock direction + for (int dim = 0; dim < num_dims; dim++) + { + shock_dir(dim) = area_normal(node_lid, dim) / mag; + } + } // end if mag_vel + + // cell divergence indicates compression or expansions + size_t mat_id = elem_mat_id(elem_gid); + if (div < 0) // element in compression + { + muc(node_lid) = elem_den(elem_gid) * + (material(mat_id).q1 * elem_sspd(elem_gid) + material(mat_id).q2 * mag_vel); + + muc_gradient(node_lid) = elem_den(elem_gid) / relative_element_densities(elem_gid) / num_nodes_in_elem * + (material(mat_id).q1 * elem_sspd(elem_gid) + material(mat_id).q2 * mag_vel); + } + else // element in expansion + { + muc(node_lid) = elem_den(elem_gid) * + (material(mat_id).q1ex * elem_sspd(elem_gid) + material(mat_id).q2ex * mag_vel); + muc_gradient(node_lid) = elem_den(elem_gid) / relative_element_densities(elem_gid) / num_nodes_in_elem * + (material(mat_id).q1ex * elem_sspd(elem_gid) + material(mat_id).q2ex * mag_vel); + } // end if on divergence sign + + size_t use_shock_dir = 0; + double mu_term; + double mu_term_gradient; + + // Coding to use shock direction + if (use_shock_dir == 1) + { + // this is denominator of the Riamann solver and the multiplier + // on velocity in the numerator. It filters on the shock + // direction + mu_term = muc(node_lid) * + fabs(shock_dir(0) * area_normal(node_lid, 0) + + shock_dir(1) * area_normal(node_lid, 1) + + shock_dir(2) * area_normal(node_lid, 2) ); + mu_term_gradient = muc_gradient(node_lid) * // amend if shock dir has dependence for gradient not captured here + fabs(shock_dir(0) * area_normal(node_lid, 0) + + shock_dir(1) * area_normal(node_lid, 1) + + shock_dir(2) * area_normal(node_lid, 2) ); + } + else + { + // Using a full tensoral Riemann jump relation + mu_term = muc(node_lid) + * sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) + + area_normal(node_lid, 1) * area_normal(node_lid, 1) + + area_normal(node_lid, 2) * area_normal(node_lid, 2) ); + mu_term_gradient = muc_gradient(node_lid) + * sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) + + area_normal(node_lid, 1) * area_normal(node_lid, 1) + + area_normal(node_lid, 2) * area_normal(node_lid, 2) ); + } + + sum(0) += mu_term * vel(0); + sum(1) += mu_term * vel(1); + sum(2) += mu_term * vel(2); + sum(3) += mu_term; + + sum_gradient(0) += mu_term_gradient * vel(0); + sum_gradient(1) += mu_term_gradient * vel(1); + sum_gradient(2) += mu_term_gradient * vel(2); + sum_gradient(3) += mu_term_gradient; + + muc(node_lid) = mu_term; // the impeadance time surface area is stored here + muc_gradient(node_lid) = mu_term_gradient; + } // end for node_lid loop over nodes of the elem + + // The Riemann velocity, called vel_star + if (sum(3) > fuzz) + { + for (size_t i = 0; i < num_dims; i++) + { + vel_star(i) = sum(i) / sum(3); + vel_star_gradient(i) = sum_gradient(i) / sum(3) - sum_gradient(3) * sum(i) / sum(3) / sum(3); + } + } + else + { + for (int i = 0; i < num_dims; i++) + { + vel_star(i) = 0.0; + vel_star_gradient(i) = 0.0; + } + } // end if + + // ---- Calculate the shock detector for the Riemann-solver ---- + // + // The dissipation from the Riemann problem is limited by phi + // phi = (1. - max( 0., min( 1. , r_face ) ))^n + // where + // r_face = (C* div(u_+)/div(u_z)) + // The plus denotes the cell center divergence of a neighbor. + // The solution will be first order when phi=1 and have + // zero dissipation when phi=0. + // phi = 0 highest-order solution + // phi = 1 first order solution + // + + double phi = 0.0; // the shock detector + double r_face = 1.0; // the ratio on the face + double r_min = 1.0; // the min ratio for the cell + double r_coef = 0.9; // 0.9; the coefficient on the ratio + // (1=minmod and 2=superbee) + double n_coef = 1.0; // the power on the limiting coefficient + // (1=nominal, and n_coeff > 1 oscillatory) + + // loop over the nieghboring cells + for (size_t elem_lid = 0; elem_lid < num_elems_in_elem(elem_gid); elem_lid++) + { + // Get global index for neighboring cell + size_t neighbor_gid = elems_in_elem(elem_gid, elem_lid); + + // calculate the velocity divergence in neighbor + double div_neighbor = elem_div(neighbor_gid); + + r_face = r_coef * (div_neighbor + small) / (div + small); + + // store the smallest face ratio + r_min = fmin(r_face, r_min); + } // end for elem_lid + + // calculate standard shock detector + phi = 1.0 - fmax(0.0, r_min); + phi = pow(phi, n_coef); + + // Mach number shock detector + double omega = 20.0; // 20.0; // weighting factor on Mach number + double third = 1.0 / 3.0; + double c_length = pow(vol, third); // characteristic length + double alpha = fmin(1.0, omega * (c_length * fabs(div)) / (elem_sspd(elem_gid) + fuzz) ); + + // use Mach based detector with standard shock detector + + // turn off dissipation in expansion + // alpha = fmax(-fabs(div0)/div0 * alpha, 0.0); // this should be if(div0<0) alpha=alpha else alpha=0 + + phi = alpha * phi; + + // curl limiter on Q + double phi_curl = fmin(1.0, 1.0 * fabs(div) / (mag_curl + fuzz)); // disable Q when vorticity is high + // phi = phi_curl*phi; + phi = 1; + + // ---- Calculate the Riemann force on each node ---- + + // loop over the each node in the elem + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + size_t corner_lid = node_lid; + + // Get corner gid + size_t corner_gid = corners_in_elem(elem_gid, corner_lid); + + // Get node gid + size_t node_gid = nodes_in_elem(elem_gid, node_lid); + + // loop over dimension + for (int dim = 0; dim < num_dims; dim++) + { + corner_vector_storage(corner_gid, dim) = + area_normal(node_lid, 0) * tau_gradient(0, dim) + + area_normal(node_lid, 1) * tau_gradient(1, dim) + + area_normal(node_lid, 2) * tau_gradient(2, dim) + + phi * muc_gradient(node_lid) * (vel_star(dim) - node_vel(rk_level, node_gid, dim)) + + phi * muc(node_lid) * (vel_star_gradient(dim)); + } // end loop over dimension + } // end for loop over nodes in elem + + // --- Update Stress --- + // calculate the new stress at the next rk level, if it is a hypo model + + size_t mat_id = elem_mat_id(elem_gid); + } + // }); // end parallel for loop over elements + + /* + //accumulate node values from corner storage + force_gradient_design->putScalar(0); + + vec_array force_gradient_design_view = force_gradient_design->getLocalView (Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { + size_t corner_id; + for(int icorner=0; icorner < num_corners_in_node(node_id); icorner++){ + corner_id = corners_in_node(node_id,icorner); + force_gradient_design_view(node_id,0) += corner_vector_storage(corner_id, 0); + force_gradient_design_view(node_id,1) += corner_vector_storage(corner_id, 1); + force_gradient_design_view(node_id,2) += corner_vector_storage(corner_id, 2); + } + }); // end parallel for + Kokkos::fence(); + */ + + return; +} // end of routine diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/force_sgh.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_sgh.cpp similarity index 54% rename from src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/force_sgh.cpp rename to src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_sgh.cpp index 2ca9ef946..e4046cddf 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/force_sgh.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_sgh.cpp @@ -1,101 +1,94 @@ - #include "mesh.h" #include "state.h" #include "Simulation_Parameters/Simulation_Parameters_Explicit.h" #include "Simulation_Parameters/FEA_Module/SGH_Parameters.h" #include "FEA_Module_SGH.h" -// ----------------------------------------------------------------------------- -// This function calculates the corner forces and the evolves stress (hypo) -//------------------------------------------------------------------------------ -void FEA_Module_SGH::get_force_sgh(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_pres, - DViewCArrayKokkos &elem_stress, - const DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &elem_mat_id, - DViewCArrayKokkos &corner_force, - const double rk_alpha, - const size_t cycle - ){ - - const_vec_array initial_node_coords = initial_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - + +void FEA_Module_SGH::get_force_sgh(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_pres, + DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + DViewCArrayKokkos& corner_force, + const double rk_alpha, + const size_t cycle + ) +{ + const_vec_array initial_node_coords = initial_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + // check to see if any material model will be run on the host bool any_host_material_model_run = false; - for(int imat = 0; imat < material.size(); imat++){ - if(material.host(imat).strength_run_location == RUN_LOCATION::host){ + for (int imat = 0; imat < material.size(); imat++) + { + if (material.host(imat).strength_run_location == RUN_LOCATION::host) + { any_host_material_model_run = true; } - } - + } + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - + // --- calculate the forces acting on the nodes from the element --- - FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { - + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { const size_t num_dims = 3; const size_t num_nodes_in_elem = 8; - + // total Cauchy stress double tau_array[9]; - + // corner area normals double area_normal_array[24]; - + // estimate of shock direction double shock_dir_array[3]; - + // the sums in the Riemann solver double sum_array[4]; - + // corner shock impeadance x |corner area normal dot shock_dir| double muc_array[8]; - + // Riemann velocity double vel_star_array[3]; - + // velocity gradient double vel_grad_array[9]; - + // --- Create views of arrays to aid the force calculation --- - - ViewCArrayKokkos tau(tau_array, num_dims, num_dims); - ViewCArrayKokkos area_normal(area_normal_array, num_nodes_in_elem, num_dims); - ViewCArrayKokkos shock_dir(shock_dir_array, num_dims); - ViewCArrayKokkos sum(sum_array, 4); - ViewCArrayKokkos muc(muc_array, num_nodes_in_elem); - ViewCArrayKokkos vel_star(vel_star_array, num_dims); - ViewCArrayKokkos vel_grad(vel_grad_array, num_dims, num_dims); - - + + ViewCArrayKokkos tau(tau_array, num_dims, num_dims); + ViewCArrayKokkos area_normal(area_normal_array, num_nodes_in_elem, num_dims); + ViewCArrayKokkos shock_dir(shock_dir_array, num_dims); + ViewCArrayKokkos sum(sum_array, 4); + ViewCArrayKokkos muc(muc_array, num_nodes_in_elem); + ViewCArrayKokkos vel_star(vel_star_array, num_dims); + ViewCArrayKokkos vel_grad(vel_grad_array, num_dims, num_dims); + // --- abviatations of variables --- - + // element volume double vol = elem_vol(elem_gid); - + // create a view of the stress_matrix - ViewCArrayKokkos stress(&elem_stress(rk_level, elem_gid, 0,0), 3, 3); - - + ViewCArrayKokkos stress(&elem_stress(rk_level, elem_gid, 0, 0), 3, 3); + // cut out the node_gids for this element - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 8); - - - + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 8); + // get the B matrix which are the OUTWARD corner area normals get_bmatrix(area_normal, elem_gid, node_coords, elem_node_gids, rk_level); - - + // --- Calculate the velocity gradient --- get_velgrad(vel_grad, elem_node_gids, @@ -104,84 +97,86 @@ void FEA_Module_SGH::get_force_sgh(const DCArrayKokkos &material, vol, elem_gid, rk_level); - - // save vel_grad in elem_vel_grad - for (size_t i = 0; i < 3; i++) { - for (size_t j = 0; j < 3; j++) { - elem_vel_grad(elem_gid,i,j) = vel_grad(i,j); + + // save vel_grad in elem_vel_grad + for (size_t i = 0; i < 3; i++) + { + for (size_t j = 0; j < 3; j++) + { + elem_vel_grad(elem_gid, i, j) = vel_grad(i, j); } } - + // the -1 is for the inward surface area normal, - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - for (size_t dim = 0; dim < num_dims; dim++){ - area_normal(node_lid, dim) = (-1.0)*area_normal(node_lid,dim); + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + for (size_t dim = 0; dim < num_dims; dim++) + { + area_normal(node_lid, dim) = (-1.0) * area_normal(node_lid, dim); } // end for } // end for - - - + double div = elem_div(elem_gid); - - + // vel = [u,v,w] // [du/dx, du/dy, du/dz] // vel_grad = [dv/dx, dv/dy, dv/dz] // [dw/dx, dw/dy, dw/dz] double curl[3]; - curl[0] = vel_grad(2,1) - vel_grad(1,2); // dw/dy - dv/dz - curl[1] = vel_grad(0,2) - vel_grad(2,0); // du/dz - dw/dx - curl[2] = vel_grad(1,0) - vel_grad(0,1); // dv/dx - du/dy - - double mag_curl = sqrt(curl[0]*curl[0] + curl[1]*curl[1] + curl[2]*curl[2]); - - + curl[0] = vel_grad(2, 1) - vel_grad(1, 2); // dw/dy - dv/dz + curl[1] = vel_grad(0, 2) - vel_grad(2, 0); // du/dz - dw/dx + curl[2] = vel_grad(1, 0) - vel_grad(0, 1); // dv/dx - du/dy + + double mag_curl = sqrt(curl[0] * curl[0] + curl[1] * curl[1] + curl[2] * curl[2]); + // --- Calculate the Cauchy stress --- - for (size_t i = 0; i < 3; i++){ - for (size_t j = 0; j < 3; j++){ - tau(i, j) = stress(i,j); + for (size_t i = 0; i < 3; i++) + { + for (size_t j = 0; j < 3; j++) + { + tau(i, j) = stress(i, j); // artificial viscosity can be added here to tau } // end for - } //end for + } // end for // add the pressure - if(elem_pres(elem_gid)!=0){ - for (int i = 0; i < num_dims; i++){ + if (elem_pres(elem_gid) != 0) + { + for (int i = 0; i < num_dims; i++) + { tau(i, i) -= elem_pres(elem_gid); } // end for } - - - // ---- Multidirectional Approximate Riemann solver (MARS) ---- // find the average velocity of the elem, it is an // estimate of the Riemann velocity - + // initialize to Riemann velocity to zero - for (size_t dim = 0; dim < num_dims; dim++){ + for (size_t dim = 0; dim < num_dims; dim++) + { vel_star(dim) = 0.0; } // loop over nodes and calculate an average velocity, which is // an estimate of Riemann velocity - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { // Get node gloabl index and create view of nodal velocity int node_gid = nodes_in_elem(elem_gid, node_lid); - - ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); - - vel_star(0) += 0.125*vel(0); - vel_star(1) += 0.125*vel(1); - vel_star(2) += 0.125*vel(2); - + + ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); + + vel_star(0) += 0.125 * vel(0); + vel_star(1) += 0.125 * vel(1); + vel_star(2) += 0.125 * vel(2); } // end for loop over nodes // find shock direction and shock impedance associated with each node - + // initialize sum term in MARS to zero - for (int i = 0; i < 4; i++){ + for (int i = 0; i < 4; i++) + { sum(i) = 0.0; } @@ -190,102 +185,101 @@ void FEA_Module_SGH::get_force_sgh(const DCArrayKokkos &material, size_t mat_id = elem_mat_id(elem_gid); // loop over the nodes of the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { - + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { // Get global node id size_t node_gid = nodes_in_elem(elem_gid, node_lid); // Create view of nodal velocity - ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); + ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); // Get an estimate of the shock direction. - mag_vel = sqrt( (vel(0) - vel_star(0) )*(vel(0) - vel_star(0) ) - + (vel(1) - vel_star(1) )*(vel(1) - vel_star(1) ) - + (vel(2) - vel_star(2) )*(vel(2) - vel_star(2) ) ); + mag_vel = sqrt( (vel(0) - vel_star(0) ) * (vel(0) - vel_star(0) ) + + (vel(1) - vel_star(1) ) * (vel(1) - vel_star(1) ) + + (vel(2) - vel_star(2) ) * (vel(2) - vel_star(2) ) ); - - if (mag_vel > small) { - + if (mag_vel > small) + { // estimate of the shock direction, a unit normal - for (int dim = 0; dim < num_dims; dim++){ + for (int dim = 0; dim < num_dims; dim++) + { shock_dir(dim) = (vel(dim) - vel_star(dim)) / mag_vel; } } - - else { - + else + { // if there is no velocity change, then use the surface area // normal as the shock direction - mag = sqrt( area_normal(node_lid, 0)*area_normal(node_lid, 0) - + area_normal(node_lid, 1)*area_normal(node_lid, 1) - + area_normal(node_lid, 2)*area_normal(node_lid, 2) ); - + mag = sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) + + area_normal(node_lid, 1) * area_normal(node_lid, 1) + + area_normal(node_lid, 2) * area_normal(node_lid, 2) ); + // estimate of the shock direction - for (int dim = 0; dim < num_dims; dim++){ - shock_dir(dim) = area_normal(node_lid, dim)/mag; + for (int dim = 0; dim < num_dims; dim++) + { + shock_dir(dim) = area_normal(node_lid, dim) / mag; } - } // end if mag_vel - // cell divergence indicates compression or expansions - if (div < 0){ // element in compression + if (div < 0) // element in compression + { muc(node_lid) = elem_den(elem_gid) * - (material(mat_id).q1*elem_sspd(elem_gid) + material(mat_id).q2*mag_vel); + (material(mat_id).q1 * elem_sspd(elem_gid) + material(mat_id).q2 * mag_vel); } - else { // element in expansion + else // element in expansion + { muc(node_lid) = elem_den(elem_gid) * - (material(mat_id).q1ex*elem_sspd(elem_gid) + material(mat_id).q2ex*mag_vel); + (material(mat_id).q1ex * elem_sspd(elem_gid) + material(mat_id).q2ex * mag_vel); } // end if on divergence sign - size_t use_shock_dir = 0; double mu_term; - + // Coding to use shock direction - if (use_shock_dir == 1){ + if (use_shock_dir == 1) + { // this is denominator of the Riamann solver and the multiplier // on velocity in the numerator. It filters on the shock // direction - mu_term = muc(node_lid)* - fabs( shock_dir(0)*area_normal(node_lid,0) - + shock_dir(1)*area_normal(node_lid,1) - + shock_dir(2)*area_normal(node_lid,2) ); + mu_term = muc(node_lid) * + fabs(shock_dir(0) * area_normal(node_lid, 0) + + shock_dir(1) * area_normal(node_lid, 1) + + shock_dir(2) * area_normal(node_lid, 2) ); } - else { - // Using a full tensoral Riemann jump relation - mu_term = muc(node_lid) - * sqrt( area_normal(node_lid, 0)*area_normal(node_lid, 0) - + area_normal(node_lid, 1)*area_normal(node_lid, 1) - + area_normal(node_lid, 2)*area_normal(node_lid, 2) ); + else + { + // Using a full tensoral Riemann jump relation + mu_term = muc(node_lid) + * sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) + + area_normal(node_lid, 1) * area_normal(node_lid, 1) + + area_normal(node_lid, 2) * area_normal(node_lid, 2) ); } - - sum(0) += mu_term*vel(0); - sum(1) += mu_term*vel(1); - sum(2) += mu_term*vel(2); + + sum(0) += mu_term * vel(0); + sum(1) += mu_term * vel(1); + sum(2) += mu_term * vel(2); sum(3) += mu_term; muc(node_lid) = mu_term; // the impeadance time surface area is stored here - } // end for node_lid loop over nodes of the elem - - - // The Riemann velocity, called vel_star - if (sum(3) > fuzz) { - for (size_t i = 0; i < num_dims; i++) { - vel_star(i) = sum(i)/sum(3); + if (sum(3) > fuzz) + { + for (size_t i = 0; i < num_dims; i++) + { + vel_star(i) = sum(i) / sum(3); } } - else { - for (int i = 0; i < num_dims; i++){ + else + { + for (int i = 0; i < num_dims; i++) + { vel_star(i) = 0.0; } } // end if - - // ---- Calculate the shock detector for the Riemann-solver ---- // // The dissipation from the Riemann problem is limited by phi @@ -298,7 +292,7 @@ void FEA_Module_SGH::get_force_sgh(const DCArrayKokkos &material, // phi = 0 highest-order solution // phi = 1 first order solution // - + double phi = 0.0; // the shock detector double r_face = 1.0; // the ratio on the face double r_min = 1.0; // the min ratio for the cell @@ -307,88 +301,84 @@ void FEA_Module_SGH::get_force_sgh(const DCArrayKokkos &material, double n_coef = 1.0; // the power on the limiting coefficient // (1=nominal, and n_coeff > 1 oscillatory) - // loop over the nieghboring cells - for (size_t elem_lid = 0; elem_lid < num_elems_in_elem(elem_gid); elem_lid++){ - + for (size_t elem_lid = 0; elem_lid < num_elems_in_elem(elem_gid); elem_lid++) + { // Get global index for neighboring cell size_t neighbor_gid = elems_in_elem(elem_gid, elem_lid); - + // calculate the velocity divergence in neighbor double div_neighbor = elem_div(neighbor_gid); - r_face = r_coef*(div_neighbor + small)/(div + small); + r_face = r_coef * (div_neighbor + small) / (div + small); // store the smallest face ratio r_min = fmin(r_face, r_min); - } // end for elem_lid - // calculate standard shock detector phi = 1.0 - fmax(0.0, r_min); phi = pow(phi, n_coef); // Mach number shock detector - double omega = 20.0; // weighting factor on Mach number - double third = 1.0/3.0; + double omega = 20.0; // weighting factor on Mach number + double third = 1.0 / 3.0; double c_length = pow(vol, third); // characteristic length - double alpha = fmin(1.0, omega * (c_length * fabs(div))/(elem_sspd(elem_gid) + fuzz) ); + double alpha = fmin(1.0, omega * (c_length * fabs(div)) / (elem_sspd(elem_gid) + fuzz) ); // curl limiter on Q double omega_curl = 1.0; // increase this to increase robustness, but as it increases, additional dissipation will be introduce, blocking bending - double phi_curl = fmin(1.0, omega_curl*fabs(div)/(mag_curl + fuzz)); // disable Q when vorticity is high - + double phi_curl = fmin(1.0, omega_curl * fabs(div) / (mag_curl + fuzz)); // disable Q when vorticity is high + phi = 1.0; // for the future case of using a slope limiter approach - phi = fmin(phi_curl*phi, alpha*phi);// if noise arrises in simulation on really smooth flows, then try something like - phi = fmax(phi,0.001); // ensuring a very small amount of dissipation for stability and robustness + phi = fmin(phi_curl * phi, alpha * phi); // if noise arrises in simulation on really smooth flows, then try something like + phi = fmax(phi, 0.001); // ensuring a very small amount of dissipation for stability and robustness - if(material(mat_id).maximum_limiter){ - phi=1; + if (material(mat_id).maximum_limiter) + { + phi = 1; } - + // ---- Calculate the Riemann force on each node ---- // loop over the each node in the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { - + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { size_t corner_lid = node_lid; // Get corner gid size_t corner_gid = corners_in_elem(elem_gid, corner_lid); - + // Get node gid size_t node_gid = nodes_in_elem(elem_gid, node_lid); - + // loop over dimension - for (int dim = 0; dim < num_dims; dim++){ + for (int dim = 0; dim < num_dims; dim++) + { corner_force(corner_gid, dim) = - area_normal(node_lid, 0)*tau(0, dim) - + area_normal(node_lid, 1)*tau(1, dim) - + area_normal(node_lid, 2)*tau(2, dim) - + phi*muc(node_lid)*(vel_star(dim) - node_vel(rk_level, node_gid, dim)); - //test clause - //corner_force(corner_gid, dim) = -0.00001*node_vel(rk_level, node_gid, dim); - //corner_force(corner_gid, dim) = 0.0001*relative_element_densities(elem_gid)-0.00001*node_vel(rk_level, node_gid, dim); - + area_normal(node_lid, 0) * tau(0, dim) + + area_normal(node_lid, 1) * tau(1, dim) + + area_normal(node_lid, 2) * tau(2, dim) + + phi * muc(node_lid) * (vel_star(dim) - node_vel(rk_level, node_gid, dim)); + // test clause + // corner_force(corner_gid, dim) = -0.00001*node_vel(rk_level, node_gid, dim); + // corner_force(corner_gid, dim) = 0.0001*relative_element_densities(elem_gid)-0.00001*node_vel(rk_level, node_gid, dim); } // end loop over dimension - } // end for loop over nodes in elem - - + // --- Update Stress --- // calculate the new stress at the next rk level, if it is a hypo model - - // hypo elastic plastic model - if(material(mat_id).strength_type == STRENGTH_TYPE::hypo){ - - if(material(mat_id).strength_run_location == RUN_LOCATION::device){ + // hypo elastic plastic model + if (material(mat_id).strength_type == STRENGTH_TYPE::hypo) + { + if (material(mat_id).strength_run_location == RUN_LOCATION::device) + { // cut out the node_gids for this element - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 8); + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 8); StrengthParent* strength_model = elem_strength(elem_gid).model; - + // --- call strength model --- strength_model->calc_stress(elem_pres, elem_stress, @@ -399,52 +389,49 @@ void FEA_Module_SGH::get_force_sgh(const DCArrayKokkos &material, elem_user_output_vars, elem_sspd, elem_den(elem_gid), - elem_sie(rk_level,elem_gid), + elem_sie(rk_level, elem_gid), vel_grad, elem_node_gids, node_coords, node_vel, elem_vol(elem_gid), - dt, + dt, rk_alpha, cycle, rk_level); } // end logical for strength run location - } // end logical on hypo strength model - - }); // end parallel for loop over elements - - if (any_host_material_model_run == true) { + if (any_host_material_model_run == true) + { // update host elem_vel_grad.update_host(); // below host updates are commented out to save time because they are not used for // the current user model. if a user model uses any of them, please uncomment it - //elem_pres.update_host(); - //elem_den.upsate_host(); - //elem_sie.update_host(); - //node_coords.update_host(); - //node_vel.update_host(); - //elem_vol.update_host(); + // elem_pres.update_host(); + // elem_den.upsate_host(); + // elem_sie.update_host(); + // node_coords.update_host(); + // node_vel.update_host(); + // elem_vol.update_host(); // calling user strength model on host - for (size_t elem_gid = 0; elem_gid < mesh.num_elems; elem_gid++) { - + for (size_t elem_gid = 0; elem_gid < mesh.num_elems; elem_gid++) + { const size_t num_dims = 3; - size_t mat_id = elem_mat_id.host(elem_gid); + size_t mat_id = elem_mat_id.host(elem_gid); // hypo elastic plastic model - if(material.host(mat_id).strength_type == STRENGTH_TYPE::hypo){ - - if(material.host(mat_id).strength_run_location == RUN_LOCATION::host){ - + if (material.host(mat_id).strength_type == STRENGTH_TYPE::hypo) + { + if (material.host(mat_id).strength_run_location == RUN_LOCATION::host) + { // cut out the node_gids for this element - ViewCArrayKokkos elem_node_gids(&nodes_in_elem.host(elem_gid, 0), 8); + ViewCArrayKokkos elem_node_gids(&nodes_in_elem.host(elem_gid, 0), 8); - // cut out vel_grad - ViewCArrayKokkos vel_grad(&elem_vel_grad.host(elem_gid,0,0), num_dims, num_dims); + // cut out vel_grad + ViewCArrayKokkos vel_grad(&elem_vel_grad.host(elem_gid, 0, 0), num_dims, num_dims); StrengthParent* strength_model = elem_strength.host(elem_gid).model; @@ -458,7 +445,7 @@ void FEA_Module_SGH::get_force_sgh(const DCArrayKokkos &material, elem_user_output_vars, elem_sspd, elem_den.host(elem_gid), - elem_sie.host(rk_level,elem_gid), + elem_sie.host(rk_level, elem_gid), vel_grad, elem_node_gids, node_coords, @@ -468,90 +455,81 @@ void FEA_Module_SGH::get_force_sgh(const DCArrayKokkos &material, rk_alpha, cycle, rk_level); - } // end logical for strength run location - } // end logical on hypo strength model - - } // end for loop over elements + } // end for loop over elements // update device elem_stress.update_device(); - } // end for if host_material_model_run == true return; - } // end of routine - // ----------------------------------------------------------------------------- // This function calculates the corner forces and the evolves stress (hypo) -//------------------------------------------------------------------------------ -void FEA_Module_SGH::get_force_sgh2D(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &elem_mat_id, - DViewCArrayKokkos &corner_force, - const double rk_alpha, - const size_t cycle - ){ - +// ------------------------------------------------------------------------------ +void FEA_Module_SGH::get_force_sgh2D(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_pres, + const DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + DViewCArrayKokkos& corner_force, + const double rk_alpha, + const size_t cycle + ) +{ const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; // --- calculate the forces acting on the nodes from the element --- - FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { - + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { const size_t num_dims = 2; const size_t num_nodes_in_elem = 4; - + // total Cauchy stress double tau_array[9]; - + // corner area normals double area_normal_array[8]; // 4 corners and 2 directions - + // estimate of shock direction double shock_dir_array[2]; - + // the sums in the Riemann solver double sum_array[4]; - + // corner shock impeadance x |corner area normal dot shock_dir| double muc_array[4]; - + // Riemann velocity double vel_star_array[2]; - + // velocity gradient double vel_grad_array[9]; - + // --- Create views of arrays to aid the force calculation --- - - ViewCArrayKokkos tau(tau_array, 3, 3); - ViewCArrayKokkos area_normal(area_normal_array, num_nodes_in_elem, num_dims); - ViewCArrayKokkos shock_dir(shock_dir_array, num_dims); - ViewCArrayKokkos sum(sum_array, 4); - ViewCArrayKokkos muc(muc_array, num_nodes_in_elem); - ViewCArrayKokkos vel_star(vel_star_array, num_dims); - ViewCArrayKokkos vel_grad(vel_grad_array, 3, 3); - - + + ViewCArrayKokkos tau(tau_array, 3, 3); + ViewCArrayKokkos area_normal(area_normal_array, num_nodes_in_elem, num_dims); + ViewCArrayKokkos shock_dir(shock_dir_array, num_dims); + ViewCArrayKokkos sum(sum_array, 4); + ViewCArrayKokkos muc(muc_array, num_nodes_in_elem); + ViewCArrayKokkos vel_star(vel_star_array, num_dims); + ViewCArrayKokkos vel_grad(vel_grad_array, 3, 3); + // create a view of the stress_matrix - ViewCArrayKokkos stress(&elem_stress(rk_level, elem_gid, 0,0), 3, 3); - - + ViewCArrayKokkos stress(&elem_stress(rk_level, elem_gid, 0, 0), 3, 3); + // cut out the node_gids for this element - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 4); - + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 4); + // get the B matrix which are the OUTWARD corner area normals get_bmatrix2D(area_normal, elem_gid, @@ -559,11 +537,10 @@ void FEA_Module_SGH::get_force_sgh2D(const DCArrayKokkos &material, elem_node_gids, rk_level); // NOTE: I added a minux in bmatrix2D, it should be outward pointing now? - + // facial area of the element double elem_area = get_area_quad(elem_gid, node_coords, elem_node_gids, rk_level); - - + // --- Calculate the velocity gradient --- get_velgrad2D(vel_grad, elem_node_gids, @@ -573,71 +550,70 @@ void FEA_Module_SGH::get_force_sgh2D(const DCArrayKokkos &material, elem_area, elem_gid, rk_level); - - + // the -1 is for the inward surface area normal, - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - for (size_t dim = 0; dim < num_dims; dim++){ - area_normal(node_lid, dim) = (-1.0)*area_normal(node_lid,dim); + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + for (size_t dim = 0; dim < num_dims; dim++) + { + area_normal(node_lid, dim) = (-1.0) * area_normal(node_lid, dim); } // end for } // end for - - - + double div = elem_div(elem_gid); - - + // vel = [u,v] // [du/dx, du/dy] // vel_grad = [dv/dx, dv/dy] double curl; - curl = vel_grad(1,0) - vel_grad(0,1); // dv/dx - du/dy - + curl = vel_grad(1, 0) - vel_grad(0, 1); // dv/dx - du/dy + double mag_curl = curl; - - + // --- Calculate the Cauchy stress --- - for (size_t i = 0; i < 3; i++){ - for (size_t j = 0; j < 3; j++){ - tau(i, j) = stress(i,j); + for (size_t i = 0; i < 3; i++) + { + for (size_t j = 0; j < 3; j++) + { + tau(i, j) = stress(i, j); // artificial viscosity can be added here to tau } // end for - } //end for + } // end for // add the pressure - for (int i = 0; i < 3; i++){ + for (int i = 0; i < 3; i++) + { tau(i, i) -= elem_pres(elem_gid); } // end for - - // ---- Multidirectional Approximate Riemann solver (MARS) ---- // find the average velocity of the elem, it is an // estimate of the Riemann velocity - + // initialize to Riemann velocity to zero - for (size_t dim = 0; dim < num_dims; dim++){ + for (size_t dim = 0; dim < num_dims; dim++) + { vel_star(dim) = 0.0; } // loop over nodes and calculate an average velocity, which is // an estimate of Riemann velocity - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { // Get node gloabl index and create view of nodal velocity int node_gid = nodes_in_elem(elem_gid, node_lid); - - ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); - - vel_star(0) += 0.25*vel(0); - vel_star(1) += 0.25*vel(1); - + + ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); + + vel_star(0) += 0.25 * vel(0); + vel_star(1) += 0.25 * vel(1); } // end for loop over nodes // find shock direction and shock impedance associated with each node - + // initialize sum term in MARS to zero - for (int i = 0; i < 4; i++){ + for (int i = 0; i < 4; i++) + { sum(i) = 0.0; } @@ -645,98 +621,97 @@ void FEA_Module_SGH::get_force_sgh2D(const DCArrayKokkos &material, double mag_vel; // magnitude of velocity // loop over the nodes of the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { - + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { // Get global node id size_t node_gid = nodes_in_elem(elem_gid, node_lid); // Create view of nodal velocity - ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); + ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); // Get an estimate of the shock direction. - mag_vel = sqrt( (vel(0) - vel_star(0) )*(vel(0) - vel_star(0) ) - + (vel(1) - vel_star(1) )*(vel(1) - vel_star(1) ) ); + mag_vel = sqrt( (vel(0) - vel_star(0) ) * (vel(0) - vel_star(0) ) + + (vel(1) - vel_star(1) ) * (vel(1) - vel_star(1) ) ); - - if (mag_vel > small) { - + if (mag_vel > small) + { // estimate of the shock direction, a unit normal - for (int dim = 0; dim < num_dims; dim++){ + for (int dim = 0; dim < num_dims; dim++) + { shock_dir(dim) = (vel(dim) - vel_star(dim)) / mag_vel; } } - - else { - + else + { // if there is no velocity change, then use the surface area // normal as the shock direction - mag = sqrt( area_normal(node_lid, 0)*area_normal(node_lid, 0) - + area_normal(node_lid, 1)*area_normal(node_lid, 1) ); - + mag = sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) + + area_normal(node_lid, 1) * area_normal(node_lid, 1) ); + // estimate of the shock direction - for (int dim = 0; dim < num_dims; dim++){ - shock_dir(dim) = area_normal(node_lid, dim)/mag; + for (int dim = 0; dim < num_dims; dim++) + { + shock_dir(dim) = area_normal(node_lid, dim) / mag; } - } // end if mag_vel - // cell divergence indicates compression or expansions size_t mat_id = elem_mat_id(elem_gid); - if (div < 0){ // element in compression + if (div < 0) // element in compression + { muc(node_lid) = elem_den(elem_gid) * - (material(mat_id).q1*elem_sspd(elem_gid) + material(mat_id).q2*mag_vel); + (material(mat_id).q1 * elem_sspd(elem_gid) + material(mat_id).q2 * mag_vel); } - else { // element in expansion + else // element in expansion + { muc(node_lid) = elem_den(elem_gid) * - (material(mat_id).q1ex*elem_sspd(elem_gid) + material(mat_id).q2ex*mag_vel); + (material(mat_id).q1ex * elem_sspd(elem_gid) + material(mat_id).q2ex * mag_vel); } // end if on divergence sign - size_t use_shock_dir = 0; double mu_term; - + // Coding to use shock direction - if (use_shock_dir == 1){ + if (use_shock_dir == 1) + { // this is denominator of the Riamann solver and the multiplier // on velocity in the numerator. It filters on the shock // direction - mu_term = muc(node_lid)* - fabs( shock_dir(0)*area_normal(0) - + shock_dir(1)*area_normal(1) ); + mu_term = muc(node_lid) * + fabs(shock_dir(0) * area_normal(0) + + shock_dir(1) * area_normal(1) ); } - else { - // Using a full tensoral Riemann jump relation - mu_term = muc(node_lid) - * sqrt( area_normal(node_lid, 0)*area_normal(node_lid, 0) - + area_normal(node_lid, 1)*area_normal(node_lid, 1) ); + else + { + // Using a full tensoral Riemann jump relation + mu_term = muc(node_lid) + * sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) + + area_normal(node_lid, 1) * area_normal(node_lid, 1) ); } - - sum(0) += mu_term*vel(0); - sum(1) += mu_term*vel(1); + + sum(0) += mu_term * vel(0); + sum(1) += mu_term * vel(1); sum(3) += mu_term; muc(node_lid) = mu_term; // the impeadance time surface area is stored here - } // end for node_lid loop over nodes of the elem - - - // The Riemann velocity, called vel_star - if (sum(3) > fuzz) { - for (size_t i = 0; i < num_dims; i++) { - vel_star(i) = sum(i)/sum(3); + if (sum(3) > fuzz) + { + for (size_t i = 0; i < num_dims; i++) + { + vel_star(i) = sum(i) / sum(3); } } - else { - for (int i = 0; i < num_dims; i++){ + else + { + for (int i = 0; i < num_dims; i++) + { vel_star(i) = 0.0; } } // end if - - - + // ---- Calculate the shock detector for the Riemann-solver ---- // // The dissipation from the Riemann problem is limited by phi @@ -749,7 +724,7 @@ void FEA_Module_SGH::get_force_sgh2D(const DCArrayKokkos &material, // phi = 0 highest-order solution // phi = 1 first order solution // - + double phi = 0.0; // the shock detector double r_face = 1.0; // the ratio on the face double r_min = 1.0; // the min ratio for the cell @@ -758,95 +733,88 @@ void FEA_Module_SGH::get_force_sgh2D(const DCArrayKokkos &material, double n_coef = 1.0; // the power on the limiting coefficient // (1=nominal, and n_coeff > 1 oscillatory) - // loop over the nieghboring cells - for (size_t elem_lid = 0; elem_lid < num_elems_in_elem(elem_gid); elem_lid++){ - + for (size_t elem_lid = 0; elem_lid < num_elems_in_elem(elem_gid); elem_lid++) + { // Get global index for neighboring cell size_t neighbor_gid = elems_in_elem(elem_gid, elem_lid); - + // calculate the velocity divergence in neighbor double div_neighbor = elem_div(neighbor_gid); - r_face = r_coef*(div_neighbor + small)/(div + small); + r_face = r_coef * (div_neighbor + small) / (div + small); // store the smallest face ratio r_min = fmin(r_face, r_min); - } // end for elem_lid - // calculate standard shock detector phi = 1.0 - fmax(0.0, r_min); phi = pow(phi, n_coef); // Mach number shock detector - double omega = 20.0;//20.0; // weighting factor on Mach number + double omega = 20.0; // 20.0; // weighting factor on Mach number double c_length = sqrt(elem_area); // characteristic length - double alpha = fmin(1.0, omega * (c_length * fabs(div))/(elem_sspd(elem_gid) + fuzz) ); - + double alpha = fmin(1.0, omega * (c_length * fabs(div)) / (elem_sspd(elem_gid) + fuzz) ); + // use Mach based detector with standard shock detector // turn off dissipation in expansion - //alpha = fmax(-fabs(div0)/div0 * alpha, 0.0); // this should be if(div0<0) alpha=alpha else alpha=0 - - phi = alpha*phi; - + // alpha = fmax(-fabs(div0)/div0 * alpha, 0.0); // this should be if(div0<0) alpha=alpha else alpha=0 + + phi = alpha * phi; + // curl limiter on Q - double phi_curl = fmin(1.0, 4.0*fabs(div)/(mag_curl + fuzz)); // disable Q when vorticity is high - //phi = phi_curl*phi; + double phi_curl = fmin(1.0, 4.0 * fabs(div) / (mag_curl + fuzz)); // disable Q when vorticity is high + // phi = phi_curl*phi; phi = 1.0; // WARNING WARNING WARNING // ---- Calculate the Riemann force on each node ---- // loop over the each node in the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { - + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { size_t corner_lid = node_lid; // Get corner gid size_t corner_gid = corners_in_elem(elem_gid, corner_lid); - + // Get node gid size_t node_gid = nodes_in_elem(elem_gid, node_lid); - - // loop over dimension - for (int dim = 0; dim < num_dims; dim++){ + // loop over dimension + for (int dim = 0; dim < num_dims; dim++) + { corner_force(corner_gid, dim) = - area_normal(node_lid, 0)*tau(0, dim) - + area_normal(node_lid, 1)*tau(1, dim) - + phi*muc(node_lid)*(vel_star(dim) - node_vel(rk_level, node_gid, dim)); - + area_normal(node_lid, 0) * tau(0, dim) + + area_normal(node_lid, 1) * tau(1, dim) + + phi * muc(node_lid) * (vel_star(dim) - node_vel(rk_level, node_gid, dim)); } // end loop over dimension - - + // ---- add hoop stress terms ---- - - double node_radius = node_coords(rk_level,node_gid,1); - - if(node_radius>1e-14){ + + double node_radius = node_coords(rk_level, node_gid, 1); + + if (node_radius > 1e-14) + { // sigma_RZ / R_p - corner_force(corner_gid, 0) += tau(1,0)*elem_area*0.25/node_radius; - + corner_force(corner_gid, 0) += tau(1, 0) * elem_area * 0.25 / node_radius; + // (sigma_RR - sigma_theta) / R_p - corner_force(corner_gid, 1) += (tau(1,1) - tau(2,2))*elem_area*0.25/node_radius; + corner_force(corner_gid, 1) += (tau(1, 1) - tau(2, 2)) * elem_area * 0.25 / node_radius; } // end if radius >0 - } // end for loop over nodes in elem - - - + // --- Update Stress --- // calculate the new stress at the next rk level, if it is a hypo model - + size_t mat_id = elem_mat_id(elem_gid); - - // hypo elastic plastic model - if(material(mat_id).strength_type == STRENGTH_TYPE::hypo){ + // hypo elastic plastic model + if (material(mat_id).strength_type == STRENGTH_TYPE::hypo) + { // cut out the node_gids for this element - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 4); + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 4); StrengthParent* strength_model = elem_strength(elem_gid).model; @@ -869,45 +837,40 @@ void FEA_Module_SGH::get_force_sgh2D(const DCArrayKokkos &material, dt, rk_alpha, cycle, - rk_level); - + rk_level); } // end logical on hypo strength model - - }); // end parallel for loop over elements - return; - } // end of routine for 2D force and stress update // ----------------------------------------------------------------------------- // This function calculates the corner forces and the evolves stress (hypo) -//------------------------------------------------------------------------------ -void FEA_Module_SGH::applied_forces(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &node_mass, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &elem_mat_id, - DViewCArrayKokkos &corner_force, - const double rk_alpha, - const size_t cycle - ){ - - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - const size_t num_dim = mesh.num_dims; - const_vec_array all_initial_node_coords = all_initial_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - const size_t num_lcs = module_params->loading.size(); - - const DCArrayKokkos mat_fill = simparam->mat_fill; - const DCArrayKokkos loading = module_params->loading; - - //debug check - //std::cout << "NUMBER OF LOADING CONDITIONS: " << num_lcs << std::endl; +// ------------------------------------------------------------------------------ +void FEA_Module_SGH::applied_forces(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_mass, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + DViewCArrayKokkos& corner_force, + const double rk_alpha, + const size_t cycle + ) +{ + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + const size_t num_dim = mesh.num_dims; + const_vec_array all_initial_node_coords = all_initial_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + const size_t num_lcs = module_params->loading.size(); + + const DCArrayKokkos mat_fill = simparam->mat_fill; + const DCArrayKokkos loading = module_params->loading; + + // debug check + // std::cout << "NUMBER OF LOADING CONDITIONS: " << num_lcs << std::endl; // walk over the nodes to update the velocity FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { @@ -916,41 +879,44 @@ void FEA_Module_SGH::applied_forces(const DCArrayKokkos &material, double node_force[3]; double applied_force[3]; double radius; - for (size_t dim = 0; dim < num_dim; dim++){ + for (size_t dim = 0; dim < num_dim; dim++) + { node_force[dim] = 0.0; current_node_coords[dim] = all_initial_node_coords(node_gid, dim); } // end for dim - radius = sqrt(current_node_coords[0]*current_node_coords[0]+current_node_coords[1]*current_node_coords[1]+current_node_coords[2]*current_node_coords[2]); - for(size_t ilc=0; ilc < num_lcs; ilc++){ - //debug check - //std::cout << "LOADING CONDITION VOLUME TYPE: " << to_string(loading(ilc).volume) << std::endl; - - bool fill_this = loading(ilc).volume.contains(current_node_coords); - if(fill_this){ - // loop over all corners around the node and calculate the nodal force - for (size_t corner_lid=0; corner_lid& node_coords, + const DViewCArrayKokkos& node_vel) +{ + const size_t rk_level = rk_num_bins - 1; + int num_dims = num_dim; + + // loop over all the nodes in the mesh + FOR_ALL_CLASS(node_gid, 0, num_nodes, { + for (int dim = 0; dim < num_dims; dim++) + { + double half_vel = (node_vel(rk_level, node_gid, dim) + node_vel(0, node_gid, dim)) * 0.5; + node_coords(rk_level, node_gid, dim) = node_coords(0, node_gid, dim) + rk_alpha * dt * half_vel; + } + }); // end parallel for over nodes +} // end subroutine + +// ----------------------------------------------------------------------------- +// This function claculates +// B_p = J^{-T} \cdot (\nabla_{xi} \phi_p w +// where +// \phi_p is the basis function for vertex p +// w is the 1 gauss point for the cell (everything is evaluted at this point) +// J^{-T} is the inverse transpose of the Jacobi matrix +// \nabla_{xi} is the gradient opperator in the reference coordinates +// +// B_p is the OUTWARD corner area normal at node p +// ------------------------------------------------------------------------------ +KOKKOS_FUNCTION +void FEA_Module_SGH::get_bmatrix(const ViewCArrayKokkos& B_matrix, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const +{ + const size_t num_nodes = 8; + + double x_array[8]; + double y_array[8]; + double z_array[8]; + + // x, y, z coordinates of elem vertices + auto x = ViewCArrayKokkos(x_array, num_nodes); + auto y = ViewCArrayKokkos(y_array, num_nodes); + auto z = ViewCArrayKokkos(z_array, num_nodes); + + // get the coordinates of the nodes(rk,elem,node) in this element + for (int node_lid = 0; node_lid < num_nodes; node_lid++) + { + x(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 0); + y(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 1); + z(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 2); + } // end for + + double twelth = 1. / 12.; + + B_matrix(0, 0) = (+y(1) * (-z(2) - z(3) + z(4) + z(5) ) + + y(2) * (+z(1) - z(3) ) + + y(3) * (+z(1) + z(2) - z(4) - z(7) ) + + y(4) * (-z(1) + z(3) - z(5) + z(7) ) + + y(5) * (-z(1) + z(4) ) + + y(7) * (+z(3) - z(4) ) ) * twelth; + + B_matrix(1, 0) = (+y(0) * (+z(2) + z(3) - z(4) - z(5) ) + + y(2) * (-z(0) - z(3) + z(5) + z(6) ) + + y(3) * (-z(0) + z(2) ) + + y(4) * (+z(0) - z(5) ) + + y(5) * (+z(0) - z(2) + z(4) - z(6) ) + + y(6) * (-z(2) + z(5) ) ) * twelth; + + B_matrix(2, 0) = (+y(0) * (-z(1) + z(3) ) + + y(1) * (+z(0) + z(3) - z(5) - z(6) ) + + y(3) * (-z(0) - z(1) + z(6) + z(7) ) + + y(5) * (+z(1) - z(6) ) + + y(6) * (+z(1) - z(3) + z(5) - z(7) ) + + y(7) * (-z(3) + z(6) ) ) * twelth; + + B_matrix(3, 0) = (+y(0) * (-z(1) - z(2) + z(4) + z(7) ) + + y(1) * (+z(0) - z(2) ) + + y(2) * (+z(0) + z(1) - z(6) - z(7) ) + + y(4) * (-z(0) + z(7) ) + + y(6) * (+z(2) - z(7) ) + + y(7) * (-z(0) + z(2) - z(4) + z(6) ) ) * twelth; + + B_matrix(4, 0) = (+y(0) * (+z(1) - z(3) + z(5) - z(7) ) + + y(1) * (-z(0) + z(5) ) + + y(3) * (+z(0) - z(7) ) + + y(5) * (-z(0) - z(1) + z(6) + z(7) ) + + y(6) * (-z(5) + z(7) ) + + y(7) * (+z(0) + z(3) - z(5) - z(6) ) ) * twelth; + + B_matrix(5, 0) = (+y(0) * (+z(1) - z(4) ) + + y(1) * (-z(0) + z(2) - z(4) + z(6) ) + + y(2) * (-z(1) + z(6) ) + + y(4) * (+z(0) + z(1) - z(6) - z(7) ) + + y(6) * (-z(1) - z(2) + z(4) + z(7) ) + + y(7) * (+z(4) - z(6) ) ) * twelth; + + B_matrix(6, 0) = (+y(1) * (+z(2) - z(5) ) + + y(2) * (-z(1) + z(3) - z(5) + z(7) ) + + y(3) * (-z(2) + z(7) ) + + y(4) * (+z(5) - z(7) ) + + y(5) * (+z(1) + z(2) - z(4) - z(7) ) + + y(7) * (-z(2) - z(3) + z(4) + z(5) ) ) * twelth; + + B_matrix(7, 0) = (+y(0) * (-z(3) + z(4) ) + + y(2) * (+z(3) - z(6) ) + + y(3) * (+z(0) - z(2) + z(4) - z(6) ) + + y(4) * (-z(0) - z(3) + z(5) + z(6) ) + + y(5) * (-z(4) + z(6) ) + + y(6) * (+z(2) + z(3) - z(4) - z(5) ) ) * twelth; + + B_matrix(0, 1) = (+z(1) * (-x(2) - x(3) + x(4) + x(5) ) + + z(2) * (+x(1) - x(3) ) + + z(3) * (+x(1) + x(2) - x(4) - x(7) ) + + z(4) * (-x(1) + x(3) - x(5) + x(7) ) + + z(5) * (-x(1) + x(4) ) + + z(7) * (+x(3) - x(4) ) ) * twelth; + + B_matrix(1, 1) = (+z(0) * (+x(2) + x(3) - x(4) - x(5) ) + + z(2) * (-x(0) - x(3) + x(5) + x(6) ) + + z(3) * (-x(0) + x(2) ) + + z(4) * (+x(0) - x(5) ) + + z(5) * (+x(0) - x(2) + x(4) - x(6) ) + + z(6) * (-x(2) + x(5) ) ) * twelth; + + B_matrix(2, 1) = (+z(0) * (-x(1) + x(3) ) + + z(1) * (+x(0) + x(3) - x(5) - x(6) ) + + z(3) * (-x(0) - x(1) + x(6) + x(7) ) + + z(5) * (+x(1) - x(6) ) + + z(6) * (+x(1) - x(3) + x(5) - x(7) ) + + z(7) * (-x(3) + x(6) ) ) * twelth; + + B_matrix(3, 1) = (+z(0) * (-x(1) - x(2) + x(4) + x(7) ) + + z(1) * (+x(0) - x(2) ) + + z(2) * (+x(0) + x(1) - x(6) - x(7) ) + + z(4) * (-x(0) + x(7) ) + + z(6) * (+x(2) - x(7) ) + + z(7) * (-x(0) + x(2) - x(4) + x(6) ) ) * twelth; + + B_matrix(4, 1) = (+z(0) * (+x(1) - x(3) + x(5) - x(7) ) + + z(1) * (-x(0) + x(5) ) + + z(3) * (+x(0) - x(7) ) + + z(5) * (-x(0) - x(1) + x(6) + x(7) ) + + z(6) * (-x(5) + x(7) ) + + z(7) * (+x(0) + x(3) - x(5) - x(6) ) ) * twelth; + + B_matrix(5, 1) = (+z(0) * (+x(1) - x(4) ) + + z(1) * (-x(0) + x(2) - x(4) + x(6) ) + + z(2) * (-x(1) + x(6) ) + + z(4) * (+x(0) + x(1) - x(6) - x(7) ) + + z(6) * (-x(1) - x(2) + x(4) + x(7) ) + + z(7) * (+x(4) - x(6) ) ) * twelth; + + B_matrix(6, 1) = (+z(1) * (+x(2) - x(5) ) + + z(2) * (-x(1) + x(3) - x(5) + x(7) ) + + z(3) * (-x(2) + x(7) ) + + z(4) * (+x(5) - x(7) ) + + z(5) * (+x(1) + x(2) - x(4) - x(7) ) + + z(7) * (-x(2) - x(3) + x(4) + x(5) ) ) * twelth; + + B_matrix(7, 1) = (+z(0) * (-x(3) + x(4) ) + + z(2) * (+x(3) - x(6) ) + + z(3) * (+x(0) - x(2) + x(4) - x(6) ) + + z(4) * (-x(0) - x(3) + x(5) + x(6) ) + + z(5) * (-x(4) + x(6) ) + + z(6) * (+x(2) + x(3) - x(4) - x(5) ) ) * twelth; + + B_matrix(0, 2) = (+x(1) * (-y(2) - y(3) + y(4) + y(5) ) + + x(2) * (+y(1) - y(3) ) + + x(3) * (+y(1) + y(2) - y(4) - y(7) ) + + x(4) * (-y(1) + y(3) - y(5) + y(7) ) + + x(5) * (-y(1) + y(4) ) + + x(7) * (+y(3) - y(4) ) ) * twelth; + + B_matrix(1, 2) = (+x(0) * (+y(2) + y(3) - y(4) - y(5) ) + + x(2) * (-y(0) - y(3) + y(5) + y(6) ) + + x(3) * (-y(0) + y(2) ) + + x(4) * (+y(0) - y(5) ) + + x(5) * (+y(0) - y(2) + y(4) - y(6) ) + + x(6) * (-y(2) + y(5) ) ) * twelth; + + B_matrix(2, 2) = (+x(0) * (-y(1) + y(3) ) + + x(1) * (+y(0) + y(3) - y(5) - y(6) ) + + x(3) * (-y(0) - y(1) + y(6) + y(7) ) + + x(5) * (+y(1) - y(6) ) + + x(6) * (+y(1) - y(3) + y(5) - y(7) ) + + x(7) * (-y(3) + y(6) ) ) * twelth; + + B_matrix(3, 2) = (+x(0) * (-y(1) - y(2) + y(4) + y(7) ) + + x(1) * (+y(0) - y(2) ) + + x(2) * (+y(0) + y(1) - y(6) - y(7) ) + + x(4) * (-y(0) + y(7) ) + + x(6) * (+y(2) - y(7) ) + + x(7) * (-y(0) + y(2) - y(4) + y(6) ) ) * twelth; + + B_matrix(4, 2) = (+x(0) * (+y(1) - y(3) + y(5) - y(7) ) + + x(1) * (-y(0) + y(5) ) + + x(3) * (+y(0) - y(7) ) + + x(5) * (-y(0) - y(1) + y(6) + y(7) ) + + x(6) * (-y(5) + y(7) ) + + x(7) * (+y(0) + y(3) - y(5) - y(6) ) ) * twelth; + + B_matrix(5, 2) = (+x(0) * (+y(1) - y(4) ) + + x(1) * (-y(0) + y(2) - y(4) + y(6) ) + + x(2) * (-y(1) + y(6) ) + + x(4) * (+y(0) + y(1) - y(6) - y(7) ) + + x(6) * (-y(1) - y(2) + y(4) + y(7) ) + + x(7) * (+y(4) - y(6) ) ) * twelth; + + B_matrix(6, 2) = (+x(1) * (+y(2) - y(5) ) + + x(2) * (-y(1) + y(3) - y(5) + y(7) ) + + x(3) * (-y(2) + y(7) ) + + x(4) * (+y(5) - y(7) ) + + x(5) * (+y(1) + y(2) - y(4) - y(7) ) + + x(7) * (-y(2) - y(3) + y(4) + y(5) ) ) * twelth; + + B_matrix(7, 2) = (+x(0) * (-y(3) + y(4) ) + + x(2) * (+y(3) - y(6) ) + + x(3) * (+y(0) - y(2) + y(4) - y(6) ) + + x(4) * (-y(0) - y(3) + y(5) + y(6) ) + + x(5) * (-y(4) + y(6) ) + + x(6) * (+y(2) + y(3) - y(4) - y(5) ) ) * twelth; +} // end subroutine + +/* ---------------------------------------------------------------------------- + Compute Volume of each finite element +------------------------------------------------------------------------------- */ + +void FEA_Module_SGH::get_vol() +{ + const size_t rk_level = rk_num_bins - 1; + const size_t num_dims = num_dim; + + if (num_dims == 2) + { + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { + // cut out the node_gids for this element + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 4); + get_vol_quad(elem_vol, elem_gid, node_coords, elem_node_gids, rk_level); + }); + Kokkos::fence(); + } + else + { + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { + // cut out the node_gids for this element + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 8); + get_vol_hex(elem_vol, elem_gid, node_coords, elem_node_gids, rk_level); + }); + Kokkos::fence(); + } // end if + + return; +} // end subroutine + +// Exact volume for a hex element +KOKKOS_INLINE_FUNCTION +void FEA_Module_SGH::get_vol_hex(const DViewCArrayKokkos& elem_vol, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const +{ + const size_t num_nodes = 8; + + double x_array[8]; + double y_array[8]; + double z_array[8]; + + // x, y, z coordinates of elem vertices + auto x = ViewCArrayKokkos(x_array, num_nodes); + auto y = ViewCArrayKokkos(y_array, num_nodes); + auto z = ViewCArrayKokkos(z_array, num_nodes); + + // get the coordinates of the nodes(rk,elem,node) in this element + for (int node_lid = 0; node_lid < num_nodes; node_lid++) + { + x(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 0); + y(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 1); + z(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 2); + } // end for + + double twelth = 1. / 12.; + + // element volume + elem_vol(elem_gid) = + (x(1) * (y(3) * (-z(0) + z(2)) + y(4) * (z(0) - z(5)) + y(0) * (z(2) + z(3) - z(4) - z(5)) + y(6) * (-z(2) + z(5)) + y(5) * (z(0) - z(2) + z(4) - z(6)) + y(2) * (-z(0) - z(3) + z(5) + z(6))) + + x(7) * (y(0) * (-z(3) + z(4)) + y(6) * (z(2) + z(3) - z(4) - z(5)) + y(2) * (z(3) - z(6)) + y(3) * (z(0) - z(2) + z(4) - z(6)) + y(5) * (-z(4) + z(6)) + y(4) * (-z(0) - z(3) + z(5) + z(6))) + + x(3) * (y(1) * (z(0) - z(2)) + y(7) * (-z(0) + z(2) - z(4) + z(6)) + y(6) * (z(2) - z(7)) + y(2) * (z(0) + z(1) - z(6) - z(7)) + y(4) * (-z(0) + z(7)) + y(0) * (-z(1) - z(2) + z(4) + z(7))) + + x(5) * (y(0) * (z(1) - z(4)) + y(7) * (z(4) - z(6)) + y(2) * (-z(1) + z(6)) + y(1) * (-z(0) + z(2) - z(4) + z(6)) + y(4) * (z(0) + z(1) - z(6) - z(7)) + y(6) * (-z(1) - z(2) + z(4) + z(7))) + + x(6) * (y(1) * (z(2) - z(5)) + y(7) * (-z(2) - z(3) + z(4) + z(5)) + y(5) * (z(1) + z(2) - z(4) - z(7)) + y(4) * (z(5) - z(7)) + y(3) * (-z(2) + z(7)) + y(2) * (-z(1) + z(3) - z(5) + z(7))) + + x(0) * (y(2) * (z(1) - z(3)) + y(7) * (z(3) - z(4)) + y(5) * (-z(1) + z(4)) + y(1) * (-z(2) - z(3) + z(4) + z(5)) + y(3) * (z(1) + z(2) - z(4) - z(7)) + y(4) * (-z(1) + z(3) - z(5) + z(7))) + + x(2) * (y(0) * (-z(1) + z(3)) + y(5) * (z(1) - z(6)) + y(1) * (z(0) + z(3) - z(5) - z(6)) + y(7) * (-z(3) + z(6)) + y(6) * (z(1) - z(3) + z(5) - z(7)) + y(3) * (-z(0) - z(1) + z(6) + z(7))) + + x(4) * + (y(1) * (-z(0) + z(5)) + y(7) * (z(0) + z(3) - z(5) - z(6)) + y(3) * (z(0) - z(7)) + y(0) * (z(1) - z(3) + z(5) - z(7)) + y(6) * (-z(5) + z(7)) + y(5) * (-z(0) - z(1) + z(6) + z(7)))) * + twelth; + + return; +} // end subroutine + +KOKKOS_FUNCTION +void FEA_Module_SGH::get_bmatrix2D(const ViewCArrayKokkos& B_matrix, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const +{ + const size_t num_nodes = 4; + + double x_array[4]; + double y_array[4]; + + // x, y coordinates of elem vertices + auto x = ViewCArrayKokkos(x_array, num_nodes); + auto y = ViewCArrayKokkos(y_array, num_nodes); + + // get the coordinates of the nodes(rk,elem,node) in this element + for (int node_lid = 0; node_lid < num_nodes; node_lid++) + { + x(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 0); + y(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 1); + } // end for + + /* ensight node order 0 1 2 3 + Flanaghan node order 3 4 1 2 + */ + + B_matrix(0, 0) = -0.5 * (y(3) - y(1)); + + B_matrix(1, 0) = -0.5 * (y(0) - y(2)); + + B_matrix(2, 0) = -0.5 * (y(1) - y(3)); + + B_matrix(3, 0) = -0.5 * (y(2) - y(0)); + + B_matrix(0, 1) = -0.5 * (x(1) - x(3)); + + B_matrix(1, 1) = -0.5 * (x(2) - x(0)); + + B_matrix(2, 1) = -0.5 * (x(3) - x(1)); + + B_matrix(3, 1) = -0.5 * (x(0) - x(2)); + + // + /* + The Flanagan and Belytschko paper has: + x y + node 1: 0.5*(y2 - y4) , 0.5*(x4 - x2) + node 2: 0.5*(y3 - y1) , 0.5*(x1 - x3) + node 3: 0.5*(y4 - y2) , 0.5*(x2 - x4) + node 4: 0.5*(y1 - y3) , 0.5*(x3 - x1) + + Ensight order would be + + node 2: 0.5*(y3 - y1) , 0.5*(x1 - x3) + node 3: 0.5*(y0 - y2) , 0.5*(x2 - x0) + node 0: 0.5*(y1 - y3) , 0.5*(x3 - x1) + node 1: 0.5*(y2 - y0) , 0.5*(x0 - x2) + + */ + + return; +} // end subroutine + +// true volume of a quad in RZ coords +KOKKOS_INLINE_FUNCTION +void FEA_Module_SGH::get_vol_quad(const DViewCArrayKokkos& elem_vol, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const +{ + // --- testing here --- + /* + double test_vol = 0.0; + // getting the corner facial area + double corner_areas_array[4]; + ViewCArrayKokkos corner_areas(&corner_areas_array[0],4); + + get_area_weights2D(corner_areas, + elem_gid, + node_coords, + elem_node_gids, + rk_level); + + + for(size_t node_lid=0; node_lid<4; node_lid++){ + double y = node_coords(rk_level, elem_node_gids(node_lid), 1); // node radius + test_vol += corner_areas(node_lid)*y; + } // end for + + test_vol matches the Barlow volume formula + */ + // ------------------- + + elem_vol(elem_gid) = 0.0; + + const size_t num_nodes = 4; + + double x_array[4]; + double y_array[4]; + + // x, y coordinates of elem vertices + auto x = ViewCArrayKokkos(x_array, num_nodes); + auto y = ViewCArrayKokkos(y_array, num_nodes); + + // get the coordinates of the nodes(rk,elem,node) in this element + for (int node_lid = 0; node_lid < num_nodes; node_lid++) + { + x(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 0); + y(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 1); + } // end for + + /* ensight node order 0 1 2 3 + Flanaghan node order 3 4 1 2 + */ + elem_vol(elem_gid) = + ( (y(2) + y(3) + y(0)) * ((y(2) - y(3)) * (x(0) - x(3)) - (y(0) - y(3)) * (x(2) - x(3)) ) + + (y(0) + y(1) + y(2)) * ((y(0) - y(1)) * (x(2) - x(1)) - (y(2) - y(1)) * (x(0) - x(1))) ) / 6.0; + + return; +} // end subroutine + +// element facial area +KOKKOS_FUNCTION +double FEA_Module_SGH::get_area_quad(const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const +{ + double elem_area = 0.0; + + const size_t num_nodes = 4; + + double x_array[4]; + double y_array[4]; + + // x, y coordinates of elem vertices + auto x = ViewCArrayKokkos(x_array, num_nodes); + auto y = ViewCArrayKokkos(y_array, num_nodes); + + // get the coordinates of the nodes(rk,elem,node) in this element + for (int node_lid = 0; node_lid < num_nodes; node_lid++) + { + x(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 0); + y(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 1); + } // end for + + /* ensight node order 0 1 2 3 + Flanaghan node order 3 4 1 2 + */ + + // element facial area + elem_area = 0.5 * ((x(0) - x(2)) * (y(1) - y(3)) + (x(3) - x(1)) * (y(0) - y(2))); + + return elem_area; +} // end subroutine + +KOKKOS_INLINE_FUNCTION +double FEA_Module_SGH::heron(const double x1, + const double y1, + const double x2, + const double y2, + const double x3, + const double y3) const +{ + double S, a, b, c, area; + + S = 0.0; + a = sqrt((x2 - x1) * (x2 - x1) + (y2 - y1) * (y2 - y1)); + S += a; + b = sqrt((x3 - x2) * (x3 - x2) + (y3 - y2) * (y3 - y2)); + S += b; + c = sqrt((x3 - x1) * (x3 - x1) + (y3 - y1) * (y3 - y1)); + S += c; + + S *= 0.5; + area = sqrt(S * (S - a) * (S - b) * (S - c)); + + return area; +} + +KOKKOS_FUNCTION +void FEA_Module_SGH::get_area_weights2D(const ViewCArrayKokkos& corner_areas, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const +{ + const size_t num_nodes = 4; + + double x_array[4]; + double y_array[4]; + + double rc, zc; + double A12, A23, A34, A41; + + // x, y coordinates of elem vertices + ViewCArrayKokkos x(x_array, num_nodes); + ViewCArrayKokkos y(y_array, num_nodes); + + // get the coordinates of the nodes(rk,elem,node) in this element + rc = zc = 0.0; + for (int node_lid = 0; node_lid < num_nodes; node_lid++) + { + x(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 0); + y(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 1); + rc += 0.25 * y(node_lid); + zc += 0.25 * x(node_lid); + } // end for + + /* ensight node order 0 1 2 3 + Barlow node order 1 2 3 4 + */ + + A12 = heron(x(0), y(0), zc, rc, x(1), y(1)); + A23 = heron(x(1), y(1), zc, rc, x(2), y(2)); + A34 = heron(x(2), y(2), zc, rc, x(3), y(3)); + A41 = heron(x(3), y(3), zc, rc, x(0), y(0)); + + corner_areas(0) = (5. * A41 + 5. * A12 + A23 + A34) / 12.; + corner_areas(1) = (A41 + 5. * A12 + 5. * A23 + A34) / 12.; + corner_areas(2) = (A41 + A12 + 5. * A23 + 5. * A34) / 12.; + corner_areas(3) = (5. * A41 + A12 + A23 + 5. * A34) / 12.; + + return; +} // end subroutine + +/* ----------------------------------------------------------------------------------- + Compute Gradient of the Volume of each finite element with respect to displacement +-------------------------------------------------------------------------------------- */ + +void FEA_Module_SGH::get_vol_ugradient(const size_t gradient_node_id, const size_t gradient_dim) +{ + const size_t rk_level = rk_num_bins - 1; + const size_t num_dims = num_dim; + + if (num_dims == 2) + { + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { + // cut out the node_gids for this element + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 4); + get_vol_quad(elem_vol, elem_gid, node_coords, elem_node_gids, rk_level); + }); + Kokkos::fence(); + } + else + { + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { + // cut out the node_gids for this element + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 8); + // get_vol_hex_ugradient(elem_vol, elem_gid, node_coords, elem_node_gids, rk_level); + }); + Kokkos::fence(); + } // end if + + return; +} // end subroutine + +// Exact volume for a hex element +KOKKOS_FUNCTION +void FEA_Module_SGH::get_vol_hex_ugradient(const ViewCArrayKokkos& elem_vol_gradients, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const +{ + const size_t num_nodes = 8; + const size_t num_dims = num_dim; + double x_array[8]; + double y_array[8]; + double z_array[8]; + double gradient_result; + + // x, y, z coordinates of elem vertices + auto x = ViewCArrayKokkos(x_array, num_nodes); + auto y = ViewCArrayKokkos(y_array, num_nodes); + auto z = ViewCArrayKokkos(z_array, num_nodes); + + // get the coordinates of the nodes(rk,elem,node) in this element + for (int node_lid = 0; node_lid < num_nodes; node_lid++) + { + x(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 0); + y(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 1); + z(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 2); + } // end for + + double twelth = 1. / 12.; + + // element volume gradient + for (int inode = 0; inode < 8; inode++) + { + for (int idim = 0; idim < num_dims; idim++) + { + switch (num_dims * inode + idim) + { + case 0: + gradient_result = + ((y(2) * (z(1) - z(3)) + y(7) * (z(3) - z(4)) + y(5) * (-z(1) + z(4)) + y(1) * (-z(2) - z(3) + z(4) + z(5)) + y(3) * (z(1) + z(2) - z(4) - z(7)) + y(4) * + (-z(1) + z(3) - z(5) + z(7)))) * twelth; + break; + case 3: + gradient_result = + ((y(3) * (-z(0) + z(2)) + y(4) * (z(0) - z(5)) + y(0) * (z(2) + z(3) - z(4) - z(5)) + y(6) * (-z(2) + z(5)) + y(5) * (z(0) - z(2) + z(4) - z(6)) + y(2) * + (-z(0) - z(3) + z(5) + z(6)))) * twelth; + break; + + case 6: + gradient_result = + ((y(0) * (-z(1) + z(3)) + y(5) * (z(1) - z(6)) + y(1) * (z(0) + z(3) - z(5) - z(6)) + y(7) * (-z(3) + z(6)) + y(6) * (z(1) - z(3) + z(5) - z(7)) + y(3) * + (-z(0) - z(1) + z(6) + z(7)))) * twelth; + break; + case 9: + gradient_result = + ((y(1) * (z(0) - z(2)) + y(7) * (-z(0) + z(2) - z(4) + z(6)) + y(6) * (z(2) - z(7)) + y(2) * (z(0) + z(1) - z(6) - z(7)) + y(4) * (-z(0) + z(7)) + y(0) * + (-z(1) - z(2) + z(4) + z(7)))) * twelth; + break; + case 12: + gradient_result = + ((y(1) * (-z(0) + z(5)) + y(7) * (z(0) + z(3) - z(5) - z(6)) + y(3) * (z(0) - z(7)) + y(0) * (z(1) - z(3) + z(5) - z(7)) + y(6) * (-z(5) + z(7)) + y(5) * + (-z(0) - z(1) + z(6) + z(7)))) * twelth; + break; + case 15: + gradient_result = + ((y(0) * (z(1) - z(4)) + y(7) * (z(4) - z(6)) + y(2) * (-z(1) + z(6)) + y(1) * (-z(0) + z(2) - z(4) + z(6)) + y(4) * (z(0) + z(1) - z(6) - z(7)) + y(6) * + (-z(1) - z(2) + z(4) + z(7)))) * twelth; + break; + case 18: + gradient_result = + ((y(1) * (z(2) - z(5)) + y(7) * (-z(2) - z(3) + z(4) + z(5)) + y(5) * (z(1) + z(2) - z(4) - z(7)) + y(4) * (z(5) - z(7)) + y(3) * (-z(2) + z(7)) + y(2) * + (-z(1) + z(3) - z(5) + z(7)))) * twelth; + break; + case 21: + gradient_result = + ((y(0) * (-z(3) + z(4)) + y(6) * (z(2) + z(3) - z(4) - z(5)) + y(2) * (z(3) - z(6)) + y(3) * (z(0) - z(2) + z(4) - z(6)) + y(5) * (-z(4) + z(6)) + y(4) * + (-z(0) - z(3) + z(5) + z(6)))) * twelth; + break; + case 1: + gradient_result = + (x(1) * ((z(2) + z(3) - z(4) - z(5))) + + x(7) * ((-z(3) + z(4))) + + x(3) * ((-z(1) - z(2) + z(4) + z(7))) + + x(5) * ((z(1) - z(4))) + + x(2) * ((-z(1) + z(3))) + + x(4) * ((z(1) - z(3) + z(5) - z(7)))) * twelth; + break; + case 4: + gradient_result = + (x(3) * ((z(0) - z(2))) + + x(5) * ((-z(0) + z(2) - z(4) + z(6))) + + x(6) * ((z(2) - z(5))) + + x(0) * ((-z(2) - z(3) + z(4) + z(5))) + + x(2) * ((z(0) + z(3) - z(5) - z(6))) + + x(4) * ((-z(0) + z(5)))) * twelth; + break; + case 7: + gradient_result = + (x(1) * ((-z(0) - z(3) + z(5) + z(6))) + + x(7) * ((z(3) - z(6))) + + x(3) * ((z(0) + z(1) - z(6) - z(7))) + + x(5) * ((-z(1) + z(6))) + + x(6) * ((-z(1) + z(3) - z(5) + z(7))) + + x(0) * ((z(1) - z(3)))) * twelth; + break; + case 10: + gradient_result = + (x(1) * ((-z(0) + z(2))) + + x(7) * ((z(0) - z(2) + z(4) - z(6))) + + x(6) * ((-z(2) + z(7))) + + x(0) * ((z(1) + z(2) - z(4) - z(7))) + + x(2) * ((-z(0) - z(1) + z(6) + z(7))) + + x(4) * ((z(0) - z(7)))) * twelth; + break; + case 13: + gradient_result = + (x(1) * ((z(0) - z(5))) + + x(7) * ((-z(0) - z(3) + z(5) + z(6))) + + x(3) * ((-z(0) + z(7))) + + x(5) * ((z(0) + z(1) - z(6) - z(7))) + + x(6) * ((z(5) - z(7))) + + x(0) * ((-z(1) + z(3) - z(5) + z(7)))) * twelth; + break; + case 16: + gradient_result = + (x(1) * ((z(0) - z(2) + z(4) - z(6))) + + x(7) * ((-z(4) + z(6))) + + x(6) * ((z(1) + z(2) - z(4) - z(7))) + + x(0) * ((-z(1) + z(4))) + + x(2) * ((z(1) - z(6))) + + x(4) * ((-z(0) - z(1) + z(6) + z(7)))) * twelth; + break; + case 19: + gradient_result = + (x(1) * ((-z(2) + z(5))) + + x(7) * ((z(2) + z(3) - z(4) - z(5))) + + x(3) * ((z(2) - z(7))) + + x(5) * ((-z(1) - z(2) + z(4) + z(7))) + + x(2) * ((z(1) - z(3) + z(5) - z(7))) + + x(4) * ((-z(5) + z(7)))) * twelth; + break; + case 22: + gradient_result = + (x(3) * ((-z(0) + z(2) - z(4) + z(6))) + + x(5) * ((z(4) - z(6))) + + x(6) * ((-z(2) - z(3) + z(4) + z(5))) + + x(0) * ((z(3) - z(4))) + + x(2) * ((-z(3) + z(6))) + + x(4) * ((z(0) + z(3) - z(5) - z(6)))) * twelth; + break; + case 2: + gradient_result = + (x(1) * (-y(3) + y(4) + y(5) - y(2)) + + x(7) * (y(3) - y(4)) + + x(3) * (y(1) - y(7) + y(2) - y(4)) + + x(5) * (-y(1) + y(4)) + + x(2) * (y(1) - y(3)) + + x(4) * (-y(1) + y(7) + y(3) - y(5))) * twelth; + break; + case 5: + gradient_result = + (x(3) * (y(2) - y(0)) + + x(5) * (y(0) - y(2) + y(4) - y(6)) + + x(6) * (y(5) - y(2)) + + x(0) * (y(2) - y(5) + y(3) - y(4)) + + x(2) * (-y(0) + y(5) + y(6) - y(3)) + + x(4) * (y(0) - y(5))) * twelth; + break; + case 8: + gradient_result = + (x(1) * (y(3) + y(0) - y(6) - y(5)) + + x(7) * (y(6) - y(3)) + + x(3) * (-y(1) + y(7) + y(6) - y(0)) + + x(5) * (y(1) - y(6)) + + x(6) * (y(1) - y(7) + y(5) - y(3)) + + x(0) * (-y(1) + y(3))) * twelth; + break; + case 11: + gradient_result = + (x(1) * (y(0) - y(2)) + + x(7) * (-y(0) + y(6) + y(2) - y(4)) + + x(6) * (-y(7) + y(2)) + + x(0) * (-y(2) + y(7) - y(1) + y(4)) + + x(2) * (y(0) + y(1) - y(7) - y(6)) + + x(4) * (y(7) - y(0))) * twelth; + break; + case 14: + gradient_result = + (x(1) * (-y(0) + y(5)) + + x(7) * (y(0) - y(6) + y(3) - y(5)) + + x(3) * (-y(7) + y(0)) + + x(5) * (-y(0) + y(7) - y(1) + y(6)) + + x(6) * (y(7) - y(5)) + + x(0) * (-y(7) + y(5) + y(1) - y(3))) * twelth; + break; + case 17: + gradient_result = + (x(1) * (-y(4) - y(0) + y(6) + y(2)) + + x(7) * (-y(6) + y(4)) + + x(6) * (-y(1) + y(7) + y(4) - y(2)) + + x(0) * (y(1) - y(4)) + + x(2) * (-y(1) + y(6)) + + x(4) * (y(1) - y(7) + y(0) - y(6))) * twelth; + break; + case 20: + gradient_result = + (x(1) * (-y(5) + y(2)) + + x(7) * (-y(2) - y(3) + y(5) + y(4)) + + x(3) * (y(7) - y(2)) + + x(5) * (-y(7) + y(2) + y(1) - y(4)) + + x(2) * (-y(5) - y(1) + y(7) + y(3)) + + x(4) * (-y(7) + y(5))) * twelth; + break; + case 23: + gradient_result = + (x(3) * (-y(6) - y(2) + y(4) + y(0)) + + x(5) * (-y(4) + y(6)) + + x(6) * (-y(5) - y(4) + y(3) + y(2)) + + x(0) * (-y(3) + y(4)) + + x(2) * (-y(6) + y(3)) + + x(4) * (-y(3) - y(0) + y(6) + y(5))) * twelth; + break; + } + elem_vol_gradients(inode, idim) = gradient_result; + } + } + return; +} // end subroutine + +KOKKOS_FUNCTION +void FEA_Module_SGH::get_bmatrix_gradients(const ViewCArrayKokkos& B_matrix_gradients, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const +{ + const size_t num_nodes = 8; + + double x_array[8]; + double y_array[8]; + double z_array[8]; + double gradient_terms_array[2 * 8]; + + // x, y, z coordinates of elem vertices + auto x = ViewCArrayKokkos(x_array, num_nodes); + auto y = ViewCArrayKokkos(y_array, num_nodes); + auto z = ViewCArrayKokkos(z_array, num_nodes); + auto gradient_terms = ViewCArrayKokkos(gradient_terms_array, 2, num_nodes); + + // get the coordinates of the nodes(rk,elem,node) in this element + for (int node_lid = 0; node_lid < num_nodes; node_lid++) + { + x(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 0); + y(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 1); + z(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 2); + } // end for + + double twelth = 1. / 12.; + + // B_matrix(0,0) = ( +y(1)*( -z(2) -z(3) +z(4) +z(5) ) + // +y(2)*( +z(1) -z(3) ) + // +y(3)*( +z(1) +z(2) -z(4) -z(7) ) + // +y(4)*( -z(1) +z(3) -z(5) +z(7) ) + // +y(5)*( -z(1) +z(4) ) + // +y(7)*( +z(3) -z(4) ) )*twelth; + + // y derivative + gradient_terms(0, 0) = 0; + gradient_terms(0, 1) = -z(2) - z(3) + z(4) + z(5); + gradient_terms(0, 2) = z(1) - z(3); + gradient_terms(0, 3) = z(1) + z(2) - z(4) - z(7); + gradient_terms(0, 4) = -z(1) + z(3) - z(5) + z(7); + gradient_terms(0, 5) = -z(1) + z(4); + gradient_terms(0, 6) = 0; + gradient_terms(0, 7) = z(3) - z(4); + + // z derivative + gradient_terms(1, 0) = 0; + gradient_terms(1, 1) = y(2) + y(3) - y(4) - y(5); + gradient_terms(1, 2) = -y(1) + y(3); + gradient_terms(1, 3) = -y(1) - y(2) + y(4) + y(7); + gradient_terms(1, 4) = y(1) - y(3) + y(5) - y(7); + gradient_terms(1, 5) = y(1) - y(4); + gradient_terms(1, 6) = 0; + gradient_terms(1, 7) = -y(3) + y(4); + + for (int inode = 0; inode < 8; inode++) + { + B_matrix_gradients(0, 0, inode, 0) = 0; + B_matrix_gradients(0, 0, inode, 1) = gradient_terms(0, inode) * twelth; + B_matrix_gradients(0, 0, inode, 2) = gradient_terms(1, inode) * twelth; + } + + // B_matrix(1,0) = ( +y(0)*( +z(2) +z(3) -z(4) -z(5) ) + // +y(2)*( -z(0) -z(3) +z(5) +z(6) ) + // +y(3)*( -z(0) +z(2) ) + // +y(4)*( +z(0) -z(5) ) + // +y(5)*( +z(0) -z(2) +z(4) -z(6) ) + // +y(6)*( -z(2) +z(5) ) )*twelth; + + // y derivative + gradient_terms(0, 0) = z(2) + z(3) - z(4) - z(5); + gradient_terms(0, 1) = 0; + gradient_terms(0, 2) = -z(0) - z(3) + z(5) + z(6); + gradient_terms(0, 3) = -z(0) + z(2); + gradient_terms(0, 4) = z(0) - z(5); + gradient_terms(0, 5) = z(0) - z(2) + z(4) - z(6); + gradient_terms(0, 6) = -z(2) + z(5); + gradient_terms(0, 7) = 0; + + // z derivative + gradient_terms(1, 0) = -y(2) - y(3) + y(4) + y(5); + gradient_terms(1, 1) = 0; + gradient_terms(1, 2) = y(0) + y(3) - y(5) - y(6); + gradient_terms(1, 3) = y(0) - y(2); + gradient_terms(1, 4) = -y(0) + y(5); + gradient_terms(1, 5) = -y(0) + y(2) - y(4) + y(6); + gradient_terms(1, 6) = y(2) - y(5); + gradient_terms(1, 7) = 0; + + for (int inode = 0; inode < 8; inode++) + { + B_matrix_gradients(1, 0, inode, 0) = 0; + B_matrix_gradients(1, 0, inode, 1) = gradient_terms(0, inode) * twelth; + B_matrix_gradients(1, 0, inode, 2) = gradient_terms(1, inode) * twelth; + } + + // B_matrix(2,0) = ( +y(0)*( -z(1) +z(3) ) + // +y(1)*( +z(0) +z(3) -z(5) -z(6) ) + // +y(3)*( -z(0) -z(1) +z(6) +z(7) ) + // +y(5)*( +z(1) -z(6) ) + // +y(6)*( +z(1) -z(3) +z(5) -z(7) ) + // +y(7)*( -z(3) +z(6) ) )*twelth; + + // y derivative + gradient_terms(0, 0) = -z(1) + z(3); + gradient_terms(0, 1) = z(0) + z(3) - z(5) - z(6); + gradient_terms(0, 2) = 0; + gradient_terms(0, 3) = -z(0) - z(1) + z(6) + z(7); + gradient_terms(0, 4) = 0; + gradient_terms(0, 5) = z(1) - z(6); + gradient_terms(0, 6) = z(1) - z(3) + z(5) - z(7); + gradient_terms(0, 7) = -z(3) + z(6); + + // z derivative + gradient_terms(1, 0) = y(1) - y(3); + gradient_terms(1, 1) = -y(0) - y(3) + y(5) + y(6); + gradient_terms(1, 2) = 0; + gradient_terms(1, 3) = y(0) + y(1) - y(6) - y(7); + gradient_terms(1, 4) = 0; + gradient_terms(1, 5) = -y(1) + y(6); + gradient_terms(1, 6) = -y(1) + y(3) - y(5) + y(7); + gradient_terms(1, 7) = y(3) - y(6); + + for (int inode = 0; inode < 8; inode++) + { + B_matrix_gradients(2, 0, inode, 0) = 0; + B_matrix_gradients(2, 0, inode, 1) = gradient_terms(0, inode) * twelth; + B_matrix_gradients(2, 0, inode, 2) = gradient_terms(1, inode) * twelth; + } + + // B_matrix(3,0) = ( +y(0)*( -z(1) -z(2) +z(4) +z(7) ) + // +y(1)*( +z(0) -z(2) ) + // +y(2)*( +z(0) +z(1) -z(6) -z(7) ) + // +y(4)*( -z(0) +z(7) ) + // +y(6)*( +z(2) -z(7) ) + // +y(7)*( -z(0) +z(2) -z(4) +z(6) ) )*twelth; + + // y derivative + gradient_terms(0, 0) = -z(1) - z(2) + z(4) + z(7); + gradient_terms(0, 1) = z(0) - z(2); + gradient_terms(0, 2) = z(0) + z(1) - z(6) - z(7); + gradient_terms(0, 3) = 0; + gradient_terms(0, 4) = -z(0) + z(7); + gradient_terms(0, 5) = 0; + gradient_terms(0, 6) = z(2) - z(7); + gradient_terms(0, 7) = -z(0) + z(2) - z(4) + z(6); + + // z derivative + gradient_terms(1, 0) = y(1) + y(2) - y(4) - y(7); + gradient_terms(1, 1) = -y(0) + y(2); + gradient_terms(1, 2) = -y(0) - y(1) + y(6) + y(7); + gradient_terms(1, 3) = 0; + gradient_terms(1, 4) = y(0) - y(7); + gradient_terms(1, 5) = 0; + gradient_terms(1, 6) = -y(2) + y(7); + gradient_terms(1, 7) = y(0) - y(2) + y(4) - y(6); + + for (int inode = 0; inode < 8; inode++) + { + B_matrix_gradients(3, 0, inode, 0) = 0; + B_matrix_gradients(3, 0, inode, 1) = gradient_terms(0, inode) * twelth; + B_matrix_gradients(3, 0, inode, 2) = gradient_terms(1, inode) * twelth; + } + + // B_matrix(4,0) = ( +y(0)*( +z(1) -z(3) +z(5) -z(7) ) + // +y(1)*( -z(0) +z(5) ) + // +y(3)*( +z(0) -z(7) ) + // +y(5)*( -z(0) -z(1) +z(6) +z(7) ) + // +y(6)*( -z(5) +z(7) ) + // +y(7)*( +z(0) +z(3) -z(5) -z(6) ) )*twelth; + + // y derivative + gradient_terms(0, 0) = z(1) - z(3) + z(5) - z(7); + gradient_terms(0, 1) = -z(0) + z(5); + gradient_terms(0, 2) = 0; + gradient_terms(0, 3) = z(0) - z(7); + gradient_terms(0, 4) = 0; + gradient_terms(0, 5) = -z(0) - z(1) + z(6) + z(7); + gradient_terms(0, 6) = -z(5) + z(7); + gradient_terms(0, 7) = z(0) + z(3) - z(5) - z(6); + + // z derivative + gradient_terms(1, 0) = -y(1) + y(3) - y(5) + y(7); + gradient_terms(1, 1) = y(0) - y(5); + gradient_terms(1, 2) = 0; + gradient_terms(1, 3) = -y(0) + y(7); + gradient_terms(1, 4) = 0; + gradient_terms(1, 5) = y(0) + y(1) - y(6) - y(7); + gradient_terms(1, 6) = y(5) - y(7); + gradient_terms(1, 7) = -y(0) - y(3) + y(5) + y(6); + + for (int inode = 0; inode < 8; inode++) + { + B_matrix_gradients(4, 0, inode, 0) = 0; + B_matrix_gradients(4, 0, inode, 1) = gradient_terms(0, inode) * twelth; + B_matrix_gradients(4, 0, inode, 2) = gradient_terms(1, inode) * twelth; + } + + // B_matrix(5,0) = ( +y(0)*( +z(1) -z(4) ) + // +y(1)*( -z(0) +z(2) -z(4) +z(6) ) + // +y(2)*( -z(1) +z(6) ) + // +y(4)*( +z(0) +z(1) -z(6) -z(7) ) + // +y(6)*( -z(1) -z(2) +z(4) +z(7) ) + // +y(7)*( +z(4) -z(6) ) )*twelth; + + // y derivative + gradient_terms(0, 0) = z(1) - z(4); + gradient_terms(0, 1) = -z(0) + z(2) - z(4) + z(6); + gradient_terms(0, 2) = -z(1) + z(6); + gradient_terms(0, 3) = 0; + gradient_terms(0, 4) = z(0) + z(1) - z(6) - z(7); + gradient_terms(0, 5) = 0; + gradient_terms(0, 6) = -z(1) - z(2) + z(4) + z(7); + gradient_terms(0, 7) = z(4) - z(6); + + // z derivative + gradient_terms(1, 0) = -y(1) + y(4); + gradient_terms(1, 1) = y(0) - y(2) + y(4) - y(6); + gradient_terms(1, 2) = y(1) - y(6); + gradient_terms(1, 3) = 0; + gradient_terms(1, 4) = -y(0) - y(1) + y(6) + y(7); + gradient_terms(1, 5) = 0; + gradient_terms(1, 6) = y(1) + y(2) - y(4) - y(7); + gradient_terms(1, 7) = -y(4) + y(6); + + for (int inode = 0; inode < 8; inode++) + { + B_matrix_gradients(5, 0, inode, 0) = 0; + B_matrix_gradients(5, 0, inode, 1) = gradient_terms(0, inode) * twelth; + B_matrix_gradients(5, 0, inode, 2) = gradient_terms(1, inode) * twelth; + } + + // B_matrix(6,0) = ( +y(1)*( +z(2) -z(5) ) + // +y(2)*( -z(1) +z(3) -z(5) +z(7) ) + // +y(3)*( -z(2) +z(7) ) + // +y(4)*( +z(5) -z(7) ) + // +y(5)*( +z(1) +z(2) -z(4) -z(7) ) + // +y(7)*( -z(2) -z(3) +z(4) +z(5) ) )*twelth; + + // y derivative + gradient_terms(0, 0) = 0; + gradient_terms(0, 1) = z(2) - z(5); + gradient_terms(0, 2) = -z(1) + z(3) - z(5) + z(7); + gradient_terms(0, 3) = -z(2) + z(7); + gradient_terms(0, 4) = z(5) - z(7); + gradient_terms(0, 5) = z(1) + z(2) - z(4) - z(7); + gradient_terms(0, 6) = 0; + gradient_terms(0, 7) = -z(2) - z(3) + z(4) + z(5); + + // z derivative + gradient_terms(1, 0) = 0; + gradient_terms(1, 1) = -y(2) + y(5); + gradient_terms(1, 2) = y(1) - y(3) + y(5) - y(7); + gradient_terms(1, 3) = y(2) - y(7); + gradient_terms(1, 4) = -y(5) + y(7); + gradient_terms(1, 5) = -y(1) - y(2) + y(4) + y(7); + gradient_terms(1, 6) = 0; + gradient_terms(1, 7) = y(2) + y(3) - y(4) - y(5); + + for (int inode = 0; inode < 8; inode++) + { + B_matrix_gradients(6, 0, inode, 0) = 0; + B_matrix_gradients(6, 0, inode, 1) = gradient_terms(0, inode) * twelth; + B_matrix_gradients(6, 0, inode, 2) = gradient_terms(1, inode) * twelth; + } + + // B_matrix(7,0) = ( +y(0)*( -z(3) +z(4) ) + // +y(2)*( +z(3) -z(6) ) + // +y(3)*( +z(0) -z(2) +z(4) -z(6) ) + // +y(4)*( -z(0) -z(3) +z(5) +z(6) ) + // +y(5)*( -z(4) +z(6) ) + // +y(6)*( +z(2) +z(3) -z(4) -z(5) ) )*twelth; + + // y derivative + gradient_terms(0, 0) = -z(3) + z(4); + gradient_terms(0, 1) = 0; + gradient_terms(0, 2) = z(3) - z(6); + gradient_terms(0, 3) = z(0) - z(2) + z(4) - z(6); + gradient_terms(0, 4) = -z(0) - z(3) + z(5) + z(6); + gradient_terms(0, 5) = -z(4) + z(6); + gradient_terms(0, 6) = z(2) + z(3) - z(4) - z(5); + gradient_terms(0, 7) = 0; + + // z derivative + gradient_terms(1, 0) = y(3) - y(4); + gradient_terms(1, 1) = 0; + gradient_terms(1, 2) = -y(3) + y(6); + gradient_terms(1, 3) = -y(0) + y(2) - y(4) + y(6); + gradient_terms(1, 4) = y(0) + y(3) - y(5) - y(6); + gradient_terms(1, 5) = y(4) - y(6); + gradient_terms(1, 6) = -y(2) - y(3) + y(4) + y(5); + gradient_terms(1, 7) = 0; + + for (int inode = 0; inode < 8; inode++) + { + B_matrix_gradients(7, 0, inode, 0) = 0; + B_matrix_gradients(7, 0, inode, 1) = gradient_terms(0, inode) * twelth; + B_matrix_gradients(7, 0, inode, 2) = gradient_terms(1, inode) * twelth; + } + + // B_matrix(0,1) = ( +z(1)*( -x(2) -x(3) +x(4) +x(5) ) + // +z(2)*( +x(1) -x(3) ) + // +z(3)*( +x(1) +x(2) -x(4) -x(7) ) + // +z(4)*( -x(1) +x(3) -x(5) +x(7) ) + // +z(5)*( -x(1) +x(4) ) + // +z(7)*( +x(3) -x(4) ) )*twelth; + + // x derivative + gradient_terms(0, 0) = 0; + gradient_terms(0, 1) = z(2) + z(3) - z(4) - z(5); + gradient_terms(0, 2) = -z(1) + z(3); + gradient_terms(0, 3) = -z(1) - z(2) + z(4) + z(7); + gradient_terms(0, 4) = z(1) - z(3) + z(5) - z(7); + gradient_terms(0, 5) = z(1) - z(4); + gradient_terms(0, 6) = 0; + gradient_terms(0, 7) = -z(3) + z(4); + + // z derivative + gradient_terms(1, 0) = 0; + gradient_terms(1, 1) = -x(2) - x(3) + x(4) + x(5); + gradient_terms(1, 2) = x(1) - x(3); + gradient_terms(1, 3) = x(1) + x(2) - x(4) - x(7); + gradient_terms(1, 4) = -x(1) + x(3) - x(5) + x(7); + gradient_terms(1, 5) = -x(1) + x(4); + gradient_terms(1, 6) = 0; + gradient_terms(1, 7) = x(3) - x(4); + + for (int inode = 0; inode < 8; inode++) + { + B_matrix_gradients(0, 1, inode, 0) = gradient_terms(0, inode) * twelth; + B_matrix_gradients(0, 1, inode, 1) = 0; + B_matrix_gradients(0, 1, inode, 2) = gradient_terms(1, inode) * twelth; + } + + // B_matrix(1,1) = ( +z(0)*( +x(2) +x(3) -x(4) -x(5) ) + // +z(2)*( -x(0) -x(3) +x(5) +x(6) ) + // +z(3)*( -x(0) +x(2) ) + // +z(4)*( +x(0) -x(5) ) + // +z(5)*( +x(0) -x(2) +x(4) -x(6) ) + // +z(6)*( -x(2) +x(5) ) )*twelth; + + // x derivative + gradient_terms(0, 0) = -z(2) - z(3) + z(4) + z(5); + gradient_terms(0, 1) = 0; + gradient_terms(0, 2) = z(0) + z(3) - z(5) - z(6); + gradient_terms(0, 3) = z(0) - z(2); + gradient_terms(0, 4) = -z(0) + z(5); + gradient_terms(0, 5) = -z(0) + z(2) - z(4) + z(6); + gradient_terms(0, 6) = z(2) - z(5); + gradient_terms(0, 7) = 0; + + // z derivative + gradient_terms(1, 0) = x(2) + x(3) - x(4) - x(5); + gradient_terms(1, 1) = 0; + gradient_terms(1, 2) = -x(0) - x(3) + x(5) + x(6); + gradient_terms(1, 3) = -x(0) + x(2); + gradient_terms(1, 4) = x(0) - x(5); + gradient_terms(1, 5) = x(0) - x(2) + x(4) - x(6); + gradient_terms(1, 6) = -x(2) + x(5); + gradient_terms(1, 7) = 0; + + for (int inode = 0; inode < 8; inode++) + { + B_matrix_gradients(1, 1, inode, 0) = gradient_terms(0, inode) * twelth; + B_matrix_gradients(1, 1, inode, 1) = 0; + B_matrix_gradients(1, 1, inode, 2) = gradient_terms(1, inode) * twelth; + } + + // B_matrix(2,1) = ( +z(0)*( -x(1) +x(3) ) + // +z(1)*( +x(0) +x(3) -x(5) -x(6) ) + // +z(3)*( -x(0) -x(1) +x(6) +x(7) ) + // +z(5)*( +x(1) -x(6) ) + // +z(6)*( +x(1) -x(3) +x(5) -x(7) ) + // +z(7)*( -x(3) +x(6) ) )*twelth; + + // x derivative + gradient_terms(0, 0) = z(1) - z(3); + gradient_terms(0, 1) = -z(0) - z(3) + z(5) + z(6); + gradient_terms(0, 2) = 0; + gradient_terms(0, 3) = z(0) + z(1) - z(6) - z(7); + gradient_terms(0, 4) = 0; + gradient_terms(0, 5) = -z(1) + z(6); + gradient_terms(0, 6) = -z(1) + z(3) - z(5) + z(7); + gradient_terms(0, 7) = z(3) - z(6); + + // z derivative + gradient_terms(1, 0) = -x(1) + x(3); + gradient_terms(1, 1) = x(0) + x(3) - x(5) - x(6); + gradient_terms(1, 2) = 0; + gradient_terms(1, 3) = -x(0) - x(1) + x(6) + x(7); + gradient_terms(1, 4) = 0; + gradient_terms(1, 5) = x(1) - x(6); + gradient_terms(1, 6) = x(1) - x(3) + x(5) - x(7); + gradient_terms(1, 7) = -x(3) + x(6); + + for (int inode = 0; inode < 8; inode++) + { + B_matrix_gradients(2, 1, inode, 0) = gradient_terms(0, inode) * twelth; + B_matrix_gradients(2, 1, inode, 1) = 0; + B_matrix_gradients(2, 1, inode, 2) = gradient_terms(1, inode) * twelth; + } + + // B_matrix(3,1) = ( +z(0)*( -x(1) -x(2) +x(4) +x(7) ) + // +z(1)*( +x(0) -x(2) ) + // +z(2)*( +x(0) +x(1) -x(6) -x(7) ) + // +z(4)*( -x(0) +x(7) ) + // +z(6)*( +x(2) -x(7) ) + // +z(7)*( -x(0) +x(2) -x(4) +x(6) ) )*twelth; + + // x derivative + gradient_terms(0, 0) = z(1) + z(2) - z(4) - z(7); + gradient_terms(0, 1) = -z(0) + z(2); + gradient_terms(0, 2) = -z(0) - z(1) + z(6) + z(7); + gradient_terms(0, 3) = 0; + gradient_terms(0, 4) = z(0) - z(7); + gradient_terms(0, 5) = 0; + gradient_terms(0, 6) = -z(2) + z(7); + gradient_terms(0, 7) = z(0) - z(2) + z(4) - z(6); + + // z derivative + gradient_terms(1, 0) = -x(1) - x(2) + x(4) + x(7); + gradient_terms(1, 1) = x(0) - x(2); + gradient_terms(1, 2) = x(0) + x(1) - x(6) - x(7); + gradient_terms(1, 3) = 0; + gradient_terms(1, 4) = -x(0) + x(7); + gradient_terms(1, 5) = 0; + gradient_terms(1, 6) = x(2) - x(7); + gradient_terms(1, 7) = -x(0) + x(2) - x(4) + x(6); + + for (int inode = 0; inode < 8; inode++) + { + B_matrix_gradients(3, 1, inode, 0) = gradient_terms(0, inode) * twelth; + B_matrix_gradients(3, 1, inode, 1) = 0; + B_matrix_gradients(3, 1, inode, 2) = gradient_terms(1, inode) * twelth; + } + + // B_matrix(4,1) = ( +z(0)*( +x(1) -x(3) +x(5) -x(7) ) + // +z(1)*( -x(0) +x(5) ) + // +z(3)*( +x(0) -x(7) ) + // +z(5)*( -x(0) -x(1) +x(6) +x(7) ) + // +z(6)*( -x(5) +x(7) ) + // +z(7)*( +x(0) +x(3) -x(5) -x(6) ) )*twelth; + + // x derivative + gradient_terms(0, 0) = -z(1) + z(3) - z(5) + z(7); + gradient_terms(0, 1) = z(0) - z(5); + gradient_terms(0, 2) = 0; + gradient_terms(0, 3) = -z(0) + z(7); + gradient_terms(0, 4) = 0; + gradient_terms(0, 5) = z(0) + z(1) - z(6) - z(7); + gradient_terms(0, 6) = z(5) - z(7); + gradient_terms(0, 7) = -z(0) - z(3) + z(5) + z(6); + + // z derivative + gradient_terms(1, 0) = x(1) - x(3) + x(5) - x(7); + gradient_terms(1, 1) = -x(0) + x(5); + gradient_terms(1, 2) = 0; + gradient_terms(1, 3) = x(0) - x(7); + gradient_terms(1, 4) = 0; + gradient_terms(1, 5) = -x(0) - x(1) + x(6) + x(7); + gradient_terms(1, 6) = -x(5) + x(7); + gradient_terms(1, 7) = x(0) + x(3) - x(5) - x(6); + + for (int inode = 0; inode < 8; inode++) + { + B_matrix_gradients(4, 1, inode, 0) = gradient_terms(0, inode) * twelth; + B_matrix_gradients(4, 1, inode, 1) = 0; + B_matrix_gradients(4, 1, inode, 2) = gradient_terms(1, inode) * twelth; + } + + // B_matrix(5,1) = ( +z(0)*( +x(1) -x(4) ) + // +z(1)*( -x(0) +x(2) -x(4) +x(6) ) + // +z(2)*( -x(1) +x(6) ) + // +z(4)*( +x(0) +x(1) -x(6) -x(7) ) + // +z(6)*( -x(1) -x(2) +x(4) +x(7) ) + // +z(7)*( +x(4) -x(6) ) )*twelth; + + // x derivative + gradient_terms(0, 0) = -z(1) + z(4); + gradient_terms(0, 1) = z(0) - z(2) + z(4) - z(6); + gradient_terms(0, 2) = z(1) - z(6); + gradient_terms(0, 3) = 0; + gradient_terms(0, 4) = -z(0) - z(1) + z(6) + z(7); + gradient_terms(0, 5) = 0; + gradient_terms(0, 6) = z(1) + z(2) - z(4) - z(7); + gradient_terms(0, 7) = -z(4) + z(6); + + // z derivative + gradient_terms(1, 0) = x(1) - x(4); + gradient_terms(1, 1) = -x(0) + x(2) - x(4) + x(6); + gradient_terms(1, 2) = -x(1) + x(6); + gradient_terms(1, 3) = 0; + gradient_terms(1, 4) = x(0) + x(1) - x(6) - x(7); + gradient_terms(1, 5) = 0; + gradient_terms(1, 6) = -x(1) - x(2) + x(4) + x(7); + gradient_terms(1, 7) = x(4) - x(6); + + for (int inode = 0; inode < 8; inode++) + { + B_matrix_gradients(5, 1, inode, 0) = gradient_terms(0, inode) * twelth; + B_matrix_gradients(5, 1, inode, 1) = 0; + B_matrix_gradients(5, 1, inode, 2) = gradient_terms(1, inode) * twelth; + } + + // B_matrix(6,1) = ( +z(1)*( +x(2) -x(5) ) + // +z(2)*( -x(1) +x(3) -x(5) +x(7) ) + // +z(3)*( -x(2) +x(7) ) + // +z(4)*( +x(5) -x(7) ) + // +z(5)*( +x(1) +x(2) -x(4) -x(7) ) + // +z(7)*( -x(2) -x(3) +x(4) +x(5) ) )*twelth; + + // x derivative + gradient_terms(0, 0) = 0; + gradient_terms(0, 1) = -z(2) + z(5); + gradient_terms(0, 2) = z(1) - z(3) + z(5) - z(7); + gradient_terms(0, 3) = z(2) - z(7); + gradient_terms(0, 4) = -z(5) + z(7); + gradient_terms(0, 5) = -z(1) - z(2) + z(4) + z(7); + gradient_terms(0, 6) = 0; + gradient_terms(0, 7) = z(2) + z(3) - z(4) - z(5); + + // z derivative + gradient_terms(1, 0) = 0; + gradient_terms(1, 1) = x(2) - x(5); + gradient_terms(1, 2) = -x(1) + x(3) - x(5) + x(7); + gradient_terms(1, 3) = -x(2) + x(7); + gradient_terms(1, 4) = x(5) - x(7); + gradient_terms(1, 5) = x(1) + x(2) - x(4) - x(7); + gradient_terms(1, 6) = 0; + gradient_terms(1, 7) = -x(2) - x(3) + x(4) + x(5); + + for (int inode = 0; inode < 8; inode++) + { + B_matrix_gradients(6, 1, inode, 0) = gradient_terms(0, inode) * twelth; + B_matrix_gradients(6, 1, inode, 1) = 0; + B_matrix_gradients(6, 1, inode, 2) = gradient_terms(1, inode) * twelth; + } + + // B_matrix(7,1) = ( +z(0)*( -x(3) +x(4) ) + // +z(2)*( +x(3) -x(6) ) + // +z(3)*( +x(0) -x(2) +x(4) -x(6) ) + // +z(4)*( -x(0) -x(3) +x(5) +x(6) ) + // +z(5)*( -x(4) +x(6) ) + // +z(6)*( +x(2) +x(3) -x(4) -x(5) ) )*twelth; + + // x derivative + gradient_terms(0, 0) = z(3) - z(4); + gradient_terms(0, 1) = 0; + gradient_terms(0, 2) = -z(3) + z(6); + gradient_terms(0, 3) = -z(0) + z(2) - z(4) + z(6); + gradient_terms(0, 4) = z(0) + z(3) - z(5) - z(6); + gradient_terms(0, 5) = z(4) - z(6); + gradient_terms(0, 6) = -z(2) - z(3) + z(4) + z(5); + gradient_terms(0, 7) = 0; + + // z derivative + gradient_terms(1, 0) = -x(3) + x(4); + gradient_terms(1, 1) = 0; + gradient_terms(1, 2) = x(3) - x(6); + gradient_terms(1, 3) = x(0) - x(2) + x(4) - x(6); + gradient_terms(1, 4) = -x(0) - x(3) + x(5) + x(6); + gradient_terms(1, 5) = -x(4) + x(6); + gradient_terms(1, 6) = x(2) + x(3) - x(4) - x(5); + gradient_terms(1, 7) = 0; + + for (int inode = 0; inode < 8; inode++) + { + B_matrix_gradients(7, 1, inode, 0) = gradient_terms(0, inode) * twelth; + B_matrix_gradients(7, 1, inode, 1) = 0; + B_matrix_gradients(7, 1, inode, 2) = gradient_terms(1, inode) * twelth; + } + + // B_matrix(0,2) = ( +x(1)*( -y(2) -y(3) +y(4) +y(5) ) + // +x(2)*( +y(1) -y(3) ) + // +x(3)*( +y(1) +y(2) -y(4) -y(7) ) + // +x(4)*( -y(1) +y(3) -y(5) +y(7) ) + // +x(5)*( -y(1) +y(4) ) + // +x(7)*( +y(3) -y(4) ) )*twelth; + + // x derivative + gradient_terms(0, 0) = 0; + gradient_terms(0, 1) = -y(2) - y(3) + y(4) + y(5); + gradient_terms(0, 2) = y(1) - y(3); + gradient_terms(0, 3) = y(1) + y(2) - y(4) - y(7); + gradient_terms(0, 4) = -y(1) + y(3) - y(5) + y(7); + gradient_terms(0, 5) = -y(1) + y(4); + gradient_terms(0, 6) = 0; + gradient_terms(0, 7) = y(3) - y(4); + + // y derivative + gradient_terms(1, 0) = 0; + gradient_terms(1, 1) = x(2) + x(3) - x(4) - x(5); + gradient_terms(1, 2) = -x(1) + x(3); + gradient_terms(1, 3) = -x(1) - x(2) + x(4) + x(7); + gradient_terms(1, 4) = x(1) - x(3) + x(5) - x(7); + gradient_terms(1, 5) = x(1) - x(4); + gradient_terms(1, 6) = 0; + gradient_terms(1, 7) = -x(3) + x(4); + + for (int inode = 0; inode < 8; inode++) + { + B_matrix_gradients(0, 2, inode, 0) = gradient_terms(0, inode) * twelth; + B_matrix_gradients(0, 2, inode, 1) = gradient_terms(1, inode) * twelth; + B_matrix_gradients(0, 2, inode, 2) = 0; + } + + // B_matrix(1,2) = ( +x(0)*( +y(2) +y(3) -y(4) -y(5) ) + // +x(2)*( -y(0) -y(3) +y(5) +y(6) ) + // +x(3)*( -y(0) +y(2) ) + // +x(4)*( +y(0) -y(5) ) + // +x(5)*( +y(0) -y(2) +y(4) -y(6) ) + // +x(6)*( -y(2) +y(5) ) )*twelth; + + // x derivative + gradient_terms(0, 0) = y(2) + y(3) - y(4) - y(5); + gradient_terms(0, 1) = 0; + gradient_terms(0, 2) = -y(0) - y(3) + y(5) + y(6); + gradient_terms(0, 3) = -y(0) + y(2); + gradient_terms(0, 4) = y(0) - y(5); + gradient_terms(0, 5) = y(0) - y(2) + y(4) - y(6); + gradient_terms(0, 6) = -y(2) + y(5); + gradient_terms(0, 7) = 0; + + // y derivative + gradient_terms(1, 0) = -x(2) - x(3) + x(4) + x(5); + gradient_terms(1, 1) = 0; + gradient_terms(1, 2) = x(0) + x(3) - x(5) - x(6); + gradient_terms(1, 3) = x(0) - x(2); + gradient_terms(1, 4) = -x(0) + x(5); + gradient_terms(1, 5) = -x(0) + x(2) - x(4) + x(6); + gradient_terms(1, 6) = x(2) - x(5); + gradient_terms(1, 7) = 0; + + for (int inode = 0; inode < 8; inode++) + { + B_matrix_gradients(1, 2, inode, 0) = gradient_terms(0, inode) * twelth; + B_matrix_gradients(1, 2, inode, 1) = gradient_terms(1, inode) * twelth; + B_matrix_gradients(1, 2, inode, 2) = 0; + } + + // B_matrix(2,2) = ( +x(0)*( -y(1) +y(3) ) + // +x(1)*( +y(0) +y(3) -y(5) -y(6) ) + // +x(3)*( -y(0) -y(1) +y(6) +y(7) ) + // +x(5)*( +y(1) -y(6) ) + // +x(6)*( +y(1) -y(3) +y(5) -y(7) ) + // +x(7)*( -y(3) +y(6) ) )*twelth; + + // x derivative + gradient_terms(0, 0) = -y(1) + y(3); + gradient_terms(0, 1) = y(0) + y(3) - y(5) - y(6); + gradient_terms(0, 2) = 0; + gradient_terms(0, 3) = -y(0) - y(1) + y(6) + y(7); + gradient_terms(0, 4) = 0; + gradient_terms(0, 5) = y(1) - y(6); + gradient_terms(0, 6) = y(1) - y(3) + y(5) - y(7); + gradient_terms(0, 7) = -y(3) + y(6); + + // y derivative + gradient_terms(1, 0) = x(1) - x(3); + gradient_terms(1, 1) = -x(0) - x(3) + x(5) + x(6); + gradient_terms(1, 2) = 0; + gradient_terms(1, 3) = x(0) + x(1) - x(6) - x(7); + gradient_terms(1, 4) = 0; + gradient_terms(1, 5) = -x(1) + x(6); + gradient_terms(1, 6) = -x(1) + x(3) - x(5) + x(7); + gradient_terms(1, 7) = x(3) - x(6); + + for (int inode = 0; inode < 8; inode++) + { + B_matrix_gradients(2, 2, inode, 0) = gradient_terms(0, inode) * twelth; + B_matrix_gradients(2, 2, inode, 1) = gradient_terms(1, inode) * twelth; + B_matrix_gradients(2, 2, inode, 2) = 0; + } + + // B_matrix(3,2) = ( +x(0)*( -y(1) -y(2) +y(4) +y(7) ) + // +x(1)*( +y(0) -y(2) ) + // +x(2)*( +y(0) +y(1) -y(6) -y(7) ) + // +x(4)*( -y(0) +y(7) ) + // +x(6)*( +y(2) -y(7) ) + // +x(7)*( -y(0) +y(2) -y(4) +y(6) ) )*twelth; + + // x derivative + gradient_terms(0, 0) = -y(1) - y(2) + y(4) + y(7); + gradient_terms(0, 1) = y(0) - y(2); + gradient_terms(0, 2) = y(0) + y(1) - y(6) - y(7); + gradient_terms(0, 3) = 0; + gradient_terms(0, 4) = -y(0) + y(7); + gradient_terms(0, 5) = 0; + gradient_terms(0, 6) = y(2) - y(7); + gradient_terms(0, 7) = -y(0) + y(2) - y(4) + y(6); + + // y derivative + gradient_terms(1, 0) = x(1) + x(2) - x(4) - x(7); + gradient_terms(1, 1) = -x(0) + x(2); + gradient_terms(1, 2) = -x(0) - x(1) + x(6) + x(7); + gradient_terms(1, 3) = 0; + gradient_terms(1, 4) = x(0) - x(7); + gradient_terms(1, 5) = 0; + gradient_terms(1, 6) = -x(2) + x(7); + gradient_terms(1, 7) = x(0) - x(2) + x(4) - x(6); + + for (int inode = 0; inode < 8; inode++) + { + B_matrix_gradients(3, 2, inode, 0) = gradient_terms(0, inode) * twelth; + B_matrix_gradients(3, 2, inode, 1) = gradient_terms(1, inode) * twelth; + B_matrix_gradients(3, 2, inode, 2) = 0; + } + + // B_matrix(4,2) = ( +x(0)*( +y(1) -y(3) +y(5) -y(7) ) + // +x(1)*( -y(0) +y(5) ) + // +x(3)*( +y(0) -y(7) ) + // +x(5)*( -y(0) -y(1) +y(6) +y(7) ) + // +x(6)*( -y(5) +y(7) ) + // +x(7)*( +y(0) +y(3) -y(5) -y(6) ) )*twelth; + + // x derivative + gradient_terms(0, 0) = y(1) - y(3) + y(5) - y(7); + gradient_terms(0, 1) = -y(0) + y(5); + gradient_terms(0, 2) = 0; + gradient_terms(0, 3) = y(0) - y(7); + gradient_terms(0, 4) = 0; + gradient_terms(0, 5) = -y(0) - y(1) + y(6) + y(7); + gradient_terms(0, 6) = -y(5) + y(7); + gradient_terms(0, 7) = y(0) + y(3) - y(5) - y(6); + + // y derivative + gradient_terms(1, 0) = -x(1) + x(3) - x(4) + x(7); + gradient_terms(1, 1) = x(0) - x(5); + gradient_terms(1, 2) = 0; + gradient_terms(1, 3) = -x(0) + x(7); + gradient_terms(1, 4) = 0; + gradient_terms(1, 5) = x(0) + x(1) - x(6) - x(7); + gradient_terms(1, 6) = x(5) - x(7); + gradient_terms(1, 7) = -x(0) - x(3) + x(5) + x(6); + + for (int inode = 0; inode < 8; inode++) + { + B_matrix_gradients(4, 2, inode, 0) = gradient_terms(0, inode) * twelth; + B_matrix_gradients(4, 2, inode, 1) = gradient_terms(1, inode) * twelth; + B_matrix_gradients(4, 2, inode, 2) = 0; + } + + // B_matrix(5,2) = ( +x(0)*( +y(1) -y(4) ) + // +x(1)*( -y(0) +y(2) -y(4) +y(6) ) + // +x(2)*( -y(1) +y(6) ) + // +x(4)*( +y(0) +y(1) -y(6) -y(7) ) + // +x(6)*( -y(1) -y(2) +y(4) +y(7) ) + // +x(7)*( +y(4) -y(6) ) )*twelth; + + // x derivative + gradient_terms(0, 0) = y(1) - y(4); + gradient_terms(0, 1) = -y(0) + y(2) - y(4) + y(6); + gradient_terms(0, 2) = -y(1) + y(6); + gradient_terms(0, 3) = 0; + gradient_terms(0, 4) = y(0) + y(1) - y(6) - y(7); + gradient_terms(0, 5) = 0; + gradient_terms(0, 6) = -y(1) - y(2) + y(4) + y(7); + gradient_terms(0, 7) = y(4) - y(6); + + // y derivative + gradient_terms(1, 0) = -x(1) + x(4); + gradient_terms(1, 1) = x(0) - x(2) + x(4) - x(6); + gradient_terms(1, 2) = x(1) - x(6); + gradient_terms(1, 3) = 0; + gradient_terms(1, 4) = -x(0) - x(1) + x(6) + x(7); + gradient_terms(1, 5) = 0; + gradient_terms(1, 6) = x(1) + x(2) - x(4) - x(7); + gradient_terms(1, 7) = -x(4) + x(6); + + for (int inode = 0; inode < 8; inode++) + { + B_matrix_gradients(5, 2, inode, 0) = gradient_terms(0, inode) * twelth; + B_matrix_gradients(5, 2, inode, 1) = gradient_terms(1, inode) * twelth; + B_matrix_gradients(5, 2, inode, 2) = 0; + } + + // B_matrix(6,2) = ( +x(1)*( +y(2) -y(5) ) + // +x(2)*( -y(1) +y(3) -y(5) +y(7) ) + // +x(3)*( -y(2) +y(7) ) + // +x(4)*( +y(5) -y(7) ) + // +x(5)*( +y(1) +y(2) -y(4) -y(7) ) + // +x(7)*( -y(2) -y(3) +y(4) +y(5) ) )*twelth; + + // x derivative + gradient_terms(0, 0) = 0; + gradient_terms(0, 1) = y(2) - y(5); + gradient_terms(0, 2) = -y(1) + y(3) - y(5) + y(7); + gradient_terms(0, 3) = -y(2) + y(7); + gradient_terms(0, 4) = y(5) - y(7); + gradient_terms(0, 5) = y(1) + y(2) - y(4) - y(7); + gradient_terms(0, 6) = 0; + gradient_terms(0, 7) = -y(2) - y(3) + y(4) + y(5); + + // y derivative + gradient_terms(1, 0) = 0; + gradient_terms(1, 1) = -x(2) + x(5); + gradient_terms(1, 2) = x(1) - x(3) + x(5) - x(7); + gradient_terms(1, 3) = x(2) - x(7); + gradient_terms(1, 4) = -x(5) + x(7); + gradient_terms(1, 5) = -x(1) - x(2) + x(4) + x(7); + gradient_terms(1, 6) = 0; + gradient_terms(1, 7) = x(2) + x(3) - x(4) - x(5); + + for (int inode = 0; inode < 8; inode++) + { + B_matrix_gradients(6, 2, inode, 0) = gradient_terms(0, inode) * twelth; + B_matrix_gradients(6, 2, inode, 1) = gradient_terms(1, inode) * twelth; + B_matrix_gradients(6, 2, inode, 2) = 0; + } + + // B_matrix(7,2) = ( +x(0)*( -y(3) +y(4) ) + // +x(2)*( +y(3) -y(6) ) + // +x(3)*( +y(0) -y(2) +y(4) -y(6) ) + // +x(4)*( -y(0) -y(3) +y(5) +y(6) ) + // +x(5)*( -y(4) +y(6) ) + // +x(6)*( +y(2) +y(3) -y(4) -y(5) ) )*twelth; + + // x derivative + gradient_terms(0, 0) = -y(3) + y(4); + gradient_terms(0, 1) = 0; + gradient_terms(0, 2) = y(3) - y(6); + gradient_terms(0, 3) = y(0) - y(2) + y(4) - y(6); + gradient_terms(0, 4) = -y(0) - y(3) + y(5) + y(6); + gradient_terms(0, 5) = -y(4) + y(6); + gradient_terms(0, 6) = y(2) + y(3) - y(4) - y(5); + gradient_terms(0, 7) = 0; + + // y derivative + gradient_terms(1, 0) = x(3) - x(4); + gradient_terms(1, 1) = 0; + gradient_terms(1, 2) = -x(3) + x(6); + gradient_terms(1, 3) = -x(0) + x(2) - x(4) + x(6); + gradient_terms(1, 4) = x(0) + x(3) - x(5) - x(6); + gradient_terms(1, 5) = x(4) - x(6); + gradient_terms(1, 6) = -x(2) - x(3) + x(4) + x(5); + gradient_terms(1, 7) = 0; + + for (int inode = 0; inode < 8; inode++) + { + B_matrix_gradients(7, 2, inode, 0) = gradient_terms(0, inode) * twelth; + B_matrix_gradients(7, 2, inode, 1) = gradient_terms(1, inode) * twelth; + B_matrix_gradients(7, 2, inode, 2) = 0; + } +} // end subroutine diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/momentum.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/momentum.cpp new file mode 100644 index 000000000..03452ccd1 --- /dev/null +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/momentum.cpp @@ -0,0 +1,391 @@ + +#include "state.h" +#include "FEA_Module_SGH.h" + +// ----------------------------------------------------------------------------- +// This function evolves the velocity at the nodes of the mesh +// ------------------------------------------------------------------------------ +void FEA_Module_SGH::update_velocity_sgh(double rk_alpha, + DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_mass, + const DViewCArrayKokkos& corner_force + ) +{ + const size_t rk_level = rk_num_bins - 1; + const size_t num_dims = num_dim; + + // walk over the nodes to update the velocity + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + double node_force[3]; + for (size_t dim = 0; dim < num_dims; dim++) + { + node_force[dim] = 0.0; + } // end for dim + + // loop over all corners around the node and calculate the nodal force + for (size_t corner_lid = 0; corner_lid < num_corners_in_node(node_gid); corner_lid++) + { + // Get corner gid + size_t corner_gid = corners_in_node(node_gid, corner_lid); + + // loop over dimension + for (size_t dim = 0; dim < num_dims; dim++) + { + node_force[dim] += corner_force(corner_gid, dim); + } // end for dim + } // end for corner_lid + + // update the velocity + for (int dim = 0; dim < num_dims; dim++) + { + node_vel(rk_level, node_gid, dim) = node_vel(0, node_gid, dim) + + rk_alpha * dt * node_force[dim] / node_mass(node_gid); + } // end for dim + }); // end for parallel for over nodes + + return; +} // end subroutine update_velocity + +// ----------------------------------------------------------------------------- +// This function calculates the velocity gradient +// ------------------------------------------------------------------------------ +KOKKOS_FUNCTION +void FEA_Module_SGH::get_velgrad(ViewCArrayKokkos& vel_grad, + const ViewCArrayKokkos& elem_node_gids, + const DViewCArrayKokkos& node_vel, + const ViewCArrayKokkos& b_matrix, + const double elem_vol, + const size_t elem_gid, + const size_t rk_level + ) const +{ + const size_t num_nodes_in_elem = 8; + + double u_array[num_nodes_in_elem]; // x-dir vel component + double v_array[num_nodes_in_elem]; // y-dir vel component + double w_array[num_nodes_in_elem]; // z-dir vel component + + ViewCArrayKokkos u(u_array, num_nodes_in_elem); + ViewCArrayKokkos v(v_array, num_nodes_in_elem); + ViewCArrayKokkos w(w_array, num_nodes_in_elem); + + // get the vertex velocities for the cell + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + // Get node gid + size_t node_gid = elem_node_gids(node_lid); + + u(node_lid) = node_vel(rk_level, node_gid, 0); + v(node_lid) = node_vel(rk_level, node_gid, 1); + w(node_lid) = node_vel(rk_level, node_gid, 2); + } // end for + + // --- calculate the velocity gradient terms --- + double inverse_vol = 1.0 / elem_vol; + + // x-dir + vel_grad(0, 0) = (u(0) * b_matrix(0, 0) + u(1) * b_matrix(1, 0) + + u(2) * b_matrix(2, 0) + u(3) * b_matrix(3, 0) + + u(4) * b_matrix(4, 0) + u(5) * b_matrix(5, 0) + + u(6) * b_matrix(6, 0) + u(7) * b_matrix(7, 0)) * inverse_vol; + + vel_grad(0, 1) = (u(0) * b_matrix(0, 1) + u(1) * b_matrix(1, 1) + + u(2) * b_matrix(2, 1) + u(3) * b_matrix(3, 1) + + u(4) * b_matrix(4, 1) + u(5) * b_matrix(5, 1) + + u(6) * b_matrix(6, 1) + u(7) * b_matrix(7, 1)) * inverse_vol; + + vel_grad(0, 2) = (u(0) * b_matrix(0, 2) + u(1) * b_matrix(1, 2) + + u(2) * b_matrix(2, 2) + u(3) * b_matrix(3, 2) + + u(4) * b_matrix(4, 2) + u(5) * b_matrix(5, 2) + + u(6) * b_matrix(6, 2) + u(7) * b_matrix(7, 2)) * inverse_vol; + + // y-dir + vel_grad(1, 0) = (v(0) * b_matrix(0, 0) + v(1) * b_matrix(1, 0) + + v(2) * b_matrix(2, 0) + v(3) * b_matrix(3, 0) + + v(4) * b_matrix(4, 0) + v(5) * b_matrix(5, 0) + + v(6) * b_matrix(6, 0) + v(7) * b_matrix(7, 0)) * inverse_vol; + + vel_grad(1, 1) = (v(0) * b_matrix(0, 1) + v(1) * b_matrix(1, 1) + + v(2) * b_matrix(2, 1) + v(3) * b_matrix(3, 1) + + v(4) * b_matrix(4, 1) + v(5) * b_matrix(5, 1) + + v(6) * b_matrix(6, 1) + v(7) * b_matrix(7, 1)) * inverse_vol; + vel_grad(1, 2) = (v(0) * b_matrix(0, 2) + v(1) * b_matrix(1, 2) + + v(2) * b_matrix(2, 2) + v(3) * b_matrix(3, 2) + + v(4) * b_matrix(4, 2) + v(5) * b_matrix(5, 2) + + v(6) * b_matrix(6, 2) + v(7) * b_matrix(7, 2)) * inverse_vol; + + // z-dir + vel_grad(2, 0) = (w(0) * b_matrix(0, 0) + w(1) * b_matrix(1, 0) + + w(2) * b_matrix(2, 0) + w(3) * b_matrix(3, 0) + + w(4) * b_matrix(4, 0) + w(5) * b_matrix(5, 0) + + w(6) * b_matrix(6, 0) + w(7) * b_matrix(7, 0)) * inverse_vol; + + vel_grad(2, 1) = (w(0) * b_matrix(0, 1) + w(1) * b_matrix(1, 1) + + w(2) * b_matrix(2, 1) + w(3) * b_matrix(3, 1) + + w(4) * b_matrix(4, 1) + w(5) * b_matrix(5, 1) + + w(6) * b_matrix(6, 1) + w(7) * b_matrix(7, 1)) * inverse_vol; + + vel_grad(2, 2) = (w(0) * b_matrix(0, 2) + w(1) * b_matrix(1, 2) + + w(2) * b_matrix(2, 2) + w(3) * b_matrix(3, 2) + + w(4) * b_matrix(4, 2) + w(5) * b_matrix(5, 2) + + w(6) * b_matrix(6, 2) + w(7) * b_matrix(7, 2)) * inverse_vol; + + return; +} // end function + +// ----------------------------------------------------------------------------- +// This function calculates the velocity gradient +// ------------------------------------------------------------------------------ +KOKKOS_FUNCTION +void FEA_Module_SGH::get_velgrad2D(ViewCArrayKokkos& vel_grad, + const ViewCArrayKokkos& elem_node_gids, + const DViewCArrayKokkos& node_vel, + const ViewCArrayKokkos& b_matrix, + const double elem_vol, + const double elem_area, + const size_t elem_gid, + const size_t rk_level + ) const +{ + const size_t num_nodes_in_elem = 4; + + double u_array[num_nodes_in_elem]; + double v_array[num_nodes_in_elem]; + ViewCArrayKokkos u(u_array, num_nodes_in_elem); // x-dir vel component + ViewCArrayKokkos v(v_array, num_nodes_in_elem); // y-dir vel component + + // get the vertex velocities for the cell + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + // Get node gid + size_t node_gid = elem_node_gids(node_lid); + + u(node_lid) = node_vel(rk_level, node_gid, 0); // x-comp + v(node_lid) = node_vel(rk_level, node_gid, 1); // y-comp + } // end for + + // initialize to zero + for (size_t i = 0; i < 3; i++) + { + for (size_t j = 0; j < 3; j++) + { + vel_grad(i, j) = 0.0; + } + } + + double mean_radius = elem_vol / elem_area; + double elem_vel_r = 0.25 * (v(0) + v(1) + v(2) + v(3)); + + // --- calculate the velocity gradient terms --- + double inverse_area = 1.0 / elem_area; + + // x-dir + vel_grad(0, 0) = (u(0) * b_matrix(0, 0) + u(1) * b_matrix(1, 0) + + u(2) * b_matrix(2, 0) + u(3) * b_matrix(3, 0)) * inverse_area; + + vel_grad(0, 1) = (u(0) * b_matrix(0, 1) + u(1) * b_matrix(1, 1) + + u(2) * b_matrix(2, 1) + u(3) * b_matrix(3, 1)) * inverse_area; + + // y-dir + vel_grad(1, 0) = (v(0) * b_matrix(0, 0) + v(1) * b_matrix(1, 0) + + v(2) * b_matrix(2, 0) + v(3) * b_matrix(3, 0)) * inverse_area; + + vel_grad(1, 1) = (v(0) * b_matrix(0, 1) + v(1) * b_matrix(1, 1) + + v(2) * b_matrix(2, 1) + v(3) * b_matrix(3, 1)) * inverse_area; + + vel_grad(2, 2) = elem_vel_r / mean_radius; // + avg(vel_R)/R + + return; +} // end function + +// ----------------------------------------------------------------------------- +// This subroutine to calculate the velocity divergence in all elements +// ------------------------------------------------------------------------------ +void FEA_Module_SGH::get_divergence(DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_vol + ) +{ + const size_t rk_level = rk_num_bins - 1; + + // --- calculate the forces acting on the nodes from the element --- + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { + const size_t num_nodes_in_elem = 8; + const size_t num_dims = 3; + + double u_array[num_nodes_in_elem]; + double v_array[num_nodes_in_elem]; + double w_array[num_nodes_in_elem]; + ViewCArrayKokkos u(u_array, num_nodes_in_elem); // x-dir vel component + ViewCArrayKokkos v(v_array, num_nodes_in_elem); // y-dir vel component + ViewCArrayKokkos w(w_array, num_nodes_in_elem); // z-dir vel component + + // cut out the node_gids for this element + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 8); + + // The b_matrix are the outward corner area normals + double b_matrix_array[24]; + ViewCArrayKokkos b_matrix(b_matrix_array, num_nodes_in_elem, num_dims); + get_bmatrix(b_matrix, + elem_gid, + node_coords, + elem_node_gids, + rk_level); + + // get the vertex velocities for the elem + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + // Get node gid + size_t node_gid = elem_node_gids(node_lid); + + u(node_lid) = node_vel(rk_level, node_gid, 0); + v(node_lid) = node_vel(rk_level, node_gid, 1); + w(node_lid) = node_vel(rk_level, node_gid, 2); + } // end for + + // --- calculate the velocity divergence terms --- + double inverse_vol = 1.0 / elem_vol(elem_gid); + + elem_div(elem_gid) = 0.0; + + // x-dir + elem_div(elem_gid) += (u(0) * b_matrix(0, 0) + u(1) * b_matrix(1, 0) + + u(2) * b_matrix(2, 0) + u(3) * b_matrix(3, 0) + + u(4) * b_matrix(4, 0) + u(5) * b_matrix(5, 0) + + u(6) * b_matrix(6, 0) + u(7) * b_matrix(7, 0)) * inverse_vol; + + // y-dir + elem_div(elem_gid) += (v(0) * b_matrix(0, 1) + v(1) * b_matrix(1, 1) + + v(2) * b_matrix(2, 1) + v(3) * b_matrix(3, 1) + + v(4) * b_matrix(4, 1) + v(5) * b_matrix(5, 1) + + v(6) * b_matrix(6, 1) + v(7) * b_matrix(7, 1)) * inverse_vol; + + // z-dir + elem_div(elem_gid) += (w(0) * b_matrix(0, 2) + w(1) * b_matrix(1, 2) + + w(2) * b_matrix(2, 2) + w(3) * b_matrix(3, 2) + + w(4) * b_matrix(4, 2) + w(5) * b_matrix(5, 2) + + w(6) * b_matrix(6, 2) + w(7) * b_matrix(7, 2)) * inverse_vol; + }); // end parallel for over elem_gid + + return; +} // end subroutine + +// ----------------------------------------------------------------------------- +// This subroutine to calculate the velocity divergence in all elements +// ------------------------------------------------------------------------------ +void FEA_Module_SGH::get_divergence2D(DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_vol + ) +{ + const size_t rk_level = rk_num_bins - 1; + + // --- calculate the forces acting on the nodes from the element --- + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { + const size_t num_nodes_in_elem = 4; + const size_t num_dims = 2; + + double u_array[num_nodes_in_elem]; + double v_array[num_nodes_in_elem]; + ViewCArrayKokkos u(u_array, num_nodes_in_elem); // x-dir vel component + ViewCArrayKokkos v(v_array, num_nodes_in_elem); // y-dir vel component + + // true volume RZ + // double r_array[num_nodes_in_elem]; + // ViewCArrayKokkos r(r_array, num_nodes_in_elem); // r-dir coordinate + + // cut out the node_gids for this element + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 4); + + // The b_matrix are the outward corner area normals + double b_matrix_array[24]; + ViewCArrayKokkos b_matrix(b_matrix_array, num_nodes_in_elem, num_dims); + get_bmatrix2D(b_matrix, + elem_gid, + node_coords, + elem_node_gids, + rk_level); + + // calculate the area of the quad + double elem_area = get_area_quad(elem_gid, node_coords, elem_node_gids, rk_level); + // true volume uses the elem_vol + + // get the vertex velocities and node coordinate for the elem + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + // Get node gid + size_t node_gid = elem_node_gids(node_lid); + + u(node_lid) = node_vel(rk_level, node_gid, 0); + v(node_lid) = node_vel(rk_level, node_gid, 1); + + // r(node_lid) = node_coords(rk_level, node_gid, 1); // true volume RZ + } // end for + + // --- calculate the velocity divergence terms --- + double inverse_area = 1.0 / elem_area; + + double mean_radius = elem_vol(elem_gid) / elem_area; + double elem_vel_r = 0.25 * (v(0) + v(1) + v(2) + v(3)); + + elem_div(elem_gid) = 0.0; + + // x-dir + elem_div(elem_gid) += (u(0) * b_matrix(0, 0) + + u(1) * b_matrix(1, 0) + + u(2) * b_matrix(2, 0) + + u(3) * b_matrix(3, 0)) * inverse_area; + + // y-dir (i.e., r direction) + elem_div(elem_gid) += (v(0) * b_matrix(0, 1) + + v(1) * b_matrix(1, 1) + + v(2) * b_matrix(2, 1) + + v(3) * b_matrix(3, 1)) * inverse_area + + elem_vel_r / mean_radius; // + avg(u_R)/R + }); // end parallel for over elem_gid + + return; +} // end subroutine + +// The velocity gradient can be decomposed into symmetric and antisymmetric tensors +// L = vel_grad +// D = sym(L) +// W = antisym(L) +KOKKOS_INLINE_FUNCTION +void FEA_Module_SGH::decompose_vel_grad(ViewCArrayKokkos& D_tensor, + ViewCArrayKokkos& W_tensor, + const ViewCArrayKokkos& vel_grad, + const ViewCArrayKokkos& elem_node_gids, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const double vol + ) const +{ + // --- Calculate the velocity gradient --- + + const size_t num_dims = 3; + + // initialize to zero + for (size_t i = 0; i < num_dims; i++) + { + for (size_t j = 0; j < num_dims; j++) + { + D_tensor(i, j) = 0.0; + W_tensor(i, j) = 0.0; + } + } // end for + + for (size_t i = 0; i < num_dims; i++) + { + for (size_t j = 0; j < num_dims; j++) + { + D_tensor(i, j) = 0.5 * (vel_grad(i, j) + vel_grad(j, i)); + W_tensor(i, j) = 0.5 * (vel_grad(i, j) - vel_grad(j, i)); + } + } // end for + + return; +} // end function to calculate D and W diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/power_gradients_sgh.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/power_gradients_sgh.cpp new file mode 100644 index 000000000..ad7e648d5 --- /dev/null +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/power_gradients_sgh.cpp @@ -0,0 +1,495 @@ + +#include "mesh.h" +#include "state.h" +#include "FEA_Module_SGH.h" +#include "Simulation_Parameters/Simulation_Parameters_Explicit.h" +#include "Simulation_Parameters/FEA_Module/SGH_Parameters.h" + +// -------------------------------------------------------------------------------------------------------- +// Computes term objective derivative term involving gradient of power with respect to the design variable +// --------------------------------------------------------------------------------------------------------- + +void FEA_Module_SGH::power_design_gradient_term(const_vec_array design_variables, vec_array design_gradients) +{ + bool element_constant_density = true; + size_t current_data_index, next_data_index; + size_t num_bdy_nodes = mesh->num_bdy_nodes; + int num_corners = rnum_elem * num_nodes_in_elem; + real_t global_dt; + + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos material = simparam->material; + const int num_dim = simparam->num_dims; + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + + CArrayKokkos current_element_adjoint = CArrayKokkos(num_nodes_in_elem, num_dim); + + DCArrayKokkos elem_power_dgradients(rnum_elem); + + // gradient contribution from gradient of Force vector with respect to design variable. + if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) + { + if (myrank == 0) + { + std::cout << "gradient term involving adjoint derivative" << std::endl; + } + } + + for (unsigned long cycle = 0; cycle < last_time_step + 1; cycle++) + { + // compute timestep from time data + global_dt = time_data[cycle + 1] - time_data[cycle]; + // print + if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) + { + if (cycle == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } + // print time step every 10 cycles + else if (cycle % 20 == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } // end if + } + + // view scope + { + const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array current_element_internal_energy = (*forward_solve_internal_energy_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array current_coord_vector = (*forward_solve_coordinate_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array current_psi_adjoint_vector = (*psi_adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_velocity_vector = (*forward_solve_velocity_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_element_internal_energy = (*forward_solve_internal_energy_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_coord_vector = (*forward_solve_coordinate_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_psi_adjoint_vector = (*psi_adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + + // first half of integration step calculation + FOR_ALL_CLASS(node_gid, 0, nall_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + node_vel(rk_level, node_gid, idim) = current_velocity_vector(node_gid, idim); + node_coords(rk_level, node_gid, idim) = current_coord_vector(node_gid, idim); + } + }); // end parallel for + Kokkos::fence(); + + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { + elem_sie(rk_level, elem_gid) = current_element_internal_energy(elem_gid, 0); + }); // end parallel for + Kokkos::fence(); + + get_vol(); + + // ---- Calculate velocity diveregence for the element ---- + if (num_dim == 2) + { + get_divergence2D(elem_div, node_coords, node_vel, elem_vol); + } + else + { + get_divergence(elem_div, node_coords, node_vel, elem_vol); + } // end if 2D + + // ---- Calculate elem state (den, pres, sound speed, stress) for next time step ---- + if (num_dim == 2) + { + update_state2D(material, + *mesh, + node_coords, + node_vel, + elem_den, + elem_pres, + elem_stress, + elem_sspd, + elem_sie, + elem_vol, + elem_mass, + elem_mat_id, + 1.0, + cycle); + } + else + { + update_state(material, + *mesh, + node_coords, + node_vel, + elem_den, + elem_pres, + elem_stress, + elem_sspd, + elem_sie, + elem_vol, + elem_mass, + elem_mat_id, + 1.0, + cycle); + } + + // ---- calculate the forces on the vertices and evolve stress (hypo model) ---- + if (num_dim == 2) + { + get_force_sgh2D(material, + *mesh, + node_coords, + node_vel, + elem_den, + elem_sie, + elem_pres, + elem_stress, + elem_sspd, + elem_vol, + elem_div, + elem_mat_id, + corner_force, + 1.0, + cycle); + } + else + { + get_force_sgh(material, + *mesh, + node_coords, + node_vel, + elem_den, + elem_sie, + elem_pres, + elem_stress, + elem_sspd, + elem_vol, + elem_div, + elem_mat_id, + corner_force, + 1.0, + cycle); + } + + get_force_dgradient_sgh(material, + *mesh, + node_coords, + node_vel, + elem_den, + elem_sie, + elem_pres, + elem_stress, + elem_sspd, + elem_vol, + elem_div, + elem_mat_id, + 1.0, + cycle); + + get_power_dgradient_sgh(1.0, + *mesh, + node_vel, + node_coords, + elem_sie, + elem_mass, + corner_force, + elem_power_dgradients); + + // derivatives of forces at corners stored in corner_vector_storage buffer by previous routine + FOR_ALL_CLASS(elem_id, 0, rnum_elem, { + size_t node_id; + size_t corner_id; + real_t inner_product; + + inner_product = current_psi_adjoint_vector(elem_id, 0) * elem_power_dgradients(elem_id); + + for (int inode = 0; inode < num_nodes_in_elem; inode++) + { + // compute gradient of local element contribution to v^t*M*v product + corner_id = elem_id * num_nodes_in_elem + inode; + corner_value_storage(corner_id) = inner_product; + } + }); // end parallel for + Kokkos::fence(); + + // accumulate node values from corner storage + // multiply + FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { + size_t corner_id; + for (int icorner = 0; icorner < num_corners_in_node(node_id); icorner++) + { + corner_id = corners_in_node(node_id, icorner); + design_gradients(node_id, 0) += -corner_value_storage(corner_id) * global_dt; + } + }); // end parallel for + Kokkos::fence(); + } // end view scope + } +} + +// --------------------------------------------------------------------------------------- +// This function calculates the gradient for element power with respect to design variable +// ---------------------------------------------------------------------------------------- + +void FEA_Module_SGH::get_power_dgradient_sgh(double rk_alpha, + const mesh_t& mesh, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& corner_force, + DCArrayKokkos elem_power_dgradients) +{ + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + int num_dims = simparam->num_dims; + + // loop over all the elements in the mesh + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { + double elem_power = 0.0; + elem_power_dgradients(elem_gid) = 0; + + // --- tally the contribution from each corner to the element --- + + // Loop over the nodes in the element + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + size_t corner_lid = node_lid; + + // Get node global id for the local node id + size_t node_gid = nodes_in_elem(elem_gid, node_lid); + + // Get the corner global id for the local corner id + size_t corner_gid = corners_in_elem(elem_gid, corner_lid); + + double node_radius = 1; + if (num_dims == 2) + { + node_radius = node_coords(rk_level, node_gid, 1); + } + + // calculate the Power=F dot V for this corner + for (size_t dim = 0; dim < num_dims; dim++) + { + double half_vel = (node_vel(rk_level, node_gid, dim) + node_vel(0, node_gid, dim)) * 0.5; + elem_power_dgradients(elem_gid) -= corner_vector_storage(corner_gid, dim) * node_radius * node_vel(rk_level, node_gid, dim); + } // end for dim + } // end for node_lid + }); // end parallel loop over the elements + + return; +} // end subroutine + +// ----------------------------------------------------------------------------- +// This function calculates the gradient for element power with respect to position +// ------------------------------------------------------------------------------ + +void FEA_Module_SGH::get_power_ugradient_sgh(double rk_alpha, + const mesh_t& mesh, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& corner_force) +{ + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + int num_dims = simparam->num_dims; + + // initialize gradient matrix + FOR_ALL_CLASS(dof_gid, 0, nlocal_nodes * num_dims, { + for (int idof = 0; idof < DOF_to_Elem_Matrix_Strides(dof_gid); idof++) + { + Power_Gradient_Positions(dof_gid, idof) = 0; + } + }); // end parallel for loop over nodes + Kokkos::fence(); + + // loop over all the elements in the mesh + for (size_t elem_gid = 0; elem_gid < rnum_elem; elem_gid++) + { + // FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { + + double elem_power = 0.0; + + // --- tally the contribution from each corner to the element --- + + // Loop over the nodes in the element + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + size_t corner_lid = node_lid; + size_t column_id; + size_t gradient_node_id; + // Get node global id for the local node id + size_t node_gid = nodes_in_elem(elem_gid, node_lid); + + // Get the corner global id for the local corner id + size_t corner_gid = corners_in_elem(elem_gid, corner_lid); + + double node_radius = 1; + if (num_dims == 2) + { + node_radius = node_coords(rk_level, node_gid, 1); + } + + // calculate the Power=F dot V for this corner + for (size_t dim = 0; dim < num_dims; dim++) + { + for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) + { + for (size_t jdim = 0; jdim < num_dims; jdim++) + { + column_id = Element_Gradient_Matrix_Assembly_Map(elem_gid, igradient); + gradient_node_id = nodes_in_elem(elem_gid, igradient); + if (!map->isNodeLocalElement(gradient_node_id)) + { + continue; + } + Power_Gradient_Positions(gradient_node_id * num_dims + jdim, column_id) -= corner_gradient_storage(corner_gid, dim, igradient, jdim) * node_vel(rk_level, node_gid, + dim) * node_radius; + } + } + } // end for dim + } // end for node_lid + + // }); // end parallel loop over the elements + } + + return; +} // end subroutine + +// ----------------------------------------------------------------------------- +// This function calculates the gradient for element power with respect to velocity +// ------------------------------------------------------------------------------ + +void FEA_Module_SGH::get_power_vgradient_sgh(double rk_alpha, + const mesh_t& mesh, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& corner_force) +{ + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + int num_dims = simparam->num_dims; + + // initialize gradient matrix + FOR_ALL_CLASS(dof_gid, 0, nlocal_nodes * num_dims, { + for (int idof = 0; idof < DOF_to_Elem_Matrix_Strides(dof_gid); idof++) + { + Power_Gradient_Velocities(dof_gid, idof) = 0; + } + }); // end parallel for loop over nodes + Kokkos::fence(); + + // loop over all the elements in the mesh + for (size_t elem_gid = 0; elem_gid < rnum_elem; elem_gid++) + { + // FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { + + double elem_power = 0.0; + + // --- tally the contribution from each corner to the element --- + + // Loop over the nodes in the element + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + size_t corner_lid = node_lid; + size_t column_id; + size_t gradient_node_id; + // Get node global id for the local node id + size_t node_gid = nodes_in_elem(elem_gid, node_lid); + + // Get the corner global id for the local corner id + size_t corner_gid = corners_in_elem(elem_gid, corner_lid); + + double node_radius = 1; + if (num_dims == 2) + { + node_radius = node_coords(rk_level, node_gid, 1); + } + + // calculate the Power=F dot V for this corner + for (size_t dim = 0; dim < num_dims; dim++) + { + for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) + { + column_id = Element_Gradient_Matrix_Assembly_Map(elem_gid, igradient); + gradient_node_id = nodes_in_elem(elem_gid, igradient); + if (!map->isNodeLocalElement(gradient_node_id)) + { + continue; + } + if (node_lid == igradient) + { + Power_Gradient_Velocities(gradient_node_id * num_dims + dim, column_id) -= corner_gradient_storage(corner_gid, dim, igradient, dim) * node_vel(rk_level, node_gid, + dim) * node_radius + + corner_force(corner_gid, dim) * node_radius; + } + else + { + Power_Gradient_Velocities(gradient_node_id * num_dims + dim, column_id) -= corner_gradient_storage(corner_gid, dim, igradient, dim) * node_vel(rk_level, node_gid, + dim) * node_radius; + } + } + } // end for dim + } // end for node_lid + + // }); // end parallel loop over the elements + } + + return; +} // end subroutine + +// ----------------------------------------------------------------------------- +// This function calculates the gradient for element power with respect to energy +// ------------------------------------------------------------------------------ + +void FEA_Module_SGH::get_power_egradient_sgh(double rk_alpha, + const mesh_t& mesh, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& corner_force) +{ + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + int num_dims = simparam->num_dims; + + // initialize gradient storage + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { + Power_Gradient_Energies(elem_gid) = 0; + }); // end parallel loop over the elements + + // loop over all the elements in the mesh + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { + double elem_power = 0.0; + + // --- tally the contribution from each corner to the element --- + + // Loop over the nodes in the element + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + size_t corner_lid = node_lid; + + // Get node global id for the local node id + size_t node_gid = nodes_in_elem(elem_gid, node_lid); + + // Get the corner global id for the local corner id + size_t corner_gid = corners_in_elem(elem_gid, corner_lid); + + double node_radius = 1; + if (num_dims == 2) + { + node_radius = node_coords(rk_level, node_gid, 1); + } + + // calculate the Power=F dot V for this corner + for (size_t dim = 0; dim < num_dims; dim++) + { + double half_vel = (node_vel(rk_level, node_gid, dim) + node_vel(0, node_gid, dim)) * 0.5; + Power_Gradient_Energies(elem_gid) -= Force_Gradient_Energies(elem_gid, node_lid * num_dims + dim) * node_radius * node_vel(rk_level, node_gid, dim); + } // end for dim + } // end for node_lid + }); // end parallel loop over the elements + + return; +} // end subroutine diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/properties.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/properties.cpp similarity index 63% rename from src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/properties.cpp rename to src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/properties.cpp index 9719963c9..f415296de 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/properties.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/properties.cpp @@ -1,73 +1,71 @@ - + // ----------------------------------------------------------------------------- // This calls the models to update state -//------------------------------------------------------------------------------ +// ------------------------------------------------------------------------------ #include "state.h" #include "mesh.h" #include "FEA_Module_SGH.h" -void FEA_Module_SGH::update_state(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &elem_den, - DViewCArrayKokkos &elem_pres, - DViewCArrayKokkos &elem_stress, - DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_mass, - const DViewCArrayKokkos &elem_mat_id, - const double rk_alpha, - const size_t cycle - ){ - +void FEA_Module_SGH::update_state(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_den, + DViewCArrayKokkos& elem_pres, + DViewCArrayKokkos& elem_stress, + DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& elem_mat_id, + const double rk_alpha, + const size_t cycle + ) +{ const size_t rk_level = rk_num_bins - 1; - int num_dims = num_dim; - + int num_dims = num_dim; + // loop over all the elements in the mesh - FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { - + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { const size_t num_dims = num_dims; const size_t num_nodes_in_elem = num_nodes_in_elem; // cut out the node_gids for this element - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), num_nodes_in_elem); - + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), num_nodes_in_elem); + // --- Density --- - elem_den(elem_gid) = elem_mass(elem_gid)/elem_vol(elem_gid); - + elem_den(elem_gid) = elem_mass(elem_gid) / elem_vol(elem_gid); + size_t mat_id = elem_mat_id(elem_gid); - //initialize elem pressure + // initialize elem pressure elem_pres(elem_gid) = 0; - + // --- Stress --- // hyper elastic plastic model - if(material(mat_id).strength_type == STRENGTH_TYPE::hyper){ - + if (material(mat_id).strength_type == STRENGTH_TYPE::hyper) + { // cut out the node_gids for this element - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), num_nodes_in_elem); - + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), num_nodes_in_elem); + // --- Density --- - elem_den(elem_gid) = elem_mass(elem_gid)/elem_vol(elem_gid); - + elem_den(elem_gid) = elem_mass(elem_gid) / elem_vol(elem_gid); + // corner area normals double area_array[24]; - ViewCArrayKokkos area(area_array, num_nodes_in_elem, num_dims); - + ViewCArrayKokkos area(area_array, num_nodes_in_elem, num_dims); + // velocity gradient double vel_grad_array[9]; - ViewCArrayKokkos vel_grad(vel_grad_array, num_dims, num_dims); - + ViewCArrayKokkos vel_grad(vel_grad_array, num_dims, num_dims); + // get the B matrix which are the OUTWARD corner area normals get_bmatrix(area, elem_gid, node_coords, elem_node_gids, rk_level); - - + // --- Calculate the velocity gradient --- get_velgrad(vel_grad, elem_node_gids, @@ -76,7 +74,7 @@ void FEA_Module_SGH::update_state(const DCArrayKokkos &material, elem_vol(elem_gid), elem_gid, rk_level); - + StrengthParent* strength_model = elem_strength(elem_gid).model; // --- call strength model --- @@ -99,7 +97,6 @@ void FEA_Module_SGH::update_state(const DCArrayKokkos &material, rk_alpha, cycle, rk_level); - } // end logical on hyper strength model EOSParent* eos_model = elem_eos(elem_gid).model; @@ -114,7 +111,7 @@ void FEA_Module_SGH::update_state(const DCArrayKokkos &material, elem_user_output_vars, elem_sspd, elem_den(elem_gid), - elem_sie(rk_level,elem_gid)); + elem_sie(rk_level, elem_gid)); // --- Sound speed --- eos_model->calc_sound_speed(elem_pres, @@ -126,79 +123,73 @@ void FEA_Module_SGH::update_state(const DCArrayKokkos &material, elem_user_output_vars, elem_sspd, elem_den(elem_gid), - elem_sie(rk_level,elem_gid)); - + elem_sie(rk_level, elem_gid)); }); // end parallel for Kokkos::fence(); - + return; - } // end method to update state - - -void FEA_Module_SGH::update_state2D(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &elem_den, - DViewCArrayKokkos &elem_pres, - DViewCArrayKokkos &elem_stress, - DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_mass, - const DViewCArrayKokkos &elem_mat_id, - const double rk_alpha, - const size_t cycle - ){ - +void FEA_Module_SGH::update_state2D(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_den, + DViewCArrayKokkos& elem_pres, + DViewCArrayKokkos& elem_stress, + DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& elem_mat_id, + const double rk_alpha, + const size_t cycle + ) +{ const size_t rk_level = rk_num_bins - 1; - int num_dims = num_dim; - + int num_dims = num_dim; + // loop over all the elements in the mesh - FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { - + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { const size_t num_dims = num_dims; const size_t num_nodes_in_elem = num_nodes_in_elem; // cut out the node_gids for this element - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), num_nodes_in_elem); - + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), num_nodes_in_elem); + // --- Density --- - elem_den(elem_gid) = elem_mass(elem_gid)/elem_vol(elem_gid); - + elem_den(elem_gid) = elem_mass(elem_gid) / elem_vol(elem_gid); + size_t mat_id = elem_mat_id(elem_gid); - //initialize elem pressure + // initialize elem pressure elem_pres(elem_gid) = 0; - + // --- Stress --- // hyper elastic plastic model - if(material(mat_id).strength_type == STRENGTH_TYPE::hyper){ - + if (material(mat_id).strength_type == STRENGTH_TYPE::hyper) + { // cut out the node_gids for this element - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), num_nodes_in_elem); - + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), num_nodes_in_elem); + // --- Density --- - elem_den(elem_gid) = elem_mass(elem_gid)/elem_vol(elem_gid); - + elem_den(elem_gid) = elem_mass(elem_gid) / elem_vol(elem_gid); + // corner area normals double area_array[8]; - ViewCArrayKokkos area(area_array, num_nodes_in_elem, num_dims); - + ViewCArrayKokkos area(area_array, num_nodes_in_elem, num_dims); + // velocity gradient double vel_grad_array[4]; - ViewCArrayKokkos vel_grad(vel_grad_array, num_dims, num_dims); - + ViewCArrayKokkos vel_grad(vel_grad_array, num_dims, num_dims); + // get the B matrix which are the OUTWARD corner area normals get_bmatrix(area, elem_gid, node_coords, elem_node_gids, rk_level); - - + // --- Calculate the velocity gradient --- get_velgrad(vel_grad, elem_node_gids, @@ -207,7 +198,7 @@ void FEA_Module_SGH::update_state2D(const DCArrayKokkos &material, elem_vol(elem_gid), elem_gid, rk_level); - + StrengthParent* strength_model = elem_strength(elem_gid).model; // --- call strength model --- @@ -230,7 +221,6 @@ void FEA_Module_SGH::update_state2D(const DCArrayKokkos &material, rk_alpha, cycle, rk_level); - } // end logical on hyper strength model EOSParent* eos_model = elem_eos(elem_gid).model; @@ -245,7 +235,7 @@ void FEA_Module_SGH::update_state2D(const DCArrayKokkos &material, elem_user_output_vars, elem_sspd, elem_den(elem_gid), - elem_sie(rk_level,elem_gid)); + elem_sie(rk_level, elem_gid)); // --- Sound speed --- eos_model->calc_sound_speed(elem_pres, @@ -257,14 +247,9 @@ void FEA_Module_SGH::update_state2D(const DCArrayKokkos &material, elem_user_output_vars, elem_sspd, elem_den(elem_gid), - elem_sie(rk_level,elem_gid)); - + elem_sie(rk_level, elem_gid)); }); // end parallel for Kokkos::fence(); - + return; - } // end method to update state - - - diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/setup_sgh.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/setup_sgh.cpp new file mode 100644 index 000000000..dd85ffe5e --- /dev/null +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/setup_sgh.cpp @@ -0,0 +1,958 @@ +/* ---------------------------------------------------------------------------- + Setup SGH solver data +------------------------------------------------------------------------------- */ +#include "matar.h" +#include "state.h" +#include "FEA_Module_SGH.h" +#include "Simulation_Parameters/Simulation_Parameters_Explicit.h" +#include "Simulation_Parameters/FEA_Module/SGH_Parameters.h" +#include "Explicit_Solver.h" + +void FEA_Module_SGH::setup() +{ + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + const size_t num_fills = simparam->regions.size(); + const size_t rk_num_bins = simparam->dynamic_options.rk_num_bins; + const size_t num_bcs = module_params->boundary_conditions.size(); + const size_t num_materials = simparam->materials.size(); + const int num_dim = simparam->num_dims; + const size_t num_lcs = module_params->loading.size(); + if (num_lcs) + { + have_loading_conditions = true; + } + + // --------------------------------------------------------------------- + // obtain mesh data + // --------------------------------------------------------------------- + sgh_interface_setup(node_interface, elem_interface, corner_interface); + mesh->build_corner_connectivity(); + // debug print of corner ids + /* + if(myrank==1){ + for(int i = 0; i < mesh.num_nodes; i++){ + + // loop over all corners around the node and calculate the nodal force + for (size_t corner_lid=0; corner_lidgetGlobalElement(i) << " " << i << " " << all_node_map->getLocalElement(all_node_map->getGlobalElement(i)) << " " << corner_gid << " " << std::endl; + + } // end for corner_lid + //std::cout << explicit_solver_pointer->all_node_map->getGlobalElement(i) << " " << node_force[0] << " " << node_force[1] << " " << node_force[2] << std::endl; + //std::cout << explicit_solver_pointer->all_node_map->getGlobalElement(i) << " " << node_mass(i) << std::endl; + } + } + */ + /* + if(myrank==1){ + for(int i = 0; i < mesh.num_elems; i++){ + + // loop over all corners around the node and calculate the nodal force + for (size_t corner_lid=0; corner_lidgetGlobalElement(mesh.nodes_in_elem(i, corner_lid)) <<" " << corner_gid << " " << std::endl; + + } // end for corner_lid + //std::cout << explicit_solver_pointer->all_node_map->getGlobalElement(i) << " " << node_force[0] << " " << node_force[1] << " " << node_force[2] << std::endl; + //std::cout << explicit_solver_pointer->all_node_map->getGlobalElement(i) << " " << node_mass(i) << std::endl; + } + } + */ + mesh->build_elem_elem_connectivity(); + mesh->num_bdy_patches = nboundary_patches; + if (num_dim == 2) + { + mesh->build_patch_connectivity(); + mesh->build_node_node_connectivity(); + } + + // --------------------------------------------------------------------- + // allocate memory + // --------------------------------------------------------------------- + + // shorthand names + const size_t num_nodes = mesh->num_nodes; + const size_t num_elems = mesh->num_elems; + const size_t num_corners = mesh->num_corners; + + // --- make dual views of data on CPU and GPU --- + // Notes: + // Instead of using a struct of dual types like the mesh type, + // individual dual views will be made for all the state + // variables. The motivation is to reduce memory movement + // when passing state into a function. Passing a struct by + // reference will copy the meta data and pointers for the + // variables held inside the struct. Since all the mesh + // variables are typically used by most functions, a single + // mesh struct or passing the arrays will be roughly equivalent + // for memory movement. + + // create Dual Views of the individual node struct variables + node_coords = DViewCArrayKokkos(node_interface.coords.get_kokkos_dual_view().view_host().data(), rk_num_bins, num_nodes, num_dim); + node_vel = DViewCArrayKokkos(node_interface.vel.get_kokkos_dual_view().view_host().data(), rk_num_bins, num_nodes, num_dim); + node_mass = DViewCArrayKokkos(node_interface.mass.get_kokkos_dual_view().view_host().data(), num_nodes); + + // create Dual Views of the individual elem struct variables + elem_den = DViewCArrayKokkos(&elem_interface.den(0), num_elems); + elem_pres = DViewCArrayKokkos(&elem_interface.pres(0), num_elems); + elem_stress = DViewCArrayKokkos(&elem_interface.stress(0, 0, 0, 0), rk_num_bins, num_elems, 3, 3); // always 3D even in 2D-RZ + elem_sspd = DViewCArrayKokkos(&elem_interface.sspd(0), num_elems); + elem_sie = DViewCArrayKokkos(&elem_interface.sie(0, 0), rk_num_bins, num_elems); + elem_vol = DViewCArrayKokkos(&elem_interface.vol(0), num_elems); + elem_div = DViewCArrayKokkos(&elem_interface.div(0), num_elems); + elem_mass = DViewCArrayKokkos(&elem_interface.mass(0), num_elems); + elem_mat_id = DViewCArrayKokkos(&elem_interface.mat_id(0), num_elems); + + // create Dual Views of the corner struct variables + corner_force = DViewCArrayKokkos(&corner_interface.force(0, 0), num_corners, num_dim); + corner_mass = DViewCArrayKokkos(&corner_interface.mass(0), num_corners); + + // allocate elem_vel_grad + elem_vel_grad = DCArrayKokkos(num_elems, 3, 3); + + // allocate material models + elem_eos = DCArrayKokkos(num_elems); + elem_strength = DCArrayKokkos(num_elems); + + // --------------------------------------------------------------------- + // calculate geometry + // --------------------------------------------------------------------- + node_coords.update_device(); + Kokkos::fence(); + + get_vol(); + + // FEA_Module boundary condition variable + num_boundary_conditions = num_bcs; + + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos mat_fill = simparam->mat_fill; + const DCArrayKokkos material = simparam->material; + global_vars = simparam->global_vars; + state_vars = DCArrayKokkos(rnum_elem, simparam->max_num_state_vars); + elem_user_output_vars = DCArrayKokkos(rnum_elem, simparam->output_options.max_num_user_output_vars); + + // --- calculate bdy sets ---// + mesh->num_nodes_in_patch = 2 * (num_dim - 1); // 2 (2D) or 4 (3D) + mesh->num_patches_in_elem = 2 * num_dim; // 4 (2D) or 6 (3D) + mesh->init_bdy_sets(num_bcs); + num_bdy_sets = mesh->num_bdy_sets; + printf("Num BC's = %lu\n", num_bcs); + + // patch ids in bdy set + bdy_patches_in_set = mesh->bdy_patches_in_set; + if (num_dim == 2) + { + bdy_nodes = mesh->bdy_nodes; + } + + // tag boundary patches in the set + tag_bdys(boundary, *mesh, node_coords); + + build_boundry_node_sets(*mesh); + + // node ids in bdy_patch set + bdy_nodes_in_set = mesh->bdy_nodes_in_set; + num_bdy_nodes_in_set = mesh->num_bdy_nodes_in_set; + + // assign mesh views needed by the FEA module + + // elem ids in elem + elems_in_elem = mesh->elems_in_elem; + num_elems_in_elem = mesh->num_elems_in_elem; + + // corners + num_corners_in_node = mesh->num_corners_in_node; + corners_in_node = mesh->corners_in_node; + corners_in_elem = mesh->corners_in_elem; + + // elem-node conn & node-node conn + elems_in_node = mesh->elems_in_node; + if (num_dim == 2) + { + nodes_in_node = mesh->nodes_in_node; + num_nodes_in_node = mesh->num_nodes_in_node; + // patch conn + + patches_in_elem = mesh->patches_in_elem; + nodes_in_patch = mesh->nodes_in_patch; + elems_in_patch = mesh->elems_in_patch; + } + + // loop over BCs + for (size_t this_bdy = 0; this_bdy < num_bcs; this_bdy++) + { + RUN_CLASS({ + printf("Boundary Condition number %lu \n", this_bdy); + printf(" Num bdy patches in this set = %lu \n", bdy_patches_in_set.stride(this_bdy)); + printf(" Num bdy nodes in this set = %lu \n", bdy_nodes_in_set.stride(this_bdy)); + }); + Kokkos::fence(); + } // end for + + // elem_mat_id needs to be initialized before initialization of material models + for (int f_id = 0; f_id < num_fills; f_id++) + { + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { + elem_mat_id(elem_gid) = mat_fill(f_id).material_id; + }); + } + elem_mat_id.update_host(); + + // function for initializing state_vars + init_state_vars(material, + elem_mat_id, + state_vars, + global_vars, + elem_user_output_vars, + rnum_elem); + + // initialize strength model + init_strength_model(elem_strength, + material, + elem_mat_id, + state_vars, + global_vars, + elem_user_output_vars, + rnum_elem); + + // initialize eos model + init_eos_model(elem_eos, + material, + elem_mat_id, + state_vars, + global_vars, + elem_user_output_vars, + rnum_elem); + + // --- apply the fill instructions over each of the Elements---// + + // initialize if topology optimization is used + if (simparam->topology_optimization_on) + { + for (int elem_id = 0; elem_id < rnum_elem; elem_id++) + { + relative_element_densities.host(elem_id) = 1; + } // for + relative_element_densities.update_device(); + } + + // loop over the fill instructures + for (int f_id = 0; f_id < num_fills; f_id++) + { + // parallel loop over elements in mesh + // for (size_t elem_gid = 0; elem_gid <= rnum_elem; elem_gid++) { + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { + // calculate the coordinates and radius of the element + double elem_coords[3]; // note:initialization with a list won't work + elem_coords[0] = 0.0; + elem_coords[1] = 0.0; + elem_coords[2] = 0.0; + + // get the coordinates of the element center + for (int node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + elem_coords[0] += node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), 0); + elem_coords[1] += node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), 1); + if (num_dim == 3) + { + elem_coords[2] += node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), 2); + } + else + { + elem_coords[2] = 0.0; + } + } // end loop over nodes in element + elem_coords[0] = elem_coords[0] / num_nodes_in_elem; + elem_coords[1] = elem_coords[1] / num_nodes_in_elem; + elem_coords[2] = elem_coords[2] / num_nodes_in_elem; + + // default is not to fill the element + bool fill_this = mat_fill(f_id).volume.contains(elem_coords); + + // paint the material state on the element + if (fill_this) + { + // density + elem_den(elem_gid) = mat_fill(f_id).den; + + // mass + elem_mass(elem_gid) = elem_den(elem_gid) * elem_vol(elem_gid); + + // specific internal energy + elem_sie(rk_level, elem_gid) = mat_fill(f_id).sie; + + size_t mat_id = elem_mat_id(elem_gid); // short name + + // --- stress tensor --- + // always 3D even for 2D-RZ + for (size_t i = 0; i < 3; i++) + { + for (size_t j = 0; j < 3; j++) + { + elem_stress(rk_level, elem_gid, i, j) = 0.0; + } + } // end for + + // short form for clean code + EOSParent* eos_model = elem_eos(elem_gid).model; + // --- Pressure --- + eos_model->calc_pressure(elem_pres, + elem_stress, + elem_gid, + elem_mat_id(elem_gid), + state_vars, + global_vars, + elem_user_output_vars, + elem_sspd, + elem_den(elem_gid), + elem_sie(rk_level, elem_gid)); + + // --- Sound speed --- + eos_model->calc_sound_speed(elem_pres, + elem_stress, + elem_gid, + elem_mat_id(elem_gid), + state_vars, + global_vars, + elem_user_output_vars, + elem_sspd, + elem_den(elem_gid), + elem_sie(rk_level, elem_gid)); + + // loop over the nodes of this element and apply velocity + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + // get the mesh node index + size_t node_gid = nodes_in_elem(elem_gid, node_lid); + + // --- Velocity --- + switch (mat_fill(f_id).velocity) + { + case VELOCITY_TYPE::cartesian: + { + node_vel(rk_level, node_gid, 0) = mat_fill(f_id).u; + node_vel(rk_level, node_gid, 1) = mat_fill(f_id).v; + if (num_dim == 3) + { + node_vel(rk_level, node_gid, 2) = mat_fill(f_id).w; + } + + break; + } + case VELOCITY_TYPE::radial: + { + // Setting up cylindrical + double dir[2]; + dir[0] = 0.0; + dir[1] = 0.0; + double radius_val = 0.0; + + for (int dim = 0; dim < 2; dim++) + { + dir[dim] = node_coords(rk_level, node_gid, dim); + radius_val += node_coords(rk_level, node_gid, dim) * node_coords(rk_level, node_gid, dim); + } // end for + radius_val = sqrt(radius_val); + + for (int dim = 0; dim < 2; dim++) + { + if (radius_val > 1.0e-14) + { + dir[dim] /= (radius_val); + } + else + { + dir[dim] = 0.0; + } + } // end for + + node_vel(rk_level, node_gid, 0) = mat_fill(f_id).speed * dir[0]; + node_vel(rk_level, node_gid, 1) = mat_fill(f_id).speed * dir[1]; + if (num_dim == 3) + { + node_vel(rk_level, node_gid, 2) = 0.0; + } + + break; + } + case VELOCITY_TYPE::spherical: + { + // Setting up spherical + double dir[3]; + dir[0] = 0.0; + dir[1] = 0.0; + dir[2] = 0.0; + double radius_val = 0.0; + + for (int dim = 0; dim < 3; dim++) + { + dir[dim] = node_coords(rk_level, node_gid, dim); + radius_val += node_coords(rk_level, node_gid, dim) * node_coords(rk_level, node_gid, dim); + } // end for + radius_val = sqrt(radius_val); + + for (int dim = 0; dim < 3; dim++) + { + if (radius_val > 1.0e-14) + { + dir[dim] /= (radius_val); + } + else + { + dir[dim] = 0.0; + } + } // end for + + node_vel(rk_level, node_gid, 0) = mat_fill(f_id).speed * dir[0]; + node_vel(rk_level, node_gid, 1) = mat_fill(f_id).speed * dir[1]; + if (num_dim == 3) + { + node_vel(rk_level, node_gid, 2) = mat_fill(f_id).speed * dir[2]; + } + + break; + } + case VELOCITY_TYPE::radial_linear: + { + break; + } + case VELOCITY_TYPE::spherical_linear: + { + break; + } + case VELOCITY_TYPE::tg_vortex: + { + node_vel(rk_level, node_gid, 0) = sin(PI * node_coords(rk_level, node_gid, 0)) * cos(PI * node_coords(rk_level, node_gid, 1)); + node_vel(rk_level, node_gid, 1) = -1.0 * cos(PI * node_coords(rk_level, node_gid, 0)) * sin(PI * node_coords(rk_level, node_gid, 1)); + if (num_dim == 3) + { + node_vel(rk_level, node_gid, 2) = 0.0; + } + + break; + } + } // end of switch + } // end loop over nodes of element + + if (mat_fill(f_id).velocity == VELOCITY_TYPE::tg_vortex) + { + elem_pres(elem_gid) = 0.25 * (cos(2.0 * PI * elem_coords[0]) + cos(2.0 * PI * elem_coords[1]) ) + 1.0; + + // p = rho*ie*(gamma - 1) + size_t mat_id = f_id; + double gamma = global_vars(mat_id, 0); // gamma value + elem_sie(rk_level, elem_gid) = + elem_pres(elem_gid) / (mat_fill(f_id).den * (gamma - 1.0)); + } // end if + } // end if fill + }); // end FOR_ALL_CLASS element loop + Kokkos::fence(); + } // end for loop over fills + + // apply BC's to velocity + FEA_Module_SGH::boundary_velocity(*mesh, boundary, node_vel); + + // calculate the corner massess if 2D + if (num_dim == 2) + { + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { + // facial area of the corners + double corner_areas_array[4]; + + ViewCArrayKokkos corner_areas(&corner_areas_array[0], 4); + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 4); + + get_area_weights2D(corner_areas, + elem_gid, + node_coords, + elem_node_gids, + rk_level); + + // loop over the corners of the element and calculate the mass + for (size_t corner_lid = 0; corner_lid < 4; corner_lid++) + { + size_t corner_gid = corners_in_elem(elem_gid, corner_lid); + corner_mass(corner_gid) = corner_areas(corner_lid) * elem_den(elem_gid); // node radius is added later + } // end for over corners + }); + } // end of + + // calculate the nodal mass + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + node_mass(node_gid) = 0.0; + + if (num_dim == 3) + { + for (size_t elem_lid = 0; elem_lid < num_corners_in_node(node_gid); elem_lid++) + { + size_t elem_gid = elems_in_node(node_gid, elem_lid); + node_mass(node_gid) += 1.0 / 8.0 * elem_mass(elem_gid); + } // end for elem_lid + } // end if dims=3 + else + { + // 2D-RZ + for (size_t corner_lid = 0; corner_lid < num_corners_in_node(node_gid); corner_lid++) + { + size_t corner_gid = corners_in_node(node_gid, corner_lid); + node_mass(node_gid) += corner_mass(corner_gid); // sans the radius so it is areal node mass + + corner_mass(corner_gid) *= node_coords(rk_level, node_gid, 1); // true corner mass now + } // end for elem_lid + } // end else + }); // end FOR_ALL_CLASS + Kokkos::fence(); + + // current interface has differing mass arrays; this equates them until we unify memory + // view scope + if (simparam->topology_optimization_on || simparam->shape_optimization_on || simparam->num_dims == 2) + { + { + vec_array node_mass_interface = node_masses_distributed->getLocalView(Tpetra::Access::ReadWrite); + + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + node_mass_interface(node_gid, 0) = node_mass(node_gid); + }); // end parallel for + } // end view scope + Kokkos::fence(); + // communicate ghost densities + comm_node_masses(); + + // this is forcing a copy to the device + // view scope + { + vec_array ghost_node_mass_interface = ghost_node_masses_distributed->getLocalView(Tpetra::Access::ReadWrite); + + FOR_ALL_CLASS(node_gid, nlocal_nodes, nall_nodes, { + node_mass(node_gid) = ghost_node_mass_interface(node_gid - nlocal_nodes, 0); + }); // end parallel for + } // end view scope + Kokkos::fence(); + } // endif + + // initialize if topology optimization is used + if (simparam->topology_optimization_on || simparam->shape_optimization_on) + { + init_assembly(); + assemble_matrix(); + } + + // update host copies of arrays modified in this function + elem_den.update_host(); + elem_mass.update_host(); + elem_sie.update_host(); + elem_stress.update_host(); + elem_pres.update_host(); + elem_sspd.update_host(); + + return; +} // end of setup + +// ----------------------------------------------------------------------------- +// Interfaces read in data with the SGH solver data; currently a hack to streamline +// ------------------------------------------------------------------------------ +void FEA_Module_SGH::sgh_interface_setup(node_t& node, + elem_t& elem, + corner_t& corner) +{ + const size_t num_dim = simparam->num_dims; + const size_t rk_num_bins = simparam->dynamic_options.rk_num_bins; + + num_nodes_in_elem = 1; + for (int dim = 0; dim < num_dim; dim++) + { + num_nodes_in_elem *= 2; + } + + // --- Read in the nodes in the mesh --- + + nall_nodes = Explicit_Solver_Pointer_->nall_nodes; + int myrank = Explicit_Solver_Pointer_->myrank; + int nranks = Explicit_Solver_Pointer_->nranks; + // printf("Num nodes assigned to MPI rank %lu is %lu\n" , myrank, nall_nodes); + + // intialize node variables + mesh->initialize_nodes(nall_nodes); + mesh->initialize_local_nodes(Explicit_Solver_Pointer_->nlocal_nodes); + node.initialize(rk_num_bins, nall_nodes, num_dim); + // std::cout << "Bin counts " << rk_num_bins << " Node counts " << nall_nodes << " Num dim " << num_dim << std::endl; + + // view scope + { + host_vec_array interface_node_coords = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + // save node data to node.coords + // std::cout << "NODE DATA ON RANK " << myrank << std::endl; + if (num_dim == 2) + { + for (int inode = 0; inode < nall_nodes; inode++) + { + // std::cout << "Node index " << inode+1 << " "; + node.coords.host(0, inode, 0) = interface_node_coords(inode, 0); + // std::cout << host_node_coords_state(0,inode,0)+1<< " "; + node.coords.host(0, inode, 1) = interface_node_coords(inode, 1); + // std::cout << host_node_coords_state(0,inode,1)+1<< " "; + } + } + else if (num_dim == 3) + { + for (int inode = 0; inode < nall_nodes; inode++) + { + // std::cout << "Node index " << inode+1 << " "; + node.coords.host(0, inode, 0) = interface_node_coords(inode, 0); + // std::cout << host_node_coords_state(0,inode,0)+1<< " "; + node.coords.host(0, inode, 1) = interface_node_coords(inode, 1); + // std::cout << host_node_coords_state(0,inode,1)+1<< " "; + + node.coords.host(0, inode, 2) = interface_node_coords(inode, 2); + // std::cout << host_node_coords_state(0,inode,2)+1<< std::endl; + } + } + } // end view scope + // --- read in the elements in the mesh --- + + rnum_elem = Explicit_Solver_Pointer_->rnum_elem; + // printf("Num elems assigned to MPI rank %lu is %lu\n" , myrank, rnum_elem); + + // intialize elem variables + mesh->initialize_elems(rnum_elem, num_dim); + elem.initialize(rk_num_bins, nall_nodes, 3); // always 3D here, even for 2D + nodes_in_elem = mesh->nodes_in_elem; + // save data to nodes_in_elem.host + // CArrayKokkos host_mesh_nodes_in_elem(rnum_elem, num_nodes_in_elem); + // view scope + { + host_elem_conn_array interface_nodes_in_elem = Explicit_Solver_Pointer_->global_nodes_in_elem_distributed->getLocalView(Tpetra::Access::ReadWrite); + // save node data to node.coords + // std::cout << "ELEMENT CONNECTIVITY ON RANK " << myrank << std::endl; + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + // std::cout << "Element index " << ielem+1 << " "; + for (int inode = 0; inode < num_nodes_in_elem; inode++) + { + nodes_in_elem.host(ielem, inode) = Explicit_Solver_Pointer_->all_node_map->getLocalElement(interface_nodes_in_elem(ielem, inode)); + // debug print + // std::cout << nodes_in_elem.get_kokkos_dual_view().h_view(ielem*num_nodes_in_elem + inode)+1<< " "; + } + // std::cout << std::endl; + } + } + // update device side + nodes_in_elem.update_device(); + + // debug print + + // CArrayKokkos device_mesh_nodes_in_elem(rnum_elem, num_nodes_in_elem); + // device_mesh_nodes_in_elem.get_kokkos_view() = nodes_in_elem.get_kokkos_dual_view().d_view; + // host_mesh_nodes_in_elem.get_kokkos_view() = nodes_in_elem.get_kokkos_dual_view().view_host(); + /* + if(myrank==1){ + std::cout << "ELEMENT CONNECTIVITY ON RANK 1 in LOCAL INDICES" << myrank << std::endl; + for(int ielem = 0; ielem < rnum_elem; ielem++){ + std::cout << "Element index " << ielem+1 << " "; + for(int inode = 0; inode < num_nodes_in_elem; inode++){ + //debug print + //device_mesh_nodes_in_elem(ielem,inode) = Explicit_Solver_Pointer_->all_node_map->getLocalElement(interface_nodes_in_elem(ielem,inode)); + std::cout << nodes_in_elem(ielem, inode)+1<< " "; + } + std::cout << std::endl; + } + } + */ + /* + std::cout.flush(); + if(myrank==1){ + std::cout << "ELEMENT CONNECTIVITY ON RANK 1 in GLOBAL INDICES" << myrank << std::endl; + std::cout << "local node index of global index 275 on rank 1 " << Explicit_Solver_Pointer_->all_node_map->getLocalElement(275) << std::endl; + for(int ielem = 0; ielem < rnum_elem; ielem++){ + std::cout << ielem << " "; + for(int inode = 0; inode < num_nodes_in_elem; inode++){ + //debug print + //device_mesh_nodes_in_elem(ielem,inode) = Explicit_Solver_Pointer_->all_node_map->getLocalElement(interface_nodes_in_elem(ielem,inode)); + std::cout << Explicit_Solver_Pointer_->all_node_map->getGlobalElement(nodes_in_elem(ielem, inode))<< " "; + } + std::cout << std::endl; + } + } + std::cout.flush(); + */ + /* + size_t nall_nodes = Explicit_Solver_Pointer_->nall_nodes; + node.all_coords = DCArrayKokkos (rk_num_bins, nall_nodes, num_dim); + node.all_vel = DCArrayKokkos (rk_num_bins, nall_nodes, num_dim); + node.all_mass = DCArrayKokkos (nall_nodes); + + //save all data (nlocal +nghost) + CArrayKokkos host_all_node_coords_state(rk_num_bins, nall_nodes, num_dim); + host_vec_array interface_all_node_coords = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); + host_all_node_coords_state.get_kokkos_view() = node.all_coords.get_kokkos_dual_view().view_host(); + //host_node_coords_state = CArrayKokkos(rk_num_bins, nall_nodes, num_dim); + //host_all_node_coords_state.get_kokkos_view() = Kokkos::View("debug", rk_num_bins*nall_nodes*num_dim); + //save node data to node.coords + + //std::cout << "ALL NODE DATA ON RANK " << myrank << std::endl; + for(int inode = 0; inode < nall_nodes; inode++){ + //std::cout << "Node index " << inode+1 << " "; + node.all_coords.host(0,inode,0) = interface_all_node_coords(inode,0); + //std::cout << host_all_node_coords_state(0,inode,0)+1<< " "; + node.all_coords.host(0,inode,1) = interface_all_node_coords(inode,1); + //std::cout << host_all_node_coords_state(0,inode,1)+1<< " "; + node.all_coords.host(0,inode,2) = interface_all_node_coords(inode,2); + //std::cout << host_all_node_coords_state(0,inode,2)+1<< std::endl; + } + */ + + // save the node coords to the current RK value + for (size_t node_gid = 0; node_gid < nall_nodes; node_gid++) + { + for (int rk = 1; rk < rk_num_bins; rk++) + { + for (int dim = 0; dim < num_dim; dim++) + { + node.coords.host(rk, node_gid, dim) = node.coords.host(0, node_gid, dim); + } // end for dim + } // end for rk + } // end parallel for + + /* + // save the node coords to the current RK value + for (size_t node_gid=0; node_gidinitialize_corners(num_corners); + corner.initialize(num_corners, num_dim); + + /* + for(int inode = 0; inode < nall_nodes; inode++){ + std::cout << "Node index " << inode+1 << " "; + for(int rk=0; rknboundary_patches; + int num_nodes_in_patch = mesh.num_nodes_in_patch; + num_bdy_nodes_in_set = mesh.num_bdy_nodes_in_set = DCArrayKokkos(num_bdy_sets, "num_bdy_nodes_in_set"); + CArrayKokkos temp_count_num_bdy_nodes_in_set(num_bdy_sets, nall_nodes, "temp_count_num_bdy_nodes_in_set"); + + DynamicRaggedRightArrayKokkos temp_nodes_in_set(mesh.num_bdy_sets, nboundary_patches * mesh.num_nodes_in_patch, "temp_nodes_in_set"); + + // Parallel loop over boundary sets on device + FOR_ALL_CLASS(bdy_set, 0, num_bdy_sets, { + // finde the number of patches_in_set + size_t num_bdy_patches_in_set = bdy_patches_in_set.stride(bdy_set); + + num_bdy_nodes_in_set(bdy_set) = 0; + + // Loop over boundary patches in boundary set + for (size_t bdy_patch_gid = 0; bdy_patch_gid < num_bdy_patches_in_set; bdy_patch_gid++) + { + // get the global id for this boundary patch + size_t patch_gid = bdy_patches_in_set(bdy_set, bdy_patch_gid); + + // apply boundary condition at nodes on boundary + for (size_t node_lid = 0; node_lid < num_nodes_in_patch; node_lid++) + { + size_t node_gid = Local_Index_Boundary_Patches(patch_gid, node_lid); + + temp_count_num_bdy_nodes_in_set(bdy_set, node_gid) = -1; + } // end for node_lid + } // end for bdy_patch_gid + + // Loop over boundary patches in boundary set + for (size_t bdy_patch_gid = 0; bdy_patch_gid < num_bdy_patches_in_set; bdy_patch_gid++) + { + // get the global id for this boundary patch + size_t patch_gid = bdy_patches_in_set(bdy_set, bdy_patch_gid); + + // apply boundary condition at nodes on boundary + for (size_t node_lid = 0; node_lid < num_nodes_in_patch; node_lid++) + { + size_t node_gid = Local_Index_Boundary_Patches(patch_gid, node_lid); + + if (temp_count_num_bdy_nodes_in_set(bdy_set, node_gid) == -1) + { + size_t num_saved = num_bdy_nodes_in_set(bdy_set); + + num_bdy_nodes_in_set(bdy_set)++; + + // replace -1 with node_gid to denote the node was already saved + temp_count_num_bdy_nodes_in_set(bdy_set, node_gid) = node_gid; + + // increment the number of saved nodes, create memory + temp_nodes_in_set.stride(bdy_set)++; + temp_nodes_in_set(bdy_set, num_saved) = node_gid; + } // end if + } // end for node_lid + } // end for bdy_patch_gid + }); // end FOR_ALL_CLASS bdy_set + Kokkos::fence(); + + // allocate the RaggedRight bdy_nodes_in_set array + bdy_nodes_in_set = mesh.bdy_nodes_in_set = RaggedRightArrayKokkos(mesh.num_bdy_nodes_in_set, "bdy_nodes_in_set"); + + FOR_ALL_CLASS(bdy_set, 0, num_bdy_sets, { + // Loop over boundary patches in boundary set + for (size_t bdy_node_lid = 0; bdy_node_lid < num_bdy_nodes_in_set(bdy_set); bdy_node_lid++) + { + // save the bdy_node_gid + bdy_nodes_in_set(bdy_set, bdy_node_lid) = temp_nodes_in_set(bdy_set, bdy_node_lid); + } // end for + }); // end FOR_ALL_CLASS bdy_set + + // update the host side for the number nodes in a bdy_set + num_bdy_nodes_in_set.update_host(); + + return; +} // end method to build boundary nodes + +/* ---------------------------------------------------------------------------- + Initialize sets of element boundary surfaces and arrays for input conditions +------------------------------------------------------------------------------- */ +void FEA_Module_SGH::init_boundaries() +{ + max_boundary_sets = module_params->boundary_conditions.size(); + int num_dim = simparam->num_dims; + + // set the number of boundary sets + if (myrank == 0) + { + std::cout << "building boundary sets " << std::endl; + } + + // initialize to 1 since there must be at least 1 boundary set anyway; read in may occure later + if (max_boundary_sets == 0) + { + max_boundary_sets = 1; + } + // std::cout << "NUM BOUNDARY CONDITIONS ON RANK " << myrank << " FOR INIT " << num_boundary_conditions <(nall_nodes * num_dim, "Node_DOF_Boundary_Condition_Type"); + + // initialize + for (int init = 0; init < nall_nodes * num_dim; init++) + { + Node_DOF_Boundary_Condition_Type(init) = NONE; + } + + Number_DOF_BCS = 0; +} + +/* ---------------------------------------------------------------------- + initialize storage for element boundary surfaces corresponding to user BCs +------------------------------------------------------------------------- */ +void FEA_Module_SGH::init_boundary_sets(int num_sets) +{ + if (num_sets == 0) + { + std::cout << " Warning: number of boundary conditions = 0"; + return; + } + // initialize maximum + max_boundary_sets = num_sets; + // std::cout << " DEBUG PRINT "<(num_sets, "Boundary_Condition_Type_List"); + NBoundary_Condition_Patches = CArrayKokkos(num_sets, "NBoundary_Condition_Patches"); + // std::cout << "NBOUNDARY PATCHES ON RANK " << myrank << " FOR INIT IS " << nboundary_patches <(num_sets, nboundary_patches, "Boundary_Condition_Patches"); + + // initialize data + for (int iset = 0; iset < num_sets; iset++) + { + NBoundary_Condition_Patches(iset) = 0; + } + + // initialize + for (int ibdy = 0; ibdy < num_sets; ibdy++) + { + Boundary_Condition_Type_List(ibdy) = NONE; + } +} + +/* ---------------------------------------------------------------------------- + Grow boundary conditions sets of element boundary surfaces +------------------------------------------------------------------------------- */ + +void FEA_Module_SGH::grow_boundary_sets(int num_sets) +{ + int num_dim = simparam->num_dims; + + if (num_sets == 0) + { + std::cout << " Warning: number of boundary conditions being set to 0"; + return; + } + + // std::cout << " DEBUG PRINT "< max_boundary_sets) + { + // temporary storage for previous data + CArrayKokkos Temp_Boundary_Condition_Type_List = Boundary_Condition_Type_List; + CArrayKokkos Temp_NBoundary_Condition_Patches = NBoundary_Condition_Patches; + CArrayKokkos Temp_Boundary_Condition_Patches = Boundary_Condition_Patches; + + max_boundary_sets = num_sets + 5; // 5 is an arbitrary buffer + Boundary_Condition_Type_List = CArrayKokkos(max_boundary_sets, "Boundary_Condition_Type_List"); + NBoundary_Condition_Patches = CArrayKokkos(max_boundary_sets, "NBoundary_Condition_Patches"); + // std::cout << "NBOUNDARY PATCHES ON RANK " << myrank << " FOR GROW " << nboundary_patches <(max_boundary_sets, nboundary_patches, "Boundary_Condition_Patches"); + + // copy previous data back over + // std::cout << "NUM BOUNDARY CONDITIONS ON RANK " << myrank << " FOR COPY " << max_boundary_sets < +#include +#include +#include +#include +#include // fmin, fmax, abs note: fminl is long +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include "Tpetra_Details_makeColMap.hpp" +#include "Tpetra_Details_DefaultTypes.hpp" +#include "Tpetra_Details_FixedHashTable.hpp" +#include "Tpetra_Import.hpp" +#include "Tpetra_Import_Util2.hpp" + +#include "elements.h" +#include "swage.h" +#include "matar.h" +#include "utilities.h" +#include "node_combination.h" +#include "FEA_Module_SGH.h" +#include "Explicit_Solver.h" +#include "Simulation_Parameters/Simulation_Parameters_Explicit.h" +#include "Simulation_Parameters/FEA_Module/SGH_Parameters.h" + +// optimization +#include "ROL_Solver.hpp" +#include "Kinetic_Energy_Minimize.h" + +/* ---------------------------------------------------------------------- + Compute new system response due to the design variable update +------------------------------------------------------------------------- */ + +void FEA_Module_SGH::update_forward_solve(Teuchos::RCP zp) +{ + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + // local variable for host view in the dual view + int num_dim = simparam->num_dims; + int nodes_per_elem = max_nodes_per_element; + int local_node_index, current_row, current_column; + int max_stride = 0; + int current_module_index; + size_t access_index, row_access_index, row_counter; + GO global_index, global_dof_index; + LO local_dof_index; + + const size_t num_fills = simparam->regions.size(); + const size_t rk_num_bins = simparam->dynamic_options.rk_num_bins; + const size_t num_bcs = module_params->boundary_conditions.size(); + const size_t num_materials = simparam->materials.size(); + real_t objective_accumulation; + + // --- Read in the nodes in the mesh --- + int myrank = Explicit_Solver_Pointer_->myrank; + int nranks = Explicit_Solver_Pointer_->nranks; + + const DCArrayKokkos mat_fill = simparam->mat_fill; + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos material = simparam->material; + CArray current_element_nodal_densities = CArray(num_nodes_in_elem); + + std::vector> FEA_Module_My_TO_Modules = simparam->FEA_Module_My_TO_Modules; + problem = Explicit_Solver_Pointer_->problem; // Pointer to ROL optimization problem object + ROL::Ptr> obj_pointer; + + // compute element averaged density ratios corresponding to nodal density design variables + { // view scope + const_host_vec_array all_node_densities = all_node_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); + // debug print + // std::cout << "NODE DENSITY TEST " << all_node_densities(0,0) << std::endl; + for (int elem_id = 0; elem_id < rnum_elem; elem_id++) + { + for (int inode = 0; inode < num_nodes_in_elem; inode++) + { + current_element_nodal_densities(inode) = all_node_densities(nodes_in_elem(elem_id, inode), 0); + } + relative_element_densities.host(elem_id) = average_element_density(num_nodes_in_elem, current_element_nodal_densities); + } // for + } // view scope + // debug print + // std::cout << "ELEMENT RELATIVE DENSITY TEST " << relative_element_densities.host(0) << std::endl; + relative_element_densities.update_device(); + + // set density vector to the current value chosen by the optimizer + test_node_densities_distributed = zp; + + // reset nodal coordinates to initial values + node_coords_distributed->assign(*initial_node_coords_distributed); + + // comms for ghosts + Explicit_Solver_Pointer_->comm_coordinates(); + + // view scope + { + const_vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array ghost_node_coords_interface = Explicit_Solver_Pointer_->ghost_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + all_node_coords_interface(node_gid, idim) = node_coords_interface(node_gid, idim); + } + }); // end parallel for + Kokkos::fence(); + + FOR_ALL_CLASS(node_gid, nlocal_nodes, nlocal_nodes + nghost_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + all_node_coords_interface(node_gid, idim) = ghost_node_coords_interface(node_gid - nlocal_nodes, idim); + } + }); // end parallel for + Kokkos::fence(); + } // end view scope + + // reset velocities to initial conditions + node_velocities_distributed->assign(*initial_node_velocities_distributed); + + // reset time accumulating objective and constraints + /* + for(int imodule = 0 ; imodule < FEA_Module_My_TO_Modules[my_fea_module_index_].size(); imodule++){ + current_module_index = FEA_Module_My_TO_Modules[my_fea_module_index_][imodule]; + //test if module needs reset + if(){ + + } + } + */ + // simple setup to just request KE for now; above loop to be expanded and used later for scanning modules + obj_pointer = problem->getObjective(); + KineticEnergyMinimize_TopOpt& kinetic_energy_minimize_function = dynamic_cast(*obj_pointer); + kinetic_energy_minimize_function.objective_accumulation = 0; + + // interface trial density vector + + // interfacing of vectors(should be removed later once made compatible) + // view scope + { + host_vec_array interface_node_coords = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + for (size_t ibin = 0; ibin < rk_num_bins; ibin++) + { + // save node data to node.coords + // std::cout << "NODE DATA ON RANK " << myrank << std::endl; + if (num_dim == 2) + { + for (int inode = 0; inode < nall_nodes; inode++) + { + // std::cout << "Node index " << inode+1 << " "; + node_coords.host(ibin, inode, 0) = interface_node_coords(inode, 0); + // std::cout << host_node_coords_state(0,inode,0)+1<< " "; + node_coords.host(ibin, inode, 1) = interface_node_coords(inode, 1); + // std::cout << host_node_coords_state(0,inode,1)+1<< " "; + } + } + else if (num_dim == 3) + { + for (int inode = 0; inode < nall_nodes; inode++) + { + // std::cout << "Node index " << inode+1 << " "; + node_coords.host(ibin, inode, 0) = interface_node_coords(inode, 0); + // std::cout << host_node_coords_state(0,inode,0)+1<< " "; + node_coords.host(ibin, inode, 1) = interface_node_coords(inode, 1); + // std::cout << host_node_coords_state(0,inode,1)+1<< " "; + + node_coords.host(ibin, inode, 2) = interface_node_coords(inode, 2); + // std::cout << host_node_coords_state(0,inode,2)+1<< std::endl; + } + } + } + } // end view scope + + // save the node coords to the current RK value + for (size_t node_gid = 0; node_gid < nall_nodes; node_gid++) + { + for (int rk = 1; rk < rk_num_bins; rk++) + { + for (int dim = 0; dim < num_dim; dim++) + { + node_coords.host(rk, node_gid, dim) = node_coords.host(0, node_gid, dim); + } // end for dim + } // end for rk + } // end parallel for + + node_coords.update_device(); + + // setup that needs repeating + get_vol(); + // --- apply the fill instructions over the Elements---// + + // loop over the fill instructures + // view scope + { + for (int f_id = 0; f_id < num_fills; f_id++) + { + // parallel loop over elements in mesh + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { + // calculate the coordinates and radius of the element + double elem_coords[3]; // note:initialization with a list won't work + elem_coords[0] = 0.0; + elem_coords[1] = 0.0; + elem_coords[2] = 0.0; + + // get the coordinates of the element center + for (int node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + elem_coords[0] += node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), 0); + elem_coords[1] += node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), 1); + if (num_dim == 3) + { + elem_coords[2] += node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), 2); + } + else + { + elem_coords[2] = 0.0; + } + } // end loop over nodes in element + elem_coords[0] = elem_coords[0] / num_nodes_in_elem; + elem_coords[1] = elem_coords[1] / num_nodes_in_elem; + elem_coords[2] = elem_coords[2] / num_nodes_in_elem; + + // default is not to fill the element + bool fill_this = mat_fill(f_id).volume.contains(elem_coords); + + // paint the material state on the element + if (fill_this) + { + // density + elem_den(elem_gid) = mat_fill(f_id).den; + + // compute element average density from initial nodal density variables used as TO design variables + elem_den(elem_gid) = elem_den(elem_gid) * relative_element_densities(elem_gid); + + // mass + elem_mass(elem_gid) = elem_den(elem_gid) * elem_vol(elem_gid); + + // specific internal energy + elem_sie(rk_level, elem_gid) = mat_fill(f_id).sie; + + elem_mat_id(elem_gid) = mat_fill(f_id).material_id; + size_t mat_id = elem_mat_id(elem_gid); // short name + + // --- stress tensor --- + // always 3D even for 2D-RZ + for (size_t i = 0; i < 3; i++) + { + for (size_t j = 0; j < 3; j++) + { + elem_stress(rk_level, elem_gid, i, j) = 0.0; + } + } // end for + + // short form for clean code + EOSParent* eos_model = elem_eos(elem_gid).model; + + // --- Pressure --- + eos_model->calc_pressure(elem_pres, + elem_stress, + elem_gid, + elem_mat_id(elem_gid), + state_vars, + global_vars, + elem_user_output_vars, + elem_sspd, + elem_den(elem_gid), + elem_sie(rk_level, elem_gid)); + + // --- Sound speed --- + eos_model->calc_sound_speed(elem_pres, + elem_stress, + elem_gid, + elem_mat_id(elem_gid), + state_vars, + global_vars, + elem_user_output_vars, + elem_sspd, + elem_den(elem_gid), + elem_sie(rk_level, elem_gid)); + + // loop over the nodes of this element and apply velocity + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + // get the mesh node index + size_t node_gid = nodes_in_elem(elem_gid, node_lid); + + // --- Velocity --- + switch (mat_fill(f_id).velocity) + { + case VELOCITY_TYPE::cartesian: + { + node_vel(rk_level, node_gid, 0) = mat_fill(f_id).u; + node_vel(rk_level, node_gid, 1) = mat_fill(f_id).v; + if (num_dim == 3) + { + node_vel(rk_level, node_gid, 2) = mat_fill(f_id).w; + } + + break; + } + case VELOCITY_TYPE::radial: + { + // Setting up cylindrical + double dir[2]; + dir[0] = 0.0; + dir[1] = 0.0; + double radius_val = 0.0; + + for (int dim = 0; dim < 2; dim++) + { + dir[dim] = node_coords(rk_level, node_gid, dim); + radius_val += node_coords(rk_level, node_gid, dim) * node_coords(rk_level, node_gid, dim); + } // end for + radius_val = sqrt(radius_val); + + for (int dim = 0; dim < 2; dim++) + { + if (radius_val > 1.0e-14) + { + dir[dim] /= (radius_val); + } + else + { + dir[dim] = 0.0; + } + } // end for + + node_vel(rk_level, node_gid, 0) = mat_fill(f_id).speed * dir[0]; + node_vel(rk_level, node_gid, 1) = mat_fill(f_id).speed * dir[1]; + if (num_dim == 3) + { + node_vel(rk_level, node_gid, 2) = 0.0; + } + + break; + } + case VELOCITY_TYPE::spherical: + { + // Setting up spherical + double dir[3]; + dir[0] = 0.0; + dir[1] = 0.0; + dir[2] = 0.0; + double radius_val = 0.0; + + for (int dim = 0; dim < 3; dim++) + { + dir[dim] = node_coords(rk_level, node_gid, dim); + radius_val += node_coords(rk_level, node_gid, dim) * node_coords(rk_level, node_gid, dim); + } // end for + radius_val = sqrt(radius_val); + + for (int dim = 0; dim < 3; dim++) + { + if (radius_val > 1.0e-14) + { + dir[dim] /= (radius_val); + } + else + { + dir[dim] = 0.0; + } + } // end for + + node_vel(rk_level, node_gid, 0) = mat_fill(f_id).speed * dir[0]; + node_vel(rk_level, node_gid, 1) = mat_fill(f_id).speed * dir[1]; + if (num_dim == 3) + { + node_vel(rk_level, node_gid, 2) = mat_fill(f_id).speed * dir[2]; + } + + break; + } + case VELOCITY_TYPE::radial_linear: + { + break; + } + case VELOCITY_TYPE::spherical_linear: + { + break; + } + case VELOCITY_TYPE::tg_vortex: + { + node_vel(rk_level, node_gid, 0) = sin(PI * node_coords(rk_level, node_gid, 0)) * cos(PI * node_coords(rk_level, node_gid, 1)); + node_vel(rk_level, node_gid, 1) = -1.0 * cos(PI * node_coords(rk_level, node_gid, 0)) * sin(PI * node_coords(rk_level, node_gid, 1)); + if (num_dim == 3) + { + node_vel(rk_level, node_gid, 2) = 0.0; + } + + break; + } + } // end of switch + } // end loop over nodes of element + + if (mat_fill(f_id).velocity == VELOCITY_TYPE::tg_vortex) + { + elem_pres(elem_gid) = 0.25 * (cos(2.0 * PI * elem_coords[0]) + cos(2.0 * PI * elem_coords[1]) ) + 1.0; + + // p = rho*ie*(gamma - 1) + size_t mat_id = f_id; + double gamma = global_vars(mat_id, 0); // gamma value + elem_sie(rk_level, elem_gid) = + elem_pres(elem_gid) / (mat_fill(f_id).den * (gamma - 1.0)); + } // end if + } // end if fill + }); // end FOR_ALL_CLASS element loop + Kokkos::fence(); + } // end for loop over fills + } // end view scope + + // apply BC's to velocity + FEA_Module_SGH::boundary_velocity(*mesh, boundary, node_vel); + + // calculate the corner massess if 2D + if (num_dim == 2) + { + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { + // facial area of the corners + double corner_areas_array[4]; + + ViewCArrayKokkos corner_areas(&corner_areas_array[0], 4); + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 4); + + get_area_weights2D(corner_areas, + elem_gid, + node_coords, + elem_node_gids, + rk_level); + + // loop over the corners of the element and calculate the mass + for (size_t corner_lid = 0; corner_lid < 4; corner_lid++) + { + size_t corner_gid = corners_in_elem(elem_gid, corner_lid); + corner_mass(corner_gid) = corner_areas(corner_lid) * elem_den(elem_gid); // node radius is added later + } // end for over corners + }); + } // end of + + // calculate the nodal mass + FOR_ALL_CLASS(node_gid, 0, nall_nodes, { + node_mass(node_gid) = 0.0; + + if (num_dim == 3) + { + for (size_t elem_lid = 0; elem_lid < num_corners_in_node(node_gid); elem_lid++) + { + size_t elem_gid = elems_in_node(node_gid, elem_lid); + node_mass(node_gid) += 1.0 / 8.0 * elem_mass(elem_gid); + } // end for elem_lid + } // end if dims=3 + else + { + // 2D-RZ + for (size_t corner_lid = 0; corner_lid < num_corners_in_node(node_gid); corner_lid++) + { + size_t corner_gid = corners_in_node(node_gid, corner_lid); + node_mass(node_gid) += corner_mass(corner_gid); // sans the radius so it is areal node mass + + corner_mass(corner_gid) *= node_coords(rk_level, node_gid, 1); // true corner mass now + } // end for elem_lid + } // end else + }); // end FOR_ALL_CLASS + Kokkos::fence(); + + // current interface has differing mass arrays; this equates them until we unify memory + // view scope + { + vec_array node_mass_interface = node_masses_distributed->getLocalView(Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + node_mass_interface(node_gid, 0) = node_mass(node_gid); + }); // end parallel for + } // end view scope + Kokkos::fence(); + // communicate ghost densities + comm_node_masses(); + + // this is forcing a copy to the device + // view scope + { + vec_array ghost_node_mass_interface = ghost_node_masses_distributed->getLocalView(Tpetra::Access::ReadWrite); + + FOR_ALL_CLASS(node_gid, nlocal_nodes, nall_nodes, { + node_mass(node_gid) = ghost_node_mass_interface(node_gid - nlocal_nodes, 0); + }); // end parallel for + } // end view scope + Kokkos::fence(); + + // update stiffness matrix + if (simparam->topology_optimization_on || simparam->shape_optimization_on) + { + // assemble_matrix(); + } + + // update host copies of arrays modified in this function + elem_den.update_host(); + elem_mass.update_host(); + elem_sie.update_host(); + elem_stress.update_host(); + elem_pres.update_host(); + elem_sspd.update_host(); + + // execute solve + sgh_solve(); +} + +/* ------------------------------------------------------------------------------------------- + Compute average density of an element from nodal densities +---------------------------------------------------------------------------------------------- */ + +double FEA_Module_SGH::average_element_density(const int nodes_per_elem, const CArray current_element_densities) const +{ + double result = 0; + for (int i = 0; i < nodes_per_elem; i++) + { + result += current_element_densities(i) / nodes_per_elem; + } + + return result; +} + +/* ------------------------------------------------------------------------------ + Coupled adjoint problem for the kinetic energy minimization problem +--------------------------------------------------------------------------------- */ + +void FEA_Module_SGH::compute_topology_optimization_adjoint_full() +{ + const int num_dim = simparam->num_dims; + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos material = simparam->material; + + real_t global_dt; + size_t current_data_index, next_data_index; + size_t num_bdy_nodes = mesh->num_bdy_nodes; + + Teuchos::RCP previous_adjoint_vector_distributed, current_adjoint_vector_distributed; + Teuchos::RCP previous_velocity_vector_distributed, current_velocity_vector_distributed; + Teuchos::RCP previous_phi_adjoint_vector_distributed, current_phi_adjoint_vector_distributed; + + // initialize first adjoint vector at last_time_step to 0 as the terminal value + (*adjoint_vector_data)[last_time_step + 1]->putScalar(0); + (*phi_adjoint_vector_data)[last_time_step + 1]->putScalar(0); + (*psi_adjoint_vector_data)[last_time_step + 1]->putScalar(0); + + // solve terminal value problem, proceeds in time backward. For simplicity, we use the same timestep data from the forward solve. + // A linear interpolant is assumed between velocity data points; velocity midpoint is used to update the adjoint. + if (myrank == 0) + { + std::cout << "Computing adjoint vector " << time_data.size() << std::endl; + } + + for (int cycle = last_time_step; cycle >= 0; cycle--) + { + // compute timestep from time data + global_dt = time_data[cycle + 1] - time_data[cycle]; + + // print + if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) + { + if (cycle == last_time_step) + { + if (myrank == 0) + { + printf("cycle = %d, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } + // print time step every 20 cycles + else if (cycle % 20 == 0) + { + if (myrank == 0) + { + printf("cycle = %d, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } // end if + } + // else if (cycle==1){ + // if(myrank==0) + // printf("cycle = %lu, time = %f, time step = %f \n", cycle-1, time_data[cycle-1], global_dt); + // } // end if + + // compute adjoint vector for this data point; use velocity midpoint + // view scope + { + // set velocity, internal energy, and position for this timestep + const_vec_array previous_velocity_vector = (*forward_solve_velocity_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + + const_vec_array previous_coordinate_vector = (*forward_solve_coordinate_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array current_coordinate_vector = (*forward_solve_coordinate_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + + const_vec_array previous_element_internal_energy = (*forward_solve_internal_energy_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array current_element_internal_energy = (*forward_solve_internal_energy_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + + // interface of arrays for current implementation of force calculation + + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes + nghost_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + node_vel(rk_level, node_gid, idim) = previous_velocity_vector(node_gid, idim); + node_coords(rk_level, node_gid, idim) = previous_coordinate_vector(node_gid, idim); + } + }); + Kokkos::fence(); + + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { + elem_sie(rk_level, elem_gid) = previous_element_internal_energy(elem_gid, 0); + }); + Kokkos::fence(); + + // set state according to phase data at this timestep + + get_vol(); + + // ---- Calculate velocity diveregence for the element ---- + if (num_dim == 2) + { + get_divergence2D(elem_div, + node_coords, + node_vel, + elem_vol); + } + else + { + get_divergence(elem_div, + node_coords, + node_vel, + elem_vol); + } // end if 2D + + // ---- Calculate elem state (den, pres, sound speed, stress) for next time step ---- + if (num_dim == 2) + { + update_state2D(material, + *mesh, + node_coords, + node_vel, + elem_den, + elem_pres, + elem_stress, + elem_sspd, + elem_sie, + elem_vol, + elem_mass, + elem_mat_id, + 1.0, + cycle); + } + else + { + update_state(material, + *mesh, + node_coords, + node_vel, + elem_den, + elem_pres, + elem_stress, + elem_sspd, + elem_sie, + elem_vol, + elem_mass, + elem_mat_id, + 1.0, + cycle); + } + + if (num_dim == 2) + { + get_force_sgh2D(material, + *mesh, + node_coords, + node_vel, + elem_den, + elem_sie, + elem_pres, + elem_stress, + elem_sspd, + elem_vol, + elem_div, + elem_mat_id, + corner_force, + 1.0, + cycle); + } + else + { + get_force_sgh(material, + *mesh, + node_coords, + node_vel, + elem_den, + elem_sie, + elem_pres, + elem_stress, + elem_sspd, + elem_vol, + elem_div, + elem_mat_id, + corner_force, + 1.0, + cycle); + } + + // compute gradient matrices + get_force_egradient_sgh(material, + *mesh, + node_coords, + node_vel, + elem_den, + elem_sie, + elem_pres, + elem_stress, + elem_sspd, + elem_vol, + elem_div, + elem_mat_id, + 1.0, + cycle); + + get_power_egradient_sgh(1.0, + *mesh, + node_vel, + node_coords, + elem_sie, + elem_mass, + corner_force); + + get_force_vgradient_sgh(material, + *mesh, + node_coords, + node_vel, + elem_den, + elem_sie, + elem_pres, + elem_stress, + elem_sspd, + elem_vol, + elem_div, + elem_mat_id, + 1.0, + cycle); + + get_power_vgradient_sgh(1.0, + *mesh, + node_vel, + node_coords, + elem_sie, + elem_mass, + corner_force); + + get_force_ugradient_sgh(material, + *mesh, + node_coords, + node_vel, + elem_den, + elem_sie, + elem_pres, + elem_stress, + elem_sspd, + elem_vol, + elem_div, + elem_mat_id, + 1.0, + cycle); + + get_power_ugradient_sgh(1.0, + *mesh, + node_vel, + node_coords, + elem_sie, + elem_mass, + corner_force); + + // force_gradient_velocity->describe(*fos,Teuchos::VERB_EXTREME); + const_vec_array previous_force_gradient_position = force_gradient_position->getLocalView(Tpetra::Access::ReadOnly); + // const_vec_array current_force_gradient_position = force_gradient_position->getLocalView (Tpetra::Access::ReadOnly); + const_vec_array previous_force_gradient_velocity = force_gradient_velocity->getLocalView(Tpetra::Access::ReadOnly); + // const_vec_array current_force_gradient_velocity = force_gradient_velocity->getLocalView (Tpetra::Access::ReadOnly); + // compute gradient of force with respect to velocity + + const_vec_array previous_adjoint_vector = (*adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array phi_previous_adjoint_vector = (*phi_adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array psi_previous_adjoint_vector = (*psi_adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + vec_array midpoint_adjoint_vector = adjoint_vector_distributed->getLocalView(Tpetra::Access::ReadWrite); + vec_array phi_midpoint_adjoint_vector = phi_adjoint_vector_distributed->getLocalView(Tpetra::Access::ReadWrite); + vec_array psi_midpoint_adjoint_vector = psi_adjoint_vector_distributed->getLocalView(Tpetra::Access::ReadWrite); + + // half step update for RK2 scheme; EQUATION 1 + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + real_t rate_of_change; + real_t matrix_contribution; + size_t dof_id; + size_t elem_id; + for (int idim = 0; idim < num_dim; idim++) + { + // EQUATION 1 + matrix_contribution = 0; + // compute resulting row of force velocity gradient matrix transpose right multiplied by adjoint vector + for (int idof = 0; idof < Gradient_Matrix_Strides(node_gid * num_dim + idim); idof++) + { + dof_id = DOF_Graph_Matrix(node_gid * num_dim + idim, idof); + matrix_contribution += previous_adjoint_vector(dof_id / num_dim, dof_id % num_dim) * Force_Gradient_Velocities(node_gid * num_dim + idim, idof); + } + + // compute resulting row of transpose of power gradient w.r.t velocity matrix right multiplied by psi adjoint vector + for (int ielem = 0; ielem < DOF_to_Elem_Matrix_Strides(node_gid * num_dim + idim); ielem++) + { + elem_id = elems_in_node(node_gid, ielem); + matrix_contribution += psi_previous_adjoint_vector(elem_id, 0) * Power_Gradient_Velocities(node_gid * num_dim + idim, ielem); + } + + rate_of_change = previous_velocity_vector(node_gid, idim) - + matrix_contribution / node_mass(node_gid) - + phi_previous_adjoint_vector(node_gid, idim) / node_mass(node_gid); + midpoint_adjoint_vector(node_gid, idim) = -rate_of_change * global_dt / 2 + previous_adjoint_vector(node_gid, idim); + } + }); // end parallel for + Kokkos::fence(); + + // apply BCs to adjoint vector, only matters for the momentum adjoint if using strictly velocity boundary conditions + boundary_adjoint(*mesh, boundary, midpoint_adjoint_vector, phi_midpoint_adjoint_vector, psi_midpoint_adjoint_vector); + + comm_adjoint_vector(cycle); + + // half step update for RK2 scheme; EQUATION 2 + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + real_t rate_of_change; + real_t matrix_contribution; + size_t dof_id; + size_t elem_id; + for (int idim = 0; idim < num_dim; idim++) + { + // EQUATION 2 + matrix_contribution = 0; + // compute resulting row of force displacement gradient matrix transpose right multiplied by adjoint vector + for (int idof = 0; idof < Gradient_Matrix_Strides(node_gid * num_dim + idim); idof++) + { + dof_id = DOF_Graph_Matrix(node_gid * num_dim + idim, idof); + matrix_contribution += previous_adjoint_vector(dof_id / num_dim, dof_id % num_dim) * Force_Gradient_Positions(node_gid * num_dim + idim, idof); + } + + // compute resulting row of transpose of power gradient w.r.t displacement matrix right multiplied by psi adjoint vector + for (int ielem = 0; ielem < DOF_to_Elem_Matrix_Strides(node_gid * num_dim + idim); ielem++) + { + elem_id = elems_in_node(node_gid, ielem); + matrix_contribution += psi_previous_adjoint_vector(elem_id, 0) * Power_Gradient_Positions(node_gid * num_dim + idim, ielem); + } + + rate_of_change = -matrix_contribution; + // rate_of_change = -0.0000001*previous_adjoint_vector(node_gid,idim); + phi_midpoint_adjoint_vector(node_gid, idim) = -rate_of_change * global_dt / 2 + phi_previous_adjoint_vector(node_gid, idim); + } + }); // end parallel for + Kokkos::fence(); + + comm_phi_adjoint_vector(cycle); + + // phi_adjoint_vector_distributed->describe(*fos,Teuchos::VERB_EXTREME); + + // half step update for RK2 scheme; EQUATION 3 + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { + real_t rate_of_change; + real_t matrix_contribution; + size_t dof_id; + size_t elem_id; + // EQUATION 3 + matrix_contribution = 0; + // compute resulting row of force displacement gradient matrix transpose right multiplied by adjoint vector + for (int idof = 0; idof < num_nodes_in_elem * num_dim; idof++) + { + dof_id = nodes_in_elem(elem_gid, idof / num_dim) * num_dim + idof % num_dim; + matrix_contribution += previous_adjoint_vector(dof_id / num_dim, dof_id % num_dim) * Force_Gradient_Energies(elem_gid, idof); + } + rate_of_change = -(matrix_contribution + psi_previous_adjoint_vector(elem_gid, 0) * Power_Gradient_Energies(elem_gid)) / elem_mass(elem_gid); + // rate_of_change = -0.0000001*previous_adjoint_vector(node_gid,idim); + psi_midpoint_adjoint_vector(elem_gid, 0) = -rate_of_change * global_dt / 2 + psi_previous_adjoint_vector(elem_gid, 0); + }); // end parallel for + Kokkos::fence(); + + // save for second half of RK + (*psi_adjoint_vector_data)[cycle]->assign(*psi_adjoint_vector_distributed); + + // swap names to get ghost nodes for the midpoint vectors + vec_array current_adjoint_vector = adjoint_vector_distributed->getLocalView(Tpetra::Access::ReadWrite); + vec_array phi_current_adjoint_vector = phi_adjoint_vector_distributed->getLocalView(Tpetra::Access::ReadWrite); + vec_array psi_current_adjoint_vector = psi_adjoint_vector_distributed->getLocalView(Tpetra::Access::ReadWrite); + midpoint_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadWrite); + phi_midpoint_adjoint_vector = (*phi_adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadWrite); + psi_midpoint_adjoint_vector = (*psi_adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadWrite); + + // full step update with midpoint gradient for RK2 scheme; EQUATION 1 + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + real_t rate_of_change; + real_t matrix_contribution; + size_t dof_id; + size_t elem_id; + for (int idim = 0; idim < num_dim; idim++) + { + // EQUATION 1 + matrix_contribution = 0; + // compute resulting row of force velocity gradient matrix transpose right multiplied by adjoint vector + + for (int idof = 0; idof < Gradient_Matrix_Strides(node_gid * num_dim + idim); idof++) + { + dof_id = DOF_Graph_Matrix(node_gid * num_dim + idim, idof); + matrix_contribution += midpoint_adjoint_vector(dof_id / num_dim, dof_id % num_dim) * Force_Gradient_Velocities(node_gid * num_dim + idim, idof); + } + + // compute resulting row of transpose of power gradient w.r.t velocity matrix right multiplied by psi adjoint vector + for (int ielem = 0; ielem < DOF_to_Elem_Matrix_Strides(node_gid * num_dim + idim); ielem++) + { + elem_id = elems_in_node(node_gid, ielem); + matrix_contribution += psi_midpoint_adjoint_vector(elem_id, 0) * Power_Gradient_Velocities(node_gid * num_dim + idim, ielem); + } + + rate_of_change = (previous_velocity_vector(node_gid, idim) + current_velocity_vector(node_gid, idim)) / 2 - + matrix_contribution / node_mass(node_gid) - + phi_midpoint_adjoint_vector(node_gid, idim) / node_mass(node_gid); + current_adjoint_vector(node_gid, idim) = -rate_of_change * global_dt + previous_adjoint_vector(node_gid, idim); + } + }); // end parallel for + Kokkos::fence(); + + boundary_adjoint(*mesh, boundary, current_adjoint_vector, phi_current_adjoint_vector, psi_midpoint_adjoint_vector); + comm_adjoint_vector(cycle); + + // full step update with midpoint gradient for RK2 scheme; EQUATION 2 + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + real_t rate_of_change; + real_t matrix_contribution; + size_t dof_id; + size_t elem_id; + for (int idim = 0; idim < num_dim; idim++) + { + // EQUATION 2 + matrix_contribution = 0; + // compute resulting row of force displacement gradient matrix transpose right multiplied by adjoint vector + for (int idof = 0; idof < Gradient_Matrix_Strides(node_gid * num_dim + idim); idof++) + { + dof_id = DOF_Graph_Matrix(node_gid * num_dim + idim, idof); + matrix_contribution += midpoint_adjoint_vector(dof_id / num_dim, dof_id % num_dim) * Force_Gradient_Positions(node_gid * num_dim + idim, idof); + } + + // compute resulting row of transpose of power gradient w.r.t displacement matrix right multiplied by psi adjoint vector + for (int ielem = 0; ielem < DOF_to_Elem_Matrix_Strides(node_gid * num_dim + idim); ielem++) + { + elem_id = elems_in_node(node_gid, ielem); + matrix_contribution += psi_midpoint_adjoint_vector(elem_id, 0) * Power_Gradient_Positions(node_gid * num_dim + idim, ielem); + } + + rate_of_change = -matrix_contribution; + // rate_of_change = -0.0000001*midpoint_adjoint_vector(node_gid,idim); + phi_current_adjoint_vector(node_gid, idim) = -rate_of_change * global_dt + phi_previous_adjoint_vector(node_gid, idim); + } + }); // end parallel for + Kokkos::fence(); + + comm_phi_adjoint_vector(cycle); + + // full step update for RK2 scheme; EQUATION 3 + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { + real_t rate_of_change; + real_t matrix_contribution; + size_t dof_id; + size_t elem_id; + // EQUATION 3 + matrix_contribution = 0; + // compute resulting row of force displacement gradient matrix transpose right multiplied by adjoint vector + for (int idof = 0; idof < num_nodes_in_elem * num_dim; idof++) + { + dof_id = nodes_in_elem(elem_gid, idof / num_dim) * num_dim + idof % num_dim; + matrix_contribution += midpoint_adjoint_vector(dof_id / num_dim, dof_id % num_dim) * Force_Gradient_Energies(elem_gid, idof); + } + rate_of_change = -(matrix_contribution + psi_midpoint_adjoint_vector(elem_gid, 0) * Power_Gradient_Energies(elem_gid)) / elem_mass(elem_gid); + // debug + // std::cout << "PSI RATE OF CHANGE " << rate_of_change << std::endl; + psi_current_adjoint_vector(elem_gid, 0) = -rate_of_change * global_dt + psi_previous_adjoint_vector(elem_gid, 0); + }); // end parallel for + Kokkos::fence(); + + // save data from time-step completion + (*psi_adjoint_vector_data)[cycle]->assign(*psi_adjoint_vector_distributed); + } // end view scope + + // phi_adjoint_vector_distributed->describe(*fos,Teuchos::VERB_EXTREME); + } +} + +/* ---------------------------------------------------------------------------- + Gradient for the (unsimplified) kinetic energy minimization problem +------------------------------------------------------------------------------- */ + +void FEA_Module_SGH::compute_topology_optimization_gradient_full(Teuchos::RCP design_densities_distributed, Teuchos::RCP design_gradients_distributed) +{ + size_t num_bdy_nodes = mesh->num_bdy_nodes; + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos material = simparam->material; + const int num_dim = simparam->num_dims; + int num_corners = rnum_elem * num_nodes_in_elem; + real_t global_dt; + bool element_constant_density = true; + size_t current_data_index, next_data_index; + CArrayKokkos current_element_velocities = CArrayKokkos(num_nodes_in_elem, num_dim); + CArrayKokkos current_element_adjoint = CArrayKokkos(num_nodes_in_elem, num_dim); + + if (myrank == 0) + { + std::cout << "Computing accumulated kinetic energy gradient" << std::endl; + } + + compute_topology_optimization_adjoint_full(); + + { // view scope + vec_array design_gradients = design_gradients_distributed->getLocalView(Tpetra::Access::ReadWrite); + const_vec_array design_densities = design_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); + // initialize design gradients + FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { + design_gradients(node_id, 0) = 0; + }); // end parallel for + Kokkos::fence(); + + // gradient contribution from kinetic energy v(dM/drho)v product. + if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) + { + if (myrank == 0) + { + std::cout << "v*dM/drho*v term" << std::endl; + } + } + + for (unsigned long cycle = 0; cycle < last_time_step + 1; cycle++) + { + // compute timestep from time data + global_dt = time_data[cycle + 1] - time_data[cycle]; + + // print + if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) + { + if (cycle == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } + // print time step every 10 cycles + else if (cycle % 20 == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } // end if + } + // view scope + { + const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_velocity_vector = (*forward_solve_velocity_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + + FOR_ALL_CLASS(elem_id, 0, rnum_elem, { + size_t node_id; + size_t corner_id; + real_t inner_product; + // std::cout << elem_mass(elem_id) <dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) + { + if (myrank == 0) + { + std::cout << "gradient term involving adjoint derivative" << std::endl; + } + } + + for (unsigned long cycle = 0; cycle < last_time_step + 1; cycle++) + { + // compute timestep from time data + global_dt = time_data[cycle + 1] - time_data[cycle]; + // print + if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) + { + if (cycle == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } + // print time step every 10 cycles + else if (cycle % 20 == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } // end if + } + + // compute adjoint vector for this data point; use velocity midpoint + // view scope + { + const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_velocity_vector = (*forward_solve_velocity_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_adjoint_vector = (*adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + + FOR_ALL_CLASS(elem_id, 0, rnum_elem, { + real_t lambda_dot_current; + real_t lambda_dot_next; + size_t node_id; + size_t corner_id; + real_t inner_product; + // std::cout << elem_mass(elem_id) <dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) + { + if (myrank == 0) + { + std::cout << "gradient term involving adjoint derivative" << std::endl; + } + } + + for (unsigned long cycle = 0; cycle < last_time_step + 1; cycle++) + { + // compute timestep from time data + global_dt = time_data[cycle + 1] - time_data[cycle]; + // print + if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) + { + if (cycle == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } + // print time step every 10 cycles + else if (cycle % 20 == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } // end if + } + + // compute adjoint vector for this data point; use velocity midpoint + // view scope + { + const_vec_array current_element_internal_energy = (*forward_solve_internal_energy_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array current_psi_adjoint_vector = (*psi_adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_element_internal_energy = (*forward_solve_internal_energy_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_psi_adjoint_vector = (*psi_adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + + FOR_ALL_CLASS(elem_id, 0, rnum_elem, { + real_t psi_dot_current; + real_t psi_dot_next; + size_t node_id; + size_t corner_id; + real_t inner_product; + // std::cout << elem_mass(elem_id) <getLocalView(Tpetra::Access::ReadOnly); + const_vec_array current_adjoint_vector = (*adjoint_vector_data)[0]->getLocalView(Tpetra::Access::ReadOnly); + + FOR_ALL_CLASS(elem_id, 0, rnum_elem, { + real_t lambda_dot; + size_t node_id; + size_t corner_id; + real_t inner_product; + // std::cout << elem_mass(elem_id) <getLocalView(Tpetra::Access::ReadOnly); + const_vec_array current_psi_adjoint_vector = (*psi_adjoint_vector_data)[0]->getLocalView(Tpetra::Access::ReadOnly); + + // (*psi_adjoint_vector_data)[100]->describe(*fos,Teuchos::VERB_EXTREME); + FOR_ALL_CLASS(elem_id, 0, rnum_elem, { + real_t lambda_dot; + size_t node_id; + size_t corner_id; + real_t inner_product; + // std::cout << elem_mass(elem_id) <getLocalView (Tpetra::Access::ReadWrite); + // const_host_vec_array host_design_variables = design_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); + vec_array design_gradients = design_gradients_distributed->getLocalView(Tpetra::Access::ReadWrite); + const_vec_array design_variables = design_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); + force_design_gradient_term(design_variables, design_gradients); + power_design_gradient_term(design_variables, design_gradients); + } // end view scope +} + +/* ---------------------------------------------------------------------- + Initialize global vectors and array maps needed for matrix assembly +------------------------------------------------------------------------- */ +void FEA_Module_SGH::init_assembly() +{ + int num_dim = simparam->num_dims; + // const_host_elem_conn_array nodes_in_elem = global_nodes_in_elem_distributed->getLocalView (Tpetra::Access::ReadOnly); + Gradient_Matrix_Strides = DCArrayKokkos(nlocal_nodes * num_dim, "Gradient_Matrix_Strides"); + DOF_to_Elem_Matrix_Strides = DCArrayKokkos(nlocal_nodes * num_dim, "Gradient_Matrix_Strides"); + Elem_to_Elem_Matrix_Strides = DCArrayKokkos(rnum_elem, "Gradient_Matrix_Strides"); + CArrayKokkos Graph_Fill(nall_nodes, "nall_nodes"); + CArrayKokkos current_row_nodes_scanned; + CArrayKokkos count_saved_corners_in_node(nall_nodes, "count_saved_corners_in_node"); + int local_node_index, current_column_index; + size_t max_stride = 0; + size_t nodes_per_element; + nodal_density_flag = simparam->nodal_density_flag; + penalty_power = simparam->optimization_options.simp_penalty_power; + + // allocate stride arrays + CArrayKokkos Graph_Matrix_Strides_initial(nlocal_nodes, "Graph_Matrix_Strides_initial"); + DCArrayKokkos Dual_Graph_Matrix_Strides_initial(nlocal_nodes, "Host_Graph_Matrix_Strides_initial"); + Graph_Matrix_Strides = DCArrayKokkos(nlocal_nodes, "Graph_Matrix_Strides"); + + // allocate storage for the sparse gradient matrix map for node to node connectivity + Global_Gradient_Matrix_Assembly_Map = DCArrayKokkos(rnum_elem, + max_nodes_per_element, max_nodes_per_element, "Global_Gradient_Matrix_Assembly_Map"); + + // allocate storage for the sparse gradient matrix map for node to element connectivity + Element_Gradient_Matrix_Assembly_Map = DCArrayKokkos(rnum_elem, + max_nodes_per_element, "Element_Gradient_Matrix_Assembly_Map"); + + // allocate array used to determine global node repeats in the sparse graph later + DCArrayKokkos node_indices_used(nall_nodes, "node_indices_used"); + + /*allocate array that stores which column the node index occured on for the current row + when removing repeats*/ + DCArrayKokkos column_index(nall_nodes, "column_index"); + + // initialize nlocal arrays + FOR_ALL_CLASS(inode, 0, nlocal_nodes, { + Graph_Matrix_Strides_initial(inode) = 0; + Graph_Matrix_Strides(inode) = 0; + Graph_Fill(inode) = 0; + }); // end parallel for + Kokkos::fence(); + + // initialize nall arrays + // initialize nlocal arrays + FOR_ALL_CLASS(inode, 0, nall_nodes, { + node_indices_used(inode) = 0; + column_index(inode) = 0; + count_saved_corners_in_node(inode) = 0; + }); // end parallel for + Kokkos::fence(); + + // count upper bound of strides for Sparse Pattern Graph by allowing repeats due to connectivity + if (num_dim == 2) + { + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + element_select->choose_2Delem_type(Element_Types(ielem), elem2D); + nodes_per_element = elem2D->num_nodes(); + for (int lnode = 0; lnode < nodes_per_element; lnode++) + { + local_node_index = nodes_in_elem(ielem, lnode); + if (local_node_index < nlocal_nodes) + { + Dual_Graph_Matrix_Strides_initial.host(local_node_index) += nodes_per_element; + } + } + } + } + + if (num_dim == 3) + { + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + element_select->choose_3Delem_type(Element_Types(ielem), elem); + nodes_per_element = elem->num_nodes(); + for (int lnode = 0; lnode < nodes_per_element; lnode++) + { + local_node_index = nodes_in_elem(ielem, lnode); + if (local_node_index < nlocal_nodes) + { + Dual_Graph_Matrix_Strides_initial.host(local_node_index) += nodes_per_element; + } + } + } + } + + Dual_Graph_Matrix_Strides_initial.update_device(); + + // equate strides for later + FOR_ALL_CLASS(inode, 0, nlocal_nodes, { + Graph_Matrix_Strides(inode) = Graph_Matrix_Strides_initial(inode) = Dual_Graph_Matrix_Strides_initial(inode); + }); // end parallel for + + // for (int inode = 0; inode < nlocal_nodes; inode++) + // std::cout << Graph_Matrix_Strides_initial(inode) << std::endl; + + // compute maximum stride + size_t update = 0; + REDUCE_MAX_CLASS(inode, 0, nlocal_nodes, update, { + if (update < Graph_Matrix_Strides_initial(inode)) + { + update = Graph_Matrix_Strides_initial(inode); + } + }, max_stride); + + // std::cout << "THE MAX STRIDE" << max_stride << std::endl; + // allocate array used in the repeat removal process + current_row_nodes_scanned = CArrayKokkos(max_stride, "current_row_nodes_scanned"); + + // allocate sparse graph with node repeats + RaggedRightArrayKokkos Repeat_Graph_Matrix(Graph_Matrix_Strides_initial); + RaggedRightArrayofVectorsKokkos Element_local_indices(Graph_Matrix_Strides_initial, num_dim); + + // Fill the initial Graph with repeats + if (num_dim == 2) + { + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + element_select->choose_2Delem_type(Element_Types(ielem), elem2D); + nodes_per_element = elem2D->num_nodes(); + for (int lnode = 0; lnode < nodes_per_element; lnode++) + { + local_node_index = nodes_in_elem(ielem, lnode); + if (local_node_index < nlocal_nodes) + { + for (int jnode = 0; jnode < nodes_per_element; jnode++) + { + current_column_index = Graph_Fill(local_node_index) + jnode; + Repeat_Graph_Matrix(local_node_index, current_column_index) = nodes_in_elem(ielem, jnode); + + // fill inverse map + Element_local_indices(local_node_index, current_column_index, 0) = ielem; + Element_local_indices(local_node_index, current_column_index, 1) = lnode; + Element_local_indices(local_node_index, current_column_index, 2) = jnode; + + // fill forward map + Global_Gradient_Matrix_Assembly_Map(ielem, lnode, jnode) = current_column_index; + } + Graph_Fill(local_node_index) += nodes_per_element; + } + } + } + } + + if (num_dim == 3) + { + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + element_select->choose_3Delem_type(Element_Types(ielem), elem); + nodes_per_element = elem->num_nodes(); + for (int lnode = 0; lnode < nodes_per_element; lnode++) + { + local_node_index = nodes_in_elem(ielem, lnode); + if (local_node_index < nlocal_nodes) + { + for (int jnode = 0; jnode < nodes_per_element; jnode++) + { + current_column_index = Graph_Fill(local_node_index) + jnode; + Repeat_Graph_Matrix(local_node_index, current_column_index) = nodes_in_elem(ielem, jnode); + + // fill inverse map + Element_local_indices(local_node_index, current_column_index, 0) = ielem; + Element_local_indices(local_node_index, current_column_index, 1) = lnode; + Element_local_indices(local_node_index, current_column_index, 2) = jnode; + + // fill forward map + Global_Gradient_Matrix_Assembly_Map(ielem, lnode, jnode) = current_column_index; + } + Graph_Fill(local_node_index) += nodes_per_element; + } + } + } + } + + // debug statement + // std::cout << "started run" << std::endl; + // std::cout << "Graph Matrix Strides Repeat on task " << myrank << std::endl; + // for (int inode = 0; inode < nlocal_nodes; inode++) + // std::cout << Graph_Matrix_Strides(inode) << std::endl; + RUN_CLASS({ + // remove repeats from the inital graph setup + int current_node; + // remove repeats from the inital graph setup + int current_element_index; + int element_row_index; + int element_column_index; + int current_stride; + int current_row_n_nodes_scanned; + for (int inode = 0; inode < nlocal_nodes; inode++) + { + current_row_n_nodes_scanned = 0; + for (int istride = 0; istride < Graph_Matrix_Strides(inode); istride++) + { + // convert global index in graph to its local index for the flagging array + current_node = Repeat_Graph_Matrix(inode, istride); + // debug + // if(current_node==-1) + // std::cout << "Graph Matrix node access on task " << myrank << std::endl; + // std::cout << Repeat_Graph_Matrix(inode,istride) << std::endl; + if (node_indices_used(current_node)) + { + // set global assembly map index to the location in the graph matrix where this global node was first found + current_element_index = Element_local_indices(inode, istride, 0); + element_row_index = Element_local_indices(inode, istride, 1); + element_column_index = Element_local_indices(inode, istride, 2); + Global_Gradient_Matrix_Assembly_Map(current_element_index, element_row_index, element_column_index) + = column_index(current_node); + + // swap current node with the end of the current row and shorten the stride of the row + // first swap information about the inverse and forward maps + + current_stride = Graph_Matrix_Strides(inode); + if (istride != current_stride - 1) + { + Element_local_indices(inode, istride, 0) = Element_local_indices(inode, current_stride - 1, 0); + Element_local_indices(inode, istride, 1) = Element_local_indices(inode, current_stride - 1, 1); + Element_local_indices(inode, istride, 2) = Element_local_indices(inode, current_stride - 1, 2); + current_element_index = Element_local_indices(inode, istride, 0); + element_row_index = Element_local_indices(inode, istride, 1); + element_column_index = Element_local_indices(inode, istride, 2); + + Global_Gradient_Matrix_Assembly_Map(current_element_index, element_row_index, element_column_index) + = istride; + + // now that the element map information has been copied, copy the global node index and delete the last index + + Repeat_Graph_Matrix(inode, istride) = Repeat_Graph_Matrix(inode, current_stride - 1); + } + istride--; + Graph_Matrix_Strides(inode)--; + } + else + { + /*this node hasn't shown up in the row before; add it to the list of nodes + that have been scanned uniquely. Use this list to reset the flag array + afterwards without having to loop over all the nodes in the system*/ + node_indices_used(current_node) = 1; + column_index(current_node) = istride; + current_row_nodes_scanned(current_row_n_nodes_scanned) = current_node; + current_row_n_nodes_scanned++; + } + } + // reset nodes used list for the next row of the sparse list + for (int node_reset = 0; node_reset < current_row_n_nodes_scanned; node_reset++) + { + node_indices_used(current_row_nodes_scanned(node_reset)) = 0; + } + } + }); + Kokkos::fence(); + + Graph_Matrix_Strides.update_host(); + // copy reduced content to non_repeat storage + Graph_Matrix = RaggedRightArrayKokkos(Graph_Matrix_Strides); + + FOR_ALL_CLASS(inode, 0, nlocal_nodes, { + for (int istride = 0; istride < Graph_Matrix_Strides(inode); istride++) + { + Graph_Matrix(inode, istride) = Repeat_Graph_Matrix(inode, istride); + } + }); // end parallel for + + // deallocate repeat matrix + + /*At this stage the sparse graph should have unique global indices on each row. + The constructed Assembly map (to the global sparse matrix) + is used to loop over each element's local stiffness matrix in the assembly process.*/ + + // expand strides for stiffness matrix by multipling by dim + FOR_ALL_CLASS(idof, 0, num_dim * nlocal_nodes, { + Gradient_Matrix_Strides(idof) = num_dim * Graph_Matrix_Strides(idof / num_dim); + }); // end parallel for + + Gradient_Matrix_Strides.update_host(); + + // build inverse map for element gradient assembly + for (size_t elem_gid = 0; elem_gid < rnum_elem; elem_gid++) + { + FOR_ALL_CLASS(node_lid, 0, num_nodes_in_elem, { + // get the global_id of the node + size_t node_gid = nodes_in_elem(elem_gid, node_lid); + + // the column index is the num corners saved + size_t j = count_saved_corners_in_node(node_gid); + Element_Gradient_Matrix_Assembly_Map(elem_gid, node_lid) = j; + + // increment the number of corners saved to this node_gid + count_saved_corners_in_node(node_gid)++; + }); // end FOR_ALL over nodes in element + Kokkos::fence(); + } // end for elem_gid + + DOF_Graph_Matrix = RaggedRightArrayKokkos(Gradient_Matrix_Strides); + Force_Gradient_Positions = RaggedRightArrayKokkos(Gradient_Matrix_Strides); + Force_Gradient_Velocities = RaggedRightArrayKokkos(Gradient_Matrix_Strides); + // needs different graph of node to elem rather than node to node + // DOF_to_Elem_Matrix_Strides.get_kokkos_dual_view().d_view = elems_in_node.mystrides_; + Kokkos::View node_to_elem_strides = elems_in_node.mystrides_; + DOF_to_Elem_Matrix_Strides = DCArrayKokkos(nlocal_nodes * num_dim); + FOR_ALL_CLASS(inode, 0, nlocal_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + DOF_to_Elem_Matrix_Strides(inode * num_dim + idim) = count_saved_corners_in_node(inode); + } + }); // end parallel for + DOF_to_Elem_Matrix_Strides.update_host(); + Force_Gradient_Energies = CArrayKokkos(rnum_elem, num_nodes_in_elem * num_dim); + Power_Gradient_Energies = CArrayKokkos(rnum_elem); + Power_Gradient_Positions = RaggedRightArrayKokkos(DOF_to_Elem_Matrix_Strides); + Power_Gradient_Velocities = RaggedRightArrayKokkos(DOF_to_Elem_Matrix_Strides); + + // set stiffness Matrix Graph + // debug print + // std::cout << "DOF GRAPH MATRIX ENTRIES ON TASK " << myrank << std::endl; + FOR_ALL_CLASS(idof, 0, num_dim * nlocal_nodes, { + for (int istride = 0; istride < Gradient_Matrix_Strides(idof); istride++) + { + DOF_Graph_Matrix(idof, istride) = Graph_Matrix(idof / num_dim, istride / num_dim) * num_dim + istride % num_dim; + } + }); // end parallel for + + /* + //construct distributed gradient matrix from local kokkos data + //build column map for the global gradient matrix + Teuchos::RCP > colmap; + const Teuchos::RCP > dommap = local_dof_map; + + Tpetra::Details::makeColMap(colmap,dommap,DOF_Graph_Matrix.get_kokkos_view(), nullptr); + + size_t nnz = DOF_Graph_Matrix.size(); + + //debug print + //std::cout << "DOF GRAPH SIZE ON RANK " << myrank << " IS " << nnz << std::endl; + + //local indices in the graph using the constructed column map + CArrayKokkos gradient_local_indices(nnz, "gradient_local_indices"); + + //row offsets with compatible template arguments + Kokkos::View row_offsets = DOF_Graph_Matrix.start_index_; + row_pointers row_offsets_pass("row_offsets", nlocal_nodes*num_dim+1); + for(int ipass = 0; ipass < nlocal_nodes*num_dim + 1; ipass++){ + row_offsets_pass(ipass) = row_offsets(ipass); + } + + size_t entrycount = 0; + for(int irow = 0; irow < nlocal_nodes*num_dim; irow++){ + for(int istride = 0; istride < Gradient_Matrix_Strides(irow); istride++){ + gradient_local_indices(entrycount) = colmap->getLocalElement(DOF_Graph_Matrix(irow,istride)); + entrycount++; + } + } + + + //sort values and indices + Tpetra::Import_Util::sortCrsEntries(row_offsets_pass, gradient_local_indices.get_kokkos_view(), Force_Gradient_Positions.get_kokkos_view()); + Tpetra::Import_Util::sortCrsEntries(row_offsets_pass, gradient_local_indices.get_kokkos_view(), Force_Gradient_Velocities.get_kokkos_view()); + + //Teuchos::RCP crs_matrix_params = Teuchos::rcp(new Teuchos::ParameterList("crsmatrix")); + //crs_matrix_params->set("sorted", false); + distributed_force_gradient_positions = Teuchos::rcp(new MAT(local_dof_map, colmap, row_offsets_pass, gradient_local_indices.get_kokkos_view(), Force_Gradient_Positions.get_kokkos_view())); + distributed_force_gradient_positions->fillComplete(); + distributed_force_gradient_velocities = Teuchos::rcp(new MAT(local_dof_map, colmap, row_offsets_pass, gradient_local_indices.get_kokkos_view(), Force_Gradient_Velocities.get_kokkos_view())); + distributed_force_gradient_velocities->fillComplete(); + */ + // distributed_force_gradient_positions->describe(*fos,Teuchos::VERB_EXTREME); + // distributed_force_gradient_velocities->describe(*fos,Teuchos::VERB_EXTREME); +} + +/* ---------------------------------------------------------------------- + Enforce boundary conditions on the adjoint vectors +------------------------------------------------------------------------- */ + +void FEA_Module_SGH::boundary_adjoint(const mesh_t& mesh, + const DCArrayKokkos& boundary, + vec_array& node_adjoint, + vec_array& node_phi_adjoint, + vec_array& node_psi_adjoint) +{ + // error and debug flag + // DCArrayKokkos print_flag(1, "print_flag"); + // print_flag.host(0) = false; + // print_flag.update_device(); + + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + int num_dims = simparam->num_dims; + // Loop over boundary sets + for (size_t bdy_set = 0; bdy_set < num_bdy_sets; bdy_set++) + { + // Loop over boundary nodes in a boundary set + FOR_ALL_CLASS(bdy_node_lid, 0, num_bdy_nodes_in_set.host(bdy_set), { + // reflected (boundary array is on the device) + if (boundary(bdy_set).type == BOUNDARY_CONDITION_TYPE::reflected) + { + // directions with hydro_bc: + // x_plane = 0, + // y_plane = 1, + // z_plane = 2, + size_t direction = boundary(bdy_set).surface.planar_surface_index(); + + size_t bdy_node_gid = bdy_nodes_in_set(bdy_set, bdy_node_lid); + + // Set velocity to zero in that directdion + if (bdy_node_gid < nlocal_nodes) + { + node_adjoint(bdy_node_gid, direction) = 0.0; + } + // node_phi_adjoint(bdy_node_gid, direction) = 0.0; + } + else if (boundary(bdy_set).type == BOUNDARY_CONDITION_TYPE::fixed_position) + { + size_t bdy_node_gid = bdy_nodes_in_set(bdy_set, bdy_node_lid); + + // debug clause + // if(bdy_node_gid==549412) print_flag(0) = true; + + for (size_t dim = 0; dim < num_dims; dim++) + { + // Set velocity to zero + if (bdy_node_gid < nlocal_nodes) + { + node_adjoint(bdy_node_gid, dim) = 0.0; + } + // node_phi_adjoint(bdy_node_gid, dim) = 0.0; + } + } + }); // end for bdy_node_lid + } // end for bdy_set + + // debug check + // print_flag.update_host(); + // if(print_flag.host(0)) std::cout << "found boundary node with id 549412" << std::endl; + return; +} // end boundary_velocity function + +/* ---------------------------------------------------------------------- + Communicate updated nodal adjoint vectors to ghost nodes +------------------------------------------------------------------------- */ + +void FEA_Module_SGH::comm_adjoint_vector(int cycle) +{ + // comms to get ghosts + (*adjoint_vector_data)[cycle]->doImport(*adjoint_vector_distributed, *importer, Tpetra::INSERT); +} + +/* ---------------------------------------------------------------------- + Communicate updated nodal adjoint vectors to ghost nodes +------------------------------------------------------------------------- */ + +void FEA_Module_SGH::comm_phi_adjoint_vector(int cycle) +{ + // comms to get ghosts + (*phi_adjoint_vector_data)[cycle]->doImport(*phi_adjoint_vector_distributed, *importer, Tpetra::INSERT); +} diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/time_integration.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/time_integration.cpp new file mode 100644 index 000000000..a934f2932 --- /dev/null +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/time_integration.cpp @@ -0,0 +1,231 @@ + +#include "mesh.h" +#include "state.h" +#include "FEA_Module_SGH.h" + +// ----------------------------------------------------------------------------- +// This function saves the variables at rk_stage = 0, which is t_n +// ------------------------------------------------------------------------------ +void FEA_Module_SGH::rk_init(DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_sie, + DViewCArrayKokkos& elem_stress, + const size_t num_elems, + const size_t num_nodes) +{ + const size_t rk_level = rk_num_bins - 1; + int num_dims = num_dim; + // save elem quantities + FOR_ALL_CLASS(elem_gid, 0, num_elems, { + // stress is always 3D even with 2D-RZ + for (size_t i = 0; i < 3; i++) + { + for (size_t j = 0; j < 3; j++) + { + elem_stress(0, elem_gid, i, j) = elem_stress(rk_level, elem_gid, i, j); + } + } // end for + + elem_sie(0, elem_gid) = elem_sie(rk_level, elem_gid); + }); // end parallel for + + // save nodal quantities + FOR_ALL_CLASS(node_gid, 0, num_nodes, { + for (size_t i = 0; i < num_dims; i++) + { + node_coords(0, node_gid, i) = node_coords(rk_level, node_gid, i); + node_vel(0, node_gid, i) = node_vel(rk_level, node_gid, i); + } + }); // end parallel for + Kokkos::fence(); + + return; +} // end rk_init + +// ----------------------------------------------------------------------------- +// This function calculates the time step by finding the shortest distance +// between any two nodes in the mesh +// ------------------------------------------------------------------------------ +// WARNING WARNING : Only works for 3D, 8 node elements +void FEA_Module_SGH::get_timestep(mesh_t& mesh, + DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_sspd, + DViewCArrayKokkos& elem_vol) +{ + const size_t rk_level = rk_num_bins - 1; + + // increase dt by 10%, that is the largest dt value + dt = dt * 1.1; + int num_dims = num_dim; + double dt_lcl; + double min_dt_calc; + REDUCE_MIN_CLASS(elem_gid, 0, rnum_elem, dt_lcl, { + double coords0[24]; // element coords + ViewCArrayKokkos coords(coords0, 8, 3); + + double distance0[28]; // array for holding distances between each node + ViewCArrayKokkos dist(distance0, 28); + + // Getting the coordinates of the element + for (size_t node_lid = 0; node_lid < 8; node_lid++) + { + for (size_t dim = 0; dim < num_dims; dim++) + { + coords(node_lid, dim) = node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), dim); + } // end for dim + } // end for loop over node_lid + + // loop conditions needed for distance calculation + size_t countA = 0; + size_t countB = 1; + size_t a; + size_t b; + size_t loop = 0; + + // Only works for 3D + // Solving for the magnitude of distance between each node + for (size_t i = 0; i < 28; i++) + { + a = countA; + b = countB; + + // returns magnitude of distance between each node, 28 total options + dist(i) = fabs(sqrt((pow((coords(b, 0) - coords(a, 0)), 2.0) + + pow((coords(b, 1) - coords(a, 1)), 2.0) + + pow((coords(b, 2) - coords(a, 2)), 2.0)))); + + countB++; + countA++; + + // tricky indexing + if (countB > 7) + { + loop++; + countB = 1 + loop; + countA = 0; + } + } // endo for i + + double dist_min = dist(0); + + for (int i = 0; i < 28; ++i) + { + dist_min = fmin(dist(i), dist_min); + } + + // local dt calc based on CFL + double dt_lcl_ = dt_cfl * dist_min / (elem_sspd(elem_gid) + fuzz); + + // make dt be in bounds + dt_lcl_ = fmin(dt_lcl_, dt_max); // make dt small than dt_max + dt_lcl_ = fmax(dt_lcl_, dt_min); // make dt larger than dt_min + + if (dt_lcl_ < dt_lcl) + { + dt_lcl = dt_lcl_; + } + }, min_dt_calc); // end parallel reduction + Kokkos::fence(); + + // save the min dt + if (min_dt_calc < dt) + { + dt = min_dt_calc; + } + + // ensure time step hits the graphics time intervals + dt = fmin(dt, (graphics_time - time_value) + fuzz); + + // make dt be exact for final time + dt = fmin(dt, time_final - time_value); + + return; +} // end get_timestep + +// ----------------------------------------------------------------------------- +// This function calculates the time step by finding the shortest distance +// between any two nodes in the mesh +// ------------------------------------------------------------------------------ +// WARNING WARNING : Only works for 3D, 8 node elements +void FEA_Module_SGH::get_timestep2D(mesh_t& mesh, + DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_sspd, + DViewCArrayKokkos& elem_vol) +{ + const size_t rk_level = rk_num_bins - 1; + + // increase dt by 10%, that is the largest dt value + dt = dt * 1.1; + int num_dims = num_dim; + double dt_lcl; + double min_dt_calc; + + REDUCE_MIN_CLASS(elem_gid, 0, rnum_elem, dt_lcl, { + double coords0[8]; // element coords + ViewCArrayKokkos coords(coords0, 4, 2); + + double distance0[6]; // array for holding distances between each node + ViewCArrayKokkos dist(distance0, 6); + + // Getting the coordinates of the nodes of the element + for (size_t node_lid = 0; node_lid < 4; node_lid++) + { + for (size_t dim = 0; dim < num_dims; dim++) + { + coords(node_lid, dim) = node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), dim); + } // end for dim + } // end for loop over node_lid + + // Only works for 2D + // Solving for the magnitude of distance between each node + size_t count = 0; + for (size_t i = 0; i < 3; i++) + { + for (size_t j = i + 1; j <= 3; j++) + { + // returns magnitude of distance between each node, 6 total options + dist(count) = fabs( + sqrt(pow((coords(i, 0) - coords(j, 0)), 2.0) + + pow((coords(i, 1) - coords(j, 1)), 2.0) ) + ); + count++; + } // end for j + } // end for i + + double dist_min = dist(0); + + for (int i = 0; i < 6; ++i) + { + dist_min = fmin(dist(i), dist_min); + } + + // local dt calc based on CFL + double dt_lcl_ = dt_cfl * dist_min / (elem_sspd(elem_gid) + fuzz); + + // make dt be in bounds + dt_lcl_ = fmin(dt_lcl_, dt_max); // make dt small than dt_max + dt_lcl_ = fmax(dt_lcl_, dt_min); // make dt larger than dt_min + + if (dt_lcl_ < dt_lcl) + { + dt_lcl = dt_lcl_; + } + }, min_dt_calc); // end parallel reduction + Kokkos::fence(); + + // save the min dt + if (min_dt_calc < dt) + { + dt = min_dt_calc; + } + + // ensure time step hits the graphics time intervals + dt = fmin(dt, (graphics_time - time_value) + fuzz); + + // make dt be exact for final time + dt = fmin(dt, time_final - time_value); + + return; +} // end get_timestep2D diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/state.h b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/state.h deleted file mode 100644 index 439f28cc2..000000000 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/state.h +++ /dev/null @@ -1,108 +0,0 @@ -#ifndef STATE_H -#define STATE_H - - -#include "matar.h" - -using namespace mtr; - -// node_state -struct node_t { - - // Position - DCArrayKokkos coords; - - // velocity - DCArrayKokkos vel; - - // mass at nodes - DCArrayKokkos mass; - - // Includes Ghost Positions - //DCArrayKokkos all_coords; - - // Includes Ghost velocities - //DCArrayKokkos all_vel; - - // Includes Ghost masses - //DCArrayKokkos all_mass; - - - // initialization method (num_rk_storage_bins, num_nodes, num_dims) - void initialize(size_t num_rk, size_t num_nodes, size_t num_dims) - { - this->coords = DCArrayKokkos (num_rk, num_nodes, num_dims); - this->vel = DCArrayKokkos (num_rk, num_nodes, num_dims); - this->mass = DCArrayKokkos (num_nodes); - }; // end method - -}; // end node_t - - -// elem_state -struct elem_t { - - // den - CArray den; - - // pres - CArray pres; - - // stress - CArray stress; - - // sspd - CArray sspd; - - // sie - CArray sie; - - // vol - CArray vol; - - // divergence of velocity - CArray div; - - // mass of elem - CArray mass; - - // mat ids - CArray mat_id; - - // initialization method (num_rk_storage_bins, num_cells, num_dims) - void initialize(size_t num_rk, size_t num_elems, size_t num_dims) - { - this->den = CArray (num_elems); - this->pres = CArray (num_elems); - this->stress = CArray (num_rk, num_elems, num_dims, num_dims); - this->sspd = CArray (num_elems); - this->sie = CArray (num_rk, num_elems); - this->vol = CArray (num_elems); - this->div = CArray (num_elems); - this->mass = CArray (num_elems); - this->mat_id = CArray (num_elems); - }; // end method - -}; // end elem_t - - -// corner_state -struct corner_t { - - // force - CArray force; - - // mass of corner - CArray mass; - - - // initialization method (num_corners, num_dims) - void initialize(size_t num_corners, size_t num_dims) - { - this->force = CArray (num_corners, num_dims); - this->mass = CArray (num_corners); - }; // end method - -}; // end corner_t - -#endif diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/time_integration.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/time_integration.cpp deleted file mode 100644 index 0d8d75ae0..000000000 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/time_integration.cpp +++ /dev/null @@ -1,237 +0,0 @@ - -#include "mesh.h" -#include "state.h" -#include "FEA_Module_SGH.h" - -// ----------------------------------------------------------------------------- -// This function saves the variables at rk_stage = 0, which is t_n -//------------------------------------------------------------------------------ -void FEA_Module_SGH::rk_init(DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &elem_sie, - DViewCArrayKokkos &elem_stress, - const size_t num_elems, - const size_t num_nodes){ - - const size_t rk_level = rk_num_bins - 1; - int num_dims = num_dim; - // save elem quantities - FOR_ALL_CLASS(elem_gid, 0, num_elems, { - - // stress is always 3D even with 2D-RZ - for(size_t i=0; i<3; i++){ - for(size_t j=0; j<3; j++){ - elem_stress(0,elem_gid,i,j) = elem_stress(rk_level,elem_gid,i,j); - } - } // end for - - elem_sie(0,elem_gid) = elem_sie(rk_level,elem_gid); - - }); // end parallel for - - - // save nodal quantities - FOR_ALL_CLASS(node_gid, 0, num_nodes, { - - for(size_t i=0; i &node_coords, - DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &elem_sspd, - DViewCArrayKokkos &elem_vol){ - - - const size_t rk_level = rk_num_bins - 1; - - // increase dt by 10%, that is the largest dt value - dt = dt*1.1; - int num_dims = num_dim; - double dt_lcl; - double min_dt_calc; - REDUCE_MIN_CLASS(elem_gid, 0, rnum_elem, dt_lcl, { - - double coords0[24]; // element coords - ViewCArrayKokkos coords(coords0, 8, 3); - - - double distance0[28]; // array for holding distances between each node - ViewCArrayKokkos dist(distance0, 28); - - // Getting the coordinates of the element - for(size_t node_lid = 0; node_lid < 8; node_lid++){ - - for (size_t dim = 0; dim < num_dims; dim++){ - coords(node_lid, dim) = node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), dim); - } // end for dim - - } // end for loop over node_lid - - // loop conditions needed for distance calculation - size_t countA = 0; - size_t countB = 1; - size_t a; - size_t b; - size_t loop = 0; - - // Only works for 3D - // Solving for the magnitude of distance between each node - for (size_t i = 0; i < 28; i++){ - - a = countA; - b = countB; - - // returns magnitude of distance between each node, 28 total options - dist(i) = fabs(sqrt(( pow((coords(b, 0) - coords(a, 0)), 2.0) - + pow((coords(b, 1) - coords(a, 1)), 2.0) - + pow((coords(b, 2) - coords(a, 2)), 2.0)))); - - countB++; - countA++; - - // tricky indexing - if (countB > 7) { - loop++; - countB = 1 + loop; - countA = 0; - } - - } // endo for i - - - double dist_min = dist(0); - - for(int i = 0; i < 28; ++i){ - dist_min = fmin(dist(i), dist_min); - } - - // local dt calc based on CFL - double dt_lcl_ = dt_cfl*dist_min/(elem_sspd(elem_gid) + fuzz); - - // make dt be in bounds - dt_lcl_ = fmin(dt_lcl_, dt_max); // make dt small than dt_max - dt_lcl_ = fmax(dt_lcl_, dt_min); // make dt larger than dt_min - - - if (dt_lcl_ < dt_lcl) dt_lcl = dt_lcl_; - - }, min_dt_calc); // end parallel reduction - Kokkos::fence(); - - // save the min dt - if(min_dt_calc < dt) dt = min_dt_calc; - - // ensure time step hits the graphics time intervals - dt = fmin(dt, (graphics_time - time_value)+fuzz); - - // make dt be exact for final time - dt = fmin(dt, time_final-time_value); - - return; - -} // end get_timestep - -// ----------------------------------------------------------------------------- -// This function calculates the time step by finding the shortest distance -// between any two nodes in the mesh -//------------------------------------------------------------------------------ -// WARNING WARNING : Only works for 3D, 8 node elements -void FEA_Module_SGH::get_timestep2D(mesh_t &mesh, - DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &elem_sspd, - DViewCArrayKokkos &elem_vol){ - - const size_t rk_level = rk_num_bins - 1; - - // increase dt by 10%, that is the largest dt value - dt = dt*1.1; - int num_dims = num_dim; - double dt_lcl; - double min_dt_calc; - - REDUCE_MIN_CLASS(elem_gid, 0, rnum_elem, dt_lcl, { - - double coords0[8]; // element coords - ViewCArrayKokkos coords(coords0, 4, 2); - - - double distance0[6]; // array for holding distances between each node - ViewCArrayKokkos dist(distance0, 6); - - // Getting the coordinates of the nodes of the element - for(size_t node_lid = 0; node_lid < 4; node_lid++){ - - for (size_t dim = 0; dim < num_dims; dim++){ - coords(node_lid, dim) = node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), dim); - } // end for dim - - } // end for loop over node_lid - - - // Only works for 2D - // Solving for the magnitude of distance between each node - size_t count = 0; - for (size_t i = 0; i < 3; i++){ - for (size_t j=i+1; j<=3; j++){ - - // returns magnitude of distance between each node, 6 total options - dist(count) = fabs( - sqrt( pow((coords(i, 0) - coords(j, 0)), 2.0) - + pow((coords(i, 1) - coords(j, 1)), 2.0) ) - ); - count ++; - } // end for j - } // end for i - - - double dist_min = dist(0); - - for(int i = 0; i < 6; ++i){ - dist_min = fmin(dist(i), dist_min); - } - - // local dt calc based on CFL - double dt_lcl_ = dt_cfl*dist_min/(elem_sspd(elem_gid) + fuzz); - - // make dt be in bounds - dt_lcl_ = fmin(dt_lcl_, dt_max); // make dt small than dt_max - dt_lcl_ = fmax(dt_lcl_, dt_min); // make dt larger than dt_min - - - if (dt_lcl_ < dt_lcl) dt_lcl = dt_lcl_; - - }, min_dt_calc); // end parallel reduction - Kokkos::fence(); - - // save the min dt - if(min_dt_calc < dt) dt = min_dt_calc; - - // ensure time step hits the graphics time intervals - dt = fmin(dt, (graphics_time - time_value)+fuzz); - - // make dt be exact for final time - dt = fmin(dt, time_final-time_value); - - return; - -} // end get_timestep2D - - diff --git a/src/Parallel-Solvers/Parallel-Explicit/Topology_Optimization/Kinetic_Energy_Minimize.h b/src/Parallel-Solvers/Parallel-Explicit/Topology_Optimization/Kinetic_Energy_Minimize.h index ff255f1f0..4ac860c6b 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Topology_Optimization/Kinetic_Energy_Minimize.h +++ b/src/Parallel-Solvers/Parallel-Explicit/Topology_Optimization/Kinetic_Energy_Minimize.h @@ -11,14 +11,14 @@ This program is open source under the BSD-3 License. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - + 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - + 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - + 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. @@ -34,7 +34,7 @@ OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. **********************************************************************************************/ - + #ifndef KINETIC_ENERGY_MINIMIZE_TOPOPT_H #define KINETIC_ENERGY_MINIMIZE_TOPOPT_H @@ -61,312 +61,360 @@ #include "FEA_Module_Dynamic_Elasticity.h" #include "Explicit_Solver.h" -class KineticEnergyMinimize_TopOpt : public ROL::Objective { - - typedef Tpetra::Map<>::local_ordinal_type LO; - typedef Tpetra::Map<>::global_ordinal_type GO; - typedef Tpetra::Map<>::node_type Node; - typedef Tpetra::Map Map; - typedef Tpetra::MultiVector MV; - typedef ROL::Vector V; - typedef ROL::TpetraMultiVector ROL_MV; - - using traits = Kokkos::ViewTraits; - using array_layout = typename traits::array_layout; - using execution_space = typename traits::execution_space; - using device_type = typename traits::device_type; - using memory_traits = typename traits::memory_traits; - using global_size_t = Tpetra::global_size_t; - - typedef Kokkos::View values_array; - typedef Kokkos::View global_indices_array; - typedef Kokkos::View indices_array; - - //typedef Kokkos::DualView::t_dev vec_array; - typedef MV::dual_view_type::t_dev vec_array; - typedef MV::dual_view_type::t_host host_vec_array; - typedef Kokkos::View const_host_vec_array; - typedef Kokkos::View const_vec_array; - typedef MV::dual_view_type dual_vec_array; +class KineticEnergyMinimize_TopOpt : public ROL::Objective +{ +typedef Tpetra::Map<>::local_ordinal_type LO; +typedef Tpetra::Map<>::global_ordinal_type GO; +typedef Tpetra::Map<>::node_type Node; +typedef Tpetra::Map Map; +typedef Tpetra::MultiVector MV; +typedef ROL::Vector V; +typedef ROL::TpetraMultiVector ROL_MV; + +using traits = Kokkos::ViewTraits; +using array_layout = typename traits::array_layout; +using execution_space = typename traits::execution_space; +using device_type = typename traits::device_type; +using memory_traits = typename traits::memory_traits; +using global_size_t = Tpetra::global_size_t; + +typedef Kokkos::View values_array; +typedef Kokkos::View global_indices_array; +typedef Kokkos::View indices_array; + +// typedef Kokkos::DualView::t_dev vec_array; +typedef MV::dual_view_type::t_dev vec_array; +typedef MV::dual_view_type::t_host host_vec_array; +typedef Kokkos::View const_host_vec_array; +typedef Kokkos::View const_vec_array; +typedef MV::dual_view_type dual_vec_array; private: - Explicit_Solver *Explicit_Solver_Pointer_; - FEA_Module_SGH *FEM_SGH_; - FEA_Module_Dynamic_Elasticity *FEM_Dynamic_Elasticity_; - ROL::Ptr ROL_Force; - ROL::Ptr ROL_Velocities; - ROL::Ptr ROL_Gradients; + Explicit_Solver* Explicit_Solver_Pointer_; + FEA_Module_SGH* FEM_SGH_; + FEA_Module_Dynamic_Elasticity* FEM_Dynamic_Elasticity_; + ROL::Ptr ROL_Force; + ROL::Ptr ROL_Velocities; + ROL::Ptr ROL_Gradients; - bool useLC_; // Use linear form of energy. Otherwise use quadratic form. + bool useLC_; // Use linear form of energy. Otherwise use quadratic form. - ROL::Ptr getVector( const V& x ) { - return dynamic_cast(x).getVector(); - } + ROL::Ptr getVector(const V& x) + { + return dynamic_cast(x).getVector(); + } - ROL::Ptr getVector( V& x ) { - return dynamic_cast(x).getVector(); - } + ROL::Ptr getVector(V& x) + { + return dynamic_cast(x).getVector(); + } public: - bool nodal_density_flag_, time_accumulation; - int last_comm_step, last_solve_step, current_step; - size_t nvalid_modules; - std::vector valid_fea_modules; //modules that may interface with this objective function - FEA_MODULE_TYPE set_module_type; - //std::string my_fea_module = "SGH"; - real_t objective_accumulation; - - KineticEnergyMinimize_TopOpt(Explicit_Solver *Explicit_Solver_Pointer, bool nodal_density_flag) - : useLC_(true) { - Explicit_Solver_Pointer_ = Explicit_Solver_Pointer; - - valid_fea_modules.push_back(FEA_MODULE_TYPE::SGH); - valid_fea_modules.push_back(FEA_MODULE_TYPE::Dynamic_Elasticity); - nvalid_modules = valid_fea_modules.size(); - - const Simulation_Parameters& simparam = Explicit_Solver_Pointer_->simparam; - for (const auto& fea_module : Explicit_Solver_Pointer_->fea_modules) { - for(int ivalid = 0; ivalid < nvalid_modules; ivalid++){ - if(fea_module->Module_Type==FEA_MODULE_TYPE::SGH){ - FEM_SGH_ = dynamic_cast(fea_module); - set_module_type = FEA_MODULE_TYPE::SGH; - } - if(fea_module->Module_Type==FEA_MODULE_TYPE::Dynamic_Elasticity){ - FEM_Dynamic_Elasticity_ = dynamic_cast(fea_module); - set_module_type = FEA_MODULE_TYPE::Dynamic_Elasticity; - } + bool nodal_density_flag_, time_accumulation; + int last_comm_step, last_solve_step, current_step; + size_t nvalid_modules; + std::vector valid_fea_modules; // modules that may interface with this objective function + FEA_MODULE_TYPE set_module_type; + // std::string my_fea_module = "SGH"; + real_t objective_accumulation; + + KineticEnergyMinimize_TopOpt(Explicit_Solver* Explicit_Solver_Pointer, bool nodal_density_flag) + : useLC_(true) + { + Explicit_Solver_Pointer_ = Explicit_Solver_Pointer; + + valid_fea_modules.push_back(FEA_MODULE_TYPE::SGH); + valid_fea_modules.push_back(FEA_MODULE_TYPE::Dynamic_Elasticity); + nvalid_modules = valid_fea_modules.size(); + + const Simulation_Parameters& simparam = Explicit_Solver_Pointer_->simparam; + for (const auto& fea_module : Explicit_Solver_Pointer_->fea_modules) + { + for (int ivalid = 0; ivalid < nvalid_modules; ivalid++) + { + if (fea_module->Module_Type == FEA_MODULE_TYPE::SGH) + { + FEM_SGH_ = dynamic_cast(fea_module); + set_module_type = FEA_MODULE_TYPE::SGH; + } + if (fea_module->Module_Type == FEA_MODULE_TYPE::Dynamic_Elasticity) + { + FEM_Dynamic_Elasticity_ = dynamic_cast(fea_module); + set_module_type = FEA_MODULE_TYPE::Dynamic_Elasticity; + } + } + } + nodal_density_flag_ = nodal_density_flag; + last_comm_step = last_solve_step = -1; + current_step = 0; + time_accumulation = true; + objective_accumulation = 0; + + // ROL_Force = ROL::makePtr(FEM_->Global_Nodal_Forces); + if (set_module_type == FEA_MODULE_TYPE::SGH) + { + ROL_Velocities = ROL::makePtr(FEM_SGH_->node_velocities_distributed); + } + if (set_module_type == FEA_MODULE_TYPE::Dynamic_Elasticity) + { + ROL_Velocities = ROL::makePtr(FEM_Dynamic_Elasticity_->node_velocities_distributed); } - } - nodal_density_flag_ = nodal_density_flag; - last_comm_step = last_solve_step = -1; - current_step = 0; - time_accumulation = true; - objective_accumulation = 0; - - //ROL_Force = ROL::makePtr(FEM_->Global_Nodal_Forces); - if(set_module_type==FEA_MODULE_TYPE::SGH) - ROL_Velocities = ROL::makePtr(FEM_SGH_->node_velocities_distributed); - if(set_module_type==FEA_MODULE_TYPE::Dynamic_Elasticity) - ROL_Velocities = ROL::makePtr(FEM_Dynamic_Elasticity_->node_velocities_distributed); - - //real_t current_kinetic_energy = ROL_Velocities->dot(*ROL_Force)/2; - //std::cout.precision(10); - //if(FEM_->myrank==0) - //std::cout << "INITIAL KINETIC ENERGY " << current_kinetic_energy << std::endl; - } - - void update(const ROL::Vector &z, ROL::UpdateType type, int iter = -1 ) { - if(set_module_type==FEA_MODULE_TYPE::SGH) - update_sgh(z,type,iter); - if(set_module_type==FEA_MODULE_TYPE::Dynamic_Elasticity) - update_elasticity(z,type,iter); - } - - void update_elasticity(const ROL::Vector &z, ROL::UpdateType type, int iter = -1 ) { - - //debug - std::ostream &out = std::cout; - Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - - current_step++; - ROL::Ptr zp = getVector(z); - const_host_vec_array design_densities = zp->getLocalView (Tpetra::Access::ReadOnly); - - if (type == ROL::UpdateType::Initial) { - // This is the first call to update - //first linear solve was done in FEA class run function already - FEM_Dynamic_Elasticity_->elastic_solve(); - //initial design density data was already communicated for ghost nodes in init_design() - //decide to output current optimization state - FEM_Dynamic_Elasticity_->Explicit_Solver_Pointer_->write_outputs(); - } - else if (type == ROL::UpdateType::Accept) { + // real_t current_kinetic_energy = ROL_Velocities->dot(*ROL_Force)/2; + // std::cout.precision(10); + // if(FEM_->myrank==0) + // std::cout << "INITIAL KINETIC ENERGY " << current_kinetic_energy << std::endl; } - else if (type == ROL::UpdateType::Revert) { - // u_ was set to u=S(x) during a trial update - // and has been rejected as the new iterate - // Revert to cached value - // This is a new value of x - //communicate density variables for ghosts - FEM_Dynamic_Elasticity_->comm_variables(zp); - //update deformation variables - FEM_Dynamic_Elasticity_->update_forward_solve(zp); - if(Explicit_Solver_Pointer_->myrank==0) - *fos << "called Revert" << std::endl; - } - else if (type == ROL::UpdateType::Trial) { - // This is a new value of x - //communicate density variables for ghosts - FEM_Dynamic_Elasticity_->comm_variables(zp); - //update deformation variables - FEM_Dynamic_Elasticity_->update_forward_solve(zp); - if(Explicit_Solver_Pointer_->myrank==0) - *fos << "called Trial" << std::endl; - - //decide to output current optimization state - FEM_Dynamic_Elasticity_->Explicit_Solver_Pointer_->write_outputs(); - } - else { // ROL::UpdateType::Temp - // This is a new value of x used for, - // e.g., finite-difference checks - if(Explicit_Solver_Pointer_->myrank==0) - *fos << "called Temp" << std::endl; - FEM_Dynamic_Elasticity_->comm_variables(zp); - FEM_Dynamic_Elasticity_->update_forward_solve(zp); - } - } - - void update_sgh(const ROL::Vector &z, ROL::UpdateType type, int iter = -1 ) { - //debug - std::ostream &out = std::cout; - Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - - current_step++; - ROL::Ptr zp = getVector(z); - const_host_vec_array design_densities = zp->getLocalView (Tpetra::Access::ReadOnly); - - if (type == ROL::UpdateType::Initial) { - // This is the first call to update - //first linear solve was done in FEA class run function already - FEM_SGH_->sgh_solve(); - //initial design density data was already communicated for ghost nodes in init_design() - //decide to output current optimization state - FEM_SGH_->Explicit_Solver_Pointer_->write_outputs(); - } - else if (type == ROL::UpdateType::Accept) { + void update(const ROL::Vector& z, ROL::UpdateType type, int iter = -1) + { + if (set_module_type == FEA_MODULE_TYPE::SGH) + { + update_sgh(z, type, iter); + } + if (set_module_type == FEA_MODULE_TYPE::Dynamic_Elasticity) + { + update_elasticity(z, type, iter); + } } - else if (type == ROL::UpdateType::Revert) { - // u_ was set to u=S(x) during a trial update - // and has been rejected as the new iterate - // Revert to cached value - // This is a new value of x - //communicate density variables for ghosts - FEM_SGH_->comm_variables(zp); - //update deformation variables - FEM_SGH_->update_forward_solve(zp); - if(Explicit_Solver_Pointer_->myrank==0) - *fos << "called Revert" << std::endl; - } - else if (type == ROL::UpdateType::Trial) { - // This is a new value of x - //communicate density variables for ghosts - FEM_SGH_->comm_variables(zp); - //update deformation variables - FEM_SGH_->update_forward_solve(zp); - if(Explicit_Solver_Pointer_->myrank==0) - *fos << "called Trial" << std::endl; - - //decide to output current optimization state - FEM_SGH_->Explicit_Solver_Pointer_->write_outputs(); + + void update_elasticity(const ROL::Vector& z, ROL::UpdateType type, int iter = -1) + { + // debug + std::ostream& out = std::cout; + Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + + current_step++; + ROL::Ptr zp = getVector(z); + const_host_vec_array design_densities = zp->getLocalView(Tpetra::Access::ReadOnly); + + if (type == ROL::UpdateType::Initial) + { + // This is the first call to update + // first linear solve was done in FEA class run function already + FEM_Dynamic_Elasticity_->elastic_solve(); + // initial design density data was already communicated for ghost nodes in init_design() + // decide to output current optimization state + FEM_Dynamic_Elasticity_->Explicit_Solver_Pointer_->write_outputs(); + } + else if (type == ROL::UpdateType::Accept) + { + } + else if (type == ROL::UpdateType::Revert) + { + // u_ was set to u=S(x) during a trial update + // and has been rejected as the new iterate + // Revert to cached value + // This is a new value of x + // communicate density variables for ghosts + FEM_Dynamic_Elasticity_->comm_variables(zp); + // update deformation variables + FEM_Dynamic_Elasticity_->update_forward_solve(zp); + if (Explicit_Solver_Pointer_->myrank == 0) + { + *fos << "called Revert" << std::endl; + } + } + else if (type == ROL::UpdateType::Trial) + { + // This is a new value of x + // communicate density variables for ghosts + FEM_Dynamic_Elasticity_->comm_variables(zp); + // update deformation variables + FEM_Dynamic_Elasticity_->update_forward_solve(zp); + if (Explicit_Solver_Pointer_->myrank == 0) + { + *fos << "called Trial" << std::endl; + } + + // decide to output current optimization state + FEM_Dynamic_Elasticity_->Explicit_Solver_Pointer_->write_outputs(); + } + else // ROL::UpdateType::Temp + // This is a new value of x used for, + // e.g., finite-difference checks + { + if (Explicit_Solver_Pointer_->myrank == 0) + { + *fos << "called Temp" << std::endl; + } + FEM_Dynamic_Elasticity_->comm_variables(zp); + FEM_Dynamic_Elasticity_->update_forward_solve(zp); + } } - else { // ROL::UpdateType::Temp - // This is a new value of x used for, - // e.g., finite-difference checks - if(Explicit_Solver_Pointer_->myrank==0) - *fos << "called Temp" << std::endl; - FEM_SGH_->comm_variables(zp); - FEM_SGH_->update_forward_solve(zp); + + void update_sgh(const ROL::Vector& z, ROL::UpdateType type, int iter = -1) + { + // debug + std::ostream& out = std::cout; + Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + + current_step++; + ROL::Ptr zp = getVector(z); + const_host_vec_array design_densities = zp->getLocalView(Tpetra::Access::ReadOnly); + + if (type == ROL::UpdateType::Initial) + { + // This is the first call to update + // first linear solve was done in FEA class run function already + FEM_SGH_->sgh_solve(); + // initial design density data was already communicated for ghost nodes in init_design() + // decide to output current optimization state + FEM_SGH_->Explicit_Solver_Pointer_->write_outputs(); + } + else if (type == ROL::UpdateType::Accept) + { + } + else if (type == ROL::UpdateType::Revert) + { + // u_ was set to u=S(x) during a trial update + // and has been rejected as the new iterate + // Revert to cached value + // This is a new value of x + // communicate density variables for ghosts + FEM_SGH_->comm_variables(zp); + // update deformation variables + FEM_SGH_->update_forward_solve(zp); + if (Explicit_Solver_Pointer_->myrank == 0) + { + *fos << "called Revert" << std::endl; + } + } + else if (type == ROL::UpdateType::Trial) + { + // This is a new value of x + // communicate density variables for ghosts + FEM_SGH_->comm_variables(zp); + // update deformation variables + FEM_SGH_->update_forward_solve(zp); + if (Explicit_Solver_Pointer_->myrank == 0) + { + *fos << "called Trial" << std::endl; + } + + // decide to output current optimization state + FEM_SGH_->Explicit_Solver_Pointer_->write_outputs(); + } + else // ROL::UpdateType::Temp + // This is a new value of x used for, + // e.g., finite-difference checks + { + if (Explicit_Solver_Pointer_->myrank == 0) + { + *fos << "called Temp" << std::endl; + } + FEM_SGH_->comm_variables(zp); + FEM_SGH_->update_forward_solve(zp); + } } - } - - real_t value(const ROL::Vector &z, real_t &tol) { - //std::cout << "Started obj value on task " <myrank << std::endl; - ROL::Ptr zp = getVector(z); - real_t c = 0.0; - - //debug print - //std::ostream &out = std::cout; - //Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - //if(FEM_->myrank==0) - //*fos << "Value function z:" << std::endl; - //zp->describe(*fos,Teuchos::VERB_EXTREME); - //*fos << std::endl; - //std::fflush(stdout); - - const_host_vec_array design_densities = zp->getLocalView (Tpetra::Access::ReadOnly); - //communicate ghosts and solve for nodal degrees of freedom as a function of the current design variables - /* - if(last_comm_step!=current_step){ - FEM_->comm_variables(zp); - last_comm_step = current_step; + + real_t value(const ROL::Vector& z, real_t& tol) + { + // std::cout << "Started obj value on task " <myrank << std::endl; + ROL::Ptr zp = getVector(z); + real_t c = 0.0; + + // debug print + // std::ostream &out = std::cout; + // Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + // if(FEM_->myrank==0) + // *fos << "Value function z:" << std::endl; + // zp->describe(*fos,Teuchos::VERB_EXTREME); + // *fos << std::endl; + // std::fflush(stdout); + + const_host_vec_array design_densities = zp->getLocalView(Tpetra::Access::ReadOnly); + // communicate ghosts and solve for nodal degrees of freedom as a function of the current design variables + /* + if(last_comm_step!=current_step){ + FEM_->comm_variables(zp); + last_comm_step = current_step; + } + + if(last_solve_step!=current_step){ + //std::cout << "UPDATED velocities" << std::endl; + FEM_->update_linear_solve(zp); + last_solve_step = current_step; + } + */ + // debug print of velocities + // std::ostream &out = std::cout; + // Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + // if(FEM_->myrank==0) + // *fos << "Displacement data :" << std::endl; + // FEM_->node_velocities_distributed->describe(*fos,Teuchos::VERB_EXTREME); + // *fos << std::endl; + // std::fflush(stdout); + + // ROL_Force = ROL::makePtr(FEM_->Global_Nodal_Forces); + if (set_module_type == FEA_MODULE_TYPE::SGH) + { + ROL_Velocities = ROL::makePtr(FEM_SGH_->node_velocities_distributed); + } + if (set_module_type == FEA_MODULE_TYPE::Dynamic_Elasticity) + { + ROL_Velocities = ROL::makePtr(FEM_Dynamic_Elasticity_->node_velocities_distributed); + } + + std::cout.precision(10); + if (Explicit_Solver_Pointer_->myrank == 0) + { + std::cout << "CURRENT TIME INTEGRAL OF KINETIC ENERGY " << objective_accumulation << std::endl; + } + + // std::cout << "Ended obj value on task " <myrank << std::endl; + return objective_accumulation; } - - if(last_solve_step!=current_step){ - //std::cout << "UPDATED velocities" << std::endl; - FEM_->update_linear_solve(zp); - last_solve_step = current_step; + + // void gradient_1( ROL::Vector &g, const ROL::Vector &u, const ROL::Vector &z, real_t &tol ) { + // g.zero(); + // } + + void gradient(ROL::Vector& g, const ROL::Vector& z, real_t& tol) + { + // std::cout << "Started obj gradient on task " <myrank << std::endl; + // get Tpetra multivector pointer from the ROL vector + ROL::Ptr zp = getVector(z); + ROL::Ptr gp = getVector(g); + + // communicate ghosts and solve for nodal degrees of freedom as a function of the current design variables + // FEM_->gradient_print_sync=1; + // FEM_->gradient_print_sync=0; + // get local view of the data + + if (set_module_type == FEA_MODULE_TYPE::SGH) + { + FEM_SGH_->compute_topology_optimization_gradient_full(zp, gp); + } + if (set_module_type == FEA_MODULE_TYPE::Dynamic_Elasticity) + { + FEM_Dynamic_Elasticity_->compute_topology_optimization_gradient_full(zp, gp); + } + // debug print of gradient + // std::ostream &out = std::cout; + // Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + // if(FEM_->myrank==0) + // *fos << "Gradient data :" << std::endl; + // gp->describe(*fos,Teuchos::VERB_EXTREME); + // *fos << std::endl; + // std::fflush(stdout); + // for(int i = 0; i < FEM_->nlocal_nodes; i++){ + // objective_gradients(i,0) *= -1; + // } + + // std::cout << "Objective Gradient called"<< std::endl; + // debug print of design variables + // std::ostream &out = std::cout; + // Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + // if(FEM_->myrank==0) + // *fos << "Gradient data :" << std::endl; + // gp->describe(*fos,Teuchos::VERB_EXTREME); + + // *fos << std::endl; + // std::fflush(stdout); + // std::cout << "ended obj gradient on task " <myrank << std::endl; } - */ - //debug print of velocities - //std::ostream &out = std::cout; - //Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - //if(FEM_->myrank==0) - //*fos << "Displacement data :" << std::endl; - //FEM_->node_velocities_distributed->describe(*fos,Teuchos::VERB_EXTREME); - //*fos << std::endl; - //std::fflush(stdout); - - //ROL_Force = ROL::makePtr(FEM_->Global_Nodal_Forces); - if(set_module_type==FEA_MODULE_TYPE::SGH) - ROL_Velocities = ROL::makePtr(FEM_SGH_->node_velocities_distributed); - if(set_module_type==FEA_MODULE_TYPE::Dynamic_Elasticity) - ROL_Velocities = ROL::makePtr(FEM_Dynamic_Elasticity_->node_velocities_distributed); - - std::cout.precision(10); - if(Explicit_Solver_Pointer_->myrank==0) - std::cout << "CURRENT TIME INTEGRAL OF KINETIC ENERGY " << objective_accumulation << std::endl; - - //std::cout << "Ended obj value on task " <myrank << std::endl; - return objective_accumulation; - } - - //void gradient_1( ROL::Vector &g, const ROL::Vector &u, const ROL::Vector &z, real_t &tol ) { - //g.zero(); - //} - - void gradient( ROL::Vector &g, const ROL::Vector &z, real_t &tol ) { - //std::cout << "Started obj gradient on task " <myrank << std::endl; - //get Tpetra multivector pointer from the ROL vector - ROL::Ptr zp = getVector(z); - ROL::Ptr gp = getVector(g); - - //communicate ghosts and solve for nodal degrees of freedom as a function of the current design variables - //FEM_->gradient_print_sync=1; - //FEM_->gradient_print_sync=0; - //get local view of the data - - if(set_module_type==FEA_MODULE_TYPE::SGH) - FEM_SGH_->compute_topology_optimization_gradient_full(zp,gp); - if(set_module_type==FEA_MODULE_TYPE::Dynamic_Elasticity) - FEM_Dynamic_Elasticity_->compute_topology_optimization_gradient_full(zp,gp); - //debug print of gradient - //std::ostream &out = std::cout; - //Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - //if(FEM_->myrank==0) - //*fos << "Gradient data :" << std::endl; - //gp->describe(*fos,Teuchos::VERB_EXTREME); - //*fos << std::endl; - //std::fflush(stdout); - //for(int i = 0; i < FEM_->nlocal_nodes; i++){ - //objective_gradients(i,0) *= -1; - //} - - //std::cout << "Objective Gradient called"<< std::endl; - //debug print of design variables - //std::ostream &out = std::cout; - //Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - //if(FEM_->myrank==0) - //*fos << "Gradient data :" << std::endl; - //gp->describe(*fos,Teuchos::VERB_EXTREME); - - //*fos << std::endl; - //std::fflush(stdout); - //std::cout << "ended obj gradient on task " <myrank << std::endl; - } - }; #endif // end header guard diff --git a/src/Parallel-Solvers/Parallel-Explicit/example_simple.yaml b/src/Parallel-Solvers/Parallel-Explicit/example_simple.yaml index 86c77388a..c3c68db69 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/example_simple.yaml +++ b/src/Parallel-Solvers/Parallel-Explicit/example_simple.yaml @@ -10,7 +10,7 @@ mesh_generation_options: type: Box origin: [0, 0, 0] length: [1.2, 1.2, 1.2] - num_elems: [32, 32, 32] + num_elems: [40, 40, 40] output_options: timer_output_level: thorough @@ -87,7 +87,7 @@ regions: - volume: type: sphere radius1: 0.0 - radius2: 0.0375 + radius2: 0.03 material_id: 0 den: 1.0 ie: 0.25833839995946534 diff --git a/src/Parallel-Solvers/Parallel-Explicit/main.cpp b/src/Parallel-Solvers/Parallel-Explicit/main.cpp index 3e1ff63dc..8f0d8151b 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/main.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/main.cpp @@ -11,14 +11,14 @@ This program is open source under the BSD-3 License. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - + 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - + 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - + 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. @@ -33,8 +33,8 @@ WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - **********************************************************************************************/ - +**********************************************************************************************/ + #include #include #include @@ -47,37 +47,49 @@ #include #include -void solver_setup(const char* filename){ - Explicit_Solver solver( - Yaml::from_file_strict(filename) - ); - //checks for optional solver routines - if(solver.setup_flag) solver.solver_setup(); - // invoke solver's run function (should perform most of the computation) - solver.run(); - //invoke optional finalize function - if(solver.finalize_flag) solver.solver_finalize(); +void solver_setup(const char* filename) +{ + Explicit_Solver solver( + Yaml::from_file_strict(filename) + ); + // checks for optional solver routines + if (solver.setup_flag) + { + solver.solver_setup(); + } + // invoke solver's run function (should perform most of the computation) + solver.run(); + // invoke optional finalize function + if (solver.finalize_flag) + { + solver.solver_finalize(); + } } -int main(int argc, char *argv[]){ - //initialize MPI - MPI_Init(&argc, &argv); +int main(int argc, char* argv[]) +{ + // initialize MPI + MPI_Init(&argc, &argv); + + Kokkos::initialize(); + if (argc < 2) + { + int myrank; + MPI_Comm_rank(MPI_COMM_WORLD, &myrank); + if (myrank == 0) + { + std::cout << "Fierro requires a yaml setup file as a command line argument" << std::endl; + } + } + else + { + solver_setup(argv[1]); + } + + MPI_Barrier(MPI_COMM_WORLD); - Kokkos::initialize(); - if(argc<2){ - int myrank; - MPI_Comm_rank(MPI_COMM_WORLD,&myrank); - if(myrank==0) - std::cout << "Fierro requires a yaml setup file as a command line argument" << std::endl; - } - else{ - solver_setup(argv[1]); - } - - MPI_Barrier(MPI_COMM_WORLD); - - Kokkos::finalize(); - MPI_Finalize(); + Kokkos::finalize(); + MPI_Finalize(); - return 0; + return 0; } diff --git a/src/Parallel-Solvers/Parallel-Explicit/outputs.cpp b/src/Parallel-Solvers/Parallel-Explicit/outputs.cpp index 2c2cf1702..6c7261e1b 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/outputs.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/outputs.cpp @@ -8,716 +8,803 @@ typedef Kokkos::LayoutRight CArrayLayout; typedef Kokkos::LayoutLeft FArrayLayout; typedef Tpetra::MultiVector<>::dual_view_type::t_host::array_layout TpetraHostViewLayout; -MPI_Offset -mpi_get_file_position_shared( - const MPI_Comm comm, - const MPI_File file_parallel); - -void -write_string_stream_to_file_mpi_all( - const std::stringstream& str_stream, - MPI_Offset rank_offset_multiplier, - const MPI_File file_parallel, - const MPI_Comm comm); - -template -void -write_data_to_string_stream( - const T* ptr, - size_t dim0, - size_t dim1, - std::stringstream& str_stream); +MPI_Offset mpi_get_file_position_shared( + const MPI_Comm comm, + const MPI_File file_parallel); + +void write_string_stream_to_file_mpi_all( + const std::stringstream& str_stream, + MPI_Offset rank_offset_multiplier, + const MPI_File file_parallel, + const MPI_Comm comm); + +template +void write_data_to_string_stream( + const T* ptr, + size_t dim0, + size_t dim1, + std::stringstream& str_stream); template -Teuchos::RCP> -sort_data( - const SC* unsorted_data_ptr, - Teuchos::RCP> unsorted_map, - size_t dim1, - size_t num_global_unique_elements, - MPI_Comm comm, - Teuchos::RCP> &sorting_importer); +Teuchos::RCP> sort_data( + const SC* unsorted_data_ptr, + Teuchos::RCP> unsorted_map, + size_t dim1, + size_t num_global_unique_elements, + MPI_Comm comm, + Teuchos::RCP>& sorting_importer); template -void -sort_and_write_data_to_file_mpi_all( - const SC* unsorted_data_ptr, - Teuchos::RCP> unsorted_map, - size_t dim1, - size_t num_global_unique_elements, - MPI_Comm comm, - MPI_File file_parallel, - Teuchos::RCP> &sorting_importer); - -Teuchos::RCP> -get_cell_nodes( - const CArray & nodes_in_elem, - size_t num_dim, - Solver::node_ordering_convention active_node_ordering_convention); - -Teuchos::RCP> -calculate_elem_speed( - const Teuchos::RCP all_node_velocities_distributed, - const Teuchos::RCP global_nodes_in_elem_distributed, - const Teuchos::RCP ghost_node_velocities_distributed, - const Teuchos::RCP> ghost_importer); - -Teuchos::RCP> -calculate_elem_switch( - Teuchos::RCP> all_element_map); - -Teuchos::RCP> -get_elem_proc_id( - Teuchos::RCP> all_element_map, - size_t myrank); - -Teuchos::RCP> -get_elem_gid( - Teuchos::RCP> all_element_map); - -Teuchos::RCP> -get_design_density( - size_t rnum_nodes, - bool topology_optimization_on, - const Teuchos::RCP design_node_densities_distributed); - -void -Explicit_Solver::write_outputs() +void sort_and_write_data_to_file_mpi_all( + const SC* unsorted_data_ptr, + Teuchos::RCP> unsorted_map, + size_t dim1, + size_t num_global_unique_elements, + MPI_Comm comm, + MPI_File file_parallel, + Teuchos::RCP>& sorting_importer); + +Teuchos::RCP> get_cell_nodes( + const CArray& nodes_in_elem, + size_t num_dim, + Solver::node_ordering_convention active_node_ordering_convention); + +Teuchos::RCP> calculate_elem_speed( + const Teuchos::RCP all_node_velocities_distributed, + const Teuchos::RCP global_nodes_in_elem_distributed, + const Teuchos::RCP ghost_node_velocities_distributed, + const Teuchos::RCP> ghost_importer); + +Teuchos::RCP> calculate_elem_switch( + Teuchos::RCP> all_element_map); + +Teuchos::RCP> get_elem_proc_id( + Teuchos::RCP> all_element_map, + size_t myrank); + +Teuchos::RCP> get_elem_gid( + Teuchos::RCP> all_element_map); + +Teuchos::RCP> get_design_density( + size_t rnum_nodes, + bool topology_optimization_on, + const Teuchos::RCP design_node_densities_distributed); + +void Explicit_Solver::write_outputs() { - // No output for OUTPUT_FORMAT::none - if (simparam.output_options.output_file_format == OUTPUT_FORMAT::none) - return; + // No output for OUTPUT_FORMAT::none + if (simparam.output_options.output_file_format == OUTPUT_FORMAT::none) + { + return; + } - const size_t rk_level = simparam.dynamic_options.rk_num_bins - 1; - Teuchos::RCP> design_density; - Teuchos::RCP> elem_switch; - Teuchos::RCP> elem_proc_id; - Teuchos::RCP> elem_gid; - Teuchos::RCP> elem_speed; + const size_t rk_level = simparam.dynamic_options.rk_num_bins - 1; + Teuchos::RCP> design_density; + Teuchos::RCP> elem_switch; + Teuchos::RCP> elem_proc_id; + Teuchos::RCP> elem_gid; + Teuchos::RCP> elem_speed; - for (const FIELD& field_name : simparam.output_options.output_fields) { - switch (field_name) + for (const FIELD& field_name : simparam.output_options.output_fields) { - case FIELD::design_density: - // node "design_density" - design_density = get_design_density(map->getLocalNumElements(), + switch (field_name) + { + case FIELD::design_density: + // node "design_density" + design_density = get_design_density(map->getLocalNumElements(), simparam.topology_optimization_on, design_node_densities_distributed); - point_data_scalars_double["design_density"] = design_density->pointer(); - break; - - case FIELD::speed: - // element "speed" - elem_speed = calculate_elem_speed(all_node_velocities_distributed, global_nodes_in_elem_distributed, - ghost_node_velocities_distributed, ghost_importer); - cell_data_scalars_double["speed"] = elem_speed->pointer(); - break; - - case FIELD::element_switch: - // element "element_switch" - elem_switch = calculate_elem_switch(all_element_map); - cell_data_scalars_int["element_switch"] = elem_switch->pointer(); - break; - - case FIELD::processor_id: - // element "processor_id" - elem_proc_id = get_elem_proc_id(all_element_map, myrank); - cell_data_scalars_int["processor_id"] = elem_proc_id->pointer(); - break; - - case FIELD::element_id: - // element "element_id" - elem_gid = get_elem_gid(all_element_map); - cell_data_scalars_int["element_id"] = elem_gid->pointer(); - break; - - default: - break; - - } // end switch - } // end if - - for(int imodule = 0; imodule < nfea_modules; imodule++){ - fea_modules[imodule]->write_data(point_data_scalars_double, point_data_vectors_double, cell_data_scalars_double, cell_data_scalars_int, cell_data_fields_double); - } + point_data_scalars_double["design_density"] = design_density->pointer(); + break; + + case FIELD::speed: + // element "speed" + elem_speed = calculate_elem_speed( + all_node_velocities_distributed, + global_nodes_in_elem_distributed, + ghost_node_velocities_distributed, + ghost_importer); + cell_data_scalars_double["speed"] = elem_speed->pointer(); + break; + + case FIELD::element_switch: + // element "element_switch" + elem_switch = calculate_elem_switch(all_element_map); + cell_data_scalars_int["element_switch"] = elem_switch->pointer(); + break; + + case FIELD::processor_id: + // element "processor_id" + elem_proc_id = get_elem_proc_id(all_element_map, myrank); + cell_data_scalars_int["processor_id"] = elem_proc_id->pointer(); + break; + + case FIELD::element_id: + // element "element_id" + elem_gid = get_elem_gid(all_element_map); + cell_data_scalars_int["element_id"] = elem_gid->pointer(); + break; + + default: + break; + } // end switch + } // end if + + for (int imodule = 0; imodule < nfea_modules; imodule++) + { + fea_modules[imodule]->write_data( + point_data_scalars_double, + point_data_vectors_double, + cell_data_scalars_double, + cell_data_scalars_int, + cell_data_fields_double); + } - switch (simparam.output_options.output_file_format) - { + switch (simparam.output_options.output_file_format) + { case OUTPUT_FORMAT::vtk: - parallel_vtk_writer_new(); - break; + parallel_vtk_writer_new(); + break; case OUTPUT_FORMAT::vtu: - parallel_vtu_writer_new(); - break; + parallel_vtu_writer_new(); + break; default: - break; - }; + break; + } + ; } -void -Explicit_Solver::parallel_vtu_writer_new() +void Explicit_Solver::parallel_vtu_writer_new() { - /* to be added... */ - throw std::runtime_error("parallel_vtu_writer_new() not yet implemented. use parallel_vtk_writer_new()"); + /* to be added... */ + throw std::runtime_error("parallel_vtu_writer_new() not yet implemented. use parallel_vtk_writer_new()"); } -std::string -construct_file_name( - size_t file_count, - int displacement_module) +std::string construct_file_name( + size_t file_count, + int displacement_module) { - bool displace_geometry = false; - std::string current_file_name; - std::string base_file_name= "VTK"; - std::string base_file_name_undeformed= "VTK_undeformed"; - std::string file_extension= ".vtk"; - - if(displace_geometry && displacement_module>=0) - current_file_name = base_file_name_undeformed + std::to_string(file_count) + file_extension; - else - current_file_name = base_file_name + std::to_string(file_count) + file_extension; - return current_file_name; + bool displace_geometry = false; + std::string current_file_name; + std::string base_file_name = "VTK"; + std::string base_file_name_undeformed = "VTK_undeformed"; + std::string file_extension = ".vtk"; + + if (displace_geometry && displacement_module >= 0) + { + current_file_name = base_file_name_undeformed + std::to_string(file_count) + file_extension; + } + else + { + current_file_name = base_file_name + std::to_string(file_count) + file_extension; + } + return current_file_name; } -void -Explicit_Solver::parallel_vtk_writer_new() +void Explicit_Solver::parallel_vtk_writer_new() { + int num_dim = simparam.num_dims; + std::stringstream str_stream; + MPI_Offset current_offset; + MPI_File myfile_parallel; + bool displace_geometry = false; - int num_dim = simparam.num_dims; - std::stringstream str_stream; - MPI_Offset current_offset; - MPI_File myfile_parallel; - bool displace_geometry = false; - - std::string vtk_dir = "vtk/"; - std::string vtk_data_dir = vtk_dir + "data/"; - - // mkdir if needed - struct stat st; - if (myrank == 0) { - if (stat(vtk_dir.c_str(), &st) != 0) { - str_stream.str(""); - str_stream << "mkdir" << " " << vtk_dir; - system(str_stream.str().c_str()); - } - if (stat(vtk_data_dir.c_str(), &st) != 0) { - str_stream.str(""); - str_stream << "mkdir" << " " << vtk_data_dir; - system(str_stream.str().c_str()); - } - } - MPI_Barrier(world); - - //construct file name - std::string current_file_name = construct_file_name(simparam.output_options.graphics_id, displacement_module); - std::string file_path = vtk_data_dir + current_file_name; - //open mpi file - int err = MPI_File_open(MPI_COMM_WORLD, file_path.c_str(), - MPI_MODE_CREATE|MPI_MODE_EXCL|MPI_MODE_WRONLY, - MPI_INFO_NULL, &myfile_parallel); - //allows overwriting the file if it already existed in the directory - if (err != MPI_SUCCESS) { - if (myrank == 0){ - MPI_File_delete(file_path.c_str(),MPI_INFO_NULL); + std::string vtk_dir = "vtk/"; + std::string vtk_data_dir = vtk_dir + "data/"; + + // mkdir if needed + struct stat st; + if (myrank == 0) + { + if (stat(vtk_dir.c_str(), &st) != 0) + { + str_stream.str(""); + str_stream << "mkdir" << " " << vtk_dir; + system(str_stream.str().c_str()); + } + if (stat(vtk_data_dir.c_str(), &st) != 0) + { + str_stream.str(""); + str_stream << "mkdir" << " " << vtk_data_dir; + system(str_stream.str().c_str()); + } } - MPI_File_open(MPI_COMM_WORLD, file_path.c_str(), - MPI_MODE_CREATE|MPI_MODE_EXCL|MPI_MODE_WRONLY, + MPI_Barrier(world); + + // construct file name + std::string current_file_name = construct_file_name(simparam.output_options.graphics_id, displacement_module); + std::string file_path = vtk_data_dir + current_file_name; + // open mpi file + int err = MPI_File_open(MPI_COMM_WORLD, file_path.c_str(), + MPI_MODE_CREATE | MPI_MODE_EXCL | MPI_MODE_WRONLY, + MPI_INFO_NULL, &myfile_parallel); + // allows overwriting the file if it already existed in the directory + if (err != MPI_SUCCESS) + { + if (myrank == 0) + { + MPI_File_delete(file_path.c_str(), MPI_INFO_NULL); + } + MPI_File_open(MPI_COMM_WORLD, file_path.c_str(), + MPI_MODE_CREATE | MPI_MODE_EXCL | MPI_MODE_WRONLY, MPI_INFO_NULL, &myfile_parallel); - } - - /*************** write header of the vtk file ***************/ - str_stream.str(""); - str_stream << "# vtk DataFile Version 2.0" << std::endl; - str_stream << "Mesh for Fierro" << std::endl; - str_stream << "ASCII" << std::endl; - str_stream << "DATASET UNSTRUCTURED_GRID" << std::endl; - current_offset = mpi_get_file_position_shared(world, myfile_parallel); - if (myrank == 0) { - MPI_File_write_at(myfile_parallel, current_offset, str_stream.str().c_str(), str_stream.str().length(), MPI_CHAR, MPI_STATUS_IGNORE); - } - - - /*************** write POINTS ***************/ - str_stream.str(""); - str_stream << std::endl << "POINTS " << num_nodes << " float" << std::endl; - current_offset = mpi_get_file_position_shared(world, myfile_parallel); - if(myrank == 0) { - MPI_File_write_at(myfile_parallel, current_offset, str_stream.str().c_str(), str_stream.str().length(), MPI_CHAR, MPI_STATUS_IGNORE); - } - //sgh_module->node_coords.update_host(); - { //view scope - host_vec_array node_coords = node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); - double* coord_data = node_coords.data(); - sort_and_write_data_to_file_mpi_all ( - coord_data, map, num_dim, num_nodes, world, myfile_parallel, node_sorting_importer); - } - - - /*************** write CELLS ***************/ - str_stream.str(""); - str_stream << std::endl << "CELLS " << num_elem << " " << num_elem*(max_nodes_per_element+1) << std::endl; - current_offset = mpi_get_file_position_shared(world, myfile_parallel); - if(myrank == 0) { - MPI_File_write_at(myfile_parallel, current_offset, str_stream.str().c_str(), str_stream.str().length(), MPI_CHAR, MPI_STATUS_IGNORE); - } - CArray nodes_in_elem (rnum_elem, max_nodes_per_element); - { //view scope - auto host_view = global_nodes_in_elem_distributed->getLocalView (Tpetra::Access::ReadOnly); - for (size_t ielem = 0; ielem < nodes_in_elem.dims(0); ielem++) { - for (size_t inode = 0; inode < nodes_in_elem.dims(1); inode++) { - nodes_in_elem(ielem, inode) = host_view(ielem, inode); - } - } - } //end view scope - auto cell_data = get_cell_nodes(nodes_in_elem, num_dim, active_node_ordering_convention); - sort_and_write_data_to_file_mpi_all ( - cell_data->pointer(), all_element_map, cell_data->dims(1), num_elem, world, myfile_parallel, element_sorting_importer); - - - /*************** write CELL_TYPES ***************/ - str_stream.str(""); - str_stream << std::endl << "CELL_TYPES " << num_elem << std::endl; - current_offset = mpi_get_file_position_shared(world, myfile_parallel); - if(myrank == 0) { - MPI_File_write_at(myfile_parallel, current_offset, str_stream.str().c_str(), str_stream.str().length(), MPI_CHAR, MPI_STATUS_IGNORE); - } - CArray cell_type (all_element_map->getLocalNumElements(), 1); - for (int i = 0; i < cell_type.dims(0); i++) - for (int j = 0; j < cell_type.dims(1); j++) - cell_type(i,j) = 12; - sort_and_write_data_to_file_mpi_all ( - cell_type.pointer(), all_element_map, cell_type.dims(1), num_elem, world, myfile_parallel, element_sorting_importer); - - - /*************** write POINT_DATA ***************/ - str_stream.str(""); - str_stream << std::endl << "POINT_DATA " << num_nodes; - current_offset = mpi_get_file_position_shared(world, myfile_parallel); - if(myrank == 0) { - MPI_File_write_at(myfile_parallel, current_offset, str_stream.str().c_str(), - str_stream.str().length(), MPI_CHAR, MPI_STATUS_IGNORE); - } + } - //SCALARS float - for (auto it = point_data_scalars_double.begin(); it != point_data_scalars_double.end(); it++) { + /*************** write header of the vtk file ***************/ str_stream.str(""); - str_stream << std::endl << "SCALARS " << it->first << " float 1" << std::endl; - str_stream << "LOOKUP_TABLE default" << std::endl; + str_stream << "# vtk DataFile Version 2.0" << std::endl; + str_stream << "Mesh for Fierro" << std::endl; + str_stream << "ASCII" << std::endl; + str_stream << "DATASET UNSTRUCTURED_GRID" << std::endl; current_offset = mpi_get_file_position_shared(world, myfile_parallel); - if(myrank == 0) { - MPI_File_write_at(myfile_parallel, current_offset, str_stream.str().c_str(), - str_stream.str().length(), MPI_CHAR, MPI_STATUS_IGNORE); + if (myrank == 0) + { + MPI_File_write_at( + myfile_parallel, + current_offset, + str_stream.str().c_str(), + str_stream.str().length(), + MPI_CHAR, + MPI_STATUS_IGNORE); } - sort_and_write_data_to_file_mpi_all ( - it->second, map, 1, num_nodes, world, myfile_parallel, node_sorting_importer); - } - //VECTORS float - for (auto it = point_data_vectors_double.begin(); it != point_data_vectors_double.end(); it++) { + /*************** write POINTS ***************/ str_stream.str(""); - str_stream << std::endl << "VECTORS " << it->first << " float" << std::endl; + str_stream << std::endl << "POINTS " << num_nodes << " float" << std::endl; current_offset = mpi_get_file_position_shared(world, myfile_parallel); - if(myrank == 0) { - MPI_File_write_at(myfile_parallel, current_offset, str_stream.str().c_str(), - str_stream.str().length(), MPI_CHAR, MPI_STATUS_IGNORE); + if (myrank == 0) + { + MPI_File_write_at( + myfile_parallel, + current_offset, + str_stream.str().c_str(), + str_stream.str().length(), + MPI_CHAR, + MPI_STATUS_IGNORE); + } + // sgh_module->node_coords.update_host(); + { // view scope + host_vec_array node_coords = node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + double* coord_data = node_coords.data(); + sort_and_write_data_to_file_mpi_all( + coord_data, + map, + num_dim, + num_nodes, + world, + myfile_parallel, + node_sorting_importer); } - sort_and_write_data_to_file_mpi_all ( - it->second, map, num_dim, num_nodes, world, myfile_parallel, node_sorting_importer); - } - - - /*************** write CELL_DATA ***************/ - str_stream.str(""); - str_stream << std::endl << "CELL_DATA " << num_elem; - current_offset = mpi_get_file_position_shared(world, myfile_parallel); - if(myrank == 0) { - MPI_File_write_at(myfile_parallel, current_offset, str_stream.str().c_str(), - str_stream.str().length(), MPI_CHAR, MPI_STATUS_IGNORE); - } - //SCALARS float - for (auto it = cell_data_scalars_double.begin(); it != cell_data_scalars_double.end(); it++) { + /*************** write CELLS ***************/ str_stream.str(""); - str_stream << std::endl << "SCALARS " << it->first << " float 1" << std::endl; - str_stream << "LOOKUP_TABLE default" << std::endl; + str_stream << std::endl << "CELLS " << num_elem << " " << num_elem * (max_nodes_per_element + 1) << std::endl; current_offset = mpi_get_file_position_shared(world, myfile_parallel); - if(myrank == 0) { - MPI_File_write_at(myfile_parallel, current_offset, str_stream.str().c_str(), + if (myrank == 0) + { + MPI_File_write_at( + myfile_parallel, + current_offset, + str_stream.str().c_str(), + str_stream.str().length(), + MPI_CHAR, + MPI_STATUS_IGNORE); + } + CArray nodes_in_elem(rnum_elem, max_nodes_per_element); + { // view scope + auto host_view = global_nodes_in_elem_distributed->getLocalView(Tpetra::Access::ReadOnly); + for (size_t ielem = 0; ielem < nodes_in_elem.dims(0); ielem++) + { + for (size_t inode = 0; inode < nodes_in_elem.dims(1); inode++) + { + nodes_in_elem(ielem, inode) = host_view(ielem, inode); + } + } + } // end view scope + auto cell_data = get_cell_nodes(nodes_in_elem, num_dim, active_node_ordering_convention); + sort_and_write_data_to_file_mpi_all( + cell_data->pointer(), + all_element_map, + cell_data->dims(1), + num_elem, + world, + myfile_parallel, + element_sorting_importer); + + /*************** write CELL_TYPES ***************/ + str_stream.str(""); + str_stream << std::endl << "CELL_TYPES " << num_elem << std::endl; + current_offset = mpi_get_file_position_shared(world, myfile_parallel); + if (myrank == 0) + { + MPI_File_write_at( + myfile_parallel, + current_offset, + str_stream.str().c_str(), + str_stream.str().length(), + MPI_CHAR, + MPI_STATUS_IGNORE); + } + CArray cell_type(all_element_map->getLocalNumElements(), 1); + for (int i = 0; i < cell_type.dims(0); i++) + { + for (int j = 0; j < cell_type.dims(1); j++) + { + cell_type(i, j) = 12; + } + } + sort_and_write_data_to_file_mpi_all( + cell_type.pointer(), + all_element_map, + cell_type.dims(1), + num_elem, + world, + myfile_parallel, + element_sorting_importer); + + /*************** write POINT_DATA ***************/ + str_stream.str(""); + str_stream << std::endl << "POINT_DATA " << num_nodes; + current_offset = mpi_get_file_position_shared(world, myfile_parallel); + if (myrank == 0) + { + MPI_File_write_at(myfile_parallel, current_offset, str_stream.str().c_str(), + str_stream.str().length(), MPI_CHAR, MPI_STATUS_IGNORE); + } + + // SCALARS float + for (auto it = point_data_scalars_double.begin(); it != point_data_scalars_double.end(); it++) + { + str_stream.str(""); + str_stream << std::endl << "SCALARS " << it->first << " float 1" << std::endl; + str_stream << "LOOKUP_TABLE default" << std::endl; + current_offset = mpi_get_file_position_shared(world, myfile_parallel); + if (myrank == 0) + { + MPI_File_write_at(myfile_parallel, current_offset, str_stream.str().c_str(), str_stream.str().length(), MPI_CHAR, MPI_STATUS_IGNORE); + } + sort_and_write_data_to_file_mpi_all( + it->second, map, 1, num_nodes, world, myfile_parallel, node_sorting_importer); } - sort_and_write_data_to_file_mpi_all ( - it->second, all_element_map, 1, num_elem, world, myfile_parallel, element_sorting_importer); - } - //SCALARS int - for (auto it = cell_data_scalars_int.begin(); it != cell_data_scalars_int.end(); it++) { + // VECTORS float + for (auto it = point_data_vectors_double.begin(); it != point_data_vectors_double.end(); it++) + { + str_stream.str(""); + str_stream << std::endl << "VECTORS " << it->first << " float" << std::endl; + current_offset = mpi_get_file_position_shared(world, myfile_parallel); + if (myrank == 0) + { + MPI_File_write_at(myfile_parallel, current_offset, str_stream.str().c_str(), + str_stream.str().length(), MPI_CHAR, MPI_STATUS_IGNORE); + } + sort_and_write_data_to_file_mpi_all( + it->second, map, num_dim, num_nodes, world, myfile_parallel, node_sorting_importer); + } + + /*************** write CELL_DATA ***************/ str_stream.str(""); - str_stream << std::endl << "SCALARS " << it->first << " int 1" << std::endl; - str_stream << "LOOKUP_TABLE default" << std::endl; + str_stream << std::endl << "CELL_DATA " << num_elem; current_offset = mpi_get_file_position_shared(world, myfile_parallel); - if(myrank == 0) { - MPI_File_write_at(myfile_parallel, current_offset, str_stream.str().c_str(), + if (myrank == 0) + { + MPI_File_write_at(myfile_parallel, current_offset, str_stream.str().c_str(), + str_stream.str().length(), MPI_CHAR, MPI_STATUS_IGNORE); + } + + // SCALARS float + for (auto it = cell_data_scalars_double.begin(); it != cell_data_scalars_double.end(); it++) + { + str_stream.str(""); + str_stream << std::endl << "SCALARS " << it->first << " float 1" << std::endl; + str_stream << "LOOKUP_TABLE default" << std::endl; + current_offset = mpi_get_file_position_shared(world, myfile_parallel); + if (myrank == 0) + { + MPI_File_write_at(myfile_parallel, current_offset, str_stream.str().c_str(), str_stream.str().length(), MPI_CHAR, MPI_STATUS_IGNORE); + } + sort_and_write_data_to_file_mpi_all( + it->second, all_element_map, 1, num_elem, world, myfile_parallel, element_sorting_importer); } - sort_and_write_data_to_file_mpi_all ( + + // SCALARS int + for (auto it = cell_data_scalars_int.begin(); it != cell_data_scalars_int.end(); it++) + { + str_stream.str(""); + str_stream << std::endl << "SCALARS " << it->first << " int 1" << std::endl; + str_stream << "LOOKUP_TABLE default" << std::endl; + current_offset = mpi_get_file_position_shared(world, myfile_parallel); + if (myrank == 0) + { + MPI_File_write_at(myfile_parallel, current_offset, str_stream.str().c_str(), + str_stream.str().length(), MPI_CHAR, MPI_STATUS_IGNORE); + } + sort_and_write_data_to_file_mpi_all( it->second, all_element_map, 1, num_elem, world, myfile_parallel, element_sorting_importer); - } - - //FIELD - str_stream.str(""); - str_stream << std::endl << "FIELD FieldData " << cell_data_fields_double.size() << std::endl; - current_offset = mpi_get_file_position_shared(world, myfile_parallel); - if(myrank == 0) { - MPI_File_write_at(myfile_parallel, current_offset, str_stream.str().c_str(), - str_stream.str().length(), MPI_CHAR, MPI_STATUS_IGNORE); - } - for (auto it = cell_data_fields_double.begin(); it != cell_data_fields_double.end(); it++) { - auto data_name = it->first; - auto [data_ptr, data_num_comps] = it->second; // Structured binding C++17 + } + + // FIELD str_stream.str(""); - str_stream << data_name << " " << data_num_comps << " " << num_elem << " float" << std::endl; + str_stream << std::endl << "FIELD FieldData " << cell_data_fields_double.size() << std::endl; current_offset = mpi_get_file_position_shared(world, myfile_parallel); - if(myrank == 0) { - MPI_File_write_at(myfile_parallel, current_offset, str_stream.str().c_str(), - str_stream.str().length(), MPI_CHAR, MPI_STATUS_IGNORE); + if (myrank == 0) + { + MPI_File_write_at(myfile_parallel, current_offset, str_stream.str().c_str(), + str_stream.str().length(), MPI_CHAR, MPI_STATUS_IGNORE); } - sort_and_write_data_to_file_mpi_all ( + for (auto it = cell_data_fields_double.begin(); it != cell_data_fields_double.end(); it++) + { + auto data_name = it->first; + auto [data_ptr, data_num_comps] = it->second; // Structured binding C++17 + str_stream.str(""); + str_stream << data_name << " " << data_num_comps << " " << num_elem << " float" << std::endl; + current_offset = mpi_get_file_position_shared(world, myfile_parallel); + if (myrank == 0) + { + MPI_File_write_at(myfile_parallel, current_offset, str_stream.str().c_str(), + str_stream.str().length(), MPI_CHAR, MPI_STATUS_IGNORE); + } + sort_and_write_data_to_file_mpi_all( data_ptr, all_element_map, data_num_comps, num_elem, world, myfile_parallel, element_sorting_importer); - } - - MPI_Barrier(world); - MPI_File_sync(myfile_parallel); - MPI_File_close(&myfile_parallel); - - - /*************** write .vtk.series file ***************/ - simparam.output_options.graphics_times(simparam.output_options.graphics_id) = time_value; - if (myrank == 0) { - FILE *myfile; - std::string filename = vtk_dir + "outputs.vtk.series"; - myfile = fopen(filename.c_str(), "w"); - - fprintf(myfile, "{\n"); - fprintf(myfile, " \"file-series-version\" : \"1.0\",\n"); - fprintf(myfile, " \"files\" : [\n"); - for (int i = 0; i <= simparam.output_options.graphics_id; i++) { - std::string vtk_filename = construct_file_name(i, displacement_module); - fprintf(myfile, " { \"name\" : \"data/%s\", \"time\" : %12.5e },\n", - vtk_filename.c_str(), simparam.output_options.graphics_times(i)); } - fprintf(myfile, " ]\n"); - fprintf(myfile, "}\n"); - fclose(myfile); - } - - simparam.output_options.graphics_id++; + MPI_Barrier(world); + MPI_File_sync(myfile_parallel); + MPI_File_close(&myfile_parallel); -} + /*************** write .vtk.series file ***************/ + simparam.output_options.graphics_times(simparam.output_options.graphics_id) = time_value; + if (myrank == 0) + { + FILE* myfile; + std::string filename = vtk_dir + "outputs.vtk.series"; + myfile = fopen(filename.c_str(), "w"); + + fprintf(myfile, "{\n"); + fprintf(myfile, " \"file-series-version\" : \"1.0\",\n"); + fprintf(myfile, " \"files\" : [\n"); + for (int i = 0; i <= simparam.output_options.graphics_id; i++) + { + std::string vtk_filename = construct_file_name(i, displacement_module); + fprintf(myfile, " { \"name\" : \"data/%s\", \"time\" : %12.5e },\n", + vtk_filename.c_str(), simparam.output_options.graphics_times(i)); + } + fprintf(myfile, " ]\n"); + fprintf(myfile, "}\n"); + fclose(myfile); + } + + simparam.output_options.graphics_id++; +} -MPI_Offset -mpi_get_file_position_shared( - const MPI_Comm comm, - const MPI_File file_parallel) +MPI_Offset mpi_get_file_position_shared( + const MPI_Comm comm, + const MPI_File file_parallel) { - MPI_Offset position; + MPI_Offset position; - MPI_Barrier(comm); - MPI_File_seek_shared(file_parallel, 0, MPI_SEEK_END); - MPI_File_get_position_shared(file_parallel, &position); - MPI_Barrier(comm); - return position; + MPI_Barrier(comm); + MPI_File_seek_shared(file_parallel, 0, MPI_SEEK_END); + MPI_File_get_position_shared(file_parallel, &position); + MPI_Barrier(comm); + return position; } -void -write_string_stream_to_file_mpi_all( - const std::stringstream& str_stream, - MPI_Offset rank_offset_multiplier, - const MPI_File file_parallel, - const MPI_Comm comm) +void write_string_stream_to_file_mpi_all( + const std::stringstream& str_stream, + MPI_Offset rank_offset_multiplier, + const MPI_File file_parallel, + const MPI_Comm comm) { - /* `str_stream`: must have the same line length - * `rank_offset_multiplier`: should be the index of the first node or element in the rank - * `file_parallel`: the parallel mpi file to write to - * `comm`: MPI_Comm that should do the write - * */ - - MPI_Offset current_offset, file_offset; - current_offset = mpi_get_file_position_shared(comm, file_parallel); - std::string line; - std::getline(const_cast(str_stream), line); - auto line_length = line.length() + 1; // the +1 is for end of line character - file_offset = line_length * rank_offset_multiplier + current_offset; - MPI_Barrier(comm); - MPI_File_write_at_all(file_parallel, file_offset, str_stream.str().c_str(), + /* `str_stream`: must have the same line length + * `rank_offset_multiplier`: should be the index of the first node or element in the rank + * `file_parallel`: the parallel mpi file to write to + * `comm`: MPI_Comm that should do the write + * */ + + MPI_Offset current_offset, file_offset; + current_offset = mpi_get_file_position_shared(comm, file_parallel); + std::string line; + std::getline(const_cast(str_stream), line); + auto line_length = line.length() + 1; // the +1 is for end of line character + file_offset = line_length * rank_offset_multiplier + current_offset; + MPI_Barrier(comm); + MPI_File_write_at_all(file_parallel, file_offset, str_stream.str().c_str(), str_stream.str().length(), MPI_CHAR, MPI_STATUS_IGNORE); - MPI_Barrier(comm); - return; + MPI_Barrier(comm); + return; } -template +template void write_data_to_string_stream( - const T* ptr, - size_t dim0, - size_t dim1, - std::stringstream& str_stream) + const T* ptr, + size_t dim0, + size_t dim1, + std::stringstream& str_stream) { - size_t w; - size_t p; - //crude way to ensure that all lines have the same length. required for MPI IO - if constexpr (std::is_same::value or - std::is_same::value) { - w = 25; - p = 8; - } - else if constexpr (std::is_same::value or - std::is_same::value or - std::is_same::value or - std::is_same::value) { - w = 10; - p = 0; - } - else { - throw std::runtime_error(std::string("unknown datatype in function ") + + size_t w; + size_t p; + // crude way to ensure that all lines have the same length. required for MPI IO + if constexpr (std::is_same::value or + std::is_same::value) + { + w = 25; + p = 8; + } + else if constexpr (std::is_same::value or + std::is_same::value or + std::is_same::value or + std::is_same::value) + { + w = 10; + p = 0; + } + else + { + throw std::runtime_error(std::string("unknown datatype in function ") + std::string(__FUNCTION__) + std::string("in file ") + std::string(__FILE__)); - } + } - // create view of unsorted_data_ptr. not using matar here. array_layout is needed - Kokkos::View + // create view of unsorted_data_ptr. not using matar here. array_layout is needed + Kokkos::View data_view(ptr, dim0, dim1); - str_stream.str(""); - str_stream << std::fixed << std::setprecision(p); - for (size_t i = 0; i < dim0; i++) { - for (size_t j = 0; j < dim1; j++) { - str_stream << std::left << std::setw(w) << data_view(i,j) << " "; + str_stream.str(""); + str_stream << std::fixed << std::setprecision(p); + for (size_t i = 0; i < dim0; i++) + { + for (size_t j = 0; j < dim1; j++) + { + str_stream << std::left << std::setw(w) << data_view(i, j) << " "; + } + str_stream << std::endl; } - str_stream << std::endl; - } - return; + return; } template -Teuchos::RCP> -sort_data( - const SC* unsorted_data_ptr, - Teuchos::RCP> unsorted_map, - size_t dim1, - size_t num_global_unique_elements, - MPI_Comm comm, - Teuchos::RCP> &sorting_importer) +Teuchos::RCP> sort_data( + const SC* unsorted_data_ptr, + Teuchos::RCP> unsorted_map, + size_t dim1, + size_t num_global_unique_elements, + MPI_Comm comm, + Teuchos::RCP>& sorting_importer) { - - // create view of unsorted_data_ptr. not using matar here. array_layout is needed - Kokkos::View + // create view of unsorted_data_ptr. not using matar here. array_layout is needed + Kokkos::View unsorted_data(unsorted_data_ptr, unsorted_map->getLocalNumElements(), dim1); - // sorted_map - Teuchos::RCP> sorted_map = - Teuchos::rcp(new Tpetra::Map( + // sorted_map + Teuchos::RCP> sorted_map = + Teuchos::rcp(new Tpetra::Map( num_global_unique_elements, unsorted_map->getIndexBase(), unsorted_map->getComm())); - // sorted storage - Teuchos::RCP> sorted_storage = - Teuchos::rcp(new Tpetra::MultiVector(sorted_map, dim1)); + // sorted storage + Teuchos::RCP> sorted_storage = + Teuchos::rcp(new Tpetra::MultiVector(sorted_map, dim1)); - // unsorted storage - Teuchos::RCP> unsorted_storage = - Teuchos::rcp(new Tpetra::MultiVector(unsorted_map, dim1)); + // unsorted storage + Teuchos::RCP> unsorted_storage = + Teuchos::rcp(new Tpetra::MultiVector(unsorted_map, dim1)); - { //view scope - auto host_view = unsorted_storage->getLocalViewHost(Tpetra::Access::ReadWrite); - for(int i = 0; i < unsorted_map->getLocalNumElements(); i++) { - for (int j = 0; j < dim1; j++) { - host_view(i,j) = unsorted_data(i,j); - } - } // for - } //end view scope + { // view scope + auto host_view = unsorted_storage->getLocalViewHost(Tpetra::Access::ReadWrite); + for (int i = 0; i < unsorted_map->getLocalNumElements(); i++) + { + for (int j = 0; j < dim1; j++) + { + host_view(i, j) = unsorted_data(i, j); + } + } // for + } // end view scope - sorted_storage->doImport(*unsorted_storage, *sorting_importer, Tpetra::INSERT); + sorted_storage->doImport(*unsorted_storage, *sorting_importer, Tpetra::INSERT); - return sorted_storage; + return sorted_storage; } template -void -sort_and_write_data_to_file_mpi_all( - const SC* unsorted_data_ptr, - Teuchos::RCP> unsorted_map, - size_t dim1, - size_t num_global_unique_elements, - MPI_Comm comm, - MPI_File file_parallel, - Teuchos::RCP> &sorting_importer) +void sort_and_write_data_to_file_mpi_all( + const SC* unsorted_data_ptr, + Teuchos::RCP> unsorted_map, + size_t dim1, + size_t num_global_unique_elements, + MPI_Comm comm, + MPI_File file_parallel, + Teuchos::RCP>& sorting_importer) { - - auto sorted_data = - sort_data (unsorted_data_ptr, unsorted_map, dim1, num_global_unique_elements, comm, sorting_importer); - - { //view scope - auto const_host_view = sorted_data->getLocalViewHost(Tpetra::Access::ReadOnly); - std::stringstream str_stream; - write_data_to_string_stream (const_host_view.data(), sorted_data->getMap()->getLocalNumElements(), dim1, str_stream); - write_string_stream_to_file_mpi_all(str_stream, sorted_data->getMap()->getGlobalElement(0), file_parallel, comm); - } //end view scope + auto sorted_data = + sort_data( + unsorted_data_ptr, + unsorted_map, + dim1, + num_global_unique_elements, + comm, + sorting_importer); + + { // view scope + auto const_host_view = sorted_data->getLocalViewHost(Tpetra::Access::ReadOnly); + std::stringstream str_stream; + write_data_to_string_stream( + const_host_view.data(), + sorted_data->getMap()->getLocalNumElements(), + dim1, + str_stream); + write_string_stream_to_file_mpi_all( + str_stream, + sorted_data->getMap()->getGlobalElement(0), + file_parallel, + comm); + } // end view scope } -Teuchos::RCP> -get_cell_nodes( - const CArray & nodes_in_elem, - size_t num_dim, - Solver::node_ordering_convention active_node_ordering_convention) +Teuchos::RCP> get_cell_nodes( + const CArray& nodes_in_elem, + size_t num_dim, + Solver::node_ordering_convention active_node_ordering_convention) { - CArray convert_ijk_to_ensight(nodes_in_elem.dims(1)); - convert_ijk_to_ensight(0) = 0; - convert_ijk_to_ensight(1) = 1; - convert_ijk_to_ensight(2) = 3; - convert_ijk_to_ensight(3) = 2; - if(num_dim==3){ - convert_ijk_to_ensight(4) = 4; - convert_ijk_to_ensight(5) = 5; - convert_ijk_to_ensight(6) = 7; - convert_ijk_to_ensight(7) = 6; - } - - Teuchos::RCP> cell_data = - Teuchos::rcp( new CArray(nodes_in_elem.dims(0), nodes_in_elem.dims(1)+1) ); - - size_t temp_convert; - for (size_t ielem = 0; ielem < nodes_in_elem.dims(0); ielem++) { - (*cell_data)(ielem,0) = 8; - for (int ii = 0; ii < nodes_in_elem.dims(1); ii++) { - if(active_node_ordering_convention == Solver::node_ordering_convention::IJK) - temp_convert = convert_ijk_to_ensight(ii); - else - temp_convert = ii; - (*cell_data)(ielem,ii+1) = nodes_in_elem(ielem,temp_convert); - //(*cell_data)(ielem,ii+1) = nodes_in_elem(ielem,ii); - } - } - - return cell_data; -} - - -Teuchos::RCP> -calculate_elem_speed( - const Teuchos::RCP all_node_velocities_distributed, - const Teuchos::RCP global_nodes_in_elem_distributed, - const Teuchos::RCP ghost_node_velocities_distributed, - const Teuchos::RCP> ghost_importer) -{ - - size_t rnum_elems = global_nodes_in_elem_distributed->getLocalLength(); - size_t num_nodes_in_elem = global_nodes_in_elem_distributed->getNumVectors(); - size_t num_dims = all_node_velocities_distributed->getNumVectors(); - - Teuchos::RCP> elem_speed = - Teuchos::rcp(new CArray(rnum_elems)); - - auto nodes_in_elem_hview = global_nodes_in_elem_distributed->getLocalViewHost(Tpetra::Access::ReadOnly); - auto all_node_vel_hview = all_node_velocities_distributed->getLocalViewHost(Tpetra::Access::ReadOnly); - auto vector_map = all_node_velocities_distributed->getMap(); - for (size_t elem_gid = 0; elem_gid < rnum_elems; elem_gid++) { - double elem_vel[3]; - elem_vel[0] = 0.0; - elem_vel[1] = 0.0; - elem_vel[2] = 0.0; - // get the coordinates of the element center - for (int node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - size_t inode = vector_map->getLocalElement(nodes_in_elem_hview(elem_gid, node_lid)); - elem_vel[0] += all_node_vel_hview(inode, 0); - elem_vel[1] += all_node_vel_hview(inode, 1); - if (num_dims == 3){ - elem_vel[2] += all_node_vel_hview(inode, 2); - } - else { - elem_vel[2] = 0.0; - } - } // end loop over nodes in element + CArray convert_ijk_to_ensight(nodes_in_elem.dims(1)); + convert_ijk_to_ensight(0) = 0; + convert_ijk_to_ensight(1) = 1; + convert_ijk_to_ensight(2) = 3; + convert_ijk_to_ensight(3) = 2; + if (num_dim == 3) + { + convert_ijk_to_ensight(4) = 4; + convert_ijk_to_ensight(5) = 5; + convert_ijk_to_ensight(6) = 7; + convert_ijk_to_ensight(7) = 6; + } - elem_vel[0] = elem_vel[0]/num_nodes_in_elem; - elem_vel[1] = elem_vel[1]/num_nodes_in_elem; - elem_vel[2] = elem_vel[2]/num_nodes_in_elem; + Teuchos::RCP> cell_data = + Teuchos::rcp(new CArray(nodes_in_elem.dims(0), nodes_in_elem.dims(1) + 1) ); - double speed_sqrd = 0.0; - for (int dim=0; dim> calculate_elem_speed( + const Teuchos::RCP all_node_velocities_distributed, + const Teuchos::RCP global_nodes_in_elem_distributed, + const Teuchos::RCP ghost_node_velocities_distributed, + const Teuchos::RCP> ghost_importer) +{ + size_t rnum_elems = global_nodes_in_elem_distributed->getLocalLength(); + size_t num_nodes_in_elem = global_nodes_in_elem_distributed->getNumVectors(); + size_t num_dims = all_node_velocities_distributed->getNumVectors(); -} + Teuchos::RCP> elem_speed = + Teuchos::rcp(new CArray(rnum_elems)); + auto nodes_in_elem_hview = global_nodes_in_elem_distributed->getLocalViewHost(Tpetra::Access::ReadOnly); + auto all_node_vel_hview = all_node_velocities_distributed->getLocalViewHost(Tpetra::Access::ReadOnly); + auto vector_map = all_node_velocities_distributed->getMap(); + for (size_t elem_gid = 0; elem_gid < rnum_elems; elem_gid++) + { + double elem_vel[3]; + elem_vel[0] = 0.0; + elem_vel[1] = 0.0; + elem_vel[2] = 0.0; + // get the coordinates of the element center + for (int node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + size_t inode = vector_map->getLocalElement(nodes_in_elem_hview(elem_gid, node_lid)); + elem_vel[0] += all_node_vel_hview(inode, 0); + elem_vel[1] += all_node_vel_hview(inode, 1); + if (num_dims == 3) + { + elem_vel[2] += all_node_vel_hview(inode, 2); + } + else + { + elem_vel[2] = 0.0; + } + } // end loop over nodes in element + + elem_vel[0] = elem_vel[0] / num_nodes_in_elem; + elem_vel[1] = elem_vel[1] / num_nodes_in_elem; + elem_vel[2] = elem_vel[2] / num_nodes_in_elem; + + double speed_sqrd = 0.0; + for (int dim = 0; dim < num_dims; dim++) + { + speed_sqrd += elem_vel[dim] * elem_vel[dim]; + } + + (*elem_speed)(elem_gid) = sqrt(speed_sqrd); + } // end for + + return elem_speed; +} -Teuchos::RCP> -calculate_elem_switch( - Teuchos::RCP> all_element_map) +Teuchos::RCP> calculate_elem_switch( + Teuchos::RCP> all_element_map) { - Teuchos::RCP> elem_switch = - Teuchos::rcp(new CArray(all_element_map->getLocalNumElements())); + Teuchos::RCP> elem_switch = + Teuchos::rcp(new CArray(all_element_map->getLocalNumElements())); - for (size_t ielem = 0; ielem < elem_switch->dims(0); ielem++) { - (*elem_switch)(ielem) = all_element_map->getGlobalElement(ielem) % 2; - } - return elem_switch; + for (size_t ielem = 0; ielem < elem_switch->dims(0); ielem++) + { + (*elem_switch)(ielem) = all_element_map->getGlobalElement(ielem) % 2; + } + return elem_switch; } -Teuchos::RCP> -get_elem_proc_id( - Teuchos::RCP> all_element_map, - size_t myrank) +Teuchos::RCP> get_elem_proc_id( + Teuchos::RCP> all_element_map, + size_t myrank) { - Teuchos::RCP> elem_proc_id = - Teuchos::rcp(new CArray(all_element_map->getLocalNumElements())); + Teuchos::RCP> elem_proc_id = + Teuchos::rcp(new CArray(all_element_map->getLocalNumElements())); - for (size_t ielem = 0; ielem < elem_proc_id->dims(0); ielem++) { - (*elem_proc_id)(ielem) = myrank; - } - return elem_proc_id; + for (size_t ielem = 0; ielem < elem_proc_id->dims(0); ielem++) + { + (*elem_proc_id)(ielem) = myrank; + } + return elem_proc_id; } -Teuchos::RCP> -get_elem_gid( - Teuchos::RCP> all_element_map) +Teuchos::RCP> get_elem_gid( + Teuchos::RCP> all_element_map) { - Teuchos::RCP> elem_gid = - Teuchos::rcp(new CArray(all_element_map->getLocalNumElements())); + Teuchos::RCP> elem_gid = + Teuchos::rcp(new CArray(all_element_map->getLocalNumElements())); - for (size_t ielem = 0; ielem < elem_gid->dims(0); ielem++) { - (*elem_gid)(ielem) = all_element_map->getGlobalElement(ielem); - } - return elem_gid; + for (size_t ielem = 0; ielem < elem_gid->dims(0); ielem++) + { + (*elem_gid)(ielem) = all_element_map->getGlobalElement(ielem); + } + return elem_gid; } -Teuchos::RCP> -get_design_density( - size_t rnum_nodes, - bool topology_optimization_on, - const Teuchos::RCP design_node_densities_distributed) +Teuchos::RCP> get_design_density( + size_t rnum_nodes, + bool topology_optimization_on, + const Teuchos::RCP design_node_densities_distributed) { - Teuchos::RCP> design_density = - Teuchos::rcp(new CArray(rnum_nodes)); + Teuchos::RCP> design_density = + Teuchos::rcp(new CArray(rnum_nodes)); - if(topology_optimization_on) { - auto host_view = design_node_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); - for (size_t inode = 0; inode < rnum_nodes; inode++) { - (*design_density)(inode) = host_view(inode,0); + if (topology_optimization_on) + { + auto host_view = design_node_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); + for (size_t inode = 0; inode < rnum_nodes; inode++) + { + (*design_density)(inode) = host_view(inode, 0); + } } - } - else { - for (size_t inode = 0; inode < rnum_nodes; inode++) { - (*design_density)(inode) = 1.0; + else + { + for (size_t inode = 0; inode < rnum_nodes; inode++) + { + (*design_density)(inode) = 1.0; + } } - } - return design_density; + return design_density; } - diff --git a/src/Parallel-Solvers/Solver.cpp b/src/Parallel-Solvers/Solver.cpp index 4631e22f8..081339dd1 100644 --- a/src/Parallel-Solvers/Solver.cpp +++ b/src/Parallel-Solvers/Solver.cpp @@ -11,14 +11,14 @@ This program is open source under the BSD-3 License. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - + 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - + 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - + 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. @@ -41,7 +41,7 @@ #include #include #include -#include +#include #include // fmin, fmax, abs note: fminl is long #include #include @@ -71,13 +71,13 @@ #include "MeshBuilder.h" #include "MeshIO.h" -//Repartition Package +// Repartition Package #include #include #include #include -//debug and performance includes +// debug and performance includes #include #include #include @@ -89,2559 +89,3016 @@ #define DENSITY_EPSILON 0.0001 #define BC_EPSILON 1.0e-8 -Solver::Solver(Simulation_Parameters& _simparam) : simparam(_simparam) { - //default flags assume optional routines are off - setup_flag = finalize_flag = 0; - communication_time = dev2host_time = host2dev_time = output_time = 0; - last_print_step = -1; +Solver::Solver(Simulation_Parameters& _simparam) : simparam(_simparam) +{ + // default flags assume optional routines are off + setup_flag = finalize_flag = 0; + communication_time = dev2host_time = host2dev_time = output_time = 0; + last_print_step = -1; + + // FEA module data init + nfea_modules = 0; + displacement_module = -1; +} - //FEA module data init - nfea_modules = 0; - displacement_module = -1; +void Solver::exit_solver(int status) +{ + Kokkos::finalize(); + MPI_Barrier(MPI_COMM_WORLD); + MPI_Finalize(); + exit(status); } -void Solver::exit_solver(int status){ - Kokkos::finalize(); - MPI_Barrier(MPI_COMM_WORLD); - MPI_Finalize(); - exit(status); +Solver::~Solver() +{ + // destroy FEA modules + for (int imodule = 0; imodule < nfea_modules; imodule++) + { + delete fea_modules[imodule]; + } } -Solver::~Solver(){ - //destroy FEA modules - for(int imodule = 0; imodule < nfea_modules; imodule++){ - delete fea_modules[imodule]; - } +namespace elements +{ +namespace elem_types +{ +elem_type from_vtk(const int& vtk_elem) +{ + switch (vtk_elem) + { + case 9: + return elem_type::Quad4; + case 12: + return elem_type::Hex8; + case 70: + return elem_type::QuadN; + case 72: + return elem_type::HexN; + default: + throw std::runtime_error("Unsupported vtk element type: " + std::to_string(vtk_elem)); + } } +} // namespace elem_types +} // namespace elements +void Solver::generate_mesh(const std::shared_ptr& mesh_generation_options) +{ + auto mesh = MeshBuilder::build_mesh(mesh_generation_options); + switch (active_node_ordering_convention) + { + case ENSIGHT: + MeshIO::_Impl::reorder_columns(mesh.element_point_index, MeshIO::_Impl::ijk_to_fea().data()); + break; + case IJK: + // Already in IJK + break; + } -namespace elements { - namespace elem_types { - elem_type from_vtk(const int& vtk_elem) { - switch (vtk_elem) { - case 9: - return elem_type::Quad4; - case 12: - return elem_type::Hex8; - case 70: - return elem_type::QuadN; - case 72: - return elem_type::HexN; - default: - throw std::runtime_error("Unsupported vtk element type: " + std::to_string(vtk_elem)); + num_nodes = mesh.points.dims(0); + num_elem = mesh.element_point_index.dims(0); + map = Teuchos::rcp(new Tpetra::Map(num_nodes, 0, comm)); + + nlocal_nodes = map->getLocalNumElements(); + + node_coords_distributed = Teuchos::rcp(new MV(map, mesh.points.dims(1))); + { + host_vec_array node_coords = node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + for (long long int i = 0; i < num_nodes; i++) + { + // set global node id (ensight specific order) + // let map decide if this node id belongs locally; if yes store data + if (map->isNodeGlobalElement(i)) + { + // set local node index in this mpi rank + long long int node_rid = map->getLocalElement(i); + for (long long int j = 0; j < mesh.points.dims(1); j++) + { + node_coords(node_rid, j) = mesh.points(i, j); + } + } } } - } -} -void Solver::generate_mesh(const std::shared_ptr& mesh_generation_options) { - auto mesh = MeshBuilder::build_mesh(mesh_generation_options); - switch (active_node_ordering_convention) { - case ENSIGHT: - MeshIO::_Impl::reorder_columns(mesh.element_point_index, MeshIO::_Impl::ijk_to_fea().data()); - break; - case IJK: - // Already in IJK - break; - } - - num_nodes = mesh.points.dims(0); - num_elem = mesh.element_point_index.dims(0); - map = Teuchos::rcp( new Tpetra::Map(num_nodes, 0, comm)); - - nlocal_nodes = map->getLocalNumElements(); - - node_coords_distributed = Teuchos::rcp(new MV(map, mesh.points.dims(1))); - { - host_vec_array node_coords = node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); - for(long long int i = 0; i < num_nodes; i++){ - //set global node id (ensight specific order) - //let map decide if this node id belongs locally; if yes store data - if(map->isNodeGlobalElement(i)){ - //set local node index in this mpi rank - long long int node_rid = map->getLocalElement(i); - for (long long int j = 0; j < mesh.points.dims(1); j++) - node_coords(node_rid, j) = mesh.points(i, j); - } + repartition_nodes(); + + max_nodes_per_element = mesh.element_point_index.dims(1); + + // Figure out which elements belong to me. + std::vector global_indices_temp; + for (size_t i = 0; i < mesh.element_point_index.dims(0); i++) + { + for (size_t j = 0; j < mesh.element_point_index.dims(1); j++) + { + if (map->isNodeGlobalElement(mesh.element_point_index(i, j))) + { + global_indices_temp.push_back(i); + break; + } + } } - } - repartition_nodes(); - - max_nodes_per_element = mesh.element_point_index.dims(1); - - // Figure out which elements belong to me. - std::vector global_indices_temp; - for (size_t i = 0; i < mesh.element_point_index.dims(0); i++) { - for (size_t j = 0; j < mesh.element_point_index.dims(1); j++) { - if (map->isNodeGlobalElement(mesh.element_point_index(i, j))) { - global_indices_temp.push_back(i); + + rnum_elem = global_indices_temp.size(); + + Element_Types = CArrayKokkos(global_indices_temp.size()); + auto element_type = elements::elem_types::from_vtk(mesh.element_types(0)); // Not from elements. From up there ^ + switch (mesh.points.dims(1)) + { + case 2: + switch (element_type) + { + case elements::elem_types::elem_type::Quad4: + max_nodes_per_patch = 2; + break; + default: + throw std::runtime_error("Higher order meshes are unsupported."); + } + break; + case 3: + switch (element_type) + { + case elements::elem_types::elem_type::Hex8: + max_nodes_per_patch = 4; + break; + default: + throw std::runtime_error("Higher order meshes are unsupported."); + } break; - } } - } - rnum_elem = global_indices_temp.size(); + for (size_t i = 0; i < global_indices_temp.size(); i++) + { + Element_Types(i) = element_type; + } - Element_Types = CArrayKokkos(global_indices_temp.size()); - auto element_type = elements::elem_types::from_vtk(mesh.element_types(0)); // Not from elements. From up there ^ - switch (mesh.points.dims(1)) { - case 2: - switch (element_type) { - case elements::elem_types::elem_type::Quad4: - max_nodes_per_patch = 2; - break; - default: - throw std::runtime_error("Higher order meshes are unsupported."); - } - break; - case 3: - switch (element_type) { - case elements::elem_types::elem_type::Hex8: - max_nodes_per_patch = 4; - break; - default: - throw std::runtime_error("Higher order meshes are unsupported."); - } - break; - } - - for (size_t i = 0; i < global_indices_temp.size(); i++) - Element_Types(i) = element_type; - - //copy temporary element storage to multivector storage - dual_nodes_in_elem = dual_elem_conn_array("dual_nodes_in_elem", global_indices_temp.size(), mesh.element_point_index.dims(1)); - host_elem_conn_array nodes_in_elem = dual_nodes_in_elem.view_host(); - dual_nodes_in_elem.modify_host(); - - for (size_t i = 0; i < global_indices_temp.size(); i++) - for (size_t j = 0; j < mesh.element_point_index.dims(1); j++) - nodes_in_elem(i, j) = mesh.element_point_index(global_indices_temp[i], j); - - //view storage for all local elements connected to local nodes on this rank - Kokkos::DualView All_Element_Global_Indices("All_Element_Global_Indices", global_indices_temp.size()); - //copy temporary global indices storage to view storage - for(int i = 0; i < global_indices_temp.size(); i++) - All_Element_Global_Indices.h_view(i) = global_indices_temp[i]; - - //construct overlapping element map (since different ranks can own the same elements due to the local node map) - All_Element_Global_Indices.modify_host(); - All_Element_Global_Indices.sync_device(); - - all_element_map = Teuchos::rcp( new Tpetra::Map(Teuchos::OrdinalTraits::invalid(), All_Element_Global_Indices.d_view, 0, comm)); + // copy temporary element storage to multivector storage + dual_nodes_in_elem = dual_elem_conn_array("dual_nodes_in_elem", global_indices_temp.size(), mesh.element_point_index.dims(1)); + host_elem_conn_array nodes_in_elem = dual_nodes_in_elem.view_host(); + dual_nodes_in_elem.modify_host(); + + for (size_t i = 0; i < global_indices_temp.size(); i++) + { + for (size_t j = 0; j < mesh.element_point_index.dims(1); j++) + { + nodes_in_elem(i, j) = mesh.element_point_index(global_indices_temp[i], j); + } + } + + // view storage for all local elements connected to local nodes on this rank + Kokkos::DualView All_Element_Global_Indices("All_Element_Global_Indices", global_indices_temp.size()); + // copy temporary global indices storage to view storage + for (int i = 0; i < global_indices_temp.size(); i++) + { + All_Element_Global_Indices.h_view(i) = global_indices_temp[i]; + } + + // construct overlapping element map (since different ranks can own the same elements due to the local node map) + All_Element_Global_Indices.modify_host(); + All_Element_Global_Indices.sync_device(); + + all_element_map = Teuchos::rcp(new Tpetra::Map(Teuchos::OrdinalTraits::invalid(), All_Element_Global_Indices.d_view, 0, comm)); } /* ---------------------------------------------------------------------- Read Ensight format mesh file ------------------------------------------------------------------------- */ -void Solver::read_mesh_ensight(const char *MESH){ - - char ch; - int num_dim = simparam.num_dims; - Input_Options input_options = simparam.input_options.value(); - int p_order = input_options.p_order; - real_t unit_scaling = input_options.unit_scaling; - int local_node_index, current_column_index; - size_t strain_count; - std::string skip_line, read_line, substring; - std::stringstream line_parse; - CArrayKokkos read_buffer; - int buffer_loop, buffer_iteration, buffer_iterations, dof_limit, scan_loop; - size_t read_index_start, node_rid, elem_gid; - GO node_gid; - real_t dof_value; - bool zero_index_base = input_options.zero_index_base; - int negative_index_found = 0; - int global_negative_index_found = 0; - //Nodes_Per_Element_Type = elements::elem_types::Nodes_Per_Element_Type; - - //read the mesh - //PLACEHOLDER: ensight_format(MESH); - // abaqus_format(MESH); - // vtk_format(MESH) - - //task 0 reads file - if(myrank==0){ - std::cout << " NUM DIM is " << num_dim << std::endl; - in = new std::ifstream(); - in->open(MESH); - if (!(*in)) throw std::runtime_error(std::string("Can't open ") + MESH); - //skip 8 lines - for (int j = 1; j <= 8; j++) { - getline(*in, skip_line); - std::cout << skip_line << std::endl; - } //for - } - - // --- Read the number of nodes in the mesh --- // - if(myrank==0){ - getline(*in, read_line); - line_parse.str(read_line); - line_parse >> num_nodes; - std::cout << "declared node count: " << num_nodes << std::endl; - } - - //broadcast number of nodes - MPI_Bcast(&num_nodes,1,MPI_LONG_LONG_INT,0,world); - - //construct contiguous parallel row map now that we know the number of nodes - map = Teuchos::rcp( new Tpetra::Map(num_nodes,0,comm)); - //map->describe(*fos,Teuchos::VERB_EXTREME); - // set the vertices in the mesh read in - nlocal_nodes = map->getLocalNumElements(); - //populate local row offset data from global data - global_size_t min_gid = map->getMinGlobalIndex(); - global_size_t max_gid = map->getMaxGlobalIndex(); - global_size_t index_base = map->getIndexBase(); - //debug print - //std::cout << "local node count on task: " << " " << nlocal_nodes << std::endl; - - //allocate node storage with dual view - //dual_node_coords = dual_vec_array("dual_node_coords", nlocal_nodes,num_dim); - - //local variable for host view in the dual view - - node_coords_distributed = Teuchos::rcp(new MV(map, num_dim)); - - //scope ensures view is destroyed for now to avoid calling a device view with an active host view later - { - host_vec_array node_coords = node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); - //host_vec_array node_coords = dual_node_coords.view_host(); - //notify that the host view is going to be modified in the file readin - //dual_node_coords.modify_host(); - - //old swage method - //mesh->init_nodes(local_nrows); // add 1 for index starting at 1 - - std::cout << "Num nodes assigned to task " << myrank << " = " << nlocal_nodes << std::endl; - - // read the initial mesh coordinates - // x-coords - /*only task 0 reads in nodes and elements from the input file - stores node data in a buffer and communicates once the buffer cap is reached - or the data ends*/ - - words_per_line = input_options.words_per_line; - elem_words_per_line = input_options.elem_words_per_line; - - //allocate read buffer - read_buffer = CArrayKokkos(BUFFER_LINES,words_per_line,MAX_WORD); - - dof_limit = num_nodes; - buffer_iterations = dof_limit/BUFFER_LINES; - if(dof_limit%BUFFER_LINES!=0) buffer_iterations++; - - //x-coords - read_index_start = 0; - for(buffer_iteration = 0; buffer_iteration < buffer_iterations; buffer_iteration++){ - //pack buffer on rank 0 - if(myrank==0&&buffer_iteration> substring; - //debug print - //std::cout<<" "<< substring < read_buffer; + + GO node_gid; + real_t dof_value; + real_t unit_scaling = input_options.unit_scaling; + + // Nodes_Per_Element_Type = elements::elem_types::Nodes_Per_Element_Type; + + // read the mesh + // PLACEHOLDER: ensight_format(MESH); + // abaqus_format(MESH); + // vtk_format(MESH) + + // task 0 reads file + if (myrank == 0) + { + std::cout << " NUM DIM is " << num_dim << std::endl; + in = new std::ifstream(); + in->open(MESH); + if (!(*in)) + { + throw std::runtime_error(std::string("Can't open ") + MESH); } - } - } - else if(myrank==0){ - buffer_loop=0; - while(buffer_iteration*BUFFER_LINES+buffer_loop < num_nodes) { - getline(*in,read_line); - line_parse.clear(); + // skip 8 lines + for (int j = 1; j <= 8; j++) + { + getline(*in, skip_line); + std::cout << skip_line << std::endl; + } // for + } + + // --- Read the number of nodes in the mesh --- // + if (myrank == 0) + { + getline(*in, read_line); line_parse.str(read_line); - for(int iword = 0; iword < words_per_line; iword++){ - //read portions of the line into the substring variable - line_parse >> substring; - //assign the substring variable as a word of the read buffer - strcpy(&read_buffer(buffer_loop,iword,0),substring.c_str()); - } - buffer_loop++; - } - - } - - //broadcast buffer to all ranks; each rank will determine which nodes in the buffer belong - MPI_Bcast(read_buffer.pointer(),BUFFER_LINES*words_per_line*MAX_WORD,MPI_CHAR,0,world); - //broadcast how many nodes were read into this buffer iteration - MPI_Bcast(&buffer_loop,1,MPI_INT,0,world); - - //debug_print - //std::cout << "NODE BUFFER LOOP IS: " << buffer_loop << std::endl; - //for(int iprint=0; iprint < buffer_loop; iprint++) - //std::cout<<"buffer packing: " << std::string(&read_buffer(iprint,0,0)) << std::endl; - //return; - - //determine which data to store in the swage mesh members (the local node data) - //loop through read buffer - for(scan_loop = 0; scan_loop < buffer_loop; scan_loop++){ - //set global node id (ensight specific order) - node_gid = read_index_start + scan_loop; - //let map decide if this node id belongs locally; if yes store data - if(map->isNodeGlobalElement(node_gid)){ - //set local node index in this mpi rank - node_rid = map->getLocalElement(node_gid); - //extract nodal position from the read buffer - //for ensight format this is just one coordinate per line - dof_value = atof(&read_buffer(scan_loop,0,0)); - node_coords(node_rid, 0) = dof_value * unit_scaling; - } + line_parse >> num_nodes; + std::cout << "declared node count: " << num_nodes << std::endl; } - read_index_start+=BUFFER_LINES; - } - - - // y-coords - read_index_start = 0; - for(buffer_iteration = 0; buffer_iteration < buffer_iterations; buffer_iteration++){ - - //pack buffer on rank 0 - if(myrank==0&&buffer_iteration> substring; - //assign the substring variable as a word of the read buffer - strcpy(&read_buffer(buffer_loop,iword,0),substring.c_str()); + + // broadcast number of nodes + MPI_Bcast(&num_nodes, 1, MPI_LONG_LONG_INT, 0, world); + + // construct contiguous parallel row map now that we know the number of nodes + map = Teuchos::rcp(new Tpetra::Map(num_nodes, 0, comm)); + // map->describe(*fos,Teuchos::VERB_EXTREME); + // set the vertices in the mesh read in + nlocal_nodes = map->getLocalNumElements(); + // populate local row offset data from global data + global_size_t min_gid = map->getMinGlobalIndex(); + global_size_t max_gid = map->getMaxGlobalIndex(); + global_size_t index_base = map->getIndexBase(); + // debug print + // std::cout << "local node count on task: " << " " << nlocal_nodes << std::endl; + + // allocate node storage with dual view + // dual_node_coords = dual_vec_array("dual_node_coords", nlocal_nodes,num_dim); + + // local variable for host view in the dual view + + node_coords_distributed = Teuchos::rcp(new MV(map, num_dim)); + + // scope ensures view is destroyed for now to avoid calling a device view with an active host view later + { + host_vec_array node_coords = node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + // host_vec_array node_coords = dual_node_coords.view_host(); + // notify that the host view is going to be modified in the file readin + // dual_node_coords.modify_host(); + + // old swage method + // mesh->init_nodes(local_nrows); // add 1 for index starting at 1 + + std::cout << "Num nodes assigned to task " << myrank << " = " << nlocal_nodes << std::endl; + + // read the initial mesh coordinates + // x-coords + /*only task 0 reads in nodes and elements from the input file + stores node data in a buffer and communicates once the buffer cap is reached + or the data ends*/ + + words_per_line = input_options.words_per_line; + elem_words_per_line = input_options.elem_words_per_line; + + // allocate read buffer + read_buffer = CArrayKokkos(BUFFER_LINES, words_per_line, MAX_WORD); + + dof_limit = num_nodes; + buffer_iterations = dof_limit / BUFFER_LINES; + if (dof_limit % BUFFER_LINES != 0) + { + buffer_iterations++; } + + // x-coords + read_index_start = 0; + for (buffer_iteration = 0; buffer_iteration < buffer_iterations; buffer_iteration++) + { + // pack buffer on rank 0 + if (myrank == 0 && buffer_iteration < buffer_iterations - 1) + { + for (buffer_loop = 0; buffer_loop < BUFFER_LINES; buffer_loop++) + { + getline(*in, read_line); + line_parse.clear(); + line_parse.str(read_line); + + for (int iword = 0; iword < words_per_line; iword++) + { + // read portions of the line into the substring variable + line_parse >> substring; + // debug print + // std::cout<<" "<< substring <> substring; + // assign the substring variable as a word of the read buffer + strcpy(&read_buffer(buffer_loop, iword, 0), substring.c_str()); + } + buffer_loop++; + } + } + + // broadcast buffer to all ranks; each rank will determine which nodes in the buffer belong + MPI_Bcast(read_buffer.pointer(), BUFFER_LINES * words_per_line * MAX_WORD, MPI_CHAR, 0, world); + // broadcast how many nodes were read into this buffer iteration + MPI_Bcast(&buffer_loop, 1, MPI_INT, 0, world); + + // debug_print + // std::cout << "NODE BUFFER LOOP IS: " << buffer_loop << std::endl; + // for(int iprint=0; iprint < buffer_loop; iprint++) + // std::cout<<"buffer packing: " << std::string(&read_buffer(iprint,0,0)) << std::endl; + // return; + + // determine which data to store in the swage mesh members (the local node data) + // loop through read buffer + for (scan_loop = 0; scan_loop < buffer_loop; scan_loop++) + { + // set global node id (ensight specific order) + node_gid = read_index_start + scan_loop; + // let map decide if this node id belongs locally; if yes store data + if (map->isNodeGlobalElement(node_gid)) + { + // set local node index in this mpi rank + node_rid = map->getLocalElement(node_gid); + // extract nodal position from the read buffer + // for ensight format this is just one coordinate per line + dof_value = atof(&read_buffer(scan_loop, 0, 0)); + node_coords(node_rid, 0) = dof_value * unit_scaling; + } + } + read_index_start += BUFFER_LINES; + } + + // y-coords + read_index_start = 0; + for (buffer_iteration = 0; buffer_iteration < buffer_iterations; buffer_iteration++) + { + // pack buffer on rank 0 + if (myrank == 0 && buffer_iteration < buffer_iterations - 1) + { + for (buffer_loop = 0; buffer_loop < BUFFER_LINES; buffer_loop++) + { + getline(*in, read_line); + line_parse.clear(); + line_parse.str(read_line); + for (int iword = 0; iword < words_per_line; iword++) + { + // read portions of the line into the substring variable + line_parse >> substring; + // assign the substring variable as a word of the read buffer + strcpy(&read_buffer(buffer_loop, iword, 0), substring.c_str()); + } + } + } + else if (myrank == 0) + { + buffer_loop = 0; + while (buffer_iteration * BUFFER_LINES + buffer_loop < num_nodes) { + getline(*in, read_line); + line_parse.clear(); + line_parse.str(read_line); + for (int iword = 0; iword < words_per_line; iword++) + { + // read portions of the line into the substring variable + line_parse >> substring; + // assign the substring variable as a word of the read buffer + strcpy(&read_buffer(buffer_loop, iword, 0), substring.c_str()); + } + buffer_loop++; + // std::cout<<" "<< node_coords(node_gid, 0)<isNodeGlobalElement(node_gid)) + { + // set local node index in this mpi rank + node_rid = map->getLocalElement(node_gid); + // extract nodal position from the read buffer + // for ensight format this is just one coordinate per line + dof_value = atof(&read_buffer(scan_loop, 0, 0)); + node_coords(node_rid, 1) = dof_value * unit_scaling; + } + } + read_index_start += BUFFER_LINES; + } + + // z-coords + read_index_start = 0; + for (buffer_iteration = 0; buffer_iteration < buffer_iterations; buffer_iteration++) + { + // pack buffer on rank 0 + if (myrank == 0 && buffer_iteration < buffer_iterations - 1) + { + for (buffer_loop = 0; buffer_loop < BUFFER_LINES; buffer_loop++) + { + getline(*in, read_line); + line_parse.clear(); + line_parse.str(read_line); + for (int iword = 0; iword < words_per_line; iword++) + { + // read portions of the line into the substring variable + line_parse >> substring; + // assign the substring variable as a word of the read buffer + strcpy(&read_buffer(buffer_loop, iword, 0), substring.c_str()); + } + } + } + else if (myrank == 0) + { + buffer_loop = 0; + while (buffer_iteration * BUFFER_LINES + buffer_loop < num_nodes) { + getline(*in, read_line); + line_parse.clear(); + line_parse.str(read_line); + for (int iword = 0; iword < words_per_line; iword++) + { + // read portions of the line into the substring variable + line_parse >> substring; + // assign the substring variable as a word of the read buffer + strcpy(&read_buffer(buffer_loop, iword, 0), substring.c_str()); + } + buffer_loop++; + // std::cout<<" "<< node_coords(node_gid, 0)<isNodeGlobalElement(node_gid)) + { + // set local node index in this mpi rank + node_rid = map->getLocalElement(node_gid); + // extract nodal position from the read buffer + // for ensight format this is just one coordinate per line + dof_value = atof(&read_buffer(scan_loop, 0, 0)); + if (num_dim == 3) + { + node_coords(node_rid, 2) = dof_value * unit_scaling; + } + } + } + read_index_start += BUFFER_LINES; + } + } // end active view scope + // repartition node distribution + repartition_nodes(); + + // synchronize device data + // dual_node_coords.sync_device(); + // dual_node_coords.modify_device(); + + // debug print of nodal data + + // debug print nodal positions and indices + /* + std::cout << " ------------NODAL POSITIONS ON TASK " << myrank << " --------------"<getGlobalElement(inode) + 1 << " { "; + for (int istride = 0; istride < num_dim; istride++){ + std::cout << node_coords(inode,istride) << " , "; } + std::cout << " }"<< std::endl; } - else if(myrank==0){ - buffer_loop=0; - while(buffer_iteration*BUFFER_LINES+buffer_loop < num_nodes) { - getline(*in,read_line); - line_parse.clear(); - line_parse.str(read_line); - for(int iword = 0; iword < words_per_line; iword++){ - //read portions of the line into the substring variable - line_parse >> substring; - //assign the substring variable as a word of the read buffer - strcpy(&read_buffer(buffer_loop,iword,0),substring.c_str()); - } - buffer_loop++; - //std::cout<<" "<< node_coords(node_gid, 0)<isNodeGlobalElement(node_gid)){ - //set local node index in this mpi rank - node_rid = map->getLocalElement(node_gid); - //extract nodal position from the read buffer - //for ensight format this is just one coordinate per line - dof_value = atof(&read_buffer(scan_loop,0,0)); - node_coords(node_rid, 1) = dof_value * unit_scaling; - } + // check that local assignments match global total + + // read in element info (ensight file format is organized in element type sections) + // loop over this later for several element type sections + + num_elem = 0; + rnum_elem = 0; + CArrayKokkos node_store(elem_words_per_line); + + if (myrank == 0) + { + // skip element type name line + getline(*in, skip_line); + std::cout << skip_line << std::endl; } - read_index_start+=BUFFER_LINES; - } - - // z-coords - read_index_start = 0; - for(buffer_iteration = 0; buffer_iteration < buffer_iterations; buffer_iteration++){ - - //pack buffer on rank 0 - if(myrank==0&&buffer_iteration> substring; - //assign the substring variable as a word of the read buffer - strcpy(&read_buffer(buffer_loop,iword,0),substring.c_str()); + line_parse >> num_elem; + std::cout << "declared element count: " << num_elem << std::endl; + if (num_elem <= 0) + { + std::cout << "ERROR, NO ELEMENTS IN MESH" << std::endl; } - } - } - else if(myrank==0){ - buffer_loop=0; - while(buffer_iteration*BUFFER_LINES+buffer_loop < num_nodes) { - getline(*in,read_line); - line_parse.clear(); - line_parse.str(read_line); - for(int iword = 0; iword < words_per_line; iword++){ - //read portions of the line into the substring variable - line_parse >> substring; - //assign the substring variable as a word of the read buffer - strcpy(&read_buffer(buffer_loop,iword,0),substring.c_str()); - } - buffer_loop++; - //std::cout<<" "<< node_coords(node_gid, 0)<isNodeGlobalElement(node_gid)){ - //set local node index in this mpi rank - node_rid = map->getLocalElement(node_gid); - //extract nodal position from the read buffer - //for ensight format this is just one coordinate per line - dof_value = atof(&read_buffer(scan_loop,0,0)); - if(num_dim==3) - node_coords(node_rid, 2) = dof_value * unit_scaling; - } - } - read_index_start+=BUFFER_LINES; - } - } //end active view scope - //repartition node distribution - repartition_nodes(); - - //synchronize device data - //dual_node_coords.sync_device(); - //dual_node_coords.modify_device(); - - //debug print of nodal data - - //debug print nodal positions and indices - /* - std::cout << " ------------NODAL POSITIONS ON TASK " << myrank << " --------------"<getGlobalElement(inode) + 1 << " { "; - for (int istride = 0; istride < num_dim; istride++){ - std::cout << node_coords(inode,istride) << " , "; - } - std::cout << " }"<< std::endl; - } - */ - - //check that local assignments match global total - - - //read in element info (ensight file format is organized in element type sections) - //loop over this later for several element type sections - - num_elem = 0; - rnum_elem = 0; - CArrayKokkos node_store(elem_words_per_line); - - if(myrank==0){ - //skip element type name line - getline(*in, skip_line); - std::cout << skip_line << std::endl; - } - - // --- read the number of cells in the mesh --- - // --- Read the number of vertices in the mesh --- // - if(myrank==0){ - getline(*in, read_line); - line_parse.clear(); - line_parse.str(read_line); - line_parse >> num_elem; - std::cout << "declared element count: " << num_elem << std::endl; - if(num_elem <= 0) std::cout << "ERROR, NO ELEMENTS IN MESH" << std::endl; - } - - //broadcast number of elements - MPI_Bcast(&num_elem,1,MPI_LONG_LONG_INT,0,world); - - if(myrank == 0) - std::cout<<"before mesh initialization"<(BUFFER_LINES,elem_words_per_line,MAX_WORD); - - //calculate buffer iterations to read number of lines - buffer_iterations = num_elem/BUFFER_LINES; - int assign_flag; - - //dynamic buffer used to store elements before we know how many this rank needs - std::vector element_temp(BUFFER_LINES*elem_words_per_line); - std::vector global_indices_temp(BUFFER_LINES); - size_t buffer_max = BUFFER_LINES*elem_words_per_line; - size_t indices_buffer_max = BUFFER_LINES; - - if(num_elem%BUFFER_LINES!=0) buffer_iterations++; - read_index_start = 0; - //std::cout << "ELEMENT BUFFER ITERATIONS: " << buffer_iterations << std::endl; - rnum_elem = 0; - for(buffer_iteration = 0; buffer_iteration < buffer_iterations; buffer_iteration++){ - //pack buffer on rank 0 - if(myrank==0&&buffer_iteration> substring; - //assign the substring variable as a word of the read buffer - strcpy(&read_buffer(buffer_loop,iword,0),substring.c_str()); + // broadcast number of elements + MPI_Bcast(&num_elem, 1, MPI_LONG_LONG_INT, 0, world); + + if (myrank == 0) + { + std::cout << "before mesh initialization" << std::endl; + } + + // read in element connectivity + // we're gonna reallocate for the words per line expected for the element connectivity + read_buffer = CArrayKokkos(BUFFER_LINES, elem_words_per_line, MAX_WORD); + + // calculate buffer iterations to read number of lines + buffer_iterations = num_elem / BUFFER_LINES; + int assign_flag; + + // dynamic buffer used to store elements before we know how many this rank needs + std::vector element_temp(BUFFER_LINES * elem_words_per_line); + std::vector global_indices_temp(BUFFER_LINES); + size_t buffer_max = BUFFER_LINES * elem_words_per_line; + size_t indices_buffer_max = BUFFER_LINES; + + if (num_elem % BUFFER_LINES != 0) + { + buffer_iterations++; + } + read_index_start = 0; + // std::cout << "ELEMENT BUFFER ITERATIONS: " << buffer_iterations << std::endl; + rnum_elem = 0; + for (buffer_iteration = 0; buffer_iteration < buffer_iterations; buffer_iteration++) + { + // pack buffer on rank 0 + if (myrank == 0 && buffer_iteration < buffer_iterations - 1) + { + for (buffer_loop = 0; buffer_loop < BUFFER_LINES; buffer_loop++) + { + getline(*in, read_line); + line_parse.clear(); + line_parse.str(read_line); + for (int iword = 0; iword < elem_words_per_line; iword++) + { + // read portions of the line into the substring variable + line_parse >> substring; + // assign the substring variable as a word of the read buffer + strcpy(&read_buffer(buffer_loop, iword, 0), substring.c_str()); + } + } } - } + else if (myrank == 0) + { + buffer_loop = 0; + while (buffer_iteration * BUFFER_LINES + buffer_loop < num_elem) { + getline(*in, read_line); + line_parse.clear(); + line_parse.str(read_line); + for (int iword = 0; iword < elem_words_per_line; iword++) + { + // read portions of the line into the substring variable + line_parse >> substring; + // assign the substring variable as a word of the read buffer + strcpy(&read_buffer(buffer_loop, iword, 0), substring.c_str()); + } + buffer_loop++; + // std::cout<<" "<< node_coords(node_gid, 0)<isNodeGlobalElement(node_gid) && !assign_flag) + { + assign_flag = 1; + rnum_elem++; + } + } + else + { + if (map->isNodeGlobalElement(node_gid - 1) && !assign_flag) + { + assign_flag = 1; + rnum_elem++; + } + } + } + + if (assign_flag) + { + for (int inode = 0; inode < elem_words_per_line; inode++) + { + if ((rnum_elem - 1) * elem_words_per_line + inode >= buffer_max) + { + element_temp.resize((rnum_elem - 1) * elem_words_per_line + inode + BUFFER_LINES * elem_words_per_line); + buffer_max = (rnum_elem - 1) * elem_words_per_line + inode + BUFFER_LINES * elem_words_per_line; + } + element_temp[(rnum_elem - 1) * elem_words_per_line + inode] = node_store(inode); + // std::cout << "VECTOR STORAGE FOR ELEM " << rnum_elem << " ON TASK " << myrank << " NODE " << inode+1 << " IS " << node_store(inode) + 1 << std::endl; + } + // assign global element id to temporary list + if (rnum_elem - 1 >= indices_buffer_max) + { + global_indices_temp.resize(rnum_elem - 1 + BUFFER_LINES); + indices_buffer_max = rnum_elem - 1 + BUFFER_LINES; + } + global_indices_temp[rnum_elem - 1] = elem_gid; + } + } + read_index_start += BUFFER_LINES; } - else if(myrank==0){ - buffer_loop=0; - while(buffer_iteration*BUFFER_LINES+buffer_loop < num_elem) { - getline(*in,read_line); - line_parse.clear(); - line_parse.str(read_line); - for(int iword = 0; iword < elem_words_per_line; iword++){ - //read portions of the line into the substring variable - line_parse >> substring; - //assign the substring variable as a word of the read buffer - strcpy(&read_buffer(buffer_loop,iword,0),substring.c_str()); - } - buffer_loop++; - //std::cout<<" "<< node_coords(node_gid, 0)<close(); } - //broadcast buffer to all ranks; each rank will determine which nodes in the buffer belong - MPI_Bcast(read_buffer.pointer(),BUFFER_LINES*elem_words_per_line*MAX_WORD,MPI_CHAR,0,world); - //broadcast how many nodes were read into this buffer iteration - MPI_Bcast(&buffer_loop,1,MPI_INT,0,world); - - //store element connectivity that belongs to this rank - //loop through read buffer - for(scan_loop = 0; scan_loop < buffer_loop; scan_loop++){ - //set global node id (ensight specific order) - elem_gid = read_index_start + scan_loop; - //add this element to the local list if any of its nodes belong to this rank according to the map - //get list of nodes for each element line and check if they belong to the map - assign_flag = 0; - for(int inode = 0; inode < elem_words_per_line; inode++){ - //as we loop through the nodes belonging to this element we store them - //if any of these nodes belongs to this rank this list is used to store the element locally - node_gid = atoi(&read_buffer(scan_loop,inode,0)); - if(zero_index_base) - node_store(inode) = node_gid; //subtract 1 since file index start is 1 but code expects 0 + // std::cout << "RNUM ELEMENTS IS: " << rnum_elem << std::endl; + + Element_Types = CArrayKokkos(rnum_elem); + + elements::elem_types::elem_type mesh_element_type; + + if (simparam.num_dims == 2) + { + if (input_options.element_type == ELEMENT_TYPE::quad4) + { + mesh_element_type = elements::elem_types::Quad4; + max_nodes_per_patch = 2; + } + else if (input_options.element_type == ELEMENT_TYPE::quad8) + { + mesh_element_type = elements::elem_types::Quad8; + max_nodes_per_patch = 3; + } + else if (input_options.element_type == ELEMENT_TYPE::quad12) + { + mesh_element_type = elements::elem_types::Quad12; + max_nodes_per_patch = 4; + } else - node_store(inode) = node_gid - 1; //subtract 1 since file index start is 1 but code expects 0 - if(node_store(inode) < 0){ - negative_index_found = 1; - } - //first we add the elements to a dynamically allocated list - if(zero_index_base){ - if(map->isNodeGlobalElement(node_gid)&&!assign_flag){ - assign_flag = 1; - rnum_elem++; - } - } - else{ - if(map->isNodeGlobalElement(node_gid-1)&&!assign_flag){ - assign_flag = 1; - rnum_elem++; - } + { + if (myrank == 0) + { + std::cout << "ELEMENT TYPE UNRECOGNIZED" << std::endl; + } + exit_solver(0); } - } - - if(assign_flag){ - for(int inode = 0; inode < elem_words_per_line; inode++){ - if((rnum_elem-1)*elem_words_per_line + inode>=buffer_max){ - element_temp.resize((rnum_elem-1)*elem_words_per_line + inode + BUFFER_LINES*elem_words_per_line); - buffer_max = (rnum_elem-1)*elem_words_per_line + inode + BUFFER_LINES*elem_words_per_line; - } - element_temp[(rnum_elem-1)*elem_words_per_line + inode] = node_store(inode); - //std::cout << "VECTOR STORAGE FOR ELEM " << rnum_elem << " ON TASK " << myrank << " NODE " << inode+1 << " IS " << node_store(inode) + 1 << std::endl; - } - //assign global element id to temporary list - if(rnum_elem-1>=indices_buffer_max){ - global_indices_temp.resize(rnum_elem-1 + BUFFER_LINES); - indices_buffer_max = rnum_elem-1 + BUFFER_LINES; - } - global_indices_temp[rnum_elem-1] = elem_gid; - } - } - read_index_start+=BUFFER_LINES; - } - - // Close mesh input file - if(myrank==0) in->close(); - - //std::cout << "RNUM ELEMENTS IS: " << rnum_elem << std::endl; - - Element_Types = CArrayKokkos(rnum_elem); - - elements::elem_types::elem_type mesh_element_type; - - if(simparam.num_dims == 2){ - if(input_options.element_type == ELEMENT_TYPE::quad4){ - mesh_element_type = elements::elem_types::Quad4; - max_nodes_per_patch = 2; - } - else if(input_options.element_type == ELEMENT_TYPE::quad8){ - mesh_element_type = elements::elem_types::Quad8; - max_nodes_per_patch = 3; - } - else if(input_options.element_type == ELEMENT_TYPE::quad12){ - mesh_element_type = elements::elem_types::Quad12; - max_nodes_per_patch = 4; - } - else{ - if(myrank==0){ - std::cout << "ELEMENT TYPE UNRECOGNIZED" << std::endl; - } - exit_solver(0); + element_select->choose_2Delem_type(mesh_element_type, elem2D); + max_nodes_per_element = elem2D->num_nodes(); } - element_select->choose_2Delem_type(mesh_element_type, elem2D); - max_nodes_per_element = elem2D->num_nodes(); - } - if(simparam.num_dims == 3){ - if(input_options.element_type == ELEMENT_TYPE::hex8){ - mesh_element_type = elements::elem_types::Hex8; - max_nodes_per_patch = 4; + if (simparam.num_dims == 3) + { + if (input_options.element_type == ELEMENT_TYPE::hex8) + { + mesh_element_type = elements::elem_types::Hex8; + max_nodes_per_patch = 4; + } + else if (input_options.element_type == ELEMENT_TYPE::hex20) + { + mesh_element_type = elements::elem_types::Hex20; + max_nodes_per_patch = 8; + } + else if (input_options.element_type == ELEMENT_TYPE::hex32) + { + mesh_element_type = elements::elem_types::Hex32; + max_nodes_per_patch = 12; + } + else + { + if (myrank == 0) + { + std::cout << "ELEMENT TYPE UNRECOGNIZED" << std::endl; + } + exit_solver(0); + } + element_select->choose_3Delem_type(mesh_element_type, elem); + max_nodes_per_element = elem->num_nodes(); } - else if(input_options.element_type == ELEMENT_TYPE::hex20){ - mesh_element_type = elements::elem_types::Hex20; - max_nodes_per_patch = 8; + + // 1 type per mesh for now + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + Element_Types(ielem) = mesh_element_type; } - else if(input_options.element_type == ELEMENT_TYPE::hex32){ - mesh_element_type = elements::elem_types::Hex32; - max_nodes_per_patch = 12; + + // copy temporary element storage to multivector storage + dual_nodes_in_elem = dual_elem_conn_array("dual_nodes_in_elem", rnum_elem, max_nodes_per_element); + host_elem_conn_array nodes_in_elem = dual_nodes_in_elem.view_host(); + dual_nodes_in_elem.modify_host(); + + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + for (int inode = 0; inode < elem_words_per_line; inode++) + { + nodes_in_elem(ielem, inode) = element_temp[ielem * elem_words_per_line + inode]; + } } - else{ - if(myrank==0){ - std::cout << "ELEMENT TYPE UNRECOGNIZED" << std::endl; - } - exit_solver(0); + + // view storage for all local elements connected to local nodes on this rank + // DCArrayKokkos All_Element_Global_Indices(rnum_elem); + Kokkos::DualView All_Element_Global_Indices("All_Element_Global_Indices", rnum_elem); + // copy temporary global indices storage to view storage + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + All_Element_Global_Indices.h_view(ielem) = global_indices_temp[ielem]; + if (global_indices_temp[ielem] < 0) + { + negative_index_found = 1; + } } - element_select->choose_3Delem_type(mesh_element_type, elem); - max_nodes_per_element = elem->num_nodes(); - } - //1 type per mesh for now - for(int ielem = 0; ielem < rnum_elem; ielem++) - Element_Types(ielem) = mesh_element_type; - - //copy temporary element storage to multivector storage - dual_nodes_in_elem = dual_elem_conn_array("dual_nodes_in_elem", rnum_elem, max_nodes_per_element); - host_elem_conn_array nodes_in_elem = dual_nodes_in_elem.view_host(); - dual_nodes_in_elem.modify_host(); - - for(int ielem = 0; ielem < rnum_elem; ielem++) - for(int inode = 0; inode < elem_words_per_line; inode++){ - nodes_in_elem(ielem, inode) = element_temp[ielem*elem_words_per_line + inode]; - } - - //view storage for all local elements connected to local nodes on this rank - //DCArrayKokkos All_Element_Global_Indices(rnum_elem); - Kokkos::DualView All_Element_Global_Indices("All_Element_Global_Indices",rnum_elem); - //copy temporary global indices storage to view storage - for(int ielem = 0; ielem < rnum_elem; ielem++){ - All_Element_Global_Indices.h_view(ielem) = global_indices_temp[ielem]; - if(global_indices_temp[ielem]<0){ - negative_index_found = 1; + MPI_Allreduce(&negative_index_found, &global_negative_index_found, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD); + if (global_negative_index_found) + { + if (myrank == 0) + { + std::cout << "Node index less than or equal to zero detected; set \"zero_index_base: true\" under \"input_options\" in your yaml file if indices start at 0" << std::endl; + } + exit_solver(0); + } + // delete temporary element connectivity and index storage + std::vector().swap(element_temp); + std::vector().swap(global_indices_temp); + + All_Element_Global_Indices.modify_host(); + All_Element_Global_Indices.sync_device(); + + // debug print + /* + Kokkos::View All_Element_Global_Indices_pass("All_Element_Global_Indices_pass",rnum_elem); + deep_copy(All_Element_Global_Indices_pass, All_Element_Global_Indices.h_view); + std::cout << " ------------ELEMENT GLOBAL INDICES ON TASK " << myrank << " --------------"<(Teuchos::OrdinalTraits::invalid(), All_Element_Global_Indices.d_view, 0, comm)); + + // element type selection (subject to change) + // ---- Set Element Type ---- // + // allocate element type memory + // elements::elem_type_t* elem_choice; + + int NE = 1; // number of element types in problem + + // Convert ensight index system to the ijk finite element numbering convention + // for vertices in cell + if (active_node_ordering_convention == IJK) + { + CArrayKokkos convert_ensight_to_ijk(max_nodes_per_element); + CArrayKokkos tmp_ijk_indx(max_nodes_per_element); + convert_ensight_to_ijk(0) = 0; + convert_ensight_to_ijk(1) = 1; + convert_ensight_to_ijk(2) = 3; + convert_ensight_to_ijk(3) = 2; + convert_ensight_to_ijk(4) = 4; + convert_ensight_to_ijk(5) = 5; + convert_ensight_to_ijk(6) = 7; + convert_ensight_to_ijk(7) = 6; + + int nodes_per_element; + + if (num_dim == 2) + { + for (int cell_rid = 0; cell_rid < rnum_elem; cell_rid++) + { + // set nodes per element + element_select->choose_2Delem_type(Element_Types(cell_rid), elem2D); + nodes_per_element = elem2D->num_nodes(); + for (int node_lid = 0; node_lid < nodes_per_element; node_lid++) + { + tmp_ijk_indx(node_lid) = nodes_in_elem(cell_rid, convert_ensight_to_ijk(node_lid)); + } + + for (int node_lid = 0; node_lid < nodes_per_element; node_lid++) + { + nodes_in_elem(cell_rid, node_lid) = tmp_ijk_indx(node_lid); + } + } + } + + if (num_dim == 3) + { + for (int cell_rid = 0; cell_rid < rnum_elem; cell_rid++) + { + // set nodes per element + element_select->choose_3Delem_type(Element_Types(cell_rid), elem); + nodes_per_element = elem->num_nodes(); + for (int node_lid = 0; node_lid < nodes_per_element; node_lid++) + { + tmp_ijk_indx(node_lid) = nodes_in_elem(cell_rid, convert_ensight_to_ijk(node_lid)); + } + + for (int node_lid = 0; node_lid < nodes_per_element; node_lid++) + { + nodes_in_elem(cell_rid, node_lid) = tmp_ijk_indx(node_lid); + } + } + } } - } - - MPI_Allreduce(&negative_index_found,&global_negative_index_found,1,MPI_INT,MPI_MAX,MPI_COMM_WORLD); - if(global_negative_index_found){ - if(myrank==0){ - std::cout << "Node index less than or equal to zero detected; set \"zero_index_base: true\" under \"input_options\" in your yaml file if indices start at 0" << std::endl; + // debug print element edof + /* + std::cout << " ------------ELEMENT EDOF ON TASK " << myrank << " --------------"<().swap(element_temp); - std::vector().swap(global_indices_temp); - - All_Element_Global_Indices.modify_host(); - All_Element_Global_Indices.sync_device(); - - //debug print - /* - Kokkos::View All_Element_Global_Indices_pass("All_Element_Global_Indices_pass",rnum_elem); - deep_copy(All_Element_Global_Indices_pass, All_Element_Global_Indices.h_view); - std::cout << " ------------ELEMENT GLOBAL INDICES ON TASK " << myrank << " --------------"<(Teuchos::OrdinalTraits::invalid(),All_Element_Global_Indices.d_view,0,comm)); - - //element type selection (subject to change) - // ---- Set Element Type ---- // - // allocate element type memory - //elements::elem_type_t* elem_choice; - - int NE = 1; // number of element types in problem - - - // Convert ensight index system to the ijk finite element numbering convention - // for vertices in cell - if(active_node_ordering_convention == IJK){ - CArrayKokkos convert_ensight_to_ijk(max_nodes_per_element); - CArrayKokkos tmp_ijk_indx(max_nodes_per_element); - convert_ensight_to_ijk(0) = 0; - convert_ensight_to_ijk(1) = 1; - convert_ensight_to_ijk(2) = 3; - convert_ensight_to_ijk(3) = 2; - convert_ensight_to_ijk(4) = 4; - convert_ensight_to_ijk(5) = 5; - convert_ensight_to_ijk(6) = 7; - convert_ensight_to_ijk(7) = 6; - - int nodes_per_element; - - if(num_dim==2) - for (int cell_rid = 0; cell_rid < rnum_elem; cell_rid++) { - //set nodes per element - element_select->choose_2Delem_type(Element_Types(cell_rid), elem2D); - nodes_per_element = elem2D->num_nodes(); - for (int node_lid = 0; node_lid < nodes_per_element; node_lid++){ - tmp_ijk_indx(node_lid) = nodes_in_elem(cell_rid, convert_ensight_to_ijk(node_lid)); - } - - for (int node_lid = 0; node_lid < nodes_per_element; node_lid++){ - nodes_in_elem(cell_rid, node_lid) = tmp_ijk_indx(node_lid); - } - } - - if(num_dim==3) - for (int cell_rid = 0; cell_rid < rnum_elem; cell_rid++) { - //set nodes per element - element_select->choose_3Delem_type(Element_Types(cell_rid), elem); - nodes_per_element = elem->num_nodes(); - for (int node_lid = 0; node_lid < nodes_per_element; node_lid++){ - tmp_ijk_indx(node_lid) = nodes_in_elem(cell_rid, convert_ensight_to_ijk(node_lid)); - } - - for (int node_lid = 0; node_lid < nodes_per_element; node_lid++){ - nodes_in_elem(cell_rid, node_lid) = tmp_ijk_indx(node_lid); - } - } - } - //debug print element edof - /* - std::cout << " ------------ELEMENT EDOF ON TASK " << myrank << " --------------"< read_buffer; - int buffer_loop, buffer_iteration, buffer_iterations, dof_limit, scan_loop; - size_t read_index_start, node_rid, elem_gid; - GO node_gid; - real_t dof_value; - bool zero_index_base = input_options.zero_index_base; - int negative_index_found = 0; - int global_negative_index_found = 0; - //Nodes_Per_Element_Type = elements::elem_types::Nodes_Per_Element_Type; - - //read the mesh - //PLACEHOLDER: ensight_format(MESH); - // abaqus_format(MESH); - // vtk_format(MESH) - - // --- Read the number of nodes in the mesh --- // - num_nodes = 0; - if(myrank==0){ - std::cout << " NUM DIM is " << num_dim << std::endl; - in = new std::ifstream(); - in->open(MESH); - - int i = 0; - bool found = false; - while (found==false) { - std::getline(*in, read_line); - line_parse.str(""); - line_parse.clear(); - line_parse << read_line; - line_parse >> substring; - - // looking for the following text: - // POINTS %d float - if(substring == "POINTS"){ - line_parse >> num_nodes; - std::cout << "declared node count: " << num_nodes << std::endl; - if(num_nodes <= 0) throw std::runtime_error("ERROR, NO NODES IN MESH"); - found=true; - } // end if - - - if (i>1000){ - throw std::runtime_error("ERROR: Failed to find POINTS"); - break; - } // end if - - i++; - } // end while - } // end if(myrank==0) - - //broadcast number of nodes - MPI_Bcast(&num_nodes,1,MPI_LONG_LONG_INT,0,world); - - //construct contiguous parallel row map now that we know the number of nodes - map = Teuchos::rcp( new Tpetra::Map(num_nodes,0,comm)); - //map->describe(*fos,Teuchos::VERB_EXTREME); - - // set the vertices in the mesh read in - nlocal_nodes = map->getLocalNumElements(); - //populate local row offset data from global data - global_size_t min_gid = map->getMinGlobalIndex(); - global_size_t max_gid = map->getMaxGlobalIndex(); - global_size_t index_base = map->getIndexBase(); - //debug print - //std::cout << "local node count on task: " << " " << nlocal_nodes << std::endl; - - //allocate node storage with dual view - //dual_node_coords = dual_vec_array("dual_node_coords", nlocal_nodes,num_dim); - - //local variable for host view in the dual view - - node_coords_distributed = Teuchos::rcp(new MV(map, num_dim)); - - //scope ensures view is destroyed for now to avoid calling a device view with an active host view later - { - host_vec_array node_coords = node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); - //host_vec_array node_coords = dual_node_coords.view_host(); - //notify that the host view is going to be modified in the file readin - //dual_node_coords.modify_host(); - - //old swage method - //mesh->init_nodes(local_nrows); // add 1 for index starting at 1 - - std::cout << "Num nodes assigned to task " << myrank << " = " << nlocal_nodes << std::endl; - - // read the initial mesh coordinates - // x-coords - /*only task 0 reads in nodes and elements from the input file - stores node data in a buffer and communicates once the buffer cap is reached - or the data ends*/ - - words_per_line = input_options.words_per_line; - elem_words_per_line = input_options.elem_words_per_line; - - //allocate read buffer - read_buffer = CArrayKokkos(BUFFER_LINES,words_per_line,MAX_WORD); - - dof_limit = num_nodes; - buffer_iterations = dof_limit/BUFFER_LINES; - if(dof_limit%BUFFER_LINES!=0) buffer_iterations++; - - //read coords, also density if restarting - read_index_start = 0; - for(buffer_iteration = 0; buffer_iteration < buffer_iterations; buffer_iteration++){ - //pack buffer on rank 0 - if(myrank==0&&buffer_iteration> substring; - //debug print - //std::cout<<" "<< substring < read_buffer; + + // Nodes_Per_Element_Type = elements::elem_types::Nodes_Per_Element_Type; + + // read the mesh + // PLACEHOLDER: ensight_format(MESH); + // abaqus_format(MESH); + // vtk_format(MESH) + + // --- Read the number of nodes in the mesh --- // + num_nodes = 0; + if (myrank == 0) + { + std::cout << " NUM DIM is " << num_dim << std::endl; + in = new std::ifstream(); + in->open(MESH); + + int i = 0; + bool found = false; + while (found == false) { + std::getline(*in, read_line); + line_parse.str(""); + line_parse.clear(); + line_parse << read_line; + line_parse >> substring; + + // looking for the following text: + // POINTS %d float + if (substring == "POINTS") + { + line_parse >> num_nodes; + std::cout << "declared node count: " << num_nodes << std::endl; + if (num_nodes <= 0) + { + throw std::runtime_error("ERROR, NO NODES IN MESH"); + } + found = true; + } // end if + + if (i > 1000) + { + throw std::runtime_error("ERROR: Failed to find POINTS"); + break; + } // end if + + i++; + } // end while + } // end if(myrank==0) + + // broadcast number of nodes + MPI_Bcast(&num_nodes, 1, MPI_LONG_LONG_INT, 0, world); + + // construct contiguous parallel row map now that we know the number of nodes + map = Teuchos::rcp(new Tpetra::Map(num_nodes, 0, comm)); + // map->describe(*fos,Teuchos::VERB_EXTREME); + + // set the vertices in the mesh read in + nlocal_nodes = map->getLocalNumElements(); + // populate local row offset data from global data + global_size_t min_gid = map->getMinGlobalIndex(); + global_size_t max_gid = map->getMaxGlobalIndex(); + global_size_t index_base = map->getIndexBase(); + // debug print + // std::cout << "local node count on task: " << " " << nlocal_nodes << std::endl; + + // allocate node storage with dual view + // dual_node_coords = dual_vec_array("dual_node_coords", nlocal_nodes,num_dim); + + // local variable for host view in the dual view + + node_coords_distributed = Teuchos::rcp(new MV(map, num_dim)); + + // scope ensures view is destroyed for now to avoid calling a device view with an active host view later + { + host_vec_array node_coords = node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + // host_vec_array node_coords = dual_node_coords.view_host(); + // notify that the host view is going to be modified in the file readin + // dual_node_coords.modify_host(); + + // old swage method + // mesh->init_nodes(local_nrows); // add 1 for index starting at 1 + + std::cout << "Num nodes assigned to task " << myrank << " = " << nlocal_nodes << std::endl; + + // read the initial mesh coordinates + // x-coords + /*only task 0 reads in nodes and elements from the input file + stores node data in a buffer and communicates once the buffer cap is reached + or the data ends*/ + + words_per_line = input_options.words_per_line; + elem_words_per_line = input_options.elem_words_per_line; + + // allocate read buffer + read_buffer = CArrayKokkos(BUFFER_LINES, words_per_line, MAX_WORD); + + dof_limit = num_nodes; + buffer_iterations = dof_limit / BUFFER_LINES; + if (dof_limit % BUFFER_LINES != 0) + { + buffer_iterations++; } + + // read coords, also density if restarting + read_index_start = 0; + for (buffer_iteration = 0; buffer_iteration < buffer_iterations; buffer_iteration++) + { + // pack buffer on rank 0 + if (myrank == 0 && buffer_iteration < buffer_iterations - 1) + { + for (buffer_loop = 0; buffer_loop < BUFFER_LINES; buffer_loop++) + { + getline(*in, read_line); + line_parse.clear(); + line_parse.str(read_line); + + for (int iword = 0; iword < words_per_line; iword++) + { + // read portions of the line into the substring variable + line_parse >> substring; + // debug print + // std::cout<<" "<< substring <> substring; + // debug print + // std::cout<<" "<< substring <isNodeGlobalElement(node_gid)) + { + // set local node index in this mpi rank + node_rid = map->getLocalElement(node_gid); + // extract nodal position from the read buffer + // for tecplot format this is the three coords in the same line + dof_value = atof(&read_buffer(scan_loop, 0, 0)); + node_coords(node_rid, 0) = dof_value * unit_scaling; + dof_value = atof(&read_buffer(scan_loop, 1, 0)); + node_coords(node_rid, 1) = dof_value * unit_scaling; + if (num_dim == 3) + { + dof_value = atof(&read_buffer(scan_loop, 2, 0)); + node_coords(node_rid, 2) = dof_value * unit_scaling; + } + // extract density if restarting + } + } + read_index_start += BUFFER_LINES; + } + } // end of coordinate readin + // repartition node distribution + repartition_nodes(); + + // synchronize device data + // dual_node_coords.sync_device(); + // dual_node_coords.modify_device(); + + // debug print of nodal data + + // debug print nodal positions and indices + /* + std::cout << " ------------NODAL POSITIONS ON TASK " << myrank << " --------------"<getGlobalElement(inode) + 1 << " { "; + for (int istride = 0; istride < num_dim; istride++){ + std::cout << node_coords(inode,istride) << " , "; } + std::cout << " }"<< std::endl; + } + */ + + // check that local assignments match global total + + // read in element info (ensight file format is organized in element type sections) + // loop over this later for several element type sections + + num_elem = 0; + rnum_elem = 0; + CArrayKokkos node_store(elem_words_per_line); + + // --- read the number of cells in the mesh --- + // --- Read the number of vertices in the mesh --- // + if (myrank == 0) + { + bool found = false; + int i = 0; + while (found == false) { + std::getline(*in, read_line); + line_parse.str(""); + line_parse.clear(); + line_parse << read_line; + line_parse >> substring; + + // looking for the following text: + // CELLS num_cells size + if (substring == "CELLS") + { + line_parse >> num_elem; + std::cout << "declared element count: " << num_elem << std::endl; + if (num_elem <= 0) + { + throw std::runtime_error("ERROR, NO ELEMENTS IN MESH"); + } + found = true; + } // end if + + if (i > 1000) + { + throw std::runtime_error("ERROR: Failed to find CELLS"); + break; + } // end if + + i++; + } // end while + } // end if(myrank==0) + + // broadcast number of elements + MPI_Bcast(&num_elem, 1, MPI_LONG_LONG_INT, 0, world); + + if (myrank == 0) + { + std::cout << "before mesh initialization" << std::endl; + } + + // read in element connectivity + // we're gonna reallocate for the words per line expected for the element connectivity + read_buffer = CArrayKokkos(BUFFER_LINES, elem_words_per_line, MAX_WORD); + + // calculate buffer iterations to read number of lines + buffer_iterations = num_elem / BUFFER_LINES; + int assign_flag; + + // dynamic buffer used to store elements before we know how many this rank needs + std::vector element_temp(BUFFER_LINES * elem_words_per_line); + std::vector global_indices_temp(BUFFER_LINES); + size_t buffer_max = BUFFER_LINES * elem_words_per_line; + size_t indices_buffer_max = BUFFER_LINES; + + if (num_elem % BUFFER_LINES != 0) + { + buffer_iterations++; + } + read_index_start = 0; + // std::cout << "ELEMENT BUFFER ITERATIONS: " << buffer_iterations << std::endl; + rnum_elem = 0; + for (buffer_iteration = 0; buffer_iteration < buffer_iterations; buffer_iteration++) + { + // pack buffer on rank 0 + if (myrank == 0 && buffer_iteration < buffer_iterations - 1) + { + for (buffer_loop = 0; buffer_loop < BUFFER_LINES; buffer_loop++) + { + getline(*in, read_line); + line_parse.clear(); + line_parse.str(read_line); + // disregard node count line since we're using one element type per mesh + line_parse >> substring; + for (int iword = 0; iword < elem_words_per_line; iword++) + { + // read portions of the line into the substring variable + line_parse >> substring; + // debug print + // std::cout<<" "<< substring; + // assign the substring variable as a word of the read buffer + strcpy(&read_buffer(buffer_loop, iword, 0), substring.c_str()); + } + // std::cout <> substring; + for (int iword = 0; iword < elem_words_per_line; iword++) + { + // read portions of the line into the substring variable + line_parse >> substring; + // debug print + // std::cout<<" "<< substring; + // assign the substring variable as a word of the read buffer + strcpy(&read_buffer(buffer_loop, iword, 0), substring.c_str()); + } + // std::cout <isNodeGlobalElement(node_gid) && !assign_flag) + { + assign_flag = 1; + rnum_elem++; + } + } + else + { + if (map->isNodeGlobalElement(node_gid - 1) && !assign_flag) + { + assign_flag = 1; + rnum_elem++; + } + } + } + + if (assign_flag) + { + for (int inode = 0; inode < elem_words_per_line; inode++) + { + if ((rnum_elem - 1) * elem_words_per_line + inode >= buffer_max) + { + element_temp.resize((rnum_elem - 1) * elem_words_per_line + inode + BUFFER_LINES * elem_words_per_line); + buffer_max = (rnum_elem - 1) * elem_words_per_line + inode + BUFFER_LINES * elem_words_per_line; + } + element_temp[(rnum_elem - 1) * elem_words_per_line + inode] = node_store(inode); + // std::cout << "VECTOR STORAGE FOR ELEM " << rnum_elem << " ON TASK " << myrank << " NODE " << inode+1 << " IS " << node_store(inode) + 1 << std::endl; + } + // assign global element id to temporary list + if (rnum_elem - 1 >= indices_buffer_max) + { + global_indices_temp.resize(rnum_elem - 1 + BUFFER_LINES); + indices_buffer_max = rnum_elem - 1 + BUFFER_LINES; + } + global_indices_temp[rnum_elem - 1] = elem_gid; + } + } + read_index_start += BUFFER_LINES; } - else if(myrank==0){ - buffer_loop=0; - while(buffer_iteration*BUFFER_LINES+buffer_loop < num_nodes) { - getline(*in,read_line); - line_parse.clear(); - line_parse.str(read_line); - for(int iword = 0; iword < words_per_line; iword++){ - //read portions of the line into the substring variable - line_parse >> substring; - //debug print - //std::cout<<" "<< substring <isNodeGlobalElement(node_gid)){ - //set local node index in this mpi rank - node_rid = map->getLocalElement(node_gid); - //extract nodal position from the read buffer - //for tecplot format this is the three coords in the same line - dof_value = atof(&read_buffer(scan_loop,0,0)); - node_coords(node_rid, 0) = dof_value * unit_scaling; - dof_value = atof(&read_buffer(scan_loop,1,0)); - node_coords(node_rid, 1) = dof_value * unit_scaling; - if(num_dim==3){ - dof_value = atof(&read_buffer(scan_loop,2,0)); - node_coords(node_rid, 2) = dof_value * unit_scaling; - } - //extract density if restarting - } - } - read_index_start+=BUFFER_LINES; - } - } //end of coordinate readin - //repartition node distribution - repartition_nodes(); - - //synchronize device data - //dual_node_coords.sync_device(); - //dual_node_coords.modify_device(); - - //debug print of nodal data - - //debug print nodal positions and indices - /* - std::cout << " ------------NODAL POSITIONS ON TASK " << myrank << " --------------"<getGlobalElement(inode) + 1 << " { "; - for (int istride = 0; istride < num_dim; istride++){ - std::cout << node_coords(inode,istride) << " , "; - } - std::cout << " }"<< std::endl; - } - */ - - //check that local assignments match global total - - - //read in element info (ensight file format is organized in element type sections) - //loop over this later for several element type sections - - num_elem = 0; - rnum_elem = 0; - CArrayKokkos node_store(elem_words_per_line); - - // --- read the number of cells in the mesh --- - // --- Read the number of vertices in the mesh --- // - if(myrank==0){ - bool found = false; - int i = 0; - while (found==false) { - std::getline(*in, read_line); - line_parse.str(""); - line_parse.clear(); - line_parse << read_line; - line_parse >> substring; - - // looking for the following text: - // CELLS num_cells size - if(substring == "CELLS"){ - line_parse >> num_elem; - std::cout << "declared element count: " << num_elem << std::endl; - if(num_elem <= 0) throw std::runtime_error("ERROR, NO ELEMENTS IN MESH"); - found=true; - } // end if - - - if (i>1000){ - throw std::runtime_error("ERROR: Failed to find CELLS"); - break; - } // end if - - i++; - } // end while - } // end if(myrank==0) - - //broadcast number of elements - MPI_Bcast(&num_elem,1,MPI_LONG_LONG_INT,0,world); - - if(myrank == 0) - std::cout<<"before mesh initialization"<(BUFFER_LINES,elem_words_per_line,MAX_WORD); - - //calculate buffer iterations to read number of lines - buffer_iterations = num_elem/BUFFER_LINES; - int assign_flag; - - //dynamic buffer used to store elements before we know how many this rank needs - std::vector element_temp(BUFFER_LINES*elem_words_per_line); - std::vector global_indices_temp(BUFFER_LINES); - size_t buffer_max = BUFFER_LINES*elem_words_per_line; - size_t indices_buffer_max = BUFFER_LINES; - - if(num_elem%BUFFER_LINES!=0) buffer_iterations++; - read_index_start = 0; - //std::cout << "ELEMENT BUFFER ITERATIONS: " << buffer_iterations << std::endl; - rnum_elem = 0; - for(buffer_iteration = 0; buffer_iteration < buffer_iterations; buffer_iteration++){ - //pack buffer on rank 0 - if(myrank==0&&buffer_iteration> substring; - for(int iword = 0; iword < elem_words_per_line; iword++){ - //read portions of the line into the substring variable - line_parse >> substring; - //debug print - //std::cout<<" "<< substring; - //assign the substring variable as a word of the read buffer - strcpy(&read_buffer(buffer_loop,iword,0),substring.c_str()); - } - //std::cout <> substring; - for(int iword = 0; iword < elem_words_per_line; iword++){ - //read portions of the line into the substring variable - line_parse >> substring; - //debug print - //std::cout<<" "<< substring; - //assign the substring variable as a word of the read buffer - strcpy(&read_buffer(buffer_loop,iword,0),substring.c_str()); - } - //std::cout <close(); } - //broadcast buffer to all ranks; each rank will determine which nodes in the buffer belong - MPI_Bcast(read_buffer.pointer(),BUFFER_LINES*elem_words_per_line*MAX_WORD,MPI_CHAR,0,world); - //broadcast how many nodes were read into this buffer iteration - MPI_Bcast(&buffer_loop,1,MPI_INT,0,world); - - //store element connectivity that belongs to this rank - //loop through read buffer - for(scan_loop = 0; scan_loop < buffer_loop; scan_loop++){ - //set global node id (ensight specific order) - elem_gid = read_index_start + scan_loop; - //add this element to the local list if any of its nodes belong to this rank according to the map - //get list of nodes for each element line and check if they belong to the map - assign_flag = 0; - for(int inode = 0; inode < elem_words_per_line; inode++){ - //as we loop through the nodes belonging to this element we store them - //if any of these nodes belongs to this rank this list is used to store the element locally - node_gid = atoi(&read_buffer(scan_loop,inode,0)); - if(zero_index_base) - node_store(inode) = node_gid; //subtract 1 since file index start is 1 but code expects 0 + // std::cout << "RNUM ELEMENTS IS: " << rnum_elem << std::endl; + + Element_Types = CArrayKokkos(rnum_elem); + + elements::elem_types::elem_type mesh_element_type; + + if (simparam.num_dims == 2) + { + if (input_options.element_type == ELEMENT_TYPE::quad4) + { + mesh_element_type = elements::elem_types::Quad4; + max_nodes_per_patch = 2; + } + else if (input_options.element_type == ELEMENT_TYPE::quad8) + { + mesh_element_type = elements::elem_types::Quad8; + max_nodes_per_patch = 3; + } + else if (input_options.element_type == ELEMENT_TYPE::quad12) + { + mesh_element_type = elements::elem_types::Quad12; + max_nodes_per_patch = 4; + } else - node_store(inode) = node_gid - 1; //subtract 1 since file index start is 1 but code expects 0 - if(node_store(inode) < 0){ - negative_index_found = 1; - } - //first we add the elements to a dynamically allocated list - if(zero_index_base){ - if(map->isNodeGlobalElement(node_gid)&&!assign_flag){ - assign_flag = 1; - rnum_elem++; - } - } - else{ - if(map->isNodeGlobalElement(node_gid-1)&&!assign_flag){ - assign_flag = 1; - rnum_elem++; - } + { + if (myrank == 0) + { + std::cout << "ELEMENT TYPE UNRECOGNIZED" << std::endl; + } + exit_solver(0); } - } - - if(assign_flag){ - for(int inode = 0; inode < elem_words_per_line; inode++){ - if((rnum_elem-1)*elem_words_per_line + inode>=buffer_max){ - element_temp.resize((rnum_elem-1)*elem_words_per_line + inode + BUFFER_LINES*elem_words_per_line); - buffer_max = (rnum_elem-1)*elem_words_per_line + inode + BUFFER_LINES*elem_words_per_line; - } - element_temp[(rnum_elem-1)*elem_words_per_line + inode] = node_store(inode); - //std::cout << "VECTOR STORAGE FOR ELEM " << rnum_elem << " ON TASK " << myrank << " NODE " << inode+1 << " IS " << node_store(inode) + 1 << std::endl; - } - //assign global element id to temporary list - if(rnum_elem-1>=indices_buffer_max){ - global_indices_temp.resize(rnum_elem-1 + BUFFER_LINES); - indices_buffer_max = rnum_elem-1 + BUFFER_LINES; - } - global_indices_temp[rnum_elem-1] = elem_gid; - } - } - read_index_start+=BUFFER_LINES; - } - - // Close mesh input file - if(myrank==0) - in->close(); - - //std::cout << "RNUM ELEMENTS IS: " << rnum_elem << std::endl; - - Element_Types = CArrayKokkos(rnum_elem); - - elements::elem_types::elem_type mesh_element_type; - - if(simparam.num_dims == 2){ - if(input_options.element_type == ELEMENT_TYPE::quad4){ - mesh_element_type = elements::elem_types::Quad4; - max_nodes_per_patch = 2; - } - else if(input_options.element_type == ELEMENT_TYPE::quad8){ - mesh_element_type = elements::elem_types::Quad8; - max_nodes_per_patch = 3; - } - else if(input_options.element_type == ELEMENT_TYPE::quad12){ - mesh_element_type = elements::elem_types::Quad12; - max_nodes_per_patch = 4; - } - else{ - if(myrank==0){ - std::cout << "ELEMENT TYPE UNRECOGNIZED" << std::endl; - } - exit_solver(0); + element_select->choose_2Delem_type(mesh_element_type, elem2D); + max_nodes_per_element = elem2D->num_nodes(); } - element_select->choose_2Delem_type(mesh_element_type, elem2D); - max_nodes_per_element = elem2D->num_nodes(); - } - if(simparam.num_dims == 3){ - if(input_options.element_type == ELEMENT_TYPE::hex8){ - mesh_element_type = elements::elem_types::Hex8; - max_nodes_per_patch = 4; + if (simparam.num_dims == 3) + { + if (input_options.element_type == ELEMENT_TYPE::hex8) + { + mesh_element_type = elements::elem_types::Hex8; + max_nodes_per_patch = 4; + } + else if (input_options.element_type == ELEMENT_TYPE::hex20) + { + mesh_element_type = elements::elem_types::Hex20; + max_nodes_per_patch = 8; + } + else if (input_options.element_type == ELEMENT_TYPE::hex32) + { + mesh_element_type = elements::elem_types::Hex32; + max_nodes_per_patch = 12; + } + else + { + if (myrank == 0) + { + std::cout << "ELEMENT TYPE UNRECOGNIZED" << std::endl; + } + exit_solver(0); + } + element_select->choose_3Delem_type(mesh_element_type, elem); + max_nodes_per_element = elem->num_nodes(); } - else if(input_options.element_type == ELEMENT_TYPE::hex20){ - mesh_element_type = elements::elem_types::Hex20; - max_nodes_per_patch = 8; + + // 1 type per mesh for now + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + Element_Types(ielem) = mesh_element_type; } - else if(input_options.element_type == ELEMENT_TYPE::hex32){ - mesh_element_type = elements::elem_types::Hex32; - max_nodes_per_patch = 12; + + // copy temporary element storage to multivector storage + dual_nodes_in_elem = dual_elem_conn_array("dual_nodes_in_elem", rnum_elem, max_nodes_per_element); + host_elem_conn_array nodes_in_elem = dual_nodes_in_elem.view_host(); + dual_nodes_in_elem.modify_host(); + + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + for (int inode = 0; inode < elem_words_per_line; inode++) + { + nodes_in_elem(ielem, inode) = element_temp[ielem * elem_words_per_line + inode]; + } } - else{ - if(myrank==0){ - std::cout << "ELEMENT TYPE UNRECOGNIZED" << std::endl; - } - exit_solver(0); + + // view storage for all local elements connected to local nodes on this rank + // DCArrayKokkos All_Element_Global_Indices(rnum_elem); + Kokkos::DualView All_Element_Global_Indices("All_Element_Global_Indices", rnum_elem); + // copy temporary global indices storage to view storage + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + All_Element_Global_Indices.h_view(ielem) = global_indices_temp[ielem]; + if (global_indices_temp[ielem] < 0) + { + negative_index_found = 1; + } } - element_select->choose_3Delem_type(mesh_element_type, elem); - max_nodes_per_element = elem->num_nodes(); - } - //1 type per mesh for now - for(int ielem = 0; ielem < rnum_elem; ielem++) - Element_Types(ielem) = mesh_element_type; - - //copy temporary element storage to multivector storage - dual_nodes_in_elem = dual_elem_conn_array("dual_nodes_in_elem", rnum_elem, max_nodes_per_element); - host_elem_conn_array nodes_in_elem = dual_nodes_in_elem.view_host(); - dual_nodes_in_elem.modify_host(); - - for(int ielem = 0; ielem < rnum_elem; ielem++) - for(int inode = 0; inode < elem_words_per_line; inode++){ - nodes_in_elem(ielem, inode) = element_temp[ielem*elem_words_per_line + inode]; - } - - //view storage for all local elements connected to local nodes on this rank - //DCArrayKokkos All_Element_Global_Indices(rnum_elem); - Kokkos::DualView All_Element_Global_Indices("All_Element_Global_Indices",rnum_elem); - //copy temporary global indices storage to view storage - for(int ielem = 0; ielem < rnum_elem; ielem++){ - All_Element_Global_Indices.h_view(ielem) = global_indices_temp[ielem]; - if(global_indices_temp[ielem]<0){ - negative_index_found = 1; + MPI_Allreduce(&negative_index_found, &global_negative_index_found, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD); + if (global_negative_index_found) + { + if (myrank == 0) + { + std::cout << "Node index less than or equal to zero detected; set \"zero_index_base: true\" under \"input_options\" in your yaml file if indices start at 0" << std::endl; + } + exit_solver(0); + } + + // delete temporary element connectivity and index storage + std::vector().swap(element_temp); + std::vector().swap(global_indices_temp); + + All_Element_Global_Indices.modify_host(); + All_Element_Global_Indices.sync_device(); + + // debug print + /* + Kokkos::View All_Element_Global_Indices_pass("All_Element_Global_Indices_pass",rnum_elem); + deep_copy(All_Element_Global_Indices_pass, All_Element_Global_Indices.h_view); + std::cout << " ------------ELEMENT GLOBAL INDICES ON TASK " << myrank << " --------------"<(Teuchos::OrdinalTraits::invalid(), All_Element_Global_Indices.d_view, 0, comm)); + + // element type selection (subject to change) + // ---- Set Element Type ---- // + // allocate element type memory + // elements::elem_type_t* elem_choice; + + int NE = 1; // number of element types in problem + + // Convert ensight index system to the ijk finite element numbering convention + // for vertices in cell + if (active_node_ordering_convention == IJK) + { + CArrayKokkos convert_ensight_to_ijk(max_nodes_per_element); + CArrayKokkos tmp_ijk_indx(max_nodes_per_element); + convert_ensight_to_ijk(0) = 0; + convert_ensight_to_ijk(1) = 1; + convert_ensight_to_ijk(2) = 3; + convert_ensight_to_ijk(3) = 2; + convert_ensight_to_ijk(4) = 4; + convert_ensight_to_ijk(5) = 5; + convert_ensight_to_ijk(6) = 7; + convert_ensight_to_ijk(7) = 6; + + int nodes_per_element; + + if (num_dim == 2) + { + for (int cell_rid = 0; cell_rid < rnum_elem; cell_rid++) + { + // set nodes per element + element_select->choose_2Delem_type(Element_Types(cell_rid), elem2D); + nodes_per_element = elem2D->num_nodes(); + for (int node_lid = 0; node_lid < nodes_per_element; node_lid++) + { + tmp_ijk_indx(node_lid) = nodes_in_elem(cell_rid, convert_ensight_to_ijk(node_lid)); + } + + for (int node_lid = 0; node_lid < nodes_per_element; node_lid++) + { + nodes_in_elem(cell_rid, node_lid) = tmp_ijk_indx(node_lid); + } + } + } + + if (num_dim == 3) + { + for (int cell_rid = 0; cell_rid < rnum_elem; cell_rid++) + { + // set nodes per element + element_select->choose_3Delem_type(Element_Types(cell_rid), elem); + nodes_per_element = elem->num_nodes(); + for (int node_lid = 0; node_lid < nodes_per_element; node_lid++) + { + tmp_ijk_indx(node_lid) = nodes_in_elem(cell_rid, convert_ensight_to_ijk(node_lid)); + } + + for (int node_lid = 0; node_lid < nodes_per_element; node_lid++) + { + nodes_in_elem(cell_rid, node_lid) = tmp_ijk_indx(node_lid); + } + } + } } - } - - MPI_Allreduce(&negative_index_found,&global_negative_index_found,1,MPI_INT,MPI_MAX,MPI_COMM_WORLD); - if(global_negative_index_found){ - if(myrank==0){ - std::cout << "Node index less than or equal to zero detected; set \"zero_index_base: true\" under \"input_options\" in your yaml file if indices start at 0" << std::endl; + // debug print element edof + /* + std::cout << " ------------ELEMENT EDOF ON TASK " << myrank << " --------------"<().swap(element_temp); - std::vector().swap(global_indices_temp); - - All_Element_Global_Indices.modify_host(); - All_Element_Global_Indices.sync_device(); - - //debug print - /* - Kokkos::View All_Element_Global_Indices_pass("All_Element_Global_Indices_pass",rnum_elem); - deep_copy(All_Element_Global_Indices_pass, All_Element_Global_Indices.h_view); - std::cout << " ------------ELEMENT GLOBAL INDICES ON TASK " << myrank << " --------------"<(Teuchos::OrdinalTraits::invalid(),All_Element_Global_Indices.d_view,0,comm)); - - //element type selection (subject to change) - // ---- Set Element Type ---- // - // allocate element type memory - //elements::elem_type_t* elem_choice; - - int NE = 1; // number of element types in problem - - - // Convert ensight index system to the ijk finite element numbering convention - // for vertices in cell - if(active_node_ordering_convention == IJK){ - CArrayKokkos convert_ensight_to_ijk(max_nodes_per_element); - CArrayKokkos tmp_ijk_indx(max_nodes_per_element); - convert_ensight_to_ijk(0) = 0; - convert_ensight_to_ijk(1) = 1; - convert_ensight_to_ijk(2) = 3; - convert_ensight_to_ijk(3) = 2; - convert_ensight_to_ijk(4) = 4; - convert_ensight_to_ijk(5) = 5; - convert_ensight_to_ijk(6) = 7; - convert_ensight_to_ijk(7) = 6; - - int nodes_per_element; - - if(num_dim==2) - for (int cell_rid = 0; cell_rid < rnum_elem; cell_rid++) { - //set nodes per element - element_select->choose_2Delem_type(Element_Types(cell_rid), elem2D); - nodes_per_element = elem2D->num_nodes(); - for (int node_lid = 0; node_lid < nodes_per_element; node_lid++){ - tmp_ijk_indx(node_lid) = nodes_in_elem(cell_rid, convert_ensight_to_ijk(node_lid)); - } - - for (int node_lid = 0; node_lid < nodes_per_element; node_lid++){ - nodes_in_elem(cell_rid, node_lid) = tmp_ijk_indx(node_lid); - } - } - - if(num_dim==3) - for (int cell_rid = 0; cell_rid < rnum_elem; cell_rid++) { - //set nodes per element - element_select->choose_3Delem_type(Element_Types(cell_rid), elem); - nodes_per_element = elem->num_nodes(); - for (int node_lid = 0; node_lid < nodes_per_element; node_lid++){ - tmp_ijk_indx(node_lid) = nodes_in_elem(cell_rid, convert_ensight_to_ijk(node_lid)); - } - - for (int node_lid = 0; node_lid < nodes_per_element; node_lid++){ - nodes_in_elem(cell_rid, node_lid) = tmp_ijk_indx(node_lid); - } - } - } - //debug print element edof - /* - std::cout << " ------------ELEMENT EDOF ON TASK " << myrank << " --------------"< read_buffer; - int buffer_loop, buffer_iteration, buffer_iterations, dof_limit, scan_loop; - size_t read_index_start, node_rid, elem_gid; - GO node_gid; - real_t dof_value; - host_vec_array node_densities; - bool zero_index_base = input_options.zero_index_base; - int negative_index_found = 0; - int global_negative_index_found = 0; - //Nodes_Per_Element_Type = elements::elem_types::Nodes_Per_Element_Type; - - //read the mesh - //PLACEHOLDER: ensight_format(MESH); - // abaqus_format(MESH); - // vtk_format(MESH) - - //task 0 reads file - if(myrank==0){ - in = new std::ifstream(); - in->open(MESH); - //skip 2 lines - for (int j = 1; j <= 2; j++) { - getline(*in, skip_line); - std::cout << skip_line << std::endl; - } //for - } - - - // --- Read the number of nodes in the mesh --- // - if(myrank==0){ - getline(*in, read_line); - line_parse.str(read_line); - //stop when the NODES= string is reached - while (!line_parse.eof()){ - line_parse >> substring; - if(!substring.compare("NODES=")){ - line_parse >> num_nodes; - } - if(!substring.compare("ELEMENTS=")){ - line_parse >> num_elem; - } - } //while - std::cout << "declared node count: " << num_nodes << std::endl; - std::cout << "declared element count: " << num_elem << std::endl; - if(num_elem <= 0) std::cout << "ERROR, NO ELEMENTS IN MESH!!!!" << std::endl; - } - - //broadcast number of nodes - MPI_Bcast(&num_nodes,1,MPI_LONG_LONG_INT,0,world); - - //construct contiguous parallel row map now that we know the number of nodes - map = Teuchos::rcp( new Tpetra::Map(num_nodes,0,comm)); - - // set the vertices in the mesh read in - nlocal_nodes = map->getLocalNumElements(); - //populate local row offset data from global data - global_size_t min_gid = map->getMinGlobalIndex(); - global_size_t max_gid = map->getMaxGlobalIndex(); - global_size_t index_base = map->getIndexBase(); - //debug print - //std::cout << "local node count on task: " << " " << nlocal_nodes << std::endl; - - //allocate node storage with dual view - //dual_node_coords = dual_vec_array("dual_node_coords", nlocal_nodes,num_dim); - //if(restart_file) - //dual_node_densities = dual_vec_array("dual_node_densities", nlocal_nodes,1); - - //local variable for host view in the dual view - node_coords_distributed = Teuchos::rcp(new MV(map, num_dim)); - //active view scrope - { - host_vec_array node_coords = node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); - if(restart_file){ - design_node_densities_distributed = Teuchos::rcp(new MV(map, 1)); - node_densities = design_node_densities_distributed->getLocalView (Tpetra::Access::ReadWrite); - } - //host_vec_array node_coords = dual_node_coords.view_host(); - //if(restart_file) - //node_densities = dual_node_densities.view_host(); - //notify that the host view is going to be modified in the file readin - //dual_node_coords.modify_host(); - //if(restart_file) - //dual_node_densities.modify_host(); - - //old swage method - //mesh->init_nodes(local_nrows); // add 1 for index starting at 1 - - std::cout << "Num nodes assigned to task " << myrank << " = " << nlocal_nodes << std::endl; - - // read the initial mesh coordinates - // x-coords - /*only task 0 reads in nodes and elements from the input file - stores node data in a buffer and communicates once the buffer cap is reached - or the data ends*/ - - words_per_line = input_options.words_per_line; - if(restart_file) words_per_line++; - elem_words_per_line = input_options.elem_words_per_line; - - //allocate read buffer - read_buffer = CArrayKokkos(BUFFER_LINES,words_per_line,MAX_WORD); - - dof_limit = num_nodes; - buffer_iterations = dof_limit/BUFFER_LINES; - if(dof_limit%BUFFER_LINES!=0) buffer_iterations++; - - //read coords, also density if restarting - read_index_start = 0; - for(buffer_iteration = 0; buffer_iteration < buffer_iterations; buffer_iteration++){ - //pack buffer on rank 0 - if(myrank==0&&buffer_iteration read_buffer; + int buffer_loop, buffer_iteration, buffer_iterations, dof_limit, scan_loop; + size_t read_index_start, node_rid, elem_gid; + GO node_gid; + real_t dof_value; + host_vec_array node_densities; + bool zero_index_base = input_options.zero_index_base; + int negative_index_found = 0; + int global_negative_index_found = 0; + // Nodes_Per_Element_Type = elements::elem_types::Nodes_Per_Element_Type; + + // read the mesh + // PLACEHOLDER: ensight_format(MESH); + // abaqus_format(MESH); + // vtk_format(MESH) + + // task 0 reads file + if (myrank == 0) + { + in = new std::ifstream(); + in->open(MESH); + // skip 2 lines + for (int j = 1; j <= 2; j++) + { + getline(*in, skip_line); + std::cout << skip_line << std::endl; + } // for + } + + // --- Read the number of nodes in the mesh --- // + if (myrank == 0) + { + getline(*in, read_line); line_parse.str(read_line); - - for(int iword = 0; iword < words_per_line; iword++){ - //read portions of the line into the substring variable - line_parse >> substring; - //debug print - //std::cout<<" "<< substring <> substring; + if (!substring.compare("NODES=")) + { + line_parse >> num_nodes; + } + if (!substring.compare("ELEMENTS=")) + { + line_parse >> num_elem; + } + } // while + std::cout << "declared node count: " << num_nodes << std::endl; + std::cout << "declared element count: " << num_elem << std::endl; + if (num_elem <= 0) + { + std::cout << "ERROR, NO ELEMENTS IN MESH!!!!" << std::endl; } - } } - else if(myrank==0){ - buffer_loop=0; - while(buffer_iteration*BUFFER_LINES+buffer_loop < num_nodes) { - getline(*in,read_line); - line_parse.clear(); - line_parse.str(read_line); - for(int iword = 0; iword < words_per_line; iword++){ - //read portions of the line into the substring variable - line_parse >> substring; - //assign the substring variable as a word of the read buffer - strcpy(&read_buffer(buffer_loop,iword,0),substring.c_str()); + + // broadcast number of nodes + MPI_Bcast(&num_nodes, 1, MPI_LONG_LONG_INT, 0, world); + + // construct contiguous parallel row map now that we know the number of nodes + map = Teuchos::rcp(new Tpetra::Map(num_nodes, 0, comm)); + + // set the vertices in the mesh read in + nlocal_nodes = map->getLocalNumElements(); + // populate local row offset data from global data + global_size_t min_gid = map->getMinGlobalIndex(); + global_size_t max_gid = map->getMaxGlobalIndex(); + global_size_t index_base = map->getIndexBase(); + // debug print + // std::cout << "local node count on task: " << " " << nlocal_nodes << std::endl; + + // allocate node storage with dual view + // dual_node_coords = dual_vec_array("dual_node_coords", nlocal_nodes,num_dim); + // if(restart_file) + // dual_node_densities = dual_vec_array("dual_node_densities", nlocal_nodes,1); + + // local variable for host view in the dual view + node_coords_distributed = Teuchos::rcp(new MV(map, num_dim)); + // active view scrope + { + host_vec_array node_coords = node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + if (restart_file) + { + design_node_densities_distributed = Teuchos::rcp(new MV(map, 1)); + node_densities = design_node_densities_distributed->getLocalView(Tpetra::Access::ReadWrite); + } + // host_vec_array node_coords = dual_node_coords.view_host(); + // if(restart_file) + // node_densities = dual_node_densities.view_host(); + // notify that the host view is going to be modified in the file readin + // dual_node_coords.modify_host(); + // if(restart_file) + // dual_node_densities.modify_host(); + + // old swage method + // mesh->init_nodes(local_nrows); // add 1 for index starting at 1 + + std::cout << "Num nodes assigned to task " << myrank << " = " << nlocal_nodes << std::endl; + + // read the initial mesh coordinates + // x-coords + /*only task 0 reads in nodes and elements from the input file + stores node data in a buffer and communicates once the buffer cap is reached + or the data ends*/ + + words_per_line = input_options.words_per_line; + if (restart_file) + { + words_per_line++; + } + elem_words_per_line = input_options.elem_words_per_line; + + // allocate read buffer + read_buffer = CArrayKokkos(BUFFER_LINES, words_per_line, MAX_WORD); + + dof_limit = num_nodes; + buffer_iterations = dof_limit / BUFFER_LINES; + if (dof_limit % BUFFER_LINES != 0) + { + buffer_iterations++; + } + + // read coords, also density if restarting + read_index_start = 0; + for (buffer_iteration = 0; buffer_iteration < buffer_iterations; buffer_iteration++) + { + // pack buffer on rank 0 + if (myrank == 0 && buffer_iteration < buffer_iterations - 1) + { + for (buffer_loop = 0; buffer_loop < BUFFER_LINES; buffer_loop++) + { + getline(*in, read_line); + line_parse.clear(); + line_parse.str(read_line); + + for (int iword = 0; iword < words_per_line; iword++) + { + // read portions of the line into the substring variable + line_parse >> substring; + // debug print + // std::cout<<" "<< substring <> substring; + // assign the substring variable as a word of the read buffer + strcpy(&read_buffer(buffer_loop, iword, 0), substring.c_str()); + } + buffer_loop++; + } + } + + // broadcast buffer to all ranks; each rank will determine which nodes in the buffer belong + MPI_Bcast(read_buffer.pointer(), BUFFER_LINES * words_per_line * MAX_WORD, MPI_CHAR, 0, world); + // broadcast how many nodes were read into this buffer iteration + MPI_Bcast(&buffer_loop, 1, MPI_INT, 0, world); + + // debug_print + // std::cout << "NODE BUFFER LOOP IS: " << buffer_loop << std::endl; + // for(int iprint=0; iprint < buffer_loop; iprint++) + // std::cout<<"buffer packing: " << std::string(&read_buffer(iprint,0,0)) << std::endl; + // return; + + // determine which data to store in the swage mesh members (the local node data) + // loop through read buffer + for (scan_loop = 0; scan_loop < buffer_loop; scan_loop++) + { + // set global node id (ensight specific order) + node_gid = read_index_start + scan_loop; + // let map decide if this node id belongs locally; if yes store data + if (map->isNodeGlobalElement(node_gid)) + { + // set local node index in this mpi rank + node_rid = map->getLocalElement(node_gid); + // extract nodal position from the read buffer + // for tecplot format this is the three coords in the same line + dof_value = atof(&read_buffer(scan_loop, 0, 0)); + node_coords(node_rid, 0) = dof_value * unit_scaling; + dof_value = atof(&read_buffer(scan_loop, 1, 0)); + node_coords(node_rid, 1) = dof_value * unit_scaling; + if (num_dim == 3) + { + dof_value = atof(&read_buffer(scan_loop, 2, 0)); + node_coords(node_rid, 2) = dof_value * unit_scaling; + } + if (restart_file) + { + dof_value = atof(&read_buffer(scan_loop, num_dim, 0)); + node_densities(node_rid, 0) = dof_value; + } + // extract density if restarting + } + } + read_index_start += BUFFER_LINES; } - buffer_loop++; - } - - } - - //broadcast buffer to all ranks; each rank will determine which nodes in the buffer belong - MPI_Bcast(read_buffer.pointer(),BUFFER_LINES*words_per_line*MAX_WORD,MPI_CHAR,0,world); - //broadcast how many nodes were read into this buffer iteration - MPI_Bcast(&buffer_loop,1,MPI_INT,0,world); - - //debug_print - //std::cout << "NODE BUFFER LOOP IS: " << buffer_loop << std::endl; - //for(int iprint=0; iprint < buffer_loop; iprint++) - //std::cout<<"buffer packing: " << std::string(&read_buffer(iprint,0,0)) << std::endl; - //return; - - //determine which data to store in the swage mesh members (the local node data) - //loop through read buffer - for(scan_loop = 0; scan_loop < buffer_loop; scan_loop++){ - //set global node id (ensight specific order) - node_gid = read_index_start + scan_loop; - //let map decide if this node id belongs locally; if yes store data - if(map->isNodeGlobalElement(node_gid)){ - //set local node index in this mpi rank - node_rid = map->getLocalElement(node_gid); - //extract nodal position from the read buffer - //for tecplot format this is the three coords in the same line - dof_value = atof(&read_buffer(scan_loop,0,0)); - node_coords(node_rid, 0) = dof_value * unit_scaling; - dof_value = atof(&read_buffer(scan_loop,1,0)); - node_coords(node_rid, 1) = dof_value * unit_scaling; - if(num_dim==3){ - dof_value = atof(&read_buffer(scan_loop,2,0)); - node_coords(node_rid, 2) = dof_value * unit_scaling; - } - if(restart_file){ - dof_value = atof(&read_buffer(scan_loop,num_dim,0)); - node_densities(node_rid, 0) = dof_value; - } - //extract density if restarting - } } - read_index_start+=BUFFER_LINES; - } - } - //repartition node distribution - repartition_nodes(); - - //synchronize device data - //dual_node_coords.sync_device(); - //dual_node_coords.modify_device(); - //if(restart_file){ - //dual_node_densities.sync_device(); - //dual_node_densities.modify_device(); - //} - - //debug print of nodal data - - //debug print nodal positions and indices - - //std::cout << " ------------NODAL POSITIONS ON TASK " << myrank << " --------------"<getGlobalElement(inode) + 1 << " { "; - //for (int istride = 0; istride < num_dim; istride++){ - //std::cout << node_coords(inode,istride) << " , "; - //} - //std::cout << node_densities(inode,0); - //std::cout << " }"<< std::endl; - //} - - - //check that local assignments match global total - - - //read in element info (supported tecplot format currently assumes one type) - - CArrayKokkos node_store(elem_words_per_line); - - //broadcast number of elements - MPI_Bcast(&num_elem,1,MPI_LONG_LONG_INT,0,world); - //std::cout<<"before initial mesh initialization"<(BUFFER_LINES,elem_words_per_line,MAX_WORD); - - //calculate buffer iterations to read number of lines - buffer_iterations = num_elem/BUFFER_LINES; - int assign_flag; - - //dynamic buffer used to store elements before we know how many this rank needs - std::vector element_temp(BUFFER_LINES*elem_words_per_line); - std::vector global_indices_temp(BUFFER_LINES); - size_t buffer_max = BUFFER_LINES*elem_words_per_line; - size_t indices_buffer_max = BUFFER_LINES; - - if(num_elem%BUFFER_LINES!=0) buffer_iterations++; - read_index_start = 0; - //std::cout << "ELEMENT BUFFER ITERATIONS: " << buffer_iterations << std::endl; - rnum_elem = 0; - //std::cout << "BUFFER ITERATIONS IS: " << buffer_iterations << std::endl; - for(buffer_iteration = 0; buffer_iteration < buffer_iterations; buffer_iteration++){ - //pack buffer on rank 0 - if(myrank==0&&buffer_iteration> substring; - //assign the substring variable as a word of the read buffer - strcpy(&read_buffer(buffer_loop,iword,0),substring.c_str()); + // repartition node distribution + repartition_nodes(); + + // synchronize device data + // dual_node_coords.sync_device(); + // dual_node_coords.modify_device(); + // if(restart_file){ + // dual_node_densities.sync_device(); + // dual_node_densities.modify_device(); + // } + + // debug print of nodal data + + // debug print nodal positions and indices + + // std::cout << " ------------NODAL POSITIONS ON TASK " << myrank << " --------------"<getGlobalElement(inode) + 1 << " { "; + // for (int istride = 0; istride < num_dim; istride++){ + // std::cout << node_coords(inode,istride) << " , "; + // } + // std::cout << node_densities(inode,0); + // std::cout << " }"<< std::endl; + // } + + // check that local assignments match global total + + // read in element info (supported tecplot format currently assumes one type) + + CArrayKokkos node_store(elem_words_per_line); + + // broadcast number of elements + MPI_Bcast(&num_elem, 1, MPI_LONG_LONG_INT, 0, world); + // std::cout<<"before initial mesh initialization"<(BUFFER_LINES, elem_words_per_line, MAX_WORD); + + // calculate buffer iterations to read number of lines + buffer_iterations = num_elem / BUFFER_LINES; + int assign_flag; + + // dynamic buffer used to store elements before we know how many this rank needs + std::vector element_temp(BUFFER_LINES * elem_words_per_line); + std::vector global_indices_temp(BUFFER_LINES); + size_t buffer_max = BUFFER_LINES * elem_words_per_line; + size_t indices_buffer_max = BUFFER_LINES; + + if (num_elem % BUFFER_LINES != 0) + { + buffer_iterations++; + } + read_index_start = 0; + // std::cout << "ELEMENT BUFFER ITERATIONS: " << buffer_iterations << std::endl; + rnum_elem = 0; + // std::cout << "BUFFER ITERATIONS IS: " << buffer_iterations << std::endl; + for (buffer_iteration = 0; buffer_iteration < buffer_iterations; buffer_iteration++) + { + // pack buffer on rank 0 + if (myrank == 0 && buffer_iteration < buffer_iterations - 1) + { + for (buffer_loop = 0; buffer_loop < BUFFER_LINES; buffer_loop++) + { + getline(*in, read_line); + line_parse.clear(); + line_parse.str(read_line); + for (int iword = 0; iword < elem_words_per_line; iword++) + { + // read portions of the line into the substring variable + line_parse >> substring; + // assign the substring variable as a word of the read buffer + strcpy(&read_buffer(buffer_loop, iword, 0), substring.c_str()); + } + } } - } + else if (myrank == 0) + { + buffer_loop = 0; + while (buffer_iteration * BUFFER_LINES + buffer_loop < num_elem) { + getline(*in, read_line); + line_parse.clear(); + line_parse.str(read_line); + for (int iword = 0; iword < elem_words_per_line; iword++) + { + // read portions of the line into the substring variable + line_parse >> substring; + // assign the substring variable as a word of the read buffer + strcpy(&read_buffer(buffer_loop, iword, 0), substring.c_str()); + } + buffer_loop++; + // std::cout<<" "<< node_coords(node_gid, 0)<isNodeGlobalElement(node_gid) && !assign_flag) + { + assign_flag = 1; + rnum_elem++; + } + } + else + { + if (map->isNodeGlobalElement(node_gid - 1) && !assign_flag) + { + assign_flag = 1; + rnum_elem++; + } + } + } + + if (assign_flag) + { + for (int inode = 0; inode < elem_words_per_line; inode++) + { + if ((rnum_elem - 1) * elem_words_per_line + inode >= buffer_max) + { + element_temp.resize((rnum_elem - 1) * elem_words_per_line + inode + BUFFER_LINES * elem_words_per_line); + buffer_max = (rnum_elem - 1) * elem_words_per_line + inode + BUFFER_LINES * elem_words_per_line; + } + element_temp[(rnum_elem - 1) * elem_words_per_line + inode] = node_store(inode); + // std::cout << "VECTOR STORAGE FOR ELEM " << rnum_elem << " ON TASK " << myrank << " NODE " << inode+1 << " IS " << node_store(inode) + 1 << std::endl; + } + // assign global element id to temporary list + if (rnum_elem - 1 >= indices_buffer_max) + { + global_indices_temp.resize(rnum_elem - 1 + BUFFER_LINES); + indices_buffer_max = rnum_elem - 1 + BUFFER_LINES; + } + global_indices_temp[rnum_elem - 1] = elem_gid; + } + } + read_index_start += BUFFER_LINES; } - else if(myrank==0){ - buffer_loop=0; - while(buffer_iteration*BUFFER_LINES+buffer_loop < num_elem) { - getline(*in,read_line); - line_parse.clear(); - line_parse.str(read_line); - for(int iword = 0; iword < elem_words_per_line; iword++){ - //read portions of the line into the substring variable - line_parse >> substring; - //assign the substring variable as a word of the read buffer - strcpy(&read_buffer(buffer_loop,iword,0),substring.c_str()); - } - buffer_loop++; - //std::cout<<" "<< node_coords(node_gid, 0)<close(); } - //broadcast buffer to all ranks; each rank will determine which nodes in the buffer belong - MPI_Bcast(read_buffer.pointer(),BUFFER_LINES*elem_words_per_line*MAX_WORD,MPI_CHAR,0,world); - //broadcast how many nodes were read into this buffer iteration - MPI_Bcast(&buffer_loop,1,MPI_INT,0,world); - - //store element connectivity that belongs to this rank - //loop through read buffer - for(scan_loop = 0; scan_loop < buffer_loop; scan_loop++){ - //set global node id (ensight specific order) - elem_gid = read_index_start + scan_loop; - //add this element to the local list if any of its nodes belong to this rank according to the map - //get list of nodes for each element line and check if they belong to the map - assign_flag = 0; - for(int inode = 0; inode < elem_words_per_line; inode++){ - //as we loop through the nodes belonging to this element we store them - //if any of these nodes belongs to this rank this list is used to store the element locally - node_gid = atoi(&read_buffer(scan_loop,inode,0)); - if(zero_index_base) - node_store(inode) = node_gid; //subtract 1 since file index start is 1 but code expects 0 + std::cout << "RNUM ELEMENTS IS: " << rnum_elem << std::endl; + // copy temporary element storage to multivector storage + Element_Types = CArrayKokkos(rnum_elem); + + elements::elem_types::elem_type mesh_element_type; + + if (simparam.num_dims == 2) + { + if (input_options.element_type == ELEMENT_TYPE::quad4) + { + mesh_element_type = elements::elem_types::Quad4; + max_nodes_per_patch = 2; + } + else if (input_options.element_type == ELEMENT_TYPE::quad8) + { + mesh_element_type = elements::elem_types::Quad8; + max_nodes_per_patch = 3; + } + else if (input_options.element_type == ELEMENT_TYPE::quad12) + { + mesh_element_type = elements::elem_types::Quad12; + max_nodes_per_patch = 4; + } else - node_store(inode) = node_gid - 1; //subtract 1 since file index start is 1 but code expects 0 - if(node_store(inode) < 0){ - negative_index_found = 1; - } - //first we add the elements to a dynamically allocated list - if(zero_index_base){ - if(map->isNodeGlobalElement(node_gid)&&!assign_flag){ - assign_flag = 1; - rnum_elem++; - } - } - else{ - if(map->isNodeGlobalElement(node_gid-1)&&!assign_flag){ - assign_flag = 1; - rnum_elem++; - } + { + if (myrank == 0) + { + std::cout << "ELEMENT TYPE UNRECOGNIZED" << std::endl; + } + exit_solver(0); } - } + element_select->choose_2Delem_type(mesh_element_type, elem2D); + max_nodes_per_element = elem2D->num_nodes(); + } - if(assign_flag){ - for(int inode = 0; inode < elem_words_per_line; inode++){ - if((rnum_elem-1)*elem_words_per_line + inode>=buffer_max){ - element_temp.resize((rnum_elem-1)*elem_words_per_line + inode + BUFFER_LINES*elem_words_per_line); - buffer_max = (rnum_elem-1)*elem_words_per_line + inode + BUFFER_LINES*elem_words_per_line; - } - element_temp[(rnum_elem-1)*elem_words_per_line + inode] = node_store(inode); - //std::cout << "VECTOR STORAGE FOR ELEM " << rnum_elem << " ON TASK " << myrank << " NODE " << inode+1 << " IS " << node_store(inode) + 1 << std::endl; - } - //assign global element id to temporary list - if(rnum_elem-1>=indices_buffer_max){ - global_indices_temp.resize(rnum_elem-1 + BUFFER_LINES); - indices_buffer_max = rnum_elem-1 + BUFFER_LINES; - } - global_indices_temp[rnum_elem-1] = elem_gid; - } + if (simparam.num_dims == 3) + { + if (input_options.element_type == ELEMENT_TYPE::hex8) + { + mesh_element_type = elements::elem_types::Hex8; + max_nodes_per_patch = 4; + } + else if (input_options.element_type == ELEMENT_TYPE::hex20) + { + mesh_element_type = elements::elem_types::Hex20; + max_nodes_per_patch = 8; + } + else if (input_options.element_type == ELEMENT_TYPE::hex32) + { + mesh_element_type = elements::elem_types::Hex32; + max_nodes_per_patch = 12; + } + else + { + if (myrank == 0) + { + std::cout << "ELEMENT TYPE UNRECOGNIZED" << std::endl; + } + exit_solver(0); + } + << << << < HEAD + global_indices_temp[rnum_elem - 1] = elem_gid; } - read_index_start+=BUFFER_LINES; - } - - // Close mesh input file - if(myrank==0) - in->close(); - - std::cout << "RNUM ELEMENTS IS: " << rnum_elem << std::endl; - //copy temporary element storage to multivector storage - Element_Types = CArrayKokkos(rnum_elem); - - elements::elem_types::elem_type mesh_element_type; - - if(simparam.num_dims == 2){ - if(input_options.element_type == ELEMENT_TYPE::quad4){ - mesh_element_type = elements::elem_types::Quad4; - max_nodes_per_patch = 2; - } - else if(input_options.element_type == ELEMENT_TYPE::quad8){ - mesh_element_type = elements::elem_types::Quad8; - max_nodes_per_patch = 3; - } - else if(input_options.element_type == ELEMENT_TYPE::quad12){ - mesh_element_type = elements::elem_types::Quad12; - max_nodes_per_patch = 4; - } - else{ - if(myrank==0){ - std::cout << "ELEMENT TYPE UNRECOGNIZED" << std::endl; - } - exit_solver(0); +} + +read_index_start += BUFFER_LINES; +} + +// Close mesh input file +if (myrank == 0) +{ + in->close(); +} + +std::cout << "RNUM ELEMENTS IS: " << rnum_elem << std::endl; +// copy temporary element storage to multivector storage +Element_Types = CArrayKokkos(rnum_elem); + +elements::elem_types::elem_type mesh_element_type; + +if (simparam.num_dims == 2) +{ + if (input_options.element_type == ELEMENT_TYPE::quad4) + { + mesh_element_type = elements::elem_types::Quad4; + max_nodes_per_patch = 2; + } + else if (input_options.element_type == ELEMENT_TYPE::quad8) + { + mesh_element_type = elements::elem_types::Quad8; + max_nodes_per_patch = 3; + } + else if (input_options.element_type == ELEMENT_TYPE::quad12) + { + mesh_element_type = elements::elem_types::Quad12; + max_nodes_per_patch = 4; + } + else + { + if (myrank == 0) + { + std::cout << "ELEMENT TYPE UNRECOGNIZED" << std::endl; + } + exit_solver(0); } element_select->choose_2Delem_type(mesh_element_type, elem2D); max_nodes_per_element = elem2D->num_nodes(); - } +} - if(simparam.num_dims == 3){ - if(input_options.element_type == ELEMENT_TYPE::hex8){ - mesh_element_type = elements::elem_types::Hex8; - max_nodes_per_patch = 4; +if (simparam.num_dims == 3) +{ + if (input_options.element_type == ELEMENT_TYPE::hex8) + { + mesh_element_type = elements::elem_types::Hex8; + max_nodes_per_patch = 4; } - else if(input_options.element_type == ELEMENT_TYPE::hex20){ - mesh_element_type = elements::elem_types::Hex20; - max_nodes_per_patch = 8; + else if (input_options.element_type == ELEMENT_TYPE::hex20) + { + mesh_element_type = elements::elem_types::Hex20; + max_nodes_per_patch = 8; } - else if(input_options.element_type == ELEMENT_TYPE::hex32){ - mesh_element_type = elements::elem_types::Hex32; - max_nodes_per_patch = 12; + else if (input_options.element_type == ELEMENT_TYPE::hex32) + { + mesh_element_type = elements::elem_types::Hex32; + max_nodes_per_patch = 12; } - else{ - if(myrank==0){ - std::cout << "ELEMENT TYPE UNRECOGNIZED" << std::endl; - } - exit_solver(0); + else + { + if (myrank == 0) + { + std::cout << "ELEMENT TYPE UNRECOGNIZED" << std::endl; + } + exit_solver(0); } element_select->choose_3Delem_type(mesh_element_type, elem); max_nodes_per_element = elem->num_nodes(); - } +} - //1 type per mesh for now - for(int ielem = 0; ielem < rnum_elem; ielem++) +// 1 type per mesh for now +for (int ielem = 0; ielem < rnum_elem; ielem++) +{ Element_Types(ielem) = mesh_element_type; +} - dual_nodes_in_elem = dual_elem_conn_array("dual_nodes_in_elem", rnum_elem, max_nodes_per_element); - host_elem_conn_array nodes_in_elem = dual_nodes_in_elem.view_host(); - dual_nodes_in_elem.modify_host(); +dual_nodes_in_elem = dual_elem_conn_array("dual_nodes_in_elem", rnum_elem, max_nodes_per_element); +host_elem_conn_array nodes_in_elem = dual_nodes_in_elem.view_host(); +dual_nodes_in_elem.modify_host(); - for(int ielem = 0; ielem < rnum_elem; ielem++) - for(int inode = 0; inode < elem_words_per_line; inode++){ - nodes_in_elem(ielem, inode) = element_temp[ielem*elem_words_per_line + inode]; +for (int ielem = 0; ielem < rnum_elem; ielem++) +{ + for (int inode = 0; inode < elem_words_per_line; inode++) + { + nodes_in_elem(ielem, inode) = element_temp[ielem * elem_words_per_line + inode]; + == == == = + element_select->choose_3Delem_type(mesh_element_type, elem); + max_nodes_per_element = elem->num_nodes(); + >> >> >> > f635e485(STYLE: Uncrustify Parallel - Explicit solvers) } +} + +dual_nodes_in_elem = dual_elem_conn_array("dual_nodes_in_elem", rnum_elem, max_nodes_per_element); +host_elem_conn_array nodes_in_elem = dual_nodes_in_elem.view_host(); +dual_nodes_in_elem.modify_host(); + +for (int ielem = 0; ielem < rnum_elem; ielem++) +{ + for (int inode = 0; inode < elem_words_per_line; inode++) + { + nodes_in_elem(ielem, inode) = element_temp[ielem * elem_words_per_line + inode]; + } +} - //view storage for all local elements connected to local nodes on this rank - Kokkos::DualView All_Element_Global_Indices("All_Element_Global_Indices",rnum_elem); - //copy temporary global indices storage to view storage - for(int ielem = 0; ielem < rnum_elem; ielem++){ +// view storage for all local elements connected to local nodes on this rank +Kokkos::DualView All_Element_Global_Indices("All_Element_Global_Indices", rnum_elem); +// copy temporary global indices storage to view storage +for (int ielem = 0; ielem < rnum_elem; ielem++) +{ All_Element_Global_Indices.h_view(ielem) = global_indices_temp[ielem]; - if(global_indices_temp[ielem]<0){ - negative_index_found = 1; + if (global_indices_temp[ielem] < 0) + { + negative_index_found = 1; } - } - - MPI_Allreduce(&negative_index_found,&global_negative_index_found,1,MPI_INT,MPI_MAX,MPI_COMM_WORLD); - if(global_negative_index_found){ - if(myrank==0){ - std::cout << "Node index less than or equal to zero detected; set \"zero_index_base: true\" under \"input_options\" in your yaml file if indices start at 0" << std::endl; +} + +MPI_Allreduce(&negative_index_found, &global_negative_index_found, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD); +if (global_negative_index_found) +{ + if (myrank == 0) + { + std::cout << "Node index less than or equal to zero detected; set \"zero_index_base: true\" under \"input_options\" in your yaml file if indices start at 0" << std::endl; } exit_solver(0); - } - - //delete temporary element connectivity and index storage - std::vector().swap(element_temp); - std::vector().swap(global_indices_temp); - - All_Element_Global_Indices.modify_host(); - All_Element_Global_Indices.sync_device(); - - //construct overlapping element map (since different ranks can own the same elements due to the local node map) - all_element_map = Teuchos::rcp( new Tpetra::Map(Teuchos::OrdinalTraits::invalid(),All_Element_Global_Indices.d_view,0,comm)); - - - //element type selection (subject to change) - // ---- Set Element Type ---- // - // allocate element type memory - //elements::elem_type_t* elem_choice; - - int NE = 1; // number of element types in problem - - // Convert ijk index system to the finite element numbering convention - // for vertices in cell - if(active_node_ordering_convention == IJK){ - CArrayKokkos convert_ensight_to_ijk(max_nodes_per_element); - CArrayKokkos tmp_ijk_indx(max_nodes_per_element); - convert_ensight_to_ijk(0) = 0; - convert_ensight_to_ijk(1) = 1; - convert_ensight_to_ijk(2) = 3; - convert_ensight_to_ijk(3) = 2; - convert_ensight_to_ijk(4) = 4; - convert_ensight_to_ijk(5) = 5; - convert_ensight_to_ijk(6) = 7; - convert_ensight_to_ijk(7) = 6; - - int nodes_per_element; - - if(num_dim==2) - for (int cell_rid = 0; cell_rid < rnum_elem; cell_rid++) { - //set nodes per element - element_select->choose_2Delem_type(Element_Types(cell_rid), elem2D); - nodes_per_element = elem2D->num_nodes(); - for (int node_lid = 0; node_lid < nodes_per_element; node_lid++){ - tmp_ijk_indx(node_lid) = nodes_in_elem(cell_rid, convert_ensight_to_ijk(node_lid)); - } - - for (int node_lid = 0; node_lid < nodes_per_element; node_lid++){ - nodes_in_elem(cell_rid, node_lid) = tmp_ijk_indx(node_lid); - } - } - - if(num_dim==3) - for (int cell_rid = 0; cell_rid < rnum_elem; cell_rid++) { - //set nodes per element - element_select->choose_3Delem_type(Element_Types(cell_rid), elem); - nodes_per_element = elem->num_nodes(); - for (int node_lid = 0; node_lid < nodes_per_element; node_lid++){ - tmp_ijk_indx(node_lid) = nodes_in_elem(cell_rid, convert_ensight_to_ijk(node_lid)); - } - - for (int node_lid = 0; node_lid < nodes_per_element; node_lid++){ - nodes_in_elem(cell_rid, node_lid) = tmp_ijk_indx(node_lid); - } - } - } - - //debug print element edof - - //std::cout << " ------------ELEMENT EDOF ON TASK " << myrank << " --------------"<().swap(element_temp); +std::vector().swap(global_indices_temp); + +All_Element_Global_Indices.modify_host(); +All_Element_Global_Indices.sync_device(); + +// construct overlapping element map (since different ranks can own the same elements due to the local node map) +all_element_map = Teuchos::rcp(new Tpetra::Map(Teuchos::OrdinalTraits::invalid(), All_Element_Global_Indices.d_view, 0, comm)); + +// element type selection (subject to change) +// ---- Set Element Type ---- // +// allocate element type memory +// elements::elem_type_t* elem_choice; + +int NE = 1; // number of element types in problem + +// Convert ijk index system to the finite element numbering convention +// for vertices in cell +if (active_node_ordering_convention == IJK) +{ + CArrayKokkos convert_ensight_to_ijk(max_nodes_per_element); + CArrayKokkos tmp_ijk_indx(max_nodes_per_element); + convert_ensight_to_ijk(0) = 0; + convert_ensight_to_ijk(1) = 1; + convert_ensight_to_ijk(2) = 3; + convert_ensight_to_ijk(3) = 2; + convert_ensight_to_ijk(4) = 4; + convert_ensight_to_ijk(5) = 5; + convert_ensight_to_ijk(6) = 7; + convert_ensight_to_ijk(7) = 6; + + int nodes_per_element; + + if (num_dim == 2) + { + for (int cell_rid = 0; cell_rid < rnum_elem; cell_rid++) + { + // set nodes per element + element_select->choose_2Delem_type(Element_Types(cell_rid), elem2D); + nodes_per_element = elem2D->num_nodes(); + for (int node_lid = 0; node_lid < nodes_per_element; node_lid++) + { + tmp_ijk_indx(node_lid) = nodes_in_elem(cell_rid, convert_ensight_to_ijk(node_lid)); + } + + for (int node_lid = 0; node_lid < nodes_per_element; node_lid++) + { + nodes_in_elem(cell_rid, node_lid) = tmp_ijk_indx(node_lid); + } + } + } + + if (num_dim == 3) + { + for (int cell_rid = 0; cell_rid < rnum_elem; cell_rid++) + { + // set nodes per element + element_select->choose_3Delem_type(Element_Types(cell_rid), elem); + nodes_per_element = elem->num_nodes(); + for (int node_lid = 0; node_lid < nodes_per_element; node_lid++) + { + tmp_ijk_indx(node_lid) = nodes_in_elem(cell_rid, convert_ensight_to_ijk(node_lid)); + } + + for (int node_lid = 0; node_lid < nodes_per_element; node_lid++) + { + nodes_in_elem(cell_rid, node_lid) = tmp_ijk_indx(node_lid); + } + } + } +} + +// debug print element edof + +// std::cout << " ------------ELEMENT EDOF ON TASK " << myrank << " --------------"< read_buffer; - int nodes_per_element; - GO node_gid; - - //construct input adapted needed by Zoltan2 problem - typedef Xpetra::MultiVector xvector_t; - typedef Zoltan2::XpetraMultiVectorAdapter inputAdapter_t; - typedef Zoltan2::EvaluatePartition quality_t; - - Teuchos::RCP xpetra_node_coords = Teuchos::rcp(new Xpetra::TpetraMultiVector(node_coords_distributed)); - Teuchos::RCP problem_adapter = Teuchos::rcp(new inputAdapter_t(xpetra_node_coords)); - - // Create parameters for an RCB problem - - double tolerance = 1.05; - - Teuchos::ParameterList params("Node Partition Params"); - params.set("debug_level", "basic_status"); - params.set("debug_procs", "0"); - params.set("error_check_level", "debug_mode_assertions"); - - //params.set("algorithm", "rcb"); - params.set("algorithm", "multijagged"); - params.set("imbalance_tolerance", tolerance ); - params.set("num_global_parts", nranks); - params.set("partitioning_objective", "minimize_cut_edge_count"); - - Teuchos::RCP > problem = - Teuchos::rcp(new Zoltan2::PartitioningProblem(&(*problem_adapter), ¶ms)); - - // Solve the problem - - problem->solve(); - - // create metric object where communicator is Teuchos default - - quality_t *metricObject1 = new quality_t(&(*problem_adapter), ¶ms, //problem1->getComm(), - &problem->getSolution()); - // Check the solution. - - if (myrank == 0) { - metricObject1->printMetrics(std::cout); - } - - if (myrank == 0){ - real_t imb = metricObject1->getObjectCountImbalance(); - if (imb <= tolerance) - std::cout << "pass: " << imb << std::endl; - else - std::cout << "fail: " << imb << std::endl; - std::cout << std::endl; - } - delete metricObject1; - - //migrate rows of the vector so they correspond to the partition recommended by Zoltan2 - Teuchos::RCP partitioned_node_coords_distributed = Teuchos::rcp(new MV(map,num_dim)); - Teuchos::RCP xpartitioned_node_coords_distributed = - Teuchos::rcp(new Xpetra::TpetraMultiVector(partitioned_node_coords_distributed)); - - problem_adapter->applyPartitioningSolution(*xpetra_node_coords, xpartitioned_node_coords_distributed, problem->getSolution()); - *partitioned_node_coords_distributed = Xpetra::toTpetra(*xpartitioned_node_coords_distributed); - Teuchos::RCP > partitioned_map = Teuchos::rcp(new Tpetra::Map(*(partitioned_node_coords_distributed->getMap()))); - Teuchos::RCP > partitioned_map_one_to_one; - partitioned_map_one_to_one = Tpetra::createOneToOne(partitioned_map); - Teuchos::RCP partitioned_node_coords_one_to_one_distributed = Teuchos::rcp(new MV(partitioned_map_one_to_one,num_dim)); - - Tpetra::Import importer_one_to_one(partitioned_map, partitioned_map_one_to_one); - partitioned_node_coords_one_to_one_distributed->doImport(*partitioned_node_coords_distributed, importer_one_to_one, Tpetra::INSERT); - node_coords_distributed = partitioned_node_coords_one_to_one_distributed; - partitioned_map = Teuchos::rcp(new Tpetra::Map(*partitioned_map_one_to_one)); - - //migrate density vector if this is a restart file read - if(simparam.restart_file){ - Teuchos::RCP partitioned_node_densities_distributed = Teuchos::rcp(new MV(partitioned_map, 1)); - - //create import object using local node indices map and all indices map - Tpetra::Import importer(map, partitioned_map); - - //comms to get ghosts - partitioned_node_densities_distributed->doImport(*design_node_densities_distributed, importer, Tpetra::INSERT); - design_node_densities_distributed = partitioned_node_densities_distributed; - } - - //update nlocal_nodes and node map - map = partitioned_map; - nlocal_nodes = map->getLocalNumElements(); - +void Solver::repartition_nodes() +{ + char ch; + int num_dim = simparam.num_dims; + int local_node_index, current_column_index; + size_t strain_count; + std::stringstream line_parse; + CArrayKokkos read_buffer; + int nodes_per_element; + GO node_gid; + + // construct input adapted needed by Zoltan2 problem + typedef Xpetra::MultiVector xvector_t; + typedef Zoltan2::XpetraMultiVectorAdapter inputAdapter_t; + typedef Zoltan2::EvaluatePartition quality_t; + + Teuchos::RCP xpetra_node_coords = Teuchos::rcp(new Xpetra::TpetraMultiVector(node_coords_distributed)); + Teuchos::RCP problem_adapter = Teuchos::rcp(new inputAdapter_t(xpetra_node_coords)); + + // Create parameters for an RCB problem + + double tolerance = 1.05; + + Teuchos::ParameterList params("Node Partition Params"); + params.set("debug_level", "basic_status"); + params.set("debug_procs", "0"); + params.set("error_check_level", "debug_mode_assertions"); + + // params.set("algorithm", "rcb"); + params.set("algorithm", "multijagged"); + params.set("imbalance_tolerance", tolerance); + params.set("num_global_parts", nranks); + params.set("partitioning_objective", "minimize_cut_edge_count"); + + Teuchos::RCP> problem = + Teuchos::rcp(new Zoltan2::PartitioningProblem(&(*problem_adapter), ¶ms)); + + // Solve the problem + + problem->solve(); + + // create metric object where communicator is Teuchos default + + quality_t* metricObject1 = new quality_t(&(*problem_adapter), ¶ms, // problem1->getComm(), + &problem->getSolution()); + // Check the solution. + + if (myrank == 0) + { + metricObject1->printMetrics(std::cout); + } + + if (myrank == 0) + { + real_t imb = metricObject1->getObjectCountImbalance(); + if (imb <= tolerance) + { + std::cout << "pass: " << imb << std::endl; + } + else + { + std::cout << "fail: " << imb << std::endl; + } + std::cout << std::endl; + } + delete metricObject1; + + // migrate rows of the vector so they correspond to the partition recommended by Zoltan2 + Teuchos::RCP partitioned_node_coords_distributed = Teuchos::rcp(new MV(map, num_dim)); + Teuchos::RCP xpartitioned_node_coords_distributed = + Teuchos::rcp(new Xpetra::TpetraMultiVector(partitioned_node_coords_distributed)); + + problem_adapter->applyPartitioningSolution(*xpetra_node_coords, xpartitioned_node_coords_distributed, problem->getSolution()); + *partitioned_node_coords_distributed = Xpetra::toTpetra(*xpartitioned_node_coords_distributed); + Teuchos::RCP> partitioned_map = Teuchos::rcp(new Tpetra::Map(*(partitioned_node_coords_distributed->getMap()))); + Teuchos::RCP> partitioned_map_one_to_one; + partitioned_map_one_to_one = Tpetra::createOneToOne(partitioned_map); + Teuchos::RCP partitioned_node_coords_one_to_one_distributed = Teuchos::rcp(new MV(partitioned_map_one_to_one, num_dim)); + + Tpetra::Import importer_one_to_one(partitioned_map, partitioned_map_one_to_one); + partitioned_node_coords_one_to_one_distributed->doImport(*partitioned_node_coords_distributed, importer_one_to_one, Tpetra::INSERT); + node_coords_distributed = partitioned_node_coords_one_to_one_distributed; + partitioned_map = Teuchos::rcp(new Tpetra::Map(*partitioned_map_one_to_one)); + + // migrate density vector if this is a restart file read + if (simparam.restart_file) + { + Teuchos::RCP partitioned_node_densities_distributed = Teuchos::rcp(new MV(partitioned_map, 1)); + + // create import object using local node indices map and all indices map + Tpetra::Import importer(map, partitioned_map); + + // comms to get ghosts + partitioned_node_densities_distributed->doImport(*design_node_densities_distributed, importer, Tpetra::INSERT); + design_node_densities_distributed = partitioned_node_densities_distributed; + } + + // update nlocal_nodes and node map + map = partitioned_map; + nlocal_nodes = map->getLocalNumElements(); } /* ---------------------------------------------------------------------- Initialize Ghost and Non-Overlapping Element Maps ------------------------------------------------------------------------- */ -void Solver::init_maps(){ - char ch; - int num_dim = simparam.num_dims; - int local_node_index, current_column_index; - int nodes_per_element; - GO node_gid; - host_elem_conn_array nodes_in_elem = dual_nodes_in_elem.view_host(); - - if(rnum_elem >= 1) { - - //Construct set of ghost nodes; start with a buffer with upper limit - size_t buffer_limit = 0; - if(num_dim==2) - for(int ielem = 0; ielem < rnum_elem; ielem++){ - element_select->choose_2Delem_type(Element_Types(ielem), elem2D); - buffer_limit += elem2D->num_nodes(); - } - - if(num_dim==3) - for(int ielem = 0; ielem < rnum_elem; ielem++){ - element_select->choose_3Delem_type(Element_Types(ielem), elem); - buffer_limit += elem->num_nodes(); - } - - CArrayKokkos ghost_node_buffer(buffer_limit); - std::set ghost_node_set; - - //search through local elements for global node indices not owned by this MPI rank - if(num_dim==2) - for (int cell_rid = 0; cell_rid < rnum_elem; cell_rid++) { - //set nodes per element - element_select->choose_2Delem_type(Element_Types(cell_rid), elem2D); - nodes_per_element = elem2D->num_nodes(); - for (int node_lid = 0; node_lid < nodes_per_element; node_lid++){ - node_gid = nodes_in_elem(cell_rid, node_lid); - if(!map->isNodeGlobalElement(node_gid)) ghost_node_set.insert(node_gid); - } +void Solver::init_maps() +{ + char ch; + int num_dim = simparam.num_dims; + int local_node_index, current_column_index; + int nodes_per_element; + GO node_gid; + host_elem_conn_array nodes_in_elem = dual_nodes_in_elem.view_host(); + + if (rnum_elem >= 1) + { + // Construct set of ghost nodes; start with a buffer with upper limit + size_t buffer_limit = 0; + if (num_dim == 2) + { + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + element_select->choose_2Delem_type(Element_Types(ielem), elem2D); + buffer_limit += elem2D->num_nodes(); + } + } + + if (num_dim == 3) + { + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + element_select->choose_3Delem_type(Element_Types(ielem), elem); + buffer_limit += elem->num_nodes(); + } + } + + CArrayKokkos ghost_node_buffer(buffer_limit); + std::set ghost_node_set; + + // search through local elements for global node indices not owned by this MPI rank + if (num_dim == 2) + { + for (int cell_rid = 0; cell_rid < rnum_elem; cell_rid++) + { + // set nodes per element + element_select->choose_2Delem_type(Element_Types(cell_rid), elem2D); + nodes_per_element = elem2D->num_nodes(); + for (int node_lid = 0; node_lid < nodes_per_element; node_lid++) + { + node_gid = nodes_in_elem(cell_rid, node_lid); + if (!map->isNodeGlobalElement(node_gid)) + { + ghost_node_set.insert(node_gid); + } + } + } + } + + if (num_dim == 3) + { + for (int cell_rid = 0; cell_rid < rnum_elem; cell_rid++) + { + // set nodes per element + element_select->choose_3Delem_type(Element_Types(cell_rid), elem); + nodes_per_element = elem->num_nodes(); + for (int node_lid = 0; node_lid < nodes_per_element; node_lid++) + { + node_gid = nodes_in_elem(cell_rid, node_lid); + if (!map->isNodeGlobalElement(node_gid)) + { + ghost_node_set.insert(node_gid); + } + } + } + } + + // by now the set contains, with no repeats, all the global node indices that are ghosts for this rank + // now pass the contents of the set over to a CArrayKokkos, then create a map to find local ghost indices from global ghost indices + nghost_nodes = ghost_node_set.size(); + ghost_nodes = Kokkos::DualView("ghost_nodes", nghost_nodes); + ghost_node_ranks = Kokkos::DualView("ghost_node_ranks", nghost_nodes); + int ighost = 0; + auto it = ghost_node_set.begin(); + while (it != ghost_node_set.end()) { + ghost_nodes.h_view(ighost++) = *it; + it++; + } + + // debug print of ghost nodes + // std::cout << " GHOST NODE SET ON TASK " << myrank << std::endl; + // for(int i = 0; i < nghost_nodes; i++) + // std::cout << "{" << i + 1 << "," << ghost_nodes(i) + 1 << "}" << std::endl; + + // find which mpi rank each ghost node belongs to and store the information in a CArrayKokkos + // allocate Teuchos Views since they are the only input available at the moment in the map definitions + Teuchos::ArrayView ghost_nodes_pass(ghost_nodes.h_view.data(), nghost_nodes); + Teuchos::ArrayView ghost_node_ranks_pass(ghost_node_ranks.h_view.data(), nghost_nodes); + map->getRemoteIndexList(ghost_nodes_pass, ghost_node_ranks_pass); + + // debug print of ghost nodes + // std::cout << " GHOST NODE MAP ON TASK " << myrank << std::endl; + // for(int i = 0; i < nghost_nodes; i++) + // std::cout << "{" << i + 1 << "," << global2local_map.get(ghost_nodes(i)) + 1 << "}" << std::endl; } - if(num_dim==3) - for (int cell_rid = 0; cell_rid < rnum_elem; cell_rid++) { - //set nodes per element - element_select->choose_3Delem_type(Element_Types(cell_rid), elem); - nodes_per_element = elem->num_nodes(); - for (int node_lid = 0; node_lid < nodes_per_element; node_lid++){ - node_gid = nodes_in_elem(cell_rid, node_lid); - if(!map->isNodeGlobalElement(node_gid)) ghost_node_set.insert(node_gid); - } + ghost_nodes.modify_host(); + ghost_nodes.sync_device(); + ghost_node_ranks.modify_host(); + ghost_node_ranks.sync_device(); + // create a Map for ghost node indices + ghost_node_map = Teuchos::rcp(new Tpetra::Map(Teuchos::OrdinalTraits::invalid(), ghost_nodes.d_view, 0, comm)); + + // Create reference element + // ref_elem->init(p_order, num_dim, elem->num_basis()); + // std::cout<<"done with ref elem"< all_node_indices(nall_nodes, "all_node_indices"); + Kokkos::DualView all_node_indices("all_node_indices", nall_nodes); + for (int i = 0; i < nall_nodes; i++) + { + if (i < nlocal_nodes) + { + all_node_indices.h_view(i) = map->getGlobalElement(i); + } + else + { + all_node_indices.h_view(i) = ghost_nodes.h_view(i - nlocal_nodes); + } + } + all_node_indices.modify_host(); + all_node_indices.sync_device(); + // debug print of node indices + // for(int inode=0; inode < index_counter; inode++) + // std::cout << " my_reduced_global_indices " << my_reduced_global_indices(inode) <(Teuchos::OrdinalTraits::invalid(), all_node_indices.d_view, 0, comm)); + + // remove elements from the local set so that each rank has a unique set of global ids + + // local elements belonging to the non-overlapping element distribution to each rank with buffer + Kokkos::DualView Initial_Element_Global_Indices("Initial_Element_Global_Indices", rnum_elem); + size_t nonoverlapping_count = 0; + int my_element_flag; + // loop through local element set + if (num_dim == 2) + { + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + element_select->choose_2Delem_type(Element_Types(ielem), elem2D); + nodes_per_element = elem2D->num_nodes(); + my_element_flag = 1; + for (int lnode = 0; lnode < nodes_per_element; lnode++) + { + node_gid = nodes_in_elem(ielem, lnode); + if (ghost_node_map->isNodeGlobalElement(node_gid)) + { + local_node_index = ghost_node_map->getLocalElement(node_gid); + if (ghost_node_ranks.h_view(local_node_index) < myrank) + { + my_element_flag = 0; + } + } + } + if (my_element_flag) + { + Initial_Element_Global_Indices.h_view(nonoverlapping_count++) = all_element_map->getGlobalElement(ielem); + } + } } - //by now the set contains, with no repeats, all the global node indices that are ghosts for this rank - //now pass the contents of the set over to a CArrayKokkos, then create a map to find local ghost indices from global ghost indices - nghost_nodes = ghost_node_set.size(); - ghost_nodes = Kokkos::DualView ("ghost_nodes", nghost_nodes); - ghost_node_ranks = Kokkos::DualView ("ghost_node_ranks", nghost_nodes); - int ighost = 0; - auto it = ghost_node_set.begin(); - while(it!=ghost_node_set.end()){ - ghost_nodes.h_view(ighost++) = *it; - it++; - } - - //debug print of ghost nodes - //std::cout << " GHOST NODE SET ON TASK " << myrank << std::endl; - //for(int i = 0; i < nghost_nodes; i++) - //std::cout << "{" << i + 1 << "," << ghost_nodes(i) + 1 << "}" << std::endl; - - //find which mpi rank each ghost node belongs to and store the information in a CArrayKokkos - //allocate Teuchos Views since they are the only input available at the moment in the map definitions - Teuchos::ArrayView ghost_nodes_pass(ghost_nodes.h_view.data(), nghost_nodes); - Teuchos::ArrayView ghost_node_ranks_pass(ghost_node_ranks.h_view.data(), nghost_nodes); - map->getRemoteIndexList(ghost_nodes_pass, ghost_node_ranks_pass); - - //debug print of ghost nodes - //std::cout << " GHOST NODE MAP ON TASK " << myrank << std::endl; - //for(int i = 0; i < nghost_nodes; i++) - //std::cout << "{" << i + 1 << "," << global2local_map.get(ghost_nodes(i)) + 1 << "}" << std::endl; - - } - - ghost_nodes.modify_host(); - ghost_nodes.sync_device(); - ghost_node_ranks.modify_host(); - ghost_node_ranks.sync_device(); - // create a Map for ghost node indices - ghost_node_map = Teuchos::rcp( new Tpetra::Map(Teuchos::OrdinalTraits::invalid(),ghost_nodes.d_view,0,comm)); - - // Create reference element - //ref_elem->init(p_order, num_dim, elem->num_basis()); - //std::cout<<"done with ref elem"< all_node_indices(nall_nodes, "all_node_indices"); - Kokkos::DualView all_node_indices("all_node_indices", nall_nodes); - for(int i = 0; i < nall_nodes; i++){ - if(igetGlobalElement(i); - else all_node_indices.h_view(i) = ghost_nodes.h_view(i-nlocal_nodes); - } - all_node_indices.modify_host(); - all_node_indices.sync_device(); - //debug print of node indices - //for(int inode=0; inode < index_counter; inode++) - //std::cout << " my_reduced_global_indices " << my_reduced_global_indices(inode) <(Teuchos::OrdinalTraits::invalid(),all_node_indices.d_view,0,comm)); - - //remove elements from the local set so that each rank has a unique set of global ids - - //local elements belonging to the non-overlapping element distribution to each rank with buffer - Kokkos::DualView Initial_Element_Global_Indices("Initial_Element_Global_Indices", rnum_elem); - size_t nonoverlapping_count = 0; - int my_element_flag; - //loop through local element set - if(num_dim == 2) - for (int ielem = 0; ielem < rnum_elem; ielem++){ - element_select->choose_2Delem_type(Element_Types(ielem), elem2D); - nodes_per_element = elem2D->num_nodes(); - my_element_flag = 1; - for (int lnode = 0; lnode < nodes_per_element; lnode++){ - node_gid = nodes_in_elem(ielem, lnode); - if(ghost_node_map->isNodeGlobalElement(node_gid)){ - local_node_index = ghost_node_map->getLocalElement(node_gid); - if(ghost_node_ranks.h_view(local_node_index) < myrank) my_element_flag = 0; - } + if (num_dim == 3) + { + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + element_select->choose_3Delem_type(Element_Types(ielem), elem); + nodes_per_element = elem->num_nodes(); + my_element_flag = 1; + for (int lnode = 0; lnode < nodes_per_element; lnode++) + { + node_gid = nodes_in_elem(ielem, lnode); + if (ghost_node_map->isNodeGlobalElement(node_gid)) + { + local_node_index = ghost_node_map->getLocalElement(node_gid); + if (ghost_node_ranks.h_view(local_node_index) < myrank) + { + my_element_flag = 0; + } + } + } + if (my_element_flag) + { + Initial_Element_Global_Indices.h_view(nonoverlapping_count++) = all_element_map->getGlobalElement(ielem); + } + } } - if(my_element_flag){ - Initial_Element_Global_Indices.h_view(nonoverlapping_count++) = all_element_map->getGlobalElement(ielem); - } - } - - if(num_dim == 3) - for (int ielem = 0; ielem < rnum_elem; ielem++){ - element_select->choose_3Delem_type(Element_Types(ielem), elem); - nodes_per_element = elem->num_nodes(); - my_element_flag = 1; - for (int lnode = 0; lnode < nodes_per_element; lnode++){ - node_gid = nodes_in_elem(ielem, lnode); - if(ghost_node_map->isNodeGlobalElement(node_gid)){ - local_node_index = ghost_node_map->getLocalElement(node_gid); - if(ghost_node_ranks.h_view(local_node_index) < myrank) my_element_flag = 0; - } + + // copy over from buffer to compressed storage + Kokkos::DualView Element_Global_Indices("Element_Global_Indices", nonoverlapping_count); + for (int ibuffer = 0; ibuffer < nonoverlapping_count; ibuffer++) + { + Element_Global_Indices.h_view(ibuffer) = Initial_Element_Global_Indices.h_view(ibuffer); + } + nlocal_elem_non_overlapping = nonoverlapping_count; + Element_Global_Indices.modify_host(); + Element_Global_Indices.sync_device(); + // create nonoverlapping element map + element_map = Teuchos::rcp(new Tpetra::Map(Teuchos::OrdinalTraits::invalid(), Element_Global_Indices.d_view, 0, comm)); + + // sort element connectivity so nonoverlaps are sequentially found first + // define initial sorting of global indices + + // element_map->describe(*fos,Teuchos::VERB_EXTREME); + + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + Initial_Element_Global_Indices.h_view(ielem) = all_element_map->getGlobalElement(ielem); + } + + // re-sort so local elements in the nonoverlapping map are first in storage + CArrayKokkos Temp_Nodes(max_nodes_per_element); + GO temp_element_gid, current_element_gid; + int last_storage_index = rnum_elem - 1; + for (int ielem = 0; ielem < nlocal_elem_non_overlapping; ielem++) + { + current_element_gid = Initial_Element_Global_Indices.h_view(ielem); + // if this element is not part of the non overlap list then send it to the end of the storage and swap the element at the end + if (!element_map->isNodeGlobalElement(current_element_gid)) + { + temp_element_gid = current_element_gid; + for (int lnode = 0; lnode < max_nodes_per_element; lnode++) + { + Temp_Nodes(lnode) = nodes_in_elem(ielem, lnode); + } + Initial_Element_Global_Indices.h_view(ielem) = Initial_Element_Global_Indices.h_view(last_storage_index); + Initial_Element_Global_Indices.h_view(last_storage_index) = temp_element_gid; + for (int lnode = 0; lnode < max_nodes_per_element; lnode++) + { + nodes_in_elem(ielem, lnode) = nodes_in_elem(last_storage_index, lnode); + nodes_in_elem(last_storage_index, lnode) = Temp_Nodes(lnode); + } + last_storage_index--; + + // test if swapped element is also not part of the non overlap map; if so lower loop counter to repeat the above + temp_element_gid = Initial_Element_Global_Indices.h_view(ielem); + if (!element_map->isNodeGlobalElement(temp_element_gid)) + { + ielem--; + } + } } - if(my_element_flag){ - Initial_Element_Global_Indices.h_view(nonoverlapping_count++) = all_element_map->getGlobalElement(ielem); - } - } - - //copy over from buffer to compressed storage - Kokkos::DualView Element_Global_Indices("Element_Global_Indices",nonoverlapping_count); - for(int ibuffer = 0; ibuffer < nonoverlapping_count; ibuffer++){ - Element_Global_Indices.h_view(ibuffer) = Initial_Element_Global_Indices.h_view(ibuffer); - } - nlocal_elem_non_overlapping = nonoverlapping_count; - Element_Global_Indices.modify_host(); - Element_Global_Indices.sync_device(); - //create nonoverlapping element map - element_map = Teuchos::rcp( new Tpetra::Map(Teuchos::OrdinalTraits::invalid(),Element_Global_Indices.d_view,0,comm)); - - //sort element connectivity so nonoverlaps are sequentially found first - //define initial sorting of global indices - - //element_map->describe(*fos,Teuchos::VERB_EXTREME); - - for (int ielem = 0; ielem < rnum_elem; ielem++){ - Initial_Element_Global_Indices.h_view(ielem) = all_element_map->getGlobalElement(ielem); - } - - //re-sort so local elements in the nonoverlapping map are first in storage - CArrayKokkos Temp_Nodes(max_nodes_per_element); - GO temp_element_gid, current_element_gid; - int last_storage_index = rnum_elem - 1; - for (int ielem = 0; ielem < nlocal_elem_non_overlapping; ielem++){ - current_element_gid = Initial_Element_Global_Indices.h_view(ielem); - //if this element is not part of the non overlap list then send it to the end of the storage and swap the element at the end - if(!element_map->isNodeGlobalElement(current_element_gid)){ - temp_element_gid = current_element_gid; - for (int lnode = 0; lnode < max_nodes_per_element; lnode++){ - Temp_Nodes(lnode) = nodes_in_elem(ielem,lnode); - } - Initial_Element_Global_Indices.h_view(ielem) = Initial_Element_Global_Indices.h_view(last_storage_index); - Initial_Element_Global_Indices.h_view(last_storage_index) = temp_element_gid; - for (int lnode = 0; lnode < max_nodes_per_element; lnode++){ - nodes_in_elem(ielem, lnode) = nodes_in_elem(last_storage_index,lnode); - nodes_in_elem(last_storage_index,lnode) = Temp_Nodes(lnode); - } - last_storage_index--; - - //test if swapped element is also not part of the non overlap map; if so lower loop counter to repeat the above - temp_element_gid = Initial_Element_Global_Indices.h_view(ielem); - if(!element_map->isNodeGlobalElement(temp_element_gid)) ielem--; - } - } - //reset all element map to its re-sorted version - Initial_Element_Global_Indices.modify_host(); - Initial_Element_Global_Indices.sync_device(); - - all_element_map = Teuchos::rcp( new Tpetra::Map(Teuchos::OrdinalTraits::invalid(),Initial_Element_Global_Indices.d_view,0,comm)); - //element_map->describe(*fos,Teuchos::VERB_EXTREME); - //all_element_map->describe(*fos,Teuchos::VERB_EXTREME); - - //all_element_map->describe(*fos,Teuchos::VERB_EXTREME); - //construct dof map that follows from the node map (used for distributed matrix and vector objects later) - Kokkos::DualView local_dof_indices("local_dof_indices", nlocal_nodes*num_dim); - for(int i = 0; i < nlocal_nodes; i++){ - for(int j = 0; j < num_dim; j++) - local_dof_indices.h_view(i*num_dim + j) = map->getGlobalElement(i)*num_dim + j; - } - - local_dof_indices.modify_host(); - local_dof_indices.sync_device(); - local_dof_map = Teuchos::rcp( new Tpetra::Map(num_nodes*num_dim,local_dof_indices.d_view,0,comm) ); - - //construct dof map that follows from the all_node map (used for distributed matrix and vector objects later) - Kokkos::DualView all_dof_indices("all_dof_indices", nall_nodes*num_dim); - for(int i = 0; i < nall_nodes; i++){ - for(int j = 0; j < num_dim; j++) - all_dof_indices.h_view(i*num_dim + j) = all_node_map->getGlobalElement(i)*num_dim + j; - } - - all_dof_indices.modify_host(); - all_dof_indices.sync_device(); - //pass invalid global count so the map reduces the global count automatically - all_dof_map = Teuchos::rcp( new Tpetra::Map(Teuchos::OrdinalTraits::invalid(),all_dof_indices.d_view,0,comm) ); - - //debug print of map - //debug print - std::ostream &out = std::cout; - Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - //if(myrank==0) - //*fos << "Ghost Node Map :" << std::endl; - //all_node_map->describe(*fos,Teuchos::VERB_EXTREME); - //*fos << std::endl; - //std::fflush(stdout); - - //Count how many elements connect to each local node - node_nconn_distributed = Teuchos::rcp(new MCONN(map, 1)); - //active view scope - { - host_elem_conn_array node_nconn = node_nconn_distributed->getLocalView (Tpetra::Access::ReadWrite); - for(int inode = 0; inode < nlocal_nodes; inode++) - node_nconn(inode,0) = 0; - - for(int ielem = 0; ielem < rnum_elem; ielem++){ - for(int inode = 0; inode < nodes_per_element; inode++){ - node_gid = nodes_in_elem(ielem, inode); - if(map->isNodeGlobalElement(node_gid)) - node_nconn(map->getLocalElement(node_gid),0)++; - } + // reset all element map to its re-sorted version + Initial_Element_Global_Indices.modify_host(); + Initial_Element_Global_Indices.sync_device(); + + all_element_map = Teuchos::rcp(new Tpetra::Map(Teuchos::OrdinalTraits::invalid(), Initial_Element_Global_Indices.d_view, 0, comm)); + // element_map->describe(*fos,Teuchos::VERB_EXTREME); + // all_element_map->describe(*fos,Teuchos::VERB_EXTREME); + + // all_element_map->describe(*fos,Teuchos::VERB_EXTREME); + // construct dof map that follows from the node map (used for distributed matrix and vector objects later) + Kokkos::DualView local_dof_indices("local_dof_indices", nlocal_nodes * num_dim); + for (int i = 0; i < nlocal_nodes; i++) + { + for (int j = 0; j < num_dim; j++) + { + local_dof_indices.h_view(i * num_dim + j) = map->getGlobalElement(i) * num_dim + j; + } } - } - - //create distributed multivector of the local node data and all (local + ghost) node storage - all_node_coords_distributed = Teuchos::rcp(new MV(all_node_map, num_dim)); - ghost_node_coords_distributed = Teuchos::rcp(new MV(ghost_node_map, num_dim)); - - //debug print - //std::ostream &out = std::cout; - //Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - //if(myrank==0) - //*fos << "Node Data :" << std::endl; - //node_coords_distributed->describe(*fos,Teuchos::VERB_EXTREME); - //*fos << std::endl; - //std::fflush(stdout); - - //create import object using local node indices map and all indices map - comm_importer_setup(); - - //comms to get ghosts - all_node_coords_distributed->doImport(*node_coords_distributed, *importer, Tpetra::INSERT); - //all_node_nconn_distributed->doImport(*node_nconn_distributed, importer, Tpetra::INSERT); - - dual_nodes_in_elem.sync_device(); - dual_nodes_in_elem.modify_device(); - //construct distributed element connectivity multivector - global_nodes_in_elem_distributed = Teuchos::rcp(new MCONN(all_element_map, dual_nodes_in_elem)); - - if(nlocal_elem_non_overlapping >= 1) { - //construct map of nodes that belong to the non-overlapping element set (contained by ghost + local node set but not all of them) - std::set nonoverlap_elem_node_set; - - //search through local elements for global node indices not owned by this MPI rank - if(num_dim==2) - for (int cell_rid = 0; cell_rid < nlocal_elem_non_overlapping; cell_rid++) { - //set nodes per element - element_select->choose_2Delem_type(Element_Types(cell_rid), elem2D); - nodes_per_element = elem2D->num_nodes(); - for (int node_lid = 0; node_lid < nodes_per_element; node_lid++){ - node_gid = nodes_in_elem(cell_rid, node_lid); - nonoverlap_elem_node_set.insert(node_gid); - } + + local_dof_indices.modify_host(); + local_dof_indices.sync_device(); + local_dof_map = Teuchos::rcp(new Tpetra::Map(num_nodes * num_dim, local_dof_indices.d_view, 0, comm) ); + + // construct dof map that follows from the all_node map (used for distributed matrix and vector objects later) + Kokkos::DualView all_dof_indices("all_dof_indices", nall_nodes * num_dim); + for (int i = 0; i < nall_nodes; i++) + { + for (int j = 0; j < num_dim; j++) + { + all_dof_indices.h_view(i * num_dim + j) = all_node_map->getGlobalElement(i) * num_dim + j; + } } - if(num_dim==3) - for (int cell_rid = 0; cell_rid < nlocal_elem_non_overlapping; cell_rid++) { - //set nodes per element - element_select->choose_3Delem_type(Element_Types(cell_rid), elem); - nodes_per_element = elem->num_nodes(); - for (int node_lid = 0; node_lid < nodes_per_element; node_lid++){ - node_gid = nodes_in_elem(cell_rid, node_lid); - nonoverlap_elem_node_set.insert(node_gid); - } + all_dof_indices.modify_host(); + all_dof_indices.sync_device(); + // pass invalid global count so the map reduces the global count automatically + all_dof_map = Teuchos::rcp(new Tpetra::Map(Teuchos::OrdinalTraits::invalid(), all_dof_indices.d_view, 0, comm) ); + + // debug print of map + // debug print + std::ostream& out = std::cout; + Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + // if(myrank==0) + // *fos << "Ghost Node Map :" << std::endl; + // all_node_map->describe(*fos,Teuchos::VERB_EXTREME); + // *fos << std::endl; + // std::fflush(stdout); + + // Count how many elements connect to each local node + node_nconn_distributed = Teuchos::rcp(new MCONN(map, 1)); + // active view scope + { + host_elem_conn_array node_nconn = node_nconn_distributed->getLocalView(Tpetra::Access::ReadWrite); + for (int inode = 0; inode < nlocal_nodes; inode++) + { + node_nconn(inode, 0) = 0; + } + + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + for (int inode = 0; inode < nodes_per_element; inode++) + { + node_gid = nodes_in_elem(ielem, inode); + if (map->isNodeGlobalElement(node_gid)) + { + node_nconn(map->getLocalElement(node_gid), 0)++; + } + } + } } - //by now the set contains, with no repeats, all the global node indices belonging to the non overlapping element list on this MPI rank - //now pass the contents of the set over to a CArrayKokkos, then create a map to find local ghost indices from global ghost indices - nnonoverlap_elem_nodes = nonoverlap_elem_node_set.size(); - nonoverlap_elem_nodes = Kokkos::DualView ("nonoverlap_elem_nodes", nnonoverlap_elem_nodes); - int inonoverlap_elem_node = 0; - auto it = nonoverlap_elem_node_set.begin(); - while(it!=nonoverlap_elem_node_set.end()){ - nonoverlap_elem_nodes.h_view(inonoverlap_elem_node++) = *it; - it++; + // create distributed multivector of the local node data and all (local + ghost) node storage + all_node_coords_distributed = Teuchos::rcp(new MV(all_node_map, num_dim)); + ghost_node_coords_distributed = Teuchos::rcp(new MV(ghost_node_map, num_dim)); + + // create import object using local node indices map and all indices map + comm_importer_setup(); + + // comms to get ghosts + all_node_coords_distributed->doImport(*node_coords_distributed, *importer, Tpetra::INSERT); + // all_node_nconn_distributed->doImport(*node_nconn_distributed, importer, Tpetra::INSERT); + + dual_nodes_in_elem.sync_device(); + dual_nodes_in_elem.modify_device(); + // construct distributed element connectivity multivector + global_nodes_in_elem_distributed = Teuchos::rcp(new MCONN(all_element_map, dual_nodes_in_elem)); + + if (nlocal_elem_non_overlapping >= 1) + { + // construct map of nodes that belong to the non-overlapping element set (contained by ghost + local node set but not all of them) + std::set nonoverlap_elem_node_set; + + // search through local elements for global node indices not owned by this MPI rank + if (num_dim == 2) + { + for (int cell_rid = 0; cell_rid < nlocal_elem_non_overlapping; cell_rid++) + { + // set nodes per element + element_select->choose_2Delem_type(Element_Types(cell_rid), elem2D); + nodes_per_element = elem2D->num_nodes(); + for (int node_lid = 0; node_lid < nodes_per_element; node_lid++) + { + node_gid = nodes_in_elem(cell_rid, node_lid); + nonoverlap_elem_node_set.insert(node_gid); + } + } + } + + if (num_dim == 3) + { + for (int cell_rid = 0; cell_rid < nlocal_elem_non_overlapping; cell_rid++) + { + // set nodes per element + element_select->choose_3Delem_type(Element_Types(cell_rid), elem); + nodes_per_element = elem->num_nodes(); + for (int node_lid = 0; node_lid < nodes_per_element; node_lid++) + { + node_gid = nodes_in_elem(cell_rid, node_lid); + nonoverlap_elem_node_set.insert(node_gid); + } + } + } + + // by now the set contains, with no repeats, all the global node indices belonging to the non overlapping element list on this MPI rank + // now pass the contents of the set over to a CArrayKokkos, then create a map to find local ghost indices from global ghost indices + nnonoverlap_elem_nodes = nonoverlap_elem_node_set.size(); + nonoverlap_elem_nodes = Kokkos::DualView("nonoverlap_elem_nodes", nnonoverlap_elem_nodes); + int inonoverlap_elem_node = 0; + auto it = nonoverlap_elem_node_set.begin(); + while (it != nonoverlap_elem_node_set.end()) { + nonoverlap_elem_nodes.h_view(inonoverlap_elem_node++) = *it; + it++; + } + + nonoverlap_elem_nodes.modify_host(); + nonoverlap_elem_nodes.sync_device(); + // create a Map for ghost node indices + nonoverlap_element_node_map = Teuchos::rcp(new Tpetra::Map(Teuchos::OrdinalTraits::invalid(), nonoverlap_elem_nodes.d_view, 0, comm)); } - - nonoverlap_elem_nodes.modify_host(); - nonoverlap_elem_nodes.sync_device(); - // create a Map for ghost node indices - nonoverlap_element_node_map = Teuchos::rcp( new Tpetra::Map(Teuchos::OrdinalTraits::invalid(),nonoverlap_elem_nodes.d_view,0,comm)); - - } - - - //debug print - //std::ostream &out = std::cout; - //Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - //if(myrank==0) - //*fos << "Element Connectivity :" << std::endl; - //global_nodes_in_elem_distributed->describe(*fos,Teuchos::VERB_EXTREME); - //*fos << std::endl; - //std::fflush(stdout); - - //debug print - //std::ostream &out = std::cout; - //Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - //if(myrank==0) - //*fos << "Number of Elements Connected to Each Node :" << std::endl; - //all_node_nconn_distributed->describe(*fos,Teuchos::VERB_EXTREME); - //*fos << std::endl; - //std::fflush(stdout); - - //debug print - //std::ostream &out = std::cout; - //Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - //if(myrank==0) - //*fos << "Node Data with Ghosts :" << std::endl; - //all_node_coords_distributed->describe(*fos,Teuchos::VERB_EXTREME); - //*fos << std::endl; - //std::fflush(stdout); - - //debug print of views node indices - //std::cout << "Local View of All Nodes on Task " << myrank <getLocalNumElements(); inode++){ - //std::cout << "node "<getGlobalElement(inode) << " } " ; - //std::cout << dual_all_node_coords.view_host()(inode,0) << " " << dual_all_node_coords.view_host()(inode,1) << " " << dual_all_node_coords.view_host()(inode,2) << " " << std::endl; - //} - - //std::cout << "number of patches = " << mesh->num_patches() << std::endl; - if(myrank == 0) - std::cout << "End of map setup " << std::endl; + // std::cout << "number of patches = " << mesh->num_patches() << std::endl; + if (myrank == 0) + { + std::cout << "End of map setup " << std::endl; + } } /* ---------------------------------------------------------------------- Find boundary surface segments that belong to this MPI rank ------------------------------------------------------------------------- */ -void Solver::Get_Boundary_Patches(){ - size_t npatches_repeat, npatches, element_npatches, num_nodes_in_patch, node_gid; - int local_node_id; - int num_dim = simparam.num_dims; - CArray Surface_Nodes; - const_host_elem_conn_array nodes_in_elem = global_nodes_in_elem_distributed->getLocalView (Tpetra::Access::ReadOnly); - //Surface_Nodes = CArrayKokkos(4, "Surface_Nodes"); - - std::set my_patches; - //inititializes type for the pair variable (finding the iterator type is annoying) - std::pair::iterator, bool> current_combination; - std::set::iterator it; - - - CArrayKokkos convert_node_order(max_nodes_per_element); - if((active_node_ordering_convention == ENSIGHT && num_dim==3)||(active_node_ordering_convention == IJK && num_dim==2)){ - convert_node_order(0) = 0; - convert_node_order(1) = 1; - convert_node_order(2) = 3; - convert_node_order(3) = 2; - if(num_dim == 3){ - convert_node_order(4) = 4; - convert_node_order(5) = 5; - convert_node_order(6) = 7; - convert_node_order(7) = 6; - } - } - else if((active_node_ordering_convention == IJK && num_dim==3)||(active_node_ordering_convention == ENSIGHT && num_dim==2)){ - convert_node_order(0) = 0; - convert_node_order(1) = 1; - convert_node_order(2) = 2; - convert_node_order(3) = 3; - if(num_dim==3){ - convert_node_order(4) = 4; - convert_node_order(5) = 5; - convert_node_order(6) = 6; - convert_node_order(7) = 7; - } - } - - //compute the number of patches in this MPI rank with repeats for adjacent cells - npatches_repeat = 0; - - if(num_dim==2) - for(int ielem = 0; ielem < rnum_elem; ielem++){ - element_select->choose_2Delem_type(Element_Types(ielem), elem2D); - element_npatches = elem2D->nsurfaces; - npatches_repeat += element_npatches; - } - - else if(num_dim==3) - for(int ielem = 0; ielem < rnum_elem; ielem++){ - element_select->choose_3Delem_type(Element_Types(ielem), elem); - element_npatches = elem->nsurfaces; - npatches_repeat += element_npatches; - } - //std::cout << "Starting boundary patch allocation of size " << npatches_repeat << std::endl < Patch_Nodes(npatches_repeat, "Patch_Nodes"); - CArrayKokkos Patch_Boundary_Flags(npatches_repeat, "Patch_Boundary_Flags"); - if(myrank == 0) - std::cout << "Done with boundary patch allocation" << std::endl <choose_2Delem_type(Element_Types(ielem), elem2D); - element_npatches = elem2D->nsurfaces; - //loop through local surfaces - for(int isurface = 0; isurface < element_npatches; isurface++){ - num_nodes_in_patch = elem2D->surface_to_dof_lid.stride(isurface); - Surface_Nodes = CArray(num_nodes_in_patch); - for(int inode = 0; inode < num_nodes_in_patch; inode++){ - local_node_id = elem2D->surface_to_dof_lid(isurface,inode); - local_node_id = convert_node_order(local_node_id); - Surface_Nodes(inode) = nodes_in_elem(ielem, local_node_id); - } - Node_Combination temp(Surface_Nodes); - //construct Node Combination object for this surface - Patch_Nodes(npatches_repeat) = temp; - Patch_Nodes(npatches_repeat).patch_id = npatches_repeat; - Patch_Nodes(npatches_repeat).element_id = ielem; - Patch_Nodes(npatches_repeat).local_patch_id = isurface; - //test if this patch has already been added; if yes set boundary flags to 0 - current_combination = my_patches.insert(Patch_Nodes(npatches_repeat)); - //if the set determines this is a duplicate, access the original element's patch id and set flag to 0 - if(current_combination.second==false){ - //set original element flag to 0 - Patch_Boundary_Flags((*current_combination.first).patch_id) = 0; - //set this current flag to 0 for the duplicate as well - Patch_Boundary_Flags(npatches_repeat) = 0; - - } - npatches_repeat++; - } - } - - if(num_dim==3) - for(int ielem = 0; ielem < rnum_elem; ielem++){ - element_select->choose_3Delem_type(Element_Types(ielem), elem); - element_npatches = elem->nsurfaces; - //loop through local surfaces - for(int isurface = 0; isurface < element_npatches; isurface++){ - num_nodes_in_patch = elem->surface_to_dof_lid.stride(isurface); - //debug print - //std::cout << "NUMBER OF PATCH NODES FOR ELEMENT " << ielem+1 << " ON LOCAL SURFACE " << isurface+1 << " IS " << num_nodes_in_patch << std::endl; - Surface_Nodes = CArray(num_nodes_in_patch); - for(int inode = 0; inode < num_nodes_in_patch; inode++){ - local_node_id = elem->surface_to_dof_lid(isurface,inode); - local_node_id = convert_node_order(local_node_id); - Surface_Nodes(inode) = nodes_in_elem(ielem, local_node_id); - } - Node_Combination temp(Surface_Nodes); - //construct Node Combination object for this surface - Patch_Nodes(npatches_repeat) = temp; - Patch_Nodes(npatches_repeat).patch_id = npatches_repeat; - Patch_Nodes(npatches_repeat).element_id = ielem; - Patch_Nodes(npatches_repeat).local_patch_id = isurface; - //test if this patch has already been added; if yes set boundary flags to 0 - current_combination = my_patches.insert(Patch_Nodes(npatches_repeat)); - //if the set determines this is a duplicate access the original element's patch id and set flag to 0 - - if(current_combination.second==false){ - //set original element flag to 0 - Patch_Boundary_Flags((*current_combination.first).patch_id) = 0; - //set this current flag to 0 for the duplicate as well - Patch_Boundary_Flags(npatches_repeat) = 0; - } - npatches_repeat++; - } - } - //debug print of all patches - /* - std::cout << " ALL PATCHES " << npatches_repeat <isNodeGlobalElement(node_gid)){ - //if(ghost_node_ranks.h_view(global2local_map.get(node_gid)) Surface_Nodes; + const_host_elem_conn_array nodes_in_elem = global_nodes_in_elem_distributed->getLocalView(Tpetra::Access::ReadOnly); + // Surface_Nodes = CArrayKokkos(4, "Surface_Nodes"); + + std::set my_patches; + // inititializes type for the pair variable (finding the iterator type is annoying) + std::pair::iterator, bool> current_combination; + std::set::iterator it; + + CArrayKokkos convert_node_order(max_nodes_per_element); + if ((active_node_ordering_convention == ENSIGHT && num_dim == 3) || (active_node_ordering_convention == IJK && num_dim == 2)) + { + convert_node_order(0) = 0; + convert_node_order(1) = 1; + convert_node_order(2) = 3; + convert_node_order(3) = 2; + if (num_dim == 3) + { + convert_node_order(4) = 4; + convert_node_order(5) = 5; + convert_node_order(6) = 7; + convert_node_order(7) = 6; + } + } + else if ((active_node_ordering_convention == IJK && num_dim == 3) || (active_node_ordering_convention == ENSIGHT && num_dim == 2)) + { + convert_node_order(0) = 0; + convert_node_order(1) = 1; + convert_node_order(2) = 2; + convert_node_order(3) = 3; + if (num_dim == 3) + { + convert_node_order(4) = 4; + convert_node_order(5) = 5; + convert_node_order(6) = 6; + convert_node_order(7) = 7; + } + } - if(remote_count == num_nodes_in_patch) my_rank_flag = false; - //all nodes were remote - //if(remote_count == num_nodes_in_patch) my_rank_flag = false; + // compute the number of patches in this MPI rank with repeats for adjacent cells + npatches_repeat = 0; - //if at least one node was local - if(my_rank_flag){ - Boundary_Patches(nboundary_patches++) = Patch_Nodes(ipatch); - boundary_patch_to_index[Patch_Nodes(ipatch)] = nboundary_patches-1; - } + if (num_dim == 2) + { + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + element_select->choose_2Delem_type(Element_Types(ielem), elem2D); + element_npatches = elem2D->nsurfaces; + npatches_repeat += element_npatches; + } } - } - - //debug print of boundary patches - /* - std::cout << " BOUNDARY PATCHES ON TASK " << myrank << " = " << nboundary_patches <choose_3Delem_type(Element_Types(ielem), elem); + element_npatches = elem->nsurfaces; + npatches_repeat += element_npatches; + } + } + // std::cout << "Starting boundary patch allocation of size " << npatches_repeat << std::endl < Patch_Nodes(npatches_repeat, "Patch_Nodes"); + CArrayKokkos Patch_Boundary_Flags(npatches_repeat, "Patch_Boundary_Flags"); + if (myrank == 0) + { + std::cout << "Done with boundary patch allocation" << std::endl << std::flush; + } + // initialize boundary patch flags + for (int init = 0; init < npatches_repeat; init++) + { + Patch_Boundary_Flags(init) = 1; + } + + if (myrank == 0) + { + std::cout << "Done with boundary patch flags init" << std::endl << std::flush; + } + // use set of nodal combinations to find boundary set + // boundary patches will not try to add nodal combinations twice + // loop through elements in this rank to find boundary patches + npatches_repeat = 0; + + if (num_dim == 2) + { + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + element_select->choose_2Delem_type(Element_Types(ielem), elem2D); + element_npatches = elem2D->nsurfaces; + // loop through local surfaces + for (int isurface = 0; isurface < element_npatches; isurface++) + { + num_nodes_in_patch = elem2D->surface_to_dof_lid.stride(isurface); + Surface_Nodes = CArray(num_nodes_in_patch); + for (int inode = 0; inode < num_nodes_in_patch; inode++) + { + local_node_id = elem2D->surface_to_dof_lid(isurface, inode); + local_node_id = convert_node_order(local_node_id); + Surface_Nodes(inode) = nodes_in_elem(ielem, local_node_id); + } + Node_Combination temp(Surface_Nodes); + // construct Node Combination object for this surface + Patch_Nodes(npatches_repeat) = temp; + Patch_Nodes(npatches_repeat).patch_id = npatches_repeat; + Patch_Nodes(npatches_repeat).element_id = ielem; + Patch_Nodes(npatches_repeat).local_patch_id = isurface; + // test if this patch has already been added; if yes set boundary flags to 0 + current_combination = my_patches.insert(Patch_Nodes(npatches_repeat)); + // if the set determines this is a duplicate, access the original element's patch id and set flag to 0 + if (current_combination.second == false) + { + // set original element flag to 0 + Patch_Boundary_Flags((*current_combination.first).patch_id) = 0; + // set this current flag to 0 for the duplicate as well + Patch_Boundary_Flags(npatches_repeat) = 0; + } + npatches_repeat++; + } + } + } + + if (num_dim == 3) + { + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + element_select->choose_3Delem_type(Element_Types(ielem), elem); + element_npatches = elem->nsurfaces; + // loop through local surfaces + for (int isurface = 0; isurface < element_npatches; isurface++) + { + num_nodes_in_patch = elem->surface_to_dof_lid.stride(isurface); + // debug print + // std::cout << "NUMBER OF PATCH NODES FOR ELEMENT " << ielem+1 << " ON LOCAL SURFACE " << isurface+1 << " IS " << num_nodes_in_patch << std::endl; + Surface_Nodes = CArray(num_nodes_in_patch); + for (int inode = 0; inode < num_nodes_in_patch; inode++) + { + local_node_id = elem->surface_to_dof_lid(isurface, inode); + local_node_id = convert_node_order(local_node_id); + Surface_Nodes(inode) = nodes_in_elem(ielem, local_node_id); + } + Node_Combination temp(Surface_Nodes); + // construct Node Combination object for this surface + Patch_Nodes(npatches_repeat) = temp; + Patch_Nodes(npatches_repeat).patch_id = npatches_repeat; + Patch_Nodes(npatches_repeat).element_id = ielem; + Patch_Nodes(npatches_repeat).local_patch_id = isurface; + // test if this patch has already been added; if yes set boundary flags to 0 + current_combination = my_patches.insert(Patch_Nodes(npatches_repeat)); + // if the set determines this is a duplicate access the original element's patch id and set flag to 0 + + if (current_combination.second == false) + { + // set original element flag to 0 + Patch_Boundary_Flags((*current_combination.first).patch_id) = 0; + // set this current flag to 0 for the duplicate as well + Patch_Boundary_Flags(npatches_repeat) = 0; + } + npatches_repeat++; + } + } + } + // debug print of all patches + /* + std::cout << " ALL PATCHES " << npatches_repeat <isNodeGlobalElement(node_gid)) + { + // if(ghost_node_ranks.h_view(global2local_map.get(node_gid))(map, all_node_map)); - ghost_importer = Teuchos::rcp( new Tpetra::Import(map, ghost_node_map)); - dof_importer = Teuchos::rcp( new Tpetra::Import(local_dof_map, all_dof_map)); - - //output map and importers - sorted_map = Teuchos::rcp( new Tpetra::Map(num_nodes,0,comm)); - node_sorting_importer = Teuchos::rcp( new Tpetra::Import(map, sorted_map)); - //sorted element mapping - sorted_element_map = Teuchos::rcp( new Tpetra::Map(num_elem,0,comm)); - element_sorting_importer = Teuchos::rcp( new Tpetra::Import(all_element_map, sorted_element_map));; - +void Solver::comm_importer_setup() +{ + // create import object using local node indices map and ghost indices map + importer = Teuchos::rcp(new Tpetra::Import(map, all_node_map)); + ghost_importer = Teuchos::rcp(new Tpetra::Import(map, ghost_node_map)); + dof_importer = Teuchos::rcp(new Tpetra::Import(local_dof_map, all_dof_map)); + + // output map and importers + sorted_map = Teuchos::rcp(new Tpetra::Map(num_nodes, 0, comm)); + node_sorting_importer = Teuchos::rcp(new Tpetra::Import(map, sorted_map)); + // sorted element mapping + sorted_element_map = Teuchos::rcp(new Tpetra::Map(num_elem, 0, comm)); + element_sorting_importer = Teuchos::rcp(new Tpetra::Import(all_element_map, sorted_element_map));; } /* ---------------------------------------------------------------------- Communicate updated nodal coordinates to ghost nodes ------------------------------------------------------------------------- */ -void Solver::comm_coordinates(){ +void Solver::comm_coordinates() +{ + // create import object using local node indices map and ghost indices map + // Tpetra::Import importer(map, ghost_node_map); - //create import object using local node indices map and ghost indices map - //Tpetra::Import importer(map, ghost_node_map); - - //comms to get ghosts - ghost_node_coords_distributed->doImport(*node_coords_distributed, *ghost_importer, Tpetra::INSERT); - //all_node_map->describe(*fos,Teuchos::VERB_EXTREME); - //all_node_coords_distributed->describe(*fos,Teuchos::VERB_EXTREME); + // comms to get ghosts + ghost_node_coords_distributed->doImport(*node_coords_distributed, *ghost_importer, Tpetra::INSERT); + // all_node_map->describe(*fos,Teuchos::VERB_EXTREME); + // all_node_coords_distributed->describe(*fos,Teuchos::VERB_EXTREME); } /* ---------------------------------------------------------------------- @@ -2651,22 +3108,23 @@ void Solver::comm_coordinates(){ double Solver::CPU_Time() { - std::chrono::system_clock::time_point zero_time; - auto zero_time_duration = zero_time.time_since_epoch(); - auto time = std::chrono::system_clock::now(); - auto time_duration = time.time_since_epoch(); - //double calc_time = std::chrono::duration_cast(diff).count(); - double calc_time = std::chrono::duration_cast< std::chrono::nanoseconds>(time_duration-zero_time_duration).count(); - calc_time *= 1e-09; - - return calc_time; + std::chrono::system_clock::time_point zero_time; + auto zero_time_duration = zero_time.time_since_epoch(); + auto time = std::chrono::system_clock::now(); + auto time_duration = time.time_since_epoch(); + // double calc_time = std::chrono::duration_cast(diff).count(); + double calc_time = std::chrono::duration_cast(time_duration - zero_time_duration).count(); + calc_time *= 1e-09; + + return calc_time; } /* ---------------------------------------------------------------------- Clock variable initialization ------------------------------------------------------------------------- */ -void Solver::init_clock(){ - double current_cpu = 0; - initial_CPU_time = CPU_Time(); +void Solver::init_clock() +{ + double current_cpu = 0; + initial_CPU_time = CPU_Time(); } diff --git a/src/Parallel-Solvers/Solver.h b/src/Parallel-Solvers/Solver.h index 7e4088c6a..01b01807e 100644 --- a/src/Parallel-Solvers/Solver.h +++ b/src/Parallel-Solvers/Solver.h @@ -11,14 +11,14 @@ This program is open source under the BSD-3 License. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - + 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - + 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - + 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. @@ -34,9 +34,9 @@ OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. **********************************************************************************************/ - + #ifndef FIERRO_SOLVER_H -#define FIERRO_SOLVER_H +#define FIERRO_SOLVER_H #include "utilities.h" #include "matar.h" @@ -64,227 +64,227 @@ using namespace mtr; -//forward declarations -namespace swage{ - class mesh_t; -} +// forward declarations +namespace swage +{ +class mesh_t; +} // namespace swage -namespace ROL{ - template - class Problem; -} +namespace ROL +{ +template +class Problem; +} // namespace ROL +class Solver +{ +public: -class Solver{ + // Trilinos type definitions + typedef Tpetra::Map<>::local_ordinal_type LO; + typedef Tpetra::Map<>::global_ordinal_type GO; -public: + typedef Tpetra::CrsMatrix MAT; + typedef const Tpetra::CrsMatrix const_MAT; + typedef Tpetra::MultiVector MV; + typedef Tpetra::MultiVector MCONN; + + typedef Kokkos::ViewTraits::size_type SizeType; + typedef Tpetra::Details::DefaultTypes::node_type node_type; + using traits = Kokkos::ViewTraits; - //Trilinos type definitions - typedef Tpetra::Map<>::local_ordinal_type LO; - typedef Tpetra::Map<>::global_ordinal_type GO; + using array_layout = typename traits::array_layout; + using execution_space = typename traits::execution_space; + using device_type = typename traits::device_type; + using memory_traits = typename traits::memory_traits; + using global_size_t = Tpetra::global_size_t; - typedef Tpetra::CrsMatrix MAT; - typedef const Tpetra::CrsMatrix const_MAT; - typedef Tpetra::MultiVector MV; - typedef Tpetra::MultiVector MCONN; - - typedef Kokkos::ViewTraits::size_type SizeType; - typedef Tpetra::Details::DefaultTypes::node_type node_type; - using traits = Kokkos::ViewTraits; - - using array_layout = typename traits::array_layout; - using execution_space = typename traits::execution_space; - using device_type = typename traits::device_type; - using memory_traits = typename traits::memory_traits; - using global_size_t = Tpetra::global_size_t; - - typedef Kokkos::View values_array; - typedef Kokkos::View global_indices_array; - typedef Kokkos::View indices_array; - //typedef Kokkos::View row_pointers; - typedef MAT::local_graph_device_type::row_map_type::non_const_type row_pointers; - //typedef Kokkos::DualView::t_dev vec_array; - typedef MV::dual_view_type::t_dev vec_array; - typedef MV::dual_view_type::t_host host_vec_array; - typedef Kokkos::View const_host_vec_array; - typedef Kokkos::View const_vec_array; - typedef Kokkos::View const_host_ivec_array; - typedef Kokkos::View host_ivec_array; - typedef MV::dual_view_type dual_vec_array; - typedef MCONN::dual_view_type dual_elem_conn_array; - typedef MCONN::dual_view_type::t_host host_elem_conn_array; - typedef MCONN::dual_view_type::t_dev elem_conn_array; - typedef Kokkos::View const_host_elem_conn_array; - typedef Kokkos::View const_elem_conn_array; - - Solver(Simulation_Parameters& _simparam); - virtual ~Solver(); - - virtual void setup() {} - - virtual void run() = 0; - - virtual void solver_setup() {} - - virtual void solver_finalize() {} - - virtual void exit_solver(int status); - - virtual void generate_mesh(const std::shared_ptr& mesh_generation_options); - - virtual void read_mesh_ensight(const char *MESH); - - virtual void init_design() {} - - virtual void read_mesh_tecplot(const char *MESH); - - virtual void read_mesh_vtk(const char *MESH); - - virtual void repartition_nodes(); - - virtual void comm_importer_setup(); - - virtual void comm_coordinates(); - - virtual void tecplot_writer() {} - - virtual void parallel_tecplot_writer() {} - - virtual void parallel_vtk_writer() {} - - virtual void output_design(int current_step) {} - - //setup ghosts and element maps - virtual void init_maps(); - - //finds the boundary element surfaces in this model - virtual void Get_Boundary_Patches(); - - int setup_flag, finalize_flag; - - //MPI data - int myrank = 0; //index of this mpi rank in the world communicator - int nranks; //number of mpi ranks in the world communicator - MPI_Comm world; //stores the default communicator object (MPI_COMM_WORLD) - Teuchos::RCP> importer; //all node comms - Teuchos::RCP> ghost_importer; //ghost node comms - Teuchos::RCP> node_sorting_importer; //sorted node comms - Teuchos::RCP> element_sorting_importer; //sorted element comms - Teuchos::RCP> dof_importer; //ghost dof comms - - //class Simulation_Parameters *simparam; - Simulation_Parameters simparam; - - //set of enabled FEA modules - std::vector fea_module_types; - std::vector fea_modules; - std::vector fea_modules_modal_analysis; - std::set fea_module_must_read; - int nfea_modules; - int displacement_module; - - //Local FEA data - size_t nlocal_nodes; - size_t nlocal_elem_non_overlapping; //used for reduction calls of per element values - dual_vec_array dual_node_coords; //coordinates of the nodes - dual_vec_array dual_node_densities; //topology optimization design variable - dual_elem_conn_array dual_nodes_in_elem; //dual view of element connectivity to nodes - //host_elem_conn_array nodes_in_elem; //host view of element connectivity to nodes - CArrayKokkos Element_Types; - CArrayKokkos Nodes_Per_Element_Type; - CArrayKokkos corner_value_storage; - CArrayKokkos corner_vector_storage; - CArrayKokkos corner_gradient_storage; - size_t max_nodes_per_element, max_nodes_per_patch; - std::shared_ptr element_select; - std::shared_ptr ref_elem; - elements::Element2D *elem2D; - elements::Element3D *elem; - - //Ghost data on this MPI rank - size_t nghost_nodes; - Kokkos::DualView ghost_nodes; - Kokkos::DualView ghost_node_ranks; - - //Node set corresponding to uniquely assigned list of elements on this MPI rank - size_t nnonoverlap_elem_nodes; - Kokkos::DualView nonoverlap_elem_nodes; - - //Local FEA data including ghosts - size_t nall_nodes; - size_t rnum_elem; - - //Global FEA data - long long int num_nodes, num_elem; - Teuchos::RCP > comm; - Teuchos::RCP > map; //map of node indices - Teuchos::RCP > sorted_map; //sorted contiguous map of node indices - Teuchos::RCP > ghost_node_map; //map of node indices with ghosts on each rank - Teuchos::RCP > all_node_map; //map of node indices with ghosts on each rank - Teuchos::RCP > nonoverlap_element_node_map; //map of node indices with ghosts on each rank - Teuchos::RCP > element_map; //non overlapping map of elements owned by each rank used in reduction ops - Teuchos::RCP > all_element_map; //overlapping map of elements connected to the local nodes in each rank - Teuchos::RCP > sorted_element_map; //sorted contiguous map of element indices owned by each rank used in parallel IO - Teuchos::RCP > local_dof_map; //map of local dofs (typically num_node_local*num_dim) - Teuchos::RCP > all_dof_map; //map of local and ghost dofs (typically num_node_all*num_dim) - Teuchos::RCP global_nodes_in_elem_distributed; //element to node connectivity table - Teuchos::RCP node_nconn_distributed; //how many elements a node is connected to - Teuchos::RCP node_coords_distributed; - Teuchos::RCP ghost_node_coords_distributed; - Teuchos::RCP initial_node_coords_distributed; - Teuchos::RCP all_initial_node_coords_distributed; - Teuchos::RCP all_node_coords_distributed; - Teuchos::RCP design_node_densities_distributed; - Teuchos::RCP filtered_node_densities_distributed; - Teuchos::RCP test_node_densities_distributed; - Teuchos::RCP all_node_densities_distributed; - Teuchos::RCP all_filtered_node_densities_distributed; - Teuchos::RCP lower_bound_node_densities_distributed; - Teuchos::RCP upper_bound_node_densities_distributed; - Teuchos::RCP Global_Element_Densities_Upper_Bound; - Teuchos::RCP Global_Element_Densities_Lower_Bound; - Teuchos::RCP Global_Element_Densities; - - //Distributions of data used to print - Teuchos::RCP collected_node_coords_distributed; - Teuchos::RCP collected_node_densities_distributed; - Teuchos::RCP collected_nodes_in_elem_distributed; - Teuchos::RCP sorted_node_coords_distributed; - Teuchos::RCP sorted_node_densities_distributed; - Teuchos::RCP sorted_nodes_in_elem_distributed; - Teuchos::RCP sorted_element_densities_distributed; - - //Boundary Conditions Data - //CArray Patch_Nodes; - size_t nboundary_patches; - size_t num_boundary_conditions; - int current_bdy_id; - CArrayKokkos Boundary_Patches; - std::map boundary_patch_to_index; //maps patches to corresponding patch index (inverse of Boundary Patches array) - - //file readin variables - std::ifstream *in = NULL; - std::streampos before_condition_header; - std::string filename; - int words_per_line, elem_words_per_line; - enum node_ordering_convention {IJK, ENSIGHT}; - node_ordering_convention active_node_ordering_convention; - - //file output variables - int file_index, nsteps_print; //file sequence index and print frequency in # of optimization steps - - //output stream - Teuchos::RCP fos; - int last_print_step; - - //debug and system functions/variables - double CPU_Time(); - void init_clock(); - double initial_CPU_time, communication_time, dev2host_time, host2dev_time, output_time; - - //Pointer to ROL Problem for optimization solves - Teuchos::RCP> problem; + typedef Kokkos::View values_array; + typedef Kokkos::View global_indices_array; + typedef Kokkos::View indices_array; + // typedef Kokkos::View row_pointers; + typedef MAT::local_graph_device_type::row_map_type::non_const_type row_pointers; + // typedef Kokkos::DualView::t_dev vec_array; + typedef MV::dual_view_type::t_dev vec_array; + typedef MV::dual_view_type::t_host host_vec_array; + typedef Kokkos::View const_host_vec_array; + typedef Kokkos::View const_vec_array; + typedef Kokkos::View const_host_ivec_array; + typedef Kokkos::View host_ivec_array; + typedef MV::dual_view_type dual_vec_array; + typedef MCONN::dual_view_type dual_elem_conn_array; + typedef MCONN::dual_view_type::t_host host_elem_conn_array; + typedef MCONN::dual_view_type::t_dev elem_conn_array; + typedef Kokkos::View const_host_elem_conn_array; + typedef Kokkos::View const_elem_conn_array; + Solver(Simulation_Parameters& _simparam); + virtual ~Solver(); + + virtual void setup() {} + + virtual void run() = 0; + + virtual void solver_setup() {} + + virtual void solver_finalize() {} + + virtual void exit_solver(int status); + + virtual void generate_mesh(const std::shared_ptr& mesh_generation_options); + + virtual void read_mesh_ensight(const char* MESH); + + virtual void init_design() {} + + virtual void read_mesh_tecplot(const char* MESH); + + virtual void read_mesh_vtk(const char* MESH); + + virtual void repartition_nodes(); + + virtual void comm_importer_setup(); + + virtual void comm_coordinates(); + + virtual void tecplot_writer() {} + + virtual void parallel_tecplot_writer() {} + + virtual void parallel_vtk_writer() {} + + virtual void output_design(int current_step) {} + + // setup ghosts and element maps + virtual void init_maps(); + + // finds the boundary element surfaces in this model + virtual void Get_Boundary_Patches(); + + int setup_flag, finalize_flag; + + // MPI data + int myrank = 0; // index of this mpi rank in the world communicator + int nranks; // number of mpi ranks in the world communicator + MPI_Comm world; // stores the default communicator object (MPI_COMM_WORLD) + Teuchos::RCP> importer; // all node comms + Teuchos::RCP> ghost_importer; // ghost node comms + Teuchos::RCP> node_sorting_importer; // sorted node comms + Teuchos::RCP> element_sorting_importer; // sorted element comms + Teuchos::RCP> dof_importer; // ghost dof comms + + // class Simulation_Parameters *simparam; + Simulation_Parameters simparam; + + // set of enabled FEA modules + std::vector fea_module_types; + std::vector fea_modules; + std::vector fea_modules_modal_analysis; + std::set fea_module_must_read; + int nfea_modules; + int displacement_module; + + // Local FEA data + size_t nlocal_nodes; + size_t nlocal_elem_non_overlapping; // used for reduction calls of per element values + dual_vec_array dual_node_coords; // coordinates of the nodes + dual_vec_array dual_node_densities; // topology optimization design variable + dual_elem_conn_array dual_nodes_in_elem; // dual view of element connectivity to nodes + // host_elem_conn_array nodes_in_elem; //host view of element connectivity to nodes + CArrayKokkos Element_Types; + CArrayKokkos Nodes_Per_Element_Type; + CArrayKokkos corner_value_storage; + CArrayKokkos corner_vector_storage; + CArrayKokkos corner_gradient_storage; + size_t max_nodes_per_element, max_nodes_per_patch; + std::shared_ptr element_select; + std::shared_ptr ref_elem; + elements::Element2D* elem2D; + elements::Element3D* elem; + + // Ghost data on this MPI rank + size_t nghost_nodes; + Kokkos::DualView ghost_nodes; + Kokkos::DualView ghost_node_ranks; + + // Node set corresponding to uniquely assigned list of elements on this MPI rank + size_t nnonoverlap_elem_nodes; + Kokkos::DualView nonoverlap_elem_nodes; + + // Local FEA data including ghosts + size_t nall_nodes; + size_t rnum_elem; + + // Global FEA data + long long int num_nodes, num_elem; + Teuchos::RCP> comm; + Teuchos::RCP> map; // map of node indices + Teuchos::RCP> sorted_map; // sorted contiguous map of node indices + Teuchos::RCP> ghost_node_map; // map of node indices with ghosts on each rank + Teuchos::RCP> all_node_map; // map of node indices with ghosts on each rank + Teuchos::RCP> nonoverlap_element_node_map; // map of node indices with ghosts on each rank + Teuchos::RCP> element_map; // non overlapping map of elements owned by each rank used in reduction ops + Teuchos::RCP> all_element_map; // overlapping map of elements connected to the local nodes in each rank + Teuchos::RCP> sorted_element_map; // sorted contiguous map of element indices owned by each rank used in parallel IO + Teuchos::RCP> local_dof_map; // map of local dofs (typically num_node_local*num_dim) + Teuchos::RCP> all_dof_map; // map of local and ghost dofs (typically num_node_all*num_dim) + Teuchos::RCP global_nodes_in_elem_distributed; // element to node connectivity table + Teuchos::RCP node_nconn_distributed; // how many elements a node is connected to + Teuchos::RCP node_coords_distributed; + Teuchos::RCP ghost_node_coords_distributed; + Teuchos::RCP initial_node_coords_distributed; + Teuchos::RCP all_initial_node_coords_distributed; + Teuchos::RCP all_node_coords_distributed; + Teuchos::RCP design_node_densities_distributed; + Teuchos::RCP filtered_node_densities_distributed; + Teuchos::RCP test_node_densities_distributed; + Teuchos::RCP all_node_densities_distributed; + Teuchos::RCP all_filtered_node_densities_distributed; + Teuchos::RCP lower_bound_node_densities_distributed; + Teuchos::RCP upper_bound_node_densities_distributed; + Teuchos::RCP Global_Element_Densities_Upper_Bound; + Teuchos::RCP Global_Element_Densities_Lower_Bound; + Teuchos::RCP Global_Element_Densities; + + // Distributions of data used to print + Teuchos::RCP collected_node_coords_distributed; + Teuchos::RCP collected_node_densities_distributed; + Teuchos::RCP collected_nodes_in_elem_distributed; + Teuchos::RCP sorted_node_coords_distributed; + Teuchos::RCP sorted_node_densities_distributed; + Teuchos::RCP sorted_nodes_in_elem_distributed; + Teuchos::RCP sorted_element_densities_distributed; + + // Boundary Conditions Data + // CArray Patch_Nodes; + size_t nboundary_patches; + size_t num_boundary_conditions; + int current_bdy_id; + CArrayKokkos Boundary_Patches; + std::map boundary_patch_to_index; // maps patches to corresponding patch index (inverse of Boundary Patches array) + + // file readin variables + std::ifstream* in = NULL; + std::streampos before_condition_header; + std::string filename; + int words_per_line, elem_words_per_line; + enum node_ordering_convention { IJK, ENSIGHT }; + node_ordering_convention active_node_ordering_convention; + + // file output variables + int file_index, nsteps_print; // file sequence index and print frequency in # of optimization steps + + // output stream + Teuchos::RCP fos; + int last_print_step; + + // debug and system functions/variables + double CPU_Time(); + void init_clock(); + double initial_CPU_time, communication_time, dev2host_time, host2dev_time, output_time; + + // Pointer to ROL Problem for optimization solves + Teuchos::RCP> problem; }; -#endif // end Header Guard +#endif // end Header Guard \ No newline at end of file diff --git a/src/Parallel-Solvers/node_combination.cpp b/src/Parallel-Solvers/node_combination.cpp index a251ba180..89433fc4e 100644 --- a/src/Parallel-Solvers/node_combination.cpp +++ b/src/Parallel-Solvers/node_combination.cpp @@ -11,14 +11,14 @@ This program is open source under the BSD-3 License. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - + 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - + 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - + 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. @@ -34,43 +34,58 @@ OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. **********************************************************************************************/ - + #include "node_combination.h" #include "utilities.h" #include "matar.h" using namespace utils; -//overload < operator - bool operator< (const Node_Combination &object1, const Node_Combination &object2){ +// overload < operator +bool operator<(const Node_Combination& object1, const Node_Combination& object2) +{ int size1 = object1.node_set.size(); - //check if this node combination is identical - //first check size of the combination; if smaller evaluate to true - //the set using this is then ordered first according to size of the combinations - if(size1object2.node_set.size()) - return false; - + // check if this node combination is identical + // first check size of the combination; if smaller evaluate to true + // the set using this is then ordered first according to size of the combinations + if (size1 < object2.node_set.size()) + { + return true; + } + else if (size1 > object2.node_set.size()) + { + return false; + } + CArray sort_set1 = object1.sort_set; CArray sort_set2 = object2.sort_set; - for(int i = 0; i < size1; i++){ - sort_set1(i) = object1.node_set(i); - sort_set2(i) = object2.node_set(i); + for (int i = 0; i < size1; i++) + { + sort_set1(i) = object1.node_set(i); + sort_set2(i) = object2.node_set(i); } - //This part sorts for segments of the set where combinations have the same size - //define < using the sort of both combinations. If the first nonequal element of the lhs combination, w.r.t to - //the corresponding element of the rhs, is less than the respective element of the rhs < evaluates to true - std::sort(sort_set1.pointer(),sort_set1.pointer()+sort_set1.size()); - std::sort(sort_set2.pointer(),sort_set2.pointer()+sort_set2.size()); + // This part sorts for segments of the set where combinations have the same size + // define < using the sort of both combinations. If the first nonequal element of the lhs combination, w.r.t to + // the corresponding element of the rhs, is less than the respective element of the rhs < evaluates to true + std::sort(sort_set1.pointer(), sort_set1.pointer() + sort_set1.size()); + std::sort(sort_set2.pointer(), sort_set2.pointer() + sort_set2.size()); - //loop through the sorted nodes to check for < - for(int i = 0; i < size1; i++){ - if(sort_set1(i)::local_ordinal_type LO; - typedef Tpetra::Map<>::global_ordinal_type GO; - typedef Kokkos::ViewTraits::size_type SizeType; - using traits = Kokkos::ViewTraits; - - using array_layout = typename traits::array_layout; - using execution_space = typename traits::execution_space; - using device_type = typename traits::device_type; - using memory_traits = typename traits::memory_traits; - - CArray node_set, sort_set; - GO patch_id, element_id; - LO local_patch_id; - - //Default Constructor - Node_Combination(){} - - //Constructor with initialization - Node_Combination(CArray &nodes_init) { - node_set = nodes_init; - sort_set = CArray(node_set.size()); - } - - //Destructor - ~Node_Combination( ) {} - - //overload = operator - Node_Combination& operator= (const Node_Combination ¬_this){ - node_set = not_this.node_set; - sort_set = not_this.sort_set; - patch_id = not_this.patch_id; - element_id = not_this.element_id; - local_patch_id = not_this.local_patch_id; - return *this; - } - - //overload = operator - bool operator== (Node_Combination ¬_this){ - int this_size = this->node_set.size(); - //check if this node combination is identical - //first check size of the combination - if(this_size!=not_this.node_set.size()) - return false; - - CArray sort_set1 = this->sort_set; - CArray sort_set2 = not_this.sort_set; - for(int i = 0; i < this_size; i++){ - sort_set1(i) = this->node_set(i); - sort_set2(i) = not_this.node_set(i); + // Trilinos type definitions + typedef Tpetra::Map<>::local_ordinal_type LO; + typedef Tpetra::Map<>::global_ordinal_type GO; + typedef Kokkos::ViewTraits::size_type SizeType; + using traits = Kokkos::ViewTraits; + + using array_layout = typename traits::array_layout; + using execution_space = typename traits::execution_space; + using device_type = typename traits::device_type; + using memory_traits = typename traits::memory_traits; + + CArray node_set, sort_set; + GO patch_id, element_id; + LO local_patch_id; + + // Default Constructor + Node_Combination() {} + + // Constructor with initialization + Node_Combination(CArray& nodes_init) + { + node_set = nodes_init; + sort_set = CArray(node_set.size()); } - //check if the nodes in the set are the same; sort them to simplify - std::sort(sort_set1.pointer(),sort_set1.pointer()+sort_set1.size()); - std::sort(sort_set2.pointer(),sort_set2.pointer()+sort_set2.size());\ - - //loop through the sorted nodes to check for equivalence - for(int i = 0; i < this_size; i++) - if(sort_set1(i)!=sort_set2(i)) return false; - - return true; - - } + // Destructor + ~Node_Combination() {} + + // overload = operator + Node_Combination& operator=(const Node_Combination& not_this) + { + node_set = not_this.node_set; + sort_set = not_this.sort_set; + patch_id = not_this.patch_id; + element_id = not_this.element_id; + local_patch_id = not_this.local_patch_id; + return *this; + } + // overload = operator + bool operator==(Node_Combination& not_this) + { + int this_size = this->node_set.size(); + // check if this node combination is identical + // first check size of the combination + if (this_size != not_this.node_set.size()) + { + return false; + } + + CArray sort_set1 = this->sort_set; + CArray sort_set2 = not_this.sort_set; + for (int i = 0; i < this_size; i++) + { + sort_set1(i) = this->node_set(i); + sort_set2(i) = not_this.node_set(i); + } + + // check if the nodes in the set are the same; sort them to simplify + std::sort(sort_set1.pointer(), sort_set1.pointer() + sort_set1.size()); + std::sort(sort_set2.pointer(), sort_set2.pointer() + sort_set2.size()); \ + + // loop through the sorted nodes to check for equivalence + for (int i = 0; i < this_size; i++) + { + if (sort_set1(i) != sort_set2(i)) { return false; } } + + return true; + } }; #endif // end STATE_H From 764e3cc4284982c5a1fff68fd127f1058f5040ac Mon Sep 17 00:00:00 2001 From: Jacob Moore Date: Fri, 16 Feb 2024 15:31:49 -0600 Subject: [PATCH 18/64] STYLE: Tidying up and wrapping debug statements in #ifdefs --- scripts/build-fierro.sh | 2 +- .../FEA_Module_Dynamic_Elasticity.cpp | 209 +++++++------- .../FEA_Module_Dynamic_Elasticity.h | 3 +- .../elastic_optimization.cpp | 240 +++++++++------- .../Dynamic_Elastic_Solver/force_elastic.cpp | 136 ++++----- .../force_gradients_elastic.cpp | 26 +- .../Dynamic_Elastic_Solver/geometry.cpp | 9 +- .../Dynamic_Elastic_Solver/momentum.cpp | 4 + .../Eulerian_Solver/FEA_Module_Eulerian.cpp | 59 ++-- .../Parallel-Explicit/Explicit_Solver.cpp | 145 +++++----- .../SGH_Solver/include/mesh.h | 8 +- .../SGH_Solver/src/FEA_Module_SGH.cpp | 262 ++++++++++-------- .../SGH_Solver/src/force_gradients_sgh.cpp | 56 ++-- .../SGH_Solver/src/force_sgh.cpp | 9 +- .../SGH_Solver/src/geometry.cpp | 7 - .../SGH_Solver/src/momentum.cpp | 20 +- .../SGH_Solver/src/setup_sgh.cpp | 162 +++++------ .../SGH_Solver/src/sgh_optimization.cpp | 100 +++---- .../SGH_Solver/src/time_integration.cpp | 2 +- 19 files changed, 787 insertions(+), 672 deletions(-) diff --git a/scripts/build-fierro.sh b/scripts/build-fierro.sh index 17293bd68..1830beacc 100755 --- a/scripts/build-fierro.sh +++ b/scripts/build-fierro.sh @@ -28,7 +28,7 @@ show_help() { echo " explicit-evpfft builds the explicit solver with the EVPFFT material model" echo " explicit-ls-evpfft builds the explicit solver with the LS-EVPFFT material model" echo " explicit-evp builds the explicit solver with the EVP material model" - echo " implicit builds the explicit solver" + echo " implicit builds the implicit solver" echo " " echo " --kokkos_build_type The desired kokkos parallel backend to use. The default is 'serial'" echo " " diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.cpp b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.cpp index 652d7e797..fdabd0bcf 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.cpp @@ -135,23 +135,27 @@ FEA_Module_Dynamic_Elasticity::FEA_Module_Dynamic_Elasticity( // set parameters Dynamic_Options dynamic_options = simparam->dynamic_options; - time_value = dynamic_options.time_value; - time_final = dynamic_options.time_final; - dt_max = dynamic_options.dt_max; - dt_min = dynamic_options.dt_min; - dt_cfl = dynamic_options.dt_cfl; + + dt = dynamic_options.dt; + time_value = dynamic_options.time_value; + time_final = dynamic_options.time_final; + dt_max = dynamic_options.dt_max; + dt_min = dynamic_options.dt_min; + dt_cfl = dynamic_options.dt_cfl; + rk_num_bins = simparam->dynamic_options.rk_num_bins; + graphics_time = simparam->output_options.graphics_time; graphics_dt_ival = simparam->output_options.graphics_dt_ival; graphics_cyc_ival = simparam->output_options.graphics_cyc_ival; - cycle_stop = dynamic_options.cycle_stop; - rk_num_stages = dynamic_options.rk_num_stages; - dt = dynamic_options.dt; + graphics_times = simparam->output_options.graphics_times; + graphics_id = simparam->output_options.graphics_id; + + cycle_stop = dynamic_options.cycle_stop; + rk_num_stages = dynamic_options.rk_num_stages; + fuzz = dynamic_options.fuzz; tiny = dynamic_options.tiny; small = dynamic_options.small; - graphics_times = simparam->output_options.graphics_times; - graphics_id = simparam->output_options.graphics_id; - rk_num_bins = simparam->dynamic_options.rk_num_bins; if (simparam->topology_optimization_on) { @@ -182,25 +186,33 @@ FEA_Module_Dynamic_Elasticity::~FEA_Module_Dynamic_Elasticity() ------------------------------------------------------------------------- */ void FEA_Module_Dynamic_Elasticity::read_conditions_ansys_dat(std::ifstream* in, std::streampos before_condition_header) { - char ch; - int num_dim = simparam->num_dims; - int buffer_lines = 1000; - int max_word = 30; - auto input_options = simparam->input_options.value(); - int p_order = input_options.p_order; - real_t unit_scaling = input_options.unit_scaling; - int local_node_index, current_column_index; - size_t strain_count; - std::string skip_line, read_line, substring, token; - std::stringstream line_parse, line_parse2; + Input_Options input_options = simparam->input_options.value(); + + char ch; + std::string skip_line, read_line, substring, token; + std::stringstream line_parse, line_parse2; + + int num_dim = simparam->num_dims; + int buffer_lines = 1000; + int max_word = 30; + int local_node_index, current_column_index; + int p_order = input_options.p_order; + int buffer_loop, buffer_iteration, buffer_iterations, scan_loop, nodes_per_element, words_per_line; + + size_t strain_count; + size_t read_index_start; + size_t node_rid; + size_t elem_gid; + + real_t unit_scaling = input_options.unit_scaling; + CArrayKokkos read_buffer; CArrayKokkos read_buffer_indices; - int buffer_loop, buffer_iteration, buffer_iterations, scan_loop, nodes_per_element, words_per_line; - size_t read_index_start, node_rid, elem_gid; - LO local_dof_id; - GO node_gid; - real_t dof_value; - host_vec_array node_densities; + + LO local_dof_id; + GO node_gid; + real_t dof_value; + host_vec_array node_densities; } // end read_conditions_ansys_dat // ----------------------------------------------------------------------------- @@ -554,8 +566,10 @@ void FEA_Module_Dynamic_Elasticity::init_output() bool output_velocity_flag = simparam->output(FIELD::velocity); bool output_strain_flag = simparam->output(FIELD::strain); bool output_stress_flag = simparam->output(FIELD::stress); - int num_dim = simparam->num_dims; - int Brows; + + int num_dim = simparam->num_dims; + int Brows; + if (num_dim == 3) { Brows = 6; @@ -842,16 +856,14 @@ void FEA_Module_Dynamic_Elasticity::comm_variables(Teuchos::RCP zp) void FEA_Module_Dynamic_Elasticity::node_density_constraints(host_vec_array& node_densities_lower_bound) { - const size_t num_dim = mesh->num_dims; const_vec_array all_initial_node_coords = all_initial_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); - const size_t num_lcs = module_params->loading_conditions.size(); + + const size_t num_dim = mesh->num_dims; + const size_t num_lcs = module_params->loading_conditions.size(); const DCArrayKokkos mat_fill = simparam->mat_fill; const DCArrayKokkos loading = module_params->loading; - // debug check - // std::cout << "NUMBER OF LOADING CONDITIONS: " << num_lcs << std::endl; - // walk over the nodes to update the velocity FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { double current_node_coords[3]; @@ -872,7 +884,7 @@ void FEA_Module_Dynamic_Elasticity::node_density_constraints(host_vec_array& nod node_densities_lower_bound(node_gid, 0) = 1; } } - }); // end for parallel for over nodes + }); // end for parallel for over nodes } /* ---------------------------------------------------------------------------- @@ -882,12 +894,14 @@ void FEA_Module_Dynamic_Elasticity::node_density_constraints(host_vec_array& nod void FEA_Module_Dynamic_Elasticity::setup() { Dynamic_Options dynamic_options = simparam->dynamic_options; - const size_t rk_level = dynamic_options.rk_num_bins - 1; - const size_t num_fills = simparam->regions.size(); - const size_t rk_num_bins = dynamic_options.rk_num_bins; - const size_t num_bcs = module_params->boundary_conditions.size(); - const size_t num_materials = simparam->materials.size(); - const int num_dim = simparam->num_dims; + + const size_t rk_level = dynamic_options.rk_num_bins - 1; + const size_t num_fills = simparam->regions.size(); + const size_t rk_num_bins = dynamic_options.rk_num_bins; + const size_t num_bcs = module_params->boundary_conditions.size(); + const size_t num_materials = simparam->materials.size(); + + const int num_dim = simparam->num_dims; // --------------------------------------------------------------------- // obtain mesh data @@ -960,50 +974,28 @@ void FEA_Module_Dynamic_Elasticity::setup() // create Dual Views of the individual node struct variables node_coords = DViewCArrayKokkos(node_interface.coords.get_kokkos_dual_view().view_host().data(), rk_num_bins, num_nodes, num_dim); - - node_vel = DViewCArrayKokkos(node_interface.vel.get_kokkos_dual_view().view_host().data(), rk_num_bins, num_nodes, num_dim); - - node_mass = DViewCArrayKokkos(node_interface.mass.get_kokkos_dual_view().view_host().data(), num_nodes); + node_vel = DViewCArrayKokkos(node_interface.vel.get_kokkos_dual_view().view_host().data(), rk_num_bins, num_nodes, num_dim); + node_mass = DViewCArrayKokkos(node_interface.mass.get_kokkos_dual_view().view_host().data(), num_nodes); // create Dual Views of the individual elem struct variables - elem_den = DViewCArrayKokkos(&elem_interface.den(0), - num_elems); - - elem_pres = DViewCArrayKokkos(&elem_interface.pres(0), - num_elems); - + elem_den = DViewCArrayKokkos(&elem_interface.den(0), num_elems); + elem_pres = DViewCArrayKokkos(&elem_interface.pres(0), num_elems); elem_stress = DViewCArrayKokkos(&elem_interface.stress(0, 0, 0, 0), - rk_num_bins, - num_elems, - 3, - 3); // always 3D even in 2D-RZ - - elem_sspd = DViewCArrayKokkos(&elem_interface.sspd(0), - num_elems); - - elem_sie = DViewCArrayKokkos(&elem_interface.sie(0, 0), rk_num_bins, - num_elems); - - elem_vol = DViewCArrayKokkos(&elem_interface.vol(0), - num_elems); - - elem_div = DViewCArrayKokkos(&elem_interface.div(0), - num_elems); - - elem_mass = DViewCArrayKokkos(&elem_interface.mass(0), - num_elems); - - elem_mat_id = DViewCArrayKokkos(&elem_interface.mat_id(0), - num_elems); + num_elems, + 3, 3); // always 3D even in 2D-RZ + elem_sspd = DViewCArrayKokkos(&elem_interface.sspd(0), num_elems); + elem_sie = DViewCArrayKokkos(&elem_interface.sie(0, 0), + rk_num_bins, + num_elems); + elem_vol = DViewCArrayKokkos(&elem_interface.vol(0), num_elems); + elem_div = DViewCArrayKokkos(&elem_interface.div(0), num_elems); + elem_mass = DViewCArrayKokkos(&elem_interface.mass(0), num_elems); + elem_mat_id = DViewCArrayKokkos(&elem_interface.mat_id(0), num_elems); // create Dual Views of the corner struct variables - corner_force = DViewCArrayKokkos(&corner_interface.force(0, 0), - num_corners, - num_dim); - - corner_mass = DViewCArrayKokkos(&corner_interface.mass(0), - num_corners); + corner_force = DViewCArrayKokkos(&corner_interface.force(0, 0), num_corners, num_dim); + corner_mass = DViewCArrayKokkos(&corner_interface.mass(0), num_corners); // allocate elem_vel_grad elem_vel_grad = DCArrayKokkos(num_elems, 3, 3); @@ -1026,6 +1018,7 @@ void FEA_Module_Dynamic_Elasticity::setup() const DCArrayKokkos boundary = module_params->boundary; const DCArrayKokkos mat_fill = simparam->mat_fill; const DCArrayKokkos material = simparam->material; + global_vars = simparam->global_vars; state_vars = DCArrayKokkos(rnum_elem, simparam->max_num_state_vars); elem_user_output_vars = DCArrayKokkos(rnum_elem, simparam->output_options.max_num_user_output_vars); @@ -1093,7 +1086,7 @@ void FEA_Module_Dynamic_Elasticity::setup() { FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { elem_mat_id(elem_gid) = mat_fill(f_id).material_id; - }); + }); } elem_mat_id.update_host(); @@ -1411,7 +1404,7 @@ void FEA_Module_Dynamic_Elasticity::setup() FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { node_mass_interface(node_gid, 0) = node_mass(node_gid); - }); // end parallel for + }); // end parallel for } // end view scope Kokkos::fence(); // communicate ghost densities @@ -1424,7 +1417,7 @@ void FEA_Module_Dynamic_Elasticity::setup() FOR_ALL_CLASS(node_gid, nlocal_nodes, nall_nodes, { node_mass(node_gid) = ghost_node_mass_interface(node_gid - nlocal_nodes, 0); - }); // end parallel for + }); // end parallel for } // end view scope Kokkos::fence(); } // endif @@ -1549,7 +1542,7 @@ void FEA_Module_Dynamic_Elasticity::tag_bdys(const DCArrayKokkos& bo bdy_patches_in_set(bdy_set, index) = bdy_patch_gid; } // end if } // end for bdy_patch - // } + // } }); // end FOR_ALL_CLASS bdy_sets // debug check @@ -1737,6 +1730,8 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve() Dynamic_Options dynamic_options = simparam->dynamic_options; const size_t rk_level = dynamic_options.rk_num_bins - 1; + const int num_dim = simparam->num_dims; + time_value = dynamic_options.time_initial; time_final = dynamic_options.time_final; dt_max = dynamic_options.dt_max; @@ -1752,15 +1747,19 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve() small = dynamic_options.small; graphics_times = simparam->output_options.graphics_times; graphics_id = simparam->output_options.graphics_id; - size_t num_bdy_nodes = mesh->num_bdy_nodes; - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos material = simparam->material; - int nTO_modules; - int old_max_forward_buffer; - unsigned long cycle; - const int num_dim = simparam->num_dims; - real_t objective_accumulation, global_objective_accumulation; - std::vector> FEA_Module_My_TO_Modules = simparam->FEA_Module_My_TO_Modules; + size_t num_bdy_nodes = mesh->num_bdy_nodes; + + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos material = simparam->material; + + int nTO_modules; + int old_max_forward_buffer; + + unsigned long cycle; + + real_t objective_accumulation, global_objective_accumulation; + + std::vector> FEA_Module_My_TO_Modules = simparam->FEA_Module_My_TO_Modules; problem = Explicit_Solver_Pointer_->problem; // Pointer to ROL optimization problem object ROL::Ptr> obj_pointer; @@ -1915,7 +1914,7 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve() node_velocities_interface(node_gid, idim) = node_vel(rk_level, node_gid, idim); node_coords_interface(node_gid, idim) = node_coords(rk_level, node_gid, idim); } - }); + }); } // end view scope Kokkos::fence(); @@ -1950,7 +1949,7 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve() all_node_velocities_interface(node_gid, idim) = node_velocities_interface(node_gid, idim); all_node_coords_interface(node_gid, idim) = node_coords_interface(node_gid, idim); } - }); // end parallel for + }); // end parallel for Kokkos::fence(); FOR_ALL_CLASS(node_gid, nlocal_nodes, nlocal_nodes + nghost_nodes, { @@ -1959,7 +1958,7 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve() all_node_velocities_interface(node_gid, idim) = ghost_node_velocities_interface(node_gid - nlocal_nodes, idim); all_node_coords_interface(node_gid, idim) = ghost_node_coords_interface(node_gid - nlocal_nodes, idim); } - }); // end parallel for + }); // end parallel for Kokkos::fence(); } // end view scope @@ -2114,7 +2113,7 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve() { node_velocities_interface(node_gid, idim) = node_vel(rk_level, node_gid, idim); } - }); // end parallel for + }); // end parallel for } // end view scope Kokkos::fence(); @@ -2138,7 +2137,7 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve() { node_vel(rk_level, node_gid, idim) = ghost_node_velocities_interface(node_gid - nlocal_nodes, idim); } - }); // end parallel for + }); // end parallel for } // end view scope Kokkos::fence(); @@ -2225,7 +2224,7 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve() vec_array node_mass_interface = node_masses_distributed->getLocalView(Tpetra::Access::ReadWrite); FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { node_mass_interface(node_gid, 0) = node_mass(node_gid); - }); // end parallel for + }); // end parallel for } // end view scope Kokkos::fence(); // communicate ghost densities @@ -2238,7 +2237,7 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve() FOR_ALL_CLASS(node_gid, nlocal_nodes, nall_nodes, { node_mass(node_gid) = ghost_node_mass_interface(node_gid - nlocal_nodes, 0); - }); // end parallel for + }); // end parallel for } // end view scope Kokkos::fence(); @@ -2379,17 +2378,19 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve() { const_vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); const_vec_array ghost_node_velocities_interface = Explicit_Solver_Pointer_->ghost_node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); - vec_array all_node_velocities_interface = Explicit_Solver_Pointer_->all_node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); const_vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); const_vec_array ghost_node_coords_interface = Explicit_Solver_Pointer_->ghost_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + vec_array all_node_velocities_interface = Explicit_Solver_Pointer_->all_node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { for (int idim = 0; idim < num_dim; idim++) { all_node_velocities_interface(node_gid, idim) = node_velocities_interface(node_gid, idim); all_node_coords_interface(node_gid, idim) = node_coords_interface(node_gid, idim); } - }); // end parallel for + }); // end parallel for Kokkos::fence(); FOR_ALL_CLASS(node_gid, nlocal_nodes, nlocal_nodes + nghost_nodes, { @@ -2398,7 +2399,7 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve() all_node_velocities_interface(node_gid, idim) = ghost_node_velocities_interface(node_gid - nlocal_nodes, idim); all_node_coords_interface(node_gid, idim) = ghost_node_coords_interface(node_gid - nlocal_nodes, idim); } - }); // end parallel for + }); // end parallel for Kokkos::fence(); } // end view scope @@ -2434,7 +2435,7 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve() { KE_loc_sum += node_mass(node_gid) * ke; } - }, KE_sum); + }, KE_sum); Kokkos::fence(); KE_sum = 0.5 * KE_sum; objective_accumulation += KE_sum * dt; diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.h b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.h index a62ab5bc7..6c10c8504 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.h +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.h @@ -179,7 +179,8 @@ class FEA_Module_Dynamic_Elasticity : public FEA_Module const double x3, const double y3) const; - double average_element_density(const int nodes_per_elem, const CArray current_element_densities) const; + double average_element_density(const int nodes_per_elem, + const CArray current_element_densities) const; void get_divergence(DViewCArrayKokkos& elem_div, const mesh_t mesh, diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/elastic_optimization.cpp b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/elastic_optimization.cpp index 7f122066d..2805208a9 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/elastic_optimization.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/elastic_optimization.cpp @@ -264,27 +264,27 @@ void FEA_Module_Dynamic_Elasticity::update_forward_solve(Teuchos::RCP // --- Pressure --- eos_model->calc_pressure(elem_pres, - elem_stress, - elem_gid, - elem_mat_id(elem_gid), - state_vars, - global_vars, - elem_user_output_vars, - elem_sspd, - elem_den(elem_gid), - elem_sie(rk_level, elem_gid)); + elem_stress, + elem_gid, + elem_mat_id(elem_gid), + state_vars, + global_vars, + elem_user_output_vars, + elem_sspd, + elem_den(elem_gid), + elem_sie(rk_level, elem_gid)); // --- Sound speed --- eos_model->calc_sound_speed(elem_pres, - elem_stress, - elem_gid, - elem_mat_id(elem_gid), - state_vars, - global_vars, - elem_user_output_vars, - elem_sspd, - elem_den(elem_gid), - elem_sie(rk_level, elem_gid)); + elem_stress, + elem_gid, + elem_mat_id(elem_gid), + state_vars, + global_vars, + elem_user_output_vars, + elem_sspd, + elem_den(elem_gid), + elem_sie(rk_level, elem_gid)); // loop over the nodes of this element and apply velocity for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) @@ -412,7 +412,7 @@ void FEA_Module_Dynamic_Elasticity::update_forward_solve(Teuchos::RCP elem_pres(elem_gid) / (mat_fill(f_id).den * (gamma - 1.0)); } // end if } // end if fill - }); // end FOR_ALL_CLASS element loop + }); // end FOR_ALL_CLASS element loop Kokkos::fence(); } // end for loop over fills } // end view scope @@ -477,7 +477,7 @@ void FEA_Module_Dynamic_Elasticity::update_forward_solve(Teuchos::RCP vec_array node_mass_interface = node_masses_distributed->getLocalView(Tpetra::Access::ReadWrite); FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { node_mass_interface(node_gid, 0) = node_mass(node_gid); - }); // end parallel for + }); // end parallel for } // end view scope Kokkos::fence(); // communicate ghost densities @@ -490,7 +490,7 @@ void FEA_Module_Dynamic_Elasticity::update_forward_solve(Teuchos::RCP FOR_ALL_CLASS(node_gid, nlocal_nodes, nall_nodes, { node_mass(node_gid) = ghost_node_mass_interface(node_gid - nlocal_nodes, 0); - }); // end parallel for + }); // end parallel for } // end view scope Kokkos::fence(); @@ -526,13 +526,17 @@ double FEA_Module_Dynamic_Elasticity::average_element_density(const int nodes_pe void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_adjoint() { - size_t num_bdy_nodes = mesh->num_bdy_nodes; - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos material = simparam->material; - const int num_dim = simparam->num_dims; - real_t global_dt; - size_t current_data_index, next_data_index; - Teuchos::RCP previous_adjoint_vector_distributed, current_adjoint_vector_distributed, previous_velocity_vector_distributed, current_velocity_vector_distributed; + size_t num_bdy_nodes = mesh->num_bdy_nodes; + size_t current_data_index, next_data_index; + real_t global_dt; + + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos material = simparam->material; + const int num_dim = simparam->num_dims; + + Teuchos::RCP previous_adjoint_vector_distributed, current_adjoint_vector_distributed; + Teuchos::RCP previous_velocity_vector_distributed, current_velocity_vector_distributed; + // initialize first adjoint vector at last_time_step to 0 as the terminal value (*adjoint_vector_data)[last_time_step + 1]->putScalar(0); @@ -585,10 +589,12 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_adjoint() for (int idim = 0; idim < num_dim; idim++) { // cancellation of half from midpoint and 2 from adjoint equation already done - current_adjoint_vector(node_gid, idim) = -0.5 * (current_velocity_vector(node_gid, idim) + previous_velocity_vector(node_gid, idim)) * global_dt + previous_adjoint_vector(node_gid, - idim); + current_adjoint_vector(node_gid, idim) = + -0.5 * (current_velocity_vector(node_gid, idim) + + previous_velocity_vector(node_gid, idim)) * global_dt + + previous_adjoint_vector(node_gid, idim); } - }); // end parallel for + }); // end parallel for Kokkos::fence(); } // end view scope } @@ -600,16 +606,21 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_adjoint() void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_adjoint_full() { - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - size_t num_bdy_nodes = mesh->num_bdy_nodes; - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos material = simparam->material; - const int num_dim = simparam->num_dims; - const real_t damping_constant = module_params->damping_constant; - real_t global_dt; - size_t current_data_index, next_data_index; - Teuchos::RCP previous_adjoint_vector_distributed, current_adjoint_vector_distributed, previous_velocity_vector_distributed, current_velocity_vector_distributed; - Teuchos::RCP previous_phi_adjoint_vector_distributed, current_phi_adjoint_vector_distributed; + const int num_dim = simparam->num_dims; + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + const real_t damping_constant = module_params->damping_constant; + + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos material = simparam->material; + + real_t global_dt; + size_t num_bdy_nodes = mesh->num_bdy_nodes; + size_t current_data_index, next_data_index; + + Teuchos::RCP revious_adjoint_vector_distributed, current_adjoint_vector_distributed; + Teuchos::RCP previous_velocity_vector_distributed, current_velocity_vector_distributed; + Teuchos::RCP previous_phi_adjoint_vector_distributed, current_phi_adjoint_vector_distributed; + // initialize first adjoint vector at last_time_step to 0 as the terminal value (*adjoint_vector_data)[last_time_step + 1]->putScalar(0); (*phi_adjoint_vector_data)[last_time_step + 1]->putScalar(0); @@ -663,7 +674,7 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_adjoint_full() node_vel(rk_level, node_gid, idim) = previous_velocity_vector(node_gid, idim); node_coords(rk_level, node_gid, idim) = previous_coordinate_vector(node_gid, idim); } - }); + }); Kokkos::fence(); get_force_vgradient_elastic(material, @@ -689,9 +700,10 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_adjoint_full() // compute gradient of force with respect to velocity const_vec_array previous_adjoint_vector = (*adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); - const_vec_array phi_previous_adjoint_vector = (*phi_adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); - vec_array midpoint_adjoint_vector = adjoint_vector_distributed->getLocalView(Tpetra::Access::ReadWrite); - vec_array phi_midpoint_adjoint_vector = phi_adjoint_vector_distributed->getLocalView(Tpetra::Access::ReadWrite); + const_vec_array phi_previous_adjoint_vector = (*phi_adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + + vec_array midpoint_adjoint_vector = adjoint_vector_distributed->getLocalView(Tpetra::Access::ReadWrite); + vec_array phi_midpoint_adjoint_vector = phi_adjoint_vector_distributed->getLocalView(Tpetra::Access::ReadWrite); // half step update for RK2 scheme FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { @@ -713,22 +725,25 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_adjoint_full() matrix_contribution / node_mass(node_gid) - phi_previous_adjoint_vector(node_gid, idim) / node_mass(node_gid); midpoint_adjoint_vector(node_gid, idim) = -rate_of_change * global_dt / 2 + previous_adjoint_vector(node_gid, idim); + matrix_contribution = 0; // compute resulting row of force displacement gradient matrix transpose right multiplied by adjoint vector for (int idof = 0; idof < Gradient_Matrix_Strides(node_gid * num_dim + idim); idof++) { dof_id = DOF_Graph_Matrix(node_gid * num_dim + idim, idof); - matrix_contribution += -previous_adjoint_vector(dof_id / num_dim, dof_id % num_dim) * Force_Gradient_Positions(node_gid * num_dim + idim, idof); + matrix_contribution += -previous_adjoint_vector(dof_id / num_dim, dof_id % num_dim) + * Force_Gradient_Positions(node_gid * num_dim + idim, idof); } rate_of_change = -matrix_contribution; // rate_of_change = -0.0000001*previous_adjoint_vector(node_gid,idim); phi_midpoint_adjoint_vector(node_gid, idim) = -rate_of_change * global_dt / 2 + phi_previous_adjoint_vector(node_gid, idim); } - }); // end parallel for + }); // end parallel for Kokkos::fence(); boundary_adjoint(*mesh, boundary, midpoint_adjoint_vector, phi_midpoint_adjoint_vector); comm_adjoint_vectors(cycle); + // swap names to get ghost nodes for the midpoint vectors vec_array current_adjoint_vector = adjoint_vector_distributed->getLocalView(Tpetra::Access::ReadWrite); vec_array phi_current_adjoint_vector = phi_adjoint_vector_distributed->getLocalView(Tpetra::Access::ReadWrite); @@ -766,7 +781,7 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_adjoint_full() // rate_of_change = -0.0000001*midpoint_adjoint_vector(node_gid,idim); phi_current_adjoint_vector(node_gid, idim) = -rate_of_change * global_dt + phi_previous_adjoint_vector(node_gid, idim); } - }); // end parallel for + }); // end parallel for Kokkos::fence(); boundary_adjoint(*mesh, boundary, current_adjoint_vector, phi_current_adjoint_vector); @@ -783,15 +798,18 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_adjoint_full() void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient(const_vec_array design_variables, vec_array design_gradients) { - size_t num_bdy_nodes = mesh->num_bdy_nodes; - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos material = simparam->material; - const int num_dim = simparam->num_dims; - int num_corners = rnum_elem * num_nodes_in_elem; - real_t global_dt; - size_t current_data_index, next_data_index; - CArrayKokkos current_element_velocities = CArrayKokkos(num_nodes_in_elem, num_dim); - CArrayKokkos current_element_adjoint = CArrayKokkos(num_nodes_in_elem, num_dim); + size_t num_bdy_nodes = mesh->num_bdy_nodes; + size_t current_data_index, next_data_index; + + int num_corners = rnum_elem * num_nodes_in_elem; + real_t global_dt; + + const int num_dim = simparam->num_dims; + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos material = simparam->material; + + auto current_element_velocities = CArrayKokkos(num_nodes_in_elem, num_dim); + auto current_element_adjoint = CArrayKokkos(num_nodes_in_elem, num_dim); if (myrank == 0) { @@ -803,7 +821,7 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient(const // compute design gradients FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { design_gradients(node_id, 0) = 0; - }); // end parallel for + }); // end parallel for Kokkos::fence(); // gradient contribution from kinetic energy vMv product. @@ -874,7 +892,7 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient(const corner_id = elem_id * num_nodes_in_elem + inode; corner_value_storage(corner_id) = inner_product * global_dt; } - }); // end parallel for + }); // end parallel for Kokkos::fence(); // accumulate node values from corner storage @@ -886,7 +904,7 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient(const corner_id = corners_in_node(node_id, icorner); design_gradients(node_id, 0) += corner_value_storage(corner_id); } - }); // end parallel for + }); // end parallel for Kokkos::fence(); // test code @@ -928,7 +946,7 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient(const FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { design_gradients(node_id, 0) *= -0.5 / (double)num_nodes_in_elem / (double)num_nodes_in_elem; // design_gradients(node_id,0) =0.00001; - }); // end parallel for + }); // end parallel for Kokkos::fence(); // gradient contribution from Force vector. @@ -1007,7 +1025,7 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient(const corner_id = elem_id * num_nodes_in_elem + inode; corner_value_storage(corner_id) = -inner_product * global_dt / (double)num_nodes_in_elem; } - }); // end parallel for + }); // end parallel for Kokkos::fence(); // accumulate node values from corner storage @@ -1019,7 +1037,7 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient(const corner_id = corners_in_node(node_id, icorner); design_gradients(node_id, 0) += corner_value_storage(corner_id); } - }); // end parallel for + }); // end parallel for Kokkos::fence(); } // end view scope } @@ -1031,16 +1049,18 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient(const void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient_full(Teuchos::RCP design_densities_distributed, Teuchos::RCP design_gradients_distributed) { - size_t num_bdy_nodes = mesh->num_bdy_nodes; - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos material = simparam->material; - const int num_dim = simparam->num_dims; - int num_corners = rnum_elem * num_nodes_in_elem; - real_t global_dt; - bool element_constant_density = true; - size_t current_data_index, next_data_index; - CArrayKokkos current_element_velocities = CArrayKokkos(num_nodes_in_elem, num_dim); - CArrayKokkos current_element_adjoint = CArrayKokkos(num_nodes_in_elem, num_dim); + bool element_constant_density = true; + size_t num_bdy_nodes = mesh->num_bdy_nodes; + size_t current_data_index, next_data_index; + int num_corners = rnum_elem * num_nodes_in_elem; + real_t global_dt; + + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos material = simparam->material; + const int num_dim = simparam->num_dims; + + auto current_element_velocities = CArrayKokkos(num_nodes_in_elem, num_dim); + auto current_element_adjoint = CArrayKokkos(num_nodes_in_elem, num_dim); if (myrank == 0) { @@ -1055,7 +1075,7 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient_full( // compute design gradients FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { design_gradients(node_id, 0) = 0; - }); // end parallel for + }); // end parallel for Kokkos::fence(); // gradient contribution from kinetic energy v(dM/drho)v product. @@ -1135,7 +1155,7 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient_full( // division by design ratio recovers nominal element mass used in the gradient operator corner_value_storage(corner_id) = inner_product * global_dt / relative_element_densities(elem_id); } - }); // end parallel for + }); // end parallel for Kokkos::fence(); // accumulate node values from corner storage @@ -1147,7 +1167,7 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient_full( corner_id = corners_in_node(node_id, icorner); design_gradients(node_id, 0) += corner_value_storage(corner_id); } - }); // end parallel for + }); // end parallel for Kokkos::fence(); } // end view scope } @@ -1156,7 +1176,7 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient_full( FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { design_gradients(node_id, 0) *= 0.5 / (double)num_nodes_in_elem / (double)num_nodes_in_elem; // design_gradients(node_id,0) =0.00001; - }); // end parallel for + }); // end parallel for Kokkos::fence(); // gradient contribution from time derivative of adjoint \dot{lambda}(dM/drho)v product. @@ -1203,7 +1223,9 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient_full( const_vec_array next_phi_adjoint_vector = (*phi_adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); const real_t damping_constant = module_params->damping_constant; - FOR_ALL_CLASS(elem_id, 0, rnum_elem, { + + FOR_ALL_CLASS(elem_id, 0, rnum_elem, + { real_t lambda_dot_current; real_t lambda_dot_next; size_t node_id; @@ -1230,10 +1252,14 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient_full( for (int idim = 0; idim < num_dim; idim++) { // lambda_dot_current = lambda_dot_next = (next_adjoint_vector(node_id,idim)-current_adjoint_vector(node_id,idim))/global_dt; - lambda_dot_current = current_velocity_vector(node_id, idim) + damping_constant * current_adjoint_vector(node_id, - idim) / node_mass(node_id) - current_phi_adjoint_vector(node_id, idim) / node_mass(node_id); - lambda_dot_next = next_velocity_vector(node_id, idim) + damping_constant * next_adjoint_vector(node_id, idim) / node_mass(node_id) - next_phi_adjoint_vector(node_id, - idim) / node_mass(node_id); + lambda_dot_current = current_velocity_vector(node_id, idim) + + damping_constant * current_adjoint_vector(node_id, idim) / node_mass(node_id) + - current_phi_adjoint_vector(node_id, idim) / node_mass(node_id); + + lambda_dot_next = next_velocity_vector(node_id, idim) + + damping_constant * next_adjoint_vector(node_id, idim) / node_mass(node_id) + - next_phi_adjoint_vector(node_id, idim) / node_mass(node_id); + inner_product += elem_mass(elem_id) * (lambda_dot_current + lambda_dot_next) * current_element_velocities(ifill, idim) / 2; } } @@ -1245,7 +1271,7 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient_full( // division by design ratio recovers nominal element mass used in the gradient operator corner_value_storage(corner_id) = inner_product * global_dt / relative_element_densities(elem_id); } - }); // end parallel for + }); // end parallel for Kokkos::fence(); // accumulate node values from corner storage @@ -1257,7 +1283,7 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient_full( corner_id = corners_in_node(node_id, icorner); design_gradients(node_id, 0) += -corner_value_storage(corner_id) / (double)num_nodes_in_elem / (double)num_nodes_in_elem; } - }); // end parallel for + }); // end parallel for Kokkos::fence(); } // end view scope } @@ -1268,7 +1294,8 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient_full( const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[0]->getLocalView(Tpetra::Access::ReadOnly); const_vec_array current_adjoint_vector = (*adjoint_vector_data)[0]->getLocalView(Tpetra::Access::ReadOnly); - FOR_ALL_CLASS(elem_id, 0, rnum_elem, { + FOR_ALL_CLASS(elem_id, 0, rnum_elem, + { real_t lambda_dot; size_t node_id; size_t corner_id; @@ -1304,7 +1331,7 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient_full( // division by design ratio recovers nominal element mass used in the gradient operator corner_value_storage(corner_id) = inner_product / relative_element_densities(elem_id); } - }); // end parallel for + }); // end parallel for Kokkos::fence(); // accumulate node values from corner storage @@ -1316,7 +1343,7 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient_full( corner_id = corners_in_node(node_id, icorner); design_gradients(node_id, 0) += -corner_value_storage(corner_id) / (double)num_nodes_in_elem / (double)num_nodes_in_elem; } - }); // end parallel for + }); // end parallel for Kokkos::fence(); } // end view scope } // end view scope @@ -1335,14 +1362,18 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient_full( ------------------------------------------------------------------------- */ void FEA_Module_Dynamic_Elasticity::init_assembly() { - int num_dim = simparam->num_dims; + int num_dim = simparam->num_dims; + int local_node_index; + int current_column_index; + size_t max_stride = 0; + size_t nodes_per_element; + // const_host_elem_conn_array nodes_in_elem = global_nodes_in_elem_distributed->getLocalView (Tpetra::Access::ReadOnly); Gradient_Matrix_Strides = DCArrayKokkos(nlocal_nodes * num_dim, "Gradient_Matrix_Strides"); + CArrayKokkos Graph_Fill(nall_nodes, "nall_nodes"); CArrayKokkos current_row_nodes_scanned; - int local_node_index, current_column_index; - size_t max_stride = 0; - size_t nodes_per_element; + nodal_density_flag = simparam->nodal_density_flag; penalty_power = simparam->optimization_options.simp_penalty_power; @@ -1352,8 +1383,8 @@ void FEA_Module_Dynamic_Elasticity::init_assembly() Graph_Matrix_Strides = DCArrayKokkos(nlocal_nodes, "Graph_Matrix_Strides"); // allocate storage for the sparse stiffness matrix map used in the assembly process - Global_Stiffness_Matrix_Assembly_Map = Global_Gradient_Matrix_Assembly_Map = DCArrayKokkos(rnum_elem, - max_nodes_per_element, max_nodes_per_element, "Global_Gradient_Matrix_Assembly_Map"); + Global_Stiffness_Matrix_Assembly_Map = Global_Gradient_Matrix_Assembly_Map = DCArrayKokkos( + rnum_elem, max_nodes_per_element, max_nodes_per_element, "Global_Gradient_Matrix_Assembly_Map"); // allocate array used to determine global node repeats in the sparse graph later DCArrayKokkos node_indices_used(nall_nodes, "node_indices_used"); @@ -1367,7 +1398,7 @@ void FEA_Module_Dynamic_Elasticity::init_assembly() Graph_Matrix_Strides_initial(inode) = 0; Graph_Matrix_Strides(inode) = 0; Graph_Fill(inode) = 0; - }); // end parallel for + }); // end parallel for Kokkos::fence(); // initialize nall arrays @@ -1375,7 +1406,7 @@ void FEA_Module_Dynamic_Elasticity::init_assembly() FOR_ALL_CLASS(inode, 0, nall_nodes, { node_indices_used(inode) = 0; column_index(inode) = 0; - }); // end parallel for + }); // end parallel for Kokkos::fence(); // count upper bound of strides for Sparse Pattern Graph by allowing repeats due to connectivity @@ -1418,7 +1449,7 @@ void FEA_Module_Dynamic_Elasticity::init_assembly() // equate strides for later FOR_ALL_CLASS(inode, 0, nlocal_nodes, { Graph_Matrix_Strides(inode) = Graph_Matrix_Strides_initial(inode) = Dual_Graph_Matrix_Strides_initial(inode); - }); // end parallel for + }); // end parallel for // for (int inode = 0; inode < nlocal_nodes; inode++) // std::cout << Graph_Matrix_Strides_initial(inode) << std::endl; @@ -1430,7 +1461,7 @@ void FEA_Module_Dynamic_Elasticity::init_assembly() { update = Graph_Matrix_Strides_initial(inode); } - }, max_stride); + }, max_stride); // std::cout << "THE MAX STRIDE" << max_stride << std::endl; // allocate array used in the repeat removal process @@ -1515,6 +1546,7 @@ void FEA_Module_Dynamic_Elasticity::init_assembly() int element_column_index; int current_stride; int current_row_n_nodes_scanned; + for (int inode = 0; inode < nlocal_nodes; inode++) { current_row_n_nodes_scanned = 0; @@ -1575,7 +1607,7 @@ void FEA_Module_Dynamic_Elasticity::init_assembly() node_indices_used(current_row_nodes_scanned(node_reset)) = 0; } } - }); + }); Kokkos::fence(); Graph_Matrix_Strides.update_host(); @@ -1587,7 +1619,7 @@ void FEA_Module_Dynamic_Elasticity::init_assembly() { Graph_Matrix(inode, istride) = Repeat_Graph_Matrix(inode, istride); } - }); // end parallel for + }); // end parallel for // deallocate repeat matrix @@ -1598,12 +1630,14 @@ void FEA_Module_Dynamic_Elasticity::init_assembly() // expand strides for stiffness matrix by multipling by dim FOR_ALL_CLASS(idof, 0, num_dim * nlocal_nodes, { Gradient_Matrix_Strides(idof) = num_dim * Graph_Matrix_Strides(idof / num_dim); - }); // end parallel for + }); // end parallel for - Stiffness_Matrix = Force_Gradient_Positions = RaggedRightArrayKokkos(Gradient_Matrix_Strides); - Force_Gradient_Velocities = RaggedRightArrayKokkos(Gradient_Matrix_Strides); + Stiffness_Matrix = RaggedRightArrayKokkos(Gradient_Matrix_Strides); DOF_Graph_Matrix = RaggedRightArrayKokkos(Gradient_Matrix_Strides); + Force_Gradient_Positions = RaggedRightArrayKokkos(Gradient_Matrix_Strides); + Force_Gradient_Velocities = RaggedRightArrayKokkos(Gradient_Matrix_Strides); + // set stiffness Matrix Graph // debug print // std::cout << "DOF GRAPH MATRIX ENTRIES ON TASK " << myrank << std::endl; @@ -1612,7 +1646,7 @@ void FEA_Module_Dynamic_Elasticity::init_assembly() { DOF_Graph_Matrix(idof, istride) = Graph_Matrix(idof / num_dim, istride / num_dim) * num_dim + istride % num_dim; } - }); // end parallel for + }); // end parallel for Stiffness_Matrix_Strides = Gradient_Matrix_Strides; diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/force_elastic.cpp b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/force_elastic.cpp index 4ef144028..edf219c35 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/force_elastic.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/force_elastic.cpp @@ -93,14 +93,15 @@ void FEA_Module_Dynamic_Elasticity::applied_forces(const DCArrayKokkosdynamic_options.rk_num_bins - 1; - const size_t num_dim = mesh.num_dims; - const_vec_array all_initial_node_coords = all_initial_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); - const size_t num_lcs = module_params->loading.size(); + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + const size_t num_dim = mesh.num_dims; + const size_t num_lcs = module_params->loading.size(); const DCArrayKokkos mat_fill = simparam->mat_fill; const DCArrayKokkos loading = module_params->loading; + const_vec_array all_initial_node_coords = all_initial_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + // debug check // std::cout << "NUMBER OF LOADING CONDITIONS: " << num_lcs << std::endl; @@ -118,7 +119,10 @@ void FEA_Module_Dynamic_Elasticity::applied_forces(const DCArrayKokkos legendre_nodes_1D(num_gauss_points); // CArrayKokkos legendre_weights_1D(num_gauss_points); - CArray legendre_nodes_1D(num_gauss_points); - CArray legendre_weights_1D(num_gauss_points); - real_t pointer_quad_coordinate[num_dim]; - real_t pointer_quad_coordinate_weight[num_dim]; - real_t pointer_interpolated_point[num_dim]; - real_t pointer_JT_row1[num_dim]; - real_t pointer_JT_row2[num_dim]; - real_t pointer_JT_row3[num_dim]; + CArray legendre_nodes_1D(num_gauss_points); + CArray legendre_weights_1D(num_gauss_points); + + real_t pointer_quad_coordinate[num_dim]; + real_t pointer_quad_coordinate_weight[num_dim]; + real_t pointer_interpolated_point[num_dim]; + real_t pointer_JT_row1[num_dim]; + real_t pointer_JT_row2[num_dim]; + real_t pointer_JT_row3[num_dim]; + real_t pointer_basis_values[elem->num_basis()]; + real_t pointer_basis_derivative_s1[elem->num_basis()]; + real_t pointer_basis_derivative_s2[elem->num_basis()]; + real_t pointer_basis_derivative_s3[elem->num_basis()]; + ViewCArray quad_coordinate(pointer_quad_coordinate, num_dim); ViewCArray quad_coordinate_weight(pointer_quad_coordinate_weight, num_dim); ViewCArray interpolated_point(pointer_interpolated_point, num_dim); ViewCArray JT_row1(pointer_JT_row1, num_dim); ViewCArray JT_row2(pointer_JT_row2, num_dim); ViewCArray JT_row3(pointer_JT_row3, num_dim); + ViewCArray basis_values(pointer_basis_values, elem->num_basis()); + ViewCArray basis_derivative_s1(pointer_basis_derivative_s1, elem->num_basis()); + ViewCArray basis_derivative_s2(pointer_basis_derivative_s2, elem->num_basis()); + ViewCArray basis_derivative_s3(pointer_basis_derivative_s3, elem->num_basis()); - real_t pointer_basis_values[elem->num_basis()]; - real_t pointer_basis_derivative_s1[elem->num_basis()]; - real_t pointer_basis_derivative_s2[elem->num_basis()]; - real_t pointer_basis_derivative_s3[elem->num_basis()]; - ViewCArray basis_values(pointer_basis_values, elem->num_basis()); - ViewCArray basis_derivative_s1(pointer_basis_derivative_s1, elem->num_basis()); - ViewCArray basis_derivative_s2(pointer_basis_derivative_s2, elem->num_basis()); - ViewCArray basis_derivative_s3(pointer_basis_derivative_s3, elem->num_basis()); CArrayKokkos nodal_positions(elem->num_basis(), num_dim); CArrayKokkos nodal_density(elem->num_basis()); @@ -857,16 +855,18 @@ void FEA_Module_Dynamic_Elasticity::compute_stiffness_gradients(const_host_vec_a { Element_Densities = Global_Element_Densities->getLocalView(Tpetra::Access::ReadOnly); } - int num_dim = simparam->num_dims; - int nodes_per_elem = elem->num_basis(); - int num_gauss_points = simparam->num_gauss_points; - int z_quad, y_quad, x_quad, direct_product_count; + int num_dim = simparam->num_dims; + int nodes_per_elem = elem->num_basis(); + int num_gauss_points = simparam->num_gauss_points; + int z_quad, y_quad, x_quad, direct_product_count; + size_t local_node_id, local_dof_idx, local_dof_idy, local_dof_idz; const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; GO current_global_index; - real_t global_dt; - size_t current_data_index, next_data_index; + real_t global_dt; + size_t current_data_index, next_data_index; + CArrayKokkos current_element_adjoint = CArrayKokkos(num_nodes_in_elem * num_dim); direct_product_count = std::pow(num_gauss_points, num_dim); @@ -875,29 +875,31 @@ void FEA_Module_Dynamic_Elasticity::compute_stiffness_gradients(const_host_vec_a real_t inner_product, matrix_term, Jacobian, invJacobian, weight_multiply; // CArrayKokkos legendre_nodes_1D(num_gauss_points); // CArrayKokkos legendre_weights_1D(num_gauss_points); - CArray legendre_nodes_1D(num_gauss_points); - CArray legendre_weights_1D(num_gauss_points); - real_t pointer_quad_coordinate[num_dim]; - real_t pointer_quad_coordinate_weight[num_dim]; - real_t pointer_interpolated_point[num_dim]; - real_t pointer_JT_row1[num_dim]; - real_t pointer_JT_row2[num_dim]; - real_t pointer_JT_row3[num_dim]; + CArray legendre_nodes_1D(num_gauss_points); + CArray legendre_weights_1D(num_gauss_points); + + real_t pointer_quad_coordinate[num_dim]; + real_t pointer_quad_coordinate_weight[num_dim]; + real_t pointer_interpolated_point[num_dim]; + real_t pointer_JT_row1[num_dim]; + real_t pointer_JT_row2[num_dim]; + real_t pointer_JT_row3[num_dim]; + real_t pointer_basis_values[elem->num_basis()]; + real_t pointer_basis_derivative_s1[elem->num_basis()]; + real_t pointer_basis_derivative_s2[elem->num_basis()]; + real_t pointer_basis_derivative_s3[elem->num_basis()]; + ViewCArray quad_coordinate(pointer_quad_coordinate, num_dim); ViewCArray quad_coordinate_weight(pointer_quad_coordinate_weight, num_dim); ViewCArray interpolated_point(pointer_interpolated_point, num_dim); ViewCArray JT_row1(pointer_JT_row1, num_dim); ViewCArray JT_row2(pointer_JT_row2, num_dim); ViewCArray JT_row3(pointer_JT_row3, num_dim); + ViewCArray basis_values(pointer_basis_values, elem->num_basis()); + ViewCArray basis_derivative_s1(pointer_basis_derivative_s1, elem->num_basis()); + ViewCArray basis_derivative_s2(pointer_basis_derivative_s2, elem->num_basis()); + ViewCArray basis_derivative_s3(pointer_basis_derivative_s3, elem->num_basis()); - real_t pointer_basis_values[elem->num_basis()]; - real_t pointer_basis_derivative_s1[elem->num_basis()]; - real_t pointer_basis_derivative_s2[elem->num_basis()]; - real_t pointer_basis_derivative_s3[elem->num_basis()]; - ViewCArray basis_values(pointer_basis_values, elem->num_basis()); - ViewCArray basis_derivative_s1(pointer_basis_derivative_s1, elem->num_basis()); - ViewCArray basis_derivative_s2(pointer_basis_derivative_s2, elem->num_basis()); - ViewCArray basis_derivative_s3(pointer_basis_derivative_s3, elem->num_basis()); CArrayKokkos nodal_positions(elem->num_basis(), num_dim); CArrayKokkos current_nodal_displacements(elem->num_basis() * num_dim); CArrayKokkos nodal_density(elem->num_basis()); @@ -923,7 +925,8 @@ void FEA_Module_Dynamic_Elasticity::compute_stiffness_gradients(const_host_vec_a elements::legendre_weights_1D(legendre_weights_1D, num_gauss_points); Solver::node_ordering_convention active_node_ordering_convention = Explicit_Solver_Pointer_->active_node_ordering_convention; - real_t current_density = 1; + real_t current_density = 1; + CArrayKokkos convert_node_order(max_nodes_per_element); if ((active_node_ordering_convention == Solver::ENSIGHT && num_dim == 3) || (active_node_ordering_convention == Solver::IJK && num_dim == 2)) { @@ -1030,10 +1033,15 @@ void FEA_Module_Dynamic_Elasticity::compute_stiffness_gradients(const_host_vec_a nodal_positions(node_loop, 0) = all_initial_node_coords(local_node_id, 0); nodal_positions(node_loop, 1) = all_initial_node_coords(local_node_id, 1); - current_nodal_displacements(node_loop * num_dim) = 0.5 * (next_coordinate_vector(local_node_id, 0) - all_initial_node_coords(local_node_id, - 0) + current_coordinate_vector(local_node_id, 0) - all_initial_node_coords(local_node_id, 0)); - current_nodal_displacements(node_loop * num_dim + 1) = 0.5 * (next_coordinate_vector(local_node_id, 1) - all_initial_node_coords(local_node_id, - 1) + current_coordinate_vector(local_node_id, 1) - all_initial_node_coords(local_node_id, 1)); + + current_nodal_displacements(node_loop * num_dim) = 0.5 + * (next_coordinate_vector(local_node_id, 0) - all_initial_node_coords(local_node_id, 0) + + current_coordinate_vector(local_node_id, 0) - all_initial_node_coords(local_node_id, 0)); + + current_nodal_displacements(node_loop * num_dim + 1) = 0.5 + * (next_coordinate_vector(local_node_id, 1) - all_initial_node_coords(local_node_id, 1) + + current_coordinate_vector(local_node_id, 1) - all_initial_node_coords(local_node_id, 1)); + current_element_adjoint(node_loop * num_dim) = 0.5 * (current_adjoint_vector(local_node_id, 0) + next_adjoint_vector(local_node_id, 0)); current_element_adjoint(node_loop * num_dim + 1) = 0.5 * (current_adjoint_vector(local_node_id, 1) + next_adjoint_vector(local_node_id, 1)); @@ -1042,7 +1050,7 @@ void FEA_Module_Dynamic_Elasticity::compute_stiffness_gradients(const_host_vec_a nodal_positions(node_loop, 2) = all_initial_node_coords(local_node_id, 2); current_nodal_displacements(node_loop * num_dim + 2) = 0.5 * (next_coordinate_vector(local_node_id, 2) - all_initial_node_coords(local_node_id, 2) + current_coordinate_vector(local_node_id, 2) - all_initial_node_coords(local_node_id, 2)); - current_element_adjoint(node_loop * num_dim + 2) = 0.5 * (current_adjoint_vector(local_node_id, 2) + next_adjoint_vector(local_node_id, 2)); + current_element_adjoint(node_loop * num_dim + 2) = 0.5 * (current_adjoint_vector(local_node_id, 2) + next_adjoint_vector(local_node_id, 2)); } if (nodal_density_flag) diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/force_gradients_elastic.cpp b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/force_gradients_elastic.cpp index 5e8999635..b88a3895e 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/force_gradients_elastic.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/force_gradients_elastic.cpp @@ -768,15 +768,17 @@ void FEA_Module_Dynamic_Elasticity::get_force_ugradient_elastic(const DCArrayKok void FEA_Module_Dynamic_Elasticity::force_design_gradient_term(const_vec_array design_variables, vec_array design_gradients) { - size_t num_bdy_nodes = mesh->num_bdy_nodes; - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos material = simparam->material; - const int num_dim = simparam->num_dims; - int num_corners = rnum_elem * num_nodes_in_elem; - real_t global_dt; - bool element_constant_density = true; - size_t current_data_index, next_data_index; - CArrayKokkos current_element_adjoint = CArrayKokkos(num_nodes_in_elem, num_dim); + bool element_constant_density = true; + int num_corners = rnum_elem * num_nodes_in_elem; + size_t num_bdy_nodes = mesh->num_bdy_nodes; + size_t current_data_index, next_data_index; + real_t global_dt; + + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos material = simparam->material; + const int num_dim = simparam->num_dims; + + auto current_element_adjoint = CArrayKokkos(num_nodes_in_elem, num_dim); // gradient contribution from gradient of Force vector with respect to design variable. for (unsigned long cycle = 0; cycle < last_time_step + 1; cycle++) @@ -819,11 +821,11 @@ void FEA_Module_Dynamic_Elasticity::force_design_gradient_term(const_vec_array d for (int inode = 0; inode < num_nodes_in_elem; inode++) { node_id = nodes_in_elem(elem_id, inode); - current_element_adjoint(inode, 0) = (current_adjoint_vector(node_id, 0) + next_adjoint_vector(node_id, 0)) / 2; - current_element_adjoint(inode, 1) = (current_adjoint_vector(node_id, 1) + next_adjoint_vector(node_id, 1)) / 2; + current_element_adjoint(inode, 0) = 0.5 * (current_adjoint_vector(node_id, 0) + next_adjoint_vector(node_id, 0)); + current_element_adjoint(inode, 1) = 0.5 * (current_adjoint_vector(node_id, 1) + next_adjoint_vector(node_id, 1)); if (num_dim == 3) { - current_element_adjoint(inode, 2) = (current_adjoint_vector(node_id, 2) + next_adjoint_vector(node_id, 2)) / 2; + current_element_adjoint(inode, 2) = 0.5 * (current_adjoint_vector(node_id, 2) + next_adjoint_vector(node_id, 2)); } } diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/geometry.cpp b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/geometry.cpp index 7e4034a8a..0f19f6580 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/geometry.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/geometry.cpp @@ -335,19 +335,12 @@ void FEA_Module_Dynamic_Elasticity::get_bmatrix2D(const ViewCArrayKokkos */ B_matrix(0, 0) = -0.5 * (y(3) - y(1)); - B_matrix(1, 0) = -0.5 * (y(0) - y(2)); - B_matrix(2, 0) = -0.5 * (y(1) - y(3)); - B_matrix(3, 0) = -0.5 * (y(2) - y(0)); - B_matrix(0, 1) = -0.5 * (x(1) - x(3)); - B_matrix(1, 1) = -0.5 * (x(2) - x(0)); - B_matrix(2, 1) = -0.5 * (x(3) - x(1)); - B_matrix(3, 1) = -0.5 * (x(0) - x(2)); // @@ -425,7 +418,7 @@ void FEA_Module_Dynamic_Elasticity::get_vol_quad(const DViewCArrayKokkos */ elem_vol(elem_gid) = ( (y(2) + y(3) + y(0)) * ((y(2) - y(3)) * (x(0) - x(3)) - (y(0) - y(3)) * (x(2) - x(3)) ) - + (y(0) + y(1) + y(2)) * ((y(0) - y(1)) * (x(2) - x(1)) - (y(2) - y(1)) * (x(0) - x(1))) ) / 6.0; + + (y(0) + y(1) + y(2)) * ((y(0) - y(1)) * (x(2) - x(1)) - (y(2) - y(1)) * (x(0) - x(1))) ) / 6.0; return; } // end subroutine diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/momentum.cpp b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/momentum.cpp index 98e83b455..be4ca4b69 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/momentum.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/momentum.cpp @@ -69,6 +69,7 @@ void FEA_Module_Dynamic_Elasticity::get_velgrad(ViewCArrayKokkos& double u_array[num_nodes_in_elem]; double v_array[num_nodes_in_elem]; double w_array[num_nodes_in_elem]; + ViewCArrayKokkos u(u_array, num_nodes_in_elem); // x-dir vel component ViewCArrayKokkos v(v_array, num_nodes_in_elem); // y-dir vel component ViewCArrayKokkos w(w_array, num_nodes_in_elem); // z-dir vel component @@ -155,6 +156,7 @@ void FEA_Module_Dynamic_Elasticity::get_velgrad2D(ViewCArrayKokkos& double u_array[num_nodes_in_elem]; double v_array[num_nodes_in_elem]; + ViewCArrayKokkos u(u_array, num_nodes_in_elem); // x-dir vel component ViewCArrayKokkos v(v_array, num_nodes_in_elem); // y-dir vel component @@ -222,6 +224,7 @@ void FEA_Module_Dynamic_Elasticity::get_divergence(DViewCArrayKokkos& double u_array[num_nodes_in_elem]; double v_array[num_nodes_in_elem]; double w_array[num_nodes_in_elem]; + ViewCArrayKokkos u(u_array, num_nodes_in_elem); // x-dir vel component ViewCArrayKokkos v(v_array, num_nodes_in_elem); // y-dir vel component ViewCArrayKokkos w(w_array, num_nodes_in_elem); // z-dir vel component @@ -295,6 +298,7 @@ void FEA_Module_Dynamic_Elasticity::get_divergence2D(DViewCArrayKokkos& double u_array[num_nodes_in_elem]; double v_array[num_nodes_in_elem]; + ViewCArrayKokkos u(u_array, num_nodes_in_elem); // x-dir vel component ViewCArrayKokkos v(v_array, num_nodes_in_elem); // y-dir vel component diff --git a/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/FEA_Module_Eulerian.cpp b/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/FEA_Module_Eulerian.cpp index d6319aa08..ee4b9d012 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/FEA_Module_Eulerian.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/FEA_Module_Eulerian.cpp @@ -207,24 +207,29 @@ FEA_Module_Eulerian::~FEA_Module_Eulerian() ------------------------------------------------------------------------- */ void FEA_Module_Eulerian::read_conditions_ansys_dat(std::ifstream* in, std::streampos before_condition_header) { - char ch; - int num_dim = simparam->num_dims; - int buffer_lines = 1000; - int max_word = 30; - int p_order = simparam->p_order; - real_t unit_scaling = simparam->get_unit_scaling(); - int local_node_index, current_column_index; - size_t strain_count; - std::string skip_line, read_line, substring, token; - std::stringstream line_parse, line_parse2; + char ch; + std::string skip_line, read_line, substring, token; + std::stringstream line_parse, line_parse2; + + int num_dim = simparam->num_dims; + int buffer_lines = 1000; + int max_word = 30; + int p_order = simparam->p_order; + int local_node_index, current_column_index; + int buffer_loop, buffer_iteration, buffer_iterations, scan_loop, nodes_per_element, words_per_line; + + real_t unit_scaling = simparam->get_unit_scaling(); + + size_t strain_count; + size_t read_index_start, node_rid, elem_gid; + CArrayKokkos read_buffer; CArrayKokkos read_buffer_indices; - int buffer_loop, buffer_iteration, buffer_iterations, scan_loop, nodes_per_element, words_per_line; - size_t read_index_start, node_rid, elem_gid; - LO local_dof_id; - GO node_gid; - real_t dof_value; - host_vec_array node_densities; + + LO local_dof_id; + GO node_gid; + real_t dof_value; + host_vec_array node_densities; } // end read_conditions_ansys_dat /* ---------------------------------------------------------------------------- @@ -902,17 +907,19 @@ void FEA_Module_Eulerian::euler_solve() { const_vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); const_vec_array ghost_node_velocities_interface = Explicit_Solver_Pointer_->ghost_node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); - vec_array all_node_velocities_interface = Explicit_Solver_Pointer_->all_node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); const_vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); const_vec_array ghost_node_coords_interface = Explicit_Solver_Pointer_->ghost_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); - vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + + vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + vec_array all_node_velocities_interface = Explicit_Solver_Pointer_->all_node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { for (int idim = 0; idim < num_dim; idim++) { all_node_velocities_interface(node_gid, idim) = node_velocities_interface(node_gid, idim); all_node_coords_interface(node_gid, idim) = node_coords_interface(node_gid, idim); } - }); // end parallel for + }); // end parallel for Kokkos::fence(); FOR_ALL_CLASS(node_gid, nlocal_nodes, nlocal_nodes + nghost_nodes, { @@ -921,7 +928,7 @@ void FEA_Module_Eulerian::euler_solve() all_node_velocities_interface(node_gid, idim) = ghost_node_velocities_interface(node_gid - nlocal_nodes, idim); all_node_coords_interface(node_gid, idim) = ghost_node_coords_interface(node_gid - nlocal_nodes, idim); } - }); // end parallel for + }); // end parallel for Kokkos::fence(); } // end view scope @@ -1026,7 +1033,7 @@ void FEA_Module_Eulerian::euler_solve() { node_vel(rk_level, node_gid, idim) = ghost_node_velocities_interface(node_gid - nlocal_nodes, idim); } - }); // end parallel for + }); // end parallel for } // end view scope Kokkos::fence(); @@ -1193,7 +1200,7 @@ void FEA_Module_Eulerian::euler_solve() node_velocities_interface(node_gid, idim) = node_vel(rk_level, node_gid, idim); node_coords_interface(node_gid, idim) = node_coords(rk_level, node_gid, idim); } - }); + }); } // end view scope Kokkos::fence(); @@ -1228,7 +1235,7 @@ void FEA_Module_Eulerian::euler_solve() all_node_velocities_interface(node_gid, idim) = node_velocities_interface(node_gid, idim); all_node_coords_interface(node_gid, idim) = node_coords_interface(node_gid, idim); } - }); // end parallel for + }); // end parallel for Kokkos::fence(); FOR_ALL_CLASS(node_gid, nlocal_nodes, nlocal_nodes + nghost_nodes, { @@ -1237,7 +1244,7 @@ void FEA_Module_Eulerian::euler_solve() all_node_velocities_interface(node_gid, idim) = ghost_node_velocities_interface(node_gid - nlocal_nodes, idim); all_node_coords_interface(node_gid, idim) = ghost_node_coords_interface(node_gid - nlocal_nodes, idim); } - }); // end parallel for + }); // end parallel for Kokkos::fence(); } // end view scope @@ -1273,12 +1280,12 @@ void FEA_Module_Eulerian::euler_solve() { KE_loc_sum += node_mass(node_gid) * ke; } - }, KE_sum); + }, KE_sum); Kokkos::fence(); KE_sum = 0.5 * KE_sum; objective_accumulation += KE_sum * dt; } - } + } // end topology optimization if check size_t write = 0; if ((cycle + 1) % graphics_cyc_ival == 0 && cycle > 0) diff --git a/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp b/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp index 13870d5ae..ab9c05781 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp @@ -409,25 +409,35 @@ void Explicit_Solver::run() ------------------------------------------------------------------------- */ void Explicit_Solver::read_mesh_ansys_dat(const char* MESH) { - char ch; - int num_dim = simparam.num_dims; - Input_Options input_options = simparam.input_options.value(); - int p_order = input_options.p_order; - real_t unit_scaling = input_options.unit_scaling; - bool restart_file = simparam.restart_file; - int local_node_index, current_column_index; - size_t strain_count; - std::string skip_line, read_line, substring, token; - std::stringstream line_parse; + Input_Options input_options = simparam.input_options.value(); + + bool restart_file = simparam.restart_file; + bool zero_index_base = input_options.zero_index_base; + + char ch; + std::string skip_line, read_line, substring, token; + std::stringstream line_parse; + + int num_dim = simparam.num_dims; + int p_order = input_options.p_order; + int local_node_index, current_column_index; + int buffer_loop, buffer_iteration, buffer_iterations; + int dof_limit, scan_loop, nodes_per_element; + int negative_index_found = 0; + int global_negative_index_found = 0; + + real_t unit_scaling = input_options.unit_scaling; + real_t dof_value; + + size_t read_index_start, node_rid, elem_gid; + size_t strain_count; + CArrayKokkos read_buffer; - int buffer_loop, buffer_iteration, buffer_iterations, dof_limit, scan_loop, nodes_per_element; - size_t read_index_start, node_rid, elem_gid; - GO node_gid; - real_t dof_value; - host_vec_array node_densities; - bool zero_index_base = input_options.zero_index_base; - int negative_index_found = 0; - int global_negative_index_found = 0; + + GO node_gid; + + host_vec_array node_densities; + // Nodes_Per_Element_Type = elements::elem_types::Nodes_Per_Element_Type; // read the mesh @@ -1216,26 +1226,29 @@ void Explicit_Solver::setup_optimization_problem() std::vector TO_Module_My_FEA_Module = simparam.TO_Module_My_FEA_Module; // std::vector Multi_Objective_Modules = simparam->Multi_Objective_Modules; // std::vector Multi_Objective_Weights = simparam->Multi_Objective_Weights; - std::vector> Function_Arguments = simparam.Function_Arguments; - std::vector TO_Function_Type = simparam.TO_Function_Type; + std::vector> Function_Arguments = simparam.Function_Arguments; + std::vector TO_Function_Type = simparam.TO_Function_Type; + std::vector>> Multi_Objective_Terms; - std::string constraint_base, constraint_name; - std::stringstream number_union; - CArray Surface_Nodes; - GO current_node_index, current_element_index; - LO local_node_index, local_element_id; - int num_bdy_patches_in_set; - size_t node_id, patch_id, module_id; - int num_boundary_sets; - int local_surface_id; + std::string constraint_base, constraint_name; + std::stringstream number_union; + + CArray Surface_Nodes; + GO current_node_index, current_element_index; + LO local_node_index, local_element_id; + int num_bdy_patches_in_set; + size_t node_id, patch_id, module_id; + int num_boundary_sets; + int local_surface_id; + const_host_vec_array design_densities; typedef ROL::TpetraMultiVector ROL_MV; const_host_elem_conn_array nodes_in_elem = global_nodes_in_elem_distributed->getLocalView(Tpetra::Access::ReadOnly); // fill parameter list with desired algorithmic options or leave as default // Read optimization input parameter list. - std::string filename = "optimization_parameters.xml"; + std::string filename = "/var/tmp/repos/Fierro/Fierro/src/Parallel-Solvers/Parallel-Explicit/optimization_parameters.xml"; auto parlist = ROL::getParametersFromXmlFile(filename); // ROL::ParameterList parlist; @@ -2054,19 +2067,22 @@ void Explicit_Solver::comm_densities() void Explicit_Solver::parallel_tecplot_writer() { - int num_dim = simparam.num_dims; - std::string current_file_name; - std::string base_file_name = "TecplotTO"; - std::string base_file_name_undeformed = "TecplotTO_undeformed"; - std::stringstream current_line_stream; - std::string current_line; - std::string file_extension = ".dat"; - std::string file_count; + int num_dim = simparam.num_dims; + int time_step = 0; + int temp_convert; + int noutput, nvector; + + std::string current_file_name; + std::string base_file_name = "TecplotTO"; + std::string base_file_name_undeformed = "TecplotTO_undeformed"; + std::string current_line; + std::string file_extension = ".dat"; + std::string file_count; + std::stringstream count_temp; - int time_step = 0; - int temp_convert; - int noutput, nvector; - bool displace_geometry = false; + std::stringstream current_line_stream; + + bool displace_geometry = false; // Convert ijk index system to the finite element numbering convention // for vertices in cell @@ -2320,27 +2336,31 @@ void Explicit_Solver::parallel_tecplot_writer() void Explicit_Solver::parallel_vtk_writer() { - int num_dim = simparam.num_dims; - std::string current_file_name; - std::string base_file_name = "VTK"; - std::string base_file_name_undeformed = "VTK_undeformed"; - std::stringstream current_line_stream; - std::string current_line; - std::string file_extension = ".vtk"; - std::string file_count; + int num_dim = simparam.num_dims; + int time_step = 0; + int temp_convert; + int noutput, nvector; + int buffer_size_per_element_line, nlocal_elements; + int default_vector_count; + int buffer_size_per_node_line; + int nlocal_sorted_nodes; + bool displace_geometry = false; + + std::string current_file_name; + std::string base_file_name = "VTK"; + std::string base_file_name_undeformed = "VTK_undeformed"; + std::string current_line; + std::string file_extension = ".vtk"; + std::string file_count; + std::stringstream count_temp; - int time_step = 0; - int temp_convert; - int noutput, nvector; - bool displace_geometry = false; - MPI_Offset current_stream_position, header_stream_offset, file_stream_offset; - MPI_File myfile_parallel; - int buffer_size_per_element_line, nlocal_elements; - GO first_element_global_id; - int default_vector_count; - int buffer_size_per_node_line; - int nlocal_sorted_nodes; - GO first_node_global_id; + std::stringstream current_line_stream; + + MPI_Offset current_stream_position, header_stream_offset, file_stream_offset; + MPI_File myfile_parallel; + + GO first_element_global_id; + GO first_node_global_id; // Convert ijk index system to the finite element numbering convention // for vertices in cell @@ -2364,6 +2384,7 @@ void Explicit_Solver::parallel_vtk_writer() convert_ijk_to_ensight(1) = 1; convert_ijk_to_ensight(2) = 3; convert_ijk_to_ensight(3) = 2; + if (num_dim == 3) { convert_ijk_to_ensight(4) = 4; diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/mesh.h b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/mesh.h index d33862889..40e9417fc 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/mesh.h +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/mesh.h @@ -67,17 +67,13 @@ void bubble_sort(size_t arr[], const size_t num) struct mesh_t { size_t num_dims; - - size_t num_nodes, num_local_nodes; - + size_t num_nodes; + size_t num_local_nodes; size_t num_elems; size_t num_nodes_in_elem; size_t num_patches_in_elem; - size_t num_corners; - size_t num_patches; - size_t num_bdy_patches; size_t num_bdy_nodes; size_t num_bdy_sets; diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/FEA_Module_SGH.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/FEA_Module_SGH.cpp index ac3a309ed..d06ceae29 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/FEA_Module_SGH.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/FEA_Module_SGH.cpp @@ -77,6 +77,8 @@ #define BC_EPSILON 1.0e-6 #define BUFFER_GROW 100 +// #define DEBUG + using namespace utils; FEA_Module_SGH::FEA_Module_SGH( @@ -500,28 +502,35 @@ void FEA_Module_SGH::compute_output() void FEA_Module_SGH::comm_node_masses() { // debug print of design vector - // std::ostream &out = std::cout; - // Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - // if(myrank==0) - // *fos << "Density data :" << std::endl; - // node_densities_distributed->describe(*fos,Teuchos::VERB_EXTREME); - // *fos << std::endl; - // std::fflush(stdout); - - // communicate design densities - // create import object using local node indices map and all indices map - // Tpetra::Import importer(map, ghost_node_map); +#ifdef DEBUG + std::ostream& out = std::cout; + Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + if (myrank == 0) + { + *fos << "Density data :" << std::endl; + } + node_densities_distributed->describe(*fos, Teuchos::VERB_EXTREME); + *fos << std::endl; + std::fflush(stdout); + communicate design densities + create import object using local node indices map and all indices map + Tpetra::Import importer(map, ghost_node_map); +#endif // comms to get ghosts ghost_node_masses_distributed->doImport(*node_masses_distributed, *ghost_importer, Tpetra::INSERT); - // all_node_map->describe(*fos,Teuchos::VERB_EXTREME); - // all_node_velocities_distributed->describe(*fos,Teuchos::VERB_EXTREME); - - // update_count++; - // if(update_count==1){ - // MPI_Barrier(world); - // MPI_Abort(world,4); - // } + +#ifdef DEBUG + all_node_map->describe(*fos, Teuchos::VERB_EXTREME); + all_node_velocities_distributed->describe(*fos, Teuchos::VERB_EXTREME); + + update_count++; + if (update_count == 1) + { + MPI_Barrier(world); + MPI_Abort(world, 4); + } +#endif } /* ------------------------------------------------------------------------------------------- @@ -534,20 +543,22 @@ void FEA_Module_SGH::comm_variables(Teuchos::RCP zp) { // set density vector to the current value chosen by the optimizer test_node_densities_distributed = zp; - +#ifdef DEBUG // debug print of design vector - // std::ostream &out = std::cout; - // Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - // if(myrank==0) - // *fos << "Density data :" << std::endl; - // node_densities_distributed->describe(*fos,Teuchos::VERB_EXTREME); - // *fos << std::endl; - // std::fflush(stdout); + std::ostream& out = std::cout; + Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + if (myrank == 0) + { + *fos << "Density data :" << std::endl; + } + node_densities_distributed->describe(*fos, Teuchos::VERB_EXTREME); + *fos << std::endl; + std::fflush(stdout); // communicate design densities // create import object using local node indices map and all indices map - // Tpetra::Import importer(map, all_node_map); - + Tpetra::Import importer(map, all_node_map); +#endif // comms to get ghosts all_node_densities_distributed->doImport(*test_node_densities_distributed, *importer, Tpetra::INSERT); } @@ -570,8 +581,9 @@ void FEA_Module_SGH::node_density_constraints(host_vec_array node_densities_lowe const DCArrayKokkos loading = module_params->loading; // debug check - // std::cout << "NUMBER OF LOADING CONDITIONS: " << num_lcs << std::endl; - +#ifdef DEBUG + std::cout << "NUMBER OF LOADING CONDITIONS: " << num_lcs << std::endl; +#endif // walk over the nodes to update the velocity FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { double current_node_coords[3]; @@ -586,8 +598,9 @@ void FEA_Module_SGH::node_density_constraints(host_vec_array node_densities_lowe for (size_t ilc = 0; ilc < num_lcs; ilc++) { // debug check - // std::cout << "LOADING CONDITION VOLUME TYPE: " << to_string(loading(ilc).volume) << std::endl; - +#ifdef DEBUG + std::cout << "LOADING CONDITION VOLUME TYPE: " << to_string(loading(ilc).volume) << std::endl; +#endif bool fill_this = loading(ilc).volume.contains(current_node_coords); if (fill_this) { @@ -647,17 +660,19 @@ void FEA_Module_SGH::tag_bdys(const DCArrayKokkos& boundary, size_t num_dim = simparam->num_dims; int nboundary_patches = Explicit_Solver_Pointer_->nboundary_patches; int num_nodes_in_patch = mesh.num_nodes_in_patch; - - // if (bdy_set == mesh.num_bdy_sets){ - // printf(" ERROR: number of boundary sets must be increased by %zu", - // bdy_set-mesh.num_bdy_sets+1); - // exit(0); - // } // end if +#ifdef DEBUG + if (bdy_set == mesh.num_bdy_sets) + { + printf(" ERROR: number of boundary sets must be increased by %zu", + bdy_set - mesh.num_bdy_sets + 1); + exit(0); + } // end if // error and debug flag - // DCArrayKokkos print_flag(1, "print_flag"); - // print_flag.host(0) = false; - // print_flag.update_device(); + DCArrayKokkos print_flag(1, "print_flag"); + print_flag.host(0) = false; + print_flag.update_device(); +#endif FOR_ALL_CLASS(bdy_set, 0, num_bdy_sets, { // tag boundaries @@ -680,12 +695,16 @@ void FEA_Module_SGH::tag_bdys(const DCArrayKokkos& boundary, rk_level); // no=0, yes=1 // debug check - /* - for (size_t patch_node_lid=0; patch_node_lid& boundary, } // end for bdy_patch }); // end FOR_ALL_CLASS bdy_sets - // debug check - // print_flag.update_host(); - // if(print_flag.host(0)) std::cout << "found boundary node with id 549412" << std::endl; +#ifdef DEBUG + print_flag.update_host(); + if (print_flag.host(0)) + { + std::cout << "found boundary node with id 549412" << std::endl; + } +#endif return; } // end tag @@ -800,31 +823,38 @@ void FEA_Module_SGH::sgh_solve() { Dynamic_Options dynamic_options = simparam->dynamic_options; + const int num_dim = simparam->num_dims; const size_t rk_level = dynamic_options.rk_num_bins - 1; - time_value = dynamic_options.time_initial; - time_final = dynamic_options.time_final; - dt_max = dynamic_options.dt_max; - dt_min = dynamic_options.dt_min; - dt_cfl = dynamic_options.dt_cfl; + + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos material = simparam->material; + + time_value = dynamic_options.time_initial; + time_final = dynamic_options.time_final; + dt_max = dynamic_options.dt_max; + dt_min = dynamic_options.dt_min; + dt = dynamic_options.dt; + dt_cfl = dynamic_options.dt_cfl; + graphics_time = simparam->output_options.graphics_step; graphics_dt_ival = simparam->output_options.graphics_step; cycle_stop = dynamic_options.cycle_stop; rk_num_stages = dynamic_options.rk_num_stages; - dt = dynamic_options.dt; + graphics_times = simparam->output_options.graphics_times; + graphics_id = simparam->output_options.graphics_id; + fuzz = dynamic_options.fuzz; tiny = dynamic_options.tiny; small = dynamic_options.small; - graphics_times = simparam->output_options.graphics_times; - graphics_id = simparam->output_options.graphics_id; - size_t num_bdy_nodes = mesh->num_bdy_nodes; - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos material = simparam->material; - int nTO_modules; - int old_max_forward_buffer; - size_t cycle; - const int num_dim = simparam->num_dims; - real_t objective_accumulation, global_objective_accumulation; - std::vector> FEA_Module_My_TO_Modules = simparam->FEA_Module_My_TO_Modules; + + size_t num_bdy_nodes = mesh->num_bdy_nodes; + size_t cycle; + real_t objective_accumulation, global_objective_accumulation; + + int nTO_modules; + int old_max_forward_buffer; + + std::vector> FEA_Module_My_TO_Modules = simparam->FEA_Module_My_TO_Modules; problem = Explicit_Solver_Pointer_->problem; // Pointer to ROL optimization problem object ROL::Ptr> obj_pointer; @@ -983,7 +1013,7 @@ void FEA_Module_SGH::sgh_solve() node_velocities_interface(node_gid, idim) = node_vel(rk_level, node_gid, idim); node_coords_interface(node_gid, idim) = node_coords(rk_level, node_gid, idim); } - }); + }); } // end view scope Kokkos::fence(); @@ -1019,7 +1049,7 @@ void FEA_Module_SGH::sgh_solve() all_node_velocities_interface(node_gid, idim) = node_velocities_interface(node_gid, idim); all_node_coords_interface(node_gid, idim) = node_coords_interface(node_gid, idim); } - }); // end parallel for + }); // end parallel for Kokkos::fence(); FOR_ALL_CLASS(node_gid, nlocal_nodes, nlocal_nodes + nghost_nodes, { @@ -1028,13 +1058,13 @@ void FEA_Module_SGH::sgh_solve() all_node_velocities_interface(node_gid, idim) = ghost_node_velocities_interface(node_gid - nlocal_nodes, idim); all_node_coords_interface(node_gid, idim) = ghost_node_coords_interface(node_gid - nlocal_nodes, idim); } - }); // end parallel for + }); // end parallel for Kokkos::fence(); // interface for element internal energies FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { element_internal_energy(elem_gid, 0) = elem_sie(rk_level, elem_gid); - }); // end parallel for + }); // end parallel for Kokkos::fence(); } // end view scope @@ -1162,40 +1192,45 @@ void FEA_Module_SGH::sgh_solve() cycle); } - /* - debug block - if(myrank==1){ - std::cout << rk_alpha << " " << dt << std::endl; - for(int i = 0; i < nall_nodes; i++){ - double node_force[3]; - for (size_t dim = 0; dim < num_dim; dim++){ - node_force[dim] = 0.0; - } // end for dim - - // loop over all corners around the node and calculate the nodal force - for (size_t corner_lid=0; corner_lidall_node_map->getGlobalElement(i) << " " << corner_gid << " " << corner_force(corner_gid, 0) << " " << corner_force(corner_gid, 1) << " " << corner_force(corner_gid, 2) << std::endl; - // loop over dimension - for (size_t dim = 0; dim < num_dim; dim++){ - node_force[dim] += corner_force(corner_gid, dim); - } // end for dim - - } // end for corner_lid - //std::cout << Explicit_Solver_Pointer_->all_node_map->getGlobalElement(i) << " " << node_force[0] << " " << node_force[1] << " " << node_force[2] << std::endl; - //std::cout << Explicit_Solver_Pointer_->all_node_map->getGlobalElement(i) << " " << node_mass(i) << std::endl; - } +#ifdef DEBUG + if (myrank == 1) + { + std::cout << "rk_alpha = " << rk_alpha << ", dt = " << dt << std::endl; + for (int i = 0; i < nall_nodes; i++) + { + double node_force[3]; + for (size_t dim = 0; dim < num_dim; dim++) + { + node_force[dim] = 0.0; + } // end for dim + + // loop over all corners around the node and calculate the nodal force + for (size_t corner_lid = 0; corner_lid < mesh.num_corners_in_node(i); corner_lid++) + { + // Get corner gid + size_t corner_gid = mesh.corners_in_node(i, corner_lid); + std::cout << Explicit_Solver_Pointer_->all_node_map->getGlobalElement(i) << " " << corner_gid << " " << corner_force(corner_gid, 0) << " " << corner_force(corner_gid, + 1) << " " << corner_force(corner_gid, 2) << std::endl; + // loop over dimension + for (size_t dim = 0; dim < num_dim; dim++) + { + node_force[dim] += corner_force(corner_gid, dim); + } // end for dim + } // end for corner_lid + } } - /* - //debug print vector values on a rank - /* - if(myrank==0) - for(int i = 0; i < nall_nodes; i++){ - std::cout << Explicit_Solver_Pointer_->all_node_map->getGlobalElement(i) << " " << node_vel(rk_level,i,0) << " " << node_vel(rk_level,i,1) << " " << node_vel(rk_level,i,2) << std::endl; - } - */ + + // debug print vector values on a rank + + if (myrank == 0) + { + for (int i = 0; i < nall_nodes; i++) + { + std::cout << Explicit_Solver_Pointer_->all_node_map->getGlobalElement(i) << " " << node_vel(rk_level, i, 0) << " " << node_vel(rk_level, i, 1) << " " << node_vel(rk_level, i, + 2) << std::endl; + } + } +#endif // ---- Update nodal velocities ---- // update_velocity_sgh(rk_alpha, @@ -1263,13 +1298,18 @@ void FEA_Module_SGH::sgh_solve() double comm_time4 = Explicit_Solver_Pointer_->CPU_Time(); Explicit_Solver_Pointer_->host2dev_time += comm_time4 - comm_time3; Explicit_Solver_Pointer_->communication_time += comm_time4 - comm_time1; + +#ifdef DEBUG // debug print vector values on a rank - /* - if(myrank==0) - for(int i = 0; i < nall_nodes; i++){ - std::cout << Explicit_Solver_Pointer_->all_node_map->getGlobalElement(i) << " " << node_vel(rk_level,i,0) << " " << node_vel(rk_level,i,1) << " " << node_vel(rk_level,i,2) << std::endl; - } - */ + if (myrank == 0) + { + for (int i = 0; i < nall_nodes; i++) + { + std::cout << Explicit_Solver_Pointer_->all_node_map->getGlobalElement(i) << " " << node_vel(rk_level, i, 0) << " " << node_vel(rk_level, i, 1) << " " << node_vel(rk_level, i, + 2) << std::endl; + } + } +#endif // ---- Update specific internal energy in the elements ---- update_energy_sgh(rk_alpha, *mesh, @@ -1572,7 +1612,7 @@ void FEA_Module_SGH::sgh_solve() { KE_loc_sum += node_mass(node_gid) * ke; } - }, KE_sum); + }, KE_sum); Kokkos::fence(); KE_sum = 0.5 * KE_sum; objective_accumulation += KE_sum * dt; diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp index 9ef23e8c1..2d5310509 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp @@ -280,7 +280,7 @@ void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& ma mu_term = muc(node_lid) * fabs(shock_dir(0) * area_normal(node_lid, 0) + shock_dir(1) * area_normal(node_lid, 1) - + shock_dir(2) * area_normal(node_lid, 2) ); // code gradient for shock dir w.r.t velocity if using shock_dir + + shock_dir(2) * area_normal(node_lid, 2) ); // code gradient for shock dir w.r.t velocity if using shock_dir } else { @@ -994,14 +994,19 @@ void FEA_Module_SGH::get_force_ugradient_sgh(const DCArrayKokkos& ma // gradients of the element volume get_vol_hex_ugradient(volume_gradients, elem_gid, node_coords, elem_node_gids, rk_level); - // //debug - // for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - // for (size_t dim = 0; dim < num_dims; dim++){ - // if(volume_gradients(node_lid, dim)>1 || volume_gradients(node_lid, dim) < -1) - // std::cout << volume_gradients(node_lid, dim) << " "; - // } // end for - // } // end for - // std::cout << std::endl; +#ifdef DEBUG + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { + for (size_t dim = 0; dim < num_dims; dim++) + { + if (volume_gradients(node_lid, dim) > 1 || volume_gradients(node_lid, dim) < -1) + { + std::cout << volume_gradients(node_lid, dim) << " "; + } + } // end for + } // end for + std::cout << std::endl; +#endif // get the B matrix which are the OUTWARD corner area normals get_bmatrix(area_normal, @@ -1483,16 +1488,21 @@ void FEA_Module_SGH::get_force_ugradient_sgh(const DCArrayKokkos& ma void FEA_Module_SGH::force_design_gradient_term(const_vec_array design_variables, vec_array design_gradients) { - size_t num_bdy_nodes = mesh->num_bdy_nodes; - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos material = simparam->material; - const int num_dim = simparam->num_dims; - size_t num_corners = rnum_elem * num_nodes_in_elem; - real_t global_dt; - bool element_constant_density = true; - size_t current_data_index, next_data_index; - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - CArrayKokkos current_element_adjoint = CArrayKokkos(num_nodes_in_elem, num_dim); + bool element_constant_density = true; + + size_t num_bdy_nodes = mesh->num_bdy_nodes; + size_t num_corners = rnum_elem * num_nodes_in_elem; + size_t current_data_index, next_data_index; + + real_t global_dt; + + const int num_dim = simparam->num_dims; + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos material = simparam->material; + + auto current_element_adjoint = CArrayKokkos(num_nodes_in_elem, num_dim); // gradient contribution from gradient of Force vector with respect to design variable. if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) @@ -1545,12 +1555,12 @@ void FEA_Module_SGH::force_design_gradient_term(const_vec_array design_variables node_vel(rk_level, node_gid, idim) = current_velocity_vector(node_gid, idim); node_coords(rk_level, node_gid, idim) = current_coord_vector(node_gid, idim); } - }); // end parallel for + }); // end parallel for Kokkos::fence(); FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { elem_sie(rk_level, elem_gid) = current_element_internal_energy(elem_gid, 0); - }); // end parallel for + }); // end parallel for Kokkos::fence(); get_vol(); @@ -1645,7 +1655,7 @@ void FEA_Module_SGH::force_design_gradient_term(const_vec_array design_variables corner_id = elem_id * num_nodes_in_elem + inode; corner_value_storage(corner_id) = inner_product; } - }); // end parallel for + }); // end parallel for Kokkos::fence(); // accumulate node values from corner storage @@ -1657,7 +1667,7 @@ void FEA_Module_SGH::force_design_gradient_term(const_vec_array design_variables corner_id = corners_in_node(node_id, icorner); design_gradients(node_id, 0) += -corner_value_storage(corner_id) * global_dt; } - }); // end parallel for + }); // end parallel for Kokkos::fence(); } // end view scope } diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_sgh.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_sgh.cpp index e4046cddf..1a338a1e0 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_sgh.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_sgh.cpp @@ -861,14 +861,15 @@ void FEA_Module_SGH::applied_forces(const DCArrayKokkos& material, const size_t cycle ) { - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - const size_t num_dim = mesh.num_dims; - const_vec_array all_initial_node_coords = all_initial_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); - const size_t num_lcs = module_params->loading.size(); + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + const size_t num_dim = mesh.num_dims; + const size_t num_lcs = module_params->loading.size(); const DCArrayKokkos mat_fill = simparam->mat_fill; const DCArrayKokkos loading = module_params->loading; + const_vec_array all_initial_node_coords = all_initial_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + // debug check // std::cout << "NUMBER OF LOADING CONDITIONS: " << num_lcs << std::endl; diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/geometry.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/geometry.cpp index b0501bf19..726c54a5b 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/geometry.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/geometry.cpp @@ -335,19 +335,12 @@ void FEA_Module_SGH::get_bmatrix2D(const ViewCArrayKokkos& B_matrix, */ B_matrix(0, 0) = -0.5 * (y(3) - y(1)); - B_matrix(1, 0) = -0.5 * (y(0) - y(2)); - B_matrix(2, 0) = -0.5 * (y(1) - y(3)); - B_matrix(3, 0) = -0.5 * (y(2) - y(0)); - B_matrix(0, 1) = -0.5 * (x(1) - x(3)); - B_matrix(1, 1) = -0.5 * (x(2) - x(0)); - B_matrix(2, 1) = -0.5 * (x(3) - x(1)); - B_matrix(3, 1) = -0.5 * (x(0) - x(2)); // diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/momentum.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/momentum.cpp index 03452ccd1..7adbfb401 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/momentum.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/momentum.cpp @@ -61,13 +61,13 @@ void FEA_Module_SGH::get_velgrad(ViewCArrayKokkos& vel_grad, { const size_t num_nodes_in_elem = 8; - double u_array[num_nodes_in_elem]; // x-dir vel component - double v_array[num_nodes_in_elem]; // y-dir vel component - double w_array[num_nodes_in_elem]; // z-dir vel component - - ViewCArrayKokkos u(u_array, num_nodes_in_elem); - ViewCArrayKokkos v(v_array, num_nodes_in_elem); - ViewCArrayKokkos w(w_array, num_nodes_in_elem); + double u_array[num_nodes_in_elem]; // x-dir vel component + double v_array[num_nodes_in_elem]; // y-dir vel component + double w_array[num_nodes_in_elem]; // z-dir vel component + + ViewCArrayKokkos u(u_array, num_nodes_in_elem); + ViewCArrayKokkos v(v_array, num_nodes_in_elem); + ViewCArrayKokkos w(w_array, num_nodes_in_elem); // get the vertex velocities for the cell for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) @@ -149,8 +149,9 @@ void FEA_Module_SGH::get_velgrad2D(ViewCArrayKokkos& vel_grad, { const size_t num_nodes_in_elem = 4; - double u_array[num_nodes_in_elem]; - double v_array[num_nodes_in_elem]; + double u_array[num_nodes_in_elem]; + double v_array[num_nodes_in_elem]; + ViewCArrayKokkos u(u_array, num_nodes_in_elem); // x-dir vel component ViewCArrayKokkos v(v_array, num_nodes_in_elem); // y-dir vel component @@ -217,6 +218,7 @@ void FEA_Module_SGH::get_divergence(DViewCArrayKokkos& elem_div, double u_array[num_nodes_in_elem]; double v_array[num_nodes_in_elem]; double w_array[num_nodes_in_elem]; + ViewCArrayKokkos u(u_array, num_nodes_in_elem); // x-dir vel component ViewCArrayKokkos v(v_array, num_nodes_in_elem); // y-dir vel component ViewCArrayKokkos w(w_array, num_nodes_in_elem); // z-dir vel component diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/setup_sgh.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/setup_sgh.cpp index dd85ffe5e..a58fac95c 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/setup_sgh.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/setup_sgh.cpp @@ -8,6 +8,8 @@ #include "Simulation_Parameters/FEA_Module/SGH_Parameters.h" #include "Explicit_Solver.h" +// #define DEBUG + void FEA_Module_SGH::setup() { const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; @@ -28,40 +30,36 @@ void FEA_Module_SGH::setup() sgh_interface_setup(node_interface, elem_interface, corner_interface); mesh->build_corner_connectivity(); // debug print of corner ids - /* - if(myrank==1){ - for(int i = 0; i < mesh.num_nodes; i++){ - +#ifdef DEBUG + if (myrank == 1) + { + for (int i = 0; i < mesh.num_nodes; i++) + { // loop over all corners around the node and calculate the nodal force - for (size_t corner_lid=0; corner_lidgetGlobalElement(i) << " " << i << " " << all_node_map->getLocalElement(all_node_map->getGlobalElement(i)) << " " << corner_gid << " " << std::endl; - + for (size_t corner_lid = 0; corner_lid < mesh.num_corners_in_node(i); corner_lid++) + { + // Get corner gid + size_t corner_gid = mesh.corners_in_node(i, corner_lid); + std::cout << map->getGlobalElement(i) << " " << i << " " << all_node_map->getLocalElement(all_node_map->getGlobalElement(i)) << " " << corner_gid << " " << std::endl; } // end for corner_lid - //std::cout << explicit_solver_pointer->all_node_map->getGlobalElement(i) << " " << node_force[0] << " " << node_force[1] << " " << node_force[2] << std::endl; - //std::cout << explicit_solver_pointer->all_node_map->getGlobalElement(i) << " " << node_mass(i) << std::endl; - } } - */ - /* - if(myrank==1){ - for(int i = 0; i < mesh.num_elems; i++){ + } + if (myrank == 1) + { + for (int i = 0; i < mesh.num_elems; i++) + { // loop over all corners around the node and calculate the nodal force - for (size_t corner_lid=0; corner_lidgetGlobalElement(mesh.nodes_in_elem(i, corner_lid)) <<" " << corner_gid << " " << std::endl; - + for (size_t corner_lid = 0; corner_lid < max_nodes_per_element; corner_lid++) + { + // Get corner gid + size_t corner_gid = mesh.corners_in_elem(i, corner_lid); + std::cout << i << " " << mesh.nodes_in_elem(i, corner_lid) << " " << all_node_map->getGlobalElement(mesh.nodes_in_elem(i, corner_lid)) << " " << corner_gid << " " << std::endl; } // end for corner_lid - //std::cout << explicit_solver_pointer->all_node_map->getGlobalElement(i) << " " << node_force[0] << " " << node_force[1] << " " << node_force[2] << std::endl; - //std::cout << explicit_solver_pointer->all_node_map->getGlobalElement(i) << " " << node_mass(i) << std::endl; - } } - */ + } +#endif + mesh->build_elem_elem_connectivity(); mesh->num_bdy_patches = nboundary_patches; if (num_dim == 2) @@ -517,7 +515,7 @@ void FEA_Module_SGH::setup() FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { node_mass_interface(node_gid, 0) = node_mass(node_gid); - }); // end parallel for + }); // end parallel for } // end view scope Kokkos::fence(); // communicate ghost densities @@ -530,7 +528,7 @@ void FEA_Module_SGH::setup() FOR_ALL_CLASS(node_gid, nlocal_nodes, nall_nodes, { node_mass(node_gid) = ghost_node_mass_interface(node_gid - nlocal_nodes, 0); - }); // end parallel for + }); // end parallel for } // end view scope Kokkos::fence(); } // endif @@ -635,76 +633,78 @@ void FEA_Module_SGH::sgh_interface_setup(node_t& node, for (int inode = 0; inode < num_nodes_in_elem; inode++) { nodes_in_elem.host(ielem, inode) = Explicit_Solver_Pointer_->all_node_map->getLocalElement(interface_nodes_in_elem(ielem, inode)); - // debug print - // std::cout << nodes_in_elem.get_kokkos_dual_view().h_view(ielem*num_nodes_in_elem + inode)+1<< " "; } - // std::cout << std::endl; } } // update device side nodes_in_elem.update_device(); // debug print +#ifdef DEBUG + CArrayKokkos device_mesh_nodes_in_elem(rnum_elem, num_nodes_in_elem); + device_mesh_nodes_in_elem.get_kokkos_view() = nodes_in_elem.get_kokkos_dual_view().d_view; + host_mesh_nodes_in_elem.get_kokkos_view() = nodes_in_elem.get_kokkos_dual_view().view_host(); - // CArrayKokkos device_mesh_nodes_in_elem(rnum_elem, num_nodes_in_elem); - // device_mesh_nodes_in_elem.get_kokkos_view() = nodes_in_elem.get_kokkos_dual_view().d_view; - // host_mesh_nodes_in_elem.get_kokkos_view() = nodes_in_elem.get_kokkos_dual_view().view_host(); - /* - if(myrank==1){ - std::cout << "ELEMENT CONNECTIVITY ON RANK 1 in LOCAL INDICES" << myrank << std::endl; - for(int ielem = 0; ielem < rnum_elem; ielem++){ - std::cout << "Element index " << ielem+1 << " "; - for(int inode = 0; inode < num_nodes_in_elem; inode++){ - //debug print - //device_mesh_nodes_in_elem(ielem,inode) = Explicit_Solver_Pointer_->all_node_map->getLocalElement(interface_nodes_in_elem(ielem,inode)); - std::cout << nodes_in_elem(ielem, inode)+1<< " "; + if (myrank == 1) + { + std::cout << "ELEMENT CONNECTIVITY ON RANK 1 in LOCAL INDICES" << myrank << std::endl; + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + std::cout << "Element index " << ielem + 1 << " "; + for (int inode = 0; inode < num_nodes_in_elem; inode++) + { + // debug print + device_mesh_nodes_in_elem(ielem, inode) = Explicit_Solver_Pointer_->all_node_map->getLocalElement(interface_nodes_in_elem(ielem, inode)); + std::cout << nodes_in_elem(ielem, inode) + 1 << " "; + } + std::cout << std::endl; } - std::cout << std::endl; - } } - */ - /* + std::cout.flush(); - if(myrank==1){ - std::cout << "ELEMENT CONNECTIVITY ON RANK 1 in GLOBAL INDICES" << myrank << std::endl; - std::cout << "local node index of global index 275 on rank 1 " << Explicit_Solver_Pointer_->all_node_map->getLocalElement(275) << std::endl; - for(int ielem = 0; ielem < rnum_elem; ielem++){ - std::cout << ielem << " "; - for(int inode = 0; inode < num_nodes_in_elem; inode++){ - //debug print - //device_mesh_nodes_in_elem(ielem,inode) = Explicit_Solver_Pointer_->all_node_map->getLocalElement(interface_nodes_in_elem(ielem,inode)); - std::cout << Explicit_Solver_Pointer_->all_node_map->getGlobalElement(nodes_in_elem(ielem, inode))<< " "; + if (myrank == 1) + { + std::cout << "ELEMENT CONNECTIVITY ON RANK 1 in GLOBAL INDICES" << myrank << std::endl; + std::cout << "local node index of global index 275 on rank 1 " << Explicit_Solver_Pointer_->all_node_map->getLocalElement(275) << std::endl; + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + std::cout << ielem << " "; + for (int inode = 0; inode < num_nodes_in_elem; inode++) + { + // debug print + device_mesh_nodes_in_elem(ielem, inode) = Explicit_Solver_Pointer_->all_node_map->getLocalElement(interface_nodes_in_elem(ielem, inode)); + std::cout << Explicit_Solver_Pointer_->all_node_map->getGlobalElement(nodes_in_elem(ielem, inode)) << " "; + } + std::cout << std::endl; } - std::cout << std::endl; - } } std::cout.flush(); - */ - /* + size_t nall_nodes = Explicit_Solver_Pointer_->nall_nodes; - node.all_coords = DCArrayKokkos (rk_num_bins, nall_nodes, num_dim); - node.all_vel = DCArrayKokkos (rk_num_bins, nall_nodes, num_dim); - node.all_mass = DCArrayKokkos (nall_nodes); + node.all_coords = DCArrayKokkos(rk_num_bins, nall_nodes, num_dim); + node.all_vel = DCArrayKokkos(rk_num_bins, nall_nodes, num_dim); + node.all_mass = DCArrayKokkos(nall_nodes); - //save all data (nlocal +nghost) + // save all data (nlocal +nghost) CArrayKokkos host_all_node_coords_state(rk_num_bins, nall_nodes, num_dim); - host_vec_array interface_all_node_coords = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); + host_vec_array interface_all_node_coords = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); host_all_node_coords_state.get_kokkos_view() = node.all_coords.get_kokkos_dual_view().view_host(); - //host_node_coords_state = CArrayKokkos(rk_num_bins, nall_nodes, num_dim); - //host_all_node_coords_state.get_kokkos_view() = Kokkos::View("debug", rk_num_bins*nall_nodes*num_dim); - //save node data to node.coords + host_node_coords_state = CArrayKokkos(rk_num_bins, nall_nodes, num_dim); + host_all_node_coords_state.get_kokkos_view() = Kokkos::View("debug", rk_num_bins * nall_nodes * num_dim); + // save node data to node.coords - //std::cout << "ALL NODE DATA ON RANK " << myrank << std::endl; - for(int inode = 0; inode < nall_nodes; inode++){ - //std::cout << "Node index " << inode+1 << " "; - node.all_coords.host(0,inode,0) = interface_all_node_coords(inode,0); - //std::cout << host_all_node_coords_state(0,inode,0)+1<< " "; - node.all_coords.host(0,inode,1) = interface_all_node_coords(inode,1); - //std::cout << host_all_node_coords_state(0,inode,1)+1<< " "; - node.all_coords.host(0,inode,2) = interface_all_node_coords(inode,2); - //std::cout << host_all_node_coords_state(0,inode,2)+1<< std::endl; + std::cout << "ALL NODE DATA ON RANK " << myrank << std::endl; + for (int inode = 0; inode < nall_nodes; inode++) + { + // std::cout << "Node index " << inode+1 << " "; + node.all_coords.host(0, inode, 0) = interface_all_node_coords(inode, 0); + // std::cout << host_all_node_coords_state(0,inode,0)+1<< " "; + node.all_coords.host(0, inode, 1) = interface_all_node_coords(inode, 1); + // std::cout << host_all_node_coords_state(0,inode,1)+1<< " "; + node.all_coords.host(0, inode, 2) = interface_all_node_coords(inode, 2); + // std::cout << host_all_node_coords_state(0,inode,2)+1<< std::endl; } - */ +#endif // save the node coords to the current RK value for (size_t node_gid = 0; node_gid < nall_nodes; node_gid++) @@ -932,7 +932,9 @@ void FEA_Module_SGH::grow_boundary_sets(int num_sets) Boundary_Condition_Patches = CArrayKokkos(max_boundary_sets, nboundary_patches, "Boundary_Condition_Patches"); // copy previous data back over - // std::cout << "NUM BOUNDARY CONDITIONS ON RANK " << myrank << " FOR COPY " << max_boundary_sets < zp) int max_stride = 0; int current_module_index; size_t access_index, row_access_index, row_counter; - GO global_index, global_dof_index; - LO local_dof_index; + + GO global_index, global_dof_index; + LO local_dof_index; const size_t num_fills = simparam->regions.size(); const size_t rk_num_bins = simparam->dynamic_options.rk_num_bins; @@ -68,7 +69,8 @@ void FEA_Module_SGH::update_forward_solve(Teuchos::RCP zp) const DCArrayKokkos mat_fill = simparam->mat_fill; const DCArrayKokkos boundary = module_params->boundary; const DCArrayKokkos material = simparam->material; - CArray current_element_nodal_densities = CArray(num_nodes_in_elem); + + CArray current_element_nodal_densities = CArray(num_nodes_in_elem); std::vector> FEA_Module_My_TO_Modules = simparam->FEA_Module_My_TO_Modules; problem = Explicit_Solver_Pointer_->problem; // Pointer to ROL optimization problem object @@ -88,7 +90,8 @@ void FEA_Module_SGH::update_forward_solve(Teuchos::RCP zp) relative_element_densities.host(elem_id) = average_element_density(num_nodes_in_elem, current_element_nodal_densities); } // for } // view scope - // debug print + // debug print + // std::cout << "ELEMENT RELATIVE DENSITY TEST " << relative_element_densities.host(0) << std::endl; relative_element_densities.update_device(); @@ -477,7 +480,7 @@ void FEA_Module_SGH::update_forward_solve(Teuchos::RCP zp) vec_array node_mass_interface = node_masses_distributed->getLocalView(Tpetra::Access::ReadWrite); FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { node_mass_interface(node_gid, 0) = node_mass(node_gid); - }); // end parallel for + }); // end parallel for } // end view scope Kokkos::fence(); // communicate ghost densities @@ -582,10 +585,6 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full() } } // end if } - // else if (cycle==1){ - // if(myrank==0) - // printf("cycle = %lu, time = %f, time step = %f \n", cycle-1, time_data[cycle-1], global_dt); - // } // end if // compute adjoint vector for this data point; use velocity midpoint // view scope @@ -789,9 +788,10 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full() const_vec_array previous_adjoint_vector = (*adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); const_vec_array phi_previous_adjoint_vector = (*phi_adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); const_vec_array psi_previous_adjoint_vector = (*psi_adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); - vec_array midpoint_adjoint_vector = adjoint_vector_distributed->getLocalView(Tpetra::Access::ReadWrite); - vec_array phi_midpoint_adjoint_vector = phi_adjoint_vector_distributed->getLocalView(Tpetra::Access::ReadWrite); - vec_array psi_midpoint_adjoint_vector = psi_adjoint_vector_distributed->getLocalView(Tpetra::Access::ReadWrite); + + vec_array midpoint_adjoint_vector = adjoint_vector_distributed->getLocalView(Tpetra::Access::ReadWrite); + vec_array phi_midpoint_adjoint_vector = phi_adjoint_vector_distributed->getLocalView(Tpetra::Access::ReadWrite); + vec_array psi_midpoint_adjoint_vector = psi_adjoint_vector_distributed->getLocalView(Tpetra::Access::ReadWrite); // half step update for RK2 scheme; EQUATION 1 FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { @@ -822,7 +822,7 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full() phi_previous_adjoint_vector(node_gid, idim) / node_mass(node_gid); midpoint_adjoint_vector(node_gid, idim) = -rate_of_change * global_dt / 2 + previous_adjoint_vector(node_gid, idim); } - }); // end parallel for + }); // end parallel for Kokkos::fence(); // apply BCs to adjoint vector, only matters for the momentum adjoint if using strictly velocity boundary conditions @@ -858,7 +858,7 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full() // rate_of_change = -0.0000001*previous_adjoint_vector(node_gid,idim); phi_midpoint_adjoint_vector(node_gid, idim) = -rate_of_change * global_dt / 2 + phi_previous_adjoint_vector(node_gid, idim); } - }); // end parallel for + }); // end parallel for Kokkos::fence(); comm_phi_adjoint_vector(cycle); @@ -882,7 +882,7 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full() rate_of_change = -(matrix_contribution + psi_previous_adjoint_vector(elem_gid, 0) * Power_Gradient_Energies(elem_gid)) / elem_mass(elem_gid); // rate_of_change = -0.0000001*previous_adjoint_vector(node_gid,idim); psi_midpoint_adjoint_vector(elem_gid, 0) = -rate_of_change * global_dt / 2 + psi_previous_adjoint_vector(elem_gid, 0); - }); // end parallel for + }); // end parallel for Kokkos::fence(); // save for second half of RK @@ -926,7 +926,7 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full() phi_midpoint_adjoint_vector(node_gid, idim) / node_mass(node_gid); current_adjoint_vector(node_gid, idim) = -rate_of_change * global_dt + previous_adjoint_vector(node_gid, idim); } - }); // end parallel for + }); // end parallel for Kokkos::fence(); boundary_adjoint(*mesh, boundary, current_adjoint_vector, phi_current_adjoint_vector, psi_midpoint_adjoint_vector); @@ -960,7 +960,7 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full() // rate_of_change = -0.0000001*midpoint_adjoint_vector(node_gid,idim); phi_current_adjoint_vector(node_gid, idim) = -rate_of_change * global_dt + phi_previous_adjoint_vector(node_gid, idim); } - }); // end parallel for + }); // end parallel for Kokkos::fence(); comm_phi_adjoint_vector(cycle); @@ -983,7 +983,7 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full() // debug // std::cout << "PSI RATE OF CHANGE " << rate_of_change << std::endl; psi_current_adjoint_vector(elem_gid, 0) = -rate_of_change * global_dt + psi_previous_adjoint_vector(elem_gid, 0); - }); // end parallel for + }); // end parallel for Kokkos::fence(); // save data from time-step completion @@ -1000,16 +1000,18 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full() void FEA_Module_SGH::compute_topology_optimization_gradient_full(Teuchos::RCP design_densities_distributed, Teuchos::RCP design_gradients_distributed) { - size_t num_bdy_nodes = mesh->num_bdy_nodes; - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos material = simparam->material; - const int num_dim = simparam->num_dims; - int num_corners = rnum_elem * num_nodes_in_elem; - real_t global_dt; - bool element_constant_density = true; - size_t current_data_index, next_data_index; - CArrayKokkos current_element_velocities = CArrayKokkos(num_nodes_in_elem, num_dim); - CArrayKokkos current_element_adjoint = CArrayKokkos(num_nodes_in_elem, num_dim); + bool element_constant_density = true; + size_t num_bdy_nodes = mesh->num_bdy_nodes; + size_t current_data_index, next_data_index; + int num_corners = rnum_elem * num_nodes_in_elem; + real_t global_dt; + + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos material = simparam->material; + const int num_dim = simparam->num_dims; + + auto current_element_velocities = CArrayKokkos(num_nodes_in_elem, num_dim); + auto current_element_adjoint = CArrayKokkos(num_nodes_in_elem, num_dim); if (myrank == 0) { @@ -1121,7 +1123,7 @@ void FEA_Module_SGH::compute_topology_optimization_gradient_full(Teuchos::RCP(rnum_elem, num_nodes_in_elem * num_dim); Power_Gradient_Energies = CArrayKokkos(rnum_elem); @@ -1726,7 +1726,7 @@ void FEA_Module_SGH::init_assembly() { DOF_Graph_Matrix(idof, istride) = Graph_Matrix(idof / num_dim, istride / num_dim) * num_dim + istride % num_dim; } - }); // end parallel for + }); // end parallel for /* //construct distributed gradient matrix from local kokkos data diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/time_integration.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/time_integration.cpp index a934f2932..aeaea1d51 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/time_integration.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/time_integration.cpp @@ -147,7 +147,7 @@ void FEA_Module_SGH::get_timestep(mesh_t& mesh, // This function calculates the time step by finding the shortest distance // between any two nodes in the mesh // ------------------------------------------------------------------------------ -// WARNING WARNING : Only works for 3D, 8 node elements +// WARNING WARNING : Only works for 2D, 4 node elements void FEA_Module_SGH::get_timestep2D(mesh_t& mesh, DViewCArrayKokkos& node_coords, DViewCArrayKokkos& node_vel, From 1b028740cb3250056d7c226c87917d6ce1554476 Mon Sep 17 00:00:00 2001 From: Jacob Moore Date: Fri, 16 Feb 2024 17:25:51 -0600 Subject: [PATCH 19/64] COMP: Adding MSU machine type to automate building on HPCC machines --- scripts/build-fierro.sh | 8 +++++--- scripts/machines/msu-env.sh | 34 ++++++++++++++++++++++++++++++++++ scripts/trilinos-install.sh | 27 +++++++++++++++++++++++++++ 3 files changed, 66 insertions(+), 3 deletions(-) create mode 100644 scripts/machines/msu-env.sh diff --git a/scripts/build-fierro.sh b/scripts/build-fierro.sh index 1830beacc..6f63f1671 100755 --- a/scripts/build-fierro.sh +++ b/scripts/build-fierro.sh @@ -43,6 +43,7 @@ show_help() { echo " darwin The darwin cluster at LANL. Uses module loads for software" echo " linux A general linux machine (that does not use modules)" echo " mac A Mac computer. This option does not allow for cuda and hip builds, and build_cores will be set to 1" + echo " msu A linux computer managed by the HPCC group at Mississippi State University" echo " " echo " --heffte_build_type The build type for the heffte installation. The default is 'fftw'" echo " " @@ -67,7 +68,7 @@ valid_build_action=("full-app" "set-env" "install-trilinos" "install-hdf5" "inst valid_solver=("all" "explicit" "explicit-evpfft" "explicit-ls-evpfft" "explicit-evp" "implicit") valid_kokkos_build_types=("serial" "openmp" "pthreads" "cuda" "hip") valid_heffte_build_types=("fftw" "cufft" "rocfft") -valid_machines=("darwin" "chicoma" "linux" "mac") +valid_machines=("darwin" "chicoma" "linux" "mac" "msu") # Parse command line arguments for arg in "$@"; do @@ -168,6 +169,7 @@ echo "Building based on these argument options:" echo "Build action - ${build_action}" echo "Solver - ${solver}" echo "Kokkos backend - ${kokkos_build_type}" +echo "Machine - ${machine}" if [ "${solver}" = "explicit-evpfft" ] || [ "${solver}" = "explicit-ls-evpfft" ]; then echo "HEFFTE - ${heffte_build_type}" fi @@ -180,14 +182,14 @@ source setup-env.sh ${machine} ${kokkos_build_type} ${build_cores} # Next, do action based on args if [ "$build_action" = "full-app" ]; then - source trilinos-install.sh ${kokkos_build_type} + source trilinos-install.sh ${kokkos_build_type} ${machine} if [ "$solver" = "explicit-evpfft" ] || [ "${solver}" = "explicit-ls-evpfft" ]; then source hdf5-install.sh source heffte-install.sh ${heffte_build_type} ${machine} fi source cmake_build.sh ${solver} ${heffte_build_type} ${kokkos_build_type} elif [ "$build_action" = "install-trilinos" ]; then - source trilinos-install.sh ${kokkos_build_type} + source trilinos-install.sh ${kokkos_build_type} ${machine} elif [ "$build_action" = "install-hdf5" ]; then source hdf5-install.sh elif [ "$build_action" = "install-heffte" ]; then diff --git a/scripts/machines/msu-env.sh b/scripts/machines/msu-env.sh new file mode 100644 index 000000000..50adf7286 --- /dev/null +++ b/scripts/machines/msu-env.sh @@ -0,0 +1,34 @@ +#!/bin/bash -e +### Load environment modules here +### Assign names as relevant + +# arg number is '4' because it's based on the original build-fierro script args +kokkos_build_type="$1" + +mygcc="gcc/12.2.0" +mycuda="cuda/11.7.0" +mympi="openmpi/4.1.4" +mypython="python/3.10.8" +mymkl="intel-oneapi-mkl/2022.2.1" +myfftw="fftw/3.3.10" + + +echo "** Purging modules and loading those necessary for Fierro **" +module purge + +module load ${mygcc} +module load ${mympi} +module load ${mypython} +module load ${mymkl} +module load ${myfftw} + +if [ "$kokkos_build_type" = "cuda" ]; then + module load ${mycuda} +elif [ "$kokkos_build_type" = "hip" ]; then + echo "Error: MSU cannot build with Kokkos HIP backend since we do not have rocm" + show_help + return 1 +fi + +module load cmake +module -t list diff --git a/scripts/trilinos-install.sh b/scripts/trilinos-install.sh index ed6f1c6cb..579416312 100644 --- a/scripts/trilinos-install.sh +++ b/scripts/trilinos-install.sh @@ -1,10 +1,12 @@ #!/bin/bash -e kokkos_build_type="${1}" +machine="${2}" # If all arguments are valid, you can use them in your script as needed echo "Trilinos Kokkos Build Type: $kokkos_build_type" + #check if Trilinos directory exists, git clone Trilinos if it doesn't [ -d "${TRILINOS_SOURCE_DIR}" ] && echo "Directory Trilinos exists, skipping Trilinos download" @@ -68,12 +70,35 @@ OPENMP_ADDITIONS=( -D Trilinos_ENABLE_OpenMP=ON ) +# Flags for building with MKL, which is supported at MSU HPCC +MSU_ADDITIONS=( +-D BLAS_LIBRARY_NAMES="libmkl_rt.so" +-D BLAS_LIBRARY_DIRS="/apps/spack-managed/gcc-11.3.1/intel-oneapi-mkl-2022.2.1-7l7jlsd56x2kljiskrcvsoenmq4y3cu7/mkl/2022.2.1/lib/intel64" +-D LAPACK_LIBRARY_NAMES="libmkl_rt.so" +-D LAPACK_LIBRARY_DIRS="/apps/spack-managed/gcc-11.3.1/intel-oneapi-mkl-2022.2.1-7l7jlsd56x2kljiskrcvsoenmq4y3cu7/mkl/2022.2.1/lib/intel64" +-D TPL_ENABLE_MKL:BOOL=ON +-D MKL_LIBRARY_DIRS:FILEPATH="/apps/spack-managed/gcc-11.3.1/intel-oneapi-mkl-2022.2.1-7l7jlsd56x2kljiskrcvsoenmq4y3cu7/mkl/2022.2.1/lib/intel64" +-D MKL_LIBRARY_NAMES:STRING="mkl_rt" +-D MKL_INCLUDE_DIRS:FILEPATH="/apps/spack-managed/gcc-11.3.1/intel-oneapi-mkl-2022.2.1-7l7jlsd56x2kljiskrcvsoenmq4y3cu7/mkl/2022.2.1/include" +) + # Configure kokkos using CMake cmake_options=( -D CMAKE_BUILD_TYPE=Release -D Trilinos_MUST_FIND_ALL_TPL_LIBS=TRUE -D CMAKE_CXX_STANDARD=17 -D TPL_ENABLE_MPI=ON +) + +echo "**** Machine = ${machine} ****" +if [ "$machine" = "msu" ]; then + echo "**** WARNING: Verify MKL path in trilinos-install.sh ****" + cmake_options+=( + ${MSU_ADDITIONS[@]} + ) +fi + +cmake_options+=( -D Trilinos_ENABLE_Kokkos=ON ${ADDITIONS[@]} -D Trilinos_ENABLE_Amesos2=ON @@ -90,6 +115,8 @@ ${ADDITIONS[@]} -D CMAKE_INSTALL_PREFIX=${TRILINOS_INSTALL_DIR} ) + + if [ "$kokkos_build_type" = "openmp" ]; then cmake_options+=( ${OPENMP_ADDITIONS[@]} From 0c3ddd4f71d3fdd0b17f334ba27b97eb16ec1e59 Mon Sep 17 00:00:00 2001 From: Jacob Moore Date: Mon, 19 Feb 2024 15:03:03 -0600 Subject: [PATCH 20/64] ENH: Basic integrated testing functionality. WIP --- .../Solver-Inputs/SGH_Sedov_12x12x12.yaml | 99 + .../SGH/Sedov_12x12x12/vtk/data/VTK0.vtk | 18252 ++++++++++++++++ .../SGH/Sedov_12x12x12/vtk/outputs.vtk.series | 6 + python/Test-scripts/test_fierro.py | 128 + 4 files changed, 18485 insertions(+) create mode 100644 python/Test-scripts/Solver-Inputs/SGH_Sedov_12x12x12.yaml create mode 100644 python/Test-scripts/standard-results/SGH/Sedov_12x12x12/vtk/data/VTK0.vtk create mode 100644 python/Test-scripts/standard-results/SGH/Sedov_12x12x12/vtk/outputs.vtk.series create mode 100644 python/Test-scripts/test_fierro.py diff --git a/python/Test-scripts/Solver-Inputs/SGH_Sedov_12x12x12.yaml b/python/Test-scripts/Solver-Inputs/SGH_Sedov_12x12x12.yaml new file mode 100644 index 000000000..a8604d82b --- /dev/null +++ b/python/Test-scripts/Solver-Inputs/SGH_Sedov_12x12x12.yaml @@ -0,0 +1,99 @@ +num_dims: 3 +dynamic_options: + time_final: 1.0 + dt_min: 1.e-8 + dt_max: 1.e-2 + dt_start: 1.e-5 + cycle_stop: 2000000 + +mesh_generation_options: + type: Box + origin: [0, 0, 0] + length: [1.2, 1.2, 1.2] + num_elems: [12, 12, 12] + +output_options: + timer_output_level: thorough + output_file_format: vtk + graphics_step: 1.0 + write_initial: false + +fea_module_parameters: + - type: SGH + material_id: 0 + boundary_conditions: + # Tag X plane + - surface: + type: x_plane + plane_position: 0.0 + type: reflected + + # Tag Y plane + - surface: + type: y_plane + plane_position: 0.0 + type: reflected + + # Tag Z plane + - surface: + type: z_plane + plane_position: 0.0 + type: reflected + + # Tag X plane + - surface: + type: x_plane + plane_position: 1.2 + type: reflected + + # Tag Y plane + - surface: + type: y_plane + plane_position: 1.2 + type: reflected + + # Tag Z plane + - surface: + type: z_plane + plane_position: 1.2 + type: reflected + +materials: + - id: 0 + eos_model: ideal_gas + strength_model: none + elastic_modulus: 10 + poisson_ratio: 0.3 + q1: 1.0 + q2: 0 + q1ex: 1.0 + q2ex: 0.0 + global_vars: + - 1.666666666666667 + - 1.0E-14 + - 1.0 + +regions: + - volume: + type: global + material_id: 0 + den: 1.0 + sie: 1.e-10 + + velocity: cartesian + u: 0.0 + v: 0.0 + w: 0.0 + # energy source initial conditions + - volume: + type: sphere + radius1: 0.0 + radius2: 0.1 + material_id: 0 + den: 1.0 + ie: 0.5 + + velocity: cartesian + u: 0.0 + v: 0.0 + w: 0.0 diff --git a/python/Test-scripts/standard-results/SGH/Sedov_12x12x12/vtk/data/VTK0.vtk b/python/Test-scripts/standard-results/SGH/Sedov_12x12x12/vtk/data/VTK0.vtk new file mode 100644 index 000000000..f5dc9b531 --- /dev/null +++ b/python/Test-scripts/standard-results/SGH/Sedov_12x12x12/vtk/data/VTK0.vtk @@ -0,0 +1,18252 @@ +# vtk DataFile Version 2.0 +Mesh for Fierro +ASCII +DATASET UNSTRUCTURED_GRID + +POINTS 2197 float +0.00000000 0.00000000 0.00000000 +0.53776191 0.00000000 0.00000000 +0.60205274 0.00000000 0.00000000 +0.69594450 0.00000000 0.00000000 +0.77599848 0.00000000 0.00000000 +0.84746937 0.00000000 0.00000000 +0.91204792 0.00000000 0.00000000 +0.96962515 0.00000000 0.00000000 +1.01908611 0.00000000 0.00000000 +1.05975035 0.00000000 0.00000000 +1.10930635 0.00000000 0.00000000 +1.11637892 0.00000000 0.00000000 +1.20000000 0.00000000 0.00000000 +0.00000000 0.53776191 0.00000000 +0.44435254 0.44435254 0.00000000 +0.61155544 0.26939912 0.00000000 +0.69846681 0.22064358 0.00000000 +0.77688181 0.18976175 0.00000000 +0.84748313 0.16967335 0.00000000 +0.91018232 0.15517265 0.00000000 +0.96483786 0.14332892 0.00000000 +1.01357716 0.13287109 0.00000000 +1.05399342 0.12283560 0.00000000 +1.09851927 0.11378558 0.00000000 +1.11184016 0.10152648 0.00000000 +1.20000000 0.10000052 0.00000000 +0.00000000 0.60205274 0.00000000 +0.26939912 0.61155544 0.00000000 +0.54203263 0.54203263 0.00000000 +0.67307734 0.41180871 0.00000000 +0.75701522 0.35501801 0.00000000 +0.82700772 0.31714318 0.00000000 +0.88817242 0.29071456 0.00000000 +0.94191103 0.27030805 0.00000000 +0.98673239 0.25193669 0.00000000 +1.02993988 0.23617460 0.00000000 +1.06148154 0.21674437 0.00000000 +1.10311427 0.20081847 0.00000000 +1.20000000 0.20000021 0.00000000 +0.00000000 0.69594450 0.00000000 +0.22064358 0.69846681 0.00000000 +0.41180871 0.67307734 0.00000000 +0.60391464 0.60391464 0.00000000 +0.71080419 0.50149623 0.00000000 +0.78868751 0.44974512 0.00000000 +0.85270186 0.41230124 0.00000000 +0.90696809 0.38368634 0.00000000 +0.95438813 0.35904395 0.00000000 +0.99927356 0.33668601 0.00000000 +1.02383514 0.30867229 0.00000000 +1.10029021 0.30009933 0.00000000 +1.20000000 0.30000001 0.00000000 +0.00000000 0.77599848 0.00000000 +0.18976175 0.77688181 0.00000000 +0.35501801 0.75701522 0.00000000 +0.50149623 0.71080419 0.00000000 +0.64970821 0.64970821 0.00000000 +0.73928284 0.56858842 0.00000000 +0.81006029 0.52387144 0.00000000 +0.86770511 0.48767065 0.00000000 +0.92067108 0.45894538 0.00000000 +0.95881249 0.42696076 0.00000000 +1.00427903 0.40189866 0.00000000 +1.10000820 0.40000322 0.00000000 +1.20000000 0.40000000 0.00000000 +0.00000000 0.84746937 0.00000000 +0.16967335 0.84748313 0.00000000 +0.31714318 0.82700772 0.00000000 +0.44974512 0.78868751 0.00000000 +0.56858842 0.73928284 0.00000000 +0.68717236 0.68717236 0.00000000 +0.76182433 0.62063065 0.00000000 +0.82630414 0.58266570 0.00000000 +0.87929698 0.54716000 0.00000000 +0.91747243 0.50972929 0.00000000 +1.00030406 0.50015749 0.00000000 +1.10000004 0.50000002 0.00000000 +1.20000000 0.50000000 0.00000000 +0.00000000 0.91204792 0.00000000 +0.15517265 0.91018232 0.00000000 +0.29071456 0.88817242 0.00000000 +0.41230124 0.85270186 0.00000000 +0.52387144 0.81006029 0.00000000 +0.62063065 0.76182433 0.00000000 +0.71717747 0.71717747 0.00000000 +0.78380908 0.66428258 0.00000000 +0.82986250 0.62120277 0.00000000 +0.90158256 0.60103535 0.00000000 +1.00000345 0.60000199 0.00000000 +1.10000000 0.60000000 0.00000000 +1.20000000 0.60000000 0.00000000 +0.00000000 0.96962515 0.00000000 +0.14332892 0.96483786 0.00000000 +0.27030805 0.94191103 0.00000000 +0.38368634 0.90696809 0.00000000 +0.48767065 0.86770511 0.00000000 +0.58266570 0.82630414 0.00000000 +0.66428258 0.78380908 0.00000000 +0.73913964 0.73913964 0.00000000 +0.80329722 0.70269171 0.00000000 +0.90002831 0.70002079 0.00000000 +1.00000000 0.70000000 0.00000000 +1.10000000 0.70000000 0.00000000 +1.20000000 0.70000000 0.00000000 +0.00000000 1.01908611 0.00000000 +0.13287109 1.01357716 0.00000000 +0.25193669 0.98673239 0.00000000 +0.35904395 0.95438813 0.00000000 +0.45894538 0.92067108 0.00000000 +0.54716000 0.87929698 0.00000000 +0.62120277 0.82986250 0.00000000 +0.70269171 0.80329722 0.00000000 +0.80011294 0.80011294 0.00000000 +0.90000003 0.80000002 0.00000000 +1.00000000 0.80000000 0.00000000 +1.10000000 0.80000000 0.00000000 +1.20000000 0.80000000 0.00000000 +0.00000000 1.05975035 0.00000000 +0.12283560 1.05399342 0.00000000 +0.23617460 1.02993988 0.00000000 +0.33668601 0.99927356 0.00000000 +0.42696076 0.95881249 0.00000000 +0.50972929 0.91747243 0.00000000 +0.60103535 0.90158256 0.00000000 +0.70002079 0.90002831 0.00000000 +0.80000002 0.90000003 0.00000000 +0.90000000 0.90000000 0.00000000 +1.00000000 0.90000000 0.00000000 +1.10000000 0.90000000 0.00000000 +1.20000000 0.90000000 0.00000000 +0.00000000 1.10930635 0.00000000 +0.11378558 1.09851927 0.00000000 +0.21674437 1.06148154 0.00000000 +0.30867229 1.02383514 0.00000000 +0.40189866 1.00427903 0.00000000 +0.50015749 1.00030406 0.00000000 +0.60000199 1.00000345 0.00000000 +0.70000000 1.00000000 0.00000000 +0.80000000 1.00000000 0.00000000 +0.90000000 1.00000000 0.00000000 +1.00000000 1.00000000 0.00000000 +1.10000000 1.00000000 0.00000000 +1.20000000 1.00000000 0.00000000 +0.00000000 1.11637892 0.00000000 +0.10152648 1.11184016 0.00000000 +0.20081847 1.10311427 0.00000000 +0.30009933 1.10029021 0.00000000 +0.40000322 1.10000820 0.00000000 +0.50000002 1.10000004 0.00000000 +0.60000000 1.10000000 0.00000000 +0.70000000 1.10000000 0.00000000 +0.80000000 1.10000000 0.00000000 +0.90000000 1.10000000 0.00000000 +1.00000000 1.10000000 0.00000000 +1.10000000 1.10000000 0.00000000 +1.20000000 1.10000000 0.00000000 +0.00000000 1.20000000 0.00000000 +0.10000052 1.20000000 0.00000000 +0.20000021 1.20000000 0.00000000 +0.30000001 1.20000000 0.00000000 +0.40000000 1.20000000 0.00000000 +0.50000000 1.20000000 0.00000000 +0.60000000 1.20000000 0.00000000 +0.70000000 1.20000000 0.00000000 +0.80000000 1.20000000 0.00000000 +0.90000000 1.20000000 0.00000000 +1.00000000 1.20000000 0.00000000 +1.10000000 1.20000000 0.00000000 +1.20000000 1.20000000 0.00000000 +0.00000000 0.00000000 0.53776191 +0.44435254 0.00000000 0.44435254 +0.61155544 0.00000000 0.26939912 +0.69846681 0.00000000 0.22064358 +0.77688181 0.00000000 0.18976175 +0.84748313 0.00000000 0.16967335 +0.91018232 0.00000000 0.15517265 +0.96483786 0.00000000 0.14332892 +1.01357716 0.00000000 0.13287109 +1.05399342 0.00000000 0.12283560 +1.09851927 0.00000000 0.11378558 +1.11184016 0.00000000 0.10152648 +1.20000000 0.00000000 0.10000052 +0.00000000 0.44435254 0.44435254 +0.40158516 0.40158516 0.40158516 +0.60529265 0.27963723 0.27963723 +0.69636507 0.21528261 0.21528261 +0.77402488 0.18567451 0.18567451 +0.84371201 0.16592356 0.16592356 +0.90540812 0.15186355 0.15186355 +0.95897612 0.14058988 0.14058988 +1.00656530 0.13066508 0.13066508 +1.04745861 0.12143171 0.12143171 +1.08708267 0.11202892 0.11202892 +1.10834589 0.10110022 0.10110022 +1.20000000 0.10000032 0.10000032 +0.00000000 0.61155544 0.26939912 +0.27963723 0.60529265 0.27963723 +0.52500961 0.52500961 0.25920378 +0.66381197 0.41240417 0.21097409 +0.74994845 0.35105276 0.17683110 +0.82057257 0.31369245 0.15782758 +0.88178372 0.28735387 0.14416070 +0.93524877 0.26714900 0.13370188 +0.98023176 0.24916134 0.12455660 +1.02428904 0.23383545 0.11668098 +1.05283153 0.21405368 0.10675696 +1.10216182 0.20056038 0.10028399 +1.20000000 0.20000013 0.10000006 +0.00000000 0.69846681 0.22064358 +0.21528261 0.69636507 0.21528261 +0.41240417 0.66381197 0.21097409 +0.59221847 0.59221847 0.19712823 +0.70343685 0.50080222 0.17093147 +0.78235981 0.44678136 0.15018738 +0.84712657 0.40975168 0.13725219 +0.90155981 0.38108807 0.12715273 +0.94953048 0.35679227 0.11879272 +0.99386503 0.33427106 0.11124799 +1.01968452 0.30706878 0.10223985 +1.10020126 0.30006789 0.10002269 +1.20000000 0.30000000 0.10000000 +0.00000000 0.77688181 0.18976175 +0.18567451 0.77402488 0.18567451 +0.35105276 0.74994845 0.17683110 +0.50080222 0.70343685 0.17093147 +0.64119052 0.64119052 0.16128471 +0.73369463 0.56719040 0.14458578 +0.80467853 0.52099842 0.13108583 +0.86286900 0.48524650 0.12165362 +0.91652877 0.45674548 0.11417539 +0.95313141 0.42424467 0.10595209 +1.00335672 0.40148402 0.10037558 +1.10000555 0.40000217 0.10000050 +1.20000000 0.40000000 0.10000000 +0.00000000 0.84748313 0.16967335 +0.16592356 0.84371201 0.16592356 +0.31369245 0.82057257 0.15782758 +0.44678136 0.78235981 0.15018738 +0.56719040 0.73369463 0.14458578 +0.68053012 0.68053012 0.13738574 +0.75739272 0.61883699 0.12555538 +0.82242886 0.58018583 0.11646131 +0.87441298 0.54424669 0.10890351 +0.91472437 0.50818514 0.10161500 +1.00022657 0.50011671 0.10002438 +1.10000003 0.50000001 0.10000000 +1.20000000 0.50000000 0.10000000 +0.00000000 0.91018232 0.15517265 +0.15186355 0.90540812 0.15186355 +0.28735387 0.88178372 0.14416070 +0.40975168 0.84712657 0.13725219 +0.52099842 0.80467853 0.13108583 +0.61883699 0.75739272 0.12555538 +0.71254499 0.71254499 0.11987896 +0.78003954 0.66232491 0.11125308 +0.82642640 0.61873750 0.10310482 +0.90125052 0.60081694 0.10014389 +1.00000241 0.60000138 0.10000021 +1.10000000 0.60000000 0.10000000 +1.20000000 0.60000000 0.10000000 +0.00000000 0.96483786 0.14332892 +0.14058988 0.95897612 0.14058988 +0.26714900 0.93524877 0.13370188 +0.38108807 0.90155981 0.12715273 +0.48524650 0.86286900 0.12165362 +0.58018583 0.82242886 0.11646131 +0.66232491 0.78003954 0.11125308 +0.73398435 0.73398435 0.10513418 +0.80271483 0.70223247 0.10034158 +0.90002082 0.70001527 0.10000221 +1.00000000 0.70000000 0.10000000 +1.10000000 0.70000000 0.10000000 +1.20000000 0.70000000 0.10000000 +0.00000000 1.01357716 0.13287109 +0.13066508 1.00656530 0.13066508 +0.24916134 0.98023176 0.12455660 +0.35679227 0.94953048 0.11879272 +0.45674548 0.91652877 0.11417539 +0.54424669 0.87441298 0.10890351 +0.61873750 0.82642640 0.10310482 +0.70223247 0.80271483 0.10034158 +0.80007988 0.80007988 0.10001129 +0.90000002 0.80000002 0.10000000 +1.00000000 0.80000000 0.10000000 +1.10000000 0.80000000 0.10000000 +1.20000000 0.80000000 0.10000000 +0.00000000 1.05399342 0.12283560 +0.12143171 1.04745861 0.12143171 +0.23383545 1.02428904 0.11668098 +0.33427106 0.99386503 0.11124799 +0.42424467 0.95313141 0.10595209 +0.50818514 0.91472437 0.10161500 +0.60081694 0.90125052 0.10014389 +0.70001527 0.90002082 0.10000221 +0.80000002 0.90000002 0.10000000 +0.90000000 0.90000000 0.10000000 +1.00000000 0.90000000 0.10000000 +1.10000000 0.90000000 0.10000000 +1.20000000 0.90000000 0.10000000 +0.00000000 1.09851927 0.11378558 +0.11202892 1.08708267 0.11202892 +0.21405368 1.05283153 0.10675696 +0.30706878 1.01968452 0.10223985 +0.40148402 1.00335672 0.10037558 +0.50011671 1.00022657 0.10002438 +0.60000138 1.00000241 0.10000021 +0.70000000 1.00000000 0.10000000 +0.80000000 1.00000000 0.10000000 +0.90000000 1.00000000 0.10000000 +1.00000000 1.00000000 0.10000000 +1.10000000 1.00000000 0.10000000 +1.20000000 1.00000000 0.10000000 +0.00000000 1.11184016 0.10152648 +0.10110022 1.10834589 0.10110022 +0.20056038 1.10216182 0.10028399 +0.30006789 1.10020126 0.10002269 +0.40000217 1.10000555 0.10000050 +0.50000001 1.10000003 0.10000000 +0.60000000 1.10000000 0.10000000 +0.70000000 1.10000000 0.10000000 +0.80000000 1.10000000 0.10000000 +0.90000000 1.10000000 0.10000000 +1.00000000 1.10000000 0.10000000 +1.10000000 1.10000000 0.10000000 +1.20000000 1.10000000 0.10000000 +0.00000000 1.20000000 0.10000052 +0.10000032 1.20000000 0.10000032 +0.20000013 1.20000000 0.10000006 +0.30000000 1.20000000 0.10000000 +0.40000000 1.20000000 0.10000000 +0.50000000 1.20000000 0.10000000 +0.60000000 1.20000000 0.10000000 +0.70000000 1.20000000 0.10000000 +0.80000000 1.20000000 0.10000000 +0.90000000 1.20000000 0.10000000 +1.00000000 1.20000000 0.10000000 +1.10000000 1.20000000 0.10000000 +1.20000000 1.20000000 0.10000000 +0.00000000 0.00000000 0.60205274 +0.26939912 0.00000000 0.61155544 +0.54203263 0.00000000 0.54203263 +0.67307734 0.00000000 0.41180871 +0.75701522 0.00000000 0.35501801 +0.82700772 0.00000000 0.31714318 +0.88817242 0.00000000 0.29071456 +0.94191103 0.00000000 0.27030805 +0.98673239 0.00000000 0.25193669 +1.02993988 0.00000000 0.23617460 +1.06148154 0.00000000 0.21674437 +1.10311427 0.00000000 0.20081847 +1.20000000 0.00000000 0.20000021 +0.00000000 0.26939912 0.61155544 +0.27963723 0.27963723 0.60529265 +0.52500961 0.25920378 0.52500961 +0.66381197 0.21097409 0.41240417 +0.74994845 0.17683110 0.35105276 +0.82057257 0.15782758 0.31369245 +0.88178372 0.14416070 0.28735387 +0.93524877 0.13370188 0.26714900 +0.98023176 0.12455660 0.24916134 +1.02428904 0.11668098 0.23383545 +1.05283153 0.10675696 0.21405368 +1.10216182 0.10028399 0.20056038 +1.20000000 0.10000006 0.20000013 +0.00000000 0.54203263 0.54203263 +0.25920378 0.52500961 0.52500961 +0.46801317 0.46801317 0.46801317 +0.62471435 0.40259738 0.40259738 +0.72359708 0.34252623 0.34252623 +0.79895918 0.30473274 0.30473274 +0.86136115 0.27879357 0.27879357 +0.91526121 0.25903270 0.25903270 +0.96161127 0.24190248 0.24190248 +1.00671014 0.22701033 0.22701033 +1.03110077 0.20766246 0.20766246 +1.10062140 0.20015072 0.20015072 +1.20000000 0.20000002 0.20000002 +0.00000000 0.67307734 0.41180871 +0.21097409 0.66381197 0.41240417 +0.40259738 0.62471435 0.40259738 +0.55777243 0.55777243 0.37130158 +0.67799702 0.49245949 0.33036339 +0.76188336 0.43867714 0.29400223 +0.82950195 0.40204041 0.26861035 +0.88510792 0.37365783 0.24924309 +0.93512570 0.35040098 0.23356272 +0.97657491 0.32704436 0.21784235 +1.01034139 0.30359058 0.20235762 +1.10006116 0.30001958 0.20001291 +1.20000000 0.30000000 0.20000000 +0.00000000 0.75701522 0.35501801 +0.17683110 0.74994845 0.35105276 +0.34252623 0.72359708 0.34252623 +0.49245949 0.67799702 0.33036339 +0.61746997 0.61746997 0.30925925 +0.71543727 0.55957630 0.28223638 +0.78816087 0.51279397 0.25768821 +0.84899828 0.47825791 0.23944822 +0.90311681 0.44979284 0.22492296 +0.93777007 0.41704032 0.20834384 +1.00154830 0.40067510 0.20034203 +1.10000159 0.40000060 0.20000028 +1.20000000 0.40000000 0.20000000 +0.00000000 0.82700772 0.31714318 +0.15782758 0.82057257 0.31369245 +0.30473274 0.79895918 0.30473274 +0.43867714 0.76188336 0.29400223 +0.55957630 0.71543727 0.28223638 +0.66254081 0.66254081 0.26630201 +0.74412487 0.61203591 0.24659868 +0.81100840 0.57321991 0.22993157 +0.85996942 0.53560066 0.21419343 +0.90841516 0.50466014 0.20186981 +1.00008969 0.50004540 0.20001849 +1.10000001 0.50000000 0.20000000 +1.20000000 0.50000000 0.20000000 +0.00000000 0.88817242 0.29071456 +0.14416070 0.88178372 0.28735387 +0.27879357 0.86136115 0.27879357 +0.40204041 0.82950195 0.26861035 +0.51279397 0.78816087 0.25768821 +0.61203591 0.74412487 0.24659868 +0.69988798 0.69988798 0.23467848 +0.76772981 0.65451792 0.21891426 +0.81767238 0.61254377 0.20417332 +0.90059787 0.60038862 0.20013549 +1.00000077 0.60000044 0.20000014 +1.10000000 0.60000000 0.20000000 +1.20000000 0.60000000 0.20000000 +0.00000000 0.94191103 0.27030805 +0.13370188 0.93524877 0.26714900 +0.25903270 0.91526121 0.25903270 +0.37365783 0.88510792 0.24924309 +0.47825791 0.84899828 0.23944822 +0.57321991 0.81100840 0.22993157 +0.65451792 0.76772981 0.21891426 +0.72157041 0.72157041 0.20635303 +0.80144842 0.70121158 0.20037105 +0.90000795 0.70000581 0.20000164 +1.00000000 0.70000000 0.20000000 +1.10000000 0.70000000 0.20000000 +1.20000000 0.70000000 0.20000000 +0.00000000 0.98673239 0.25193669 +0.12455660 0.98023176 0.24916134 +0.24190248 0.96161127 0.24190248 +0.35040098 0.93512570 0.23356272 +0.44979284 0.90311681 0.22492296 +0.53560066 0.85996942 0.21419343 +0.61254377 0.81767238 0.20417332 +0.70121158 0.80144842 0.20037105 +0.80002681 0.80002681 0.20000706 +0.90000001 0.80000001 0.20000000 +1.00000000 0.80000000 0.20000000 +1.10000000 0.80000000 0.20000000 +1.20000000 0.80000000 0.20000000 +0.00000000 1.02993988 0.23617460 +0.11668098 1.02428904 0.23383545 +0.22701033 1.00671014 0.22701033 +0.32704436 0.97657491 0.21784235 +0.41704032 0.93777007 0.20834384 +0.50466014 0.90841516 0.20186981 +0.60038862 0.90059787 0.20013549 +0.70000581 0.90000795 0.20000164 +0.80000001 0.90000001 0.20000000 +0.90000000 0.90000000 0.20000000 +1.00000000 0.90000000 0.20000000 +1.10000000 0.90000000 0.20000000 +1.20000000 0.90000000 0.20000000 +0.00000000 1.06148154 0.21674437 +0.10675696 1.05283153 0.21405368 +0.20766246 1.03110077 0.20766246 +0.30359058 1.01034139 0.20235762 +0.40067510 1.00154830 0.20034203 +0.50004540 1.00008969 0.20001849 +0.60000044 1.00000077 0.20000014 +0.70000000 1.00000000 0.20000000 +0.80000000 1.00000000 0.20000000 +0.90000000 1.00000000 0.20000000 +1.00000000 1.00000000 0.20000000 +1.10000000 1.00000000 0.20000000 +1.20000000 1.00000000 0.20000000 +0.00000000 1.10311427 0.20081847 +0.10028399 1.10216182 0.20056038 +0.20015072 1.10062140 0.20015072 +0.30001958 1.10006116 0.20001291 +0.40000060 1.10000159 0.20000028 +0.50000000 1.10000001 0.20000000 +0.60000000 1.10000000 0.20000000 +0.70000000 1.10000000 0.20000000 +0.80000000 1.10000000 0.20000000 +0.90000000 1.10000000 0.20000000 +1.00000000 1.10000000 0.20000000 +1.10000000 1.10000000 0.20000000 +1.20000000 1.10000000 0.20000000 +0.00000000 1.20000000 0.20000021 +0.10000006 1.20000000 0.20000013 +0.20000002 1.20000000 0.20000002 +0.30000000 1.20000000 0.20000000 +0.40000000 1.20000000 0.20000000 +0.50000000 1.20000000 0.20000000 +0.60000000 1.20000000 0.20000000 +0.70000000 1.20000000 0.20000000 +0.80000000 1.20000000 0.20000000 +0.90000000 1.20000000 0.20000000 +1.00000000 1.20000000 0.20000000 +1.10000000 1.20000000 0.20000000 +1.20000000 1.20000000 0.20000000 +0.00000000 0.00000000 0.69594450 +0.22064358 0.00000000 0.69846681 +0.41180871 0.00000000 0.67307734 +0.60391464 0.00000000 0.60391464 +0.71080419 0.00000000 0.50149623 +0.78868751 0.00000000 0.44974512 +0.85270186 0.00000000 0.41230124 +0.90696809 0.00000000 0.38368634 +0.95438813 0.00000000 0.35904395 +0.99927356 0.00000000 0.33668601 +1.02383514 0.00000000 0.30867229 +1.10029021 0.00000000 0.30009933 +1.20000000 0.00000000 0.30000001 +0.00000000 0.22064358 0.69846681 +0.21528261 0.21528261 0.69636507 +0.41240417 0.21097409 0.66381197 +0.59221847 0.19712823 0.59221847 +0.70343685 0.17093147 0.50080222 +0.78235981 0.15018738 0.44678136 +0.84712657 0.13725219 0.40975168 +0.90155981 0.12715273 0.38108807 +0.94953048 0.11879272 0.35679227 +0.99386503 0.11124799 0.33427106 +1.01968452 0.10223985 0.30706878 +1.10020126 0.10002269 0.30006789 +1.20000000 0.10000000 0.30000000 +0.00000000 0.41180871 0.67307734 +0.21097409 0.41240417 0.66381197 +0.40259738 0.40259738 0.62471435 +0.55777243 0.37130158 0.55777243 +0.67799702 0.33036339 0.49245949 +0.76188336 0.29400223 0.43867714 +0.82950195 0.26861035 0.40204041 +0.88510792 0.24924309 0.37365783 +0.93512570 0.23356272 0.35040098 +0.97657491 0.21784235 0.32704436 +1.01034139 0.20235762 0.30359058 +1.10006116 0.20001291 0.30001958 +1.20000000 0.20000000 0.30000000 +0.00000000 0.60391464 0.60391464 +0.19712823 0.59221847 0.59221847 +0.37130158 0.55777243 0.55777243 +0.51573703 0.51573703 0.51573703 +0.63468021 0.47026829 0.47026829 +0.72882359 0.42634679 0.42634679 +0.80031590 0.39037523 0.39037523 +0.85896581 0.36286548 0.36286548 +0.91301195 0.34124155 0.34124155 +0.94805255 0.31627253 0.31627253 +1.00287541 0.30096251 0.30096251 +1.10000642 0.30000187 0.30000187 +1.20000000 0.30000000 0.30000000 +0.00000000 0.71080419 0.50149623 +0.17093147 0.70343685 0.50080222 +0.33036339 0.67799702 0.49245949 +0.47026829 0.63468021 0.47026829 +0.58727758 0.58727758 0.43999065 +0.68544422 0.54231789 0.40765797 +0.76328972 0.50064654 0.37623684 +0.82731919 0.46741977 0.35081514 +0.87928885 0.43790516 0.32839894 +0.91846729 0.40823181 0.30610938 +1.00037561 0.40015870 0.30011969 +1.10000015 0.40000005 0.30000004 +1.20000000 0.40000000 0.30000000 +0.00000000 0.78868751 0.44974512 +0.15018738 0.78235981 0.44678136 +0.29400223 0.76188336 0.43867714 +0.42634679 0.72882359 0.42634679 +0.54231789 0.68544422 0.40765797 +0.63909979 0.63909979 0.38385260 +0.72252282 0.59824110 0.36006378 +0.79135631 0.56136548 0.33729954 +0.83815065 0.52260788 0.31343113 +0.90288377 0.50158998 0.30096622 +1.00001657 0.50000815 0.30000484 +1.10000000 0.50000000 0.30000000 +1.20000000 0.50000000 0.30000000 +0.00000000 0.85270186 0.41230124 +0.13725219 0.84712657 0.40975168 +0.26861035 0.82950195 0.40204041 +0.39037523 0.80031590 0.39037523 +0.50064654 0.76328972 0.37623684 +0.59824110 0.72252282 0.36006378 +0.68192803 0.68192803 0.34163275 +0.74630639 0.63839074 0.31941918 +0.80785710 0.60564149 0.30285894 +0.90015817 0.60010177 0.30005217 +1.00000009 0.60000005 0.30000003 +1.10000000 0.60000000 0.30000000 +1.20000000 0.60000000 0.30000000 +0.00000000 0.90696809 0.38368634 +0.12715273 0.90155981 0.38108807 +0.24924309 0.88510792 0.37365783 +0.36286548 0.85896581 0.36286548 +0.46741977 0.82731919 0.35081514 +0.56136548 0.79135631 0.33729954 +0.63839074 0.74630639 0.31941918 +0.70939520 0.70939520 0.30409247 +0.80044005 0.70037406 0.30016913 +0.90000138 0.70000101 0.30000042 +1.00000000 0.70000000 0.30000000 +1.10000000 0.70000000 0.30000000 +1.20000000 0.70000000 0.30000000 +0.00000000 0.95438813 0.35904395 +0.11879272 0.94953048 0.35679227 +0.23356272 0.93512570 0.35040098 +0.34124155 0.91301195 0.34124155 +0.43790516 0.87928885 0.32839894 +0.52260788 0.83815065 0.31343113 +0.60564149 0.80785710 0.30285894 +0.70037406 0.80044005 0.30016913 +0.80000384 0.80000384 0.30000142 +0.90000000 0.80000000 0.30000000 +1.00000000 0.80000000 0.30000000 +1.10000000 0.80000000 0.30000000 +1.20000000 0.80000000 0.30000000 +0.00000000 0.99927356 0.33668601 +0.11124799 0.99386503 0.33427106 +0.21784235 0.97657491 0.32704436 +0.31627253 0.94805255 0.31627253 +0.40823181 0.91846729 0.30610938 +0.50158998 0.90288377 0.30096622 +0.60010177 0.90015817 0.30005217 +0.70000101 0.90000138 0.30000042 +0.80000000 0.90000000 0.30000000 +0.90000000 0.90000000 0.30000000 +1.00000000 0.90000000 0.30000000 +1.10000000 0.90000000 0.30000000 +1.20000000 0.90000000 0.30000000 +0.00000000 1.02383514 0.30867229 +0.10223985 1.01968452 0.30706878 +0.20235762 1.01034139 0.30359058 +0.30096251 1.00287541 0.30096251 +0.40015870 1.00037561 0.30011969 +0.50000815 1.00001657 0.30000484 +0.60000005 1.00000009 0.30000003 +0.70000000 1.00000000 0.30000000 +0.80000000 1.00000000 0.30000000 +0.90000000 1.00000000 0.30000000 +1.00000000 1.00000000 0.30000000 +1.10000000 1.00000000 0.30000000 +1.20000000 1.00000000 0.30000000 +0.00000000 1.10029021 0.30009933 +0.10002269 1.10020126 0.30006789 +0.20001291 1.10006116 0.30001958 +0.30000187 1.10000642 0.30000187 +0.40000005 1.10000015 0.30000004 +0.50000000 1.10000000 0.30000000 +0.60000000 1.10000000 0.30000000 +0.70000000 1.10000000 0.30000000 +0.80000000 1.10000000 0.30000000 +0.90000000 1.10000000 0.30000000 +1.00000000 1.10000000 0.30000000 +1.10000000 1.10000000 0.30000000 +1.20000000 1.10000000 0.30000000 +0.00000000 1.20000000 0.30000001 +0.10000000 1.20000000 0.30000000 +0.20000000 1.20000000 0.30000000 +0.30000000 1.20000000 0.30000000 +0.40000000 1.20000000 0.30000000 +0.50000000 1.20000000 0.30000000 +0.60000000 1.20000000 0.30000000 +0.70000000 1.20000000 0.30000000 +0.80000000 1.20000000 0.30000000 +0.90000000 1.20000000 0.30000000 +1.00000000 1.20000000 0.30000000 +1.10000000 1.20000000 0.30000000 +1.20000000 1.20000000 0.30000000 +0.00000000 0.00000000 0.77599848 +0.18976175 0.00000000 0.77688181 +0.35501801 0.00000000 0.75701522 +0.50149623 0.00000000 0.71080419 +0.64970821 0.00000000 0.64970821 +0.73928284 0.00000000 0.56858842 +0.81006029 0.00000000 0.52387144 +0.86770511 0.00000000 0.48767065 +0.92067108 0.00000000 0.45894538 +0.95881249 0.00000000 0.42696076 +1.00427903 0.00000000 0.40189866 +1.10000820 0.00000000 0.40000322 +1.20000000 0.00000000 0.40000000 +0.00000000 0.18976175 0.77688181 +0.18567451 0.18567451 0.77402488 +0.35105276 0.17683110 0.74994845 +0.50080222 0.17093147 0.70343685 +0.64119052 0.16128471 0.64119052 +0.73369463 0.14458578 0.56719040 +0.80467853 0.13108583 0.52099842 +0.86286900 0.12165362 0.48524650 +0.91652877 0.11417539 0.45674548 +0.95313141 0.10595209 0.42424467 +1.00335672 0.10037558 0.40148402 +1.10000555 0.10000050 0.40000217 +1.20000000 0.10000000 0.40000000 +0.00000000 0.35501801 0.75701522 +0.17683110 0.35105276 0.74994845 +0.34252623 0.34252623 0.72359708 +0.49245949 0.33036339 0.67799702 +0.61746997 0.30925925 0.61746997 +0.71543727 0.28223638 0.55957630 +0.78816087 0.25768821 0.51279397 +0.84899828 0.23944822 0.47825791 +0.90311681 0.22492296 0.44979284 +0.93777007 0.20834384 0.41704032 +1.00154830 0.20034203 0.40067510 +1.10000159 0.20000028 0.40000060 +1.20000000 0.20000000 0.40000000 +0.00000000 0.50149623 0.71080419 +0.17093147 0.50080222 0.70343685 +0.33036339 0.49245949 0.67799702 +0.47026829 0.47026829 0.63468021 +0.58727758 0.43999065 0.58727758 +0.68544422 0.40765797 0.54231789 +0.76328972 0.37623684 0.50064654 +0.82731919 0.35081514 0.46741977 +0.87928885 0.32839894 0.43790516 +0.91846729 0.30610938 0.40823181 +1.00037561 0.30011969 0.40015870 +1.10000015 0.30000004 0.40000005 +1.20000000 0.30000000 0.40000000 +0.00000000 0.64970821 0.64970821 +0.16128471 0.64119052 0.64119052 +0.30925925 0.61746997 0.61746997 +0.43999065 0.58727758 0.58727758 +0.55432823 0.55432823 0.55432823 +0.64933117 0.51735904 0.51735904 +0.73125472 0.48386966 0.48386966 +0.80015564 0.45385923 0.45385923 +0.84633911 0.42186536 0.42186536 +0.90474355 0.40210769 0.40210769 +1.00003915 0.40001565 0.40001565 +1.10000000 0.40000000 0.40000000 +1.20000000 0.40000000 0.40000000 +0.00000000 0.73928284 0.56858842 +0.14458578 0.73369463 0.56719040 +0.28223638 0.71543727 0.55957630 +0.40765797 0.68544422 0.54231789 +0.51735904 0.64933117 0.51735904 +0.61430584 0.61430584 0.49101416 +0.69689697 0.57911331 0.46343371 +0.76247824 0.54287421 0.43423088 +0.81599699 0.50953910 0.40759358 +0.90052141 0.50028459 0.40022936 +1.00000107 0.50000051 0.40000040 +1.10000000 0.50000000 0.40000000 +1.20000000 0.50000000 0.40000000 +0.00000000 0.81006029 0.52387144 +0.13108583 0.80467853 0.52099842 +0.25768821 0.78816087 0.51279397 +0.37623684 0.76328972 0.50064654 +0.48386966 0.73125472 0.48386966 +0.57911331 0.69689697 0.46343371 +0.65872595 0.65872595 0.43876962 +0.72246813 0.61888951 0.41248886 +0.80192020 0.60140266 0.40095016 +0.90001954 0.60001237 0.40000825 +1.00000000 0.60000000 0.40000000 +1.10000000 0.60000000 0.40000000 +1.20000000 0.60000000 0.40000000 +0.00000000 0.86770511 0.48767065 +0.12165362 0.86286900 0.48524650 +0.23944822 0.84899828 0.47825791 +0.35081514 0.82731919 0.46741977 +0.45385923 0.80015564 0.45385923 +0.54287421 0.76247824 0.43423088 +0.61888951 0.72246813 0.41248886 +0.70244509 0.70244509 0.40141285 +0.80006445 0.70005521 0.40003218 +0.90000008 0.70000006 0.40000003 +1.00000000 0.70000000 0.40000000 +1.10000000 0.70000000 0.40000000 +1.20000000 0.70000000 0.40000000 +0.00000000 0.92067108 0.45894538 +0.11417539 0.91652877 0.45674548 +0.22492296 0.90311681 0.44979284 +0.32839894 0.87928885 0.43790516 +0.42186536 0.84633911 0.42186536 +0.50953910 0.81599699 0.40759358 +0.60140266 0.80192020 0.40095016 +0.70005521 0.80006445 0.40003218 +0.80000021 0.80000021 0.40000010 +0.90000000 0.80000000 0.40000000 +1.00000000 0.80000000 0.40000000 +1.10000000 0.80000000 0.40000000 +1.20000000 0.80000000 0.40000000 +0.00000000 0.95881249 0.42696076 +0.10595209 0.95313141 0.42424467 +0.20834384 0.93777007 0.41704032 +0.30610938 0.91846729 0.40823181 +0.40210769 0.90474355 0.40210769 +0.50028459 0.90052141 0.40022936 +0.60001237 0.90001954 0.40000825 +0.70000006 0.90000008 0.40000003 +0.80000000 0.90000000 0.40000000 +0.90000000 0.90000000 0.40000000 +1.00000000 0.90000000 0.40000000 +1.10000000 0.90000000 0.40000000 +1.20000000 0.90000000 0.40000000 +0.00000000 1.00427903 0.40189866 +0.10037558 1.00335672 0.40148402 +0.20034203 1.00154830 0.40067510 +0.30011969 1.00037561 0.40015870 +0.40001565 1.00003915 0.40001565 +0.50000051 1.00000107 0.40000040 +0.60000000 1.00000000 0.40000000 +0.70000000 1.00000000 0.40000000 +0.80000000 1.00000000 0.40000000 +0.90000000 1.00000000 0.40000000 +1.00000000 1.00000000 0.40000000 +1.10000000 1.00000000 0.40000000 +1.20000000 1.00000000 0.40000000 +0.00000000 1.10000820 0.40000322 +0.10000050 1.10000555 0.40000217 +0.20000028 1.10000159 0.40000060 +0.30000004 1.10000015 0.40000005 +0.40000000 1.10000000 0.40000000 +0.50000000 1.10000000 0.40000000 +0.60000000 1.10000000 0.40000000 +0.70000000 1.10000000 0.40000000 +0.80000000 1.10000000 0.40000000 +0.90000000 1.10000000 0.40000000 +1.00000000 1.10000000 0.40000000 +1.10000000 1.10000000 0.40000000 +1.20000000 1.10000000 0.40000000 +0.00000000 1.20000000 0.40000000 +0.10000000 1.20000000 0.40000000 +0.20000000 1.20000000 0.40000000 +0.30000000 1.20000000 0.40000000 +0.40000000 1.20000000 0.40000000 +0.50000000 1.20000000 0.40000000 +0.60000000 1.20000000 0.40000000 +0.70000000 1.20000000 0.40000000 +0.80000000 1.20000000 0.40000000 +0.90000000 1.20000000 0.40000000 +1.00000000 1.20000000 0.40000000 +1.10000000 1.20000000 0.40000000 +1.20000000 1.20000000 0.40000000 +0.00000000 0.00000000 0.84746937 +0.16967335 0.00000000 0.84748313 +0.31714318 0.00000000 0.82700772 +0.44974512 0.00000000 0.78868751 +0.56858842 0.00000000 0.73928284 +0.68717236 0.00000000 0.68717236 +0.76182433 0.00000000 0.62063065 +0.82630414 0.00000000 0.58266570 +0.87929698 0.00000000 0.54716000 +0.91747243 0.00000000 0.50972929 +1.00030406 0.00000000 0.50015749 +1.10000004 0.00000000 0.50000002 +1.20000000 0.00000000 0.50000000 +0.00000000 0.16967335 0.84748313 +0.16592356 0.16592356 0.84371201 +0.31369245 0.15782758 0.82057257 +0.44678136 0.15018738 0.78235981 +0.56719040 0.14458578 0.73369463 +0.68053012 0.13738574 0.68053012 +0.75739272 0.12555538 0.61883699 +0.82242886 0.11646131 0.58018583 +0.87441298 0.10890351 0.54424669 +0.91472437 0.10161500 0.50818514 +1.00022657 0.10002438 0.50011671 +1.10000003 0.10000000 0.50000001 +1.20000000 0.10000000 0.50000000 +0.00000000 0.31714318 0.82700772 +0.15782758 0.31369245 0.82057257 +0.30473274 0.30473274 0.79895918 +0.43867714 0.29400223 0.76188336 +0.55957630 0.28223638 0.71543727 +0.66254081 0.26630201 0.66254081 +0.74412487 0.24659868 0.61203591 +0.81100840 0.22993157 0.57321991 +0.85996942 0.21419343 0.53560066 +0.90841516 0.20186981 0.50466014 +1.00008969 0.20001849 0.50004540 +1.10000001 0.20000000 0.50000000 +1.20000000 0.20000000 0.50000000 +0.00000000 0.44974512 0.78868751 +0.15018738 0.44678136 0.78235981 +0.29400223 0.43867714 0.76188336 +0.42634679 0.42634679 0.72882359 +0.54231789 0.40765797 0.68544422 +0.63909979 0.38385260 0.63909979 +0.72252282 0.36006378 0.59824110 +0.79135631 0.33729954 0.56136548 +0.83815065 0.31343113 0.52260788 +0.90288377 0.30096622 0.50158998 +1.00001657 0.30000484 0.50000815 +1.10000000 0.30000000 0.50000000 +1.20000000 0.30000000 0.50000000 +0.00000000 0.56858842 0.73928284 +0.14458578 0.56719040 0.73369463 +0.28223638 0.55957630 0.71543727 +0.40765797 0.54231789 0.68544422 +0.51735904 0.51735904 0.64933117 +0.61430584 0.49101416 0.61430584 +0.69689697 0.46343371 0.57911331 +0.76247824 0.43423088 0.54287421 +0.81599699 0.40759358 0.50953910 +0.90052141 0.40022936 0.50028459 +1.00000107 0.40000040 0.50000051 +1.10000000 0.40000000 0.50000000 +1.20000000 0.40000000 0.50000000 +0.00000000 0.68717236 0.68717236 +0.13738574 0.68053012 0.68053012 +0.26630201 0.66254081 0.66254081 +0.38385260 0.63909979 0.63909979 +0.49101416 0.61430584 0.61430584 +0.58559764 0.58559764 0.58559764 +0.66643254 0.55462614 0.55462614 +0.72846227 0.51973761 0.51973761 +0.80311288 0.50189877 0.50189877 +0.90004039 0.50002147 0.50002147 +1.00000001 0.50000001 0.50000001 +1.10000000 0.50000000 0.50000000 +1.20000000 0.50000000 0.50000000 +0.00000000 0.76182433 0.62063065 +0.12555538 0.75739272 0.61883699 +0.24659868 0.74412487 0.61203591 +0.36006378 0.72252282 0.59824110 +0.46343371 0.69689697 0.57911331 +0.55462614 0.66643254 0.55462614 +0.62980716 0.62980716 0.52456904 +0.70574649 0.60486862 0.50404594 +0.80022001 0.60016189 0.50013560 +0.90000077 0.60000048 0.50000040 +1.00000000 0.60000000 0.50000000 +1.10000000 0.60000000 0.50000000 +1.20000000 0.60000000 0.50000000 +0.00000000 0.82630414 0.58266570 +0.11646131 0.82242886 0.58018583 +0.22993157 0.81100840 0.57321991 +0.33729954 0.79135631 0.56136548 +0.43423088 0.76247824 0.54287421 +0.51973761 0.72846227 0.51973761 +0.60486862 0.70574649 0.50404594 +0.70032266 0.70032266 0.50023117 +0.80000353 0.70000302 0.50000213 +0.90000000 0.70000000 0.50000000 +1.00000000 0.70000000 0.50000000 +1.10000000 0.70000000 0.50000000 +1.20000000 0.70000000 0.50000000 +0.00000000 0.87929698 0.54716000 +0.10890351 0.87441298 0.54424669 +0.21419343 0.85996942 0.53560066 +0.31343113 0.83815065 0.52260788 +0.40759358 0.81599699 0.50953910 +0.50189877 0.80311288 0.50189877 +0.60016189 0.80022001 0.50013560 +0.70000302 0.80000353 0.50000213 +0.80000000 0.80000000 0.50000000 +0.90000000 0.80000000 0.50000000 +1.00000000 0.80000000 0.50000000 +1.10000000 0.80000000 0.50000000 +1.20000000 0.80000000 0.50000000 +0.00000000 0.91747243 0.50972929 +0.10161500 0.91472437 0.50818514 +0.20186981 0.90841516 0.50466014 +0.30096622 0.90288377 0.50158998 +0.40022936 0.90052141 0.50028459 +0.50002147 0.90004039 0.50002147 +0.60000048 0.90000077 0.50000040 +0.70000000 0.90000000 0.50000000 +0.80000000 0.90000000 0.50000000 +0.90000000 0.90000000 0.50000000 +1.00000000 0.90000000 0.50000000 +1.10000000 0.90000000 0.50000000 +1.20000000 0.90000000 0.50000000 +0.00000000 1.00030406 0.50015749 +0.10002438 1.00022657 0.50011671 +0.20001849 1.00008969 0.50004540 +0.30000484 1.00001657 0.50000815 +0.40000040 1.00000107 0.50000051 +0.50000001 1.00000001 0.50000001 +0.60000000 1.00000000 0.50000000 +0.70000000 1.00000000 0.50000000 +0.80000000 1.00000000 0.50000000 +0.90000000 1.00000000 0.50000000 +1.00000000 1.00000000 0.50000000 +1.10000000 1.00000000 0.50000000 +1.20000000 1.00000000 0.50000000 +0.00000000 1.10000004 0.50000002 +0.10000000 1.10000003 0.50000001 +0.20000000 1.10000001 0.50000000 +0.30000000 1.10000000 0.50000000 +0.40000000 1.10000000 0.50000000 +0.50000000 1.10000000 0.50000000 +0.60000000 1.10000000 0.50000000 +0.70000000 1.10000000 0.50000000 +0.80000000 1.10000000 0.50000000 +0.90000000 1.10000000 0.50000000 +1.00000000 1.10000000 0.50000000 +1.10000000 1.10000000 0.50000000 +1.20000000 1.10000000 0.50000000 +0.00000000 1.20000000 0.50000000 +0.10000000 1.20000000 0.50000000 +0.20000000 1.20000000 0.50000000 +0.30000000 1.20000000 0.50000000 +0.40000000 1.20000000 0.50000000 +0.50000000 1.20000000 0.50000000 +0.60000000 1.20000000 0.50000000 +0.70000000 1.20000000 0.50000000 +0.80000000 1.20000000 0.50000000 +0.90000000 1.20000000 0.50000000 +1.00000000 1.20000000 0.50000000 +1.10000000 1.20000000 0.50000000 +1.20000000 1.20000000 0.50000000 +0.00000000 0.00000000 0.91204792 +0.15517265 0.00000000 0.91018232 +0.29071456 0.00000000 0.88817242 +0.41230124 0.00000000 0.85270186 +0.52387144 0.00000000 0.81006029 +0.62063065 0.00000000 0.76182433 +0.71717747 0.00000000 0.71717747 +0.78380908 0.00000000 0.66428258 +0.82986250 0.00000000 0.62120277 +0.90158256 0.00000000 0.60103535 +1.00000345 0.00000000 0.60000199 +1.10000000 0.00000000 0.60000000 +1.20000000 0.00000000 0.60000000 +0.00000000 0.15517265 0.91018232 +0.15186355 0.15186355 0.90540812 +0.28735387 0.14416070 0.88178372 +0.40975168 0.13725219 0.84712657 +0.52099842 0.13108583 0.80467853 +0.61883699 0.12555538 0.75739272 +0.71254499 0.11987896 0.71254499 +0.78003954 0.11125308 0.66232491 +0.82642640 0.10310482 0.61873750 +0.90125052 0.10014389 0.60081694 +1.00000241 0.10000021 0.60000138 +1.10000000 0.10000000 0.60000000 +1.20000000 0.10000000 0.60000000 +0.00000000 0.29071456 0.88817242 +0.14416070 0.28735387 0.88178372 +0.27879357 0.27879357 0.86136115 +0.40204041 0.26861035 0.82950195 +0.51279397 0.25768821 0.78816087 +0.61203591 0.24659868 0.74412487 +0.69988798 0.23467848 0.69988798 +0.76772981 0.21891426 0.65451792 +0.81767238 0.20417332 0.61254377 +0.90059787 0.20013549 0.60038862 +1.00000077 0.20000014 0.60000044 +1.10000000 0.20000000 0.60000000 +1.20000000 0.20000000 0.60000000 +0.00000000 0.41230124 0.85270186 +0.13725219 0.40975168 0.84712657 +0.26861035 0.40204041 0.82950195 +0.39037523 0.39037523 0.80031590 +0.50064654 0.37623684 0.76328972 +0.59824110 0.36006378 0.72252282 +0.68192803 0.34163275 0.68192803 +0.74630639 0.31941918 0.63839074 +0.80785710 0.30285894 0.60564149 +0.90015817 0.30005217 0.60010177 +1.00000009 0.30000003 0.60000005 +1.10000000 0.30000000 0.60000000 +1.20000000 0.30000000 0.60000000 +0.00000000 0.52387144 0.81006029 +0.13108583 0.52099842 0.80467853 +0.25768821 0.51279397 0.78816087 +0.37623684 0.50064654 0.76328972 +0.48386966 0.48386966 0.73125472 +0.57911331 0.46343371 0.69689697 +0.65872595 0.43876962 0.65872595 +0.72246813 0.41248886 0.61888951 +0.80192020 0.40095016 0.60140266 +0.90001954 0.40000825 0.60001237 +1.00000000 0.40000000 0.60000000 +1.10000000 0.40000000 0.60000000 +1.20000000 0.40000000 0.60000000 +0.00000000 0.62063065 0.76182433 +0.12555538 0.61883699 0.75739272 +0.24659868 0.61203591 0.74412487 +0.36006378 0.59824110 0.72252282 +0.46343371 0.57911331 0.69689697 +0.55462614 0.55462614 0.66643254 +0.62980716 0.52456904 0.62980716 +0.70574649 0.50404594 0.60486862 +0.80022001 0.50013560 0.60016189 +0.90000077 0.50000040 0.60000048 +1.00000000 0.50000000 0.60000000 +1.10000000 0.50000000 0.60000000 +1.20000000 0.50000000 0.60000000 +0.00000000 0.71717747 0.71717747 +0.11987896 0.71254499 0.71254499 +0.23467848 0.69988798 0.69988798 +0.34163275 0.68192803 0.68192803 +0.43876962 0.65872595 0.65872595 +0.52456904 0.62980716 0.62980716 +0.60700341 0.60700341 0.60700341 +0.70056154 0.60047943 0.60047943 +0.80000815 0.60000594 0.60000594 +0.90000001 0.60000000 0.60000000 +1.00000000 0.60000000 0.60000000 +1.10000000 0.60000000 0.60000000 +1.20000000 0.60000000 0.60000000 +0.00000000 0.78380908 0.66428258 +0.11125308 0.78003954 0.66232491 +0.21891426 0.76772981 0.65451792 +0.31941918 0.74630639 0.63839074 +0.41248886 0.72246813 0.61888951 +0.50404594 0.70574649 0.60486862 +0.60047943 0.70056154 0.60047943 +0.70001539 0.70001539 0.60001308 +0.80000004 0.70000003 0.60000003 +0.90000000 0.70000000 0.60000000 +1.00000000 0.70000000 0.60000000 +1.10000000 0.70000000 0.60000000 +1.20000000 0.70000000 0.60000000 +0.00000000 0.82986250 0.62120277 +0.10310482 0.82642640 0.61873750 +0.20417332 0.81767238 0.61254377 +0.30285894 0.80785710 0.60564149 +0.40095016 0.80192020 0.60140266 +0.50013560 0.80022001 0.60016189 +0.60000594 0.80000815 0.60000594 +0.70000003 0.80000004 0.60000003 +0.80000000 0.80000000 0.60000000 +0.90000000 0.80000000 0.60000000 +1.00000000 0.80000000 0.60000000 +1.10000000 0.80000000 0.60000000 +1.20000000 0.80000000 0.60000000 +0.00000000 0.90158256 0.60103535 +0.10014389 0.90125052 0.60081694 +0.20013549 0.90059787 0.60038862 +0.30005217 0.90015817 0.60010177 +0.40000825 0.90001954 0.60001237 +0.50000040 0.90000077 0.60000048 +0.60000000 0.90000001 0.60000000 +0.70000000 0.90000000 0.60000000 +0.80000000 0.90000000 0.60000000 +0.90000000 0.90000000 0.60000000 +1.00000000 0.90000000 0.60000000 +1.10000000 0.90000000 0.60000000 +1.20000000 0.90000000 0.60000000 +0.00000000 1.00000345 0.60000199 +0.10000021 1.00000241 0.60000138 +0.20000014 1.00000077 0.60000044 +0.30000003 1.00000009 0.60000005 +0.40000000 1.00000000 0.60000000 +0.50000000 1.00000000 0.60000000 +0.60000000 1.00000000 0.60000000 +0.70000000 1.00000000 0.60000000 +0.80000000 1.00000000 0.60000000 +0.90000000 1.00000000 0.60000000 +1.00000000 1.00000000 0.60000000 +1.10000000 1.00000000 0.60000000 +1.20000000 1.00000000 0.60000000 +0.00000000 1.10000000 0.60000000 +0.10000000 1.10000000 0.60000000 +0.20000000 1.10000000 0.60000000 +0.30000000 1.10000000 0.60000000 +0.40000000 1.10000000 0.60000000 +0.50000000 1.10000000 0.60000000 +0.60000000 1.10000000 0.60000000 +0.70000000 1.10000000 0.60000000 +0.80000000 1.10000000 0.60000000 +0.90000000 1.10000000 0.60000000 +1.00000000 1.10000000 0.60000000 +1.10000000 1.10000000 0.60000000 +1.20000000 1.10000000 0.60000000 +0.00000000 1.20000000 0.60000000 +0.10000000 1.20000000 0.60000000 +0.20000000 1.20000000 0.60000000 +0.30000000 1.20000000 0.60000000 +0.40000000 1.20000000 0.60000000 +0.50000000 1.20000000 0.60000000 +0.60000000 1.20000000 0.60000000 +0.70000000 1.20000000 0.60000000 +0.80000000 1.20000000 0.60000000 +0.90000000 1.20000000 0.60000000 +1.00000000 1.20000000 0.60000000 +1.10000000 1.20000000 0.60000000 +1.20000000 1.20000000 0.60000000 +0.00000000 0.00000000 0.96962515 +0.14332892 0.00000000 0.96483786 +0.27030805 0.00000000 0.94191103 +0.38368634 0.00000000 0.90696809 +0.48767065 0.00000000 0.86770511 +0.58266570 0.00000000 0.82630414 +0.66428258 0.00000000 0.78380908 +0.73913964 0.00000000 0.73913964 +0.80329722 0.00000000 0.70269171 +0.90002831 0.00000000 0.70002079 +1.00000000 0.00000000 0.70000000 +1.10000000 0.00000000 0.70000000 +1.20000000 0.00000000 0.70000000 +0.00000000 0.14332892 0.96483786 +0.14058988 0.14058988 0.95897612 +0.26714900 0.13370188 0.93524877 +0.38108807 0.12715273 0.90155981 +0.48524650 0.12165362 0.86286900 +0.58018583 0.11646131 0.82242886 +0.66232491 0.11125308 0.78003954 +0.73398435 0.10513418 0.73398435 +0.80271483 0.10034158 0.70223247 +0.90002082 0.10000221 0.70001527 +1.00000000 0.10000000 0.70000000 +1.10000000 0.10000000 0.70000000 +1.20000000 0.10000000 0.70000000 +0.00000000 0.27030805 0.94191103 +0.13370188 0.26714900 0.93524877 +0.25903270 0.25903270 0.91526121 +0.37365783 0.24924309 0.88510792 +0.47825791 0.23944822 0.84899828 +0.57321991 0.22993157 0.81100840 +0.65451792 0.21891426 0.76772981 +0.72157041 0.20635303 0.72157041 +0.80144842 0.20037105 0.70121158 +0.90000795 0.20000164 0.70000581 +1.00000000 0.20000000 0.70000000 +1.10000000 0.20000000 0.70000000 +1.20000000 0.20000000 0.70000000 +0.00000000 0.38368634 0.90696809 +0.12715273 0.38108807 0.90155981 +0.24924309 0.37365783 0.88510792 +0.36286548 0.36286548 0.85896581 +0.46741977 0.35081514 0.82731919 +0.56136548 0.33729954 0.79135631 +0.63839074 0.31941918 0.74630639 +0.70939520 0.30409247 0.70939520 +0.80044005 0.30016913 0.70037406 +0.90000138 0.30000042 0.70000101 +1.00000000 0.30000000 0.70000000 +1.10000000 0.30000000 0.70000000 +1.20000000 0.30000000 0.70000000 +0.00000000 0.48767065 0.86770511 +0.12165362 0.48524650 0.86286900 +0.23944822 0.47825791 0.84899828 +0.35081514 0.46741977 0.82731919 +0.45385923 0.45385923 0.80015564 +0.54287421 0.43423088 0.76247824 +0.61888951 0.41248886 0.72246813 +0.70244509 0.40141285 0.70244509 +0.80006445 0.40003218 0.70005521 +0.90000008 0.40000003 0.70000006 +1.00000000 0.40000000 0.70000000 +1.10000000 0.40000000 0.70000000 +1.20000000 0.40000000 0.70000000 +0.00000000 0.58266570 0.82630414 +0.11646131 0.58018583 0.82242886 +0.22993157 0.57321991 0.81100840 +0.33729954 0.56136548 0.79135631 +0.43423088 0.54287421 0.76247824 +0.51973761 0.51973761 0.72846227 +0.60486862 0.50404594 0.70574649 +0.70032266 0.50023117 0.70032266 +0.80000353 0.50000213 0.70000302 +0.90000000 0.50000000 0.70000000 +1.00000000 0.50000000 0.70000000 +1.10000000 0.50000000 0.70000000 +1.20000000 0.50000000 0.70000000 +0.00000000 0.66428258 0.78380908 +0.11125308 0.66232491 0.78003954 +0.21891426 0.65451792 0.76772981 +0.31941918 0.63839074 0.74630639 +0.41248886 0.61888951 0.72246813 +0.50404594 0.60486862 0.70574649 +0.60047943 0.60047943 0.70056154 +0.70001539 0.60001308 0.70001539 +0.80000004 0.60000003 0.70000003 +0.90000000 0.60000000 0.70000000 +1.00000000 0.60000000 0.70000000 +1.10000000 0.60000000 0.70000000 +1.20000000 0.60000000 0.70000000 +0.00000000 0.73913964 0.73913964 +0.10513418 0.73398435 0.73398435 +0.20635303 0.72157041 0.72157041 +0.30409247 0.70939520 0.70939520 +0.40141285 0.70244509 0.70244509 +0.50023117 0.70032266 0.70032266 +0.60001308 0.70001539 0.70001539 +0.70000010 0.70000010 0.70000010 +0.80000000 0.70000000 0.70000000 +0.90000000 0.70000000 0.70000000 +1.00000000 0.70000000 0.70000000 +1.10000000 0.70000000 0.70000000 +1.20000000 0.70000000 0.70000000 +0.00000000 0.80329722 0.70269171 +0.10034158 0.80271483 0.70223247 +0.20037105 0.80144842 0.70121158 +0.30016913 0.80044005 0.70037406 +0.40003218 0.80006445 0.70005521 +0.50000213 0.80000353 0.70000302 +0.60000003 0.80000004 0.70000003 +0.70000000 0.80000000 0.70000000 +0.80000000 0.80000000 0.70000000 +0.90000000 0.80000000 0.70000000 +1.00000000 0.80000000 0.70000000 +1.10000000 0.80000000 0.70000000 +1.20000000 0.80000000 0.70000000 +0.00000000 0.90002831 0.70002079 +0.10000221 0.90002082 0.70001527 +0.20000164 0.90000795 0.70000581 +0.30000042 0.90000138 0.70000101 +0.40000003 0.90000008 0.70000006 +0.50000000 0.90000000 0.70000000 +0.60000000 0.90000000 0.70000000 +0.70000000 0.90000000 0.70000000 +0.80000000 0.90000000 0.70000000 +0.90000000 0.90000000 0.70000000 +1.00000000 0.90000000 0.70000000 +1.10000000 0.90000000 0.70000000 +1.20000000 0.90000000 0.70000000 +0.00000000 1.00000000 0.70000000 +0.10000000 1.00000000 0.70000000 +0.20000000 1.00000000 0.70000000 +0.30000000 1.00000000 0.70000000 +0.40000000 1.00000000 0.70000000 +0.50000000 1.00000000 0.70000000 +0.60000000 1.00000000 0.70000000 +0.70000000 1.00000000 0.70000000 +0.80000000 1.00000000 0.70000000 +0.90000000 1.00000000 0.70000000 +1.00000000 1.00000000 0.70000000 +1.10000000 1.00000000 0.70000000 +1.20000000 1.00000000 0.70000000 +0.00000000 1.10000000 0.70000000 +0.10000000 1.10000000 0.70000000 +0.20000000 1.10000000 0.70000000 +0.30000000 1.10000000 0.70000000 +0.40000000 1.10000000 0.70000000 +0.50000000 1.10000000 0.70000000 +0.60000000 1.10000000 0.70000000 +0.70000000 1.10000000 0.70000000 +0.80000000 1.10000000 0.70000000 +0.90000000 1.10000000 0.70000000 +1.00000000 1.10000000 0.70000000 +1.10000000 1.10000000 0.70000000 +1.20000000 1.10000000 0.70000000 +0.00000000 1.20000000 0.70000000 +0.10000000 1.20000000 0.70000000 +0.20000000 1.20000000 0.70000000 +0.30000000 1.20000000 0.70000000 +0.40000000 1.20000000 0.70000000 +0.50000000 1.20000000 0.70000000 +0.60000000 1.20000000 0.70000000 +0.70000000 1.20000000 0.70000000 +0.80000000 1.20000000 0.70000000 +0.90000000 1.20000000 0.70000000 +1.00000000 1.20000000 0.70000000 +1.10000000 1.20000000 0.70000000 +1.20000000 1.20000000 0.70000000 +0.00000000 0.00000000 1.01908611 +0.13287109 0.00000000 1.01357716 +0.25193669 0.00000000 0.98673239 +0.35904395 0.00000000 0.95438813 +0.45894538 0.00000000 0.92067108 +0.54716000 0.00000000 0.87929698 +0.62120277 0.00000000 0.82986250 +0.70269171 0.00000000 0.80329722 +0.80011294 0.00000000 0.80011294 +0.90000003 0.00000000 0.80000002 +1.00000000 0.00000000 0.80000000 +1.10000000 0.00000000 0.80000000 +1.20000000 0.00000000 0.80000000 +0.00000000 0.13287109 1.01357716 +0.13066508 0.13066508 1.00656530 +0.24916134 0.12455660 0.98023176 +0.35679227 0.11879272 0.94953048 +0.45674548 0.11417539 0.91652877 +0.54424669 0.10890351 0.87441298 +0.61873750 0.10310482 0.82642640 +0.70223247 0.10034158 0.80271483 +0.80007988 0.10001129 0.80007988 +0.90000002 0.10000000 0.80000002 +1.00000000 0.10000000 0.80000000 +1.10000000 0.10000000 0.80000000 +1.20000000 0.10000000 0.80000000 +0.00000000 0.25193669 0.98673239 +0.12455660 0.24916134 0.98023176 +0.24190248 0.24190248 0.96161127 +0.35040098 0.23356272 0.93512570 +0.44979284 0.22492296 0.90311681 +0.53560066 0.21419343 0.85996942 +0.61254377 0.20417332 0.81767238 +0.70121158 0.20037105 0.80144842 +0.80002681 0.20000706 0.80002681 +0.90000001 0.20000000 0.80000001 +1.00000000 0.20000000 0.80000000 +1.10000000 0.20000000 0.80000000 +1.20000000 0.20000000 0.80000000 +0.00000000 0.35904395 0.95438813 +0.11879272 0.35679227 0.94953048 +0.23356272 0.35040098 0.93512570 +0.34124155 0.34124155 0.91301195 +0.43790516 0.32839894 0.87928885 +0.52260788 0.31343113 0.83815065 +0.60564149 0.30285894 0.80785710 +0.70037406 0.30016913 0.80044005 +0.80000384 0.30000142 0.80000384 +0.90000000 0.30000000 0.80000000 +1.00000000 0.30000000 0.80000000 +1.10000000 0.30000000 0.80000000 +1.20000000 0.30000000 0.80000000 +0.00000000 0.45894538 0.92067108 +0.11417539 0.45674548 0.91652877 +0.22492296 0.44979284 0.90311681 +0.32839894 0.43790516 0.87928885 +0.42186536 0.42186536 0.84633911 +0.50953910 0.40759358 0.81599699 +0.60140266 0.40095016 0.80192020 +0.70005521 0.40003218 0.80006445 +0.80000021 0.40000010 0.80000021 +0.90000000 0.40000000 0.80000000 +1.00000000 0.40000000 0.80000000 +1.10000000 0.40000000 0.80000000 +1.20000000 0.40000000 0.80000000 +0.00000000 0.54716000 0.87929698 +0.10890351 0.54424669 0.87441298 +0.21419343 0.53560066 0.85996942 +0.31343113 0.52260788 0.83815065 +0.40759358 0.50953910 0.81599699 +0.50189877 0.50189877 0.80311288 +0.60016189 0.50013560 0.80022001 +0.70000302 0.50000213 0.80000353 +0.80000000 0.50000000 0.80000000 +0.90000000 0.50000000 0.80000000 +1.00000000 0.50000000 0.80000000 +1.10000000 0.50000000 0.80000000 +1.20000000 0.50000000 0.80000000 +0.00000000 0.62120277 0.82986250 +0.10310482 0.61873750 0.82642640 +0.20417332 0.61254377 0.81767238 +0.30285894 0.60564149 0.80785710 +0.40095016 0.60140266 0.80192020 +0.50013560 0.60016189 0.80022001 +0.60000594 0.60000594 0.80000815 +0.70000003 0.60000003 0.80000004 +0.80000000 0.60000000 0.80000000 +0.90000000 0.60000000 0.80000000 +1.00000000 0.60000000 0.80000000 +1.10000000 0.60000000 0.80000000 +1.20000000 0.60000000 0.80000000 +0.00000000 0.70269171 0.80329722 +0.10034158 0.70223247 0.80271483 +0.20037105 0.70121158 0.80144842 +0.30016913 0.70037406 0.80044005 +0.40003218 0.70005521 0.80006445 +0.50000213 0.70000302 0.80000353 +0.60000003 0.70000003 0.80000004 +0.70000000 0.70000000 0.80000000 +0.80000000 0.70000000 0.80000000 +0.90000000 0.70000000 0.80000000 +1.00000000 0.70000000 0.80000000 +1.10000000 0.70000000 0.80000000 +1.20000000 0.70000000 0.80000000 +0.00000000 0.80011294 0.80011294 +0.10001129 0.80007988 0.80007988 +0.20000706 0.80002681 0.80002681 +0.30000142 0.80000384 0.80000384 +0.40000010 0.80000021 0.80000021 +0.50000000 0.80000000 0.80000000 +0.60000000 0.80000000 0.80000000 +0.70000000 0.80000000 0.80000000 +0.80000000 0.80000000 0.80000000 +0.90000000 0.80000000 0.80000000 +1.00000000 0.80000000 0.80000000 +1.10000000 0.80000000 0.80000000 +1.20000000 0.80000000 0.80000000 +0.00000000 0.90000003 0.80000002 +0.10000000 0.90000002 0.80000002 +0.20000000 0.90000001 0.80000001 +0.30000000 0.90000000 0.80000000 +0.40000000 0.90000000 0.80000000 +0.50000000 0.90000000 0.80000000 +0.60000000 0.90000000 0.80000000 +0.70000000 0.90000000 0.80000000 +0.80000000 0.90000000 0.80000000 +0.90000000 0.90000000 0.80000000 +1.00000000 0.90000000 0.80000000 +1.10000000 0.90000000 0.80000000 +1.20000000 0.90000000 0.80000000 +0.00000000 1.00000000 0.80000000 +0.10000000 1.00000000 0.80000000 +0.20000000 1.00000000 0.80000000 +0.30000000 1.00000000 0.80000000 +0.40000000 1.00000000 0.80000000 +0.50000000 1.00000000 0.80000000 +0.60000000 1.00000000 0.80000000 +0.70000000 1.00000000 0.80000000 +0.80000000 1.00000000 0.80000000 +0.90000000 1.00000000 0.80000000 +1.00000000 1.00000000 0.80000000 +1.10000000 1.00000000 0.80000000 +1.20000000 1.00000000 0.80000000 +0.00000000 1.10000000 0.80000000 +0.10000000 1.10000000 0.80000000 +0.20000000 1.10000000 0.80000000 +0.30000000 1.10000000 0.80000000 +0.40000000 1.10000000 0.80000000 +0.50000000 1.10000000 0.80000000 +0.60000000 1.10000000 0.80000000 +0.70000000 1.10000000 0.80000000 +0.80000000 1.10000000 0.80000000 +0.90000000 1.10000000 0.80000000 +1.00000000 1.10000000 0.80000000 +1.10000000 1.10000000 0.80000000 +1.20000000 1.10000000 0.80000000 +0.00000000 1.20000000 0.80000000 +0.10000000 1.20000000 0.80000000 +0.20000000 1.20000000 0.80000000 +0.30000000 1.20000000 0.80000000 +0.40000000 1.20000000 0.80000000 +0.50000000 1.20000000 0.80000000 +0.60000000 1.20000000 0.80000000 +0.70000000 1.20000000 0.80000000 +0.80000000 1.20000000 0.80000000 +0.90000000 1.20000000 0.80000000 +1.00000000 1.20000000 0.80000000 +1.10000000 1.20000000 0.80000000 +1.20000000 1.20000000 0.80000000 +0.00000000 0.00000000 1.05975035 +0.12283560 0.00000000 1.05399342 +0.23617460 0.00000000 1.02993988 +0.33668601 0.00000000 0.99927356 +0.42696076 0.00000000 0.95881249 +0.50972929 0.00000000 0.91747243 +0.60103535 0.00000000 0.90158256 +0.70002079 0.00000000 0.90002831 +0.80000002 0.00000000 0.90000003 +0.90000000 0.00000000 0.90000000 +1.00000000 0.00000000 0.90000000 +1.10000000 0.00000000 0.90000000 +1.20000000 0.00000000 0.90000000 +0.00000000 0.12283560 1.05399342 +0.12143171 0.12143171 1.04745861 +0.23383545 0.11668098 1.02428904 +0.33427106 0.11124799 0.99386503 +0.42424467 0.10595209 0.95313141 +0.50818514 0.10161500 0.91472437 +0.60081694 0.10014389 0.90125052 +0.70001527 0.10000221 0.90002082 +0.80000002 0.10000000 0.90000002 +0.90000000 0.10000000 0.90000000 +1.00000000 0.10000000 0.90000000 +1.10000000 0.10000000 0.90000000 +1.20000000 0.10000000 0.90000000 +0.00000000 0.23617460 1.02993988 +0.11668098 0.23383545 1.02428904 +0.22701033 0.22701033 1.00671014 +0.32704436 0.21784235 0.97657491 +0.41704032 0.20834384 0.93777007 +0.50466014 0.20186981 0.90841516 +0.60038862 0.20013549 0.90059787 +0.70000581 0.20000164 0.90000795 +0.80000001 0.20000000 0.90000001 +0.90000000 0.20000000 0.90000000 +1.00000000 0.20000000 0.90000000 +1.10000000 0.20000000 0.90000000 +1.20000000 0.20000000 0.90000000 +0.00000000 0.33668601 0.99927356 +0.11124799 0.33427106 0.99386503 +0.21784235 0.32704436 0.97657491 +0.31627253 0.31627253 0.94805255 +0.40823181 0.30610938 0.91846729 +0.50158998 0.30096622 0.90288377 +0.60010177 0.30005217 0.90015817 +0.70000101 0.30000042 0.90000138 +0.80000000 0.30000000 0.90000000 +0.90000000 0.30000000 0.90000000 +1.00000000 0.30000000 0.90000000 +1.10000000 0.30000000 0.90000000 +1.20000000 0.30000000 0.90000000 +0.00000000 0.42696076 0.95881249 +0.10595209 0.42424467 0.95313141 +0.20834384 0.41704032 0.93777007 +0.30610938 0.40823181 0.91846729 +0.40210769 0.40210769 0.90474355 +0.50028459 0.40022936 0.90052141 +0.60001237 0.40000825 0.90001954 +0.70000006 0.40000003 0.90000008 +0.80000000 0.40000000 0.90000000 +0.90000000 0.40000000 0.90000000 +1.00000000 0.40000000 0.90000000 +1.10000000 0.40000000 0.90000000 +1.20000000 0.40000000 0.90000000 +0.00000000 0.50972929 0.91747243 +0.10161500 0.50818514 0.91472437 +0.20186981 0.50466014 0.90841516 +0.30096622 0.50158998 0.90288377 +0.40022936 0.50028459 0.90052141 +0.50002147 0.50002147 0.90004039 +0.60000048 0.50000040 0.90000077 +0.70000000 0.50000000 0.90000000 +0.80000000 0.50000000 0.90000000 +0.90000000 0.50000000 0.90000000 +1.00000000 0.50000000 0.90000000 +1.10000000 0.50000000 0.90000000 +1.20000000 0.50000000 0.90000000 +0.00000000 0.60103535 0.90158256 +0.10014389 0.60081694 0.90125052 +0.20013549 0.60038862 0.90059787 +0.30005217 0.60010177 0.90015817 +0.40000825 0.60001237 0.90001954 +0.50000040 0.60000048 0.90000077 +0.60000000 0.60000000 0.90000001 +0.70000000 0.60000000 0.90000000 +0.80000000 0.60000000 0.90000000 +0.90000000 0.60000000 0.90000000 +1.00000000 0.60000000 0.90000000 +1.10000000 0.60000000 0.90000000 +1.20000000 0.60000000 0.90000000 +0.00000000 0.70002079 0.90002831 +0.10000221 0.70001527 0.90002082 +0.20000164 0.70000581 0.90000795 +0.30000042 0.70000101 0.90000138 +0.40000003 0.70000006 0.90000008 +0.50000000 0.70000000 0.90000000 +0.60000000 0.70000000 0.90000000 +0.70000000 0.70000000 0.90000000 +0.80000000 0.70000000 0.90000000 +0.90000000 0.70000000 0.90000000 +1.00000000 0.70000000 0.90000000 +1.10000000 0.70000000 0.90000000 +1.20000000 0.70000000 0.90000000 +0.00000000 0.80000002 0.90000003 +0.10000000 0.80000002 0.90000002 +0.20000000 0.80000001 0.90000001 +0.30000000 0.80000000 0.90000000 +0.40000000 0.80000000 0.90000000 +0.50000000 0.80000000 0.90000000 +0.60000000 0.80000000 0.90000000 +0.70000000 0.80000000 0.90000000 +0.80000000 0.80000000 0.90000000 +0.90000000 0.80000000 0.90000000 +1.00000000 0.80000000 0.90000000 +1.10000000 0.80000000 0.90000000 +1.20000000 0.80000000 0.90000000 +0.00000000 0.90000000 0.90000000 +0.10000000 0.90000000 0.90000000 +0.20000000 0.90000000 0.90000000 +0.30000000 0.90000000 0.90000000 +0.40000000 0.90000000 0.90000000 +0.50000000 0.90000000 0.90000000 +0.60000000 0.90000000 0.90000000 +0.70000000 0.90000000 0.90000000 +0.80000000 0.90000000 0.90000000 +0.90000000 0.90000000 0.90000000 +1.00000000 0.90000000 0.90000000 +1.10000000 0.90000000 0.90000000 +1.20000000 0.90000000 0.90000000 +0.00000000 1.00000000 0.90000000 +0.10000000 1.00000000 0.90000000 +0.20000000 1.00000000 0.90000000 +0.30000000 1.00000000 0.90000000 +0.40000000 1.00000000 0.90000000 +0.50000000 1.00000000 0.90000000 +0.60000000 1.00000000 0.90000000 +0.70000000 1.00000000 0.90000000 +0.80000000 1.00000000 0.90000000 +0.90000000 1.00000000 0.90000000 +1.00000000 1.00000000 0.90000000 +1.10000000 1.00000000 0.90000000 +1.20000000 1.00000000 0.90000000 +0.00000000 1.10000000 0.90000000 +0.10000000 1.10000000 0.90000000 +0.20000000 1.10000000 0.90000000 +0.30000000 1.10000000 0.90000000 +0.40000000 1.10000000 0.90000000 +0.50000000 1.10000000 0.90000000 +0.60000000 1.10000000 0.90000000 +0.70000000 1.10000000 0.90000000 +0.80000000 1.10000000 0.90000000 +0.90000000 1.10000000 0.90000000 +1.00000000 1.10000000 0.90000000 +1.10000000 1.10000000 0.90000000 +1.20000000 1.10000000 0.90000000 +0.00000000 1.20000000 0.90000000 +0.10000000 1.20000000 0.90000000 +0.20000000 1.20000000 0.90000000 +0.30000000 1.20000000 0.90000000 +0.40000000 1.20000000 0.90000000 +0.50000000 1.20000000 0.90000000 +0.60000000 1.20000000 0.90000000 +0.70000000 1.20000000 0.90000000 +0.80000000 1.20000000 0.90000000 +0.90000000 1.20000000 0.90000000 +1.00000000 1.20000000 0.90000000 +1.10000000 1.20000000 0.90000000 +1.20000000 1.20000000 0.90000000 +0.00000000 0.00000000 1.10930635 +0.11378558 0.00000000 1.09851927 +0.21674437 0.00000000 1.06148154 +0.30867229 0.00000000 1.02383514 +0.40189866 0.00000000 1.00427903 +0.50015749 0.00000000 1.00030406 +0.60000199 0.00000000 1.00000345 +0.70000000 0.00000000 1.00000000 +0.80000000 0.00000000 1.00000000 +0.90000000 0.00000000 1.00000000 +1.00000000 0.00000000 1.00000000 +1.10000000 0.00000000 1.00000000 +1.20000000 0.00000000 1.00000000 +0.00000000 0.11378558 1.09851927 +0.11202892 0.11202892 1.08708267 +0.21405368 0.10675696 1.05283153 +0.30706878 0.10223985 1.01968452 +0.40148402 0.10037558 1.00335672 +0.50011671 0.10002438 1.00022657 +0.60000138 0.10000021 1.00000241 +0.70000000 0.10000000 1.00000000 +0.80000000 0.10000000 1.00000000 +0.90000000 0.10000000 1.00000000 +1.00000000 0.10000000 1.00000000 +1.10000000 0.10000000 1.00000000 +1.20000000 0.10000000 1.00000000 +0.00000000 0.21674437 1.06148154 +0.10675696 0.21405368 1.05283153 +0.20766246 0.20766246 1.03110077 +0.30359058 0.20235762 1.01034139 +0.40067510 0.20034203 1.00154830 +0.50004540 0.20001849 1.00008969 +0.60000044 0.20000014 1.00000077 +0.70000000 0.20000000 1.00000000 +0.80000000 0.20000000 1.00000000 +0.90000000 0.20000000 1.00000000 +1.00000000 0.20000000 1.00000000 +1.10000000 0.20000000 1.00000000 +1.20000000 0.20000000 1.00000000 +0.00000000 0.30867229 1.02383514 +0.10223985 0.30706878 1.01968452 +0.20235762 0.30359058 1.01034139 +0.30096251 0.30096251 1.00287541 +0.40015870 0.30011969 1.00037561 +0.50000815 0.30000484 1.00001657 +0.60000005 0.30000003 1.00000009 +0.70000000 0.30000000 1.00000000 +0.80000000 0.30000000 1.00000000 +0.90000000 0.30000000 1.00000000 +1.00000000 0.30000000 1.00000000 +1.10000000 0.30000000 1.00000000 +1.20000000 0.30000000 1.00000000 +0.00000000 0.40189866 1.00427903 +0.10037558 0.40148402 1.00335672 +0.20034203 0.40067510 1.00154830 +0.30011969 0.40015870 1.00037561 +0.40001565 0.40001565 1.00003915 +0.50000051 0.40000040 1.00000107 +0.60000000 0.40000000 1.00000000 +0.70000000 0.40000000 1.00000000 +0.80000000 0.40000000 1.00000000 +0.90000000 0.40000000 1.00000000 +1.00000000 0.40000000 1.00000000 +1.10000000 0.40000000 1.00000000 +1.20000000 0.40000000 1.00000000 +0.00000000 0.50015749 1.00030406 +0.10002438 0.50011671 1.00022657 +0.20001849 0.50004540 1.00008969 +0.30000484 0.50000815 1.00001657 +0.40000040 0.50000051 1.00000107 +0.50000001 0.50000001 1.00000001 +0.60000000 0.50000000 1.00000000 +0.70000000 0.50000000 1.00000000 +0.80000000 0.50000000 1.00000000 +0.90000000 0.50000000 1.00000000 +1.00000000 0.50000000 1.00000000 +1.10000000 0.50000000 1.00000000 +1.20000000 0.50000000 1.00000000 +0.00000000 0.60000199 1.00000345 +0.10000021 0.60000138 1.00000241 +0.20000014 0.60000044 1.00000077 +0.30000003 0.60000005 1.00000009 +0.40000000 0.60000000 1.00000000 +0.50000000 0.60000000 1.00000000 +0.60000000 0.60000000 1.00000000 +0.70000000 0.60000000 1.00000000 +0.80000000 0.60000000 1.00000000 +0.90000000 0.60000000 1.00000000 +1.00000000 0.60000000 1.00000000 +1.10000000 0.60000000 1.00000000 +1.20000000 0.60000000 1.00000000 +0.00000000 0.70000000 1.00000000 +0.10000000 0.70000000 1.00000000 +0.20000000 0.70000000 1.00000000 +0.30000000 0.70000000 1.00000000 +0.40000000 0.70000000 1.00000000 +0.50000000 0.70000000 1.00000000 +0.60000000 0.70000000 1.00000000 +0.70000000 0.70000000 1.00000000 +0.80000000 0.70000000 1.00000000 +0.90000000 0.70000000 1.00000000 +1.00000000 0.70000000 1.00000000 +1.10000000 0.70000000 1.00000000 +1.20000000 0.70000000 1.00000000 +0.00000000 0.80000000 1.00000000 +0.10000000 0.80000000 1.00000000 +0.20000000 0.80000000 1.00000000 +0.30000000 0.80000000 1.00000000 +0.40000000 0.80000000 1.00000000 +0.50000000 0.80000000 1.00000000 +0.60000000 0.80000000 1.00000000 +0.70000000 0.80000000 1.00000000 +0.80000000 0.80000000 1.00000000 +0.90000000 0.80000000 1.00000000 +1.00000000 0.80000000 1.00000000 +1.10000000 0.80000000 1.00000000 +1.20000000 0.80000000 1.00000000 +0.00000000 0.90000000 1.00000000 +0.10000000 0.90000000 1.00000000 +0.20000000 0.90000000 1.00000000 +0.30000000 0.90000000 1.00000000 +0.40000000 0.90000000 1.00000000 +0.50000000 0.90000000 1.00000000 +0.60000000 0.90000000 1.00000000 +0.70000000 0.90000000 1.00000000 +0.80000000 0.90000000 1.00000000 +0.90000000 0.90000000 1.00000000 +1.00000000 0.90000000 1.00000000 +1.10000000 0.90000000 1.00000000 +1.20000000 0.90000000 1.00000000 +0.00000000 1.00000000 1.00000000 +0.10000000 1.00000000 1.00000000 +0.20000000 1.00000000 1.00000000 +0.30000000 1.00000000 1.00000000 +0.40000000 1.00000000 1.00000000 +0.50000000 1.00000000 1.00000000 +0.60000000 1.00000000 1.00000000 +0.70000000 1.00000000 1.00000000 +0.80000000 1.00000000 1.00000000 +0.90000000 1.00000000 1.00000000 +1.00000000 1.00000000 1.00000000 +1.10000000 1.00000000 1.00000000 +1.20000000 1.00000000 1.00000000 +0.00000000 1.10000000 1.00000000 +0.10000000 1.10000000 1.00000000 +0.20000000 1.10000000 1.00000000 +0.30000000 1.10000000 1.00000000 +0.40000000 1.10000000 1.00000000 +0.50000000 1.10000000 1.00000000 +0.60000000 1.10000000 1.00000000 +0.70000000 1.10000000 1.00000000 +0.80000000 1.10000000 1.00000000 +0.90000000 1.10000000 1.00000000 +1.00000000 1.10000000 1.00000000 +1.10000000 1.10000000 1.00000000 +1.20000000 1.10000000 1.00000000 +0.00000000 1.20000000 1.00000000 +0.10000000 1.20000000 1.00000000 +0.20000000 1.20000000 1.00000000 +0.30000000 1.20000000 1.00000000 +0.40000000 1.20000000 1.00000000 +0.50000000 1.20000000 1.00000000 +0.60000000 1.20000000 1.00000000 +0.70000000 1.20000000 1.00000000 +0.80000000 1.20000000 1.00000000 +0.90000000 1.20000000 1.00000000 +1.00000000 1.20000000 1.00000000 +1.10000000 1.20000000 1.00000000 +1.20000000 1.20000000 1.00000000 +0.00000000 0.00000000 1.11637892 +0.10152648 0.00000000 1.11184016 +0.20081847 0.00000000 1.10311427 +0.30009933 0.00000000 1.10029021 +0.40000322 0.00000000 1.10000820 +0.50000002 0.00000000 1.10000004 +0.60000000 0.00000000 1.10000000 +0.70000000 0.00000000 1.10000000 +0.80000000 0.00000000 1.10000000 +0.90000000 0.00000000 1.10000000 +1.00000000 0.00000000 1.10000000 +1.10000000 0.00000000 1.10000000 +1.20000000 0.00000000 1.10000000 +0.00000000 0.10152648 1.11184016 +0.10110022 0.10110022 1.10834589 +0.20056038 0.10028399 1.10216182 +0.30006789 0.10002269 1.10020126 +0.40000217 0.10000050 1.10000555 +0.50000001 0.10000000 1.10000003 +0.60000000 0.10000000 1.10000000 +0.70000000 0.10000000 1.10000000 +0.80000000 0.10000000 1.10000000 +0.90000000 0.10000000 1.10000000 +1.00000000 0.10000000 1.10000000 +1.10000000 0.10000000 1.10000000 +1.20000000 0.10000000 1.10000000 +0.00000000 0.20081847 1.10311427 +0.10028399 0.20056038 1.10216182 +0.20015072 0.20015072 1.10062140 +0.30001958 0.20001291 1.10006116 +0.40000060 0.20000028 1.10000159 +0.50000000 0.20000000 1.10000001 +0.60000000 0.20000000 1.10000000 +0.70000000 0.20000000 1.10000000 +0.80000000 0.20000000 1.10000000 +0.90000000 0.20000000 1.10000000 +1.00000000 0.20000000 1.10000000 +1.10000000 0.20000000 1.10000000 +1.20000000 0.20000000 1.10000000 +0.00000000 0.30009933 1.10029021 +0.10002269 0.30006789 1.10020126 +0.20001291 0.30001958 1.10006116 +0.30000187 0.30000187 1.10000642 +0.40000005 0.30000004 1.10000015 +0.50000000 0.30000000 1.10000000 +0.60000000 0.30000000 1.10000000 +0.70000000 0.30000000 1.10000000 +0.80000000 0.30000000 1.10000000 +0.90000000 0.30000000 1.10000000 +1.00000000 0.30000000 1.10000000 +1.10000000 0.30000000 1.10000000 +1.20000000 0.30000000 1.10000000 +0.00000000 0.40000322 1.10000820 +0.10000050 0.40000217 1.10000555 +0.20000028 0.40000060 1.10000159 +0.30000004 0.40000005 1.10000015 +0.40000000 0.40000000 1.10000000 +0.50000000 0.40000000 1.10000000 +0.60000000 0.40000000 1.10000000 +0.70000000 0.40000000 1.10000000 +0.80000000 0.40000000 1.10000000 +0.90000000 0.40000000 1.10000000 +1.00000000 0.40000000 1.10000000 +1.10000000 0.40000000 1.10000000 +1.20000000 0.40000000 1.10000000 +0.00000000 0.50000002 1.10000004 +0.10000000 0.50000001 1.10000003 +0.20000000 0.50000000 1.10000001 +0.30000000 0.50000000 1.10000000 +0.40000000 0.50000000 1.10000000 +0.50000000 0.50000000 1.10000000 +0.60000000 0.50000000 1.10000000 +0.70000000 0.50000000 1.10000000 +0.80000000 0.50000000 1.10000000 +0.90000000 0.50000000 1.10000000 +1.00000000 0.50000000 1.10000000 +1.10000000 0.50000000 1.10000000 +1.20000000 0.50000000 1.10000000 +0.00000000 0.60000000 1.10000000 +0.10000000 0.60000000 1.10000000 +0.20000000 0.60000000 1.10000000 +0.30000000 0.60000000 1.10000000 +0.40000000 0.60000000 1.10000000 +0.50000000 0.60000000 1.10000000 +0.60000000 0.60000000 1.10000000 +0.70000000 0.60000000 1.10000000 +0.80000000 0.60000000 1.10000000 +0.90000000 0.60000000 1.10000000 +1.00000000 0.60000000 1.10000000 +1.10000000 0.60000000 1.10000000 +1.20000000 0.60000000 1.10000000 +0.00000000 0.70000000 1.10000000 +0.10000000 0.70000000 1.10000000 +0.20000000 0.70000000 1.10000000 +0.30000000 0.70000000 1.10000000 +0.40000000 0.70000000 1.10000000 +0.50000000 0.70000000 1.10000000 +0.60000000 0.70000000 1.10000000 +0.70000000 0.70000000 1.10000000 +0.80000000 0.70000000 1.10000000 +0.90000000 0.70000000 1.10000000 +1.00000000 0.70000000 1.10000000 +1.10000000 0.70000000 1.10000000 +1.20000000 0.70000000 1.10000000 +0.00000000 0.80000000 1.10000000 +0.10000000 0.80000000 1.10000000 +0.20000000 0.80000000 1.10000000 +0.30000000 0.80000000 1.10000000 +0.40000000 0.80000000 1.10000000 +0.50000000 0.80000000 1.10000000 +0.60000000 0.80000000 1.10000000 +0.70000000 0.80000000 1.10000000 +0.80000000 0.80000000 1.10000000 +0.90000000 0.80000000 1.10000000 +1.00000000 0.80000000 1.10000000 +1.10000000 0.80000000 1.10000000 +1.20000000 0.80000000 1.10000000 +0.00000000 0.90000000 1.10000000 +0.10000000 0.90000000 1.10000000 +0.20000000 0.90000000 1.10000000 +0.30000000 0.90000000 1.10000000 +0.40000000 0.90000000 1.10000000 +0.50000000 0.90000000 1.10000000 +0.60000000 0.90000000 1.10000000 +0.70000000 0.90000000 1.10000000 +0.80000000 0.90000000 1.10000000 +0.90000000 0.90000000 1.10000000 +1.00000000 0.90000000 1.10000000 +1.10000000 0.90000000 1.10000000 +1.20000000 0.90000000 1.10000000 +0.00000000 1.00000000 1.10000000 +0.10000000 1.00000000 1.10000000 +0.20000000 1.00000000 1.10000000 +0.30000000 1.00000000 1.10000000 +0.40000000 1.00000000 1.10000000 +0.50000000 1.00000000 1.10000000 +0.60000000 1.00000000 1.10000000 +0.70000000 1.00000000 1.10000000 +0.80000000 1.00000000 1.10000000 +0.90000000 1.00000000 1.10000000 +1.00000000 1.00000000 1.10000000 +1.10000000 1.00000000 1.10000000 +1.20000000 1.00000000 1.10000000 +0.00000000 1.10000000 1.10000000 +0.10000000 1.10000000 1.10000000 +0.20000000 1.10000000 1.10000000 +0.30000000 1.10000000 1.10000000 +0.40000000 1.10000000 1.10000000 +0.50000000 1.10000000 1.10000000 +0.60000000 1.10000000 1.10000000 +0.70000000 1.10000000 1.10000000 +0.80000000 1.10000000 1.10000000 +0.90000000 1.10000000 1.10000000 +1.00000000 1.10000000 1.10000000 +1.10000000 1.10000000 1.10000000 +1.20000000 1.10000000 1.10000000 +0.00000000 1.20000000 1.10000000 +0.10000000 1.20000000 1.10000000 +0.20000000 1.20000000 1.10000000 +0.30000000 1.20000000 1.10000000 +0.40000000 1.20000000 1.10000000 +0.50000000 1.20000000 1.10000000 +0.60000000 1.20000000 1.10000000 +0.70000000 1.20000000 1.10000000 +0.80000000 1.20000000 1.10000000 +0.90000000 1.20000000 1.10000000 +1.00000000 1.20000000 1.10000000 +1.10000000 1.20000000 1.10000000 +1.20000000 1.20000000 1.10000000 +0.00000000 0.00000000 1.20000000 +0.10000052 0.00000000 1.20000000 +0.20000021 0.00000000 1.20000000 +0.30000001 0.00000000 1.20000000 +0.40000000 0.00000000 1.20000000 +0.50000000 0.00000000 1.20000000 +0.60000000 0.00000000 1.20000000 +0.70000000 0.00000000 1.20000000 +0.80000000 0.00000000 1.20000000 +0.90000000 0.00000000 1.20000000 +1.00000000 0.00000000 1.20000000 +1.10000000 0.00000000 1.20000000 +1.20000000 0.00000000 1.20000000 +0.00000000 0.10000052 1.20000000 +0.10000032 0.10000032 1.20000000 +0.20000013 0.10000006 1.20000000 +0.30000000 0.10000000 1.20000000 +0.40000000 0.10000000 1.20000000 +0.50000000 0.10000000 1.20000000 +0.60000000 0.10000000 1.20000000 +0.70000000 0.10000000 1.20000000 +0.80000000 0.10000000 1.20000000 +0.90000000 0.10000000 1.20000000 +1.00000000 0.10000000 1.20000000 +1.10000000 0.10000000 1.20000000 +1.20000000 0.10000000 1.20000000 +0.00000000 0.20000021 1.20000000 +0.10000006 0.20000013 1.20000000 +0.20000002 0.20000002 1.20000000 +0.30000000 0.20000000 1.20000000 +0.40000000 0.20000000 1.20000000 +0.50000000 0.20000000 1.20000000 +0.60000000 0.20000000 1.20000000 +0.70000000 0.20000000 1.20000000 +0.80000000 0.20000000 1.20000000 +0.90000000 0.20000000 1.20000000 +1.00000000 0.20000000 1.20000000 +1.10000000 0.20000000 1.20000000 +1.20000000 0.20000000 1.20000000 +0.00000000 0.30000001 1.20000000 +0.10000000 0.30000000 1.20000000 +0.20000000 0.30000000 1.20000000 +0.30000000 0.30000000 1.20000000 +0.40000000 0.30000000 1.20000000 +0.50000000 0.30000000 1.20000000 +0.60000000 0.30000000 1.20000000 +0.70000000 0.30000000 1.20000000 +0.80000000 0.30000000 1.20000000 +0.90000000 0.30000000 1.20000000 +1.00000000 0.30000000 1.20000000 +1.10000000 0.30000000 1.20000000 +1.20000000 0.30000000 1.20000000 +0.00000000 0.40000000 1.20000000 +0.10000000 0.40000000 1.20000000 +0.20000000 0.40000000 1.20000000 +0.30000000 0.40000000 1.20000000 +0.40000000 0.40000000 1.20000000 +0.50000000 0.40000000 1.20000000 +0.60000000 0.40000000 1.20000000 +0.70000000 0.40000000 1.20000000 +0.80000000 0.40000000 1.20000000 +0.90000000 0.40000000 1.20000000 +1.00000000 0.40000000 1.20000000 +1.10000000 0.40000000 1.20000000 +1.20000000 0.40000000 1.20000000 +0.00000000 0.50000000 1.20000000 +0.10000000 0.50000000 1.20000000 +0.20000000 0.50000000 1.20000000 +0.30000000 0.50000000 1.20000000 +0.40000000 0.50000000 1.20000000 +0.50000000 0.50000000 1.20000000 +0.60000000 0.50000000 1.20000000 +0.70000000 0.50000000 1.20000000 +0.80000000 0.50000000 1.20000000 +0.90000000 0.50000000 1.20000000 +1.00000000 0.50000000 1.20000000 +1.10000000 0.50000000 1.20000000 +1.20000000 0.50000000 1.20000000 +0.00000000 0.60000000 1.20000000 +0.10000000 0.60000000 1.20000000 +0.20000000 0.60000000 1.20000000 +0.30000000 0.60000000 1.20000000 +0.40000000 0.60000000 1.20000000 +0.50000000 0.60000000 1.20000000 +0.60000000 0.60000000 1.20000000 +0.70000000 0.60000000 1.20000000 +0.80000000 0.60000000 1.20000000 +0.90000000 0.60000000 1.20000000 +1.00000000 0.60000000 1.20000000 +1.10000000 0.60000000 1.20000000 +1.20000000 0.60000000 1.20000000 +0.00000000 0.70000000 1.20000000 +0.10000000 0.70000000 1.20000000 +0.20000000 0.70000000 1.20000000 +0.30000000 0.70000000 1.20000000 +0.40000000 0.70000000 1.20000000 +0.50000000 0.70000000 1.20000000 +0.60000000 0.70000000 1.20000000 +0.70000000 0.70000000 1.20000000 +0.80000000 0.70000000 1.20000000 +0.90000000 0.70000000 1.20000000 +1.00000000 0.70000000 1.20000000 +1.10000000 0.70000000 1.20000000 +1.20000000 0.70000000 1.20000000 +0.00000000 0.80000000 1.20000000 +0.10000000 0.80000000 1.20000000 +0.20000000 0.80000000 1.20000000 +0.30000000 0.80000000 1.20000000 +0.40000000 0.80000000 1.20000000 +0.50000000 0.80000000 1.20000000 +0.60000000 0.80000000 1.20000000 +0.70000000 0.80000000 1.20000000 +0.80000000 0.80000000 1.20000000 +0.90000000 0.80000000 1.20000000 +1.00000000 0.80000000 1.20000000 +1.10000000 0.80000000 1.20000000 +1.20000000 0.80000000 1.20000000 +0.00000000 0.90000000 1.20000000 +0.10000000 0.90000000 1.20000000 +0.20000000 0.90000000 1.20000000 +0.30000000 0.90000000 1.20000000 +0.40000000 0.90000000 1.20000000 +0.50000000 0.90000000 1.20000000 +0.60000000 0.90000000 1.20000000 +0.70000000 0.90000000 1.20000000 +0.80000000 0.90000000 1.20000000 +0.90000000 0.90000000 1.20000000 +1.00000000 0.90000000 1.20000000 +1.10000000 0.90000000 1.20000000 +1.20000000 0.90000000 1.20000000 +0.00000000 1.00000000 1.20000000 +0.10000000 1.00000000 1.20000000 +0.20000000 1.00000000 1.20000000 +0.30000000 1.00000000 1.20000000 +0.40000000 1.00000000 1.20000000 +0.50000000 1.00000000 1.20000000 +0.60000000 1.00000000 1.20000000 +0.70000000 1.00000000 1.20000000 +0.80000000 1.00000000 1.20000000 +0.90000000 1.00000000 1.20000000 +1.00000000 1.00000000 1.20000000 +1.10000000 1.00000000 1.20000000 +1.20000000 1.00000000 1.20000000 +0.00000000 1.10000000 1.20000000 +0.10000000 1.10000000 1.20000000 +0.20000000 1.10000000 1.20000000 +0.30000000 1.10000000 1.20000000 +0.40000000 1.10000000 1.20000000 +0.50000000 1.10000000 1.20000000 +0.60000000 1.10000000 1.20000000 +0.70000000 1.10000000 1.20000000 +0.80000000 1.10000000 1.20000000 +0.90000000 1.10000000 1.20000000 +1.00000000 1.10000000 1.20000000 +1.10000000 1.10000000 1.20000000 +1.20000000 1.10000000 1.20000000 +0.00000000 1.20000000 1.20000000 +0.10000000 1.20000000 1.20000000 +0.20000000 1.20000000 1.20000000 +0.30000000 1.20000000 1.20000000 +0.40000000 1.20000000 1.20000000 +0.50000000 1.20000000 1.20000000 +0.60000000 1.20000000 1.20000000 +0.70000000 1.20000000 1.20000000 +0.80000000 1.20000000 1.20000000 +0.90000000 1.20000000 1.20000000 +1.00000000 1.20000000 1.20000000 +1.10000000 1.20000000 1.20000000 +1.20000000 1.20000000 1.20000000 + +CELLS 1728 15552 +8 0 1 14 13 169 170 183 182 +8 1 2 15 14 170 171 184 183 +8 2 3 16 15 171 172 185 184 +8 3 4 17 16 172 173 186 185 +8 4 5 18 17 173 174 187 186 +8 5 6 19 18 174 175 188 187 +8 6 7 20 19 175 176 189 188 +8 7 8 21 20 176 177 190 189 +8 8 9 22 21 177 178 191 190 +8 9 10 23 22 178 179 192 191 +8 10 11 24 23 179 180 193 192 +8 11 12 25 24 180 181 194 193 +8 13 14 27 26 182 183 196 195 +8 14 15 28 27 183 184 197 196 +8 15 16 29 28 184 185 198 197 +8 16 17 30 29 185 186 199 198 +8 17 18 31 30 186 187 200 199 +8 18 19 32 31 187 188 201 200 +8 19 20 33 32 188 189 202 201 +8 20 21 34 33 189 190 203 202 +8 21 22 35 34 190 191 204 203 +8 22 23 36 35 191 192 205 204 +8 23 24 37 36 192 193 206 205 +8 24 25 38 37 193 194 207 206 +8 26 27 40 39 195 196 209 208 +8 27 28 41 40 196 197 210 209 +8 28 29 42 41 197 198 211 210 +8 29 30 43 42 198 199 212 211 +8 30 31 44 43 199 200 213 212 +8 31 32 45 44 200 201 214 213 +8 32 33 46 45 201 202 215 214 +8 33 34 47 46 202 203 216 215 +8 34 35 48 47 203 204 217 216 +8 35 36 49 48 204 205 218 217 +8 36 37 50 49 205 206 219 218 +8 37 38 51 50 206 207 220 219 +8 39 40 53 52 208 209 222 221 +8 40 41 54 53 209 210 223 222 +8 41 42 55 54 210 211 224 223 +8 42 43 56 55 211 212 225 224 +8 43 44 57 56 212 213 226 225 +8 44 45 58 57 213 214 227 226 +8 45 46 59 58 214 215 228 227 +8 46 47 60 59 215 216 229 228 +8 47 48 61 60 216 217 230 229 +8 48 49 62 61 217 218 231 230 +8 49 50 63 62 218 219 232 231 +8 50 51 64 63 219 220 233 232 +8 52 53 66 65 221 222 235 234 +8 53 54 67 66 222 223 236 235 +8 54 55 68 67 223 224 237 236 +8 55 56 69 68 224 225 238 237 +8 56 57 70 69 225 226 239 238 +8 57 58 71 70 226 227 240 239 +8 58 59 72 71 227 228 241 240 +8 59 60 73 72 228 229 242 241 +8 60 61 74 73 229 230 243 242 +8 61 62 75 74 230 231 244 243 +8 62 63 76 75 231 232 245 244 +8 63 64 77 76 232 233 246 245 +8 65 66 79 78 234 235 248 247 +8 66 67 80 79 235 236 249 248 +8 67 68 81 80 236 237 250 249 +8 68 69 82 81 237 238 251 250 +8 69 70 83 82 238 239 252 251 +8 70 71 84 83 239 240 253 252 +8 71 72 85 84 240 241 254 253 +8 72 73 86 85 241 242 255 254 +8 73 74 87 86 242 243 256 255 +8 74 75 88 87 243 244 257 256 +8 75 76 89 88 244 245 258 257 +8 76 77 90 89 245 246 259 258 +8 78 79 92 91 247 248 261 260 +8 79 80 93 92 248 249 262 261 +8 80 81 94 93 249 250 263 262 +8 81 82 95 94 250 251 264 263 +8 82 83 96 95 251 252 265 264 +8 83 84 97 96 252 253 266 265 +8 84 85 98 97 253 254 267 266 +8 85 86 99 98 254 255 268 267 +8 86 87 100 99 255 256 269 268 +8 87 88 101 100 256 257 270 269 +8 88 89 102 101 257 258 271 270 +8 89 90 103 102 258 259 272 271 +8 91 92 105 104 260 261 274 273 +8 92 93 106 105 261 262 275 274 +8 93 94 107 106 262 263 276 275 +8 94 95 108 107 263 264 277 276 +8 95 96 109 108 264 265 278 277 +8 96 97 110 109 265 266 279 278 +8 97 98 111 110 266 267 280 279 +8 98 99 112 111 267 268 281 280 +8 99 100 113 112 268 269 282 281 +8 100 101 114 113 269 270 283 282 +8 101 102 115 114 270 271 284 283 +8 102 103 116 115 271 272 285 284 +8 104 105 118 117 273 274 287 286 +8 105 106 119 118 274 275 288 287 +8 106 107 120 119 275 276 289 288 +8 107 108 121 120 276 277 290 289 +8 108 109 122 121 277 278 291 290 +8 109 110 123 122 278 279 292 291 +8 110 111 124 123 279 280 293 292 +8 111 112 125 124 280 281 294 293 +8 112 113 126 125 281 282 295 294 +8 113 114 127 126 282 283 296 295 +8 114 115 128 127 283 284 297 296 +8 115 116 129 128 284 285 298 297 +8 117 118 131 130 286 287 300 299 +8 118 119 132 131 287 288 301 300 +8 119 120 133 132 288 289 302 301 +8 120 121 134 133 289 290 303 302 +8 121 122 135 134 290 291 304 303 +8 122 123 136 135 291 292 305 304 +8 123 124 137 136 292 293 306 305 +8 124 125 138 137 293 294 307 306 +8 125 126 139 138 294 295 308 307 +8 126 127 140 139 295 296 309 308 +8 127 128 141 140 296 297 310 309 +8 128 129 142 141 297 298 311 310 +8 130 131 144 143 299 300 313 312 +8 131 132 145 144 300 301 314 313 +8 132 133 146 145 301 302 315 314 +8 133 134 147 146 302 303 316 315 +8 134 135 148 147 303 304 317 316 +8 135 136 149 148 304 305 318 317 +8 136 137 150 149 305 306 319 318 +8 137 138 151 150 306 307 320 319 +8 138 139 152 151 307 308 321 320 +8 139 140 153 152 308 309 322 321 +8 140 141 154 153 309 310 323 322 +8 141 142 155 154 310 311 324 323 +8 143 144 157 156 312 313 326 325 +8 144 145 158 157 313 314 327 326 +8 145 146 159 158 314 315 328 327 +8 146 147 160 159 315 316 329 328 +8 147 148 161 160 316 317 330 329 +8 148 149 162 161 317 318 331 330 +8 149 150 163 162 318 319 332 331 +8 150 151 164 163 319 320 333 332 +8 151 152 165 164 320 321 334 333 +8 152 153 166 165 321 322 335 334 +8 153 154 167 166 322 323 336 335 +8 154 155 168 167 323 324 337 336 +8 169 170 183 182 338 339 352 351 +8 170 171 184 183 339 340 353 352 +8 171 172 185 184 340 341 354 353 +8 172 173 186 185 341 342 355 354 +8 173 174 187 186 342 343 356 355 +8 174 175 188 187 343 344 357 356 +8 175 176 189 188 344 345 358 357 +8 176 177 190 189 345 346 359 358 +8 177 178 191 190 346 347 360 359 +8 178 179 192 191 347 348 361 360 +8 179 180 193 192 348 349 362 361 +8 180 181 194 193 349 350 363 362 +8 182 183 196 195 351 352 365 364 +8 183 184 197 196 352 353 366 365 +8 184 185 198 197 353 354 367 366 +8 185 186 199 198 354 355 368 367 +8 186 187 200 199 355 356 369 368 +8 187 188 201 200 356 357 370 369 +8 188 189 202 201 357 358 371 370 +8 189 190 203 202 358 359 372 371 +8 190 191 204 203 359 360 373 372 +8 191 192 205 204 360 361 374 373 +8 192 193 206 205 361 362 375 374 +8 193 194 207 206 362 363 376 375 +8 195 196 209 208 364 365 378 377 +8 196 197 210 209 365 366 379 378 +8 197 198 211 210 366 367 380 379 +8 198 199 212 211 367 368 381 380 +8 199 200 213 212 368 369 382 381 +8 200 201 214 213 369 370 383 382 +8 201 202 215 214 370 371 384 383 +8 202 203 216 215 371 372 385 384 +8 203 204 217 216 372 373 386 385 +8 204 205 218 217 373 374 387 386 +8 205 206 219 218 374 375 388 387 +8 206 207 220 219 375 376 389 388 +8 208 209 222 221 377 378 391 390 +8 209 210 223 222 378 379 392 391 +8 210 211 224 223 379 380 393 392 +8 211 212 225 224 380 381 394 393 +8 212 213 226 225 381 382 395 394 +8 213 214 227 226 382 383 396 395 +8 214 215 228 227 383 384 397 396 +8 215 216 229 228 384 385 398 397 +8 216 217 230 229 385 386 399 398 +8 217 218 231 230 386 387 400 399 +8 218 219 232 231 387 388 401 400 +8 219 220 233 232 388 389 402 401 +8 221 222 235 234 390 391 404 403 +8 222 223 236 235 391 392 405 404 +8 223 224 237 236 392 393 406 405 +8 224 225 238 237 393 394 407 406 +8 225 226 239 238 394 395 408 407 +8 226 227 240 239 395 396 409 408 +8 227 228 241 240 396 397 410 409 +8 228 229 242 241 397 398 411 410 +8 229 230 243 242 398 399 412 411 +8 230 231 244 243 399 400 413 412 +8 231 232 245 244 400 401 414 413 +8 232 233 246 245 401 402 415 414 +8 234 235 248 247 403 404 417 416 +8 235 236 249 248 404 405 418 417 +8 236 237 250 249 405 406 419 418 +8 237 238 251 250 406 407 420 419 +8 238 239 252 251 407 408 421 420 +8 239 240 253 252 408 409 422 421 +8 240 241 254 253 409 410 423 422 +8 241 242 255 254 410 411 424 423 +8 242 243 256 255 411 412 425 424 +8 243 244 257 256 412 413 426 425 +8 244 245 258 257 413 414 427 426 +8 245 246 259 258 414 415 428 427 +8 247 248 261 260 416 417 430 429 +8 248 249 262 261 417 418 431 430 +8 249 250 263 262 418 419 432 431 +8 250 251 264 263 419 420 433 432 +8 251 252 265 264 420 421 434 433 +8 252 253 266 265 421 422 435 434 +8 253 254 267 266 422 423 436 435 +8 254 255 268 267 423 424 437 436 +8 255 256 269 268 424 425 438 437 +8 256 257 270 269 425 426 439 438 +8 257 258 271 270 426 427 440 439 +8 258 259 272 271 427 428 441 440 +8 260 261 274 273 429 430 443 442 +8 261 262 275 274 430 431 444 443 +8 262 263 276 275 431 432 445 444 +8 263 264 277 276 432 433 446 445 +8 264 265 278 277 433 434 447 446 +8 265 266 279 278 434 435 448 447 +8 266 267 280 279 435 436 449 448 +8 267 268 281 280 436 437 450 449 +8 268 269 282 281 437 438 451 450 +8 269 270 283 282 438 439 452 451 +8 270 271 284 283 439 440 453 452 +8 271 272 285 284 440 441 454 453 +8 273 274 287 286 442 443 456 455 +8 274 275 288 287 443 444 457 456 +8 275 276 289 288 444 445 458 457 +8 276 277 290 289 445 446 459 458 +8 277 278 291 290 446 447 460 459 +8 278 279 292 291 447 448 461 460 +8 279 280 293 292 448 449 462 461 +8 280 281 294 293 449 450 463 462 +8 281 282 295 294 450 451 464 463 +8 282 283 296 295 451 452 465 464 +8 283 284 297 296 452 453 466 465 +8 284 285 298 297 453 454 467 466 +8 286 287 300 299 455 456 469 468 +8 287 288 301 300 456 457 470 469 +8 288 289 302 301 457 458 471 470 +8 289 290 303 302 458 459 472 471 +8 290 291 304 303 459 460 473 472 +8 291 292 305 304 460 461 474 473 +8 292 293 306 305 461 462 475 474 +8 293 294 307 306 462 463 476 475 +8 294 295 308 307 463 464 477 476 +8 295 296 309 308 464 465 478 477 +8 296 297 310 309 465 466 479 478 +8 297 298 311 310 466 467 480 479 +8 299 300 313 312 468 469 482 481 +8 300 301 314 313 469 470 483 482 +8 301 302 315 314 470 471 484 483 +8 302 303 316 315 471 472 485 484 +8 303 304 317 316 472 473 486 485 +8 304 305 318 317 473 474 487 486 +8 305 306 319 318 474 475 488 487 +8 306 307 320 319 475 476 489 488 +8 307 308 321 320 476 477 490 489 +8 308 309 322 321 477 478 491 490 +8 309 310 323 322 478 479 492 491 +8 310 311 324 323 479 480 493 492 +8 312 313 326 325 481 482 495 494 +8 313 314 327 326 482 483 496 495 +8 314 315 328 327 483 484 497 496 +8 315 316 329 328 484 485 498 497 +8 316 317 330 329 485 486 499 498 +8 317 318 331 330 486 487 500 499 +8 318 319 332 331 487 488 501 500 +8 319 320 333 332 488 489 502 501 +8 320 321 334 333 489 490 503 502 +8 321 322 335 334 490 491 504 503 +8 322 323 336 335 491 492 505 504 +8 323 324 337 336 492 493 506 505 +8 338 339 352 351 507 508 521 520 +8 339 340 353 352 508 509 522 521 +8 340 341 354 353 509 510 523 522 +8 341 342 355 354 510 511 524 523 +8 342 343 356 355 511 512 525 524 +8 343 344 357 356 512 513 526 525 +8 344 345 358 357 513 514 527 526 +8 345 346 359 358 514 515 528 527 +8 346 347 360 359 515 516 529 528 +8 347 348 361 360 516 517 530 529 +8 348 349 362 361 517 518 531 530 +8 349 350 363 362 518 519 532 531 +8 351 352 365 364 520 521 534 533 +8 352 353 366 365 521 522 535 534 +8 353 354 367 366 522 523 536 535 +8 354 355 368 367 523 524 537 536 +8 355 356 369 368 524 525 538 537 +8 356 357 370 369 525 526 539 538 +8 357 358 371 370 526 527 540 539 +8 358 359 372 371 527 528 541 540 +8 359 360 373 372 528 529 542 541 +8 360 361 374 373 529 530 543 542 +8 361 362 375 374 530 531 544 543 +8 362 363 376 375 531 532 545 544 +8 364 365 378 377 533 534 547 546 +8 365 366 379 378 534 535 548 547 +8 366 367 380 379 535 536 549 548 +8 367 368 381 380 536 537 550 549 +8 368 369 382 381 537 538 551 550 +8 369 370 383 382 538 539 552 551 +8 370 371 384 383 539 540 553 552 +8 371 372 385 384 540 541 554 553 +8 372 373 386 385 541 542 555 554 +8 373 374 387 386 542 543 556 555 +8 374 375 388 387 543 544 557 556 +8 375 376 389 388 544 545 558 557 +8 377 378 391 390 546 547 560 559 +8 378 379 392 391 547 548 561 560 +8 379 380 393 392 548 549 562 561 +8 380 381 394 393 549 550 563 562 +8 381 382 395 394 550 551 564 563 +8 382 383 396 395 551 552 565 564 +8 383 384 397 396 552 553 566 565 +8 384 385 398 397 553 554 567 566 +8 385 386 399 398 554 555 568 567 +8 386 387 400 399 555 556 569 568 +8 387 388 401 400 556 557 570 569 +8 388 389 402 401 557 558 571 570 +8 390 391 404 403 559 560 573 572 +8 391 392 405 404 560 561 574 573 +8 392 393 406 405 561 562 575 574 +8 393 394 407 406 562 563 576 575 +8 394 395 408 407 563 564 577 576 +8 395 396 409 408 564 565 578 577 +8 396 397 410 409 565 566 579 578 +8 397 398 411 410 566 567 580 579 +8 398 399 412 411 567 568 581 580 +8 399 400 413 412 568 569 582 581 +8 400 401 414 413 569 570 583 582 +8 401 402 415 414 570 571 584 583 +8 403 404 417 416 572 573 586 585 +8 404 405 418 417 573 574 587 586 +8 405 406 419 418 574 575 588 587 +8 406 407 420 419 575 576 589 588 +8 407 408 421 420 576 577 590 589 +8 408 409 422 421 577 578 591 590 +8 409 410 423 422 578 579 592 591 +8 410 411 424 423 579 580 593 592 +8 411 412 425 424 580 581 594 593 +8 412 413 426 425 581 582 595 594 +8 413 414 427 426 582 583 596 595 +8 414 415 428 427 583 584 597 596 +8 416 417 430 429 585 586 599 598 +8 417 418 431 430 586 587 600 599 +8 418 419 432 431 587 588 601 600 +8 419 420 433 432 588 589 602 601 +8 420 421 434 433 589 590 603 602 +8 421 422 435 434 590 591 604 603 +8 422 423 436 435 591 592 605 604 +8 423 424 437 436 592 593 606 605 +8 424 425 438 437 593 594 607 606 +8 425 426 439 438 594 595 608 607 +8 426 427 440 439 595 596 609 608 +8 427 428 441 440 596 597 610 609 +8 429 430 443 442 598 599 612 611 +8 430 431 444 443 599 600 613 612 +8 431 432 445 444 600 601 614 613 +8 432 433 446 445 601 602 615 614 +8 433 434 447 446 602 603 616 615 +8 434 435 448 447 603 604 617 616 +8 435 436 449 448 604 605 618 617 +8 436 437 450 449 605 606 619 618 +8 437 438 451 450 606 607 620 619 +8 438 439 452 451 607 608 621 620 +8 439 440 453 452 608 609 622 621 +8 440 441 454 453 609 610 623 622 +8 442 443 456 455 611 612 625 624 +8 443 444 457 456 612 613 626 625 +8 444 445 458 457 613 614 627 626 +8 445 446 459 458 614 615 628 627 +8 446 447 460 459 615 616 629 628 +8 447 448 461 460 616 617 630 629 +8 448 449 462 461 617 618 631 630 +8 449 450 463 462 618 619 632 631 +8 450 451 464 463 619 620 633 632 +8 451 452 465 464 620 621 634 633 +8 452 453 466 465 621 622 635 634 +8 453 454 467 466 622 623 636 635 +8 455 456 469 468 624 625 638 637 +8 456 457 470 469 625 626 639 638 +8 457 458 471 470 626 627 640 639 +8 458 459 472 471 627 628 641 640 +8 459 460 473 472 628 629 642 641 +8 460 461 474 473 629 630 643 642 +8 461 462 475 474 630 631 644 643 +8 462 463 476 475 631 632 645 644 +8 463 464 477 476 632 633 646 645 +8 464 465 478 477 633 634 647 646 +8 465 466 479 478 634 635 648 647 +8 466 467 480 479 635 636 649 648 +8 468 469 482 481 637 638 651 650 +8 469 470 483 482 638 639 652 651 +8 470 471 484 483 639 640 653 652 +8 471 472 485 484 640 641 654 653 +8 472 473 486 485 641 642 655 654 +8 473 474 487 486 642 643 656 655 +8 474 475 488 487 643 644 657 656 +8 475 476 489 488 644 645 658 657 +8 476 477 490 489 645 646 659 658 +8 477 478 491 490 646 647 660 659 +8 478 479 492 491 647 648 661 660 +8 479 480 493 492 648 649 662 661 +8 481 482 495 494 650 651 664 663 +8 482 483 496 495 651 652 665 664 +8 483 484 497 496 652 653 666 665 +8 484 485 498 497 653 654 667 666 +8 485 486 499 498 654 655 668 667 +8 486 487 500 499 655 656 669 668 +8 487 488 501 500 656 657 670 669 +8 488 489 502 501 657 658 671 670 +8 489 490 503 502 658 659 672 671 +8 490 491 504 503 659 660 673 672 +8 491 492 505 504 660 661 674 673 +8 492 493 506 505 661 662 675 674 +8 507 508 521 520 676 677 690 689 +8 508 509 522 521 677 678 691 690 +8 509 510 523 522 678 679 692 691 +8 510 511 524 523 679 680 693 692 +8 511 512 525 524 680 681 694 693 +8 512 513 526 525 681 682 695 694 +8 513 514 527 526 682 683 696 695 +8 514 515 528 527 683 684 697 696 +8 515 516 529 528 684 685 698 697 +8 516 517 530 529 685 686 699 698 +8 517 518 531 530 686 687 700 699 +8 518 519 532 531 687 688 701 700 +8 520 521 534 533 689 690 703 702 +8 521 522 535 534 690 691 704 703 +8 522 523 536 535 691 692 705 704 +8 523 524 537 536 692 693 706 705 +8 524 525 538 537 693 694 707 706 +8 525 526 539 538 694 695 708 707 +8 526 527 540 539 695 696 709 708 +8 527 528 541 540 696 697 710 709 +8 528 529 542 541 697 698 711 710 +8 529 530 543 542 698 699 712 711 +8 530 531 544 543 699 700 713 712 +8 531 532 545 544 700 701 714 713 +8 533 534 547 546 702 703 716 715 +8 534 535 548 547 703 704 717 716 +8 535 536 549 548 704 705 718 717 +8 536 537 550 549 705 706 719 718 +8 537 538 551 550 706 707 720 719 +8 538 539 552 551 707 708 721 720 +8 539 540 553 552 708 709 722 721 +8 540 541 554 553 709 710 723 722 +8 541 542 555 554 710 711 724 723 +8 542 543 556 555 711 712 725 724 +8 543 544 557 556 712 713 726 725 +8 544 545 558 557 713 714 727 726 +8 546 547 560 559 715 716 729 728 +8 547 548 561 560 716 717 730 729 +8 548 549 562 561 717 718 731 730 +8 549 550 563 562 718 719 732 731 +8 550 551 564 563 719 720 733 732 +8 551 552 565 564 720 721 734 733 +8 552 553 566 565 721 722 735 734 +8 553 554 567 566 722 723 736 735 +8 554 555 568 567 723 724 737 736 +8 555 556 569 568 724 725 738 737 +8 556 557 570 569 725 726 739 738 +8 557 558 571 570 726 727 740 739 +8 559 560 573 572 728 729 742 741 +8 560 561 574 573 729 730 743 742 +8 561 562 575 574 730 731 744 743 +8 562 563 576 575 731 732 745 744 +8 563 564 577 576 732 733 746 745 +8 564 565 578 577 733 734 747 746 +8 565 566 579 578 734 735 748 747 +8 566 567 580 579 735 736 749 748 +8 567 568 581 580 736 737 750 749 +8 568 569 582 581 737 738 751 750 +8 569 570 583 582 738 739 752 751 +8 570 571 584 583 739 740 753 752 +8 572 573 586 585 741 742 755 754 +8 573 574 587 586 742 743 756 755 +8 574 575 588 587 743 744 757 756 +8 575 576 589 588 744 745 758 757 +8 576 577 590 589 745 746 759 758 +8 577 578 591 590 746 747 760 759 +8 578 579 592 591 747 748 761 760 +8 579 580 593 592 748 749 762 761 +8 580 581 594 593 749 750 763 762 +8 581 582 595 594 750 751 764 763 +8 582 583 596 595 751 752 765 764 +8 583 584 597 596 752 753 766 765 +8 585 586 599 598 754 755 768 767 +8 586 587 600 599 755 756 769 768 +8 587 588 601 600 756 757 770 769 +8 588 589 602 601 757 758 771 770 +8 589 590 603 602 758 759 772 771 +8 590 591 604 603 759 760 773 772 +8 591 592 605 604 760 761 774 773 +8 592 593 606 605 761 762 775 774 +8 593 594 607 606 762 763 776 775 +8 594 595 608 607 763 764 777 776 +8 595 596 609 608 764 765 778 777 +8 596 597 610 609 765 766 779 778 +8 598 599 612 611 767 768 781 780 +8 599 600 613 612 768 769 782 781 +8 600 601 614 613 769 770 783 782 +8 601 602 615 614 770 771 784 783 +8 602 603 616 615 771 772 785 784 +8 603 604 617 616 772 773 786 785 +8 604 605 618 617 773 774 787 786 +8 605 606 619 618 774 775 788 787 +8 606 607 620 619 775 776 789 788 +8 607 608 621 620 776 777 790 789 +8 608 609 622 621 777 778 791 790 +8 609 610 623 622 778 779 792 791 +8 611 612 625 624 780 781 794 793 +8 612 613 626 625 781 782 795 794 +8 613 614 627 626 782 783 796 795 +8 614 615 628 627 783 784 797 796 +8 615 616 629 628 784 785 798 797 +8 616 617 630 629 785 786 799 798 +8 617 618 631 630 786 787 800 799 +8 618 619 632 631 787 788 801 800 +8 619 620 633 632 788 789 802 801 +8 620 621 634 633 789 790 803 802 +8 621 622 635 634 790 791 804 803 +8 622 623 636 635 791 792 805 804 +8 624 625 638 637 793 794 807 806 +8 625 626 639 638 794 795 808 807 +8 626 627 640 639 795 796 809 808 +8 627 628 641 640 796 797 810 809 +8 628 629 642 641 797 798 811 810 +8 629 630 643 642 798 799 812 811 +8 630 631 644 643 799 800 813 812 +8 631 632 645 644 800 801 814 813 +8 632 633 646 645 801 802 815 814 +8 633 634 647 646 802 803 816 815 +8 634 635 648 647 803 804 817 816 +8 635 636 649 648 804 805 818 817 +8 637 638 651 650 806 807 820 819 +8 638 639 652 651 807 808 821 820 +8 639 640 653 652 808 809 822 821 +8 640 641 654 653 809 810 823 822 +8 641 642 655 654 810 811 824 823 +8 642 643 656 655 811 812 825 824 +8 643 644 657 656 812 813 826 825 +8 644 645 658 657 813 814 827 826 +8 645 646 659 658 814 815 828 827 +8 646 647 660 659 815 816 829 828 +8 647 648 661 660 816 817 830 829 +8 648 649 662 661 817 818 831 830 +8 650 651 664 663 819 820 833 832 +8 651 652 665 664 820 821 834 833 +8 652 653 666 665 821 822 835 834 +8 653 654 667 666 822 823 836 835 +8 654 655 668 667 823 824 837 836 +8 655 656 669 668 824 825 838 837 +8 656 657 670 669 825 826 839 838 +8 657 658 671 670 826 827 840 839 +8 658 659 672 671 827 828 841 840 +8 659 660 673 672 828 829 842 841 +8 660 661 674 673 829 830 843 842 +8 661 662 675 674 830 831 844 843 +8 676 677 690 689 845 846 859 858 +8 677 678 691 690 846 847 860 859 +8 678 679 692 691 847 848 861 860 +8 679 680 693 692 848 849 862 861 +8 680 681 694 693 849 850 863 862 +8 681 682 695 694 850 851 864 863 +8 682 683 696 695 851 852 865 864 +8 683 684 697 696 852 853 866 865 +8 684 685 698 697 853 854 867 866 +8 685 686 699 698 854 855 868 867 +8 686 687 700 699 855 856 869 868 +8 687 688 701 700 856 857 870 869 +8 689 690 703 702 858 859 872 871 +8 690 691 704 703 859 860 873 872 +8 691 692 705 704 860 861 874 873 +8 692 693 706 705 861 862 875 874 +8 693 694 707 706 862 863 876 875 +8 694 695 708 707 863 864 877 876 +8 695 696 709 708 864 865 878 877 +8 696 697 710 709 865 866 879 878 +8 697 698 711 710 866 867 880 879 +8 698 699 712 711 867 868 881 880 +8 699 700 713 712 868 869 882 881 +8 700 701 714 713 869 870 883 882 +8 702 703 716 715 871 872 885 884 +8 703 704 717 716 872 873 886 885 +8 704 705 718 717 873 874 887 886 +8 705 706 719 718 874 875 888 887 +8 706 707 720 719 875 876 889 888 +8 707 708 721 720 876 877 890 889 +8 708 709 722 721 877 878 891 890 +8 709 710 723 722 878 879 892 891 +8 710 711 724 723 879 880 893 892 +8 711 712 725 724 880 881 894 893 +8 712 713 726 725 881 882 895 894 +8 713 714 727 726 882 883 896 895 +8 715 716 729 728 884 885 898 897 +8 716 717 730 729 885 886 899 898 +8 717 718 731 730 886 887 900 899 +8 718 719 732 731 887 888 901 900 +8 719 720 733 732 888 889 902 901 +8 720 721 734 733 889 890 903 902 +8 721 722 735 734 890 891 904 903 +8 722 723 736 735 891 892 905 904 +8 723 724 737 736 892 893 906 905 +8 724 725 738 737 893 894 907 906 +8 725 726 739 738 894 895 908 907 +8 726 727 740 739 895 896 909 908 +8 728 729 742 741 897 898 911 910 +8 729 730 743 742 898 899 912 911 +8 730 731 744 743 899 900 913 912 +8 731 732 745 744 900 901 914 913 +8 732 733 746 745 901 902 915 914 +8 733 734 747 746 902 903 916 915 +8 734 735 748 747 903 904 917 916 +8 735 736 749 748 904 905 918 917 +8 736 737 750 749 905 906 919 918 +8 737 738 751 750 906 907 920 919 +8 738 739 752 751 907 908 921 920 +8 739 740 753 752 908 909 922 921 +8 741 742 755 754 910 911 924 923 +8 742 743 756 755 911 912 925 924 +8 743 744 757 756 912 913 926 925 +8 744 745 758 757 913 914 927 926 +8 745 746 759 758 914 915 928 927 +8 746 747 760 759 915 916 929 928 +8 747 748 761 760 916 917 930 929 +8 748 749 762 761 917 918 931 930 +8 749 750 763 762 918 919 932 931 +8 750 751 764 763 919 920 933 932 +8 751 752 765 764 920 921 934 933 +8 752 753 766 765 921 922 935 934 +8 754 755 768 767 923 924 937 936 +8 755 756 769 768 924 925 938 937 +8 756 757 770 769 925 926 939 938 +8 757 758 771 770 926 927 940 939 +8 758 759 772 771 927 928 941 940 +8 759 760 773 772 928 929 942 941 +8 760 761 774 773 929 930 943 942 +8 761 762 775 774 930 931 944 943 +8 762 763 776 775 931 932 945 944 +8 763 764 777 776 932 933 946 945 +8 764 765 778 777 933 934 947 946 +8 765 766 779 778 934 935 948 947 +8 767 768 781 780 936 937 950 949 +8 768 769 782 781 937 938 951 950 +8 769 770 783 782 938 939 952 951 +8 770 771 784 783 939 940 953 952 +8 771 772 785 784 940 941 954 953 +8 772 773 786 785 941 942 955 954 +8 773 774 787 786 942 943 956 955 +8 774 775 788 787 943 944 957 956 +8 775 776 789 788 944 945 958 957 +8 776 777 790 789 945 946 959 958 +8 777 778 791 790 946 947 960 959 +8 778 779 792 791 947 948 961 960 +8 780 781 794 793 949 950 963 962 +8 781 782 795 794 950 951 964 963 +8 782 783 796 795 951 952 965 964 +8 783 784 797 796 952 953 966 965 +8 784 785 798 797 953 954 967 966 +8 785 786 799 798 954 955 968 967 +8 786 787 800 799 955 956 969 968 +8 787 788 801 800 956 957 970 969 +8 788 789 802 801 957 958 971 970 +8 789 790 803 802 958 959 972 971 +8 790 791 804 803 959 960 973 972 +8 791 792 805 804 960 961 974 973 +8 793 794 807 806 962 963 976 975 +8 794 795 808 807 963 964 977 976 +8 795 796 809 808 964 965 978 977 +8 796 797 810 809 965 966 979 978 +8 797 798 811 810 966 967 980 979 +8 798 799 812 811 967 968 981 980 +8 799 800 813 812 968 969 982 981 +8 800 801 814 813 969 970 983 982 +8 801 802 815 814 970 971 984 983 +8 802 803 816 815 971 972 985 984 +8 803 804 817 816 972 973 986 985 +8 804 805 818 817 973 974 987 986 +8 806 807 820 819 975 976 989 988 +8 807 808 821 820 976 977 990 989 +8 808 809 822 821 977 978 991 990 +8 809 810 823 822 978 979 992 991 +8 810 811 824 823 979 980 993 992 +8 811 812 825 824 980 981 994 993 +8 812 813 826 825 981 982 995 994 +8 813 814 827 826 982 983 996 995 +8 814 815 828 827 983 984 997 996 +8 815 816 829 828 984 985 998 997 +8 816 817 830 829 985 986 999 998 +8 817 818 831 830 986 987 1000 999 +8 819 820 833 832 988 989 1002 1001 +8 820 821 834 833 989 990 1003 1002 +8 821 822 835 834 990 991 1004 1003 +8 822 823 836 835 991 992 1005 1004 +8 823 824 837 836 992 993 1006 1005 +8 824 825 838 837 993 994 1007 1006 +8 825 826 839 838 994 995 1008 1007 +8 826 827 840 839 995 996 1009 1008 +8 827 828 841 840 996 997 1010 1009 +8 828 829 842 841 997 998 1011 1010 +8 829 830 843 842 998 999 1012 1011 +8 830 831 844 843 999 1000 1013 1012 +8 845 846 859 858 1014 1015 1028 1027 +8 846 847 860 859 1015 1016 1029 1028 +8 847 848 861 860 1016 1017 1030 1029 +8 848 849 862 861 1017 1018 1031 1030 +8 849 850 863 862 1018 1019 1032 1031 +8 850 851 864 863 1019 1020 1033 1032 +8 851 852 865 864 1020 1021 1034 1033 +8 852 853 866 865 1021 1022 1035 1034 +8 853 854 867 866 1022 1023 1036 1035 +8 854 855 868 867 1023 1024 1037 1036 +8 855 856 869 868 1024 1025 1038 1037 +8 856 857 870 869 1025 1026 1039 1038 +8 858 859 872 871 1027 1028 1041 1040 +8 859 860 873 872 1028 1029 1042 1041 +8 860 861 874 873 1029 1030 1043 1042 +8 861 862 875 874 1030 1031 1044 1043 +8 862 863 876 875 1031 1032 1045 1044 +8 863 864 877 876 1032 1033 1046 1045 +8 864 865 878 877 1033 1034 1047 1046 +8 865 866 879 878 1034 1035 1048 1047 +8 866 867 880 879 1035 1036 1049 1048 +8 867 868 881 880 1036 1037 1050 1049 +8 868 869 882 881 1037 1038 1051 1050 +8 869 870 883 882 1038 1039 1052 1051 +8 871 872 885 884 1040 1041 1054 1053 +8 872 873 886 885 1041 1042 1055 1054 +8 873 874 887 886 1042 1043 1056 1055 +8 874 875 888 887 1043 1044 1057 1056 +8 875 876 889 888 1044 1045 1058 1057 +8 876 877 890 889 1045 1046 1059 1058 +8 877 878 891 890 1046 1047 1060 1059 +8 878 879 892 891 1047 1048 1061 1060 +8 879 880 893 892 1048 1049 1062 1061 +8 880 881 894 893 1049 1050 1063 1062 +8 881 882 895 894 1050 1051 1064 1063 +8 882 883 896 895 1051 1052 1065 1064 +8 884 885 898 897 1053 1054 1067 1066 +8 885 886 899 898 1054 1055 1068 1067 +8 886 887 900 899 1055 1056 1069 1068 +8 887 888 901 900 1056 1057 1070 1069 +8 888 889 902 901 1057 1058 1071 1070 +8 889 890 903 902 1058 1059 1072 1071 +8 890 891 904 903 1059 1060 1073 1072 +8 891 892 905 904 1060 1061 1074 1073 +8 892 893 906 905 1061 1062 1075 1074 +8 893 894 907 906 1062 1063 1076 1075 +8 894 895 908 907 1063 1064 1077 1076 +8 895 896 909 908 1064 1065 1078 1077 +8 897 898 911 910 1066 1067 1080 1079 +8 898 899 912 911 1067 1068 1081 1080 +8 899 900 913 912 1068 1069 1082 1081 +8 900 901 914 913 1069 1070 1083 1082 +8 901 902 915 914 1070 1071 1084 1083 +8 902 903 916 915 1071 1072 1085 1084 +8 903 904 917 916 1072 1073 1086 1085 +8 904 905 918 917 1073 1074 1087 1086 +8 905 906 919 918 1074 1075 1088 1087 +8 906 907 920 919 1075 1076 1089 1088 +8 907 908 921 920 1076 1077 1090 1089 +8 908 909 922 921 1077 1078 1091 1090 +8 910 911 924 923 1079 1080 1093 1092 +8 911 912 925 924 1080 1081 1094 1093 +8 912 913 926 925 1081 1082 1095 1094 +8 913 914 927 926 1082 1083 1096 1095 +8 914 915 928 927 1083 1084 1097 1096 +8 915 916 929 928 1084 1085 1098 1097 +8 916 917 930 929 1085 1086 1099 1098 +8 917 918 931 930 1086 1087 1100 1099 +8 918 919 932 931 1087 1088 1101 1100 +8 919 920 933 932 1088 1089 1102 1101 +8 920 921 934 933 1089 1090 1103 1102 +8 921 922 935 934 1090 1091 1104 1103 +8 923 924 937 936 1092 1093 1106 1105 +8 924 925 938 937 1093 1094 1107 1106 +8 925 926 939 938 1094 1095 1108 1107 +8 926 927 940 939 1095 1096 1109 1108 +8 927 928 941 940 1096 1097 1110 1109 +8 928 929 942 941 1097 1098 1111 1110 +8 929 930 943 942 1098 1099 1112 1111 +8 930 931 944 943 1099 1100 1113 1112 +8 931 932 945 944 1100 1101 1114 1113 +8 932 933 946 945 1101 1102 1115 1114 +8 933 934 947 946 1102 1103 1116 1115 +8 934 935 948 947 1103 1104 1117 1116 +8 936 937 950 949 1105 1106 1119 1118 +8 937 938 951 950 1106 1107 1120 1119 +8 938 939 952 951 1107 1108 1121 1120 +8 939 940 953 952 1108 1109 1122 1121 +8 940 941 954 953 1109 1110 1123 1122 +8 941 942 955 954 1110 1111 1124 1123 +8 942 943 956 955 1111 1112 1125 1124 +8 943 944 957 956 1112 1113 1126 1125 +8 944 945 958 957 1113 1114 1127 1126 +8 945 946 959 958 1114 1115 1128 1127 +8 946 947 960 959 1115 1116 1129 1128 +8 947 948 961 960 1116 1117 1130 1129 +8 949 950 963 962 1118 1119 1132 1131 +8 950 951 964 963 1119 1120 1133 1132 +8 951 952 965 964 1120 1121 1134 1133 +8 952 953 966 965 1121 1122 1135 1134 +8 953 954 967 966 1122 1123 1136 1135 +8 954 955 968 967 1123 1124 1137 1136 +8 955 956 969 968 1124 1125 1138 1137 +8 956 957 970 969 1125 1126 1139 1138 +8 957 958 971 970 1126 1127 1140 1139 +8 958 959 972 971 1127 1128 1141 1140 +8 959 960 973 972 1128 1129 1142 1141 +8 960 961 974 973 1129 1130 1143 1142 +8 962 963 976 975 1131 1132 1145 1144 +8 963 964 977 976 1132 1133 1146 1145 +8 964 965 978 977 1133 1134 1147 1146 +8 965 966 979 978 1134 1135 1148 1147 +8 966 967 980 979 1135 1136 1149 1148 +8 967 968 981 980 1136 1137 1150 1149 +8 968 969 982 981 1137 1138 1151 1150 +8 969 970 983 982 1138 1139 1152 1151 +8 970 971 984 983 1139 1140 1153 1152 +8 971 972 985 984 1140 1141 1154 1153 +8 972 973 986 985 1141 1142 1155 1154 +8 973 974 987 986 1142 1143 1156 1155 +8 975 976 989 988 1144 1145 1158 1157 +8 976 977 990 989 1145 1146 1159 1158 +8 977 978 991 990 1146 1147 1160 1159 +8 978 979 992 991 1147 1148 1161 1160 +8 979 980 993 992 1148 1149 1162 1161 +8 980 981 994 993 1149 1150 1163 1162 +8 981 982 995 994 1150 1151 1164 1163 +8 982 983 996 995 1151 1152 1165 1164 +8 983 984 997 996 1152 1153 1166 1165 +8 984 985 998 997 1153 1154 1167 1166 +8 985 986 999 998 1154 1155 1168 1167 +8 986 987 1000 999 1155 1156 1169 1168 +8 988 989 1002 1001 1157 1158 1171 1170 +8 989 990 1003 1002 1158 1159 1172 1171 +8 990 991 1004 1003 1159 1160 1173 1172 +8 991 992 1005 1004 1160 1161 1174 1173 +8 992 993 1006 1005 1161 1162 1175 1174 +8 993 994 1007 1006 1162 1163 1176 1175 +8 994 995 1008 1007 1163 1164 1177 1176 +8 995 996 1009 1008 1164 1165 1178 1177 +8 996 997 1010 1009 1165 1166 1179 1178 +8 997 998 1011 1010 1166 1167 1180 1179 +8 998 999 1012 1011 1167 1168 1181 1180 +8 999 1000 1013 1012 1168 1169 1182 1181 +8 1014 1015 1028 1027 1183 1184 1197 1196 +8 1015 1016 1029 1028 1184 1185 1198 1197 +8 1016 1017 1030 1029 1185 1186 1199 1198 +8 1017 1018 1031 1030 1186 1187 1200 1199 +8 1018 1019 1032 1031 1187 1188 1201 1200 +8 1019 1020 1033 1032 1188 1189 1202 1201 +8 1020 1021 1034 1033 1189 1190 1203 1202 +8 1021 1022 1035 1034 1190 1191 1204 1203 +8 1022 1023 1036 1035 1191 1192 1205 1204 +8 1023 1024 1037 1036 1192 1193 1206 1205 +8 1024 1025 1038 1037 1193 1194 1207 1206 +8 1025 1026 1039 1038 1194 1195 1208 1207 +8 1027 1028 1041 1040 1196 1197 1210 1209 +8 1028 1029 1042 1041 1197 1198 1211 1210 +8 1029 1030 1043 1042 1198 1199 1212 1211 +8 1030 1031 1044 1043 1199 1200 1213 1212 +8 1031 1032 1045 1044 1200 1201 1214 1213 +8 1032 1033 1046 1045 1201 1202 1215 1214 +8 1033 1034 1047 1046 1202 1203 1216 1215 +8 1034 1035 1048 1047 1203 1204 1217 1216 +8 1035 1036 1049 1048 1204 1205 1218 1217 +8 1036 1037 1050 1049 1205 1206 1219 1218 +8 1037 1038 1051 1050 1206 1207 1220 1219 +8 1038 1039 1052 1051 1207 1208 1221 1220 +8 1040 1041 1054 1053 1209 1210 1223 1222 +8 1041 1042 1055 1054 1210 1211 1224 1223 +8 1042 1043 1056 1055 1211 1212 1225 1224 +8 1043 1044 1057 1056 1212 1213 1226 1225 +8 1044 1045 1058 1057 1213 1214 1227 1226 +8 1045 1046 1059 1058 1214 1215 1228 1227 +8 1046 1047 1060 1059 1215 1216 1229 1228 +8 1047 1048 1061 1060 1216 1217 1230 1229 +8 1048 1049 1062 1061 1217 1218 1231 1230 +8 1049 1050 1063 1062 1218 1219 1232 1231 +8 1050 1051 1064 1063 1219 1220 1233 1232 +8 1051 1052 1065 1064 1220 1221 1234 1233 +8 1053 1054 1067 1066 1222 1223 1236 1235 +8 1054 1055 1068 1067 1223 1224 1237 1236 +8 1055 1056 1069 1068 1224 1225 1238 1237 +8 1056 1057 1070 1069 1225 1226 1239 1238 +8 1057 1058 1071 1070 1226 1227 1240 1239 +8 1058 1059 1072 1071 1227 1228 1241 1240 +8 1059 1060 1073 1072 1228 1229 1242 1241 +8 1060 1061 1074 1073 1229 1230 1243 1242 +8 1061 1062 1075 1074 1230 1231 1244 1243 +8 1062 1063 1076 1075 1231 1232 1245 1244 +8 1063 1064 1077 1076 1232 1233 1246 1245 +8 1064 1065 1078 1077 1233 1234 1247 1246 +8 1066 1067 1080 1079 1235 1236 1249 1248 +8 1067 1068 1081 1080 1236 1237 1250 1249 +8 1068 1069 1082 1081 1237 1238 1251 1250 +8 1069 1070 1083 1082 1238 1239 1252 1251 +8 1070 1071 1084 1083 1239 1240 1253 1252 +8 1071 1072 1085 1084 1240 1241 1254 1253 +8 1072 1073 1086 1085 1241 1242 1255 1254 +8 1073 1074 1087 1086 1242 1243 1256 1255 +8 1074 1075 1088 1087 1243 1244 1257 1256 +8 1075 1076 1089 1088 1244 1245 1258 1257 +8 1076 1077 1090 1089 1245 1246 1259 1258 +8 1077 1078 1091 1090 1246 1247 1260 1259 +8 1079 1080 1093 1092 1248 1249 1262 1261 +8 1080 1081 1094 1093 1249 1250 1263 1262 +8 1081 1082 1095 1094 1250 1251 1264 1263 +8 1082 1083 1096 1095 1251 1252 1265 1264 +8 1083 1084 1097 1096 1252 1253 1266 1265 +8 1084 1085 1098 1097 1253 1254 1267 1266 +8 1085 1086 1099 1098 1254 1255 1268 1267 +8 1086 1087 1100 1099 1255 1256 1269 1268 +8 1087 1088 1101 1100 1256 1257 1270 1269 +8 1088 1089 1102 1101 1257 1258 1271 1270 +8 1089 1090 1103 1102 1258 1259 1272 1271 +8 1090 1091 1104 1103 1259 1260 1273 1272 +8 1092 1093 1106 1105 1261 1262 1275 1274 +8 1093 1094 1107 1106 1262 1263 1276 1275 +8 1094 1095 1108 1107 1263 1264 1277 1276 +8 1095 1096 1109 1108 1264 1265 1278 1277 +8 1096 1097 1110 1109 1265 1266 1279 1278 +8 1097 1098 1111 1110 1266 1267 1280 1279 +8 1098 1099 1112 1111 1267 1268 1281 1280 +8 1099 1100 1113 1112 1268 1269 1282 1281 +8 1100 1101 1114 1113 1269 1270 1283 1282 +8 1101 1102 1115 1114 1270 1271 1284 1283 +8 1102 1103 1116 1115 1271 1272 1285 1284 +8 1103 1104 1117 1116 1272 1273 1286 1285 +8 1105 1106 1119 1118 1274 1275 1288 1287 +8 1106 1107 1120 1119 1275 1276 1289 1288 +8 1107 1108 1121 1120 1276 1277 1290 1289 +8 1108 1109 1122 1121 1277 1278 1291 1290 +8 1109 1110 1123 1122 1278 1279 1292 1291 +8 1110 1111 1124 1123 1279 1280 1293 1292 +8 1111 1112 1125 1124 1280 1281 1294 1293 +8 1112 1113 1126 1125 1281 1282 1295 1294 +8 1113 1114 1127 1126 1282 1283 1296 1295 +8 1114 1115 1128 1127 1283 1284 1297 1296 +8 1115 1116 1129 1128 1284 1285 1298 1297 +8 1116 1117 1130 1129 1285 1286 1299 1298 +8 1118 1119 1132 1131 1287 1288 1301 1300 +8 1119 1120 1133 1132 1288 1289 1302 1301 +8 1120 1121 1134 1133 1289 1290 1303 1302 +8 1121 1122 1135 1134 1290 1291 1304 1303 +8 1122 1123 1136 1135 1291 1292 1305 1304 +8 1123 1124 1137 1136 1292 1293 1306 1305 +8 1124 1125 1138 1137 1293 1294 1307 1306 +8 1125 1126 1139 1138 1294 1295 1308 1307 +8 1126 1127 1140 1139 1295 1296 1309 1308 +8 1127 1128 1141 1140 1296 1297 1310 1309 +8 1128 1129 1142 1141 1297 1298 1311 1310 +8 1129 1130 1143 1142 1298 1299 1312 1311 +8 1131 1132 1145 1144 1300 1301 1314 1313 +8 1132 1133 1146 1145 1301 1302 1315 1314 +8 1133 1134 1147 1146 1302 1303 1316 1315 +8 1134 1135 1148 1147 1303 1304 1317 1316 +8 1135 1136 1149 1148 1304 1305 1318 1317 +8 1136 1137 1150 1149 1305 1306 1319 1318 +8 1137 1138 1151 1150 1306 1307 1320 1319 +8 1138 1139 1152 1151 1307 1308 1321 1320 +8 1139 1140 1153 1152 1308 1309 1322 1321 +8 1140 1141 1154 1153 1309 1310 1323 1322 +8 1141 1142 1155 1154 1310 1311 1324 1323 +8 1142 1143 1156 1155 1311 1312 1325 1324 +8 1144 1145 1158 1157 1313 1314 1327 1326 +8 1145 1146 1159 1158 1314 1315 1328 1327 +8 1146 1147 1160 1159 1315 1316 1329 1328 +8 1147 1148 1161 1160 1316 1317 1330 1329 +8 1148 1149 1162 1161 1317 1318 1331 1330 +8 1149 1150 1163 1162 1318 1319 1332 1331 +8 1150 1151 1164 1163 1319 1320 1333 1332 +8 1151 1152 1165 1164 1320 1321 1334 1333 +8 1152 1153 1166 1165 1321 1322 1335 1334 +8 1153 1154 1167 1166 1322 1323 1336 1335 +8 1154 1155 1168 1167 1323 1324 1337 1336 +8 1155 1156 1169 1168 1324 1325 1338 1337 +8 1157 1158 1171 1170 1326 1327 1340 1339 +8 1158 1159 1172 1171 1327 1328 1341 1340 +8 1159 1160 1173 1172 1328 1329 1342 1341 +8 1160 1161 1174 1173 1329 1330 1343 1342 +8 1161 1162 1175 1174 1330 1331 1344 1343 +8 1162 1163 1176 1175 1331 1332 1345 1344 +8 1163 1164 1177 1176 1332 1333 1346 1345 +8 1164 1165 1178 1177 1333 1334 1347 1346 +8 1165 1166 1179 1178 1334 1335 1348 1347 +8 1166 1167 1180 1179 1335 1336 1349 1348 +8 1167 1168 1181 1180 1336 1337 1350 1349 +8 1168 1169 1182 1181 1337 1338 1351 1350 +8 1183 1184 1197 1196 1352 1353 1366 1365 +8 1184 1185 1198 1197 1353 1354 1367 1366 +8 1185 1186 1199 1198 1354 1355 1368 1367 +8 1186 1187 1200 1199 1355 1356 1369 1368 +8 1187 1188 1201 1200 1356 1357 1370 1369 +8 1188 1189 1202 1201 1357 1358 1371 1370 +8 1189 1190 1203 1202 1358 1359 1372 1371 +8 1190 1191 1204 1203 1359 1360 1373 1372 +8 1191 1192 1205 1204 1360 1361 1374 1373 +8 1192 1193 1206 1205 1361 1362 1375 1374 +8 1193 1194 1207 1206 1362 1363 1376 1375 +8 1194 1195 1208 1207 1363 1364 1377 1376 +8 1196 1197 1210 1209 1365 1366 1379 1378 +8 1197 1198 1211 1210 1366 1367 1380 1379 +8 1198 1199 1212 1211 1367 1368 1381 1380 +8 1199 1200 1213 1212 1368 1369 1382 1381 +8 1200 1201 1214 1213 1369 1370 1383 1382 +8 1201 1202 1215 1214 1370 1371 1384 1383 +8 1202 1203 1216 1215 1371 1372 1385 1384 +8 1203 1204 1217 1216 1372 1373 1386 1385 +8 1204 1205 1218 1217 1373 1374 1387 1386 +8 1205 1206 1219 1218 1374 1375 1388 1387 +8 1206 1207 1220 1219 1375 1376 1389 1388 +8 1207 1208 1221 1220 1376 1377 1390 1389 +8 1209 1210 1223 1222 1378 1379 1392 1391 +8 1210 1211 1224 1223 1379 1380 1393 1392 +8 1211 1212 1225 1224 1380 1381 1394 1393 +8 1212 1213 1226 1225 1381 1382 1395 1394 +8 1213 1214 1227 1226 1382 1383 1396 1395 +8 1214 1215 1228 1227 1383 1384 1397 1396 +8 1215 1216 1229 1228 1384 1385 1398 1397 +8 1216 1217 1230 1229 1385 1386 1399 1398 +8 1217 1218 1231 1230 1386 1387 1400 1399 +8 1218 1219 1232 1231 1387 1388 1401 1400 +8 1219 1220 1233 1232 1388 1389 1402 1401 +8 1220 1221 1234 1233 1389 1390 1403 1402 +8 1222 1223 1236 1235 1391 1392 1405 1404 +8 1223 1224 1237 1236 1392 1393 1406 1405 +8 1224 1225 1238 1237 1393 1394 1407 1406 +8 1225 1226 1239 1238 1394 1395 1408 1407 +8 1226 1227 1240 1239 1395 1396 1409 1408 +8 1227 1228 1241 1240 1396 1397 1410 1409 +8 1228 1229 1242 1241 1397 1398 1411 1410 +8 1229 1230 1243 1242 1398 1399 1412 1411 +8 1230 1231 1244 1243 1399 1400 1413 1412 +8 1231 1232 1245 1244 1400 1401 1414 1413 +8 1232 1233 1246 1245 1401 1402 1415 1414 +8 1233 1234 1247 1246 1402 1403 1416 1415 +8 1235 1236 1249 1248 1404 1405 1418 1417 +8 1236 1237 1250 1249 1405 1406 1419 1418 +8 1237 1238 1251 1250 1406 1407 1420 1419 +8 1238 1239 1252 1251 1407 1408 1421 1420 +8 1239 1240 1253 1252 1408 1409 1422 1421 +8 1240 1241 1254 1253 1409 1410 1423 1422 +8 1241 1242 1255 1254 1410 1411 1424 1423 +8 1242 1243 1256 1255 1411 1412 1425 1424 +8 1243 1244 1257 1256 1412 1413 1426 1425 +8 1244 1245 1258 1257 1413 1414 1427 1426 +8 1245 1246 1259 1258 1414 1415 1428 1427 +8 1246 1247 1260 1259 1415 1416 1429 1428 +8 1248 1249 1262 1261 1417 1418 1431 1430 +8 1249 1250 1263 1262 1418 1419 1432 1431 +8 1250 1251 1264 1263 1419 1420 1433 1432 +8 1251 1252 1265 1264 1420 1421 1434 1433 +8 1252 1253 1266 1265 1421 1422 1435 1434 +8 1253 1254 1267 1266 1422 1423 1436 1435 +8 1254 1255 1268 1267 1423 1424 1437 1436 +8 1255 1256 1269 1268 1424 1425 1438 1437 +8 1256 1257 1270 1269 1425 1426 1439 1438 +8 1257 1258 1271 1270 1426 1427 1440 1439 +8 1258 1259 1272 1271 1427 1428 1441 1440 +8 1259 1260 1273 1272 1428 1429 1442 1441 +8 1261 1262 1275 1274 1430 1431 1444 1443 +8 1262 1263 1276 1275 1431 1432 1445 1444 +8 1263 1264 1277 1276 1432 1433 1446 1445 +8 1264 1265 1278 1277 1433 1434 1447 1446 +8 1265 1266 1279 1278 1434 1435 1448 1447 +8 1266 1267 1280 1279 1435 1436 1449 1448 +8 1267 1268 1281 1280 1436 1437 1450 1449 +8 1268 1269 1282 1281 1437 1438 1451 1450 +8 1269 1270 1283 1282 1438 1439 1452 1451 +8 1270 1271 1284 1283 1439 1440 1453 1452 +8 1271 1272 1285 1284 1440 1441 1454 1453 +8 1272 1273 1286 1285 1441 1442 1455 1454 +8 1274 1275 1288 1287 1443 1444 1457 1456 +8 1275 1276 1289 1288 1444 1445 1458 1457 +8 1276 1277 1290 1289 1445 1446 1459 1458 +8 1277 1278 1291 1290 1446 1447 1460 1459 +8 1278 1279 1292 1291 1447 1448 1461 1460 +8 1279 1280 1293 1292 1448 1449 1462 1461 +8 1280 1281 1294 1293 1449 1450 1463 1462 +8 1281 1282 1295 1294 1450 1451 1464 1463 +8 1282 1283 1296 1295 1451 1452 1465 1464 +8 1283 1284 1297 1296 1452 1453 1466 1465 +8 1284 1285 1298 1297 1453 1454 1467 1466 +8 1285 1286 1299 1298 1454 1455 1468 1467 +8 1287 1288 1301 1300 1456 1457 1470 1469 +8 1288 1289 1302 1301 1457 1458 1471 1470 +8 1289 1290 1303 1302 1458 1459 1472 1471 +8 1290 1291 1304 1303 1459 1460 1473 1472 +8 1291 1292 1305 1304 1460 1461 1474 1473 +8 1292 1293 1306 1305 1461 1462 1475 1474 +8 1293 1294 1307 1306 1462 1463 1476 1475 +8 1294 1295 1308 1307 1463 1464 1477 1476 +8 1295 1296 1309 1308 1464 1465 1478 1477 +8 1296 1297 1310 1309 1465 1466 1479 1478 +8 1297 1298 1311 1310 1466 1467 1480 1479 +8 1298 1299 1312 1311 1467 1468 1481 1480 +8 1300 1301 1314 1313 1469 1470 1483 1482 +8 1301 1302 1315 1314 1470 1471 1484 1483 +8 1302 1303 1316 1315 1471 1472 1485 1484 +8 1303 1304 1317 1316 1472 1473 1486 1485 +8 1304 1305 1318 1317 1473 1474 1487 1486 +8 1305 1306 1319 1318 1474 1475 1488 1487 +8 1306 1307 1320 1319 1475 1476 1489 1488 +8 1307 1308 1321 1320 1476 1477 1490 1489 +8 1308 1309 1322 1321 1477 1478 1491 1490 +8 1309 1310 1323 1322 1478 1479 1492 1491 +8 1310 1311 1324 1323 1479 1480 1493 1492 +8 1311 1312 1325 1324 1480 1481 1494 1493 +8 1313 1314 1327 1326 1482 1483 1496 1495 +8 1314 1315 1328 1327 1483 1484 1497 1496 +8 1315 1316 1329 1328 1484 1485 1498 1497 +8 1316 1317 1330 1329 1485 1486 1499 1498 +8 1317 1318 1331 1330 1486 1487 1500 1499 +8 1318 1319 1332 1331 1487 1488 1501 1500 +8 1319 1320 1333 1332 1488 1489 1502 1501 +8 1320 1321 1334 1333 1489 1490 1503 1502 +8 1321 1322 1335 1334 1490 1491 1504 1503 +8 1322 1323 1336 1335 1491 1492 1505 1504 +8 1323 1324 1337 1336 1492 1493 1506 1505 +8 1324 1325 1338 1337 1493 1494 1507 1506 +8 1326 1327 1340 1339 1495 1496 1509 1508 +8 1327 1328 1341 1340 1496 1497 1510 1509 +8 1328 1329 1342 1341 1497 1498 1511 1510 +8 1329 1330 1343 1342 1498 1499 1512 1511 +8 1330 1331 1344 1343 1499 1500 1513 1512 +8 1331 1332 1345 1344 1500 1501 1514 1513 +8 1332 1333 1346 1345 1501 1502 1515 1514 +8 1333 1334 1347 1346 1502 1503 1516 1515 +8 1334 1335 1348 1347 1503 1504 1517 1516 +8 1335 1336 1349 1348 1504 1505 1518 1517 +8 1336 1337 1350 1349 1505 1506 1519 1518 +8 1337 1338 1351 1350 1506 1507 1520 1519 +8 1352 1353 1366 1365 1521 1522 1535 1534 +8 1353 1354 1367 1366 1522 1523 1536 1535 +8 1354 1355 1368 1367 1523 1524 1537 1536 +8 1355 1356 1369 1368 1524 1525 1538 1537 +8 1356 1357 1370 1369 1525 1526 1539 1538 +8 1357 1358 1371 1370 1526 1527 1540 1539 +8 1358 1359 1372 1371 1527 1528 1541 1540 +8 1359 1360 1373 1372 1528 1529 1542 1541 +8 1360 1361 1374 1373 1529 1530 1543 1542 +8 1361 1362 1375 1374 1530 1531 1544 1543 +8 1362 1363 1376 1375 1531 1532 1545 1544 +8 1363 1364 1377 1376 1532 1533 1546 1545 +8 1365 1366 1379 1378 1534 1535 1548 1547 +8 1366 1367 1380 1379 1535 1536 1549 1548 +8 1367 1368 1381 1380 1536 1537 1550 1549 +8 1368 1369 1382 1381 1537 1538 1551 1550 +8 1369 1370 1383 1382 1538 1539 1552 1551 +8 1370 1371 1384 1383 1539 1540 1553 1552 +8 1371 1372 1385 1384 1540 1541 1554 1553 +8 1372 1373 1386 1385 1541 1542 1555 1554 +8 1373 1374 1387 1386 1542 1543 1556 1555 +8 1374 1375 1388 1387 1543 1544 1557 1556 +8 1375 1376 1389 1388 1544 1545 1558 1557 +8 1376 1377 1390 1389 1545 1546 1559 1558 +8 1378 1379 1392 1391 1547 1548 1561 1560 +8 1379 1380 1393 1392 1548 1549 1562 1561 +8 1380 1381 1394 1393 1549 1550 1563 1562 +8 1381 1382 1395 1394 1550 1551 1564 1563 +8 1382 1383 1396 1395 1551 1552 1565 1564 +8 1383 1384 1397 1396 1552 1553 1566 1565 +8 1384 1385 1398 1397 1553 1554 1567 1566 +8 1385 1386 1399 1398 1554 1555 1568 1567 +8 1386 1387 1400 1399 1555 1556 1569 1568 +8 1387 1388 1401 1400 1556 1557 1570 1569 +8 1388 1389 1402 1401 1557 1558 1571 1570 +8 1389 1390 1403 1402 1558 1559 1572 1571 +8 1391 1392 1405 1404 1560 1561 1574 1573 +8 1392 1393 1406 1405 1561 1562 1575 1574 +8 1393 1394 1407 1406 1562 1563 1576 1575 +8 1394 1395 1408 1407 1563 1564 1577 1576 +8 1395 1396 1409 1408 1564 1565 1578 1577 +8 1396 1397 1410 1409 1565 1566 1579 1578 +8 1397 1398 1411 1410 1566 1567 1580 1579 +8 1398 1399 1412 1411 1567 1568 1581 1580 +8 1399 1400 1413 1412 1568 1569 1582 1581 +8 1400 1401 1414 1413 1569 1570 1583 1582 +8 1401 1402 1415 1414 1570 1571 1584 1583 +8 1402 1403 1416 1415 1571 1572 1585 1584 +8 1404 1405 1418 1417 1573 1574 1587 1586 +8 1405 1406 1419 1418 1574 1575 1588 1587 +8 1406 1407 1420 1419 1575 1576 1589 1588 +8 1407 1408 1421 1420 1576 1577 1590 1589 +8 1408 1409 1422 1421 1577 1578 1591 1590 +8 1409 1410 1423 1422 1578 1579 1592 1591 +8 1410 1411 1424 1423 1579 1580 1593 1592 +8 1411 1412 1425 1424 1580 1581 1594 1593 +8 1412 1413 1426 1425 1581 1582 1595 1594 +8 1413 1414 1427 1426 1582 1583 1596 1595 +8 1414 1415 1428 1427 1583 1584 1597 1596 +8 1415 1416 1429 1428 1584 1585 1598 1597 +8 1417 1418 1431 1430 1586 1587 1600 1599 +8 1418 1419 1432 1431 1587 1588 1601 1600 +8 1419 1420 1433 1432 1588 1589 1602 1601 +8 1420 1421 1434 1433 1589 1590 1603 1602 +8 1421 1422 1435 1434 1590 1591 1604 1603 +8 1422 1423 1436 1435 1591 1592 1605 1604 +8 1423 1424 1437 1436 1592 1593 1606 1605 +8 1424 1425 1438 1437 1593 1594 1607 1606 +8 1425 1426 1439 1438 1594 1595 1608 1607 +8 1426 1427 1440 1439 1595 1596 1609 1608 +8 1427 1428 1441 1440 1596 1597 1610 1609 +8 1428 1429 1442 1441 1597 1598 1611 1610 +8 1430 1431 1444 1443 1599 1600 1613 1612 +8 1431 1432 1445 1444 1600 1601 1614 1613 +8 1432 1433 1446 1445 1601 1602 1615 1614 +8 1433 1434 1447 1446 1602 1603 1616 1615 +8 1434 1435 1448 1447 1603 1604 1617 1616 +8 1435 1436 1449 1448 1604 1605 1618 1617 +8 1436 1437 1450 1449 1605 1606 1619 1618 +8 1437 1438 1451 1450 1606 1607 1620 1619 +8 1438 1439 1452 1451 1607 1608 1621 1620 +8 1439 1440 1453 1452 1608 1609 1622 1621 +8 1440 1441 1454 1453 1609 1610 1623 1622 +8 1441 1442 1455 1454 1610 1611 1624 1623 +8 1443 1444 1457 1456 1612 1613 1626 1625 +8 1444 1445 1458 1457 1613 1614 1627 1626 +8 1445 1446 1459 1458 1614 1615 1628 1627 +8 1446 1447 1460 1459 1615 1616 1629 1628 +8 1447 1448 1461 1460 1616 1617 1630 1629 +8 1448 1449 1462 1461 1617 1618 1631 1630 +8 1449 1450 1463 1462 1618 1619 1632 1631 +8 1450 1451 1464 1463 1619 1620 1633 1632 +8 1451 1452 1465 1464 1620 1621 1634 1633 +8 1452 1453 1466 1465 1621 1622 1635 1634 +8 1453 1454 1467 1466 1622 1623 1636 1635 +8 1454 1455 1468 1467 1623 1624 1637 1636 +8 1456 1457 1470 1469 1625 1626 1639 1638 +8 1457 1458 1471 1470 1626 1627 1640 1639 +8 1458 1459 1472 1471 1627 1628 1641 1640 +8 1459 1460 1473 1472 1628 1629 1642 1641 +8 1460 1461 1474 1473 1629 1630 1643 1642 +8 1461 1462 1475 1474 1630 1631 1644 1643 +8 1462 1463 1476 1475 1631 1632 1645 1644 +8 1463 1464 1477 1476 1632 1633 1646 1645 +8 1464 1465 1478 1477 1633 1634 1647 1646 +8 1465 1466 1479 1478 1634 1635 1648 1647 +8 1466 1467 1480 1479 1635 1636 1649 1648 +8 1467 1468 1481 1480 1636 1637 1650 1649 +8 1469 1470 1483 1482 1638 1639 1652 1651 +8 1470 1471 1484 1483 1639 1640 1653 1652 +8 1471 1472 1485 1484 1640 1641 1654 1653 +8 1472 1473 1486 1485 1641 1642 1655 1654 +8 1473 1474 1487 1486 1642 1643 1656 1655 +8 1474 1475 1488 1487 1643 1644 1657 1656 +8 1475 1476 1489 1488 1644 1645 1658 1657 +8 1476 1477 1490 1489 1645 1646 1659 1658 +8 1477 1478 1491 1490 1646 1647 1660 1659 +8 1478 1479 1492 1491 1647 1648 1661 1660 +8 1479 1480 1493 1492 1648 1649 1662 1661 +8 1480 1481 1494 1493 1649 1650 1663 1662 +8 1482 1483 1496 1495 1651 1652 1665 1664 +8 1483 1484 1497 1496 1652 1653 1666 1665 +8 1484 1485 1498 1497 1653 1654 1667 1666 +8 1485 1486 1499 1498 1654 1655 1668 1667 +8 1486 1487 1500 1499 1655 1656 1669 1668 +8 1487 1488 1501 1500 1656 1657 1670 1669 +8 1488 1489 1502 1501 1657 1658 1671 1670 +8 1489 1490 1503 1502 1658 1659 1672 1671 +8 1490 1491 1504 1503 1659 1660 1673 1672 +8 1491 1492 1505 1504 1660 1661 1674 1673 +8 1492 1493 1506 1505 1661 1662 1675 1674 +8 1493 1494 1507 1506 1662 1663 1676 1675 +8 1495 1496 1509 1508 1664 1665 1678 1677 +8 1496 1497 1510 1509 1665 1666 1679 1678 +8 1497 1498 1511 1510 1666 1667 1680 1679 +8 1498 1499 1512 1511 1667 1668 1681 1680 +8 1499 1500 1513 1512 1668 1669 1682 1681 +8 1500 1501 1514 1513 1669 1670 1683 1682 +8 1501 1502 1515 1514 1670 1671 1684 1683 +8 1502 1503 1516 1515 1671 1672 1685 1684 +8 1503 1504 1517 1516 1672 1673 1686 1685 +8 1504 1505 1518 1517 1673 1674 1687 1686 +8 1505 1506 1519 1518 1674 1675 1688 1687 +8 1506 1507 1520 1519 1675 1676 1689 1688 +8 1521 1522 1535 1534 1690 1691 1704 1703 +8 1522 1523 1536 1535 1691 1692 1705 1704 +8 1523 1524 1537 1536 1692 1693 1706 1705 +8 1524 1525 1538 1537 1693 1694 1707 1706 +8 1525 1526 1539 1538 1694 1695 1708 1707 +8 1526 1527 1540 1539 1695 1696 1709 1708 +8 1527 1528 1541 1540 1696 1697 1710 1709 +8 1528 1529 1542 1541 1697 1698 1711 1710 +8 1529 1530 1543 1542 1698 1699 1712 1711 +8 1530 1531 1544 1543 1699 1700 1713 1712 +8 1531 1532 1545 1544 1700 1701 1714 1713 +8 1532 1533 1546 1545 1701 1702 1715 1714 +8 1534 1535 1548 1547 1703 1704 1717 1716 +8 1535 1536 1549 1548 1704 1705 1718 1717 +8 1536 1537 1550 1549 1705 1706 1719 1718 +8 1537 1538 1551 1550 1706 1707 1720 1719 +8 1538 1539 1552 1551 1707 1708 1721 1720 +8 1539 1540 1553 1552 1708 1709 1722 1721 +8 1540 1541 1554 1553 1709 1710 1723 1722 +8 1541 1542 1555 1554 1710 1711 1724 1723 +8 1542 1543 1556 1555 1711 1712 1725 1724 +8 1543 1544 1557 1556 1712 1713 1726 1725 +8 1544 1545 1558 1557 1713 1714 1727 1726 +8 1545 1546 1559 1558 1714 1715 1728 1727 +8 1547 1548 1561 1560 1716 1717 1730 1729 +8 1548 1549 1562 1561 1717 1718 1731 1730 +8 1549 1550 1563 1562 1718 1719 1732 1731 +8 1550 1551 1564 1563 1719 1720 1733 1732 +8 1551 1552 1565 1564 1720 1721 1734 1733 +8 1552 1553 1566 1565 1721 1722 1735 1734 +8 1553 1554 1567 1566 1722 1723 1736 1735 +8 1554 1555 1568 1567 1723 1724 1737 1736 +8 1555 1556 1569 1568 1724 1725 1738 1737 +8 1556 1557 1570 1569 1725 1726 1739 1738 +8 1557 1558 1571 1570 1726 1727 1740 1739 +8 1558 1559 1572 1571 1727 1728 1741 1740 +8 1560 1561 1574 1573 1729 1730 1743 1742 +8 1561 1562 1575 1574 1730 1731 1744 1743 +8 1562 1563 1576 1575 1731 1732 1745 1744 +8 1563 1564 1577 1576 1732 1733 1746 1745 +8 1564 1565 1578 1577 1733 1734 1747 1746 +8 1565 1566 1579 1578 1734 1735 1748 1747 +8 1566 1567 1580 1579 1735 1736 1749 1748 +8 1567 1568 1581 1580 1736 1737 1750 1749 +8 1568 1569 1582 1581 1737 1738 1751 1750 +8 1569 1570 1583 1582 1738 1739 1752 1751 +8 1570 1571 1584 1583 1739 1740 1753 1752 +8 1571 1572 1585 1584 1740 1741 1754 1753 +8 1573 1574 1587 1586 1742 1743 1756 1755 +8 1574 1575 1588 1587 1743 1744 1757 1756 +8 1575 1576 1589 1588 1744 1745 1758 1757 +8 1576 1577 1590 1589 1745 1746 1759 1758 +8 1577 1578 1591 1590 1746 1747 1760 1759 +8 1578 1579 1592 1591 1747 1748 1761 1760 +8 1579 1580 1593 1592 1748 1749 1762 1761 +8 1580 1581 1594 1593 1749 1750 1763 1762 +8 1581 1582 1595 1594 1750 1751 1764 1763 +8 1582 1583 1596 1595 1751 1752 1765 1764 +8 1583 1584 1597 1596 1752 1753 1766 1765 +8 1584 1585 1598 1597 1753 1754 1767 1766 +8 1586 1587 1600 1599 1755 1756 1769 1768 +8 1587 1588 1601 1600 1756 1757 1770 1769 +8 1588 1589 1602 1601 1757 1758 1771 1770 +8 1589 1590 1603 1602 1758 1759 1772 1771 +8 1590 1591 1604 1603 1759 1760 1773 1772 +8 1591 1592 1605 1604 1760 1761 1774 1773 +8 1592 1593 1606 1605 1761 1762 1775 1774 +8 1593 1594 1607 1606 1762 1763 1776 1775 +8 1594 1595 1608 1607 1763 1764 1777 1776 +8 1595 1596 1609 1608 1764 1765 1778 1777 +8 1596 1597 1610 1609 1765 1766 1779 1778 +8 1597 1598 1611 1610 1766 1767 1780 1779 +8 1599 1600 1613 1612 1768 1769 1782 1781 +8 1600 1601 1614 1613 1769 1770 1783 1782 +8 1601 1602 1615 1614 1770 1771 1784 1783 +8 1602 1603 1616 1615 1771 1772 1785 1784 +8 1603 1604 1617 1616 1772 1773 1786 1785 +8 1604 1605 1618 1617 1773 1774 1787 1786 +8 1605 1606 1619 1618 1774 1775 1788 1787 +8 1606 1607 1620 1619 1775 1776 1789 1788 +8 1607 1608 1621 1620 1776 1777 1790 1789 +8 1608 1609 1622 1621 1777 1778 1791 1790 +8 1609 1610 1623 1622 1778 1779 1792 1791 +8 1610 1611 1624 1623 1779 1780 1793 1792 +8 1612 1613 1626 1625 1781 1782 1795 1794 +8 1613 1614 1627 1626 1782 1783 1796 1795 +8 1614 1615 1628 1627 1783 1784 1797 1796 +8 1615 1616 1629 1628 1784 1785 1798 1797 +8 1616 1617 1630 1629 1785 1786 1799 1798 +8 1617 1618 1631 1630 1786 1787 1800 1799 +8 1618 1619 1632 1631 1787 1788 1801 1800 +8 1619 1620 1633 1632 1788 1789 1802 1801 +8 1620 1621 1634 1633 1789 1790 1803 1802 +8 1621 1622 1635 1634 1790 1791 1804 1803 +8 1622 1623 1636 1635 1791 1792 1805 1804 +8 1623 1624 1637 1636 1792 1793 1806 1805 +8 1625 1626 1639 1638 1794 1795 1808 1807 +8 1626 1627 1640 1639 1795 1796 1809 1808 +8 1627 1628 1641 1640 1796 1797 1810 1809 +8 1628 1629 1642 1641 1797 1798 1811 1810 +8 1629 1630 1643 1642 1798 1799 1812 1811 +8 1630 1631 1644 1643 1799 1800 1813 1812 +8 1631 1632 1645 1644 1800 1801 1814 1813 +8 1632 1633 1646 1645 1801 1802 1815 1814 +8 1633 1634 1647 1646 1802 1803 1816 1815 +8 1634 1635 1648 1647 1803 1804 1817 1816 +8 1635 1636 1649 1648 1804 1805 1818 1817 +8 1636 1637 1650 1649 1805 1806 1819 1818 +8 1638 1639 1652 1651 1807 1808 1821 1820 +8 1639 1640 1653 1652 1808 1809 1822 1821 +8 1640 1641 1654 1653 1809 1810 1823 1822 +8 1641 1642 1655 1654 1810 1811 1824 1823 +8 1642 1643 1656 1655 1811 1812 1825 1824 +8 1643 1644 1657 1656 1812 1813 1826 1825 +8 1644 1645 1658 1657 1813 1814 1827 1826 +8 1645 1646 1659 1658 1814 1815 1828 1827 +8 1646 1647 1660 1659 1815 1816 1829 1828 +8 1647 1648 1661 1660 1816 1817 1830 1829 +8 1648 1649 1662 1661 1817 1818 1831 1830 +8 1649 1650 1663 1662 1818 1819 1832 1831 +8 1651 1652 1665 1664 1820 1821 1834 1833 +8 1652 1653 1666 1665 1821 1822 1835 1834 +8 1653 1654 1667 1666 1822 1823 1836 1835 +8 1654 1655 1668 1667 1823 1824 1837 1836 +8 1655 1656 1669 1668 1824 1825 1838 1837 +8 1656 1657 1670 1669 1825 1826 1839 1838 +8 1657 1658 1671 1670 1826 1827 1840 1839 +8 1658 1659 1672 1671 1827 1828 1841 1840 +8 1659 1660 1673 1672 1828 1829 1842 1841 +8 1660 1661 1674 1673 1829 1830 1843 1842 +8 1661 1662 1675 1674 1830 1831 1844 1843 +8 1662 1663 1676 1675 1831 1832 1845 1844 +8 1664 1665 1678 1677 1833 1834 1847 1846 +8 1665 1666 1679 1678 1834 1835 1848 1847 +8 1666 1667 1680 1679 1835 1836 1849 1848 +8 1667 1668 1681 1680 1836 1837 1850 1849 +8 1668 1669 1682 1681 1837 1838 1851 1850 +8 1669 1670 1683 1682 1838 1839 1852 1851 +8 1670 1671 1684 1683 1839 1840 1853 1852 +8 1671 1672 1685 1684 1840 1841 1854 1853 +8 1672 1673 1686 1685 1841 1842 1855 1854 +8 1673 1674 1687 1686 1842 1843 1856 1855 +8 1674 1675 1688 1687 1843 1844 1857 1856 +8 1675 1676 1689 1688 1844 1845 1858 1857 +8 1690 1691 1704 1703 1859 1860 1873 1872 +8 1691 1692 1705 1704 1860 1861 1874 1873 +8 1692 1693 1706 1705 1861 1862 1875 1874 +8 1693 1694 1707 1706 1862 1863 1876 1875 +8 1694 1695 1708 1707 1863 1864 1877 1876 +8 1695 1696 1709 1708 1864 1865 1878 1877 +8 1696 1697 1710 1709 1865 1866 1879 1878 +8 1697 1698 1711 1710 1866 1867 1880 1879 +8 1698 1699 1712 1711 1867 1868 1881 1880 +8 1699 1700 1713 1712 1868 1869 1882 1881 +8 1700 1701 1714 1713 1869 1870 1883 1882 +8 1701 1702 1715 1714 1870 1871 1884 1883 +8 1703 1704 1717 1716 1872 1873 1886 1885 +8 1704 1705 1718 1717 1873 1874 1887 1886 +8 1705 1706 1719 1718 1874 1875 1888 1887 +8 1706 1707 1720 1719 1875 1876 1889 1888 +8 1707 1708 1721 1720 1876 1877 1890 1889 +8 1708 1709 1722 1721 1877 1878 1891 1890 +8 1709 1710 1723 1722 1878 1879 1892 1891 +8 1710 1711 1724 1723 1879 1880 1893 1892 +8 1711 1712 1725 1724 1880 1881 1894 1893 +8 1712 1713 1726 1725 1881 1882 1895 1894 +8 1713 1714 1727 1726 1882 1883 1896 1895 +8 1714 1715 1728 1727 1883 1884 1897 1896 +8 1716 1717 1730 1729 1885 1886 1899 1898 +8 1717 1718 1731 1730 1886 1887 1900 1899 +8 1718 1719 1732 1731 1887 1888 1901 1900 +8 1719 1720 1733 1732 1888 1889 1902 1901 +8 1720 1721 1734 1733 1889 1890 1903 1902 +8 1721 1722 1735 1734 1890 1891 1904 1903 +8 1722 1723 1736 1735 1891 1892 1905 1904 +8 1723 1724 1737 1736 1892 1893 1906 1905 +8 1724 1725 1738 1737 1893 1894 1907 1906 +8 1725 1726 1739 1738 1894 1895 1908 1907 +8 1726 1727 1740 1739 1895 1896 1909 1908 +8 1727 1728 1741 1740 1896 1897 1910 1909 +8 1729 1730 1743 1742 1898 1899 1912 1911 +8 1730 1731 1744 1743 1899 1900 1913 1912 +8 1731 1732 1745 1744 1900 1901 1914 1913 +8 1732 1733 1746 1745 1901 1902 1915 1914 +8 1733 1734 1747 1746 1902 1903 1916 1915 +8 1734 1735 1748 1747 1903 1904 1917 1916 +8 1735 1736 1749 1748 1904 1905 1918 1917 +8 1736 1737 1750 1749 1905 1906 1919 1918 +8 1737 1738 1751 1750 1906 1907 1920 1919 +8 1738 1739 1752 1751 1907 1908 1921 1920 +8 1739 1740 1753 1752 1908 1909 1922 1921 +8 1740 1741 1754 1753 1909 1910 1923 1922 +8 1742 1743 1756 1755 1911 1912 1925 1924 +8 1743 1744 1757 1756 1912 1913 1926 1925 +8 1744 1745 1758 1757 1913 1914 1927 1926 +8 1745 1746 1759 1758 1914 1915 1928 1927 +8 1746 1747 1760 1759 1915 1916 1929 1928 +8 1747 1748 1761 1760 1916 1917 1930 1929 +8 1748 1749 1762 1761 1917 1918 1931 1930 +8 1749 1750 1763 1762 1918 1919 1932 1931 +8 1750 1751 1764 1763 1919 1920 1933 1932 +8 1751 1752 1765 1764 1920 1921 1934 1933 +8 1752 1753 1766 1765 1921 1922 1935 1934 +8 1753 1754 1767 1766 1922 1923 1936 1935 +8 1755 1756 1769 1768 1924 1925 1938 1937 +8 1756 1757 1770 1769 1925 1926 1939 1938 +8 1757 1758 1771 1770 1926 1927 1940 1939 +8 1758 1759 1772 1771 1927 1928 1941 1940 +8 1759 1760 1773 1772 1928 1929 1942 1941 +8 1760 1761 1774 1773 1929 1930 1943 1942 +8 1761 1762 1775 1774 1930 1931 1944 1943 +8 1762 1763 1776 1775 1931 1932 1945 1944 +8 1763 1764 1777 1776 1932 1933 1946 1945 +8 1764 1765 1778 1777 1933 1934 1947 1946 +8 1765 1766 1779 1778 1934 1935 1948 1947 +8 1766 1767 1780 1779 1935 1936 1949 1948 +8 1768 1769 1782 1781 1937 1938 1951 1950 +8 1769 1770 1783 1782 1938 1939 1952 1951 +8 1770 1771 1784 1783 1939 1940 1953 1952 +8 1771 1772 1785 1784 1940 1941 1954 1953 +8 1772 1773 1786 1785 1941 1942 1955 1954 +8 1773 1774 1787 1786 1942 1943 1956 1955 +8 1774 1775 1788 1787 1943 1944 1957 1956 +8 1775 1776 1789 1788 1944 1945 1958 1957 +8 1776 1777 1790 1789 1945 1946 1959 1958 +8 1777 1778 1791 1790 1946 1947 1960 1959 +8 1778 1779 1792 1791 1947 1948 1961 1960 +8 1779 1780 1793 1792 1948 1949 1962 1961 +8 1781 1782 1795 1794 1950 1951 1964 1963 +8 1782 1783 1796 1795 1951 1952 1965 1964 +8 1783 1784 1797 1796 1952 1953 1966 1965 +8 1784 1785 1798 1797 1953 1954 1967 1966 +8 1785 1786 1799 1798 1954 1955 1968 1967 +8 1786 1787 1800 1799 1955 1956 1969 1968 +8 1787 1788 1801 1800 1956 1957 1970 1969 +8 1788 1789 1802 1801 1957 1958 1971 1970 +8 1789 1790 1803 1802 1958 1959 1972 1971 +8 1790 1791 1804 1803 1959 1960 1973 1972 +8 1791 1792 1805 1804 1960 1961 1974 1973 +8 1792 1793 1806 1805 1961 1962 1975 1974 +8 1794 1795 1808 1807 1963 1964 1977 1976 +8 1795 1796 1809 1808 1964 1965 1978 1977 +8 1796 1797 1810 1809 1965 1966 1979 1978 +8 1797 1798 1811 1810 1966 1967 1980 1979 +8 1798 1799 1812 1811 1967 1968 1981 1980 +8 1799 1800 1813 1812 1968 1969 1982 1981 +8 1800 1801 1814 1813 1969 1970 1983 1982 +8 1801 1802 1815 1814 1970 1971 1984 1983 +8 1802 1803 1816 1815 1971 1972 1985 1984 +8 1803 1804 1817 1816 1972 1973 1986 1985 +8 1804 1805 1818 1817 1973 1974 1987 1986 +8 1805 1806 1819 1818 1974 1975 1988 1987 +8 1807 1808 1821 1820 1976 1977 1990 1989 +8 1808 1809 1822 1821 1977 1978 1991 1990 +8 1809 1810 1823 1822 1978 1979 1992 1991 +8 1810 1811 1824 1823 1979 1980 1993 1992 +8 1811 1812 1825 1824 1980 1981 1994 1993 +8 1812 1813 1826 1825 1981 1982 1995 1994 +8 1813 1814 1827 1826 1982 1983 1996 1995 +8 1814 1815 1828 1827 1983 1984 1997 1996 +8 1815 1816 1829 1828 1984 1985 1998 1997 +8 1816 1817 1830 1829 1985 1986 1999 1998 +8 1817 1818 1831 1830 1986 1987 2000 1999 +8 1818 1819 1832 1831 1987 1988 2001 2000 +8 1820 1821 1834 1833 1989 1990 2003 2002 +8 1821 1822 1835 1834 1990 1991 2004 2003 +8 1822 1823 1836 1835 1991 1992 2005 2004 +8 1823 1824 1837 1836 1992 1993 2006 2005 +8 1824 1825 1838 1837 1993 1994 2007 2006 +8 1825 1826 1839 1838 1994 1995 2008 2007 +8 1826 1827 1840 1839 1995 1996 2009 2008 +8 1827 1828 1841 1840 1996 1997 2010 2009 +8 1828 1829 1842 1841 1997 1998 2011 2010 +8 1829 1830 1843 1842 1998 1999 2012 2011 +8 1830 1831 1844 1843 1999 2000 2013 2012 +8 1831 1832 1845 1844 2000 2001 2014 2013 +8 1833 1834 1847 1846 2002 2003 2016 2015 +8 1834 1835 1848 1847 2003 2004 2017 2016 +8 1835 1836 1849 1848 2004 2005 2018 2017 +8 1836 1837 1850 1849 2005 2006 2019 2018 +8 1837 1838 1851 1850 2006 2007 2020 2019 +8 1838 1839 1852 1851 2007 2008 2021 2020 +8 1839 1840 1853 1852 2008 2009 2022 2021 +8 1840 1841 1854 1853 2009 2010 2023 2022 +8 1841 1842 1855 1854 2010 2011 2024 2023 +8 1842 1843 1856 1855 2011 2012 2025 2024 +8 1843 1844 1857 1856 2012 2013 2026 2025 +8 1844 1845 1858 1857 2013 2014 2027 2026 +8 1859 1860 1873 1872 2028 2029 2042 2041 +8 1860 1861 1874 1873 2029 2030 2043 2042 +8 1861 1862 1875 1874 2030 2031 2044 2043 +8 1862 1863 1876 1875 2031 2032 2045 2044 +8 1863 1864 1877 1876 2032 2033 2046 2045 +8 1864 1865 1878 1877 2033 2034 2047 2046 +8 1865 1866 1879 1878 2034 2035 2048 2047 +8 1866 1867 1880 1879 2035 2036 2049 2048 +8 1867 1868 1881 1880 2036 2037 2050 2049 +8 1868 1869 1882 1881 2037 2038 2051 2050 +8 1869 1870 1883 1882 2038 2039 2052 2051 +8 1870 1871 1884 1883 2039 2040 2053 2052 +8 1872 1873 1886 1885 2041 2042 2055 2054 +8 1873 1874 1887 1886 2042 2043 2056 2055 +8 1874 1875 1888 1887 2043 2044 2057 2056 +8 1875 1876 1889 1888 2044 2045 2058 2057 +8 1876 1877 1890 1889 2045 2046 2059 2058 +8 1877 1878 1891 1890 2046 2047 2060 2059 +8 1878 1879 1892 1891 2047 2048 2061 2060 +8 1879 1880 1893 1892 2048 2049 2062 2061 +8 1880 1881 1894 1893 2049 2050 2063 2062 +8 1881 1882 1895 1894 2050 2051 2064 2063 +8 1882 1883 1896 1895 2051 2052 2065 2064 +8 1883 1884 1897 1896 2052 2053 2066 2065 +8 1885 1886 1899 1898 2054 2055 2068 2067 +8 1886 1887 1900 1899 2055 2056 2069 2068 +8 1887 1888 1901 1900 2056 2057 2070 2069 +8 1888 1889 1902 1901 2057 2058 2071 2070 +8 1889 1890 1903 1902 2058 2059 2072 2071 +8 1890 1891 1904 1903 2059 2060 2073 2072 +8 1891 1892 1905 1904 2060 2061 2074 2073 +8 1892 1893 1906 1905 2061 2062 2075 2074 +8 1893 1894 1907 1906 2062 2063 2076 2075 +8 1894 1895 1908 1907 2063 2064 2077 2076 +8 1895 1896 1909 1908 2064 2065 2078 2077 +8 1896 1897 1910 1909 2065 2066 2079 2078 +8 1898 1899 1912 1911 2067 2068 2081 2080 +8 1899 1900 1913 1912 2068 2069 2082 2081 +8 1900 1901 1914 1913 2069 2070 2083 2082 +8 1901 1902 1915 1914 2070 2071 2084 2083 +8 1902 1903 1916 1915 2071 2072 2085 2084 +8 1903 1904 1917 1916 2072 2073 2086 2085 +8 1904 1905 1918 1917 2073 2074 2087 2086 +8 1905 1906 1919 1918 2074 2075 2088 2087 +8 1906 1907 1920 1919 2075 2076 2089 2088 +8 1907 1908 1921 1920 2076 2077 2090 2089 +8 1908 1909 1922 1921 2077 2078 2091 2090 +8 1909 1910 1923 1922 2078 2079 2092 2091 +8 1911 1912 1925 1924 2080 2081 2094 2093 +8 1912 1913 1926 1925 2081 2082 2095 2094 +8 1913 1914 1927 1926 2082 2083 2096 2095 +8 1914 1915 1928 1927 2083 2084 2097 2096 +8 1915 1916 1929 1928 2084 2085 2098 2097 +8 1916 1917 1930 1929 2085 2086 2099 2098 +8 1917 1918 1931 1930 2086 2087 2100 2099 +8 1918 1919 1932 1931 2087 2088 2101 2100 +8 1919 1920 1933 1932 2088 2089 2102 2101 +8 1920 1921 1934 1933 2089 2090 2103 2102 +8 1921 1922 1935 1934 2090 2091 2104 2103 +8 1922 1923 1936 1935 2091 2092 2105 2104 +8 1924 1925 1938 1937 2093 2094 2107 2106 +8 1925 1926 1939 1938 2094 2095 2108 2107 +8 1926 1927 1940 1939 2095 2096 2109 2108 +8 1927 1928 1941 1940 2096 2097 2110 2109 +8 1928 1929 1942 1941 2097 2098 2111 2110 +8 1929 1930 1943 1942 2098 2099 2112 2111 +8 1930 1931 1944 1943 2099 2100 2113 2112 +8 1931 1932 1945 1944 2100 2101 2114 2113 +8 1932 1933 1946 1945 2101 2102 2115 2114 +8 1933 1934 1947 1946 2102 2103 2116 2115 +8 1934 1935 1948 1947 2103 2104 2117 2116 +8 1935 1936 1949 1948 2104 2105 2118 2117 +8 1937 1938 1951 1950 2106 2107 2120 2119 +8 1938 1939 1952 1951 2107 2108 2121 2120 +8 1939 1940 1953 1952 2108 2109 2122 2121 +8 1940 1941 1954 1953 2109 2110 2123 2122 +8 1941 1942 1955 1954 2110 2111 2124 2123 +8 1942 1943 1956 1955 2111 2112 2125 2124 +8 1943 1944 1957 1956 2112 2113 2126 2125 +8 1944 1945 1958 1957 2113 2114 2127 2126 +8 1945 1946 1959 1958 2114 2115 2128 2127 +8 1946 1947 1960 1959 2115 2116 2129 2128 +8 1947 1948 1961 1960 2116 2117 2130 2129 +8 1948 1949 1962 1961 2117 2118 2131 2130 +8 1950 1951 1964 1963 2119 2120 2133 2132 +8 1951 1952 1965 1964 2120 2121 2134 2133 +8 1952 1953 1966 1965 2121 2122 2135 2134 +8 1953 1954 1967 1966 2122 2123 2136 2135 +8 1954 1955 1968 1967 2123 2124 2137 2136 +8 1955 1956 1969 1968 2124 2125 2138 2137 +8 1956 1957 1970 1969 2125 2126 2139 2138 +8 1957 1958 1971 1970 2126 2127 2140 2139 +8 1958 1959 1972 1971 2127 2128 2141 2140 +8 1959 1960 1973 1972 2128 2129 2142 2141 +8 1960 1961 1974 1973 2129 2130 2143 2142 +8 1961 1962 1975 1974 2130 2131 2144 2143 +8 1963 1964 1977 1976 2132 2133 2146 2145 +8 1964 1965 1978 1977 2133 2134 2147 2146 +8 1965 1966 1979 1978 2134 2135 2148 2147 +8 1966 1967 1980 1979 2135 2136 2149 2148 +8 1967 1968 1981 1980 2136 2137 2150 2149 +8 1968 1969 1982 1981 2137 2138 2151 2150 +8 1969 1970 1983 1982 2138 2139 2152 2151 +8 1970 1971 1984 1983 2139 2140 2153 2152 +8 1971 1972 1985 1984 2140 2141 2154 2153 +8 1972 1973 1986 1985 2141 2142 2155 2154 +8 1973 1974 1987 1986 2142 2143 2156 2155 +8 1974 1975 1988 1987 2143 2144 2157 2156 +8 1976 1977 1990 1989 2145 2146 2159 2158 +8 1977 1978 1991 1990 2146 2147 2160 2159 +8 1978 1979 1992 1991 2147 2148 2161 2160 +8 1979 1980 1993 1992 2148 2149 2162 2161 +8 1980 1981 1994 1993 2149 2150 2163 2162 +8 1981 1982 1995 1994 2150 2151 2164 2163 +8 1982 1983 1996 1995 2151 2152 2165 2164 +8 1983 1984 1997 1996 2152 2153 2166 2165 +8 1984 1985 1998 1997 2153 2154 2167 2166 +8 1985 1986 1999 1998 2154 2155 2168 2167 +8 1986 1987 2000 1999 2155 2156 2169 2168 +8 1987 1988 2001 2000 2156 2157 2170 2169 +8 1989 1990 2003 2002 2158 2159 2172 2171 +8 1990 1991 2004 2003 2159 2160 2173 2172 +8 1991 1992 2005 2004 2160 2161 2174 2173 +8 1992 1993 2006 2005 2161 2162 2175 2174 +8 1993 1994 2007 2006 2162 2163 2176 2175 +8 1994 1995 2008 2007 2163 2164 2177 2176 +8 1995 1996 2009 2008 2164 2165 2178 2177 +8 1996 1997 2010 2009 2165 2166 2179 2178 +8 1997 1998 2011 2010 2166 2167 2180 2179 +8 1998 1999 2012 2011 2167 2168 2181 2180 +8 1999 2000 2013 2012 2168 2169 2182 2181 +8 2000 2001 2014 2013 2169 2170 2183 2182 +8 2002 2003 2016 2015 2171 2172 2185 2184 +8 2003 2004 2017 2016 2172 2173 2186 2185 +8 2004 2005 2018 2017 2173 2174 2187 2186 +8 2005 2006 2019 2018 2174 2175 2188 2187 +8 2006 2007 2020 2019 2175 2176 2189 2188 +8 2007 2008 2021 2020 2176 2177 2190 2189 +8 2008 2009 2022 2021 2177 2178 2191 2190 +8 2009 2010 2023 2022 2178 2179 2192 2191 +8 2010 2011 2024 2023 2179 2180 2193 2192 +8 2011 2012 2025 2024 2180 2181 2194 2193 +8 2012 2013 2026 2025 2181 2182 2195 2194 +8 2013 2014 2027 2026 2182 2183 2196 2195 + +CELL_TYPES 1728 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 + +POINT_DATA 2197 +VECTORS velocity float +0.00000000 0.00000000 0.00000000 +0.15129286 0.00000000 0.00000000 +0.15323531 0.00000000 0.00000000 +0.18052624 0.00000000 0.00000000 +0.19922659 0.00000000 0.00000000 +0.21962195 0.00000000 0.00000000 +0.24196835 0.00000000 0.00000000 +0.26345823 0.00000000 0.00000000 +0.27808604 0.00000000 0.00000000 +0.35128500 0.00000000 0.00000000 +0.25113715 0.00000000 0.00000000 +0.38181582 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.15129286 0.00000000 +0.12109740 0.12109740 0.00000000 +0.16706916 0.08253341 0.00000000 +0.18507825 0.07255161 0.00000000 +0.20510401 0.06409840 0.00000000 +0.22557832 0.05784992 0.00000000 +0.24812410 0.05315190 0.00000000 +0.26935568 0.05031494 0.00000000 +0.29184205 0.05226268 0.00000000 +0.32796197 0.04779386 0.00000000 +0.34155637 0.05854756 0.00000000 +0.29245201 0.03622370 0.00000000 +0.00000000 0.00004101 0.00000000 +0.00000000 0.15323531 0.00000000 +0.08253341 0.16706916 0.00000000 +0.15129872 0.15129872 0.00000000 +0.18661483 0.12387973 0.00000000 +0.20744356 0.11091403 0.00000000 +0.23059254 0.09877480 0.00000000 +0.25292777 0.08951714 0.00000000 +0.27422715 0.08372217 0.00000000 +0.30646293 0.08576194 0.00000000 +0.28483792 0.07437695 0.00000000 +0.43878482 0.12180165 0.00000000 +0.08015231 0.02176584 0.00000000 +0.00000000 0.00001641 0.00000000 +0.00000000 0.18052624 0.00000000 +0.07255161 0.18507825 0.00000000 +0.12387973 0.18661483 0.00000000 +0.17156448 0.17156448 0.00000000 +0.20161696 0.14816375 0.00000000 +0.22568042 0.13415464 0.00000000 +0.24815988 0.12082245 0.00000000 +0.27809616 0.11779382 0.00000000 +0.28558967 0.10768783 0.00000000 +0.34061828 0.12922569 0.00000000 +0.32820362 0.11853629 0.00000000 +0.00927076 0.00327665 0.00000000 +0.00000000 0.00000048 0.00000000 +0.00000000 0.19922659 0.00000000 +0.06409840 0.20510401 0.00000000 +0.11091403 0.20744356 0.00000000 +0.14816375 0.20161696 0.00000000 +0.18839642 0.18839642 0.00000000 +0.21255306 0.16465196 0.00000000 +0.24407471 0.15550298 0.00000000 +0.26798333 0.14539135 0.00000000 +0.26576622 0.13184502 0.00000000 +0.39499686 0.18176438 0.00000000 +0.09442587 0.04187089 0.00000000 +0.00039903 0.00015905 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.21962195 0.00000000 +0.05784992 0.22557832 0.00000000 +0.09877480 0.23059254 0.00000000 +0.13415464 0.22568042 0.00000000 +0.16465196 0.21255306 0.00000000 +0.19805544 0.19805544 0.00000000 +0.23584271 0.18550201 0.00000000 +0.22778616 0.15266555 0.00000000 +0.33580943 0.20439457 0.00000000 +0.24988166 0.13969963 0.00000000 +0.00901370 0.00475284 0.00000000 +0.00000239 0.00000109 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.24196835 0.00000000 +0.05315190 0.24812410 0.00000000 +0.08951714 0.25292777 0.00000000 +0.12082245 0.24815988 0.00000000 +0.15550298 0.24407471 0.00000000 +0.18550201 0.23584271 0.00000000 +0.19670448 0.19670448 0.00000000 +0.27828749 0.22263260 0.00000000 +0.29186871 0.20940311 0.00000000 +0.03569997 0.02352921 0.00000000 +0.00016711 0.00009720 0.00000000 +0.00000001 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.26345823 0.00000000 +0.05031494 0.26935568 0.00000000 +0.08372217 0.27422715 0.00000000 +0.11779382 0.27809616 0.00000000 +0.14539135 0.26798333 0.00000000 +0.15266555 0.22778616 0.00000000 +0.22263260 0.27828749 0.00000000 +0.28020504 0.28020504 0.00000000 +0.06343861 0.05159365 0.00000000 +0.00103623 0.00076839 0.00000000 +0.00000017 0.00000011 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.27808604 0.00000000 +0.05226268 0.29184205 0.00000000 +0.08576194 0.30646293 0.00000000 +0.10768783 0.28558967 0.00000000 +0.13184502 0.26576622 0.00000000 +0.20439457 0.33580943 0.00000000 +0.20940311 0.29186871 0.00000000 +0.05159365 0.06343861 0.00000000 +0.00335127 0.00335127 0.00000000 +0.00000143 0.00000119 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.35128500 0.00000000 +0.04779386 0.32796197 0.00000000 +0.07437695 0.28483792 0.00000000 +0.12922569 0.34061828 0.00000000 +0.18176438 0.39499686 0.00000000 +0.13969963 0.24988166 0.00000000 +0.02352921 0.03569997 0.00000000 +0.00076839 0.00103623 0.00000000 +0.00000119 0.00000143 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.25113715 0.00000000 +0.05854756 0.34155637 0.00000000 +0.12180165 0.43878482 0.00000000 +0.11853629 0.32820362 0.00000000 +0.04187089 0.09442587 0.00000000 +0.00475284 0.00901370 0.00000000 +0.00009720 0.00016711 0.00000000 +0.00000011 0.00000017 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.38181582 0.00000000 +0.03622370 0.29245201 0.00000000 +0.02176584 0.08015231 0.00000000 +0.00327665 0.00927076 0.00000000 +0.00015905 0.00039903 0.00000000 +0.00000109 0.00000239 0.00000000 +0.00000000 0.00000001 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00004101 0.00000000 0.00000000 +0.00001641 0.00000000 0.00000000 +0.00000048 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.15129286 +0.12109740 0.00000000 0.12109740 +0.16706916 0.00000000 0.08253341 +0.18507825 0.00000000 0.07255161 +0.20510401 0.00000000 0.06409840 +0.22557832 0.00000000 0.05784992 +0.24812410 0.00000000 0.05315190 +0.26935568 0.00000000 0.05031494 +0.29184205 0.00000000 0.05226268 +0.32796197 0.00000000 0.04779386 +0.34155637 0.00000000 0.05854756 +0.29245201 0.00000000 0.03622370 +0.00000000 0.00000000 0.00004101 +0.00000000 0.12109740 0.12109740 +0.11194433 0.11194433 0.11194433 +0.16763493 0.08447348 0.08447348 +0.18818094 0.06881720 0.06881720 +0.20793413 0.06125917 0.06125917 +0.22912333 0.05484865 0.05484865 +0.25170547 0.05018152 0.05018152 +0.27224313 0.04698754 0.04698754 +0.30077655 0.04897950 0.04897950 +0.30800102 0.04250931 0.04250931 +0.40165640 0.06440826 0.06440826 +0.21450057 0.02766431 0.02766431 +0.00000000 0.00002538 0.00002538 +0.00000000 0.16706916 0.08253341 +0.08447348 0.16763493 0.08447348 +0.14671206 0.14671206 0.07661738 +0.18541418 0.12388847 0.06505718 +0.20769496 0.10849415 0.05558558 +0.23095378 0.09698467 0.04975907 +0.25306158 0.08754438 0.04486165 +0.27552492 0.08255333 0.04216991 +0.30537290 0.08302718 0.04225252 +0.28730540 0.07481661 0.03779717 +0.42864392 0.11485281 0.05668998 +0.05712571 0.01532577 0.00770589 +0.00000000 0.00001000 0.00000477 +0.00000000 0.18507825 0.07255161 +0.06881720 0.18818094 0.06881720 +0.12388847 0.18541418 0.06505718 +0.16862699 0.16862699 0.05971093 +0.20078594 0.14850555 0.05284890 +0.22513003 0.13297281 0.04626450 +0.24786893 0.12034648 0.04185713 +0.27864928 0.11728339 0.04033039 +0.27970791 0.10485952 0.03571615 +0.35622202 0.13373729 0.04466998 +0.29339533 0.10428998 0.03244443 +0.00661588 0.00230056 0.00077655 +0.00000000 0.00000030 0.00000008 +0.00000000 0.20510401 0.06409840 +0.06125917 0.20793413 0.06125917 +0.10849415 0.20769496 0.05558558 +0.14850555 0.20078594 0.05284890 +0.18594880 0.18594880 0.04921526 +0.21173705 0.16466959 0.04398982 +0.24448037 0.15544591 0.04078769 +0.26429520 0.14306836 0.03713853 +0.27336582 0.13597825 0.03482393 +0.38848962 0.17779283 0.04530383 +0.07612365 0.03370150 0.00840840 +0.00027532 0.00010882 0.00002590 +0.00000000 0.00000000 0.00000000 +0.00000000 0.22557832 0.05784992 +0.05484865 0.22912333 0.05484865 +0.09698467 0.23095378 0.04975907 +0.13297281 0.22513003 0.04626450 +0.16466959 0.21173705 0.04398982 +0.19891678 0.19891678 0.04227551 +0.23331275 0.18492540 0.03964287 +0.22853000 0.15384842 0.03262551 +0.34080524 0.20669146 0.04322289 +0.22244465 0.12399210 0.02378979 +0.00688265 0.00360923 0.00076146 +0.00000159 0.00000072 0.00000012 +0.00000000 0.00000000 0.00000000 +0.00000000 0.24812410 0.05315190 +0.05018152 0.25170547 0.05018152 +0.08754438 0.25306158 0.04486165 +0.12034648 0.24786893 0.04185713 +0.15544591 0.24448037 0.04078769 +0.18492540 0.23331275 0.03964287 +0.19466515 0.19466515 0.03375700 +0.28234193 0.22783320 0.04143619 +0.27650887 0.19774171 0.03245509 +0.02890315 0.01903327 0.00332244 +0.00011812 0.00006852 0.00001094 +0.00000001 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.26935568 0.05031494 +0.04698754 0.27224313 0.04698754 +0.08255333 0.27552492 0.04216991 +0.11728339 0.27864928 0.04033039 +0.14306836 0.26429520 0.03713853 +0.15384842 0.22853000 0.03262551 +0.22783320 0.28234193 0.04143619 +0.26686098 0.26686098 0.04015244 +0.05361946 0.04401921 0.00666538 +0.00077884 0.00057648 0.00008618 +0.00000012 0.00000008 0.00000001 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.29184205 0.05226268 +0.04897950 0.30077655 0.04897950 +0.08302718 0.30537290 0.04225252 +0.10485952 0.27970791 0.03571615 +0.13597825 0.27336582 0.03482393 +0.20669146 0.34080524 0.04322289 +0.19774171 0.27650887 0.03245509 +0.04401921 0.05361946 0.00666538 +0.00242573 0.00242573 0.00035678 +0.00000100 0.00000083 0.00000009 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.32796197 0.04779386 +0.04250931 0.30800102 0.04250931 +0.07481661 0.28730540 0.03779717 +0.13373729 0.35622202 0.04466998 +0.17779283 0.38848962 0.04530383 +0.12399210 0.22244465 0.02378979 +0.01903327 0.02890315 0.00332244 +0.00057648 0.00077884 0.00008618 +0.00000083 0.00000100 0.00000009 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.34155637 0.05854756 +0.06440826 0.40165640 0.06440826 +0.11485281 0.42864392 0.05668998 +0.10428998 0.29339533 0.03244443 +0.03370150 0.07612365 0.00840840 +0.00360923 0.00688265 0.00076146 +0.00006852 0.00011812 0.00001094 +0.00000008 0.00000012 0.00000001 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.29245201 0.03622370 +0.02766431 0.21450057 0.02766431 +0.01532577 0.05712571 0.00770589 +0.00230056 0.00661588 0.00077655 +0.00010882 0.00027532 0.00002590 +0.00000072 0.00000159 0.00000012 +0.00000000 0.00000001 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.00004101 +0.00002538 0.00000000 0.00002538 +0.00001000 0.00000000 0.00000477 +0.00000030 0.00000000 0.00000008 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.15323531 +0.08253341 0.00000000 0.16706916 +0.15129872 0.00000000 0.15129872 +0.18661483 0.00000000 0.12387973 +0.20744356 0.00000000 0.11091403 +0.23059254 0.00000000 0.09877480 +0.25292777 0.00000000 0.08951714 +0.27422715 0.00000000 0.08372217 +0.30646293 0.00000000 0.08576194 +0.28483792 0.00000000 0.07437695 +0.43878482 0.00000000 0.12180165 +0.08015231 0.00000000 0.02176584 +0.00000000 0.00000000 0.00001641 +0.00000000 0.08253341 0.16706916 +0.08447348 0.08447348 0.16763493 +0.14671206 0.07661738 0.14671206 +0.18541418 0.06505718 0.12388847 +0.20769496 0.05558558 0.10849415 +0.23095378 0.04975907 0.09698467 +0.25306158 0.04486165 0.08754438 +0.27552492 0.04216991 0.08255333 +0.30537290 0.04225252 0.08302718 +0.28730540 0.03779717 0.07481661 +0.42864392 0.05668998 0.11485281 +0.05712571 0.00770589 0.01532577 +0.00000000 0.00000477 0.00001000 +0.00000000 0.15129872 0.15129872 +0.07661738 0.14671206 0.14671206 +0.13524621 0.13524621 0.13524621 +0.17594823 0.11939755 0.11939755 +0.20477920 0.10411901 0.10411901 +0.22897201 0.09274519 0.09274519 +0.25098717 0.08373253 0.08373253 +0.27829336 0.08042743 0.08042743 +0.29232812 0.07512978 0.07512978 +0.32324693 0.08311482 0.08311482 +0.35807989 0.08667964 0.08667964 +0.01778447 0.00447407 0.00447407 +0.00000000 0.00000178 0.00000178 +0.00000000 0.18661483 0.12387973 +0.06505718 0.18541418 0.12388847 +0.11939755 0.17594823 0.11939755 +0.16106826 0.16106826 0.11029892 +0.19508786 0.14601426 0.09944481 +0.22159286 0.13001978 0.08845597 +0.24729509 0.11936413 0.08111633 +0.27596054 0.11453460 0.07738642 +0.26793238 0.09981428 0.06718600 +0.39079255 0.14162973 0.09486673 +0.18972336 0.06513445 0.04209815 +0.00221287 0.00072722 0.00048335 +0.00000000 0.00000006 0.00000004 +0.00000000 0.20744356 0.11091403 +0.05558558 0.20769496 0.10849415 +0.10411901 0.20477920 0.10411901 +0.14601426 0.19508786 0.09944481 +0.17985915 0.17985915 0.09249949 +0.20934788 0.16423396 0.08491019 +0.24378950 0.15508359 0.08008805 +0.25175385 0.13583805 0.06946288 +0.29942689 0.14900888 0.07502505 +0.35117617 0.15836951 0.07863162 +0.03790964 0.01665534 0.00841665 +0.00008359 0.00003212 0.00001515 +0.00000000 0.00000000 0.00000000 +0.00000000 0.23059254 0.09877480 +0.04975907 0.23095378 0.09698467 +0.09274519 0.22897201 0.09274519 +0.13001978 0.22159286 0.08845597 +0.16423396 0.20934788 0.08491019 +0.19957410 0.19957410 0.08257578 +0.22329079 0.17940160 0.07444742 +0.23871448 0.16256273 0.06735030 +0.34498469 0.20762137 0.08568348 +0.14694051 0.08145713 0.03193184 +0.00293995 0.00151442 0.00062559 +0.00000044 0.00000020 0.00000007 +0.00000000 0.00000000 0.00000000 +0.00000000 0.25292777 0.08951714 +0.04486165 0.25306158 0.08754438 +0.08373253 0.25098717 0.08373253 +0.11936413 0.24729509 0.08111633 +0.15508359 0.24378950 0.08008805 +0.17940160 0.22329079 0.07444742 +0.19769699 0.19769699 0.06773175 +0.29264331 0.24083547 0.08576193 +0.22308379 0.15910566 0.05178263 +0.01485261 0.00974796 0.00339455 +0.00003945 0.00002269 0.00000711 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.27422715 0.08372217 +0.04216991 0.27552492 0.08255333 +0.08042743 0.27829336 0.08042743 +0.11453460 0.27596054 0.07738642 +0.13583805 0.25175385 0.06946288 +0.16256273 0.23871448 0.06735030 +0.24083547 0.29264331 0.08576193 +0.21988464 0.21988464 0.06371968 +0.03079889 0.02582956 0.00786198 +0.00031769 0.00023404 0.00006772 +0.00000004 0.00000003 0.00000001 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.30646293 0.08576194 +0.04225252 0.30537290 0.08302718 +0.07512978 0.29232812 0.07512978 +0.09981428 0.26793238 0.06718600 +0.14900888 0.29942689 0.07502505 +0.20762137 0.34498469 0.08568348 +0.15910566 0.22308379 0.05178263 +0.02582956 0.03079889 0.00786198 +0.00088028 0.00088028 0.00023977 +0.00000033 0.00000028 0.00000006 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.28483792 0.07437695 +0.03779717 0.28730540 0.07481661 +0.08311482 0.32324693 0.08311482 +0.14162973 0.39079255 0.09486673 +0.15836951 0.35117617 0.07863162 +0.08145713 0.14694051 0.03193184 +0.00974796 0.01485261 0.00339455 +0.00023404 0.00031769 0.00006772 +0.00000028 0.00000033 0.00000006 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.43878482 0.12180165 +0.05668998 0.42864392 0.11485281 +0.08667964 0.35807989 0.08667964 +0.06513445 0.18972336 0.04209815 +0.01665534 0.03790964 0.00841665 +0.00151442 0.00293995 0.00062559 +0.00002269 0.00003945 0.00000711 +0.00000003 0.00000004 0.00000001 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.08015231 0.02176584 +0.00770589 0.05712571 0.01532577 +0.00447407 0.01778447 0.00447407 +0.00072722 0.00221287 0.00048335 +0.00003212 0.00008359 0.00001515 +0.00000020 0.00000044 0.00000007 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.00001641 +0.00000477 0.00000000 0.00001000 +0.00000178 0.00000000 0.00000178 +0.00000006 0.00000000 0.00000004 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.18052624 +0.07255161 0.00000000 0.18507825 +0.12387973 0.00000000 0.18661483 +0.17156448 0.00000000 0.17156448 +0.20161696 0.00000000 0.14816375 +0.22568042 0.00000000 0.13415464 +0.24815988 0.00000000 0.12082245 +0.27809616 0.00000000 0.11779382 +0.28558967 0.00000000 0.10768783 +0.34061828 0.00000000 0.12922569 +0.32820362 0.00000000 0.11853629 +0.00927076 0.00000000 0.00327665 +0.00000000 0.00000000 0.00000048 +0.00000000 0.07255161 0.18507825 +0.06881720 0.06881720 0.18818094 +0.12388847 0.06505718 0.18541418 +0.16862699 0.05971093 0.16862699 +0.20078594 0.05284890 0.14850555 +0.22513003 0.04626450 0.13297281 +0.24786893 0.04185713 0.12034648 +0.27864928 0.04033039 0.11728339 +0.27970791 0.03571615 0.10485952 +0.35622202 0.04466998 0.13373729 +0.29339533 0.03244443 0.10428998 +0.00661588 0.00077655 0.00230056 +0.00000000 0.00000008 0.00000030 +0.00000000 0.12387973 0.18661483 +0.06505718 0.12388847 0.18541418 +0.11939755 0.11939755 0.17594823 +0.16106826 0.11029892 0.16106826 +0.19508786 0.09944481 0.14601426 +0.22159286 0.08845597 0.13001978 +0.24729509 0.08111633 0.11936413 +0.27596054 0.07738642 0.11453460 +0.26793238 0.06718600 0.09981428 +0.39079255 0.09486673 0.14162973 +0.18972336 0.04209815 0.06513445 +0.00221287 0.00048335 0.00072722 +0.00000000 0.00000004 0.00000006 +0.00000000 0.17156448 0.17156448 +0.05971093 0.16862699 0.16862699 +0.11029892 0.16106826 0.16106826 +0.15140484 0.15140484 0.15140484 +0.18473471 0.13903198 0.13903198 +0.21415373 0.12654261 0.12654261 +0.24571981 0.11846918 0.11846918 +0.26056065 0.10657151 0.10657151 +0.28272496 0.10587090 0.10587090 +0.37633366 0.12899042 0.12899042 +0.06562444 0.02195203 0.02195203 +0.00028331 0.00008430 0.00008430 +0.00000000 0.00000000 0.00000000 +0.00000000 0.20161696 0.14816375 +0.05284890 0.20078594 0.14850555 +0.09944481 0.19508786 0.14601426 +0.13903198 0.18473471 0.13903198 +0.17280630 0.17280630 0.13037117 +0.20593067 0.16267392 0.12321924 +0.23536902 0.15087424 0.11447137 +0.23783199 0.12929001 0.09792657 +0.34029210 0.16756172 0.12658670 +0.24698436 0.10955503 0.08071337 +0.01040427 0.00446799 0.00337684 +0.00000867 0.00000318 0.00000229 +0.00000000 0.00000000 0.00000000 +0.00000000 0.22568042 0.13415464 +0.04626450 0.22513003 0.13297281 +0.08845597 0.22159286 0.13001978 +0.12654261 0.21415373 0.12654261 +0.16267392 0.20593067 0.12321924 +0.19476779 0.19476779 0.11779403 +0.20768059 0.16858893 0.10276335 +0.26752042 0.18489086 0.11274378 +0.31261553 0.18628836 0.11184057 +0.05968429 0.03299667 0.01991438 +0.00062314 0.00031089 0.00018728 +0.00000005 0.00000002 0.00000001 +0.00000000 0.00000000 0.00000000 +0.00000000 0.24815988 0.12082245 +0.04185713 0.24786893 0.12034648 +0.08111633 0.24729509 0.11936413 +0.11846918 0.24571981 0.11846918 +0.15087424 0.23536902 0.11447137 +0.16858893 0.20768059 0.10276335 +0.21943567 0.21943567 0.11144995 +0.28722793 0.24046303 0.12382638 +0.12780462 0.09195493 0.04575108 +0.00443936 0.00288967 0.00149160 +0.00000523 0.00000297 0.00000139 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.27809616 0.11779382 +0.04033039 0.27864928 0.11728339 +0.07738642 0.27596054 0.11453460 +0.10657151 0.26056065 0.10657151 +0.12929001 0.23783199 0.09792657 +0.18489086 0.26752042 0.11274378 +0.24046303 0.28722793 0.12382638 +0.13479603 0.13479603 0.05758260 +0.01050494 0.00897675 0.00406497 +0.00006176 0.00004529 0.00001896 +0.00000001 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.28558967 0.10768783 +0.03571615 0.27970791 0.10485952 +0.06718600 0.26793238 0.09981428 +0.10587090 0.28272496 0.10587090 +0.16756172 0.34029210 0.12658670 +0.18628836 0.31261553 0.11184057 +0.09195493 0.12780462 0.04575108 +0.00897675 0.01050494 0.00406497 +0.00014707 0.00014707 0.00005548 +0.00000005 0.00000004 0.00000001 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.34061828 0.12922569 +0.04466998 0.35622202 0.13373729 +0.09486673 0.39079255 0.14162973 +0.12899042 0.37633366 0.12899042 +0.10955503 0.24698436 0.08071337 +0.03299667 0.05968429 0.01991438 +0.00288967 0.00443936 0.00149160 +0.00004529 0.00006176 0.00001896 +0.00000004 0.00000005 0.00000001 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.32820362 0.11853629 +0.03244443 0.29339533 0.10428998 +0.04209815 0.18972336 0.06513445 +0.02195203 0.06562444 0.02195203 +0.00446799 0.01040427 0.00337684 +0.00031089 0.00062314 0.00018728 +0.00000297 0.00000523 0.00000139 +0.00000000 0.00000001 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00927076 0.00327665 +0.00077655 0.00661588 0.00230056 +0.00048335 0.00221287 0.00072722 +0.00008430 0.00028331 0.00008430 +0.00000318 0.00000867 0.00000229 +0.00000002 0.00000005 0.00000001 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.00000048 +0.00000008 0.00000000 0.00000030 +0.00000004 0.00000000 0.00000006 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.19922659 +0.06409840 0.00000000 0.20510401 +0.11091403 0.00000000 0.20744356 +0.14816375 0.00000000 0.20161696 +0.18839642 0.00000000 0.18839642 +0.21255306 0.00000000 0.16465196 +0.24407471 0.00000000 0.15550298 +0.26798333 0.00000000 0.14539135 +0.26576622 0.00000000 0.13184502 +0.39499686 0.00000000 0.18176438 +0.09442587 0.00000000 0.04187089 +0.00039903 0.00000000 0.00015905 +0.00000000 0.00000000 0.00000000 +0.00000000 0.06409840 0.20510401 +0.06125917 0.06125917 0.20793413 +0.10849415 0.05558558 0.20769496 +0.14850555 0.05284890 0.20078594 +0.18594880 0.04921526 0.18594880 +0.21173705 0.04398982 0.16466959 +0.24448037 0.04078769 0.15544591 +0.26429520 0.03713853 0.14306836 +0.27336582 0.03482393 0.13597825 +0.38848962 0.04530383 0.17779283 +0.07612365 0.00840840 0.03370150 +0.00027532 0.00002590 0.00010882 +0.00000000 0.00000000 0.00000000 +0.00000000 0.11091403 0.20744356 +0.05558558 0.10849415 0.20769496 +0.10411901 0.10411901 0.20477920 +0.14601426 0.09944481 0.19508786 +0.17985915 0.09249949 0.17985915 +0.20934788 0.08491019 0.16423396 +0.24378950 0.08008805 0.15508359 +0.25175385 0.06946288 0.13583805 +0.29942689 0.07502505 0.14900888 +0.35117617 0.07863162 0.15836951 +0.03790964 0.00841665 0.01665534 +0.00008359 0.00001515 0.00003212 +0.00000000 0.00000000 0.00000000 +0.00000000 0.14816375 0.20161696 +0.05284890 0.14850555 0.20078594 +0.09944481 0.14601426 0.19508786 +0.13903198 0.13903198 0.18473471 +0.17280630 0.13037117 0.17280630 +0.20593067 0.12321924 0.16267392 +0.23536902 0.11447137 0.15087424 +0.23783199 0.09792657 0.12929001 +0.34029210 0.12658670 0.16756172 +0.24698436 0.08071337 0.10955503 +0.01040427 0.00337684 0.00446799 +0.00000867 0.00000229 0.00000318 +0.00000000 0.00000000 0.00000000 +0.00000000 0.18839642 0.18839642 +0.04921526 0.18594880 0.18594880 +0.09249949 0.17985915 0.17985915 +0.13037117 0.17280630 0.17280630 +0.16611896 0.16611896 0.16611896 +0.19930738 0.15831015 0.15831015 +0.21424615 0.13877265 0.13877265 +0.25662287 0.14200517 0.14200517 +0.32983805 0.15821767 0.15821767 +0.09124718 0.04040967 0.04040967 +0.00132483 0.00054062 0.00054062 +0.00000020 0.00000007 0.00000007 +0.00000000 0.00000000 0.00000000 +0.00000000 0.21255306 0.16465196 +0.04398982 0.21173705 0.16466959 +0.08491019 0.20934788 0.16423396 +0.12321924 0.20593067 0.16267392 +0.15831015 0.19930738 0.15831015 +0.18007570 0.18007570 0.14379405 +0.21044762 0.17305007 0.13879825 +0.29860258 0.20935749 0.16849324 +0.20675213 0.12296526 0.09726934 +0.01280627 0.00705675 0.00569046 +0.00004980 0.00002382 0.00001877 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.24407471 0.15550298 +0.04078769 0.24448037 0.15544591 +0.08008805 0.24378950 0.15508359 +0.11447137 0.23536902 0.15087424 +0.13877265 0.21424615 0.13877265 +0.17305007 0.21044762 0.13879825 +0.25132266 0.25132266 0.16847290 +0.22475615 0.18922115 0.12425884 +0.03895527 0.02854692 0.01923163 +0.00066014 0.00042264 0.00028438 +0.00000021 0.00000012 0.00000008 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.26798333 0.14539135 +0.03713853 0.26429520 0.14306836 +0.06946288 0.25175385 0.13583805 +0.09792657 0.23783199 0.12929001 +0.14200517 0.25662287 0.14200517 +0.20935749 0.29860258 0.16849324 +0.18922115 0.22475615 0.12425884 +0.04653578 0.04653578 0.02666223 +0.00183246 0.00157772 0.00092690 +0.00000432 0.00000317 0.00000174 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.26576622 0.13184502 +0.03482393 0.27336582 0.13597825 +0.07502505 0.29942689 0.14900888 +0.12658670 0.34029210 0.16756172 +0.15821767 0.32983805 0.15821767 +0.12296526 0.20675213 0.09726934 +0.02854692 0.03895527 0.01923163 +0.00157772 0.00183246 0.00092690 +0.00000966 0.00000966 0.00000463 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.39499686 0.18176438 +0.04530383 0.38848962 0.17779283 +0.07863162 0.35117617 0.15836951 +0.08071337 0.24698436 0.10955503 +0.04040967 0.09124718 0.04040967 +0.00705675 0.01280627 0.00569046 +0.00042264 0.00066014 0.00028438 +0.00000317 0.00000432 0.00000174 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 -0.00000000 +0.00000000 0.09442587 0.04187089 +0.00840840 0.07612365 0.03370150 +0.00841665 0.03790964 0.01665534 +0.00337684 0.01040427 0.00446799 +0.00054062 0.00132483 0.00054062 +0.00002382 0.00004980 0.00001877 +0.00000012 0.00000021 0.00000008 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00039903 0.00015905 +0.00002590 0.00027532 0.00010882 +0.00001515 0.00008359 0.00003212 +0.00000229 0.00000867 0.00000318 +0.00000007 0.00000020 0.00000007 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 -0.00000000 +0.00000000 -0.00000000 -0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.21962195 +0.05784992 0.00000000 0.22557832 +0.09877480 0.00000000 0.23059254 +0.13415464 0.00000000 0.22568042 +0.16465196 0.00000000 0.21255306 +0.19805544 0.00000000 0.19805544 +0.23584271 0.00000000 0.18550201 +0.22778616 0.00000000 0.15266555 +0.33580943 0.00000000 0.20439457 +0.24988166 0.00000000 0.13969963 +0.00901370 0.00000000 0.00475284 +0.00000239 0.00000000 0.00000109 +0.00000000 0.00000000 0.00000000 +0.00000000 0.05784992 0.22557832 +0.05484865 0.05484865 0.22912333 +0.09698467 0.04975907 0.23095378 +0.13297281 0.04626450 0.22513003 +0.16466959 0.04398982 0.21173705 +0.19891678 0.04227551 0.19891678 +0.23331275 0.03964287 0.18492540 +0.22853000 0.03262551 0.15384842 +0.34080524 0.04322289 0.20669146 +0.22244465 0.02378979 0.12399210 +0.00688265 0.00076146 0.00360923 +0.00000159 0.00000012 0.00000072 +0.00000000 0.00000000 0.00000000 +0.00000000 0.09877480 0.23059254 +0.04975907 0.09698467 0.23095378 +0.09274519 0.09274519 0.22897201 +0.13001978 0.08845597 0.22159286 +0.16423396 0.08491019 0.20934788 +0.19957410 0.08257578 0.19957410 +0.22329079 0.07444742 0.17940160 +0.23871448 0.06735030 0.16256273 +0.34498469 0.08568348 0.20762137 +0.14694051 0.03193184 0.08145713 +0.00293995 0.00062559 0.00151442 +0.00000044 0.00000007 0.00000020 +0.00000000 0.00000000 0.00000000 +0.00000000 0.13415464 0.22568042 +0.04626450 0.13297281 0.22513003 +0.08845597 0.13001978 0.22159286 +0.12654261 0.12654261 0.21415373 +0.16267392 0.12321924 0.20593067 +0.19476779 0.11779403 0.19476779 +0.20768059 0.10276335 0.16858893 +0.26752042 0.11274378 0.18489086 +0.31261553 0.11184057 0.18628836 +0.05968429 0.01991438 0.03299667 +0.00062314 0.00018728 0.00031089 +0.00000005 0.00000001 0.00000002 +0.00000000 0.00000000 0.00000000 +0.00000000 0.16465196 0.21255306 +0.04398982 0.16466959 0.21173705 +0.08491019 0.16423396 0.20934788 +0.12321924 0.16267392 0.20593067 +0.15831015 0.15831015 0.19930738 +0.18007570 0.14379405 0.18007570 +0.21044762 0.13879825 0.17305007 +0.29860258 0.16849324 0.20935749 +0.20675213 0.09726934 0.12296526 +0.01280627 0.00569046 0.00705675 +0.00004980 0.00001877 0.00002382 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.19805544 0.19805544 +0.04227551 0.19891678 0.19891678 +0.08257578 0.19957410 0.19957410 +0.11779403 0.19476779 0.19476779 +0.14379405 0.18007570 0.18007570 +0.17328577 0.17328577 0.17328577 +0.24761344 0.20555657 0.20555657 +0.24707821 0.17159958 0.17159958 +0.05937251 0.03616259 0.03616259 +0.00125378 0.00067638 0.00067638 +0.00000080 0.00000037 0.00000037 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.23584271 0.18550201 +0.03964287 0.23331275 0.18492540 +0.07444742 0.22329079 0.17940160 +0.10276335 0.20768059 0.16858893 +0.13879825 0.21044762 0.17305007 +0.20555657 0.24761344 0.20555657 +0.22620541 0.22620541 0.18680260 +0.09448152 0.07989574 0.06604124 +0.00548834 0.00406603 0.00340836 +0.00003419 0.00002144 0.00001768 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.22778616 0.15266555 +0.03262551 0.22853000 0.15384842 +0.06735030 0.23871448 0.16256273 +0.11274378 0.26752042 0.18489086 +0.16849324 0.29860258 0.20935749 +0.17159958 0.24707821 0.17159958 +0.07989574 0.09448152 0.06604124 +0.00763276 0.00763276 0.00547505 +0.00013148 0.00011296 0.00008002 +0.00000008 0.00000006 0.00000004 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.33580943 0.20439457 +0.04322289 0.34080524 0.20669146 +0.08568348 0.34498469 0.20762137 +0.11184057 0.31261553 0.18628836 +0.09726934 0.20675213 0.12296526 +0.03616259 0.05937251 0.03616259 +0.00406603 0.00548834 0.00340836 +0.00011296 0.00013148 0.00008002 +0.00000018 0.00000018 0.00000011 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.24988166 0.13969963 +0.02378979 0.22244465 0.12399210 +0.03193184 0.14694051 0.08145713 +0.01991438 0.05968429 0.03299667 +0.00569046 0.01280627 0.00705675 +0.00067638 0.00125378 0.00067638 +0.00002144 0.00003419 0.00001768 +0.00000006 0.00000008 0.00000004 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00901370 0.00475284 +0.00076146 0.00688265 0.00360923 +0.00062559 0.00293995 0.00151442 +0.00018728 0.00062314 0.00031089 +0.00001877 0.00004980 0.00002382 +0.00000037 0.00000080 0.00000037 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000239 0.00000109 +0.00000012 0.00000159 0.00000072 +0.00000007 0.00000044 0.00000020 +0.00000001 0.00000005 0.00000002 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 -0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.24196835 +0.05315190 0.00000000 0.24812410 +0.08951714 0.00000000 0.25292777 +0.12082245 0.00000000 0.24815988 +0.15550298 0.00000000 0.24407471 +0.18550201 0.00000000 0.23584271 +0.19670448 0.00000000 0.19670448 +0.27828749 0.00000000 0.22263260 +0.29186871 0.00000000 0.20940311 +0.03569997 0.00000000 0.02352921 +0.00016711 0.00000000 0.00009720 +0.00000001 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.05315190 0.24812410 +0.05018152 0.05018152 0.25170547 +0.08754438 0.04486165 0.25306158 +0.12034648 0.04185713 0.24786893 +0.15544591 0.04078769 0.24448037 +0.18492540 0.03964287 0.23331275 +0.19466515 0.03375700 0.19466515 +0.28234193 0.04143619 0.22783320 +0.27650887 0.03245509 0.19774171 +0.02890315 0.00332244 0.01903327 +0.00011812 0.00001094 0.00006852 +0.00000001 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.08951714 0.25292777 +0.04486165 0.08754438 0.25306158 +0.08373253 0.08373253 0.25098717 +0.11936413 0.08111633 0.24729509 +0.15508359 0.08008805 0.24378950 +0.17940160 0.07444742 0.22329079 +0.19769699 0.06773175 0.19769699 +0.29264331 0.08576193 0.24083547 +0.22308379 0.05178263 0.15910566 +0.01485261 0.00339455 0.00974796 +0.00003945 0.00000711 0.00002269 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.12082245 0.24815988 +0.04185713 0.12034648 0.24786893 +0.08111633 0.11936413 0.24729509 +0.11846918 0.11846918 0.24571981 +0.15087424 0.11447137 0.23536902 +0.16858893 0.10276335 0.20768059 +0.21943567 0.11144995 0.21943567 +0.28722793 0.12382638 0.24046303 +0.12780462 0.04575108 0.09195493 +0.00443936 0.00149160 0.00288967 +0.00000523 0.00000139 0.00000297 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.15550298 0.24407471 +0.04078769 0.15544591 0.24448037 +0.08008805 0.15508359 0.24378950 +0.11447137 0.15087424 0.23536902 +0.13877265 0.13877265 0.21424615 +0.17305007 0.13879825 0.21044762 +0.25132266 0.16847290 0.25132266 +0.22475615 0.12425884 0.18922115 +0.03895527 0.01923163 0.02854692 +0.00066014 0.00028438 0.00042264 +0.00000021 0.00000008 0.00000012 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.18550201 0.23584271 +0.03964287 0.18492540 0.23331275 +0.07444742 0.17940160 0.22329079 +0.10276335 0.16858893 0.20768059 +0.13879825 0.17305007 0.21044762 +0.20555657 0.20555657 0.24761344 +0.22620541 0.18680260 0.22620541 +0.09448152 0.06604124 0.07989574 +0.00548834 0.00340836 0.00406603 +0.00003419 0.00001768 0.00002144 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.19670448 0.19670448 +0.03375700 0.19466515 0.19466515 +0.06773175 0.19769699 0.19769699 +0.11144995 0.21943567 0.21943567 +0.16847290 0.25132266 0.25132266 +0.18680260 0.22620541 0.22620541 +0.10426029 0.10426029 0.10426029 +0.01253251 0.01070927 0.01070927 +0.00027520 0.00020192 0.00020192 +0.00000027 0.00000017 0.00000017 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.27828749 0.22263260 +0.04143619 0.28234193 0.22783320 +0.08576193 0.29264331 0.24083547 +0.12382638 0.28722793 0.24046303 +0.12425884 0.22475615 0.18922115 +0.06604124 0.09448152 0.07989574 +0.01070927 0.01253251 0.01070927 +0.00047962 0.00047962 0.00040935 +0.00000196 0.00000168 0.00000141 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.29186871 0.20940311 +0.03245509 0.27650887 0.19774171 +0.05178263 0.22308379 0.15910566 +0.04575108 0.12780462 0.09195493 +0.01923163 0.03895527 0.02854692 +0.00340836 0.00548834 0.00406603 +0.00020192 0.00027520 0.00020192 +0.00000168 0.00000196 0.00000141 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.03569997 0.02352921 +0.00332244 0.02890315 0.01903327 +0.00339455 0.01485261 0.00974796 +0.00149160 0.00443936 0.00288967 +0.00028438 0.00066014 0.00042264 +0.00001768 0.00003419 0.00002144 +0.00000017 0.00000027 0.00000017 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00016711 0.00009720 +0.00001094 0.00011812 0.00006852 +0.00000711 0.00003945 0.00002269 +0.00000139 0.00000523 0.00000297 +0.00000008 0.00000021 0.00000012 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000001 0.00000000 +0.00000000 0.00000001 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 -0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.26345823 +0.05031494 0.00000000 0.26935568 +0.08372217 0.00000000 0.27422715 +0.11779382 0.00000000 0.27809616 +0.14539135 0.00000000 0.26798333 +0.15266555 0.00000000 0.22778616 +0.22263260 0.00000000 0.27828749 +0.28020504 0.00000000 0.28020504 +0.06343861 0.00000000 0.05159365 +0.00103623 0.00000000 0.00076839 +0.00000017 0.00000000 0.00000011 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.05031494 0.26935568 +0.04698754 0.04698754 0.27224313 +0.08255333 0.04216991 0.27552492 +0.11728339 0.04033039 0.27864928 +0.14306836 0.03713853 0.26429520 +0.15384842 0.03262551 0.22853000 +0.22783320 0.04143619 0.28234193 +0.26686098 0.04015244 0.26686098 +0.05361946 0.00666538 0.04401921 +0.00077884 0.00008618 0.00057648 +0.00000012 0.00000001 0.00000008 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.08372217 0.27422715 +0.04216991 0.08255333 0.27552492 +0.08042743 0.08042743 0.27829336 +0.11453460 0.07738642 0.27596054 +0.13583805 0.06946288 0.25175385 +0.16256273 0.06735030 0.23871448 +0.24083547 0.08576193 0.29264331 +0.21988464 0.06371968 0.21988464 +0.03079889 0.00786198 0.02582956 +0.00031769 0.00006772 0.00023404 +0.00000004 0.00000001 0.00000003 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.11779382 0.27809616 +0.04033039 0.11728339 0.27864928 +0.07738642 0.11453460 0.27596054 +0.10657151 0.10657151 0.26056065 +0.12929001 0.09792657 0.23783199 +0.18489086 0.11274378 0.26752042 +0.24046303 0.12382638 0.28722793 +0.13479603 0.05758260 0.13479603 +0.01050494 0.00406497 0.00897675 +0.00006176 0.00001896 0.00004529 +0.00000001 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.14539135 0.26798333 +0.03713853 0.14306836 0.26429520 +0.06946288 0.13583805 0.25175385 +0.09792657 0.12929001 0.23783199 +0.14200517 0.14200517 0.25662287 +0.20935749 0.16849324 0.29860258 +0.18922115 0.12425884 0.22475615 +0.04653578 0.02666223 0.04653578 +0.00183246 0.00092690 0.00157772 +0.00000432 0.00000174 0.00000317 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.15266555 0.22778616 +0.03262551 0.15384842 0.22853000 +0.06735030 0.16256273 0.23871448 +0.11274378 0.18489086 0.26752042 +0.16849324 0.20935749 0.29860258 +0.17159958 0.17159958 0.24707821 +0.07989574 0.06604124 0.09448152 +0.00763276 0.00547505 0.00763276 +0.00013148 0.00008002 0.00011296 +0.00000008 0.00000004 0.00000006 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.22263260 0.27828749 +0.04143619 0.22783320 0.28234193 +0.08576193 0.24083547 0.29264331 +0.12382638 0.24046303 0.28722793 +0.12425884 0.18922115 0.22475615 +0.06604124 0.07989574 0.09448152 +0.01070927 0.01070927 0.01253251 +0.00047962 0.00040935 0.00047962 +0.00000196 0.00000141 0.00000168 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.28020504 0.28020504 +0.04015244 0.26686098 0.26686098 +0.06371968 0.21988464 0.21988464 +0.05758260 0.13479603 0.13479603 +0.02666223 0.04653578 0.04653578 +0.00547505 0.00763276 0.00763276 +0.00040935 0.00047962 0.00047962 +0.00000479 0.00000479 0.00000479 +0.00000001 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.06343861 0.05159365 +0.00666538 0.05361946 0.04401921 +0.00786198 0.03079889 0.02582956 +0.00406497 0.01050494 0.00897675 +0.00092690 0.00183246 0.00157772 +0.00008002 0.00013148 0.00011296 +0.00000141 0.00000196 0.00000168 +0.00000000 0.00000001 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00103623 0.00076839 +0.00008618 0.00077884 0.00057648 +0.00006772 0.00031769 0.00023404 +0.00001896 0.00006176 0.00004529 +0.00000174 0.00000432 0.00000317 +0.00000004 0.00000008 0.00000006 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000017 0.00000011 +0.00000001 0.00000012 0.00000008 +0.00000001 0.00000004 0.00000003 +0.00000000 0.00000001 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 -0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.27808604 +0.05226268 0.00000000 0.29184205 +0.08576194 0.00000000 0.30646293 +0.10768783 0.00000000 0.28558967 +0.13184502 0.00000000 0.26576622 +0.20439457 0.00000000 0.33580943 +0.20940311 0.00000000 0.29186871 +0.05159365 0.00000000 0.06343861 +0.00335127 0.00000000 0.00335127 +0.00000143 0.00000000 0.00000119 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.05226268 0.29184205 +0.04897950 0.04897950 0.30077655 +0.08302718 0.04225252 0.30537290 +0.10485952 0.03571615 0.27970791 +0.13597825 0.03482393 0.27336582 +0.20669146 0.04322289 0.34080524 +0.19774171 0.03245509 0.27650887 +0.04401921 0.00666538 0.05361946 +0.00242573 0.00035678 0.00242573 +0.00000100 0.00000009 0.00000083 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.08576194 0.30646293 +0.04225252 0.08302718 0.30537290 +0.07512978 0.07512978 0.29232812 +0.09981428 0.06718600 0.26793238 +0.14900888 0.07502505 0.29942689 +0.20762137 0.08568348 0.34498469 +0.15910566 0.05178263 0.22308379 +0.02582956 0.00786198 0.03079889 +0.00088028 0.00023977 0.00088028 +0.00000033 0.00000006 0.00000028 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.10768783 0.28558967 +0.03571615 0.10485952 0.27970791 +0.06718600 0.09981428 0.26793238 +0.10587090 0.10587090 0.28272496 +0.16756172 0.12658670 0.34029210 +0.18628836 0.11184057 0.31261553 +0.09195493 0.04575108 0.12780462 +0.00897675 0.00406497 0.01050494 +0.00014707 0.00005548 0.00014707 +0.00000005 0.00000001 0.00000004 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.13184502 0.26576622 +0.03482393 0.13597825 0.27336582 +0.07502505 0.14900888 0.29942689 +0.12658670 0.16756172 0.34029210 +0.15821767 0.15821767 0.32983805 +0.12296526 0.09726934 0.20675213 +0.02854692 0.01923163 0.03895527 +0.00157772 0.00092690 0.00183246 +0.00000966 0.00000463 0.00000966 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.20439457 0.33580943 +0.04322289 0.20669146 0.34080524 +0.08568348 0.20762137 0.34498469 +0.11184057 0.18628836 0.31261553 +0.09726934 0.12296526 0.20675213 +0.03616259 0.03616259 0.05937251 +0.00406603 0.00340836 0.00548834 +0.00011296 0.00008002 0.00013148 +0.00000018 0.00000011 0.00000018 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.20940311 0.29186871 +0.03245509 0.19774171 0.27650887 +0.05178263 0.15910566 0.22308379 +0.04575108 0.09195493 0.12780462 +0.01923163 0.02854692 0.03895527 +0.00340836 0.00406603 0.00548834 +0.00020192 0.00020192 0.00027520 +0.00000168 0.00000141 0.00000196 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.05159365 0.06343861 +0.00666538 0.04401921 0.05361946 +0.00786198 0.02582956 0.03079889 +0.00406497 0.00897675 0.01050494 +0.00092690 0.00157772 0.00183246 +0.00008002 0.00011296 0.00013148 +0.00000141 0.00000168 0.00000196 +0.00000000 0.00000000 0.00000001 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 -0.00000000 +0.00000000 0.00335127 0.00335127 +0.00035678 0.00242573 0.00242573 +0.00023977 0.00088028 0.00088028 +0.00005548 0.00014707 0.00014707 +0.00000463 0.00000966 0.00000966 +0.00000011 0.00000018 0.00000018 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000143 0.00000119 +0.00000009 0.00000100 0.00000083 +0.00000006 0.00000033 0.00000028 +0.00000001 0.00000005 0.00000004 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 -0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 -0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 -0.00000000 -0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.35128500 +0.04779386 0.00000000 0.32796197 +0.07437695 0.00000000 0.28483792 +0.12922569 0.00000000 0.34061828 +0.18176438 0.00000000 0.39499686 +0.13969963 0.00000000 0.24988166 +0.02352921 0.00000000 0.03569997 +0.00076839 0.00000000 0.00103623 +0.00000119 0.00000000 0.00000143 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.04779386 0.32796197 +0.04250931 0.04250931 0.30800102 +0.07481661 0.03779717 0.28730540 +0.13373729 0.04466998 0.35622202 +0.17779283 0.04530383 0.38848962 +0.12399210 0.02378979 0.22244465 +0.01903327 0.00332244 0.02890315 +0.00057648 0.00008618 0.00077884 +0.00000083 0.00000009 0.00000100 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.07437695 0.28483792 +0.03779717 0.07481661 0.28730540 +0.08311482 0.08311482 0.32324693 +0.14162973 0.09486673 0.39079255 +0.15836951 0.07863162 0.35117617 +0.08145713 0.03193184 0.14694051 +0.00974796 0.00339455 0.01485261 +0.00023404 0.00006772 0.00031769 +0.00000028 0.00000006 0.00000033 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.12922569 0.34061828 +0.04466998 0.13373729 0.35622202 +0.09486673 0.14162973 0.39079255 +0.12899042 0.12899042 0.37633366 +0.10955503 0.08071337 0.24698436 +0.03299667 0.01991438 0.05968429 +0.00288967 0.00149160 0.00443936 +0.00004529 0.00001896 0.00006176 +0.00000004 0.00000001 0.00000005 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.18176438 0.39499686 +0.04530383 0.17779283 0.38848962 +0.07863162 0.15836951 0.35117617 +0.08071337 0.10955503 0.24698436 +0.04040967 0.04040967 0.09124718 +0.00705675 0.00569046 0.01280627 +0.00042264 0.00028438 0.00066014 +0.00000317 0.00000174 0.00000432 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.13969963 0.24988166 +0.02378979 0.12399210 0.22244465 +0.03193184 0.08145713 0.14694051 +0.01991438 0.03299667 0.05968429 +0.00569046 0.00705675 0.01280627 +0.00067638 0.00067638 0.00125378 +0.00002144 0.00001768 0.00003419 +0.00000006 0.00000004 0.00000008 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.02352921 0.03569997 +0.00332244 0.01903327 0.02890315 +0.00339455 0.00974796 0.01485261 +0.00149160 0.00288967 0.00443936 +0.00028438 0.00042264 0.00066014 +0.00001768 0.00002144 0.00003419 +0.00000017 0.00000017 0.00000027 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00076839 0.00103623 +0.00008618 0.00057648 0.00077884 +0.00006772 0.00023404 0.00031769 +0.00001896 0.00004529 0.00006176 +0.00000174 0.00000317 0.00000432 +0.00000004 0.00000006 0.00000008 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000119 0.00000143 +0.00000009 0.00000083 0.00000100 +0.00000006 0.00000028 0.00000033 +0.00000001 0.00000004 0.00000005 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 -0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 -0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.25113715 +0.05854756 0.00000000 0.34155637 +0.12180165 0.00000000 0.43878482 +0.11853629 0.00000000 0.32820362 +0.04187089 0.00000000 0.09442587 +0.00475284 0.00000000 0.00901370 +0.00009720 0.00000000 0.00016711 +0.00000011 0.00000000 0.00000017 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.05854756 0.34155637 +0.06440826 0.06440826 0.40165640 +0.11485281 0.05668998 0.42864392 +0.10428998 0.03244443 0.29339533 +0.03370150 0.00840840 0.07612365 +0.00360923 0.00076146 0.00688265 +0.00006852 0.00001094 0.00011812 +0.00000008 0.00000001 0.00000012 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.12180165 0.43878482 +0.05668998 0.11485281 0.42864392 +0.08667964 0.08667964 0.35807989 +0.06513445 0.04209815 0.18972336 +0.01665534 0.00841665 0.03790964 +0.00151442 0.00062559 0.00293995 +0.00002269 0.00000711 0.00003945 +0.00000003 0.00000001 0.00000004 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.11853629 0.32820362 +0.03244443 0.10428998 0.29339533 +0.04209815 0.06513445 0.18972336 +0.02195203 0.02195203 0.06562444 +0.00446799 0.00337684 0.01040427 +0.00031089 0.00018728 0.00062314 +0.00000297 0.00000139 0.00000523 +0.00000000 0.00000000 0.00000001 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.04187089 0.09442587 +0.00840840 0.03370150 0.07612365 +0.00841665 0.01665534 0.03790964 +0.00337684 0.00446799 0.01040427 +0.00054062 0.00054062 0.00132483 +0.00002382 0.00001877 0.00004980 +0.00000012 0.00000008 0.00000021 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00475284 0.00901370 +0.00076146 0.00360923 0.00688265 +0.00062559 0.00151442 0.00293995 +0.00018728 0.00031089 0.00062314 +0.00001877 0.00002382 0.00004980 +0.00000037 0.00000037 0.00000080 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00009720 0.00016711 +0.00001094 0.00006852 0.00011812 +0.00000711 0.00002269 0.00003945 +0.00000139 0.00000297 0.00000523 +0.00000008 0.00000012 0.00000021 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000011 0.00000017 +0.00000001 0.00000008 0.00000012 +0.00000001 0.00000003 0.00000004 +0.00000000 0.00000000 0.00000001 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 -0.00000000 -0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.38181582 +0.03622370 0.00000000 0.29245201 +0.02176584 0.00000000 0.08015231 +0.00327665 0.00000000 0.00927076 +0.00015905 0.00000000 0.00039903 +0.00000109 0.00000000 0.00000239 +0.00000000 0.00000000 0.00000001 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.03622370 0.29245201 +0.02766431 0.02766431 0.21450057 +0.01532577 0.00770589 0.05712571 +0.00230056 0.00077655 0.00661588 +0.00010882 0.00002590 0.00027532 +0.00000072 0.00000012 0.00000159 +0.00000000 0.00000000 0.00000001 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.02176584 0.08015231 +0.00770589 0.01532577 0.05712571 +0.00447407 0.00447407 0.01778447 +0.00072722 0.00048335 0.00221287 +0.00003212 0.00001515 0.00008359 +0.00000020 0.00000007 0.00000044 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00327665 0.00927076 +0.00077655 0.00230056 0.00661588 +0.00048335 0.00072722 0.00221287 +0.00008430 0.00008430 0.00028331 +0.00000318 0.00000229 0.00000867 +0.00000002 0.00000001 0.00000005 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00015905 0.00039903 +0.00002590 0.00010882 0.00027532 +0.00001515 0.00003212 0.00008359 +0.00000229 0.00000318 0.00000867 +0.00000007 0.00000007 0.00000020 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000109 0.00000239 +0.00000012 0.00000072 0.00000159 +0.00000007 0.00000020 0.00000044 +0.00000001 0.00000002 0.00000005 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000001 +0.00000000 0.00000000 0.00000001 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 -0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 -0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 -0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 -0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.00000000 +0.00004101 0.00000000 0.00000000 +0.00001641 0.00000000 0.00000000 +0.00000048 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00004101 0.00000000 +0.00002538 0.00002538 0.00000000 +0.00001000 0.00000477 0.00000000 +0.00000030 0.00000008 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00001641 0.00000000 +0.00000477 0.00001000 0.00000000 +0.00000178 0.00000178 0.00000000 +0.00000006 0.00000004 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000048 0.00000000 +0.00000008 0.00000030 0.00000000 +0.00000004 0.00000006 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 + +CELL_DATA 1728 +SCALARS SIE float 1 +LOOKUP_TABLE default +11.96970310 +1.95000345 +0.68059159 +0.42679413 +0.31956721 +0.25884397 +0.21675030 +0.18791042 +0.17026011 +0.12799510 +0.15550975 +0.00010527 +1.95000345 +1.28520793 +0.55813255 +0.32283128 +0.23218801 +0.18530510 +0.15376736 +0.13124374 +0.11335403 +0.09892888 +0.04855330 +0.00002110 +0.68059159 +0.55813255 +0.42481935 +0.24743849 +0.17598073 +0.13956042 +0.11737831 +0.10329029 +0.08427341 +0.09175639 +0.00607623 +0.00000013 +0.42679413 +0.32283128 +0.24743849 +0.21151579 +0.14386465 +0.11536829 +0.10075016 +0.08506177 +0.08084794 +0.04458657 +0.00024090 +0.00000000 +0.31956721 +0.23218801 +0.17598073 +0.14386465 +0.12803331 +0.09721032 +0.08233922 +0.07372138 +0.07780606 +0.00588790 +0.00000049 +0.00000000 +0.25884397 +0.18530510 +0.13956042 +0.11536829 +0.09721032 +0.08964409 +0.06747199 +0.07294920 +0.01873712 +0.00009796 +0.00000000 +0.00000000 +0.21675030 +0.15376736 +0.11737831 +0.10075016 +0.08233922 +0.06747199 +0.07085256 +0.02962047 +0.00074974 +0.00000002 +0.00000000 +0.00000000 +0.18791042 +0.13124374 +0.10329029 +0.08506177 +0.07372138 +0.07294920 +0.02962047 +0.00298788 +0.00000029 +0.00000000 +0.00000000 +0.00000000 +0.17026011 +0.11335403 +0.08427341 +0.08084794 +0.07780606 +0.01873712 +0.00074974 +0.00000029 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.12799510 +0.09892888 +0.09175639 +0.04458657 +0.00588790 +0.00009796 +0.00000002 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.15550975 +0.04855330 +0.00607623 +0.00024090 +0.00000049 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00010527 +0.00002110 +0.00000013 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +1.95000345 +1.28520793 +0.55813255 +0.32283128 +0.23218801 +0.18530510 +0.15376736 +0.13124374 +0.11335403 +0.09892888 +0.04855330 +0.00002110 +1.28520793 +0.79807819 +0.46034581 +0.26637332 +0.19434385 +0.15332891 +0.12794963 +0.11231419 +0.09231118 +0.09251083 +0.01764984 +0.00000355 +0.55813255 +0.46034581 +0.31839076 +0.22030975 +0.15722323 +0.12704809 +0.10856596 +0.09405160 +0.08071688 +0.08188588 +0.00236939 +0.00000002 +0.32283128 +0.26637332 +0.22030975 +0.17199862 +0.13291264 +0.10597756 +0.09412617 +0.07907574 +0.08021050 +0.02804596 +0.00007821 +0.00000000 +0.23218801 +0.19434385 +0.15722323 +0.13291264 +0.11010145 +0.09350691 +0.07665995 +0.07183175 +0.06574932 +0.00305855 +0.00000013 +0.00000000 +0.18530510 +0.15332891 +0.12704809 +0.10597756 +0.09350691 +0.07775026 +0.06706071 +0.07144920 +0.01193449 +0.00003456 +0.00000000 +0.00000000 +0.15376736 +0.12794963 +0.10856596 +0.09412617 +0.07665995 +0.06706071 +0.06813314 +0.02153258 +0.00035406 +0.00000000 +0.00000000 +0.00000000 +0.13124374 +0.11231419 +0.09405160 +0.07907574 +0.07183175 +0.07144920 +0.02153258 +0.00124650 +0.00000009 +0.00000000 +0.00000000 +0.00000000 +0.11335403 +0.09231118 +0.08071688 +0.08021050 +0.06574932 +0.01193449 +0.00035406 +0.00000009 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.09892888 +0.09251083 +0.08188588 +0.02804596 +0.00305855 +0.00003456 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.04855330 +0.01764984 +0.00236939 +0.00007821 +0.00000013 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00002110 +0.00000355 +0.00000002 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.68059159 +0.55813255 +0.42481935 +0.24743849 +0.17598073 +0.13956042 +0.11737831 +0.10329029 +0.08427341 +0.09175639 +0.00607623 +0.00000013 +0.55813255 +0.46034581 +0.31839076 +0.22030975 +0.15722323 +0.12704809 +0.10856596 +0.09405160 +0.08071688 +0.08188588 +0.00236939 +0.00000002 +0.42481935 +0.31839076 +0.22944566 +0.17527137 +0.13735527 +0.11084416 +0.09659286 +0.08275941 +0.07860557 +0.04355621 +0.00030393 +0.00000000 +0.24743849 +0.22030975 +0.17527137 +0.13789615 +0.11555518 +0.09723650 +0.08277858 +0.07262965 +0.07833357 +0.00983444 +0.00000628 +0.00000000 +0.17598073 +0.15722323 +0.13735527 +0.11555518 +0.09525889 +0.08377748 +0.07027104 +0.07119741 +0.03692349 +0.00072355 +0.00000001 +0.00000000 +0.13956042 +0.12704809 +0.11084416 +0.09723650 +0.08377748 +0.06818122 +0.06616563 +0.05853347 +0.00451546 +0.00000354 +0.00000000 +0.00000000 +0.11737831 +0.10856596 +0.09659286 +0.08277858 +0.07027104 +0.06616563 +0.06025065 +0.00972799 +0.00006690 +0.00000000 +0.00000000 +0.00000000 +0.10329029 +0.09405160 +0.08275941 +0.07262965 +0.07119741 +0.05853347 +0.00972799 +0.00020339 +0.00000001 +0.00000000 +0.00000000 +0.00000000 +0.08427341 +0.08071688 +0.07860557 +0.07833357 +0.03692349 +0.00451546 +0.00006690 +0.00000001 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.09175639 +0.08188588 +0.04355621 +0.00983444 +0.00072355 +0.00000354 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00607623 +0.00236939 +0.00030393 +0.00000628 +0.00000001 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000013 +0.00000002 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.42679413 +0.32283128 +0.24743849 +0.21151579 +0.14386465 +0.11536829 +0.10075016 +0.08506177 +0.08084794 +0.04458657 +0.00024090 +0.00000000 +0.32283128 +0.26637332 +0.22030975 +0.17199862 +0.13291264 +0.10597756 +0.09412617 +0.07907574 +0.08021050 +0.02804596 +0.00007821 +0.00000000 +0.24743849 +0.22030975 +0.17527137 +0.13789615 +0.11555518 +0.09723650 +0.08277858 +0.07262965 +0.07833357 +0.00983444 +0.00000628 +0.00000000 +0.21151579 +0.17199862 +0.13789615 +0.11829215 +0.09811864 +0.08655123 +0.07309248 +0.07001552 +0.04925510 +0.00151216 +0.00000006 +0.00000000 +0.14386465 +0.13291264 +0.11555518 +0.09811864 +0.08753088 +0.07339695 +0.06683084 +0.07014299 +0.01118626 +0.00005207 +0.00000000 +0.00000000 +0.11536829 +0.10597756 +0.09723650 +0.08655123 +0.07339695 +0.06558138 +0.06847082 +0.02693209 +0.00080374 +0.00000007 +0.00000000 +0.00000000 +0.10075016 +0.09412617 +0.08277858 +0.07309248 +0.06683084 +0.06847082 +0.03245148 +0.00225449 +0.00000332 +0.00000000 +0.00000000 +0.00000000 +0.08506177 +0.07907574 +0.07262965 +0.07001552 +0.07014299 +0.02693209 +0.00225449 +0.00001003 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.08084794 +0.08021050 +0.07833357 +0.04925510 +0.01118626 +0.00080374 +0.00000332 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.04458657 +0.02804596 +0.00983444 +0.00151216 +0.00005207 +0.00000007 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00024090 +0.00007821 +0.00000628 +0.00000006 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.31956721 +0.23218801 +0.17598073 +0.14386465 +0.12803331 +0.09721032 +0.08233922 +0.07372138 +0.07780606 +0.00588790 +0.00000049 +0.00000000 +0.23218801 +0.19434385 +0.15722323 +0.13291264 +0.11010145 +0.09350691 +0.07665995 +0.07183175 +0.06574932 +0.00305855 +0.00000013 +0.00000000 +0.17598073 +0.15722323 +0.13735527 +0.11555518 +0.09525889 +0.08377748 +0.07027104 +0.07119741 +0.03692349 +0.00072355 +0.00000001 +0.00000000 +0.14386465 +0.13291264 +0.11555518 +0.09811864 +0.08753088 +0.07339695 +0.06683084 +0.07014299 +0.01118626 +0.00005207 +0.00000000 +0.00000000 +0.12803331 +0.11010145 +0.09525889 +0.08753088 +0.07550642 +0.06670611 +0.06800883 +0.03625927 +0.00145810 +0.00000040 +0.00000000 +0.00000000 +0.09721032 +0.09350691 +0.08377748 +0.07339695 +0.06670611 +0.06780607 +0.05087100 +0.00582269 +0.00003846 +0.00000000 +0.00000000 +0.00000000 +0.08233922 +0.07665995 +0.07027104 +0.06683084 +0.06800883 +0.05087100 +0.00845280 +0.00018220 +0.00000002 +0.00000000 +0.00000000 +0.00000000 +0.07372138 +0.07183175 +0.07119741 +0.07014299 +0.03625927 +0.00582269 +0.00018220 +0.00000008 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.07780606 +0.06574932 +0.03692349 +0.01118626 +0.00145810 +0.00003846 +0.00000002 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00588790 +0.00305855 +0.00072355 +0.00005207 +0.00000040 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000049 +0.00000013 +0.00000001 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.25884397 +0.18530510 +0.13956042 +0.11536829 +0.09721032 +0.08964409 +0.06747199 +0.07294920 +0.01873712 +0.00009796 +0.00000000 +0.00000000 +0.18530510 +0.15332891 +0.12704809 +0.10597756 +0.09350691 +0.07775026 +0.06706071 +0.07144920 +0.01193449 +0.00003456 +0.00000000 +0.00000000 +0.13956042 +0.12704809 +0.11084416 +0.09723650 +0.08377748 +0.06818122 +0.06616563 +0.05853347 +0.00451546 +0.00000354 +0.00000000 +0.00000000 +0.11536829 +0.10597756 +0.09723650 +0.08655123 +0.07339695 +0.06558138 +0.06847082 +0.02693209 +0.00080374 +0.00000007 +0.00000000 +0.00000000 +0.09721032 +0.09350691 +0.08377748 +0.07339695 +0.06670611 +0.06780607 +0.05087100 +0.00582269 +0.00003846 +0.00000000 +0.00000000 +0.00000000 +0.08964409 +0.07775026 +0.06818122 +0.06558138 +0.06780607 +0.05775727 +0.01246553 +0.00037585 +0.00000012 +0.00000000 +0.00000000 +0.00000000 +0.06747199 +0.06706071 +0.06616563 +0.06847082 +0.05087100 +0.01246553 +0.00074932 +0.00000165 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.07294920 +0.07144920 +0.05853347 +0.02693209 +0.00582269 +0.00037585 +0.00000165 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.01873712 +0.01193449 +0.00451546 +0.00080374 +0.00003846 +0.00000012 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00009796 +0.00003456 +0.00000354 +0.00000007 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.21675030 +0.15376736 +0.11737831 +0.10075016 +0.08233922 +0.06747199 +0.07085256 +0.02962047 +0.00074974 +0.00000002 +0.00000000 +0.00000000 +0.15376736 +0.12794963 +0.10856596 +0.09412617 +0.07665995 +0.06706071 +0.06813314 +0.02153258 +0.00035406 +0.00000000 +0.00000000 +0.00000000 +0.11737831 +0.10856596 +0.09659286 +0.08277858 +0.07027104 +0.06616563 +0.06025065 +0.00972799 +0.00006690 +0.00000000 +0.00000000 +0.00000000 +0.10075016 +0.09412617 +0.08277858 +0.07309248 +0.06683084 +0.06847082 +0.03245148 +0.00225449 +0.00000332 +0.00000000 +0.00000000 +0.00000000 +0.08233922 +0.07665995 +0.07027104 +0.06683084 +0.06800883 +0.05087100 +0.00845280 +0.00018220 +0.00000002 +0.00000000 +0.00000000 +0.00000000 +0.06747199 +0.06706071 +0.06616563 +0.06847082 +0.05087100 +0.01246553 +0.00074932 +0.00000165 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.07085256 +0.06813314 +0.06025065 +0.03245148 +0.00845280 +0.00074932 +0.00000578 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.02962047 +0.02153258 +0.00972799 +0.00225449 +0.00018220 +0.00000165 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00074974 +0.00035406 +0.00006690 +0.00000332 +0.00000002 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000002 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.18791042 +0.13124374 +0.10329029 +0.08506177 +0.07372138 +0.07294920 +0.02962047 +0.00298788 +0.00000029 +0.00000000 +0.00000000 +0.00000000 +0.13124374 +0.11231419 +0.09405160 +0.07907574 +0.07183175 +0.07144920 +0.02153258 +0.00124650 +0.00000009 +0.00000000 +0.00000000 +0.00000000 +0.10329029 +0.09405160 +0.08275941 +0.07262965 +0.07119741 +0.05853347 +0.00972799 +0.00020339 +0.00000001 +0.00000000 +0.00000000 +0.00000000 +0.08506177 +0.07907574 +0.07262965 +0.07001552 +0.07014299 +0.02693209 +0.00225449 +0.00001003 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.07372138 +0.07183175 +0.07119741 +0.07014299 +0.03625927 +0.00582269 +0.00018220 +0.00000008 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.07294920 +0.07144920 +0.05853347 +0.02693209 +0.00582269 +0.00037585 +0.00000165 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.02962047 +0.02153258 +0.00972799 +0.00225449 +0.00018220 +0.00000165 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00298788 +0.00124650 +0.00020339 +0.00001003 +0.00000008 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000029 +0.00000009 +0.00000001 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.17026011 +0.11335403 +0.08427341 +0.08084794 +0.07780606 +0.01873712 +0.00074974 +0.00000029 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.11335403 +0.09231118 +0.08071688 +0.08021050 +0.06574932 +0.01193449 +0.00035406 +0.00000009 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.08427341 +0.08071688 +0.07860557 +0.07833357 +0.03692349 +0.00451546 +0.00006690 +0.00000001 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.08084794 +0.08021050 +0.07833357 +0.04925510 +0.01118626 +0.00080374 +0.00000332 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.07780606 +0.06574932 +0.03692349 +0.01118626 +0.00145810 +0.00003846 +0.00000002 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.01873712 +0.01193449 +0.00451546 +0.00080374 +0.00003846 +0.00000012 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00074974 +0.00035406 +0.00006690 +0.00000332 +0.00000002 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000029 +0.00000009 +0.00000001 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.12799510 +0.09892888 +0.09175639 +0.04458657 +0.00588790 +0.00009796 +0.00000002 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.09892888 +0.09251083 +0.08188588 +0.02804596 +0.00305855 +0.00003456 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.09175639 +0.08188588 +0.04355621 +0.00983444 +0.00072355 +0.00000354 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.04458657 +0.02804596 +0.00983444 +0.00151216 +0.00005207 +0.00000007 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00588790 +0.00305855 +0.00072355 +0.00005207 +0.00000040 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00009796 +0.00003456 +0.00000354 +0.00000007 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000002 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.15550975 +0.04855330 +0.00607623 +0.00024090 +0.00000049 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.04855330 +0.01764984 +0.00236939 +0.00007821 +0.00000013 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00607623 +0.00236939 +0.00030393 +0.00000628 +0.00000001 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00024090 +0.00007821 +0.00000628 +0.00000006 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000049 +0.00000013 +0.00000001 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00010527 +0.00002110 +0.00000013 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00002110 +0.00000355 +0.00000002 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000013 +0.00000002 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 + +SCALARS element_density float 1 +LOOKUP_TABLE default +0.01059896 +0.06028598 +0.18275629 +0.30874663 +0.44891561 +0.61765647 +0.83598131 +1.10711539 +1.54571030 +1.64889211 +6.74836785 +1.12321467 +0.06028598 +0.09454520 +0.20220073 +0.37708219 +0.56522550 +0.78305577 +1.05302032 +1.41794008 +1.78819213 +2.55789130 +3.34525938 +1.06814764 +0.18275629 +0.20220073 +0.28684617 +0.45987870 +0.72089895 +0.99970957 +1.34702602 +1.80964877 +2.13732651 +4.92673785 +1.71008559 +1.01698812 +0.30874663 +0.37708219 +0.45987870 +0.60254824 +0.85104435 +1.20884303 +1.64579808 +1.99289978 +3.09972265 +3.69851764 +1.18071033 +1.00161247 +0.44891561 +0.56522550 +0.72089895 +0.85104435 +1.07338231 +1.47642710 +1.91607685 +2.48729715 +5.02047567 +1.74250470 +1.02797014 +1.00004667 +0.61765647 +0.78305577 +0.99970957 +1.20884303 +1.47642710 +1.84572537 +2.19363884 +4.46617088 +2.56923277 +1.14090797 +1.00196071 +1.00000022 +0.83598131 +1.05302032 +1.34702602 +1.64579808 +1.91607685 +2.19363884 +3.52467473 +3.18444579 +1.30733317 +1.01151158 +1.00002255 +1.00000000 +1.10711539 +1.41794008 +1.80964877 +1.99289978 +2.48729715 +4.46617088 +3.18444579 +1.54735240 +1.02717675 +1.00020750 +1.00000002 +1.00000000 +1.54571030 +1.78819213 +2.13732651 +3.09972265 +5.02047567 +2.56923277 +1.30733317 +1.02717675 +1.00093673 +1.00000020 +1.00000000 +1.00000000 +1.64889211 +2.55789130 +4.92673785 +3.69851764 +1.74250470 +1.14090797 +1.01151158 +1.00020750 +1.00000020 +1.00000000 +1.00000000 +1.00000000 +6.74836785 +3.34525938 +1.71008559 +1.18071033 +1.02797014 +1.00196071 +1.00002255 +1.00000002 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.12321467 +1.06814764 +1.01698812 +1.00161247 +1.00004667 +1.00000022 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +0.06028598 +0.09454520 +0.20220073 +0.37708219 +0.56522550 +0.78305577 +1.05302032 +1.41794008 +1.78819213 +2.55789130 +3.34525938 +1.06814764 +0.09454520 +0.13707712 +0.25363317 +0.43653918 +0.65601811 +0.91535002 +1.22495668 +1.65556195 +1.96413896 +3.79775217 +2.24473568 +1.03810236 +0.20220073 +0.25363317 +0.35047190 +0.53841255 +0.78764869 +1.08616431 +1.46680475 +1.91035799 +2.37151402 +5.21745587 +1.45588968 +1.00962730 +0.37708219 +0.43653918 +0.53841255 +0.68630633 +0.95276250 +1.30058326 +1.74284788 +2.08683938 +3.68913863 +2.96038837 +1.12195513 +1.00091153 +0.56522550 +0.65601811 +0.78764869 +0.95276250 +1.18653545 +1.60397360 +1.96337513 +2.79495075 +4.78716176 +1.54063711 +1.01848518 +1.00002536 +0.78305577 +0.91535002 +1.08616431 +1.30058326 +1.60397360 +1.90461577 +2.40758071 +4.82522598 +2.20193738 +1.10012297 +1.00121571 +1.00000012 +1.05302032 +1.22495668 +1.46680475 +1.74284788 +1.96337513 +2.40758071 +4.16408894 +2.80143960 +1.23126187 +1.00773513 +1.00001270 +1.00000000 +1.41794008 +1.65556195 +1.91035799 +2.08683938 +2.79495075 +4.82522598 +2.80143960 +1.37979954 +1.01932031 +1.00012613 +1.00000001 +1.00000000 +1.78819213 +1.96413896 +2.37151402 +3.68913863 +4.78716176 +2.20193738 +1.23126187 +1.01932031 +1.00054429 +1.00000011 +1.00000000 +1.00000000 +2.55789130 +3.79775217 +5.21745587 +2.96038837 +1.54063711 +1.10012297 +1.00773513 +1.00012613 +1.00000011 +1.00000000 +1.00000000 +1.00000000 +3.34525938 +2.24473568 +1.45588968 +1.12195513 +1.01848518 +1.00121571 +1.00001270 +1.00000001 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.06814764 +1.03810236 +1.00962730 +1.00091153 +1.00002536 +1.00000012 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +0.18275629 +0.20220073 +0.28684617 +0.45987870 +0.72089895 +0.99970957 +1.34702602 +1.80964877 +2.13732651 +4.92673785 +1.71008559 +1.01698812 +0.20220073 +0.25363317 +0.35047190 +0.53841255 +0.78764869 +1.08616431 +1.46680475 +1.91035799 +2.37151402 +5.21745587 +1.45588968 +1.00962730 +0.28684617 +0.35047190 +0.48503526 +0.67607895 +0.92000432 +1.23670716 +1.67408651 +2.03637404 +3.17013000 +3.74258616 +1.19708047 +1.00259320 +0.45987870 +0.53841255 +0.67607895 +0.85444168 +1.11636939 +1.47792318 +1.89455570 +2.36028267 +4.85497505 +2.02148892 +1.05398342 +1.00025353 +0.72089895 +0.78764869 +0.92000432 +1.11636939 +1.39929604 +1.78482881 +2.11214051 +3.62383286 +3.59078365 +1.29006240 +1.00767405 +1.00000659 +0.99970957 +1.08616431 +1.23670716 +1.47792318 +1.78482881 +2.05001846 +3.03428814 +4.69553640 +1.70798913 +1.04930793 +1.00043503 +1.00000003 +1.34702602 +1.46680475 +1.67408651 +1.89455570 +2.11214051 +3.03428814 +4.69121168 +2.11845737 +1.12863248 +1.00334449 +1.00000367 +1.00000000 +1.80964877 +1.91035799 +2.03637404 +2.36028267 +3.62383286 +4.69553640 +2.11845737 +1.19206755 +1.00928909 +1.00004347 +1.00000000 +1.00000000 +2.13732651 +2.37151402 +3.17013000 +4.85497505 +3.59078365 +1.70798913 +1.12863248 +1.00928909 +1.00016737 +1.00000004 +1.00000000 +1.00000000 +4.92673785 +5.21745587 +3.74258616 +2.02148892 +1.29006240 +1.04930793 +1.00334449 +1.00004347 +1.00000004 +1.00000000 +1.00000000 +1.00000000 +1.71008559 +1.45588968 +1.19708047 +1.05398342 +1.00767405 +1.00043503 +1.00000367 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.01698812 +1.00962730 +1.00259320 +1.00025353 +1.00000659 +1.00000003 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +0.30874663 +0.37708219 +0.45987870 +0.60254824 +0.85104435 +1.20884303 +1.64579808 +1.99289978 +3.09972265 +3.69851764 +1.18071033 +1.00161247 +0.37708219 +0.43653918 +0.53841255 +0.68630633 +0.95276250 +1.30058326 +1.74284788 +2.08683938 +3.68913863 +2.96038837 +1.12195513 +1.00091153 +0.45987870 +0.53841255 +0.67607895 +0.85444168 +1.11636939 +1.47792318 +1.89455570 +2.36028267 +4.85497505 +2.02148892 +1.05398342 +1.00025353 +0.60254824 +0.68630633 +0.85444168 +1.06799788 +1.34884632 +1.72646359 +2.05676383 +3.23815309 +4.23041524 +1.39968435 +1.01389135 +1.00002607 +0.85104435 +0.95276250 +1.11636939 +1.34884632 +1.66271014 +1.95104280 +2.56601603 +4.89786332 +2.19501261 +1.11057793 +1.00176949 +1.00000061 +1.20884303 +1.30058326 +1.47792318 +1.72646359 +1.95104280 +2.43868827 +4.31104219 +3.18754429 +1.32061019 +1.01554183 +1.00007703 +1.00000000 +1.64579808 +1.74284788 +1.89455570 +2.05676383 +2.56601603 +4.31104219 +3.53444834 +1.52323853 +1.04972809 +1.00084205 +1.00000045 +1.00000000 +1.99289978 +2.08683938 +2.36028267 +3.23815309 +4.89786332 +3.18754429 +1.52323853 +1.07104992 +1.00268739 +1.00000730 +1.00000000 +1.00000000 +3.09972265 +3.68913863 +4.85497505 +4.23041524 +2.19501261 +1.32061019 +1.04972809 +1.00268739 +1.00002353 +1.00000001 +1.00000000 +1.00000000 +3.69851764 +2.96038837 +2.02148892 +1.39968435 +1.11057793 +1.01554183 +1.00084205 +1.00000730 +1.00000001 +1.00000000 +1.00000000 +1.00000000 +1.18071033 +1.12195513 +1.05398342 +1.01389135 +1.00176949 +1.00007703 +1.00000045 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00161247 +1.00091153 +1.00025353 +1.00002607 +1.00000061 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +0.44891561 +0.56522550 +0.72089895 +0.85104435 +1.07338231 +1.47642710 +1.91607685 +2.48729715 +5.02047567 +1.74250470 +1.02797014 +1.00004667 +0.56522550 +0.65601811 +0.78764869 +0.95276250 +1.18653545 +1.60397360 +1.96337513 +2.79495075 +4.78716176 +1.54063711 +1.01848518 +1.00002536 +0.72089895 +0.78764869 +0.92000432 +1.11636939 +1.39929604 +1.78482881 +2.11214051 +3.62383286 +3.59078365 +1.29006240 +1.00767405 +1.00000659 +0.85104435 +0.95276250 +1.11636939 +1.34884632 +1.66271014 +1.95104280 +2.56601603 +4.89786332 +2.19501261 +1.11057793 +1.00176949 +1.00000061 +1.07338231 +1.18653545 +1.39929604 +1.66271014 +1.90393974 +2.29248911 +3.94722752 +3.71408451 +1.41865170 +1.02533557 +1.00018095 +1.00000001 +1.47642710 +1.60397360 +1.78482881 +1.95104280 +2.29248911 +3.58481503 +4.46756253 +1.85074347 +1.10400191 +1.00273473 +1.00000498 +1.00000000 +1.91607685 +1.96337513 +2.11214051 +2.56601603 +3.94722752 +4.46756253 +2.05894465 +1.18392652 +1.01153195 +1.00010268 +1.00000002 +1.00000000 +2.48729715 +2.79495075 +3.62383286 +4.89786332 +3.71408451 +1.85074347 +1.18392652 +1.01718902 +1.00039108 +1.00000045 +1.00000000 +1.00000000 +5.02047567 +4.78716176 +3.59078365 +2.19501261 +1.41865170 +1.10400191 +1.01153195 +1.00039108 +1.00000128 +1.00000000 +1.00000000 +1.00000000 +1.74250470 +1.54063711 +1.29006240 +1.11057793 +1.02533557 +1.00273473 +1.00010268 +1.00000045 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.02797014 +1.01848518 +1.00767405 +1.00176949 +1.00018095 +1.00000498 +1.00000002 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00004667 +1.00002536 +1.00000659 +1.00000061 +1.00000001 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +0.61765647 +0.78305577 +0.99970957 +1.20884303 +1.47642710 +1.84572537 +2.19363884 +4.46617088 +2.56923277 +1.14090797 +1.00196071 +1.00000022 +0.78305577 +0.91535002 +1.08616431 +1.30058326 +1.60397360 +1.90461577 +2.40758071 +4.82522598 +2.20193738 +1.10012297 +1.00121571 +1.00000012 +0.99970957 +1.08616431 +1.23670716 +1.47792318 +1.78482881 +2.05001846 +3.03428814 +4.69553640 +1.70798913 +1.04930793 +1.00043503 +1.00000003 +1.20884303 +1.30058326 +1.47792318 +1.72646359 +1.95104280 +2.43868827 +4.31104219 +3.18754429 +1.32061019 +1.01554183 +1.00007703 +1.00000000 +1.47642710 +1.60397360 +1.78482881 +1.95104280 +2.29248911 +3.58481503 +4.46756253 +1.85074347 +1.10400191 +1.00273473 +1.00000498 +1.00000000 +1.84572537 +1.90461577 +2.05001846 +2.43868827 +3.58481503 +4.69561911 +2.34453450 +1.24335023 +1.01858146 +1.00021178 +1.00000007 +1.00000000 +2.19363884 +2.40758071 +3.03428814 +4.31104219 +4.46756253 +2.34453450 +1.32099117 +1.04035687 +1.00132267 +1.00000413 +1.00000000 +1.00000000 +4.46617088 +4.82522598 +4.69553640 +3.18754429 +1.85074347 +1.24335023 +1.04035687 +1.00224842 +1.00002182 +1.00000001 +1.00000000 +1.00000000 +2.56923277 +2.20193738 +1.70798913 +1.32061019 +1.10400191 +1.01858146 +1.00132267 +1.00002182 +1.00000002 +1.00000000 +1.00000000 +1.00000000 +1.14090797 +1.10012297 +1.04930793 +1.01554183 +1.00273473 +1.00021178 +1.00000413 +1.00000001 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00196071 +1.00121571 +1.00043503 +1.00007703 +1.00000498 +1.00000007 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000022 +1.00000012 +1.00000003 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +0.83598131 +1.05302032 +1.34702602 +1.64579808 +1.91607685 +2.19363884 +3.52467473 +3.18444579 +1.30733317 +1.01151158 +1.00002255 +1.00000000 +1.05302032 +1.22495668 +1.46680475 +1.74284788 +1.96337513 +2.40758071 +4.16408894 +2.80143960 +1.23126187 +1.00773513 +1.00001270 +1.00000000 +1.34702602 +1.46680475 +1.67408651 +1.89455570 +2.11214051 +3.03428814 +4.69121168 +2.11845737 +1.12863248 +1.00334449 +1.00000367 +1.00000000 +1.64579808 +1.74284788 +1.89455570 +2.05676383 +2.56601603 +4.31104219 +3.53444834 +1.52323853 +1.04972809 +1.00084205 +1.00000045 +1.00000000 +1.91607685 +1.96337513 +2.11214051 +2.56601603 +3.94722752 +4.46756253 +2.05894465 +1.18392652 +1.01153195 +1.00010268 +1.00000002 +1.00000000 +2.19363884 +2.40758071 +3.03428814 +4.31104219 +4.46756253 +2.34453450 +1.32099117 +1.04035687 +1.00132267 +1.00000413 +1.00000000 +1.00000000 +3.52467473 +4.16408894 +4.69121168 +3.53444834 +2.05894465 +1.32099117 +1.05868539 +1.00389046 +1.00005026 +1.00000003 +1.00000000 +1.00000000 +3.18444579 +2.80143960 +2.11845737 +1.52323853 +1.18392652 +1.04035687 +1.00389046 +1.00011001 +1.00000025 +1.00000000 +1.00000000 +1.00000000 +1.30733317 +1.23126187 +1.12863248 +1.04972809 +1.01153195 +1.00132267 +1.00005026 +1.00000025 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.01151158 +1.00773513 +1.00334449 +1.00084205 +1.00010268 +1.00000413 +1.00000003 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00002255 +1.00001270 +1.00000367 +1.00000045 +1.00000002 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.10711539 +1.41794008 +1.80964877 +1.99289978 +2.48729715 +4.46617088 +3.18444579 +1.54735240 +1.02717675 +1.00020750 +1.00000002 +1.00000000 +1.41794008 +1.65556195 +1.91035799 +2.08683938 +2.79495075 +4.82522598 +2.80143960 +1.37979954 +1.01932031 +1.00012613 +1.00000001 +1.00000000 +1.80964877 +1.91035799 +2.03637404 +2.36028267 +3.62383286 +4.69553640 +2.11845737 +1.19206755 +1.00928909 +1.00004347 +1.00000000 +1.00000000 +1.99289978 +2.08683938 +2.36028267 +3.23815309 +4.89786332 +3.18754429 +1.52323853 +1.07104992 +1.00268739 +1.00000730 +1.00000000 +1.00000000 +2.48729715 +2.79495075 +3.62383286 +4.89786332 +3.71408451 +1.85074347 +1.18392652 +1.01718902 +1.00039108 +1.00000045 +1.00000000 +1.00000000 +4.46617088 +4.82522598 +4.69553640 +3.18754429 +1.85074347 +1.24335023 +1.04035687 +1.00224842 +1.00002182 +1.00000001 +1.00000000 +1.00000000 +3.18444579 +2.80143960 +2.11845737 +1.52323853 +1.18392652 +1.04035687 +1.00389046 +1.00011001 +1.00000025 +1.00000000 +1.00000000 +1.00000000 +1.54735240 +1.37979954 +1.19206755 +1.07104992 +1.01718902 +1.00224842 +1.00011001 +1.00000075 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.02717675 +1.01932031 +1.00928909 +1.00268739 +1.00039108 +1.00002182 +1.00000025 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00020750 +1.00012613 +1.00004347 +1.00000730 +1.00000045 +1.00000001 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000002 +1.00000001 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.54571030 +1.78819213 +2.13732651 +3.09972265 +5.02047567 +2.56923277 +1.30733317 +1.02717675 +1.00093673 +1.00000020 +1.00000000 +1.00000000 +1.78819213 +1.96413896 +2.37151402 +3.68913863 +4.78716176 +2.20193738 +1.23126187 +1.01932031 +1.00054429 +1.00000011 +1.00000000 +1.00000000 +2.13732651 +2.37151402 +3.17013000 +4.85497505 +3.59078365 +1.70798913 +1.12863248 +1.00928909 +1.00016737 +1.00000004 +1.00000000 +1.00000000 +3.09972265 +3.68913863 +4.85497505 +4.23041524 +2.19501261 +1.32061019 +1.04972809 +1.00268739 +1.00002353 +1.00000001 +1.00000000 +1.00000000 +5.02047567 +4.78716176 +3.59078365 +2.19501261 +1.41865170 +1.10400191 +1.01153195 +1.00039108 +1.00000128 +1.00000000 +1.00000000 +1.00000000 +2.56923277 +2.20193738 +1.70798913 +1.32061019 +1.10400191 +1.01858146 +1.00132267 +1.00002182 +1.00000002 +1.00000000 +1.00000000 +1.00000000 +1.30733317 +1.23126187 +1.12863248 +1.04972809 +1.01153195 +1.00132267 +1.00005026 +1.00000025 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.02717675 +1.01932031 +1.00928909 +1.00268739 +1.00039108 +1.00002182 +1.00000025 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00093673 +1.00054429 +1.00016737 +1.00002353 +1.00000128 +1.00000002 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000020 +1.00000011 +1.00000004 +1.00000001 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.64889211 +2.55789130 +4.92673785 +3.69851764 +1.74250470 +1.14090797 +1.01151158 +1.00020750 +1.00000020 +1.00000000 +1.00000000 +1.00000000 +2.55789130 +3.79775217 +5.21745587 +2.96038837 +1.54063711 +1.10012297 +1.00773513 +1.00012613 +1.00000011 +1.00000000 +1.00000000 +1.00000000 +4.92673785 +5.21745587 +3.74258616 +2.02148892 +1.29006240 +1.04930793 +1.00334449 +1.00004347 +1.00000004 +1.00000000 +1.00000000 +1.00000000 +3.69851764 +2.96038837 +2.02148892 +1.39968435 +1.11057793 +1.01554183 +1.00084205 +1.00000730 +1.00000001 +1.00000000 +1.00000000 +1.00000000 +1.74250470 +1.54063711 +1.29006240 +1.11057793 +1.02533557 +1.00273473 +1.00010268 +1.00000045 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.14090797 +1.10012297 +1.04930793 +1.01554183 +1.00273473 +1.00021178 +1.00000413 +1.00000001 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.01151158 +1.00773513 +1.00334449 +1.00084205 +1.00010268 +1.00000413 +1.00000003 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00020750 +1.00012613 +1.00004347 +1.00000730 +1.00000045 +1.00000001 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000020 +1.00000011 +1.00000004 +1.00000001 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +6.74836785 +3.34525938 +1.71008559 +1.18071033 +1.02797014 +1.00196071 +1.00002255 +1.00000002 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +3.34525938 +2.24473568 +1.45588968 +1.12195513 +1.01848518 +1.00121571 +1.00001270 +1.00000001 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.71008559 +1.45588968 +1.19708047 +1.05398342 +1.00767405 +1.00043503 +1.00000367 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.18071033 +1.12195513 +1.05398342 +1.01389135 +1.00176949 +1.00007703 +1.00000045 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.02797014 +1.01848518 +1.00767405 +1.00176949 +1.00018095 +1.00000498 +1.00000002 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00196071 +1.00121571 +1.00043503 +1.00007703 +1.00000498 +1.00000007 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00002255 +1.00001270 +1.00000367 +1.00000045 +1.00000002 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000002 +1.00000001 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.12321467 +1.06814764 +1.01698812 +1.00161247 +1.00004667 +1.00000022 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.06814764 +1.03810236 +1.00962730 +1.00091153 +1.00002536 +1.00000012 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.01698812 +1.00962730 +1.00259320 +1.00025353 +1.00000659 +1.00000003 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00161247 +1.00091153 +1.00025353 +1.00002607 +1.00000061 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00004667 +1.00002536 +1.00000659 +1.00000061 +1.00000001 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000022 +1.00000012 +1.00000003 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 + +SCALARS mass float 1 +LOOKUP_TABLE default +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 + +SCALARS pressure float 1 +LOOKUP_TABLE default +0.08457759 +0.07837191 +0.08292159 +0.08784750 +0.09563914 +0.10658444 +0.12079947 +0.13869234 +0.17544854 +0.14070008 +0.69962468 +0.00007882 +0.07837191 +0.08100683 +0.07523654 +0.08115595 +0.08749239 +0.09673615 +0.10794677 +0.12406384 +0.13513252 +0.16869955 +0.10828226 +0.00001502 +0.08292159 +0.07523654 +0.08123854 +0.07586113 +0.08457621 +0.09301326 +0.10540776 +0.12461277 +0.12007987 +0.30137312 +0.00692725 +0.00000009 +0.08784750 +0.08115595 +0.07586113 +0.08496564 +0.08162346 +0.09297477 +0.11054294 +0.11301305 +0.16707078 +0.10993614 +0.00018963 +0.00000000 +0.09563914 +0.08749239 +0.08457621 +0.08162346 +0.09161912 +0.09568263 +0.10517885 +0.12224465 +0.26041562 +0.00683980 +0.00000034 +0.00000000 +0.10658444 +0.09673615 +0.09301326 +0.09297477 +0.09568263 +0.11030559 +0.09867278 +0.21720238 +0.03209334 +0.00007451 +0.00000000 +0.00000000 +0.12079947 +0.10794677 +0.10540776 +0.11054294 +0.10517885 +0.09867278 +0.16648816 +0.06288320 +0.00065344 +0.00000001 +0.00000000 +0.00000000 +0.13869234 +0.12406384 +0.12461277 +0.11301305 +0.12224465 +0.21720238 +0.06288320 +0.00308220 +0.00000020 +0.00000000 +0.00000000 +0.00000000 +0.17544854 +0.13513252 +0.12007987 +0.16707078 +0.26041562 +0.03209334 +0.00065344 +0.00000020 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.14070008 +0.16869955 +0.30137312 +0.10993614 +0.00683980 +0.00007451 +0.00000001 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.69962468 +0.10828226 +0.00692725 +0.00018963 +0.00000034 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00007882 +0.00001502 +0.00000009 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.07837191 +0.08100683 +0.07523654 +0.08115595 +0.08749239 +0.09673615 +0.10794677 +0.12406384 +0.13513252 +0.16869955 +0.10828226 +0.00001502 +0.08100683 +0.07293218 +0.07783931 +0.07752159 +0.08499539 +0.09356641 +0.10448850 +0.12396206 +0.12087465 +0.23422214 +0.02641282 +0.00000246 +0.07523654 +0.07783931 +0.07439134 +0.07907836 +0.08255778 +0.09199674 +0.10616337 +0.11978148 +0.12761413 +0.28482399 +0.00229972 +0.00000001 +0.08115595 +0.07752159 +0.07907836 +0.07869583 +0.08442278 +0.09188843 +0.10936506 +0.11001225 +0.19727178 +0.05535130 +0.00005850 +0.00000000 +0.08749239 +0.08499539 +0.08255778 +0.08442278 +0.08709285 +0.09998841 +0.10034150 +0.13384413 +0.20983509 +0.00314141 +0.00000009 +0.00000000 +0.09673615 +0.09356641 +0.09199674 +0.09188843 +0.09998841 +0.09872292 +0.10763605 +0.22983901 +0.01751933 +0.00002535 +0.00000000 +0.00000000 +0.10794677 +0.10448850 +0.10616337 +0.10936506 +0.10034150 +0.10763605 +0.18914162 +0.04021482 +0.00029063 +0.00000000 +0.00000000 +0.00000000 +0.12406384 +0.12396206 +0.11978148 +0.11001225 +0.13384413 +0.22983901 +0.04021482 +0.00114662 +0.00000006 +0.00000000 +0.00000000 +0.00000000 +0.13513252 +0.12087465 +0.12761413 +0.19727178 +0.20983509 +0.01751933 +0.00029063 +0.00000006 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.16869955 +0.23422214 +0.28482399 +0.05535130 +0.00314141 +0.00002535 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.10828226 +0.02641282 +0.00229972 +0.00005850 +0.00000009 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00001502 +0.00000246 +0.00000001 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.08292159 +0.07523654 +0.08123854 +0.07586113 +0.08457621 +0.09301326 +0.10540776 +0.12461277 +0.12007987 +0.30137312 +0.00692725 +0.00000009 +0.07523654 +0.07783931 +0.07439134 +0.07907836 +0.08255778 +0.09199674 +0.10616337 +0.11978148 +0.12761413 +0.28482399 +0.00229972 +0.00000001 +0.08123854 +0.07439134 +0.07419282 +0.07899819 +0.08424496 +0.09138784 +0.10780320 +0.11235274 +0.16612659 +0.10867524 +0.00024256 +0.00000000 +0.07586113 +0.07907836 +0.07899819 +0.07854948 +0.08600151 +0.09580539 +0.10455243 +0.11428434 +0.25353834 +0.01325347 +0.00000441 +0.00000000 +0.08457621 +0.08255778 +0.08424496 +0.08600151 +0.08886359 +0.09968564 +0.09894821 +0.17200500 +0.08838950 +0.00062229 +0.00000000 +0.00000000 +0.09301326 +0.09199674 +0.09138784 +0.09580539 +0.09968564 +0.09318184 +0.13384373 +0.18323069 +0.00514157 +0.00000248 +0.00000000 +0.00000000 +0.10540776 +0.10616337 +0.10780320 +0.10455243 +0.09894821 +0.13384373 +0.18843237 +0.01373889 +0.00005033 +0.00000000 +0.00000000 +0.00000000 +0.12461277 +0.11978148 +0.11235274 +0.11428434 +0.17200500 +0.18323069 +0.01373889 +0.00016164 +0.00000001 +0.00000000 +0.00000000 +0.00000000 +0.12007987 +0.12761413 +0.16612659 +0.25353834 +0.08838950 +0.00514157 +0.00005033 +0.00000001 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.30137312 +0.28482399 +0.10867524 +0.01325347 +0.00062229 +0.00000248 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00692725 +0.00229972 +0.00024256 +0.00000441 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000009 +0.00000001 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.08784750 +0.08115595 +0.07586113 +0.08496564 +0.08162346 +0.09297477 +0.11054294 +0.11301305 +0.16707078 +0.10993614 +0.00018963 +0.00000000 +0.08115595 +0.07752159 +0.07907836 +0.07869583 +0.08442278 +0.09188843 +0.10936506 +0.11001225 +0.19727178 +0.05535130 +0.00005850 +0.00000000 +0.07586113 +0.07907836 +0.07899819 +0.07854948 +0.08600151 +0.09580539 +0.10455243 +0.11428434 +0.25353834 +0.01325347 +0.00000441 +0.00000000 +0.08496564 +0.07869583 +0.07854948 +0.08422384 +0.08823131 +0.09961837 +0.10022264 +0.15114731 +0.13891303 +0.00141103 +0.00000004 +0.00000000 +0.08162346 +0.08442278 +0.08600151 +0.08823131 +0.09702565 +0.09546706 +0.11432600 +0.22903386 +0.01636932 +0.00003855 +0.00000000 +0.00000000 +0.09297477 +0.09188843 +0.09580539 +0.09961837 +0.09546706 +0.10662170 +0.19678706 +0.05723148 +0.00070762 +0.00000005 +0.00000000 +0.00000000 +0.11054294 +0.10936506 +0.10455243 +0.10022264 +0.11432600 +0.19678706 +0.07646540 +0.00228942 +0.00000232 +0.00000000 +0.00000000 +0.00000000 +0.11301305 +0.11001225 +0.11428434 +0.15114731 +0.22903386 +0.05723148 +0.00228942 +0.00000716 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.16707078 +0.19727178 +0.25353834 +0.13891303 +0.01636932 +0.00070762 +0.00000232 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.10993614 +0.05535130 +0.01325347 +0.00141103 +0.00003855 +0.00000005 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00018963 +0.00005850 +0.00000441 +0.00000004 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.09563914 +0.08749239 +0.08457621 +0.08162346 +0.09161912 +0.09568263 +0.10517885 +0.12224465 +0.26041562 +0.00683980 +0.00000034 +0.00000000 +0.08749239 +0.08499539 +0.08255778 +0.08442278 +0.08709285 +0.09998841 +0.10034150 +0.13384413 +0.20983509 +0.00314141 +0.00000009 +0.00000000 +0.08457621 +0.08255778 +0.08424496 +0.08600151 +0.08886359 +0.09968564 +0.09894821 +0.17200500 +0.08838950 +0.00062229 +0.00000000 +0.00000000 +0.08162346 +0.08442278 +0.08600151 +0.08823131 +0.09702565 +0.09546706 +0.11432600 +0.22903386 +0.01636932 +0.00003855 +0.00000000 +0.00000000 +0.09161912 +0.08709285 +0.08886359 +0.09702565 +0.09583979 +0.10194868 +0.17896421 +0.08978000 +0.00137903 +0.00000027 +0.00000000 +0.00000000 +0.09568263 +0.09998841 +0.09968564 +0.09546706 +0.10194868 +0.16204816 +0.15151291 +0.00718420 +0.00002830 +0.00000000 +0.00000000 +0.00000000 +0.10517885 +0.10034150 +0.09894821 +0.11432600 +0.17896421 +0.15151291 +0.01160257 +0.00014381 +0.00000001 +0.00000000 +0.00000000 +0.00000000 +0.12224465 +0.13384413 +0.17200500 +0.22903386 +0.08978000 +0.00718420 +0.00014381 +0.00000005 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.26041562 +0.20983509 +0.08838950 +0.01636932 +0.00137903 +0.00002830 +0.00000001 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00683980 +0.00314141 +0.00062229 +0.00003855 +0.00000027 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000034 +0.00000009 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.10658444 +0.09673615 +0.09301326 +0.09297477 +0.09568263 +0.11030559 +0.09867278 +0.21720238 +0.03209334 +0.00007451 +0.00000000 +0.00000000 +0.09673615 +0.09356641 +0.09199674 +0.09188843 +0.09998841 +0.09872292 +0.10763605 +0.22983901 +0.01751933 +0.00002535 +0.00000000 +0.00000000 +0.09301326 +0.09199674 +0.09138784 +0.09580539 +0.09968564 +0.09318184 +0.13384373 +0.18323069 +0.00514157 +0.00000248 +0.00000000 +0.00000000 +0.09297477 +0.09188843 +0.09580539 +0.09961837 +0.09546706 +0.10662170 +0.19678706 +0.05723148 +0.00070762 +0.00000005 +0.00000000 +0.00000000 +0.09568263 +0.09998841 +0.09968564 +0.09546706 +0.10194868 +0.16204816 +0.15151291 +0.00718420 +0.00002830 +0.00000000 +0.00000000 +0.00000000 +0.11030559 +0.09872292 +0.09318184 +0.10662170 +0.16204816 +0.18080410 +0.01948391 +0.00031154 +0.00000008 +0.00000000 +0.00000000 +0.00000000 +0.09867278 +0.10763605 +0.13384373 +0.19678706 +0.15151291 +0.01948391 +0.00065989 +0.00000114 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.21720238 +0.22983901 +0.18323069 +0.05723148 +0.00718420 +0.00031154 +0.00000114 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.03209334 +0.01751933 +0.00514157 +0.00070762 +0.00002830 +0.00000008 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00007451 +0.00002535 +0.00000248 +0.00000005 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.12079947 +0.10794677 +0.10540776 +0.11054294 +0.10517885 +0.09867278 +0.16648816 +0.06288320 +0.00065344 +0.00000001 +0.00000000 +0.00000000 +0.10794677 +0.10448850 +0.10616337 +0.10936506 +0.10034150 +0.10763605 +0.18914162 +0.04021482 +0.00029063 +0.00000000 +0.00000000 +0.00000000 +0.10540776 +0.10616337 +0.10780320 +0.10455243 +0.09894821 +0.13384373 +0.18843237 +0.01373889 +0.00005033 +0.00000000 +0.00000000 +0.00000000 +0.11054294 +0.10936506 +0.10455243 +0.10022264 +0.11432600 +0.19678706 +0.07646540 +0.00228942 +0.00000232 +0.00000000 +0.00000000 +0.00000000 +0.10517885 +0.10034150 +0.09894821 +0.11432600 +0.17896421 +0.15151291 +0.01160257 +0.00014381 +0.00000001 +0.00000000 +0.00000000 +0.00000000 +0.09867278 +0.10763605 +0.13384373 +0.19678706 +0.15151291 +0.01948391 +0.00065989 +0.00000114 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.16648816 +0.18914162 +0.18843237 +0.07646540 +0.01160257 +0.00065989 +0.00000408 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.06288320 +0.04021482 +0.01373889 +0.00228942 +0.00014381 +0.00000114 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00065344 +0.00029063 +0.00005033 +0.00000232 +0.00000001 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000001 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.13869234 +0.12406384 +0.12461277 +0.11301305 +0.12224465 +0.21720238 +0.06288320 +0.00308220 +0.00000020 +0.00000000 +0.00000000 +0.00000000 +0.12406384 +0.12396206 +0.11978148 +0.11001225 +0.13384413 +0.22983901 +0.04021482 +0.00114662 +0.00000006 +0.00000000 +0.00000000 +0.00000000 +0.12461277 +0.11978148 +0.11235274 +0.11428434 +0.17200500 +0.18323069 +0.01373889 +0.00016164 +0.00000001 +0.00000000 +0.00000000 +0.00000000 +0.11301305 +0.11001225 +0.11428434 +0.15114731 +0.22903386 +0.05723148 +0.00228942 +0.00000716 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.12224465 +0.13384413 +0.17200500 +0.22903386 +0.08978000 +0.00718420 +0.00014381 +0.00000005 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.21720238 +0.22983901 +0.18323069 +0.05723148 +0.00718420 +0.00031154 +0.00000114 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.06288320 +0.04021482 +0.01373889 +0.00228942 +0.00014381 +0.00000114 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00308220 +0.00114662 +0.00016164 +0.00000716 +0.00000005 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000020 +0.00000006 +0.00000001 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.17544854 +0.13513252 +0.12007987 +0.16707078 +0.26041562 +0.03209334 +0.00065344 +0.00000020 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.13513252 +0.12087465 +0.12761413 +0.19727178 +0.20983509 +0.01751933 +0.00029063 +0.00000006 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.12007987 +0.12761413 +0.16612659 +0.25353834 +0.08838950 +0.00514157 +0.00005033 +0.00000001 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.16707078 +0.19727178 +0.25353834 +0.13891303 +0.01636932 +0.00070762 +0.00000232 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.26041562 +0.20983509 +0.08838950 +0.01636932 +0.00137903 +0.00002830 +0.00000001 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.03209334 +0.01751933 +0.00514157 +0.00070762 +0.00002830 +0.00000008 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00065344 +0.00029063 +0.00005033 +0.00000232 +0.00000001 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000020 +0.00000006 +0.00000001 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.14070008 +0.16869955 +0.30137312 +0.10993614 +0.00683980 +0.00007451 +0.00000001 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.16869955 +0.23422214 +0.28482399 +0.05535130 +0.00314141 +0.00002535 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.30137312 +0.28482399 +0.10867524 +0.01325347 +0.00062229 +0.00000248 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.10993614 +0.05535130 +0.01325347 +0.00141103 +0.00003855 +0.00000005 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00683980 +0.00314141 +0.00062229 +0.00003855 +0.00000027 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00007451 +0.00002535 +0.00000248 +0.00000005 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000001 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.69962468 +0.10828226 +0.00692725 +0.00018963 +0.00000034 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.10828226 +0.02641282 +0.00229972 +0.00005850 +0.00000009 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00692725 +0.00229972 +0.00024256 +0.00000441 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00018963 +0.00005850 +0.00000441 +0.00000004 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000034 +0.00000009 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00007882 +0.00001502 +0.00000009 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00001502 +0.00000246 +0.00000001 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000009 +0.00000001 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 + +SCALARS sound_speed float 1 +LOOKUP_TABLE default +3.64687128 +1.47196145 +0.86960501 +0.68863321 +0.59588143 +0.53628762 +0.49074807 +0.45693474 +0.43494586 +0.37711640 +0.41567850 +0.01081493 +1.47196145 +1.19499322 +0.78749431 +0.59891687 +0.50792389 +0.45375605 +0.41334323 +0.38187220 +0.35489283 +0.33154333 +0.23226734 +0.00484158 +0.86960501 +0.78749431 +0.68703821 +0.52433926 +0.44219243 +0.39378564 +0.36113757 +0.33877278 +0.30600184 +0.31929852 +0.08216671 +0.00037403 +0.68863321 +0.59891687 +0.52433926 +0.48478608 +0.39981197 +0.35803210 +0.33458126 +0.30742979 +0.29971827 +0.22257725 +0.01636066 +0.00001481 +0.59588143 +0.50792389 +0.44219243 +0.39981197 +0.37717268 +0.32865098 +0.30246987 +0.28620385 +0.29402581 +0.08088334 +0.00073983 +0.00001055 +0.53628762 +0.45375605 +0.39378564 +0.35803210 +0.32865098 +0.31560188 +0.27380444 +0.28470101 +0.14428797 +0.01043263 +0.00001580 +0.00001054 +0.49074807 +0.41334323 +0.36113757 +0.33458126 +0.30246987 +0.27380444 +0.28057988 +0.18141565 +0.02886257 +0.00013649 +0.00001054 +0.00001054 +0.45693474 +0.38187220 +0.33877278 +0.30742979 +0.28620385 +0.28470101 +0.18141565 +0.05761829 +0.00057044 +0.00001061 +0.00001054 +0.00001054 +0.43494586 +0.35489283 +0.30600184 +0.29971827 +0.29402581 +0.14428797 +0.02886257 +0.00057044 +0.00001166 +0.00001054 +0.00001054 +0.00001054 +0.37711640 +0.33154333 +0.31929852 +0.22257725 +0.08088334 +0.01043263 +0.00013649 +0.00001061 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.41567850 +0.23226734 +0.08216671 +0.01636066 +0.00073983 +0.00001580 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.01081493 +0.00484158 +0.00037403 +0.00001481 +0.00001055 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +1.47196145 +1.19499322 +0.78749431 +0.59891687 +0.50792389 +0.45375605 +0.41334323 +0.38187220 +0.35489283 +0.33154333 +0.23226734 +0.00484158 +1.19499322 +0.94167592 +0.71518902 +0.54403158 +0.46469088 +0.41275350 +0.37704941 +0.35326129 +0.32026235 +0.32060850 +0.14003905 +0.00198558 +0.78749431 +0.71518902 +0.59478358 +0.49476117 +0.41796229 +0.37571871 +0.34731663 +0.32326735 +0.29947524 +0.30163606 +0.05130944 +0.00015391 +0.59891687 +0.54403158 +0.49476117 +0.43716082 +0.38429248 +0.34315135 +0.32339547 +0.29641515 +0.29853439 +0.17652814 +0.00932219 +0.00001221 +0.50792389 +0.46469088 +0.41796229 +0.38429248 +0.34976413 +0.32232991 +0.29185223 +0.28251204 +0.27028652 +0.05829573 +0.00037442 +0.00001054 +0.45375605 +0.41275350 +0.37571871 +0.34315135 +0.32232991 +0.29392036 +0.27296868 +0.28175875 +0.11515441 +0.00619698 +0.00001289 +0.00001054 +0.41334323 +0.37704941 +0.34731663 +0.32339547 +0.29185223 +0.27296868 +0.27514266 +0.15467737 +0.01983436 +0.00007437 +0.00001054 +0.00001054 +0.38187220 +0.35326129 +0.32326735 +0.29641515 +0.28251204 +0.28175875 +0.15467737 +0.03721566 +0.00032433 +0.00001057 +0.00001054 +0.00001054 +0.35489283 +0.32026235 +0.29947524 +0.29853439 +0.27028652 +0.11515441 +0.01983436 +0.00032433 +0.00001099 +0.00001054 +0.00001054 +0.00001054 +0.33154333 +0.32060850 +0.30163606 +0.17652814 +0.05829573 +0.00619698 +0.00007437 +0.00001057 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.23226734 +0.14003905 +0.05130944 +0.00932219 +0.00037442 +0.00001289 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00484158 +0.00198558 +0.00015391 +0.00001221 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.86960501 +0.78749431 +0.68703821 +0.52433926 +0.44219243 +0.39378564 +0.36113757 +0.33877278 +0.30600184 +0.31929852 +0.08216671 +0.00037403 +0.78749431 +0.71518902 +0.59478358 +0.49476117 +0.41796229 +0.37571871 +0.34731663 +0.32326735 +0.29947524 +0.30163606 +0.05130944 +0.00015391 +0.68703821 +0.59478358 +0.50491546 +0.44130032 +0.39066221 +0.35094184 +0.32760556 +0.30324065 +0.29553262 +0.21999042 +0.01837672 +0.00002368 +0.52433926 +0.49476117 +0.44130032 +0.39143064 +0.35832198 +0.32869523 +0.30327579 +0.28407677 +0.29502084 +0.10453302 +0.00264075 +0.00001071 +0.44219243 +0.41796229 +0.39066221 +0.35832198 +0.32533553 +0.30510013 +0.27942608 +0.28126185 +0.20254900 +0.02835399 +0.00008876 +0.00001054 +0.39378564 +0.37571871 +0.35094184 +0.32869523 +0.30510013 +0.27523973 +0.27114086 +0.25502390 +0.07083202 +0.00198369 +0.00001092 +0.00001054 +0.36113757 +0.34731663 +0.32760556 +0.30327579 +0.27942608 +0.27114086 +0.25873764 +0.10396576 +0.00862150 +0.00002511 +0.00001054 +0.00001054 +0.33877278 +0.32326735 +0.30324065 +0.28407677 +0.28126185 +0.25502390 +0.10396576 +0.01503287 +0.00009890 +0.00001055 +0.00001054 +0.00001054 +0.30600184 +0.29947524 +0.29553262 +0.29502084 +0.20254900 +0.07083202 +0.00862150 +0.00009890 +0.00001059 +0.00001054 +0.00001054 +0.00001054 +0.31929852 +0.30163606 +0.21999042 +0.10453302 +0.02835399 +0.00198369 +0.00002511 +0.00001055 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.08216671 +0.05130944 +0.01837672 +0.00264075 +0.00008876 +0.00001092 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00037403 +0.00015391 +0.00002368 +0.00001071 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.68863321 +0.59891687 +0.52433926 +0.48478608 +0.39981197 +0.35803210 +0.33458126 +0.30742979 +0.29971827 +0.22257725 +0.01636066 +0.00001481 +0.59891687 +0.54403158 +0.49476117 +0.43716082 +0.38429248 +0.34315135 +0.32339547 +0.29641515 +0.29853439 +0.17652814 +0.00932219 +0.00001221 +0.52433926 +0.49476117 +0.44130032 +0.39143064 +0.35832198 +0.32869523 +0.30327579 +0.28407677 +0.29502084 +0.10453302 +0.00264075 +0.00001071 +0.48478608 +0.43716082 +0.39143064 +0.36254065 +0.33018285 +0.31010972 +0.28498046 +0.27891759 +0.23393994 +0.04098991 +0.00025368 +0.00001054 +0.39981197 +0.38429248 +0.35832198 +0.33018285 +0.31185979 +0.28557340 +0.27250044 +0.27917138 +0.11148623 +0.00760629 +0.00001593 +0.00001054 +0.35803210 +0.34315135 +0.32869523 +0.31010972 +0.28557340 +0.26994111 +0.27582365 +0.17298712 +0.02988380 +0.00027286 +0.00001056 +0.00001054 +0.33458126 +0.32339547 +0.30327579 +0.28498046 +0.27250044 +0.27582365 +0.18988735 +0.05004989 +0.00191934 +0.00001171 +0.00001054 +0.00001054 +0.30742979 +0.29641515 +0.28407677 +0.27891759 +0.27917138 +0.17298712 +0.05004989 +0.00333814 +0.00001997 +0.00001054 +0.00001054 +0.00001054 +0.29971827 +0.29853439 +0.29502084 +0.23393994 +0.11148623 +0.02988380 +0.00191934 +0.00001997 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.22257725 +0.17652814 +0.10453302 +0.04098991 +0.00760629 +0.00027286 +0.00001171 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.01636066 +0.00932219 +0.00264075 +0.00025368 +0.00001593 +0.00001056 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001481 +0.00001221 +0.00001071 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.59588143 +0.50792389 +0.44219243 +0.39981197 +0.37717268 +0.32865098 +0.30246987 +0.28620385 +0.29402581 +0.08088334 +0.00073983 +0.00001055 +0.50792389 +0.46469088 +0.41796229 +0.38429248 +0.34976413 +0.32232991 +0.29185223 +0.28251204 +0.27028652 +0.05829573 +0.00037442 +0.00001054 +0.44219243 +0.41796229 +0.39066221 +0.35832198 +0.32533553 +0.30510013 +0.27942608 +0.28126185 +0.20254900 +0.02835399 +0.00008876 +0.00001054 +0.39981197 +0.38429248 +0.35832198 +0.33018285 +0.31185979 +0.28557340 +0.27250044 +0.27917138 +0.11148623 +0.00760629 +0.00001593 +0.00001054 +0.37717268 +0.34976413 +0.32533553 +0.31185979 +0.28964811 +0.27224602 +0.27489155 +0.20071891 +0.04025066 +0.00066411 +0.00001062 +0.00001054 +0.32865098 +0.32232991 +0.30510013 +0.28557340 +0.27224602 +0.27448148 +0.23774636 +0.08043416 +0.00653668 +0.00002130 +0.00001054 +0.00001054 +0.30246987 +0.29185223 +0.27942608 +0.27250044 +0.27489155 +0.23774636 +0.09691234 +0.01422822 +0.00015379 +0.00001056 +0.00001054 +0.00001054 +0.28620385 +0.28251204 +0.28126185 +0.27917138 +0.20071891 +0.08043416 +0.01422822 +0.00029793 +0.00001080 +0.00001054 +0.00001054 +0.00001054 +0.29402581 +0.27028652 +0.20254900 +0.11148623 +0.04025066 +0.00653668 +0.00015379 +0.00001080 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.08088334 +0.05829573 +0.02835399 +0.00760629 +0.00066411 +0.00002130 +0.00001056 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00073983 +0.00037442 +0.00008876 +0.00001593 +0.00001062 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001055 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.53628762 +0.45375605 +0.39378564 +0.35803210 +0.32865098 +0.31560188 +0.27380444 +0.28470101 +0.14428797 +0.01043263 +0.00001580 +0.00001054 +0.45375605 +0.41275350 +0.37571871 +0.34315135 +0.32232991 +0.29392036 +0.27296868 +0.28175875 +0.11515441 +0.00619698 +0.00001289 +0.00001054 +0.39378564 +0.37571871 +0.35094184 +0.32869523 +0.30510013 +0.27523973 +0.27114086 +0.25502390 +0.07083202 +0.00198369 +0.00001092 +0.00001054 +0.35803210 +0.34315135 +0.32869523 +0.31010972 +0.28557340 +0.26994111 +0.27582365 +0.17298712 +0.02988380 +0.00027286 +0.00001056 +0.00001054 +0.32865098 +0.32232991 +0.30510013 +0.28557340 +0.27224602 +0.27448148 +0.23774636 +0.08043416 +0.00653668 +0.00002130 +0.00001054 +0.00001054 +0.31560188 +0.29392036 +0.27523973 +0.26994111 +0.27448148 +0.25332735 +0.11768853 +0.02043551 +0.00035919 +0.00001063 +0.00001054 +0.00001054 +0.27380444 +0.27296868 +0.27114086 +0.27582365 +0.23774636 +0.11768853 +0.02885434 +0.00135211 +0.00001308 +0.00001054 +0.00001054 +0.00001054 +0.28470101 +0.28175875 +0.25502390 +0.17298712 +0.08043416 +0.02043551 +0.00135211 +0.00001727 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.14428797 +0.11515441 +0.07083202 +0.02988380 +0.00653668 +0.00035919 +0.00001308 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.01043263 +0.00619698 +0.00198369 +0.00027286 +0.00002130 +0.00001063 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001580 +0.00001289 +0.00001092 +0.00001056 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.49074807 +0.41334323 +0.36113757 +0.33458126 +0.30246987 +0.27380444 +0.28057988 +0.18141565 +0.02886257 +0.00013649 +0.00001054 +0.00001054 +0.41334323 +0.37704941 +0.34731663 +0.32339547 +0.29185223 +0.27296868 +0.27514266 +0.15467737 +0.01983436 +0.00007437 +0.00001054 +0.00001054 +0.36113757 +0.34731663 +0.32760556 +0.30327579 +0.27942608 +0.27114086 +0.25873764 +0.10396576 +0.00862150 +0.00002511 +0.00001054 +0.00001054 +0.33458126 +0.32339547 +0.30327579 +0.28498046 +0.27250044 +0.27582365 +0.18988735 +0.05004989 +0.00191934 +0.00001171 +0.00001054 +0.00001054 +0.30246987 +0.29185223 +0.27942608 +0.27250044 +0.27489155 +0.23774636 +0.09691234 +0.01422822 +0.00015379 +0.00001056 +0.00001054 +0.00001054 +0.27380444 +0.27296868 +0.27114086 +0.27582365 +0.23774636 +0.11768853 +0.02885434 +0.00135211 +0.00001308 +0.00001054 +0.00001054 +0.00001054 +0.28057988 +0.27514266 +0.25873764 +0.18988735 +0.09691234 +0.02885434 +0.00253416 +0.00002930 +0.00001055 +0.00001054 +0.00001054 +0.00001054 +0.18141565 +0.15467737 +0.10396576 +0.05004989 +0.01422822 +0.00135211 +0.00002930 +0.00001056 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.02886257 +0.01983436 +0.00862150 +0.00191934 +0.00015379 +0.00001308 +0.00001055 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00013649 +0.00007437 +0.00002511 +0.00001171 +0.00001056 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.45693474 +0.38187220 +0.33877278 +0.30742979 +0.28620385 +0.28470101 +0.18141565 +0.05761829 +0.00057044 +0.00001061 +0.00001054 +0.00001054 +0.38187220 +0.35326129 +0.32326735 +0.29641515 +0.28251204 +0.28175875 +0.15467737 +0.03721566 +0.00032433 +0.00001057 +0.00001054 +0.00001054 +0.33877278 +0.32326735 +0.30324065 +0.28407677 +0.28126185 +0.25502390 +0.10396576 +0.01503287 +0.00009890 +0.00001055 +0.00001054 +0.00001054 +0.30742979 +0.29641515 +0.28407677 +0.27891759 +0.27917138 +0.17298712 +0.05004989 +0.00333814 +0.00001997 +0.00001054 +0.00001054 +0.00001054 +0.28620385 +0.28251204 +0.28126185 +0.27917138 +0.20071891 +0.08043416 +0.01422822 +0.00029793 +0.00001080 +0.00001054 +0.00001054 +0.00001054 +0.28470101 +0.28175875 +0.25502390 +0.17298712 +0.08043416 +0.02043551 +0.00135211 +0.00001727 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.18141565 +0.15467737 +0.10396576 +0.05004989 +0.01422822 +0.00135211 +0.00002930 +0.00001056 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.05761829 +0.03721566 +0.01503287 +0.00333814 +0.00029793 +0.00001727 +0.00001056 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00057044 +0.00032433 +0.00009890 +0.00001997 +0.00001080 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001061 +0.00001057 +0.00001055 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.43494586 +0.35489283 +0.30600184 +0.29971827 +0.29402581 +0.14428797 +0.02886257 +0.00057044 +0.00001166 +0.00001054 +0.00001054 +0.00001054 +0.35489283 +0.32026235 +0.29947524 +0.29853439 +0.27028652 +0.11515441 +0.01983436 +0.00032433 +0.00001099 +0.00001054 +0.00001054 +0.00001054 +0.30600184 +0.29947524 +0.29553262 +0.29502084 +0.20254900 +0.07083202 +0.00862150 +0.00009890 +0.00001059 +0.00001054 +0.00001054 +0.00001054 +0.29971827 +0.29853439 +0.29502084 +0.23393994 +0.11148623 +0.02988380 +0.00191934 +0.00001997 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.29402581 +0.27028652 +0.20254900 +0.11148623 +0.04025066 +0.00653668 +0.00015379 +0.00001080 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.14428797 +0.11515441 +0.07083202 +0.02988380 +0.00653668 +0.00035919 +0.00001308 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.02886257 +0.01983436 +0.00862150 +0.00191934 +0.00015379 +0.00001308 +0.00001055 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00057044 +0.00032433 +0.00009890 +0.00001997 +0.00001080 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001166 +0.00001099 +0.00001059 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.37711640 +0.33154333 +0.31929852 +0.22257725 +0.08088334 +0.01043263 +0.00013649 +0.00001061 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.33154333 +0.32060850 +0.30163606 +0.17652814 +0.05829573 +0.00619698 +0.00007437 +0.00001057 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.31929852 +0.30163606 +0.21999042 +0.10453302 +0.02835399 +0.00198369 +0.00002511 +0.00001055 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.22257725 +0.17652814 +0.10453302 +0.04098991 +0.00760629 +0.00027286 +0.00001171 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.08088334 +0.05829573 +0.02835399 +0.00760629 +0.00066411 +0.00002130 +0.00001056 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.01043263 +0.00619698 +0.00198369 +0.00027286 +0.00002130 +0.00001063 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00013649 +0.00007437 +0.00002511 +0.00001171 +0.00001056 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001061 +0.00001057 +0.00001055 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.41567850 +0.23226734 +0.08216671 +0.01636066 +0.00073983 +0.00001580 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.23226734 +0.14003905 +0.05130944 +0.00932219 +0.00037442 +0.00001289 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.08216671 +0.05130944 +0.01837672 +0.00264075 +0.00008876 +0.00001092 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.01636066 +0.00932219 +0.00264075 +0.00025368 +0.00001593 +0.00001056 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00073983 +0.00037442 +0.00008876 +0.00001593 +0.00001062 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001580 +0.00001289 +0.00001092 +0.00001056 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.01081493 +0.00484158 +0.00037403 +0.00001481 +0.00001055 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00484158 +0.00198558 +0.00015391 +0.00001221 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00037403 +0.00015391 +0.00002368 +0.00001071 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001481 +0.00001221 +0.00001071 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001055 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 + +SCALARS volume float 1 +LOOKUP_TABLE default +0.09434889 +0.01658761 +0.00547177 +0.00323890 +0.00222759 +0.00161902 +0.00119620 +0.00090325 +0.00064695 +0.00060647 +0.00014818 +0.00089030 +0.01658761 +0.01057695 +0.00494558 +0.00265194 +0.00176921 +0.00127705 +0.00094965 +0.00070525 +0.00055922 +0.00039095 +0.00029893 +0.00093620 +0.00547177 +0.00494558 +0.00348619 +0.00217449 +0.00138716 +0.00100029 +0.00074238 +0.00055259 +0.00046787 +0.00020297 +0.00058477 +0.00098330 +0.00323890 +0.00265194 +0.00217449 +0.00165962 +0.00117503 +0.00082724 +0.00060761 +0.00050178 +0.00032261 +0.00027038 +0.00084695 +0.00099839 +0.00222759 +0.00176921 +0.00138716 +0.00117503 +0.00093163 +0.00067731 +0.00052190 +0.00040204 +0.00019918 +0.00057389 +0.00097279 +0.00099995 +0.00161902 +0.00127705 +0.00100029 +0.00082724 +0.00067731 +0.00054179 +0.00045586 +0.00022391 +0.00038922 +0.00087649 +0.00099804 +0.00100000 +0.00119620 +0.00094965 +0.00074238 +0.00060761 +0.00052190 +0.00045586 +0.00028371 +0.00031403 +0.00076492 +0.00098862 +0.00099998 +0.00100000 +0.00090325 +0.00070525 +0.00055259 +0.00050178 +0.00040204 +0.00022391 +0.00031403 +0.00064627 +0.00097354 +0.00099979 +0.00100000 +0.00100000 +0.00064695 +0.00055922 +0.00046787 +0.00032261 +0.00019918 +0.00038922 +0.00076492 +0.00097354 +0.00099906 +0.00100000 +0.00100000 +0.00100000 +0.00060647 +0.00039095 +0.00020297 +0.00027038 +0.00057389 +0.00087649 +0.00098862 +0.00099979 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00014818 +0.00029893 +0.00058477 +0.00084695 +0.00097279 +0.00099804 +0.00099998 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00089030 +0.00093620 +0.00098330 +0.00099839 +0.00099995 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.01658761 +0.01057695 +0.00494558 +0.00265194 +0.00176921 +0.00127705 +0.00094965 +0.00070525 +0.00055922 +0.00039095 +0.00029893 +0.00093620 +0.01057695 +0.00729516 +0.00394270 +0.00229075 +0.00152435 +0.00109248 +0.00081636 +0.00060402 +0.00050913 +0.00026331 +0.00044549 +0.00096330 +0.00494558 +0.00394270 +0.00285330 +0.00185731 +0.00126960 +0.00092067 +0.00068175 +0.00052346 +0.00042167 +0.00019166 +0.00068687 +0.00099046 +0.00265194 +0.00229075 +0.00185731 +0.00145708 +0.00104958 +0.00076889 +0.00057377 +0.00047919 +0.00027107 +0.00033779 +0.00089130 +0.00099909 +0.00176921 +0.00152435 +0.00126960 +0.00104958 +0.00084279 +0.00062345 +0.00050933 +0.00035779 +0.00020889 +0.00064908 +0.00098185 +0.00099997 +0.00127705 +0.00109248 +0.00092067 +0.00076889 +0.00062345 +0.00052504 +0.00041535 +0.00020724 +0.00045415 +0.00090899 +0.00099879 +0.00100000 +0.00094965 +0.00081636 +0.00068175 +0.00057377 +0.00050933 +0.00041535 +0.00024015 +0.00035696 +0.00081217 +0.00099232 +0.00099999 +0.00100000 +0.00070525 +0.00060402 +0.00052346 +0.00047919 +0.00035779 +0.00020724 +0.00035696 +0.00072474 +0.00098105 +0.00099987 +0.00100000 +0.00100000 +0.00055922 +0.00050913 +0.00042167 +0.00027107 +0.00020889 +0.00045415 +0.00081217 +0.00098105 +0.00099946 +0.00100000 +0.00100000 +0.00100000 +0.00039095 +0.00026331 +0.00019166 +0.00033779 +0.00064908 +0.00090899 +0.00099232 +0.00099987 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00029893 +0.00044549 +0.00068687 +0.00089130 +0.00098185 +0.00099879 +0.00099999 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00093620 +0.00096330 +0.00099046 +0.00099909 +0.00099997 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00547177 +0.00494558 +0.00348619 +0.00217449 +0.00138716 +0.00100029 +0.00074238 +0.00055259 +0.00046787 +0.00020297 +0.00058477 +0.00098330 +0.00494558 +0.00394270 +0.00285330 +0.00185731 +0.00126960 +0.00092067 +0.00068175 +0.00052346 +0.00042167 +0.00019166 +0.00068687 +0.00099046 +0.00348619 +0.00285330 +0.00206171 +0.00147912 +0.00108695 +0.00080860 +0.00059734 +0.00049107 +0.00031544 +0.00026719 +0.00083537 +0.00099741 +0.00217449 +0.00185731 +0.00147912 +0.00117035 +0.00089576 +0.00067663 +0.00052783 +0.00042368 +0.00020597 +0.00049468 +0.00094878 +0.00099975 +0.00138716 +0.00126960 +0.00108695 +0.00089576 +0.00071465 +0.00056028 +0.00047345 +0.00027595 +0.00027849 +0.00077516 +0.00099238 +0.00099999 +0.00100029 +0.00092067 +0.00080860 +0.00067663 +0.00056028 +0.00048780 +0.00032957 +0.00021297 +0.00058548 +0.00095301 +0.00099957 +0.00100000 +0.00074238 +0.00068175 +0.00059734 +0.00052783 +0.00047345 +0.00032957 +0.00021316 +0.00047204 +0.00088603 +0.00099667 +0.00100000 +0.00100000 +0.00055259 +0.00052346 +0.00049107 +0.00042368 +0.00027595 +0.00021297 +0.00047204 +0.00083888 +0.00099080 +0.00099996 +0.00100000 +0.00100000 +0.00046787 +0.00042167 +0.00031544 +0.00020597 +0.00027849 +0.00058548 +0.00088603 +0.00099080 +0.00099983 +0.00100000 +0.00100000 +0.00100000 +0.00020297 +0.00019166 +0.00026719 +0.00049468 +0.00077516 +0.00095301 +0.00099667 +0.00099996 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00058477 +0.00068687 +0.00083537 +0.00094878 +0.00099238 +0.00099957 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00098330 +0.00099046 +0.00099741 +0.00099975 +0.00099999 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00323890 +0.00265194 +0.00217449 +0.00165962 +0.00117503 +0.00082724 +0.00060761 +0.00050178 +0.00032261 +0.00027038 +0.00084695 +0.00099839 +0.00265194 +0.00229075 +0.00185731 +0.00145708 +0.00104958 +0.00076889 +0.00057377 +0.00047919 +0.00027107 +0.00033779 +0.00089130 +0.00099909 +0.00217449 +0.00185731 +0.00147912 +0.00117035 +0.00089576 +0.00067663 +0.00052783 +0.00042368 +0.00020597 +0.00049468 +0.00094878 +0.00099975 +0.00165962 +0.00145708 +0.00117035 +0.00093633 +0.00074137 +0.00057922 +0.00048620 +0.00030882 +0.00023638 +0.00071445 +0.00098630 +0.00099997 +0.00117503 +0.00104958 +0.00089576 +0.00074137 +0.00060143 +0.00051255 +0.00038971 +0.00020417 +0.00045558 +0.00090043 +0.00099823 +0.00100000 +0.00082724 +0.00076889 +0.00067663 +0.00057922 +0.00051255 +0.00041006 +0.00023196 +0.00031372 +0.00075723 +0.00098470 +0.00099992 +0.00100000 +0.00060761 +0.00057377 +0.00052783 +0.00048620 +0.00038971 +0.00023196 +0.00028293 +0.00065650 +0.00095263 +0.00099916 +0.00100000 +0.00100000 +0.00050178 +0.00047919 +0.00042368 +0.00030882 +0.00020417 +0.00031372 +0.00065650 +0.00093366 +0.00099732 +0.00099999 +0.00100000 +0.00100000 +0.00032261 +0.00027107 +0.00020597 +0.00023638 +0.00045558 +0.00075723 +0.00095263 +0.00099732 +0.00099998 +0.00100000 +0.00100000 +0.00100000 +0.00027038 +0.00033779 +0.00049468 +0.00071445 +0.00090043 +0.00098470 +0.00099916 +0.00099999 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00084695 +0.00089130 +0.00094878 +0.00098630 +0.00099823 +0.00099992 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00099839 +0.00099909 +0.00099975 +0.00099997 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00222759 +0.00176921 +0.00138716 +0.00117503 +0.00093163 +0.00067731 +0.00052190 +0.00040204 +0.00019918 +0.00057389 +0.00097279 +0.00099995 +0.00176921 +0.00152435 +0.00126960 +0.00104958 +0.00084279 +0.00062345 +0.00050933 +0.00035779 +0.00020889 +0.00064908 +0.00098185 +0.00099997 +0.00138716 +0.00126960 +0.00108695 +0.00089576 +0.00071465 +0.00056028 +0.00047345 +0.00027595 +0.00027849 +0.00077516 +0.00099238 +0.00099999 +0.00117503 +0.00104958 +0.00089576 +0.00074137 +0.00060143 +0.00051255 +0.00038971 +0.00020417 +0.00045558 +0.00090043 +0.00099823 +0.00100000 +0.00093163 +0.00084279 +0.00071465 +0.00060143 +0.00052523 +0.00043621 +0.00025334 +0.00026925 +0.00070489 +0.00097529 +0.00099982 +0.00100000 +0.00067731 +0.00062345 +0.00056028 +0.00051255 +0.00043621 +0.00027895 +0.00022384 +0.00054032 +0.00090580 +0.00099727 +0.00100000 +0.00100000 +0.00052190 +0.00050933 +0.00047345 +0.00038971 +0.00025334 +0.00022384 +0.00048569 +0.00084465 +0.00098860 +0.00099990 +0.00100000 +0.00100000 +0.00040204 +0.00035779 +0.00027595 +0.00020417 +0.00026925 +0.00054032 +0.00084465 +0.00098310 +0.00099961 +0.00100000 +0.00100000 +0.00100000 +0.00019918 +0.00020889 +0.00027849 +0.00045558 +0.00070489 +0.00090580 +0.00098860 +0.00099961 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00057389 +0.00064908 +0.00077516 +0.00090043 +0.00097529 +0.00099727 +0.00099990 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00097279 +0.00098185 +0.00099238 +0.00099823 +0.00099982 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00099995 +0.00099997 +0.00099999 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00161902 +0.00127705 +0.00100029 +0.00082724 +0.00067731 +0.00054179 +0.00045586 +0.00022391 +0.00038922 +0.00087649 +0.00099804 +0.00100000 +0.00127705 +0.00109248 +0.00092067 +0.00076889 +0.00062345 +0.00052504 +0.00041535 +0.00020724 +0.00045415 +0.00090899 +0.00099879 +0.00100000 +0.00100029 +0.00092067 +0.00080860 +0.00067663 +0.00056028 +0.00048780 +0.00032957 +0.00021297 +0.00058548 +0.00095301 +0.00099957 +0.00100000 +0.00082724 +0.00076889 +0.00067663 +0.00057922 +0.00051255 +0.00041006 +0.00023196 +0.00031372 +0.00075723 +0.00098470 +0.00099992 +0.00100000 +0.00067731 +0.00062345 +0.00056028 +0.00051255 +0.00043621 +0.00027895 +0.00022384 +0.00054032 +0.00090580 +0.00099727 +0.00100000 +0.00100000 +0.00054179 +0.00052504 +0.00048780 +0.00041006 +0.00027895 +0.00021296 +0.00042652 +0.00080428 +0.00098176 +0.00099979 +0.00100000 +0.00100000 +0.00045586 +0.00041535 +0.00032957 +0.00023196 +0.00022384 +0.00042652 +0.00075701 +0.00096121 +0.00099868 +0.00100000 +0.00100000 +0.00100000 +0.00022391 +0.00020724 +0.00021297 +0.00031372 +0.00054032 +0.00080428 +0.00096121 +0.00099776 +0.00099998 +0.00100000 +0.00100000 +0.00100000 +0.00038922 +0.00045415 +0.00058548 +0.00075723 +0.00090580 +0.00098176 +0.00099868 +0.00099998 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00087649 +0.00090899 +0.00095301 +0.00098470 +0.00099727 +0.00099979 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00099804 +0.00099879 +0.00099957 +0.00099992 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00119620 +0.00094965 +0.00074238 +0.00060761 +0.00052190 +0.00045586 +0.00028371 +0.00031403 +0.00076492 +0.00098862 +0.00099998 +0.00100000 +0.00094965 +0.00081636 +0.00068175 +0.00057377 +0.00050933 +0.00041535 +0.00024015 +0.00035696 +0.00081217 +0.00099232 +0.00099999 +0.00100000 +0.00074238 +0.00068175 +0.00059734 +0.00052783 +0.00047345 +0.00032957 +0.00021316 +0.00047204 +0.00088603 +0.00099667 +0.00100000 +0.00100000 +0.00060761 +0.00057377 +0.00052783 +0.00048620 +0.00038971 +0.00023196 +0.00028293 +0.00065650 +0.00095263 +0.00099916 +0.00100000 +0.00100000 +0.00052190 +0.00050933 +0.00047345 +0.00038971 +0.00025334 +0.00022384 +0.00048569 +0.00084465 +0.00098860 +0.00099990 +0.00100000 +0.00100000 +0.00045586 +0.00041535 +0.00032957 +0.00023196 +0.00022384 +0.00042652 +0.00075701 +0.00096121 +0.00099868 +0.00100000 +0.00100000 +0.00100000 +0.00028371 +0.00024015 +0.00021316 +0.00028293 +0.00048569 +0.00075701 +0.00094457 +0.00099612 +0.00099995 +0.00100000 +0.00100000 +0.00100000 +0.00031403 +0.00035696 +0.00047204 +0.00065650 +0.00084465 +0.00096121 +0.00099612 +0.00099989 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00076492 +0.00081217 +0.00088603 +0.00095263 +0.00098860 +0.00099868 +0.00099995 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00098862 +0.00099232 +0.00099667 +0.00099916 +0.00099990 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00099998 +0.00099999 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00090325 +0.00070525 +0.00055259 +0.00050178 +0.00040204 +0.00022391 +0.00031403 +0.00064627 +0.00097354 +0.00099979 +0.00100000 +0.00100000 +0.00070525 +0.00060402 +0.00052346 +0.00047919 +0.00035779 +0.00020724 +0.00035696 +0.00072474 +0.00098105 +0.00099987 +0.00100000 +0.00100000 +0.00055259 +0.00052346 +0.00049107 +0.00042368 +0.00027595 +0.00021297 +0.00047204 +0.00083888 +0.00099080 +0.00099996 +0.00100000 +0.00100000 +0.00050178 +0.00047919 +0.00042368 +0.00030882 +0.00020417 +0.00031372 +0.00065650 +0.00093366 +0.00099732 +0.00099999 +0.00100000 +0.00100000 +0.00040204 +0.00035779 +0.00027595 +0.00020417 +0.00026925 +0.00054032 +0.00084465 +0.00098310 +0.00099961 +0.00100000 +0.00100000 +0.00100000 +0.00022391 +0.00020724 +0.00021297 +0.00031372 +0.00054032 +0.00080428 +0.00096121 +0.00099776 +0.00099998 +0.00100000 +0.00100000 +0.00100000 +0.00031403 +0.00035696 +0.00047204 +0.00065650 +0.00084465 +0.00096121 +0.00099612 +0.00099989 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00064627 +0.00072474 +0.00083888 +0.00093366 +0.00098310 +0.00099776 +0.00099989 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00097354 +0.00098105 +0.00099080 +0.00099732 +0.00099961 +0.00099998 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00099979 +0.00099987 +0.00099996 +0.00099999 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00064695 +0.00055922 +0.00046787 +0.00032261 +0.00019918 +0.00038922 +0.00076492 +0.00097354 +0.00099906 +0.00100000 +0.00100000 +0.00100000 +0.00055922 +0.00050913 +0.00042167 +0.00027107 +0.00020889 +0.00045415 +0.00081217 +0.00098105 +0.00099946 +0.00100000 +0.00100000 +0.00100000 +0.00046787 +0.00042167 +0.00031544 +0.00020597 +0.00027849 +0.00058548 +0.00088603 +0.00099080 +0.00099983 +0.00100000 +0.00100000 +0.00100000 +0.00032261 +0.00027107 +0.00020597 +0.00023638 +0.00045558 +0.00075723 +0.00095263 +0.00099732 +0.00099998 +0.00100000 +0.00100000 +0.00100000 +0.00019918 +0.00020889 +0.00027849 +0.00045558 +0.00070489 +0.00090580 +0.00098860 +0.00099961 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00038922 +0.00045415 +0.00058548 +0.00075723 +0.00090580 +0.00098176 +0.00099868 +0.00099998 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00076492 +0.00081217 +0.00088603 +0.00095263 +0.00098860 +0.00099868 +0.00099995 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00097354 +0.00098105 +0.00099080 +0.00099732 +0.00099961 +0.00099998 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00099906 +0.00099946 +0.00099983 +0.00099998 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00060647 +0.00039095 +0.00020297 +0.00027038 +0.00057389 +0.00087649 +0.00098862 +0.00099979 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00039095 +0.00026331 +0.00019166 +0.00033779 +0.00064908 +0.00090899 +0.00099232 +0.00099987 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00020297 +0.00019166 +0.00026719 +0.00049468 +0.00077516 +0.00095301 +0.00099667 +0.00099996 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00027038 +0.00033779 +0.00049468 +0.00071445 +0.00090043 +0.00098470 +0.00099916 +0.00099999 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00057389 +0.00064908 +0.00077516 +0.00090043 +0.00097529 +0.00099727 +0.00099990 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00087649 +0.00090899 +0.00095301 +0.00098470 +0.00099727 +0.00099979 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00098862 +0.00099232 +0.00099667 +0.00099916 +0.00099990 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00099979 +0.00099987 +0.00099996 +0.00099999 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00014818 +0.00029893 +0.00058477 +0.00084695 +0.00097279 +0.00099804 +0.00099998 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00029893 +0.00044549 +0.00068687 +0.00089130 +0.00098185 +0.00099879 +0.00099999 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00058477 +0.00068687 +0.00083537 +0.00094878 +0.00099238 +0.00099957 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00084695 +0.00089130 +0.00094878 +0.00098630 +0.00099823 +0.00099992 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00097279 +0.00098185 +0.00099238 +0.00099823 +0.00099982 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00099804 +0.00099879 +0.00099957 +0.00099992 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00099998 +0.00099999 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00089030 +0.00093620 +0.00098330 +0.00099839 +0.00099995 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00093620 +0.00096330 +0.00099046 +0.00099909 +0.00099997 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00098330 +0.00099046 +0.00099741 +0.00099975 +0.00099999 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00099839 +0.00099909 +0.00099975 +0.00099997 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00099995 +0.00099997 +0.00099999 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 + +FIELD FieldData 0 diff --git a/python/Test-scripts/standard-results/SGH/Sedov_12x12x12/vtk/outputs.vtk.series b/python/Test-scripts/standard-results/SGH/Sedov_12x12x12/vtk/outputs.vtk.series new file mode 100644 index 000000000..e75ac9e43 --- /dev/null +++ b/python/Test-scripts/standard-results/SGH/Sedov_12x12x12/vtk/outputs.vtk.series @@ -0,0 +1,6 @@ +{ + "file-series-version" : "1.0", + "files" : [ + { "name" : "data/VTK0.vtk", "time" : 1.00000e+00 }, + ] +} diff --git a/python/Test-scripts/test_fierro.py b/python/Test-scripts/test_fierro.py new file mode 100644 index 000000000..db50cc0f2 --- /dev/null +++ b/python/Test-scripts/test_fierro.py @@ -0,0 +1,128 @@ +#!/usr/bin/python3 + +import os +import os.path +import sys +import math + +ground_truth_path = "" + + +# print("you passed in", len(sys.argv), "argument(s)") +# print("those arguments are: ", str(sys.argv)) + +# print("hello", sys.argv[0]) + +# os.system('pwd') + +executable = "./../../build-fierro-openmp/bin/fierro-parallel-explicit" +sim_input = "Solver-Inputs/SGH_Sedov_12x12x12.yaml" + +# Run simulation +os.system(executable + ' ' + sim_input) + + +# Functions for reading results from vtk file + +def extract_vector_data(filename, keyword): + data = [] + found_keyword = False + + with open(filename, 'r') as file: + for line in file: + if found_keyword: + if line.strip(): # If the line is not blank + # Split each line into fields. + fields = line.split() + xx = float(fields[0]) + yy = float(fields[1]) + zz = float(fields[2]) + # Group and append to results. + vec = [xx, yy, zz] + data.append(vec) + else: + break # Exit the loop when encountering a blank line + elif keyword in line: + found_keyword = True + + return data + +def extract_scalar_data(filename, keyword): + data = [] + found_keyword = False + + with open(filename, 'r') as file: + for line in file: + if found_keyword: + if line.strip(): # If the line is not blank + fields = line.split() + if fields[0] != "LOOKUP_TABLE": + data.append(float(fields[0])) + else: + break # Exit the loop when encountering a blank line + elif keyword in line: + found_keyword = True + # file.next() # skip first line "LOOKUP_TABLE" + + return data + + + +def percent_difference_scalars(array1, array2): + if len(array1) != len(array2): + raise ValueError("Arrays must have the same length") + + percent_diff = [] + for i in range(len(array1)): + diff = array2[i] - array1[i] + percent_diff.append((diff / array1[i]) * 100 if array1[i] != 0 else 0) + + return percent_diff + +def percent_difference_vectors(array1, array2): + if len(array1) != len(array2): + raise ValueError("Arrays must have the same length") + + percent_diff = [] + for i in range(len(array1)): + if len(array1[i]) != 3 or len(array2[i]) != 3: + raise ValueError("Subarrays must have length 3") + + diff = [array2[i][j] - array1[i][j] for j in range(3)] + percent_diff.append([(diff[j] / array1[i][j]) * 100 if array1[i][j] != 0 else 0 for j in range(3)]) + + return percent_diff + +def magnitude(array): + mag = math.sqrt(sum(x**2 for x in array)) + return mag + + +# Read ground truth results for sedov +GT_filename = "standard-results/SGH/Sedov_12x12x12/vtk/data/VTK0.vtk" + +velocity_keyword = "VECTORS velocity float" +position_keyword = "POINTS 2197 float" +SIE_keyword = "SCALARS SIE float 1" +density_keyword = "SCALARS element_density float 1" + +GT_positions = extract_vector_data(GT_filename, position_keyword) +GT_velocities = extract_vector_data(GT_filename, velocity_keyword) +GT_SIE = extract_scalar_data(GT_filename, SIE_keyword) +GT_densities = extract_scalar_data(GT_filename, density_keyword) + + +# Read simulation results +results_filename = "vtk/data/VTK0.vtk" + +results_positions = extract_vector_data(results_filename, position_keyword) +results_velocities = extract_vector_data(results_filename, velocity_keyword) +results_SIE = extract_scalar_data(results_filename, SIE_keyword) +results_densities = extract_scalar_data(results_filename, density_keyword) + + +density_diff = percent_difference_scalars(GT_densities, results_densities) + +print("Density difference: ") +print(magnitude(density_diff)) + From cbddb94dfda11422dac308aa7107642f9534f94d Mon Sep 17 00:00:00 2001 From: Jacob Moore Date: Wed, 21 Feb 2024 14:29:40 -0600 Subject: [PATCH 21/64] STYLE: Tidying up python test script --- python/Test-scripts/test_fierro.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python/Test-scripts/test_fierro.py b/python/Test-scripts/test_fierro.py index db50cc0f2..6baab8cb2 100644 --- a/python/Test-scripts/test_fierro.py +++ b/python/Test-scripts/test_fierro.py @@ -5,7 +5,6 @@ import sys import math -ground_truth_path = "" # print("you passed in", len(sys.argv), "argument(s)") From 19769490e6f493cf13f8bc43230199d193fc9b01 Mon Sep 17 00:00:00 2001 From: Jacob Moore Date: Wed, 21 Feb 2024 16:13:53 -0600 Subject: [PATCH 22/64] COMP: Fix compiler issue on debug check --- .../Parallel-Explicit/Explicit_Solver.cpp | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp b/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp index ab9c05781..1d603ae5e 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp @@ -1579,22 +1579,22 @@ void Explicit_Solver::setup_optimization_problem() // problem->check(true,std::cout); // debug checks - ROL::Ptr> rol_x = - ROL::makePtr>(design_node_densities_distributed); - // construct direction vector for check - Teuchos::RCP directions_distributed = Teuchos::rcp(new MV(map, 1)); - directions_distributed->putScalar(-0.1); - directions_distributed->randomize(-0.8, 1); - Kokkos::View direction_norm("gradient norm", 1); - directions_distributed->norm2(direction_norm); - directions_distributed->scale(1 / direction_norm(0)); - // set all but first component to 0 for debug - host_vec_array directions = directions_distributed->getLocalView(Tpetra::Access::ReadWrite); - // for(int init = 1; init < nlocal_nodes; init++) - // directions(4,0) = -0.3; - ROL::Ptr> rol_d = - ROL::makePtr>(directions_distributed); - obj->checkGradient(*rol_x, *rol_d); + // ROL::Ptr> rol_x = + // ROL::makePtr>(design_node_densities_distributed); + // // construct direction vector for check + // Teuchos::RCP directions_distributed = Teuchos::rcp(new MV(map, 1)); + // directions_distributed->putScalar(-0.1); + // directions_distributed->randomize(-0.8, 1); + // Kokkos::View direction_norm("gradient norm", 1); + // directions_distributed->norm2(direction_norm); + // directions_distributed->scale(1 / direction_norm(0)); + // // set all but first component to 0 for debug + // host_vec_array directions = directions_distributed->getLocalView(Tpetra::Access::ReadWrite); + // // for(int init = 1; init < nlocal_nodes; init++) + // // directions(4,0) = -0.3; + // ROL::Ptr> rol_d = + // ROL::makePtr>(directions_distributed); + // obj->checkGradient(*rol_x, *rol_d); // obj->checkHessVec(*rol_x, *rol_d); // directions_distributed->putScalar(-0.000001); // obj->checkGradient(*rol_x, *rol_d); From 93b2892ec457fe76c8f40df87edece1e5bbe5485 Mon Sep 17 00:00:00 2001 From: Jacob Moore Date: Wed, 21 Feb 2024 16:46:49 -0600 Subject: [PATCH 23/64] STYLE: Spot checking for residual strange formatting --- .../Parallel-Explicit/Explicit_Solver.cpp | 80 ++++++++++--------- .../SGH_Solver/src/power_gradients_sgh.cpp | 14 ++-- 2 files changed, 49 insertions(+), 45 deletions(-) diff --git a/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp b/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp index 1d603ae5e..4c7d83fef 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp @@ -1338,30 +1338,28 @@ void Explicit_Solver::setup_optimization_problem() // ROL::Ptr> emul = ROL::makePtr>(); // problem.addConstraint("Equality Constraint",econ,emul); - //debug checks - ROL::Ptr> rol_x = - ROL::makePtr>(design_node_densities_distributed); - //construct direction vector for check + // debug checks + ROL::Ptr> rol_x = + ROL::makePtr>(design_node_densities_distributed); + // construct direction vector for check Teuchos::RCP directions_distributed = Teuchos::rcp(new MV(map, 1)); directions_distributed->putScalar(-0.1); - directions_distributed->randomize(-0.8,1); - Kokkos::View direction_norm("gradient norm",1); + directions_distributed->randomize(-0.8, 1); + Kokkos::View direction_norm("gradient norm", 1); directions_distributed->norm2(direction_norm); - directions_distributed->scale(1/direction_norm(0)); - //set all but first component to 0 for debug - host_vec_array directions = directions_distributed->getLocalView (Tpetra::Access::ReadWrite); - //for(int init = 1; init < nlocal_nodes; init++) - //directions(4,0) = -0.3; - ROL::Ptr> rol_d = - ROL::makePtr>(directions_distributed); + directions_distributed->scale(1 / direction_norm(0)); + // set all but first component to 0 for debug + host_vec_array directions = directions_distributed->getLocalView(Tpetra::Access::ReadWrite); + // for(int init = 1; init < nlocal_nodes; init++) + // directions(4,0) = -0.3; + ROL::Ptr> rol_d = + ROL::makePtr>(directions_distributed); obj->checkGradient(*rol_x, *rol_d); - //obj->checkHessVec(*rol_x, *rol_d); - //directions_distributed->putScalar(-0.000001); - //obj->checkGradient(*rol_x, *rol_d); - //directions_distributed->putScalar(-0.0000001); - //obj->checkGradient(*rol_x, *rol_d); - - + // obj->checkHessVec(*rol_x, *rol_d); + // directions_distributed->putScalar(-0.000001); + // obj->checkGradient(*rol_x, *rol_d); + // directions_distributed->putScalar(-0.0000001); + // obj->checkGradient(*rol_x, *rol_d); // ROL::Ptr> ineq_constraint = ROL::makePtr(fea_elasticity, nodal_density_flag); // problem->addConstraint("Inequality Constraint",ineq_constraint,constraint_mul,constraint_bnd); @@ -2208,12 +2206,14 @@ void Explicit_Solver::parallel_tecplot_writer() // output nodal data // compute buffer output size and file stream offset for this MPI rank - int default_vector_count = 2; - int buffer_size_per_node_line = 26 * default_vector_count * num_dim + 1; // 25 width per number + 1 space times 6 entries plus line terminator - int nlocal_sorted_nodes = sorted_map->getLocalNumElements(); - GO first_node_global_id = sorted_map->getGlobalElement(0); + int default_vector_count = 2; + int buffer_size_per_node_line = 26 * default_vector_count * num_dim + 1; // 25 width per number + 1 space times 6 entries plus line terminator + int nlocal_sorted_nodes = sorted_map->getLocalNumElements(); + GO first_node_global_id = sorted_map->getGlobalElement(0); + CArrayKokkos print_buffer(buffer_size_per_node_line * nlocal_sorted_nodes); - MPI_Offset file_stream_offset = buffer_size_per_node_line * first_node_global_id; + + MPI_Offset file_stream_offset = buffer_size_per_node_line * first_node_global_id; // populate buffer long long current_buffer_position = 0; @@ -2845,21 +2845,25 @@ void Explicit_Solver::parallel_vtk_writer() void Explicit_Solver::tecplot_writer() { - int num_dim = simparam.num_dims; - std::string current_file_name; - std::string base_file_name = "TecplotTO"; - std::string base_file_name_undeformed = "TecplotTO_undeformed"; - std::stringstream ss; - std::string file_extension = ".dat"; - std::string file_count; - std::stringstream count_temp; - int time_step = 0; - int temp_convert; - int noutput, nvector; - bool displace_geometry = false; + int num_dim = simparam.num_dims; + int time_step = 0; + int temp_convert; + int noutput, nvector; + bool displace_geometry = false; + int displacement_index; + + std::string current_file_name; + std::string base_file_name = "TecplotTO"; + std::string base_file_name_undeformed = "TecplotTO_undeformed"; + std::string file_extension = ".dat"; + std::string file_count; + std::stringstream count_temp; + std::stringstream ss; + const_host_vec_array current_collected_output; - int displacement_index; + collect_information(); + // set host views of the collected data to print out from const_host_vec_array collected_node_coords = collected_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); const_host_vec_array collected_node_velocities = collected_node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/power_gradients_sgh.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/power_gradients_sgh.cpp index ad7e648d5..ed2c3166d 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/power_gradients_sgh.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/power_gradients_sgh.cpp @@ -343,8 +343,8 @@ void FEA_Module_SGH::get_power_ugradient_sgh(double rk { continue; } - Power_Gradient_Positions(gradient_node_id * num_dims + jdim, column_id) -= corner_gradient_storage(corner_gid, dim, igradient, jdim) * node_vel(rk_level, node_gid, - dim) * node_radius; + Power_Gradient_Positions(gradient_node_id * num_dims + jdim, column_id) -= + corner_gradient_storage(corner_gid, dim, igradient, jdim) * node_vel(rk_level, node_gid, dim) * node_radius; } } } // end for dim @@ -420,14 +420,14 @@ void FEA_Module_SGH::get_power_vgradient_sgh(double rk } if (node_lid == igradient) { - Power_Gradient_Velocities(gradient_node_id * num_dims + dim, column_id) -= corner_gradient_storage(corner_gid, dim, igradient, dim) * node_vel(rk_level, node_gid, - dim) * node_radius + - corner_force(corner_gid, dim) * node_radius; + Power_Gradient_Velocities(gradient_node_id * num_dims + dim, column_id) -= + corner_gradient_storage(corner_gid, dim, igradient, dim) * node_vel(rk_level, node_gid, dim) + * node_radius + corner_force(corner_gid, dim) * node_radius; } else { - Power_Gradient_Velocities(gradient_node_id * num_dims + dim, column_id) -= corner_gradient_storage(corner_gid, dim, igradient, dim) * node_vel(rk_level, node_gid, - dim) * node_radius; + Power_Gradient_Velocities(gradient_node_id * num_dims + dim, column_id) -= + corner_gradient_storage(corner_gid, dim, igradient, dim) * node_vel(rk_level, node_gid, dim) * node_radius; } } } // end for dim From 7fd87561e5a00491805283516d23b62a9a8679a1 Mon Sep 17 00:00:00 2001 From: Evan Lieberman - 270561 Date: Thu, 22 Feb 2024 09:56:34 -0700 Subject: [PATCH 24/64] ENH: Added partitioned VTU output option and writer --- .../SGH_Solver/FEA_Module_SGH.cpp | 20 +- .../Parallel-Explicit/outputs.cpp | 576 +++++++++++++++++- .../Simulation_Parameters/Output_Options.h | 2 +- 3 files changed, 585 insertions(+), 13 deletions(-) diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/FEA_Module_SGH.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/FEA_Module_SGH.cpp index 648cfb3d1..21fee4585 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/FEA_Module_SGH.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/FEA_Module_SGH.cpp @@ -1419,7 +1419,7 @@ void FEA_Module_SGH::setup(){ //initialize if topology optimization is used if(simparam->topology_optimization_on || simparam->shape_optimization_on){ init_assembly(); - //assemble_matrix(); + assemble_matrix(); } // update host copies of arrays modified in this function @@ -1787,7 +1787,7 @@ void FEA_Module_SGH::sgh_solve(){ nTO_modules = simparam->TO_Module_List.size(); int myrank = Explicit_Solver_Pointer_->myrank; - if(simparam->output_options.output_file_format==OUTPUT_FORMAT::vtk&&simparam->output_options.write_initial) + if(simparam->output_options.write_initial) { if(myrank==0) printf("Writing outputs to file at %f \n", time_value); @@ -2507,17 +2507,15 @@ void FEA_Module_SGH::sgh_solve(){ } }); // end parallel for } //end view scope - if(simparam->output_options.output_file_format==OUTPUT_FORMAT::vtk){ - if(myrank==0){ - printf("Writing outputs to file at %f \n", graphics_time); - } + if(myrank==0){ + printf("Writing outputs to file at %f \n", graphics_time); + } - double comm_time1 = Explicit_Solver_Pointer_->CPU_Time(); - Explicit_Solver_Pointer_->write_outputs(); + double comm_time1 = Explicit_Solver_Pointer_->CPU_Time(); + Explicit_Solver_Pointer_->write_outputs(); - double comm_time2 = Explicit_Solver_Pointer_->CPU_Time(); - Explicit_Solver_Pointer_->output_time += comm_time2 - comm_time1; - } + double comm_time2 = Explicit_Solver_Pointer_->CPU_Time(); + Explicit_Solver_Pointer_->output_time += comm_time2 - comm_time1; graphics_time = time_value + graphics_dt_ival; } // end if diff --git a/src/Parallel-Solvers/Parallel-Explicit/outputs.cpp b/src/Parallel-Solvers/Parallel-Explicit/outputs.cpp index 2c2cf1702..4952dbf38 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/outputs.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/outputs.cpp @@ -159,7 +159,581 @@ void Explicit_Solver::parallel_vtu_writer_new() { /* to be added... */ - throw std::runtime_error("parallel_vtu_writer_new() not yet implemented. use parallel_vtk_writer_new()"); + //throw std::runtime_error("parallel_vtu_writer_new() not yet implemented. use parallel_vtk_writer_new()"); + + std::string tmp_str; + std::stringstream mixed_str; + + char name[128]; + char pfilename[128]; + char filename[128]; + char pvtu_dir[128]; + char dirname[128]; + char subdirname[128]; + char tmp[128]; + + int num_dims = simparam.num_dims; + int graphics_idx = simparam.output_options.graphics_id; + int num_nodes_in_elem = 8; + int num_points = nnonoverlap_elem_nodes; + int num_cells = nlocal_elem_non_overlapping; + host_vec_array node_coords = all_node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); + + CArray nodes_in_elem (rnum_elem, max_nodes_per_element); + { + auto host_view = global_nodes_in_elem_distributed->getLocalView (Tpetra::Access::ReadOnly); + for (size_t ielem = 0; ielem < nodes_in_elem.dims(0); ielem++) { + for (size_t inode = 0; inode < nodes_in_elem.dims(1); inode++) { + nodes_in_elem(ielem, inode) = nonoverlap_element_node_map->getLocalElement(host_view(ielem, inode)); + } + } + } + + sprintf(name,"FierroOut"); + sprintf(pvtu_dir, "vtu"); + sprintf(subdirname, "%s_%05lu", name, graphics_idx); + + // mkdir if needed + struct stat st; + if (myrank == 0) { + if (stat(pvtu_dir, &st) != 0) { + // str_stream.str(""); + // str_stream << "mkdir" << " " << pvtu_dir; + // system(str_stream.str().c_str()); + //tmp = ""; + sprintf(tmp, "mkdir %s",pvtu_dir); + system(tmp); + } + sprintf(dirname,"%s/%s",pvtu_dir,subdirname); + if (stat(dirname, &st) != 0) { + // str_stream.str(""); + // str_stream << "mkdir" << " " << vtu_dir; + // system(str_stream.str().c_str()); + //tmp = ""; + sprintf(tmp, "mkdir %s", dirname); + system(tmp); + } + } + MPI_Barrier(world); + + // --------------------------------------------------------------------------- + // Write the PVTU file (only done by a single rank) + // --------------------------------------------------------------------------- + unsigned long int byte_offset = 0; + int block_header_size = sizeof(unsigned long int); + unsigned long int data_block_size; + + if (myrank == 0){ + sprintf(pfilename, "%s/%s_%05lu.pvtu",pvtu_dir, name, graphics_idx); + std::ofstream pout; // FILE *out; + pout.open(pfilename,std::ofstream::binary); + + // Write Header + tmp_str = "\n"; + pout.write(tmp_str.c_str(), tmp_str.length()); + tmp_str = "\n"; //unsure of exact correct usage of ghostlevel, leaving as 1 for now + pout.write(tmp_str.c_str(), tmp_str.length()); + // Write Field Data (only part in pvtu using byte_offset) + tmp_str = "\n"; + pout.write(tmp_str.c_str(), tmp_str.length()); + mixed_str.str(""); + mixed_str << "\n"; + byte_offset += sizeof(double)+block_header_size; + tmp_str = mixed_str.str(); + pout.write(tmp_str.c_str(), tmp_str.length()); + tmp_str = "\n"; + pout.write(tmp_str.c_str(), tmp_str.length()); + // Write PPoints Section + tmp_str = "\n"; + pout.write(tmp_str.c_str(), tmp_str.length()); + mixed_str.str(""); + mixed_str << "\n"; + tmp_str = mixed_str.str(); + pout.write(tmp_str.c_str(), tmp_str.length()); + tmp_str = "\n"; + pout.write(tmp_str.c_str(), tmp_str.length()); + // Write PPoint Data List + tmp_str = "\n"; + pout.write(tmp_str.c_str(), tmp_str.length()); + // + //SCALARS float + for (auto it = point_data_scalars_double.begin(); it != point_data_scalars_double.end(); it++) { + mixed_str.str(""); + mixed_str << "first << "\"/>\n"; + tmp_str = mixed_str.str(); + pout.write(tmp_str.c_str(), tmp_str.length()); + } + //VECTORS float + for (auto it = point_data_vectors_double.begin(); it != point_data_vectors_double.end(); it++) { + mixed_str.str(""); + mixed_str << "first << "\" NumberOfComponents=\"" << num_dims << "\"/>\n"; + tmp_str = mixed_str.str(); + pout.write(tmp_str.c_str(), tmp_str.length()); + } + // // Velocity + // mixed_str.str(""); + // mixed_str << "\n"; + // tmp_str = mixed_str.str(); + // pout.write(tmp_str.c_str(), tmp_str.length()); + // + tmp_str = "\n"; + pout.write(tmp_str.c_str(), tmp_str.length()); + // Write PCell Data List + tmp_str = "\n"; + pout.write(tmp_str.c_str(), tmp_str.length()); + // + //SCALARS float + for (auto it = cell_data_scalars_double.begin(); it != cell_data_scalars_double.end(); it++) { + mixed_str.str(""); + mixed_str << "first << "\"/>\n"; + tmp_str = mixed_str.str(); + pout.write(tmp_str.c_str(), tmp_str.length()); + } + //SCALARS int + for (auto it = cell_data_scalars_int.begin(); it != cell_data_scalars_int.end(); it++) { + mixed_str.str(""); + mixed_str << "first << "\"/>\n"; + tmp_str = mixed_str.str(); + pout.write(tmp_str.c_str(), tmp_str.length()); + } + // NON-SCALARS float + for (auto it = cell_data_fields_double.begin(); it != cell_data_fields_double.end(); it++) { + auto data_name = it->first; + auto [data_ptr, data_num_comps] = it->second; // Structured binding C++17 + mixed_str.str(""); + mixed_str << "\n"; + tmp_str = mixed_str.str(); + pout.write(tmp_str.c_str(), tmp_str.length()); + } + // Rank + mixed_str.str(""); + mixed_str << "\n"; + tmp_str = mixed_str.str(); + pout.write(tmp_str.c_str(), tmp_str.length()); + // // Stress + // mixed_str.str(""); + // mixed_str << "\n"; + // tmp_str = mixed_str.str(); + // pout.write(tmp_str.c_str(), tmp_str.length()); + // // Density + // mixed_str.str(""); + // mixed_str << "\n"; + // tmp_str = mixed_str.str(); + // pout.write(tmp_str.c_str(), tmp_str.length()); + // // IPFColor + // mixed_str.str(""); + // mixed_str << "\n"; + // tmp_str = mixed_str.str(); + // pout.write(tmp_str.c_str(), tmp_str.length()); + // + tmp_str = "\n"; + pout.write(tmp_str.c_str(), tmp_str.length()); + // Write Partition Piece List + char piecename[128]; + for (int rank = 0; rank < nranks; rank++){ + sprintf(piecename, "%s/%s_%05lu_%05lu.vtu",subdirname, name, graphics_idx, rank); + mixed_str.str(""); + mixed_str << "\n"; + tmp_str = mixed_str.str(); + pout.write(tmp_str.c_str(), tmp_str.length()); + } + // + tmp_str = "\n"; + pout.write(tmp_str.c_str(), tmp_str.length()); + // Write Appended Data + tmp_str = "\n_"; + pout.write(tmp_str.c_str(), tmp_str.length()); + // Write Time Value + data_block_size = sizeof(double); + pout.write((char *) &data_block_size,block_header_size); + pout.write((char *) &time_value,sizeof(time_value)); + pout.put('\n'); + tmp_str = "\n"; + pout.write(tmp_str.c_str(), tmp_str.length()); + + tmp_str = "\n"; + pout.write(tmp_str.c_str(), tmp_str.length()); + + pout.close(); + } + + // --------------------------------------------------------------------------- + // Write the VTU file + // --------------------------------------------------------------------------- + sprintf(filename, "%s/%s/%s_%05lu_%05lu.vtu",pvtu_dir ,subdirname, name, graphics_idx, myrank); + // filename has the full string + + std::ofstream out; // FILE *out; + out.open(filename,std::ofstream::binary); + + byte_offset = 0; + double crap = 0.0; + + // Write Header + //tmp_str = "\n"; + //out.write(tmp_str.c_str(), tmp_str.length()); + tmp_str = "\n"; + out.write(tmp_str.c_str(), tmp_str.length()); + tmp_str = "\n"; + out.write(tmp_str.c_str(), tmp_str.length()); + + // Write Time Value Header + tmp_str = "\n"; + out.write(tmp_str.c_str(), tmp_str.length()); + mixed_str.str(""); + mixed_str << "\n"; + byte_offset += sizeof(double)+block_header_size; + tmp_str = mixed_str.str(); + out.write(tmp_str.c_str(), tmp_str.length()); + tmp_str = "\n"; + out.write(tmp_str.c_str(), tmp_str.length()); + + mixed_str.str(""); + mixed_str << "\n"; + tmp_str = mixed_str.str(); + out.write(tmp_str.c_str(), tmp_str.length()); + + // **Write Points Header** + tmp_str = "\n"; + out.write(tmp_str.c_str(), tmp_str.length()); + //tmp_str = "\n"; + mixed_str.str(""); + mixed_str << "\n"; + byte_offset += num_points*num_dims*sizeof(double)+block_header_size; + tmp_str = mixed_str.str(); + out.write(tmp_str.c_str(), tmp_str.length()); + tmp_str = "\n"; + out.write(tmp_str.c_str(), tmp_str.length()); + // tmp_str = "\n"; + // out.write(tmp_str.c_str(), tmp_str.length()); + tmp_str = "\n"; + out.write(tmp_str.c_str(), tmp_str.length()); + + // **Write Cells Header** + tmp_str = "\n"; + out.write(tmp_str.c_str(), tmp_str.length()); + // Connectivity + mixed_str.str(""); + mixed_str << "\n"; + byte_offset += num_cells*num_nodes_in_elem*sizeof(int)+block_header_size; + tmp_str = mixed_str.str(); + out.write(tmp_str.c_str(), tmp_str.length()); + // Offsets + mixed_str.str(""); + mixed_str << "\n"; + byte_offset += num_cells*sizeof(int)+block_header_size; + tmp_str = mixed_str.str(); + out.write(tmp_str.c_str(), tmp_str.length()); + // Types + mixed_str.str(""); + mixed_str << "\n"; + byte_offset += num_cells*sizeof(int)+block_header_size; + tmp_str = mixed_str.str(); + out.write(tmp_str.c_str(), tmp_str.length()); + tmp_str = "\n"; + out.write(tmp_str.c_str(), tmp_str.length()); + + // **Write Point Data Headers** + tmp_str = "\n"; + out.write(tmp_str.c_str(), tmp_str.length()); + // + //SCALARS float + for (auto it = point_data_scalars_double.begin(); it != point_data_scalars_double.end(); it++) { + mixed_str.str(""); + mixed_str << "first << "\" format=\"appended\" offset=\"" << byte_offset << "\">\n"; + byte_offset += num_points*sizeof(double)+block_header_size; + tmp_str = mixed_str.str(); + out.write(tmp_str.c_str(), tmp_str.length()); + tmp_str = "\n"; + out.write(tmp_str.c_str(), tmp_str.length()); + } + //VECTORS float + for (auto it = point_data_vectors_double.begin(); it != point_data_vectors_double.end(); it++) { + mixed_str.str(""); + mixed_str << "first << "\" NumberOfComponents=\"" << num_dims << "\" format=\"appended\" offset=\"" << byte_offset << "\">\n"; + byte_offset += num_points*num_dims*sizeof(double)+block_header_size; + tmp_str = mixed_str.str(); + out.write(tmp_str.c_str(), tmp_str.length()); + tmp_str = "\n"; + out.write(tmp_str.c_str(), tmp_str.length()); + } + // // Velocity + // mixed_str.str(""); + // mixed_str << "\n"; + // byte_offset += num_points*num_dims*sizeof(double)+block_header_size; + // tmp_str = mixed_str.str(); + // out.write(tmp_str.c_str(), tmp_str.length()); + // tmp_str = "\n"; + // out.write(tmp_str.c_str(), tmp_str.length()); + tmp_str = "\n"; + out.write(tmp_str.c_str(), tmp_str.length()); + + // **Write Cell Data Headers** + tmp_str = "\n"; + out.write(tmp_str.c_str(), tmp_str.length()); + // + //SCALARS float + for (auto it = cell_data_scalars_double.begin(); it != cell_data_scalars_double.end(); it++) { + mixed_str.str(""); + mixed_str << "first << "\" format=\"appended\" offset=\"" << byte_offset << "\">\n"; + byte_offset += num_cells*sizeof(double)+block_header_size; + tmp_str = mixed_str.str(); + out.write(tmp_str.c_str(), tmp_str.length()); + tmp_str = "\n"; + out.write(tmp_str.c_str(), tmp_str.length()); + } + //SCALARS int + for (auto it = cell_data_scalars_int.begin(); it != cell_data_scalars_int.end(); it++) { + mixed_str.str(""); + mixed_str << "first << "\" format=\"appended\" offset=\"" << byte_offset << "\">\n"; + byte_offset += num_cells*sizeof(int)+block_header_size; + tmp_str = mixed_str.str(); + out.write(tmp_str.c_str(), tmp_str.length()); + tmp_str = "\n"; + out.write(tmp_str.c_str(), tmp_str.length()); + } + // NON-SCALARS float + for (auto it = cell_data_fields_double.begin(); it != cell_data_fields_double.end(); it++) { + auto data_name = it->first; + auto [data_ptr, data_num_comps] = it->second; // Structured binding C++17 + mixed_str.str(""); + mixed_str << "\n"; + byte_offset += num_cells*data_num_comps*sizeof(double)+block_header_size; + tmp_str = mixed_str.str(); + out.write(tmp_str.c_str(), tmp_str.length()); + tmp_str = "\n"; + out.write(tmp_str.c_str(), tmp_str.length()); + } + // Rank + mixed_str.str(""); + mixed_str << "\n"; + byte_offset += num_cells*sizeof(int)+block_header_size; + tmp_str = mixed_str.str(); + out.write(tmp_str.c_str(), tmp_str.length()); + tmp_str = "\n"; + out.write(tmp_str.c_str(), tmp_str.length()); + // // Stress + // mixed_str.str(""); + // mixed_str << "\n"; + // byte_offset += num_cells*sizeof(double)+block_header_size; + // tmp_str = mixed_str.str(); + // out.write(tmp_str.c_str(), tmp_str.length()); + // tmp_str = "\n"; + // out.write(tmp_str.c_str(), tmp_str.length()); + // // Density + // mixed_str.str(""); + // mixed_str << "\n"; + // byte_offset += num_cells*sizeof(double)+block_header_size; + // tmp_str = mixed_str.str(); + // out.write(tmp_str.c_str(), tmp_str.length()); + // tmp_str = "\n"; + // out.write(tmp_str.c_str(), tmp_str.length()); + // // IPFColor + // mixed_str.str(""); + // mixed_str << "\n"; + // byte_offset += num_cells*num_dims*sizeof(unsigned char)+block_header_size; + // tmp_str = mixed_str.str(); + // out.write(tmp_str.c_str(), tmp_str.length()); + // tmp_str = "\n"; + // out.write(tmp_str.c_str(), tmp_str.length()); + // + tmp_str = "\n"; + out.write(tmp_str.c_str(), tmp_str.length()); + + // Write Mesh Close + tmp_str = "\n"; + out.write(tmp_str.c_str(), tmp_str.length()); + tmp_str = "\n"; + out.write(tmp_str.c_str(), tmp_str.length()); + + // Write Appended Data + tmp_str = "\n_"; + out.write(tmp_str.c_str(), tmp_str.length()); + + // Write Time Value + data_block_size = sizeof(double); + out.write((char *) &data_block_size,block_header_size); + out.write((char *) &time_value,sizeof(time_value)); + + // **Write Points Data** + //double coords[num_points*num_dims]; + data_block_size = num_points*num_dims*sizeof(double); + out.write((char *) &data_block_size,block_header_size); + double coord_tmp; + // for (int node_gid = 0; node_gid < num_points; node_gid++){ + // for (int dim = 0; dim < num_dims; dim++){ + // coord_tmp = node_coords(node_gid,dim); + // out.write((char *) &coord_tmp,sizeof(coord_tmp)); + // } + // } + for (int node_id = 0; node_id < num_points; node_id++){ + int node_gid = nonoverlap_element_node_map->getGlobalElement(node_id); + int node_lid = all_node_map->getLocalElement(node_gid); + for (int dim = 0; dim < num_dims; dim++){ + //printf("rank %d point %d coord %d = %g \n",myrank,node_lid,dim,node_coords(node_lid,dim)); + coord_tmp = node_coords(node_lid,dim); + out.write((char *) &coord_tmp,sizeof(coord_tmp)); + } + } + + // **Write Cells Data** + //int connect[num_cells*mesh.num_nodes_in_elem]; + data_block_size = num_cells*num_nodes_in_elem*sizeof(int); + out.write((char *) &data_block_size,block_header_size); + int connect_tmp; + for (int elem_gid = 0; elem_gid < num_cells; elem_gid++){ + for (int node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ + connect_tmp = nodes_in_elem(elem_gid, node_lid); + out.write((char *) &connect_tmp,sizeof(connect_tmp)); + } + } + //int offsets[num_cells]; + data_block_size = num_cells*sizeof(int); + out.write((char *) &data_block_size,block_header_size); + int offset_tmp; + for (int elem_gid = 0; elem_gid < num_cells; elem_gid++){ + offset_tmp = (elem_gid+1)*num_nodes_in_elem; + out.write((char *) &offset_tmp,sizeof(offset_tmp)); + } + //int types[num_cells]; + data_block_size = num_cells*sizeof(int); + out.write((char *) &data_block_size,block_header_size); + int type_tmp; + for (int elem_gid = 0; elem_gid < num_cells; elem_gid++){ + type_tmp = 12; + out.write((char *) &type_tmp,sizeof(type_tmp)); + } + + // **Write Point Data** + //SCALARS float + for (auto it = point_data_scalars_double.begin(); it != point_data_scalars_double.end(); it++) { + auto data_ptr = it->second; + ViewCArrayKokkos data_tmp(data_ptr,nall_nodes); + + data_block_size = num_points*sizeof(double); + out.write((char *) &data_block_size,block_header_size); + double pscalar_tmp; + for (int node_id = 0; node_id < num_points; node_id++){ + int node_gid = nonoverlap_element_node_map->getGlobalElement(node_id); + int node_lid = all_node_map->getLocalElement(node_gid); + //printf("rank %d point %d coord %d = %g \n",myrank,node_lid,dim,node_coords(node_lid,dim)); + pscalar_tmp = data_tmp(node_lid); + out.write((char *) &pscalar_tmp,sizeof(pscalar_tmp)); + } + } + //VECTORS float + for (auto it = point_data_vectors_double.begin(); it != point_data_vectors_double.end(); it++) { + auto data_ptr = it->second; + ViewCArrayKokkos data_tmp(data_ptr,nall_nodes,num_dims); + + data_block_size = num_points*num_dims*sizeof(double); + out.write((char *) &data_block_size,block_header_size); + double pvector_tmp; + for (int node_id = 0; node_id < num_points; node_id++){ + int node_gid = nonoverlap_element_node_map->getGlobalElement(node_id); + int node_lid = all_node_map->getLocalElement(node_gid); + for (int dim = 0; dim < num_dims; dim++){ + //printf("rank %d point %d coord %d = %g \n",myrank,node_lid,dim,node_coords(node_lid,dim)); + pvector_tmp = data_tmp(node_lid,dim); + out.write((char *) &pvector_tmp,sizeof(pvector_tmp)); + } + } + } + // // Velocity + // data_block_size = num_points*num_dims*sizeof(double); + // out.write((char *) &data_block_size,block_header_size); + // for (int node_gid = 0; node_gid < num_points; node_gid++){ + // for (int dim = 0; dim < num_dims; dim++){ + // //out.write((char *) &node_vel.host(1, node_gid, dim),sizeof(double)); + // out.write((char *) &crap,sizeof(double)); + // } + // } + + // **Write Cell Data** + //SCALARS float + for (auto it = cell_data_scalars_double.begin(); it != cell_data_scalars_double.end(); it++) { + auto data_ptr = it->second; + ViewCArrayKokkos data_tmp(data_ptr,rnum_elem); + + data_block_size = num_cells*sizeof(double); + out.write((char *) &data_block_size,block_header_size); + double cscalar_ftmp; + for (int elem_gid = 0; elem_gid < num_cells; elem_gid++){ + cscalar_ftmp = data_tmp(elem_gid); + out.write((char *) &cscalar_ftmp,sizeof(double)); + } + } + //SCALARS int + for (auto it = cell_data_scalars_int.begin(); it != cell_data_scalars_int.end(); it++) { + auto data_ptr = it->second; + ViewCArrayKokkos data_tmp(data_ptr,rnum_elem); + + data_block_size = num_cells*sizeof(int); + out.write((char *) &data_block_size,block_header_size); + double cscalar_itmp; + for (int elem_gid = 0; elem_gid < num_cells; elem_gid++){ + cscalar_itmp = data_tmp(elem_gid); + out.write((char *) &cscalar_itmp,sizeof(int)); + } + } + // NON-SCALARS float + for (auto it = cell_data_fields_double.begin(); it != cell_data_fields_double.end(); it++) { + auto data_name = it->first; + auto [data_ptr, data_num_comps] = it->second; // Structured binding C++17 + ViewCArrayKokkos data_tmp(data_ptr,rnum_elem,data_num_comps); + + data_block_size = num_cells*data_num_comps*sizeof(double); + out.write((char *) &data_block_size,block_header_size); + double cfield_ftmp; + for (int elem_gid = 0; elem_gid < num_cells; elem_gid++){ + for (int comp = 0; comp < data_num_comps; comp++){ + cfield_ftmp = data_tmp(elem_gid,comp); + out.write((char *) &cfield_ftmp,sizeof(double)); + } + } + } + // Rank + data_block_size = num_cells*sizeof(int); + out.write((char *) &data_block_size,block_header_size); + for (int elem_gid = 0; elem_gid < num_cells; elem_gid++){ + out.write((char *) &myrank,sizeof(int)); + } + // // Stress + // data_block_size = num_cells*sizeof(double); + // out.write((char *) &data_block_size,block_header_size); + // for (int elem_gid = 0; elem_gid < num_cells; elem_gid++){ + // for (int ii = 0; ii < 9; ii++){ + // //out.write((char *) &elem_stress.host(1,elem_gid,ii),sizeof(double)); + // out.write((char *) &crap,sizeof(double)); + // } + // } + // // Density + // data_block_size = num_cells*sizeof(double); + // out.write((char *) &data_block_size,block_header_size); + // for (int elem_gid = 0; elem_gid < num_cells; elem_gid++){ + // out.write((char *) &elem_den.host(elem_gid),sizeof(double)); + // } + // // IPFColor + // data_block_size = num_cells*num_dims*sizeof(unsigned char); + // out.write((char *) &data_block_size,block_header_size); + // for (int elem_gid = 0; elem_gid < num_cells; elem_gid++){ + // for (int dim = 0; dim < num_dims; dim++){ + // out.write((char *) &elem_IPF_colors.host(elem_gid,dim),sizeof(unsigned char)); + // } + // } + + out.put('\n'); + tmp_str = "\n"; + out.write(tmp_str.c_str(), tmp_str.length()); + + // Write File Close + tmp_str = "\n"; + out.write(tmp_str.c_str(), tmp_str.length()); + + out.close(); + + simparam.output_options.graphics_id++; + } std::string diff --git a/src/Parallel-Solvers/Simulation_Parameters/Output_Options.h b/src/Parallel-Solvers/Simulation_Parameters/Output_Options.h index bb444f18a..f07fdef98 100644 --- a/src/Parallel-Solvers/Simulation_Parameters/Output_Options.h +++ b/src/Parallel-Solvers/Simulation_Parameters/Output_Options.h @@ -3,7 +3,7 @@ #include "Fields.h" #include -SERIALIZABLE_ENUM(OUTPUT_FORMAT, vtk, vtu, none) +SERIALIZABLE_ENUM(OUTPUT_FORMAT, vtk, pvtu, none) SERIALIZABLE_ENUM(TIMER_VERBOSITY, standard, thorough) struct Output_Options : Yaml::DerivedFields, Yaml::ValidatedYaml { TIMER_VERBOSITY timer_output_level = TIMER_VERBOSITY::standard; From 18a22e5b167193f9024dcdef117fed9b0b4af9ef Mon Sep 17 00:00:00 2001 From: Evan Lieberman - 270561 Date: Thu, 22 Feb 2024 12:58:15 -0700 Subject: [PATCH 25/64] BUG: Minor fixes on changes made for PVTU writer --- .../Parallel-Explicit/SGH_Solver/FEA_Module_SGH.cpp | 2 +- src/Parallel-Solvers/Parallel-Explicit/outputs.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/FEA_Module_SGH.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/FEA_Module_SGH.cpp index 21fee4585..dea62e1a6 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/FEA_Module_SGH.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/FEA_Module_SGH.cpp @@ -1419,7 +1419,7 @@ void FEA_Module_SGH::setup(){ //initialize if topology optimization is used if(simparam->topology_optimization_on || simparam->shape_optimization_on){ init_assembly(); - assemble_matrix(); + //assemble_matrix(); } // update host copies of arrays modified in this function diff --git a/src/Parallel-Solvers/Parallel-Explicit/outputs.cpp b/src/Parallel-Solvers/Parallel-Explicit/outputs.cpp index 4952dbf38..94e8186ab 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/outputs.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/outputs.cpp @@ -146,7 +146,7 @@ Explicit_Solver::write_outputs() parallel_vtk_writer_new(); break; - case OUTPUT_FORMAT::vtu: + case OUTPUT_FORMAT::pvtu: parallel_vtu_writer_new(); break; From c650b8a1e2ad7c8c1d59ea0379744375f8b6038f Mon Sep 17 00:00:00 2001 From: Jacob Moore Date: Thu, 22 Feb 2024 19:01:32 -0600 Subject: [PATCH 26/64] STYLE: Modifying uncrustify config, and automating --- docs/formatting/HeaderForSources.txt | 34 ++ docs/formatting/uncrustify.cfg | 30 +- scripts/uncrustify.sh | 76 +++++ .../FEA_Module_Dynamic_Elasticity.cpp | 97 +++--- .../FEA_Module_Dynamic_Elasticity.h | 187 +++++----- .../Dynamic_Elastic_Solver/boundary.cpp | 41 ++- .../elastic_optimization.cpp | 88 +++-- .../Dynamic_Elastic_Solver/force_elastic.cpp | 96 ++++-- .../force_gradients_elastic.cpp | 120 ++++--- .../Dynamic_Elastic_Solver/geometry.cpp | 110 +++--- .../Dynamic_Elastic_Solver/mesh.h | 39 ++- .../Dynamic_Elastic_Solver/momentum.cpp | 128 ++++--- .../Dynamic_Elastic_Solver/properties.cpp | 94 +++-- .../Dynamic_Elastic_Solver/state.h | 33 ++ .../time_integration.cpp | 66 +++- .../Eulerian_Solver/FEA_Module_Eulerian.cpp | 61 ++-- .../Eulerian_Solver/FEA_Module_Eulerian.h | 25 +- .../Simulation_Parameters_Eulerian.h | 3 - .../Eulerian_Solver/eulerian_geometry.cpp | 20 +- .../eulerian_time_integration.cpp | 66 +++- .../SGH_Solver/include/FEA_Module_SGH.h | 274 ++++++++------- .../SGH_Solver/include/mesh.h | 41 ++- .../SGH_Solver/include/state.h | 33 ++ .../SGH_Solver/src/FEA_Module_SGH.cpp | 114 +++---- .../SGH_Solver/src/boundary.cpp | 49 ++- .../SGH_Solver/src/energy_sgh.cpp | 55 ++- .../SGH_Solver/src/force_gradients_sgh.cpp | 207 ++++++----- .../SGH_Solver/src/force_sgh.cpp | 119 ++++--- .../SGH_Solver/src/geometry.cpp | 136 +++++--- .../SGH_Solver/src/momentum.cpp | 116 ++++--- .../SGH_Solver/src/power_gradients_sgh.cpp | 112 +++--- .../SGH_Solver/src/properties.cpp | 94 +++-- .../SGH_Solver/src/setup_sgh.cpp | 53 ++- .../SGH_Solver/src/sgh_optimization.cpp | 88 +++-- .../SGH_Solver/src/time_integration.cpp | 66 +++- .../Topology_Optimization/Area_Normals.h | 322 +++++++++--------- .../Kinetic_Energy_Minimize.h | 16 +- .../Parallel-Explicit/main.cpp | 5 +- 38 files changed, 2109 insertions(+), 1205 deletions(-) create mode 100644 docs/formatting/HeaderForSources.txt mode change 100644 => 100755 docs/formatting/uncrustify.cfg create mode 100755 scripts/uncrustify.sh diff --git a/docs/formatting/HeaderForSources.txt b/docs/formatting/HeaderForSources.txt new file mode 100644 index 000000000..2cc9b2f00 --- /dev/null +++ b/docs/formatting/HeaderForSources.txt @@ -0,0 +1,34 @@ + /********************************************************************************************** + © 2020. Triad National Security, LLC. All rights reserved. + This program was produced under U.S. Government contract 89233218CNA000001 for Los Alamos + National Laboratory (LANL), which is operated by Triad National Security, LLC for the U.S. + Department of Energy/National Nuclear Security Administration. All rights in the program are + reserved by Triad National Security, LLC, and the U.S. Department of Energy/National Nuclear + Security Administration. The Government is granted for itself and others acting on its behalf a + nonexclusive, paid-up, irrevocable worldwide license in this material to reproduce, prepare + derivative works, distribute copies to the public, perform publicly and display publicly, and + to permit others to do so. + This program is open source under the BSD-3 License. + Redistribution and use in source and binary forms, with or without modification, are permitted + provided that the following conditions are met: + 1. Redistributions of source code must retain the above copyright notice, this list of + conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright notice, this list of + conditions and the following disclaimer in the documentation and/or other materials + provided with the distribution. + 3. Neither the name of the copyright holder nor the names of its contributors may be used + to endorse or promote products derived from this software without specific prior + written permission. + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS + IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **********************************************************************************************/ + \ No newline at end of file diff --git a/docs/formatting/uncrustify.cfg b/docs/formatting/uncrustify.cfg old mode 100644 new mode 100755 index 043d49f3f..7f086a6b0 --- a/docs/formatting/uncrustify.cfg +++ b/docs/formatting/uncrustify.cfg @@ -26,6 +26,12 @@ indent_func_call_param = true # false/true # Same as indent_func_call_param, but for function defs indent_func_def_param = false # false/true +# for function definitions, only if indent_func_def_param is false +# Allows to align params when appropriate and indent them when not +# behave as if it was true if paren position is more than this value +# if paren position is more than the option value +indent_func_def_param_paren_pos_threshold = 6 # unsigned number + # Same as indent_func_call_param, but for function protos indent_func_proto_param = false # false/true @@ -369,18 +375,36 @@ align_var_class_span = 1 # number align_var_class_thresh = 6 # number # Whether to align variable definitions in prototypes and functions. -align_func_params = true # true/false +align_func_params = false # true/false + +# The span for aligning parameter definitions in function on parameter name. +# +# 0: Don't align (default). +align_func_params_span = 1 # unsigned number + + +# The threshold for aligning function parameter definitions. +# Use a negative number for absolute thresholds. +# +# 0: No limit (default). +align_func_params_thresh = -5 # number # The span for aligning on '=' in assignments. # # 0 = Don't align (default). align_assign_span = 1 # unsigned number +# The span for aligning function prototypes (0=don't align) +align_func_proto_span = 0 # number + # The threshold for aligning on '=' in assignments. # Use a negative number for absolute thresholds. # # 0 = No limit (default). -align_assign_thresh = 6 # number +align_assign_thresh = 4 # number + +# The threshold for aligning variable definitions (0=no limit) +align_var_def_thresh = 4 # number # Whether to align lines that start with '<<' with previous '<<'. # @@ -440,7 +464,7 @@ cmt_indent_multi = false # false/true # The filename that contains text to insert at the head of a file if the file doesn't start with a C/C++ comment. # Will substitute $(filename) with the current file's name. -# cmt_insert_file_header = "../HeaderForSources.txt" +cmt_insert_file_header = "HeaderForSources.txt" # If a namespace body exceeds the specified number of newlines and doesn't have a comment after # the close brace, a comment will be added. diff --git a/scripts/uncrustify.sh b/scripts/uncrustify.sh new file mode 100755 index 000000000..becdde2c2 --- /dev/null +++ b/scripts/uncrustify.sh @@ -0,0 +1,76 @@ +#!/bin/bash -e + +# This script must be run from the top level Fierro directory +# This script assumes uncrustify has been installed + + +uncrustifyConfigDir="docs/formatting" +# Save the directory the user had before calling +startDir=$(pwd) +thisScriptDir=$(dirname "$0") +echo "Start Directory: $startDir" +echo "This Script Directory: $thisScriptDir" +echo "Uncrustify config directory: $uncrustifyConfigDir" + +# Get the full path to the config needed by uncrustify +cd $uncrustifyConfigDir +fullUncrustifyDir=$(pwd) +cd $startDir + +# Get the source directory +cd src +sourceDir=$(pwd) + +# Function to walk through files +treeProcess() { + + echo "calling tree in $(pwd)" + # For every file in the directory + for file in *.cpp; do + if [ -f "$file" ]; then + # echo "Using: $(pwd)/$f" + uncrustify -c "$fullUncrustifyDir"/uncrustify.cfg --no-backup "$(pwd)/$file" + fi + done + + for file in *.h; do + if [ -f "$file" ]; then + # echo "Using: $(pwd)/$f" + uncrustify -c "$fullUncrustifyDir"/uncrustify.cfg --no-backup "$(pwd)/$file" + fi + done + + # For every directory, recurse + for dir in */; do + if [ -d "$dir" ]; then + cd "$dir" || exit + treeProcess + cd .. + fi + + done +} + + +# Uncrustify SGH Solver +echo "Uncrusting: $sourceDir/Parallel-Solvers/Parallel-Explicit/SGH_Solver" +cd "$sourceDir/Parallel-Solvers/Parallel-Explicit/SGH_Solver" || exit +treeProcess + +# Uncrustify Topology Optimization +echo "Uncrusting: $sourceDir/Parallel-Solvers/Parallel-Explicit/Topology_Optimization" +cd "$sourceDir/Parallel-Solvers/Parallel-Explicit/Topology_Optimization" || exit +treeProcess + +# Uncrustify Eulerian Solver +echo "Uncrusting: $sourceDir/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver" +cd "$sourceDir/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver" || exit +treeProcess + +# Uncrustify Dynamic Elastic +echo "Uncrusting: $sourceDir/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver" +cd "$sourceDir/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver" || exit +treeProcess + +exit + diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.cpp b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.cpp index fdabd0bcf..fa370af16 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.cpp @@ -11,14 +11,11 @@ This program is open source under the BSD-3 License. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. @@ -91,7 +88,7 @@ FEA_Module_Dynamic_Elasticity::FEA_Module_Dynamic_Elasticity( // recast solver pointer for non-base class access Explicit_Solver_Pointer_ = dynamic_cast(Solver_Pointer); module_params = ¶ms; - simparam = &(Explicit_Solver_Pointer_->simparam); + simparam = &(Explicit_Solver_Pointer_->simparam); mesh = mesh_in; @@ -120,7 +117,7 @@ FEA_Module_Dynamic_Elasticity::FEA_Module_Dynamic_Elasticity( if (simparam->topology_optimization_on || simparam->shape_optimization_on || simparam->num_dims == 2) { - node_masses_distributed = Teuchos::rcp(new MV(map, 1)); + node_masses_distributed = Teuchos::rcp(new MV(map, 1)); ghost_node_masses_distributed = Teuchos::rcp(new MV(ghost_node_map, 1)); adjoint_vector_distributed = Teuchos::rcp(new MV(map, simparam->num_dims)); phi_adjoint_vector_distributed = Teuchos::rcp(new MV(map, simparam->num_dims)); @@ -148,7 +145,7 @@ FEA_Module_Dynamic_Elasticity::FEA_Module_Dynamic_Elasticity( graphics_dt_ival = simparam->output_options.graphics_dt_ival; graphics_cyc_ival = simparam->output_options.graphics_cyc_ival; graphics_times = simparam->output_options.graphics_times; - graphics_id = simparam->output_options.graphics_id; + graphics_id = simparam->output_options.graphics_id; cycle_stop = dynamic_options.cycle_stop; rk_num_stages = dynamic_options.rk_num_stages; @@ -188,11 +185,11 @@ void FEA_Module_Dynamic_Elasticity::read_conditions_ansys_dat(std::ifstream* in, { Input_Options input_options = simparam->input_options.value(); - char ch; - std::string skip_line, read_line, substring, token; + char ch; + std::string skip_line, read_line, substring, token; std::stringstream line_parse, line_parse2; - int num_dim = simparam->num_dims; + int num_dim = simparam->num_dims; int buffer_lines = 1000; int max_word = 30; int local_node_index, current_column_index; @@ -206,21 +203,21 @@ void FEA_Module_Dynamic_Elasticity::read_conditions_ansys_dat(std::ifstream* in, real_t unit_scaling = input_options.unit_scaling; - CArrayKokkos read_buffer; + CArrayKokkos read_buffer; CArrayKokkos read_buffer_indices; - LO local_dof_id; - GO node_gid; - real_t dof_value; + LO local_dof_id; + GO node_gid; + real_t dof_value; host_vec_array node_densities; } // end read_conditions_ansys_dat // ----------------------------------------------------------------------------- // Interfaces read in data with the elastic solver data; currently a hack to streamline // ------------------------------------------------------------------------------ -void FEA_Module_Dynamic_Elasticity::elastic_interface_setup(node_t& node, - elem_t& elem, - corner_t& corner) +void FEA_Module_Dynamic_Elasticity::elastic_interface_setup(node_t& node, + elem_t& elem, + corner_t& corner) { const size_t num_dim = simparam->num_dims; const size_t rk_num_bins = simparam->dynamic_options.rk_num_bins; @@ -503,9 +500,9 @@ void FEA_Module_Dynamic_Elasticity::grow_boundary_sets(int num_sets) if (num_sets > max_boundary_sets) { // temporary storage for previous data - CArrayKokkos Temp_Boundary_Condition_Type_List = Boundary_Condition_Type_List; - CArrayKokkos Temp_NBoundary_Condition_Patches = NBoundary_Condition_Patches; - CArrayKokkos Temp_Boundary_Condition_Patches = Boundary_Condition_Patches; + CArrayKokkos Temp_Boundary_Condition_Type_List = Boundary_Condition_Type_List; + CArrayKokkos Temp_NBoundary_Condition_Patches = NBoundary_Condition_Patches; + CArrayKokkos Temp_Boundary_Condition_Patches = Boundary_Condition_Patches; max_boundary_sets = num_sets + 5; // 5 is an arbitrary buffer Boundary_Condition_Type_List = CArrayKokkos(max_boundary_sets, "Boundary_Condition_Type_List"); @@ -675,11 +672,11 @@ void FEA_Module_Dynamic_Elasticity::sort_output(Teuchos::RCP& point_data_scalars_double, - std::map& point_data_vectors_double, - std::map& cell_data_scalars_double, - std::map& cell_data_scalars_int, - std::map>& cell_data_fields_double) +void FEA_Module_Dynamic_Elasticity::write_data(std::map& point_data_scalars_double, + std::map& point_data_vectors_double, + std::map& cell_data_scalars_double, + std::map& cell_data_scalars_int, + std::map>& cell_data_fields_double) { const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; @@ -1001,7 +998,7 @@ void FEA_Module_Dynamic_Elasticity::setup() elem_vel_grad = DCArrayKokkos(num_elems, 3, 3); // allocate material models - elem_eos = DCArrayKokkos(num_elems); + elem_eos = DCArrayKokkos(num_elems); elem_strength = DCArrayKokkos(num_elems); // --------------------------------------------------------------------- @@ -1363,7 +1360,7 @@ void FEA_Module_Dynamic_Elasticity::setup() // loop over the corners of the element and calculate the mass for (size_t corner_lid = 0; corner_lid < 4; corner_lid++) { - size_t corner_gid = corners_in_elem(elem_gid, corner_lid); + size_t corner_gid = corners_in_elem(elem_gid, corner_lid); corner_mass(corner_gid) = corner_areas(corner_lid) * elem_den(elem_gid); // node radius is added later } // end for over corners }); @@ -1483,13 +1480,13 @@ void FEA_Module_Dynamic_Elasticity::cleanup_material_models() * Modifies: bdy_patches_in_set */ void FEA_Module_Dynamic_Elasticity::tag_bdys(const DCArrayKokkos& boundary, - mesh_t& mesh, - const DViewCArrayKokkos& node_coords) + mesh_t& mesh, + const DViewCArrayKokkos& node_coords) { const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - size_t num_dim = simparam->num_dims; - int nboundary_patches = Explicit_Solver_Pointer_->nboundary_patches; - int num_nodes_in_patch = mesh.num_nodes_in_patch; + size_t num_dim = simparam->num_dims; + int nboundary_patches = Explicit_Solver_Pointer_->nboundary_patches; + int num_nodes_in_patch = mesh.num_nodes_in_patch; // if (bdy_set == mesh.num_bdy_sets){ // printf(" ERROR: number of boundary sets must be increased by %zu", @@ -1553,13 +1550,13 @@ void FEA_Module_Dynamic_Elasticity::tag_bdys(const DCArrayKokkos& bo } // end tag KOKKOS_INLINE_FUNCTION -bool FEA_Module_Dynamic_Elasticity::check_bdy(const size_t patch_gid, - const int num_dim, - const int num_nodes_in_patch, - const BOUNDARY_TYPE bc_type, - const double val, - const DViewCArrayKokkos& node_coords, - const size_t rk_level) const +bool FEA_Module_Dynamic_Elasticity::check_bdy(const size_t patch_gid, + const int num_dim, + const int num_nodes_in_patch, + const BOUNDARY_TYPE bc_type, + const double val, + const DViewCArrayKokkos& node_coords, + const size_t rk_level) const { // default bool is not on the boundary size_t is_on_bdy = 0; @@ -1780,7 +1777,7 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve() KineticEnergyMinimize_TopOpt& kinetic_energy_minimize_function = dynamic_cast(*obj_pointer); kinetic_energy_minimize_function.objective_accumulation = 0; global_objective_accumulation = objective_accumulation = 0; - kinetic_energy_objective = true; + kinetic_energy_objective = true; if (max_time_steps + 1 > forward_solve_velocity_data->size()) { old_max_forward_buffer = forward_solve_velocity_data->size(); @@ -1937,12 +1934,12 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve() // view scope { - const_vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); const_vec_array ghost_node_velocities_interface = Explicit_Solver_Pointer_->ghost_node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); - vec_array all_node_velocities_interface = Explicit_Solver_Pointer_->all_node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); + vec_array all_node_velocities_interface = Explicit_Solver_Pointer_->all_node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); const_vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); const_vec_array ghost_node_coords_interface = Explicit_Solver_Pointer_->ghost_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); - vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { for (int idim = 0; idim < num_dim; idim++) { @@ -2142,7 +2139,7 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve() Kokkos::fence(); double comm_time4 = Explicit_Solver_Pointer_->CPU_Time(); - Explicit_Solver_Pointer_->host2dev_time += comm_time4 - comm_time3; + Explicit_Solver_Pointer_->host2dev_time += comm_time4 - comm_time3; Explicit_Solver_Pointer_->communication_time += comm_time4 - comm_time1; // debug print vector values on a rank /* @@ -2376,14 +2373,14 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve() // view scope { - const_vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); const_vec_array ghost_node_velocities_interface = Explicit_Solver_Pointer_->ghost_node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); - const_vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); const_vec_array ghost_node_coords_interface = Explicit_Solver_Pointer_->ghost_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); - - vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); - vec_array all_node_velocities_interface = Explicit_Solver_Pointer_->all_node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); - + + vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + vec_array all_node_velocities_interface = Explicit_Solver_Pointer_->all_node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { for (int idim = 0; idim < num_dim; idim++) { @@ -2404,7 +2401,7 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve() } // end view scope double comm_time4 = Explicit_Solver_Pointer_->CPU_Time(); - Explicit_Solver_Pointer_->host2dev_time += comm_time4 - comm_time3; + Explicit_Solver_Pointer_->host2dev_time += comm_time4 - comm_time3; Explicit_Solver_Pointer_->communication_time += comm_time4 - comm_time1; (*forward_solve_velocity_data)[cycle + 1]->assign(*Explicit_Solver_Pointer_->all_node_velocities_distributed); diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.h b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.h index 6c10c8504..068db5540 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.h +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.h @@ -11,14 +11,11 @@ This program is open source under the BSD-3 License. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. @@ -75,7 +72,7 @@ class FEA_Module_Dynamic_Elasticity : public FEA_Module void elastic_solve(); void get_force_vgradient_elastic(const DCArrayKokkos& material, - const mesh_t& mesh, + const mesh_t& mesh, const DViewCArrayKokkos& node_coords, const DViewCArrayKokkos& node_vel, const DViewCArrayKokkos& elem_den, @@ -86,11 +83,11 @@ class FEA_Module_Dynamic_Elasticity : public FEA_Module const DViewCArrayKokkos& elem_vol, const DViewCArrayKokkos& elem_div, const DViewCArrayKokkos& elem_mat_id, - const double rk_alpha, - const size_t cycle); + const double rk_alpha, + const size_t cycle); void get_force_ugradient_elastic(const DCArrayKokkos& material, - const mesh_t& mesh, + const mesh_t& mesh, const DViewCArrayKokkos& node_coords, const DViewCArrayKokkos& node_vel, const DViewCArrayKokkos& elem_den, @@ -101,11 +98,11 @@ class FEA_Module_Dynamic_Elasticity : public FEA_Module const DViewCArrayKokkos& elem_vol, const DViewCArrayKokkos& elem_div, const DViewCArrayKokkos& elem_mat_id, - const double rk_alpha, - const size_t cycle); + const double rk_alpha, + const size_t cycle); void get_force_dgradient_elastic(const DCArrayKokkos& material, - const mesh_t& mesh, + const mesh_t& mesh, const DViewCArrayKokkos& node_coords, const DViewCArrayKokkos& node_vel, const DViewCArrayKokkos& elem_den, @@ -116,14 +113,14 @@ class FEA_Module_Dynamic_Elasticity : public FEA_Module const DViewCArrayKokkos& elem_vol, const DViewCArrayKokkos& elem_div, const DViewCArrayKokkos& elem_mat_id, - const double rk_alpha, - const size_t cycle); + const double rk_alpha, + const size_t cycle); void force_design_gradient_term(const_vec_array design_variables, vec_array design_gradients); - void update_position_elastic(double rk_alpha, - const size_t num_nodes, - DViewCArrayKokkos& node_coords, + void update_position_elastic(double rk_alpha, + const size_t num_nodes, + DViewCArrayKokkos& node_coords, const DViewCArrayKokkos& node_vel); void get_vol(); @@ -132,44 +129,44 @@ class FEA_Module_Dynamic_Elasticity : public FEA_Module KOKKOS_INLINE_FUNCTION void get_vol_hex(const DViewCArrayKokkos& elem_vol, - const size_t elem_gid, + const size_t elem_gid, const DViewCArrayKokkos& node_coords, const ViewCArrayKokkos& elem_node_gids, - const size_t rk_level) const; + const size_t rk_level) const; KOKKOS_INLINE_FUNCTION void get_vol_quad(const DViewCArrayKokkos& elem_vol, - const size_t elem_gid, + const size_t elem_gid, const DViewCArrayKokkos& node_coords, const ViewCArrayKokkos& elem_node_gids, - const size_t rk_level) const; + const size_t rk_level) const; KOKKOS_FUNCTION - double get_area_quad(const size_t elem_gid, + double get_area_quad(const size_t elem_gid, const DViewCArrayKokkos& node_coords, const ViewCArrayKokkos& elem_node_gids, - const size_t rk_level) const; + const size_t rk_level) const; KOKKOS_FUNCTION - void get_bmatrix(const ViewCArrayKokkos& B_matrix, - const size_t elem_gid, + void get_bmatrix(const ViewCArrayKokkos& B_matrix, + const size_t elem_gid, const DViewCArrayKokkos& node_coords, const ViewCArrayKokkos& elem_node_gids, - const size_t rk_level) const; + const size_t rk_level) const; KOKKOS_FUNCTION - void get_bmatrix2D(const ViewCArrayKokkos& B_matrix, - const size_t elem_gid, + void get_bmatrix2D(const ViewCArrayKokkos& B_matrix, + const size_t elem_gid, const DViewCArrayKokkos& node_coords, const ViewCArrayKokkos& elem_node_gids, - const size_t rk_level) const; + const size_t rk_level) const; KOKKOS_FUNCTION - void get_area_weights2D(const ViewCArrayKokkos& corner_areas, - const size_t elem_gid, + void get_area_weights2D(const ViewCArrayKokkos& corner_areas, + const size_t elem_gid, const DViewCArrayKokkos& node_coords, const ViewCArrayKokkos& elem_node_gids, - const size_t rk_level) const; + const size_t rk_level) const; KOKKOS_INLINE_FUNCTION double heron(const double x1, @@ -179,120 +176,120 @@ class FEA_Module_Dynamic_Elasticity : public FEA_Module const double x3, const double y3) const; - double average_element_density(const int nodes_per_elem, + double average_element_density(const int nodes_per_elem, const CArray current_element_densities) const; - void get_divergence(DViewCArrayKokkos& elem_div, - const mesh_t mesh, + void get_divergence(DViewCArrayKokkos& elem_div, + const mesh_t mesh, const DViewCArrayKokkos& node_coords, const DViewCArrayKokkos& node_vel, const DViewCArrayKokkos& elem_vol); - void get_divergence2D(DViewCArrayKokkos& elem_div, - const mesh_t mesh, + void get_divergence2D(DViewCArrayKokkos& elem_div, + const mesh_t mesh, const DViewCArrayKokkos& node_coords, const DViewCArrayKokkos& node_vel, const DViewCArrayKokkos& elem_vol); KOKKOS_FUNCTION - void get_velgrad(ViewCArrayKokkos& vel_grad, + void get_velgrad(ViewCArrayKokkos& vel_grad, const ViewCArrayKokkos& elem_node_gids, const DViewCArrayKokkos& node_vel, const ViewCArrayKokkos& b_matrix, - const double elem_vol, - const size_t elem_gid, - const size_t rk_level) const; + const double elem_vol, + const size_t elem_gid, + const size_t rk_level) const; KOKKOS_FUNCTION - void get_velgrad2D(ViewCArrayKokkos& vel_grad, + void get_velgrad2D(ViewCArrayKokkos& vel_grad, const ViewCArrayKokkos& elem_node_gids, const DViewCArrayKokkos& node_vel, const ViewCArrayKokkos& b_matrix, - const double elem_vol, - const double elem_area, - const size_t elem_gid, - const size_t rk_level) const; + const double elem_vol, + const double elem_area, + const size_t elem_gid, + const size_t rk_level) const; KOKKOS_INLINE_FUNCTION - void decompose_vel_grad(ViewCArrayKokkos& D_tensor, - ViewCArrayKokkos& W_tensor, - const ViewCArrayKokkos& vel_grad, - const ViewCArrayKokkos& elem_node_gids, - const size_t elem_gid, + void decompose_vel_grad(ViewCArrayKokkos& D_tensor, + ViewCArrayKokkos& W_tensor, + const ViewCArrayKokkos& vel_grad, + const ViewCArrayKokkos& elem_node_gids, + const size_t elem_gid, const DViewCArrayKokkos& node_coords, const DViewCArrayKokkos& node_vel, - const double vol) const; + const double vol) const; - void boundary_velocity(const mesh_t& mesh, + void boundary_velocity(const mesh_t& mesh, const DCArrayKokkos& boundary, - DViewCArrayKokkos& node_vel); + DViewCArrayKokkos& node_vel); - void boundary_adjoint(const mesh_t& mesh, + void boundary_adjoint(const mesh_t& mesh, const DCArrayKokkos& boundary, - vec_array& node_adjoint, - vec_array& node_phi_adjoint); + vec_array& node_adjoint, + vec_array& node_phi_adjoint); void tag_bdys(const DCArrayKokkos& boundary, - mesh_t& mesh, + mesh_t& mesh, const DViewCArrayKokkos& node_coords); KOKKOS_INLINE_FUNCTION - bool check_bdy(const size_t patch_gid, - const int num_dim, - const int num_nodes_in_patch, - const BOUNDARY_TYPE this_bc_tag, - const double val, + bool check_bdy(const size_t patch_gid, + const int num_dim, + const int num_nodes_in_patch, + const BOUNDARY_TYPE this_bc_tag, + const double val, const DViewCArrayKokkos& node_coords, - const size_t rk_level) const; + const size_t rk_level) const; void rk_init(DViewCArrayKokkos& node_coords, DViewCArrayKokkos& node_vel, DViewCArrayKokkos& elem_sie, DViewCArrayKokkos& elem_stress, - const size_t num_elems, - const size_t num_nodes); + const size_t num_elems, + const size_t num_nodes); - void get_timestep(mesh_t& mesh, + void get_timestep(mesh_t& mesh, DViewCArrayKokkos& node_coords, DViewCArrayKokkos& node_vel, DViewCArrayKokkos& elem_sspd, DViewCArrayKokkos& elem_vol); - void get_timestep2D(mesh_t& mesh, + void get_timestep2D(mesh_t& mesh, DViewCArrayKokkos& node_coords, DViewCArrayKokkos& node_vel, DViewCArrayKokkos& elem_sspd, DViewCArrayKokkos& elem_vol); void update_state(const DCArrayKokkos& material, - const mesh_t& mesh, + const mesh_t& mesh, const DViewCArrayKokkos& node_coords, const DViewCArrayKokkos& node_vel, - DViewCArrayKokkos& elem_den, - DViewCArrayKokkos& elem_pres, - DViewCArrayKokkos& elem_stress, - DViewCArrayKokkos& elem_sspd, + DViewCArrayKokkos& elem_den, + DViewCArrayKokkos& elem_pres, + DViewCArrayKokkos& elem_stress, + DViewCArrayKokkos& elem_sspd, const DViewCArrayKokkos& elem_sie, const DViewCArrayKokkos& elem_vol, const DViewCArrayKokkos& elem_mass, const DViewCArrayKokkos& elem_mat_id, - const double rk_alpha, - const size_t cycle); + const double rk_alpha, + const size_t cycle); void update_state2D(const DCArrayKokkos& material, - const mesh_t& mesh, + const mesh_t& mesh, const DViewCArrayKokkos& node_coords, const DViewCArrayKokkos& node_vel, - DViewCArrayKokkos& elem_den, - DViewCArrayKokkos& elem_pres, - DViewCArrayKokkos& elem_stress, - DViewCArrayKokkos& elem_sspd, + DViewCArrayKokkos& elem_den, + DViewCArrayKokkos& elem_pres, + DViewCArrayKokkos& elem_stress, + DViewCArrayKokkos& elem_sspd, const DViewCArrayKokkos& elem_sie, const DViewCArrayKokkos& elem_vol, const DViewCArrayKokkos& elem_mass, const DViewCArrayKokkos& elem_mat_id, - const double rk_alpha, - const size_t cycle); + const double rk_alpha, + const size_t cycle); void build_boundry_node_sets(mesh_t& mesh); @@ -328,13 +325,13 @@ class FEA_Module_Dynamic_Elasticity : public FEA_Module void collect_output(Teuchos::RCP> global_reduce_map); - void write_data(std::map& point_data_scalars_double, - std::map& point_data_vectors_double, - std::map& cell_data_scalars_double, - std::map& cell_data_scalars_int, + void write_data(std::map& point_data_scalars_double, + std::map& point_data_vectors_double, + std::map& cell_data_scalars_double, + std::map& cell_data_scalars_int, std::map>& cell_data_fields_double); - void write_outputs(const mesh_t& mesh, + void write_outputs(const mesh_t& mesh, DViewCArrayKokkos& node_coords, DViewCArrayKokkos& node_vel, DViewCArrayKokkos& node_mass, @@ -347,7 +344,7 @@ class FEA_Module_Dynamic_Elasticity : public FEA_Module DViewCArrayKokkos& elem_mass, DViewCArrayKokkos& elem_mat_id); - void ensight(const mesh_t& mesh, + void ensight(const mesh_t& mesh, const DViewCArrayKokkos& node_coords, const DViewCArrayKokkos& node_vel, const DViewCArrayKokkos& node_mass, @@ -360,7 +357,7 @@ class FEA_Module_Dynamic_Elasticity : public FEA_Module const DViewCArrayKokkos& elem_mass, const DViewCArrayKokkos& elem_mat_id); - void state_file(const mesh_t& mesh, + void state_file(const mesh_t& mesh, const DViewCArrayKokkos& node_coords, const DViewCArrayKokkos& node_vel, const DViewCArrayKokkos& node_mass, @@ -385,7 +382,7 @@ class FEA_Module_Dynamic_Elasticity : public FEA_Module // elastic TO stuff void get_force_elastic(const DCArrayKokkos& material, - const mesh_t& mesh, + const mesh_t& mesh, const DViewCArrayKokkos& node_coords, const DViewCArrayKokkos& node_vel, const DViewCArrayKokkos& node_mass, @@ -393,12 +390,12 @@ class FEA_Module_Dynamic_Elasticity : public FEA_Module const DViewCArrayKokkos& elem_vol, const DViewCArrayKokkos& elem_div, const DViewCArrayKokkos& elem_mat_id, - DViewCArrayKokkos& corner_force, - const double rk_alpha, - const size_t cycle); + DViewCArrayKokkos& corner_force, + const double rk_alpha, + const size_t cycle); void applied_forces(const DCArrayKokkos& material, - const mesh_t& mesh, + const mesh_t& mesh, const DViewCArrayKokkos& node_coords, const DViewCArrayKokkos& node_vel, const DViewCArrayKokkos& node_mass, @@ -406,9 +403,9 @@ class FEA_Module_Dynamic_Elasticity : public FEA_Module const DViewCArrayKokkos& elem_vol, const DViewCArrayKokkos& elem_div, const DViewCArrayKokkos& elem_mat_id, - DViewCArrayKokkos& corner_force, - const double rk_alpha, - const size_t cycle); + DViewCArrayKokkos& corner_force, + const double rk_alpha, + const size_t cycle); void Element_Material_Properties(size_t ielem, real_t& Element_Modulus, real_t& Poisson_Ratio, real_t density); diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/boundary.cpp b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/boundary.cpp index 9f3a5c7a0..f45d6d532 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/boundary.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/boundary.cpp @@ -1,3 +1,36 @@ +/********************************************************************************************** + 2020. Triad National Security, LLC. All rights reserved. + This program was produced under U.S. Government contract 89233218CNA000001 for Los Alamos + National Laboratory (LANL), which is operated by Triad National Security, LLC for the U.S. + Department of Energy/National Nuclear Security Administration. All rights in the program are + reserved by Triad National Security, LLC, and the U.S. Department of Energy/National Nuclear + Security Administration. The Government is granted for itself and others acting on its behalf a + nonexclusive, paid-up, irrevocable worldwide license in this material to reproduce, prepare + derivative works, distribute copies to the public, perform publicly and display publicly, and + to permit others to do so. + This program is open source under the BSD-3 License. + Redistribution and use in source and binary forms, with or without modification, are permitted + provided that the following conditions are met: + 1. Redistributions of source code must retain the above copyright notice, this list of + conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright notice, this list of + conditions and the following disclaimer in the documentation and/or other materials + provided with the distribution. + 3. Neither the name of the copyright holder nor the names of its contributors may be used + to endorse or promote products derived from this software without specific prior + written permission. + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS + IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **********************************************************************************************/ // ------------------------------------------------------- // This function applys the boundary condition // to points on a list of patches created at setup @@ -8,9 +41,9 @@ #include "FEA_Module_Dynamic_Elasticity.h" #include "Simulation_Parameters/FEA_Module/Boundary_Conditions.h" -void FEA_Module_Dynamic_Elasticity::boundary_velocity(const mesh_t& mesh, - const DCArrayKokkos& boundary, - DViewCArrayKokkos& node_vel) +void FEA_Module_Dynamic_Elasticity::boundary_velocity(const mesh_t& mesh, + const DCArrayKokkos& boundary, + DViewCArrayKokkos& node_vel) { // error and debug flag // DCArrayKokkos print_flag(1, "print_flag"); @@ -18,7 +51,7 @@ void FEA_Module_Dynamic_Elasticity::boundary_velocity(const mesh_t& // print_flag.update_device(); const size_t rk_level = rk_num_bins - 1; - int num_dims = num_dim; + int num_dims = num_dim; // Loop over boundary sets for (size_t bdy_set = 0; bdy_set < num_bdy_sets; bdy_set++) { diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/elastic_optimization.cpp b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/elastic_optimization.cpp index 2805208a9..5a209cfb8 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/elastic_optimization.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/elastic_optimization.cpp @@ -1,4 +1,36 @@ - +/********************************************************************************************** + 2020. Triad National Security, LLC. All rights reserved. + This program was produced under U.S. Government contract 89233218CNA000001 for Los Alamos + National Laboratory (LANL), which is operated by Triad National Security, LLC for the U.S. + Department of Energy/National Nuclear Security Administration. All rights in the program are + reserved by Triad National Security, LLC, and the U.S. Department of Energy/National Nuclear + Security Administration. The Government is granted for itself and others acting on its behalf a + nonexclusive, paid-up, irrevocable worldwide license in this material to reproduce, prepare + derivative works, distribute copies to the public, perform publicly and display publicly, and + to permit others to do so. + This program is open source under the BSD-3 License. + Redistribution and use in source and binary forms, with or without modification, are permitted + provided that the following conditions are met: + 1. Redistributions of source code must retain the above copyright notice, this list of + conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright notice, this list of + conditions and the following disclaimer in the documentation and/or other materials + provided with the distribution. + 3. Neither the name of the copyright holder nor the names of its contributors may be used + to endorse or promote products derived from this software without specific prior + written permission. + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS + IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **********************************************************************************************/ #include "mesh.h" #include "state.h" #include @@ -59,7 +91,7 @@ void FEA_Module_Dynamic_Elasticity::update_forward_solve(Teuchos::RCP const size_t rk_num_bins = simparam->dynamic_options.rk_num_bins; const size_t num_bcs = module_params->boundary_conditions.size(); const size_t num_materials = simparam->materials.size(); - real_t objective_accumulation; + real_t objective_accumulation; // --- Read in the nodes in the mesh --- int myrank = Explicit_Solver_Pointer_->myrank; @@ -68,7 +100,7 @@ void FEA_Module_Dynamic_Elasticity::update_forward_solve(Teuchos::RCP const DCArrayKokkos mat_fill = simparam->mat_fill; const DCArrayKokkos boundary = module_params->boundary; const DCArrayKokkos material = simparam->material; - CArray current_element_nodal_densities = CArray(num_nodes_in_elem); + CArray current_element_nodal_densities = CArray(num_nodes_in_elem); std::vector> FEA_Module_My_TO_Modules = simparam->FEA_Module_My_TO_Modules; problem = Explicit_Solver_Pointer_->problem; // Pointer to ROL optimization problem object @@ -103,9 +135,9 @@ void FEA_Module_Dynamic_Elasticity::update_forward_solve(Teuchos::RCP // view scope { - const_vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); const_vec_array ghost_node_coords_interface = Explicit_Solver_Pointer_->ghost_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); - vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { for (int idim = 0; idim < num_dim; idim++) { @@ -439,7 +471,7 @@ void FEA_Module_Dynamic_Elasticity::update_forward_solve(Teuchos::RCP // loop over the corners of the element and calculate the mass for (size_t corner_lid = 0; corner_lid < 4; corner_lid++) { - size_t corner_gid = corners_in_elem(elem_gid, corner_lid); + size_t corner_gid = corners_in_elem(elem_gid, corner_lid); corner_mass(corner_gid) = corner_areas(corner_lid) * elem_den(elem_gid); // node radius is added later } // end for over corners }); @@ -532,7 +564,7 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_adjoint() const DCArrayKokkos boundary = module_params->boundary; const DCArrayKokkos material = simparam->material; - const int num_dim = simparam->num_dims; + const int num_dim = simparam->num_dims; Teuchos::RCP previous_adjoint_vector_distributed, current_adjoint_vector_distributed; Teuchos::RCP previous_velocity_vector_distributed, current_velocity_vector_distributed; @@ -583,7 +615,7 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_adjoint() const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); const_vec_array previous_adjoint_vector = (*adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); - vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadWrite); + vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadWrite); FOR_ALL_CLASS(node_gid, 0, nlocal_nodes + nghost_nodes, { for (int idim = 0; idim < num_dim; idim++) @@ -721,9 +753,9 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_adjoint_full() } */ matrix_contribution = -damping_constant * previous_adjoint_vector(node_gid, idim); - rate_of_change = previous_velocity_vector(node_gid, idim) - - matrix_contribution / node_mass(node_gid) - - phi_previous_adjoint_vector(node_gid, idim) / node_mass(node_gid); + rate_of_change = previous_velocity_vector(node_gid, idim) - + matrix_contribution / node_mass(node_gid) - + phi_previous_adjoint_vector(node_gid, idim) / node_mass(node_gid); midpoint_adjoint_vector(node_gid, idim) = -rate_of_change * global_dt / 2 + previous_adjoint_vector(node_gid, idim); matrix_contribution = 0; @@ -766,9 +798,9 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_adjoint_full() } */ matrix_contribution = -damping_constant * midpoint_adjoint_vector(node_gid, idim); - rate_of_change = (previous_velocity_vector(node_gid, idim) + current_velocity_vector(node_gid, idim)) / 2 - - matrix_contribution / node_mass(node_gid) - - phi_midpoint_adjoint_vector(node_gid, idim) / node_mass(node_gid); + rate_of_change = (previous_velocity_vector(node_gid, idim) + current_velocity_vector(node_gid, idim)) / 2 - + matrix_contribution / node_mass(node_gid) - + phi_midpoint_adjoint_vector(node_gid, idim) / node_mass(node_gid); current_adjoint_vector(node_gid, idim) = -rate_of_change * global_dt + previous_adjoint_vector(node_gid, idim); matrix_contribution = 0; // compute resulting row of force displacement gradient matrix transpose right multiplied by adjoint vector @@ -804,7 +836,7 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient(const int num_corners = rnum_elem * num_nodes_in_elem; real_t global_dt; - const int num_dim = simparam->num_dims; + const int num_dim = simparam->num_dims; const DCArrayKokkos boundary = module_params->boundary; const DCArrayKokkos material = simparam->material; @@ -1057,7 +1089,7 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient_full( const DCArrayKokkos boundary = module_params->boundary; const DCArrayKokkos material = simparam->material; - const int num_dim = simparam->num_dims; + const int num_dim = simparam->num_dims; auto current_element_velocities = CArrayKokkos(num_nodes_in_elem, num_dim); auto current_element_adjoint = CArrayKokkos(num_nodes_in_elem, num_dim); @@ -1070,7 +1102,7 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient_full( compute_topology_optimization_adjoint_full(); { // view scope - vec_array design_gradients = design_gradients_distributed->getLocalView(Tpetra::Access::ReadWrite); + vec_array design_gradients = design_gradients_distributed->getLocalView(Tpetra::Access::ReadWrite); const_vec_array design_densities = design_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); // compute design gradients FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { @@ -1218,7 +1250,7 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient_full( const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); const_vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); const_vec_array current_phi_adjoint_vector = (*phi_adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); - const_vec_array next_velocity_vector = (*forward_solve_velocity_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_velocity_vector = (*forward_solve_velocity_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); const_vec_array next_adjoint_vector = (*adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); const_vec_array next_phi_adjoint_vector = (*phi_adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); @@ -1351,7 +1383,7 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient_full( // force_design_gradient_term(design_variables, design_gradients); // view scope { - host_vec_array host_design_gradients = design_gradients_distributed->getLocalView(Tpetra::Access::ReadWrite); + host_vec_array host_design_gradients = design_gradients_distributed->getLocalView(Tpetra::Access::ReadWrite); const_host_vec_array host_design_variables = design_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); compute_stiffness_gradients(host_design_variables, host_design_gradients); } // end view scope @@ -1375,7 +1407,7 @@ void FEA_Module_Dynamic_Elasticity::init_assembly() CArrayKokkos current_row_nodes_scanned; nodal_density_flag = simparam->nodal_density_flag; - penalty_power = simparam->optimization_options.simp_penalty_power; + penalty_power = simparam->optimization_options.simp_penalty_power; // allocate stride arrays CArrayKokkos Graph_Matrix_Strides_initial(nlocal_nodes, "Graph_Matrix_Strides_initial"); @@ -1405,7 +1437,7 @@ void FEA_Module_Dynamic_Elasticity::init_assembly() // initialize nlocal arrays FOR_ALL_CLASS(inode, 0, nall_nodes, { node_indices_used(inode) = 0; - column_index(inode) = 0; + column_index(inode) = 0; }); // end parallel for Kokkos::fence(); @@ -1468,7 +1500,7 @@ void FEA_Module_Dynamic_Elasticity::init_assembly() current_row_nodes_scanned = CArrayKokkos(max_stride, "current_row_nodes_scanned"); // allocate sparse graph with node repeats - RaggedRightArrayKokkos Repeat_Graph_Matrix(Graph_Matrix_Strides_initial); + RaggedRightArrayKokkos Repeat_Graph_Matrix(Graph_Matrix_Strides_initial); RaggedRightArrayofVectorsKokkos Element_local_indices(Graph_Matrix_Strides_initial, num_dim); // Fill the initial Graph with repeats @@ -1596,7 +1628,7 @@ void FEA_Module_Dynamic_Elasticity::init_assembly() that have been scanned uniquely. Use this list to reset the flag array afterwards without having to loop over all the nodes in the system*/ node_indices_used(current_node) = 1; - column_index(current_node) = istride; + column_index(current_node) = istride; current_row_nodes_scanned(current_row_n_nodes_scanned) = current_node; current_row_n_nodes_scanned++; } @@ -1701,10 +1733,10 @@ void FEA_Module_Dynamic_Elasticity::init_assembly() Enforce boundary conditions on the adjoint vectors ------------------------------------------------------------------------- */ -void FEA_Module_Dynamic_Elasticity::boundary_adjoint(const mesh_t& mesh, - const DCArrayKokkos& boundary, - vec_array& node_adjoint, - vec_array& node_phi_adjoint) +void FEA_Module_Dynamic_Elasticity::boundary_adjoint(const mesh_t& mesh, + const DCArrayKokkos& boundary, + vec_array& node_adjoint, + vec_array& node_phi_adjoint) { // error and debug flag // DCArrayKokkos print_flag(1, "print_flag"); @@ -1712,7 +1744,7 @@ void FEA_Module_Dynamic_Elasticity::boundary_adjoint(const mesh_t& // print_flag.update_device(); const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - int num_dims = simparam->num_dims; + int num_dims = simparam->num_dims; // Loop over boundary sets for (size_t bdy_set = 0; bdy_set < num_bdy_sets; bdy_set++) { diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/force_elastic.cpp b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/force_elastic.cpp index edf219c35..256efaa35 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/force_elastic.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/force_elastic.cpp @@ -1,4 +1,36 @@ - +/********************************************************************************************** + 2020. Triad National Security, LLC. All rights reserved. + This program was produced under U.S. Government contract 89233218CNA000001 for Los Alamos + National Laboratory (LANL), which is operated by Triad National Security, LLC for the U.S. + Department of Energy/National Nuclear Security Administration. All rights in the program are + reserved by Triad National Security, LLC, and the U.S. Department of Energy/National Nuclear + Security Administration. The Government is granted for itself and others acting on its behalf a + nonexclusive, paid-up, irrevocable worldwide license in this material to reproduce, prepare + derivative works, distribute copies to the public, perform publicly and display publicly, and + to permit others to do so. + This program is open source under the BSD-3 License. + Redistribution and use in source and binary forms, with or without modification, are permitted + provided that the following conditions are met: + 1. Redistributions of source code must retain the above copyright notice, this list of + conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright notice, this list of + conditions and the following disclaimer in the documentation and/or other materials + provided with the distribution. + 3. Neither the name of the copyright holder nor the names of its contributors may be used + to endorse or promote products derived from this software without specific prior + written permission. + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS + IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **********************************************************************************************/ #include // fmin, fmax, abs note: fminl is long #include "mesh.h" #include "state.h" @@ -11,18 +43,18 @@ // This function calculates the corner forces and the evolves stress (hypo) // ------------------------------------------------------------------------------ void FEA_Module_Dynamic_Elasticity::get_force_elastic(const DCArrayKokkos& material, - const mesh_t& mesh, - const DViewCArrayKokkos& node_coords, - const DViewCArrayKokkos& node_vel, - const DViewCArrayKokkos& node_mass, - const DViewCArrayKokkos& elem_den, - const DViewCArrayKokkos& elem_vol, - const DViewCArrayKokkos& elem_div, - const DViewCArrayKokkos& elem_mat_id, - DViewCArrayKokkos& corner_force, - const double rk_alpha, - const size_t cycle - ) + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_mass, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + DViewCArrayKokkos& corner_force, + const double rk_alpha, + const size_t cycle + ) { const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; const size_t num_dim = mesh.num_dims; @@ -80,18 +112,18 @@ void FEA_Module_Dynamic_Elasticity::get_force_elastic(const DCArrayKokkos& material, - const mesh_t& mesh, - const DViewCArrayKokkos& node_coords, - const DViewCArrayKokkos& node_vel, - const DViewCArrayKokkos& node_mass, - const DViewCArrayKokkos& elem_den, - const DViewCArrayKokkos& elem_vol, - const DViewCArrayKokkos& elem_div, - const DViewCArrayKokkos& elem_mat_id, - DViewCArrayKokkos& corner_force, - const double rk_alpha, - const size_t cycle - ) + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_mass, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + DViewCArrayKokkos& corner_force, + const double rk_alpha, + const size_t cycle + ) { const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; const size_t num_dim = mesh.num_dims; @@ -430,7 +462,7 @@ void FEA_Module_Dynamic_Elasticity::local_matrix_multiply(int ielem, CArrayKokko elements::legendre_weights_1D(legendre_weights_1D, num_gauss_points); Solver::node_ordering_convention active_node_ordering_convention = Explicit_Solver_Pointer_->active_node_ordering_convention; - real_t current_density = 1; + real_t current_density = 1; CArrayKokkos convert_node_order(max_nodes_per_element); if ((active_node_ordering_convention == Solver::ENSIGHT && num_dim == 3) || (active_node_ordering_convention == Solver::IJK && num_dim == 2)) { @@ -551,8 +583,8 @@ void FEA_Module_Dynamic_Elasticity::local_matrix_multiply(int ielem, CArrayKokko // look up element material properties at this point as a function of density Element_Material_Properties((size_t) ielem, Element_Modulus, Poisson_Ratio, current_density); Elastic_Constant = Element_Modulus / ((1 + Poisson_Ratio) * (1 - 2 * Poisson_Ratio)); - Shear_Term = 0.5 - Poisson_Ratio; - Pressure_Term = 1 - Poisson_Ratio; + Shear_Term = 0.5 - Poisson_Ratio; + Pressure_Term = 1 - Poisson_Ratio; // debug print // std::cout << "Element Material Params " << Elastic_Constant << std::endl; @@ -860,9 +892,9 @@ void FEA_Module_Dynamic_Elasticity::compute_stiffness_gradients(const_host_vec_a int num_gauss_points = simparam->num_gauss_points; int z_quad, y_quad, x_quad, direct_product_count; - size_t local_node_id, local_dof_idx, local_dof_idy, local_dof_idz; + size_t local_node_id, local_dof_idx, local_dof_idy, local_dof_idz; const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - GO current_global_index; + GO current_global_index; real_t global_dt; size_t current_data_index, next_data_index; @@ -1254,8 +1286,8 @@ void FEA_Module_Dynamic_Elasticity::compute_stiffness_gradients(const_host_vec_a // look up element material properties at this point as a function of density Gradient_Element_Material_Properties(ielem, Element_Modulus_Gradient, Poisson_Ratio, current_density); Elastic_Constant = Element_Modulus_Gradient / ((1 + Poisson_Ratio) * (1 - 2 * Poisson_Ratio)); - Shear_Term = 0.5 - Poisson_Ratio; - Pressure_Term = 1 - Poisson_Ratio; + Shear_Term = 0.5 - Poisson_Ratio; + Pressure_Term = 1 - Poisson_Ratio; // debug print // std::cout << "Element Material Params " << Elastic_Constant << std::endl; diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/force_gradients_elastic.cpp b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/force_gradients_elastic.cpp index b88a3895e..fc33b514f 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/force_gradients_elastic.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/force_gradients_elastic.cpp @@ -1,4 +1,36 @@ - +/********************************************************************************************** + 2020. Triad National Security, LLC. All rights reserved. + This program was produced under U.S. Government contract 89233218CNA000001 for Los Alamos + National Laboratory (LANL), which is operated by Triad National Security, LLC for the U.S. + Department of Energy/National Nuclear Security Administration. All rights in the program are + reserved by Triad National Security, LLC, and the U.S. Department of Energy/National Nuclear + Security Administration. The Government is granted for itself and others acting on its behalf a + nonexclusive, paid-up, irrevocable worldwide license in this material to reproduce, prepare + derivative works, distribute copies to the public, perform publicly and display publicly, and + to permit others to do so. + This program is open source under the BSD-3 License. + Redistribution and use in source and binary forms, with or without modification, are permitted + provided that the following conditions are met: + 1. Redistributions of source code must retain the above copyright notice, this list of + conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright notice, this list of + conditions and the following disclaimer in the documentation and/or other materials + provided with the distribution. + 3. Neither the name of the copyright holder nor the names of its contributors may be used + to endorse or promote products derived from this software without specific prior + written permission. + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS + IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **********************************************************************************************/ #include "mesh.h" #include "state.h" #include @@ -26,20 +58,20 @@ // This function calculates the corner forces and the evolves stress (hypo) // ------------------------------------------------------------------------------ void FEA_Module_Dynamic_Elasticity::get_force_vgradient_elastic(const DCArrayKokkos& material, - const mesh_t& mesh, - const DViewCArrayKokkos& node_coords, - const DViewCArrayKokkos& node_vel, - const DViewCArrayKokkos& elem_den, - const DViewCArrayKokkos& elem_sie, - const DViewCArrayKokkos& elem_pres, - const DViewCArrayKokkos& elem_stress, - const DViewCArrayKokkos& elem_sspd, - const DViewCArrayKokkos& elem_vol, - const DViewCArrayKokkos& elem_div, - const DViewCArrayKokkos& elem_mat_id, - const double rk_alpha, - const size_t cycle - ) + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_pres, + const DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + const double rk_alpha, + const size_t cycle + ) { const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; const size_t num_dims = simparam->num_dims; @@ -401,20 +433,20 @@ void FEA_Module_Dynamic_Elasticity::get_force_vgradient_elastic(const DCArrayKok // This function calculates the corner forces and the evolves stress (hypo) // ------------------------------------------------------------------------------ void FEA_Module_Dynamic_Elasticity::get_force_ugradient_elastic(const DCArrayKokkos& material, - const mesh_t& mesh, - const DViewCArrayKokkos& node_coords, - const DViewCArrayKokkos& node_vel, - const DViewCArrayKokkos& elem_den, - const DViewCArrayKokkos& elem_sie, - const DViewCArrayKokkos& elem_pres, - const DViewCArrayKokkos& elem_stress, - const DViewCArrayKokkos& elem_sspd, - const DViewCArrayKokkos& elem_vol, - const DViewCArrayKokkos& elem_div, - const DViewCArrayKokkos& elem_mat_id, - const double rk_alpha, - const size_t cycle - ) + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_pres, + const DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + const double rk_alpha, + const size_t cycle + ) { const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; const size_t num_dims = simparam->num_dims; @@ -776,7 +808,7 @@ void FEA_Module_Dynamic_Elasticity::force_design_gradient_term(const_vec_array d const DCArrayKokkos boundary = module_params->boundary; const DCArrayKokkos material = simparam->material; - const int num_dim = simparam->num_dims; + const int num_dim = simparam->num_dims; auto current_element_adjoint = CArrayKokkos(num_nodes_in_elem, num_dim); @@ -896,20 +928,20 @@ void FEA_Module_Dynamic_Elasticity::force_design_gradient_term(const_vec_array d // This function calculates the corner forces and the evolves stress (hypo) // ------------------------------------------------------------------------------ void FEA_Module_Dynamic_Elasticity::get_force_dgradient_elastic(const DCArrayKokkos& material, - const mesh_t& mesh, - const DViewCArrayKokkos& node_coords, - const DViewCArrayKokkos& node_vel, - const DViewCArrayKokkos& elem_den, - const DViewCArrayKokkos& elem_sie, - const DViewCArrayKokkos& elem_pres, - const DViewCArrayKokkos& elem_stress, - const DViewCArrayKokkos& elem_sspd, - const DViewCArrayKokkos& elem_vol, - const DViewCArrayKokkos& elem_div, - const DViewCArrayKokkos& elem_mat_id, - const double rk_alpha, - const size_t cycle - ) + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_pres, + const DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + const double rk_alpha, + const size_t cycle + ) { const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; const size_t num_dims = simparam->num_dims; diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/geometry.cpp b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/geometry.cpp index 0f19f6580..a79f8fe56 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/geometry.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/geometry.cpp @@ -1,3 +1,37 @@ +/********************************************************************************************** + 2020. Triad National Security, LLC. All rights reserved. + This program was produced under U.S. Government contract 89233218CNA000001 for Los Alamos + National Laboratory (LANL), which is operated by Triad National Security, LLC for the U.S. + Department of Energy/National Nuclear Security Administration. All rights in the program are + reserved by Triad National Security, LLC, and the U.S. Department of Energy/National Nuclear + Security Administration. The Government is granted for itself and others acting on its behalf a + nonexclusive, paid-up, irrevocable worldwide license in this material to reproduce, prepare + derivative works, distribute copies to the public, perform publicly and display publicly, and + to permit others to do so. + This program is open source under the BSD-3 License. + Redistribution and use in source and binary forms, with or without modification, are permitted + provided that the following conditions are met: + 1. Redistributions of source code must retain the above copyright notice, this list of + conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright notice, this list of + conditions and the following disclaimer in the documentation and/or other materials + provided with the distribution. + 3. Neither the name of the copyright holder nor the names of its contributors may be used + to endorse or promote products derived from this software without specific prior + written permission. + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS + IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **********************************************************************************************/ + // ----------------------------------------------------------------------------- // This code handles the geometric information for the mesh for the SHG solver // ------------------------------------------------------------------------------ @@ -5,13 +39,13 @@ #include "state.h" #include "FEA_Module_Dynamic_Elasticity.h" -void FEA_Module_Dynamic_Elasticity::update_position_elastic(double rk_alpha, - const size_t num_nodes, - DViewCArrayKokkos& node_coords, - const DViewCArrayKokkos& node_vel) +void FEA_Module_Dynamic_Elasticity::update_position_elastic(double rk_alpha, + const size_t num_nodes, + DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel) { const size_t rk_level = rk_num_bins - 1; - int num_dims = num_dims; + int num_dims = num_dims; // loop over all the nodes in the mesh FOR_ALL_CLASS(node_gid, 0, num_nodes, { @@ -35,11 +69,11 @@ void FEA_Module_Dynamic_Elasticity::update_position_elastic(double // B_p is the OUTWARD corner area normal at node p // ------------------------------------------------------------------------------ KOKKOS_FUNCTION -void FEA_Module_Dynamic_Elasticity::get_bmatrix(const ViewCArrayKokkos& B_matrix, - const size_t elem_gid, - const DViewCArrayKokkos& node_coords, - const ViewCArrayKokkos& elem_node_gids, - const size_t rk_level) const +void FEA_Module_Dynamic_Elasticity::get_bmatrix(const ViewCArrayKokkos& B_matrix, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const { const size_t num_nodes = 8; @@ -265,10 +299,10 @@ void FEA_Module_Dynamic_Elasticity::get_vol() // Exact volume for a hex element KOKKOS_INLINE_FUNCTION void FEA_Module_Dynamic_Elasticity::get_vol_hex(const DViewCArrayKokkos& elem_vol, - const size_t elem_gid, - const DViewCArrayKokkos& node_coords, - const ViewCArrayKokkos& elem_node_gids, - const size_t rk_level) const + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const { const size_t num_nodes = 8; @@ -308,11 +342,11 @@ void FEA_Module_Dynamic_Elasticity::get_vol_hex(const DViewCArrayKokkos& } // end subroutine KOKKOS_FUNCTION -void FEA_Module_Dynamic_Elasticity::get_bmatrix2D(const ViewCArrayKokkos& B_matrix, - const size_t elem_gid, - const DViewCArrayKokkos& node_coords, - const ViewCArrayKokkos& elem_node_gids, - const size_t rk_level) const +void FEA_Module_Dynamic_Elasticity::get_bmatrix2D(const ViewCArrayKokkos& B_matrix, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const { const size_t num_nodes = 4; @@ -367,10 +401,10 @@ void FEA_Module_Dynamic_Elasticity::get_bmatrix2D(const ViewCArrayKokkos // true volume of a quad in RZ coords KOKKOS_INLINE_FUNCTION void FEA_Module_Dynamic_Elasticity::get_vol_quad(const DViewCArrayKokkos& elem_vol, - const size_t elem_gid, - const DViewCArrayKokkos& node_coords, - const ViewCArrayKokkos& elem_node_gids, - const size_t rk_level) const + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const { // --- testing here --- /* @@ -418,17 +452,17 @@ void FEA_Module_Dynamic_Elasticity::get_vol_quad(const DViewCArrayKokkos */ elem_vol(elem_gid) = ( (y(2) + y(3) + y(0)) * ((y(2) - y(3)) * (x(0) - x(3)) - (y(0) - y(3)) * (x(2) - x(3)) ) - + (y(0) + y(1) + y(2)) * ((y(0) - y(1)) * (x(2) - x(1)) - (y(2) - y(1)) * (x(0) - x(1))) ) / 6.0; + + (y(0) + y(1) + y(2)) * ((y(0) - y(1)) * (x(2) - x(1)) - (y(2) - y(1)) * (x(0) - x(1))) ) / 6.0; return; } // end subroutine // element facial area KOKKOS_FUNCTION -double FEA_Module_Dynamic_Elasticity::get_area_quad(const size_t elem_gid, - const DViewCArrayKokkos& node_coords, - const ViewCArrayKokkos& elem_node_gids, - const size_t rk_level) const +double FEA_Module_Dynamic_Elasticity::get_area_quad(const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const { double elem_area = 0.0; @@ -460,11 +494,11 @@ double FEA_Module_Dynamic_Elasticity::get_area_quad(const size_t KOKKOS_INLINE_FUNCTION double FEA_Module_Dynamic_Elasticity::heron(const double x1, - const double y1, - const double x2, - const double y2, - const double x3, - const double y3) const + const double y1, + const double x2, + const double y2, + const double x3, + const double y3) const { double S, a, b, c, area; @@ -483,11 +517,11 @@ double FEA_Module_Dynamic_Elasticity::heron(const double x1, } KOKKOS_FUNCTION -void FEA_Module_Dynamic_Elasticity::get_area_weights2D(const ViewCArrayKokkos& corner_areas, - const size_t elem_gid, - const DViewCArrayKokkos& node_coords, - const ViewCArrayKokkos& elem_node_gids, - const size_t rk_level) const +void FEA_Module_Dynamic_Elasticity::get_area_weights2D(const ViewCArrayKokkos& corner_areas, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const { const size_t num_nodes = 4; diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/mesh.h b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/mesh.h index 2a4d852c8..4fac52d37 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/mesh.h +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/mesh.h @@ -1,3 +1,36 @@ +/********************************************************************************************** + 2020. Triad National Security, LLC. All rights reserved. + This program was produced under U.S. Government contract 89233218CNA000001 for Los Alamos + National Laboratory (LANL), which is operated by Triad National Security, LLC for the U.S. + Department of Energy/National Nuclear Security Administration. All rights in the program are + reserved by Triad National Security, LLC, and the U.S. Department of Energy/National Nuclear + Security Administration. The Government is granted for itself and others acting on its behalf a + nonexclusive, paid-up, irrevocable worldwide license in this material to reproduce, prepare + derivative works, distribute copies to the public, perform publicly and display publicly, and + to permit others to do so. + This program is open source under the BSD-3 License. + Redistribution and use in source and binary forms, with or without modification, are permitted + provided that the following conditions are met: + 1. Redistributions of source code must retain the above copyright notice, this list of + conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright notice, this list of + conditions and the following disclaimer in the documentation and/or other materials + provided with the distribution. + 3. Neither the name of the copyright holder nor the names of its contributors may be used + to endorse or promote products derived from this software without specific prior + written permission. + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS + IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **********************************************************************************************/ #ifndef MESH_H #define MESH_H @@ -485,7 +518,7 @@ struct mesh_t // neighboring_side_lids(elem_gid, patch_lid) = patch_lid; patches_in_elem(elem_gid, patch_lid) = patch_gid; - temp_bdy_patches(bdy_patch_gid) = patch_gid; + temp_bdy_patches(bdy_patch_gid) = patch_gid; patch_gid++; bdy_patch_gid++; @@ -556,7 +589,7 @@ struct mesh_t }); // end FOR_ALL bdy_patch_gid // find and store the boundary nodes - CArrayKokkos temp_bdy_nodes(num_nodes, "temp_bdy_nodes"); + CArrayKokkos temp_bdy_nodes(num_nodes, "temp_bdy_nodes"); CArrayKokkos hash_bdy_nodes(num_nodes, "hash_bdy_nodes"); FOR_ALL_CLASS(node_gid, 0, num_nodes, { @@ -756,7 +789,7 @@ struct mesh_t printf("ERROR: number of boundary sets = 0, set it = 1"); num_bcs = 1; } - num_bdy_sets = num_bcs; + num_bdy_sets = num_bcs; bdy_patches_in_set = DynamicRaggedRightArrayKokkos(num_bcs, num_bdy_patches, "bdy_patches_in_set"); return; diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/momentum.cpp b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/momentum.cpp index be4ca4b69..dff8ef928 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/momentum.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/momentum.cpp @@ -1,4 +1,36 @@ - +/********************************************************************************************** + 2020. Triad National Security, LLC. All rights reserved. + This program was produced under U.S. Government contract 89233218CNA000001 for Los Alamos + National Laboratory (LANL), which is operated by Triad National Security, LLC for the U.S. + Department of Energy/National Nuclear Security Administration. All rights in the program are + reserved by Triad National Security, LLC, and the U.S. Department of Energy/National Nuclear + Security Administration. The Government is granted for itself and others acting on its behalf a + nonexclusive, paid-up, irrevocable worldwide license in this material to reproduce, prepare + derivative works, distribute copies to the public, perform publicly and display publicly, and + to permit others to do so. + This program is open source under the BSD-3 License. + Redistribution and use in source and binary forms, with or without modification, are permitted + provided that the following conditions are met: + 1. Redistributions of source code must retain the above copyright notice, this list of + conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright notice, this list of + conditions and the following disclaimer in the documentation and/or other materials + provided with the distribution. + 3. Neither the name of the copyright holder nor the names of its contributors may be used + to endorse or promote products derived from this software without specific prior + written permission. + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS + IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **********************************************************************************************/ #include "mesh.h" #include "state.h" #include "FEA_Module_Dynamic_Elasticity.h" @@ -55,21 +87,21 @@ void FEA_Module_Dynamic_Elasticity::update_velocity_elastic(double rk_alpha, // This function calculates the velocity gradient // ------------------------------------------------------------------------------ KOKKOS_FUNCTION -void FEA_Module_Dynamic_Elasticity::get_velgrad(ViewCArrayKokkos& vel_grad, - const ViewCArrayKokkos& elem_node_gids, - const DViewCArrayKokkos& node_vel, - const ViewCArrayKokkos& b_matrix, - const double elem_vol, - const size_t elem_gid, - const size_t rk_level - ) const +void FEA_Module_Dynamic_Elasticity::get_velgrad(ViewCArrayKokkos& vel_grad, + const ViewCArrayKokkos& elem_node_gids, + const DViewCArrayKokkos& node_vel, + const ViewCArrayKokkos& b_matrix, + const double elem_vol, + const size_t elem_gid, + const size_t rk_level + ) const { const size_t num_nodes_in_elem = 8; - double u_array[num_nodes_in_elem]; - double v_array[num_nodes_in_elem]; - double w_array[num_nodes_in_elem]; - + double u_array[num_nodes_in_elem]; + double v_array[num_nodes_in_elem]; + double w_array[num_nodes_in_elem]; + ViewCArrayKokkos u(u_array, num_nodes_in_elem); // x-dir vel component ViewCArrayKokkos v(v_array, num_nodes_in_elem); // y-dir vel component ViewCArrayKokkos w(w_array, num_nodes_in_elem); // z-dir vel component @@ -142,21 +174,21 @@ void FEA_Module_Dynamic_Elasticity::get_velgrad(ViewCArrayKokkos& // This function calculates the velocity gradient // ------------------------------------------------------------------------------ KOKKOS_FUNCTION -void FEA_Module_Dynamic_Elasticity::get_velgrad2D(ViewCArrayKokkos& vel_grad, - const ViewCArrayKokkos& elem_node_gids, - const DViewCArrayKokkos& node_vel, - const ViewCArrayKokkos& b_matrix, - const double elem_vol, - const double elem_area, - const size_t elem_gid, - const size_t rk_level - ) const +void FEA_Module_Dynamic_Elasticity::get_velgrad2D(ViewCArrayKokkos& vel_grad, + const ViewCArrayKokkos& elem_node_gids, + const DViewCArrayKokkos& node_vel, + const ViewCArrayKokkos& b_matrix, + const double elem_vol, + const double elem_area, + const size_t elem_gid, + const size_t rk_level + ) const { const size_t num_nodes_in_elem = 4; - double u_array[num_nodes_in_elem]; - double v_array[num_nodes_in_elem]; - + double u_array[num_nodes_in_elem]; + double v_array[num_nodes_in_elem]; + ViewCArrayKokkos u(u_array, num_nodes_in_elem); // x-dir vel component ViewCArrayKokkos v(v_array, num_nodes_in_elem); // y-dir vel component @@ -207,12 +239,12 @@ void FEA_Module_Dynamic_Elasticity::get_velgrad2D(ViewCArrayKokkos& // ----------------------------------------------------------------------------- // This subroutine to calculate the velocity divergence in all elements // ------------------------------------------------------------------------------ -void FEA_Module_Dynamic_Elasticity::get_divergence(DViewCArrayKokkos& elem_div, - const mesh_t mesh, - const DViewCArrayKokkos& node_coords, - const DViewCArrayKokkos& node_vel, - const DViewCArrayKokkos& elem_vol - ) +void FEA_Module_Dynamic_Elasticity::get_divergence(DViewCArrayKokkos& elem_div, + const mesh_t mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_vol + ) { const size_t rk_level = rk_num_bins - 1; @@ -224,7 +256,7 @@ void FEA_Module_Dynamic_Elasticity::get_divergence(DViewCArrayKokkos& double u_array[num_nodes_in_elem]; double v_array[num_nodes_in_elem]; double w_array[num_nodes_in_elem]; - + ViewCArrayKokkos u(u_array, num_nodes_in_elem); // x-dir vel component ViewCArrayKokkos v(v_array, num_nodes_in_elem); // y-dir vel component ViewCArrayKokkos w(w_array, num_nodes_in_elem); // z-dir vel component @@ -282,12 +314,12 @@ void FEA_Module_Dynamic_Elasticity::get_divergence(DViewCArrayKokkos& // ----------------------------------------------------------------------------- // This subroutine to calculate the velocity divergence in all elements // ------------------------------------------------------------------------------ -void FEA_Module_Dynamic_Elasticity::get_divergence2D(DViewCArrayKokkos& elem_div, - const mesh_t mesh, - const DViewCArrayKokkos& node_coords, - const DViewCArrayKokkos& node_vel, - const DViewCArrayKokkos& elem_vol - ) +void FEA_Module_Dynamic_Elasticity::get_divergence2D(DViewCArrayKokkos& elem_div, + const mesh_t mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_vol + ) { const size_t rk_level = rk_num_bins - 1; @@ -298,7 +330,7 @@ void FEA_Module_Dynamic_Elasticity::get_divergence2D(DViewCArrayKokkos& double u_array[num_nodes_in_elem]; double v_array[num_nodes_in_elem]; - + ViewCArrayKokkos u(u_array, num_nodes_in_elem); // x-dir vel component ViewCArrayKokkos v(v_array, num_nodes_in_elem); // y-dir vel component @@ -364,15 +396,15 @@ void FEA_Module_Dynamic_Elasticity::get_divergence2D(DViewCArrayKokkos& // D = sym(L) // W = antisym(L) KOKKOS_INLINE_FUNCTION -void FEA_Module_Dynamic_Elasticity::decompose_vel_grad(ViewCArrayKokkos& D_tensor, - ViewCArrayKokkos& W_tensor, - const ViewCArrayKokkos& vel_grad, - const ViewCArrayKokkos& elem_node_gids, - const size_t elem_gid, - const DViewCArrayKokkos& node_coords, - const DViewCArrayKokkos& node_vel, - const double vol - ) const +void FEA_Module_Dynamic_Elasticity::decompose_vel_grad(ViewCArrayKokkos& D_tensor, + ViewCArrayKokkos& W_tensor, + const ViewCArrayKokkos& vel_grad, + const ViewCArrayKokkos& elem_node_gids, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const double vol + ) const { // --- Calculate the velocity gradient --- diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/properties.cpp b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/properties.cpp index cc57dd5c0..4e0d5c52e 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/properties.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/properties.cpp @@ -1,4 +1,36 @@ - +/********************************************************************************************** + 2020. Triad National Security, LLC. All rights reserved. + This program was produced under U.S. Government contract 89233218CNA000001 for Los Alamos + National Laboratory (LANL), which is operated by Triad National Security, LLC for the U.S. + Department of Energy/National Nuclear Security Administration. All rights in the program are + reserved by Triad National Security, LLC, and the U.S. Department of Energy/National Nuclear + Security Administration. The Government is granted for itself and others acting on its behalf a + nonexclusive, paid-up, irrevocable worldwide license in this material to reproduce, prepare + derivative works, distribute copies to the public, perform publicly and display publicly, and + to permit others to do so. + This program is open source under the BSD-3 License. + Redistribution and use in source and binary forms, with or without modification, are permitted + provided that the following conditions are met: + 1. Redistributions of source code must retain the above copyright notice, this list of + conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright notice, this list of + conditions and the following disclaimer in the documentation and/or other materials + provided with the distribution. + 3. Neither the name of the copyright holder nor the names of its contributors may be used + to endorse or promote products derived from this software without specific prior + written permission. + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS + IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **********************************************************************************************/ // ----------------------------------------------------------------------------- // This calls the models to update state // ------------------------------------------------------------------------------ @@ -7,23 +39,23 @@ #include "FEA_Module_Dynamic_Elasticity.h" void FEA_Module_Dynamic_Elasticity::update_state(const DCArrayKokkos& material, - const mesh_t& mesh, - const DViewCArrayKokkos& node_coords, - const DViewCArrayKokkos& node_vel, - DViewCArrayKokkos& elem_den, - DViewCArrayKokkos& elem_pres, - DViewCArrayKokkos& elem_stress, - DViewCArrayKokkos& elem_sspd, - const DViewCArrayKokkos& elem_sie, - const DViewCArrayKokkos& elem_vol, - const DViewCArrayKokkos& elem_mass, - const DViewCArrayKokkos& elem_mat_id, - const double rk_alpha, - const size_t cycle - ) + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_den, + DViewCArrayKokkos& elem_pres, + DViewCArrayKokkos& elem_stress, + DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& elem_mat_id, + const double rk_alpha, + const size_t cycle + ) { const size_t rk_level = rk_num_bins - 1; - int num_dims = num_dim; + int num_dims = num_dim; // loop over all the elements in the mesh FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { @@ -140,23 +172,23 @@ void FEA_Module_Dynamic_Elasticity::update_state(const DCArrayKokkos } // end method to update state void FEA_Module_Dynamic_Elasticity::update_state2D(const DCArrayKokkos& material, - const mesh_t& mesh, - const DViewCArrayKokkos& node_coords, - const DViewCArrayKokkos& node_vel, - DViewCArrayKokkos& elem_den, - DViewCArrayKokkos& elem_pres, - DViewCArrayKokkos& elem_stress, - DViewCArrayKokkos& elem_sspd, - const DViewCArrayKokkos& elem_sie, - const DViewCArrayKokkos& elem_vol, - const DViewCArrayKokkos& elem_mass, - const DViewCArrayKokkos& elem_mat_id, - const double rk_alpha, - const size_t cycle - ) + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_den, + DViewCArrayKokkos& elem_pres, + DViewCArrayKokkos& elem_stress, + DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& elem_mat_id, + const double rk_alpha, + const size_t cycle + ) { const size_t rk_level = rk_num_bins - 1; - int num_dims = num_dim; + int num_dims = num_dim; // loop over all the elements in the mesh FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/state.h b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/state.h index b7679c89a..ceb575ff1 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/state.h +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/state.h @@ -1,3 +1,36 @@ +/********************************************************************************************** + 2020. Triad National Security, LLC. All rights reserved. + This program was produced under U.S. Government contract 89233218CNA000001 for Los Alamos + National Laboratory (LANL), which is operated by Triad National Security, LLC for the U.S. + Department of Energy/National Nuclear Security Administration. All rights in the program are + reserved by Triad National Security, LLC, and the U.S. Department of Energy/National Nuclear + Security Administration. The Government is granted for itself and others acting on its behalf a + nonexclusive, paid-up, irrevocable worldwide license in this material to reproduce, prepare + derivative works, distribute copies to the public, perform publicly and display publicly, and + to permit others to do so. + This program is open source under the BSD-3 License. + Redistribution and use in source and binary forms, with or without modification, are permitted + provided that the following conditions are met: + 1. Redistributions of source code must retain the above copyright notice, this list of + conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright notice, this list of + conditions and the following disclaimer in the documentation and/or other materials + provided with the distribution. + 3. Neither the name of the copyright holder nor the names of its contributors may be used + to endorse or promote products derived from this software without specific prior + written permission. + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS + IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **********************************************************************************************/ #ifndef STATE_H #define STATE_H diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/time_integration.cpp b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/time_integration.cpp index 76d9eda73..e77705940 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/time_integration.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/time_integration.cpp @@ -1,4 +1,36 @@ - +/********************************************************************************************** + 2020. Triad National Security, LLC. All rights reserved. + This program was produced under U.S. Government contract 89233218CNA000001 for Los Alamos + National Laboratory (LANL), which is operated by Triad National Security, LLC for the U.S. + Department of Energy/National Nuclear Security Administration. All rights in the program are + reserved by Triad National Security, LLC, and the U.S. Department of Energy/National Nuclear + Security Administration. The Government is granted for itself and others acting on its behalf a + nonexclusive, paid-up, irrevocable worldwide license in this material to reproduce, prepare + derivative works, distribute copies to the public, perform publicly and display publicly, and + to permit others to do so. + This program is open source under the BSD-3 License. + Redistribution and use in source and binary forms, with or without modification, are permitted + provided that the following conditions are met: + 1. Redistributions of source code must retain the above copyright notice, this list of + conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright notice, this list of + conditions and the following disclaimer in the documentation and/or other materials + provided with the distribution. + 3. Neither the name of the copyright holder nor the names of its contributors may be used + to endorse or promote products derived from this software without specific prior + written permission. + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS + IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **********************************************************************************************/ #include "mesh.h" #include "state.h" #include "FEA_Module_Dynamic_Elasticity.h" @@ -7,14 +39,14 @@ // This function saves the variables at rk_stage = 0, which is t_n // ------------------------------------------------------------------------------ void FEA_Module_Dynamic_Elasticity::rk_init(DViewCArrayKokkos& node_coords, - DViewCArrayKokkos& node_vel, - DViewCArrayKokkos& elem_sie, - DViewCArrayKokkos& elem_stress, - const size_t num_elems, - const size_t num_nodes) + DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_sie, + DViewCArrayKokkos& elem_stress, + const size_t num_elems, + const size_t num_nodes) { const size_t rk_level = rk_num_bins - 1; - int num_dims = num_dim; + int num_dims = num_dim; // save elem quantities FOR_ALL_CLASS(elem_gid, 0, num_elems, { // stress is always 3D even with 2D-RZ @@ -47,11 +79,11 @@ void FEA_Module_Dynamic_Elasticity::rk_init(DViewCArrayKokkos& node_coor // between any two nodes in the mesh // ------------------------------------------------------------------------------ // WARNING WARNING : Only works for 3D, 8 node elements -void FEA_Module_Dynamic_Elasticity::get_timestep(mesh_t& mesh, - DViewCArrayKokkos& node_coords, - DViewCArrayKokkos& node_vel, - DViewCArrayKokkos& elem_sspd, - DViewCArrayKokkos& elem_vol) +void FEA_Module_Dynamic_Elasticity::get_timestep(mesh_t& mesh, + DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_sspd, + DViewCArrayKokkos& elem_vol) { const size_t rk_level = rk_num_bins - 1; @@ -148,11 +180,11 @@ void FEA_Module_Dynamic_Elasticity::get_timestep(mesh_t& mesh // between any two nodes in the mesh // ------------------------------------------------------------------------------ // WARNING WARNING : Only works for 3D, 8 node elements -void FEA_Module_Dynamic_Elasticity::get_timestep2D(mesh_t& mesh, - DViewCArrayKokkos& node_coords, - DViewCArrayKokkos& node_vel, - DViewCArrayKokkos& elem_sspd, - DViewCArrayKokkos& elem_vol) +void FEA_Module_Dynamic_Elasticity::get_timestep2D(mesh_t& mesh, + DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_sspd, + DViewCArrayKokkos& elem_vol) { const size_t rk_level = rk_num_bins - 1; diff --git a/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/FEA_Module_Eulerian.cpp b/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/FEA_Module_Eulerian.cpp index ee4b9d012..c7c17409a 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/FEA_Module_Eulerian.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/FEA_Module_Eulerian.cpp @@ -11,14 +11,11 @@ This program is open source under the BSD-3 License. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. @@ -146,7 +143,7 @@ FEA_Module_Eulerian::FEA_Module_Eulerian(Solver* Solver_Pointer, mesh_t& mesh, c if (simparam_dynamic_opt.topology_optimization_on || simparam_dynamic_opt.shape_optimization_on || simparam->num_dims == 2) { - node_masses_distributed = Teuchos::rcp(new MV(map, 1)); + node_masses_distributed = Teuchos::rcp(new MV(map, 1)); ghost_node_masses_distributed = Teuchos::rcp(new MV(ghost_node_map, 1)); adjoint_vector_distributed = Teuchos::rcp(new MV(map, simparam->num_dims)); phi_adjoint_vector_distributed = Teuchos::rcp(new MV(map, simparam->num_dims)); @@ -207,14 +204,14 @@ FEA_Module_Eulerian::~FEA_Module_Eulerian() ------------------------------------------------------------------------- */ void FEA_Module_Eulerian::read_conditions_ansys_dat(std::ifstream* in, std::streampos before_condition_header) { - char ch; - std::string skip_line, read_line, substring, token; + char ch; + std::string skip_line, read_line, substring, token; std::stringstream line_parse, line_parse2; - int num_dim = simparam->num_dims; + int num_dim = simparam->num_dims; int buffer_lines = 1000; int max_word = 30; - int p_order = simparam->p_order; + int p_order = simparam->p_order; int local_node_index, current_column_index; int buffer_loop, buffer_iteration, buffer_iterations, scan_loop, nodes_per_element, words_per_line; @@ -223,12 +220,12 @@ void FEA_Module_Eulerian::read_conditions_ansys_dat(std::ifstream* in, std::stre size_t strain_count; size_t read_index_start, node_rid, elem_gid; - CArrayKokkos read_buffer; + CArrayKokkos read_buffer; CArrayKokkos read_buffer_indices; - LO local_dof_id; - GO node_gid; - real_t dof_value; + LO local_dof_id; + GO node_gid; + real_t dof_value; host_vec_array node_densities; } // end read_conditions_ansys_dat @@ -317,9 +314,9 @@ void FEA_Module_Eulerian::grow_boundary_sets(int num_sets) if (num_sets > max_boundary_sets) { // temporary storage for previous data - CArrayKokkos Temp_Boundary_Condition_Type_List = Boundary_Condition_Type_List; - CArrayKokkos Temp_NBoundary_Condition_Patches = NBoundary_Condition_Patches; - CArrayKokkos Temp_Boundary_Condition_Patches = Boundary_Condition_Patches; + CArrayKokkos Temp_Boundary_Condition_Type_List = Boundary_Condition_Type_List; + CArrayKokkos Temp_NBoundary_Condition_Patches = NBoundary_Condition_Patches; + CArrayKokkos Temp_Boundary_Condition_Patches = Boundary_Condition_Patches; max_boundary_sets = num_sets + 5; // 5 is an arbitrary buffer Boundary_Condition_Type_List = CArrayKokkos(max_boundary_sets, "Boundary_Condition_Type_List"); @@ -701,15 +698,15 @@ void FEA_Module_Eulerian::euler_solve() small = tv.small; graphics_times = simparam->graphics_options.graphics_times; graphics_id = simparam->graphics_options.graphics_id; - size_t num_bdy_nodes = mesh.num_bdy_nodes; - const DCArrayKokkos boundary = simparam->boundary; - const DCArrayKokkos material = simparam->material; - int nTO_modules; - int old_max_forward_buffer; - size_t cycle; - const int num_dim = simparam->num_dims; - real_t objective_accumulation, global_objective_accumulation; - std::vector> FEA_Module_My_TO_Modules = simparam_dynamic_opt.FEA_Module_My_TO_Modules; + size_t num_bdy_nodes = mesh.num_bdy_nodes; + const DCArrayKokkos boundary = simparam->boundary; + const DCArrayKokkos material = simparam->material; + int nTO_modules; + int old_max_forward_buffer; + size_t cycle; + const int num_dim = simparam->num_dims; + real_t objective_accumulation, global_objective_accumulation; + std::vector> FEA_Module_My_TO_Modules = simparam_dynamic_opt.FEA_Module_My_TO_Modules; problem = Explicit_Solver_Pointer_->problem; // Pointer to ROL optimization problem object ROL::Ptr> obj_pointer; @@ -730,7 +727,7 @@ void FEA_Module_Eulerian::euler_solve() KineticEnergyMinimize_TopOpt& kinetic_energy_minimize_function = dynamic_cast(*obj_pointer); kinetic_energy_minimize_function.objective_accumulation = 0; global_objective_accumulation = objective_accumulation = 0; - kinetic_energy_objective = true; + kinetic_energy_objective = true; if (max_time_steps + 1 > forward_solve_velocity_data->size()) { old_max_forward_buffer = forward_solve_velocity_data->size(); @@ -905,9 +902,9 @@ void FEA_Module_Eulerian::euler_solve() // view scope { - const_vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); const_vec_array ghost_node_velocities_interface = Explicit_Solver_Pointer_->ghost_node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); - const_vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); const_vec_array ghost_node_coords_interface = Explicit_Solver_Pointer_->ghost_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); @@ -1038,7 +1035,7 @@ void FEA_Module_Eulerian::euler_solve() Kokkos::fence(); double comm_time4 = Explicit_Solver_Pointer_->CPU_Time(); - Explicit_Solver_Pointer_->host2dev_time += comm_time4 - comm_time3; + Explicit_Solver_Pointer_->host2dev_time += comm_time4 - comm_time3; Explicit_Solver_Pointer_->communication_time += comm_time4 - comm_time1; // debug print vector values on a rank /* @@ -1223,12 +1220,12 @@ void FEA_Module_Eulerian::euler_solve() // view scope { - const_vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); const_vec_array ghost_node_velocities_interface = Explicit_Solver_Pointer_->ghost_node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); - vec_array all_node_velocities_interface = Explicit_Solver_Pointer_->all_node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); + vec_array all_node_velocities_interface = Explicit_Solver_Pointer_->all_node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); const_vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); const_vec_array ghost_node_coords_interface = Explicit_Solver_Pointer_->ghost_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); - vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { for (int idim = 0; idim < num_dim; idim++) { @@ -1249,7 +1246,7 @@ void FEA_Module_Eulerian::euler_solve() } // end view scope double comm_time4 = Explicit_Solver_Pointer_->CPU_Time(); - Explicit_Solver_Pointer_->host2dev_time += comm_time4 - comm_time3; + Explicit_Solver_Pointer_->host2dev_time += comm_time4 - comm_time3; Explicit_Solver_Pointer_->communication_time += comm_time4 - comm_time1; (*forward_solve_velocity_data)[cycle + 1]->assign(*Explicit_Solver_Pointer_->all_node_velocities_distributed); diff --git a/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/FEA_Module_Eulerian.h b/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/FEA_Module_Eulerian.h index 2e3692735..788285415 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/FEA_Module_Eulerian.h +++ b/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/FEA_Module_Eulerian.h @@ -11,14 +11,11 @@ This program is open source under the BSD-3 License. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. @@ -101,22 +98,22 @@ class FEA_Module_Eulerian : public FEA_Module DViewCArrayKokkos& node_vel, DViewCArrayKokkos& elem_sie, DViewCArrayKokkos& elem_stress, - const size_t num_elems, - const size_t num_nodes); + const size_t num_elems, + const size_t num_nodes); - void get_timestep(mesh_t& mesh, + void get_timestep(mesh_t& mesh, DViewCArrayKokkos& node_coords, DViewCArrayKokkos& node_vel, DViewCArrayKokkos& elem_sspd, DViewCArrayKokkos& elem_vol); - void get_timestep2D(mesh_t& mesh, + void get_timestep2D(mesh_t& mesh, DViewCArrayKokkos& node_coords, DViewCArrayKokkos& node_vel, DViewCArrayKokkos& elem_sspd, DViewCArrayKokkos& elem_vol); - void write_outputs(const mesh_t& mesh, + void write_outputs(const mesh_t& mesh, DViewCArrayKokkos& node_coords, DViewCArrayKokkos& node_vel, DViewCArrayKokkos& node_mass, @@ -131,17 +128,17 @@ class FEA_Module_Eulerian : public FEA_Module void node_density_constraints(host_vec_array node_densities_lower_bound); - void example_function(double rk_alpha, - const size_t num_nodes, - DViewCArrayKokkos& node_coords, + void example_function(double rk_alpha, + const size_t num_nodes, + DViewCArrayKokkos& node_coords, const DViewCArrayKokkos& node_vel); KOKKOS_FUNCTION - void example_device_function(const ViewCArrayKokkos& B_matrix, - const size_t elem_gid, + void example_device_function(const ViewCArrayKokkos& B_matrix, + const size_t elem_gid, const DViewCArrayKokkos& node_coords, const ViewCArrayKokkos& elem_node_gids, - const size_t rk_level) const; + const size_t rk_level) const; bool nodal_density_flag; real_t penalty_power; diff --git a/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/Simulation_Parameters_Eulerian.h b/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/Simulation_Parameters_Eulerian.h index 87017b8fa..aef19a467 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/Simulation_Parameters_Eulerian.h +++ b/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/Simulation_Parameters_Eulerian.h @@ -11,14 +11,11 @@ This program is open source under the BSD-3 License. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. diff --git a/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/eulerian_geometry.cpp b/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/eulerian_geometry.cpp index 174ac6b14..025655c73 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/eulerian_geometry.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/eulerian_geometry.cpp @@ -6,13 +6,13 @@ #include "state.h" #include "FEA_Module_Eulerian.h" -void FEA_Module_Eulerian::example_function(double rk_alpha, - const size_t num_nodes, - DViewCArrayKokkos& node_coords, - const DViewCArrayKokkos& node_vel) +void FEA_Module_Eulerian::example_function(double rk_alpha, + const size_t num_nodes, + DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel) { const size_t rk_level = rk_num_bins - 1; - int num_dims = num_dim; + int num_dims = num_dim; // example device loop FOR_ALL_CLASS(node_gid, 0, num_nodes, { @@ -36,10 +36,10 @@ void FEA_Module_Eulerian::example_function(double rk_a // B_p is the OUTWARD corner area normal at node p // ------------------------------------------------------------------------------ KOKKOS_FUNCTION -void FEA_Module_Eulerian::example_device_function(const ViewCArrayKokkos& B_matrix, - const size_t elem_gid, - const DViewCArrayKokkos& node_coords, - const ViewCArrayKokkos& elem_node_gids, - const size_t rk_level) const +void FEA_Module_Eulerian::example_device_function(const ViewCArrayKokkos& B_matrix, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const { } // end subroutine diff --git a/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/eulerian_time_integration.cpp b/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/eulerian_time_integration.cpp index 2e99a4718..29f0624ae 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/eulerian_time_integration.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/eulerian_time_integration.cpp @@ -1,4 +1,36 @@ - +/********************************************************************************************** + 2020. Triad National Security, LLC. All rights reserved. + This program was produced under U.S. Government contract 89233218CNA000001 for Los Alamos + National Laboratory (LANL), which is operated by Triad National Security, LLC for the U.S. + Department of Energy/National Nuclear Security Administration. All rights in the program are + reserved by Triad National Security, LLC, and the U.S. Department of Energy/National Nuclear + Security Administration. The Government is granted for itself and others acting on its behalf a + nonexclusive, paid-up, irrevocable worldwide license in this material to reproduce, prepare + derivative works, distribute copies to the public, perform publicly and display publicly, and + to permit others to do so. + This program is open source under the BSD-3 License. + Redistribution and use in source and binary forms, with or without modification, are permitted + provided that the following conditions are met: + 1. Redistributions of source code must retain the above copyright notice, this list of + conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright notice, this list of + conditions and the following disclaimer in the documentation and/or other materials + provided with the distribution. + 3. Neither the name of the copyright holder nor the names of its contributors may be used + to endorse or promote products derived from this software without specific prior + written permission. + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS + IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **********************************************************************************************/ #include "mesh.h" #include "state.h" #include "FEA_Module_Eulerian.h" @@ -7,14 +39,14 @@ // This function saves the variables at rk_stage = 0, which is t_n // ------------------------------------------------------------------------------ void FEA_Module_Eulerian::rk_init(DViewCArrayKokkos& node_coords, - DViewCArrayKokkos& node_vel, - DViewCArrayKokkos& elem_sie, - DViewCArrayKokkos& elem_stress, - const size_t num_elems, - const size_t num_nodes) + DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_sie, + DViewCArrayKokkos& elem_stress, + const size_t num_elems, + const size_t num_nodes) { const size_t rk_level = rk_num_bins - 1; - int num_dims = num_dim; + int num_dims = num_dim; // save elem quantities FOR_ALL_CLASS(elem_gid, 0, num_elems, { // stress is always 3D even with 2D-RZ @@ -47,11 +79,11 @@ void FEA_Module_Eulerian::rk_init(DViewCArrayKokkos& node_coords, // between any two nodes in the mesh // ------------------------------------------------------------------------------ // WARNING WARNING : Only works for 3D, 8 node elements -void FEA_Module_Eulerian::get_timestep(mesh_t& mesh, - DViewCArrayKokkos& node_coords, - DViewCArrayKokkos& node_vel, - DViewCArrayKokkos& elem_sspd, - DViewCArrayKokkos& elem_vol) +void FEA_Module_Eulerian::get_timestep(mesh_t& mesh, + DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_sspd, + DViewCArrayKokkos& elem_vol) { ->const size_t rk_level = rk_num_bins - 1; @@ -148,11 +180,11 @@ void FEA_Module_Eulerian::get_timestep(mesh_t& mesh, // between any two nodes in the mesh // ------------------------------------------------------------------------------ // WARNING WARNING : Only works for 3D, 8 node elements -void FEA_Module_Eulerian::get_timestep2D(mesh_t& mesh, - DViewCArrayKokkos& node_coords, - DViewCArrayKokkos& node_vel, - DViewCArrayKokkos& elem_sspd, - DViewCArrayKokkos& elem_vol) +void FEA_Module_Eulerian::get_timestep2D(mesh_t& mesh, + DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_sspd, + DViewCArrayKokkos& elem_vol) { ->const size_t rk_level = rk_num_bins - 1; diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/FEA_Module_SGH.h b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/FEA_Module_SGH.h index a9cc158bd..db9c1e3e2 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/FEA_Module_SGH.h +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/FEA_Module_SGH.h @@ -1,5 +1,5 @@ /********************************************************************************************** - © 2020. Triad National Security, LLC. All rights reserved. + 2020. Triad National Security, LLC. All rights reserved. This program was produced under U.S. Government contract 89233218CNA000001 for Los Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC for the U.S. Department of Energy/National Nuclear Security Administration. All rights in the program are @@ -11,14 +11,11 @@ This program is open source under the BSD-3 License. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. @@ -34,7 +31,6 @@ OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. **********************************************************************************************/ - #ifndef FEA_MODULE_SGH_H #define FEA_MODULE_SGH_H @@ -69,7 +65,7 @@ class FEA_Module_SGH : public FEA_Module void cleanup_material_models(); - int solve(); + int solve(); void module_cleanup(); @@ -99,23 +95,23 @@ class FEA_Module_SGH : public FEA_Module * */ void get_force_sgh(const DCArrayKokkos& material, - const mesh_t& mesh, + const mesh_t& mesh, const DViewCArrayKokkos& node_coords, const DViewCArrayKokkos& node_vel, const DViewCArrayKokkos& elem_den, const DViewCArrayKokkos& elem_sie, const DViewCArrayKokkos& elem_pres, - DViewCArrayKokkos& elem_stress, + DViewCArrayKokkos& elem_stress, const DViewCArrayKokkos& elem_sspd, const DViewCArrayKokkos& elem_vol, const DViewCArrayKokkos& elem_div, const DViewCArrayKokkos& elem_mat_id, - DViewCArrayKokkos& corner_force, - const double rk_alpha, - const size_t cycle); + DViewCArrayKokkos& corner_force, + const double rk_alpha, + const size_t cycle); void get_force_vgradient_sgh(const DCArrayKokkos& material, - const mesh_t& mesh, + const mesh_t& mesh, const DViewCArrayKokkos& node_coords, const DViewCArrayKokkos& node_vel, const DViewCArrayKokkos& elem_den, @@ -126,11 +122,11 @@ class FEA_Module_SGH : public FEA_Module const DViewCArrayKokkos& elem_vol, const DViewCArrayKokkos& elem_div, const DViewCArrayKokkos& elem_mat_id, - const double rk_alpha, - const size_t cycle); + const double rk_alpha, + const size_t cycle); void get_force_ugradient_sgh(const DCArrayKokkos& material, - const mesh_t& mesh, + const mesh_t& mesh, const DViewCArrayKokkos& node_coords, const DViewCArrayKokkos& node_vel, const DViewCArrayKokkos& elem_den, @@ -141,11 +137,11 @@ class FEA_Module_SGH : public FEA_Module const DViewCArrayKokkos& elem_vol, const DViewCArrayKokkos& elem_div, const DViewCArrayKokkos& elem_mat_id, - const double rk_alpha, - const size_t cycle); + const double rk_alpha, + const size_t cycle); void get_force_egradient_sgh(const DCArrayKokkos& material, - const mesh_t& mesh, + const mesh_t& mesh, const DViewCArrayKokkos& node_coords, const DViewCArrayKokkos& node_vel, const DViewCArrayKokkos& elem_den, @@ -156,11 +152,11 @@ class FEA_Module_SGH : public FEA_Module const DViewCArrayKokkos& elem_vol, const DViewCArrayKokkos& elem_div, const DViewCArrayKokkos& elem_mat_id, - const double rk_alpha, - const size_t cycle); + const double rk_alpha, + const size_t cycle); void get_force_dgradient_sgh(const DCArrayKokkos& material, - const mesh_t& mesh, + const mesh_t& mesh, const DViewCArrayKokkos& node_coords, const DViewCArrayKokkos& node_vel, const DViewCArrayKokkos& elem_den, @@ -171,13 +167,13 @@ class FEA_Module_SGH : public FEA_Module const DViewCArrayKokkos& elem_vol, const DViewCArrayKokkos& elem_div, const DViewCArrayKokkos& elem_mat_id, - const double rk_alpha, - const size_t cycle); + const double rk_alpha, + const size_t cycle); void force_design_gradient_term(const_vec_array design_variables, vec_array design_gradients); void get_force_sgh2D(const DCArrayKokkos& material, - const mesh_t& mesh, + const mesh_t& mesh, const DViewCArrayKokkos& node_coords, const DViewCArrayKokkos& node_vel, const DViewCArrayKokkos& elem_den, @@ -188,13 +184,13 @@ class FEA_Module_SGH : public FEA_Module const DViewCArrayKokkos& elem_vol, const DViewCArrayKokkos& elem_div, const DViewCArrayKokkos& elem_mat_id, - DViewCArrayKokkos& corner_force, - const double rk_alpha, - const size_t cycle); + DViewCArrayKokkos& corner_force, + const double rk_alpha, + const size_t cycle); - void update_position_sgh(double rk_alpha, - const size_t num_nodes, - DViewCArrayKokkos& node_coords, + void update_position_sgh(double rk_alpha, + const size_t num_nodes, + DViewCArrayKokkos& node_coords, const DViewCArrayKokkos& node_vel); void get_vol(); @@ -205,58 +201,58 @@ class FEA_Module_SGH : public FEA_Module KOKKOS_INLINE_FUNCTION void get_vol_hex(const DViewCArrayKokkos& elem_vol, - const size_t elem_gid, + const size_t elem_gid, const DViewCArrayKokkos& node_coords, const ViewCArrayKokkos& elem_node_gids, - const size_t rk_level) const; + const size_t rk_level) const; KOKKOS_FUNCTION - void get_vol_hex_ugradient(const ViewCArrayKokkos& elem_vol_gradients, - const size_t elem_gid, + void get_vol_hex_ugradient(const ViewCArrayKokkos& elem_vol_gradients, + const size_t elem_gid, const DViewCArrayKokkos& node_coords, const ViewCArrayKokkos& elem_node_gids, - const size_t rk_level) const; + const size_t rk_level) const; KOKKOS_INLINE_FUNCTION void get_vol_quad(const DViewCArrayKokkos& elem_vol, - const size_t elem_gid, + const size_t elem_gid, const DViewCArrayKokkos& node_coords, const ViewCArrayKokkos& elem_node_gids, - const size_t rk_level) const; + const size_t rk_level) const; KOKKOS_FUNCTION - double get_area_quad(const size_t elem_gid, + double get_area_quad(const size_t elem_gid, const DViewCArrayKokkos& node_coords, const ViewCArrayKokkos& elem_node_gids, - const size_t rk_level) const; + const size_t rk_level) const; KOKKOS_FUNCTION - void get_bmatrix(const ViewCArrayKokkos& B_matrix, - const size_t elem_gid, + void get_bmatrix(const ViewCArrayKokkos& B_matrix, + const size_t elem_gid, const DViewCArrayKokkos& node_coords, const ViewCArrayKokkos& elem_node_gids, - const size_t rk_level) const; + const size_t rk_level) const; KOKKOS_FUNCTION - void get_bmatrix_gradients(const ViewCArrayKokkos& B_matrix_gradients, - const size_t elem_gid, + void get_bmatrix_gradients(const ViewCArrayKokkos& B_matrix_gradients, + const size_t elem_gid, const DViewCArrayKokkos& node_coords, const ViewCArrayKokkos& elem_node_gids, - const size_t rk_level) const; + const size_t rk_level) const; KOKKOS_FUNCTION - void get_bmatrix2D(const ViewCArrayKokkos& B_matrix, - const size_t elem_gid, + void get_bmatrix2D(const ViewCArrayKokkos& B_matrix, + const size_t elem_gid, const DViewCArrayKokkos& node_coords, const ViewCArrayKokkos& elem_node_gids, - const size_t rk_level) const; + const size_t rk_level) const; KOKKOS_FUNCTION - void get_area_weights2D(const ViewCArrayKokkos& corner_areas, - const size_t elem_gid, + void get_area_weights2D(const ViewCArrayKokkos& corner_areas, + const size_t elem_gid, const DViewCArrayKokkos& node_coords, const ViewCArrayKokkos& elem_node_gids, - const size_t rk_level) const; + const size_t rk_level) const; KOKKOS_INLINE_FUNCTION double heron(const double x1, @@ -268,158 +264,158 @@ class FEA_Module_SGH : public FEA_Module double average_element_density(const int nodes_per_elem, const CArray current_element_densities) const; - void get_divergence(DViewCArrayKokkos& elem_div, - const DViewCArrayKokkos& node_coords, - const DViewCArrayKokkos& node_vel, - const DViewCArrayKokkos& elem_vol); - - void get_divergence2D(DViewCArrayKokkos& elem_div, + void get_divergence(DViewCArrayKokkos& elem_div, const DViewCArrayKokkos& node_coords, const DViewCArrayKokkos& node_vel, const DViewCArrayKokkos& elem_vol); + void get_divergence2D(DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_vol); + KOKKOS_FUNCTION - void get_velgrad(ViewCArrayKokkos& vel_grad, + void get_velgrad(ViewCArrayKokkos& vel_grad, const ViewCArrayKokkos& elem_node_gids, const DViewCArrayKokkos& node_vel, const ViewCArrayKokkos& b_matrix, - const double elem_vol, - const size_t elem_gid, - const size_t rk_level) const; + const double elem_vol, + const size_t elem_gid, + const size_t rk_level) const; KOKKOS_FUNCTION - void get_velgrad2D(ViewCArrayKokkos& vel_grad, + void get_velgrad2D(ViewCArrayKokkos& vel_grad, const ViewCArrayKokkos& elem_node_gids, const DViewCArrayKokkos& node_vel, const ViewCArrayKokkos& b_matrix, - const double elem_vol, - const double elem_area, - const size_t elem_gid, - const size_t rk_level) const; + const double elem_vol, + const double elem_area, + const size_t elem_gid, + const size_t rk_level) const; KOKKOS_INLINE_FUNCTION - void decompose_vel_grad(ViewCArrayKokkos& D_tensor, - ViewCArrayKokkos& W_tensor, - const ViewCArrayKokkos& vel_grad, - const ViewCArrayKokkos& elem_node_gids, - const size_t elem_gid, + void decompose_vel_grad(ViewCArrayKokkos& D_tensor, + ViewCArrayKokkos& W_tensor, + const ViewCArrayKokkos& vel_grad, + const ViewCArrayKokkos& elem_node_gids, + const size_t elem_gid, const DViewCArrayKokkos& node_coords, const DViewCArrayKokkos& node_vel, - const double vol) const; + const double vol) const; - void update_velocity_sgh(double rk_alpha, - DViewCArrayKokkos& node_vel, + void update_velocity_sgh(double rk_alpha, + DViewCArrayKokkos& node_vel, const DViewCArrayKokkos& node_mass, const DViewCArrayKokkos& corner_force); void tag_bdys(const DCArrayKokkos& boundary, - mesh_t& mesh, + mesh_t& mesh, const DViewCArrayKokkos& node_coords); - void boundary_velocity(const mesh_t& mesh, + void boundary_velocity(const mesh_t& mesh, const DCArrayKokkos& boundary, - DViewCArrayKokkos& node_vel); + DViewCArrayKokkos& node_vel); KOKKOS_INLINE_FUNCTION - bool check_bdy(const size_t patch_gid, - const int num_dim, - const int num_nodes_in_patch, - const BOUNDARY_TYPE this_bc_tag, - const double val, + bool check_bdy(const size_t patch_gid, + const int num_dim, + const int num_nodes_in_patch, + const BOUNDARY_TYPE this_bc_tag, + const double val, const DViewCArrayKokkos& node_coords, - const size_t rk_level) const; + const size_t rk_level) const; void rk_init(DViewCArrayKokkos& node_coords, DViewCArrayKokkos& node_vel, DViewCArrayKokkos& elem_sie, DViewCArrayKokkos& elem_stress, - const size_t num_elems, - const size_t num_nodes); + const size_t num_elems, + const size_t num_nodes); - void get_timestep(mesh_t& mesh, + void get_timestep(mesh_t& mesh, DViewCArrayKokkos& node_coords, DViewCArrayKokkos& node_vel, DViewCArrayKokkos& elem_sspd, DViewCArrayKokkos& elem_vol); - void get_timestep2D(mesh_t& mesh, + void get_timestep2D(mesh_t& mesh, DViewCArrayKokkos& node_coords, DViewCArrayKokkos& node_vel, DViewCArrayKokkos& elem_sspd, DViewCArrayKokkos& elem_vol); - void update_energy_sgh(double rk_alpha, - const mesh_t& mesh, + void update_energy_sgh(double rk_alpha, + const mesh_t& mesh, const DViewCArrayKokkos& node_vel, const DViewCArrayKokkos& node_coords, - DViewCArrayKokkos& elem_sie, + DViewCArrayKokkos& elem_sie, const DViewCArrayKokkos& elem_mass, const DViewCArrayKokkos& corner_force); void power_design_gradient_term(const_vec_array design_variables, vec_array design_gradients); - void get_power_dgradient_sgh(double rk_alpha, - const mesh_t& mesh, + void get_power_dgradient_sgh(double rk_alpha, + const mesh_t& mesh, const DViewCArrayKokkos& node_vel, const DViewCArrayKokkos& node_coords, - DViewCArrayKokkos& elem_sie, + DViewCArrayKokkos& elem_sie, const DViewCArrayKokkos& elem_mass, const DViewCArrayKokkos& corner_force, - DCArrayKokkos elem_power_dgradients); + DCArrayKokkos elem_power_dgradients); - void get_power_ugradient_sgh(double rk_alpha, - const mesh_t& mesh, + void get_power_ugradient_sgh(double rk_alpha, + const mesh_t& mesh, const DViewCArrayKokkos& node_vel, const DViewCArrayKokkos& node_coords, - DViewCArrayKokkos& elem_sie, + DViewCArrayKokkos& elem_sie, const DViewCArrayKokkos& elem_mass, const DViewCArrayKokkos& corner_force); - void get_power_vgradient_sgh(double rk_alpha, - const mesh_t& mesh, + void get_power_vgradient_sgh(double rk_alpha, + const mesh_t& mesh, const DViewCArrayKokkos& node_vel, const DViewCArrayKokkos& node_coords, - DViewCArrayKokkos& elem_sie, + DViewCArrayKokkos& elem_sie, const DViewCArrayKokkos& elem_mass, const DViewCArrayKokkos& corner_force); - void get_power_egradient_sgh(double rk_alpha, - const mesh_t& mesh, + void get_power_egradient_sgh(double rk_alpha, + const mesh_t& mesh, const DViewCArrayKokkos& node_vel, const DViewCArrayKokkos& node_coords, - DViewCArrayKokkos& elem_sie, + DViewCArrayKokkos& elem_sie, const DViewCArrayKokkos& elem_mass, const DViewCArrayKokkos& corner_force); void update_state(const DCArrayKokkos& material, - const mesh_t& mesh, + const mesh_t& mesh, const DViewCArrayKokkos& node_coords, const DViewCArrayKokkos& node_vel, - DViewCArrayKokkos& elem_den, - DViewCArrayKokkos& elem_pres, - DViewCArrayKokkos& elem_stress, - DViewCArrayKokkos& elem_sspd, + DViewCArrayKokkos& elem_den, + DViewCArrayKokkos& elem_pres, + DViewCArrayKokkos& elem_stress, + DViewCArrayKokkos& elem_sspd, const DViewCArrayKokkos& elem_sie, const DViewCArrayKokkos& elem_vol, const DViewCArrayKokkos& elem_mass, const DViewCArrayKokkos& elem_mat_id, - const double rk_alpha, - const size_t cycle); + const double rk_alpha, + const size_t cycle); void update_state2D(const DCArrayKokkos& material, - const mesh_t& mesh, + const mesh_t& mesh, const DViewCArrayKokkos& node_coords, const DViewCArrayKokkos& node_vel, - DViewCArrayKokkos& elem_den, - DViewCArrayKokkos& elem_pres, - DViewCArrayKokkos& elem_stress, - DViewCArrayKokkos& elem_sspd, + DViewCArrayKokkos& elem_den, + DViewCArrayKokkos& elem_pres, + DViewCArrayKokkos& elem_stress, + DViewCArrayKokkos& elem_sspd, const DViewCArrayKokkos& elem_sie, const DViewCArrayKokkos& elem_vol, const DViewCArrayKokkos& elem_mass, const DViewCArrayKokkos& elem_mat_id, - const double rk_alpha, - const size_t cycle); + const double rk_alpha, + const size_t cycle); void build_boundry_node_sets(mesh_t& mesh); @@ -459,13 +455,13 @@ class FEA_Module_SGH : public FEA_Module void collect_output(Teuchos::RCP> global_reduce_map); - void write_data(std::map& point_data_scalars_double, - std::map& point_data_vectors_double, - std::map& cell_data_scalars_double, - std::map& cell_data_scalars_int, + void write_data(std::map& point_data_scalars_double, + std::map& point_data_vectors_double, + std::map& cell_data_scalars_double, + std::map& cell_data_scalars_int, std::map>& cell_data_fields_double); - void write_outputs(const mesh_t& mesh, + void write_outputs(const mesh_t& mesh, DViewCArrayKokkos& node_coords, DViewCArrayKokkos& node_vel, DViewCArrayKokkos& node_mass, @@ -478,7 +474,7 @@ class FEA_Module_SGH : public FEA_Module DViewCArrayKokkos& elem_mass, DViewCArrayKokkos& elem_mat_id); - void ensight(const mesh_t& mesh, + void ensight(const mesh_t& mesh, const DViewCArrayKokkos& node_coords, const DViewCArrayKokkos& node_vel, const DViewCArrayKokkos& node_mass, @@ -491,7 +487,7 @@ class FEA_Module_SGH : public FEA_Module const DViewCArrayKokkos& elem_mass, const DViewCArrayKokkos& elem_mat_id); - void state_file(const mesh_t& mesh, + void state_file(const mesh_t& mesh, const DViewCArrayKokkos& node_coords, const DViewCArrayKokkos& node_vel, const DViewCArrayKokkos& node_mass, @@ -510,14 +506,14 @@ class FEA_Module_SGH : public FEA_Module void compute_topology_optimization_gradient_full(Teuchos::RCP design_densities_distributed, Teuchos::RCP design_gradients_distributed); - void boundary_adjoint(const mesh_t& mesh, + void boundary_adjoint(const mesh_t& mesh, const DCArrayKokkos& boundary, - vec_array& node_adjoint, - vec_array& node_phi_adjoint, - vec_array& node_psi_adjoint); + vec_array& node_adjoint, + vec_array& node_phi_adjoint, + vec_array& node_psi_adjoint); void applied_forces(const DCArrayKokkos& material, - const mesh_t& mesh, + const mesh_t& mesh, const DViewCArrayKokkos& node_coords, const DViewCArrayKokkos& node_vel, const DViewCArrayKokkos& node_mass, @@ -525,9 +521,9 @@ class FEA_Module_SGH : public FEA_Module const DViewCArrayKokkos& elem_vol, const DViewCArrayKokkos& elem_div, const DViewCArrayKokkos& elem_mat_id, - DViewCArrayKokkos& corner_force, - const double rk_alpha, - const size_t cycle); + DViewCArrayKokkos& corner_force, + const double rk_alpha, + const size_t cycle); bool have_loading_conditions; bool nodal_density_flag; @@ -614,14 +610,16 @@ class FEA_Module_SGH : public FEA_Module CArrayKokkos Force_Gradient_Energies; // transposed such that elem ids correspond to rows RaggedRightArrayKokkos Power_Gradient_Positions; // transposed such that node dofs correspond to rows RaggedRightArrayKokkos Power_Gradient_Velocities; // transposed such that node dofs correspond to rows - CArrayKokkos Power_Gradient_Energies; - DCArrayKokkos Gradient_Matrix_Strides; - DCArrayKokkos DOF_to_Elem_Matrix_Strides; - DCArrayKokkos Elem_to_Elem_Matrix_Strides; - DCArrayKokkos Graph_Matrix_Strides; + + CArrayKokkos Power_Gradient_Energies; + DCArrayKokkos Gradient_Matrix_Strides; + DCArrayKokkos DOF_to_Elem_Matrix_Strides; + DCArrayKokkos Elem_to_Elem_Matrix_Strides; + DCArrayKokkos Graph_Matrix_Strides; + RaggedRightArrayKokkos Original_Gradient_Entries; RaggedRightArrayKokkos Original_Gradient_Entry_Indices; - DCArrayKokkos Original_Gradient_Entries_Strides; + DCArrayKokkos Original_Gradient_Entries_Strides; // distributed matrices Teuchos::RCP distributed_force_gradient_positions; diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/mesh.h b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/mesh.h index 40e9417fc..d28d54457 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/mesh.h +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/mesh.h @@ -1,3 +1,36 @@ +/********************************************************************************************** + 2020. Triad National Security, LLC. All rights reserved. + This program was produced under U.S. Government contract 89233218CNA000001 for Los Alamos + National Laboratory (LANL), which is operated by Triad National Security, LLC for the U.S. + Department of Energy/National Nuclear Security Administration. All rights in the program are + reserved by Triad National Security, LLC, and the U.S. Department of Energy/National Nuclear + Security Administration. The Government is granted for itself and others acting on its behalf a + nonexclusive, paid-up, irrevocable worldwide license in this material to reproduce, prepare + derivative works, distribute copies to the public, perform publicly and display publicly, and + to permit others to do so. + This program is open source under the BSD-3 License. + Redistribution and use in source and binary forms, with or without modification, are permitted + provided that the following conditions are met: + 1. Redistributions of source code must retain the above copyright notice, this list of + conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright notice, this list of + conditions and the following disclaimer in the documentation and/or other materials + provided with the distribution. + 3. Neither the name of the copyright holder nor the names of its contributors may be used + to endorse or promote products derived from this software without specific prior + written permission. + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS + IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **********************************************************************************************/ #ifndef MESH_H #define MESH_H @@ -218,7 +251,7 @@ struct mesh_t size_t corner_gid = node_lid + elem_gid * num_nodes_in_elem; corners_in_node(node_gid, j) = corner_gid; - elems_in_node(node_gid, j) = elem_gid; // save the elem_gid + elems_in_node(node_gid, j) = elem_gid; // save the elem_gid // Save corner index to element size_t corner_lid = node_lid; @@ -481,7 +514,7 @@ struct mesh_t // neighboring_side_lids(elem_gid, patch_lid) = patch_lid; patches_in_elem(elem_gid, patch_lid) = patch_gid; - temp_bdy_patches(bdy_patch_gid) = patch_gid; + temp_bdy_patches(bdy_patch_gid) = patch_gid; patch_gid++; bdy_patch_gid++; @@ -552,7 +585,7 @@ struct mesh_t }); // end FOR_ALL bdy_patch_gid // find and store the boundary nodes - CArrayKokkos temp_bdy_nodes(num_nodes, "temp_bdy_nodes"); + CArrayKokkos temp_bdy_nodes(num_nodes, "temp_bdy_nodes"); CArrayKokkos hash_bdy_nodes(num_nodes, "hash_bdy_nodes"); FOR_ALL_CLASS(node_gid, 0, num_nodes, { @@ -752,7 +785,7 @@ struct mesh_t printf("ERROR: number of boundary sets = 0, set it = 1"); num_bcs = 1; } - num_bdy_sets = num_bcs; + num_bdy_sets = num_bcs; bdy_patches_in_set = DynamicRaggedRightArrayKokkos(num_bcs, num_bdy_patches, "bdy_patches_in_set"); return; diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/state.h b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/state.h index 8200848b6..0150235ac 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/state.h +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/state.h @@ -1,3 +1,36 @@ +/********************************************************************************************** + 2020. Triad National Security, LLC. All rights reserved. + This program was produced under U.S. Government contract 89233218CNA000001 for Los Alamos + National Laboratory (LANL), which is operated by Triad National Security, LLC for the U.S. + Department of Energy/National Nuclear Security Administration. All rights in the program are + reserved by Triad National Security, LLC, and the U.S. Department of Energy/National Nuclear + Security Administration. The Government is granted for itself and others acting on its behalf a + nonexclusive, paid-up, irrevocable worldwide license in this material to reproduce, prepare + derivative works, distribute copies to the public, perform publicly and display publicly, and + to permit others to do so. + This program is open source under the BSD-3 License. + Redistribution and use in source and binary forms, with or without modification, are permitted + provided that the following conditions are met: + 1. Redistributions of source code must retain the above copyright notice, this list of + conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright notice, this list of + conditions and the following disclaimer in the documentation and/or other materials + provided with the distribution. + 3. Neither the name of the copyright holder nor the names of its contributors may be used + to endorse or promote products derived from this software without specific prior + written permission. + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS + IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **********************************************************************************************/ #ifndef STATE_H #define STATE_H diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/FEA_Module_SGH.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/FEA_Module_SGH.cpp index d06ceae29..b82b243b5 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/FEA_Module_SGH.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/FEA_Module_SGH.cpp @@ -92,7 +92,7 @@ FEA_Module_SGH::FEA_Module_SGH( // recast solver pointer for non-base class access Explicit_Solver_Pointer_ = dynamic_cast(Solver_Pointer); - simparam = &(Explicit_Solver_Pointer_->simparam); + simparam = &(Explicit_Solver_Pointer_->simparam); module_params = ¶ms; // create ref element object @@ -133,7 +133,7 @@ FEA_Module_SGH::FEA_Module_SGH( if (simparam->topology_optimization_on || simparam->shape_optimization_on || simparam->num_dims == 2) { - node_masses_distributed = Teuchos::rcp(new MV(map, 1)); + node_masses_distributed = Teuchos::rcp(new MV(map, 1)); ghost_node_masses_distributed = Teuchos::rcp(new MV(ghost_node_map, 1)); } @@ -169,8 +169,8 @@ FEA_Module_SGH::FEA_Module_SGH( max_time_steps = BUFFER_GROW; time_data.resize(max_time_steps + 1); element_internal_energy_distributed = Teuchos::rcp(new MV(all_element_map, 1)); - forward_solve_velocity_data = Teuchos::rcp(new std::vector>(max_time_steps + 1)); - forward_solve_coordinate_data = Teuchos::rcp(new std::vector>(max_time_steps + 1)); + forward_solve_velocity_data = Teuchos::rcp(new std::vector>(max_time_steps + 1)); + forward_solve_coordinate_data = Teuchos::rcp(new std::vector>(max_time_steps + 1)); forward_solve_internal_energy_data = Teuchos::rcp(new std::vector>(max_time_steps + 1)); adjoint_vector_data = Teuchos::rcp(new std::vector>(max_time_steps + 1)); phi_adjoint_vector_data = Teuchos::rcp(new std::vector>(max_time_steps + 1)); @@ -179,8 +179,8 @@ FEA_Module_SGH::FEA_Module_SGH( // assign a multivector of corresponding size to each new timestep in the buffer for (int istep = 0; istep < max_time_steps + 1; istep++) { - (*forward_solve_velocity_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - (*forward_solve_coordinate_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*forward_solve_velocity_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*forward_solve_coordinate_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); (*forward_solve_internal_energy_data)[istep] = Teuchos::rcp(new MV(all_element_map, 1)); (*adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); (*phi_adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); @@ -203,11 +203,11 @@ void FEA_Module_SGH::read_conditions_ansys_dat(std::ifstream* in, std::streampos { auto input_options = simparam->input_options.value(); - char ch; - std::string skip_line, read_line, substring, token; + char ch; + std::string skip_line, read_line, substring, token; std::stringstream line_parse, line_parse2; - int num_dim = simparam->num_dims; + int num_dim = simparam->num_dims; int buffer_lines = 1000; int max_word = 30; int local_node_index, current_column_index; @@ -220,7 +220,7 @@ void FEA_Module_SGH::read_conditions_ansys_dat(std::ifstream* in, std::streampos real_t unit_scaling = input_options.unit_scaling; real_t dof_value; - CArrayKokkos read_buffer; + CArrayKokkos read_buffer; CArrayKokkos read_buffer_indices; LO local_dof_id; @@ -372,11 +372,11 @@ void FEA_Module_SGH::sort_output(Teuchos::RCP> so populate requests this module makes for output data ---------------------------------------------------------------------------------------------- */ -void FEA_Module_SGH::write_data(std::map& point_data_scalars_double, - std::map& point_data_vectors_double, - std::map& cell_data_scalars_double, - std::map& cell_data_scalars_int, - std::map>& cell_data_fields_double) +void FEA_Module_SGH::write_data(std::map& point_data_scalars_double, + std::map& point_data_vectors_double, + std::map& cell_data_scalars_double, + std::map& cell_data_scalars_int, + std::map>& cell_data_fields_double) { const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; @@ -503,7 +503,7 @@ void FEA_Module_SGH::comm_node_masses() { // debug print of design vector #ifdef DEBUG - std::ostream& out = std::cout; + std::ostream& out = std::cout; Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); if (myrank == 0) { @@ -513,7 +513,7 @@ void FEA_Module_SGH::comm_node_masses() *fos << std::endl; std::fflush(stdout); communicate design densities - create import object using local node indices map and all indices map + create import object using local node indices map and all indices map Tpetra::Import importer(map, ghost_node_map); #endif @@ -545,7 +545,7 @@ void FEA_Module_SGH::comm_variables(Teuchos::RCP zp) test_node_densities_distributed = zp; #ifdef DEBUG // debug print of design vector - std::ostream& out = std::cout; + std::ostream& out = std::cout; Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); if (myrank == 0) { @@ -653,13 +653,13 @@ void FEA_Module_SGH::cleanup_material_models() * Modifies: bdy_patches_in_set */ void FEA_Module_SGH::tag_bdys(const DCArrayKokkos& boundary, - mesh_t& mesh, - const DViewCArrayKokkos& node_coords) + mesh_t& mesh, + const DViewCArrayKokkos& node_coords) { const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - size_t num_dim = simparam->num_dims; - int nboundary_patches = Explicit_Solver_Pointer_->nboundary_patches; - int num_nodes_in_patch = mesh.num_nodes_in_patch; + size_t num_dim = simparam->num_dims; + int nboundary_patches = Explicit_Solver_Pointer_->nboundary_patches; + int num_nodes_in_patch = mesh.num_nodes_in_patch; #ifdef DEBUG if (bdy_set == mesh.num_bdy_sets) { @@ -730,13 +730,13 @@ void FEA_Module_SGH::tag_bdys(const DCArrayKokkos& boundary, } // end tag KOKKOS_INLINE_FUNCTION -bool FEA_Module_SGH::check_bdy(const size_t patch_gid, - const int num_dim, - const int num_nodes_in_patch, - const BOUNDARY_TYPE bc_type, - const double val, - const DViewCArrayKokkos& node_coords, - const size_t rk_level) const +bool FEA_Module_SGH::check_bdy(const size_t patch_gid, + const int num_dim, + const int num_nodes_in_patch, + const BOUNDARY_TYPE bc_type, + const double val, + const DViewCArrayKokkos& node_coords, + const size_t rk_level) const { // default bool is not on the boundary size_t is_on_bdy = 0; @@ -838,10 +838,10 @@ void FEA_Module_SGH::sgh_solve() graphics_time = simparam->output_options.graphics_step; graphics_dt_ival = simparam->output_options.graphics_step; - cycle_stop = dynamic_options.cycle_stop; - rk_num_stages = dynamic_options.rk_num_stages; - graphics_times = simparam->output_options.graphics_times; - graphics_id = simparam->output_options.graphics_id; + cycle_stop = dynamic_options.cycle_stop; + rk_num_stages = dynamic_options.rk_num_stages; + graphics_times = simparam->output_options.graphics_times; + graphics_id = simparam->output_options.graphics_id; fuzz = dynamic_options.fuzz; tiny = dynamic_options.tiny; @@ -875,7 +875,7 @@ void FEA_Module_SGH::sgh_solve() KineticEnergyMinimize_TopOpt& kinetic_energy_minimize_function = dynamic_cast(*obj_pointer); kinetic_energy_minimize_function.objective_accumulation = 0; global_objective_accumulation = objective_accumulation = 0; - kinetic_energy_objective = true; + kinetic_energy_objective = true; if (max_time_steps + 1 > forward_solve_velocity_data->size()) { old_max_forward_buffer = forward_solve_velocity_data->size(); @@ -889,8 +889,8 @@ void FEA_Module_SGH::sgh_solve() // assign a multivector of corresponding size to each new timestep in the buffer for (int istep = old_max_forward_buffer; istep < max_time_steps + 1; istep++) { - (*forward_solve_velocity_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - (*forward_solve_coordinate_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*forward_solve_velocity_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*forward_solve_coordinate_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); (*forward_solve_internal_energy_data)[istep] = Teuchos::rcp(new MV(all_element_map, 1)); (*adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); (*phi_adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); @@ -1036,13 +1036,13 @@ void FEA_Module_SGH::sgh_solve() // view scope { - const_vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); const_vec_array ghost_node_velocities_interface = Explicit_Solver_Pointer_->ghost_node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); - vec_array all_node_velocities_interface = Explicit_Solver_Pointer_->all_node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); - vec_array element_internal_energy = element_internal_energy_distributed->getLocalView(Tpetra::Access::ReadWrite); - const_vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + vec_array all_node_velocities_interface = Explicit_Solver_Pointer_->all_node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); + vec_array element_internal_energy = element_internal_energy_distributed->getLocalView(Tpetra::Access::ReadWrite); + const_vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); const_vec_array ghost_node_coords_interface = Explicit_Solver_Pointer_->ghost_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); - vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { for (int idim = 0; idim < num_dim; idim++) { @@ -1296,9 +1296,9 @@ void FEA_Module_SGH::sgh_solve() Kokkos::fence(); double comm_time4 = Explicit_Solver_Pointer_->CPU_Time(); - Explicit_Solver_Pointer_->host2dev_time += comm_time4 - comm_time3; + Explicit_Solver_Pointer_->host2dev_time += comm_time4 - comm_time3; Explicit_Solver_Pointer_->communication_time += comm_time4 - comm_time1; - + #ifdef DEBUG // debug print vector values on a rank if (myrank == 0) @@ -1502,8 +1502,8 @@ void FEA_Module_SGH::sgh_solve() // assign a multivector of corresponding size to each new timestep in the buffer for (int istep = old_max_forward_buffer; istep < max_time_steps + BUFFER_GROW + 1; istep++) { - (*forward_solve_velocity_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); - (*forward_solve_coordinate_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*forward_solve_velocity_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); + (*forward_solve_coordinate_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); (*forward_solve_internal_energy_data)[istep] = Teuchos::rcp(new MV(all_element_map, 1)); (*adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); (*phi_adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); @@ -1524,7 +1524,7 @@ void FEA_Module_SGH::sgh_solve() node_velocities_interface(node_gid, idim) = node_vel(rk_level, node_gid, idim); node_coords_interface(node_gid, idim) = node_coords(rk_level, node_gid, idim); } - }); + }); } // end view scope Kokkos::fence(); @@ -1547,20 +1547,20 @@ void FEA_Module_SGH::sgh_solve() // view scope { - const_vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); const_vec_array ghost_node_velocities_interface = Explicit_Solver_Pointer_->ghost_node_velocities_distributed->getLocalView(Tpetra::Access::ReadOnly); - vec_array all_node_velocities_interface = Explicit_Solver_Pointer_->all_node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); - vec_array element_internal_energy = element_internal_energy_distributed->getLocalView(Tpetra::Access::ReadWrite); - const_vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + vec_array all_node_velocities_interface = Explicit_Solver_Pointer_->all_node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); + vec_array element_internal_energy = element_internal_energy_distributed->getLocalView(Tpetra::Access::ReadWrite); + const_vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); const_vec_array ghost_node_coords_interface = Explicit_Solver_Pointer_->ghost_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); - vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { for (int idim = 0; idim < num_dim; idim++) { all_node_velocities_interface(node_gid, idim) = node_velocities_interface(node_gid, idim); all_node_coords_interface(node_gid, idim) = node_coords_interface(node_gid, idim); } - }); // end parallel for + }); // end parallel for Kokkos::fence(); FOR_ALL_CLASS(node_gid, nlocal_nodes, nlocal_nodes + nghost_nodes, { @@ -1569,18 +1569,18 @@ void FEA_Module_SGH::sgh_solve() all_node_velocities_interface(node_gid, idim) = ghost_node_velocities_interface(node_gid - nlocal_nodes, idim); all_node_coords_interface(node_gid, idim) = ghost_node_coords_interface(node_gid - nlocal_nodes, idim); } - }); // end parallel for + }); // end parallel for Kokkos::fence(); // interface for element internal energies FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { element_internal_energy(elem_gid, 0) = elem_sie(rk_level, elem_gid); - }); // end parallel for + }); // end parallel for Kokkos::fence(); } // end view scope double comm_time4 = Explicit_Solver_Pointer_->CPU_Time(); - Explicit_Solver_Pointer_->host2dev_time += comm_time4 - comm_time3; + Explicit_Solver_Pointer_->host2dev_time += comm_time4 - comm_time3; Explicit_Solver_Pointer_->communication_time += comm_time4 - comm_time1; (*forward_solve_internal_energy_data)[cycle + 1]->assign(*element_internal_energy_distributed); @@ -1649,7 +1649,7 @@ void FEA_Module_SGH::sgh_solve() { node_coords_interface(node_gid, idim) = node_coords(rk_level, node_gid, idim); } - }); // end parallel for + }); // end parallel for } // end view scope if (simparam->output_options.output_file_format == OUTPUT_FORMAT::vtk) { diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/boundary.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/boundary.cpp index 5996ca246..2107eeb46 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/boundary.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/boundary.cpp @@ -1,19 +1,47 @@ +/********************************************************************************************** + 2020. Triad National Security, LLC. All rights reserved. + This program was produced under U.S. Government contract 89233218CNA000001 for Los Alamos + National Laboratory (LANL), which is operated by Triad National Security, LLC for the U.S. + Department of Energy/National Nuclear Security Administration. All rights in the program are + reserved by Triad National Security, LLC, and the U.S. Department of Energy/National Nuclear + Security Administration. The Government is granted for itself and others acting on its behalf a + nonexclusive, paid-up, irrevocable worldwide license in this material to reproduce, prepare + derivative works, distribute copies to the public, perform publicly and display publicly, and + to permit others to do so. + This program is open source under the BSD-3 License. + Redistribution and use in source and binary forms, with or without modification, are permitted + provided that the following conditions are met: + 1. Redistributions of source code must retain the above copyright notice, this list of + conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright notice, this list of + conditions and the following disclaimer in the documentation and/or other materials + provided with the distribution. + 3. Neither the name of the copyright holder nor the names of its contributors may be used + to endorse or promote products derived from this software without specific prior + written permission. + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS + IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **********************************************************************************************/ #include "mesh.h" #include "state.h" #include "FEA_Module_SGH.h" #include "Simulation_Parameters/FEA_Module/Boundary_Conditions.h" -void FEA_Module_SGH::boundary_velocity(const mesh_t& mesh, - const DCArrayKokkos& boundary, - DViewCArrayKokkos& node_vel) +void FEA_Module_SGH::boundary_velocity(const mesh_t& mesh, + const DCArrayKokkos& boundary, + DViewCArrayKokkos& node_vel) { - // error and debug flag - // DCArrayKokkos print_flag(1, "print_flag"); - // print_flag.host(0) = false; - // print_flag.update_device(); - const size_t rk_level = rk_num_bins - 1; - int num_dims = num_dim; + int num_dims = num_dim; // Loop over boundary sets for (size_t bdy_set = 0; bdy_set < num_bdy_sets; bdy_set++) { @@ -60,8 +88,5 @@ void FEA_Module_SGH::boundary_velocity(const mesh_t& mesh, }); // end for bdy_node_lid } // end for bdy_set - // debug check - // print_flag.update_host(); - // if(print_flag.host(0)) std::cout << "found boundary node with id 549412" << std::endl; return; } // end boundary_velocity function \ No newline at end of file diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/energy_sgh.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/energy_sgh.cpp index 682eb355f..a22d68310 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/energy_sgh.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/energy_sgh.cpp @@ -1,21 +1,54 @@ - +/********************************************************************************************** + 2020. Triad National Security, LLC. All rights reserved. + This program was produced under U.S. Government contract 89233218CNA000001 for Los Alamos + National Laboratory (LANL), which is operated by Triad National Security, LLC for the U.S. + Department of Energy/National Nuclear Security Administration. All rights in the program are + reserved by Triad National Security, LLC, and the U.S. Department of Energy/National Nuclear + Security Administration. The Government is granted for itself and others acting on its behalf a + nonexclusive, paid-up, irrevocable worldwide license in this material to reproduce, prepare + derivative works, distribute copies to the public, perform publicly and display publicly, and + to permit others to do so. + This program is open source under the BSD-3 License. + Redistribution and use in source and binary forms, with or without modification, are permitted + provided that the following conditions are met: + 1. Redistributions of source code must retain the above copyright notice, this list of + conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright notice, this list of + conditions and the following disclaimer in the documentation and/or other materials + provided with the distribution. + 3. Neither the name of the copyright holder nor the names of its contributors may be used + to endorse or promote products derived from this software without specific prior + written permission. + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS + IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **********************************************************************************************/ #include "mesh.h" #include "state.h" #include "FEA_Module_SGH.h" -void FEA_Module_SGH::update_energy_sgh(double rk_alpha, - const mesh_t& mesh, - const DViewCArrayKokkos& node_vel, - const DViewCArrayKokkos& node_coords, - DViewCArrayKokkos& elem_sie, - const DViewCArrayKokkos& elem_mass, - const DViewCArrayKokkos& corner_force) +void FEA_Module_SGH::update_energy_sgh(double rk_alpha, + const mesh_t& mesh, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& corner_force) { const size_t rk_level = rk_num_bins - 1; - int num_dims = num_dim; + int num_dims = num_dim; // loop over all the elements in the mesh - FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, + { double elem_power = 0.0; // --- tally the contribution from each corner to the element --- @@ -41,7 +74,7 @@ void FEA_Module_SGH::update_energy_sgh(double rk_alpha for (size_t dim = 0; dim < num_dims; dim++) { double half_vel = (node_vel(rk_level, node_gid, dim) + node_vel(0, node_gid, dim)) * 0.5; - elem_power += corner_force(corner_gid, dim) * node_radius * half_vel; + elem_power += corner_force(corner_gid, dim) * node_radius * half_vel; } // end for dim } // end for node_lid diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp index 2d5310509..d87a86759 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp @@ -1,4 +1,36 @@ - +/********************************************************************************************** + 2020. Triad National Security, LLC. All rights reserved. + This program was produced under U.S. Government contract 89233218CNA000001 for Los Alamos + National Laboratory (LANL), which is operated by Triad National Security, LLC for the U.S. + Department of Energy/National Nuclear Security Administration. All rights in the program are + reserved by Triad National Security, LLC, and the U.S. Department of Energy/National Nuclear + Security Administration. The Government is granted for itself and others acting on its behalf a + nonexclusive, paid-up, irrevocable worldwide license in this material to reproduce, prepare + derivative works, distribute copies to the public, perform publicly and display publicly, and + to permit others to do so. + This program is open source under the BSD-3 License. + Redistribution and use in source and binary forms, with or without modification, are permitted + provided that the following conditions are met: + 1. Redistributions of source code must retain the above copyright notice, this list of + conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright notice, this list of + conditions and the following disclaimer in the documentation and/or other materials + provided with the distribution. + 3. Neither the name of the copyright holder nor the names of its contributors may be used + to endorse or promote products derived from this software without specific prior + written permission. + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS + IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **********************************************************************************************/ #include "mesh.h" #include "state.h" #include @@ -27,24 +59,24 @@ // current implementation assumes material.q2 = 0 // ------------------------------------------------------------------------------ void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& material, - const mesh_t& mesh, - const DViewCArrayKokkos& node_coords, - const DViewCArrayKokkos& node_vel, - const DViewCArrayKokkos& elem_den, - const DViewCArrayKokkos& elem_sie, - const DViewCArrayKokkos& elem_pres, - const DViewCArrayKokkos& elem_stress, - const DViewCArrayKokkos& elem_sspd, - const DViewCArrayKokkos& elem_vol, - const DViewCArrayKokkos& elem_div, - const DViewCArrayKokkos& elem_mat_id, - const double rk_alpha, - const size_t cycle - ) + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_pres, + const DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + const double rk_alpha, + const size_t cycle + ) { - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - const size_t num_dims = simparam->num_dims; - size_t num_corners = rnum_elem * num_nodes_in_elem; + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + const size_t num_dims = simparam->num_dims; + size_t num_corners = rnum_elem * num_nodes_in_elem; // initialize gradient matrix FOR_ALL_CLASS(dof_gid, 0, nlocal_nodes * num_dims, { @@ -77,7 +109,7 @@ void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& ma // FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { const size_t num_nodes_in_elem = 8; - real_t gradient_result[num_dims]; + real_t gradient_result[num_dims]; /// total Cauchy stress double tau_array[9]; double tau_gradient_array[9]; @@ -466,20 +498,20 @@ void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& ma // This function calculates the corner forces and the evolves stress (hypo) // ------------------------------------------------------------------------------ void FEA_Module_SGH::get_force_egradient_sgh(const DCArrayKokkos& material, - const mesh_t& mesh, - const DViewCArrayKokkos& node_coords, - const DViewCArrayKokkos& node_vel, - const DViewCArrayKokkos& elem_den, - const DViewCArrayKokkos& elem_sie, - const DViewCArrayKokkos& elem_pres, - const DViewCArrayKokkos& elem_stress, - const DViewCArrayKokkos& elem_sspd, - const DViewCArrayKokkos& elem_vol, - const DViewCArrayKokkos& elem_div, - const DViewCArrayKokkos& elem_mat_id, - const double rk_alpha, - const size_t cycle - ) + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_pres, + const DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + const double rk_alpha, + const size_t cycle + ) { const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; const size_t num_dims = simparam->num_dims; @@ -500,7 +532,7 @@ void FEA_Module_SGH::get_force_egradient_sgh(const DCArrayKokkos& ma // FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { const size_t num_nodes_in_elem = 8; - real_t gradient_result[num_dims]; + real_t gradient_result[num_dims]; // total Cauchy stress double tau_array[9]; double tau_gradient_array[9]; @@ -898,20 +930,20 @@ void FEA_Module_SGH::get_force_egradient_sgh(const DCArrayKokkos& ma // This function calculates the corner forces and the evolves stress (hypo) // ------------------------------------------------------------------------------ void FEA_Module_SGH::get_force_ugradient_sgh(const DCArrayKokkos& material, - const mesh_t& mesh, - const DViewCArrayKokkos& node_coords, - const DViewCArrayKokkos& node_vel, - const DViewCArrayKokkos& elem_den, - const DViewCArrayKokkos& elem_sie, - const DViewCArrayKokkos& elem_pres, - const DViewCArrayKokkos& elem_stress, - const DViewCArrayKokkos& elem_sspd, - const DViewCArrayKokkos& elem_vol, - const DViewCArrayKokkos& elem_div, - const DViewCArrayKokkos& elem_mat_id, - const double rk_alpha, - const size_t cycle - ) + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_pres, + const DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + const double rk_alpha, + const size_t cycle + ) { const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; const size_t num_dims = simparam->num_dims; @@ -931,7 +963,7 @@ void FEA_Module_SGH::get_force_ugradient_sgh(const DCArrayKokkos& ma // FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { const size_t num_nodes_in_elem = 8; - real_t gradient_result[num_dims]; + real_t gradient_result[num_dims]; // total Cauchy stress double tau_array[9]; double tau_gradient_array[9 * max_nodes_per_element * num_dims]; @@ -1232,9 +1264,9 @@ void FEA_Module_SGH::get_force_ugradient_sgh(const DCArrayKokkos& ma // on velocity in the numerator. It filters on the shock // direction mu_term = muc(node_lid) * - fabs(shock_dir(0) * area_normal(node_lid, 0) - + shock_dir(1) * area_normal(node_lid, 1) - + shock_dir(2) * area_normal(node_lid, 2) ); + fabs(shock_dir(0) * area_normal(node_lid, 0) + + shock_dir(1) * area_normal(node_lid, 1) + + shock_dir(2) * area_normal(node_lid, 2) ); // muc_gradient(node_lid) = muc_gradient(node_lid)* // fabs( shock_dir(0)*area_normal(node_lid,0) // + shock_dir(1)*area_normal(node_lid,1) @@ -1422,24 +1454,25 @@ void FEA_Module_SGH::get_force_ugradient_sgh(const DCArrayKokkos& ma column_index = num_dims * Global_Gradient_Matrix_Assembly_Map(elem_gid, igradient, node_lid); if (map->isNodeLocalElement(gradient_node_gid)) { - Force_Gradient_Positions(gradient_node_gid * num_dims + jdim, column_index + dim) += area_normal(node_lid, 0) * tau_gradient(0, dim, igradient, jdim) - + area_normal(node_lid, 1) * tau_gradient(1, dim, igradient, jdim) - + area_normal(node_lid, 2) * tau_gradient(2, dim, igradient, jdim) - + area_normal_gradients(node_lid, 0, igradient, jdim) * tau(0, dim) - + area_normal_gradients(node_lid, 1, igradient, jdim) * tau(1, dim) - + area_normal_gradients(node_lid, 2, igradient, jdim) * tau(2, dim) - + phi * muc_gradient(node_lid, igradient, - jdim) * (vel_star(dim) - node_vel(rk_level, node_gid, dim)) - + phi * muc(node_lid) * (vel_star_gradient(dim, igradient, jdim)); + Force_Gradient_Positions(gradient_node_gid * num_dims + jdim, column_index + dim) += + area_normal(node_lid, 0) * tau_gradient(0, dim, igradient, jdim) + + area_normal(node_lid, 1) * tau_gradient(1, dim, igradient, jdim) + + area_normal(node_lid, 2) * tau_gradient(2, dim, igradient, jdim) + + area_normal_gradients(node_lid, 0, igradient, jdim) * tau(0, dim) + + area_normal_gradients(node_lid, 1, igradient, jdim) * tau(1, dim) + + area_normal_gradients(node_lid, 2, igradient, jdim) * tau(2, dim) + + phi * muc_gradient(node_lid, igradient, jdim) * (vel_star(dim) - node_vel(rk_level, node_gid, dim)) + + phi * muc(node_lid) * (vel_star_gradient(dim, igradient, jdim)); } - corner_gradient_storage(corner_gid, dim, igradient, jdim) = area_normal(node_lid, 0) * tau_gradient(0, dim, igradient, jdim) - + area_normal(node_lid, 1) * tau_gradient(1, dim, igradient, jdim) - + area_normal(node_lid, 2) * tau_gradient(2, dim, igradient, jdim) - + area_normal_gradients(node_lid, 0, igradient, jdim) * tau(0, dim) - + area_normal_gradients(node_lid, 1, igradient, jdim) * tau(1, dim) - + area_normal_gradients(node_lid, 2, igradient, jdim) * tau(2, dim) - + phi * muc_gradient(node_lid, igradient, jdim) * (vel_star(dim) - node_vel(rk_level, node_gid, dim)) - + phi * muc(node_lid) * (vel_star_gradient(dim, igradient, jdim)); + corner_gradient_storage(corner_gid, dim, igradient, jdim) = + area_normal(node_lid, 0) * tau_gradient(0, dim, igradient, jdim) + + area_normal(node_lid, 1) * tau_gradient(1, dim, igradient, jdim) + + area_normal(node_lid, 2) * tau_gradient(2, dim, igradient, jdim) + + area_normal_gradients(node_lid, 0, igradient, jdim) * tau(0, dim) + + area_normal_gradients(node_lid, 1, igradient, jdim) * tau(1, dim) + + area_normal_gradients(node_lid, 2, igradient, jdim) * tau(2, dim) + + phi * muc_gradient(node_lid, igradient, jdim) * (vel_star(dim) - node_vel(rk_level, node_gid, dim)) + + phi * muc(node_lid) * (vel_star_gradient(dim, igradient, jdim)); // if(map->isNodeLocalElement(gradient_node_gid)){ // Force_Gradient_Positions(gradient_node_gid*num_dims+jdim, column_index+dim) += // + area_normal_gradients(node_lid, 0, igradient, jdim)*tau(0, dim) @@ -1541,9 +1574,9 @@ void FEA_Module_SGH::force_design_gradient_term(const_vec_array design_variables { const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); const_vec_array current_element_internal_energy = (*forward_solve_internal_energy_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); - const_vec_array current_coord_vector = (*forward_solve_coordinate_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); - const_vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); - const_vec_array next_velocity_vector = (*forward_solve_velocity_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array current_coord_vector = (*forward_solve_coordinate_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_velocity_vector = (*forward_solve_velocity_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); const_vec_array next_element_internal_energy = (*forward_solve_internal_energy_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); const_vec_array next_coord_vector = (*forward_solve_coordinate_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); const_vec_array next_adjoint_vector = (*adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); @@ -1677,20 +1710,20 @@ void FEA_Module_SGH::force_design_gradient_term(const_vec_array design_variables // This function calculates the corner forces and the evolves stress (hypo) // ------------------------------------------------------------------------------ void FEA_Module_SGH::get_force_dgradient_sgh(const DCArrayKokkos& material, - const mesh_t& mesh, - const DViewCArrayKokkos& node_coords, - const DViewCArrayKokkos& node_vel, - const DViewCArrayKokkos& elem_den, - const DViewCArrayKokkos& elem_sie, - const DViewCArrayKokkos& elem_pres, - const DViewCArrayKokkos& elem_stress, - const DViewCArrayKokkos& elem_sspd, - const DViewCArrayKokkos& elem_vol, - const DViewCArrayKokkos& elem_div, - const DViewCArrayKokkos& elem_mat_id, - const double rk_alpha, - const size_t cycle - ) + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_pres, + const DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + const double rk_alpha, + const size_t cycle + ) { const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; const size_t num_dims = simparam->num_dims; @@ -1700,7 +1733,7 @@ void FEA_Module_SGH::get_force_dgradient_sgh(const DCArrayKokkos& ma // FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { const size_t num_nodes_in_elem = 8; - real_t gradient_result[num_dims]; + real_t gradient_result[num_dims]; // total Cauchy stress double tau_array[9]; double tau_gradient_array[9]; diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_sgh.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_sgh.cpp index 1a338a1e0..7ce9c9e77 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_sgh.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_sgh.cpp @@ -1,3 +1,36 @@ +/********************************************************************************************** + 2020. Triad National Security, LLC. All rights reserved. + This program was produced under U.S. Government contract 89233218CNA000001 for Los Alamos + National Laboratory (LANL), which is operated by Triad National Security, LLC for the U.S. + Department of Energy/National Nuclear Security Administration. All rights in the program are + reserved by Triad National Security, LLC, and the U.S. Department of Energy/National Nuclear + Security Administration. The Government is granted for itself and others acting on its behalf a + nonexclusive, paid-up, irrevocable worldwide license in this material to reproduce, prepare + derivative works, distribute copies to the public, perform publicly and display publicly, and + to permit others to do so. + This program is open source under the BSD-3 License. + Redistribution and use in source and binary forms, with or without modification, are permitted + provided that the following conditions are met: + 1. Redistributions of source code must retain the above copyright notice, this list of + conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright notice, this list of + conditions and the following disclaimer in the documentation and/or other materials + provided with the distribution. + 3. Neither the name of the copyright holder nor the names of its contributors may be used + to endorse or promote products derived from this software without specific prior + written permission. + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS + IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **********************************************************************************************/ #include "mesh.h" #include "state.h" #include "Simulation_Parameters/Simulation_Parameters_Explicit.h" @@ -5,21 +38,21 @@ #include "FEA_Module_SGH.h" void FEA_Module_SGH::get_force_sgh(const DCArrayKokkos& material, - const mesh_t& mesh, - const DViewCArrayKokkos& node_coords, - const DViewCArrayKokkos& node_vel, - const DViewCArrayKokkos& elem_den, - const DViewCArrayKokkos& elem_sie, - const DViewCArrayKokkos& elem_pres, - DViewCArrayKokkos& elem_stress, - const DViewCArrayKokkos& elem_sspd, - const DViewCArrayKokkos& elem_vol, - const DViewCArrayKokkos& elem_div, - const DViewCArrayKokkos& elem_mat_id, - DViewCArrayKokkos& corner_force, - const double rk_alpha, - const size_t cycle - ) + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_pres, + DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + DViewCArrayKokkos& corner_force, + const double rk_alpha, + const size_t cycle + ) { const_vec_array initial_node_coords = initial_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); @@ -420,7 +453,7 @@ void FEA_Module_SGH::get_force_sgh(const DCArrayKokkos& material, for (size_t elem_gid = 0; elem_gid < mesh.num_elems; elem_gid++) { const size_t num_dims = 3; - size_t mat_id = elem_mat_id.host(elem_gid); + size_t mat_id = elem_mat_id.host(elem_gid); // hypo elastic plastic model if (material.host(mat_id).strength_type == STRENGTH_TYPE::hypo) @@ -470,21 +503,21 @@ void FEA_Module_SGH::get_force_sgh(const DCArrayKokkos& material, // This function calculates the corner forces and the evolves stress (hypo) // ------------------------------------------------------------------------------ void FEA_Module_SGH::get_force_sgh2D(const DCArrayKokkos& material, - const mesh_t& mesh, - const DViewCArrayKokkos& node_coords, - const DViewCArrayKokkos& node_vel, - const DViewCArrayKokkos& elem_den, - const DViewCArrayKokkos& elem_sie, - const DViewCArrayKokkos& elem_pres, - const DViewCArrayKokkos& elem_stress, - const DViewCArrayKokkos& elem_sspd, - const DViewCArrayKokkos& elem_vol, - const DViewCArrayKokkos& elem_div, - const DViewCArrayKokkos& elem_mat_id, - DViewCArrayKokkos& corner_force, - const double rk_alpha, - const size_t cycle - ) + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_pres, + const DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + DViewCArrayKokkos& corner_force, + const double rk_alpha, + const size_t cycle + ) { const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; @@ -848,18 +881,18 @@ void FEA_Module_SGH::get_force_sgh2D(const DCArrayKokkos& material, // This function calculates the corner forces and the evolves stress (hypo) // ------------------------------------------------------------------------------ void FEA_Module_SGH::applied_forces(const DCArrayKokkos& material, - const mesh_t& mesh, - const DViewCArrayKokkos& node_coords, - const DViewCArrayKokkos& node_vel, - const DViewCArrayKokkos& node_mass, - const DViewCArrayKokkos& elem_den, - const DViewCArrayKokkos& elem_vol, - const DViewCArrayKokkos& elem_div, - const DViewCArrayKokkos& elem_mat_id, - DViewCArrayKokkos& corner_force, - const double rk_alpha, - const size_t cycle - ) + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_mass, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + DViewCArrayKokkos& corner_force, + const double rk_alpha, + const size_t cycle + ) { const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; const size_t num_dim = mesh.num_dims; diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/geometry.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/geometry.cpp index 726c54a5b..517094125 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/geometry.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/geometry.cpp @@ -1,3 +1,37 @@ +/********************************************************************************************** + 2020. Triad National Security, LLC. All rights reserved. + This program was produced under U.S. Government contract 89233218CNA000001 for Los Alamos + National Laboratory (LANL), which is operated by Triad National Security, LLC for the U.S. + Department of Energy/National Nuclear Security Administration. All rights in the program are + reserved by Triad National Security, LLC, and the U.S. Department of Energy/National Nuclear + Security Administration. The Government is granted for itself and others acting on its behalf a + nonexclusive, paid-up, irrevocable worldwide license in this material to reproduce, prepare + derivative works, distribute copies to the public, perform publicly and display publicly, and + to permit others to do so. + This program is open source under the BSD-3 License. + Redistribution and use in source and binary forms, with or without modification, are permitted + provided that the following conditions are met: + 1. Redistributions of source code must retain the above copyright notice, this list of + conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright notice, this list of + conditions and the following disclaimer in the documentation and/or other materials + provided with the distribution. + 3. Neither the name of the copyright holder nor the names of its contributors may be used + to endorse or promote products derived from this software without specific prior + written permission. + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS + IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **********************************************************************************************/ + // ----------------------------------------------------------------------------- // This code handles the geometric information for the mesh for the SHG solver // ------------------------------------------------------------------------------ @@ -5,13 +39,13 @@ #include "state.h" #include "FEA_Module_SGH.h" -void FEA_Module_SGH::update_position_sgh(double rk_alpha, - const size_t num_nodes, - DViewCArrayKokkos& node_coords, - const DViewCArrayKokkos& node_vel) +void FEA_Module_SGH::update_position_sgh(double rk_alpha, + const size_t num_nodes, + DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel) { const size_t rk_level = rk_num_bins - 1; - int num_dims = num_dim; + int num_dims = num_dim; // loop over all the nodes in the mesh FOR_ALL_CLASS(node_gid, 0, num_nodes, { @@ -35,11 +69,11 @@ void FEA_Module_SGH::update_position_sgh(double rk_alp // B_p is the OUTWARD corner area normal at node p // ------------------------------------------------------------------------------ KOKKOS_FUNCTION -void FEA_Module_SGH::get_bmatrix(const ViewCArrayKokkos& B_matrix, - const size_t elem_gid, - const DViewCArrayKokkos& node_coords, - const ViewCArrayKokkos& elem_node_gids, - const size_t rk_level) const +void FEA_Module_SGH::get_bmatrix(const ViewCArrayKokkos& B_matrix, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const { const size_t num_nodes = 8; @@ -265,10 +299,10 @@ void FEA_Module_SGH::get_vol() // Exact volume for a hex element KOKKOS_INLINE_FUNCTION void FEA_Module_SGH::get_vol_hex(const DViewCArrayKokkos& elem_vol, - const size_t elem_gid, - const DViewCArrayKokkos& node_coords, - const ViewCArrayKokkos& elem_node_gids, - const size_t rk_level) const + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const { const size_t num_nodes = 8; @@ -308,11 +342,11 @@ void FEA_Module_SGH::get_vol_hex(const DViewCArrayKokkos& elem_vol, } // end subroutine KOKKOS_FUNCTION -void FEA_Module_SGH::get_bmatrix2D(const ViewCArrayKokkos& B_matrix, - const size_t elem_gid, - const DViewCArrayKokkos& node_coords, - const ViewCArrayKokkos& elem_node_gids, - const size_t rk_level) const +void FEA_Module_SGH::get_bmatrix2D(const ViewCArrayKokkos& B_matrix, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const { const size_t num_nodes = 4; @@ -367,10 +401,10 @@ void FEA_Module_SGH::get_bmatrix2D(const ViewCArrayKokkos& B_matrix, // true volume of a quad in RZ coords KOKKOS_INLINE_FUNCTION void FEA_Module_SGH::get_vol_quad(const DViewCArrayKokkos& elem_vol, - const size_t elem_gid, - const DViewCArrayKokkos& node_coords, - const ViewCArrayKokkos& elem_node_gids, - const size_t rk_level) const + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const { // --- testing here --- /* @@ -425,10 +459,10 @@ void FEA_Module_SGH::get_vol_quad(const DViewCArrayKokkos& elem_vol, // element facial area KOKKOS_FUNCTION -double FEA_Module_SGH::get_area_quad(const size_t elem_gid, - const DViewCArrayKokkos& node_coords, - const ViewCArrayKokkos& elem_node_gids, - const size_t rk_level) const +double FEA_Module_SGH::get_area_quad(const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const { double elem_area = 0.0; @@ -460,11 +494,11 @@ double FEA_Module_SGH::get_area_quad(const size_t elem_gid, KOKKOS_INLINE_FUNCTION double FEA_Module_SGH::heron(const double x1, - const double y1, - const double x2, - const double y2, - const double x3, - const double y3) const + const double y1, + const double x2, + const double y2, + const double x3, + const double y3) const { double S, a, b, c, area; @@ -483,11 +517,11 @@ double FEA_Module_SGH::heron(const double x1, } KOKKOS_FUNCTION -void FEA_Module_SGH::get_area_weights2D(const ViewCArrayKokkos& corner_areas, - const size_t elem_gid, - const DViewCArrayKokkos& node_coords, - const ViewCArrayKokkos& elem_node_gids, - const size_t rk_level) const +void FEA_Module_SGH::get_area_weights2D(const ViewCArrayKokkos& corner_areas, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const { const size_t num_nodes = 4; @@ -561,18 +595,18 @@ void FEA_Module_SGH::get_vol_ugradient(const size_t gradient_node_id, const size // Exact volume for a hex element KOKKOS_FUNCTION -void FEA_Module_SGH::get_vol_hex_ugradient(const ViewCArrayKokkos& elem_vol_gradients, - const size_t elem_gid, - const DViewCArrayKokkos& node_coords, - const ViewCArrayKokkos& elem_node_gids, - const size_t rk_level) const +void FEA_Module_SGH::get_vol_hex_ugradient(const ViewCArrayKokkos& elem_vol_gradients, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const { const size_t num_nodes = 8; const size_t num_dims = num_dim; - double x_array[8]; - double y_array[8]; - double z_array[8]; - double gradient_result; + double x_array[8]; + double y_array[8]; + double z_array[8]; + double gradient_result; // x, y, z coordinates of elem vertices auto x = ViewCArrayKokkos(x_array, num_nodes); @@ -789,11 +823,11 @@ void FEA_Module_SGH::get_vol_hex_ugradient(const ViewCArrayKokkos& elem } // end subroutine KOKKOS_FUNCTION -void FEA_Module_SGH::get_bmatrix_gradients(const ViewCArrayKokkos& B_matrix_gradients, - const size_t elem_gid, - const DViewCArrayKokkos& node_coords, - const ViewCArrayKokkos& elem_node_gids, - const size_t rk_level) const +void FEA_Module_SGH::get_bmatrix_gradients(const ViewCArrayKokkos& B_matrix_gradients, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const { const size_t num_nodes = 8; diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/momentum.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/momentum.cpp index 7adbfb401..848da2d69 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/momentum.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/momentum.cpp @@ -1,15 +1,47 @@ - +/********************************************************************************************** + 2020. Triad National Security, LLC. All rights reserved. + This program was produced under U.S. Government contract 89233218CNA000001 for Los Alamos + National Laboratory (LANL), which is operated by Triad National Security, LLC for the U.S. + Department of Energy/National Nuclear Security Administration. All rights in the program are + reserved by Triad National Security, LLC, and the U.S. Department of Energy/National Nuclear + Security Administration. The Government is granted for itself and others acting on its behalf a + nonexclusive, paid-up, irrevocable worldwide license in this material to reproduce, prepare + derivative works, distribute copies to the public, perform publicly and display publicly, and + to permit others to do so. + This program is open source under the BSD-3 License. + Redistribution and use in source and binary forms, with or without modification, are permitted + provided that the following conditions are met: + 1. Redistributions of source code must retain the above copyright notice, this list of + conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright notice, this list of + conditions and the following disclaimer in the documentation and/or other materials + provided with the distribution. + 3. Neither the name of the copyright holder nor the names of its contributors may be used + to endorse or promote products derived from this software without specific prior + written permission. + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS + IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **********************************************************************************************/ #include "state.h" #include "FEA_Module_SGH.h" // ----------------------------------------------------------------------------- // This function evolves the velocity at the nodes of the mesh // ------------------------------------------------------------------------------ -void FEA_Module_SGH::update_velocity_sgh(double rk_alpha, - DViewCArrayKokkos& node_vel, - const DViewCArrayKokkos& node_mass, - const DViewCArrayKokkos& corner_force - ) +void FEA_Module_SGH::update_velocity_sgh(double rk_alpha, + DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_mass, + const DViewCArrayKokkos& corner_force + ) { const size_t rk_level = rk_num_bins - 1; const size_t num_dims = num_dim; @@ -50,14 +82,14 @@ void FEA_Module_SGH::update_velocity_sgh(double rk_alp // This function calculates the velocity gradient // ------------------------------------------------------------------------------ KOKKOS_FUNCTION -void FEA_Module_SGH::get_velgrad(ViewCArrayKokkos& vel_grad, - const ViewCArrayKokkos& elem_node_gids, - const DViewCArrayKokkos& node_vel, - const ViewCArrayKokkos& b_matrix, - const double elem_vol, - const size_t elem_gid, - const size_t rk_level - ) const +void FEA_Module_SGH::get_velgrad(ViewCArrayKokkos& vel_grad, + const ViewCArrayKokkos& elem_node_gids, + const DViewCArrayKokkos& node_vel, + const ViewCArrayKokkos& b_matrix, + const double elem_vol, + const size_t elem_gid, + const size_t rk_level + ) const { const size_t num_nodes_in_elem = 8; @@ -137,15 +169,15 @@ void FEA_Module_SGH::get_velgrad(ViewCArrayKokkos& vel_grad, // This function calculates the velocity gradient // ------------------------------------------------------------------------------ KOKKOS_FUNCTION -void FEA_Module_SGH::get_velgrad2D(ViewCArrayKokkos& vel_grad, - const ViewCArrayKokkos& elem_node_gids, - const DViewCArrayKokkos& node_vel, - const ViewCArrayKokkos& b_matrix, - const double elem_vol, - const double elem_area, - const size_t elem_gid, - const size_t rk_level - ) const +void FEA_Module_SGH::get_velgrad2D(ViewCArrayKokkos& vel_grad, + const ViewCArrayKokkos& elem_node_gids, + const DViewCArrayKokkos& node_vel, + const ViewCArrayKokkos& b_matrix, + const double elem_vol, + const double elem_area, + const size_t elem_gid, + const size_t rk_level + ) const { const size_t num_nodes_in_elem = 4; @@ -202,11 +234,11 @@ void FEA_Module_SGH::get_velgrad2D(ViewCArrayKokkos& vel_grad, // ----------------------------------------------------------------------------- // This subroutine to calculate the velocity divergence in all elements // ------------------------------------------------------------------------------ -void FEA_Module_SGH::get_divergence(DViewCArrayKokkos& elem_div, - const DViewCArrayKokkos& node_coords, - const DViewCArrayKokkos& node_vel, - const DViewCArrayKokkos& elem_vol - ) +void FEA_Module_SGH::get_divergence(DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_vol + ) { const size_t rk_level = rk_num_bins - 1; @@ -276,11 +308,11 @@ void FEA_Module_SGH::get_divergence(DViewCArrayKokkos& elem_div, // ----------------------------------------------------------------------------- // This subroutine to calculate the velocity divergence in all elements // ------------------------------------------------------------------------------ -void FEA_Module_SGH::get_divergence2D(DViewCArrayKokkos& elem_div, - const DViewCArrayKokkos& node_coords, - const DViewCArrayKokkos& node_vel, - const DViewCArrayKokkos& elem_vol - ) +void FEA_Module_SGH::get_divergence2D(DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_vol + ) { const size_t rk_level = rk_num_bins - 1; @@ -356,15 +388,15 @@ void FEA_Module_SGH::get_divergence2D(DViewCArrayKokkos& elem_div, // D = sym(L) // W = antisym(L) KOKKOS_INLINE_FUNCTION -void FEA_Module_SGH::decompose_vel_grad(ViewCArrayKokkos& D_tensor, - ViewCArrayKokkos& W_tensor, - const ViewCArrayKokkos& vel_grad, - const ViewCArrayKokkos& elem_node_gids, - const size_t elem_gid, - const DViewCArrayKokkos& node_coords, - const DViewCArrayKokkos& node_vel, - const double vol - ) const +void FEA_Module_SGH::decompose_vel_grad(ViewCArrayKokkos& D_tensor, + ViewCArrayKokkos& W_tensor, + const ViewCArrayKokkos& vel_grad, + const ViewCArrayKokkos& elem_node_gids, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const double vol + ) const { // --- Calculate the velocity gradient --- diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/power_gradients_sgh.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/power_gradients_sgh.cpp index ed2c3166d..8cfb2b1df 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/power_gradients_sgh.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/power_gradients_sgh.cpp @@ -1,4 +1,36 @@ - +/********************************************************************************************** + 2020. Triad National Security, LLC. All rights reserved. + This program was produced under U.S. Government contract 89233218CNA000001 for Los Alamos + National Laboratory (LANL), which is operated by Triad National Security, LLC for the U.S. + Department of Energy/National Nuclear Security Administration. All rights in the program are + reserved by Triad National Security, LLC, and the U.S. Department of Energy/National Nuclear + Security Administration. The Government is granted for itself and others acting on its behalf a + nonexclusive, paid-up, irrevocable worldwide license in this material to reproduce, prepare + derivative works, distribute copies to the public, perform publicly and display publicly, and + to permit others to do so. + This program is open source under the BSD-3 License. + Redistribution and use in source and binary forms, with or without modification, are permitted + provided that the following conditions are met: + 1. Redistributions of source code must retain the above copyright notice, this list of + conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright notice, this list of + conditions and the following disclaimer in the documentation and/or other materials + provided with the distribution. + 3. Neither the name of the copyright holder nor the names of its contributors may be used + to endorse or promote products derived from this software without specific prior + written permission. + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS + IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **********************************************************************************************/ #include "mesh.h" #include "state.h" #include "FEA_Module_SGH.h" @@ -19,8 +51,8 @@ void FEA_Module_SGH::power_design_gradient_term(const_vec_array design_variables const DCArrayKokkos boundary = module_params->boundary; const DCArrayKokkos material = simparam->material; - const int num_dim = simparam->num_dims; - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + const int num_dim = simparam->num_dims; + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; CArrayKokkos current_element_adjoint = CArrayKokkos(num_nodes_in_elem, num_dim); @@ -63,11 +95,11 @@ void FEA_Module_SGH::power_design_gradient_term(const_vec_array design_variables { const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); const_vec_array current_element_internal_energy = (*forward_solve_internal_energy_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); - const_vec_array current_coord_vector = (*forward_solve_coordinate_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); - const_vec_array current_psi_adjoint_vector = (*psi_adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); - const_vec_array next_velocity_vector = (*forward_solve_velocity_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array current_coord_vector = (*forward_solve_coordinate_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array current_psi_adjoint_vector = (*psi_adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_velocity_vector = (*forward_solve_velocity_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); const_vec_array next_element_internal_energy = (*forward_solve_internal_energy_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); - const_vec_array next_coord_vector = (*forward_solve_coordinate_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_coord_vector = (*forward_solve_coordinate_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); const_vec_array next_psi_adjoint_vector = (*psi_adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); // first half of integration step calculation @@ -231,17 +263,17 @@ void FEA_Module_SGH::power_design_gradient_term(const_vec_array design_variables // This function calculates the gradient for element power with respect to design variable // ---------------------------------------------------------------------------------------- -void FEA_Module_SGH::get_power_dgradient_sgh(double rk_alpha, - const mesh_t& mesh, - const DViewCArrayKokkos& node_vel, - const DViewCArrayKokkos& node_coords, - DViewCArrayKokkos& elem_sie, - const DViewCArrayKokkos& elem_mass, - const DViewCArrayKokkos& corner_force, - DCArrayKokkos elem_power_dgradients) +void FEA_Module_SGH::get_power_dgradient_sgh(double rk_alpha, + const mesh_t& mesh, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& corner_force, + DCArrayKokkos elem_power_dgradients) { const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - int num_dims = simparam->num_dims; + int num_dims = simparam->num_dims; // loop over all the elements in the mesh FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { @@ -283,16 +315,16 @@ void FEA_Module_SGH::get_power_dgradient_sgh(double rk // This function calculates the gradient for element power with respect to position // ------------------------------------------------------------------------------ -void FEA_Module_SGH::get_power_ugradient_sgh(double rk_alpha, - const mesh_t& mesh, - const DViewCArrayKokkos& node_vel, - const DViewCArrayKokkos& node_coords, - DViewCArrayKokkos& elem_sie, - const DViewCArrayKokkos& elem_mass, - const DViewCArrayKokkos& corner_force) +void FEA_Module_SGH::get_power_ugradient_sgh(double rk_alpha, + const mesh_t& mesh, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& corner_force) { const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - int num_dims = simparam->num_dims; + int num_dims = simparam->num_dims; // initialize gradient matrix FOR_ALL_CLASS(dof_gid, 0, nlocal_nodes * num_dims, { @@ -360,16 +392,16 @@ void FEA_Module_SGH::get_power_ugradient_sgh(double rk // This function calculates the gradient for element power with respect to velocity // ------------------------------------------------------------------------------ -void FEA_Module_SGH::get_power_vgradient_sgh(double rk_alpha, - const mesh_t& mesh, - const DViewCArrayKokkos& node_vel, - const DViewCArrayKokkos& node_coords, - DViewCArrayKokkos& elem_sie, - const DViewCArrayKokkos& elem_mass, - const DViewCArrayKokkos& corner_force) +void FEA_Module_SGH::get_power_vgradient_sgh(double rk_alpha, + const mesh_t& mesh, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& corner_force) { const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - int num_dims = simparam->num_dims; + int num_dims = simparam->num_dims; // initialize gradient matrix FOR_ALL_CLASS(dof_gid, 0, nlocal_nodes * num_dims, { @@ -443,16 +475,16 @@ void FEA_Module_SGH::get_power_vgradient_sgh(double rk // This function calculates the gradient for element power with respect to energy // ------------------------------------------------------------------------------ -void FEA_Module_SGH::get_power_egradient_sgh(double rk_alpha, - const mesh_t& mesh, - const DViewCArrayKokkos& node_vel, - const DViewCArrayKokkos& node_coords, - DViewCArrayKokkos& elem_sie, - const DViewCArrayKokkos& elem_mass, - const DViewCArrayKokkos& corner_force) +void FEA_Module_SGH::get_power_egradient_sgh(double rk_alpha, + const mesh_t& mesh, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& corner_force) { const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - int num_dims = simparam->num_dims; + int num_dims = simparam->num_dims; // initialize gradient storage FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/properties.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/properties.cpp index f415296de..a4c6c80d9 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/properties.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/properties.cpp @@ -1,4 +1,36 @@ - +/********************************************************************************************** + 2020. Triad National Security, LLC. All rights reserved. + This program was produced under U.S. Government contract 89233218CNA000001 for Los Alamos + National Laboratory (LANL), which is operated by Triad National Security, LLC for the U.S. + Department of Energy/National Nuclear Security Administration. All rights in the program are + reserved by Triad National Security, LLC, and the U.S. Department of Energy/National Nuclear + Security Administration. The Government is granted for itself and others acting on its behalf a + nonexclusive, paid-up, irrevocable worldwide license in this material to reproduce, prepare + derivative works, distribute copies to the public, perform publicly and display publicly, and + to permit others to do so. + This program is open source under the BSD-3 License. + Redistribution and use in source and binary forms, with or without modification, are permitted + provided that the following conditions are met: + 1. Redistributions of source code must retain the above copyright notice, this list of + conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright notice, this list of + conditions and the following disclaimer in the documentation and/or other materials + provided with the distribution. + 3. Neither the name of the copyright holder nor the names of its contributors may be used + to endorse or promote products derived from this software without specific prior + written permission. + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS + IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **********************************************************************************************/ // ----------------------------------------------------------------------------- // This calls the models to update state // ------------------------------------------------------------------------------ @@ -7,23 +39,23 @@ #include "FEA_Module_SGH.h" void FEA_Module_SGH::update_state(const DCArrayKokkos& material, - const mesh_t& mesh, - const DViewCArrayKokkos& node_coords, - const DViewCArrayKokkos& node_vel, - DViewCArrayKokkos& elem_den, - DViewCArrayKokkos& elem_pres, - DViewCArrayKokkos& elem_stress, - DViewCArrayKokkos& elem_sspd, - const DViewCArrayKokkos& elem_sie, - const DViewCArrayKokkos& elem_vol, - const DViewCArrayKokkos& elem_mass, - const DViewCArrayKokkos& elem_mat_id, - const double rk_alpha, - const size_t cycle - ) + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_den, + DViewCArrayKokkos& elem_pres, + DViewCArrayKokkos& elem_stress, + DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& elem_mat_id, + const double rk_alpha, + const size_t cycle + ) { const size_t rk_level = rk_num_bins - 1; - int num_dims = num_dim; + int num_dims = num_dim; // loop over all the elements in the mesh FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { @@ -131,23 +163,23 @@ void FEA_Module_SGH::update_state(const DCArrayKokkos& material, } // end method to update state void FEA_Module_SGH::update_state2D(const DCArrayKokkos& material, - const mesh_t& mesh, - const DViewCArrayKokkos& node_coords, - const DViewCArrayKokkos& node_vel, - DViewCArrayKokkos& elem_den, - DViewCArrayKokkos& elem_pres, - DViewCArrayKokkos& elem_stress, - DViewCArrayKokkos& elem_sspd, - const DViewCArrayKokkos& elem_sie, - const DViewCArrayKokkos& elem_vol, - const DViewCArrayKokkos& elem_mass, - const DViewCArrayKokkos& elem_mat_id, - const double rk_alpha, - const size_t cycle - ) + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_den, + DViewCArrayKokkos& elem_pres, + DViewCArrayKokkos& elem_stress, + DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& elem_mat_id, + const double rk_alpha, + const size_t cycle + ) { const size_t rk_level = rk_num_bins - 1; - int num_dims = num_dim; + int num_dims = num_dim; // loop over all the elements in the mesh FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/setup_sgh.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/setup_sgh.cpp index a58fac95c..54cccd4f5 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/setup_sgh.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/setup_sgh.cpp @@ -1,3 +1,36 @@ +/********************************************************************************************** + 2020. Triad National Security, LLC. All rights reserved. + This program was produced under U.S. Government contract 89233218CNA000001 for Los Alamos + National Laboratory (LANL), which is operated by Triad National Security, LLC for the U.S. + Department of Energy/National Nuclear Security Administration. All rights in the program are + reserved by Triad National Security, LLC, and the U.S. Department of Energy/National Nuclear + Security Administration. The Government is granted for itself and others acting on its behalf a + nonexclusive, paid-up, irrevocable worldwide license in this material to reproduce, prepare + derivative works, distribute copies to the public, perform publicly and display publicly, and + to permit others to do so. + This program is open source under the BSD-3 License. + Redistribution and use in source and binary forms, with or without modification, are permitted + provided that the following conditions are met: + 1. Redistributions of source code must retain the above copyright notice, this list of + conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright notice, this list of + conditions and the following disclaimer in the documentation and/or other materials + provided with the distribution. + 3. Neither the name of the copyright holder nor the names of its contributors may be used + to endorse or promote products derived from this software without specific prior + written permission. + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS + IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **********************************************************************************************/ /* ---------------------------------------------------------------------------- Setup SGH solver data ------------------------------------------------------------------------------- */ @@ -113,7 +146,7 @@ void FEA_Module_SGH::setup() elem_vel_grad = DCArrayKokkos(num_elems, 3, 3); // allocate material models - elem_eos = DCArrayKokkos(num_elems); + elem_eos = DCArrayKokkos(num_elems); elem_strength = DCArrayKokkos(num_elems); // --------------------------------------------------------------------- @@ -474,7 +507,7 @@ void FEA_Module_SGH::setup() // loop over the corners of the element and calculate the mass for (size_t corner_lid = 0; corner_lid < 4; corner_lid++) { - size_t corner_gid = corners_in_elem(elem_gid, corner_lid); + size_t corner_gid = corners_in_elem(elem_gid, corner_lid); corner_mass(corner_gid) = corner_areas(corner_lid) * elem_den(elem_gid); // node radius is added later } // end for over corners }); @@ -554,9 +587,9 @@ void FEA_Module_SGH::setup() // ----------------------------------------------------------------------------- // Interfaces read in data with the SGH solver data; currently a hack to streamline // ------------------------------------------------------------------------------ -void FEA_Module_SGH::sgh_interface_setup(node_t& node, - elem_t& elem, - corner_t& corner) +void FEA_Module_SGH::sgh_interface_setup(node_t& node, + elem_t& elem, + corner_t& corner) { const size_t num_dim = simparam->num_dims; const size_t rk_num_bins = simparam->dynamic_options.rk_num_bins; @@ -687,7 +720,7 @@ void FEA_Module_SGH::sgh_interface_setup(node_t& node, // save all data (nlocal +nghost) CArrayKokkos host_all_node_coords_state(rk_num_bins, nall_nodes, num_dim); - host_vec_array interface_all_node_coords = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + host_vec_array interface_all_node_coords = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); host_all_node_coords_state.get_kokkos_view() = node.all_coords.get_kokkos_dual_view().view_host(); host_node_coords_state = CArrayKokkos(rk_num_bins, nall_nodes, num_dim); host_all_node_coords_state.get_kokkos_view() = Kokkos::View("debug", rk_num_bins * nall_nodes * num_dim); @@ -921,9 +954,9 @@ void FEA_Module_SGH::grow_boundary_sets(int num_sets) if (num_sets > max_boundary_sets) { // temporary storage for previous data - CArrayKokkos Temp_Boundary_Condition_Type_List = Boundary_Condition_Type_List; - CArrayKokkos Temp_NBoundary_Condition_Patches = NBoundary_Condition_Patches; - CArrayKokkos Temp_Boundary_Condition_Patches = Boundary_Condition_Patches; + CArrayKokkos Temp_Boundary_Condition_Type_List = Boundary_Condition_Type_List; + CArrayKokkos Temp_NBoundary_Condition_Patches = NBoundary_Condition_Patches; + CArrayKokkos Temp_Boundary_Condition_Patches = Boundary_Condition_Patches; max_boundary_sets = num_sets + 5; // 5 is an arbitrary buffer Boundary_Condition_Type_List = CArrayKokkos(max_boundary_sets, "Boundary_Condition_Type_List"); @@ -933,7 +966,7 @@ void FEA_Module_SGH::grow_boundary_sets(int num_sets) // copy previous data back over #ifdef DEBUG - std::cout << "NUM BOUNDARY CONDITIONS ON RANK " << myrank << " FOR COPY " << max_boundary_sets < @@ -60,7 +92,7 @@ void FEA_Module_SGH::update_forward_solve(Teuchos::RCP zp) const size_t rk_num_bins = simparam->dynamic_options.rk_num_bins; const size_t num_bcs = module_params->boundary_conditions.size(); const size_t num_materials = simparam->materials.size(); - real_t objective_accumulation; + real_t objective_accumulation; // --- Read in the nodes in the mesh --- int myrank = Explicit_Solver_Pointer_->myrank; @@ -106,9 +138,9 @@ void FEA_Module_SGH::update_forward_solve(Teuchos::RCP zp) // view scope { - const_vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); const_vec_array ghost_node_coords_interface = Explicit_Solver_Pointer_->ghost_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); - vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { for (int idim = 0; idim < num_dim; idim++) { @@ -442,7 +474,7 @@ void FEA_Module_SGH::update_forward_solve(Teuchos::RCP zp) // loop over the corners of the element and calculate the mass for (size_t corner_lid = 0; corner_lid < 4; corner_lid++) { - size_t corner_gid = corners_in_elem(elem_gid, corner_lid); + size_t corner_gid = corners_in_elem(elem_gid, corner_lid); corner_mass(corner_gid) = corner_areas(corner_lid) * elem_den(elem_gid); // node radius is added later } // end for over corners }); @@ -536,8 +568,8 @@ double FEA_Module_SGH::average_element_density(const int nodes_per_elem, const C void FEA_Module_SGH::compute_topology_optimization_adjoint_full() { - const int num_dim = simparam->num_dims; - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + const int num_dim = simparam->num_dims; + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; const DCArrayKokkos boundary = module_params->boundary; const DCArrayKokkos material = simparam->material; @@ -1008,7 +1040,7 @@ void FEA_Module_SGH::compute_topology_optimization_gradient_full(Teuchos::RCP boundary = module_params->boundary; const DCArrayKokkos material = simparam->material; - const int num_dim = simparam->num_dims; + const int num_dim = simparam->num_dims; auto current_element_velocities = CArrayKokkos(num_nodes_in_elem, num_dim); auto current_element_adjoint = CArrayKokkos(num_nodes_in_elem, num_dim); @@ -1021,7 +1053,7 @@ void FEA_Module_SGH::compute_topology_optimization_gradient_full(Teuchos::RCPgetLocalView(Tpetra::Access::ReadWrite); + vec_array design_gradients = design_gradients_distributed->getLocalView(Tpetra::Access::ReadWrite); const_vec_array design_densities = design_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); // initialize design gradients FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { @@ -1260,8 +1292,8 @@ void FEA_Module_SGH::compute_topology_optimization_gradient_full(Teuchos::RCPgetLocalView(Tpetra::Access::ReadOnly); - const_vec_array current_psi_adjoint_vector = (*psi_adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); - const_vec_array next_element_internal_energy = (*forward_solve_internal_energy_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array current_psi_adjoint_vector = (*psi_adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_element_internal_energy = (*forward_solve_internal_energy_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); const_vec_array next_psi_adjoint_vector = (*psi_adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); FOR_ALL_CLASS(elem_id, 0, rnum_elem, { @@ -1359,7 +1391,7 @@ void FEA_Module_SGH::compute_topology_optimization_gradient_full(Teuchos::RCPgetLocalView(Tpetra::Access::ReadOnly); - const_vec_array current_psi_adjoint_vector = (*psi_adjoint_vector_data)[0]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array current_psi_adjoint_vector = (*psi_adjoint_vector_data)[0]->getLocalView(Tpetra::Access::ReadOnly); // (*psi_adjoint_vector_data)[100]->describe(*fos,Teuchos::VERB_EXTREME); FOR_ALL_CLASS(elem_id, 0, rnum_elem, { @@ -1397,7 +1429,7 @@ void FEA_Module_SGH::compute_topology_optimization_gradient_full(Teuchos::RCPgetLocalView (Tpetra::Access::ReadWrite); // const_host_vec_array host_design_variables = design_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); - vec_array design_gradients = design_gradients_distributed->getLocalView(Tpetra::Access::ReadWrite); + vec_array design_gradients = design_gradients_distributed->getLocalView(Tpetra::Access::ReadWrite); const_vec_array design_variables = design_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); force_design_gradient_term(design_variables, design_gradients); power_design_gradient_term(design_variables, design_gradients); @@ -1416,12 +1448,12 @@ void FEA_Module_SGH::init_assembly() Elem_to_Elem_Matrix_Strides = DCArrayKokkos(rnum_elem, "Gradient_Matrix_Strides"); CArrayKokkos Graph_Fill(nall_nodes, "nall_nodes"); CArrayKokkos current_row_nodes_scanned; - CArrayKokkos count_saved_corners_in_node(nall_nodes, "count_saved_corners_in_node"); - int local_node_index, current_column_index; - size_t max_stride = 0; - size_t nodes_per_element; + CArrayKokkos count_saved_corners_in_node(nall_nodes, "count_saved_corners_in_node"); + int local_node_index, current_column_index; + size_t max_stride = 0; + size_t nodes_per_element; nodal_density_flag = simparam->nodal_density_flag; - penalty_power = simparam->optimization_options.simp_penalty_power; + penalty_power = simparam->optimization_options.simp_penalty_power; // allocate stride arrays CArrayKokkos Graph_Matrix_Strides_initial(nlocal_nodes, "Graph_Matrix_Strides_initial"); @@ -1455,7 +1487,7 @@ void FEA_Module_SGH::init_assembly() // initialize nlocal arrays FOR_ALL_CLASS(inode, 0, nall_nodes, { node_indices_used(inode) = 0; - column_index(inode) = 0; + column_index(inode) = 0; count_saved_corners_in_node(inode) = 0; }); // end parallel for Kokkos::fence(); @@ -1516,7 +1548,7 @@ void FEA_Module_SGH::init_assembly() current_row_nodes_scanned = CArrayKokkos(max_stride, "current_row_nodes_scanned"); // allocate sparse graph with node repeats - RaggedRightArrayKokkos Repeat_Graph_Matrix(Graph_Matrix_Strides_initial); + RaggedRightArrayKokkos Repeat_Graph_Matrix(Graph_Matrix_Strides_initial); RaggedRightArrayofVectorsKokkos Element_local_indices(Graph_Matrix_Strides_initial, num_dim); // Fill the initial Graph with repeats @@ -1594,7 +1626,7 @@ void FEA_Module_SGH::init_assembly() int element_column_index; int current_stride; int current_row_n_nodes_scanned; - + for (int inode = 0; inode < nlocal_nodes; inode++) { current_row_n_nodes_scanned = 0; @@ -1644,7 +1676,7 @@ void FEA_Module_SGH::init_assembly() that have been scanned uniquely. Use this list to reset the flag array afterwards without having to loop over all the nodes in the system*/ node_indices_used(current_node) = 1; - column_index(current_node) = istride; + column_index(current_node) = istride; current_row_nodes_scanned(current_row_n_nodes_scanned) = current_node; current_row_n_nodes_scanned++; } @@ -1779,11 +1811,11 @@ void FEA_Module_SGH::init_assembly() Enforce boundary conditions on the adjoint vectors ------------------------------------------------------------------------- */ -void FEA_Module_SGH::boundary_adjoint(const mesh_t& mesh, - const DCArrayKokkos& boundary, - vec_array& node_adjoint, - vec_array& node_phi_adjoint, - vec_array& node_psi_adjoint) +void FEA_Module_SGH::boundary_adjoint(const mesh_t& mesh, + const DCArrayKokkos& boundary, + vec_array& node_adjoint, + vec_array& node_phi_adjoint, + vec_array& node_psi_adjoint) { // error and debug flag // DCArrayKokkos print_flag(1, "print_flag"); @@ -1791,7 +1823,7 @@ void FEA_Module_SGH::boundary_adjoint(const mesh_t& mesh, // print_flag.update_device(); const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - int num_dims = simparam->num_dims; + int num_dims = simparam->num_dims; // Loop over boundary sets for (size_t bdy_set = 0; bdy_set < num_bdy_sets; bdy_set++) { diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/time_integration.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/time_integration.cpp index aeaea1d51..93628c4ec 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/time_integration.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/time_integration.cpp @@ -1,4 +1,36 @@ - +/********************************************************************************************** + 2020. Triad National Security, LLC. All rights reserved. + This program was produced under U.S. Government contract 89233218CNA000001 for Los Alamos + National Laboratory (LANL), which is operated by Triad National Security, LLC for the U.S. + Department of Energy/National Nuclear Security Administration. All rights in the program are + reserved by Triad National Security, LLC, and the U.S. Department of Energy/National Nuclear + Security Administration. The Government is granted for itself and others acting on its behalf a + nonexclusive, paid-up, irrevocable worldwide license in this material to reproduce, prepare + derivative works, distribute copies to the public, perform publicly and display publicly, and + to permit others to do so. + This program is open source under the BSD-3 License. + Redistribution and use in source and binary forms, with or without modification, are permitted + provided that the following conditions are met: + 1. Redistributions of source code must retain the above copyright notice, this list of + conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright notice, this list of + conditions and the following disclaimer in the documentation and/or other materials + provided with the distribution. + 3. Neither the name of the copyright holder nor the names of its contributors may be used + to endorse or promote products derived from this software without specific prior + written permission. + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS + IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **********************************************************************************************/ #include "mesh.h" #include "state.h" #include "FEA_Module_SGH.h" @@ -7,14 +39,14 @@ // This function saves the variables at rk_stage = 0, which is t_n // ------------------------------------------------------------------------------ void FEA_Module_SGH::rk_init(DViewCArrayKokkos& node_coords, - DViewCArrayKokkos& node_vel, - DViewCArrayKokkos& elem_sie, - DViewCArrayKokkos& elem_stress, - const size_t num_elems, - const size_t num_nodes) + DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_sie, + DViewCArrayKokkos& elem_stress, + const size_t num_elems, + const size_t num_nodes) { const size_t rk_level = rk_num_bins - 1; - int num_dims = num_dim; + int num_dims = num_dim; // save elem quantities FOR_ALL_CLASS(elem_gid, 0, num_elems, { // stress is always 3D even with 2D-RZ @@ -47,11 +79,11 @@ void FEA_Module_SGH::rk_init(DViewCArrayKokkos& node_coords, // between any two nodes in the mesh // ------------------------------------------------------------------------------ // WARNING WARNING : Only works for 3D, 8 node elements -void FEA_Module_SGH::get_timestep(mesh_t& mesh, - DViewCArrayKokkos& node_coords, - DViewCArrayKokkos& node_vel, - DViewCArrayKokkos& elem_sspd, - DViewCArrayKokkos& elem_vol) +void FEA_Module_SGH::get_timestep(mesh_t& mesh, + DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_sspd, + DViewCArrayKokkos& elem_vol) { const size_t rk_level = rk_num_bins - 1; @@ -148,11 +180,11 @@ void FEA_Module_SGH::get_timestep(mesh_t& mesh, // between any two nodes in the mesh // ------------------------------------------------------------------------------ // WARNING WARNING : Only works for 2D, 4 node elements -void FEA_Module_SGH::get_timestep2D(mesh_t& mesh, - DViewCArrayKokkos& node_coords, - DViewCArrayKokkos& node_vel, - DViewCArrayKokkos& elem_sspd, - DViewCArrayKokkos& elem_vol) +void FEA_Module_SGH::get_timestep2D(mesh_t& mesh, + DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_sspd, + DViewCArrayKokkos& elem_vol) { const size_t rk_level = rk_num_bins - 1; diff --git a/src/Parallel-Solvers/Parallel-Explicit/Topology_Optimization/Area_Normals.h b/src/Parallel-Solvers/Parallel-Explicit/Topology_Optimization/Area_Normals.h index f4ad28896..e880c1a75 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Topology_Optimization/Area_Normals.h +++ b/src/Parallel-Solvers/Parallel-Explicit/Topology_Optimization/Area_Normals.h @@ -11,14 +11,14 @@ This program is open source under the BSD-3 License. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - + 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - + 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - + 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. @@ -34,7 +34,7 @@ OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. **********************************************************************************************/ - + #ifndef AREA_NORMALS_SHAPEOPT_H #define AREA_NORMALS_SHAPEOPT_H @@ -61,182 +61,196 @@ #include "FEA_Module_Dynamic_Elasticity.h" #include "Explicit_Solver.h" -class AreaNormals : public ROL::Objective { - - typedef Tpetra::Map<>::local_ordinal_type LO; - typedef Tpetra::Map<>::global_ordinal_type GO; - typedef Tpetra::Map<>::node_type Node; - typedef Tpetra::Map Map; - typedef Tpetra::MultiVector MV; - typedef ROL::Vector V; - typedef ROL::TpetraMultiVector ROL_MV; - - using traits = Kokkos::ViewTraits; - using array_layout = typename traits::array_layout; - using execution_space = typename traits::execution_space; - using device_type = typename traits::device_type; - using memory_traits = typename traits::memory_traits; - using global_size_t = Tpetra::global_size_t; - - typedef Kokkos::View values_array; - typedef Kokkos::View global_indices_array; - typedef Kokkos::View indices_array; - - //typedef Kokkos::DualView::t_dev vec_array; - typedef MV::dual_view_type::t_dev vec_array; - typedef MV::dual_view_type::t_host host_vec_array; - typedef Kokkos::View const_host_vec_array; - typedef Kokkos::View const_vec_array; - typedef MV::dual_view_type dual_vec_array; +class AreaNormals : public ROL::Objective +{ +typedef Tpetra::Map<>::local_ordinal_type LO; +typedef Tpetra::Map<>::global_ordinal_type GO; +typedef Tpetra::Map<>::node_type Node; +typedef Tpetra::Map Map; +typedef Tpetra::MultiVector MV; +typedef ROL::Vector V; +typedef ROL::TpetraMultiVector ROL_MV; + +using traits = Kokkos::ViewTraits; +using array_layout = typename traits::array_layout; +using execution_space = typename traits::execution_space; +using device_type = typename traits::device_type; +using memory_traits = typename traits::memory_traits; +using global_size_t = Tpetra::global_size_t; + +typedef Kokkos::View values_array; +typedef Kokkos::View global_indices_array; +typedef Kokkos::View indices_array; + +// typedef Kokkos::DualView::t_dev vec_array; +typedef MV::dual_view_type::t_dev vec_array; +typedef MV::dual_view_type::t_host host_vec_array; +typedef Kokkos::View const_host_vec_array; +typedef Kokkos::View const_vec_array; +typedef MV::dual_view_type dual_vec_array; private: - Explicit_Solver *Explicit_Solver_Pointer_; - FEA_Module_SGH *FEM_SGH_; - FEA_Module_Dynamic_Elasticity *FEM_Dynamic_Elasticity_; - ROL::Ptr ROL_Force; - ROL::Ptr ROL_Velocities; - ROL::Ptr ROL_Gradients; + Explicit_Solver* Explicit_Solver_Pointer_; + FEA_Module_SGH* FEM_SGH_; + FEA_Module_Dynamic_Elasticity* FEM_Dynamic_Elasticity_; + ROL::Ptr ROL_Force; + ROL::Ptr ROL_Velocities; + ROL::Ptr ROL_Gradients; - ROL::Ptr getVector( const V& x ) { - return dynamic_cast(x).getVector(); - } + ROL::Ptr getVector(const V& x) + { + return dynamic_cast(x).getVector(); + } - ROL::Ptr getVector( V& x ) { - return dynamic_cast(x).getVector(); - } + ROL::Ptr getVector(V& x) + { + return dynamic_cast(x).getVector(); + } public: - bool nodal_density_flag_; - int last_comm_step, last_solve_step, current_step; - size_t nvalid_modules; - std::vector valid_fea_modules; //modules that may interface with this objective function - FEA_MODULE_TYPE set_module_type; - int set_node_, set_dim_; - //std::string my_fea_module = "SGH"; - - AreaNormals(Explicit_Solver *Explicit_Solver_Pointer, int set_node, int set_dim, bool nodal_density_flag) - { - Explicit_Solver_Pointer_ = Explicit_Solver_Pointer; - - valid_fea_modules.push_back(FEA_MODULE_TYPE::SGH); - valid_fea_modules.push_back(FEA_MODULE_TYPE::Dynamic_Elasticity); - nvalid_modules = valid_fea_modules.size(); - - const Simulation_Parameters& simparam = Explicit_Solver_Pointer_->simparam; - for (const auto& fea_module : Explicit_Solver_Pointer_->fea_modules) { - for(int ivalid = 0; ivalid < nvalid_modules; ivalid++){ - if(fea_module->Module_Type==FEA_MODULE_TYPE::SGH){ - FEM_SGH_ = dynamic_cast(fea_module); - set_module_type = FEA_MODULE_TYPE::SGH; - } - if(fea_module->Module_Type==FEA_MODULE_TYPE::Dynamic_Elasticity){ - FEM_Dynamic_Elasticity_ = dynamic_cast(fea_module); - set_module_type = FEA_MODULE_TYPE::Dynamic_Elasticity; - } + bool nodal_density_flag_; + int last_comm_step, last_solve_step, current_step; + size_t nvalid_modules; + std::vector valid_fea_modules; // modules that may interface with this objective function + FEA_MODULE_TYPE set_module_type; + int set_node_, set_dim_; + // std::string my_fea_module = "SGH"; + + AreaNormals(Explicit_Solver* Explicit_Solver_Pointer, int set_node, int set_dim, bool nodal_density_flag) + { + Explicit_Solver_Pointer_ = Explicit_Solver_Pointer; + + valid_fea_modules.push_back(FEA_MODULE_TYPE::SGH); + valid_fea_modules.push_back(FEA_MODULE_TYPE::Dynamic_Elasticity); + nvalid_modules = valid_fea_modules.size(); + + const Simulation_Parameters& simparam = Explicit_Solver_Pointer_->simparam; + for (const auto& fea_module : Explicit_Solver_Pointer_->fea_modules) + { + for (int ivalid = 0; ivalid < nvalid_modules; ivalid++) + { + if (fea_module->Module_Type == FEA_MODULE_TYPE::SGH) + { + FEM_SGH_ = dynamic_cast(fea_module); + set_module_type = FEA_MODULE_TYPE::SGH; + } + if (fea_module->Module_Type == FEA_MODULE_TYPE::Dynamic_Elasticity) + { + FEM_Dynamic_Elasticity_ = dynamic_cast(fea_module); + set_module_type = FEA_MODULE_TYPE::Dynamic_Elasticity; + } + } } - } - nodal_density_flag_ = nodal_density_flag; - last_comm_step = last_solve_step = -1; - current_step = 0; - set_node_ = set_node; - set_dim_ = set_dim; - - //real_t current_kinetic_energy = ROL_Velocities->dot(*ROL_Force)/2; - //std::cout.precision(10); - //if(FEM_->myrank==0) - //std::cout << "INITIAL KINETIC ENERGY " << current_kinetic_energy << std::endl; - } - - void update(const ROL::Vector &z, ROL::UpdateType type, int iter = -1 ) { - } - - real_t value(const ROL::Vector &z, real_t &tol) { - - double area_normal_array[24]; - size_t nodes_in_elem_array[8]; - double node_coords_array[2*8*3]; - ViewCArrayKokkos elem_node_gids(nodes_in_elem_array, 8); - for(int init = 0; init < 8; init++){ - elem_node_gids(init) = init; + nodal_density_flag_ = nodal_density_flag; + last_comm_step = last_solve_step = -1; + current_step = 0; + set_node_ = set_node; + set_dim_ = set_dim; + + // real_t current_kinetic_energy = ROL_Velocities->dot(*ROL_Force)/2; + // std::cout.precision(10); + // if(FEM_->myrank==0) + // std::cout << "INITIAL KINETIC ENERGY " << current_kinetic_energy << std::endl; } - ViewCArrayKokkos area_normal(area_normal_array, 8, 3); - const size_t rk_level = FEM_SGH_->simparam->dynamic_options.rk_num_bins - 1; - //std::cout << "Started obj value on task " <myrank << std::endl; - ROL::Ptr zp = getVector(z); - - const_host_vec_array design_coordinates = zp->getLocalView (Tpetra::Access::ReadOnly); - DViewCArrayKokkos node_coords(node_coords_array,2,8,3); - for(int inode = 0; inode < 8; inode++){ - size_t local_node_id = FEM_SGH_->nodes_in_elem(0,inode); - node_coords(1,inode,0) = design_coordinates(local_node_id,0); - node_coords(1,inode,1) = design_coordinates(local_node_id,1); - node_coords(1,inode,2) = design_coordinates(local_node_id,2); + + void update(const ROL::Vector& z, ROL::UpdateType type, int iter = -1) + { } - // std::cout.precision(10); - // if(Explicit_Solver_Pointer_->myrank==0) - // std::cout << "CURRENT TIME INTEGRAL OF KINETIC ENERGY " << objective_accumulation << std::endl; - FEM_SGH_->get_bmatrix(area_normal, + real_t value(const ROL::Vector& z, real_t& tol) + { + double area_normal_array[24]; + size_t nodes_in_elem_array[8]; + double node_coords_array[2 * 8 * 3]; + ViewCArrayKokkos elem_node_gids(nodes_in_elem_array, 8); + for (int init = 0; init < 8; init++) + { + elem_node_gids(init) = init; + } + ViewCArrayKokkos area_normal(area_normal_array, 8, 3); + const size_t rk_level = FEM_SGH_->simparam->dynamic_options.rk_num_bins - 1; + // std::cout << "Started obj value on task " <myrank << std::endl; + ROL::Ptr zp = getVector(z); + + const_host_vec_array design_coordinates = zp->getLocalView(Tpetra::Access::ReadOnly); + DViewCArrayKokkos node_coords(node_coords_array, 2, 8, 3); + for (int inode = 0; inode < 8; inode++) + { + size_t local_node_id = FEM_SGH_->nodes_in_elem(0, inode); + node_coords(1, inode, 0) = design_coordinates(local_node_id, 0); + node_coords(1, inode, 1) = design_coordinates(local_node_id, 1); + node_coords(1, inode, 2) = design_coordinates(local_node_id, 2); + } + + // std::cout.precision(10); + // if(Explicit_Solver_Pointer_->myrank==0) + // std::cout << "CURRENT TIME INTEGRAL OF KINETIC ENERGY " << objective_accumulation << std::endl; + FEM_SGH_->get_bmatrix(area_normal, 0, node_coords, elem_node_gids, 1); - real_t objective_value = -area_normal(set_node_, set_dim_); - //std::cout << "Ended obj value on task " <myrank << std::endl; - return objective_value; - } - - //void gradient_1( ROL::Vector &g, const ROL::Vector &u, const ROL::Vector &z, real_t &tol ) { - //g.zero(); - //} - - void gradient( ROL::Vector &g, const ROL::Vector &z, real_t &tol ) { - //std::cout << "Started obj gradient on task " <myrank << std::endl; - //get Tpetra multivector pointer from the ROL vector - double area_normal_gradients_array[8*8*3*3]; - size_t nodes_in_elem[8]; - double node_coords_array[2*8*3]; - ViewCArrayKokkos elem_node_gids(nodes_in_elem, 8); - for(int init = 0; init < 8; init++){ - elem_node_gids(init) = init; - } - - ViewCArrayKokkos area_normal_gradients(area_normal_gradients_array, 8, 3, 8, 3); - const size_t rk_level = FEM_SGH_->simparam->dynamic_options.rk_num_bins - 1; - ROL::Ptr zp = getVector(z); - ROL::Ptr gp = getVector(g); - const_host_vec_array design_coordinates = zp->getLocalView (Tpetra::Access::ReadOnly); - host_vec_array design_gradients = gp->getLocalView (Tpetra::Access::ReadWrite); - DViewCArrayKokkos node_coords(node_coords_array,2,8,3); - for(int inode = 0; inode < 8; inode++){ - size_t local_node_id = FEM_SGH_->nodes_in_elem(0,inode); - node_coords(1,inode,0) = design_coordinates(local_node_id,0); - node_coords(1,inode,1) = design_coordinates(local_node_id,1); - node_coords(1,inode,2) = design_coordinates(local_node_id,2); + real_t objective_value = -area_normal(set_node_, set_dim_); + // std::cout << "Ended obj value on task " <myrank << std::endl; + return objective_value; } - for (int igradient = 0; igradient < FEM_SGH_->nlocal_nodes+FEM_SGH_->nghost_nodes; igradient++){ - design_gradients(igradient,0) = 0; - design_gradients(igradient,1) = 0; - design_gradients(igradient,2) = 0; - } + // void gradient_1( ROL::Vector &g, const ROL::Vector &u, const ROL::Vector &z, real_t &tol ) { + // g.zero(); + // } + void gradient(ROL::Vector& g, const ROL::Vector& z, real_t& tol) + { + // std::cout << "Started obj gradient on task " <myrank << std::endl; + // get Tpetra multivector pointer from the ROL vector + double area_normal_gradients_array[8 * 8 * 3 * 3]; + size_t nodes_in_elem[8]; + double node_coords_array[2 * 8 * 3]; + ViewCArrayKokkos elem_node_gids(nodes_in_elem, 8); + for (int init = 0; init < 8; init++) + { + elem_node_gids(init) = init; + } + + ViewCArrayKokkos area_normal_gradients(area_normal_gradients_array, 8, 3, 8, 3); + const size_t rk_level = FEM_SGH_->simparam->dynamic_options.rk_num_bins - 1; + ROL::Ptr zp = getVector(z); + ROL::Ptr gp = getVector(g); + const_host_vec_array design_coordinates = zp->getLocalView(Tpetra::Access::ReadOnly); + host_vec_array design_gradients = gp->getLocalView(Tpetra::Access::ReadWrite); + DViewCArrayKokkos node_coords(node_coords_array, 2, 8, 3); + for (int inode = 0; inode < 8; inode++) + { + size_t local_node_id = FEM_SGH_->nodes_in_elem(0, inode); + node_coords(1, inode, 0) = design_coordinates(local_node_id, 0); + node_coords(1, inode, 1) = design_coordinates(local_node_id, 1); + node_coords(1, inode, 2) = design_coordinates(local_node_id, 2); + } - FEM_SGH_->get_bmatrix_gradients(area_normal_gradients, + for (int igradient = 0; igradient < FEM_SGH_->nlocal_nodes + FEM_SGH_->nghost_nodes; igradient++) + { + design_gradients(igradient, 0) = 0; + design_gradients(igradient, 1) = 0; + design_gradients(igradient, 2) = 0; + } + + FEM_SGH_->get_bmatrix_gradients(area_normal_gradients, 0, node_coords, elem_node_gids, 1); - for(int inode = 0; inode < 8; inode++){ - size_t local_node_id = FEM_SGH_->nodes_in_elem(0,inode); - for(int idim = 0; idim < 3; idim++) - design_gradients(local_node_id,idim) = -area_normal_gradients(set_node_,set_dim_,inode,idim); + for (int inode = 0; inode < 8; inode++) + { + size_t local_node_id = FEM_SGH_->nodes_in_elem(0, inode); + for (int idim = 0; idim < 3; idim++) + { + design_gradients(local_node_id, idim) = -area_normal_gradients(set_node_, set_dim_, inode, idim); + } + } } - } - }; #endif // end header guard diff --git a/src/Parallel-Solvers/Parallel-Explicit/Topology_Optimization/Kinetic_Energy_Minimize.h b/src/Parallel-Solvers/Parallel-Explicit/Topology_Optimization/Kinetic_Energy_Minimize.h index 4ac860c6b..4799a8b8a 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Topology_Optimization/Kinetic_Energy_Minimize.h +++ b/src/Parallel-Solvers/Parallel-Explicit/Topology_Optimization/Kinetic_Energy_Minimize.h @@ -71,7 +71,7 @@ typedef Tpetra::MultiVector MV; typedef ROL::Vector V; typedef ROL::TpetraMultiVector ROL_MV; -using traits = Kokkos::ViewTraits; +using traits = Kokkos::ViewTraits; using array_layout = typename traits::array_layout; using execution_space = typename traits::execution_space; using device_type = typename traits::device_type; @@ -146,9 +146,9 @@ typedef MV::dual_view_type dual_vec_array; } } nodal_density_flag_ = nodal_density_flag; - last_comm_step = last_solve_step = -1; - current_step = 0; - time_accumulation = true; + last_comm_step = last_solve_step = -1; + current_step = 0; + time_accumulation = true; objective_accumulation = 0; // ROL_Force = ROL::makePtr(FEM_->Global_Nodal_Forces); @@ -182,7 +182,7 @@ typedef MV::dual_view_type dual_vec_array; void update_elasticity(const ROL::Vector& z, ROL::UpdateType type, int iter = -1) { // debug - std::ostream& out = std::cout; + std::ostream& out = std::cout; Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); current_step++; @@ -247,7 +247,7 @@ typedef MV::dual_view_type dual_vec_array; void update_sgh(const ROL::Vector& z, ROL::UpdateType type, int iter = -1) { // debug - std::ostream& out = std::cout; + std::ostream& out = std::cout; Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); current_step++; @@ -313,7 +313,7 @@ typedef MV::dual_view_type dual_vec_array; { // std::cout << "Started obj value on task " <myrank << std::endl; ROL::Ptr zp = getVector(z); - real_t c = 0.0; + real_t c = 0.0; // debug print // std::ostream &out = std::cout; @@ -376,7 +376,7 @@ typedef MV::dual_view_type dual_vec_array; // std::cout << "Started obj gradient on task " <myrank << std::endl; // get Tpetra multivector pointer from the ROL vector ROL::Ptr zp = getVector(z); - ROL::Ptr gp = getVector(g); + ROL::Ptr gp = getVector(g); // communicate ghosts and solve for nodal degrees of freedom as a function of the current design variables // FEM_->gradient_print_sync=1; diff --git a/src/Parallel-Solvers/Parallel-Explicit/main.cpp b/src/Parallel-Solvers/Parallel-Explicit/main.cpp index 8f0d8151b..eec328b07 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/main.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/main.cpp @@ -49,9 +49,8 @@ void solver_setup(const char* filename) { - Explicit_Solver solver( - Yaml::from_file_strict(filename) - ); + Explicit_Solver solver(Yaml::from_file_strict(filename)); + // checks for optional solver routines if (solver.setup_flag) { From 5648cd5399b8de24606a12988fb324959fefed72 Mon Sep 17 00:00:00 2001 From: Jacob Moore Date: Thu, 22 Feb 2024 19:29:37 -0600 Subject: [PATCH 27/64] STYLE: Uncrustify --- .../FEA_Module_Dynamic_Elasticity.cpp | 31 +- .../FEA_Module_Dynamic_Elasticity.h | 10 +- .../elastic_optimization.cpp | 2822 +++++++++-------- .../Dynamic_Elastic_Solver/properties.cpp | 3 - .../Eulerian_Solver/FEA_Module_Eulerian.cpp | 21 +- .../Eulerian_Solver/FEA_Module_Eulerian.h | 10 +- .../Simulation_Parameters_Eulerian.h | 177 +- .../SGH_Solver/src/force_sgh.cpp | 8 +- .../SGH_Solver/src/properties.cpp | 4 - 9 files changed, 1595 insertions(+), 1491 deletions(-) diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.cpp b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.cpp index 45f3e5a7a..0561d4d5a 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.cpp @@ -1012,19 +1012,18 @@ void FEA_Module_Dynamic_Elasticity::setup() // FEA_Module bc variable num_boundary_conditions = num_bcs; - - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos mat_fill = simparam->mat_fill; - const DCArrayKokkos material = simparam->material; + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos mat_fill = simparam->mat_fill; + const DCArrayKokkos material = simparam->material; eos_global_vars = simparam->eos_global_vars; strength_global_vars = simparam->strength_global_vars; - eos_state_vars = DCArrayKokkos (rnum_elem, simparam->max_num_eos_state_vars); - strength_state_vars = DCArrayKokkos (rnum_elem, simparam->max_num_strength_state_vars); - elem_user_output_vars = DCArrayKokkos (rnum_elem, simparam->output_options.max_num_user_output_vars); - - //--- calculate bdy sets ---// - mesh->num_nodes_in_patch = 2*(num_dim-1); // 2 (2D) or 4 (3D) - mesh->num_patches_in_elem = 2*num_dim; // 4 (2D) or 6 (3D) + eos_state_vars = DCArrayKokkos(rnum_elem, simparam->max_num_eos_state_vars); + strength_state_vars = DCArrayKokkos(rnum_elem, simparam->max_num_strength_state_vars); + elem_user_output_vars = DCArrayKokkos(rnum_elem, simparam->output_options.max_num_user_output_vars); + + // --- calculate bdy sets ---// + mesh->num_nodes_in_patch = 2 * (num_dim - 1); // 2 (2D) or 4 (3D) + mesh->num_patches_in_elem = 2 * num_dim; // 4 (2D) or 6 (3D) mesh->init_bdy_sets(num_bcs); num_bdy_sets = mesh->num_bdy_sets; @@ -1339,11 +1338,11 @@ void FEA_Module_Dynamic_Elasticity::setup() // p = rho*ie*(gamma - 1) size_t mat_id = f_id; -<<<<<<< HEAD - double gamma = global_vars(mat_id, 0); // gamma value -======= - double gamma = eos_global_vars(mat_id,0); // gamma value ->>>>>>> main + << << << < HEAD + double gamma = global_vars(mat_id, 0); // gamma value + == == == = + double gamma = eos_global_vars(mat_id, 0); // gamma value + >> >> >> > main elem_sie(rk_level, elem_gid) = elem_pres(elem_gid) / (mat_fill(f_id).den * (gamma - 1.0)); } // end if diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.h b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.h index acb693882..e0ed923e6 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.h +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.h @@ -438,15 +438,13 @@ class FEA_Module_Dynamic_Elasticity : public FEA_Module RaggedRightArrayKokkos corners_in_node; CArrayKokkos num_corners_in_node; - // for storing global variables used in user material model - DCArrayKokkos eos_global_vars; - DCArrayKokkos strength_global_vars; + DCArrayKokkos eos_global_vars; + DCArrayKokkos strength_global_vars; // for storing state variables used in user material model - DCArrayKokkos eos_state_vars; - DCArrayKokkos strength_state_vars; - + DCArrayKokkos eos_state_vars; + DCArrayKokkos strength_state_vars; // node ids in elem DCArrayKokkos nodes_in_elem; diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/elastic_optimization.cpp b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/elastic_optimization.cpp index c83144f97..490e6cb6d 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/elastic_optimization.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/elastic_optimization.cpp @@ -38,7 +38,7 @@ #include #include #include -#include +#include #include // fmin, fmax, abs note: fminl is long #include #include @@ -67,7 +67,7 @@ #include "FEA_Module_Dynamic_Elasticity.h" #include "Explicit_Solver.h" -//optimization +// optimization #include "ROL_Solver.hpp" #include "Kinetic_Energy_Minimize.h" @@ -75,210 +75,224 @@ Compute new system response due to the design variable update ------------------------------------------------------------------------- */ -void FEA_Module_Dynamic_Elasticity::update_forward_solve(Teuchos::RCP zp){ - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - //local variable for host view in the dual view - int num_dim = simparam->num_dims; - int nodes_per_elem = max_nodes_per_element; - int local_node_index, current_row, current_column; - int max_stride = 0; - int current_module_index; - size_t access_index, row_access_index, row_counter; - GO global_index, global_dof_index; - LO local_dof_index; - const size_t num_fills = simparam->regions.size(); - const size_t rk_num_bins = simparam->dynamic_options.rk_num_bins; - const size_t num_bcs = module_params->boundary_conditions.size(); - const size_t num_materials = simparam->materials.size(); - real_t objective_accumulation; - - // --- Read in the nodes in the mesh --- - int myrank = Explicit_Solver_Pointer_->myrank; - int nranks = Explicit_Solver_Pointer_->nranks; - - const DCArrayKokkos mat_fill = simparam->mat_fill; - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos material = simparam->material; - CArray current_element_nodal_densities = CArray(num_nodes_in_elem); - - std::vector> FEA_Module_My_TO_Modules = simparam->FEA_Module_My_TO_Modules; - problem = Explicit_Solver_Pointer_->problem; //Pointer to ROL optimization problem object - ROL::Ptr> obj_pointer; - - //compute element averaged density ratios corresponding to nodal density design variables - {//view scope - const_host_vec_array all_node_densities = all_node_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); - //debug print - //std::cout << "NODE DENSITY TEST " << all_node_densities(0,0) << std::endl; - for(int elem_id = 0; elem_id < rnum_elem; elem_id++){ - for(int inode = 0; inode < num_nodes_in_elem; inode++){ - current_element_nodal_densities(inode) = all_node_densities(nodes_in_elem(elem_id,inode),0); - } - relative_element_densities.host(elem_id) = average_element_density(num_nodes_in_elem, current_element_nodal_densities); - }//for - } //view scope - //debug print - //std::cout << "ELEMENT RELATIVE DENSITY TEST " << relative_element_densities.host(0) << std::endl; - relative_element_densities.update_device(); - - //set density vector to the current value chosen by the optimizer - test_node_densities_distributed = zp; - - //reset nodal coordinates to initial values - node_coords_distributed->assign(*initial_node_coords_distributed); - - //comms for ghosts - Explicit_Solver_Pointer_->comm_coordinates(); - - //view scope - { - const_vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array ghost_node_coords_interface = Explicit_Solver_Pointer_->ghost_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - all_node_coords_interface(node_gid,idim) = node_coords_interface(node_gid,idim); - } +void FEA_Module_Dynamic_Elasticity::update_forward_solve(Teuchos::RCP zp) +{ + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + // local variable for host view in the dual view + int num_dim = simparam->num_dims; + int nodes_per_elem = max_nodes_per_element; + int local_node_index, current_row, current_column; + int max_stride = 0; + int current_module_index; + size_t access_index, row_access_index, row_counter; + GO global_index, global_dof_index; + LO local_dof_index; + const size_t num_fills = simparam->regions.size(); + const size_t rk_num_bins = simparam->dynamic_options.rk_num_bins; + const size_t num_bcs = module_params->boundary_conditions.size(); + const size_t num_materials = simparam->materials.size(); + real_t objective_accumulation; + + // --- Read in the nodes in the mesh --- + int myrank = Explicit_Solver_Pointer_->myrank; + int nranks = Explicit_Solver_Pointer_->nranks; + + const DCArrayKokkos mat_fill = simparam->mat_fill; + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos material = simparam->material; + CArray current_element_nodal_densities = CArray(num_nodes_in_elem); + + std::vector> FEA_Module_My_TO_Modules = simparam->FEA_Module_My_TO_Modules; + problem = Explicit_Solver_Pointer_->problem; // Pointer to ROL optimization problem object + ROL::Ptr> obj_pointer; + + // compute element averaged density ratios corresponding to nodal density design variables + { // view scope + const_host_vec_array all_node_densities = all_node_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); + // debug print + // std::cout << "NODE DENSITY TEST " << all_node_densities(0,0) << std::endl; + for (int elem_id = 0; elem_id < rnum_elem; elem_id++) + { + for (int inode = 0; inode < num_nodes_in_elem; inode++) + { + current_element_nodal_densities(inode) = all_node_densities(nodes_in_elem(elem_id, inode), 0); + } + relative_element_densities.host(elem_id) = average_element_density(num_nodes_in_elem, current_element_nodal_densities); + } // for + } // view scope + // debug print + // std::cout << "ELEMENT RELATIVE DENSITY TEST " << relative_element_densities.host(0) << std::endl; + relative_element_densities.update_device(); + + // set density vector to the current value chosen by the optimizer + test_node_densities_distributed = zp; + + // reset nodal coordinates to initial values + node_coords_distributed->assign(*initial_node_coords_distributed); + + // comms for ghosts + Explicit_Solver_Pointer_->comm_coordinates(); + + // view scope + { + const_vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array ghost_node_coords_interface = Explicit_Solver_Pointer_->ghost_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + all_node_coords_interface(node_gid, idim) = node_coords_interface(node_gid, idim); + } }); // end parallel for - Kokkos::fence(); + Kokkos::fence(); - FOR_ALL_CLASS(node_gid, nlocal_nodes, nlocal_nodes+nghost_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - all_node_coords_interface(node_gid,idim) = ghost_node_coords_interface(node_gid-nlocal_nodes,idim); - } + FOR_ALL_CLASS(node_gid, nlocal_nodes, nlocal_nodes + nghost_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + all_node_coords_interface(node_gid, idim) = ghost_node_coords_interface(node_gid - nlocal_nodes, idim); + } }); // end parallel for - Kokkos::fence(); - } //end view scope - - //reset velocities to initial conditions - node_velocities_distributed->assign(*initial_node_velocities_distributed); - - //reset time accumulating objective and constraints - /* - for(int imodule = 0 ; imodule < FEA_Module_My_TO_Modules[my_fea_module_index_].size(); imodule++){ - current_module_index = FEA_Module_My_TO_Modules[my_fea_module_index_][imodule]; - //test if module needs reset - if(){ - - } - } - */ - //simple setup to just request KE for now; above loop to be expanded and used later for scanning modules - obj_pointer = problem->getObjective(); - KineticEnergyMinimize_TopOpt& kinetic_energy_minimize_function = dynamic_cast(*obj_pointer); - kinetic_energy_minimize_function.objective_accumulation = 0; - - //interface trial density vector - - //interfacing of vectors(should be removed later once made compatible) - //view scope - { - host_vec_array interface_node_coords = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); - for(size_t ibin = 0; ibin < rk_num_bins; ibin++){ - //save node data to node.coords - //std::cout << "NODE DATA ON RANK " << myrank << std::endl; - if(num_dim==2){ - for(int inode = 0; inode < nall_nodes; inode++){ - //std::cout << "Node index " << inode+1 << " "; - node_coords.host(ibin,inode,0) = interface_node_coords(inode,0); - //std::cout << host_node_coords_state(0,inode,0)+1<< " "; - node_coords.host(ibin,inode,1) = interface_node_coords(inode,1); - //std::cout << host_node_coords_state(0,inode,1)+1<< " "; - } - } - else if(num_dim==3){ - for(int inode = 0; inode < nall_nodes; inode++){ - //std::cout << "Node index " << inode+1 << " "; - node_coords.host(ibin,inode,0) = interface_node_coords(inode,0); - //std::cout << host_node_coords_state(0,inode,0)+1<< " "; - node_coords.host(ibin,inode,1) = interface_node_coords(inode,1); - //std::cout << host_node_coords_state(0,inode,1)+1<< " "; - - node_coords.host(ibin,inode,2) = interface_node_coords(inode,2); - //std::cout << host_node_coords_state(0,inode,2)+1<< std::endl; - } + Kokkos::fence(); + } // end view scope + + // reset velocities to initial conditions + node_velocities_distributed->assign(*initial_node_velocities_distributed); + + // reset time accumulating objective and constraints + /* + for(int imodule = 0 ; imodule < FEA_Module_My_TO_Modules[my_fea_module_index_].size(); imodule++){ + current_module_index = FEA_Module_My_TO_Modules[my_fea_module_index_][imodule]; + //test if module needs reset + if(){ + } } - } //end view scope + */ + // simple setup to just request KE for now; above loop to be expanded and used later for scanning modules + obj_pointer = problem->getObjective(); + KineticEnergyMinimize_TopOpt& kinetic_energy_minimize_function = dynamic_cast(*obj_pointer); + kinetic_energy_minimize_function.objective_accumulation = 0; + + // interface trial density vector + + // interfacing of vectors(should be removed later once made compatible) + // view scope + { + host_vec_array interface_node_coords = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + for (size_t ibin = 0; ibin < rk_num_bins; ibin++) + { + // save node data to node.coords + // std::cout << "NODE DATA ON RANK " << myrank << std::endl; + if (num_dim == 2) + { + for (int inode = 0; inode < nall_nodes; inode++) + { + // std::cout << "Node index " << inode+1 << " "; + node_coords.host(ibin, inode, 0) = interface_node_coords(inode, 0); + // std::cout << host_node_coords_state(0,inode,0)+1<< " "; + node_coords.host(ibin, inode, 1) = interface_node_coords(inode, 1); + // std::cout << host_node_coords_state(0,inode,1)+1<< " "; + } + } + else if (num_dim == 3) + { + for (int inode = 0; inode < nall_nodes; inode++) + { + // std::cout << "Node index " << inode+1 << " "; + node_coords.host(ibin, inode, 0) = interface_node_coords(inode, 0); + // std::cout << host_node_coords_state(0,inode,0)+1<< " "; + node_coords.host(ibin, inode, 1) = interface_node_coords(inode, 1); + // std::cout << host_node_coords_state(0,inode,1)+1<< " "; + + node_coords.host(ibin, inode, 2) = interface_node_coords(inode, 2); + // std::cout << host_node_coords_state(0,inode,2)+1<< std::endl; + } + } + } + } // end view scope // save the node coords to the current RK value - for (size_t node_gid=0; node_gid < nall_nodes; node_gid++){ - - for(int rk=1; rk elem_user_output_vars, elem_sspd, elem_den(elem_gid), - elem_sie(rk_level,elem_gid)); + elem_sie(rk_level, elem_gid)); - // --- Sound speed --- - elem_eos(elem_gid).calc_sound_speed(elem_pres, + // --- Sound speed --- + elem_eos(elem_gid).calc_sound_speed(elem_pres, elem_stress, elem_gid, elem_mat_id(elem_gid), @@ -303,229 +317,224 @@ void FEA_Module_Dynamic_Elasticity::update_forward_solve(Teuchos::RCP elem_user_output_vars, elem_sspd, elem_den(elem_gid), - elem_sie(rk_level,elem_gid)); - - // loop over the nodes of this element and apply velocity - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - - // get the mesh node index - size_t node_gid = nodes_in_elem(elem_gid, node_lid); + elem_sie(rk_level, elem_gid)); - - // --- Velocity --- - switch(mat_fill(f_id).velocity) + // loop over the nodes of this element and apply velocity + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { - case VELOCITY_TYPE::cartesian: - { - - node_vel(rk_level, node_gid, 0) = mat_fill(f_id).u; - node_vel(rk_level, node_gid, 1) = mat_fill(f_id).v; - if (num_dim == 3) node_vel(rk_level, node_gid, 2) = mat_fill(f_id).w; - - - break; - } - case VELOCITY_TYPE::radial: + // get the mesh node index + size_t node_gid = nodes_in_elem(elem_gid, node_lid); + + // --- Velocity --- + switch (mat_fill(f_id).velocity) { - // Setting up cylindrical - double dir[2]; - dir[0] = 0.0; - dir[1] = 0.0; - double radius_val = 0.0; - - for(int dim=0; dim<2; dim++){ - dir[dim] = node_coords(rk_level, node_gid, dim); - radius_val += node_coords(rk_level, node_gid, dim)*node_coords(rk_level, node_gid, dim); - } // end for - radius_val = sqrt(radius_val); - - for(int dim=0; dim<2; dim++){ - if (radius_val > 1.0e-14){ - dir[dim] /= (radius_val); + case VELOCITY_TYPE::cartesian: + { + node_vel(rk_level, node_gid, 0) = mat_fill(f_id).u; + node_vel(rk_level, node_gid, 1) = mat_fill(f_id).v; + if (num_dim == 3) + { + node_vel(rk_level, node_gid, 2) = mat_fill(f_id).w; + } + + break; } - else{ - dir[dim] = 0.0; + case VELOCITY_TYPE::radial: + { + // Setting up cylindrical + double dir[2]; + dir[0] = 0.0; + dir[1] = 0.0; + double radius_val = 0.0; + + for (int dim = 0; dim < 2; dim++) + { + dir[dim] = node_coords(rk_level, node_gid, dim); + radius_val += node_coords(rk_level, node_gid, dim) * node_coords(rk_level, node_gid, dim); + } // end for + radius_val = sqrt(radius_val); + + for (int dim = 0; dim < 2; dim++) + { + if (radius_val > 1.0e-14) + { + dir[dim] /= (radius_val); + } + else + { + dir[dim] = 0.0; + } + } // end for + + node_vel(rk_level, node_gid, 0) = mat_fill(f_id).speed * dir[0]; + node_vel(rk_level, node_gid, 1) = mat_fill(f_id).speed * dir[1]; + if (num_dim == 3) + { + node_vel(rk_level, node_gid, 2) = 0.0; + } + + break; } - } // end for - - - node_vel(rk_level, node_gid, 0) = mat_fill(f_id).speed*dir[0]; - node_vel(rk_level, node_gid, 1) = mat_fill(f_id).speed*dir[1]; - if (num_dim == 3) node_vel(rk_level, node_gid, 2) = 0.0; - - break; - } - case VELOCITY_TYPE::spherical: - { - - // Setting up spherical - double dir[3]; - dir[0] = 0.0; - dir[1] = 0.0; - dir[2] = 0.0; - double radius_val = 0.0; - - for(int dim=0; dim<3; dim++){ - dir[dim] = node_coords(rk_level, node_gid, dim); - radius_val += node_coords(rk_level, node_gid, dim)*node_coords(rk_level, node_gid, dim); - } // end for - radius_val = sqrt(radius_val); - - for(int dim=0; dim<3; dim++){ - if (radius_val > 1.0e-14){ - dir[dim] /= (radius_val); + case VELOCITY_TYPE::spherical: + { + // Setting up spherical + double dir[3]; + dir[0] = 0.0; + dir[1] = 0.0; + dir[2] = 0.0; + double radius_val = 0.0; + + for (int dim = 0; dim < 3; dim++) + { + dir[dim] = node_coords(rk_level, node_gid, dim); + radius_val += node_coords(rk_level, node_gid, dim) * node_coords(rk_level, node_gid, dim); + } // end for + radius_val = sqrt(radius_val); + + for (int dim = 0; dim < 3; dim++) + { + if (radius_val > 1.0e-14) + { + dir[dim] /= (radius_val); + } + else + { + dir[dim] = 0.0; + } + } // end for + + node_vel(rk_level, node_gid, 0) = mat_fill(f_id).speed * dir[0]; + node_vel(rk_level, node_gid, 1) = mat_fill(f_id).speed * dir[1]; + if (num_dim == 3) + { + node_vel(rk_level, node_gid, 2) = mat_fill(f_id).speed * dir[2]; + } + + break; } - else{ - dir[dim] = 0.0; + case VELOCITY_TYPE::radial_linear: + { + break; } - } // end for - - node_vel(rk_level, node_gid, 0) = mat_fill(f_id).speed*dir[0]; - node_vel(rk_level, node_gid, 1) = mat_fill(f_id).speed*dir[1]; - if (num_dim == 3) node_vel(rk_level, node_gid, 2) = mat_fill(f_id).speed*dir[2]; - - break; - } - case VELOCITY_TYPE::radial_linear: - { - - break; - } - case VELOCITY_TYPE::spherical_linear: - { - - break; - } - case VELOCITY_TYPE::tg_vortex: - { - - node_vel(rk_level, node_gid, 0) = sin(PI * node_coords(rk_level,node_gid, 0)) * cos(PI * node_coords(rk_level,node_gid, 1)); - node_vel(rk_level, node_gid, 1) = -1.0*cos(PI * node_coords(rk_level,node_gid, 0)) * sin(PI * node_coords(rk_level,node_gid, 1)); - if (num_dim == 3) node_vel(rk_level, node_gid, 2) = 0.0; - - break; - } - } // end of switch - - }// end loop over nodes of element - - - if(mat_fill(f_id).velocity == VELOCITY_TYPE::tg_vortex) - { - elem_pres(elem_gid) = 0.25*( cos(2.0*PI*elem_coords[0]) + cos(2.0*PI*elem_coords[1]) ) + 1.0; - - // p = rho*ie*(gamma - 1) - size_t mat_id = f_id; - double gamma = eos_global_vars(mat_id,0); // gamma value - elem_sie(rk_level, elem_gid) = - elem_pres(elem_gid)/(mat_fill(f_id).den*(gamma - 1.0)); - } // end if + case VELOCITY_TYPE::spherical_linear: + { + break; + } + case VELOCITY_TYPE::tg_vortex: + { + node_vel(rk_level, node_gid, 0) = sin(PI * node_coords(rk_level, node_gid, 0)) * cos(PI * node_coords(rk_level, node_gid, 1)); + node_vel(rk_level, node_gid, 1) = -1.0 * cos(PI * node_coords(rk_level, node_gid, 0)) * sin(PI * node_coords(rk_level, node_gid, 1)); + if (num_dim == 3) + { + node_vel(rk_level, node_gid, 2) = 0.0; + } + + break; + } + } // end of switch + } // end loop over nodes of element - } // end if fill - + if (mat_fill(f_id).velocity == VELOCITY_TYPE::tg_vortex) + { + elem_pres(elem_gid) = 0.25 * (cos(2.0 * PI * elem_coords[0]) + cos(2.0 * PI * elem_coords[1]) ) + 1.0; + + // p = rho*ie*(gamma - 1) + size_t mat_id = f_id; + double gamma = eos_global_vars(mat_id, 0); // gamma value + elem_sie(rk_level, elem_gid) = + elem_pres(elem_gid) / (mat_fill(f_id).den * (gamma - 1.0)); + } // end if + } // end if fill }); // end FOR_ALL_CLASS element loop - Kokkos::fence(); - - - } // end for loop over fills - }//end view scope - - - + Kokkos::fence(); + } // end for loop over fills + } // end view scope + // apply BC's to velocity FEA_Module_Dynamic_Elasticity::boundary_velocity(*mesh, boundary, node_vel); - - + // calculate the corner massess if 2D - if(num_dim==2){ - + if (num_dim == 2) + { FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { - // facial area of the corners double corner_areas_array[4]; - - ViewCArrayKokkos corner_areas(&corner_areas_array[0],4); - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 4); - + + ViewCArrayKokkos corner_areas(&corner_areas_array[0], 4); + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 4); + get_area_weights2D(corner_areas, elem_gid, node_coords, elem_node_gids, rk_level); - + // loop over the corners of the element and calculate the mass - for (size_t corner_lid=0; corner_lid<4; corner_lid++){ - + for (size_t corner_lid = 0; corner_lid < 4; corner_lid++) + { size_t corner_gid = corners_in_elem(elem_gid, corner_lid); - corner_mass(corner_gid) = corner_areas(corner_lid)*elem_den(elem_gid); // node radius is added later - + corner_mass(corner_gid) = corner_areas(corner_lid) * elem_den(elem_gid); // node radius is added later } // end for over corners }); - } // end of - - + // calculate the nodal mass FOR_ALL_CLASS(node_gid, 0, nall_nodes, { - node_mass(node_gid) = 0.0; - - if(num_dim==3){ - - for(size_t elem_lid=0; elem_lid < num_corners_in_node(node_gid); elem_lid++){ - size_t elem_gid = elems_in_node(node_gid,elem_lid); - node_mass(node_gid) += 1.0/8.0*elem_mass(elem_gid); + + if (num_dim == 3) + { + for (size_t elem_lid = 0; elem_lid < num_corners_in_node(node_gid); elem_lid++) + { + size_t elem_gid = elems_in_node(node_gid, elem_lid); + node_mass(node_gid) += 1.0 / 8.0 * elem_mass(elem_gid); } // end for elem_lid - - }// end if dims=3 - else { - + } // end if dims=3 + else + { // 2D-RZ - for(size_t corner_lid=0; corner_lid < num_corners_in_node(node_gid); corner_lid++){ - - size_t corner_gid = corners_in_node(node_gid, corner_lid); + for (size_t corner_lid = 0; corner_lid < num_corners_in_node(node_gid); corner_lid++) + { + size_t corner_gid = corners_in_node(node_gid, corner_lid); node_mass(node_gid) += corner_mass(corner_gid); // sans the radius so it is areal node mass - - corner_mass(corner_gid) *= node_coords(rk_level,node_gid,1); // true corner mass now + + corner_mass(corner_gid) *= node_coords(rk_level, node_gid, 1); // true corner mass now } // end for elem_lid - } // end else - }); // end FOR_ALL_CLASS Kokkos::fence(); - - //current interface has differing mass arrays; this equates them until we unify memory - //view scope + // current interface has differing mass arrays; this equates them until we unify memory + // view scope { - vec_array node_mass_interface = node_masses_distributed->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - node_mass_interface(node_gid,0) = node_mass(node_gid); + vec_array node_mass_interface = node_masses_distributed->getLocalView(Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + node_mass_interface(node_gid, 0) = node_mass(node_gid); }); // end parallel for - } //end view scope + } // end view scope Kokkos::fence(); - //communicate ghost densities + // communicate ghost densities comm_node_masses(); - //this is forcing a copy to the device - //view scope + // this is forcing a copy to the device + // view scope { - vec_array ghost_node_mass_interface = ghost_node_masses_distributed->getLocalView (Tpetra::Access::ReadWrite); + vec_array ghost_node_mass_interface = ghost_node_masses_distributed->getLocalView(Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, nlocal_nodes, nall_nodes, { - node_mass(node_gid) = ghost_node_mass_interface(node_gid-nlocal_nodes,0); + FOR_ALL_CLASS(node_gid, nlocal_nodes, nall_nodes, { + node_mass(node_gid) = ghost_node_mass_interface(node_gid - nlocal_nodes, 0); }); // end parallel for - } //end view scope + } // end view scope Kokkos::fence(); - //update stiffness matrix - if(simparam->topology_optimization_on || simparam->shape_optimization_on){ - assemble_matrix(); + // update stiffness matrix + if (simparam->topology_optimization_on || simparam->shape_optimization_on) + { + assemble_matrix(); } - - //execute solve - elastic_solve(); + // execute solve + elastic_solve(); } /* ------------------------------------------------------------------------------------------- @@ -534,141 +543,163 @@ void FEA_Module_Dynamic_Elasticity::update_forward_solve(Teuchos::RCP double FEA_Module_Dynamic_Elasticity::average_element_density(const int nodes_per_elem, const CArray current_element_densities) const { - double result = 0; - for(int i=0; i < nodes_per_elem; i++){ - result += current_element_densities(i)/nodes_per_elem; - } + double result = 0; + for (int i = 0; i < nodes_per_elem; i++) + { + result += current_element_densities(i) / nodes_per_elem; + } - return result; + return result; } /* --------------------------------------------------------------------------------------------------------------- - Simpler adjoint vector solve for the kinetic energy minimization problem + Simpler adjoint vector solve for the kinetic energy minimization problem when force does not depend on u and v. ------------------------------------------------------------------------------------------------------------------ */ -void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_adjoint(){ - - size_t num_bdy_nodes = mesh->num_bdy_nodes; - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos material = simparam->material; - const int num_dim = simparam->num_dims; - real_t global_dt; - size_t current_data_index, next_data_index; - Teuchos::RCP previous_adjoint_vector_distributed, current_adjoint_vector_distributed, previous_velocity_vector_distributed, current_velocity_vector_distributed; - //initialize first adjoint vector at last_time_step to 0 as the terminal value - (*adjoint_vector_data)[last_time_step+1]->putScalar(0); - - //solve terminal value problem, proceeds in time backward. For simplicity, we use the same timestep data from the forward solve. - //A linear interpolant is assumed between velocity data points; velocity midpoint is used to update the adjoint. - if(myrank==0) - std::cout << "Computing adjoint vector " << time_data.size() << std::endl; - - for (int cycle = last_time_step; cycle >= 0; cycle--) { - //compute timestep from time data - global_dt = time_data[cycle+1] - time_data[cycle]; - - //print - if(simparam->dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ - if (cycle==last_time_step){ - if(myrank==0) - printf("cycle = %d, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } - // print time step every 10 cycles - else if (cycle%20==0){ - if(myrank==0) - printf("cycle = %d, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } // end if +void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_adjoint() +{ + size_t num_bdy_nodes = mesh->num_bdy_nodes; + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos material = simparam->material; + const int num_dim = simparam->num_dims; + real_t global_dt; + size_t current_data_index, next_data_index; + Teuchos::RCP previous_adjoint_vector_distributed, current_adjoint_vector_distributed, previous_velocity_vector_distributed, current_velocity_vector_distributed; + // initialize first adjoint vector at last_time_step to 0 as the terminal value + (*adjoint_vector_data)[last_time_step + 1]->putScalar(0); + + // solve terminal value problem, proceeds in time backward. For simplicity, we use the same timestep data from the forward solve. + // A linear interpolant is assumed between velocity data points; velocity midpoint is used to update the adjoint. + if (myrank == 0) + { + std::cout << "Computing adjoint vector " << time_data.size() << std::endl; } - //else if (cycle==1){ - //if(myrank==0) - //printf("cycle = %lu, time = %f, time step = %f \n", cycle-1, time_data[cycle-1], global_dt); - //} // end if - //compute adjoint vector for this data point; use velocity midpoint - //view scope + for (int cycle = last_time_step; cycle >= 0; cycle--) { - const_vec_array previous_velocity_vector = (*forward_solve_velocity_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - - const_vec_array previous_adjoint_vector = (*adjoint_vector_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView (Tpetra::Access::ReadWrite); - - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes + nghost_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - //cancellation of half from midpoint and 2 from adjoint equation already done - current_adjoint_vector(node_gid,idim) = -0.5*(current_velocity_vector(node_gid,idim)+previous_velocity_vector(node_gid,idim))*global_dt + previous_adjoint_vector(node_gid,idim); - } + // compute timestep from time data + global_dt = time_data[cycle + 1] - time_data[cycle]; + + // print + if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) + { + if (cycle == last_time_step) + { + if (myrank == 0) + { + printf("cycle = %d, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } + // print time step every 10 cycles + else if (cycle % 20 == 0) + { + if (myrank == 0) + { + printf("cycle = %d, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } // end if + } + // else if (cycle==1){ + // if(myrank==0) + // printf("cycle = %lu, time = %f, time step = %f \n", cycle-1, time_data[cycle-1], global_dt); + // } // end if + + // compute adjoint vector for this data point; use velocity midpoint + // view scope + { + const_vec_array previous_velocity_vector = (*forward_solve_velocity_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + + const_vec_array previous_adjoint_vector = (*adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadWrite); + + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes + nghost_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + // cancellation of half from midpoint and 2 from adjoint equation already done + current_adjoint_vector(node_gid, idim) = -0.5 * (current_velocity_vector(node_gid, idim) + previous_velocity_vector(node_gid, idim)) * global_dt + previous_adjoint_vector(node_gid, + idim); + } }); // end parallel for - Kokkos::fence(); - } //end view scope - - } + Kokkos::fence(); + } // end view scope + } } - /* ------------------------------------------------------------------------------ Coupled adjoint problem for the kinetic energy minimization problem --------------------------------------------------------------------------------- */ -void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_adjoint_full(){ - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - size_t num_bdy_nodes = mesh->num_bdy_nodes; - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos material = simparam->material; - const int num_dim = simparam->num_dims; - const real_t damping_constant = module_params->damping_constant; - real_t global_dt; - size_t current_data_index, next_data_index; - Teuchos::RCP previous_adjoint_vector_distributed, current_adjoint_vector_distributed, previous_velocity_vector_distributed, current_velocity_vector_distributed; - Teuchos::RCP previous_phi_adjoint_vector_distributed, current_phi_adjoint_vector_distributed; - //initialize first adjoint vector at last_time_step to 0 as the terminal value - (*adjoint_vector_data)[last_time_step+1]->putScalar(0); - (*phi_adjoint_vector_data)[last_time_step+1]->putScalar(0); - - //solve terminal value problem, proceeds in time backward. For simplicity, we use the same timestep data from the forward solve. - //A linear interpolant is assumed between velocity data points; velocity midpoint is used to update the adjoint. - if(myrank==0) - std::cout << "Computing adjoint vector " << time_data.size() << std::endl; - - for (int cycle = last_time_step; cycle >= 0; cycle--) { - //compute timestep from time data - global_dt = time_data[cycle+1] - time_data[cycle]; - - //print - if(simparam->dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ - if (cycle==last_time_step){ - if(myrank==0) - printf("cycle = %d, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } - // print time step every 10 cycles - else if (cycle%20==0){ - if(myrank==0) - printf("cycle = %d, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } // end if +void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_adjoint_full() +{ + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + size_t num_bdy_nodes = mesh->num_bdy_nodes; + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos material = simparam->material; + const int num_dim = simparam->num_dims; + const real_t damping_constant = module_params->damping_constant; + real_t global_dt; + size_t current_data_index, next_data_index; + Teuchos::RCP previous_adjoint_vector_distributed, current_adjoint_vector_distributed, previous_velocity_vector_distributed, current_velocity_vector_distributed; + Teuchos::RCP previous_phi_adjoint_vector_distributed, current_phi_adjoint_vector_distributed; + // initialize first adjoint vector at last_time_step to 0 as the terminal value + (*adjoint_vector_data)[last_time_step + 1]->putScalar(0); + (*phi_adjoint_vector_data)[last_time_step + 1]->putScalar(0); + + // solve terminal value problem, proceeds in time backward. For simplicity, we use the same timestep data from the forward solve. + // A linear interpolant is assumed between velocity data points; velocity midpoint is used to update the adjoint. + if (myrank == 0) + { + std::cout << "Computing adjoint vector " << time_data.size() << std::endl; } - - //view scope + + for (int cycle = last_time_step; cycle >= 0; cycle--) { - - //set velocity and position for this timestep - const_vec_array previous_velocity_vector = (*forward_solve_velocity_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - - const_vec_array previous_coordinate_vector = (*forward_solve_coordinate_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_coordinate_vector = (*forward_solve_coordinate_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - - //interface of arrays for current implementation of force calculation - - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes+nghost_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - node_vel(rk_level,node_gid,idim) = previous_velocity_vector(node_gid,idim); - node_coords(rk_level,node_gid,idim) = previous_coordinate_vector(node_gid,idim); + // compute timestep from time data + global_dt = time_data[cycle + 1] - time_data[cycle]; + + // print + if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) + { + if (cycle == last_time_step) + { + if (myrank == 0) + { + printf("cycle = %d, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } + // print time step every 10 cycles + else if (cycle % 20 == 0) + { + if (myrank == 0) + { + printf("cycle = %d, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } // end if } + + // view scope + { + // set velocity and position for this timestep + const_vec_array previous_velocity_vector = (*forward_solve_velocity_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + + const_vec_array previous_coordinate_vector = (*forward_solve_coordinate_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array current_coordinate_vector = (*forward_solve_coordinate_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + + // interface of arrays for current implementation of force calculation + + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes + nghost_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + node_vel(rk_level, node_gid, idim) = previous_velocity_vector(node_gid, idim); + node_coords(rk_level, node_gid, idim) = previous_coordinate_vector(node_gid, idim); + } }); - Kokkos::fence(); + Kokkos::fence(); - get_force_vgradient_elastic(material, + get_force_vgradient_elastic(material, *mesh, node_coords, node_vel, @@ -683,965 +714,1050 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_adjoint_full() 1, cycle); - //force_gradient_velocity->describe(*fos,Teuchos::VERB_EXTREME); - const_vec_array previous_force_gradient_position = force_gradient_position->getLocalView (Tpetra::Access::ReadOnly); - //const_vec_array current_force_gradient_position = force_gradient_position->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array previous_force_gradient_velocity = force_gradient_velocity->getLocalView (Tpetra::Access::ReadOnly); - //const_vec_array current_force_gradient_velocity = force_gradient_velocity->getLocalView (Tpetra::Access::ReadOnly); - //compute gradient of force with respect to velocity - - const_vec_array previous_adjoint_vector = (*adjoint_vector_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array phi_previous_adjoint_vector = (*phi_adjoint_vector_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - vec_array midpoint_adjoint_vector = adjoint_vector_distributed->getLocalView (Tpetra::Access::ReadWrite); - vec_array phi_midpoint_adjoint_vector = phi_adjoint_vector_distributed->getLocalView (Tpetra::Access::ReadWrite); - - //half step update for RK2 scheme - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - real_t rate_of_change; - real_t matrix_contribution; - size_t dof_id; - for (int idim = 0; idim < num_dim; idim++){ - matrix_contribution = 0; - //compute resulting row of force velocity gradient matrix transpose right multiplied by adjoint vector - /* - for(int idof = 0; idof < Gradient_Matrix_Strides(node_gid*num_dim+idim); idof++){ - dof_id = DOF_Graph_Matrix(node_gid*num_dim+idim,idof); - matrix_contribution += previous_adjoint_vector(dof_id/num_dim,dof_id%num_dim)*Force_Gradient_Velocities(node_gid*num_dim+idim,idof); - } - */ - matrix_contribution = -damping_constant*previous_adjoint_vector(node_gid,idim); - rate_of_change = previous_velocity_vector(node_gid,idim)- - matrix_contribution/node_mass(node_gid)- - phi_previous_adjoint_vector(node_gid,idim)/node_mass(node_gid); - midpoint_adjoint_vector(node_gid,idim) = -rate_of_change*global_dt/2 + previous_adjoint_vector(node_gid,idim); - matrix_contribution = 0; - //compute resulting row of force displacement gradient matrix transpose right multiplied by adjoint vector - for(int idof = 0; idof < Gradient_Matrix_Strides(node_gid*num_dim+idim); idof++){ - dof_id = DOF_Graph_Matrix(node_gid*num_dim+idim,idof); - matrix_contribution += -previous_adjoint_vector(dof_id/num_dim,dof_id%num_dim)*Force_Gradient_Positions(node_gid*num_dim+idim,idof); - } - rate_of_change = -matrix_contribution; - //rate_of_change = -0.0000001*previous_adjoint_vector(node_gid,idim); - phi_midpoint_adjoint_vector(node_gid,idim) = -rate_of_change*global_dt/2 + phi_previous_adjoint_vector(node_gid,idim); - } + // force_gradient_velocity->describe(*fos,Teuchos::VERB_EXTREME); + const_vec_array previous_force_gradient_position = force_gradient_position->getLocalView(Tpetra::Access::ReadOnly); + // const_vec_array current_force_gradient_position = force_gradient_position->getLocalView (Tpetra::Access::ReadOnly); + const_vec_array previous_force_gradient_velocity = force_gradient_velocity->getLocalView(Tpetra::Access::ReadOnly); + // const_vec_array current_force_gradient_velocity = force_gradient_velocity->getLocalView (Tpetra::Access::ReadOnly); + // compute gradient of force with respect to velocity + + const_vec_array previous_adjoint_vector = (*adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array phi_previous_adjoint_vector = (*phi_adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + vec_array midpoint_adjoint_vector = adjoint_vector_distributed->getLocalView(Tpetra::Access::ReadWrite); + vec_array phi_midpoint_adjoint_vector = phi_adjoint_vector_distributed->getLocalView(Tpetra::Access::ReadWrite); + + // half step update for RK2 scheme + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + real_t rate_of_change; + real_t matrix_contribution; + size_t dof_id; + for (int idim = 0; idim < num_dim; idim++) + { + matrix_contribution = 0; + // compute resulting row of force velocity gradient matrix transpose right multiplied by adjoint vector + /* + for(int idof = 0; idof < Gradient_Matrix_Strides(node_gid*num_dim+idim); idof++){ + dof_id = DOF_Graph_Matrix(node_gid*num_dim+idim,idof); + matrix_contribution += previous_adjoint_vector(dof_id/num_dim,dof_id%num_dim)*Force_Gradient_Velocities(node_gid*num_dim+idim,idof); + } + */ + matrix_contribution = -damping_constant * previous_adjoint_vector(node_gid, idim); + rate_of_change = previous_velocity_vector(node_gid, idim) - + matrix_contribution / node_mass(node_gid) - + phi_previous_adjoint_vector(node_gid, idim) / node_mass(node_gid); + midpoint_adjoint_vector(node_gid, idim) = -rate_of_change * global_dt / 2 + previous_adjoint_vector(node_gid, idim); + matrix_contribution = 0; + // compute resulting row of force displacement gradient matrix transpose right multiplied by adjoint vector + for (int idof = 0; idof < Gradient_Matrix_Strides(node_gid * num_dim + idim); idof++) + { + dof_id = DOF_Graph_Matrix(node_gid * num_dim + idim, idof); + matrix_contribution += -previous_adjoint_vector(dof_id / num_dim, dof_id % num_dim) * Force_Gradient_Positions(node_gid * num_dim + idim, idof); + } + rate_of_change = -matrix_contribution; + // rate_of_change = -0.0000001*previous_adjoint_vector(node_gid,idim); + phi_midpoint_adjoint_vector(node_gid, idim) = -rate_of_change * global_dt / 2 + phi_previous_adjoint_vector(node_gid, idim); + } }); // end parallel for - Kokkos::fence(); - - boundary_adjoint(*mesh, boundary,midpoint_adjoint_vector, phi_midpoint_adjoint_vector); - comm_adjoint_vectors(cycle); - //swap names to get ghost nodes for the midpoint vectors - vec_array current_adjoint_vector = adjoint_vector_distributed->getLocalView (Tpetra::Access::ReadWrite); - vec_array phi_current_adjoint_vector = phi_adjoint_vector_distributed->getLocalView (Tpetra::Access::ReadWrite); - midpoint_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView (Tpetra::Access::ReadWrite); - phi_midpoint_adjoint_vector = (*phi_adjoint_vector_data)[cycle]->getLocalView (Tpetra::Access::ReadWrite); - - //full step update with midpoint gradient for RK2 scheme - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - real_t rate_of_change; - real_t matrix_contribution; - size_t dof_id; - for (int idim = 0; idim < num_dim; idim++){ - matrix_contribution = 0; - //compute resulting row of force velocity gradient matrix transpose right multiplied by adjoint vector - /* - for(int idof = 0; idof < Gradient_Matrix_Strides(node_gid*num_dim+idim); idof++){ - dof_id = DOF_Graph_Matrix(node_gid*num_dim+idim,idof); - matrix_contribution += midpoint_adjoint_vector(dof_id/num_dim,dof_id%num_dim)*Force_Gradient_Velocities(node_gid*num_dim+idim,idof); - } - */ - matrix_contribution = -damping_constant*midpoint_adjoint_vector(node_gid,idim); - rate_of_change = (previous_velocity_vector(node_gid,idim) + current_velocity_vector(node_gid,idim))/2- - matrix_contribution/node_mass(node_gid)- - phi_midpoint_adjoint_vector(node_gid,idim)/node_mass(node_gid); - current_adjoint_vector(node_gid,idim) = -rate_of_change*global_dt + previous_adjoint_vector(node_gid,idim); - matrix_contribution = 0; - //compute resulting row of force displacement gradient matrix transpose right multiplied by adjoint vector - for(int idof = 0; idof < Gradient_Matrix_Strides(node_gid*num_dim+idim); idof++){ - dof_id = DOF_Graph_Matrix(node_gid*num_dim+idim,idof); - matrix_contribution += -midpoint_adjoint_vector(dof_id/num_dim,dof_id%num_dim)*Force_Gradient_Positions(node_gid*num_dim+idim,idof); - } - rate_of_change = -matrix_contribution; - //rate_of_change = -0.0000001*midpoint_adjoint_vector(node_gid,idim); - phi_current_adjoint_vector(node_gid,idim) = -rate_of_change*global_dt + phi_previous_adjoint_vector(node_gid,idim); - } + Kokkos::fence(); + + boundary_adjoint(*mesh, boundary, midpoint_adjoint_vector, phi_midpoint_adjoint_vector); + comm_adjoint_vectors(cycle); + // swap names to get ghost nodes for the midpoint vectors + vec_array current_adjoint_vector = adjoint_vector_distributed->getLocalView(Tpetra::Access::ReadWrite); + vec_array phi_current_adjoint_vector = phi_adjoint_vector_distributed->getLocalView(Tpetra::Access::ReadWrite); + midpoint_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadWrite); + phi_midpoint_adjoint_vector = (*phi_adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadWrite); + + // full step update with midpoint gradient for RK2 scheme + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + real_t rate_of_change; + real_t matrix_contribution; + size_t dof_id; + for (int idim = 0; idim < num_dim; idim++) + { + matrix_contribution = 0; + // compute resulting row of force velocity gradient matrix transpose right multiplied by adjoint vector + /* + for(int idof = 0; idof < Gradient_Matrix_Strides(node_gid*num_dim+idim); idof++){ + dof_id = DOF_Graph_Matrix(node_gid*num_dim+idim,idof); + matrix_contribution += midpoint_adjoint_vector(dof_id/num_dim,dof_id%num_dim)*Force_Gradient_Velocities(node_gid*num_dim+idim,idof); + } + */ + matrix_contribution = -damping_constant * midpoint_adjoint_vector(node_gid, idim); + rate_of_change = (previous_velocity_vector(node_gid, idim) + current_velocity_vector(node_gid, idim)) / 2 - + matrix_contribution / node_mass(node_gid) - + phi_midpoint_adjoint_vector(node_gid, idim) / node_mass(node_gid); + current_adjoint_vector(node_gid, idim) = -rate_of_change * global_dt + previous_adjoint_vector(node_gid, idim); + matrix_contribution = 0; + // compute resulting row of force displacement gradient matrix transpose right multiplied by adjoint vector + for (int idof = 0; idof < Gradient_Matrix_Strides(node_gid * num_dim + idim); idof++) + { + dof_id = DOF_Graph_Matrix(node_gid * num_dim + idim, idof); + matrix_contribution += -midpoint_adjoint_vector(dof_id / num_dim, dof_id % num_dim) * Force_Gradient_Positions(node_gid * num_dim + idim, idof); + } + rate_of_change = -matrix_contribution; + // rate_of_change = -0.0000001*midpoint_adjoint_vector(node_gid,idim); + phi_current_adjoint_vector(node_gid, idim) = -rate_of_change * global_dt + phi_previous_adjoint_vector(node_gid, idim); + } }); // end parallel for - Kokkos::fence(); - - - boundary_adjoint(*mesh, boundary,current_adjoint_vector, phi_current_adjoint_vector); + Kokkos::fence(); - } //end view scope + boundary_adjoint(*mesh, boundary, current_adjoint_vector, phi_current_adjoint_vector); + } // end view scope - comm_adjoint_vectors(cycle); - //phi_adjoint_vector_distributed->describe(*fos,Teuchos::VERB_EXTREME); - } + comm_adjoint_vectors(cycle); + // phi_adjoint_vector_distributed->describe(*fos,Teuchos::VERB_EXTREME); + } } - /* ---------------------------------------------------------------------------- Gradient calculation for the kinetic energy minimization problem ------------------------------------------------------------------------------- */ -void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient(const_vec_array design_variables, vec_array design_gradients){ - - size_t num_bdy_nodes = mesh->num_bdy_nodes; - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos material = simparam->material; - const int num_dim = simparam->num_dims; - int num_corners = rnum_elem*num_nodes_in_elem; - real_t global_dt; - size_t current_data_index, next_data_index; - CArrayKokkos current_element_velocities = CArrayKokkos(num_nodes_in_elem,num_dim); - CArrayKokkos current_element_adjoint = CArrayKokkos(num_nodes_in_elem,num_dim); - - if(myrank==0) - std::cout << "Computing accumulated kinetic energy gradient" << std::endl; +void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient(const_vec_array design_variables, vec_array design_gradients) +{ + size_t num_bdy_nodes = mesh->num_bdy_nodes; + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos material = simparam->material; + const int num_dim = simparam->num_dims; + int num_corners = rnum_elem * num_nodes_in_elem; + real_t global_dt; + size_t current_data_index, next_data_index; + CArrayKokkos current_element_velocities = CArrayKokkos(num_nodes_in_elem, num_dim); + CArrayKokkos current_element_adjoint = CArrayKokkos(num_nodes_in_elem, num_dim); + + if (myrank == 0) + { + std::cout << "Computing accumulated kinetic energy gradient" << std::endl; + } - compute_topology_optimization_adjoint(); + compute_topology_optimization_adjoint(); - //compute design gradients - FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { - design_gradients(node_id,0) = 0; + // compute design gradients + FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { + design_gradients(node_id, 0) = 0; }); // end parallel for - Kokkos::fence(); - - //gradient contribution from kinetic energy vMv product. - for (unsigned long cycle = 0; cycle < last_time_step+1; cycle++) { - //compute timestep from time data - global_dt = time_data[cycle+1] - time_data[cycle]; - - //print - if(simparam->dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ - if (cycle==0){ - if(myrank==0) - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } - // print time step every 10 cycles - else if (cycle%20==0){ - if(myrank==0) - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } // end if - } + Kokkos::fence(); - //compute adjoint vector for this data point; use velocity midpoint - //view scope - { - const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array next_velocity_vector = (*forward_solve_velocity_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array next_adjoint_vector = (*adjoint_vector_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - - FOR_ALL_CLASS(elem_id, 0, rnum_elem, { - size_t node_id; - size_t corner_id; - real_t inner_product; - //std::cout << elem_mass(elem_id) <dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) + { + if (cycle == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } } - } - - for (int inode = 0; inode < num_nodes_in_elem; inode++){ - //compute gradient of local element contribution to v^t*M*v product - corner_id = elem_id*num_nodes_in_elem + inode; - corner_value_storage(corner_id) = inner_product*global_dt; - } - + // print time step every 10 cycles + else if (cycle % 20 == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } // end if + } + + // compute adjoint vector for this data point; use velocity midpoint + // view scope + { + const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_velocity_vector = (*forward_solve_velocity_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_adjoint_vector = (*adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + + FOR_ALL_CLASS(elem_id, 0, rnum_elem, { + size_t node_id; + size_t corner_id; + real_t inner_product; + // std::cout << elem_mass(elem_id) <dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ - if (cycle==0){ - if(myrank==0) - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } - // print time step every 10 cycles - else if (cycle%20==0){ - if(myrank==0) - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } // end if - } + Kokkos::fence(); - //compute adjoint vector for this data point; use velocity midpoint - //view scope - { - //const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array next_adjoint_vector = (*adjoint_vector_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - //const_vec_array current_coord_vector = forward_solve_coordinate_data[cycle]->getLocalView (Tpetra::Access::ReadOnly); - //const_vec_array final_coordinates = forward_solve_coordinate_data[last_time_step+1]->getLocalView (Tpetra::Access::ReadOnly); - - FOR_ALL_CLASS(elem_id, 0, rnum_elem, { - size_t node_id; - size_t corner_id; - real_t inner_product; - //std::cout << elem_mass(elem_id) <dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) + { + if (cycle == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } + // print time step every 10 cycles + else if (cycle % 20 == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } // end if + } -void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient_full(Teuchos::RCP design_densities_distributed, Teuchos::RCP design_gradients_distributed){ + // compute adjoint vector for this data point; use velocity midpoint + // view scope + { + // const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); + const_vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_adjoint_vector = (*adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + // const_vec_array current_coord_vector = forward_solve_coordinate_data[cycle]->getLocalView (Tpetra::Access::ReadOnly); + // const_vec_array final_coordinates = forward_solve_coordinate_data[last_time_step+1]->getLocalView (Tpetra::Access::ReadOnly); + + FOR_ALL_CLASS(elem_id, 0, rnum_elem, { + size_t node_id; + size_t corner_id; + real_t inner_product; + // std::cout << elem_mass(elem_id) <num_bdy_nodes; - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos material = simparam->material; - const int num_dim = simparam->num_dims; - int num_corners = rnum_elem*num_nodes_in_elem; - real_t global_dt; - bool element_constant_density = true; - size_t current_data_index, next_data_index; - CArrayKokkos current_element_velocities = CArrayKokkos(num_nodes_in_elem,num_dim); - CArrayKokkos current_element_adjoint = CArrayKokkos(num_nodes_in_elem,num_dim); + inner_product = 0; + for (int ifill = 0; ifill < num_nodes_in_elem; ifill++) + { + node_id = nodes_in_elem(elem_id, ifill); + for (int idim = 0; idim < num_dim; idim++) + { + inner_product += 0.00001 * current_element_adjoint(ifill, idim); + // inner_product += 0.0001; + } + } - if(myrank==0) - std::cout << "Computing accumulated kinetic energy gradient" << std::endl; + for (int inode = 0; inode < num_nodes_in_elem; inode++) + { + // compute gradient of local element contribution to v^t*M*v product + corner_id = elem_id * num_nodes_in_elem + inode; + corner_value_storage(corner_id) = -inner_product * global_dt / (double)num_nodes_in_elem; + } + }); // end parallel for + Kokkos::fence(); - compute_topology_optimization_adjoint_full(); + // accumulate node values from corner storage + // multiply + FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { + size_t corner_id; + for (int icorner = 0; icorner < num_corners_in_node(node_id); icorner++) + { + corner_id = corners_in_node(node_id, icorner); + design_gradients(node_id, 0) += corner_value_storage(corner_id); + } + }); // end parallel for + Kokkos::fence(); + } // end view scope + } +} - { //view scope - vec_array design_gradients = design_gradients_distributed->getLocalView (Tpetra::Access::ReadWrite); - const_vec_array design_densities = design_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); - //compute design gradients - FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { - design_gradients(node_id,0) = 0; - }); // end parallel for - Kokkos::fence(); +/* ---------------------------------------------------------------------------- + Gradient for the (unsimplified) kinetic energy minimization problem +------------------------------------------------------------------------------- */ - //gradient contribution from kinetic energy v(dM/drho)v product. - if(simparam->dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ - if(myrank==0){ - std::cout << "v*dM/drho*v term" << std::endl; - } +void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient_full(Teuchos::RCP design_densities_distributed, Teuchos::RCP design_gradients_distributed) +{ + size_t num_bdy_nodes = mesh->num_bdy_nodes; + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos material = simparam->material; + const int num_dim = simparam->num_dims; + int num_corners = rnum_elem * num_nodes_in_elem; + real_t global_dt; + bool element_constant_density = true; + size_t current_data_index, next_data_index; + CArrayKokkos current_element_velocities = CArrayKokkos(num_nodes_in_elem, num_dim); + CArrayKokkos current_element_adjoint = CArrayKokkos(num_nodes_in_elem, num_dim); + + if (myrank == 0) + { + std::cout << "Computing accumulated kinetic energy gradient" << std::endl; } - for (unsigned long cycle = 0; cycle < last_time_step+1; cycle++) { - //compute timestep from time data - global_dt = time_data[cycle+1] - time_data[cycle]; - - //print - if(simparam->dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ + compute_topology_optimization_adjoint_full(); - if (cycle==0){ - if(myrank==0) - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } - // print time step every 10 cycles - else if (cycle%20==0){ - if(myrank==0) - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } // end if - } + { // view scope + vec_array design_gradients = design_gradients_distributed->getLocalView(Tpetra::Access::ReadWrite); + const_vec_array design_densities = design_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); + // compute design gradients + FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { + design_gradients(node_id, 0) = 0; + }); // end parallel for + Kokkos::fence(); - //compute adjoint vector for this data point; use velocity midpoint - //view scope + // gradient contribution from kinetic energy v(dM/drho)v product. + if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) { - const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array next_velocity_vector = (*forward_solve_velocity_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array next_adjoint_vector = (*adjoint_vector_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - - FOR_ALL_CLASS(elem_id, 0, rnum_elem, { - size_t node_id; - size_t corner_id; - real_t inner_product; - //std::cout << elem_mass(elem_id) <dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) + { + if (cycle == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } + // print time step every 10 cycles + else if (cycle % 20 == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } // end if } - }); // end parallel for - Kokkos::fence(); - } //end view scope + // compute adjoint vector for this data point; use velocity midpoint + // view scope + { + const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_velocity_vector = (*forward_solve_velocity_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_adjoint_vector = (*adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + + FOR_ALL_CLASS(elem_id, 0, rnum_elem, { + size_t node_id; + size_t corner_id; + real_t inner_product; + // std::cout << elem_mass(elem_id) <dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ - if(myrank==0){ - std::cout << "gradient term involving adjoint derivative" << std::endl; + // accumulate node values from corner storage + // multiply + FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { + size_t corner_id; + for (int icorner = 0; icorner < num_corners_in_node(node_id); icorner++) + { + corner_id = corners_in_node(node_id, icorner); + design_gradients(node_id, 0) += corner_value_storage(corner_id); + } + }); // end parallel for + Kokkos::fence(); + } // end view scope } - } - for (unsigned long cycle = 0; cycle < last_time_step+1; cycle++) { - //compute timestep from time data - global_dt = time_data[cycle+1] - time_data[cycle]; - //print - if(simparam->dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ + // multiply by Hex8 constants (the diagonlization here only works for Hex8 anyway) + FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { + design_gradients(node_id, 0) *= 0.5 / (double)num_nodes_in_elem / (double)num_nodes_in_elem; + // design_gradients(node_id,0) =0.00001; + }); // end parallel for + Kokkos::fence(); - if (cycle==0){ - if(myrank==0) - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + // gradient contribution from time derivative of adjoint \dot{lambda}(dM/drho)v product. + if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) + { + if (myrank == 0) + { + std::cout << "gradient term involving adjoint derivative" << std::endl; + } } - // print time step every 10 cycles - else if (cycle%20==0){ - if(myrank==0) - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } // end if - } - //compute adjoint vector for this data point; use velocity midpoint - //view scope + for (unsigned long cycle = 0; cycle < last_time_step + 1; cycle++) { - const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_phi_adjoint_vector = (*phi_adjoint_vector_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array next_velocity_vector = (*forward_solve_velocity_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array next_adjoint_vector = (*adjoint_vector_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array next_phi_adjoint_vector = (*phi_adjoint_vector_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - - const real_t damping_constant = module_params->damping_constant; - FOR_ALL_CLASS(elem_id, 0, rnum_elem, { - real_t lambda_dot_current; - real_t lambda_dot_next; - size_t node_id; - size_t corner_id; - real_t inner_product; - //std::cout << elem_mass(elem_id) <dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) + { + if (cycle == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } + // print time step every 10 cycles + else if (cycle % 20 == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } // end if } - inner_product = 0; - for(int ifill=0; ifill < num_nodes_in_elem; ifill++){ - node_id = nodes_in_elem(elem_id, ifill); - for(int idim=0; idim < num_dim; idim++){ - //lambda_dot_current = lambda_dot_next = (next_adjoint_vector(node_id,idim)-current_adjoint_vector(node_id,idim))/global_dt; - lambda_dot_current = current_velocity_vector(node_id,idim) + damping_constant*current_adjoint_vector(node_id,idim)/node_mass(node_id) - current_phi_adjoint_vector(node_id,idim)/node_mass(node_id); - lambda_dot_next = next_velocity_vector(node_id,idim) + damping_constant*next_adjoint_vector(node_id,idim)/node_mass(node_id) - next_phi_adjoint_vector(node_id,idim)/node_mass(node_id); - inner_product += elem_mass(elem_id)*(lambda_dot_current+lambda_dot_next)*current_element_velocities(ifill,idim)/2; - } - } + // compute adjoint vector for this data point; use velocity midpoint + // view scope + { + const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array current_phi_adjoint_vector = (*phi_adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_velocity_vector = (*forward_solve_velocity_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_adjoint_vector = (*adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_phi_adjoint_vector = (*phi_adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + + const real_t damping_constant = module_params->damping_constant; + FOR_ALL_CLASS(elem_id, 0, rnum_elem, { + real_t lambda_dot_current; + real_t lambda_dot_next; + size_t node_id; + size_t corner_id; + real_t inner_product; + // std::cout << elem_mass(elem_id) <getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_adjoint_vector = (*adjoint_vector_data)[0]->getLocalView (Tpetra::Access::ReadOnly); - - FOR_ALL_CLASS(elem_id, 0, rnum_elem, { - real_t lambda_dot; - size_t node_id; - size_t corner_id; - real_t inner_product; - //std::cout << elem_mass(elem_id) <getLocalView(Tpetra::Access::ReadOnly); + const_vec_array current_adjoint_vector = (*adjoint_vector_data)[0]->getLocalView(Tpetra::Access::ReadOnly); + + FOR_ALL_CLASS(elem_id, 0, rnum_elem, { + real_t lambda_dot; + size_t node_id; + size_t corner_id; + real_t inner_product; + // std::cout << elem_mass(elem_id) <getLocalView (Tpetra::Access::ReadWrite); - const_host_vec_array host_design_variables = design_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); - compute_stiffness_gradients(host_design_variables, host_design_gradients); - }//end view scope + Kokkos::fence(); + } // end view scope + } // end view scope + // force_design_gradient_term(design_variables, design_gradients); + // view scope + { + host_vec_array host_design_gradients = design_gradients_distributed->getLocalView(Tpetra::Access::ReadWrite); + const_host_vec_array host_design_variables = design_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); + compute_stiffness_gradients(host_design_variables, host_design_gradients); + } // end view scope } /* ---------------------------------------------------------------------- Initialize global vectors and array maps needed for matrix assembly ------------------------------------------------------------------------- */ -void FEA_Module_Dynamic_Elasticity::init_assembly(){ - int num_dim = simparam->num_dims; - //const_host_elem_conn_array nodes_in_elem = global_nodes_in_elem_distributed->getLocalView (Tpetra::Access::ReadOnly); - Gradient_Matrix_Strides = DCArrayKokkos (nlocal_nodes*num_dim, "Gradient_Matrix_Strides"); - CArrayKokkos Graph_Fill(nall_nodes, "nall_nodes"); - CArrayKokkos current_row_nodes_scanned; - int local_node_index, current_column_index; - size_t max_stride = 0; - size_t nodes_per_element; - nodal_density_flag = simparam->nodal_density_flag; - penalty_power = simparam->optimization_options.simp_penalty_power; - - //allocate stride arrays - CArrayKokkos Graph_Matrix_Strides_initial(nlocal_nodes, "Graph_Matrix_Strides_initial"); - DCArrayKokkos Dual_Graph_Matrix_Strides_initial(nlocal_nodes, "Host_Graph_Matrix_Strides_initial"); - Graph_Matrix_Strides = DCArrayKokkos(nlocal_nodes, "Graph_Matrix_Strides"); - - //allocate storage for the sparse stiffness matrix map used in the assembly process - Global_Stiffness_Matrix_Assembly_Map = Global_Gradient_Matrix_Assembly_Map = DCArrayKokkos(rnum_elem, - max_nodes_per_element,max_nodes_per_element, "Global_Gradient_Matrix_Assembly_Map"); - - //allocate array used to determine global node repeats in the sparse graph later - DCArrayKokkos node_indices_used(nall_nodes, "node_indices_used"); - - /*allocate array that stores which column the node index occured on for the current row - when removing repeats*/ - DCArrayKokkos column_index(nall_nodes, "column_index"); - - //initialize nlocal arrays - FOR_ALL_CLASS(inode, 0, nlocal_nodes, { - Graph_Matrix_Strides_initial(inode) = 0; - Graph_Matrix_Strides(inode) = 0; - Graph_Fill(inode) = 0; +void FEA_Module_Dynamic_Elasticity::init_assembly() +{ + int num_dim = simparam->num_dims; + // const_host_elem_conn_array nodes_in_elem = global_nodes_in_elem_distributed->getLocalView (Tpetra::Access::ReadOnly); + Gradient_Matrix_Strides = DCArrayKokkos(nlocal_nodes * num_dim, "Gradient_Matrix_Strides"); + CArrayKokkos Graph_Fill(nall_nodes, "nall_nodes"); + CArrayKokkos current_row_nodes_scanned; + int local_node_index, current_column_index; + size_t max_stride = 0; + size_t nodes_per_element; + nodal_density_flag = simparam->nodal_density_flag; + penalty_power = simparam->optimization_options.simp_penalty_power; + + // allocate stride arrays + CArrayKokkos Graph_Matrix_Strides_initial(nlocal_nodes, "Graph_Matrix_Strides_initial"); + DCArrayKokkos Dual_Graph_Matrix_Strides_initial(nlocal_nodes, "Host_Graph_Matrix_Strides_initial"); + Graph_Matrix_Strides = DCArrayKokkos(nlocal_nodes, "Graph_Matrix_Strides"); + + // allocate storage for the sparse stiffness matrix map used in the assembly process + Global_Stiffness_Matrix_Assembly_Map = Global_Gradient_Matrix_Assembly_Map = DCArrayKokkos(rnum_elem, + max_nodes_per_element, max_nodes_per_element, "Global_Gradient_Matrix_Assembly_Map"); + + // allocate array used to determine global node repeats in the sparse graph later + DCArrayKokkos node_indices_used(nall_nodes, "node_indices_used"); + + /*allocate array that stores which column the node index occured on for the current row + when removing repeats*/ + DCArrayKokkos column_index(nall_nodes, "column_index"); + + // initialize nlocal arrays + FOR_ALL_CLASS(inode, 0, nlocal_nodes, { + Graph_Matrix_Strides_initial(inode) = 0; + Graph_Matrix_Strides(inode) = 0; + Graph_Fill(inode) = 0; }); // end parallel for - Kokkos::fence(); + Kokkos::fence(); - //initialize nall arrays - //initialize nlocal arrays - FOR_ALL_CLASS(inode, 0, nall_nodes, { - node_indices_used(inode) = 0; - column_index(inode) = 0; + // initialize nall arrays + // initialize nlocal arrays + FOR_ALL_CLASS(inode, 0, nall_nodes, { + node_indices_used(inode) = 0; + column_index(inode) = 0; }); // end parallel for - Kokkos::fence(); - - //count upper bound of strides for Sparse Pattern Graph by allowing repeats due to connectivity - if(num_dim == 2) - for (int ielem = 0; ielem < rnum_elem; ielem++){ - element_select->choose_2Delem_type(Element_Types(ielem), elem2D); - nodes_per_element = elem2D->num_nodes(); - for (int lnode = 0; lnode < nodes_per_element; lnode++){ - local_node_index = nodes_in_elem(ielem, lnode); - if(local_node_index < nlocal_nodes){ - Dual_Graph_Matrix_Strides_initial.host(local_node_index) += nodes_per_element; + Kokkos::fence(); + + // count upper bound of strides for Sparse Pattern Graph by allowing repeats due to connectivity + if (num_dim == 2) + { + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + element_select->choose_2Delem_type(Element_Types(ielem), elem2D); + nodes_per_element = elem2D->num_nodes(); + for (int lnode = 0; lnode < nodes_per_element; lnode++) + { + local_node_index = nodes_in_elem(ielem, lnode); + if (local_node_index < nlocal_nodes) + { + Dual_Graph_Matrix_Strides_initial.host(local_node_index) += nodes_per_element; + } + } } - } } - if(num_dim == 3) - for (int ielem = 0; ielem < rnum_elem; ielem++){ - element_select->choose_3Delem_type(Element_Types(ielem), elem); - nodes_per_element = elem->num_nodes(); - for (int lnode = 0; lnode < nodes_per_element; lnode++){ - local_node_index = nodes_in_elem(ielem, lnode); - if(local_node_index < nlocal_nodes){ - Dual_Graph_Matrix_Strides_initial.host(local_node_index) += nodes_per_element; + if (num_dim == 3) + { + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + element_select->choose_3Delem_type(Element_Types(ielem), elem); + nodes_per_element = elem->num_nodes(); + for (int lnode = 0; lnode < nodes_per_element; lnode++) + { + local_node_index = nodes_in_elem(ielem, lnode); + if (local_node_index < nlocal_nodes) + { + Dual_Graph_Matrix_Strides_initial.host(local_node_index) += nodes_per_element; + } + } } - } } - - Dual_Graph_Matrix_Strides_initial.update_device(); - //equate strides for later - FOR_ALL_CLASS(inode, 0, nlocal_nodes, { - Graph_Matrix_Strides(inode) = Graph_Matrix_Strides_initial(inode) = Dual_Graph_Matrix_Strides_initial(inode); + Dual_Graph_Matrix_Strides_initial.update_device(); + + // equate strides for later + FOR_ALL_CLASS(inode, 0, nlocal_nodes, { + Graph_Matrix_Strides(inode) = Graph_Matrix_Strides_initial(inode) = Dual_Graph_Matrix_Strides_initial(inode); }); // end parallel for - - //for (int inode = 0; inode < nlocal_nodes; inode++) - //std::cout << Graph_Matrix_Strides_initial(inode) << std::endl; - - //compute maximum stride - size_t update = 0; - REDUCE_MAX_CLASS(inode, 0, nlocal_nodes, update, { - if(update < Graph_Matrix_Strides_initial(inode)) - update = Graph_Matrix_Strides_initial(inode); + + // for (int inode = 0; inode < nlocal_nodes; inode++) + // std::cout << Graph_Matrix_Strides_initial(inode) << std::endl; + + // compute maximum stride + size_t update = 0; + REDUCE_MAX_CLASS(inode, 0, nlocal_nodes, update, { + if (update < Graph_Matrix_Strides_initial(inode)) + { + update = Graph_Matrix_Strides_initial(inode); + } }, max_stride); - - //std::cout << "THE MAX STRIDE" << max_stride << std::endl; - //allocate array used in the repeat removal process - current_row_nodes_scanned = CArrayKokkos(max_stride, "current_row_nodes_scanned"); - - //allocate sparse graph with node repeats - RaggedRightArrayKokkos Repeat_Graph_Matrix(Graph_Matrix_Strides_initial); - RaggedRightArrayofVectorsKokkos Element_local_indices(Graph_Matrix_Strides_initial,num_dim); - - //Fill the initial Graph with repeats - if(num_dim == 2){ - for (int ielem = 0; ielem < rnum_elem; ielem++){ - element_select->choose_2Delem_type(Element_Types(ielem), elem2D); - nodes_per_element = elem2D->num_nodes(); - for (int lnode = 0; lnode < nodes_per_element; lnode++){ - local_node_index = nodes_in_elem(ielem, lnode); - if(local_node_index < nlocal_nodes){ - for (int jnode = 0; jnode < nodes_per_element; jnode++){ - current_column_index = Graph_Fill(local_node_index)+jnode; - Repeat_Graph_Matrix(local_node_index, current_column_index) = nodes_in_elem(ielem,jnode); - - //fill inverse map - Element_local_indices(local_node_index,current_column_index,0) = ielem; - Element_local_indices(local_node_index,current_column_index,1) = lnode; - Element_local_indices(local_node_index,current_column_index,2) = jnode; - - //fill forward map - Global_Gradient_Matrix_Assembly_Map(ielem,lnode,jnode) = current_column_index; - } - Graph_Fill(local_node_index) += nodes_per_element; + + // std::cout << "THE MAX STRIDE" << max_stride << std::endl; + // allocate array used in the repeat removal process + current_row_nodes_scanned = CArrayKokkos(max_stride, "current_row_nodes_scanned"); + + // allocate sparse graph with node repeats + RaggedRightArrayKokkos Repeat_Graph_Matrix(Graph_Matrix_Strides_initial); + RaggedRightArrayofVectorsKokkos Element_local_indices(Graph_Matrix_Strides_initial, num_dim); + + // Fill the initial Graph with repeats + if (num_dim == 2) + { + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + element_select->choose_2Delem_type(Element_Types(ielem), elem2D); + nodes_per_element = elem2D->num_nodes(); + for (int lnode = 0; lnode < nodes_per_element; lnode++) + { + local_node_index = nodes_in_elem(ielem, lnode); + if (local_node_index < nlocal_nodes) + { + for (int jnode = 0; jnode < nodes_per_element; jnode++) + { + current_column_index = Graph_Fill(local_node_index) + jnode; + Repeat_Graph_Matrix(local_node_index, current_column_index) = nodes_in_elem(ielem, jnode); + + // fill inverse map + Element_local_indices(local_node_index, current_column_index, 0) = ielem; + Element_local_indices(local_node_index, current_column_index, 1) = lnode; + Element_local_indices(local_node_index, current_column_index, 2) = jnode; + + // fill forward map + Global_Gradient_Matrix_Assembly_Map(ielem, lnode, jnode) = current_column_index; + } + Graph_Fill(local_node_index) += nodes_per_element; + } + } } - } } - } - - if(num_dim == 3){ - for (int ielem = 0; ielem < rnum_elem; ielem++){ - element_select->choose_3Delem_type(Element_Types(ielem), elem); - nodes_per_element = elem->num_nodes(); - for (int lnode = 0; lnode < nodes_per_element; lnode++){ - local_node_index = nodes_in_elem(ielem, lnode); - if(local_node_index < nlocal_nodes){ - for (int jnode = 0; jnode < nodes_per_element; jnode++){ - current_column_index = Graph_Fill(local_node_index)+jnode; - Repeat_Graph_Matrix(local_node_index, current_column_index) = nodes_in_elem(ielem,jnode); - - //fill inverse map - Element_local_indices(local_node_index,current_column_index,0) = ielem; - Element_local_indices(local_node_index,current_column_index,1) = lnode; - Element_local_indices(local_node_index,current_column_index,2) = jnode; - - //fill forward map - Global_Gradient_Matrix_Assembly_Map(ielem,lnode,jnode) = current_column_index; - } - Graph_Fill(local_node_index) += nodes_per_element; + + if (num_dim == 3) + { + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + element_select->choose_3Delem_type(Element_Types(ielem), elem); + nodes_per_element = elem->num_nodes(); + for (int lnode = 0; lnode < nodes_per_element; lnode++) + { + local_node_index = nodes_in_elem(ielem, lnode); + if (local_node_index < nlocal_nodes) + { + for (int jnode = 0; jnode < nodes_per_element; jnode++) + { + current_column_index = Graph_Fill(local_node_index) + jnode; + Repeat_Graph_Matrix(local_node_index, current_column_index) = nodes_in_elem(ielem, jnode); + + // fill inverse map + Element_local_indices(local_node_index, current_column_index, 0) = ielem; + Element_local_indices(local_node_index, current_column_index, 1) = lnode; + Element_local_indices(local_node_index, current_column_index, 2) = jnode; + + // fill forward map + Global_Gradient_Matrix_Assembly_Map(ielem, lnode, jnode) = current_column_index; + } + Graph_Fill(local_node_index) += nodes_per_element; + } + } } - } } - } - - //debug statement - //std::cout << "started run" << std::endl; - //std::cout << "Graph Matrix Strides Repeat on task " << myrank << std::endl; - //for (int inode = 0; inode < nlocal_nodes; inode++) - //std::cout << Graph_Matrix_Strides(inode) << std::endl; - RUN_CLASS({ - //remove repeats from the inital graph setup - int current_node; - //remove repeats from the inital graph setup - int current_element_index; - int element_row_index; - int element_column_index; - int current_stride; - int current_row_n_nodes_scanned; - for (int inode = 0; inode < nlocal_nodes; inode++){ - current_row_n_nodes_scanned = 0; - for (int istride = 0; istride < Graph_Matrix_Strides(inode); istride++){ - //convert global index in graph to its local index for the flagging array - current_node = Repeat_Graph_Matrix(inode,istride); - //debug - //if(current_node==-1) - //std::cout << "Graph Matrix node access on task " << myrank << std::endl; - //std::cout << Repeat_Graph_Matrix(inode,istride) << std::endl; - if(node_indices_used(current_node)){ - //set global assembly map index to the location in the graph matrix where this global node was first found - current_element_index = Element_local_indices(inode,istride,0); - element_row_index = Element_local_indices(inode,istride,1); - element_column_index = Element_local_indices(inode,istride,2); - Global_Gradient_Matrix_Assembly_Map(current_element_index,element_row_index, element_column_index) - = column_index(current_node); - - - //swap current node with the end of the current row and shorten the stride of the row - //first swap information about the inverse and forward maps - - current_stride = Graph_Matrix_Strides(inode); - if(istride!=current_stride-1){ - Element_local_indices(inode,istride,0) = Element_local_indices(inode,current_stride-1,0); - Element_local_indices(inode,istride,1) = Element_local_indices(inode,current_stride-1,1); - Element_local_indices(inode,istride,2) = Element_local_indices(inode,current_stride-1,2); - current_element_index = Element_local_indices(inode,istride,0); - element_row_index = Element_local_indices(inode,istride,1); - element_column_index = Element_local_indices(inode,istride,2); - - Global_Gradient_Matrix_Assembly_Map(current_element_index,element_row_index, element_column_index) - = istride; - - //now that the element map information has been copied, copy the global node index and delete the last index - - Repeat_Graph_Matrix(inode,istride) = Repeat_Graph_Matrix(inode,current_stride-1); - } - istride--; - Graph_Matrix_Strides(inode)--; + + // debug statement + // std::cout << "started run" << std::endl; + // std::cout << "Graph Matrix Strides Repeat on task " << myrank << std::endl; + // for (int inode = 0; inode < nlocal_nodes; inode++) + // std::cout << Graph_Matrix_Strides(inode) << std::endl; + RUN_CLASS({ + // remove repeats from the inital graph setup + int current_node; + // remove repeats from the inital graph setup + int current_element_index; + int element_row_index; + int element_column_index; + int current_stride; + int current_row_n_nodes_scanned; + for (int inode = 0; inode < nlocal_nodes; inode++) + { + current_row_n_nodes_scanned = 0; + for (int istride = 0; istride < Graph_Matrix_Strides(inode); istride++) + { + // convert global index in graph to its local index for the flagging array + current_node = Repeat_Graph_Matrix(inode, istride); + // debug + // if(current_node==-1) + // std::cout << "Graph Matrix node access on task " << myrank << std::endl; + // std::cout << Repeat_Graph_Matrix(inode,istride) << std::endl; + if (node_indices_used(current_node)) + { + // set global assembly map index to the location in the graph matrix where this global node was first found + current_element_index = Element_local_indices(inode, istride, 0); + element_row_index = Element_local_indices(inode, istride, 1); + element_column_index = Element_local_indices(inode, istride, 2); + Global_Gradient_Matrix_Assembly_Map(current_element_index, element_row_index, element_column_index) + = column_index(current_node); + + // swap current node with the end of the current row and shorten the stride of the row + // first swap information about the inverse and forward maps + + current_stride = Graph_Matrix_Strides(inode); + if (istride != current_stride - 1) + { + Element_local_indices(inode, istride, 0) = Element_local_indices(inode, current_stride - 1, 0); + Element_local_indices(inode, istride, 1) = Element_local_indices(inode, current_stride - 1, 1); + Element_local_indices(inode, istride, 2) = Element_local_indices(inode, current_stride - 1, 2); + current_element_index = Element_local_indices(inode, istride, 0); + element_row_index = Element_local_indices(inode, istride, 1); + element_column_index = Element_local_indices(inode, istride, 2); + + Global_Gradient_Matrix_Assembly_Map(current_element_index, element_row_index, element_column_index) + = istride; + + // now that the element map information has been copied, copy the global node index and delete the last index + + Repeat_Graph_Matrix(inode, istride) = Repeat_Graph_Matrix(inode, current_stride - 1); + } + istride--; + Graph_Matrix_Strides(inode)--; + } + else + { + /*this node hasn't shown up in the row before; add it to the list of nodes + that have been scanned uniquely. Use this list to reset the flag array + afterwards without having to loop over all the nodes in the system*/ + node_indices_used(current_node) = 1; + column_index(current_node) = istride; + current_row_nodes_scanned(current_row_n_nodes_scanned) = current_node; + current_row_n_nodes_scanned++; + } + } + // reset nodes used list for the next row of the sparse list + for (int node_reset = 0; node_reset < current_row_n_nodes_scanned; node_reset++) + { + node_indices_used(current_row_nodes_scanned(node_reset)) = 0; + } } - else{ - /*this node hasn't shown up in the row before; add it to the list of nodes - that have been scanned uniquely. Use this list to reset the flag array - afterwards without having to loop over all the nodes in the system*/ - node_indices_used(current_node) = 1; - column_index(current_node) = istride; - current_row_nodes_scanned(current_row_n_nodes_scanned) = current_node; - current_row_n_nodes_scanned++; + }); + Kokkos::fence(); + + Graph_Matrix_Strides.update_host(); + // copy reduced content to non_repeat storage + Graph_Matrix = RaggedRightArrayKokkos(Graph_Matrix_Strides); + + FOR_ALL_CLASS(inode, 0, nlocal_nodes, { + for (int istride = 0; istride < Graph_Matrix_Strides(inode); istride++) + { + Graph_Matrix(inode, istride) = Repeat_Graph_Matrix(inode, istride); } - } - //reset nodes used list for the next row of the sparse list - for(int node_reset = 0; node_reset < current_row_n_nodes_scanned; node_reset++) - node_indices_used(current_row_nodes_scanned(node_reset)) = 0; + }); // end parallel for - } - }); - Kokkos::fence(); - - Graph_Matrix_Strides.update_host(); - //copy reduced content to non_repeat storage - Graph_Matrix = RaggedRightArrayKokkos(Graph_Matrix_Strides); - - FOR_ALL_CLASS(inode, 0, nlocal_nodes, { - for(int istride = 0; istride < Graph_Matrix_Strides(inode); istride++){ - Graph_Matrix(inode,istride) = Repeat_Graph_Matrix(inode,istride); - } + // deallocate repeat matrix + + /*At this stage the sparse graph should have unique global indices on each row. + The constructed Assembly map (to the global sparse matrix) + is used to loop over each element's local stiffness matrix in the assembly process.*/ + + // expand strides for stiffness matrix by multipling by dim + FOR_ALL_CLASS(idof, 0, num_dim * nlocal_nodes, { + Gradient_Matrix_Strides(idof) = num_dim * Graph_Matrix_Strides(idof / num_dim); }); // end parallel for - //deallocate repeat matrix - - /*At this stage the sparse graph should have unique global indices on each row. - The constructed Assembly map (to the global sparse matrix) - is used to loop over each element's local stiffness matrix in the assembly process.*/ - - //expand strides for stiffness matrix by multipling by dim - FOR_ALL_CLASS(idof, 0, num_dim*nlocal_nodes, { - Gradient_Matrix_Strides(idof) = num_dim*Graph_Matrix_Strides(idof/num_dim); + Stiffness_Matrix = Force_Gradient_Positions = RaggedRightArrayKokkos(Gradient_Matrix_Strides); + Force_Gradient_Velocities = RaggedRightArrayKokkos(Gradient_Matrix_Strides); + DOF_Graph_Matrix = RaggedRightArrayKokkos(Gradient_Matrix_Strides); + + // set stiffness Matrix Graph + // debug print + // std::cout << "DOF GRAPH MATRIX ENTRIES ON TASK " << myrank << std::endl; + FOR_ALL_CLASS(idof, 0, num_dim * nlocal_nodes, { + for (int istride = 0; istride < Gradient_Matrix_Strides(idof); istride++) + { + DOF_Graph_Matrix(idof, istride) = Graph_Matrix(idof / num_dim, istride / num_dim) * num_dim + istride % num_dim; + } }); // end parallel for - Stiffness_Matrix = Force_Gradient_Positions = RaggedRightArrayKokkos(Gradient_Matrix_Strides); - Force_Gradient_Velocities = RaggedRightArrayKokkos(Gradient_Matrix_Strides); - DOF_Graph_Matrix = RaggedRightArrayKokkos (Gradient_Matrix_Strides); + Stiffness_Matrix_Strides = Gradient_Matrix_Strides; + + /* + //construct distributed gradient matrix from local kokkos data + //build column map for the global gradient matrix + Teuchos::RCP > colmap; + const Teuchos::RCP > dommap = local_dof_map; + + Tpetra::Details::makeColMap(colmap,dommap,DOF_Graph_Matrix.get_kokkos_view(), nullptr); + + size_t nnz = DOF_Graph_Matrix.size(); + + //debug print + //std::cout << "DOF GRAPH SIZE ON RANK " << myrank << " IS " << nnz << std::endl; - //set stiffness Matrix Graph - //debug print - //std::cout << "DOF GRAPH MATRIX ENTRIES ON TASK " << myrank << std::endl; - FOR_ALL_CLASS(idof, 0, num_dim*nlocal_nodes, { - for (int istride = 0; istride < Gradient_Matrix_Strides(idof); istride++){ - DOF_Graph_Matrix(idof,istride) = Graph_Matrix(idof/num_dim,istride/num_dim)*num_dim + istride%num_dim; + //local indices in the graph using the constructed column map + CArrayKokkos gradient_local_indices(nnz, "gradient_local_indices"); + + //row offsets with compatible template arguments + Kokkos::View row_offsets = DOF_Graph_Matrix.start_index_; + row_pointers row_offsets_pass("row_offsets", nlocal_nodes*num_dim+1); + for(int ipass = 0; ipass < nlocal_nodes*num_dim + 1; ipass++){ + row_offsets_pass(ipass) = row_offsets(ipass); } - }); // end parallel for - Stiffness_Matrix_Strides = Gradient_Matrix_Strides; - - /* - //construct distributed gradient matrix from local kokkos data - //build column map for the global gradient matrix - Teuchos::RCP > colmap; - const Teuchos::RCP > dommap = local_dof_map; - - Tpetra::Details::makeColMap(colmap,dommap,DOF_Graph_Matrix.get_kokkos_view(), nullptr); - - size_t nnz = DOF_Graph_Matrix.size(); - - //debug print - //std::cout << "DOF GRAPH SIZE ON RANK " << myrank << " IS " << nnz << std::endl; - - //local indices in the graph using the constructed column map - CArrayKokkos gradient_local_indices(nnz, "gradient_local_indices"); - - //row offsets with compatible template arguments - Kokkos::View row_offsets = DOF_Graph_Matrix.start_index_; - row_pointers row_offsets_pass("row_offsets", nlocal_nodes*num_dim+1); - for(int ipass = 0; ipass < nlocal_nodes*num_dim + 1; ipass++){ - row_offsets_pass(ipass) = row_offsets(ipass); - } - - size_t entrycount = 0; - for(int irow = 0; irow < nlocal_nodes*num_dim; irow++){ - for(int istride = 0; istride < Gradient_Matrix_Strides(irow); istride++){ - gradient_local_indices(entrycount) = colmap->getLocalElement(DOF_Graph_Matrix(irow,istride)); - entrycount++; + size_t entrycount = 0; + for(int irow = 0; irow < nlocal_nodes*num_dim; irow++){ + for(int istride = 0; istride < Gradient_Matrix_Strides(irow); istride++){ + gradient_local_indices(entrycount) = colmap->getLocalElement(DOF_Graph_Matrix(irow,istride)); + entrycount++; + } } - } - - - //sort values and indices - Tpetra::Import_Util::sortCrsEntries(row_offsets_pass, gradient_local_indices.get_kokkos_view(), Force_Gradient_Positions.get_kokkos_view()); - Tpetra::Import_Util::sortCrsEntries(row_offsets_pass, gradient_local_indices.get_kokkos_view(), Force_Gradient_Velocities.get_kokkos_view()); - - //Teuchos::RCP crs_matrix_params = Teuchos::rcp(new Teuchos::ParameterList("crsmatrix")); - //crs_matrix_params->set("sorted", false); - distributed_force_gradient_positions = Teuchos::rcp(new MAT(local_dof_map, colmap, row_offsets_pass, gradient_local_indices.get_kokkos_view(), Force_Gradient_Positions.get_kokkos_view())); - distributed_force_gradient_positions->fillComplete(); - distributed_force_gradient_velocities = Teuchos::rcp(new MAT(local_dof_map, colmap, row_offsets_pass, gradient_local_indices.get_kokkos_view(), Force_Gradient_Velocities.get_kokkos_view())); - distributed_force_gradient_velocities->fillComplete(); - */ - //distributed_force_gradient_positions->describe(*fos,Teuchos::VERB_EXTREME); - //distributed_force_gradient_velocities->describe(*fos,Teuchos::VERB_EXTREME); + + + //sort values and indices + Tpetra::Import_Util::sortCrsEntries(row_offsets_pass, gradient_local_indices.get_kokkos_view(), Force_Gradient_Positions.get_kokkos_view()); + Tpetra::Import_Util::sortCrsEntries(row_offsets_pass, gradient_local_indices.get_kokkos_view(), Force_Gradient_Velocities.get_kokkos_view()); + + //Teuchos::RCP crs_matrix_params = Teuchos::rcp(new Teuchos::ParameterList("crsmatrix")); + //crs_matrix_params->set("sorted", false); + distributed_force_gradient_positions = Teuchos::rcp(new MAT(local_dof_map, colmap, row_offsets_pass, gradient_local_indices.get_kokkos_view(), Force_Gradient_Positions.get_kokkos_view())); + distributed_force_gradient_positions->fillComplete(); + distributed_force_gradient_velocities = Teuchos::rcp(new MAT(local_dof_map, colmap, row_offsets_pass, gradient_local_indices.get_kokkos_view(), Force_Gradient_Velocities.get_kokkos_view())); + distributed_force_gradient_velocities->fillComplete(); + */ + // distributed_force_gradient_positions->describe(*fos,Teuchos::VERB_EXTREME); + // distributed_force_gradient_velocities->describe(*fos,Teuchos::VERB_EXTREME); } /* ---------------------------------------------------------------------- Enforce boundary conditions on the adjoint vectors ------------------------------------------------------------------------- */ -void FEA_Module_Dynamic_Elasticity::boundary_adjoint(const mesh_t &mesh, - const DCArrayKokkos &boundary, - vec_array &node_adjoint, - vec_array &node_phi_adjoint){ - - //error and debug flag - //DCArrayKokkos print_flag(1, "print_flag"); - //print_flag.host(0) = false; - //print_flag.update_device(); - - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; +void FEA_Module_Dynamic_Elasticity::boundary_adjoint(const mesh_t& mesh, + const DCArrayKokkos& boundary, + vec_array& node_adjoint, + vec_array& node_phi_adjoint) +{ + // error and debug flag + // DCArrayKokkos print_flag(1, "print_flag"); + // print_flag.host(0) = false; + // print_flag.update_device(); + + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; int num_dims = simparam->num_dims; // Loop over boundary sets - for (size_t bdy_set=0; bdy_set fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - //if(myrank==0) - //*fos << "Density data :" << std::endl; - //node_densities_distributed->describe(*fos,Teuchos::VERB_EXTREME); - //*fos << std::endl; - //std::fflush(stdout); - - //communicate design densities - //create import object using local node indices map and all indices map - //Tpetra::Import importer(map, all_node_map); - - //comms to get ghosts - (*adjoint_vector_data)[cycle]->doImport(*adjoint_vector_distributed, *importer, Tpetra::INSERT); - (*phi_adjoint_vector_data)[cycle]->doImport(*phi_adjoint_vector_distributed, *importer, Tpetra::INSERT); - //all_node_map->describe(*fos,Teuchos::VERB_EXTREME); - //all_node_velocities_distributed->describe(*fos,Teuchos::VERB_EXTREME); - - //update_count++; - //if(update_count==1){ - //MPI_Barrier(world); - //MPI_Abort(world,4); - //} +void FEA_Module_Dynamic_Elasticity::comm_adjoint_vectors(int cycle) +{ + // debug print of design vector + // std::ostream &out = std::cout; + // Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + // if(myrank==0) + // *fos << "Density data :" << std::endl; + // node_densities_distributed->describe(*fos,Teuchos::VERB_EXTREME); + // *fos << std::endl; + // std::fflush(stdout); + + // communicate design densities + // create import object using local node indices map and all indices map + // Tpetra::Import importer(map, all_node_map); + + // comms to get ghosts + (*adjoint_vector_data)[cycle]->doImport(*adjoint_vector_distributed, *importer, Tpetra::INSERT); + (*phi_adjoint_vector_data)[cycle]->doImport(*phi_adjoint_vector_distributed, *importer, Tpetra::INSERT); + // all_node_map->describe(*fos,Teuchos::VERB_EXTREME); + // all_node_velocities_distributed->describe(*fos,Teuchos::VERB_EXTREME); + + // update_count++; + // if(update_count==1){ + // MPI_Barrier(world); + // MPI_Abort(world,4); + // } } diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/properties.cpp b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/properties.cpp index 0d8779959..f0c6f14ff 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/properties.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/properties.cpp @@ -139,7 +139,6 @@ void FEA_Module_Dynamic_Elasticity::update_state(const DCArrayKokkos cycle, rk_level, time_value); - } // end logical on hyper strength model // --- Pressure --- @@ -244,7 +243,6 @@ void FEA_Module_Dynamic_Elasticity::update_state2D(const DCArrayKokkosmaterial_options.size(); const int num_dim = simparam->num_dims; -<<<<<<< HEAD const DCArrayKokkos mat_fill = simparam->mat_fill; const DCArrayKokkos boundary = simparam->boundary; const DCArrayKokkos material = simparam->material; - global_vars = simparam->global_vars; - state_vars = DCArrayKokkos(rnum_elem, simparam->max_num_state_vars); + eos_global_vars = simparam->eos_global_vars; + strength_global_vars = simparam->strength_global_vars; + eos_state_vars = DCArrayKokkos(rnum_elem, simparam->max_num_eos_state_vars); + strength_state_vars = DCArrayKokkos(rnum_elem, simparam->max_num_strength_state_vars); // --- calculate bdy sets ---// mesh.num_nodes_in_patch = 2 * (num_dim - 1); // 2 (2D) or 4 (3D) mesh.num_patches_in_elem = 2 * num_dim; // 4 (2D) or 6 (3D) -======= - const DCArrayKokkos mat_fill = simparam->mat_fill; - const DCArrayKokkos boundary = simparam->boundary; - const DCArrayKokkos material = simparam->material; - eos_global_vars = simparam->eos_global_vars; - strength_global_vars = simparam->strength_global_vars; - eos_state_vars = DCArrayKokkos (rnum_elem, simparam->max_num_eos_state_vars); - strength_state_vars = DCArrayKokkos (rnum_elem, simparam->max_num_strength_state_vars); - - //--- calculate bdy sets ---// - mesh.num_nodes_in_patch = 2*(num_dim-1); // 2 (2D) or 4 (3D) - mesh.num_patches_in_elem = 2*num_dim; // 4 (2D) or 6 (3D) ->>>>>>> main + mesh.init_bdy_sets(num_bcs); num_bdy_sets = mesh.num_bdy_sets; printf("Num BC's = %lu\n", num_bcs); diff --git a/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/FEA_Module_Eulerian.h b/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/FEA_Module_Eulerian.h index 5a4e42fd7..e6fcf3a66 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/FEA_Module_Eulerian.h +++ b/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/FEA_Module_Eulerian.h @@ -175,7 +175,6 @@ class FEA_Module_Eulerian : public FEA_Module // corner ids in elem CArrayKokkos corners_in_elem; - // elem ids in elem RaggedRightArrayKokkos elems_in_elem; CArrayKokkos num_elems_in_elem; @@ -184,13 +183,12 @@ class FEA_Module_Eulerian : public FEA_Module CArrayKokkos patches_in_elem; // for storing global variables used in user material model - DCArrayKokkos eos_global_vars; - DCArrayKokkos strength_global_vars; + DCArrayKokkos eos_global_vars; + DCArrayKokkos strength_global_vars; // for storing global variables used in user material model - DCArrayKokkos eos_state_vars; - DCArrayKokkos strength_state_vars; - + DCArrayKokkos eos_state_vars; + DCArrayKokkos strength_state_vars; // node ids in a patch CArrayKokkos nodes_in_patch; diff --git a/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/Simulation_Parameters_Eulerian.h b/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/Simulation_Parameters_Eulerian.h index 48f2d12fb..94aa7cd67 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/Simulation_Parameters_Eulerian.h +++ b/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/Simulation_Parameters_Eulerian.h @@ -11,14 +11,14 @@ This program is open source under the BSD-3 License. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - + 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - + 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - + 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. @@ -52,92 +52,105 @@ using namespace mtr; -struct Simulation_Parameters_Eulerian : Simulation_Parameters { - Time_Variables time_variables; - std::vector region_options; - std::vector material_options; - std::vector boundary_conditions; - Graphics_Options graphics_options; - - bool gravity_flag = false; - bool report_runtime = true; - - size_t rk_num_stages = 2; - int NB = 6; // number of boundaries - int NBSF = 4; //number of surface density force conditions - int NBV = 2; //number of surface sets used to specify a fixed displacement on nodes belonging to respective surfaces - - //Non-serialized fields - int num_gauss_points = 2; - size_t max_num_eos_global_vars; - size_t max_num_strength_global_vars; - size_t rk_num_bins; - double time_value = 0.0; - DCArrayKokkos eos_global_vars; - DCArrayKokkos strength_global_vars; - - DCArrayKokkos mat_fill; - DCArrayKokkos material; - DCArrayKokkos boundary; - std::vector gravity_vector {9.81, 0., 0.}; - - void init_material_variable_arrays(size_t num_eos_global_vars, size_t num_strength_global_vars) { - eos_global_vars = DCArrayKokkos (material_options.size(), num_eos_global_vars); - strength_global_vars = DCArrayKokkos (material_options.size(), num_strength_global_vars); - - for (size_t i = 0; i < material_options.size(); i++) { - auto mat = material_options[i]; - - for (size_t j = 0; j < mat.eos_global_vars.size(); j++) { - eos_global_vars.host(i, j) = mat.eos_global_vars[j]; - } - - for (size_t j = 0; j < mat.strength_global_vars.size(); j++) { - strength_global_vars.host(i, j) = mat.strength_global_vars[j]; - } +struct Simulation_Parameters_Eulerian : Simulation_Parameters +{ + Time_Variables time_variables; + std::vector region_options; + std::vector material_options; + std::vector boundary_conditions; + Graphics_Options graphics_options; + + bool gravity_flag = false; + bool report_runtime = true; + + size_t rk_num_stages = 2; + int NB = 6; // number of boundaries + int NBSF = 4; // number of surface density force conditions + int NBV = 2; // number of surface sets used to specify a fixed displacement on nodes belonging to respective surfaces + + // Non-serialized fields + int num_gauss_points = 2; + size_t max_num_eos_global_vars; + size_t max_num_strength_global_vars; + size_t rk_num_bins; + double time_value = 0.0; + DCArrayKokkos eos_global_vars; + DCArrayKokkos strength_global_vars; + + DCArrayKokkos mat_fill; + DCArrayKokkos material; + DCArrayKokkos boundary; + std::vector gravity_vector { 9.81, 0., 0. }; + + void init_material_variable_arrays(size_t num_eos_global_vars, size_t num_strength_global_vars) + { + eos_global_vars = DCArrayKokkos(material_options.size(), num_eos_global_vars); + strength_global_vars = DCArrayKokkos(material_options.size(), num_strength_global_vars); + + for (size_t i = 0; i < material_options.size(); i++) + { + auto mat = material_options[i]; + + for (size_t j = 0; j < mat.eos_global_vars.size(); j++) + { + eos_global_vars.host(i, j) = mat.eos_global_vars[j]; + } + + for (size_t j = 0; j < mat.strength_global_vars.size(); j++) + { + strength_global_vars.host(i, j) = mat.strength_global_vars[j]; + } + } } - } - - template void from_vector(DCArrayKokkos& array, const std::vector& vec) { - array = DCArrayKokkos(vec.size()); - for (size_t i = 0; i < vec.size(); i++) - array.host(i) = *(T*)&vec[i]; - } - void derive_kokkos_arrays() { - max_num_eos_global_vars = 0; - max_num_strength_global_vars = 0; - for (auto mo : material_options) { - max_num_eos_global_vars = std::max(max_num_eos_global_vars, mo.eos_global_vars.size()); - max_num_strength_global_vars = std::max(max_num_strength_global_vars, mo.strength_global_vars.size()); - } - - init_material_variable_arrays(max_num_eos_global_vars, max_num_strength_global_vars); - from_vector(mat_fill, region_options); - from_vector(material, material_options); - from_vector(boundary, boundary_conditions); + template void from_vector(DCArrayKokkos& array, const std::vector& vec) + { + array = DCArrayKokkos(vec.size()); + for (size_t i = 0; i < vec.size(); i++) + { + array.host(i) = *(T*)&vec[i]; + } + } - // Send to device. - mat_fill.update_device(); - boundary.update_device(); - material.update_device(); - eos_global_vars.update_device(); - strength_global_vars.update_device(); + void derive_kokkos_arrays() + { + max_num_eos_global_vars = 0; + max_num_strength_global_vars = 0; + for (auto mo : material_options) + { + max_num_eos_global_vars = std::max(max_num_eos_global_vars, mo.eos_global_vars.size()); + max_num_strength_global_vars = std::max(max_num_strength_global_vars, mo.strength_global_vars.size()); + } + + init_material_variable_arrays(max_num_eos_global_vars, max_num_strength_global_vars); + + from_vector(mat_fill, region_options); + from_vector(material, material_options); + from_vector(boundary, boundary_conditions); + + // Send to device. + mat_fill.update_device(); + boundary.update_device(); + material.update_device(); + eos_global_vars.update_device(); + strength_global_vars.update_device(); + } - } + void derive() + { + derive_kokkos_arrays(); + rk_num_bins = rk_num_stages; + } - void derive() { - derive_kokkos_arrays(); - rk_num_bins = rk_num_stages; - } - void validate() { - validate_module_is_specified(FEA_MODULE_TYPE::Eulerian); - } + void validate() + { + validate_module_is_specified(FEA_MODULE_TYPE::Eulerian); + } }; -IMPL_YAML_SERIALIZABLE_WITH_BASE(Simulation_Parameters_Eulerian, Simulation_Parameters, - time_variables, material_options, region_options, +IMPL_YAML_SERIALIZABLE_WITH_BASE(Simulation_Parameters_Eulerian, Simulation_Parameters, + time_variables, material_options, region_options, boundary_conditions, gravity_flag, report_runtime, rk_num_stages, NB, NBSF, NBV, graphics_options -) + ) #endif // end HEADER_H diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_sgh.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_sgh.cpp index debe2c4d3..1c367ba6e 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_sgh.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_sgh.cpp @@ -408,8 +408,8 @@ void FEA_Module_SGH::get_force_sgh(const DCArrayKokkos& material, if (material(mat_id).strength_run_location == RUN_LOCATION::device) { // cut out the node_gids for this element - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 8); - + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 8); + // --- call strength model --- elem_strength(elem_gid).calc_stress(elem_pres, elem_stress, @@ -490,7 +490,6 @@ void FEA_Module_SGH::get_force_sgh(const DCArrayKokkos& material, cycle, rk_level, time_value); - } // end logical for strength run location } // end logical on hypo strength model } // end for loop over elements @@ -874,8 +873,7 @@ void FEA_Module_SGH::get_force_sgh2D(const DCArrayKokkos& material, rk_alpha, cycle, rk_level, - time_value); - + time_value); } // end logical on hypo strength model }); // end parallel for loop over elements diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/properties.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/properties.cpp index 2ec6d273c..0340aa0b3 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/properties.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/properties.cpp @@ -107,7 +107,6 @@ void FEA_Module_SGH::update_state(const DCArrayKokkos& material, elem_gid, rk_level); - // --- call strength model --- elem_strength(elem_gid).calc_stress(elem_pres, elem_stress, @@ -131,7 +130,6 @@ void FEA_Module_SGH::update_state(const DCArrayKokkos& material, cycle, rk_level, time_value); - } // end logical on hyper strength model // --- Pressure --- @@ -259,8 +257,6 @@ void FEA_Module_SGH::update_state2D(const DCArrayKokkos& material, cycle, rk_level, time_value); - - } // end logical on hyper strength model // --- Pressure --- From c519599ba476499a3742ed7caa8d5eaf01f03385 Mon Sep 17 00:00:00 2001 From: Jacob Moore Date: Fri, 23 Feb 2024 11:16:05 -0600 Subject: [PATCH 28/64] ENH: Auto build uncrustify and modify scripts --- .gitmodules | 3 +++ dev-utils/uncrustify | 1 + docs/formatting/ClassDefinitionComment.txt | 11 +++++++++++ docs/formatting/FunctionDefinitionComment.txt | 16 ++++++++++++++++ docs/formatting/uncrustify.cfg | 13 +++++++++++++ scripts/build-fierro.sh | 15 ++++++++++++++- scripts/setup-env.sh | 12 ++++++++++++ 7 files changed, 70 insertions(+), 1 deletion(-) create mode 160000 dev-utils/uncrustify create mode 100644 docs/formatting/ClassDefinitionComment.txt create mode 100644 docs/formatting/FunctionDefinitionComment.txt diff --git a/.gitmodules b/.gitmodules index 2df858fde..3b51d6710 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,3 +2,6 @@ path = lib/Elements url = https://github.com/lanl/ELEMENTS.git branch = master +[submodule "dev-utils/uncrustify"] + path = dev-utils/uncrustify + url = https://github.com/uncrustify/uncrustify.git diff --git a/dev-utils/uncrustify b/dev-utils/uncrustify new file mode 160000 index 000000000..e312b0e41 --- /dev/null +++ b/dev-utils/uncrustify @@ -0,0 +1 @@ +Subproject commit e312b0e410dc41814b69c78e4b3d51b5311f7a49 diff --git a/docs/formatting/ClassDefinitionComment.txt b/docs/formatting/ClassDefinitionComment.txt new file mode 100644 index 000000000..858bc3ed7 --- /dev/null +++ b/docs/formatting/ClassDefinitionComment.txt @@ -0,0 +1,11 @@ +///////////////////////////////////////////////////////////////////////////// +/// +/// \class $(class) +/// +/// \brief +/// +/// < Insert longer more detailed +/// description which can span +/// multiple lines if needed > +/// +///////////////////////////////////////////////////////////////////////////// diff --git a/docs/formatting/FunctionDefinitionComment.txt b/docs/formatting/FunctionDefinitionComment.txt new file mode 100644 index 000000000..142862205 --- /dev/null +++ b/docs/formatting/FunctionDefinitionComment.txt @@ -0,0 +1,16 @@ +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn $(function) +/// +/// \brief +/// +/// +/// +/// \param +/// \param +/// \param +/// +/// \return +/// +///////////////////////////////////////////////////////////////////////////// diff --git a/docs/formatting/uncrustify.cfg b/docs/formatting/uncrustify.cfg index 7f086a6b0..e12185b98 100755 --- a/docs/formatting/uncrustify.cfg +++ b/docs/formatting/uncrustify.cfg @@ -466,6 +466,19 @@ cmt_indent_multi = false # false/true # Will substitute $(filename) with the current file's name. cmt_insert_file_header = "HeaderForSources.txt" +# Path to a file that contains text to insert before a function definition if +# the function isn't preceded by a C/C++ comment. If the inserted text +# contains '$(function)', '$(javaparam)' or '$(fclass)', these will be +# replaced with, respectively, the name of the function, the javadoc '@param' +# and '@return' stuff, or the name of the class to which the member function +# belongs. +cmt_insert_func_header = "FunctionDefinitionComment.txt" # string + +# Path to a file that contains text to insert before a class if the class +# isn't preceded by a C/C++ comment. If the inserted text contains '$(class)', +# that will be replaced with the class name. +cmt_insert_class_header = "" # string + # If a namespace body exceeds the specified number of newlines and doesn't have a comment after # the close brace, a comment will be added. mod_add_long_namespace_closebrace_comment = 1 # number diff --git a/scripts/build-fierro.sh b/scripts/build-fierro.sh index 8ba8b2b99..bedef4c51 100755 --- a/scripts/build-fierro.sh +++ b/scripts/build-fierro.sh @@ -138,7 +138,7 @@ for arg in "$@"; do return 1 ;; *) - echo "Error: Invalid argument or value specified." + echo "Error: Invalid argument or value specified.scriptdir" show_help return 1 ;; @@ -180,6 +180,19 @@ cd "$( dirname "${BASH_SOURCE[0]}" )" # Always setup the environment source setup-env.sh ${machine} ${kokkos_build_type} ${build_cores} +# Install Uncrustify +if [ -d "${UNCRUSTIFY_BUILD_DIR}" ]; then + cd ${topdir} + echo "Installing Uncrustify..." + cd ${UNCRUSTIFY_BUILD_DIR} + echo $(pwd) + cmake -DCMAKE_BUILD_TYPE=Release .. + make -j ${build_cores} + cd ${topdir} +fi + +cd "$( dirname "${BASH_SOURCE[0]}" )" + # Next, do action based on args if [ "$build_action" = "full-app" ]; then source trilinos-install.sh ${kokkos_build_type} ${machine} diff --git a/scripts/setup-env.sh b/scripts/setup-env.sh index d638f4f38..b96a0bbae 100644 --- a/scripts/setup-env.sh +++ b/scripts/setup-env.sh @@ -18,6 +18,18 @@ export trilinosdir=${libdir} export builddir=${basedir}/${my_build} export installdir=${basedir}/install +export devutilsdir=${topdir}/dev-utils + +if { [ ! -d "${devutilsdir}/uncrustify/build" ] ;} +then + echo "Missing uncrustify build directory, making it now " + mkdir ${devutilsdir}/uncrustify/build +fi + +export UNCRUSTIFY_SOURCE_DIR=${devutilsdir}/uncrustify +export UNCRUSTIFY_BUILD_DIR=${devutilsdir}/uncrustify/build +export UNCRUSTIFY_INSTALL_DIR=${devutilsdir}/uncrustify/build + export FIERRO_BASE_DIR=${basedir} export FIERRO_SOURCE_DIR=${srcdir} export FIERRO_EXPLICIT_SOURCE_DIR=${srcdir}/Parallel-Solvers/Parallel-Explicit From 1d15e9f5f01b07a28e9d2864ffc337b2f302b7ee Mon Sep 17 00:00:00 2001 From: Jacob Moore Date: Fri, 23 Feb 2024 11:17:36 -0600 Subject: [PATCH 29/64] COMP: Fix merge bug --- .../FEA_Module_Dynamic_Elasticity.cpp | 8 +- .../FEA_Module_Dynamic_Elasticity.h | 1034 +++--- .../elastic_optimization.cpp | 2855 ++++++++--------- .../SGH_Solver/include/FEA_Module_SGH.h | 1246 +++---- .../SGH_Solver/src/FEA_Module_SGH.cpp | 19 +- .../SGH_Solver/src/boundary.cpp | 16 + .../SGH_Solver/src/energy_sgh.cpp | 16 + .../SGH_Solver/src/force_gradients_sgh.cpp | 132 +- .../SGH_Solver/src/force_sgh.cpp | 14 +- .../SGH_Solver/src/setup_sgh.cpp | 735 +++-- .../SGH_Solver/src/sgh_optimization.cpp | 342 +- 11 files changed, 3260 insertions(+), 3157 deletions(-) diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.cpp b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.cpp index 0561d4d5a..87417e713 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.cpp @@ -1338,11 +1338,9 @@ void FEA_Module_Dynamic_Elasticity::setup() // p = rho*ie*(gamma - 1) size_t mat_id = f_id; - << << << < HEAD - double gamma = global_vars(mat_id, 0); // gamma value - == == == = - double gamma = eos_global_vars(mat_id, 0); // gamma value - >> >> >> > main + + double gamma = eos_global_vars(mat_id, 0); // gamma value + elem_sie(rk_level, elem_gid) = elem_pres(elem_gid) / (mat_fill(f_id).den * (gamma - 1.0)); } // end if diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.h b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.h index e0ed923e6..ac1c0e6a2 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.h +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.h @@ -11,11 +11,14 @@ This program is open source under the BSD-3 License. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. @@ -44,539 +47,556 @@ #include "material_models.h" class Explicit_Solver; + class Solver; + class Simulation_Parameters_Explicit; + class Dynamic_Elasticity_Parameters; struct material_t; + struct boundary_t; -class FEA_Module_Dynamic_Elasticity : public FEA_Module -{ +class FEA_Module_Dynamic_Elasticity: public FEA_Module { + public: + + FEA_Module_Dynamic_Elasticity( Dynamic_Elasticity_Parameters& params, Solver *Solver_Pointer, std::shared_ptr mesh_in, const int my_fea_module_index = 0); + ~FEA_Module_Dynamic_Elasticity(); + + //initialize data for boundaries of the model and storage for boundary conditions and applied loads + void elastic_interface_setup(node_t &node, elem_t &elem, corner_t &corner); + + void setup(); + + void cleanup_material_models(); + + int solve(); + + void module_cleanup(); + + void elastic_solve(); + + void get_force_vgradient_elastic(const DCArrayKokkos &material, + const mesh_t &mesh, + const DViewCArrayKokkos &node_coords, + const DViewCArrayKokkos &node_vel, + const DViewCArrayKokkos &elem_den, + const DViewCArrayKokkos &elem_sie, + const DViewCArrayKokkos &elem_pres, + const DViewCArrayKokkos &elem_stress, + const DViewCArrayKokkos &elem_sspd, + const DViewCArrayKokkos &elem_vol, + const DViewCArrayKokkos &elem_div, + const DViewCArrayKokkos &elem_mat_id, + const double rk_alpha, + const size_t cycle); + + void get_force_ugradient_elastic(const DCArrayKokkos &material, + const mesh_t &mesh, + const DViewCArrayKokkos &node_coords, + const DViewCArrayKokkos &node_vel, + const DViewCArrayKokkos &elem_den, + const DViewCArrayKokkos &elem_sie, + const DViewCArrayKokkos &elem_pres, + const DViewCArrayKokkos &elem_stress, + const DViewCArrayKokkos &elem_sspd, + const DViewCArrayKokkos &elem_vol, + const DViewCArrayKokkos &elem_div, + const DViewCArrayKokkos &elem_mat_id, + const double rk_alpha, + const size_t cycle); + + void get_force_dgradient_elastic(const DCArrayKokkos &material, + const mesh_t &mesh, + const DViewCArrayKokkos &node_coords, + const DViewCArrayKokkos &node_vel, + const DViewCArrayKokkos &elem_den, + const DViewCArrayKokkos &elem_sie, + const DViewCArrayKokkos &elem_pres, + const DViewCArrayKokkos &elem_stress, + const DViewCArrayKokkos &elem_sspd, + const DViewCArrayKokkos &elem_vol, + const DViewCArrayKokkos &elem_div, + const DViewCArrayKokkos &elem_mat_id, + const double rk_alpha, + const size_t cycle); + + void force_design_gradient_term(const_vec_array design_variables, vec_array design_gradients); + + void update_position_elastic(double rk_alpha, + const size_t num_nodes, + DViewCArrayKokkos &node_coords, + const DViewCArrayKokkos &node_vel); + + void get_vol(); + + void init_assembly(); + + KOKKOS_INLINE_FUNCTION + void get_vol_hex(const DViewCArrayKokkos &elem_vol, + const size_t elem_gid, + const DViewCArrayKokkos &node_coords, + const ViewCArrayKokkos &elem_node_gids, + const size_t rk_level) const; + + + KOKKOS_INLINE_FUNCTION + void get_vol_quad(const DViewCArrayKokkos &elem_vol, + const size_t elem_gid, + const DViewCArrayKokkos &node_coords, + const ViewCArrayKokkos &elem_node_gids, + const size_t rk_level) const; + + + KOKKOS_FUNCTION + double get_area_quad(const size_t elem_gid, + const DViewCArrayKokkos &node_coords, + const ViewCArrayKokkos &elem_node_gids, + const size_t rk_level) const; + + + KOKKOS_FUNCTION + void get_bmatrix(const ViewCArrayKokkos &B_matrix, + const size_t elem_gid, + const DViewCArrayKokkos &node_coords, + const ViewCArrayKokkos &elem_node_gids, + const size_t rk_level) const; - FEA_Module_Dynamic_Elasticity(Dynamic_Elasticity_Parameters& params, Solver* Solver_Pointer, std::shared_ptr mesh_in, const int my_fea_module_index = 0); - ~FEA_Module_Dynamic_Elasticity(); - - // initialize data for boundaries of the model and storage for boundary conditions and applied loads - void elastic_interface_setup(node_t& node, elem_t& elem, corner_t& corner); - - void setup(); - - void cleanup_material_models(); - - int solve(); - - void module_cleanup(); - - void elastic_solve(); - - void get_force_vgradient_elastic(const DCArrayKokkos& material, - const mesh_t& mesh, - const DViewCArrayKokkos& node_coords, - const DViewCArrayKokkos& node_vel, - const DViewCArrayKokkos& elem_den, - const DViewCArrayKokkos& elem_sie, - const DViewCArrayKokkos& elem_pres, - const DViewCArrayKokkos& elem_stress, - const DViewCArrayKokkos& elem_sspd, - const DViewCArrayKokkos& elem_vol, - const DViewCArrayKokkos& elem_div, - const DViewCArrayKokkos& elem_mat_id, - const double rk_alpha, - const size_t cycle); - - void get_force_ugradient_elastic(const DCArrayKokkos& material, - const mesh_t& mesh, - const DViewCArrayKokkos& node_coords, - const DViewCArrayKokkos& node_vel, - const DViewCArrayKokkos& elem_den, - const DViewCArrayKokkos& elem_sie, - const DViewCArrayKokkos& elem_pres, - const DViewCArrayKokkos& elem_stress, - const DViewCArrayKokkos& elem_sspd, - const DViewCArrayKokkos& elem_vol, - const DViewCArrayKokkos& elem_div, - const DViewCArrayKokkos& elem_mat_id, - const double rk_alpha, - const size_t cycle); - - void get_force_dgradient_elastic(const DCArrayKokkos& material, - const mesh_t& mesh, - const DViewCArrayKokkos& node_coords, - const DViewCArrayKokkos& node_vel, - const DViewCArrayKokkos& elem_den, - const DViewCArrayKokkos& elem_sie, - const DViewCArrayKokkos& elem_pres, - const DViewCArrayKokkos& elem_stress, - const DViewCArrayKokkos& elem_sspd, - const DViewCArrayKokkos& elem_vol, - const DViewCArrayKokkos& elem_div, - const DViewCArrayKokkos& elem_mat_id, - const double rk_alpha, - const size_t cycle); - - void force_design_gradient_term(const_vec_array design_variables, vec_array design_gradients); - - void update_position_elastic(double rk_alpha, - const size_t num_nodes, - DViewCArrayKokkos& node_coords, - const DViewCArrayKokkos& node_vel); - - void get_vol(); - - void init_assembly(); - - KOKKOS_INLINE_FUNCTION - void get_vol_hex(const DViewCArrayKokkos& elem_vol, - const size_t elem_gid, - const DViewCArrayKokkos& node_coords, - const ViewCArrayKokkos& elem_node_gids, - const size_t rk_level) const; - KOKKOS_INLINE_FUNCTION - void get_vol_quad(const DViewCArrayKokkos& elem_vol, - const size_t elem_gid, - const DViewCArrayKokkos& node_coords, - const ViewCArrayKokkos& elem_node_gids, - const size_t rk_level) const; - - KOKKOS_FUNCTION - double get_area_quad(const size_t elem_gid, - const DViewCArrayKokkos& node_coords, - const ViewCArrayKokkos& elem_node_gids, - const size_t rk_level) const; - - KOKKOS_FUNCTION - void get_bmatrix(const ViewCArrayKokkos& B_matrix, + KOKKOS_FUNCTION + void get_bmatrix2D(const ViewCArrayKokkos &B_matrix, const size_t elem_gid, - const DViewCArrayKokkos& node_coords, - const ViewCArrayKokkos& elem_node_gids, + const DViewCArrayKokkos &node_coords, + const ViewCArrayKokkos &elem_node_gids, const size_t rk_level) const; - KOKKOS_FUNCTION - void get_bmatrix2D(const ViewCArrayKokkos& B_matrix, - const size_t elem_gid, - const DViewCArrayKokkos& node_coords, - const ViewCArrayKokkos& elem_node_gids, - const size_t rk_level) const; + KOKKOS_FUNCTION + void get_area_weights2D(const ViewCArrayKokkos &corner_areas, + const size_t elem_gid, + const DViewCArrayKokkos &node_coords, + const ViewCArrayKokkos &elem_node_gids, + const size_t rk_level) const; + + + KOKKOS_INLINE_FUNCTION + double heron(const double x1, + const double y1, + const double x2, + const double y2, + const double x3, + const double y3) const; + + double average_element_density(const int nodes_per_elem, const CArray current_element_densities) const; + + void get_divergence(DViewCArrayKokkos &elem_div, + const mesh_t mesh, + const DViewCArrayKokkos &node_coords, + const DViewCArrayKokkos &node_vel, + const DViewCArrayKokkos &elem_vol); - KOKKOS_FUNCTION - void get_area_weights2D(const ViewCArrayKokkos& corner_areas, - const size_t elem_gid, - const DViewCArrayKokkos& node_coords, - const ViewCArrayKokkos& elem_node_gids, - const size_t rk_level) const; - - KOKKOS_INLINE_FUNCTION - double heron(const double x1, - const double y1, - const double x2, - const double y2, - const double x3, - const double y3) const; - - double average_element_density(const int nodes_per_elem, - const CArray current_element_densities) const; - - void get_divergence(DViewCArrayKokkos& elem_div, + + void get_divergence2D(DViewCArrayKokkos &elem_div, const mesh_t mesh, - const DViewCArrayKokkos& node_coords, - const DViewCArrayKokkos& node_vel, - const DViewCArrayKokkos& elem_vol); - - void get_divergence2D(DViewCArrayKokkos& elem_div, - const mesh_t mesh, - const DViewCArrayKokkos& node_coords, - const DViewCArrayKokkos& node_vel, - const DViewCArrayKokkos& elem_vol); - - KOKKOS_FUNCTION - void get_velgrad(ViewCArrayKokkos& vel_grad, - const ViewCArrayKokkos& elem_node_gids, - const DViewCArrayKokkos& node_vel, - const ViewCArrayKokkos& b_matrix, + const DViewCArrayKokkos &node_coords, + const DViewCArrayKokkos &node_vel, + const DViewCArrayKokkos &elem_vol); + + + KOKKOS_FUNCTION + void get_velgrad(ViewCArrayKokkos &vel_grad, + const ViewCArrayKokkos &elem_node_gids, + const DViewCArrayKokkos &node_vel, + const ViewCArrayKokkos &b_matrix, + const double elem_vol, + const size_t elem_gid, + const size_t rk_level) const; + + + KOKKOS_FUNCTION + void get_velgrad2D(ViewCArrayKokkos &vel_grad, + const ViewCArrayKokkos &elem_node_gids, + const DViewCArrayKokkos &node_vel, + const ViewCArrayKokkos &b_matrix, const double elem_vol, + const double elem_area, const size_t elem_gid, const size_t rk_level) const; - KOKKOS_FUNCTION - void get_velgrad2D(ViewCArrayKokkos& vel_grad, - const ViewCArrayKokkos& elem_node_gids, - const DViewCArrayKokkos& node_vel, - const ViewCArrayKokkos& b_matrix, - const double elem_vol, - const double elem_area, - const size_t elem_gid, - const size_t rk_level) const; - - KOKKOS_INLINE_FUNCTION - void decompose_vel_grad(ViewCArrayKokkos& D_tensor, - ViewCArrayKokkos& W_tensor, - const ViewCArrayKokkos& vel_grad, - const ViewCArrayKokkos& elem_node_gids, - const size_t elem_gid, - const DViewCArrayKokkos& node_coords, - const DViewCArrayKokkos& node_vel, - const double vol) const; - - void boundary_velocity(const mesh_t& mesh, - const DCArrayKokkos& boundary, - DViewCArrayKokkos& node_vel); - - void boundary_adjoint(const mesh_t& mesh, - const DCArrayKokkos& boundary, - vec_array& node_adjoint, - vec_array& node_phi_adjoint); - - void tag_bdys(const DCArrayKokkos& boundary, - mesh_t& mesh, - const DViewCArrayKokkos& node_coords); - - KOKKOS_INLINE_FUNCTION - bool check_bdy(const size_t patch_gid, - const int num_dim, - const int num_nodes_in_patch, + KOKKOS_INLINE_FUNCTION + void decompose_vel_grad(ViewCArrayKokkos &D_tensor, + ViewCArrayKokkos &W_tensor, + const ViewCArrayKokkos &vel_grad, + const ViewCArrayKokkos &elem_node_gids, + const size_t elem_gid, + const DViewCArrayKokkos &node_coords, + const DViewCArrayKokkos &node_vel, + const double vol) const; + + void boundary_velocity(const mesh_t &mesh, + const DCArrayKokkos &boundary, + DViewCArrayKokkos &node_vel); + + void boundary_adjoint(const mesh_t &mesh, + const DCArrayKokkos &boundary, + vec_array &node_adjoint, + vec_array &node_phi_adjoint); + + void tag_bdys(const DCArrayKokkos &boundary, + mesh_t &mesh, + const DViewCArrayKokkos &node_coords); + + KOKKOS_INLINE_FUNCTION + bool check_bdy(const size_t patch_gid, + const int num_dim, + const int num_nodes_in_patch, const BOUNDARY_TYPE this_bc_tag, const double val, - const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos &node_coords, const size_t rk_level) const; - void rk_init(DViewCArrayKokkos& node_coords, - DViewCArrayKokkos& node_vel, - DViewCArrayKokkos& elem_sie, - DViewCArrayKokkos& elem_stress, - const size_t num_elems, - const size_t num_nodes); - - void get_timestep(mesh_t& mesh, - DViewCArrayKokkos& node_coords, - DViewCArrayKokkos& node_vel, - DViewCArrayKokkos& elem_sspd, - DViewCArrayKokkos& elem_vol); - - void get_timestep2D(mesh_t& mesh, - DViewCArrayKokkos& node_coords, - DViewCArrayKokkos& node_vel, - DViewCArrayKokkos& elem_sspd, - DViewCArrayKokkos& elem_vol); - - void update_state(const DCArrayKokkos& material, - const mesh_t& mesh, - const DViewCArrayKokkos& node_coords, - const DViewCArrayKokkos& node_vel, - DViewCArrayKokkos& elem_den, - DViewCArrayKokkos& elem_pres, - DViewCArrayKokkos& elem_stress, - DViewCArrayKokkos& elem_sspd, - const DViewCArrayKokkos& elem_sie, - const DViewCArrayKokkos& elem_vol, - const DViewCArrayKokkos& elem_mass, - const DViewCArrayKokkos& elem_mat_id, + void rk_init(DViewCArrayKokkos &node_coords, + DViewCArrayKokkos &node_vel, + DViewCArrayKokkos &elem_sie, + DViewCArrayKokkos &elem_stress, + const size_t num_elems, + const size_t num_nodes); + + + void get_timestep(mesh_t &mesh, + DViewCArrayKokkos &node_coords, + DViewCArrayKokkos &node_vel, + DViewCArrayKokkos &elem_sspd, + DViewCArrayKokkos &elem_vol); + + + void get_timestep2D(mesh_t &mesh, + DViewCArrayKokkos &node_coords, + DViewCArrayKokkos &node_vel, + DViewCArrayKokkos &elem_sspd, + DViewCArrayKokkos &elem_vol); + + void update_state(const DCArrayKokkos &material, + const mesh_t &mesh, + const DViewCArrayKokkos &node_coords, + const DViewCArrayKokkos &node_vel, + DViewCArrayKokkos &elem_den, + DViewCArrayKokkos &elem_pres, + DViewCArrayKokkos &elem_stress, + DViewCArrayKokkos &elem_sspd, + const DViewCArrayKokkos &elem_sie, + const DViewCArrayKokkos &elem_vol, + const DViewCArrayKokkos &elem_mass, + const DViewCArrayKokkos &elem_mat_id, + const double rk_alpha, + const size_t cycle); + + + void update_state2D(const DCArrayKokkos &material, + const mesh_t &mesh, + const DViewCArrayKokkos &node_coords, + const DViewCArrayKokkos &node_vel, + DViewCArrayKokkos &elem_den, + DViewCArrayKokkos &elem_pres, + DViewCArrayKokkos &elem_stress, + DViewCArrayKokkos &elem_sspd, + const DViewCArrayKokkos &elem_sie, + const DViewCArrayKokkos &elem_vol, + const DViewCArrayKokkos &elem_mass, + const DViewCArrayKokkos &elem_mat_id, const double rk_alpha, const size_t cycle); - void update_state2D(const DCArrayKokkos& material, - const mesh_t& mesh, - const DViewCArrayKokkos& node_coords, - const DViewCArrayKokkos& node_vel, - DViewCArrayKokkos& elem_den, - DViewCArrayKokkos& elem_pres, - DViewCArrayKokkos& elem_stress, - DViewCArrayKokkos& elem_sspd, - const DViewCArrayKokkos& elem_sie, - const DViewCArrayKokkos& elem_vol, - const DViewCArrayKokkos& elem_mass, - const DViewCArrayKokkos& elem_mat_id, - const double rk_alpha, - const size_t cycle); - - void build_boundry_node_sets(mesh_t& mesh); - - void init_boundaries(); - - // initializes memory for arrays used in the global stiffness matrix assembly - void init_boundary_sets(int num_boundary_sets); - - void grow_boundary_sets(int num_boundary_sets); - - virtual void update_forward_solve(Teuchos::RCP zp); - - void comm_node_masses(); - - void comm_adjoint_vectors(int cycle); - - void comm_variables(Teuchos::RCP zp); - - void read_conditions_ansys_dat(std::ifstream* in, std::streampos before_condition_header); - - // interfaces between user input and creating data structures for bcs - void generate_bcs(); - - void Displacement_Boundary_Conditions(); - - void init_output(); - - void compute_output(); - - void sort_output(Teuchos::RCP> sorted_map); - - void sort_element_output(Teuchos::RCP> sorted_map); - - void collect_output(Teuchos::RCP> global_reduce_map); - - void write_data(std::map& point_data_scalars_double, - std::map& point_data_vectors_double, - std::map& cell_data_scalars_double, - std::map& cell_data_scalars_int, - std::map>& cell_data_fields_double); - - void write_outputs(const mesh_t& mesh, - DViewCArrayKokkos& node_coords, - DViewCArrayKokkos& node_vel, - DViewCArrayKokkos& node_mass, - DViewCArrayKokkos& elem_den, - DViewCArrayKokkos& elem_pres, - DViewCArrayKokkos& elem_stress, - DViewCArrayKokkos& elem_sspd, - DViewCArrayKokkos& elem_sie, - DViewCArrayKokkos& elem_vol, - DViewCArrayKokkos& elem_mass, - DViewCArrayKokkos& elem_mat_id); - - void ensight(const mesh_t& mesh, - const DViewCArrayKokkos& node_coords, - const DViewCArrayKokkos& node_vel, - const DViewCArrayKokkos& node_mass, - const DViewCArrayKokkos& elem_den, - const DViewCArrayKokkos& elem_pres, - const DViewCArrayKokkos& elem_stress, - const DViewCArrayKokkos& elem_sspd, - const DViewCArrayKokkos& elem_sie, - const DViewCArrayKokkos& elem_vol, - const DViewCArrayKokkos& elem_mass, - const DViewCArrayKokkos& elem_mat_id); - - void state_file(const mesh_t& mesh, - const DViewCArrayKokkos& node_coords, - const DViewCArrayKokkos& node_vel, - const DViewCArrayKokkos& node_mass, - const DViewCArrayKokkos& elem_den, - const DViewCArrayKokkos& elem_pres, - const DViewCArrayKokkos& elem_stress, - const DViewCArrayKokkos& elem_sspd, - const DViewCArrayKokkos& elem_sie, - const DViewCArrayKokkos& elem_vol, - const DViewCArrayKokkos& elem_mass, - const DViewCArrayKokkos& elem_mat_id); - - void node_density_constraints(host_vec_array& node_densities_lower_bound); - - void compute_topology_optimization_adjoint(); // Force does not depend on node coords and velocity - - void compute_topology_optimization_adjoint_full(); // Force depends on node coords and velocity - - void compute_topology_optimization_gradient(const_vec_array design_densities, vec_array gradients); - - void compute_topology_optimization_gradient_full(Teuchos::RCP design_densities_distributed, Teuchos::RCP design_gradients_distributed); - - // elastic TO stuff - void get_force_elastic(const DCArrayKokkos& material, - const mesh_t& mesh, - const DViewCArrayKokkos& node_coords, - const DViewCArrayKokkos& node_vel, - const DViewCArrayKokkos& node_mass, - const DViewCArrayKokkos& elem_den, - const DViewCArrayKokkos& elem_vol, - const DViewCArrayKokkos& elem_div, - const DViewCArrayKokkos& elem_mat_id, - DViewCArrayKokkos& corner_force, - const double rk_alpha, - const size_t cycle); - - void applied_forces(const DCArrayKokkos& material, - const mesh_t& mesh, - const DViewCArrayKokkos& node_coords, - const DViewCArrayKokkos& node_vel, - const DViewCArrayKokkos& node_mass, - const DViewCArrayKokkos& elem_den, - const DViewCArrayKokkos& elem_vol, - const DViewCArrayKokkos& elem_div, - const DViewCArrayKokkos& elem_mat_id, - DViewCArrayKokkos& corner_force, - const double rk_alpha, - const size_t cycle); - - void Element_Material_Properties(size_t ielem, real_t& Element_Modulus, real_t& Poisson_Ratio, real_t density); - - void compute_stiffness_gradients(const_host_vec_array& design_densities, host_vec_array& gradients); - - void Gradient_Element_Material_Properties(size_t ielem, real_t& Element_Modulus, real_t& Poisson_Ratio, real_t density); - - void local_matrix_multiply(int ielem, CArrayKokkos& Local_Matrix); - - void assemble_matrix(); - - bool nodal_density_flag; - real_t penalty_power; - Teuchos::RCP Global_Stiffness_Matrix; - RaggedRightArrayKokkos Stiffness_Matrix; - DCArrayKokkos Stiffness_Matrix_Strides; - DCArrayKokkos Global_Stiffness_Matrix_Assembly_Map; - // end elastic TO data - - Dynamic_Elasticity_Parameters* module_params; - Simulation_Parameters_Explicit* simparam; - Explicit_Solver* Explicit_Solver_Pointer_; - - elements::ref_element* ref_elem; - - std::shared_ptr mesh; - // shallow copies of mesh class views - size_t num_nodes_in_elem; - // corner ids in node - RaggedRightArrayKokkos corners_in_node; - CArrayKokkos num_corners_in_node; - - // for storing global variables used in user material model - DCArrayKokkos eos_global_vars; - DCArrayKokkos strength_global_vars; - - // for storing state variables used in user material model - DCArrayKokkos eos_state_vars; - DCArrayKokkos strength_state_vars; - - // node ids in elem - DCArrayKokkos nodes_in_elem; - - // corner ids in elem - CArrayKokkos corners_in_elem; - - // elem ids in elem - RaggedRightArrayKokkos elems_in_elem; - CArrayKokkos num_elems_in_elem; - - // patch ids in elem - CArrayKokkos patches_in_elem; - - // node ids in a patch - CArrayKokkos nodes_in_patch; - - // element ids in a patch - CArrayKokkos elems_in_patch; - - // bdy nodes - CArrayKokkos bdy_nodes; - - // Topology optimization filter variable - DCArrayKokkos relative_element_densities; - - // Local FEA data - host_elem_conn_array interface_nodes_in_elem; // host view of element connectivity to nodes - - // Global FEA data - Teuchos::RCP node_velocities_distributed; - Teuchos::RCP initial_node_coords_distributed; - Teuchos::RCP all_initial_node_coords_distributed; - Teuchos::RCP initial_node_velocities_distributed; - Teuchos::RCP all_node_velocities_distributed; - Teuchos::RCP all_cached_node_velocities_distributed; - Teuchos::RCP node_masses_distributed; - Teuchos::RCP ghost_node_masses_distributed; - Teuchos::RCP adjoint_vector_distributed; - Teuchos::RCP phi_adjoint_vector_distributed; - Teuchos::RCP>> forward_solve_velocity_data; - Teuchos::RCP>> forward_solve_coordinate_data; - Teuchos::RCP>> adjoint_vector_data; - Teuchos::RCP>> phi_adjoint_vector_data; - Teuchos::RCP force_gradient_design; - Teuchos::RCP force_gradient_position; - Teuchos::RCP force_gradient_velocity; - - // Local FEA data - DCArrayKokkos Global_Gradient_Matrix_Assembly_Map; - RaggedRightArrayKokkos Graph_Matrix; // stores global indices - RaggedRightArrayKokkos DOF_Graph_Matrix; // stores global indices - RaggedRightArrayKokkos Force_Gradient_Positions; - RaggedRightArrayKokkos Force_Gradient_Velocities; - DCArrayKokkos Gradient_Matrix_Strides; - DCArrayKokkos Graph_Matrix_Strides; - RaggedRightArrayKokkos Original_Gradient_Entries; - RaggedRightArrayKokkos Original_Gradient_Entry_Indices; - DCArrayKokkos Original_Gradient_Entries_Strides; - - // distributed matrices - Teuchos::RCP distributed_force_gradient_positions; - Teuchos::RCP distributed_force_gradient_velocities; - - std::vector time_data; - unsigned long max_time_steps, last_time_step; - - // --------------------------------------------------------------------- - // state data type declarations (must stay in scope for output after run) - // --------------------------------------------------------------------- - node_t node_interface; - elem_t elem_interface; - corner_t corner_interface; - - // Dual View wrappers - // Dual Views of the individual node struct variables - DViewCArrayKokkos node_coords; - DViewCArrayKokkos node_vel; - DViewCArrayKokkos node_mass; - - // Dual Views of the individual elem struct variables - DViewCArrayKokkos elem_den; - DViewCArrayKokkos elem_pres; - DViewCArrayKokkos elem_stress; // always 3D even in 2D-RZ - DViewCArrayKokkos elem_sspd; - DViewCArrayKokkos elem_sie; - DViewCArrayKokkos elem_vol; - DViewCArrayKokkos elem_div; - DViewCArrayKokkos elem_mass; - DViewCArrayKokkos elem_mat_id; - - // Element velocity gradient - DCArrayKokkos elem_vel_grad; - - // for storing global variables used in user material model - DCArrayKokkos global_vars; - - // for storing state variables used in user material model - DCArrayKokkos state_vars; - - // elem_user_output_vars allow users to output variables of interest per element - DCArrayKokkos elem_user_output_vars; - - // material models - DCArrayKokkos elem_eos; - DCArrayKokkos elem_strength; - - // Dual Views of the corner struct variables - DViewCArrayKokkos corner_force; - DViewCArrayKokkos corner_mass; - - // Boundary Conditions Data - DCArrayKokkos Local_Index_Boundary_Patches; - // CArray Patch_Nodes; - enum bc_type { NONE, POINT_LOADING_CONDITION, LINE_LOADING_CONDITION, SURFACE_LOADING_CONDITION }; - - // Boundary Conditions Data - int max_boundary_sets; - - // output dof data - // Global arrays with collected data used to print - int output_velocity_index, output_strain_index, output_stress_index; - - // parameters - double time_value, time_final, dt, dt_max, dt_min, dt_cfl, graphics_time, graphics_dt_ival; - size_t graphics_cyc_ival, cycle_stop, rk_num_stages, graphics_id; - double fuzz, tiny, small; - CArray graphics_times; - int rk_num_bins; - - // optimization flags - bool kinetic_energy_objective; + void build_boundry_node_sets(mesh_t &mesh); + + void init_boundaries(); + + //initializes memory for arrays used in the global stiffness matrix assembly + void init_boundary_sets(int num_boundary_sets); + + void grow_boundary_sets(int num_boundary_sets); + + virtual void update_forward_solve(Teuchos::RCP zp); + + void comm_node_masses(); + + void comm_adjoint_vectors(int cycle); + + void comm_variables(Teuchos::RCP zp); + + void read_conditions_ansys_dat(std::ifstream *in, std::streampos before_condition_header); + + //interfaces between user input and creating data structures for bcs + void generate_bcs(); + + void Displacement_Boundary_Conditions(); + + void init_output(); + + void compute_output(); + + void sort_output(Teuchos::RCP > sorted_map); + + void sort_element_output(Teuchos::RCP > sorted_map); + + void collect_output(Teuchos::RCP > global_reduce_map); + + void write_data(std::map &point_data_scalars_double, + std::map &point_data_vectors_double, + std::map &cell_data_scalars_double, + std::map &cell_data_scalars_int, + std::map > &cell_data_fields_double); + + void write_outputs (const mesh_t &mesh, + DViewCArrayKokkos &node_coords, + DViewCArrayKokkos &node_vel, + DViewCArrayKokkos &node_mass, + DViewCArrayKokkos &elem_den, + DViewCArrayKokkos &elem_pres, + DViewCArrayKokkos &elem_stress, + DViewCArrayKokkos &elem_sspd, + DViewCArrayKokkos &elem_sie, + DViewCArrayKokkos &elem_vol, + DViewCArrayKokkos &elem_mass, + DViewCArrayKokkos &elem_mat_id); + + + void ensight(const mesh_t &mesh, + const DViewCArrayKokkos &node_coords, + const DViewCArrayKokkos &node_vel, + const DViewCArrayKokkos &node_mass, + const DViewCArrayKokkos &elem_den, + const DViewCArrayKokkos &elem_pres, + const DViewCArrayKokkos &elem_stress, + const DViewCArrayKokkos &elem_sspd, + const DViewCArrayKokkos &elem_sie, + const DViewCArrayKokkos &elem_vol, + const DViewCArrayKokkos &elem_mass, + const DViewCArrayKokkos &elem_mat_id); + + + void state_file(const mesh_t &mesh, + const DViewCArrayKokkos &node_coords, + const DViewCArrayKokkos &node_vel, + const DViewCArrayKokkos &node_mass, + const DViewCArrayKokkos &elem_den, + const DViewCArrayKokkos &elem_pres, + const DViewCArrayKokkos &elem_stress, + const DViewCArrayKokkos &elem_sspd, + const DViewCArrayKokkos &elem_sie, + const DViewCArrayKokkos &elem_vol, + const DViewCArrayKokkos &elem_mass, + const DViewCArrayKokkos &elem_mat_id); + + void node_density_constraints(host_vec_array &node_densities_lower_bound); + + void compute_topology_optimization_adjoint(); //Force does not depend on node coords and velocity + + void compute_topology_optimization_adjoint_full(); //Force depends on node coords and velocity + + void compute_topology_optimization_gradient(const_vec_array design_densities, vec_array gradients); + + void compute_topology_optimization_gradient_full(Teuchos::RCP design_densities_distributed, Teuchos::RCP design_gradients_distributed); + + //elastic TO stuff + void get_force_elastic(const DCArrayKokkos &material, + const mesh_t &mesh, + const DViewCArrayKokkos &node_coords, + const DViewCArrayKokkos &node_vel, + const DViewCArrayKokkos &node_mass, + const DViewCArrayKokkos &elem_den, + const DViewCArrayKokkos &elem_vol, + const DViewCArrayKokkos &elem_div, + const DViewCArrayKokkos &elem_mat_id, + DViewCArrayKokkos &corner_force, + const double rk_alpha, + const size_t cycle); + + void applied_forces(const DCArrayKokkos &material, + const mesh_t &mesh, + const DViewCArrayKokkos &node_coords, + const DViewCArrayKokkos &node_vel, + const DViewCArrayKokkos &node_mass, + const DViewCArrayKokkos &elem_den, + const DViewCArrayKokkos &elem_vol, + const DViewCArrayKokkos &elem_div, + const DViewCArrayKokkos &elem_mat_id, + DViewCArrayKokkos &corner_force, + const double rk_alpha, + const size_t cycle); + + void Element_Material_Properties(size_t ielem, real_t &Element_Modulus, real_t &Poisson_Ratio, real_t density); + + void compute_stiffness_gradients(const_host_vec_array &design_densities, host_vec_array &gradients); + + void Gradient_Element_Material_Properties(size_t ielem, real_t &Element_Modulus, real_t &Poisson_Ratio, real_t density); + + void local_matrix_multiply(int ielem, CArrayKokkos &Local_Matrix); + + void assemble_matrix(); + + bool nodal_density_flag; + real_t penalty_power; + Teuchos::RCP Global_Stiffness_Matrix; + RaggedRightArrayKokkos Stiffness_Matrix; + DCArrayKokkos Stiffness_Matrix_Strides; + DCArrayKokkos Global_Stiffness_Matrix_Assembly_Map; + //end elastic TO data + + Dynamic_Elasticity_Parameters *module_params; + Simulation_Parameters_Explicit *simparam; + Explicit_Solver *Explicit_Solver_Pointer_; + + elements::ref_element *ref_elem; + + std::shared_ptr mesh; + //shallow copies of mesh class views + size_t num_nodes_in_elem; + // corner ids in node + RaggedRightArrayKokkos corners_in_node; + CArrayKokkos num_corners_in_node; + + // elem ids in node + RaggedRightArrayKokkos elems_in_node; + + // node ids in node + RaggedRightArrayKokkos nodes_in_node; + CArrayKokkos num_nodes_in_node; + + // node ids in elem + DCArrayKokkos nodes_in_elem; + + // corner ids in elem + CArrayKokkos corners_in_elem; + + // elem ids in elem + RaggedRightArrayKokkos elems_in_elem; + CArrayKokkos num_elems_in_elem; + + // patch ids in elem + CArrayKokkos patches_in_elem; + + // node ids in a patch + CArrayKokkos nodes_in_patch; + + // element ids in a patch + CArrayKokkos elems_in_patch; + + // bdy nodes + CArrayKokkos bdy_nodes; + + //Topology optimization filter variable + DCArrayKokkos relative_element_densities; + + //Local FEA data + host_elem_conn_array interface_nodes_in_elem; //host view of element connectivity to nodes + + //Global FEA data + Teuchos::RCP node_velocities_distributed; + Teuchos::RCP initial_node_coords_distributed; + Teuchos::RCP all_initial_node_coords_distributed; + Teuchos::RCP initial_node_velocities_distributed; + Teuchos::RCP all_node_velocities_distributed; + Teuchos::RCP all_cached_node_velocities_distributed; + Teuchos::RCP node_masses_distributed; + Teuchos::RCP ghost_node_masses_distributed; + Teuchos::RCP adjoint_vector_distributed; + Teuchos::RCP phi_adjoint_vector_distributed; + Teuchos::RCP>> forward_solve_velocity_data; + Teuchos::RCP>> forward_solve_coordinate_data; + Teuchos::RCP>> adjoint_vector_data; + Teuchos::RCP>> phi_adjoint_vector_data; + Teuchos::RCP force_gradient_design; + Teuchos::RCP force_gradient_position; + Teuchos::RCP force_gradient_velocity; + + //Local FEA data + DCArrayKokkos Global_Gradient_Matrix_Assembly_Map; + RaggedRightArrayKokkos Graph_Matrix; //stores global indices + RaggedRightArrayKokkos DOF_Graph_Matrix; //stores global indices + RaggedRightArrayKokkos Force_Gradient_Positions; + RaggedRightArrayKokkos Force_Gradient_Velocities; + DCArrayKokkos Gradient_Matrix_Strides; + DCArrayKokkos Graph_Matrix_Strides; + RaggedRightArrayKokkos Original_Gradient_Entries; + RaggedRightArrayKokkos Original_Gradient_Entry_Indices; + DCArrayKokkos Original_Gradient_Entries_Strides; + + //distributed matrices + Teuchos::RCP distributed_force_gradient_positions; + Teuchos::RCP distributed_force_gradient_velocities; + + std::vector time_data; + unsigned long max_time_steps, last_time_step; + + // --------------------------------------------------------------------- + // state data type declarations (must stay in scope for output after run) + // --------------------------------------------------------------------- + node_t node_interface; + elem_t elem_interface; + corner_t corner_interface; + + //Dual View wrappers + // Dual Views of the individual node struct variables + DViewCArrayKokkos node_coords; + DViewCArrayKokkos node_vel; + DViewCArrayKokkos node_mass; + + // Dual Views of the individual elem struct variables + DViewCArrayKokkos elem_den; + DViewCArrayKokkos elem_pres; + DViewCArrayKokkos elem_stress; // always 3D even in 2D-RZ + DViewCArrayKokkos elem_sspd; + DViewCArrayKokkos elem_sie; + DViewCArrayKokkos elem_vol; + DViewCArrayKokkos elem_div; + DViewCArrayKokkos elem_mass; + DViewCArrayKokkos elem_mat_id; + + // Element velocity gradient + DCArrayKokkos elem_vel_grad; + + // for storing global variables used in user material model + DCArrayKokkos eos_global_vars; + DCArrayKokkos strength_global_vars; + + // for storing state variables used in user material model + DCArrayKokkos eos_state_vars; + DCArrayKokkos strength_state_vars; + + //elem_user_output_vars allow users to output variables of interest per element + DCArrayKokkos elem_user_output_vars; + + //material models + DCArrayKokkos elem_eos; + DCArrayKokkos elem_strength; + + // Dual Views of the corner struct variables + DViewCArrayKokkos corner_force; + DViewCArrayKokkos corner_mass; + + //Boundary Conditions Data + DCArrayKokkos Local_Index_Boundary_Patches; + //CArray Patch_Nodes; + enum bc_type {NONE, POINT_LOADING_CONDITION, LINE_LOADING_CONDITION, SURFACE_LOADING_CONDITION}; + + //Boundary Conditions Data + int max_boundary_sets; + + //output dof data + //Global arrays with collected data used to print + int output_velocity_index, output_strain_index, output_stress_index; + + //parameters + double time_value, time_final, dt, dt_max, dt_min, dt_cfl, graphics_time, graphics_dt_ival; + size_t graphics_cyc_ival, cycle_stop, rk_num_stages, graphics_id; + double fuzz, tiny, small; + CArray graphics_times; + int rk_num_bins; + + //optimization flags + bool kinetic_energy_objective; }; #endif // end HEADER_H diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/elastic_optimization.cpp b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/elastic_optimization.cpp index 490e6cb6d..6addef615 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/elastic_optimization.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/elastic_optimization.cpp @@ -1,36 +1,3 @@ -/********************************************************************************************** - 2020. Triad National Security, LLC. All rights reserved. - This program was produced under U.S. Government contract 89233218CNA000001 for Los Alamos - National Laboratory (LANL), which is operated by Triad National Security, LLC for the U.S. - Department of Energy/National Nuclear Security Administration. All rights in the program are - reserved by Triad National Security, LLC, and the U.S. Department of Energy/National Nuclear - Security Administration. The Government is granted for itself and others acting on its behalf a - nonexclusive, paid-up, irrevocable worldwide license in this material to reproduce, prepare - derivative works, distribute copies to the public, perform publicly and display publicly, and - to permit others to do so. - This program is open source under the BSD-3 License. - Redistribution and use in source and binary forms, with or without modification, are permitted - provided that the following conditions are met: - 1. Redistributions of source code must retain the above copyright notice, this list of - conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright notice, this list of - conditions and the following disclaimer in the documentation and/or other materials - provided with the distribution. - 3. Neither the name of the copyright holder nor the names of its contributors may be used - to endorse or promote products derived from this software without specific prior - written permission. - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS - IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR - CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, - WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR - OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - **********************************************************************************************/ #include "mesh.h" #include "state.h" @@ -38,7 +5,7 @@ #include #include #include -#include +#include #include // fmin, fmax, abs note: fminl is long #include #include @@ -67,7 +34,7 @@ #include "FEA_Module_Dynamic_Elasticity.h" #include "Explicit_Solver.h" -// optimization +//optimization #include "ROL_Solver.hpp" #include "Kinetic_Energy_Minimize.h" @@ -75,224 +42,210 @@ Compute new system response due to the design variable update ------------------------------------------------------------------------- */ -void FEA_Module_Dynamic_Elasticity::update_forward_solve(Teuchos::RCP zp) -{ - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - // local variable for host view in the dual view - int num_dim = simparam->num_dims; - int nodes_per_elem = max_nodes_per_element; - int local_node_index, current_row, current_column; - int max_stride = 0; - int current_module_index; - size_t access_index, row_access_index, row_counter; - GO global_index, global_dof_index; - LO local_dof_index; - const size_t num_fills = simparam->regions.size(); - const size_t rk_num_bins = simparam->dynamic_options.rk_num_bins; - const size_t num_bcs = module_params->boundary_conditions.size(); - const size_t num_materials = simparam->materials.size(); - real_t objective_accumulation; - - // --- Read in the nodes in the mesh --- - int myrank = Explicit_Solver_Pointer_->myrank; - int nranks = Explicit_Solver_Pointer_->nranks; - - const DCArrayKokkos mat_fill = simparam->mat_fill; - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos material = simparam->material; - CArray current_element_nodal_densities = CArray(num_nodes_in_elem); - - std::vector> FEA_Module_My_TO_Modules = simparam->FEA_Module_My_TO_Modules; - problem = Explicit_Solver_Pointer_->problem; // Pointer to ROL optimization problem object - ROL::Ptr> obj_pointer; - - // compute element averaged density ratios corresponding to nodal density design variables - { // view scope - const_host_vec_array all_node_densities = all_node_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); - // debug print - // std::cout << "NODE DENSITY TEST " << all_node_densities(0,0) << std::endl; - for (int elem_id = 0; elem_id < rnum_elem; elem_id++) - { - for (int inode = 0; inode < num_nodes_in_elem; inode++) - { - current_element_nodal_densities(inode) = all_node_densities(nodes_in_elem(elem_id, inode), 0); - } - relative_element_densities.host(elem_id) = average_element_density(num_nodes_in_elem, current_element_nodal_densities); - } // for - } // view scope - // debug print - // std::cout << "ELEMENT RELATIVE DENSITY TEST " << relative_element_densities.host(0) << std::endl; - relative_element_densities.update_device(); - - // set density vector to the current value chosen by the optimizer - test_node_densities_distributed = zp; - - // reset nodal coordinates to initial values - node_coords_distributed->assign(*initial_node_coords_distributed); - - // comms for ghosts - Explicit_Solver_Pointer_->comm_coordinates(); - - // view scope - { - const_vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); - const_vec_array ghost_node_coords_interface = Explicit_Solver_Pointer_->ghost_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); - vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (int idim = 0; idim < num_dim; idim++) - { - all_node_coords_interface(node_gid, idim) = node_coords_interface(node_gid, idim); - } - }); // end parallel for - Kokkos::fence(); - - FOR_ALL_CLASS(node_gid, nlocal_nodes, nlocal_nodes + nghost_nodes, { - for (int idim = 0; idim < num_dim; idim++) - { - all_node_coords_interface(node_gid, idim) = ghost_node_coords_interface(node_gid - nlocal_nodes, idim); - } +void FEA_Module_Dynamic_Elasticity::update_forward_solve(Teuchos::RCP zp){ + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + //local variable for host view in the dual view + int num_dim = simparam->num_dims; + int nodes_per_elem = max_nodes_per_element; + int local_node_index, current_row, current_column; + int max_stride = 0; + int current_module_index; + size_t access_index, row_access_index, row_counter; + GO global_index, global_dof_index; + LO local_dof_index; + const size_t num_fills = simparam->regions.size(); + const size_t rk_num_bins = simparam->dynamic_options.rk_num_bins; + const size_t num_bcs = module_params->boundary_conditions.size(); + const size_t num_materials = simparam->materials.size(); + real_t objective_accumulation; + + // --- Read in the nodes in the mesh --- + int myrank = Explicit_Solver_Pointer_->myrank; + int nranks = Explicit_Solver_Pointer_->nranks; + + const DCArrayKokkos mat_fill = simparam->mat_fill; + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos material = simparam->material; + CArray current_element_nodal_densities = CArray(num_nodes_in_elem); + + std::vector> FEA_Module_My_TO_Modules = simparam->FEA_Module_My_TO_Modules; + problem = Explicit_Solver_Pointer_->problem; //Pointer to ROL optimization problem object + ROL::Ptr> obj_pointer; + + //compute element averaged density ratios corresponding to nodal density design variables + {//view scope + const_host_vec_array all_node_densities = all_node_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); + //debug print + //std::cout << "NODE DENSITY TEST " << all_node_densities(0,0) << std::endl; + for(int elem_id = 0; elem_id < rnum_elem; elem_id++){ + for(int inode = 0; inode < num_nodes_in_elem; inode++){ + current_element_nodal_densities(inode) = all_node_densities(nodes_in_elem(elem_id,inode),0); + } + relative_element_densities.host(elem_id) = average_element_density(num_nodes_in_elem, current_element_nodal_densities); + }//for + } //view scope + //debug print + //std::cout << "ELEMENT RELATIVE DENSITY TEST " << relative_element_densities.host(0) << std::endl; + relative_element_densities.update_device(); + + //set density vector to the current value chosen by the optimizer + test_node_densities_distributed = zp; + + //reset nodal coordinates to initial values + node_coords_distributed->assign(*initial_node_coords_distributed); + + //comms for ghosts + Explicit_Solver_Pointer_->comm_coordinates(); + + //view scope + { + const_vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); + const_vec_array ghost_node_coords_interface = Explicit_Solver_Pointer_->ghost_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); + vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + for (int idim = 0; idim < num_dim; idim++){ + all_node_coords_interface(node_gid,idim) = node_coords_interface(node_gid,idim); + } }); // end parallel for - Kokkos::fence(); - } // end view scope - - // reset velocities to initial conditions - node_velocities_distributed->assign(*initial_node_velocities_distributed); - - // reset time accumulating objective and constraints - /* - for(int imodule = 0 ; imodule < FEA_Module_My_TO_Modules[my_fea_module_index_].size(); imodule++){ - current_module_index = FEA_Module_My_TO_Modules[my_fea_module_index_][imodule]; - //test if module needs reset - if(){ + Kokkos::fence(); + FOR_ALL_CLASS(node_gid, nlocal_nodes, nlocal_nodes+nghost_nodes, { + for (int idim = 0; idim < num_dim; idim++){ + all_node_coords_interface(node_gid,idim) = ghost_node_coords_interface(node_gid-nlocal_nodes,idim); } + }); // end parallel for + Kokkos::fence(); + } //end view scope + + //reset velocities to initial conditions + node_velocities_distributed->assign(*initial_node_velocities_distributed); + + //reset time accumulating objective and constraints + /* + for(int imodule = 0 ; imodule < FEA_Module_My_TO_Modules[my_fea_module_index_].size(); imodule++){ + current_module_index = FEA_Module_My_TO_Modules[my_fea_module_index_][imodule]; + //test if module needs reset + if(){ + } - */ - // simple setup to just request KE for now; above loop to be expanded and used later for scanning modules - obj_pointer = problem->getObjective(); - KineticEnergyMinimize_TopOpt& kinetic_energy_minimize_function = dynamic_cast(*obj_pointer); - kinetic_energy_minimize_function.objective_accumulation = 0; - - // interface trial density vector - - // interfacing of vectors(should be removed later once made compatible) - // view scope - { - host_vec_array interface_node_coords = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); - for (size_t ibin = 0; ibin < rk_num_bins; ibin++) - { - // save node data to node.coords - // std::cout << "NODE DATA ON RANK " << myrank << std::endl; - if (num_dim == 2) - { - for (int inode = 0; inode < nall_nodes; inode++) - { - // std::cout << "Node index " << inode+1 << " "; - node_coords.host(ibin, inode, 0) = interface_node_coords(inode, 0); - // std::cout << host_node_coords_state(0,inode,0)+1<< " "; - node_coords.host(ibin, inode, 1) = interface_node_coords(inode, 1); - // std::cout << host_node_coords_state(0,inode,1)+1<< " "; - } - } - else if (num_dim == 3) - { - for (int inode = 0; inode < nall_nodes; inode++) - { - // std::cout << "Node index " << inode+1 << " "; - node_coords.host(ibin, inode, 0) = interface_node_coords(inode, 0); - // std::cout << host_node_coords_state(0,inode,0)+1<< " "; - node_coords.host(ibin, inode, 1) = interface_node_coords(inode, 1); - // std::cout << host_node_coords_state(0,inode,1)+1<< " "; - - node_coords.host(ibin, inode, 2) = interface_node_coords(inode, 2); - // std::cout << host_node_coords_state(0,inode,2)+1<< std::endl; - } - } + } + */ + //simple setup to just request KE for now; above loop to be expanded and used later for scanning modules + obj_pointer = problem->getObjective(); + KineticEnergyMinimize_TopOpt& kinetic_energy_minimize_function = dynamic_cast(*obj_pointer); + kinetic_energy_minimize_function.objective_accumulation = 0; + + //interface trial density vector + + //interfacing of vectors(should be removed later once made compatible) + //view scope + { + host_vec_array interface_node_coords = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); + for(size_t ibin = 0; ibin < rk_num_bins; ibin++){ + //save node data to node.coords + //std::cout << "NODE DATA ON RANK " << myrank << std::endl; + if(num_dim==2){ + for(int inode = 0; inode < nall_nodes; inode++){ + //std::cout << "Node index " << inode+1 << " "; + node_coords.host(ibin,inode,0) = interface_node_coords(inode,0); + //std::cout << host_node_coords_state(0,inode,0)+1<< " "; + node_coords.host(ibin,inode,1) = interface_node_coords(inode,1); + //std::cout << host_node_coords_state(0,inode,1)+1<< " "; } - } // end view scope + } + else if(num_dim==3){ + for(int inode = 0; inode < nall_nodes; inode++){ + //std::cout << "Node index " << inode+1 << " "; + node_coords.host(ibin,inode,0) = interface_node_coords(inode,0); + //std::cout << host_node_coords_state(0,inode,0)+1<< " "; + node_coords.host(ibin,inode,1) = interface_node_coords(inode,1); + //std::cout << host_node_coords_state(0,inode,1)+1<< " "; + + node_coords.host(ibin,inode,2) = interface_node_coords(inode,2); + //std::cout << host_node_coords_state(0,inode,2)+1<< std::endl; + } + } + } + } //end view scope // save the node coords to the current RK value - for (size_t node_gid = 0; node_gid < nall_nodes; node_gid++) - { - for (int rk = 1; rk < rk_num_bins; rk++) - { - for (int dim = 0; dim < num_dim; dim++) - { - node_coords.host(rk, node_gid, dim) = node_coords.host(0, node_gid, dim); - } // end for dim - } // end for rk + for (size_t node_gid=0; node_gid < nall_nodes; node_gid++){ + + for(int rk=1; rk elem_user_output_vars, elem_sspd, elem_den(elem_gid), - elem_sie(rk_level, elem_gid)); + elem_sie(rk_level,elem_gid)); - // --- Sound speed --- - elem_eos(elem_gid).calc_sound_speed(elem_pres, + // --- Sound speed --- + elem_eos(elem_gid).calc_sound_speed(elem_pres, elem_stress, elem_gid, elem_mat_id(elem_gid), @@ -317,224 +270,229 @@ void FEA_Module_Dynamic_Elasticity::update_forward_solve(Teuchos::RCP elem_user_output_vars, elem_sspd, elem_den(elem_gid), - elem_sie(rk_level, elem_gid)); + elem_sie(rk_level,elem_gid)); + + // loop over the nodes of this element and apply velocity + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - // loop over the nodes of this element and apply velocity - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) - { - // get the mesh node index - size_t node_gid = nodes_in_elem(elem_gid, node_lid); + // get the mesh node index + size_t node_gid = nodes_in_elem(elem_gid, node_lid); - // --- Velocity --- - switch (mat_fill(f_id).velocity) + + // --- Velocity --- + switch(mat_fill(f_id).velocity) + { + case VELOCITY_TYPE::cartesian: { - case VELOCITY_TYPE::cartesian: - { - node_vel(rk_level, node_gid, 0) = mat_fill(f_id).u; - node_vel(rk_level, node_gid, 1) = mat_fill(f_id).v; - if (num_dim == 3) - { - node_vel(rk_level, node_gid, 2) = mat_fill(f_id).w; - } - - break; - } - case VELOCITY_TYPE::radial: - { - // Setting up cylindrical - double dir[2]; - dir[0] = 0.0; - dir[1] = 0.0; - double radius_val = 0.0; - - for (int dim = 0; dim < 2; dim++) - { - dir[dim] = node_coords(rk_level, node_gid, dim); - radius_val += node_coords(rk_level, node_gid, dim) * node_coords(rk_level, node_gid, dim); - } // end for - radius_val = sqrt(radius_val); - - for (int dim = 0; dim < 2; dim++) - { - if (radius_val > 1.0e-14) - { - dir[dim] /= (radius_val); - } - else - { - dir[dim] = 0.0; - } - } // end for - - node_vel(rk_level, node_gid, 0) = mat_fill(f_id).speed * dir[0]; - node_vel(rk_level, node_gid, 1) = mat_fill(f_id).speed * dir[1]; - if (num_dim == 3) - { - node_vel(rk_level, node_gid, 2) = 0.0; - } - - break; - } - case VELOCITY_TYPE::spherical: - { - // Setting up spherical - double dir[3]; - dir[0] = 0.0; - dir[1] = 0.0; - dir[2] = 0.0; - double radius_val = 0.0; - - for (int dim = 0; dim < 3; dim++) - { - dir[dim] = node_coords(rk_level, node_gid, dim); - radius_val += node_coords(rk_level, node_gid, dim) * node_coords(rk_level, node_gid, dim); - } // end for - radius_val = sqrt(radius_val); - - for (int dim = 0; dim < 3; dim++) - { - if (radius_val > 1.0e-14) - { - dir[dim] /= (radius_val); - } - else - { - dir[dim] = 0.0; - } - } // end for - - node_vel(rk_level, node_gid, 0) = mat_fill(f_id).speed * dir[0]; - node_vel(rk_level, node_gid, 1) = mat_fill(f_id).speed * dir[1]; - if (num_dim == 3) - { - node_vel(rk_level, node_gid, 2) = mat_fill(f_id).speed * dir[2]; - } - - break; + + node_vel(rk_level, node_gid, 0) = mat_fill(f_id).u; + node_vel(rk_level, node_gid, 1) = mat_fill(f_id).v; + if (num_dim == 3) node_vel(rk_level, node_gid, 2) = mat_fill(f_id).w; + + + break; + } + case VELOCITY_TYPE::radial: + { + // Setting up cylindrical + double dir[2]; + dir[0] = 0.0; + dir[1] = 0.0; + double radius_val = 0.0; + + for(int dim=0; dim<2; dim++){ + dir[dim] = node_coords(rk_level, node_gid, dim); + radius_val += node_coords(rk_level, node_gid, dim)*node_coords(rk_level, node_gid, dim); + } // end for + radius_val = sqrt(radius_val); + + for(int dim=0; dim<2; dim++){ + if (radius_val > 1.0e-14){ + dir[dim] /= (radius_val); } - case VELOCITY_TYPE::radial_linear: - { - break; + else{ + dir[dim] = 0.0; } - case VELOCITY_TYPE::spherical_linear: - { - break; + } // end for + + + node_vel(rk_level, node_gid, 0) = mat_fill(f_id).speed*dir[0]; + node_vel(rk_level, node_gid, 1) = mat_fill(f_id).speed*dir[1]; + if (num_dim == 3) node_vel(rk_level, node_gid, 2) = 0.0; + + break; + } + case VELOCITY_TYPE::spherical: + { + + // Setting up spherical + double dir[3]; + dir[0] = 0.0; + dir[1] = 0.0; + dir[2] = 0.0; + double radius_val = 0.0; + + for(int dim=0; dim<3; dim++){ + dir[dim] = node_coords(rk_level, node_gid, dim); + radius_val += node_coords(rk_level, node_gid, dim)*node_coords(rk_level, node_gid, dim); + } // end for + radius_val = sqrt(radius_val); + + for(int dim=0; dim<3; dim++){ + if (radius_val > 1.0e-14){ + dir[dim] /= (radius_val); } - case VELOCITY_TYPE::tg_vortex: - { - node_vel(rk_level, node_gid, 0) = sin(PI * node_coords(rk_level, node_gid, 0)) * cos(PI * node_coords(rk_level, node_gid, 1)); - node_vel(rk_level, node_gid, 1) = -1.0 * cos(PI * node_coords(rk_level, node_gid, 0)) * sin(PI * node_coords(rk_level, node_gid, 1)); - if (num_dim == 3) - { - node_vel(rk_level, node_gid, 2) = 0.0; - } - - break; + else{ + dir[dim] = 0.0; } - } // end of switch - } // end loop over nodes of element + } // end for + + node_vel(rk_level, node_gid, 0) = mat_fill(f_id).speed*dir[0]; + node_vel(rk_level, node_gid, 1) = mat_fill(f_id).speed*dir[1]; + if (num_dim == 3) node_vel(rk_level, node_gid, 2) = mat_fill(f_id).speed*dir[2]; - if (mat_fill(f_id).velocity == VELOCITY_TYPE::tg_vortex) - { - elem_pres(elem_gid) = 0.25 * (cos(2.0 * PI * elem_coords[0]) + cos(2.0 * PI * elem_coords[1]) ) + 1.0; - - // p = rho*ie*(gamma - 1) - size_t mat_id = f_id; - double gamma = eos_global_vars(mat_id, 0); // gamma value - elem_sie(rk_level, elem_gid) = - elem_pres(elem_gid) / (mat_fill(f_id).den * (gamma - 1.0)); - } // end if - } // end if fill - }); // end FOR_ALL_CLASS element loop - Kokkos::fence(); - } // end for loop over fills - } // end view scope + break; + } + case VELOCITY_TYPE::radial_linear: + { + + break; + } + case VELOCITY_TYPE::spherical_linear: + { + + break; + } + case VELOCITY_TYPE::tg_vortex: + { + + node_vel(rk_level, node_gid, 0) = sin(PI * node_coords(rk_level,node_gid, 0)) * cos(PI * node_coords(rk_level,node_gid, 1)); + node_vel(rk_level, node_gid, 1) = -1.0*cos(PI * node_coords(rk_level,node_gid, 0)) * sin(PI * node_coords(rk_level,node_gid, 1)); + if (num_dim == 3) node_vel(rk_level, node_gid, 2) = 0.0; + + break; + } + } // end of switch + + }// end loop over nodes of element + + + if(mat_fill(f_id).velocity == VELOCITY_TYPE::tg_vortex) + { + elem_pres(elem_gid) = 0.25*( cos(2.0*PI*elem_coords[0]) + cos(2.0*PI*elem_coords[1]) ) + 1.0; + + // p = rho*ie*(gamma - 1) + size_t mat_id = f_id; + double gamma = eos_global_vars(mat_id,0); // gamma value + elem_sie(rk_level, elem_gid) = + elem_pres(elem_gid)/(mat_fill(f_id).den*(gamma - 1.0)); + } // end if + } // end if fill + + }); // end FOR_ALL_CLASS element loop + Kokkos::fence(); + + + } // end for loop over fills + }//end view scope + + + // apply BC's to velocity FEA_Module_Dynamic_Elasticity::boundary_velocity(*mesh, boundary, node_vel); - + + // calculate the corner massess if 2D - if (num_dim == 2) - { + if(num_dim==2){ + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { + // facial area of the corners double corner_areas_array[4]; - - ViewCArrayKokkos corner_areas(&corner_areas_array[0], 4); - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 4); - + + ViewCArrayKokkos corner_areas(&corner_areas_array[0],4); + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 4); + get_area_weights2D(corner_areas, elem_gid, node_coords, elem_node_gids, rk_level); - + // loop over the corners of the element and calculate the mass - for (size_t corner_lid = 0; corner_lid < 4; corner_lid++) - { + for (size_t corner_lid=0; corner_lid<4; corner_lid++){ + size_t corner_gid = corners_in_elem(elem_gid, corner_lid); - corner_mass(corner_gid) = corner_areas(corner_lid) * elem_den(elem_gid); // node radius is added later + corner_mass(corner_gid) = corner_areas(corner_lid)*elem_den(elem_gid); // node radius is added later + } // end for over corners }); + } // end of - + + // calculate the nodal mass FOR_ALL_CLASS(node_gid, 0, nall_nodes, { + node_mass(node_gid) = 0.0; - - if (num_dim == 3) - { - for (size_t elem_lid = 0; elem_lid < num_corners_in_node(node_gid); elem_lid++) - { - size_t elem_gid = elems_in_node(node_gid, elem_lid); - node_mass(node_gid) += 1.0 / 8.0 * elem_mass(elem_gid); + + if(num_dim==3){ + + for(size_t elem_lid=0; elem_lid < num_corners_in_node(node_gid); elem_lid++){ + size_t elem_gid = elems_in_node(node_gid,elem_lid); + node_mass(node_gid) += 1.0/8.0*elem_mass(elem_gid); } // end for elem_lid - } // end if dims=3 - else - { + + }// end if dims=3 + else { + // 2D-RZ - for (size_t corner_lid = 0; corner_lid < num_corners_in_node(node_gid); corner_lid++) - { - size_t corner_gid = corners_in_node(node_gid, corner_lid); + for(size_t corner_lid=0; corner_lid < num_corners_in_node(node_gid); corner_lid++){ + + size_t corner_gid = corners_in_node(node_gid, corner_lid); node_mass(node_gid) += corner_mass(corner_gid); // sans the radius so it is areal node mass - - corner_mass(corner_gid) *= node_coords(rk_level, node_gid, 1); // true corner mass now + + corner_mass(corner_gid) *= node_coords(rk_level,node_gid,1); // true corner mass now } // end for elem_lid + } // end else + }); // end FOR_ALL_CLASS Kokkos::fence(); - // current interface has differing mass arrays; this equates them until we unify memory - // view scope + + //current interface has differing mass arrays; this equates them until we unify memory + //view scope { - vec_array node_mass_interface = node_masses_distributed->getLocalView(Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - node_mass_interface(node_gid, 0) = node_mass(node_gid); + vec_array node_mass_interface = node_masses_distributed->getLocalView (Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + node_mass_interface(node_gid,0) = node_mass(node_gid); }); // end parallel for - } // end view scope + } //end view scope Kokkos::fence(); - // communicate ghost densities + //communicate ghost densities comm_node_masses(); - // this is forcing a copy to the device - // view scope + //this is forcing a copy to the device + //view scope { - vec_array ghost_node_mass_interface = ghost_node_masses_distributed->getLocalView(Tpetra::Access::ReadWrite); + vec_array ghost_node_mass_interface = ghost_node_masses_distributed->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, nlocal_nodes, nall_nodes, { - node_mass(node_gid) = ghost_node_mass_interface(node_gid - nlocal_nodes, 0); + FOR_ALL_CLASS(node_gid, nlocal_nodes, nall_nodes, { + node_mass(node_gid) = ghost_node_mass_interface(node_gid-nlocal_nodes,0); }); // end parallel for - } // end view scope + } //end view scope Kokkos::fence(); - // update stiffness matrix - if (simparam->topology_optimization_on || simparam->shape_optimization_on) - { - assemble_matrix(); + //update stiffness matrix + if(simparam->topology_optimization_on || simparam->shape_optimization_on){ + assemble_matrix(); } - - // execute solve + + //execute solve elastic_solve(); + } /* ------------------------------------------------------------------------------------------- @@ -543,163 +501,141 @@ void FEA_Module_Dynamic_Elasticity::update_forward_solve(Teuchos::RCP double FEA_Module_Dynamic_Elasticity::average_element_density(const int nodes_per_elem, const CArray current_element_densities) const { - double result = 0; - for (int i = 0; i < nodes_per_elem; i++) - { - result += current_element_densities(i) / nodes_per_elem; - } + double result = 0; + for(int i=0; i < nodes_per_elem; i++){ + result += current_element_densities(i)/nodes_per_elem; + } - return result; + return result; } /* --------------------------------------------------------------------------------------------------------------- - Simpler adjoint vector solve for the kinetic energy minimization problem + Simpler adjoint vector solve for the kinetic energy minimization problem when force does not depend on u and v. ------------------------------------------------------------------------------------------------------------------ */ -void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_adjoint() -{ - size_t num_bdy_nodes = mesh->num_bdy_nodes; - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos material = simparam->material; - const int num_dim = simparam->num_dims; - real_t global_dt; - size_t current_data_index, next_data_index; - Teuchos::RCP previous_adjoint_vector_distributed, current_adjoint_vector_distributed, previous_velocity_vector_distributed, current_velocity_vector_distributed; - // initialize first adjoint vector at last_time_step to 0 as the terminal value - (*adjoint_vector_data)[last_time_step + 1]->putScalar(0); - - // solve terminal value problem, proceeds in time backward. For simplicity, we use the same timestep data from the forward solve. - // A linear interpolant is assumed between velocity data points; velocity midpoint is used to update the adjoint. - if (myrank == 0) - { - std::cout << "Computing adjoint vector " << time_data.size() << std::endl; +void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_adjoint(){ + + size_t num_bdy_nodes = mesh->num_bdy_nodes; + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos material = simparam->material; + const int num_dim = simparam->num_dims; + real_t global_dt; + size_t current_data_index, next_data_index; + Teuchos::RCP previous_adjoint_vector_distributed, current_adjoint_vector_distributed, previous_velocity_vector_distributed, current_velocity_vector_distributed; + //initialize first adjoint vector at last_time_step to 0 as the terminal value + (*adjoint_vector_data)[last_time_step+1]->putScalar(0); + + //solve terminal value problem, proceeds in time backward. For simplicity, we use the same timestep data from the forward solve. + //A linear interpolant is assumed between velocity data points; velocity midpoint is used to update the adjoint. + if(myrank==0) + std::cout << "Computing adjoint vector " << time_data.size() << std::endl; + + for (int cycle = last_time_step; cycle >= 0; cycle--) { + //compute timestep from time data + global_dt = time_data[cycle+1] - time_data[cycle]; + + //print + if(simparam->dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ + if (cycle==last_time_step){ + if(myrank==0) + printf("cycle = %d, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + // print time step every 10 cycles + else if (cycle%20==0){ + if(myrank==0) + printf("cycle = %d, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } // end if } + //else if (cycle==1){ + //if(myrank==0) + //printf("cycle = %lu, time = %f, time step = %f \n", cycle-1, time_data[cycle-1], global_dt); + //} // end if - for (int cycle = last_time_step; cycle >= 0; cycle--) + //compute adjoint vector for this data point; use velocity midpoint + //view scope { - // compute timestep from time data - global_dt = time_data[cycle + 1] - time_data[cycle]; - - // print - if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) - { - if (cycle == last_time_step) - { - if (myrank == 0) - { - printf("cycle = %d, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } - } - // print time step every 10 cycles - else if (cycle % 20 == 0) - { - if (myrank == 0) - { - printf("cycle = %d, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } - } // end if - } - // else if (cycle==1){ - // if(myrank==0) - // printf("cycle = %lu, time = %f, time step = %f \n", cycle-1, time_data[cycle-1], global_dt); - // } // end if - - // compute adjoint vector for this data point; use velocity midpoint - // view scope - { - const_vec_array previous_velocity_vector = (*forward_solve_velocity_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); - const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); - - const_vec_array previous_adjoint_vector = (*adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); - vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadWrite); - - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes + nghost_nodes, { - for (int idim = 0; idim < num_dim; idim++) - { - // cancellation of half from midpoint and 2 from adjoint equation already done - current_adjoint_vector(node_gid, idim) = -0.5 * (current_velocity_vector(node_gid, idim) + previous_velocity_vector(node_gid, idim)) * global_dt + previous_adjoint_vector(node_gid, - idim); - } + const_vec_array previous_velocity_vector = (*forward_solve_velocity_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); + const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); + + const_vec_array previous_adjoint_vector = (*adjoint_vector_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); + vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView (Tpetra::Access::ReadWrite); + + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes + nghost_nodes, { + for (int idim = 0; idim < num_dim; idim++){ + //cancellation of half from midpoint and 2 from adjoint equation already done + current_adjoint_vector(node_gid,idim) = -0.5*(current_velocity_vector(node_gid,idim)+previous_velocity_vector(node_gid,idim))*global_dt + previous_adjoint_vector(node_gid,idim); + } }); // end parallel for - Kokkos::fence(); - } // end view scope - } + Kokkos::fence(); + } //end view scope + + } } + /* ------------------------------------------------------------------------------ Coupled adjoint problem for the kinetic energy minimization problem --------------------------------------------------------------------------------- */ -void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_adjoint_full() -{ - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - size_t num_bdy_nodes = mesh->num_bdy_nodes; - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos material = simparam->material; - const int num_dim = simparam->num_dims; - const real_t damping_constant = module_params->damping_constant; - real_t global_dt; - size_t current_data_index, next_data_index; - Teuchos::RCP previous_adjoint_vector_distributed, current_adjoint_vector_distributed, previous_velocity_vector_distributed, current_velocity_vector_distributed; - Teuchos::RCP previous_phi_adjoint_vector_distributed, current_phi_adjoint_vector_distributed; - // initialize first adjoint vector at last_time_step to 0 as the terminal value - (*adjoint_vector_data)[last_time_step + 1]->putScalar(0); - (*phi_adjoint_vector_data)[last_time_step + 1]->putScalar(0); - - // solve terminal value problem, proceeds in time backward. For simplicity, we use the same timestep data from the forward solve. - // A linear interpolant is assumed between velocity data points; velocity midpoint is used to update the adjoint. - if (myrank == 0) - { - std::cout << "Computing adjoint vector " << time_data.size() << std::endl; +void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_adjoint_full(){ + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + size_t num_bdy_nodes = mesh->num_bdy_nodes; + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos material = simparam->material; + const int num_dim = simparam->num_dims; + const real_t damping_constant = module_params->damping_constant; + real_t global_dt; + size_t current_data_index, next_data_index; + Teuchos::RCP previous_adjoint_vector_distributed, current_adjoint_vector_distributed, previous_velocity_vector_distributed, current_velocity_vector_distributed; + Teuchos::RCP previous_phi_adjoint_vector_distributed, current_phi_adjoint_vector_distributed; + //initialize first adjoint vector at last_time_step to 0 as the terminal value + (*adjoint_vector_data)[last_time_step+1]->putScalar(0); + (*phi_adjoint_vector_data)[last_time_step+1]->putScalar(0); + + //solve terminal value problem, proceeds in time backward. For simplicity, we use the same timestep data from the forward solve. + //A linear interpolant is assumed between velocity data points; velocity midpoint is used to update the adjoint. + if(myrank==0) + std::cout << "Computing adjoint vector " << time_data.size() << std::endl; + + for (int cycle = last_time_step; cycle >= 0; cycle--) { + //compute timestep from time data + global_dt = time_data[cycle+1] - time_data[cycle]; + + //print + if(simparam->dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ + if (cycle==last_time_step){ + if(myrank==0) + printf("cycle = %d, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + // print time step every 10 cycles + else if (cycle%20==0){ + if(myrank==0) + printf("cycle = %d, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } // end if } - - for (int cycle = last_time_step; cycle >= 0; cycle--) + + //view scope { - // compute timestep from time data - global_dt = time_data[cycle + 1] - time_data[cycle]; - - // print - if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) - { - if (cycle == last_time_step) - { - if (myrank == 0) - { - printf("cycle = %d, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } - } - // print time step every 10 cycles - else if (cycle % 20 == 0) - { - if (myrank == 0) - { - printf("cycle = %d, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } - } // end if + + //set velocity and position for this timestep + const_vec_array previous_velocity_vector = (*forward_solve_velocity_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); + const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); + + const_vec_array previous_coordinate_vector = (*forward_solve_coordinate_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); + const_vec_array current_coordinate_vector = (*forward_solve_coordinate_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); + + //interface of arrays for current implementation of force calculation + + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes+nghost_nodes, { + for (int idim = 0; idim < num_dim; idim++){ + node_vel(rk_level,node_gid,idim) = previous_velocity_vector(node_gid,idim); + node_coords(rk_level,node_gid,idim) = previous_coordinate_vector(node_gid,idim); } - - // view scope - { - // set velocity and position for this timestep - const_vec_array previous_velocity_vector = (*forward_solve_velocity_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); - const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); - - const_vec_array previous_coordinate_vector = (*forward_solve_coordinate_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); - const_vec_array current_coordinate_vector = (*forward_solve_coordinate_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); - - // interface of arrays for current implementation of force calculation - - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes + nghost_nodes, { - for (int idim = 0; idim < num_dim; idim++) - { - node_vel(rk_level, node_gid, idim) = previous_velocity_vector(node_gid, idim); - node_coords(rk_level, node_gid, idim) = previous_coordinate_vector(node_gid, idim); - } }); - Kokkos::fence(); + Kokkos::fence(); - get_force_vgradient_elastic(material, + get_force_vgradient_elastic(material, *mesh, node_coords, node_vel, @@ -714,1050 +650,965 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_adjoint_full() 1, cycle); - // force_gradient_velocity->describe(*fos,Teuchos::VERB_EXTREME); - const_vec_array previous_force_gradient_position = force_gradient_position->getLocalView(Tpetra::Access::ReadOnly); - // const_vec_array current_force_gradient_position = force_gradient_position->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array previous_force_gradient_velocity = force_gradient_velocity->getLocalView(Tpetra::Access::ReadOnly); - // const_vec_array current_force_gradient_velocity = force_gradient_velocity->getLocalView (Tpetra::Access::ReadOnly); - // compute gradient of force with respect to velocity - - const_vec_array previous_adjoint_vector = (*adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); - const_vec_array phi_previous_adjoint_vector = (*phi_adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); - vec_array midpoint_adjoint_vector = adjoint_vector_distributed->getLocalView(Tpetra::Access::ReadWrite); - vec_array phi_midpoint_adjoint_vector = phi_adjoint_vector_distributed->getLocalView(Tpetra::Access::ReadWrite); - - // half step update for RK2 scheme - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - real_t rate_of_change; - real_t matrix_contribution; - size_t dof_id; - for (int idim = 0; idim < num_dim; idim++) - { - matrix_contribution = 0; - // compute resulting row of force velocity gradient matrix transpose right multiplied by adjoint vector - /* - for(int idof = 0; idof < Gradient_Matrix_Strides(node_gid*num_dim+idim); idof++){ - dof_id = DOF_Graph_Matrix(node_gid*num_dim+idim,idof); - matrix_contribution += previous_adjoint_vector(dof_id/num_dim,dof_id%num_dim)*Force_Gradient_Velocities(node_gid*num_dim+idim,idof); - } - */ - matrix_contribution = -damping_constant * previous_adjoint_vector(node_gid, idim); - rate_of_change = previous_velocity_vector(node_gid, idim) - - matrix_contribution / node_mass(node_gid) - - phi_previous_adjoint_vector(node_gid, idim) / node_mass(node_gid); - midpoint_adjoint_vector(node_gid, idim) = -rate_of_change * global_dt / 2 + previous_adjoint_vector(node_gid, idim); - matrix_contribution = 0; - // compute resulting row of force displacement gradient matrix transpose right multiplied by adjoint vector - for (int idof = 0; idof < Gradient_Matrix_Strides(node_gid * num_dim + idim); idof++) - { - dof_id = DOF_Graph_Matrix(node_gid * num_dim + idim, idof); - matrix_contribution += -previous_adjoint_vector(dof_id / num_dim, dof_id % num_dim) * Force_Gradient_Positions(node_gid * num_dim + idim, idof); - } - rate_of_change = -matrix_contribution; - // rate_of_change = -0.0000001*previous_adjoint_vector(node_gid,idim); - phi_midpoint_adjoint_vector(node_gid, idim) = -rate_of_change * global_dt / 2 + phi_previous_adjoint_vector(node_gid, idim); - } + //force_gradient_velocity->describe(*fos,Teuchos::VERB_EXTREME); + const_vec_array previous_force_gradient_position = force_gradient_position->getLocalView (Tpetra::Access::ReadOnly); + //const_vec_array current_force_gradient_position = force_gradient_position->getLocalView (Tpetra::Access::ReadOnly); + const_vec_array previous_force_gradient_velocity = force_gradient_velocity->getLocalView (Tpetra::Access::ReadOnly); + //const_vec_array current_force_gradient_velocity = force_gradient_velocity->getLocalView (Tpetra::Access::ReadOnly); + //compute gradient of force with respect to velocity + + const_vec_array previous_adjoint_vector = (*adjoint_vector_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); + const_vec_array phi_previous_adjoint_vector = (*phi_adjoint_vector_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); + vec_array midpoint_adjoint_vector = adjoint_vector_distributed->getLocalView (Tpetra::Access::ReadWrite); + vec_array phi_midpoint_adjoint_vector = phi_adjoint_vector_distributed->getLocalView (Tpetra::Access::ReadWrite); + + //half step update for RK2 scheme + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + real_t rate_of_change; + real_t matrix_contribution; + size_t dof_id; + for (int idim = 0; idim < num_dim; idim++){ + matrix_contribution = 0; + //compute resulting row of force velocity gradient matrix transpose right multiplied by adjoint vector + /* + for(int idof = 0; idof < Gradient_Matrix_Strides(node_gid*num_dim+idim); idof++){ + dof_id = DOF_Graph_Matrix(node_gid*num_dim+idim,idof); + matrix_contribution += previous_adjoint_vector(dof_id/num_dim,dof_id%num_dim)*Force_Gradient_Velocities(node_gid*num_dim+idim,idof); + } + */ + matrix_contribution = -damping_constant*previous_adjoint_vector(node_gid,idim); + rate_of_change = previous_velocity_vector(node_gid,idim)- + matrix_contribution/node_mass(node_gid)- + phi_previous_adjoint_vector(node_gid,idim)/node_mass(node_gid); + midpoint_adjoint_vector(node_gid,idim) = -rate_of_change*global_dt/2 + previous_adjoint_vector(node_gid,idim); + matrix_contribution = 0; + //compute resulting row of force displacement gradient matrix transpose right multiplied by adjoint vector + for(int idof = 0; idof < Gradient_Matrix_Strides(node_gid*num_dim+idim); idof++){ + dof_id = DOF_Graph_Matrix(node_gid*num_dim+idim,idof); + matrix_contribution += -previous_adjoint_vector(dof_id/num_dim,dof_id%num_dim)*Force_Gradient_Positions(node_gid*num_dim+idim,idof); + } + rate_of_change = -matrix_contribution; + //rate_of_change = -0.0000001*previous_adjoint_vector(node_gid,idim); + phi_midpoint_adjoint_vector(node_gid,idim) = -rate_of_change*global_dt/2 + phi_previous_adjoint_vector(node_gid,idim); + } }); // end parallel for - Kokkos::fence(); - - boundary_adjoint(*mesh, boundary, midpoint_adjoint_vector, phi_midpoint_adjoint_vector); - comm_adjoint_vectors(cycle); - // swap names to get ghost nodes for the midpoint vectors - vec_array current_adjoint_vector = adjoint_vector_distributed->getLocalView(Tpetra::Access::ReadWrite); - vec_array phi_current_adjoint_vector = phi_adjoint_vector_distributed->getLocalView(Tpetra::Access::ReadWrite); - midpoint_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadWrite); - phi_midpoint_adjoint_vector = (*phi_adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadWrite); - - // full step update with midpoint gradient for RK2 scheme - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - real_t rate_of_change; - real_t matrix_contribution; - size_t dof_id; - for (int idim = 0; idim < num_dim; idim++) - { - matrix_contribution = 0; - // compute resulting row of force velocity gradient matrix transpose right multiplied by adjoint vector - /* - for(int idof = 0; idof < Gradient_Matrix_Strides(node_gid*num_dim+idim); idof++){ - dof_id = DOF_Graph_Matrix(node_gid*num_dim+idim,idof); - matrix_contribution += midpoint_adjoint_vector(dof_id/num_dim,dof_id%num_dim)*Force_Gradient_Velocities(node_gid*num_dim+idim,idof); - } - */ - matrix_contribution = -damping_constant * midpoint_adjoint_vector(node_gid, idim); - rate_of_change = (previous_velocity_vector(node_gid, idim) + current_velocity_vector(node_gid, idim)) / 2 - - matrix_contribution / node_mass(node_gid) - - phi_midpoint_adjoint_vector(node_gid, idim) / node_mass(node_gid); - current_adjoint_vector(node_gid, idim) = -rate_of_change * global_dt + previous_adjoint_vector(node_gid, idim); - matrix_contribution = 0; - // compute resulting row of force displacement gradient matrix transpose right multiplied by adjoint vector - for (int idof = 0; idof < Gradient_Matrix_Strides(node_gid * num_dim + idim); idof++) - { - dof_id = DOF_Graph_Matrix(node_gid * num_dim + idim, idof); - matrix_contribution += -midpoint_adjoint_vector(dof_id / num_dim, dof_id % num_dim) * Force_Gradient_Positions(node_gid * num_dim + idim, idof); - } - rate_of_change = -matrix_contribution; - // rate_of_change = -0.0000001*midpoint_adjoint_vector(node_gid,idim); - phi_current_adjoint_vector(node_gid, idim) = -rate_of_change * global_dt + phi_previous_adjoint_vector(node_gid, idim); - } + Kokkos::fence(); + + boundary_adjoint(*mesh, boundary,midpoint_adjoint_vector, phi_midpoint_adjoint_vector); + comm_adjoint_vectors(cycle); + //swap names to get ghost nodes for the midpoint vectors + vec_array current_adjoint_vector = adjoint_vector_distributed->getLocalView (Tpetra::Access::ReadWrite); + vec_array phi_current_adjoint_vector = phi_adjoint_vector_distributed->getLocalView (Tpetra::Access::ReadWrite); + midpoint_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView (Tpetra::Access::ReadWrite); + phi_midpoint_adjoint_vector = (*phi_adjoint_vector_data)[cycle]->getLocalView (Tpetra::Access::ReadWrite); + + //full step update with midpoint gradient for RK2 scheme + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + real_t rate_of_change; + real_t matrix_contribution; + size_t dof_id; + for (int idim = 0; idim < num_dim; idim++){ + matrix_contribution = 0; + //compute resulting row of force velocity gradient matrix transpose right multiplied by adjoint vector + /* + for(int idof = 0; idof < Gradient_Matrix_Strides(node_gid*num_dim+idim); idof++){ + dof_id = DOF_Graph_Matrix(node_gid*num_dim+idim,idof); + matrix_contribution += midpoint_adjoint_vector(dof_id/num_dim,dof_id%num_dim)*Force_Gradient_Velocities(node_gid*num_dim+idim,idof); + } + */ + matrix_contribution = -damping_constant*midpoint_adjoint_vector(node_gid,idim); + rate_of_change = (previous_velocity_vector(node_gid,idim) + current_velocity_vector(node_gid,idim))/2- + matrix_contribution/node_mass(node_gid)- + phi_midpoint_adjoint_vector(node_gid,idim)/node_mass(node_gid); + current_adjoint_vector(node_gid,idim) = -rate_of_change*global_dt + previous_adjoint_vector(node_gid,idim); + matrix_contribution = 0; + //compute resulting row of force displacement gradient matrix transpose right multiplied by adjoint vector + for(int idof = 0; idof < Gradient_Matrix_Strides(node_gid*num_dim+idim); idof++){ + dof_id = DOF_Graph_Matrix(node_gid*num_dim+idim,idof); + matrix_contribution += -midpoint_adjoint_vector(dof_id/num_dim,dof_id%num_dim)*Force_Gradient_Positions(node_gid*num_dim+idim,idof); + } + rate_of_change = -matrix_contribution; + //rate_of_change = -0.0000001*midpoint_adjoint_vector(node_gid,idim); + phi_current_adjoint_vector(node_gid,idim) = -rate_of_change*global_dt + phi_previous_adjoint_vector(node_gid,idim); + } }); // end parallel for - Kokkos::fence(); + Kokkos::fence(); - boundary_adjoint(*mesh, boundary, current_adjoint_vector, phi_current_adjoint_vector); - } // end view scope + + boundary_adjoint(*mesh, boundary,current_adjoint_vector, phi_current_adjoint_vector); - comm_adjoint_vectors(cycle); - // phi_adjoint_vector_distributed->describe(*fos,Teuchos::VERB_EXTREME); - } + } //end view scope + + comm_adjoint_vectors(cycle); + //phi_adjoint_vector_distributed->describe(*fos,Teuchos::VERB_EXTREME); + } } + /* ---------------------------------------------------------------------------- Gradient calculation for the kinetic energy minimization problem ------------------------------------------------------------------------------- */ -void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient(const_vec_array design_variables, vec_array design_gradients) -{ - size_t num_bdy_nodes = mesh->num_bdy_nodes; - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos material = simparam->material; - const int num_dim = simparam->num_dims; - int num_corners = rnum_elem * num_nodes_in_elem; - real_t global_dt; - size_t current_data_index, next_data_index; - CArrayKokkos current_element_velocities = CArrayKokkos(num_nodes_in_elem, num_dim); - CArrayKokkos current_element_adjoint = CArrayKokkos(num_nodes_in_elem, num_dim); - - if (myrank == 0) - { - std::cout << "Computing accumulated kinetic energy gradient" << std::endl; - } +void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient(const_vec_array design_variables, vec_array design_gradients){ - compute_topology_optimization_adjoint(); + size_t num_bdy_nodes = mesh->num_bdy_nodes; + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos material = simparam->material; + const int num_dim = simparam->num_dims; + int num_corners = rnum_elem*num_nodes_in_elem; + real_t global_dt; + size_t current_data_index, next_data_index; + CArrayKokkos current_element_velocities = CArrayKokkos(num_nodes_in_elem,num_dim); + CArrayKokkos current_element_adjoint = CArrayKokkos(num_nodes_in_elem,num_dim); - // compute design gradients - FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { - design_gradients(node_id, 0) = 0; - }); // end parallel for - Kokkos::fence(); + if(myrank==0) + std::cout << "Computing accumulated kinetic energy gradient" << std::endl; - // gradient contribution from kinetic energy vMv product. - for (unsigned long cycle = 0; cycle < last_time_step + 1; cycle++) - { - // compute timestep from time data - global_dt = time_data[cycle + 1] - time_data[cycle]; + compute_topology_optimization_adjoint(); - // print - if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) - { - if (cycle == 0) - { - if (myrank == 0) - { - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } - } - // print time step every 10 cycles - else if (cycle % 20 == 0) - { - if (myrank == 0) - { - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } - } // end if - } - - // compute adjoint vector for this data point; use velocity midpoint - // view scope - { - const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); - const_vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); - const_vec_array next_velocity_vector = (*forward_solve_velocity_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); - const_vec_array next_adjoint_vector = (*adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); - - FOR_ALL_CLASS(elem_id, 0, rnum_elem, { - size_t node_id; - size_t corner_id; - real_t inner_product; - // std::cout << elem_mass(elem_id) <dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ + if (cycle==0){ + if(myrank==0) + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + // print time step every 10 cycles + else if (cycle%20==0){ + if(myrank==0) + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } // end if + } - for (int inode = 0; inode < num_nodes_in_elem; inode++) - { - // compute gradient of local element contribution to v^t*M*v product - corner_id = elem_id * num_nodes_in_elem + inode; - corner_value_storage(corner_id) = inner_product * global_dt; - } + //compute adjoint vector for this data point; use velocity midpoint + //view scope + { + const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); + const_vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); + const_vec_array next_velocity_vector = (*forward_solve_velocity_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); + const_vec_array next_adjoint_vector = (*adjoint_vector_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); + + FOR_ALL_CLASS(elem_id, 0, rnum_elem, { + size_t node_id; + size_t corner_id; + real_t inner_product; + //std::cout << elem_mass(elem_id) <dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ + if (cycle==0){ + if(myrank==0) + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + // print time step every 10 cycles + else if (cycle%20==0){ + if(myrank==0) + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } // end if + } - // print - if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) - { - if (cycle == 0) - { - if (myrank == 0) - { - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } + //compute adjoint vector for this data point; use velocity midpoint + //view scope + { + //const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); + const_vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); + const_vec_array next_adjoint_vector = (*adjoint_vector_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); + //const_vec_array current_coord_vector = forward_solve_coordinate_data[cycle]->getLocalView (Tpetra::Access::ReadOnly); + //const_vec_array final_coordinates = forward_solve_coordinate_data[last_time_step+1]->getLocalView (Tpetra::Access::ReadOnly); + + FOR_ALL_CLASS(elem_id, 0, rnum_elem, { + size_t node_id; + size_t corner_id; + real_t inner_product; + //std::cout << elem_mass(elem_id) <getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); - const_vec_array next_adjoint_vector = (*adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); - // const_vec_array current_coord_vector = forward_solve_coordinate_data[cycle]->getLocalView (Tpetra::Access::ReadOnly); - // const_vec_array final_coordinates = forward_solve_coordinate_data[last_time_step+1]->getLocalView (Tpetra::Access::ReadOnly); - - FOR_ALL_CLASS(elem_id, 0, rnum_elem, { - size_t node_id; - size_t corner_id; - real_t inner_product; - // std::cout << elem_mass(elem_id) < design_densities_distributed, Teuchos::RCP design_gradients_distributed) -{ - size_t num_bdy_nodes = mesh->num_bdy_nodes; - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos material = simparam->material; - const int num_dim = simparam->num_dims; - int num_corners = rnum_elem * num_nodes_in_elem; - real_t global_dt; - bool element_constant_density = true; - size_t current_data_index, next_data_index; - CArrayKokkos current_element_velocities = CArrayKokkos(num_nodes_in_elem, num_dim); - CArrayKokkos current_element_adjoint = CArrayKokkos(num_nodes_in_elem, num_dim); - - if (myrank == 0) - { - std::cout << "Computing accumulated kinetic energy gradient" << std::endl; - } +void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient_full(Teuchos::RCP design_densities_distributed, Teuchos::RCP design_gradients_distributed){ - compute_topology_optimization_adjoint_full(); + size_t num_bdy_nodes = mesh->num_bdy_nodes; + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos material = simparam->material; + const int num_dim = simparam->num_dims; + int num_corners = rnum_elem*num_nodes_in_elem; + real_t global_dt; + bool element_constant_density = true; + size_t current_data_index, next_data_index; + CArrayKokkos current_element_velocities = CArrayKokkos(num_nodes_in_elem,num_dim); + CArrayKokkos current_element_adjoint = CArrayKokkos(num_nodes_in_elem,num_dim); - { // view scope - vec_array design_gradients = design_gradients_distributed->getLocalView(Tpetra::Access::ReadWrite); - const_vec_array design_densities = design_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); - // compute design gradients - FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { - design_gradients(node_id, 0) = 0; + if(myrank==0) + std::cout << "Computing accumulated kinetic energy gradient" << std::endl; + + compute_topology_optimization_adjoint_full(); + + { //view scope + vec_array design_gradients = design_gradients_distributed->getLocalView (Tpetra::Access::ReadWrite); + const_vec_array design_densities = design_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); + //compute design gradients + FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { + design_gradients(node_id,0) = 0; }); // end parallel for - Kokkos::fence(); + Kokkos::fence(); - // gradient contribution from kinetic energy v(dM/drho)v product. - if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) - { - if (myrank == 0) - { - std::cout << "v*dM/drho*v term" << std::endl; - } + //gradient contribution from kinetic energy v(dM/drho)v product. + if(simparam->dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ + if(myrank==0){ + std::cout << "v*dM/drho*v term" << std::endl; } + } - for (unsigned long cycle = 0; cycle < last_time_step + 1; cycle++) + for (unsigned long cycle = 0; cycle < last_time_step+1; cycle++) { + //compute timestep from time data + global_dt = time_data[cycle+1] - time_data[cycle]; + + //print + if(simparam->dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ + + if (cycle==0){ + if(myrank==0) + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + // print time step every 10 cycles + else if (cycle%20==0){ + if(myrank==0) + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } // end if + } + + //compute adjoint vector for this data point; use velocity midpoint + //view scope { - // compute timestep from time data - global_dt = time_data[cycle + 1] - time_data[cycle]; + const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); + const_vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); + const_vec_array next_velocity_vector = (*forward_solve_velocity_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); + const_vec_array next_adjoint_vector = (*adjoint_vector_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); + + FOR_ALL_CLASS(elem_id, 0, rnum_elem, { + size_t node_id; + size_t corner_id; + real_t inner_product; + //std::cout << elem_mass(elem_id) <dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) - { - if (cycle == 0) - { - if (myrank == 0) - { - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } - } - // print time step every 10 cycles - else if (cycle % 20 == 0) - { - if (myrank == 0) - { - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } - } // end if + inner_product = 0; + for(int ifill=0; ifill < num_nodes_in_elem; ifill++){ + node_id = nodes_in_elem(elem_id, ifill); + for(int idim=0; idim < num_dim; idim++){ + inner_product += elem_mass(elem_id)*current_element_velocities(ifill,idim)*current_element_velocities(ifill,idim); + } } - // compute adjoint vector for this data point; use velocity midpoint - // view scope - { - const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); - const_vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); - const_vec_array next_velocity_vector = (*forward_solve_velocity_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); - const_vec_array next_adjoint_vector = (*adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); - - FOR_ALL_CLASS(elem_id, 0, rnum_elem, { - size_t node_id; - size_t corner_id; - real_t inner_product; - // std::cout << elem_mass(elem_id) <dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ + if(myrank==0){ + std::cout << "gradient term involving adjoint derivative" << std::endl; } + } - // multiply by Hex8 constants (the diagonlization here only works for Hex8 anyway) - FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { - design_gradients(node_id, 0) *= 0.5 / (double)num_nodes_in_elem / (double)num_nodes_in_elem; - // design_gradients(node_id,0) =0.00001; - }); // end parallel for - Kokkos::fence(); + for (unsigned long cycle = 0; cycle < last_time_step+1; cycle++) { + //compute timestep from time data + global_dt = time_data[cycle+1] - time_data[cycle]; + //print + if(simparam->dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ - // gradient contribution from time derivative of adjoint \dot{lambda}(dM/drho)v product. - if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) - { - if (myrank == 0) - { - std::cout << "gradient term involving adjoint derivative" << std::endl; - } + if (cycle==0){ + if(myrank==0) + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); } + // print time step every 10 cycles + else if (cycle%20==0){ + if(myrank==0) + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } // end if + } - for (unsigned long cycle = 0; cycle < last_time_step + 1; cycle++) + //compute adjoint vector for this data point; use velocity midpoint + //view scope { - // compute timestep from time data - global_dt = time_data[cycle + 1] - time_data[cycle]; - // print - if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) - { - if (cycle == 0) - { - if (myrank == 0) - { - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } - } - // print time step every 10 cycles - else if (cycle % 20 == 0) - { - if (myrank == 0) - { - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } - } // end if + const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); + const_vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); + const_vec_array current_phi_adjoint_vector = (*phi_adjoint_vector_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); + const_vec_array next_velocity_vector = (*forward_solve_velocity_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); + const_vec_array next_adjoint_vector = (*adjoint_vector_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); + const_vec_array next_phi_adjoint_vector = (*phi_adjoint_vector_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); + + const real_t damping_constant = module_params->damping_constant; + FOR_ALL_CLASS(elem_id, 0, rnum_elem, { + real_t lambda_dot_current; + real_t lambda_dot_next; + size_t node_id; + size_t corner_id; + real_t inner_product; + //std::cout << elem_mass(elem_id) <getLocalView(Tpetra::Access::ReadOnly); - const_vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); - const_vec_array current_phi_adjoint_vector = (*phi_adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); - const_vec_array next_velocity_vector = (*forward_solve_velocity_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); - const_vec_array next_adjoint_vector = (*adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); - const_vec_array next_phi_adjoint_vector = (*phi_adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); - - const real_t damping_constant = module_params->damping_constant; - FOR_ALL_CLASS(elem_id, 0, rnum_elem, { - real_t lambda_dot_current; - real_t lambda_dot_next; - size_t node_id; - size_t corner_id; - real_t inner_product; - // std::cout << elem_mass(elem_id) <getLocalView(Tpetra::Access::ReadOnly); - const_vec_array current_adjoint_vector = (*adjoint_vector_data)[0]->getLocalView(Tpetra::Access::ReadOnly); - - FOR_ALL_CLASS(elem_id, 0, rnum_elem, { - real_t lambda_dot; - size_t node_id; - size_t corner_id; - real_t inner_product; - // std::cout << elem_mass(elem_id) <getLocalView (Tpetra::Access::ReadOnly); + const_vec_array current_adjoint_vector = (*adjoint_vector_data)[0]->getLocalView (Tpetra::Access::ReadOnly); + + FOR_ALL_CLASS(elem_id, 0, rnum_elem, { + real_t lambda_dot; + size_t node_id; + size_t corner_id; + real_t inner_product; + //std::cout << elem_mass(elem_id) <getLocalView (Tpetra::Access::ReadWrite); + const_host_vec_array host_design_variables = design_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); + compute_stiffness_gradients(host_design_variables, host_design_gradients); + }//end view scope - // force_design_gradient_term(design_variables, design_gradients); - // view scope - { - host_vec_array host_design_gradients = design_gradients_distributed->getLocalView(Tpetra::Access::ReadWrite); - const_host_vec_array host_design_variables = design_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); - compute_stiffness_gradients(host_design_variables, host_design_gradients); - } // end view scope } /* ---------------------------------------------------------------------- Initialize global vectors and array maps needed for matrix assembly ------------------------------------------------------------------------- */ -void FEA_Module_Dynamic_Elasticity::init_assembly() -{ - int num_dim = simparam->num_dims; - // const_host_elem_conn_array nodes_in_elem = global_nodes_in_elem_distributed->getLocalView (Tpetra::Access::ReadOnly); - Gradient_Matrix_Strides = DCArrayKokkos(nlocal_nodes * num_dim, "Gradient_Matrix_Strides"); - CArrayKokkos Graph_Fill(nall_nodes, "nall_nodes"); - CArrayKokkos current_row_nodes_scanned; - int local_node_index, current_column_index; - size_t max_stride = 0; - size_t nodes_per_element; - nodal_density_flag = simparam->nodal_density_flag; - penalty_power = simparam->optimization_options.simp_penalty_power; - - // allocate stride arrays - CArrayKokkos Graph_Matrix_Strides_initial(nlocal_nodes, "Graph_Matrix_Strides_initial"); - DCArrayKokkos Dual_Graph_Matrix_Strides_initial(nlocal_nodes, "Host_Graph_Matrix_Strides_initial"); - Graph_Matrix_Strides = DCArrayKokkos(nlocal_nodes, "Graph_Matrix_Strides"); - - // allocate storage for the sparse stiffness matrix map used in the assembly process - Global_Stiffness_Matrix_Assembly_Map = Global_Gradient_Matrix_Assembly_Map = DCArrayKokkos(rnum_elem, - max_nodes_per_element, max_nodes_per_element, "Global_Gradient_Matrix_Assembly_Map"); - - // allocate array used to determine global node repeats in the sparse graph later - DCArrayKokkos node_indices_used(nall_nodes, "node_indices_used"); - - /*allocate array that stores which column the node index occured on for the current row - when removing repeats*/ - DCArrayKokkos column_index(nall_nodes, "column_index"); - - // initialize nlocal arrays - FOR_ALL_CLASS(inode, 0, nlocal_nodes, { - Graph_Matrix_Strides_initial(inode) = 0; - Graph_Matrix_Strides(inode) = 0; - Graph_Fill(inode) = 0; +void FEA_Module_Dynamic_Elasticity::init_assembly(){ + int num_dim = simparam->num_dims; + //const_host_elem_conn_array nodes_in_elem = global_nodes_in_elem_distributed->getLocalView (Tpetra::Access::ReadOnly); + Gradient_Matrix_Strides = DCArrayKokkos (nlocal_nodes*num_dim, "Gradient_Matrix_Strides"); + CArrayKokkos Graph_Fill(nall_nodes, "nall_nodes"); + CArrayKokkos current_row_nodes_scanned; + int local_node_index, current_column_index; + size_t max_stride = 0; + size_t nodes_per_element; + nodal_density_flag = simparam->nodal_density_flag; + penalty_power = simparam->optimization_options.simp_penalty_power; + + //allocate stride arrays + CArrayKokkos Graph_Matrix_Strides_initial(nlocal_nodes, "Graph_Matrix_Strides_initial"); + DCArrayKokkos Dual_Graph_Matrix_Strides_initial(nlocal_nodes, "Host_Graph_Matrix_Strides_initial"); + Graph_Matrix_Strides = DCArrayKokkos(nlocal_nodes, "Graph_Matrix_Strides"); + + //allocate storage for the sparse stiffness matrix map used in the assembly process + Global_Stiffness_Matrix_Assembly_Map = Global_Gradient_Matrix_Assembly_Map = DCArrayKokkos(rnum_elem, + max_nodes_per_element,max_nodes_per_element, "Global_Gradient_Matrix_Assembly_Map"); + + //allocate array used to determine global node repeats in the sparse graph later + DCArrayKokkos node_indices_used(nall_nodes, "node_indices_used"); + + /*allocate array that stores which column the node index occured on for the current row + when removing repeats*/ + DCArrayKokkos column_index(nall_nodes, "column_index"); + + //initialize nlocal arrays + FOR_ALL_CLASS(inode, 0, nlocal_nodes, { + Graph_Matrix_Strides_initial(inode) = 0; + Graph_Matrix_Strides(inode) = 0; + Graph_Fill(inode) = 0; }); // end parallel for - Kokkos::fence(); + Kokkos::fence(); - // initialize nall arrays - // initialize nlocal arrays - FOR_ALL_CLASS(inode, 0, nall_nodes, { - node_indices_used(inode) = 0; - column_index(inode) = 0; + //initialize nall arrays + //initialize nlocal arrays + FOR_ALL_CLASS(inode, 0, nall_nodes, { + node_indices_used(inode) = 0; + column_index(inode) = 0; }); // end parallel for - Kokkos::fence(); - - // count upper bound of strides for Sparse Pattern Graph by allowing repeats due to connectivity - if (num_dim == 2) - { - for (int ielem = 0; ielem < rnum_elem; ielem++) - { - element_select->choose_2Delem_type(Element_Types(ielem), elem2D); - nodes_per_element = elem2D->num_nodes(); - for (int lnode = 0; lnode < nodes_per_element; lnode++) - { - local_node_index = nodes_in_elem(ielem, lnode); - if (local_node_index < nlocal_nodes) - { - Dual_Graph_Matrix_Strides_initial.host(local_node_index) += nodes_per_element; - } - } + Kokkos::fence(); + + //count upper bound of strides for Sparse Pattern Graph by allowing repeats due to connectivity + if(num_dim == 2) + for (int ielem = 0; ielem < rnum_elem; ielem++){ + element_select->choose_2Delem_type(Element_Types(ielem), elem2D); + nodes_per_element = elem2D->num_nodes(); + for (int lnode = 0; lnode < nodes_per_element; lnode++){ + local_node_index = nodes_in_elem(ielem, lnode); + if(local_node_index < nlocal_nodes){ + Dual_Graph_Matrix_Strides_initial.host(local_node_index) += nodes_per_element; } + } } - if (num_dim == 3) - { - for (int ielem = 0; ielem < rnum_elem; ielem++) - { - element_select->choose_3Delem_type(Element_Types(ielem), elem); - nodes_per_element = elem->num_nodes(); - for (int lnode = 0; lnode < nodes_per_element; lnode++) - { - local_node_index = nodes_in_elem(ielem, lnode); - if (local_node_index < nlocal_nodes) - { - Dual_Graph_Matrix_Strides_initial.host(local_node_index) += nodes_per_element; - } - } + if(num_dim == 3) + for (int ielem = 0; ielem < rnum_elem; ielem++){ + element_select->choose_3Delem_type(Element_Types(ielem), elem); + nodes_per_element = elem->num_nodes(); + for (int lnode = 0; lnode < nodes_per_element; lnode++){ + local_node_index = nodes_in_elem(ielem, lnode); + if(local_node_index < nlocal_nodes){ + Dual_Graph_Matrix_Strides_initial.host(local_node_index) += nodes_per_element; } + } } + + Dual_Graph_Matrix_Strides_initial.update_device(); - Dual_Graph_Matrix_Strides_initial.update_device(); - - // equate strides for later - FOR_ALL_CLASS(inode, 0, nlocal_nodes, { - Graph_Matrix_Strides(inode) = Graph_Matrix_Strides_initial(inode) = Dual_Graph_Matrix_Strides_initial(inode); + //equate strides for later + FOR_ALL_CLASS(inode, 0, nlocal_nodes, { + Graph_Matrix_Strides(inode) = Graph_Matrix_Strides_initial(inode) = Dual_Graph_Matrix_Strides_initial(inode); }); // end parallel for - - // for (int inode = 0; inode < nlocal_nodes; inode++) - // std::cout << Graph_Matrix_Strides_initial(inode) << std::endl; - - // compute maximum stride - size_t update = 0; - REDUCE_MAX_CLASS(inode, 0, nlocal_nodes, update, { - if (update < Graph_Matrix_Strides_initial(inode)) - { - update = Graph_Matrix_Strides_initial(inode); - } + + //for (int inode = 0; inode < nlocal_nodes; inode++) + //std::cout << Graph_Matrix_Strides_initial(inode) << std::endl; + + //compute maximum stride + size_t update = 0; + REDUCE_MAX_CLASS(inode, 0, nlocal_nodes, update, { + if(update < Graph_Matrix_Strides_initial(inode)) + update = Graph_Matrix_Strides_initial(inode); }, max_stride); - - // std::cout << "THE MAX STRIDE" << max_stride << std::endl; - // allocate array used in the repeat removal process - current_row_nodes_scanned = CArrayKokkos(max_stride, "current_row_nodes_scanned"); - - // allocate sparse graph with node repeats - RaggedRightArrayKokkos Repeat_Graph_Matrix(Graph_Matrix_Strides_initial); - RaggedRightArrayofVectorsKokkos Element_local_indices(Graph_Matrix_Strides_initial, num_dim); - - // Fill the initial Graph with repeats - if (num_dim == 2) - { - for (int ielem = 0; ielem < rnum_elem; ielem++) - { - element_select->choose_2Delem_type(Element_Types(ielem), elem2D); - nodes_per_element = elem2D->num_nodes(); - for (int lnode = 0; lnode < nodes_per_element; lnode++) - { - local_node_index = nodes_in_elem(ielem, lnode); - if (local_node_index < nlocal_nodes) - { - for (int jnode = 0; jnode < nodes_per_element; jnode++) - { - current_column_index = Graph_Fill(local_node_index) + jnode; - Repeat_Graph_Matrix(local_node_index, current_column_index) = nodes_in_elem(ielem, jnode); - - // fill inverse map - Element_local_indices(local_node_index, current_column_index, 0) = ielem; - Element_local_indices(local_node_index, current_column_index, 1) = lnode; - Element_local_indices(local_node_index, current_column_index, 2) = jnode; - - // fill forward map - Global_Gradient_Matrix_Assembly_Map(ielem, lnode, jnode) = current_column_index; - } - Graph_Fill(local_node_index) += nodes_per_element; - } - } + + //std::cout << "THE MAX STRIDE" << max_stride << std::endl; + //allocate array used in the repeat removal process + current_row_nodes_scanned = CArrayKokkos(max_stride, "current_row_nodes_scanned"); + + //allocate sparse graph with node repeats + RaggedRightArrayKokkos Repeat_Graph_Matrix(Graph_Matrix_Strides_initial); + RaggedRightArrayofVectorsKokkos Element_local_indices(Graph_Matrix_Strides_initial,num_dim); + + //Fill the initial Graph with repeats + if(num_dim == 2){ + for (int ielem = 0; ielem < rnum_elem; ielem++){ + element_select->choose_2Delem_type(Element_Types(ielem), elem2D); + nodes_per_element = elem2D->num_nodes(); + for (int lnode = 0; lnode < nodes_per_element; lnode++){ + local_node_index = nodes_in_elem(ielem, lnode); + if(local_node_index < nlocal_nodes){ + for (int jnode = 0; jnode < nodes_per_element; jnode++){ + current_column_index = Graph_Fill(local_node_index)+jnode; + Repeat_Graph_Matrix(local_node_index, current_column_index) = nodes_in_elem(ielem,jnode); + + //fill inverse map + Element_local_indices(local_node_index,current_column_index,0) = ielem; + Element_local_indices(local_node_index,current_column_index,1) = lnode; + Element_local_indices(local_node_index,current_column_index,2) = jnode; + + //fill forward map + Global_Gradient_Matrix_Assembly_Map(ielem,lnode,jnode) = current_column_index; + } + Graph_Fill(local_node_index) += nodes_per_element; } + } } - - if (num_dim == 3) - { - for (int ielem = 0; ielem < rnum_elem; ielem++) - { - element_select->choose_3Delem_type(Element_Types(ielem), elem); - nodes_per_element = elem->num_nodes(); - for (int lnode = 0; lnode < nodes_per_element; lnode++) - { - local_node_index = nodes_in_elem(ielem, lnode); - if (local_node_index < nlocal_nodes) - { - for (int jnode = 0; jnode < nodes_per_element; jnode++) - { - current_column_index = Graph_Fill(local_node_index) + jnode; - Repeat_Graph_Matrix(local_node_index, current_column_index) = nodes_in_elem(ielem, jnode); - - // fill inverse map - Element_local_indices(local_node_index, current_column_index, 0) = ielem; - Element_local_indices(local_node_index, current_column_index, 1) = lnode; - Element_local_indices(local_node_index, current_column_index, 2) = jnode; - - // fill forward map - Global_Gradient_Matrix_Assembly_Map(ielem, lnode, jnode) = current_column_index; - } - Graph_Fill(local_node_index) += nodes_per_element; - } - } + } + + if(num_dim == 3){ + for (int ielem = 0; ielem < rnum_elem; ielem++){ + element_select->choose_3Delem_type(Element_Types(ielem), elem); + nodes_per_element = elem->num_nodes(); + for (int lnode = 0; lnode < nodes_per_element; lnode++){ + local_node_index = nodes_in_elem(ielem, lnode); + if(local_node_index < nlocal_nodes){ + for (int jnode = 0; jnode < nodes_per_element; jnode++){ + current_column_index = Graph_Fill(local_node_index)+jnode; + Repeat_Graph_Matrix(local_node_index, current_column_index) = nodes_in_elem(ielem,jnode); + + //fill inverse map + Element_local_indices(local_node_index,current_column_index,0) = ielem; + Element_local_indices(local_node_index,current_column_index,1) = lnode; + Element_local_indices(local_node_index,current_column_index,2) = jnode; + + //fill forward map + Global_Gradient_Matrix_Assembly_Map(ielem,lnode,jnode) = current_column_index; + } + Graph_Fill(local_node_index) += nodes_per_element; } + } } - - // debug statement - // std::cout << "started run" << std::endl; - // std::cout << "Graph Matrix Strides Repeat on task " << myrank << std::endl; - // for (int inode = 0; inode < nlocal_nodes; inode++) - // std::cout << Graph_Matrix_Strides(inode) << std::endl; - RUN_CLASS({ - // remove repeats from the inital graph setup - int current_node; - // remove repeats from the inital graph setup - int current_element_index; - int element_row_index; - int element_column_index; - int current_stride; - int current_row_n_nodes_scanned; - for (int inode = 0; inode < nlocal_nodes; inode++) - { - current_row_n_nodes_scanned = 0; - for (int istride = 0; istride < Graph_Matrix_Strides(inode); istride++) - { - // convert global index in graph to its local index for the flagging array - current_node = Repeat_Graph_Matrix(inode, istride); - // debug - // if(current_node==-1) - // std::cout << "Graph Matrix node access on task " << myrank << std::endl; - // std::cout << Repeat_Graph_Matrix(inode,istride) << std::endl; - if (node_indices_used(current_node)) - { - // set global assembly map index to the location in the graph matrix where this global node was first found - current_element_index = Element_local_indices(inode, istride, 0); - element_row_index = Element_local_indices(inode, istride, 1); - element_column_index = Element_local_indices(inode, istride, 2); - Global_Gradient_Matrix_Assembly_Map(current_element_index, element_row_index, element_column_index) - = column_index(current_node); - - // swap current node with the end of the current row and shorten the stride of the row - // first swap information about the inverse and forward maps - - current_stride = Graph_Matrix_Strides(inode); - if (istride != current_stride - 1) - { - Element_local_indices(inode, istride, 0) = Element_local_indices(inode, current_stride - 1, 0); - Element_local_indices(inode, istride, 1) = Element_local_indices(inode, current_stride - 1, 1); - Element_local_indices(inode, istride, 2) = Element_local_indices(inode, current_stride - 1, 2); - current_element_index = Element_local_indices(inode, istride, 0); - element_row_index = Element_local_indices(inode, istride, 1); - element_column_index = Element_local_indices(inode, istride, 2); - - Global_Gradient_Matrix_Assembly_Map(current_element_index, element_row_index, element_column_index) - = istride; - - // now that the element map information has been copied, copy the global node index and delete the last index - - Repeat_Graph_Matrix(inode, istride) = Repeat_Graph_Matrix(inode, current_stride - 1); - } - istride--; - Graph_Matrix_Strides(inode)--; - } - else - { - /*this node hasn't shown up in the row before; add it to the list of nodes - that have been scanned uniquely. Use this list to reset the flag array - afterwards without having to loop over all the nodes in the system*/ - node_indices_used(current_node) = 1; - column_index(current_node) = istride; - current_row_nodes_scanned(current_row_n_nodes_scanned) = current_node; - current_row_n_nodes_scanned++; - } - } - // reset nodes used list for the next row of the sparse list - for (int node_reset = 0; node_reset < current_row_n_nodes_scanned; node_reset++) - { - node_indices_used(current_row_nodes_scanned(node_reset)) = 0; - } + } + + //debug statement + //std::cout << "started run" << std::endl; + //std::cout << "Graph Matrix Strides Repeat on task " << myrank << std::endl; + //for (int inode = 0; inode < nlocal_nodes; inode++) + //std::cout << Graph_Matrix_Strides(inode) << std::endl; + RUN_CLASS({ + //remove repeats from the inital graph setup + int current_node; + //remove repeats from the inital graph setup + int current_element_index; + int element_row_index; + int element_column_index; + int current_stride; + int current_row_n_nodes_scanned; + for (int inode = 0; inode < nlocal_nodes; inode++){ + current_row_n_nodes_scanned = 0; + for (int istride = 0; istride < Graph_Matrix_Strides(inode); istride++){ + //convert global index in graph to its local index for the flagging array + current_node = Repeat_Graph_Matrix(inode,istride); + //debug + //if(current_node==-1) + //std::cout << "Graph Matrix node access on task " << myrank << std::endl; + //std::cout << Repeat_Graph_Matrix(inode,istride) << std::endl; + if(node_indices_used(current_node)){ + //set global assembly map index to the location in the graph matrix where this global node was first found + current_element_index = Element_local_indices(inode,istride,0); + element_row_index = Element_local_indices(inode,istride,1); + element_column_index = Element_local_indices(inode,istride,2); + Global_Gradient_Matrix_Assembly_Map(current_element_index,element_row_index, element_column_index) + = column_index(current_node); + + + //swap current node with the end of the current row and shorten the stride of the row + //first swap information about the inverse and forward maps + + current_stride = Graph_Matrix_Strides(inode); + if(istride!=current_stride-1){ + Element_local_indices(inode,istride,0) = Element_local_indices(inode,current_stride-1,0); + Element_local_indices(inode,istride,1) = Element_local_indices(inode,current_stride-1,1); + Element_local_indices(inode,istride,2) = Element_local_indices(inode,current_stride-1,2); + current_element_index = Element_local_indices(inode,istride,0); + element_row_index = Element_local_indices(inode,istride,1); + element_column_index = Element_local_indices(inode,istride,2); + + Global_Gradient_Matrix_Assembly_Map(current_element_index,element_row_index, element_column_index) + = istride; + + //now that the element map information has been copied, copy the global node index and delete the last index + + Repeat_Graph_Matrix(inode,istride) = Repeat_Graph_Matrix(inode,current_stride-1); + } + istride--; + Graph_Matrix_Strides(inode)--; } - }); - Kokkos::fence(); - - Graph_Matrix_Strides.update_host(); - // copy reduced content to non_repeat storage - Graph_Matrix = RaggedRightArrayKokkos(Graph_Matrix_Strides); - - FOR_ALL_CLASS(inode, 0, nlocal_nodes, { - for (int istride = 0; istride < Graph_Matrix_Strides(inode); istride++) - { - Graph_Matrix(inode, istride) = Repeat_Graph_Matrix(inode, istride); + else{ + /*this node hasn't shown up in the row before; add it to the list of nodes + that have been scanned uniquely. Use this list to reset the flag array + afterwards without having to loop over all the nodes in the system*/ + node_indices_used(current_node) = 1; + column_index(current_node) = istride; + current_row_nodes_scanned(current_row_n_nodes_scanned) = current_node; + current_row_n_nodes_scanned++; } - }); // end parallel for - - // deallocate repeat matrix - - /*At this stage the sparse graph should have unique global indices on each row. - The constructed Assembly map (to the global sparse matrix) - is used to loop over each element's local stiffness matrix in the assembly process.*/ + } + //reset nodes used list for the next row of the sparse list + for(int node_reset = 0; node_reset < current_row_n_nodes_scanned; node_reset++) + node_indices_used(current_row_nodes_scanned(node_reset)) = 0; - // expand strides for stiffness matrix by multipling by dim - FOR_ALL_CLASS(idof, 0, num_dim * nlocal_nodes, { - Gradient_Matrix_Strides(idof) = num_dim * Graph_Matrix_Strides(idof / num_dim); + } + }); + Kokkos::fence(); + + Graph_Matrix_Strides.update_host(); + //copy reduced content to non_repeat storage + Graph_Matrix = RaggedRightArrayKokkos(Graph_Matrix_Strides); + + FOR_ALL_CLASS(inode, 0, nlocal_nodes, { + for(int istride = 0; istride < Graph_Matrix_Strides(inode); istride++){ + Graph_Matrix(inode,istride) = Repeat_Graph_Matrix(inode,istride); + } }); // end parallel for - Stiffness_Matrix = Force_Gradient_Positions = RaggedRightArrayKokkos(Gradient_Matrix_Strides); - Force_Gradient_Velocities = RaggedRightArrayKokkos(Gradient_Matrix_Strides); - DOF_Graph_Matrix = RaggedRightArrayKokkos(Gradient_Matrix_Strides); - - // set stiffness Matrix Graph - // debug print - // std::cout << "DOF GRAPH MATRIX ENTRIES ON TASK " << myrank << std::endl; - FOR_ALL_CLASS(idof, 0, num_dim * nlocal_nodes, { - for (int istride = 0; istride < Gradient_Matrix_Strides(idof); istride++) - { - DOF_Graph_Matrix(idof, istride) = Graph_Matrix(idof / num_dim, istride / num_dim) * num_dim + istride % num_dim; - } + //deallocate repeat matrix + + /*At this stage the sparse graph should have unique global indices on each row. + The constructed Assembly map (to the global sparse matrix) + is used to loop over each element's local stiffness matrix in the assembly process.*/ + + //expand strides for stiffness matrix by multipling by dim + FOR_ALL_CLASS(idof, 0, num_dim*nlocal_nodes, { + Gradient_Matrix_Strides(idof) = num_dim*Graph_Matrix_Strides(idof/num_dim); }); // end parallel for - Stiffness_Matrix_Strides = Gradient_Matrix_Strides; - - /* - //construct distributed gradient matrix from local kokkos data - //build column map for the global gradient matrix - Teuchos::RCP > colmap; - const Teuchos::RCP > dommap = local_dof_map; - - Tpetra::Details::makeColMap(colmap,dommap,DOF_Graph_Matrix.get_kokkos_view(), nullptr); - - size_t nnz = DOF_Graph_Matrix.size(); - - //debug print - //std::cout << "DOF GRAPH SIZE ON RANK " << myrank << " IS " << nnz << std::endl; + Stiffness_Matrix = Force_Gradient_Positions = RaggedRightArrayKokkos(Gradient_Matrix_Strides); + Force_Gradient_Velocities = RaggedRightArrayKokkos(Gradient_Matrix_Strides); + DOF_Graph_Matrix = RaggedRightArrayKokkos (Gradient_Matrix_Strides); - //local indices in the graph using the constructed column map - CArrayKokkos gradient_local_indices(nnz, "gradient_local_indices"); - - //row offsets with compatible template arguments - Kokkos::View row_offsets = DOF_Graph_Matrix.start_index_; - row_pointers row_offsets_pass("row_offsets", nlocal_nodes*num_dim+1); - for(int ipass = 0; ipass < nlocal_nodes*num_dim + 1; ipass++){ - row_offsets_pass(ipass) = row_offsets(ipass); + //set stiffness Matrix Graph + //debug print + //std::cout << "DOF GRAPH MATRIX ENTRIES ON TASK " << myrank << std::endl; + FOR_ALL_CLASS(idof, 0, num_dim*nlocal_nodes, { + for (int istride = 0; istride < Gradient_Matrix_Strides(idof); istride++){ + DOF_Graph_Matrix(idof,istride) = Graph_Matrix(idof/num_dim,istride/num_dim)*num_dim + istride%num_dim; } + }); // end parallel for - size_t entrycount = 0; - for(int irow = 0; irow < nlocal_nodes*num_dim; irow++){ - for(int istride = 0; istride < Gradient_Matrix_Strides(irow); istride++){ - gradient_local_indices(entrycount) = colmap->getLocalElement(DOF_Graph_Matrix(irow,istride)); - entrycount++; - } + Stiffness_Matrix_Strides = Gradient_Matrix_Strides; + + /* + //construct distributed gradient matrix from local kokkos data + //build column map for the global gradient matrix + Teuchos::RCP > colmap; + const Teuchos::RCP > dommap = local_dof_map; + + Tpetra::Details::makeColMap(colmap,dommap,DOF_Graph_Matrix.get_kokkos_view(), nullptr); + + size_t nnz = DOF_Graph_Matrix.size(); + + //debug print + //std::cout << "DOF GRAPH SIZE ON RANK " << myrank << " IS " << nnz << std::endl; + + //local indices in the graph using the constructed column map + CArrayKokkos gradient_local_indices(nnz, "gradient_local_indices"); + + //row offsets with compatible template arguments + Kokkos::View row_offsets = DOF_Graph_Matrix.start_index_; + row_pointers row_offsets_pass("row_offsets", nlocal_nodes*num_dim+1); + for(int ipass = 0; ipass < nlocal_nodes*num_dim + 1; ipass++){ + row_offsets_pass(ipass) = row_offsets(ipass); + } + + size_t entrycount = 0; + for(int irow = 0; irow < nlocal_nodes*num_dim; irow++){ + for(int istride = 0; istride < Gradient_Matrix_Strides(irow); istride++){ + gradient_local_indices(entrycount) = colmap->getLocalElement(DOF_Graph_Matrix(irow,istride)); + entrycount++; } - - - //sort values and indices - Tpetra::Import_Util::sortCrsEntries(row_offsets_pass, gradient_local_indices.get_kokkos_view(), Force_Gradient_Positions.get_kokkos_view()); - Tpetra::Import_Util::sortCrsEntries(row_offsets_pass, gradient_local_indices.get_kokkos_view(), Force_Gradient_Velocities.get_kokkos_view()); - - //Teuchos::RCP crs_matrix_params = Teuchos::rcp(new Teuchos::ParameterList("crsmatrix")); - //crs_matrix_params->set("sorted", false); - distributed_force_gradient_positions = Teuchos::rcp(new MAT(local_dof_map, colmap, row_offsets_pass, gradient_local_indices.get_kokkos_view(), Force_Gradient_Positions.get_kokkos_view())); - distributed_force_gradient_positions->fillComplete(); - distributed_force_gradient_velocities = Teuchos::rcp(new MAT(local_dof_map, colmap, row_offsets_pass, gradient_local_indices.get_kokkos_view(), Force_Gradient_Velocities.get_kokkos_view())); - distributed_force_gradient_velocities->fillComplete(); - */ - // distributed_force_gradient_positions->describe(*fos,Teuchos::VERB_EXTREME); - // distributed_force_gradient_velocities->describe(*fos,Teuchos::VERB_EXTREME); + } + + + //sort values and indices + Tpetra::Import_Util::sortCrsEntries(row_offsets_pass, gradient_local_indices.get_kokkos_view(), Force_Gradient_Positions.get_kokkos_view()); + Tpetra::Import_Util::sortCrsEntries(row_offsets_pass, gradient_local_indices.get_kokkos_view(), Force_Gradient_Velocities.get_kokkos_view()); + + //Teuchos::RCP crs_matrix_params = Teuchos::rcp(new Teuchos::ParameterList("crsmatrix")); + //crs_matrix_params->set("sorted", false); + distributed_force_gradient_positions = Teuchos::rcp(new MAT(local_dof_map, colmap, row_offsets_pass, gradient_local_indices.get_kokkos_view(), Force_Gradient_Positions.get_kokkos_view())); + distributed_force_gradient_positions->fillComplete(); + distributed_force_gradient_velocities = Teuchos::rcp(new MAT(local_dof_map, colmap, row_offsets_pass, gradient_local_indices.get_kokkos_view(), Force_Gradient_Velocities.get_kokkos_view())); + distributed_force_gradient_velocities->fillComplete(); + */ + //distributed_force_gradient_positions->describe(*fos,Teuchos::VERB_EXTREME); + //distributed_force_gradient_velocities->describe(*fos,Teuchos::VERB_EXTREME); } /* ---------------------------------------------------------------------- Enforce boundary conditions on the adjoint vectors ------------------------------------------------------------------------- */ -void FEA_Module_Dynamic_Elasticity::boundary_adjoint(const mesh_t& mesh, - const DCArrayKokkos& boundary, - vec_array& node_adjoint, - vec_array& node_phi_adjoint) -{ - // error and debug flag - // DCArrayKokkos print_flag(1, "print_flag"); - // print_flag.host(0) = false; - // print_flag.update_device(); - - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; +void FEA_Module_Dynamic_Elasticity::boundary_adjoint(const mesh_t &mesh, + const DCArrayKokkos &boundary, + vec_array &node_adjoint, + vec_array &node_phi_adjoint){ + + //error and debug flag + //DCArrayKokkos print_flag(1, "print_flag"); + //print_flag.host(0) = false; + //print_flag.update_device(); + + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; int num_dims = simparam->num_dims; // Loop over boundary sets - for (size_t bdy_set = 0; bdy_set < num_bdy_sets; bdy_set++) - { + for (size_t bdy_set=0; bdy_set fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - // if(myrank==0) - // *fos << "Density data :" << std::endl; - // node_densities_distributed->describe(*fos,Teuchos::VERB_EXTREME); - // *fos << std::endl; - // std::fflush(stdout); - - // communicate design densities - // create import object using local node indices map and all indices map - // Tpetra::Import importer(map, all_node_map); - - // comms to get ghosts - (*adjoint_vector_data)[cycle]->doImport(*adjoint_vector_distributed, *importer, Tpetra::INSERT); - (*phi_adjoint_vector_data)[cycle]->doImport(*phi_adjoint_vector_distributed, *importer, Tpetra::INSERT); - // all_node_map->describe(*fos,Teuchos::VERB_EXTREME); - // all_node_velocities_distributed->describe(*fos,Teuchos::VERB_EXTREME); - - // update_count++; - // if(update_count==1){ - // MPI_Barrier(world); - // MPI_Abort(world,4); - // } +void FEA_Module_Dynamic_Elasticity::comm_adjoint_vectors(int cycle){ + + //debug print of design vector + //std::ostream &out = std::cout; + //Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + //if(myrank==0) + //*fos << "Density data :" << std::endl; + //node_densities_distributed->describe(*fos,Teuchos::VERB_EXTREME); + //*fos << std::endl; + //std::fflush(stdout); + + //communicate design densities + //create import object using local node indices map and all indices map + //Tpetra::Import importer(map, all_node_map); + + //comms to get ghosts + (*adjoint_vector_data)[cycle]->doImport(*adjoint_vector_distributed, *importer, Tpetra::INSERT); + (*phi_adjoint_vector_data)[cycle]->doImport(*phi_adjoint_vector_distributed, *importer, Tpetra::INSERT); + //all_node_map->describe(*fos,Teuchos::VERB_EXTREME); + //all_node_velocities_distributed->describe(*fos,Teuchos::VERB_EXTREME); + + //update_count++; + //if(update_count==1){ + //MPI_Barrier(world); + //MPI_Abort(world,4); + //} } diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/FEA_Module_SGH.h b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/FEA_Module_SGH.h index db9c1e3e2..b233802d2 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/FEA_Module_SGH.h +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/FEA_Module_SGH.h @@ -11,11 +11,14 @@ This program is open source under the BSD-3 License. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. @@ -31,6 +34,7 @@ OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. **********************************************************************************************/ + #ifndef FEA_MODULE_SGH_H #define FEA_MODULE_SGH_H @@ -42,657 +46,655 @@ #include "FEA_Module.h" #include "material_models.h" -// Forward declarations class Explicit_Solver; + class Solver; + class Simulation_Parameters_Explicit; + class SGH_Parameters; struct material_t; + struct boundary_t; -class FEA_Module_SGH : public FEA_Module -{ -public: - FEA_Module_SGH(SGH_Parameters& params, Solver* Solver_Pointer, std::shared_ptr mesh_in, const int my_fea_module_index = 0); - ~FEA_Module_SGH(); - - // initialize data for boundaries of the model and storage for boundary conditions and applied loads - void sgh_interface_setup(node_t& node, elem_t& elem, corner_t& corner); - - void setup(); - - void cleanup_material_models(); - - int solve(); - - void module_cleanup(); - - void sgh_solve(); - - /** - * \brief Calculate forces - * - * This function calculates the corner forces and the evolves stress - * according to the chosen material constitutive model - * - * \param material Array of material types - * \param mesh Simulation mesh - * \param node_coords View into nodal position array - * \param node_vel View into the nodal velocity array - * \param elem_den View into the element density array - * \param elem_sie View into the element specific internal energy array - * \param elem_pres View into the element pressure array - * \param elem_stress View into the element stress array - * \param elem_sspd View into the element sound speed array - * \param elem_vol View into the element volume array - * \param elem_div View into the element divergence array - * \param elem_mat_id View into the element material id array - * \param corner_force View into the corner forces - * \param rk_alpha Runge Kutta alpha value - * \param cycle Simulation cycle - * - */ - void get_force_sgh(const DCArrayKokkos& material, - const mesh_t& mesh, - const DViewCArrayKokkos& node_coords, - const DViewCArrayKokkos& node_vel, - const DViewCArrayKokkos& elem_den, - const DViewCArrayKokkos& elem_sie, - const DViewCArrayKokkos& elem_pres, - DViewCArrayKokkos& elem_stress, - const DViewCArrayKokkos& elem_sspd, - const DViewCArrayKokkos& elem_vol, - const DViewCArrayKokkos& elem_div, - const DViewCArrayKokkos& elem_mat_id, - DViewCArrayKokkos& corner_force, +class FEA_Module_SGH: public FEA_Module{ + +public: + + FEA_Module_SGH(SGH_Parameters& params, Solver *Solver_Pointer, std::shared_ptr mesh_in, const int my_fea_module_index = 0); + ~FEA_Module_SGH(); + + //initialize data for boundaries of the model and storage for boundary conditions and applied loads + void sgh_interface_setup(node_t &node, elem_t &elem, corner_t &corner); + + void setup(); + + void cleanup_material_models(); + + int solve(); + + void module_cleanup(); + + void sgh_solve(); + + void get_force_sgh(const DCArrayKokkos &material, + const mesh_t &mesh, + const DViewCArrayKokkos &node_coords, + const DViewCArrayKokkos &node_vel, + const DViewCArrayKokkos &elem_den, + const DViewCArrayKokkos &elem_sie, + const DViewCArrayKokkos &elem_pres, + DViewCArrayKokkos &elem_stress, + const DViewCArrayKokkos &elem_sspd, + const DViewCArrayKokkos &elem_vol, + const DViewCArrayKokkos &elem_div, + const DViewCArrayKokkos &elem_mat_id, + DViewCArrayKokkos &corner_force, + const double rk_alpha, + const size_t cycle); + + void get_force_vgradient_sgh(const DCArrayKokkos &material, + const mesh_t &mesh, + const DViewCArrayKokkos &node_coords, + const DViewCArrayKokkos &node_vel, + const DViewCArrayKokkos &elem_den, + const DViewCArrayKokkos &elem_sie, + const DViewCArrayKokkos &elem_pres, + const DViewCArrayKokkos &elem_stress, + const DViewCArrayKokkos &elem_sspd, + const DViewCArrayKokkos &elem_vol, + const DViewCArrayKokkos &elem_div, + const DViewCArrayKokkos &elem_mat_id, + const double rk_alpha, + const size_t cycle); + + void get_force_ugradient_sgh(const DCArrayKokkos &material, + const mesh_t &mesh, + const DViewCArrayKokkos &node_coords, + const DViewCArrayKokkos &node_vel, + const DViewCArrayKokkos &elem_den, + const DViewCArrayKokkos &elem_sie, + const DViewCArrayKokkos &elem_pres, + const DViewCArrayKokkos &elem_stress, + const DViewCArrayKokkos &elem_sspd, + const DViewCArrayKokkos &elem_vol, + const DViewCArrayKokkos &elem_div, + const DViewCArrayKokkos &elem_mat_id, + const double rk_alpha, + const size_t cycle); + + void get_force_egradient_sgh(const DCArrayKokkos &material, + const mesh_t &mesh, + const DViewCArrayKokkos &node_coords, + const DViewCArrayKokkos &node_vel, + const DViewCArrayKokkos &elem_den, + const DViewCArrayKokkos &elem_sie, + const DViewCArrayKokkos &elem_pres, + const DViewCArrayKokkos &elem_stress, + const DViewCArrayKokkos &elem_sspd, + const DViewCArrayKokkos &elem_vol, + const DViewCArrayKokkos &elem_div, + const DViewCArrayKokkos &elem_mat_id, + const double rk_alpha, + const size_t cycle); + + void get_force_dgradient_sgh(const DCArrayKokkos &material, + const mesh_t &mesh, + const DViewCArrayKokkos &node_coords, + const DViewCArrayKokkos &node_vel, + const DViewCArrayKokkos &elem_den, + const DViewCArrayKokkos &elem_sie, + const DViewCArrayKokkos &elem_pres, + const DViewCArrayKokkos &elem_stress, + const DViewCArrayKokkos &elem_sspd, + const DViewCArrayKokkos &elem_vol, + const DViewCArrayKokkos &elem_div, + const DViewCArrayKokkos &elem_mat_id, + const double rk_alpha, + const size_t cycle); + + void force_design_gradient_term(const_vec_array design_variables, vec_array design_gradients); + + + void get_force_sgh2D(const DCArrayKokkos &material, + const mesh_t &mesh, + const DViewCArrayKokkos &node_coords, + const DViewCArrayKokkos &node_vel, + const DViewCArrayKokkos &elem_den, + const DViewCArrayKokkos &elem_sie, + const DViewCArrayKokkos &elem_pres, + const DViewCArrayKokkos &elem_stress, + const DViewCArrayKokkos &elem_sspd, + const DViewCArrayKokkos &elem_vol, + const DViewCArrayKokkos &elem_div, + const DViewCArrayKokkos &elem_mat_id, + DViewCArrayKokkos &corner_force, const double rk_alpha, const size_t cycle); + + void update_position_sgh(double rk_alpha, + const size_t num_nodes, + DViewCArrayKokkos &node_coords, + const DViewCArrayKokkos &node_vel); + + void get_vol(); + + void get_vol_ugradient(const size_t gradient_node_id, const size_t gradient_dim); + + void init_assembly(); + + KOKKOS_INLINE_FUNCTION + void get_vol_hex(const DViewCArrayKokkos &elem_vol, + const size_t elem_gid, + const DViewCArrayKokkos &node_coords, + const ViewCArrayKokkos &elem_node_gids, + const size_t rk_level) const; + + KOKKOS_FUNCTION + void get_vol_hex_ugradient(const ViewCArrayKokkos &elem_vol_gradients, + const size_t elem_gid, + const DViewCArrayKokkos &node_coords, + const ViewCArrayKokkos &elem_node_gids, + const size_t rk_level) const; + + + KOKKOS_INLINE_FUNCTION + void get_vol_quad(const DViewCArrayKokkos &elem_vol, + const size_t elem_gid, + const DViewCArrayKokkos &node_coords, + const ViewCArrayKokkos &elem_node_gids, + const size_t rk_level) const; + + + KOKKOS_FUNCTION + double get_area_quad(const size_t elem_gid, + const DViewCArrayKokkos &node_coords, + const ViewCArrayKokkos &elem_node_gids, + const size_t rk_level) const; + + + KOKKOS_FUNCTION + void get_bmatrix(const ViewCArrayKokkos &B_matrix, + const size_t elem_gid, + const DViewCArrayKokkos &node_coords, + const ViewCArrayKokkos &elem_node_gids, + const size_t rk_level) const; + + KOKKOS_FUNCTION + void get_bmatrix_gradients(const ViewCArrayKokkos &B_matrix_gradients, + const size_t elem_gid, + const DViewCArrayKokkos &node_coords, + const ViewCArrayKokkos &elem_node_gids, + const size_t rk_level) const; - void get_force_vgradient_sgh(const DCArrayKokkos& material, - const mesh_t& mesh, - const DViewCArrayKokkos& node_coords, - const DViewCArrayKokkos& node_vel, - const DViewCArrayKokkos& elem_den, - const DViewCArrayKokkos& elem_sie, - const DViewCArrayKokkos& elem_pres, - const DViewCArrayKokkos& elem_stress, - const DViewCArrayKokkos& elem_sspd, - const DViewCArrayKokkos& elem_vol, - const DViewCArrayKokkos& elem_div, - const DViewCArrayKokkos& elem_mat_id, - const double rk_alpha, - const size_t cycle); - - void get_force_ugradient_sgh(const DCArrayKokkos& material, - const mesh_t& mesh, - const DViewCArrayKokkos& node_coords, - const DViewCArrayKokkos& node_vel, - const DViewCArrayKokkos& elem_den, - const DViewCArrayKokkos& elem_sie, - const DViewCArrayKokkos& elem_pres, - const DViewCArrayKokkos& elem_stress, - const DViewCArrayKokkos& elem_sspd, - const DViewCArrayKokkos& elem_vol, - const DViewCArrayKokkos& elem_div, - const DViewCArrayKokkos& elem_mat_id, - const double rk_alpha, - const size_t cycle); - - void get_force_egradient_sgh(const DCArrayKokkos& material, - const mesh_t& mesh, - const DViewCArrayKokkos& node_coords, - const DViewCArrayKokkos& node_vel, - const DViewCArrayKokkos& elem_den, - const DViewCArrayKokkos& elem_sie, - const DViewCArrayKokkos& elem_pres, - const DViewCArrayKokkos& elem_stress, - const DViewCArrayKokkos& elem_sspd, - const DViewCArrayKokkos& elem_vol, - const DViewCArrayKokkos& elem_div, - const DViewCArrayKokkos& elem_mat_id, - const double rk_alpha, - const size_t cycle); - - void get_force_dgradient_sgh(const DCArrayKokkos& material, - const mesh_t& mesh, - const DViewCArrayKokkos& node_coords, - const DViewCArrayKokkos& node_vel, - const DViewCArrayKokkos& elem_den, - const DViewCArrayKokkos& elem_sie, - const DViewCArrayKokkos& elem_pres, - const DViewCArrayKokkos& elem_stress, - const DViewCArrayKokkos& elem_sspd, - const DViewCArrayKokkos& elem_vol, - const DViewCArrayKokkos& elem_div, - const DViewCArrayKokkos& elem_mat_id, - const double rk_alpha, - const size_t cycle); - - void force_design_gradient_term(const_vec_array design_variables, vec_array design_gradients); - - void get_force_sgh2D(const DCArrayKokkos& material, - const mesh_t& mesh, - const DViewCArrayKokkos& node_coords, - const DViewCArrayKokkos& node_vel, - const DViewCArrayKokkos& elem_den, - const DViewCArrayKokkos& elem_sie, - const DViewCArrayKokkos& elem_pres, - const DViewCArrayKokkos& elem_stress, - const DViewCArrayKokkos& elem_sspd, - const DViewCArrayKokkos& elem_vol, - const DViewCArrayKokkos& elem_div, - const DViewCArrayKokkos& elem_mat_id, - DViewCArrayKokkos& corner_force, - const double rk_alpha, - const size_t cycle); - - void update_position_sgh(double rk_alpha, - const size_t num_nodes, - DViewCArrayKokkos& node_coords, - const DViewCArrayKokkos& node_vel); - - void get_vol(); - - void get_vol_ugradient(const size_t gradient_node_id, const size_t gradient_dim); - - void init_assembly(); - - KOKKOS_INLINE_FUNCTION - void get_vol_hex(const DViewCArrayKokkos& elem_vol, - const size_t elem_gid, - const DViewCArrayKokkos& node_coords, - const ViewCArrayKokkos& elem_node_gids, - const size_t rk_level) const; - KOKKOS_FUNCTION - void get_vol_hex_ugradient(const ViewCArrayKokkos& elem_vol_gradients, - const size_t elem_gid, - const DViewCArrayKokkos& node_coords, - const ViewCArrayKokkos& elem_node_gids, - const size_t rk_level) const; - - KOKKOS_INLINE_FUNCTION - void get_vol_quad(const DViewCArrayKokkos& elem_vol, - const size_t elem_gid, - const DViewCArrayKokkos& node_coords, - const ViewCArrayKokkos& elem_node_gids, - const size_t rk_level) const; - - KOKKOS_FUNCTION - double get_area_quad(const size_t elem_gid, - const DViewCArrayKokkos& node_coords, - const ViewCArrayKokkos& elem_node_gids, - const size_t rk_level) const; - - KOKKOS_FUNCTION - void get_bmatrix(const ViewCArrayKokkos& B_matrix, + KOKKOS_FUNCTION + void get_bmatrix2D(const ViewCArrayKokkos &B_matrix, const size_t elem_gid, - const DViewCArrayKokkos& node_coords, - const ViewCArrayKokkos& elem_node_gids, + const DViewCArrayKokkos &node_coords, + const ViewCArrayKokkos &elem_node_gids, const size_t rk_level) const; - KOKKOS_FUNCTION - void get_bmatrix_gradients(const ViewCArrayKokkos& B_matrix_gradients, - const size_t elem_gid, - const DViewCArrayKokkos& node_coords, - const ViewCArrayKokkos& elem_node_gids, - const size_t rk_level) const; - - KOKKOS_FUNCTION - void get_bmatrix2D(const ViewCArrayKokkos& B_matrix, - const size_t elem_gid, - const DViewCArrayKokkos& node_coords, - const ViewCArrayKokkos& elem_node_gids, - const size_t rk_level) const; + KOKKOS_FUNCTION + void get_area_weights2D(const ViewCArrayKokkos &corner_areas, + const size_t elem_gid, + const DViewCArrayKokkos &node_coords, + const ViewCArrayKokkos &elem_node_gids, + const size_t rk_level) const; + + + KOKKOS_INLINE_FUNCTION + double heron(const double x1, + const double y1, + const double x2, + const double y2, + const double x3, + const double y3) const; - KOKKOS_FUNCTION - void get_area_weights2D(const ViewCArrayKokkos& corner_areas, - const size_t elem_gid, - const DViewCArrayKokkos& node_coords, - const ViewCArrayKokkos& elem_node_gids, - const size_t rk_level) const; - - KOKKOS_INLINE_FUNCTION - double heron(const double x1, - const double y1, - const double x2, - const double y2, - const double x3, - const double y3) const; - - double average_element_density(const int nodes_per_elem, const CArray current_element_densities) const; - - void get_divergence(DViewCArrayKokkos& elem_div, - const DViewCArrayKokkos& node_coords, - const DViewCArrayKokkos& node_vel, - const DViewCArrayKokkos& elem_vol); - - void get_divergence2D(DViewCArrayKokkos& elem_div, - const DViewCArrayKokkos& node_coords, - const DViewCArrayKokkos& node_vel, - const DViewCArrayKokkos& elem_vol); - - KOKKOS_FUNCTION - void get_velgrad(ViewCArrayKokkos& vel_grad, - const ViewCArrayKokkos& elem_node_gids, - const DViewCArrayKokkos& node_vel, - const ViewCArrayKokkos& b_matrix, + double average_element_density(const int nodes_per_elem, const CArray current_element_densities) const; + + void get_divergence(DViewCArrayKokkos &elem_div, + const DViewCArrayKokkos &node_coords, + const DViewCArrayKokkos &node_vel, + const DViewCArrayKokkos &elem_vol); + + + void get_divergence2D(DViewCArrayKokkos &elem_div, + const DViewCArrayKokkos &node_coords, + const DViewCArrayKokkos &node_vel, + const DViewCArrayKokkos &elem_vol); + + + KOKKOS_FUNCTION + void get_velgrad(ViewCArrayKokkos &vel_grad, + const ViewCArrayKokkos &elem_node_gids, + const DViewCArrayKokkos &node_vel, + const ViewCArrayKokkos &b_matrix, + const double elem_vol, + const size_t elem_gid, + const size_t rk_level) const; + + + KOKKOS_FUNCTION + void get_velgrad2D(ViewCArrayKokkos &vel_grad, + const ViewCArrayKokkos &elem_node_gids, + const DViewCArrayKokkos &node_vel, + const ViewCArrayKokkos &b_matrix, const double elem_vol, + const double elem_area, const size_t elem_gid, const size_t rk_level) const; - KOKKOS_FUNCTION - void get_velgrad2D(ViewCArrayKokkos& vel_grad, - const ViewCArrayKokkos& elem_node_gids, - const DViewCArrayKokkos& node_vel, - const ViewCArrayKokkos& b_matrix, - const double elem_vol, - const double elem_area, - const size_t elem_gid, - const size_t rk_level) const; - - KOKKOS_INLINE_FUNCTION - void decompose_vel_grad(ViewCArrayKokkos& D_tensor, - ViewCArrayKokkos& W_tensor, - const ViewCArrayKokkos& vel_grad, - const ViewCArrayKokkos& elem_node_gids, - const size_t elem_gid, - const DViewCArrayKokkos& node_coords, - const DViewCArrayKokkos& node_vel, - const double vol) const; - - void update_velocity_sgh(double rk_alpha, - DViewCArrayKokkos& node_vel, - const DViewCArrayKokkos& node_mass, - const DViewCArrayKokkos& corner_force); - - void tag_bdys(const DCArrayKokkos& boundary, - mesh_t& mesh, - const DViewCArrayKokkos& node_coords); - - void boundary_velocity(const mesh_t& mesh, - const DCArrayKokkos& boundary, - DViewCArrayKokkos& node_vel); - - KOKKOS_INLINE_FUNCTION - bool check_bdy(const size_t patch_gid, - const int num_dim, - const int num_nodes_in_patch, + KOKKOS_INLINE_FUNCTION + void decompose_vel_grad(ViewCArrayKokkos &D_tensor, + ViewCArrayKokkos &W_tensor, + const ViewCArrayKokkos &vel_grad, + const ViewCArrayKokkos &elem_node_gids, + const size_t elem_gid, + const DViewCArrayKokkos &node_coords, + const DViewCArrayKokkos &node_vel, + const double vol) const; + + + void update_velocity_sgh(double rk_alpha, + DViewCArrayKokkos &node_vel, + const DViewCArrayKokkos &node_mass, + const DViewCArrayKokkos &corner_force); + + void tag_bdys(const DCArrayKokkos &boundary, + mesh_t &mesh, + const DViewCArrayKokkos &node_coords); + + void boundary_velocity(const mesh_t &mesh, + const DCArrayKokkos &boundary, + DViewCArrayKokkos &node_vel); + + KOKKOS_INLINE_FUNCTION + bool check_bdy(const size_t patch_gid, + const int num_dim, + const int num_nodes_in_patch, const BOUNDARY_TYPE this_bc_tag, const double val, - const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos &node_coords, const size_t rk_level) const; - void rk_init(DViewCArrayKokkos& node_coords, - DViewCArrayKokkos& node_vel, - DViewCArrayKokkos& elem_sie, - DViewCArrayKokkos& elem_stress, - const size_t num_elems, - const size_t num_nodes); - - void get_timestep(mesh_t& mesh, - DViewCArrayKokkos& node_coords, - DViewCArrayKokkos& node_vel, - DViewCArrayKokkos& elem_sspd, - DViewCArrayKokkos& elem_vol); - - void get_timestep2D(mesh_t& mesh, - DViewCArrayKokkos& node_coords, - DViewCArrayKokkos& node_vel, - DViewCArrayKokkos& elem_sspd, - DViewCArrayKokkos& elem_vol); - - void update_energy_sgh(double rk_alpha, - const mesh_t& mesh, - const DViewCArrayKokkos& node_vel, - const DViewCArrayKokkos& node_coords, - DViewCArrayKokkos& elem_sie, - const DViewCArrayKokkos& elem_mass, - const DViewCArrayKokkos& corner_force); - - void power_design_gradient_term(const_vec_array design_variables, vec_array design_gradients); - - void get_power_dgradient_sgh(double rk_alpha, - const mesh_t& mesh, - const DViewCArrayKokkos& node_vel, - const DViewCArrayKokkos& node_coords, - DViewCArrayKokkos& elem_sie, - const DViewCArrayKokkos& elem_mass, - const DViewCArrayKokkos& corner_force, - DCArrayKokkos elem_power_dgradients); - - void get_power_ugradient_sgh(double rk_alpha, - const mesh_t& mesh, - const DViewCArrayKokkos& node_vel, - const DViewCArrayKokkos& node_coords, - DViewCArrayKokkos& elem_sie, - const DViewCArrayKokkos& elem_mass, - const DViewCArrayKokkos& corner_force); - - void get_power_vgradient_sgh(double rk_alpha, - const mesh_t& mesh, - const DViewCArrayKokkos& node_vel, - const DViewCArrayKokkos& node_coords, - DViewCArrayKokkos& elem_sie, - const DViewCArrayKokkos& elem_mass, - const DViewCArrayKokkos& corner_force); - - void get_power_egradient_sgh(double rk_alpha, - const mesh_t& mesh, - const DViewCArrayKokkos& node_vel, - const DViewCArrayKokkos& node_coords, - DViewCArrayKokkos& elem_sie, - const DViewCArrayKokkos& elem_mass, - const DViewCArrayKokkos& corner_force); - - void update_state(const DCArrayKokkos& material, - const mesh_t& mesh, - const DViewCArrayKokkos& node_coords, - const DViewCArrayKokkos& node_vel, - DViewCArrayKokkos& elem_den, - DViewCArrayKokkos& elem_pres, - DViewCArrayKokkos& elem_stress, - DViewCArrayKokkos& elem_sspd, - const DViewCArrayKokkos& elem_sie, - const DViewCArrayKokkos& elem_vol, - const DViewCArrayKokkos& elem_mass, - const DViewCArrayKokkos& elem_mat_id, + void rk_init(DViewCArrayKokkos &node_coords, + DViewCArrayKokkos &node_vel, + DViewCArrayKokkos &elem_sie, + DViewCArrayKokkos &elem_stress, + const size_t num_elems, + const size_t num_nodes); + + + void get_timestep(mesh_t &mesh, + DViewCArrayKokkos &node_coords, + DViewCArrayKokkos &node_vel, + DViewCArrayKokkos &elem_sspd, + DViewCArrayKokkos &elem_vol); + + + void get_timestep2D(mesh_t &mesh, + DViewCArrayKokkos &node_coords, + DViewCArrayKokkos &node_vel, + DViewCArrayKokkos &elem_sspd, + DViewCArrayKokkos &elem_vol); + + void update_energy_sgh(double rk_alpha, + const mesh_t &mesh, + const DViewCArrayKokkos &node_vel, + const DViewCArrayKokkos &node_coords, + DViewCArrayKokkos &elem_sie, + const DViewCArrayKokkos &elem_mass, + const DViewCArrayKokkos &corner_force); + + + void power_design_gradient_term(const_vec_array design_variables, vec_array design_gradients); + + void get_power_dgradient_sgh(double rk_alpha, + const mesh_t &mesh, + const DViewCArrayKokkos &node_vel, + const DViewCArrayKokkos &node_coords, + DViewCArrayKokkos &elem_sie, + const DViewCArrayKokkos &elem_mass, + const DViewCArrayKokkos &corner_force, + DCArrayKokkos elem_power_dgradients); + + void get_power_ugradient_sgh(double rk_alpha, + const mesh_t &mesh, + const DViewCArrayKokkos &node_vel, + const DViewCArrayKokkos &node_coords, + DViewCArrayKokkos &elem_sie, + const DViewCArrayKokkos &elem_mass, + const DViewCArrayKokkos &corner_force); + + void get_power_vgradient_sgh(double rk_alpha, + const mesh_t &mesh, + const DViewCArrayKokkos &node_vel, + const DViewCArrayKokkos &node_coords, + DViewCArrayKokkos &elem_sie, + const DViewCArrayKokkos &elem_mass, + const DViewCArrayKokkos &corner_force); + + void get_power_egradient_sgh(double rk_alpha, + const mesh_t &mesh, + const DViewCArrayKokkos &node_vel, + const DViewCArrayKokkos &node_coords, + DViewCArrayKokkos &elem_sie, + const DViewCArrayKokkos &elem_mass, + const DViewCArrayKokkos &corner_force); + + void update_state(const DCArrayKokkos &material, + const mesh_t &mesh, + const DViewCArrayKokkos &node_coords, + const DViewCArrayKokkos &node_vel, + DViewCArrayKokkos &elem_den, + DViewCArrayKokkos &elem_pres, + DViewCArrayKokkos &elem_stress, + DViewCArrayKokkos &elem_sspd, + const DViewCArrayKokkos &elem_sie, + const DViewCArrayKokkos &elem_vol, + const DViewCArrayKokkos &elem_mass, + const DViewCArrayKokkos &elem_mat_id, + const double rk_alpha, + const size_t cycle); + + + void update_state2D(const DCArrayKokkos &material, + const mesh_t &mesh, + const DViewCArrayKokkos &node_coords, + const DViewCArrayKokkos &node_vel, + DViewCArrayKokkos &elem_den, + DViewCArrayKokkos &elem_pres, + DViewCArrayKokkos &elem_stress, + DViewCArrayKokkos &elem_sspd, + const DViewCArrayKokkos &elem_sie, + const DViewCArrayKokkos &elem_vol, + const DViewCArrayKokkos &elem_mass, + const DViewCArrayKokkos &elem_mat_id, const double rk_alpha, const size_t cycle); - void update_state2D(const DCArrayKokkos& material, - const mesh_t& mesh, - const DViewCArrayKokkos& node_coords, - const DViewCArrayKokkos& node_vel, - DViewCArrayKokkos& elem_den, - DViewCArrayKokkos& elem_pres, - DViewCArrayKokkos& elem_stress, - DViewCArrayKokkos& elem_sspd, - const DViewCArrayKokkos& elem_sie, - const DViewCArrayKokkos& elem_vol, - const DViewCArrayKokkos& elem_mass, - const DViewCArrayKokkos& elem_mat_id, - const double rk_alpha, - const size_t cycle); - - void build_boundry_node_sets(mesh_t& mesh); - - void init_boundaries(); - - // initializes memory for arrays used in the global stiffness matrix assembly - void init_boundary_sets(int num_boundary_sets); - - void grow_boundary_sets(int num_boundary_sets); - - virtual void update_forward_solve(Teuchos::RCP zp); - - void comm_node_masses(); - - void comm_adjoint_vector(int cycle); - - void comm_phi_adjoint_vector(int cycle); - - void comm_variables(Teuchos::RCP zp); - - void read_conditions_ansys_dat(std::ifstream* in, std::streampos before_condition_header); - - // interfaces between user input and creating data structures for bcs - void generate_bcs(); - - void Displacement_Boundary_Conditions(); - - void init_output(); - - void compute_output(); - - void output_control(); - - void sort_output(Teuchos::RCP> sorted_map); - - void sort_element_output(Teuchos::RCP> sorted_map); - - void collect_output(Teuchos::RCP> global_reduce_map); - - void write_data(std::map& point_data_scalars_double, - std::map& point_data_vectors_double, - std::map& cell_data_scalars_double, - std::map& cell_data_scalars_int, - std::map>& cell_data_fields_double); - - void write_outputs(const mesh_t& mesh, - DViewCArrayKokkos& node_coords, - DViewCArrayKokkos& node_vel, - DViewCArrayKokkos& node_mass, - DViewCArrayKokkos& elem_den, - DViewCArrayKokkos& elem_pres, - DViewCArrayKokkos& elem_stress, - DViewCArrayKokkos& elem_sspd, - DViewCArrayKokkos& elem_sie, - DViewCArrayKokkos& elem_vol, - DViewCArrayKokkos& elem_mass, - DViewCArrayKokkos& elem_mat_id); + void build_boundry_node_sets(mesh_t &mesh); + + void init_boundaries(); - void ensight(const mesh_t& mesh, - const DViewCArrayKokkos& node_coords, - const DViewCArrayKokkos& node_vel, - const DViewCArrayKokkos& node_mass, - const DViewCArrayKokkos& elem_den, - const DViewCArrayKokkos& elem_pres, - const DViewCArrayKokkos& elem_stress, - const DViewCArrayKokkos& elem_sspd, - const DViewCArrayKokkos& elem_sie, - const DViewCArrayKokkos& elem_vol, - const DViewCArrayKokkos& elem_mass, - const DViewCArrayKokkos& elem_mat_id); - - void state_file(const mesh_t& mesh, - const DViewCArrayKokkos& node_coords, - const DViewCArrayKokkos& node_vel, - const DViewCArrayKokkos& node_mass, - const DViewCArrayKokkos& elem_den, - const DViewCArrayKokkos& elem_pres, - const DViewCArrayKokkos& elem_stress, - const DViewCArrayKokkos& elem_sspd, - const DViewCArrayKokkos& elem_sie, - const DViewCArrayKokkos& elem_vol, - const DViewCArrayKokkos& elem_mass, - const DViewCArrayKokkos& elem_mat_id); - - void node_density_constraints(host_vec_array node_densities_lower_bound); - - void compute_topology_optimization_adjoint_full(); // Force depends on node coords and velocity - - void compute_topology_optimization_gradient_full(Teuchos::RCP design_densities_distributed, Teuchos::RCP design_gradients_distributed); - - void boundary_adjoint(const mesh_t& mesh, - const DCArrayKokkos& boundary, - vec_array& node_adjoint, - vec_array& node_phi_adjoint, - vec_array& node_psi_adjoint); - - void applied_forces(const DCArrayKokkos& material, - const mesh_t& mesh, - const DViewCArrayKokkos& node_coords, - const DViewCArrayKokkos& node_vel, - const DViewCArrayKokkos& node_mass, - const DViewCArrayKokkos& elem_den, - const DViewCArrayKokkos& elem_vol, - const DViewCArrayKokkos& elem_div, - const DViewCArrayKokkos& elem_mat_id, - DViewCArrayKokkos& corner_force, - const double rk_alpha, - const size_t cycle); - - bool have_loading_conditions; - bool nodal_density_flag; - real_t penalty_power; - - Simulation_Parameters_Explicit* simparam; - SGH_Parameters* module_params; - Explicit_Solver* Explicit_Solver_Pointer_; - - elements::ref_element* ref_elem; - - std::shared_ptr mesh; - // shallow copies of mesh class views - size_t num_nodes_in_elem; - // corner ids in node - RaggedRightArrayKokkos corners_in_node; - CArrayKokkos num_corners_in_node; - - // elem ids in node - RaggedRightArrayKokkos elems_in_node; - - // node ids in node - RaggedRightArrayKokkos nodes_in_node; - CArrayKokkos num_nodes_in_node; - - // node ids in elem - DCArrayKokkos nodes_in_elem; - - // corner ids in elem - CArrayKokkos corners_in_elem; - - // elem ids in elem - RaggedRightArrayKokkos elems_in_elem; - CArrayKokkos num_elems_in_elem; - - // patch ids in elem - CArrayKokkos patches_in_elem; - - // node ids in a patch - CArrayKokkos nodes_in_patch; - - // element ids in a patch - CArrayKokkos elems_in_patch; - - // bdy nodes - CArrayKokkos bdy_nodes; - - // Topology optimization filter variable - DCArrayKokkos relative_element_densities; - - // Local FEA data - host_elem_conn_array interface_nodes_in_elem; // host view of element connectivity to nodes - - // Global FEA data - Teuchos::RCP node_velocities_distributed; - Teuchos::RCP initial_node_coords_distributed; - Teuchos::RCP all_initial_node_coords_distributed; - Teuchos::RCP initial_node_velocities_distributed; - Teuchos::RCP all_node_velocities_distributed; - Teuchos::RCP all_cached_node_velocities_distributed; - Teuchos::RCP node_masses_distributed; - Teuchos::RCP ghost_node_masses_distributed; - Teuchos::RCP adjoint_vector_distributed; - Teuchos::RCP phi_adjoint_vector_distributed; - Teuchos::RCP psi_adjoint_vector_distributed; - Teuchos::RCP element_internal_energy_distributed; - Teuchos::RCP>> forward_solve_velocity_data; - Teuchos::RCP>> forward_solve_coordinate_data; - Teuchos::RCP>> forward_solve_internal_energy_data; - Teuchos::RCP>> adjoint_vector_data; - Teuchos::RCP>> phi_adjoint_vector_data; - Teuchos::RCP>> psi_adjoint_vector_data; - Teuchos::RCP force_gradient_design; - Teuchos::RCP force_gradient_position; - Teuchos::RCP force_gradient_velocity; - - // Local FEA data - DCArrayKokkos Global_Gradient_Matrix_Assembly_Map; // Maps element local nodes to columns on ragged right node connectivity graph - DCArrayKokkos Element_Gradient_Matrix_Assembly_Map; // Maps element-node pair to columns on ragged right node to element connectivity - RaggedRightArrayKokkos Graph_Matrix; // stores global indices - RaggedRightArrayKokkos DOF_Graph_Matrix; // stores global indices - RaggedRightArrayKokkos Force_Gradient_Positions; - RaggedRightArrayKokkos Force_Gradient_Velocities; - CArrayKokkos Force_Gradient_Energies; // transposed such that elem ids correspond to rows - RaggedRightArrayKokkos Power_Gradient_Positions; // transposed such that node dofs correspond to rows - RaggedRightArrayKokkos Power_Gradient_Velocities; // transposed such that node dofs correspond to rows - - CArrayKokkos Power_Gradient_Energies; - DCArrayKokkos Gradient_Matrix_Strides; - DCArrayKokkos DOF_to_Elem_Matrix_Strides; - DCArrayKokkos Elem_to_Elem_Matrix_Strides; - DCArrayKokkos Graph_Matrix_Strides; - - RaggedRightArrayKokkos Original_Gradient_Entries; - RaggedRightArrayKokkos Original_Gradient_Entry_Indices; - DCArrayKokkos Original_Gradient_Entries_Strides; - - // distributed matrices - Teuchos::RCP distributed_force_gradient_positions; - Teuchos::RCP distributed_force_gradient_velocities; - - std::vector time_data; - int max_time_steps, last_time_step; - - // --------------------------------------------------------------------- - // state data type declarations (must stay in scope for output after run) - // --------------------------------------------------------------------- - node_t node_interface; - elem_t elem_interface; - corner_t corner_interface; - - // Dual View wrappers - // Dual Views of the individual node struct variables - DViewCArrayKokkos node_coords; - DViewCArrayKokkos node_vel; - DViewCArrayKokkos node_mass; - - // Dual Views of the individual elem struct variables - DViewCArrayKokkos elem_den; - DViewCArrayKokkos elem_pres; - DViewCArrayKokkos elem_stress; // always 3D even in 2D-RZ - DViewCArrayKokkos elem_sspd; - DViewCArrayKokkos elem_sie; - DViewCArrayKokkos elem_vol; - DViewCArrayKokkos elem_div; - DViewCArrayKokkos elem_mass; - DViewCArrayKokkos elem_mat_id; - - // Element velocity gradient - DCArrayKokkos elem_vel_grad; - - // for storing global variables used in user material model - DCArrayKokkos global_vars; - - // for storing state variables used in user material model - DCArrayKokkos state_vars; - - // elem_user_output_vars allow users to output variables of interest per element - DCArrayKokkos elem_user_output_vars; - - // material models - DCArrayKokkos elem_eos; - DCArrayKokkos elem_strength; - - // Dual Views of the corner struct variables - DViewCArrayKokkos corner_force; - DViewCArrayKokkos corner_mass; - - // Boundary Conditions Data - DCArrayKokkos Local_Index_Boundary_Patches; - // CArray Patch_Nodes; - enum bc_type { NONE, POINT_LOADING_CONDITION, LINE_LOADING_CONDITION, SURFACE_LOADING_CONDITION }; - - // Boundary Conditions Data - int max_boundary_sets; - - // output dof data - // Global arrays with collected data used to print - int output_velocity_index, output_strain_index, output_stress_index; - - // parameters - double time_value, time_final, dt, dt_max, dt_min, dt_cfl, graphics_time, graphics_dt_ival; - size_t graphics_cyc_ival, cycle_stop, rk_num_stages, graphics_id; - double fuzz, tiny, small; - CArray graphics_times; - int rk_num_bins; - - // optimization flags - bool kinetic_energy_objective; + //initializes memory for arrays used in the global stiffness matrix assembly + void init_boundary_sets(int num_boundary_sets); + + void grow_boundary_sets(int num_boundary_sets); + + virtual void update_forward_solve(Teuchos::RCP zp); + + void comm_node_masses(); + + void comm_adjoint_vector(int cycle); + + void comm_phi_adjoint_vector(int cycle); + + void comm_variables(Teuchos::RCP zp); + + void read_conditions_ansys_dat(std::ifstream *in, std::streampos before_condition_header); + + //interfaces between user input and creating data structures for bcs + void generate_bcs(); + + void Displacement_Boundary_Conditions(); + + void init_output(); + + void compute_output(); + + void output_control(); + + void sort_output(Teuchos::RCP > sorted_map); + + void sort_element_output(Teuchos::RCP > sorted_map); + + void collect_output(Teuchos::RCP > global_reduce_map); + + void write_data(std::map &point_data_scalars_double, + std::map &point_data_vectors_double, + std::map &cell_data_scalars_double, + std::map &cell_data_scalars_int, + std::map > &cell_data_fields_double); + + void write_outputs (const mesh_t &mesh, + DViewCArrayKokkos &node_coords, + DViewCArrayKokkos &node_vel, + DViewCArrayKokkos &node_mass, + DViewCArrayKokkos &elem_den, + DViewCArrayKokkos &elem_pres, + DViewCArrayKokkos &elem_stress, + DViewCArrayKokkos &elem_sspd, + DViewCArrayKokkos &elem_sie, + DViewCArrayKokkos &elem_vol, + DViewCArrayKokkos &elem_mass, + DViewCArrayKokkos &elem_mat_id); + + + void ensight(const mesh_t &mesh, + const DViewCArrayKokkos &node_coords, + const DViewCArrayKokkos &node_vel, + const DViewCArrayKokkos &node_mass, + const DViewCArrayKokkos &elem_den, + const DViewCArrayKokkos &elem_pres, + const DViewCArrayKokkos &elem_stress, + const DViewCArrayKokkos &elem_sspd, + const DViewCArrayKokkos &elem_sie, + const DViewCArrayKokkos &elem_vol, + const DViewCArrayKokkos &elem_mass, + const DViewCArrayKokkos &elem_mat_id); + + + void state_file(const mesh_t &mesh, + const DViewCArrayKokkos &node_coords, + const DViewCArrayKokkos &node_vel, + const DViewCArrayKokkos &node_mass, + const DViewCArrayKokkos &elem_den, + const DViewCArrayKokkos &elem_pres, + const DViewCArrayKokkos &elem_stress, + const DViewCArrayKokkos &elem_sspd, + const DViewCArrayKokkos &elem_sie, + const DViewCArrayKokkos &elem_vol, + const DViewCArrayKokkos &elem_mass, + const DViewCArrayKokkos &elem_mat_id); + + void node_density_constraints(host_vec_array node_densities_lower_bound); + + void compute_topology_optimization_adjoint_full(); //Force depends on node coords and velocity + + void compute_topology_optimization_gradient_full(Teuchos::RCP design_densities_distributed, Teuchos::RCP design_gradients_distributed); + + void boundary_adjoint(const mesh_t &mesh, + const DCArrayKokkos &boundary, + vec_array &node_adjoint, + vec_array &node_phi_adjoint, + vec_array &node_psi_adjoint); + + void applied_forces(const DCArrayKokkos &material, + const mesh_t &mesh, + const DViewCArrayKokkos &node_coords, + const DViewCArrayKokkos &node_vel, + const DViewCArrayKokkos &node_mass, + const DViewCArrayKokkos &elem_den, + const DViewCArrayKokkos &elem_vol, + const DViewCArrayKokkos &elem_div, + const DViewCArrayKokkos &elem_mat_id, + DViewCArrayKokkos &corner_force, + const double rk_alpha, + const size_t cycle); + + bool have_loading_conditions; + bool nodal_density_flag; + real_t penalty_power; + + Simulation_Parameters_Explicit *simparam; + SGH_Parameters *module_params; + Explicit_Solver *Explicit_Solver_Pointer_; + + elements::ref_element *ref_elem; + + std::shared_ptr mesh; + //shallow copies of mesh class views + size_t num_nodes_in_elem; + // corner ids in node + RaggedRightArrayKokkos corners_in_node; + CArrayKokkos num_corners_in_node; + + // elem ids in node + RaggedRightArrayKokkos elems_in_node; + + // node ids in node + RaggedRightArrayKokkos nodes_in_node; + CArrayKokkos num_nodes_in_node; + + // node ids in elem + DCArrayKokkos nodes_in_elem; + + // corner ids in elem + CArrayKokkos corners_in_elem; + + // elem ids in elem + RaggedRightArrayKokkos elems_in_elem; + CArrayKokkos num_elems_in_elem; + + // patch ids in elem + CArrayKokkos patches_in_elem; + + // node ids in a patch + CArrayKokkos nodes_in_patch; + + // element ids in a patch + CArrayKokkos elems_in_patch; + + // bdy nodes + CArrayKokkos bdy_nodes; + + //Topology optimization filter variable + DCArrayKokkos relative_element_densities; + + //Local FEA data + host_elem_conn_array interface_nodes_in_elem; //host view of element connectivity to nodes + + //Global FEA data + Teuchos::RCP node_velocities_distributed; + Teuchos::RCP initial_node_coords_distributed; + Teuchos::RCP all_initial_node_coords_distributed; + Teuchos::RCP initial_node_velocities_distributed; + Teuchos::RCP all_node_velocities_distributed; + Teuchos::RCP all_cached_node_velocities_distributed; + Teuchos::RCP node_masses_distributed; + Teuchos::RCP ghost_node_masses_distributed; + Teuchos::RCP adjoint_vector_distributed; + Teuchos::RCP phi_adjoint_vector_distributed; + Teuchos::RCP psi_adjoint_vector_distributed; + Teuchos::RCP element_internal_energy_distributed; + Teuchos::RCP>> forward_solve_velocity_data; + Teuchos::RCP>> forward_solve_coordinate_data; + Teuchos::RCP>> forward_solve_internal_energy_data; + Teuchos::RCP>> adjoint_vector_data; + Teuchos::RCP>> phi_adjoint_vector_data; + Teuchos::RCP>> psi_adjoint_vector_data; + Teuchos::RCP force_gradient_design; + Teuchos::RCP force_gradient_position; + Teuchos::RCP force_gradient_velocity; + + //Local FEA data + DCArrayKokkos Global_Gradient_Matrix_Assembly_Map; //Maps element local nodes to columns on ragged right node connectivity graph + DCArrayKokkos Element_Gradient_Matrix_Assembly_Map; //Maps element-node pair to columns on ragged right node to element connectivity + RaggedRightArrayKokkos Graph_Matrix; //stores local indices + RaggedRightArrayKokkos DOF_Graph_Matrix; //stores local indices + RaggedRightArrayKokkos Force_Gradient_Positions; + RaggedRightArrayKokkos Force_Gradient_Velocities; + CArrayKokkos Force_Gradient_Energies; //transposed such that elem ids correspond to rows + RaggedRightArrayKokkos Power_Gradient_Positions; //transposed such that node dofs correspond to rows + RaggedRightArrayKokkos Power_Gradient_Velocities; //transposed such that node dofs correspond to rows + CArrayKokkos Power_Gradient_Energies; + DCArrayKokkos Gradient_Matrix_Strides; + DCArrayKokkos DOF_to_Elem_Matrix_Strides; + DCArrayKokkos Elem_to_Elem_Matrix_Strides; + DCArrayKokkos Graph_Matrix_Strides; + RaggedRightArrayKokkos Original_Gradient_Entries; + RaggedRightArrayKokkos Original_Gradient_Entry_Indices; + DCArrayKokkos Original_Gradient_Entries_Strides; + + //distributed matrices + Teuchos::RCP distributed_force_gradient_positions; + Teuchos::RCP distributed_force_gradient_velocities; + + std::vector time_data; + int max_time_steps, last_time_step; + + // --------------------------------------------------------------------- + // state data type declarations (must stay in scope for output after run) + // --------------------------------------------------------------------- + node_t node_interface; + elem_t elem_interface; + corner_t corner_interface; + + //Dual View wrappers + // Dual Views of the individual node struct variables + DViewCArrayKokkos node_coords; + DViewCArrayKokkos node_vel; + DViewCArrayKokkos node_mass; + + // Dual Views of the individual elem struct variables + DViewCArrayKokkos elem_den; + DViewCArrayKokkos elem_pres; + DViewCArrayKokkos elem_stress; // always 3D even in 2D-RZ + DViewCArrayKokkos elem_sspd; + DViewCArrayKokkos elem_sie; + DViewCArrayKokkos elem_vol; + DViewCArrayKokkos elem_div; + DViewCArrayKokkos elem_mass; + DViewCArrayKokkos elem_mat_id; + + // Element velocity gradient + DCArrayKokkos elem_vel_grad; + + // for storing global variables used in user material model + DCArrayKokkos eos_global_vars; + DCArrayKokkos strength_global_vars; + + // for storing state variables used in user material model + DCArrayKokkos eos_state_vars; + DCArrayKokkos strength_state_vars; + + + //elem_user_output_vars allow users to output variables of interest per element + DCArrayKokkos elem_user_output_vars; + + //material models + DCArrayKokkos elem_eos; + DCArrayKokkos elem_strength; + + // Dual Views of the corner struct variables + DViewCArrayKokkos corner_force; + DViewCArrayKokkos corner_mass; + + //Boundary Conditions Data + DCArrayKokkos Local_Index_Boundary_Patches; + //CArray Patch_Nodes; + enum bc_type {NONE, POINT_LOADING_CONDITION, LINE_LOADING_CONDITION, SURFACE_LOADING_CONDITION}; + + //Boundary Conditions Data + int max_boundary_sets; + + //output dof data + //Global arrays with collected data used to print + int output_velocity_index, output_strain_index, output_stress_index; + + //parameters + double time_value, time_final, dt, dt_max, dt_min, dt_cfl, graphics_time, graphics_dt_ival; + size_t graphics_cyc_ival, cycle_stop, rk_num_stages, graphics_id; + double fuzz, tiny, small; + CArray graphics_times; + int rk_num_bins; + + //optimization flags + bool kinetic_energy_objective; }; #endif // end HEADER_H diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/FEA_Module_SGH.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/FEA_Module_SGH.cpp index b82b243b5..967cf01a4 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/FEA_Module_SGH.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/FEA_Module_SGH.cpp @@ -623,16 +623,18 @@ void FEA_Module_SGH::module_cleanup() Deallocate memory used for material models ------------------------------------------------------------------------------- */ -void FEA_Module_SGH::cleanup_material_models() -{ - const DCArrayKokkos material = simparam->material; +void FEA_Module_SGH::cleanup_material_models() { + + const DCArrayKokkos material = simparam->material; // destroy strength model destroy_strength_model(elem_strength, material, elem_mat_id, - state_vars, - global_vars, + eos_state_vars, + strength_state_vars, + eos_global_vars, + strength_global_vars, elem_user_output_vars, rnum_elem); @@ -640,11 +642,14 @@ void FEA_Module_SGH::cleanup_material_models() destroy_eos_model(elem_eos, material, elem_mat_id, - state_vars, - global_vars, + eos_state_vars, + strength_state_vars, + eos_global_vars, + strength_global_vars, elem_user_output_vars, rnum_elem); return; + } // end cleanup_user_strength_model; /** diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/boundary.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/boundary.cpp index 2107eeb46..e65bec3d2 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/boundary.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/boundary.cpp @@ -36,6 +36,22 @@ #include "FEA_Module_SGH.h" #include "Simulation_Parameters/FEA_Module/Boundary_Conditions.h" +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn boundary_velocity +/// +/// \brief +/// +/// +/// +/// \param +/// \param +/// \param +/// +/// \return +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_SGH::boundary_velocity(const mesh_t& mesh, const DCArrayKokkos& boundary, DViewCArrayKokkos& node_vel) diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/energy_sgh.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/energy_sgh.cpp index a22d68310..9d649d6ec 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/energy_sgh.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/energy_sgh.cpp @@ -35,6 +35,22 @@ #include "state.h" #include "FEA_Module_SGH.h" +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn update_energy_sgh +/// +/// \brief +/// +/// +/// +/// \param +/// \param +/// \param +/// +/// \return +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_SGH::update_energy_sgh(double rk_alpha, const mesh_t& mesh, const DViewCArrayKokkos& node_vel, diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp index d87a86759..3b869c123 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp @@ -110,28 +110,28 @@ void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& ma const size_t num_nodes_in_elem = 8; real_t gradient_result[num_dims]; - /// total Cauchy stress + // total Cauchy stress double tau_array[9]; double tau_gradient_array[9]; - /// corner area normals + // corner area normals double area_normal_array[24]; - /// estimate of shock direction + // estimate of shock direction double shock_dir_array[3]; - /// the sums in the Riemann solver + // the sums in the Riemann solver double sum_array[4]; - /// corner shock impeadance x |corner area normal dot shock_dir| + // corner shock impeadance x |corner area normal dot shock_dir| double muc_array[8]; double muc_gradient_array[8]; - /// Riemann velocity + // Riemann velocity double vel_star_array[3]; double vel_star_gradient_array[3 * num_nodes_in_elem]; - /// velocity gradient + // velocity gradient double vel_grad_array[9]; // --- Create views of arrays to aid the force calculation --- @@ -147,8 +147,6 @@ void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& ma ViewCArrayKokkos vel_star_gradient(vel_star_gradient_array, num_nodes_in_elem, num_dims); ViewCArrayKokkos vel_grad(vel_grad_array, num_dims, num_dims); - EOSParent* eos_model = elem_eos(elem_gid).model; - // --- abviatations of variables --- // element volume @@ -312,7 +310,7 @@ void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& ma mu_term = muc(node_lid) * fabs(shock_dir(0) * area_normal(node_lid, 0) + shock_dir(1) * area_normal(node_lid, 1) - + shock_dir(2) * area_normal(node_lid, 2) ); // code gradient for shock dir w.r.t velocity if using shock_dir + + shock_dir(2) * area_normal(node_lid, 2) ); // code gradient for shock dir w.r.t velocity if using shock_dir } else { @@ -572,8 +570,6 @@ void FEA_Module_SGH::get_force_egradient_sgh(const DCArrayKokkos& ma ViewCArrayKokkos vel_star_gradient(vel_star_gradient_array, num_dims); ViewCArrayKokkos vel_grad(vel_grad_array, num_dims, num_dims); - EOSParent* eos_model = elem_eos(elem_gid).model; - // --- abviatations of variables --- // element volume @@ -638,12 +634,14 @@ void FEA_Module_SGH::get_force_egradient_sgh(const DCArrayKokkos& ma for (int i = 0; i < num_dims; i++) { tau(i, i) -= elem_pres(elem_gid); - tau_gradient(i, i) = -eos_model->calc_pressure_gradient_internal_energy(elem_pres, + tau_gradient(i, i) = -elem_eos(elem_gid).calc_pressure_gradient_internal_energy(elem_pres, elem_stress, elem_gid, elem_mat_id(elem_gid), - state_vars, - global_vars, + eos_state_vars, + strength_state_vars, + eos_global_vars, + strength_global_vars, elem_user_output_vars, elem_sspd, elem_den(elem_gid), @@ -687,12 +685,14 @@ void FEA_Module_SGH::get_force_egradient_sgh(const DCArrayKokkos& ma // --- Sound speed --- real_t sound_speed_gradient_energy = - eos_model->calc_sound_speed_gradient_internal_energy(elem_pres, + elem_eos(elem_gid).calc_sound_speed_gradient_internal_energy(elem_pres, elem_stress, elem_gid, elem_mat_id(elem_gid), - state_vars, - global_vars, + eos_state_vars, + strength_state_vars, + eos_global_vars, + strength_global_vars, elem_user_output_vars, elem_sspd, elem_den(elem_gid), @@ -956,7 +956,6 @@ void FEA_Module_SGH::get_force_ugradient_sgh(const DCArrayKokkos& ma } }); // end parallel for loop over nodes Kokkos::fence(); - // --- calculate the forces acting on the nodes from the element --- for (size_t elem_gid = 0; elem_gid < rnum_elem; elem_gid++) { @@ -981,7 +980,7 @@ void FEA_Module_SGH::get_force_ugradient_sgh(const DCArrayKokkos& ma // the sums in the Riemann solver double sum_array[4]; - double sum_gradient_array[4 * max_nodes_per_element * num_dim]; + double sum_gradient_array[4 * max_nodes_per_element * num_dims]; // corner shock impeadance x |corner area normal dot shock_dir| double muc_array[8]; @@ -1010,8 +1009,6 @@ void FEA_Module_SGH::get_force_ugradient_sgh(const DCArrayKokkos& ma ViewCArrayKokkos vel_star_gradient(vel_star_gradient_array, num_dims, num_nodes_in_elem, num_dims); ViewCArrayKokkos vel_grad(vel_grad_array, num_dims, num_dims); - EOSParent* eos_model = elem_eos(elem_gid).model; - // --- abviatations of variables --- // element volume @@ -1026,19 +1023,14 @@ void FEA_Module_SGH::get_force_ugradient_sgh(const DCArrayKokkos& ma // gradients of the element volume get_vol_hex_ugradient(volume_gradients, elem_gid, node_coords, elem_node_gids, rk_level); -#ifdef DEBUG - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) - { - for (size_t dim = 0; dim < num_dims; dim++) - { - if (volume_gradients(node_lid, dim) > 1 || volume_gradients(node_lid, dim) < -1) - { - std::cout << volume_gradients(node_lid, dim) << " "; - } - } // end for - } // end for - std::cout << std::endl; -#endif + // //debug + // for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ + // for (size_t dim = 0; dim < num_dims; dim++){ + // if(volume_gradients(node_lid, dim)>1 || volume_gradients(node_lid, dim) < -1) + // std::cout << volume_gradients(node_lid, dim) << " "; + // } // end for + // } // end for + // std::cout << std::endl; // get the B matrix which are the OUTWARD corner area normals get_bmatrix(area_normal, @@ -1264,9 +1256,9 @@ void FEA_Module_SGH::get_force_ugradient_sgh(const DCArrayKokkos& ma // on velocity in the numerator. It filters on the shock // direction mu_term = muc(node_lid) * - fabs(shock_dir(0) * area_normal(node_lid, 0) + - shock_dir(1) * area_normal(node_lid, 1) + - shock_dir(2) * area_normal(node_lid, 2) ); + fabs(shock_dir(0) * area_normal(node_lid, 0) + + shock_dir(1) * area_normal(node_lid, 1) + + shock_dir(2) * area_normal(node_lid, 2) ); // muc_gradient(node_lid) = muc_gradient(node_lid)* // fabs( shock_dir(0)*area_normal(node_lid,0) // + shock_dir(1)*area_normal(node_lid,1) @@ -1454,25 +1446,24 @@ void FEA_Module_SGH::get_force_ugradient_sgh(const DCArrayKokkos& ma column_index = num_dims * Global_Gradient_Matrix_Assembly_Map(elem_gid, igradient, node_lid); if (map->isNodeLocalElement(gradient_node_gid)) { - Force_Gradient_Positions(gradient_node_gid * num_dims + jdim, column_index + dim) += - area_normal(node_lid, 0) * tau_gradient(0, dim, igradient, jdim) - + area_normal(node_lid, 1) * tau_gradient(1, dim, igradient, jdim) - + area_normal(node_lid, 2) * tau_gradient(2, dim, igradient, jdim) - + area_normal_gradients(node_lid, 0, igradient, jdim) * tau(0, dim) - + area_normal_gradients(node_lid, 1, igradient, jdim) * tau(1, dim) - + area_normal_gradients(node_lid, 2, igradient, jdim) * tau(2, dim) - + phi * muc_gradient(node_lid, igradient, jdim) * (vel_star(dim) - node_vel(rk_level, node_gid, dim)) - + phi * muc(node_lid) * (vel_star_gradient(dim, igradient, jdim)); + Force_Gradient_Positions(gradient_node_gid * num_dims + jdim, column_index + dim) += area_normal(node_lid, 0) * tau_gradient(0, dim, igradient, jdim) + + area_normal(node_lid, 1) * tau_gradient(1, dim, igradient, jdim) + + area_normal(node_lid, 2) * tau_gradient(2, dim, igradient, jdim) + + area_normal_gradients(node_lid, 0, igradient, jdim) * tau(0, dim) + + area_normal_gradients(node_lid, 1, igradient, jdim) * tau(1, dim) + + area_normal_gradients(node_lid, 2, igradient, jdim) * tau(2, dim) + + phi * muc_gradient(node_lid, igradient, + jdim) * (vel_star(dim) - node_vel(rk_level, node_gid, dim)) + + phi * muc(node_lid) * (vel_star_gradient(dim, igradient, jdim)); } - corner_gradient_storage(corner_gid, dim, igradient, jdim) = - area_normal(node_lid, 0) * tau_gradient(0, dim, igradient, jdim) - + area_normal(node_lid, 1) * tau_gradient(1, dim, igradient, jdim) - + area_normal(node_lid, 2) * tau_gradient(2, dim, igradient, jdim) - + area_normal_gradients(node_lid, 0, igradient, jdim) * tau(0, dim) - + area_normal_gradients(node_lid, 1, igradient, jdim) * tau(1, dim) - + area_normal_gradients(node_lid, 2, igradient, jdim) * tau(2, dim) - + phi * muc_gradient(node_lid, igradient, jdim) * (vel_star(dim) - node_vel(rk_level, node_gid, dim)) - + phi * muc(node_lid) * (vel_star_gradient(dim, igradient, jdim)); + corner_gradient_storage(corner_gid, dim, igradient, jdim) = area_normal(node_lid, 0) * tau_gradient(0, dim, igradient, jdim) + + area_normal(node_lid, 1) * tau_gradient(1, dim, igradient, jdim) + + area_normal(node_lid, 2) * tau_gradient(2, dim, igradient, jdim) + + area_normal_gradients(node_lid, 0, igradient, jdim) * tau(0, dim) + + area_normal_gradients(node_lid, 1, igradient, jdim) * tau(1, dim) + + area_normal_gradients(node_lid, 2, igradient, jdim) * tau(2, dim) + + phi * muc_gradient(node_lid, igradient, jdim) * (vel_star(dim) - node_vel(rk_level, node_gid, dim)) + + phi * muc(node_lid) * (vel_star_gradient(dim, igradient, jdim)); // if(map->isNodeLocalElement(gradient_node_gid)){ // Force_Gradient_Positions(gradient_node_gid*num_dims+jdim, column_index+dim) += // + area_normal_gradients(node_lid, 0, igradient, jdim)*tau(0, dim) @@ -1521,21 +1512,16 @@ void FEA_Module_SGH::get_force_ugradient_sgh(const DCArrayKokkos& ma void FEA_Module_SGH::force_design_gradient_term(const_vec_array design_variables, vec_array design_gradients) { - bool element_constant_density = true; - size_t num_bdy_nodes = mesh->num_bdy_nodes; - size_t num_corners = rnum_elem * num_nodes_in_elem; - size_t current_data_index, next_data_index; - - real_t global_dt; - - const int num_dim = simparam->num_dims; - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - const DCArrayKokkos boundary = module_params->boundary; const DCArrayKokkos material = simparam->material; - - auto current_element_adjoint = CArrayKokkos(num_nodes_in_elem, num_dim); + const int num_dim = simparam->num_dims; + size_t num_corners = rnum_elem * num_nodes_in_elem; + real_t global_dt; + bool element_constant_density = true; + size_t current_data_index, next_data_index; + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + CArrayKokkos current_element_adjoint = CArrayKokkos(num_nodes_in_elem, num_dim); // gradient contribution from gradient of Force vector with respect to design variable. if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) @@ -1588,12 +1574,12 @@ void FEA_Module_SGH::force_design_gradient_term(const_vec_array design_variables node_vel(rk_level, node_gid, idim) = current_velocity_vector(node_gid, idim); node_coords(rk_level, node_gid, idim) = current_coord_vector(node_gid, idim); } - }); // end parallel for + }); // end parallel for Kokkos::fence(); FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { elem_sie(rk_level, elem_gid) = current_element_internal_energy(elem_gid, 0); - }); // end parallel for + }); // end parallel for Kokkos::fence(); get_vol(); @@ -1688,7 +1674,7 @@ void FEA_Module_SGH::force_design_gradient_term(const_vec_array design_variables corner_id = elem_id * num_nodes_in_elem + inode; corner_value_storage(corner_id) = inner_product; } - }); // end parallel for + }); // end parallel for Kokkos::fence(); // accumulate node values from corner storage @@ -1700,7 +1686,7 @@ void FEA_Module_SGH::force_design_gradient_term(const_vec_array design_variables corner_id = corners_in_node(node_id, icorner); design_gradients(node_id, 0) += -corner_value_storage(corner_id) * global_dt; } - }); // end parallel for + }); // end parallel for Kokkos::fence(); } // end view scope } @@ -1773,8 +1759,6 @@ void FEA_Module_SGH::get_force_dgradient_sgh(const DCArrayKokkos& ma ViewCArrayKokkos vel_star_gradient(vel_star_gradient_array, num_dims); ViewCArrayKokkos vel_grad(vel_grad_array, num_dims, num_dims); - EOSParent* eos_model = elem_eos(elem_gid).model; - // --- abviatations of variables --- // element volume diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_sgh.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_sgh.cpp index 1c367ba6e..80ba6a835 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_sgh.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_sgh.cpp @@ -36,7 +36,9 @@ #include "Simulation_Parameters/Simulation_Parameters_Explicit.h" #include "Simulation_Parameters/FEA_Module/SGH_Parameters.h" #include "FEA_Module_SGH.h" - +// ----------------------------------------------------------------------------- +// This function calculates the corner forces and the evolves stress (hypo) +// ------------------------------------------------------------------------------ void FEA_Module_SGH::get_force_sgh(const DCArrayKokkos& material, const mesh_t& mesh, const DViewCArrayKokkos& node_coords, @@ -883,6 +885,7 @@ void FEA_Module_SGH::get_force_sgh2D(const DCArrayKokkos& material, // ----------------------------------------------------------------------------- // This function calculates the corner forces and the evolves stress (hypo) // ------------------------------------------------------------------------------ + void FEA_Module_SGH::applied_forces(const DCArrayKokkos& material, const mesh_t& mesh, const DViewCArrayKokkos& node_coords, @@ -897,15 +900,14 @@ void FEA_Module_SGH::applied_forces(const DCArrayKokkos& material, const size_t cycle ) { - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - const size_t num_dim = mesh.num_dims; - const size_t num_lcs = module_params->loading.size(); + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + const size_t num_dim = mesh.num_dims; + const_vec_array all_initial_node_coords = all_initial_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + const size_t num_lcs = module_params->loading.size(); const DCArrayKokkos mat_fill = simparam->mat_fill; const DCArrayKokkos loading = module_params->loading; - const_vec_array all_initial_node_coords = all_initial_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); - // debug check // std::cout << "NUMBER OF LOADING CONDITIONS: " << num_lcs << std::endl; diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/setup_sgh.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/setup_sgh.cpp index 54cccd4f5..9f56e993c 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/setup_sgh.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/setup_sgh.cpp @@ -43,112 +43,149 @@ // #define DEBUG -void FEA_Module_SGH::setup() -{ - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - const size_t num_fills = simparam->regions.size(); - const size_t rk_num_bins = simparam->dynamic_options.rk_num_bins; - const size_t num_bcs = module_params->boundary_conditions.size(); +/* ---------------------------------------------------------------------------- + Setup SGH solver data +------------------------------------------------------------------------------- */ + +void FEA_Module_SGH::setup(){ + + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + const size_t num_fills = simparam->regions.size(); + const size_t rk_num_bins = simparam->dynamic_options.rk_num_bins; + const size_t num_bcs = module_params->boundary_conditions.size(); const size_t num_materials = simparam->materials.size(); - const int num_dim = simparam->num_dims; - const size_t num_lcs = module_params->loading.size(); - if (num_lcs) - { - have_loading_conditions = true; - } + const int num_dim = simparam->num_dims; + const size_t num_lcs = module_params->loading.size(); + if(num_lcs) + have_loading_conditions = true; + // --------------------------------------------------------------------- // obtain mesh data - // --------------------------------------------------------------------- + // --------------------------------------------------------------------- sgh_interface_setup(node_interface, elem_interface, corner_interface); mesh->build_corner_connectivity(); - // debug print of corner ids -#ifdef DEBUG - if (myrank == 1) - { - for (int i = 0; i < mesh.num_nodes; i++) - { - // loop over all corners around the node and calculate the nodal force - for (size_t corner_lid = 0; corner_lid < mesh.num_corners_in_node(i); corner_lid++) - { - // Get corner gid - size_t corner_gid = mesh.corners_in_node(i, corner_lid); - std::cout << map->getGlobalElement(i) << " " << i << " " << all_node_map->getLocalElement(all_node_map->getGlobalElement(i)) << " " << corner_gid << " " << std::endl; - } // end for corner_lid - } - } - - if (myrank == 1) - { - for (int i = 0; i < mesh.num_elems; i++) - { - // loop over all corners around the node and calculate the nodal force - for (size_t corner_lid = 0; corner_lid < max_nodes_per_element; corner_lid++) - { - // Get corner gid - size_t corner_gid = mesh.corners_in_elem(i, corner_lid); - std::cout << i << " " << mesh.nodes_in_elem(i, corner_lid) << " " << all_node_map->getGlobalElement(mesh.nodes_in_elem(i, corner_lid)) << " " << corner_gid << " " << std::endl; - } // end for corner_lid - } - } -#endif - + //debug print of corner ids + /* + if(myrank==1){ + for(int i = 0; i < mesh.num_nodes; i++){ + + // loop over all corners around the node and calculate the nodal force + for (size_t corner_lid=0; corner_lidgetGlobalElement(i) << " " << i << " " << all_node_map->getLocalElement(all_node_map->getGlobalElement(i)) << " " << corner_gid << " " << std::endl; + + } // end for corner_lid + //std::cout << explicit_solver_pointer->all_node_map->getGlobalElement(i) << " " << node_force[0] << " " << node_force[1] << " " << node_force[2] << std::endl; + //std::cout << explicit_solver_pointer->all_node_map->getGlobalElement(i) << " " << node_mass(i) << std::endl; + } + } + */ + /* + if(myrank==1){ + for(int i = 0; i < mesh.num_elems; i++){ + + // loop over all corners around the node and calculate the nodal force + for (size_t corner_lid=0; corner_lidgetGlobalElement(mesh.nodes_in_elem(i, corner_lid)) <<" " << corner_gid << " " << std::endl; + + } // end for corner_lid + //std::cout << explicit_solver_pointer->all_node_map->getGlobalElement(i) << " " << node_force[0] << " " << node_force[1] << " " << node_force[2] << std::endl; + //std::cout << explicit_solver_pointer->all_node_map->getGlobalElement(i) << " " << node_mass(i) << std::endl; + } + } + */ mesh->build_elem_elem_connectivity(); mesh->num_bdy_patches = nboundary_patches; - if (num_dim == 2) - { - mesh->build_patch_connectivity(); - mesh->build_node_node_connectivity(); + if(num_dim==2){ + mesh->build_patch_connectivity(); + mesh->build_node_node_connectivity(); } - - // --------------------------------------------------------------------- - // allocate memory - // --------------------------------------------------------------------- - - // shorthand names - const size_t num_nodes = mesh->num_nodes; - const size_t num_elems = mesh->num_elems; + + // --------------------------------------------------------------------- + // allocate memory + // --------------------------------------------------------------------- + + // shorthand names + const size_t num_nodes = mesh->num_nodes; + const size_t num_elems = mesh->num_elems; const size_t num_corners = mesh->num_corners; - // --- make dual views of data on CPU and GPU --- - // Notes: - // Instead of using a struct of dual types like the mesh type, - // individual dual views will be made for all the state - // variables. The motivation is to reduce memory movement - // when passing state into a function. Passing a struct by - // reference will copy the meta data and pointers for the - // variables held inside the struct. Since all the mesh - // variables are typically used by most functions, a single - // mesh struct or passing the arrays will be roughly equivalent - // for memory movement. - + + // --- make dual views of data on CPU and GPU --- + // Notes: + // Instead of using a struct of dual types like the mesh type, + // individual dual views will be made for all the state + // variables. The motivation is to reduce memory movement + // when passing state into a function. Passing a struct by + // reference will copy the meta data and pointers for the + // variables held inside the struct. Since all the mesh + // variables are typically used by most functions, a single + // mesh struct or passing the arrays will be roughly equivalent + // for memory movement. + + // create Dual Views of the individual node struct variables - node_coords = DViewCArrayKokkos(node_interface.coords.get_kokkos_dual_view().view_host().data(), rk_num_bins, num_nodes, num_dim); - node_vel = DViewCArrayKokkos(node_interface.vel.get_kokkos_dual_view().view_host().data(), rk_num_bins, num_nodes, num_dim); - node_mass = DViewCArrayKokkos(node_interface.mass.get_kokkos_dual_view().view_host().data(), num_nodes); + node_coords = DViewCArrayKokkos(node_interface.coords.get_kokkos_dual_view().view_host().data(),rk_num_bins,num_nodes,num_dim); - // create Dual Views of the individual elem struct variables - elem_den = DViewCArrayKokkos(&elem_interface.den(0), num_elems); - elem_pres = DViewCArrayKokkos(&elem_interface.pres(0), num_elems); - elem_stress = DViewCArrayKokkos(&elem_interface.stress(0, 0, 0, 0), rk_num_bins, num_elems, 3, 3); // always 3D even in 2D-RZ - elem_sspd = DViewCArrayKokkos(&elem_interface.sspd(0), num_elems); - elem_sie = DViewCArrayKokkos(&elem_interface.sie(0, 0), rk_num_bins, num_elems); - elem_vol = DViewCArrayKokkos(&elem_interface.vol(0), num_elems); - elem_div = DViewCArrayKokkos(&elem_interface.div(0), num_elems); - elem_mass = DViewCArrayKokkos(&elem_interface.mass(0), num_elems); - elem_mat_id = DViewCArrayKokkos(&elem_interface.mat_id(0), num_elems); + node_vel = DViewCArrayKokkos(node_interface.vel.get_kokkos_dual_view().view_host().data(),rk_num_bins,num_nodes,num_dim); + node_mass = DViewCArrayKokkos(node_interface.mass.get_kokkos_dual_view().view_host().data(),num_nodes); + + + // create Dual Views of the individual elem struct variables + elem_den= DViewCArrayKokkos(&elem_interface.den(0), + num_elems); + + elem_pres = DViewCArrayKokkos(&elem_interface.pres(0), + num_elems); + + elem_stress = DViewCArrayKokkos(&elem_interface.stress(0,0,0,0), + rk_num_bins, + num_elems, + 3, + 3); // always 3D even in 2D-RZ + + elem_sspd = DViewCArrayKokkos(&elem_interface.sspd(0), + num_elems); + + elem_sie = DViewCArrayKokkos(&elem_interface.sie(0,0), + rk_num_bins, + num_elems); + + elem_vol = DViewCArrayKokkos(&elem_interface.vol(0), + num_elems); + + elem_div = DViewCArrayKokkos(&elem_interface.div(0), + num_elems); + + + elem_mass = DViewCArrayKokkos(&elem_interface.mass(0), + num_elems); + + elem_mat_id = DViewCArrayKokkos(&elem_interface.mat_id(0), + num_elems); + // create Dual Views of the corner struct variables - corner_force = DViewCArrayKokkos(&corner_interface.force(0, 0), num_corners, num_dim); - corner_mass = DViewCArrayKokkos(&corner_interface.mass(0), num_corners); + corner_force = DViewCArrayKokkos (&corner_interface.force(0,0), + num_corners, + num_dim); + corner_mass = DViewCArrayKokkos (&corner_interface.mass(0), + num_corners); + // allocate elem_vel_grad - elem_vel_grad = DCArrayKokkos(num_elems, 3, 3); + elem_vel_grad = DCArrayKokkos (num_elems,3,3); // allocate material models - elem_eos = DCArrayKokkos(num_elems); - elem_strength = DCArrayKokkos(num_elems); - + elem_eos = DCArrayKokkos (num_elems); + elem_strength = DCArrayKokkos (num_elems); + // --------------------------------------------------------------------- // calculate geometry // --------------------------------------------------------------------- @@ -157,79 +194,78 @@ void FEA_Module_SGH::setup() get_vol(); - // FEA_Module boundary condition variable + //FEA_Module bc variable num_boundary_conditions = num_bcs; - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos mat_fill = simparam->mat_fill; - const DCArrayKokkos material = simparam->material; - global_vars = simparam->global_vars; - state_vars = DCArrayKokkos(rnum_elem, simparam->max_num_state_vars); - elem_user_output_vars = DCArrayKokkos(rnum_elem, simparam->output_options.max_num_user_output_vars); - - // --- calculate bdy sets ---// - mesh->num_nodes_in_patch = 2 * (num_dim - 1); // 2 (2D) or 4 (3D) - mesh->num_patches_in_elem = 2 * num_dim; // 4 (2D) or 6 (3D) + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos mat_fill = simparam->mat_fill; + const DCArrayKokkos material = simparam->material; + eos_global_vars = simparam->eos_global_vars; + strength_global_vars = simparam->strength_global_vars; + eos_state_vars = DCArrayKokkos (rnum_elem, simparam->max_num_eos_state_vars); + strength_state_vars = DCArrayKokkos (rnum_elem, simparam->max_num_strength_state_vars); + elem_user_output_vars = DCArrayKokkos (rnum_elem, simparam->output_options.max_num_user_output_vars); + + //--- calculate bdy sets ---// + mesh->num_nodes_in_patch = 2*(num_dim-1); // 2 (2D) or 4 (3D) + mesh->num_patches_in_elem = 2*num_dim; // 4 (2D) or 6 (3D) mesh->init_bdy_sets(num_bcs); num_bdy_sets = mesh->num_bdy_sets; printf("Num BC's = %lu\n", num_bcs); // patch ids in bdy set bdy_patches_in_set = mesh->bdy_patches_in_set; - if (num_dim == 2) - { - bdy_nodes = mesh->bdy_nodes; - } + if(num_dim==2) + bdy_nodes = mesh->bdy_nodes; // tag boundary patches in the set tag_bdys(boundary, *mesh, node_coords); build_boundry_node_sets(*mesh); - + // node ids in bdy_patch set - bdy_nodes_in_set = mesh->bdy_nodes_in_set; + bdy_nodes_in_set = mesh->bdy_nodes_in_set; num_bdy_nodes_in_set = mesh->num_bdy_nodes_in_set; - - // assign mesh views needed by the FEA module + + //assign mesh views needed by the FEA module // elem ids in elem - elems_in_elem = mesh->elems_in_elem; + elems_in_elem = mesh->elems_in_elem; num_elems_in_elem = mesh->num_elems_in_elem; - // corners + //corners num_corners_in_node = mesh->num_corners_in_node; - corners_in_node = mesh->corners_in_node; - corners_in_elem = mesh->corners_in_elem; + corners_in_node = mesh->corners_in_node; + corners_in_elem = mesh->corners_in_elem; - // elem-node conn & node-node conn + //elem-node conn & node-node conn elems_in_node = mesh->elems_in_node; - if (num_dim == 2) - { - nodes_in_node = mesh->nodes_in_node; - num_nodes_in_node = mesh->num_nodes_in_node; - // patch conn - - patches_in_elem = mesh->patches_in_elem; - nodes_in_patch = mesh->nodes_in_patch; - elems_in_patch = mesh->elems_in_patch; + if(num_dim==2){ + nodes_in_node = mesh->nodes_in_node; + num_nodes_in_node = mesh->num_nodes_in_node; + //patch conn + + patches_in_elem = mesh->patches_in_elem; + nodes_in_patch = mesh->nodes_in_patch; + elems_in_patch = mesh->elems_in_patch; } // loop over BCs - for (size_t this_bdy = 0; this_bdy < num_bcs; this_bdy++) - { + for (size_t this_bdy = 0; this_bdy < num_bcs; this_bdy++){ + RUN_CLASS({ printf("Boundary Condition number %lu \n", this_bdy); printf(" Num bdy patches in this set = %lu \n", bdy_patches_in_set.stride(this_bdy)); printf(" Num bdy nodes in this set = %lu \n", bdy_nodes_in_set.stride(this_bdy)); }); Kokkos::fence(); - } // end for + + }// end for // elem_mat_id needs to be initialized before initialization of material models - for (int f_id = 0; f_id < num_fills; f_id++) - { - FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { - elem_mat_id(elem_gid) = mat_fill(f_id).material_id; + for (int f_id = 0; f_id < num_fills; f_id++){ + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { + elem_mat_id(elem_gid) = mat_fill(f_id).material_id; }); } elem_mat_id.update_host(); @@ -237,17 +273,21 @@ void FEA_Module_SGH::setup() // function for initializing state_vars init_state_vars(material, elem_mat_id, - state_vars, - global_vars, + eos_state_vars, + strength_state_vars, + eos_global_vars, + strength_global_vars, elem_user_output_vars, rnum_elem); - + // initialize strength model init_strength_model(elem_strength, material, elem_mat_id, - state_vars, - global_vars, + eos_state_vars, + strength_state_vars, + eos_global_vars, + strength_global_vars, elem_user_output_vars, rnum_elem); @@ -255,29 +295,30 @@ void FEA_Module_SGH::setup() init_eos_model(elem_eos, material, elem_mat_id, - state_vars, - global_vars, + eos_state_vars, + strength_state_vars, + eos_global_vars, + strength_global_vars, elem_user_output_vars, rnum_elem); - - // --- apply the fill instructions over each of the Elements---// - - // initialize if topology optimization is used - if (simparam->topology_optimization_on) - { - for (int elem_id = 0; elem_id < rnum_elem; elem_id++) - { - relative_element_densities.host(elem_id) = 1; - } // for - relative_element_densities.update_device(); + + //--- apply the fill instructions over each of the Elements---// + + //initialize if topology optimization is used + if(simparam->topology_optimization_on){ + for(int elem_id = 0; elem_id < rnum_elem; elem_id++){ + relative_element_densities.host(elem_id) = 1; + }//for + relative_element_densities.update_device(); } - + // loop over the fill instructures - for (int f_id = 0; f_id < num_fills; f_id++) - { + for (int f_id = 0; f_id < num_fills; f_id++){ + // parallel loop over elements in mesh - // for (size_t elem_gid = 0; elem_gid <= rnum_elem; elem_gid++) { + //for (size_t elem_gid = 0; elem_gid <= rnum_elem; elem_gid++) { FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { + // calculate the coordinates and radius of the element double elem_coords[3]; // note:initialization with a list won't work elem_coords[0] = 0.0; @@ -285,292 +326,291 @@ void FEA_Module_SGH::setup() elem_coords[2] = 0.0; // get the coordinates of the element center - for (int node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) - { + for (int node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ elem_coords[0] += node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), 0); elem_coords[1] += node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), 1); - if (num_dim == 3) - { + if (num_dim == 3){ elem_coords[2] += node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), 2); - } - else + } else { elem_coords[2] = 0.0; } } // end loop over nodes in element - elem_coords[0] = elem_coords[0] / num_nodes_in_elem; - elem_coords[1] = elem_coords[1] / num_nodes_in_elem; - elem_coords[2] = elem_coords[2] / num_nodes_in_elem; + elem_coords[0] = elem_coords[0]/num_nodes_in_elem; + elem_coords[1] = elem_coords[1]/num_nodes_in_elem; + elem_coords[2] = elem_coords[2]/num_nodes_in_elem; // default is not to fill the element bool fill_this = mat_fill(f_id).volume.contains(elem_coords); // paint the material state on the element - if (fill_this) - { + if (fill_this){ + // density elem_den(elem_gid) = mat_fill(f_id).den; - + // mass - elem_mass(elem_gid) = elem_den(elem_gid) * elem_vol(elem_gid); - + elem_mass(elem_gid) = elem_den(elem_gid)*elem_vol(elem_gid); + // specific internal energy elem_sie(rk_level, elem_gid) = mat_fill(f_id).sie; - + size_t mat_id = elem_mat_id(elem_gid); // short name - + // --- stress tensor --- // always 3D even for 2D-RZ - for (size_t i = 0; i < 3; i++) - { - for (size_t j = 0; j < 3; j++) - { - elem_stress(rk_level, elem_gid, i, j) = 0.0; - } + for (size_t i=0; i<3; i++){ + for (size_t j=0; j<3; j++){ + elem_stress(rk_level,elem_gid,i,j) = 0.0; + } } // end for - - // short form for clean code - EOSParent* eos_model = elem_eos(elem_gid).model; + // --- Pressure --- - eos_model->calc_pressure(elem_pres, + elem_eos(elem_gid).calc_pressure(elem_pres, elem_stress, elem_gid, elem_mat_id(elem_gid), - state_vars, - global_vars, + eos_state_vars, + strength_state_vars, + eos_global_vars, + strength_global_vars, elem_user_output_vars, elem_sspd, elem_den(elem_gid), - elem_sie(rk_level, elem_gid)); + elem_sie(rk_level,elem_gid)); // --- Sound speed --- - eos_model->calc_sound_speed(elem_pres, + elem_eos(elem_gid).calc_sound_speed(elem_pres, elem_stress, elem_gid, elem_mat_id(elem_gid), - state_vars, - global_vars, + eos_state_vars, + strength_state_vars, + eos_global_vars, + strength_global_vars, elem_user_output_vars, elem_sspd, elem_den(elem_gid), - elem_sie(rk_level, elem_gid)); - + elem_sie(rk_level,elem_gid)); + // loop over the nodes of this element and apply velocity - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) - { + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ + // get the mesh node index size_t node_gid = nodes_in_elem(elem_gid, node_lid); + // --- Velocity --- - switch (mat_fill(f_id).velocity) + switch(mat_fill(f_id).velocity) { case VELOCITY_TYPE::cartesian: - { - node_vel(rk_level, node_gid, 0) = mat_fill(f_id).u; - node_vel(rk_level, node_gid, 1) = mat_fill(f_id).v; - if (num_dim == 3) - { - node_vel(rk_level, node_gid, 2) = mat_fill(f_id).w; - } - - break; - } + { + + node_vel(rk_level, node_gid, 0) = mat_fill(f_id).u; + node_vel(rk_level, node_gid, 1) = mat_fill(f_id).v; + if (num_dim == 3) node_vel(rk_level, node_gid, 2) = mat_fill(f_id).w; + + + break; + } case VELOCITY_TYPE::radial: - { - // Setting up cylindrical - double dir[2]; - dir[0] = 0.0; - dir[1] = 0.0; - double radius_val = 0.0; - - for (int dim = 0; dim < 2; dim++) - { - dir[dim] = node_coords(rk_level, node_gid, dim); - radius_val += node_coords(rk_level, node_gid, dim) * node_coords(rk_level, node_gid, dim); - } // end for - radius_val = sqrt(radius_val); - - for (int dim = 0; dim < 2; dim++) - { - if (radius_val > 1.0e-14) - { - dir[dim] /= (radius_val); - } - else - { - dir[dim] = 0.0; - } - } // end for - - node_vel(rk_level, node_gid, 0) = mat_fill(f_id).speed * dir[0]; - node_vel(rk_level, node_gid, 1) = mat_fill(f_id).speed * dir[1]; - if (num_dim == 3) - { - node_vel(rk_level, node_gid, 2) = 0.0; + { + // Setting up cylindrical + double dir[2]; + dir[0] = 0.0; + dir[1] = 0.0; + double radius_val = 0.0; + + for(int dim=0; dim<2; dim++){ + dir[dim] = node_coords(rk_level, node_gid, dim); + radius_val += node_coords(rk_level, node_gid, dim)*node_coords(rk_level, node_gid, dim); + } // end for + radius_val = sqrt(radius_val); + + for(int dim=0; dim<2; dim++){ + if (radius_val > 1.0e-14){ + dir[dim] /= (radius_val); } - - break; - } + else{ + dir[dim] = 0.0; + } + } // end for + + + node_vel(rk_level, node_gid, 0) = mat_fill(f_id).speed*dir[0]; + node_vel(rk_level, node_gid, 1) = mat_fill(f_id).speed*dir[1]; + if (num_dim == 3) node_vel(rk_level, node_gid, 2) = 0.0; + + break; + } case VELOCITY_TYPE::spherical: - { - // Setting up spherical - double dir[3]; - dir[0] = 0.0; - dir[1] = 0.0; - dir[2] = 0.0; - double radius_val = 0.0; - - for (int dim = 0; dim < 3; dim++) - { - dir[dim] = node_coords(rk_level, node_gid, dim); - radius_val += node_coords(rk_level, node_gid, dim) * node_coords(rk_level, node_gid, dim); - } // end for - radius_val = sqrt(radius_val); - - for (int dim = 0; dim < 3; dim++) - { - if (radius_val > 1.0e-14) - { - dir[dim] /= (radius_val); - } - else - { - dir[dim] = 0.0; - } - } // end for - - node_vel(rk_level, node_gid, 0) = mat_fill(f_id).speed * dir[0]; - node_vel(rk_level, node_gid, 1) = mat_fill(f_id).speed * dir[1]; - if (num_dim == 3) - { - node_vel(rk_level, node_gid, 2) = mat_fill(f_id).speed * dir[2]; + { + + // Setting up spherical + double dir[3]; + dir[0] = 0.0; + dir[1] = 0.0; + dir[2] = 0.0; + double radius_val = 0.0; + + for(int dim=0; dim<3; dim++){ + dir[dim] = node_coords(rk_level, node_gid, dim); + radius_val += node_coords(rk_level, node_gid, dim)*node_coords(rk_level, node_gid, dim); + } // end for + radius_val = sqrt(radius_val); + + for(int dim=0; dim<3; dim++){ + if (radius_val > 1.0e-14){ + dir[dim] /= (radius_val); } - - break; - } + else{ + dir[dim] = 0.0; + } + } // end for + + node_vel(rk_level, node_gid, 0) = mat_fill(f_id).speed*dir[0]; + node_vel(rk_level, node_gid, 1) = mat_fill(f_id).speed*dir[1]; + if (num_dim == 3) node_vel(rk_level, node_gid, 2) = mat_fill(f_id).speed*dir[2]; + + break; + } case VELOCITY_TYPE::radial_linear: - { - break; - } + { + + break; + } case VELOCITY_TYPE::spherical_linear: - { - break; - } + { + + break; + } case VELOCITY_TYPE::tg_vortex: - { - node_vel(rk_level, node_gid, 0) = sin(PI * node_coords(rk_level, node_gid, 0)) * cos(PI * node_coords(rk_level, node_gid, 1)); - node_vel(rk_level, node_gid, 1) = -1.0 * cos(PI * node_coords(rk_level, node_gid, 0)) * sin(PI * node_coords(rk_level, node_gid, 1)); - if (num_dim == 3) - { - node_vel(rk_level, node_gid, 2) = 0.0; - } - - break; - } + { + + node_vel(rk_level, node_gid, 0) = sin(PI * node_coords(rk_level,node_gid, 0)) * cos(PI * node_coords(rk_level,node_gid, 1)); + node_vel(rk_level, node_gid, 1) = -1.0*cos(PI * node_coords(rk_level,node_gid, 0)) * sin(PI * node_coords(rk_level,node_gid, 1)); + if (num_dim == 3) node_vel(rk_level, node_gid, 2) = 0.0; + + break; + } } // end of switch - } // end loop over nodes of element - if (mat_fill(f_id).velocity == VELOCITY_TYPE::tg_vortex) + }// end loop over nodes of element + + + if(mat_fill(f_id).velocity == VELOCITY_TYPE::tg_vortex) { - elem_pres(elem_gid) = 0.25 * (cos(2.0 * PI * elem_coords[0]) + cos(2.0 * PI * elem_coords[1]) ) + 1.0; - + elem_pres(elem_gid) = 0.25*( cos(2.0*PI*elem_coords[0]) + cos(2.0*PI*elem_coords[1]) ) + 1.0; + // p = rho*ie*(gamma - 1) size_t mat_id = f_id; - double gamma = global_vars(mat_id, 0); // gamma value + double gamma = eos_global_vars(mat_id,0); // gamma value elem_sie(rk_level, elem_gid) = - elem_pres(elem_gid) / (mat_fill(f_id).den * (gamma - 1.0)); + elem_pres(elem_gid)/(mat_fill(f_id).den*(gamma - 1.0)); } // end if + } // end if fill + }); // end FOR_ALL_CLASS element loop Kokkos::fence(); + + } // end for loop over fills + // apply BC's to velocity FEA_Module_SGH::boundary_velocity(*mesh, boundary, node_vel); - + + // calculate the corner massess if 2D - if (num_dim == 2) - { + if(num_dim==2){ + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { + // facial area of the corners double corner_areas_array[4]; - - ViewCArrayKokkos corner_areas(&corner_areas_array[0], 4); - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 4); - + + ViewCArrayKokkos corner_areas(&corner_areas_array[0],4); + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 4); + get_area_weights2D(corner_areas, elem_gid, node_coords, elem_node_gids, rk_level); - + // loop over the corners of the element and calculate the mass - for (size_t corner_lid = 0; corner_lid < 4; corner_lid++) - { + for (size_t corner_lid=0; corner_lid<4; corner_lid++){ + size_t corner_gid = corners_in_elem(elem_gid, corner_lid); - corner_mass(corner_gid) = corner_areas(corner_lid) * elem_den(elem_gid); // node radius is added later + corner_mass(corner_gid) = corner_areas(corner_lid)*elem_den(elem_gid); // node radius is added later + } // end for over corners }); + } // end of - + + // calculate the nodal mass FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + node_mass(node_gid) = 0.0; - - if (num_dim == 3) - { - for (size_t elem_lid = 0; elem_lid < num_corners_in_node(node_gid); elem_lid++) - { - size_t elem_gid = elems_in_node(node_gid, elem_lid); - node_mass(node_gid) += 1.0 / 8.0 * elem_mass(elem_gid); + + if(num_dim==3){ + + for(size_t elem_lid=0; elem_lidtopology_optimization_on || simparam->shape_optimization_on || simparam->num_dims == 2) - { - { - vec_array node_mass_interface = node_masses_distributed->getLocalView(Tpetra::Access::ReadWrite); - - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - node_mass_interface(node_gid, 0) = node_mass(node_gid); - }); // end parallel for - } // end view scope - Kokkos::fence(); - // communicate ghost densities - comm_node_masses(); - - // this is forcing a copy to the device - // view scope - { - vec_array ghost_node_mass_interface = ghost_node_masses_distributed->getLocalView(Tpetra::Access::ReadWrite); - - FOR_ALL_CLASS(node_gid, nlocal_nodes, nall_nodes, { - node_mass(node_gid) = ghost_node_mass_interface(node_gid - nlocal_nodes, 0); - }); // end parallel for - } // end view scope - Kokkos::fence(); - } // endif - - // initialize if topology optimization is used - if (simparam->topology_optimization_on || simparam->shape_optimization_on) - { - init_assembly(); - assemble_matrix(); + //current interface has differing mass arrays; this equates them until we unify memory + //view scope + if(simparam->topology_optimization_on||simparam->shape_optimization_on||simparam->num_dims==2){ + { + vec_array node_mass_interface = node_masses_distributed->getLocalView (Tpetra::Access::ReadWrite); + + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + node_mass_interface(node_gid,0) = node_mass(node_gid); + }); // end parallel for + } //end view scope + Kokkos::fence(); + //communicate ghost densities + comm_node_masses(); + + //this is forcing a copy to the device + //view scope + { + vec_array ghost_node_mass_interface = ghost_node_masses_distributed->getLocalView (Tpetra::Access::ReadWrite); + + + FOR_ALL_CLASS(node_gid, nlocal_nodes, nall_nodes, { + node_mass(node_gid) = ghost_node_mass_interface(node_gid-nlocal_nodes,0); + }); // end parallel for + } //end view scope + Kokkos::fence(); + } //endif + + //initialize if topology optimization is used + if(simparam->topology_optimization_on || simparam->shape_optimization_on){ + init_assembly(); + //assemble_matrix(); } // update host copies of arrays modified in this function @@ -579,9 +619,10 @@ void FEA_Module_SGH::setup() elem_sie.update_host(); elem_stress.update_host(); elem_pres.update_host(); - elem_sspd.update_host(); + elem_sspd.update_host(); return; + } // end of setup // ----------------------------------------------------------------------------- diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/sgh_optimization.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/sgh_optimization.cpp index 5fbd0c5af..549c0f8ed 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/sgh_optimization.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/sgh_optimization.cpp @@ -84,10 +84,8 @@ void FEA_Module_SGH::update_forward_solve(Teuchos::RCP zp) int max_stride = 0; int current_module_index; size_t access_index, row_access_index, row_counter; - - GO global_index, global_dof_index; - LO local_dof_index; - + GO global_index, global_dof_index; + LO local_dof_index; const size_t num_fills = simparam->regions.size(); const size_t rk_num_bins = simparam->dynamic_options.rk_num_bins; const size_t num_bcs = module_params->boundary_conditions.size(); @@ -101,7 +99,6 @@ void FEA_Module_SGH::update_forward_solve(Teuchos::RCP zp) const DCArrayKokkos mat_fill = simparam->mat_fill; const DCArrayKokkos boundary = module_params->boundary; const DCArrayKokkos material = simparam->material; - CArray current_element_nodal_densities = CArray(num_nodes_in_elem); std::vector> FEA_Module_My_TO_Modules = simparam->FEA_Module_My_TO_Modules; @@ -122,8 +119,7 @@ void FEA_Module_SGH::update_forward_solve(Teuchos::RCP zp) relative_element_densities.host(elem_id) = average_element_density(num_nodes_in_elem, current_element_nodal_densities); } // for } // view scope - // debug print - + // debug print // std::cout << "ELEMENT RELATIVE DENSITY TEST " << relative_element_densities.host(0) << std::endl; relative_element_densities.update_device(); @@ -294,28 +290,29 @@ void FEA_Module_SGH::update_forward_solve(Teuchos::RCP zp) } } // end for - // short form for clean code - EOSParent* eos_model = elem_eos(elem_gid).model; - // --- Pressure --- - eos_model->calc_pressure(elem_pres, + elem_eos(elem_gid).calc_pressure(elem_pres, elem_stress, elem_gid, elem_mat_id(elem_gid), - state_vars, - global_vars, + eos_state_vars, + strength_state_vars, + eos_global_vars, + strength_global_vars, elem_user_output_vars, elem_sspd, elem_den(elem_gid), elem_sie(rk_level, elem_gid)); // --- Sound speed --- - eos_model->calc_sound_speed(elem_pres, + elem_eos(elem_gid).calc_sound_speed(elem_pres, elem_stress, elem_gid, elem_mat_id(elem_gid), - state_vars, - global_vars, + eos_state_vars, + strength_state_vars, + eos_global_vars, + strength_global_vars, elem_user_output_vars, elem_sspd, elem_den(elem_gid), @@ -442,7 +439,7 @@ void FEA_Module_SGH::update_forward_solve(Teuchos::RCP zp) // p = rho*ie*(gamma - 1) size_t mat_id = f_id; - double gamma = global_vars(mat_id, 0); // gamma value + double gamma = eos_global_vars(mat_id, 0); // gamma value elem_sie(rk_level, elem_gid) = elem_pres(elem_gid) / (mat_fill(f_id).den * (gamma - 1.0)); } // end if @@ -512,7 +509,7 @@ void FEA_Module_SGH::update_forward_solve(Teuchos::RCP zp) vec_array node_mass_interface = node_masses_distributed->getLocalView(Tpetra::Access::ReadWrite); FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { node_mass_interface(node_gid, 0) = node_mass(node_gid); - }); // end parallel for + }); // end parallel for } // end view scope Kokkos::fence(); // communicate ghost densities @@ -568,19 +565,15 @@ double FEA_Module_SGH::average_element_density(const int nodes_per_elem, const C void FEA_Module_SGH::compute_topology_optimization_adjoint_full() { - const int num_dim = simparam->num_dims; const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + size_t num_bdy_nodes = mesh->num_bdy_nodes; const DCArrayKokkos boundary = module_params->boundary; const DCArrayKokkos material = simparam->material; - - real_t global_dt; - size_t current_data_index, next_data_index; - size_t num_bdy_nodes = mesh->num_bdy_nodes; - - Teuchos::RCP previous_adjoint_vector_distributed, current_adjoint_vector_distributed; - Teuchos::RCP previous_velocity_vector_distributed, current_velocity_vector_distributed; + const int num_dim = simparam->num_dims; + real_t global_dt; + size_t current_data_index, next_data_index; + Teuchos::RCP previous_adjoint_vector_distributed, current_adjoint_vector_distributed, previous_velocity_vector_distributed, current_velocity_vector_distributed; Teuchos::RCP previous_phi_adjoint_vector_distributed, current_phi_adjoint_vector_distributed; - // initialize first adjoint vector at last_time_step to 0 as the terminal value (*adjoint_vector_data)[last_time_step + 1]->putScalar(0); (*phi_adjoint_vector_data)[last_time_step + 1]->putScalar(0); @@ -597,7 +590,6 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full() { // compute timestep from time data global_dt = time_data[cycle + 1] - time_data[cycle]; - // print if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) { @@ -617,6 +609,10 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full() } } // end if } + // else if (cycle==1){ + // if(myrank==0) + // printf("cycle = %lu, time = %f, time step = %f \n", cycle-1, time_data[cycle-1], global_dt); + // } // end if // compute adjoint vector for this data point; use velocity midpoint // view scope @@ -639,12 +635,12 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full() node_vel(rk_level, node_gid, idim) = previous_velocity_vector(node_gid, idim); node_coords(rk_level, node_gid, idim) = previous_coordinate_vector(node_gid, idim); } - }); + }); Kokkos::fence(); FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { elem_sie(rk_level, elem_gid) = previous_element_internal_energy(elem_gid, 0); - }); + }); Kokkos::fence(); // set state according to phase data at this timestep @@ -820,7 +816,6 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full() const_vec_array previous_adjoint_vector = (*adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); const_vec_array phi_previous_adjoint_vector = (*phi_adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); const_vec_array psi_previous_adjoint_vector = (*psi_adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); - vec_array midpoint_adjoint_vector = adjoint_vector_distributed->getLocalView(Tpetra::Access::ReadWrite); vec_array phi_midpoint_adjoint_vector = phi_adjoint_vector_distributed->getLocalView(Tpetra::Access::ReadWrite); vec_array psi_midpoint_adjoint_vector = psi_adjoint_vector_distributed->getLocalView(Tpetra::Access::ReadWrite); @@ -848,20 +843,14 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full() elem_id = elems_in_node(node_gid, ielem); matrix_contribution += psi_previous_adjoint_vector(elem_id, 0) * Power_Gradient_Velocities(node_gid * num_dim + idim, ielem); } - rate_of_change = previous_velocity_vector(node_gid, idim) - matrix_contribution / node_mass(node_gid) - phi_previous_adjoint_vector(node_gid, idim) / node_mass(node_gid); midpoint_adjoint_vector(node_gid, idim) = -rate_of_change * global_dt / 2 + previous_adjoint_vector(node_gid, idim); } - }); // end parallel for + }); // end parallel for Kokkos::fence(); - // apply BCs to adjoint vector, only matters for the momentum adjoint if using strictly velocity boundary conditions - boundary_adjoint(*mesh, boundary, midpoint_adjoint_vector, phi_midpoint_adjoint_vector, psi_midpoint_adjoint_vector); - - comm_adjoint_vector(cycle); - // half step update for RK2 scheme; EQUATION 2 FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { real_t rate_of_change; @@ -890,11 +879,9 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full() // rate_of_change = -0.0000001*previous_adjoint_vector(node_gid,idim); phi_midpoint_adjoint_vector(node_gid, idim) = -rate_of_change * global_dt / 2 + phi_previous_adjoint_vector(node_gid, idim); } - }); // end parallel for + }); // end parallel for Kokkos::fence(); - comm_phi_adjoint_vector(cycle); - // phi_adjoint_vector_distributed->describe(*fos,Teuchos::VERB_EXTREME); // half step update for RK2 scheme; EQUATION 3 @@ -914,9 +901,14 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full() rate_of_change = -(matrix_contribution + psi_previous_adjoint_vector(elem_gid, 0) * Power_Gradient_Energies(elem_gid)) / elem_mass(elem_gid); // rate_of_change = -0.0000001*previous_adjoint_vector(node_gid,idim); psi_midpoint_adjoint_vector(elem_gid, 0) = -rate_of_change * global_dt / 2 + psi_previous_adjoint_vector(elem_gid, 0); - }); // end parallel for + }); // end parallel for Kokkos::fence(); + // apply BCs to adjoint vector, only matters for the momentum adjoint if using strictly velocity boundary conditions + boundary_adjoint(*mesh, boundary, midpoint_adjoint_vector, phi_midpoint_adjoint_vector, psi_midpoint_adjoint_vector); + comm_adjoint_vector(cycle); + comm_phi_adjoint_vector(cycle); + // save for second half of RK (*psi_adjoint_vector_data)[cycle]->assign(*psi_adjoint_vector_distributed); @@ -928,6 +920,184 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full() phi_midpoint_adjoint_vector = (*phi_adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadWrite); psi_midpoint_adjoint_vector = (*psi_adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadWrite); + // compute gradients at midpoint + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes + nghost_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + node_vel(rk_level, node_gid, idim) = 0.5 * (previous_velocity_vector(node_gid, idim) + current_velocity_vector(node_gid, idim)); + node_coords(rk_level, node_gid, idim) = 0.5 * (previous_coordinate_vector(node_gid, idim) + current_coordinate_vector(node_gid, idim)); + } + }); + Kokkos::fence(); + + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { + elem_sie(rk_level, elem_gid) = 0.5 * (previous_element_internal_energy(elem_gid, 0) + current_element_internal_energy(elem_gid, 0)); + }); + Kokkos::fence(); + + // set state according to phase data at this timestep + + get_vol(); + + // ---- Calculate velocity diveregence for the element ---- + if (num_dim == 2) + { + get_divergence2D(elem_div, + node_coords, + node_vel, + elem_vol); + } + else + { + get_divergence(elem_div, + node_coords, + node_vel, + elem_vol); + } // end if 2D + + // ---- Calculate elem state (den, pres, sound speed, stress) for next time step ---- + if (num_dim == 2) + { + update_state2D(material, + *mesh, + node_coords, + node_vel, + elem_den, + elem_pres, + elem_stress, + elem_sspd, + elem_sie, + elem_vol, + elem_mass, + elem_mat_id, + 1.0, + cycle); + } + else + { + update_state(material, + *mesh, + node_coords, + node_vel, + elem_den, + elem_pres, + elem_stress, + elem_sspd, + elem_sie, + elem_vol, + elem_mass, + elem_mat_id, + 1.0, + cycle); + } + + if (num_dim == 2) + { + get_force_sgh2D(material, + *mesh, + node_coords, + node_vel, + elem_den, + elem_sie, + elem_pres, + elem_stress, + elem_sspd, + elem_vol, + elem_div, + elem_mat_id, + corner_force, + 1.0, + cycle); + } + else + { + get_force_sgh(material, + *mesh, + node_coords, + node_vel, + elem_den, + elem_sie, + elem_pres, + elem_stress, + elem_sspd, + elem_vol, + elem_div, + elem_mat_id, + corner_force, + 1.0, + cycle); + } + + // compute gradient matrices + get_force_egradient_sgh(material, + *mesh, + node_coords, + node_vel, + elem_den, + elem_sie, + elem_pres, + elem_stress, + elem_sspd, + elem_vol, + elem_div, + elem_mat_id, + 1.0, + cycle); + + get_power_egradient_sgh(1.0, + *mesh, + node_vel, + node_coords, + elem_sie, + elem_mass, + corner_force); + + get_force_vgradient_sgh(material, + *mesh, + node_coords, + node_vel, + elem_den, + elem_sie, + elem_pres, + elem_stress, + elem_sspd, + elem_vol, + elem_div, + elem_mat_id, + 1.0, + cycle); + + get_power_vgradient_sgh(1.0, + *mesh, + node_vel, + node_coords, + elem_sie, + elem_mass, + corner_force); + + get_force_ugradient_sgh(material, + *mesh, + node_coords, + node_vel, + elem_den, + elem_sie, + elem_pres, + elem_stress, + elem_sspd, + elem_vol, + elem_div, + elem_mat_id, + 1.0, + cycle); + + get_power_ugradient_sgh(1.0, + *mesh, + node_vel, + node_coords, + elem_sie, + elem_mass, + corner_force); + // full step update with midpoint gradient for RK2 scheme; EQUATION 1 FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { real_t rate_of_change; @@ -958,12 +1128,9 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full() phi_midpoint_adjoint_vector(node_gid, idim) / node_mass(node_gid); current_adjoint_vector(node_gid, idim) = -rate_of_change * global_dt + previous_adjoint_vector(node_gid, idim); } - }); // end parallel for + }); // end parallel for Kokkos::fence(); - boundary_adjoint(*mesh, boundary, current_adjoint_vector, phi_current_adjoint_vector, psi_midpoint_adjoint_vector); - comm_adjoint_vector(cycle); - // full step update with midpoint gradient for RK2 scheme; EQUATION 2 FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { real_t rate_of_change; @@ -992,11 +1159,9 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full() // rate_of_change = -0.0000001*midpoint_adjoint_vector(node_gid,idim); phi_current_adjoint_vector(node_gid, idim) = -rate_of_change * global_dt + phi_previous_adjoint_vector(node_gid, idim); } - }); // end parallel for + }); // end parallel for Kokkos::fence(); - comm_phi_adjoint_vector(cycle); - // full step update for RK2 scheme; EQUATION 3 FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { real_t rate_of_change; @@ -1015,9 +1180,12 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full() // debug // std::cout << "PSI RATE OF CHANGE " << rate_of_change << std::endl; psi_current_adjoint_vector(elem_gid, 0) = -rate_of_change * global_dt + psi_previous_adjoint_vector(elem_gid, 0); - }); // end parallel for + }); // end parallel for Kokkos::fence(); + boundary_adjoint(*mesh, boundary, current_adjoint_vector, phi_current_adjoint_vector, psi_current_adjoint_vector); + comm_adjoint_vector(cycle); + comm_phi_adjoint_vector(cycle); // save data from time-step completion (*psi_adjoint_vector_data)[cycle]->assign(*psi_adjoint_vector_distributed); } // end view scope @@ -1032,18 +1200,16 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full() void FEA_Module_SGH::compute_topology_optimization_gradient_full(Teuchos::RCP design_densities_distributed, Teuchos::RCP design_gradients_distributed) { - bool element_constant_density = true; size_t num_bdy_nodes = mesh->num_bdy_nodes; - size_t current_data_index, next_data_index; - int num_corners = rnum_elem * num_nodes_in_elem; - real_t global_dt; - const DCArrayKokkos boundary = module_params->boundary; const DCArrayKokkos material = simparam->material; - const int num_dim = simparam->num_dims; - - auto current_element_velocities = CArrayKokkos(num_nodes_in_elem, num_dim); - auto current_element_adjoint = CArrayKokkos(num_nodes_in_elem, num_dim); + const int num_dim = simparam->num_dims; + int num_corners = rnum_elem * num_nodes_in_elem; + real_t global_dt; + bool element_constant_density = true; + size_t current_data_index, next_data_index; + CArrayKokkos current_element_velocities = CArrayKokkos(num_nodes_in_elem, num_dim); + CArrayKokkos current_element_adjoint = CArrayKokkos(num_nodes_in_elem, num_dim); if (myrank == 0) { @@ -1154,8 +1320,7 @@ void FEA_Module_SGH::compute_topology_optimization_gradient_full(Teuchos::RCP(max_stride, "current_row_nodes_scanned"); // allocate sparse graph with node repeats - RaggedRightArrayKokkos Repeat_Graph_Matrix(Graph_Matrix_Strides_initial); - RaggedRightArrayofVectorsKokkos Element_local_indices(Graph_Matrix_Strides_initial, num_dim); + RaggedRightArrayKokkos Repeat_Graph_Matrix(Graph_Matrix_Strides_initial); + RaggedRightArrayofVectorsKokkos Element_local_indices(Graph_Matrix_Strides_initial, 3); // Fill the initial Graph with repeats if (num_dim == 2) @@ -1626,7 +1793,6 @@ void FEA_Module_SGH::init_assembly() int element_column_index; int current_stride; int current_row_n_nodes_scanned; - for (int inode = 0; inode < nlocal_nodes; inode++) { current_row_n_nodes_scanned = 0; @@ -1687,19 +1853,19 @@ void FEA_Module_SGH::init_assembly() node_indices_used(current_row_nodes_scanned(node_reset)) = 0; } } - }); + }); Kokkos::fence(); Graph_Matrix_Strides.update_host(); // copy reduced content to non_repeat storage - Graph_Matrix = RaggedRightArrayKokkos(Graph_Matrix_Strides); + Graph_Matrix = RaggedRightArrayKokkos(Graph_Matrix_Strides); FOR_ALL_CLASS(inode, 0, nlocal_nodes, { for (int istride = 0; istride < Graph_Matrix_Strides(inode); istride++) { Graph_Matrix(inode, istride) = Repeat_Graph_Matrix(inode, istride); } - }); // end parallel for + }); // end parallel for // deallocate repeat matrix @@ -1710,7 +1876,7 @@ void FEA_Module_SGH::init_assembly() // expand strides for stiffness matrix by multipling by dim FOR_ALL_CLASS(idof, 0, num_dim * nlocal_nodes, { Gradient_Matrix_Strides(idof) = num_dim * Graph_Matrix_Strides(idof / num_dim); - }); // end parallel for + }); // end parallel for Gradient_Matrix_Strides.update_host(); @@ -1727,11 +1893,11 @@ void FEA_Module_SGH::init_assembly() // increment the number of corners saved to this node_gid count_saved_corners_in_node(node_gid)++; - }); // end FOR_ALL over nodes in element + }); // end FOR_ALL over nodes in element Kokkos::fence(); } // end for elem_gid - DOF_Graph_Matrix = RaggedRightArrayKokkos(Gradient_Matrix_Strides); + DOF_Graph_Matrix = RaggedRightArrayKokkos(Gradient_Matrix_Strides); Force_Gradient_Positions = RaggedRightArrayKokkos(Gradient_Matrix_Strides); Force_Gradient_Velocities = RaggedRightArrayKokkos(Gradient_Matrix_Strides); // needs different graph of node to elem rather than node to node @@ -1743,7 +1909,7 @@ void FEA_Module_SGH::init_assembly() { DOF_to_Elem_Matrix_Strides(inode * num_dim + idim) = count_saved_corners_in_node(inode); } - }); // end parallel for + }); // end parallel for DOF_to_Elem_Matrix_Strides.update_host(); Force_Gradient_Energies = CArrayKokkos(rnum_elem, num_nodes_in_elem * num_dim); Power_Gradient_Energies = CArrayKokkos(rnum_elem); @@ -1758,7 +1924,7 @@ void FEA_Module_SGH::init_assembly() { DOF_Graph_Matrix(idof, istride) = Graph_Matrix(idof / num_dim, istride / num_dim) * num_dim + istride % num_dim; } - }); // end parallel for + }); // end parallel for /* //construct distributed gradient matrix from local kokkos data @@ -1843,7 +2009,8 @@ void FEA_Module_SGH::boundary_adjoint(const mesh_t& mesh, // Set velocity to zero in that directdion if (bdy_node_gid < nlocal_nodes) { - node_adjoint(bdy_node_gid, direction) = 0.0; + node_adjoint(bdy_node_gid, direction) = 0.0; + node_phi_adjoint(bdy_node_gid, direction) = 0.0; } // node_phi_adjoint(bdy_node_gid, direction) = 0.0; } @@ -1859,7 +2026,8 @@ void FEA_Module_SGH::boundary_adjoint(const mesh_t& mesh, // Set velocity to zero if (bdy_node_gid < nlocal_nodes) { - node_adjoint(bdy_node_gid, dim) = 0.0; + node_adjoint(bdy_node_gid, dim) = 0.0; + node_phi_adjoint(bdy_node_gid, dim) = 0.0; } // node_phi_adjoint(bdy_node_gid, dim) = 0.0; } From ba1f6e846480d18a43dcd56279620b2d1223b45e Mon Sep 17 00:00:00 2001 From: Jacob Moore Date: Fri, 23 Feb 2024 19:43:24 -0600 Subject: [PATCH 30/64] DOCS: Adding doxygen documentation to SGH coding --- .../SGH_Solver/include/FEA_Module_SGH.h | 1216 ++++++++--------- .../SGH_Solver/include/mesh.h | 96 +- .../SGH_Solver/include/state.h | 30 +- .../SGH_Solver/src/FEA_Module_SGH.cpp | 239 ++-- .../SGH_Solver/src/boundary.cpp | 13 +- .../SGH_Solver/src/energy_sgh.cpp | 16 +- .../SGH_Solver/src/force_gradients_sgh.cpp | 130 +- .../SGH_Solver/src/force_sgh.cpp | 77 +- .../SGH_Solver/src/geometry.cpp | 180 ++- .../SGH_Solver/src/momentum.cpp | 109 +- .../SGH_Solver/src/power_gradients_sgh.cpp | 89 +- .../SGH_Solver/src/properties.cpp | 47 +- .../SGH_Solver/src/setup_sgh.cpp | 733 +++++----- .../SGH_Solver/src/sgh_optimization.cpp | 111 +- .../SGH_Solver/src/time_integration.cpp | 59 +- 15 files changed, 1905 insertions(+), 1240 deletions(-) diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/FEA_Module_SGH.h b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/FEA_Module_SGH.h index b233802d2..31b1fd76d 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/FEA_Module_SGH.h +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/FEA_Module_SGH.h @@ -11,14 +11,11 @@ This program is open source under the BSD-3 License. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. @@ -58,643 +55,636 @@ struct material_t; struct boundary_t; - -class FEA_Module_SGH: public FEA_Module{ - +///////////////////////////////////////////////////////////////////////////// +/// +/// \class FEA_Module_SGH +/// +/// \brief Class for containing functions required to perform SGH +/// +/// This class containts the requisite functions requited to perform +/// staggered grid hydrodynamics (SGH) which is equivalent to a lumped +/// mass finite element (FE) scheme. +/// +///////////////////////////////////////////////////////////////////////////// +class FEA_Module_SGH : public FEA_Module +{ public: - - FEA_Module_SGH(SGH_Parameters& params, Solver *Solver_Pointer, std::shared_ptr mesh_in, const int my_fea_module_index = 0); - ~FEA_Module_SGH(); - - //initialize data for boundaries of the model and storage for boundary conditions and applied loads - void sgh_interface_setup(node_t &node, elem_t &elem, corner_t &corner); - - void setup(); - - void cleanup_material_models(); - - int solve(); - - void module_cleanup(); - - void sgh_solve(); - - void get_force_sgh(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_pres, - DViewCArrayKokkos &elem_stress, - const DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &elem_mat_id, - DViewCArrayKokkos &corner_force, - const double rk_alpha, - const size_t cycle); - - void get_force_vgradient_sgh(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &elem_mat_id, - const double rk_alpha, - const size_t cycle); - - void get_force_ugradient_sgh(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &elem_mat_id, - const double rk_alpha, - const size_t cycle); - - void get_force_egradient_sgh(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &elem_mat_id, - const double rk_alpha, - const size_t cycle); - - void get_force_dgradient_sgh(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &elem_mat_id, - const double rk_alpha, - const size_t cycle); - - void force_design_gradient_term(const_vec_array design_variables, vec_array design_gradients); - - - void get_force_sgh2D(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &elem_mat_id, - DViewCArrayKokkos &corner_force, - const double rk_alpha, - const size_t cycle); - - void update_position_sgh(double rk_alpha, - const size_t num_nodes, - DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel); - void get_vol(); + FEA_Module_SGH(SGH_Parameters& params, Solver* Solver_Pointer, std::shared_ptr mesh_in, const int my_fea_module_index = 0); + ~FEA_Module_SGH(); - void get_vol_ugradient(const size_t gradient_node_id, const size_t gradient_dim); + // initialize data for boundaries of the model and storage for boundary conditions and applied loads + void sgh_interface_setup(node_t& node, elem_t& elem, corner_t& corner); - void init_assembly(); + void setup(); - KOKKOS_INLINE_FUNCTION - void get_vol_hex(const DViewCArrayKokkos &elem_vol, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const; - - KOKKOS_FUNCTION - void get_vol_hex_ugradient(const ViewCArrayKokkos &elem_vol_gradients, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const; + void cleanup_material_models(); + int solve(); - KOKKOS_INLINE_FUNCTION - void get_vol_quad(const DViewCArrayKokkos &elem_vol, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const; + void module_cleanup(); + void sgh_solve(); - KOKKOS_FUNCTION - double get_area_quad(const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const; - - - KOKKOS_FUNCTION - void get_bmatrix(const ViewCArrayKokkos &B_matrix, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const; - - KOKKOS_FUNCTION - void get_bmatrix_gradients(const ViewCArrayKokkos &B_matrix_gradients, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const; - + void get_force_sgh(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_pres, + DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + DViewCArrayKokkos& corner_force, + const double rk_alpha, + const size_t cycle); - KOKKOS_FUNCTION - void get_bmatrix2D(const ViewCArrayKokkos &B_matrix, + void get_force_vgradient_sgh(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_pres, + const DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + const double rk_alpha, + const size_t cycle); + + void get_force_ugradient_sgh(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_pres, + const DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + const double rk_alpha, + const size_t cycle); + + void get_force_egradient_sgh(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_pres, + const DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + const double rk_alpha, + const size_t cycle); + + void get_force_dgradient_sgh(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_pres, + const DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + const double rk_alpha, + const size_t cycle); + + void force_design_gradient_term(const_vec_array design_variables, vec_array design_gradients); + + void get_force_sgh2D(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_pres, + const DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + DViewCArrayKokkos& corner_force, + const double rk_alpha, + const size_t cycle); + + void update_position_sgh(double rk_alpha, + const size_t num_nodes, + DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel); + + void get_vol(); + + void get_vol_ugradient(const size_t gradient_node_id, const size_t gradient_dim); + + void init_assembly(); + + KOKKOS_INLINE_FUNCTION + void get_vol_hex(const DViewCArrayKokkos& elem_vol, const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, const size_t rk_level) const; - KOKKOS_FUNCTION - void get_area_weights2D(const ViewCArrayKokkos &corner_areas, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const; - - - KOKKOS_INLINE_FUNCTION - double heron(const double x1, - const double y1, - const double x2, - const double y2, - const double x3, - const double y3) const; - - double average_element_density(const int nodes_per_elem, const CArray current_element_densities) const; - - void get_divergence(DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_vol); - - - void get_divergence2D(DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_vol); - - - KOKKOS_FUNCTION - void get_velgrad(ViewCArrayKokkos &vel_grad, - const ViewCArrayKokkos &elem_node_gids, - const DViewCArrayKokkos &node_vel, - const ViewCArrayKokkos &b_matrix, - const double elem_vol, - const size_t elem_gid, - const size_t rk_level) const; + KOKKOS_FUNCTION + void get_vol_hex_ugradient(const ViewCArrayKokkos& elem_vol_gradients, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const; + + KOKKOS_INLINE_FUNCTION + void get_vol_quad(const DViewCArrayKokkos& elem_vol, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const; + + KOKKOS_FUNCTION + double get_area_quad(const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const; + + KOKKOS_FUNCTION + void get_bmatrix(const ViewCArrayKokkos& B_matrix, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const; + KOKKOS_FUNCTION + void get_bmatrix_gradients(const ViewCArrayKokkos& B_matrix_gradients, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const; + + KOKKOS_FUNCTION + void get_bmatrix2D(const ViewCArrayKokkos& B_matrix, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const; - KOKKOS_FUNCTION - void get_velgrad2D(ViewCArrayKokkos &vel_grad, - const ViewCArrayKokkos &elem_node_gids, - const DViewCArrayKokkos &node_vel, - const ViewCArrayKokkos &b_matrix, + KOKKOS_FUNCTION + void get_area_weights2D(const ViewCArrayKokkos& corner_areas, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const; + + KOKKOS_INLINE_FUNCTION + double heron(const double x1, + const double y1, + const double x2, + const double y2, + const double x3, + const double y3) const; + + double average_element_density(const int nodes_per_elem, const CArray current_element_densities) const; + + void get_divergence(DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_vol); + + void get_divergence2D(DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_vol); + + KOKKOS_FUNCTION + void get_velgrad(ViewCArrayKokkos& vel_grad, + const ViewCArrayKokkos& elem_node_gids, + const DViewCArrayKokkos& node_vel, + const ViewCArrayKokkos& b_matrix, const double elem_vol, - const double elem_area, const size_t elem_gid, const size_t rk_level) const; - KOKKOS_INLINE_FUNCTION - void decompose_vel_grad(ViewCArrayKokkos &D_tensor, - ViewCArrayKokkos &W_tensor, - const ViewCArrayKokkos &vel_grad, - const ViewCArrayKokkos &elem_node_gids, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const double vol) const; - - - void update_velocity_sgh(double rk_alpha, - DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &node_mass, - const DViewCArrayKokkos &corner_force); - - void tag_bdys(const DCArrayKokkos &boundary, - mesh_t &mesh, - const DViewCArrayKokkos &node_coords); - - void boundary_velocity(const mesh_t &mesh, - const DCArrayKokkos &boundary, - DViewCArrayKokkos &node_vel); - - KOKKOS_INLINE_FUNCTION - bool check_bdy(const size_t patch_gid, - const int num_dim, - const int num_nodes_in_patch, + KOKKOS_FUNCTION + void get_velgrad2D(ViewCArrayKokkos& vel_grad, + const ViewCArrayKokkos& elem_node_gids, + const DViewCArrayKokkos& node_vel, + const ViewCArrayKokkos& b_matrix, + const double elem_vol, + const double elem_area, + const size_t elem_gid, + const size_t rk_level) const; + + KOKKOS_INLINE_FUNCTION + void decompose_vel_grad(ViewCArrayKokkos& D_tensor, + ViewCArrayKokkos& W_tensor, + const ViewCArrayKokkos& vel_grad, + const ViewCArrayKokkos& elem_node_gids, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const double vol) const; + + void update_velocity_sgh(double rk_alpha, + DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_mass, + const DViewCArrayKokkos& corner_force); + + void tag_bdys(const DCArrayKokkos& boundary, + mesh_t& mesh, + const DViewCArrayKokkos& node_coords); + + void boundary_velocity(const mesh_t& mesh, + const DCArrayKokkos& boundary, + DViewCArrayKokkos& node_vel); + + KOKKOS_INLINE_FUNCTION + bool check_bdy(const size_t patch_gid, + const int num_dim, + const int num_nodes_in_patch, const BOUNDARY_TYPE this_bc_tag, const double val, - const DViewCArrayKokkos &node_coords, + const DViewCArrayKokkos& node_coords, const size_t rk_level) const; - void rk_init(DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &elem_sie, - DViewCArrayKokkos &elem_stress, - const size_t num_elems, - const size_t num_nodes); - - - void get_timestep(mesh_t &mesh, - DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &elem_sspd, - DViewCArrayKokkos &elem_vol); - - - void get_timestep2D(mesh_t &mesh, - DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &elem_sspd, - DViewCArrayKokkos &elem_vol); - - void update_energy_sgh(double rk_alpha, - const mesh_t &mesh, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_mass, - const DViewCArrayKokkos &corner_force); - - - void power_design_gradient_term(const_vec_array design_variables, vec_array design_gradients); - - void get_power_dgradient_sgh(double rk_alpha, - const mesh_t &mesh, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_mass, - const DViewCArrayKokkos &corner_force, - DCArrayKokkos elem_power_dgradients); - - void get_power_ugradient_sgh(double rk_alpha, - const mesh_t &mesh, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_mass, - const DViewCArrayKokkos &corner_force); - - void get_power_vgradient_sgh(double rk_alpha, - const mesh_t &mesh, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_mass, - const DViewCArrayKokkos &corner_force); - - void get_power_egradient_sgh(double rk_alpha, - const mesh_t &mesh, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_mass, - const DViewCArrayKokkos &corner_force); - - void update_state(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &elem_den, - DViewCArrayKokkos &elem_pres, - DViewCArrayKokkos &elem_stress, - DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_mass, - const DViewCArrayKokkos &elem_mat_id, - const double rk_alpha, - const size_t cycle); - - - void update_state2D(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &elem_den, - DViewCArrayKokkos &elem_pres, - DViewCArrayKokkos &elem_stress, - DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_mass, - const DViewCArrayKokkos &elem_mat_id, + void rk_init(DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_sie, + DViewCArrayKokkos& elem_stress, + const size_t num_elems, + const size_t num_nodes); + + void get_timestep(mesh_t& mesh, + DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_sspd, + DViewCArrayKokkos& elem_vol); + + void get_timestep2D(mesh_t& mesh, + DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_sspd, + DViewCArrayKokkos& elem_vol); + + void update_energy_sgh(double rk_alpha, + const mesh_t& mesh, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& corner_force); + + void power_design_gradient_term(const_vec_array design_variables, vec_array design_gradients); + + void get_power_dgradient_sgh(double rk_alpha, + const mesh_t& mesh, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& corner_force, + DCArrayKokkos elem_power_dgradients); + + void get_power_ugradient_sgh(double rk_alpha, + const mesh_t& mesh, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& corner_force); + + void get_power_vgradient_sgh(double rk_alpha, + const mesh_t& mesh, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& corner_force); + + void get_power_egradient_sgh(double rk_alpha, + const mesh_t& mesh, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& corner_force); + + void update_state(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_den, + DViewCArrayKokkos& elem_pres, + DViewCArrayKokkos& elem_stress, + DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& elem_mat_id, const double rk_alpha, const size_t cycle); - void build_boundry_node_sets(mesh_t &mesh); - - void init_boundaries(); + void update_state2D(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_den, + DViewCArrayKokkos& elem_pres, + DViewCArrayKokkos& elem_stress, + DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& elem_mat_id, + const double rk_alpha, + const size_t cycle); + + void build_boundry_node_sets(mesh_t& mesh); + + void init_boundaries(); + + // initializes memory for arrays used in the global stiffness matrix assembly + void init_boundary_sets(int num_boundary_sets); + + void grow_boundary_sets(int num_boundary_sets); + + virtual void update_forward_solve(Teuchos::RCP zp); + + void comm_node_masses(); + + void comm_adjoint_vector(int cycle); + + void comm_phi_adjoint_vector(int cycle); + + void comm_variables(Teuchos::RCP zp); + + void read_conditions_ansys_dat(std::ifstream* in, std::streampos before_condition_header); + + // interfaces between user input and creating data structures for bcs + void generate_bcs(); + + void Displacement_Boundary_Conditions(); + + void init_output(); + + void compute_output(); + + void output_control(); + + void sort_output(Teuchos::RCP> sorted_map); + + void sort_element_output(Teuchos::RCP> sorted_map); + + void collect_output(Teuchos::RCP> global_reduce_map); + + void write_data(std::map& point_data_scalars_double, + std::map& point_data_vectors_double, + std::map& cell_data_scalars_double, + std::map& cell_data_scalars_int, + std::map>& cell_data_fields_double); + + void write_outputs(const mesh_t& mesh, + DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& node_mass, + DViewCArrayKokkos& elem_den, + DViewCArrayKokkos& elem_pres, + DViewCArrayKokkos& elem_stress, + DViewCArrayKokkos& elem_sspd, + DViewCArrayKokkos& elem_sie, + DViewCArrayKokkos& elem_vol, + DViewCArrayKokkos& elem_mass, + DViewCArrayKokkos& elem_mat_id); - //initializes memory for arrays used in the global stiffness matrix assembly - void init_boundary_sets(int num_boundary_sets); - - void grow_boundary_sets(int num_boundary_sets); - - virtual void update_forward_solve(Teuchos::RCP zp); - - void comm_node_masses(); - - void comm_adjoint_vector(int cycle); - - void comm_phi_adjoint_vector(int cycle); - - void comm_variables(Teuchos::RCP zp); - - void read_conditions_ansys_dat(std::ifstream *in, std::streampos before_condition_header); - - //interfaces between user input and creating data structures for bcs - void generate_bcs(); - - void Displacement_Boundary_Conditions(); - - void init_output(); - - void compute_output(); - - void output_control(); - - void sort_output(Teuchos::RCP > sorted_map); - - void sort_element_output(Teuchos::RCP > sorted_map); - - void collect_output(Teuchos::RCP > global_reduce_map); - - void write_data(std::map &point_data_scalars_double, - std::map &point_data_vectors_double, - std::map &cell_data_scalars_double, - std::map &cell_data_scalars_int, - std::map > &cell_data_fields_double); - - void write_outputs (const mesh_t &mesh, - DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &node_mass, - DViewCArrayKokkos &elem_den, - DViewCArrayKokkos &elem_pres, - DViewCArrayKokkos &elem_stress, - DViewCArrayKokkos &elem_sspd, - DViewCArrayKokkos &elem_sie, - DViewCArrayKokkos &elem_vol, - DViewCArrayKokkos &elem_mass, - DViewCArrayKokkos &elem_mat_id); - - - void ensight(const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &node_mass, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_mass, - const DViewCArrayKokkos &elem_mat_id); - - - void state_file(const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &node_mass, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_mass, - const DViewCArrayKokkos &elem_mat_id); - - void node_density_constraints(host_vec_array node_densities_lower_bound); - - void compute_topology_optimization_adjoint_full(); //Force depends on node coords and velocity - - void compute_topology_optimization_gradient_full(Teuchos::RCP design_densities_distributed, Teuchos::RCP design_gradients_distributed); - - void boundary_adjoint(const mesh_t &mesh, - const DCArrayKokkos &boundary, - vec_array &node_adjoint, - vec_array &node_phi_adjoint, - vec_array &node_psi_adjoint); - - void applied_forces(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &node_mass, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &elem_mat_id, - DViewCArrayKokkos &corner_force, - const double rk_alpha, - const size_t cycle); - - bool have_loading_conditions; - bool nodal_density_flag; - real_t penalty_power; - - Simulation_Parameters_Explicit *simparam; - SGH_Parameters *module_params; - Explicit_Solver *Explicit_Solver_Pointer_; - - elements::ref_element *ref_elem; - - std::shared_ptr mesh; - //shallow copies of mesh class views - size_t num_nodes_in_elem; - // corner ids in node - RaggedRightArrayKokkos corners_in_node; - CArrayKokkos num_corners_in_node; - - // elem ids in node - RaggedRightArrayKokkos elems_in_node; - - // node ids in node - RaggedRightArrayKokkos nodes_in_node; - CArrayKokkos num_nodes_in_node; - - // node ids in elem - DCArrayKokkos nodes_in_elem; - - // corner ids in elem - CArrayKokkos corners_in_elem; - - // elem ids in elem - RaggedRightArrayKokkos elems_in_elem; - CArrayKokkos num_elems_in_elem; - - // patch ids in elem - CArrayKokkos patches_in_elem; - - // node ids in a patch - CArrayKokkos nodes_in_patch; - - // element ids in a patch - CArrayKokkos elems_in_patch; - - // bdy nodes - CArrayKokkos bdy_nodes; - - //Topology optimization filter variable - DCArrayKokkos relative_element_densities; - - //Local FEA data - host_elem_conn_array interface_nodes_in_elem; //host view of element connectivity to nodes - - //Global FEA data - Teuchos::RCP node_velocities_distributed; - Teuchos::RCP initial_node_coords_distributed; - Teuchos::RCP all_initial_node_coords_distributed; - Teuchos::RCP initial_node_velocities_distributed; - Teuchos::RCP all_node_velocities_distributed; - Teuchos::RCP all_cached_node_velocities_distributed; - Teuchos::RCP node_masses_distributed; - Teuchos::RCP ghost_node_masses_distributed; - Teuchos::RCP adjoint_vector_distributed; - Teuchos::RCP phi_adjoint_vector_distributed; - Teuchos::RCP psi_adjoint_vector_distributed; - Teuchos::RCP element_internal_energy_distributed; - Teuchos::RCP>> forward_solve_velocity_data; - Teuchos::RCP>> forward_solve_coordinate_data; - Teuchos::RCP>> forward_solve_internal_energy_data; - Teuchos::RCP>> adjoint_vector_data; - Teuchos::RCP>> phi_adjoint_vector_data; - Teuchos::RCP>> psi_adjoint_vector_data; - Teuchos::RCP force_gradient_design; - Teuchos::RCP force_gradient_position; - Teuchos::RCP force_gradient_velocity; - - //Local FEA data - DCArrayKokkos Global_Gradient_Matrix_Assembly_Map; //Maps element local nodes to columns on ragged right node connectivity graph - DCArrayKokkos Element_Gradient_Matrix_Assembly_Map; //Maps element-node pair to columns on ragged right node to element connectivity - RaggedRightArrayKokkos Graph_Matrix; //stores local indices - RaggedRightArrayKokkos DOF_Graph_Matrix; //stores local indices - RaggedRightArrayKokkos Force_Gradient_Positions; - RaggedRightArrayKokkos Force_Gradient_Velocities; - CArrayKokkos Force_Gradient_Energies; //transposed such that elem ids correspond to rows - RaggedRightArrayKokkos Power_Gradient_Positions; //transposed such that node dofs correspond to rows - RaggedRightArrayKokkos Power_Gradient_Velocities; //transposed such that node dofs correspond to rows - CArrayKokkos Power_Gradient_Energies; - DCArrayKokkos Gradient_Matrix_Strides; - DCArrayKokkos DOF_to_Elem_Matrix_Strides; - DCArrayKokkos Elem_to_Elem_Matrix_Strides; - DCArrayKokkos Graph_Matrix_Strides; - RaggedRightArrayKokkos Original_Gradient_Entries; - RaggedRightArrayKokkos Original_Gradient_Entry_Indices; - DCArrayKokkos Original_Gradient_Entries_Strides; - - //distributed matrices - Teuchos::RCP distributed_force_gradient_positions; - Teuchos::RCP distributed_force_gradient_velocities; - - std::vector time_data; - int max_time_steps, last_time_step; - - // --------------------------------------------------------------------- - // state data type declarations (must stay in scope for output after run) - // --------------------------------------------------------------------- - node_t node_interface; - elem_t elem_interface; - corner_t corner_interface; - - //Dual View wrappers - // Dual Views of the individual node struct variables - DViewCArrayKokkos node_coords; - DViewCArrayKokkos node_vel; - DViewCArrayKokkos node_mass; - - // Dual Views of the individual elem struct variables - DViewCArrayKokkos elem_den; - DViewCArrayKokkos elem_pres; - DViewCArrayKokkos elem_stress; // always 3D even in 2D-RZ - DViewCArrayKokkos elem_sspd; - DViewCArrayKokkos elem_sie; - DViewCArrayKokkos elem_vol; - DViewCArrayKokkos elem_div; - DViewCArrayKokkos elem_mass; - DViewCArrayKokkos elem_mat_id; - - // Element velocity gradient - DCArrayKokkos elem_vel_grad; - - // for storing global variables used in user material model - DCArrayKokkos eos_global_vars; - DCArrayKokkos strength_global_vars; - - // for storing state variables used in user material model - DCArrayKokkos eos_state_vars; - DCArrayKokkos strength_state_vars; - - - //elem_user_output_vars allow users to output variables of interest per element - DCArrayKokkos elem_user_output_vars; - - //material models - DCArrayKokkos elem_eos; - DCArrayKokkos elem_strength; - - // Dual Views of the corner struct variables - DViewCArrayKokkos corner_force; - DViewCArrayKokkos corner_mass; - - //Boundary Conditions Data - DCArrayKokkos Local_Index_Boundary_Patches; - //CArray Patch_Nodes; - enum bc_type {NONE, POINT_LOADING_CONDITION, LINE_LOADING_CONDITION, SURFACE_LOADING_CONDITION}; - - //Boundary Conditions Data - int max_boundary_sets; - - //output dof data - //Global arrays with collected data used to print - int output_velocity_index, output_strain_index, output_stress_index; - - //parameters - double time_value, time_final, dt, dt_max, dt_min, dt_cfl, graphics_time, graphics_dt_ival; - size_t graphics_cyc_ival, cycle_stop, rk_num_stages, graphics_id; - double fuzz, tiny, small; - CArray graphics_times; - int rk_num_bins; - - //optimization flags - bool kinetic_energy_objective; + void ensight(const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_mass, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_pres, + const DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& elem_mat_id); + + void state_file(const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_mass, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_pres, + const DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& elem_mat_id); + + void node_density_constraints(host_vec_array node_densities_lower_bound); + + void compute_topology_optimization_adjoint_full(); // Force depends on node coords and velocity + + void compute_topology_optimization_gradient_full(Teuchos::RCP design_densities_distributed, Teuchos::RCP design_gradients_distributed); + + void boundary_adjoint(const mesh_t& mesh, + const DCArrayKokkos& boundary, + vec_array& node_adjoint, + vec_array& node_phi_adjoint, + vec_array& node_psi_adjoint); + + void applied_forces(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_mass, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + DViewCArrayKokkos& corner_force, + const double rk_alpha, + const size_t cycle); + + bool have_loading_conditions; + bool nodal_density_flag; + real_t penalty_power; + + Simulation_Parameters_Explicit* simparam; + SGH_Parameters* module_params; + Explicit_Solver* Explicit_Solver_Pointer_; + + elements::ref_element* ref_elem; + + std::shared_ptr mesh; + // shallow copies of mesh class views + size_t num_nodes_in_elem; + // corner ids in node + RaggedRightArrayKokkos corners_in_node; + CArrayKokkos num_corners_in_node; + + // elem ids in node + RaggedRightArrayKokkos elems_in_node; + + // node ids in node + RaggedRightArrayKokkos nodes_in_node; + CArrayKokkos num_nodes_in_node; + + // node ids in elem + DCArrayKokkos nodes_in_elem; + + // corner ids in elem + CArrayKokkos corners_in_elem; + + // elem ids in elem + RaggedRightArrayKokkos elems_in_elem; + CArrayKokkos num_elems_in_elem; + + // patch ids in elem + CArrayKokkos patches_in_elem; + + // node ids in a patch + CArrayKokkos nodes_in_patch; + + // element ids in a patch + CArrayKokkos elems_in_patch; + + // bdy nodes + CArrayKokkos bdy_nodes; + + // Topology optimization filter variable + DCArrayKokkos relative_element_densities; + + // Local FEA data + host_elem_conn_array interface_nodes_in_elem; // host view of element connectivity to nodes + + // Global FEA data + Teuchos::RCP node_velocities_distributed; + Teuchos::RCP initial_node_coords_distributed; + Teuchos::RCP all_initial_node_coords_distributed; + Teuchos::RCP initial_node_velocities_distributed; + Teuchos::RCP all_node_velocities_distributed; + Teuchos::RCP all_cached_node_velocities_distributed; + Teuchos::RCP node_masses_distributed; + Teuchos::RCP ghost_node_masses_distributed; + Teuchos::RCP adjoint_vector_distributed; + Teuchos::RCP phi_adjoint_vector_distributed; + Teuchos::RCP psi_adjoint_vector_distributed; + Teuchos::RCP element_internal_energy_distributed; + Teuchos::RCP>> forward_solve_velocity_data; + Teuchos::RCP>> forward_solve_coordinate_data; + Teuchos::RCP>> forward_solve_internal_energy_data; + Teuchos::RCP>> adjoint_vector_data; + Teuchos::RCP>> phi_adjoint_vector_data; + Teuchos::RCP>> psi_adjoint_vector_data; + Teuchos::RCP force_gradient_design; + Teuchos::RCP force_gradient_position; + Teuchos::RCP force_gradient_velocity; + + // Local FEA data + DCArrayKokkos Global_Gradient_Matrix_Assembly_Map; // Maps element local nodes to columns on ragged right node connectivity graph + DCArrayKokkos Element_Gradient_Matrix_Assembly_Map; // Maps element-node pair to columns on ragged right node to element connectivity + RaggedRightArrayKokkos Graph_Matrix; // stores local indices + RaggedRightArrayKokkos DOF_Graph_Matrix; // stores local indices + RaggedRightArrayKokkos Force_Gradient_Positions; + RaggedRightArrayKokkos Force_Gradient_Velocities; + CArrayKokkos Force_Gradient_Energies; // transposed such that elem ids correspond to rows + RaggedRightArrayKokkos Power_Gradient_Positions; // transposed such that node dofs correspond to rows + RaggedRightArrayKokkos Power_Gradient_Velocities; // transposed such that node dofs correspond to rows + CArrayKokkos Power_Gradient_Energies; + DCArrayKokkos Gradient_Matrix_Strides; + DCArrayKokkos DOF_to_Elem_Matrix_Strides; + DCArrayKokkos Elem_to_Elem_Matrix_Strides; + DCArrayKokkos Graph_Matrix_Strides; + RaggedRightArrayKokkos Original_Gradient_Entries; + RaggedRightArrayKokkos Original_Gradient_Entry_Indices; + DCArrayKokkos Original_Gradient_Entries_Strides; + + // distributed matrices + Teuchos::RCP distributed_force_gradient_positions; + Teuchos::RCP distributed_force_gradient_velocities; + + std::vector time_data; + int max_time_steps, last_time_step; + + // --------------------------------------------------------------------- + // state data type declarations (must stay in scope for output after run) + // --------------------------------------------------------------------- + node_t node_interface; + elem_t elem_interface; + corner_t corner_interface; + + // Dual View wrappers + // Dual Views of the individual node struct variables + DViewCArrayKokkos node_coords; + DViewCArrayKokkos node_vel; + DViewCArrayKokkos node_mass; + + // Dual Views of the individual elem struct variables + DViewCArrayKokkos elem_den; + DViewCArrayKokkos elem_pres; + DViewCArrayKokkos elem_stress; // always 3D even in 2D-RZ + DViewCArrayKokkos elem_sspd; + DViewCArrayKokkos elem_sie; + DViewCArrayKokkos elem_vol; + DViewCArrayKokkos elem_div; + DViewCArrayKokkos elem_mass; + DViewCArrayKokkos elem_mat_id; + + // Element velocity gradient + DCArrayKokkos elem_vel_grad; + + // for storing global variables used in user material model + DCArrayKokkos eos_global_vars; + DCArrayKokkos strength_global_vars; + + // for storing state variables used in user material model + DCArrayKokkos eos_state_vars; + DCArrayKokkos strength_state_vars; + + // elem_user_output_vars allow users to output variables of interest per element + DCArrayKokkos elem_user_output_vars; + + // material models + DCArrayKokkos elem_eos; + DCArrayKokkos elem_strength; + + // Dual Views of the corner struct variables + DViewCArrayKokkos corner_force; + DViewCArrayKokkos corner_mass; + + // Boundary Conditions Data + DCArrayKokkos Local_Index_Boundary_Patches; + // CArray Patch_Nodes; + enum bc_type { NONE, POINT_LOADING_CONDITION, LINE_LOADING_CONDITION, SURFACE_LOADING_CONDITION }; + + // Boundary Conditions Data + int max_boundary_sets; + + // output dof data + // Global arrays with collected data used to print + int output_velocity_index, output_strain_index, output_stress_index; + + // parameters + double time_value, time_final, dt, dt_max, dt_min, dt_cfl, graphics_time, graphics_dt_ival; + size_t graphics_cyc_ival, cycle_stop, rk_num_stages, graphics_id; + double fuzz, tiny, small; + CArray graphics_times; + int rk_num_bins; + + // optimization flags + bool kinetic_energy_objective; }; #endif // end HEADER_H diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/mesh.h b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/mesh.h index d28d54457..09fb070ff 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/mesh.h +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/mesh.h @@ -163,7 +163,15 @@ struct mesh_t RaggedRightArrayKokkos bdy_nodes_in_set; DCArrayKokkos num_bdy_nodes_in_set; - // initialization methods + ///////////////////////////////////////////////////////////////////////////// + /// + /// \fn initialize_nodes + /// + /// \brief Initialize the number of nodes in the mesh + /// + /// \param The number of nodes in the mesh + /// + ///////////////////////////////////////////////////////////////////////////// void initialize_nodes(const size_t num_nodes_inp) { num_nodes = num_nodes_inp; @@ -171,7 +179,16 @@ struct mesh_t return; }; // end method - // initialization methods + ///////////////////////////////////////////////////////////////////////////// + /// + /// \fn initialize_local_nodes + /// + /// \brief Initialize the number of nodes in the mesh owned by a particular + // MPI rank + /// + /// \param The number of nodes in the mesh on this MPI rank + /// + ///////////////////////////////////////////////////////////////////////////// void initialize_local_nodes(const size_t num_nodes_inp) { num_local_nodes = num_nodes_inp; @@ -179,7 +196,20 @@ struct mesh_t return; }; // end method - // initialization methods + ///////////////////////////////////////////////////////////////////////////// + /// + /// \fn initialize_elems + /// + /// \brief Initialize the elements and some associated connectivity memory + /// + /// Saves the number of elements and dimensions, as well as creates + /// storage for the data structures for nodes-element connectivity and + /// corner-element connectivity + /// + /// \param Number of elements in the mesh + /// \param The number of spacial dimensions the mesh containts (3D, 2D) + /// + ///////////////////////////////////////////////////////////////////////////// void initialize_elems(const size_t num_elems_inp, const size_t num_dims_inp) { num_dims = num_dims_inp; @@ -195,7 +225,15 @@ struct mesh_t return; }; // end method - // initialization methods + ///////////////////////////////////////////////////////////////////////////// + /// + /// \fn initialize_corners + /// + /// \brief Initialize num_corners data + /// + /// \param Number of corners read in + /// + ///////////////////////////////////////////////////////////////////////////// void initialize_corners(const size_t num_corners_inp) { num_corners = num_corners_inp; @@ -203,7 +241,16 @@ struct mesh_t return; }; // end method - // build the corner mesh connectivity arrays + ///////////////////////////////////////////////////////////////////////////// + /// + /// \fn build_corner_connectivity + /// + /// \brief This class builds the corners-node, element-node, and + /// corner-element data structures. + /// + /// A corner is defined as a cell-node pair. + /// + ///////////////////////////////////////////////////////////////////////////// void build_corner_connectivity() { num_corners_in_node = CArrayKokkos(num_nodes); // stride sizes @@ -234,7 +281,7 @@ struct mesh_t count_saved_corners_in_node(node_gid) = 0; }); - // he elems_in_elem data type + // The elems_in_elem data type elems_in_node = RaggedRightArrayKokkos(num_corners_in_node, "elems_in_node"); // populate the elems connected to a node list and corners in a node @@ -265,7 +312,13 @@ struct mesh_t return; } // end of build_corner_connectivity - // build elem connectivity arrays + ///////////////////////////////////////////////////////////////////////////// + /// + /// \fn build_elem_elem_connectivity + /// + /// \brief This class builds the element-element connectivity structure + /// + ///////////////////////////////////////////////////////////////////////////// void build_elem_elem_connectivity() { // find the max number of elems around a node @@ -353,7 +406,17 @@ struct mesh_t return; } // end of build_elem_elem_connectivity - // build the patches + ///////////////////////////////////////////////////////////////////////////// + /// + /// \fn build_patch_connectivity + /// + /// \brief Builds the patch-element, node-patch, and element-patch as well as + /// tags all boundary nodes. + /// + /// A patch is defined a the surface of a linear element, for high order elements + /// a patch is the surface of a zone that is on the surface of an element. + /// + ///////////////////////////////////////////////////////////////////////////// void build_patch_connectivity() { // building patches @@ -638,7 +701,13 @@ struct mesh_t return; } // end patch connectivity method - // build the node-node connectivity data + ///////////////////////////////////////////////////////////////////////////// + /// + /// \fn build_node_node_connectivity + /// + /// \brief Builds the nodes-node connectivity structure + /// + ///////////////////////////////////////////////////////////////////////////// void build_node_node_connectivity() { // find the max number of elems around a node @@ -778,6 +847,15 @@ struct mesh_t }); // end parallel for over nodes } // end of node node connectivity + ///////////////////////////////////////////////////////////////////////////// + /// + /// \fn init_bdy_sets + /// + /// \brief Initialize boundary sets as dynamic ragged right arrays + /// + /// \param Number of boundary conditions + /// + ///////////////////////////////////////////////////////////////////////////// void init_bdy_sets(size_t num_bcs) { if (num_bcs == 0) diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/state.h b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/state.h index 0150235ac..ad614f172 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/state.h +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/state.h @@ -81,7 +81,21 @@ struct elem_t CArray mass; ///< element mass CArray mat_id; ///< element material id - // initialization method (num_rk_storage_bins, num_cells, num_dims) + ///////////////////////////////////////////////////////////////////////////// + /// + /// \fn initialize + /// + /// \brief Initialize state data associated with the element + /// + /// This function initializes storage for density, pressure, stress, + /// sound speed, specific internal energy, volume, divergence of velocity, + /// mass, and a material ID for each element in the mesh. + /// + /// \param Number of time integration bins required for the Runge Kutta scheme + /// \param Number of elements in the mesh + /// \param Number of spatial dimnesions + /// + ///////////////////////////////////////////////////////////////////////////// void initialize(size_t num_rk, size_t num_elems, size_t num_dims) { this->den = CArray(num_elems); @@ -102,7 +116,19 @@ struct corner_t CArray force; ///< Force acting on a corner CArray mass; ///< Partitioned mass of the corner - // initialization method (num_corners, num_dims) + ///////////////////////////////////////////////////////////////////////////// + /// + /// \fn initialize + /// + /// \brief Initialize state data associated with the corners + /// + /// This function initializes storage for force and mass associated with each + /// corner + /// + /// \param Number of corners in the mesh + /// \param Number of spatial dimnesions + /// + ///////////////////////////////////////////////////////////////////////////// void initialize(size_t num_corners, size_t num_dims) { this->force = CArray(num_corners, num_dims); diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/FEA_Module_SGH.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/FEA_Module_SGH.cpp index 967cf01a4..adfcf0bc7 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/FEA_Module_SGH.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/FEA_Module_SGH.cpp @@ -196,9 +196,13 @@ FEA_Module_SGH::~FEA_Module_SGH() // delete simparam; } -/* ---------------------------------------------------------------------- - Read ANSYS dat format mesh file -------------------------------------------------------------------------- */ +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn read_conditions_ansys_dat +/// +/// \brief Read ANSYS dat format mesh file +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_SGH::read_conditions_ansys_dat(std::ifstream* in, std::streampos before_condition_header) { auto input_options = simparam->input_options.value(); @@ -229,35 +233,46 @@ void FEA_Module_SGH::read_conditions_ansys_dat(std::ifstream* in, std::streampos host_vec_array node_densities; } // end read_conditions_ansys_dat -/* ---------------------------------------------------------------------- - Assign sets of element boundary surfaces corresponding to user BCs -------------------------------------------------------------------------- */ - +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn generate_bcs +/// +/// \brief Assign sets of element boundary surfaces corresponding to user BCs +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_SGH::generate_bcs() { } // end generate_bcs -/* ---------------------------------------------------------------------- - Loop through applied boundary conditions and tag node ids to remove - necessary rows and columns from the assembled linear system -------------------------------------------------------------------------- */ - +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn Displacement_Boundary_Conditions +/// +/// \brief Apply displacement boundary conditions +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_SGH::Displacement_Boundary_Conditions() { } -/* ---------------------------------------------------------------------------- - Output field settings and file settings -------------------------------------------------------------------------------- */ - +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn output_control +/// +/// \brief Output field settings and file settings +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_SGH::output_control() { } -/* ---------------------------------------------------------------------------- - Initialize output data structures -------------------------------------------------------------------------------- */ - +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn init_output +/// +/// \brief Initialize output data structures +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_SGH::init_output() { // check user parameters for output @@ -360,18 +375,31 @@ void FEA_Module_SGH::init_output() } } -/* ------------------------------------------------------------------------------------------- - Prompts sorting for elastic response output data. For now, nodal strains. ----------------------------------------------------------------------------------------------- */ - +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn sort_output +/// +/// \brief Prompts sorting for elastic response output data. For now, nodal strains. +/// Inactive in this context +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_SGH::sort_output(Teuchos::RCP> sorted_map) { } -/* ------------------------------------------------------------------------------------------- - populate requests this module makes for output data ----------------------------------------------------------------------------------------------- */ - +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn write_data +/// +/// \brief Populate requests this module makes for output data +/// +/// \param Scalar point data +/// \param Vector point data +/// \param Scalar cell data (double) +/// \param Scalar cell data (int) +/// \param Cell field data +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_SGH::write_data(std::map& point_data_scalars_double, std::map& point_data_vectors_double, std::map& cell_data_scalars_double, @@ -462,10 +490,13 @@ void FEA_Module_SGH::write_data(std::map& point_data // cell_data_fields_double["stress"] = std::make_pair(&sgh_module->elem_stress.host(rk_level,0,0,0), 9); } -/* ------------------------------------------------------------------------------------------- - Prompts sorting for elastic response output data. For now, nodal strains. ----------------------------------------------------------------------------------------------- */ - +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn sort_element_output +/// +/// \brief Prompts sorting for elementoutput data. For now, density. +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_SGH::sort_element_output(Teuchos::RCP> sorted_map) { // interface element density data @@ -479,26 +510,35 @@ void FEA_Module_SGH::sort_element_output(Teuchos::RCP> global_reduce_map) { } -/* ------------------------------------------------------------------------------------------- - Prompts computation of elastic response output data. For now, nodal strains. ----------------------------------------------------------------------------------------------- */ - +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn compute_output +/// +/// \brief Inactive in this context +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_SGH::compute_output() { } -/* ---------------------------------------------------------------------- - Communicate updated nodal velocities to ghost nodes -------------------------------------------------------------------------- */ - +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn comm_node_masses +/// +/// \brief Communicate updated nodal mass to ghost nodes +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_SGH::comm_node_masses() { // debug print of design vector @@ -533,10 +573,13 @@ void FEA_Module_SGH::comm_node_masses() #endif } -/* ------------------------------------------------------------------------------------------- - Communicate ghosts using the current optimization design data ----------------------------------------------------------------------------------------------- */ - +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn comm_variables +/// +/// \brief Communicate ghosts using the current optimization design data +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_SGH::comm_variables(Teuchos::RCP zp) { if (simparam->topology_optimization_on) @@ -568,9 +611,13 @@ void FEA_Module_SGH::comm_variables(Teuchos::RCP zp) } } -/* ------------------------------------------------------------------------------------------- - enforce density constraints on nodes due to BCS ----------------------------------------------------------------------------------------------- */ +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn node_density_constraints +/// +/// \brief Enforce density constraints on nodes due to BCS +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_SGH::node_density_constraints(host_vec_array node_densities_lower_bound) { const size_t num_dim = mesh->num_dims; @@ -610,22 +657,28 @@ void FEA_Module_SGH::node_density_constraints(host_vec_array node_densities_lowe }); // end for parallel for over nodes } -/* ---------------------------------------------------------------------------- - solve function called by solver -------------------------------------------------------------------------------- */ - +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn module_cleanup +/// +/// \brief Cleanup function called by solver +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_SGH::module_cleanup() { cleanup_material_models(); } -/* ---------------------------------------------------------------------------- - Deallocate memory used for material models -------------------------------------------------------------------------------- */ - -void FEA_Module_SGH::cleanup_material_models() { - - const DCArrayKokkos material = simparam->material; +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn cleanup_material_models +/// +/// \brief Deallocate memory used for material models +/// +///////////////////////////////////////////////////////////////////////////// +void FEA_Module_SGH::cleanup_material_models() +{ + const DCArrayKokkos material = simparam->material; // destroy strength model destroy_strength_model(elem_strength, @@ -649,14 +702,23 @@ void FEA_Module_SGH::cleanup_material_models() { elem_user_output_vars, rnum_elem); return; - } // end cleanup_user_strength_model; -/** - * Determines which of the boundary patches are associated with which boundary. - * - * Modifies: bdy_patches_in_set -*/ +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn tag_bdys +/// +/// \brief Determines which of the boundary patches are associated with which boundary. +/// +/// Modifies: bdy_patches_in_set +/// +/// \param Array of boundary sets +/// \param Simulation mesh +/// \param View into nodal position data +/// +/// \return +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_SGH::tag_bdys(const DCArrayKokkos& boundary, mesh_t& mesh, const DViewCArrayKokkos& node_coords) @@ -734,6 +796,25 @@ void FEA_Module_SGH::tag_bdys(const DCArrayKokkos& boundary, return; } // end tag +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn check_bdy +/// +/// \brief Checks if a given patch is on a defined boundary +/// +/// The test is done by checking if the centroid of the patch is within +/// 1.0e-7 of the boundary +/// +/// \param Global index of a patch +/// \param Number of spatial dimensions +/// \param Number of nodes in a patch +/// \param Type of boundary condition applied +/// \param View of nodal position data +/// \param Current Runge Kutta time integration step +/// +/// \return True if patch is on the surface of a defined boundary, else false +/// +///////////////////////////////////////////////////////////////////////////// KOKKOS_INLINE_FUNCTION bool FEA_Module_SGH::check_bdy(const size_t patch_gid, const int num_dim, @@ -809,10 +890,13 @@ bool FEA_Module_SGH::check_bdy(const size_t patch_gid, return is_on_bdy == num_nodes_in_patch; } // end method to check bdy -/* ---------------------------------------------------------------------------- - solve function called by solver -------------------------------------------------------------------------------- */ - +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn solve +/// +/// \brief Solve function called by solver +/// +///////////////////////////////////////////////////////////////////////////// int FEA_Module_SGH::solve() { sgh_solve(); @@ -820,10 +904,13 @@ int FEA_Module_SGH::solve() return 0; } -/* ---------------------------------------------------------------------------- - SGH solver loop -------------------------------------------------------------------------------- */ - +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn sgh_solve +/// +/// \brief SGH solver loop +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_SGH::sgh_solve() { Dynamic_Options dynamic_options = simparam->dynamic_options; diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/boundary.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/boundary.cpp index e65bec3d2..3c07262ae 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/boundary.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/boundary.cpp @@ -40,16 +40,11 @@ /// /// \fn boundary_velocity /// -/// \brief +/// \brief Evolves the boundary according to a give velocity /// -/// -/// -/// \param -/// \param -/// \param -/// -/// \return +/// \param The simulation mesh +/// \param An array of boundary_t that contain information about BCs +/// \param A view into the nodal velocity array /// ///////////////////////////////////////////////////////////////////////////// void FEA_Module_SGH::boundary_velocity(const mesh_t& mesh, diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/energy_sgh.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/energy_sgh.cpp index 9d649d6ec..e08af3db2 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/energy_sgh.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/energy_sgh.cpp @@ -39,16 +39,14 @@ /// /// \fn update_energy_sgh /// -/// \brief +/// \brief Evolves the specific internal energy /// -/// -/// -/// \param -/// \param -/// \param -/// -/// \return +/// \param The current Runge Kutta alpha value +/// \param The simulation mesh +/// \param A view into the nodal velocity data +/// \param A view into the nodal position data +/// \param A view into the element specific internal energy data +/// \param A view into the corner force data /// ///////////////////////////////////////////////////////////////////////////// void FEA_Module_SGH::update_energy_sgh(double rk_alpha, diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp index 3b869c123..e8defe6cd 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp @@ -54,10 +54,30 @@ #include "Simulation_Parameters/Simulation_Parameters_Explicit.h" #include "Simulation_Parameters/FEA_Module/SGH_Parameters.h" -// ----------------------------------------------------------------------------- -// This function calculates the corner force gradients w.r.t velocity; the -// current implementation assumes material.q2 = 0 -// ------------------------------------------------------------------------------ +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn get_force_vgradient_sgh +/// +/// \brief This function calculates the corner force gradients w.r.t velocity +/// +/// The current implementation assumes material.q2 = 0 +/// +/// \param An array of material_t that contains material specific data +/// \param The simulation mesh +/// \param A view into the nodal position array +/// \param A view into the nodal velocity array +/// \param A view into the element density array +/// \param A view into the element specific internal energy array +/// \param A view into the element pressure array +/// \param A view into the element stress array +/// \param A view into the element sound speed array +/// \param A view into the element volume array +/// \param A view into the element divergence of velocity array +/// \param A view into the element material identifier array +/// \param The current Runge Kutta integration alpha value +/// \param The current cycle index +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& material, const mesh_t& mesh, const DViewCArrayKokkos& node_coords, @@ -492,9 +512,30 @@ void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& ma return; } // end of routine -// ----------------------------------------------------------------------------- -// This function calculates the corner forces and the evolves stress (hypo) -// ------------------------------------------------------------------------------ +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn get_force_egradient_sgh +/// +/// \brief This function calculates the corner force gradients w.r.t energy +/// +/// The current implementation assumes material.q2 = 0 +/// +/// \param An array of material_t that contains material specific data +/// \param The simulation mesh +/// \param A view into the nodal position array +/// \param A view into the nodal velocity array +/// \param A view into the element density array +/// \param A view into the element specific internal energy array +/// \param A view into the element pressure array +/// \param A view into the element stress array +/// \param A view into the element sound speed array +/// \param A view into the element volume array +/// \param A view into the element divergence of velocity array +/// \param A view into the element material identifier array +/// \param The current Runge Kutta integration alpha value +/// \param The current cycle index +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_SGH::get_force_egradient_sgh(const DCArrayKokkos& material, const mesh_t& mesh, const DViewCArrayKokkos& node_coords, @@ -926,9 +967,30 @@ void FEA_Module_SGH::get_force_egradient_sgh(const DCArrayKokkos& ma return; } // end of routine -// ----------------------------------------------------------------------------- -// This function calculates the corner forces and the evolves stress (hypo) -// ------------------------------------------------------------------------------ +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn get_force_ugradient_sgh +/// +/// \brief This function calculates the corner force gradients w.r.t position +/// +/// The current implementation assumes material.q2 = 0 +/// +/// \param An array of material_t that contains material specific data +/// \param The simulation mesh +/// \param A view into the nodal position array +/// \param A view into the nodal velocity array +/// \param A view into the element density array +/// \param A view into the element specific internal energy array +/// \param A view into the element pressure array +/// \param A view into the element stress array +/// \param A view into the element sound speed array +/// \param A view into the element volume array +/// \param A view into the element divergence of velocity array +/// \param A view into the element material identifier array +/// \param The current Runge Kutta integration alpha value +/// \param The current cycle index +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_SGH::get_force_ugradient_sgh(const DCArrayKokkos& material, const mesh_t& mesh, const DViewCArrayKokkos& node_coords, @@ -1506,10 +1568,17 @@ void FEA_Module_SGH::get_force_ugradient_sgh(const DCArrayKokkos& ma return; } // end of routine -// -------------------------------------------------------------------------------------------------------- -// Computes term objective derivative term involving gradient of power with respect to the design variable -// --------------------------------------------------------------------------------------------------------- - +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn force_design_gradient_term +/// +/// \brief Computes term objective derivative term involving gradient of power +/// with respect to the design variable +/// +/// \param Design variables +/// \param Design gradients +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_SGH::force_design_gradient_term(const_vec_array design_variables, vec_array design_gradients) { size_t num_bdy_nodes = mesh->num_bdy_nodes; @@ -1574,12 +1643,12 @@ void FEA_Module_SGH::force_design_gradient_term(const_vec_array design_variables node_vel(rk_level, node_gid, idim) = current_velocity_vector(node_gid, idim); node_coords(rk_level, node_gid, idim) = current_coord_vector(node_gid, idim); } - }); // end parallel for + }); // end parallel for Kokkos::fence(); FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { elem_sie(rk_level, elem_gid) = current_element_internal_energy(elem_gid, 0); - }); // end parallel for + }); // end parallel for Kokkos::fence(); get_vol(); @@ -1674,7 +1743,7 @@ void FEA_Module_SGH::force_design_gradient_term(const_vec_array design_variables corner_id = elem_id * num_nodes_in_elem + inode; corner_value_storage(corner_id) = inner_product; } - }); // end parallel for + }); // end parallel for Kokkos::fence(); // accumulate node values from corner storage @@ -1686,15 +1755,34 @@ void FEA_Module_SGH::force_design_gradient_term(const_vec_array design_variables corner_id = corners_in_node(node_id, icorner); design_gradients(node_id, 0) += -corner_value_storage(corner_id) * global_dt; } - }); // end parallel for + }); // end parallel for Kokkos::fence(); } // end view scope } } -// ----------------------------------------------------------------------------- -// This function calculates the corner forces and the evolves stress (hypo) -// ------------------------------------------------------------------------------ +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn get_force_dgradient_sgh +/// +/// \brief This function calculates the force design gradients +/// +/// \param An array of material_t that contains material specific data +/// \param The simulation mesh +/// \param A view into the nodal position array +/// \param A view into the nodal velocity array +/// \param A view into the element density array +/// \param A view into the element specific internal energy array +/// \param A view into the element pressure array +/// \param A view into the element stress array +/// \param A view into the element sound speed array +/// \param A view into the element volume array +/// \param A view into the element divergence of velocity array +/// \param A view into the element material identifier array +/// \param The current Runge Kutta integration alpha value +/// \param The current cycle index +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_SGH::get_force_dgradient_sgh(const DCArrayKokkos& material, const mesh_t& mesh, const DViewCArrayKokkos& node_coords, diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_sgh.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_sgh.cpp index 80ba6a835..71d2adbc8 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_sgh.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_sgh.cpp @@ -36,9 +36,29 @@ #include "Simulation_Parameters/Simulation_Parameters_Explicit.h" #include "Simulation_Parameters/FEA_Module/SGH_Parameters.h" #include "FEA_Module_SGH.h" -// ----------------------------------------------------------------------------- -// This function calculates the corner forces and the evolves stress (hypo) -// ------------------------------------------------------------------------------ + +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn get_force_sgh +/// +/// \brief This function calculates the corner forces and the evolves stress (hypo) +/// +/// \param An array of material_t that contains material specific data +/// \param The simulation mesh +/// \param A view into the nodal position array +/// \param A view into the nodal velocity array +/// \param A view into the element density array +/// \param A view into the element specific internal energy array +/// \param A view into the element pressure array +/// \param A view into the element stress array +/// \param A view into the element sound speed array +/// \param A view into the element volume array +/// \param A view into the element divergence of velocity array +/// \param A view into the element material identifier array +/// \param The current Runge Kutta integration alpha value +/// \param The current cycle index +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_SGH::get_force_sgh(const DCArrayKokkos& material, const mesh_t& mesh, const DViewCArrayKokkos& node_coords, @@ -503,9 +523,28 @@ void FEA_Module_SGH::get_force_sgh(const DCArrayKokkos& material, return; } // end of routine -// ----------------------------------------------------------------------------- -// This function calculates the corner forces and the evolves stress (hypo) -// ------------------------------------------------------------------------------ +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn get_force_sgh2D +/// +/// \brief This function calculates the corner forces and the evolves stress (hypo) +/// +/// \param An array of material_t that contains material specific data +/// \param The simulation mesh +/// \param A view into the nodal position array +/// \param A view into the nodal velocity array +/// \param A view into the element density array +/// \param A view into the element specific internal energy array +/// \param A view into the element pressure array +/// \param A view into the element stress array +/// \param A view into the element sound speed array +/// \param A view into the element volume array +/// \param A view into the element divergence of velocity array +/// \param A view into the element material identifier array +/// \param The current Runge Kutta integration alpha value +/// \param The current cycle index +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_SGH::get_force_sgh2D(const DCArrayKokkos& material, const mesh_t& mesh, const DViewCArrayKokkos& node_coords, @@ -882,10 +921,28 @@ void FEA_Module_SGH::get_force_sgh2D(const DCArrayKokkos& material, return; } // end of routine for 2D force and stress update -// ----------------------------------------------------------------------------- -// This function calculates the corner forces and the evolves stress (hypo) -// ------------------------------------------------------------------------------ - +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn applied_forces +/// +/// \brief This function apploes force loading conditions +/// +/// \param An array of material_t that contains material specific data +/// \param The simulation mesh +/// \param A view into the nodal position array +/// \param A view into the nodal velocity array +/// \param A view into the element density array +/// \param A view into the element specific internal energy array +/// \param A view into the element pressure array +/// \param A view into the element stress array +/// \param A view into the element sound speed array +/// \param A view into the element volume array +/// \param A view into the element divergence of velocity array +/// \param A view into the element material identifier array +/// \param The current Runge Kutta integration alpha value +/// \param The current cycle index +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_SGH::applied_forces(const DCArrayKokkos& material, const mesh_t& mesh, const DViewCArrayKokkos& node_coords, diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/geometry.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/geometry.cpp index 517094125..d098a8d89 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/geometry.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/geometry.cpp @@ -39,6 +39,18 @@ #include "state.h" #include "FEA_Module_SGH.h" +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn update_position_sgh +/// +/// \brief Updates the nodal positions based on the nodal velocity +/// +/// \param Runge Kutta time integration alpha value +/// \param Number of nodes in the mesh +/// \param View of nodal position data +/// \param View of nodal velocity data +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_SGH::update_position_sgh(double rk_alpha, const size_t num_nodes, DViewCArrayKokkos& node_coords, @@ -57,17 +69,26 @@ void FEA_Module_SGH::update_position_sgh(double rk_alpha, }); // end parallel for over nodes } // end subroutine -// ----------------------------------------------------------------------------- -// This function claculates -// B_p = J^{-T} \cdot (\nabla_{xi} \phi_p w -// where -// \phi_p is the basis function for vertex p -// w is the 1 gauss point for the cell (everything is evaluted at this point) -// J^{-T} is the inverse transpose of the Jacobi matrix -// \nabla_{xi} is the gradient opperator in the reference coordinates -// -// B_p is the OUTWARD corner area normal at node p -// ------------------------------------------------------------------------------ +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn get_bmatrix +/// +/// \brief Theis function calculate the finite element B matrix: +/// +/// B_p = J^{-T} \cdot (\nabla_{xi} \phi_p w, where: +/// \phi_p is the basis function for vertex p +/// w is the 1 gauss point for the cell (everything is evaluted at this point) +/// J^{-T} is the inverse transpose of the Jacobi matrix +/// \nabla_{xi} is the gradient opperator in the reference coordinates +/// B_p is the OUTWARD corner area normal at node p +/// +/// \param B matrix +/// \param Global index of the element +/// \param View of nodal position data +/// \param View of the elements node ids +/// \param Runge Kutta time integration level +/// +///////////////////////////////////////////////////////////////////////////// KOKKOS_FUNCTION void FEA_Module_SGH::get_bmatrix(const ViewCArrayKokkos& B_matrix, const size_t elem_gid, @@ -265,10 +286,13 @@ void FEA_Module_SGH::get_bmatrix(const ViewCArrayKokkos& B_matrix, + x(6) * (+y(2) + y(3) - y(4) - y(5) ) ) * twelth; } // end subroutine -/* ---------------------------------------------------------------------------- - Compute Volume of each finite element -------------------------------------------------------------------------------- */ - +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn get_vol +/// +/// \brief Compute Volume of each finite element +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_SGH::get_vol() { const size_t rk_level = rk_num_bins - 1; @@ -296,7 +320,19 @@ void FEA_Module_SGH::get_vol() return; } // end subroutine -// Exact volume for a hex element +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn get_vol_hex +/// +/// \brief Exact volume for a hex element +/// +/// \param View of element volume data +/// \param Global element index +/// \param View into nodal position data +/// \param Runge Kutta time integration level +/// WARNING: How does this work with const elem_vol? +/// +///////////////////////////////////////////////////////////////////////////// KOKKOS_INLINE_FUNCTION void FEA_Module_SGH::get_vol_hex(const DViewCArrayKokkos& elem_vol, const size_t elem_gid, @@ -341,6 +377,22 @@ void FEA_Module_SGH::get_vol_hex(const DViewCArrayKokkos& elem_vol, return; } // end subroutine +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn get_bmatrix2D +/// +/// \brief Calculate the 2D finite element B matrix +/// +/// +/// +/// \param B Matrix +/// \param Global index of the element +/// \param Nodal coordinates +/// \param Global indices of the nodes of this element +/// \param Runge Kutta time integration step +/// +///////////////////////////////////////////////////////////////////////////// KOKKOS_FUNCTION void FEA_Module_SGH::get_bmatrix2D(const ViewCArrayKokkos& B_matrix, const size_t elem_gid, @@ -398,7 +450,19 @@ void FEA_Module_SGH::get_bmatrix2D(const ViewCArrayKokkos& B_matrix, return; } // end subroutine -// true volume of a quad in RZ coords +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn get_vol_quad +/// +/// \brief True volume of a quad in RZ coords +/// +/// \param Element volume +/// \param Global index of the element +/// \param Nodal coordinates +/// \param Global ids of the nodes in this element +/// \param Runge Kutta time integration level +/// +///////////////////////////////////////////////////////////////////////////// KOKKOS_INLINE_FUNCTION void FEA_Module_SGH::get_vol_quad(const DViewCArrayKokkos& elem_vol, const size_t elem_gid, @@ -457,7 +521,20 @@ void FEA_Module_SGH::get_vol_quad(const DViewCArrayKokkos& elem_vol, return; } // end subroutine -// element facial area +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn get_area_quad +/// +/// \brief Calculate the area of a elements face +/// +/// \param Global index of the element +/// \param Nodal coordinates +/// \param Global ids of the nodes in this element +/// \param Runge Kutta time integration level +/// +/// \return Elements face area (double) +/// +///////////////////////////////////////////////////////////////////////////// KOKKOS_FUNCTION double FEA_Module_SGH::get_area_quad(const size_t elem_gid, const DViewCArrayKokkos& node_coords, @@ -492,6 +569,23 @@ double FEA_Module_SGH::get_area_quad(const size_t elem_gid, return elem_area; } // end subroutine +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn heron +/// +/// \brief Calculate the area of a triangle using the heron algorithm +/// +/// +/// \param Node 1 X coordinate +/// \param Node 1 Y coordinate +/// \param Node 2 X coordinate +/// \param Node 2 Y coordinate +/// \param Node 3 X coordinate +/// \param Node 3 Y coordinate +/// +/// \return Triangle area +/// +///////////////////////////////////////////////////////////////////////////// KOKKOS_INLINE_FUNCTION double FEA_Module_SGH::heron(const double x1, const double y1, @@ -516,6 +610,13 @@ double FEA_Module_SGH::heron(const double x1, return area; } +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn get_area_weights2D +/// +/// \brief Calculate the corner weighted area +/// +///////////////////////////////////////////////////////////////////////////// KOKKOS_FUNCTION void FEA_Module_SGH::get_area_weights2D(const ViewCArrayKokkos& corner_areas, const size_t elem_gid, @@ -562,10 +663,17 @@ void FEA_Module_SGH::get_area_weights2D(const ViewCArrayKokkos& corner_a return; } // end subroutine -/* ----------------------------------------------------------------------------------- - Compute Gradient of the Volume of each finite element with respect to displacement --------------------------------------------------------------------------------------- */ - +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn get_vol_ugradient +/// +/// \brief Compute Gradient of the Volume of each finite element with +/// respect to displacement +/// +/// \param Gradient node index +/// \param Gradient dimension +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_SGH::get_vol_ugradient(const size_t gradient_node_id, const size_t gradient_dim) { const size_t rk_level = rk_num_bins - 1; @@ -593,7 +701,20 @@ void FEA_Module_SGH::get_vol_ugradient(const size_t gradient_node_id, const size return; } // end subroutine -// Exact volume for a hex element +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn get_vol_hex_ugradient +/// +/// \brief Calculate the gradient of volume +/// +/// +/// \param Volume gradients +/// \param Global index of the element +/// \param Nodal coordinates +/// \param Global ids of the nodes in this element +/// \param Runge Kutta time integration level +/// +///////////////////////////////////////////////////////////////////////////// KOKKOS_FUNCTION void FEA_Module_SGH::get_vol_hex_ugradient(const ViewCArrayKokkos& elem_vol_gradients, const size_t elem_gid, @@ -822,6 +943,19 @@ void FEA_Module_SGH::get_vol_hex_ugradient(const ViewCArrayKokkos& elem_ return; } // end subroutine +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn get_bmatrix_gradients +/// +/// \brief Calculate the B matrix gradient +/// +/// \param B matrix gradients +/// \param Global index of the element +/// \param Nodal coordinates +/// \param Global ids of the nodes in this element +/// \param Runge Kutta time integration level +/// +///////////////////////////////////////////////////////////////////////////// KOKKOS_FUNCTION void FEA_Module_SGH::get_bmatrix_gradients(const ViewCArrayKokkos& B_matrix_gradients, const size_t elem_gid, diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/momentum.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/momentum.cpp index 848da2d69..e0f3d4220 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/momentum.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/momentum.cpp @@ -34,9 +34,18 @@ #include "state.h" #include "FEA_Module_SGH.h" -// ----------------------------------------------------------------------------- -// This function evolves the velocity at the nodes of the mesh -// ------------------------------------------------------------------------------ +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn update_velocity_sgh +/// +/// \brief This function evolves the velocity at the nodes of the mesh +/// +/// \param Runge Kutta time integration alpha +/// \param View of the nodal velocity array +/// \param View of the nodal mass array +/// \param View of the corner forces +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_SGH::update_velocity_sgh(double rk_alpha, DViewCArrayKokkos& node_vel, const DViewCArrayKokkos& node_mass, @@ -78,9 +87,23 @@ void FEA_Module_SGH::update_velocity_sgh(double rk_alpha, return; } // end subroutine update_velocity -// ----------------------------------------------------------------------------- -// This function calculates the velocity gradient -// ------------------------------------------------------------------------------ +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn get_velgrad +/// +/// \brief This function calculates the velocity gradient +/// +/// \param Velocity gradient +/// \param Global ids of the nodes in this element +/// \param View of the nodal velocity data +/// \param The finite element B matrix +/// \param The volume of the particular element +/// \param The global id of this particular element +/// \param The Runge Kutta time integration level +/// +/// \return +/// +///////////////////////////////////////////////////////////////////////////// KOKKOS_FUNCTION void FEA_Module_SGH::get_velgrad(ViewCArrayKokkos& vel_grad, const ViewCArrayKokkos& elem_node_gids, @@ -165,9 +188,24 @@ void FEA_Module_SGH::get_velgrad(ViewCArrayKokkos& vel_grad, return; } // end function -// ----------------------------------------------------------------------------- -// This function calculates the velocity gradient -// ------------------------------------------------------------------------------ +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn get_velgrad +/// +/// \brief This function calculates the velocity gradient for a 2D element +/// +/// \param Velocity gradient +/// \param Global ids of the nodes in this element +/// \param View of the nodal velocity data +/// \param The finite element B matrix +/// \param The volume of the particular element +/// \param The elements surface area +/// \param The global id of this particular element +/// \param The Runge Kutta time integration level +/// +/// \return +/// +///////////////////////////////////////////////////////////////////////////// KOKKOS_FUNCTION void FEA_Module_SGH::get_velgrad2D(ViewCArrayKokkos& vel_grad, const ViewCArrayKokkos& elem_node_gids, @@ -231,9 +269,18 @@ void FEA_Module_SGH::get_velgrad2D(ViewCArrayKokkos& vel_grad, return; } // end function -// ----------------------------------------------------------------------------- -// This subroutine to calculate the velocity divergence in all elements -// ------------------------------------------------------------------------------ +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn get_divergence +/// +/// \brief This function calculates the divergence of velocity for all elements +/// +/// \param Divergence of velocity for all elements +/// \param iew of the nodal position data +/// \param View of the nodal velocity data +/// \param View of the volumes of each element +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_SGH::get_divergence(DViewCArrayKokkos& elem_div, const DViewCArrayKokkos& node_coords, const DViewCArrayKokkos& node_vel, @@ -305,9 +352,18 @@ void FEA_Module_SGH::get_divergence(DViewCArrayKokkos& elem_div, return; } // end subroutine -// ----------------------------------------------------------------------------- -// This subroutine to calculate the velocity divergence in all elements -// ------------------------------------------------------------------------------ +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn get_divergence +/// +/// \brief This function calculates the divergence of velocity for all 2D elements +/// +/// \param Divergence of velocity for all elements +/// \param iew of the nodal position data +/// \param View of the nodal velocity data +/// \param View of the volumes of each element +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_SGH::get_divergence2D(DViewCArrayKokkos& elem_div, const DViewCArrayKokkos& node_coords, const DViewCArrayKokkos& node_vel, @@ -383,10 +439,25 @@ void FEA_Module_SGH::get_divergence2D(DViewCArrayKokkos& elem_div, return; } // end subroutine -// The velocity gradient can be decomposed into symmetric and antisymmetric tensors -// L = vel_grad -// D = sym(L) -// W = antisym(L) +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn decompose_vel_grad +/// +/// \brief Decomposes the velocity gradient into symmetric and antisymmetric tensors +/// +/// L = D*W, where L = vel_grad, D = sym(L), W = antisym(L) +/// can span multiple lines if needed> +/// +/// \param Symmetric decomposition of velocity gradient +/// \param Antisymmetric decomposition of velocity gradient +/// \param Gradient of velocity +/// \param Global ids of the nodes associated with this element +/// \param Global id of a specific element +/// \param View of the nodal coordinate data +/// \param View of the nodal velocity data +/// \param Volume of the element +/// +///////////////////////////////////////////////////////////////////////////// KOKKOS_INLINE_FUNCTION void FEA_Module_SGH::decompose_vel_grad(ViewCArrayKokkos& D_tensor, ViewCArrayKokkos& W_tensor, diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/power_gradients_sgh.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/power_gradients_sgh.cpp index 8cfb2b1df..21d5483d0 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/power_gradients_sgh.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/power_gradients_sgh.cpp @@ -37,10 +37,17 @@ #include "Simulation_Parameters/Simulation_Parameters_Explicit.h" #include "Simulation_Parameters/FEA_Module/SGH_Parameters.h" -// -------------------------------------------------------------------------------------------------------- -// Computes term objective derivative term involving gradient of power with respect to the design variable -// --------------------------------------------------------------------------------------------------------- - +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn power_design_gradient_term +/// +/// \brief Computes term objective derivative term involving gradient of power +/// with respect to the design variable +/// +/// \param Vector of design variables +/// \param Vector of design gradients +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_SGH::power_design_gradient_term(const_vec_array design_variables, vec_array design_gradients) { bool element_constant_density = true; @@ -259,10 +266,23 @@ void FEA_Module_SGH::power_design_gradient_term(const_vec_array design_variables } } -// --------------------------------------------------------------------------------------- -// This function calculates the gradient for element power with respect to design variable -// ---------------------------------------------------------------------------------------- - +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn get_power_dgradient_sgh +/// +/// \brief This function calculates the gradient for element power with +/// respect to design variable +/// +/// \param The current Runge Kutta alpha value +/// \param The simulation mesh +/// \param A view into the nodal velocity data +/// \param A view into the nodal position data +/// \param A view into the element specific internal energy data +/// \param A view into the element mass data +/// \param A view into the corner force data +/// \param Array of design gradients +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_SGH::get_power_dgradient_sgh(double rk_alpha, const mesh_t& mesh, const DViewCArrayKokkos& node_vel, @@ -311,10 +331,24 @@ void FEA_Module_SGH::get_power_dgradient_sgh(double rk_alpha, return; } // end subroutine -// ----------------------------------------------------------------------------- -// This function calculates the gradient for element power with respect to position -// ------------------------------------------------------------------------------ - +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn get_power_ugradient_sgh +/// +/// \brief This function calculates the gradient for element power with +/// respect to position +/// +/// \param The current Runge Kutta alpha value +/// \param The simulation mesh +/// \param A view into the nodal velocity data +/// \param A view into the nodal position data +/// \param A view into the element specific internal energy data +/// \param A view into the element mass data +/// \param A view into the corner force data +/// +/// \return +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_SGH::get_power_ugradient_sgh(double rk_alpha, const mesh_t& mesh, const DViewCArrayKokkos& node_vel, @@ -388,10 +422,15 @@ void FEA_Module_SGH::get_power_ugradient_sgh(double rk_alpha, return; } // end subroutine -// ----------------------------------------------------------------------------- -// This function calculates the gradient for element power with respect to velocity -// ------------------------------------------------------------------------------ +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn get_power_vgradient_sgh +/// +/// \brief This function calculates the gradient for element power with respect to velocity +/// +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_SGH::get_power_vgradient_sgh(double rk_alpha, const mesh_t& mesh, const DViewCArrayKokkos& node_vel, @@ -471,10 +510,22 @@ void FEA_Module_SGH::get_power_vgradient_sgh(double rk_alpha, return; } // end subroutine -// ----------------------------------------------------------------------------- -// This function calculates the gradient for element power with respect to energy -// ------------------------------------------------------------------------------ - +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn get_power_egradient_sgh +/// +/// \brief This function calculates the gradient for element power with +/// respect to energy +/// +/// \param The current Runge Kutta alpha value +/// \param The simulation mesh +/// \param A view into the nodal velocity data +/// \param A view into the nodal position data +/// \param A view into the element specific internal energy data +/// \param A view into the element mass data +/// \param A view into the corner force data +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_SGH::get_power_egradient_sgh(double rk_alpha, const mesh_t& mesh, const DViewCArrayKokkos& node_vel, diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/properties.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/properties.cpp index 0340aa0b3..c06d53247 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/properties.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/properties.cpp @@ -31,13 +31,32 @@ OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. **********************************************************************************************/ -// ----------------------------------------------------------------------------- -// This calls the models to update state -// ------------------------------------------------------------------------------ #include "state.h" #include "mesh.h" #include "FEA_Module_SGH.h" +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn update_state +/// +/// \brief This calls the models to update state +/// +/// \param An array of material_t that contains material specific data +/// \param The simulation mesh +/// \param A view into the nodal position array +/// \param A view into the nodal velocity array +/// \param A view into the element density array +/// \param A view into the element specific internal energy array +/// \param A view into the element pressure array +/// \param A view into the element stress array +/// \param A view into the element sound speed array +/// \param A view into the element volume array +/// \param A view into the element divergence of velocity array +/// \param A view into the element material identifier array +/// \param The current Runge Kutta integration alpha value +/// \param The current cycle index +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_SGH::update_state(const DCArrayKokkos& material, const mesh_t& mesh, const DViewCArrayKokkos& node_coords, @@ -165,6 +184,28 @@ void FEA_Module_SGH::update_state(const DCArrayKokkos& material, return; } // end method to update state +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn update_state2D +/// +/// \brief Updates the state for 2D elements +/// +/// \param An array of material_t that contains material specific data +/// \param The simulation mesh +/// \param A view into the nodal position array +/// \param A view into the nodal velocity array +/// \param A view into the element density array +/// \param A view into the element specific internal energy array +/// \param A view into the element pressure array +/// \param A view into the element stress array +/// \param A view into the element sound speed array +/// \param A view into the element volume array +/// \param A view into the element divergence of velocity array +/// \param A view into the element material identifier array +/// \param The current Runge Kutta integration alpha value +/// \param The current cycle index +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_SGH::update_state2D(const DCArrayKokkos& material, const mesh_t& mesh, const DViewCArrayKokkos& node_coords, diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/setup_sgh.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/setup_sgh.cpp index 9f56e993c..d9a4cfb6e 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/setup_sgh.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/setup_sgh.cpp @@ -31,9 +31,6 @@ OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. **********************************************************************************************/ -/* ---------------------------------------------------------------------------- - Setup SGH solver data -------------------------------------------------------------------------------- */ #include "matar.h" #include "state.h" #include "FEA_Module_SGH.h" @@ -43,149 +40,88 @@ // #define DEBUG -/* ---------------------------------------------------------------------------- - Setup SGH solver data -------------------------------------------------------------------------------- */ - -void FEA_Module_SGH::setup(){ - - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - const size_t num_fills = simparam->regions.size(); - const size_t rk_num_bins = simparam->dynamic_options.rk_num_bins; - const size_t num_bcs = module_params->boundary_conditions.size(); +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn setup +/// +/// \brief Setup SGH solver data +/// +///////////////////////////////////////////////////////////////////////////// +void FEA_Module_SGH::setup() +{ + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + const size_t num_fills = simparam->regions.size(); + const size_t rk_num_bins = simparam->dynamic_options.rk_num_bins; + const size_t num_bcs = module_params->boundary_conditions.size(); const size_t num_materials = simparam->materials.size(); - const int num_dim = simparam->num_dims; - const size_t num_lcs = module_params->loading.size(); - if(num_lcs) - have_loading_conditions = true; - + const int num_dim = simparam->num_dims; + const size_t num_lcs = module_params->loading.size(); + if (num_lcs) + { + have_loading_conditions = true; + } // --------------------------------------------------------------------- // obtain mesh data - // --------------------------------------------------------------------- + // --------------------------------------------------------------------- sgh_interface_setup(node_interface, elem_interface, corner_interface); mesh->build_corner_connectivity(); - //debug print of corner ids - /* - if(myrank==1){ - for(int i = 0; i < mesh.num_nodes; i++){ - - // loop over all corners around the node and calculate the nodal force - for (size_t corner_lid=0; corner_lidgetGlobalElement(i) << " " << i << " " << all_node_map->getLocalElement(all_node_map->getGlobalElement(i)) << " " << corner_gid << " " << std::endl; - - } // end for corner_lid - //std::cout << explicit_solver_pointer->all_node_map->getGlobalElement(i) << " " << node_force[0] << " " << node_force[1] << " " << node_force[2] << std::endl; - //std::cout << explicit_solver_pointer->all_node_map->getGlobalElement(i) << " " << node_mass(i) << std::endl; - } - } - */ - /* - if(myrank==1){ - for(int i = 0; i < mesh.num_elems; i++){ - - // loop over all corners around the node and calculate the nodal force - for (size_t corner_lid=0; corner_lidgetGlobalElement(mesh.nodes_in_elem(i, corner_lid)) <<" " << corner_gid << " " << std::endl; - - } // end for corner_lid - //std::cout << explicit_solver_pointer->all_node_map->getGlobalElement(i) << " " << node_force[0] << " " << node_force[1] << " " << node_force[2] << std::endl; - //std::cout << explicit_solver_pointer->all_node_map->getGlobalElement(i) << " " << node_mass(i) << std::endl; - } - } - */ mesh->build_elem_elem_connectivity(); mesh->num_bdy_patches = nboundary_patches; - if(num_dim==2){ - mesh->build_patch_connectivity(); - mesh->build_node_node_connectivity(); + if (num_dim == 2) + { + mesh->build_patch_connectivity(); + mesh->build_node_node_connectivity(); } - - // --------------------------------------------------------------------- - // allocate memory - // --------------------------------------------------------------------- - - // shorthand names - const size_t num_nodes = mesh->num_nodes; - const size_t num_elems = mesh->num_elems; + + // --------------------------------------------------------------------- + // allocate memory + // --------------------------------------------------------------------- + + // shorthand names + const size_t num_nodes = mesh->num_nodes; + const size_t num_elems = mesh->num_elems; const size_t num_corners = mesh->num_corners; - - // --- make dual views of data on CPU and GPU --- - // Notes: - // Instead of using a struct of dual types like the mesh type, - // individual dual views will be made for all the state - // variables. The motivation is to reduce memory movement - // when passing state into a function. Passing a struct by - // reference will copy the meta data and pointers for the - // variables held inside the struct. Since all the mesh - // variables are typically used by most functions, a single - // mesh struct or passing the arrays will be roughly equivalent - // for memory movement. - - - // create Dual Views of the individual node struct variables - node_coords = DViewCArrayKokkos(node_interface.coords.get_kokkos_dual_view().view_host().data(),rk_num_bins,num_nodes,num_dim); + // --- make dual views of data on CPU and GPU --- + // Notes: + // Instead of using a struct of dual types like the mesh type, + // individual dual views will be made for all the state + // variables. The motivation is to reduce memory movement + // when passing state into a function. Passing a struct by + // reference will copy the meta data and pointers for the + // variables held inside the struct. Since all the mesh + // variables are typically used by most functions, a single + // mesh struct or passing the arrays will be roughly equivalent + // for memory movement. - node_vel = DViewCArrayKokkos(node_interface.vel.get_kokkos_dual_view().view_host().data(),rk_num_bins,num_nodes,num_dim); + // create Dual Views of the individual node struct variables + node_coords = DViewCArrayKokkos(node_interface.coords.get_kokkos_dual_view().view_host().data(), rk_num_bins, num_nodes, num_dim); + node_vel = DViewCArrayKokkos(node_interface.vel.get_kokkos_dual_view().view_host().data(), rk_num_bins, num_nodes, num_dim); + node_mass = DViewCArrayKokkos(node_interface.mass.get_kokkos_dual_view().view_host().data(), num_nodes); - node_mass = DViewCArrayKokkos(node_interface.mass.get_kokkos_dual_view().view_host().data(),num_nodes); - - // create Dual Views of the individual elem struct variables - elem_den= DViewCArrayKokkos(&elem_interface.den(0), - num_elems); - - elem_pres = DViewCArrayKokkos(&elem_interface.pres(0), - num_elems); - - elem_stress = DViewCArrayKokkos(&elem_interface.stress(0,0,0,0), - rk_num_bins, - num_elems, - 3, - 3); // always 3D even in 2D-RZ - - elem_sspd = DViewCArrayKokkos(&elem_interface.sspd(0), - num_elems); - - elem_sie = DViewCArrayKokkos(&elem_interface.sie(0,0), - rk_num_bins, - num_elems); - - elem_vol = DViewCArrayKokkos(&elem_interface.vol(0), - num_elems); - - elem_div = DViewCArrayKokkos(&elem_interface.div(0), - num_elems); - - - elem_mass = DViewCArrayKokkos(&elem_interface.mass(0), - num_elems); - - elem_mat_id = DViewCArrayKokkos(&elem_interface.mat_id(0), - num_elems); - + elem_den = DViewCArrayKokkos(&elem_interface.den(0), num_elems); + elem_pres = DViewCArrayKokkos(&elem_interface.pres(0), num_elems); + elem_stress = DViewCArrayKokkos(&elem_interface.stress(0, 0, 0, 0), rk_num_bins, num_elems, 3, 3); // always 3D even in 2D-RZ + elem_sspd = DViewCArrayKokkos(&elem_interface.sspd(0), num_elems); + elem_sie = DViewCArrayKokkos(&elem_interface.sie(0, 0), rk_num_bins, num_elems); + elem_vol = DViewCArrayKokkos(&elem_interface.vol(0), num_elems); + elem_div = DViewCArrayKokkos(&elem_interface.div(0), num_elems); + elem_mass = DViewCArrayKokkos(&elem_interface.mass(0), num_elems); + elem_mat_id = DViewCArrayKokkos(&elem_interface.mat_id(0), num_elems); + // create Dual Views of the corner struct variables - corner_force = DViewCArrayKokkos (&corner_interface.force(0,0), - num_corners, - num_dim); + corner_force = DViewCArrayKokkos(&corner_interface.force(0, 0), num_corners, num_dim); + corner_mass = DViewCArrayKokkos(&corner_interface.mass(0), num_corners); - corner_mass = DViewCArrayKokkos (&corner_interface.mass(0), - num_corners); - // allocate elem_vel_grad - elem_vel_grad = DCArrayKokkos (num_elems,3,3); + elem_vel_grad = DCArrayKokkos(num_elems, 3, 3); // allocate material models - elem_eos = DCArrayKokkos (num_elems); - elem_strength = DCArrayKokkos (num_elems); - + elem_eos = DCArrayKokkos(num_elems); + elem_strength = DCArrayKokkos(num_elems); + // --------------------------------------------------------------------- // calculate geometry // --------------------------------------------------------------------- @@ -194,78 +130,83 @@ void FEA_Module_SGH::setup(){ get_vol(); - //FEA_Module bc variable + // FEA_Module bc variable num_boundary_conditions = num_bcs; - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos mat_fill = simparam->mat_fill; - const DCArrayKokkos material = simparam->material; + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos mat_fill = simparam->mat_fill; + const DCArrayKokkos material = simparam->material; + + // Constitutive model data eos_global_vars = simparam->eos_global_vars; strength_global_vars = simparam->strength_global_vars; - eos_state_vars = DCArrayKokkos (rnum_elem, simparam->max_num_eos_state_vars); - strength_state_vars = DCArrayKokkos (rnum_elem, simparam->max_num_strength_state_vars); - elem_user_output_vars = DCArrayKokkos (rnum_elem, simparam->output_options.max_num_user_output_vars); - - //--- calculate bdy sets ---// - mesh->num_nodes_in_patch = 2*(num_dim-1); // 2 (2D) or 4 (3D) - mesh->num_patches_in_elem = 2*num_dim; // 4 (2D) or 6 (3D) + eos_state_vars = DCArrayKokkos(rnum_elem, simparam->max_num_eos_state_vars); + strength_state_vars = DCArrayKokkos(rnum_elem, simparam->max_num_strength_state_vars); + elem_user_output_vars = DCArrayKokkos(rnum_elem, simparam->output_options.max_num_user_output_vars); + + // --- calculate bdy sets ---// + mesh->num_nodes_in_patch = 2 * (num_dim - 1); // 2 (2D) or 4 (3D) + mesh->num_patches_in_elem = 2 * num_dim; // 4 (2D) or 6 (3D) mesh->init_bdy_sets(num_bcs); num_bdy_sets = mesh->num_bdy_sets; printf("Num BC's = %lu\n", num_bcs); // patch ids in bdy set bdy_patches_in_set = mesh->bdy_patches_in_set; - if(num_dim==2) - bdy_nodes = mesh->bdy_nodes; + if (num_dim == 2) + { + bdy_nodes = mesh->bdy_nodes; + } // tag boundary patches in the set tag_bdys(boundary, *mesh, node_coords); build_boundry_node_sets(*mesh); - + // node ids in bdy_patch set - bdy_nodes_in_set = mesh->bdy_nodes_in_set; + bdy_nodes_in_set = mesh->bdy_nodes_in_set; num_bdy_nodes_in_set = mesh->num_bdy_nodes_in_set; - - //assign mesh views needed by the FEA module + + // assign mesh views needed by the FEA module // elem ids in elem - elems_in_elem = mesh->elems_in_elem; + elems_in_elem = mesh->elems_in_elem; num_elems_in_elem = mesh->num_elems_in_elem; - //corners + // corners num_corners_in_node = mesh->num_corners_in_node; - corners_in_node = mesh->corners_in_node; - corners_in_elem = mesh->corners_in_elem; + corners_in_node = mesh->corners_in_node; + corners_in_elem = mesh->corners_in_elem; - //elem-node conn & node-node conn + // elem-node conn & node-node conn elems_in_node = mesh->elems_in_node; - if(num_dim==2){ - nodes_in_node = mesh->nodes_in_node; - num_nodes_in_node = mesh->num_nodes_in_node; - //patch conn - - patches_in_elem = mesh->patches_in_elem; - nodes_in_patch = mesh->nodes_in_patch; - elems_in_patch = mesh->elems_in_patch; + if (num_dim == 2) + { + nodes_in_node = mesh->nodes_in_node; + num_nodes_in_node = mesh->num_nodes_in_node; + // patch conn + + patches_in_elem = mesh->patches_in_elem; + nodes_in_patch = mesh->nodes_in_patch; + elems_in_patch = mesh->elems_in_patch; } // loop over BCs - for (size_t this_bdy = 0; this_bdy < num_bcs; this_bdy++){ - + for (size_t this_bdy = 0; this_bdy < num_bcs; this_bdy++) + { RUN_CLASS({ printf("Boundary Condition number %lu \n", this_bdy); printf(" Num bdy patches in this set = %lu \n", bdy_patches_in_set.stride(this_bdy)); printf(" Num bdy nodes in this set = %lu \n", bdy_nodes_in_set.stride(this_bdy)); }); Kokkos::fence(); - - }// end for + } // end for // elem_mat_id needs to be initialized before initialization of material models - for (int f_id = 0; f_id < num_fills; f_id++){ - FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { - elem_mat_id(elem_gid) = mat_fill(f_id).material_id; + for (int f_id = 0; f_id < num_fills; f_id++) + { + FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { + elem_mat_id(elem_gid) = mat_fill(f_id).material_id; }); } elem_mat_id.update_host(); @@ -279,7 +220,7 @@ void FEA_Module_SGH::setup(){ strength_global_vars, elem_user_output_vars, rnum_elem); - + // initialize strength model init_strength_model(elem_strength, material, @@ -301,24 +242,25 @@ void FEA_Module_SGH::setup(){ strength_global_vars, elem_user_output_vars, rnum_elem); - - //--- apply the fill instructions over each of the Elements---// - - //initialize if topology optimization is used - if(simparam->topology_optimization_on){ - for(int elem_id = 0; elem_id < rnum_elem; elem_id++){ - relative_element_densities.host(elem_id) = 1; - }//for - relative_element_densities.update_device(); + + // --- apply the fill instructions over each of the Elements---// + + // initialize if topology optimization is used + if (simparam->topology_optimization_on) + { + for (int elem_id = 0; elem_id < rnum_elem; elem_id++) + { + relative_element_densities.host(elem_id) = 1; + } // for + relative_element_densities.update_device(); } - + // loop over the fill instructures - for (int f_id = 0; f_id < num_fills; f_id++){ - + for (int f_id = 0; f_id < num_fills; f_id++) + { // parallel loop over elements in mesh - //for (size_t elem_gid = 0; elem_gid <= rnum_elem; elem_gid++) { + // for (size_t elem_gid = 0; elem_gid <= rnum_elem; elem_gid++) { FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { - // calculate the coordinates and radius of the element double elem_coords[3]; // note:initialization with a list won't work elem_coords[0] = 0.0; @@ -326,45 +268,50 @@ void FEA_Module_SGH::setup(){ elem_coords[2] = 0.0; // get the coordinates of the element center - for (int node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ + for (int node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { elem_coords[0] += node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), 0); elem_coords[1] += node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), 1); - if (num_dim == 3){ + if (num_dim == 3) + { elem_coords[2] += node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), 2); - } else + } + else { elem_coords[2] = 0.0; } } // end loop over nodes in element - elem_coords[0] = elem_coords[0]/num_nodes_in_elem; - elem_coords[1] = elem_coords[1]/num_nodes_in_elem; - elem_coords[2] = elem_coords[2]/num_nodes_in_elem; + elem_coords[0] = elem_coords[0] / num_nodes_in_elem; + elem_coords[1] = elem_coords[1] / num_nodes_in_elem; + elem_coords[2] = elem_coords[2] / num_nodes_in_elem; // default is not to fill the element bool fill_this = mat_fill(f_id).volume.contains(elem_coords); // paint the material state on the element - if (fill_this){ - + if (fill_this) + { // density elem_den(elem_gid) = mat_fill(f_id).den; - + // mass - elem_mass(elem_gid) = elem_den(elem_gid)*elem_vol(elem_gid); - + elem_mass(elem_gid) = elem_den(elem_gid) * elem_vol(elem_gid); + // specific internal energy elem_sie(rk_level, elem_gid) = mat_fill(f_id).sie; - + size_t mat_id = elem_mat_id(elem_gid); // short name - + // --- stress tensor --- // always 3D even for 2D-RZ - for (size_t i=0; i<3; i++){ - for (size_t j=0; j<3; j++){ - elem_stress(rk_level,elem_gid,i,j) = 0.0; - } + for (size_t i = 0; i < 3; i++) + { + for (size_t j = 0; j < 3; j++) + { + elem_stress(rk_level, elem_gid, i, j) = 0.0; + } } // end for - + // --- Pressure --- elem_eos(elem_gid).calc_pressure(elem_pres, elem_stress, @@ -377,7 +324,7 @@ void FEA_Module_SGH::setup(){ elem_user_output_vars, elem_sspd, elem_den(elem_gid), - elem_sie(rk_level,elem_gid)); + elem_sie(rk_level, elem_gid)); // --- Sound speed --- elem_eos(elem_gid).calc_sound_speed(elem_pres, @@ -391,226 +338,224 @@ void FEA_Module_SGH::setup(){ elem_user_output_vars, elem_sspd, elem_den(elem_gid), - elem_sie(rk_level,elem_gid)); - - // loop over the nodes of this element and apply velocity - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ + elem_sie(rk_level, elem_gid)); + // loop over the nodes of this element and apply velocity + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + { // get the mesh node index size_t node_gid = nodes_in_elem(elem_gid, node_lid); - // --- Velocity --- - switch(mat_fill(f_id).velocity) + switch (mat_fill(f_id).velocity) { case VELOCITY_TYPE::cartesian: - { - - node_vel(rk_level, node_gid, 0) = mat_fill(f_id).u; - node_vel(rk_level, node_gid, 1) = mat_fill(f_id).v; - if (num_dim == 3) node_vel(rk_level, node_gid, 2) = mat_fill(f_id).w; - - - break; - } - case VELOCITY_TYPE::radial: - { - // Setting up cylindrical - double dir[2]; - dir[0] = 0.0; - dir[1] = 0.0; - double radius_val = 0.0; - - for(int dim=0; dim<2; dim++){ - dir[dim] = node_coords(rk_level, node_gid, dim); - radius_val += node_coords(rk_level, node_gid, dim)*node_coords(rk_level, node_gid, dim); - } // end for - radius_val = sqrt(radius_val); - - for(int dim=0; dim<2; dim++){ - if (radius_val > 1.0e-14){ - dir[dim] /= (radius_val); + { + node_vel(rk_level, node_gid, 0) = mat_fill(f_id).u; + node_vel(rk_level, node_gid, 1) = mat_fill(f_id).v; + if (num_dim == 3) + { + node_vel(rk_level, node_gid, 2) = mat_fill(f_id).w; } - else{ - dir[dim] = 0.0; + + break; + } + case VELOCITY_TYPE::radial: + { + // Setting up cylindrical + double dir[2]; + dir[0] = 0.0; + dir[1] = 0.0; + double radius_val = 0.0; + + for (int dim = 0; dim < 2; dim++) + { + dir[dim] = node_coords(rk_level, node_gid, dim); + radius_val += node_coords(rk_level, node_gid, dim) * node_coords(rk_level, node_gid, dim); + } // end for + radius_val = sqrt(radius_val); + + for (int dim = 0; dim < 2; dim++) + { + if (radius_val > 1.0e-14) + { + dir[dim] /= (radius_val); + } + else + { + dir[dim] = 0.0; + } + } // end for + + node_vel(rk_level, node_gid, 0) = mat_fill(f_id).speed * dir[0]; + node_vel(rk_level, node_gid, 1) = mat_fill(f_id).speed * dir[1]; + if (num_dim == 3) + { + node_vel(rk_level, node_gid, 2) = 0.0; } - } // end for - - - node_vel(rk_level, node_gid, 0) = mat_fill(f_id).speed*dir[0]; - node_vel(rk_level, node_gid, 1) = mat_fill(f_id).speed*dir[1]; - if (num_dim == 3) node_vel(rk_level, node_gid, 2) = 0.0; - - break; - } + + break; + } case VELOCITY_TYPE::spherical: - { - - // Setting up spherical - double dir[3]; - dir[0] = 0.0; - dir[1] = 0.0; - dir[2] = 0.0; - double radius_val = 0.0; - - for(int dim=0; dim<3; dim++){ - dir[dim] = node_coords(rk_level, node_gid, dim); - radius_val += node_coords(rk_level, node_gid, dim)*node_coords(rk_level, node_gid, dim); - } // end for - radius_val = sqrt(radius_val); - - for(int dim=0; dim<3; dim++){ - if (radius_val > 1.0e-14){ - dir[dim] /= (radius_val); + { + // Setting up spherical + double dir[3]; + dir[0] = 0.0; + dir[1] = 0.0; + dir[2] = 0.0; + double radius_val = 0.0; + + for (int dim = 0; dim < 3; dim++) + { + dir[dim] = node_coords(rk_level, node_gid, dim); + radius_val += node_coords(rk_level, node_gid, dim) * node_coords(rk_level, node_gid, dim); + } // end for + radius_val = sqrt(radius_val); + + for (int dim = 0; dim < 3; dim++) + { + if (radius_val > 1.0e-14) + { + dir[dim] /= (radius_val); + } + else + { + dir[dim] = 0.0; + } + } // end for + + node_vel(rk_level, node_gid, 0) = mat_fill(f_id).speed * dir[0]; + node_vel(rk_level, node_gid, 1) = mat_fill(f_id).speed * dir[1]; + if (num_dim == 3) + { + node_vel(rk_level, node_gid, 2) = mat_fill(f_id).speed * dir[2]; } - else{ - dir[dim] = 0.0; - } - } // end for - - node_vel(rk_level, node_gid, 0) = mat_fill(f_id).speed*dir[0]; - node_vel(rk_level, node_gid, 1) = mat_fill(f_id).speed*dir[1]; - if (num_dim == 3) node_vel(rk_level, node_gid, 2) = mat_fill(f_id).speed*dir[2]; - - break; - } + + break; + } case VELOCITY_TYPE::radial_linear: - { - - break; - } + { + break; + } case VELOCITY_TYPE::spherical_linear: - { - - break; - } + { + break; + } case VELOCITY_TYPE::tg_vortex: - { - - node_vel(rk_level, node_gid, 0) = sin(PI * node_coords(rk_level,node_gid, 0)) * cos(PI * node_coords(rk_level,node_gid, 1)); - node_vel(rk_level, node_gid, 1) = -1.0*cos(PI * node_coords(rk_level,node_gid, 0)) * sin(PI * node_coords(rk_level,node_gid, 1)); - if (num_dim == 3) node_vel(rk_level, node_gid, 2) = 0.0; - - break; - } + { + node_vel(rk_level, node_gid, 0) = sin(PI * node_coords(rk_level, node_gid, 0)) * cos(PI * node_coords(rk_level, node_gid, 1)); + node_vel(rk_level, node_gid, 1) = -1.0 * cos(PI * node_coords(rk_level, node_gid, 0)) * sin(PI * node_coords(rk_level, node_gid, 1)); + if (num_dim == 3) + { + node_vel(rk_level, node_gid, 2) = 0.0; + } + + break; + } } // end of switch + } // end loop over nodes of element - }// end loop over nodes of element - - - if(mat_fill(f_id).velocity == VELOCITY_TYPE::tg_vortex) + if (mat_fill(f_id).velocity == VELOCITY_TYPE::tg_vortex) { - elem_pres(elem_gid) = 0.25*( cos(2.0*PI*elem_coords[0]) + cos(2.0*PI*elem_coords[1]) ) + 1.0; - + elem_pres(elem_gid) = 0.25 * (cos(2.0 * PI * elem_coords[0]) + cos(2.0 * PI * elem_coords[1]) ) + 1.0; + // p = rho*ie*(gamma - 1) size_t mat_id = f_id; - double gamma = eos_global_vars(mat_id,0); // gamma value + double gamma = eos_global_vars(mat_id, 0); // gamma value elem_sie(rk_level, elem_gid) = - elem_pres(elem_gid)/(mat_fill(f_id).den*(gamma - 1.0)); + elem_pres(elem_gid) / (mat_fill(f_id).den * (gamma - 1.0)); } // end if - } // end if fill - }); // end FOR_ALL_CLASS element loop Kokkos::fence(); - - } // end for loop over fills - // apply BC's to velocity FEA_Module_SGH::boundary_velocity(*mesh, boundary, node_vel); - - + // calculate the corner massess if 2D - if(num_dim==2){ - + if (num_dim == 2) + { FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { - // facial area of the corners double corner_areas_array[4]; - - ViewCArrayKokkos corner_areas(&corner_areas_array[0],4); - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 4); - + + ViewCArrayKokkos corner_areas(&corner_areas_array[0], 4); + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 4); + get_area_weights2D(corner_areas, elem_gid, node_coords, elem_node_gids, rk_level); - + // loop over the corners of the element and calculate the mass - for (size_t corner_lid=0; corner_lid<4; corner_lid++){ - + for (size_t corner_lid = 0; corner_lid < 4; corner_lid++) + { size_t corner_gid = corners_in_elem(elem_gid, corner_lid); - corner_mass(corner_gid) = corner_areas(corner_lid)*elem_den(elem_gid); // node radius is added later - + corner_mass(corner_gid) = corner_areas(corner_lid) * elem_den(elem_gid); // node radius is added later } // end for over corners }); - } // end of - - + // calculate the nodal mass FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - node_mass(node_gid) = 0.0; - - if(num_dim==3){ - - for(size_t elem_lid=0; elem_lidtopology_optimization_on||simparam->shape_optimization_on||simparam->num_dims==2){ - { - vec_array node_mass_interface = node_masses_distributed->getLocalView (Tpetra::Access::ReadWrite); + // current interface has differing mass arrays; this equates them until we unify memory + // view scope + if (simparam->topology_optimization_on || simparam->shape_optimization_on || simparam->num_dims == 2) + { + { + vec_array node_mass_interface = node_masses_distributed->getLocalView(Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - node_mass_interface(node_gid,0) = node_mass(node_gid); + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + node_mass_interface(node_gid, 0) = node_mass(node_gid); }); // end parallel for - } //end view scope - Kokkos::fence(); - //communicate ghost densities - comm_node_masses(); - - //this is forcing a copy to the device - //view scope - { - vec_array ghost_node_mass_interface = ghost_node_masses_distributed->getLocalView (Tpetra::Access::ReadWrite); + } // end view scope + Kokkos::fence(); + // communicate ghost densities + comm_node_masses(); + // this is forcing a copy to the device + // view scope + { + vec_array ghost_node_mass_interface = ghost_node_masses_distributed->getLocalView(Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, nlocal_nodes, nall_nodes, { - node_mass(node_gid) = ghost_node_mass_interface(node_gid-nlocal_nodes,0); + FOR_ALL_CLASS(node_gid, nlocal_nodes, nall_nodes, { + node_mass(node_gid) = ghost_node_mass_interface(node_gid - nlocal_nodes, 0); }); // end parallel for - } //end view scope - Kokkos::fence(); - } //endif - - //initialize if topology optimization is used - if(simparam->topology_optimization_on || simparam->shape_optimization_on){ - init_assembly(); - //assemble_matrix(); + } // end view scope + Kokkos::fence(); + } // endif + + // initialize if topology optimization is used + if (simparam->topology_optimization_on || simparam->shape_optimization_on) + { + init_assembly(); + // assemble_matrix(); } // update host copies of arrays modified in this function @@ -619,18 +564,23 @@ void FEA_Module_SGH::setup(){ elem_sie.update_host(); elem_stress.update_host(); elem_pres.update_host(); - elem_sspd.update_host(); + elem_sspd.update_host(); return; - } // end of setup -// ----------------------------------------------------------------------------- -// Interfaces read in data with the SGH solver data; currently a hack to streamline -// ------------------------------------------------------------------------------ -void FEA_Module_SGH::sgh_interface_setup(node_t& node, - elem_t& elem, - corner_t& corner) +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn sgh_interface_setup +/// +/// \brief Interfaces read in data with the SGH solver data; currently a hack to streamline +/// +/// \param State data for the nodes +/// \param State data for the elements +/// \param State data for the corners +/// +///////////////////////////////////////////////////////////////////////////// +void FEA_Module_SGH::sgh_interface_setup(node_t& node, elem_t& elem, corner_t& corner) { const size_t num_dim = simparam->num_dims; const size_t rk_num_bins = simparam->dynamic_options.rk_num_bins; @@ -830,9 +780,15 @@ void FEA_Module_SGH::sgh_interface_setup(node_t& node, return; } -/* ---------------------------------------------------------------------------- - Build set of nodes assigned to each boundary condition -------------------------------------------------------------------------------- */ +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn build_boundry_node_sets +/// +/// \brief Build set of nodes assigned to each boundary condition +/// +/// \param The simulation mesh +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_SGH::build_boundry_node_sets(mesh_t& mesh) { // build boundary nodes in each boundary set @@ -912,9 +868,13 @@ void FEA_Module_SGH::build_boundry_node_sets(mesh_t& mesh) return; } // end method to build boundary nodes -/* ---------------------------------------------------------------------------- - Initialize sets of element boundary surfaces and arrays for input conditions -------------------------------------------------------------------------------- */ +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn init_boundaries +/// +/// \brief Initialize sets of element boundary surfaces and arrays for input conditions +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_SGH::init_boundaries() { max_boundary_sets = module_params->boundary_conditions.size(); @@ -946,9 +906,15 @@ void FEA_Module_SGH::init_boundaries() Number_DOF_BCS = 0; } -/* ---------------------------------------------------------------------- - initialize storage for element boundary surfaces corresponding to user BCs -------------------------------------------------------------------------- */ +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn init_boundary_sets +/// +/// \brief Initialize storage for element boundary surfaces corresponding to user BCs +/// +/// \param Number of boundary sets +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_SGH::init_boundary_sets(int num_sets) { if (num_sets == 0) @@ -977,10 +943,15 @@ void FEA_Module_SGH::init_boundary_sets(int num_sets) } } -/* ---------------------------------------------------------------------------- - Grow boundary conditions sets of element boundary surfaces -------------------------------------------------------------------------------- */ - +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn grow_boundary_sets +/// +/// \brief Grow boundary conditions sets of element boundary surfaces +/// +/// \param Number of boundary sets +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_SGH::grow_boundary_sets(int num_sets) { int num_dim = simparam->num_dims; diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/sgh_optimization.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/sgh_optimization.cpp index 549c0f8ed..21b2c53ba 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/sgh_optimization.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/sgh_optimization.cpp @@ -70,10 +70,15 @@ #include "ROL_Solver.hpp" #include "Kinetic_Energy_Minimize.h" -/* ---------------------------------------------------------------------- - Compute new system response due to the design variable update -------------------------------------------------------------------------- */ - +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn update_forward_solve +/// +/// \brief Compute new system response due to the design variable update +/// +/// \param Current density value vector chosen by the optimizer +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_SGH::update_forward_solve(Teuchos::RCP zp) { const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; @@ -544,25 +549,36 @@ void FEA_Module_SGH::update_forward_solve(Teuchos::RCP zp) sgh_solve(); } -/* ------------------------------------------------------------------------------------------- - Compute average density of an element from nodal densities ----------------------------------------------------------------------------------------------- */ - -double FEA_Module_SGH::average_element_density(const int nodes_per_elem, const CArray current_element_densities) const +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn average_element_density +/// +/// \brief Compute average density of an element from nodal densities +/// +/// \param Nodes per elements +/// \param Current element densities +/// +/// \return average element density as a double +/// +///////////////////////////////////////////////////////////////////////////// +double FEA_Module_SGH::average_element_density(const int nodes_per_elem, const CArray current_nodal_densities) const { double result = 0; for (int i = 0; i < nodes_per_elem; i++) { - result += current_element_densities(i) / nodes_per_elem; + result += current_nodal_densities(i) / nodes_per_elem; } return result; } -/* ------------------------------------------------------------------------------ - Coupled adjoint problem for the kinetic energy minimization problem ---------------------------------------------------------------------------------- */ - +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn compute_topology_optimization_adjoint_full +/// +/// \brief Coupled adjoint problem for the kinetic energy minimization problem +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_SGH::compute_topology_optimization_adjoint_full() { const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; @@ -1194,10 +1210,16 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full() } } -/* ---------------------------------------------------------------------------- - Gradient for the (unsimplified) kinetic energy minimization problem -------------------------------------------------------------------------------- */ - +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn compute_topology_optimization_gradient_full +/// +/// \brief Gradient for the (unsimplified) kinetic energy minimization problem +/// +/// \param Distributed design densities +/// \param Distributed design gradients +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_SGH::compute_topology_optimization_gradient_full(Teuchos::RCP design_densities_distributed, Teuchos::RCP design_gradients_distributed) { size_t num_bdy_nodes = mesh->num_bdy_nodes; @@ -1600,9 +1622,13 @@ void FEA_Module_SGH::compute_topology_optimization_gradient_full(Teuchos::RCPnum_dims; @@ -1973,10 +1999,19 @@ void FEA_Module_SGH::init_assembly() // distributed_force_gradient_velocities->describe(*fos,Teuchos::VERB_EXTREME); } -/* ---------------------------------------------------------------------- - Enforce boundary conditions on the adjoint vectors -------------------------------------------------------------------------- */ - +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn boundary_adjoint +/// +/// \brief Enforce boundary conditions on the adjoint vectors +/// +/// \param Simulation mesh +/// \param Boundary condition array +/// \param Adjoint associated with the nodes +/// \param Phi adjoint associated with the nodes +/// \param Psi adjoint associated with the nodes +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_SGH::boundary_adjoint(const mesh_t& mesh, const DCArrayKokkos& boundary, vec_array& node_adjoint, @@ -2041,20 +2076,30 @@ void FEA_Module_SGH::boundary_adjoint(const mesh_t& mesh, return; } // end boundary_velocity function -/* ---------------------------------------------------------------------- - Communicate updated nodal adjoint vectors to ghost nodes -------------------------------------------------------------------------- */ - +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn comm_adjoint_vector +/// +/// \brief Communicate updated nodal adjoint vectors to ghost nodes +/// +/// \param Simulation cycle +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_SGH::comm_adjoint_vector(int cycle) { // comms to get ghosts (*adjoint_vector_data)[cycle]->doImport(*adjoint_vector_distributed, *importer, Tpetra::INSERT); } -/* ---------------------------------------------------------------------- - Communicate updated nodal adjoint vectors to ghost nodes -------------------------------------------------------------------------- */ - +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn comm_phi_adjoint_vector +/// +/// \brief Communicate updated nodal adjoint vectors to ghost nodes +/// +/// \param Simulation cycle +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_SGH::comm_phi_adjoint_vector(int cycle) { // comms to get ghosts diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/time_integration.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/time_integration.cpp index 93628c4ec..dd1d74577 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/time_integration.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/time_integration.cpp @@ -35,9 +35,20 @@ #include "state.h" #include "FEA_Module_SGH.h" -// ----------------------------------------------------------------------------- -// This function saves the variables at rk_stage = 0, which is t_n -// ------------------------------------------------------------------------------ +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn rk_init +/// +/// \brief This function saves the variables at rk_stage = 0, which is t_n +/// +/// \param View of nodal position data +/// \param View of nodal velocity data +/// \param View of element specific internal energy data +/// \param View of element stress +/// \param Number of elements +/// \param Number of nodes +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_SGH::rk_init(DViewCArrayKokkos& node_coords, DViewCArrayKokkos& node_vel, DViewCArrayKokkos& elem_sie, @@ -74,11 +85,22 @@ void FEA_Module_SGH::rk_init(DViewCArrayKokkos& node_coords, return; } // end rk_init -// ----------------------------------------------------------------------------- -// This function calculates the time step by finding the shortest distance -// between any two nodes in the mesh -// ------------------------------------------------------------------------------ -// WARNING WARNING : Only works for 3D, 8 node elements +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn get_timestep +/// +/// \brief This function calculates the time step by finding the shortest distance +/// between any two nodes in the mesh. +/// +/// WARNING WARNING : Only works for 3D, 8 node elements +/// +/// \param Simulation mesh +/// \param View of nodal position data +/// \param View of nodal velocity data +/// \param View of element sound speed +/// \param View of element volume +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_SGH::get_timestep(mesh_t& mesh, DViewCArrayKokkos& node_coords, DViewCArrayKokkos& node_vel, @@ -175,11 +197,22 @@ void FEA_Module_SGH::get_timestep(mesh_t& mesh, return; } // end get_timestep -// ----------------------------------------------------------------------------- -// This function calculates the time step by finding the shortest distance -// between any two nodes in the mesh -// ------------------------------------------------------------------------------ -// WARNING WARNING : Only works for 2D, 4 node elements +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn get_timestep2D +/// +/// \brief This function calculates the time step by finding the shortest distance +/// between any two nodes in the mesh. +/// +/// WARNING WARNING : Only works for 2D, 4 node elements +/// +/// \param Simulation mesh +/// \param View of nodal position data +/// \param View of nodal velocity data +/// \param View of element sound speed +/// \param View of element volume +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_SGH::get_timestep2D(mesh_t& mesh, DViewCArrayKokkos& node_coords, DViewCArrayKokkos& node_vel, From c0c1ab9fe1a8d01e9f7299dc35da56a1b77ef52b Mon Sep 17 00:00:00 2001 From: Jacob Moore Date: Sat, 24 Feb 2024 13:11:25 -0600 Subject: [PATCH 31/64] STYLE: Uncrustify --- python/Test-scripts/test_fierro.py | 10 -- scripts/build-fierro.sh | 2 +- scripts/trilinos-install.sh | 3 - src/Parallel-Solvers/FEA_Module.cpp | 3 - src/Parallel-Solvers/FEA_Module.h | 3 - src/Parallel-Solvers/FEA_Module_Inertial.cpp | 71 +++++------ src/Parallel-Solvers/FEA_Module_Inertial.h | 3 - src/Parallel-Solvers/Solver.cpp | 121 ++++++++++++------- src/Parallel-Solvers/Solver.h | 5 - src/Parallel-Solvers/node_combination.cpp | 3 - src/Parallel-Solvers/node_combination.h | 3 - 11 files changed, 114 insertions(+), 113 deletions(-) diff --git a/python/Test-scripts/test_fierro.py b/python/Test-scripts/test_fierro.py index 6baab8cb2..fbee4121e 100644 --- a/python/Test-scripts/test_fierro.py +++ b/python/Test-scripts/test_fierro.py @@ -5,15 +5,6 @@ import sys import math - - -# print("you passed in", len(sys.argv), "argument(s)") -# print("those arguments are: ", str(sys.argv)) - -# print("hello", sys.argv[0]) - -# os.system('pwd') - executable = "./../../build-fierro-openmp/bin/fierro-parallel-explicit" sim_input = "Solver-Inputs/SGH_Sedov_12x12x12.yaml" @@ -22,7 +13,6 @@ # Functions for reading results from vtk file - def extract_vector_data(filename, keyword): data = [] found_keyword = False diff --git a/scripts/build-fierro.sh b/scripts/build-fierro.sh index bedef4c51..0f0491f57 100755 --- a/scripts/build-fierro.sh +++ b/scripts/build-fierro.sh @@ -138,7 +138,7 @@ for arg in "$@"; do return 1 ;; *) - echo "Error: Invalid argument or value specified.scriptdir" + echo "Error: Invalid argument or value specified." show_help return 1 ;; diff --git a/scripts/trilinos-install.sh b/scripts/trilinos-install.sh index 579416312..d7d7877e1 100644 --- a/scripts/trilinos-install.sh +++ b/scripts/trilinos-install.sh @@ -6,7 +6,6 @@ machine="${2}" # If all arguments are valid, you can use them in your script as needed echo "Trilinos Kokkos Build Type: $kokkos_build_type" - #check if Trilinos directory exists, git clone Trilinos if it doesn't [ -d "${TRILINOS_SOURCE_DIR}" ] && echo "Directory Trilinos exists, skipping Trilinos download" @@ -115,8 +114,6 @@ ${ADDITIONS[@]} -D CMAKE_INSTALL_PREFIX=${TRILINOS_INSTALL_DIR} ) - - if [ "$kokkos_build_type" = "openmp" ]; then cmake_options+=( ${OPENMP_ADDITIONS[@]} diff --git a/src/Parallel-Solvers/FEA_Module.cpp b/src/Parallel-Solvers/FEA_Module.cpp index 3cf5165ca..ce756310a 100644 --- a/src/Parallel-Solvers/FEA_Module.cpp +++ b/src/Parallel-Solvers/FEA_Module.cpp @@ -11,14 +11,11 @@ This program is open source under the BSD-3 License. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. diff --git a/src/Parallel-Solvers/FEA_Module.h b/src/Parallel-Solvers/FEA_Module.h index 93c3fbb85..c1dacd9f2 100644 --- a/src/Parallel-Solvers/FEA_Module.h +++ b/src/Parallel-Solvers/FEA_Module.h @@ -11,14 +11,11 @@ This program is open source under the BSD-3 License. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. diff --git a/src/Parallel-Solvers/FEA_Module_Inertial.cpp b/src/Parallel-Solvers/FEA_Module_Inertial.cpp index e84945ea7..27bc9779d 100644 --- a/src/Parallel-Solvers/FEA_Module_Inertial.cpp +++ b/src/Parallel-Solvers/FEA_Module_Inertial.cpp @@ -11,14 +11,11 @@ This program is open source under the BSD-3 License. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. @@ -161,12 +158,14 @@ void FEA_Module_Inertial::compute_element_masses(const_host_vec_array design_den // CArrayKokkos legendre_weights_1D(num_gauss_points); CArray legendre_nodes_1D(num_gauss_points); CArray legendre_weights_1D(num_gauss_points); - real_t pointer_quad_coordinate[num_dim]; - real_t pointer_quad_coordinate_weight[num_dim]; - real_t pointer_interpolated_point[num_dim]; - real_t pointer_JT_row1[num_dim]; - real_t pointer_JT_row2[num_dim]; - real_t pointer_JT_row3[num_dim]; + + real_t pointer_quad_coordinate[num_dim]; + real_t pointer_quad_coordinate_weight[num_dim]; + real_t pointer_interpolated_point[num_dim]; + real_t pointer_JT_row1[num_dim]; + real_t pointer_JT_row2[num_dim]; + real_t pointer_JT_row3[num_dim]; + ViewCArray quad_coordinate(pointer_quad_coordinate, num_dim); ViewCArray quad_coordinate_weight(pointer_quad_coordinate_weight, num_dim); ViewCArray interpolated_point(pointer_interpolated_point, num_dim); @@ -174,14 +173,16 @@ void FEA_Module_Inertial::compute_element_masses(const_host_vec_array design_den ViewCArray JT_row2(pointer_JT_row2, num_dim); ViewCArray JT_row3(pointer_JT_row3, num_dim); - real_t pointer_basis_values[elem->num_basis()]; - real_t pointer_basis_derivative_s1[elem->num_basis()]; - real_t pointer_basis_derivative_s2[elem->num_basis()]; - real_t pointer_basis_derivative_s3[elem->num_basis()]; - ViewCArray basis_values(pointer_basis_values, elem->num_basis()); - ViewCArray basis_derivative_s1(pointer_basis_derivative_s1, elem->num_basis()); - ViewCArray basis_derivative_s2(pointer_basis_derivative_s2, elem->num_basis()); - ViewCArray basis_derivative_s3(pointer_basis_derivative_s3, elem->num_basis()); + real_t pointer_basis_values[elem->num_basis()]; + real_t pointer_basis_derivative_s1[elem->num_basis()]; + real_t pointer_basis_derivative_s2[elem->num_basis()]; + real_t pointer_basis_derivative_s3[elem->num_basis()]; + + ViewCArray basis_values(pointer_basis_values, elem->num_basis()); + ViewCArray basis_derivative_s1(pointer_basis_derivative_s1, elem->num_basis()); + ViewCArray basis_derivative_s2(pointer_basis_derivative_s2, elem->num_basis()); + ViewCArray basis_derivative_s3(pointer_basis_derivative_s3, elem->num_basis()); + CArrayKokkos nodal_positions(elem->num_basis(), num_dim); CArrayKokkos nodal_density(elem->num_basis()); @@ -189,8 +190,10 @@ void FEA_Module_Inertial::compute_element_masses(const_host_vec_array design_den elements::legendre_nodes_1D(legendre_nodes_1D, num_gauss_points); elements::legendre_weights_1D(legendre_weights_1D, num_gauss_points); - Solver::node_ordering_convention active_node_ordering_convention = Solver_Pointer_->active_node_ordering_convention; + Solver::node_ordering_convention active_node_ordering_convention = Solver_Pointer_->active_node_ordering_convention; + CArrayKokkos convert_node_order(max_nodes_per_element); + if ((active_node_ordering_convention == Solver::ENSIGHT && num_dim == 3) || (active_node_ordering_convention == Solver::IJK && num_dim == 2)) { convert_node_order(0) = 0; @@ -414,14 +417,14 @@ void FEA_Module_Inertial::compute_nodal_gradients(const_host_vec_array design_va real_t Jacobian, weight_multiply; // CArrayKokkos legendre_nodes_1D(num_gauss_points); // CArrayKokkos legendre_weights_1D(num_gauss_points); - CArray legendre_nodes_1D(num_gauss_points); - CArray legendre_weights_1D(num_gauss_points); - real_t pointer_quad_coordinate[num_dim]; - real_t pointer_quad_coordinate_weight[num_dim]; - real_t pointer_interpolated_point[num_dim]; - real_t pointer_JT_row1[num_dim]; - real_t pointer_JT_row2[num_dim]; - real_t pointer_JT_row3[num_dim]; + CArray legendre_nodes_1D(num_gauss_points); + CArray legendre_weights_1D(num_gauss_points); + real_t pointer_quad_coordinate[num_dim]; + real_t pointer_quad_coordinate_weight[num_dim]; + real_t pointer_interpolated_point[num_dim]; + real_t pointer_JT_row1[num_dim]; + real_t pointer_JT_row2[num_dim]; + real_t pointer_JT_row3[num_dim]; ViewCArray quad_coordinate(pointer_quad_coordinate, num_dim); ViewCArray quad_coordinate_weight(pointer_quad_coordinate_weight, num_dim); ViewCArray interpolated_point(pointer_interpolated_point, num_dim); @@ -429,14 +432,14 @@ void FEA_Module_Inertial::compute_nodal_gradients(const_host_vec_array design_va ViewCArray JT_row2(pointer_JT_row2, num_dim); ViewCArray JT_row3(pointer_JT_row3, num_dim); - real_t pointer_basis_values[elem->num_basis()]; - real_t pointer_basis_derivative_s1[elem->num_basis()]; - real_t pointer_basis_derivative_s2[elem->num_basis()]; - real_t pointer_basis_derivative_s3[elem->num_basis()]; - ViewCArray basis_values(pointer_basis_values, elem->num_basis()); - ViewCArray basis_derivative_s1(pointer_basis_derivative_s1, elem->num_basis()); - ViewCArray basis_derivative_s2(pointer_basis_derivative_s2, elem->num_basis()); - ViewCArray basis_derivative_s3(pointer_basis_derivative_s3, elem->num_basis()); + real_t pointer_basis_values[elem->num_basis()]; + real_t pointer_basis_derivative_s1[elem->num_basis()]; + real_t pointer_basis_derivative_s2[elem->num_basis()]; + real_t pointer_basis_derivative_s3[elem->num_basis()]; + ViewCArray basis_values(pointer_basis_values, elem->num_basis()); + ViewCArray basis_derivative_s1(pointer_basis_derivative_s1, elem->num_basis()); + ViewCArray basis_derivative_s2(pointer_basis_derivative_s2, elem->num_basis()); + ViewCArray basis_derivative_s3(pointer_basis_derivative_s3, elem->num_basis()); CArrayKokkos nodal_positions(elem->num_basis(), num_dim); CArrayKokkos nodal_density(elem->num_basis()); diff --git a/src/Parallel-Solvers/FEA_Module_Inertial.h b/src/Parallel-Solvers/FEA_Module_Inertial.h index 265adf32b..252de74a5 100644 --- a/src/Parallel-Solvers/FEA_Module_Inertial.h +++ b/src/Parallel-Solvers/FEA_Module_Inertial.h @@ -11,14 +11,11 @@ This program is open source under the BSD-3 License. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. diff --git a/src/Parallel-Solvers/Solver.cpp b/src/Parallel-Solvers/Solver.cpp index 5879eadbe..aedf41d40 100644 --- a/src/Parallel-Solvers/Solver.cpp +++ b/src/Parallel-Solvers/Solver.cpp @@ -11,14 +11,11 @@ This program is open source under the BSD-3 License. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. @@ -101,6 +98,13 @@ Solver::Solver(Simulation_Parameters& _simparam) : simparam(_simparam) displacement_module = -1; } +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn exit_solver +/// +/// \brief End the solve +/// +///////////////////////////////////////////////////////////////////////////// void Solver::exit_solver(int status) { Kokkos::finalize(); @@ -122,6 +126,13 @@ namespace elements { namespace elem_types { +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn from_vtk +/// +/// \brief Check the VTK element type read in +/// +///////////////////////////////////////////////////////////////////////////// elem_type from_vtk(const int& vtk_elem) { switch (vtk_elem) @@ -141,6 +152,15 @@ elem_type from_vtk(const int& vtk_elem) } // namespace elem_types } // namespace elements +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn generate_mesh +/// +/// \brief Generate simulation mesh +/// +/// \param Options for generating the mesh +/// +///////////////////////////////////////////////////////////////////////////// void Solver::generate_mesh(const std::shared_ptr& mesh_generation_options) { auto mesh = MeshBuilder::build_mesh(mesh_generation_options); @@ -156,7 +176,7 @@ void Solver::generate_mesh(const std::shared_ptr& mesh_generat num_nodes = mesh.points.dims(0); num_elem = mesh.element_point_index.dims(0); - map = Teuchos::rcp(new Tpetra::Map(num_nodes, 0, comm)); + map = Teuchos::rcp(new Tpetra::Map(num_nodes, 0, comm)); nlocal_nodes = map->getLocalNumElements(); @@ -279,7 +299,7 @@ void Solver::read_mesh_ensight(const char* MESH) size_t strain_count; size_t read_index_start, node_rid, elem_gid; - std::string skip_line, read_line, substring; + std::string skip_line, read_line, substring; std::stringstream line_parse; CArrayKokkos read_buffer; @@ -362,7 +382,7 @@ void Solver::read_mesh_ensight(const char* MESH) stores node data in a buffer and communicates once the buffer cap is reached or the data ends*/ - words_per_line = input_options.words_per_line; + words_per_line = input_options.words_per_line; elem_words_per_line = input_options.elem_words_per_line; // allocate read buffer @@ -651,8 +671,8 @@ void Solver::read_mesh_ensight(const char* MESH) // dynamic buffer used to store elements before we know how many this rank needs std::vector element_temp(BUFFER_LINES * elem_words_per_line); std::vector global_indices_temp(BUFFER_LINES); - size_t buffer_max = BUFFER_LINES * elem_words_per_line; - size_t indices_buffer_max = BUFFER_LINES; + size_t buffer_max = BUFFER_LINES * elem_words_per_line; + size_t indices_buffer_max = BUFFER_LINES; if (num_elem % BUFFER_LINES != 0) { @@ -992,8 +1012,8 @@ void Solver::read_mesh_vtk(const char* MESH) { Input_Options input_options = simparam.input_options.value(); - char ch; - std::string skip_line, read_line, substring; + char ch; + std::string skip_line, read_line, substring; std::stringstream line_parse; int num_dim = simparam.num_dims; @@ -1101,7 +1121,7 @@ void Solver::read_mesh_vtk(const char* MESH) stores node data in a buffer and communicates once the buffer cap is reached or the data ends*/ - words_per_line = input_options.words_per_line; + words_per_line = input_options.words_per_line; elem_words_per_line = input_options.elem_words_per_line; // allocate read buffer @@ -1282,8 +1302,8 @@ void Solver::read_mesh_vtk(const char* MESH) // dynamic buffer used to store elements before we know how many this rank needs std::vector element_temp(BUFFER_LINES * elem_words_per_line); std::vector global_indices_temp(BUFFER_LINES); - size_t buffer_max = BUFFER_LINES * elem_words_per_line; - size_t indices_buffer_max = BUFFER_LINES; + size_t buffer_max = BUFFER_LINES * elem_words_per_line; + size_t indices_buffer_max = BUFFER_LINES; if (num_elem % BUFFER_LINES != 0) { @@ -1630,25 +1650,33 @@ void Solver::read_mesh_vtk(const char* MESH) ------------------------------------------------------------------------- */ void Solver::read_mesh_tecplot(const char* MESH) { - char ch; - int num_dim = simparam.num_dims; - Input_Options input_options = simparam.input_options.value(); - int p_order = input_options.p_order; - real_t unit_scaling = input_options.unit_scaling; - bool restart_file = simparam.restart_file; - int local_node_index, current_column_index; - size_t strain_count; - std::string skip_line, read_line, substring; - std::stringstream line_parse; + Input_Options input_options = simparam.input_options.value(); + + char ch; + int num_dim = simparam.num_dims; + int p_order = input_options.p_order; + int negative_index_found = 0; + int global_negative_index_found = 0; + int buffer_loop, buffer_iteration, buffer_iterations, dof_limit, scan_loop; + int local_node_index, current_column_index; + + bool restart_file = simparam.restart_file; + bool zero_index_base = input_options.zero_index_base; + + size_t strain_count, read_index_start, node_rid, elem_gid; + + std::string skip_line, read_line, substring; + std::stringstream line_parse; + CArrayKokkos read_buffer; - int buffer_loop, buffer_iteration, buffer_iterations, dof_limit, scan_loop; - size_t read_index_start, node_rid, elem_gid; - GO node_gid; - real_t dof_value; - host_vec_array node_densities; - bool zero_index_base = input_options.zero_index_base; - int negative_index_found = 0; - int global_negative_index_found = 0; + + GO node_gid; + + real_t unit_scaling = input_options.unit_scaling; + real_t dof_value; + + host_vec_array node_densities; + // Nodes_Per_Element_Type = elements::elem_types::Nodes_Per_Element_Type; // read the mesh @@ -1892,8 +1920,8 @@ void Solver::read_mesh_tecplot(const char* MESH) // dynamic buffer used to store elements before we know how many this rank needs std::vector element_temp(BUFFER_LINES * elem_words_per_line); std::vector global_indices_temp(BUFFER_LINES); - size_t buffer_max = BUFFER_LINES * elem_words_per_line; - size_t indices_buffer_max = BUFFER_LINES; + size_t buffer_max = BUFFER_LINES * elem_words_per_line; + size_t indices_buffer_max = BUFFER_LINES; if (num_elem % BUFFER_LINES != 0) { @@ -2214,14 +2242,17 @@ void Solver::read_mesh_tecplot(const char* MESH) void Solver::repartition_nodes() { - char ch; - int num_dim = simparam.num_dims; - int local_node_index, current_column_index; - size_t strain_count; - std::stringstream line_parse; + char ch; + + int num_dim = simparam.num_dims; + int local_node_index, current_column_index, nodes_per_element; + + size_t strain_count; + + std::stringstream line_parse; CArrayKokkos read_buffer; - int nodes_per_element; - GO node_gid; + + GO node_gid; // construct input adapted needed by Zoltan2 problem typedef Xpetra::MultiVector xvector_t; @@ -2842,7 +2873,7 @@ void Solver::Get_Boundary_Patches() for (int isurface = 0; isurface < element_npatches; isurface++) { num_nodes_in_patch = elem2D->surface_to_dof_lid.stride(isurface); - Surface_Nodes = CArray(num_nodes_in_patch); + Surface_Nodes = CArray(num_nodes_in_patch); for (int inode = 0; inode < num_nodes_in_patch; inode++) { local_node_id = elem2D->surface_to_dof_lid(isurface, inode); @@ -2952,8 +2983,8 @@ void Solver::Get_Boundary_Patches() If only a subset of the nodes are local it must be a boundary patch; this case assigns the patch to the lowest mpi rank index the nodes in this patch belong to */ num_nodes_in_patch = Patch_Nodes(ipatch).node_set.size(); - my_rank_flag = true; - remote_count = 0; + my_rank_flag = true; + remote_count = 0; // assign as a local boundary patch if any of the nodes on the patch are local // only the local nodes on the patch will contribute to the equation assembly on this rank @@ -3005,7 +3036,7 @@ void Solver::Get_Boundary_Patches() void Solver::comm_importer_setup() { // create import object using local node indices map and ghost indices map - importer = Teuchos::rcp(new Tpetra::Import(map, all_node_map)); + importer = Teuchos::rcp(new Tpetra::Import(map, all_node_map)); ghost_importer = Teuchos::rcp(new Tpetra::Import(map, ghost_node_map)); dof_importer = Teuchos::rcp(new Tpetra::Import(local_dof_map, all_dof_map)); @@ -3013,7 +3044,7 @@ void Solver::comm_importer_setup() sorted_map = Teuchos::rcp(new Tpetra::Map(num_nodes, 0, comm)); node_sorting_importer = Teuchos::rcp(new Tpetra::Import(map, sorted_map)); // sorted element mapping - sorted_element_map = Teuchos::rcp(new Tpetra::Map(num_elem, 0, comm)); + sorted_element_map = Teuchos::rcp(new Tpetra::Map(num_elem, 0, comm)); element_sorting_importer = Teuchos::rcp(new Tpetra::Import(all_element_map, sorted_element_map));; } diff --git a/src/Parallel-Solvers/Solver.h b/src/Parallel-Solvers/Solver.h index b849d6a0a..c638108e7 100644 --- a/src/Parallel-Solvers/Solver.h +++ b/src/Parallel-Solvers/Solver.h @@ -11,14 +11,11 @@ This program is open source under the BSD-3 License. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. @@ -207,7 +204,6 @@ class Solver Kokkos::DualView ghost_nodes; Kokkos::DualView ghost_node_ranks; - // Node set corresponding to uniquely assigned list of elements on this MPI rank size_t nnonoverlap_elem_nodes; Kokkos::DualView nonoverlap_elem_nodes; @@ -264,7 +260,6 @@ class Solver CArrayKokkos Boundary_Patches; std::map boundary_patch_to_index; // maps patches to corresponding patch index (inverse of Boundary Patches array) - // file readin variables std::ifstream* in = NULL; std::streampos before_condition_header; diff --git a/src/Parallel-Solvers/node_combination.cpp b/src/Parallel-Solvers/node_combination.cpp index 89433fc4e..0e5862a52 100644 --- a/src/Parallel-Solvers/node_combination.cpp +++ b/src/Parallel-Solvers/node_combination.cpp @@ -11,14 +11,11 @@ This program is open source under the BSD-3 License. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. diff --git a/src/Parallel-Solvers/node_combination.h b/src/Parallel-Solvers/node_combination.h index 1b26696c5..72054c772 100644 --- a/src/Parallel-Solvers/node_combination.h +++ b/src/Parallel-Solvers/node_combination.h @@ -11,14 +11,11 @@ This program is open source under the BSD-3 License. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. From 3d98f169d6807c4bb36e6d735e0364e7a757c53b Mon Sep 17 00:00:00 2001 From: Jacob Moore Date: Sat, 24 Feb 2024 13:37:03 -0600 Subject: [PATCH 32/64] STYLE: Uncrustify --- src/Parallel-Solvers/FEA_Module.cpp | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/Parallel-Solvers/FEA_Module.cpp b/src/Parallel-Solvers/FEA_Module.cpp index ce756310a..30531e176 100644 --- a/src/Parallel-Solvers/FEA_Module.cpp +++ b/src/Parallel-Solvers/FEA_Module.cpp @@ -72,10 +72,10 @@ FEA_Module::FEA_Module(Solver* Solver_Pointer) (*fos).setOutputToRootOnly(0); // MPI Data copy - myrank = Solver_Pointer->myrank; - nranks = Solver_Pointer->nranks; - world = Solver_Pointer->world; - importer = Solver_Pointer->importer; + myrank = Solver_Pointer->myrank; + nranks = Solver_Pointer->nranks; + world = Solver_Pointer->world; + importer = Solver_Pointer->importer; ghost_importer = Solver_Pointer->ghost_importer; node_sorting_importer = Solver_Pointer->node_sorting_importer; element_sorting_importer = Solver_Pointer->element_sorting_importer; @@ -244,16 +244,16 @@ void FEA_Module::comm_filtered_densities() int FEA_Module::check_boundary(Node_Combination& Patch_Nodes, int bc_tag, real_t val, real_t* patch_limits) { - int is_on_set = 1; + int is_on_set = 1; const_host_vec_array all_node_coords = all_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); // Nodes on the Patch - auto node_list = Patch_Nodes.node_set; - int num_dim = simparam->num_dims; - size_t nnodes = node_list.size(); - size_t node_rid; - real_t node_coord[num_dim]; - int dim_other1, dim_other2; + auto node_list = Patch_Nodes.node_set; + int num_dim = simparam->num_dims; + size_t nnodes = node_list.size(); + size_t node_rid; + real_t node_coord[num_dim]; + int dim_other1, dim_other2; CArrayKokkos node_on_flags(nnodes, "node_on_flags"); // initialize From 9a4db305cc97defe1d648f4aacd65b26298c0992 Mon Sep 17 00:00:00 2001 From: Jacob Moore Date: Sat, 24 Feb 2024 13:50:55 -0600 Subject: [PATCH 33/64] BUG: Fix sedov input for tests --- python/Test-scripts/Solver-Inputs/SGH_Sedov_12x12x12.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/Test-scripts/Solver-Inputs/SGH_Sedov_12x12x12.yaml b/python/Test-scripts/Solver-Inputs/SGH_Sedov_12x12x12.yaml index a8604d82b..2adbcdfa6 100644 --- a/python/Test-scripts/Solver-Inputs/SGH_Sedov_12x12x12.yaml +++ b/python/Test-scripts/Solver-Inputs/SGH_Sedov_12x12x12.yaml @@ -68,7 +68,7 @@ materials: q2: 0 q1ex: 1.0 q2ex: 0.0 - global_vars: + eos_global_vars: - 1.666666666666667 - 1.0E-14 - 1.0 From 4e46c5358349c9d9d569f6acc3e7ce4f1c589bf6 Mon Sep 17 00:00:00 2001 From: Jacob Moore Date: Mon, 26 Feb 2024 09:36:07 -0600 Subject: [PATCH 34/64] STYLE: Uncrustify Dynamic_Elastic_Solver --- scripts/uncrustify.sh | 16 ++-- .../SGH_Solver/src/FEA_Module_SGH.cpp | 13 ++- .../Topology_Optimization/Area_Normals.h | 65 ++++++++++++-- .../Kinetic_Energy_Minimize.h | 85 +++++++++++++++++-- 4 files changed, 150 insertions(+), 29 deletions(-) diff --git a/scripts/uncrustify.sh b/scripts/uncrustify.sh index becdde2c2..47ed2b0fb 100755 --- a/scripts/uncrustify.sh +++ b/scripts/uncrustify.sh @@ -62,14 +62,14 @@ echo "Uncrusting: $sourceDir/Parallel-Solvers/Parallel-Explicit/Topology_Optimiz cd "$sourceDir/Parallel-Solvers/Parallel-Explicit/Topology_Optimization" || exit treeProcess -# Uncrustify Eulerian Solver -echo "Uncrusting: $sourceDir/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver" -cd "$sourceDir/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver" || exit -treeProcess - -# Uncrustify Dynamic Elastic -echo "Uncrusting: $sourceDir/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver" -cd "$sourceDir/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver" || exit +# # Uncrustify Eulerian Solver +# echo "Uncrusting: $sourceDir/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver" +# cd "$sourceDir/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver" || exit +# treeProcess + +# # Uncrustify Dynamic Elastic +# echo "Uncrusting: $sourceDir/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver" +# cd "$sourceDir/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver" || exit treeProcess exit diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/FEA_Module_SGH.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/FEA_Module_SGH.cpp index c6e8d19f5..6791ec4ef 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/FEA_Module_SGH.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/FEA_Module_SGH.cpp @@ -11,14 +11,11 @@ This program is open source under the BSD-3 License. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. @@ -997,7 +994,7 @@ void FEA_Module_SGH::sgh_solve() } int myrank = Explicit_Solver_Pointer_->myrank; - if(simparam->output_options.write_initial) + if (simparam->output_options.write_initial) { if (myrank == 0) { @@ -1743,9 +1740,10 @@ void FEA_Module_SGH::sgh_solve() } }); // end parallel for } // end view scope - - if(myrank==0){ - printf("Writing outputs to file at %f \n", graphics_time); + + if (myrank == 0) + { + printf("Writing outputs to file at %f \n", graphics_time); } double comm_time1 = Explicit_Solver_Pointer_->CPU_Time(); @@ -1754,7 +1752,6 @@ void FEA_Module_SGH::sgh_solve() double comm_time2 = Explicit_Solver_Pointer_->CPU_Time(); Explicit_Solver_Pointer_->output_time += comm_time2 - comm_time1; - graphics_time = time_value + graphics_dt_ival; } // end if diff --git a/src/Parallel-Solvers/Parallel-Explicit/Topology_Optimization/Area_Normals.h b/src/Parallel-Solvers/Parallel-Explicit/Topology_Optimization/Area_Normals.h index e880c1a75..4255258b5 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Topology_Optimization/Area_Normals.h +++ b/src/Parallel-Solvers/Parallel-Explicit/Topology_Optimization/Area_Normals.h @@ -11,14 +11,11 @@ This program is open source under the BSD-3 License. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. @@ -98,11 +95,33 @@ typedef MV::dual_view_type dual_vec_array; ROL::Ptr ROL_Velocities; ROL::Ptr ROL_Gradients; + ///////////////////////////////////////////////////////////////////////////// + /// + /// \fn getVector + /// + /// \brief Retrieves ROL vector at desired location + /// + /// \param Pointer to desired ROL vector + /// + /// \return Returns const ROL MV vector + /// + ///////////////////////////////////////////////////////////////////////////// ROL::Ptr getVector(const V& x) { return dynamic_cast(x).getVector(); } + ///////////////////////////////////////////////////////////////////////////// + /// + /// \fn getVector + /// + /// \brief Retrieves ROL vector at desired location + /// + /// \param Pointer to desired ROL vector + /// + /// \return Returns ROL MV vector + /// + ///////////////////////////////////////////////////////////////////////////// ROL::Ptr getVector(V& x) { return dynamic_cast(x).getVector(); @@ -154,10 +173,35 @@ typedef MV::dual_view_type dual_vec_array; // std::cout << "INITIAL KINETIC ENERGY " << current_kinetic_energy << std::endl; } + ///////////////////////////////////////////////////////////////////////////// + /// + /// \fn update + /// + /// \brief Unused update function + /// + /// \param Unused in this context + /// \param Unused in this context + /// \param Unused in this context + /// + /// + ///////////////////////////////////////////////////////////////////////////// void update(const ROL::Vector& z, ROL::UpdateType type, int iter = -1) { } + ///////////////////////////////////////////////////////////////////////////// + /// + /// \fn value + /// + /// \brief Returns objective value for optimiation + /// + /// + /// \param Objective value vector + /// \param Value tolerance + /// + /// \return Objective value + /// + ///////////////////////////////////////////////////////////////////////////// real_t value(const ROL::Vector& z, real_t& tol) { double area_normal_array[24]; @@ -197,10 +241,17 @@ typedef MV::dual_view_type dual_vec_array; return objective_value; } - // void gradient_1( ROL::Vector &g, const ROL::Vector &u, const ROL::Vector &z, real_t &tol ) { - // g.zero(); - // } - + ///////////////////////////////////////////////////////////////////////////// + /// + /// \fn gradient + /// + /// \brief Calculate design gradient + /// + /// \param Vector of gradient values + /// \param Objective value vector + /// \param Design tolerance + /// + ///////////////////////////////////////////////////////////////////////////// void gradient(ROL::Vector& g, const ROL::Vector& z, real_t& tol) { // std::cout << "Started obj gradient on task " <myrank << std::endl; diff --git a/src/Parallel-Solvers/Parallel-Explicit/Topology_Optimization/Kinetic_Energy_Minimize.h b/src/Parallel-Solvers/Parallel-Explicit/Topology_Optimization/Kinetic_Energy_Minimize.h index 4799a8b8a..0fcbe8618 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Topology_Optimization/Kinetic_Energy_Minimize.h +++ b/src/Parallel-Solvers/Parallel-Explicit/Topology_Optimization/Kinetic_Energy_Minimize.h @@ -11,14 +11,11 @@ This program is open source under the BSD-3 License. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. @@ -100,11 +97,33 @@ typedef MV::dual_view_type dual_vec_array; bool useLC_; // Use linear form of energy. Otherwise use quadratic form. + ///////////////////////////////////////////////////////////////////////////// + /// + /// \fn getVector + /// + /// \brief Retrieves ROL vector at desired location + /// + /// \param Pointer to desired ROL vector + /// + /// \return Returns ROL MV vector + /// + ///////////////////////////////////////////////////////////////////////////// ROL::Ptr getVector(const V& x) { return dynamic_cast(x).getVector(); } + ///////////////////////////////////////////////////////////////////////////// + /// + /// \fn getVector + /// + /// \brief Retrieves ROL vector at desired location + /// + /// \param Pointer to desired ROL vector + /// + /// \return Returns const ROL MV vector + /// + ///////////////////////////////////////////////////////////////////////////// ROL::Ptr getVector(V& x) { return dynamic_cast(x).getVector(); @@ -167,6 +186,17 @@ typedef MV::dual_view_type dual_vec_array; // std::cout << "INITIAL KINETIC ENERGY " << current_kinetic_energy << std::endl; } + ///////////////////////////////////////////////////////////////////////////// + /// + /// \fn update + /// + /// \brief Evolve the physical simulation + /// + /// \param Design vector + /// \param Update type + /// \param Iteration + /// + ///////////////////////////////////////////////////////////////////////////// void update(const ROL::Vector& z, ROL::UpdateType type, int iter = -1) { if (set_module_type == FEA_MODULE_TYPE::SGH) @@ -179,6 +209,17 @@ typedef MV::dual_view_type dual_vec_array; } } + ///////////////////////////////////////////////////////////////////////////// + /// + /// \fn update_elasticity + /// + /// \brief Evolve the simulation using dynamic elasticity + /// + /// \param Design vector + /// \param Update type + /// \param Iteration + /// + ///////////////////////////////////////////////////////////////////////////// void update_elasticity(const ROL::Vector& z, ROL::UpdateType type, int iter = -1) { // debug @@ -244,6 +285,17 @@ typedef MV::dual_view_type dual_vec_array; } } + ///////////////////////////////////////////////////////////////////////////// + /// + /// \fn update_elasticity + /// + /// \brief Evolve the simulation using staggered grid hydrodynamics + /// + /// \param Design vector + /// \param Update type + /// \param Iteration + /// + ///////////////////////////////////////////////////////////////////////////// void update_sgh(const ROL::Vector& z, ROL::UpdateType type, int iter = -1) { // debug @@ -309,6 +361,19 @@ typedef MV::dual_view_type dual_vec_array; } } + ///////////////////////////////////////////////////////////////////////////// + /// + /// \fn value + /// + /// \brief Returns objective value for optimiation + /// + /// + /// \param Objective value vector + /// \param Value tolerance + /// + /// \return Objective value + /// + ///////////////////////////////////////////////////////////////////////////// real_t value(const ROL::Vector& z, real_t& tol) { // std::cout << "Started obj value on task " <myrank << std::endl; @@ -367,10 +432,18 @@ typedef MV::dual_view_type dual_vec_array; return objective_accumulation; } - // void gradient_1( ROL::Vector &g, const ROL::Vector &u, const ROL::Vector &z, real_t &tol ) { - // g.zero(); - // } + ///////////////////////////////////////////////////////////////////////////// + /// + /// \fn gradient + /// + /// \brief Calculate design gradient + /// + /// \param Vector of gradient values + /// \param Objective value vector + /// \param Design tolerance + /// + ///////////////////////////////////////////////////////////////////////////// void gradient(ROL::Vector& g, const ROL::Vector& z, real_t& tol) { // std::cout << "Started obj gradient on task " <myrank << std::endl; From 966e62d77e68a7fe28dd4311851a66a1ae14ff1d Mon Sep 17 00:00:00 2001 From: Jacob Moore Date: Mon, 26 Feb 2024 10:29:45 -0600 Subject: [PATCH 35/64] STYLE: Uncrustify and document Eulerian Solver --- .../Eulerian_Solver/FEA_Module_Eulerian.cpp | 149 ++++++++++++------ .../Simulation_Parameters_Eulerian.h | 44 +++++- .../Eulerian_Solver/eulerian_geometry.cpp | 38 +++-- .../eulerian_time_integration.cpp | 63 ++++++-- .../Kinetic_Energy_Minimize.h | 1 - 5 files changed, 219 insertions(+), 76 deletions(-) diff --git a/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/FEA_Module_Eulerian.cpp b/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/FEA_Module_Eulerian.cpp index c690afa28..d7f57b7bf 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/FEA_Module_Eulerian.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/FEA_Module_Eulerian.cpp @@ -199,9 +199,16 @@ FEA_Module_Eulerian::~FEA_Module_Eulerian() // delete simparam; } -/* ---------------------------------------------------------------------- - Read ANSYS dat format mesh file -------------------------------------------------------------------------- */ +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn read_conditions_ansys_dat +/// +/// \brief Read ANSYS dat format mesh file +/// +/// \param Input file path +/// \param Before conditions +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_Eulerian::read_conditions_ansys_dat(std::ifstream* in, std::streampos before_condition_header) { char ch; @@ -229,10 +236,13 @@ void FEA_Module_Eulerian::read_conditions_ansys_dat(std::ifstream* in, std::stre host_vec_array node_densities; } // end read_conditions_ansys_dat -/* ---------------------------------------------------------------------------- - Initialize sets of element boundary surfaces and arrays for input conditions -------------------------------------------------------------------------------- */ - +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn init_boundaries +/// +/// \brief Initialize sets of element boundary surfaces and arrays for input conditions +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_Eulerian::init_boundaries() { max_boundary_sets = simparam->NB; @@ -264,10 +274,16 @@ void FEA_Module_Eulerian::init_boundaries() Number_DOF_BCS = 0; } -/* ---------------------------------------------------------------------- - initialize storage for element boundary surfaces corresponding to user BCs -------------------------------------------------------------------------- */ - +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn init_boundary_sets +/// +/// \brief initialize storage for element boundary surfaces corresponding +/// to user BCs +/// +/// \param Number of boundary sets +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_Eulerian::init_boundary_sets(int num_sets) { if (num_sets == 0) @@ -296,10 +312,15 @@ void FEA_Module_Eulerian::init_boundary_sets(int num_sets) } } -/* ---------------------------------------------------------------------------- - Grow boundary conditions sets of element boundary surfaces -------------------------------------------------------------------------------- */ - +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn grow_boundary_sets +/// +/// \brief Grow boundary conditions sets of element boundary surfaces +/// +/// \param Number of boundary sets +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_Eulerian::grow_boundary_sets(int num_sets) { int num_dim = simparam->num_dims; @@ -350,27 +371,40 @@ void FEA_Module_Eulerian::grow_boundary_sets(int num_sets) } } -/* ---------------------------------------------------------------------- - Assign sets of element boundary surfaces corresponding to user BCs -------------------------------------------------------------------------- */ - +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn generate_bcs +/// +/// \brief Assign sets of element boundary surfaces corresponding to user BCs +/// +/// Unused in this context +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_Eulerian::generate_bcs() { } // end generate_bcs -/* ---------------------------------------------------------------------- - Loop through applied boundary conditions and tag node ids to remove - necessary rows and columns from the assembled linear system -------------------------------------------------------------------------- */ - +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn generate_bcs +/// +/// \brief Loop through applied boundary conditions and tag node ids to remove +/// necessary rows and columns from the assembled linear system +/// +/// Unused in this context +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_Eulerian::Displacement_Boundary_Conditions() { } -/* ---------------------------------------------------------------------------- - Initialize output data structures -------------------------------------------------------------------------------- */ - +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn init_output +/// +/// \brief Initialize output data structures +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_Eulerian::init_output() { // check user parameters for output @@ -496,10 +530,13 @@ void FEA_Module_Eulerian::compute_output() { } -/* ---------------------------------------------------------------------- - Communicate updated nodal velocities to ghost nodes -------------------------------------------------------------------------- */ - +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn comm_node_masses +/// +/// \brief Communicate updated nodal velocities to ghost nodes +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_Eulerian::comm_node_masses() { // debug print of design vector @@ -527,10 +564,15 @@ void FEA_Module_Eulerian::comm_node_masses() // } } -/* ---------------------------------------------------------------------- - Communicate updated nodal adjoint vectors to ghost nodes -------------------------------------------------------------------------- */ - +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn comm_adjoint_vectors +/// +/// \brief Communicate updated nodal adjoint vectors to ghost nodes +/// +/// \param Optimization iteration cycle count +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_Eulerian::comm_adjoint_vectors(int cycle) { // debug print of design vector @@ -559,10 +601,15 @@ void FEA_Module_Eulerian::comm_adjoint_vectors(int cycle) // } } -/* ------------------------------------------------------------------------------------------- - Communicate ghosts using the current optimization design data ----------------------------------------------------------------------------------------------- */ - +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn comm_variables +/// +/// \brief Communicate ghosts using the current optimization design data +/// +/// \param Design variables +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_Eulerian::comm_variables(Teuchos::RCP zp) { if (simparam_dynamic_opt.topology_optimization_on) @@ -600,10 +647,13 @@ void FEA_Module_Eulerian::node_density_constraints(host_vec_array node_densities { } -/* ---------------------------------------------------------------------------- - Setup Eulerian solver data -------------------------------------------------------------------------------- */ - +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn setup +/// +/// \brief Setup Eulerian solver data +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_Eulerian::setup() { const size_t rk_level = simparam->rk_num_bins - 1; @@ -676,10 +726,13 @@ void FEA_Module_Eulerian::setup() return; } // end of setup -/* ---------------------------------------------------------------------------- - Eulerian solver loop -------------------------------------------------------------------------------- */ - +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn euler_solve +/// +/// \brief Eulerian solver loop +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_Eulerian::euler_solve() { Time_Variables tv = simparam->time_variables; diff --git a/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/Simulation_Parameters_Eulerian.h b/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/Simulation_Parameters_Eulerian.h index 94aa7cd67..9bb4ad56d 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/Simulation_Parameters_Eulerian.h +++ b/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/Simulation_Parameters_Eulerian.h @@ -11,14 +11,11 @@ This program is open source under the BSD-3 License. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. @@ -82,6 +79,16 @@ struct Simulation_Parameters_Eulerian : Simulation_Parameters DCArrayKokkos boundary; std::vector gravity_vector { 9.81, 0., 0. }; + ///////////////////////////////////////////////////////////////////////////// + /// + /// \fn init_material_variable_arrays + /// + /// \brief Initialize data for EOS and material strength models + /// + /// \param Number of global EOS variables + /// \param Number of global strength variables + /// + ///////////////////////////////////////////////////////////////////////////// void init_material_variable_arrays(size_t num_eos_global_vars, size_t num_strength_global_vars) { eos_global_vars = DCArrayKokkos(material_options.size(), num_eos_global_vars); @@ -103,6 +110,16 @@ struct Simulation_Parameters_Eulerian : Simulation_Parameters } } + ///////////////////////////////////////////////////////////////////////////// + /// + /// \fn from_vector + /// + /// \brief Copies data from a vector on the host device + /// + /// \param Array data is coped into + /// \param Vector on the host device + /// + ///////////////////////////////////////////////////////////////////////////// template void from_vector(DCArrayKokkos& array, const std::vector& vec) { array = DCArrayKokkos(vec.size()); @@ -112,6 +129,13 @@ struct Simulation_Parameters_Eulerian : Simulation_Parameters } } + ///////////////////////////////////////////////////////////////////////////// + /// + /// \fn derive_kokkos_arrays + /// + /// \brief Updates arrays on the device + /// + ///////////////////////////////////////////////////////////////////////////// void derive_kokkos_arrays() { max_num_eos_global_vars = 0; @@ -136,12 +160,26 @@ struct Simulation_Parameters_Eulerian : Simulation_Parameters strength_global_vars.update_device(); } + ///////////////////////////////////////////////////////////////////////////// + /// + /// \fn derive + /// + /// \brief Calls derive_kokkos_arrays and sets the number or Runge Kutta integration + /// + ///////////////////////////////////////////////////////////////////////////// void derive() { derive_kokkos_arrays(); rk_num_bins = rk_num_stages; } + ///////////////////////////////////////////////////////////////////////////// + /// + /// \fn validate + /// + /// \brief Validates the specified FEA module + /// + ///////////////////////////////////////////////////////////////////////////// void validate() { validate_module_is_specified(FEA_MODULE_TYPE::Eulerian); diff --git a/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/eulerian_geometry.cpp b/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/eulerian_geometry.cpp index 025655c73..c7522856a 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/eulerian_geometry.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/eulerian_geometry.cpp @@ -6,6 +6,20 @@ #include "state.h" #include "FEA_Module_Eulerian.h" +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn example_function +/// +/// \brief Example showing how to do parallel loops using MATAR +/// +/// \param Runge Kutta integration alpha +/// \param Number of nodes +/// \param Nodal positions +/// \param Nodal velocity +/// +/// \return +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_Eulerian::example_function(double rk_alpha, const size_t num_nodes, DViewCArrayKokkos& node_coords, @@ -24,17 +38,19 @@ void FEA_Module_Eulerian::example_function(double rk_alpha, }); // end parallel for over nodes } // end subroutine -// ----------------------------------------------------------------------------- -// This function claculates -// B_p = J^{-T} \cdot (\nabla_{xi} \phi_p w -// where -// \phi_p is the basis function for vertex p -// w is the 1 gauss point for the cell (everything is evaluted at this point) -// J^{-T} is the inverse transpose of the Jacobi matrix -// \nabla_{xi} is the gradient opperator in the reference coordinates -// -// B_p is the OUTWARD corner area normal at node p -// ------------------------------------------------------------------------------ +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn example_function +/// +/// \brief Example showing how to do device loops using MATAR +/// +/// \param Finite Element B Matrix +/// \param Global element ID +/// \param Nodal positions +/// \param Nodal Global indices of nodes in the element +/// \param Runge Kutta time integration level +/// +///////////////////////////////////////////////////////////////////////////// KOKKOS_FUNCTION void FEA_Module_Eulerian::example_device_function(const ViewCArrayKokkos& B_matrix, const size_t elem_gid, diff --git a/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/eulerian_time_integration.cpp b/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/eulerian_time_integration.cpp index 29f0624ae..a65205ad8 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/eulerian_time_integration.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/eulerian_time_integration.cpp @@ -35,9 +35,20 @@ #include "state.h" #include "FEA_Module_Eulerian.h" -// ----------------------------------------------------------------------------- -// This function saves the variables at rk_stage = 0, which is t_n -// ------------------------------------------------------------------------------ +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn rk_init +/// +/// \brief This function saves the variables at rk_stage = 0, which is t_n +/// +/// \param Nodal positions +/// \param Nodal velocities +/// \param Element specific internal energy +/// \param Element stress +/// \param Number of elements +/// \param Number of nodes +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_Eulerian::rk_init(DViewCArrayKokkos& node_coords, DViewCArrayKokkos& node_vel, DViewCArrayKokkos& elem_sie, @@ -74,11 +85,24 @@ void FEA_Module_Eulerian::rk_init(DViewCArrayKokkos& node_coords, return; } // end rk_init -// ----------------------------------------------------------------------------- -// This function calculates the time step by finding the shortest distance -// between any two nodes in the mesh -// ------------------------------------------------------------------------------ -// WARNING WARNING : Only works for 3D, 8 node elements +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn get_timestep +/// +/// \brief Calculates CFL compliant timestep +/// +/// This function calculates the time step by finding the shortest distance +/// between any two nodes in the mesh +/// +/// WARNING WARNING : Only works for 3D, 8 node elements +/// +/// \param Simulation mesh +/// \param Nodal positions +/// \param Nodal velocities +/// \param Element sound speed +/// \param Element volume +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_Eulerian::get_timestep(mesh_t& mesh, DViewCArrayKokkos& node_coords, DViewCArrayKokkos& node_vel, @@ -175,11 +199,24 @@ void FEA_Module_Eulerian::get_timestep(mesh_t& mesh, return; } // end get_timestep -// ----------------------------------------------------------------------------- -// This function calculates the time step by finding the shortest distance -// between any two nodes in the mesh -// ------------------------------------------------------------------------------ -// WARNING WARNING : Only works for 3D, 8 node elements +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn get_timestep2D +/// +/// \brief Calculates CFL compliant timestep +/// +/// This function calculates the time step by finding the shortest distance +/// between any two nodes in the mesh +/// +/// WARNING WARNING : Only works for 2D, 4 node elements +/// +/// \param Simulation mesh +/// \param Nodal positions +/// \param Nodal velocities +/// \param Element sound speed +/// \param Element volume +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_Eulerian::get_timestep2D(mesh_t& mesh, DViewCArrayKokkos& node_coords, DViewCArrayKokkos& node_vel, diff --git a/src/Parallel-Solvers/Parallel-Explicit/Topology_Optimization/Kinetic_Energy_Minimize.h b/src/Parallel-Solvers/Parallel-Explicit/Topology_Optimization/Kinetic_Energy_Minimize.h index 0fcbe8618..556c4c3b0 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Topology_Optimization/Kinetic_Energy_Minimize.h +++ b/src/Parallel-Solvers/Parallel-Explicit/Topology_Optimization/Kinetic_Energy_Minimize.h @@ -432,7 +432,6 @@ typedef MV::dual_view_type dual_vec_array; return objective_accumulation; } - ///////////////////////////////////////////////////////////////////////////// /// /// \fn gradient From b9b330e7e752cf936fb2492131e35a220887b8af Mon Sep 17 00:00:00 2001 From: Jacob Moore Date: Mon, 26 Feb 2024 12:10:39 -0600 Subject: [PATCH 36/64] DOC: Add uncrustify documentation to Dynamic Elastic solver --- .../FEA_Module_Dynamic_Elasticity.cpp | 410 ++- .../FEA_Module_Dynamic_Elasticity.h | 1024 +++--- .../Dynamic_Elastic_Solver/boundary.cpp | 16 +- .../elastic_optimization.cpp | 2990 +++++++++-------- .../Dynamic_Elastic_Solver/force_elastic.cpp | 142 +- .../force_gradients_elastic.cpp | 101 +- .../Dynamic_Elastic_Solver/geometry.cpp | 142 +- .../Dynamic_Elastic_Solver/mesh.h | 94 +- .../Dynamic_Elastic_Solver/momentum.cpp | 138 +- .../Dynamic_Elastic_Solver/properties.cpp | 47 +- .../time_integration.cpp | 58 +- .../SGH_Solver/src/momentum.cpp | 6 +- 12 files changed, 2886 insertions(+), 2282 deletions(-) diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.cpp b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.cpp index 87417e713..a35f0bc82 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.cpp @@ -178,9 +178,15 @@ FEA_Module_Dynamic_Elasticity::~FEA_Module_Dynamic_Elasticity() // delete simparam; } -/* ---------------------------------------------------------------------- - Read ANSYS dat format mesh file -------------------------------------------------------------------------- */ +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn read_conditions_ansys_dat +/// +/// \brief Read ANSYS dat format mesh file +/// +/// \param Initial conditions header +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_Dynamic_Elasticity::read_conditions_ansys_dat(std::ifstream* in, std::streampos before_condition_header) { Input_Options input_options = simparam->input_options.value(); @@ -212,9 +218,17 @@ void FEA_Module_Dynamic_Elasticity::read_conditions_ansys_dat(std::ifstream* in, host_vec_array node_densities; } // end read_conditions_ansys_dat -// ----------------------------------------------------------------------------- -// Interfaces read in data with the elastic solver data; currently a hack to streamline -// ------------------------------------------------------------------------------ +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn elastic_interface_setup +/// +/// \brief Interfaces read in data with the elastic solver data; currently a hack to streamline +/// +/// \param Nodal state data +/// \param Element state data +/// \param Corner state data +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_Dynamic_Elasticity::elastic_interface_setup(node_t& node, elem_t& elem, corner_t& corner) @@ -415,10 +429,13 @@ void FEA_Module_Dynamic_Elasticity::elastic_interface_setup(node_t& node, return; } -/* ---------------------------------------------------------------------------- - Initialize sets of element boundary surfaces and arrays for input conditions -------------------------------------------------------------------------------- */ - +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn init_boundaries +/// +/// \brief Initialize sets of element boundary surfaces and arrays for input conditions +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_Dynamic_Elasticity::init_boundaries() { max_boundary_sets = module_params->boundary_conditions.size(); @@ -450,10 +467,15 @@ void FEA_Module_Dynamic_Elasticity::init_boundaries() Number_DOF_BCS = 0; } -/* ---------------------------------------------------------------------- - initialize storage for element boundary surfaces corresponding to user BCs -------------------------------------------------------------------------- */ - +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn init_boundary_sets +/// +/// \brief Initialize storage for element boundary surfaces corresponding to user BCs +/// +/// \param Number of boundary sets +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_Dynamic_Elasticity::init_boundary_sets(int num_sets) { if (num_sets == 0) @@ -482,10 +504,15 @@ void FEA_Module_Dynamic_Elasticity::init_boundary_sets(int num_sets) } } -/* ---------------------------------------------------------------------------- - Grow boundary conditions sets of element boundary surfaces -------------------------------------------------------------------------------- */ - +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn grow_boundary_sets +/// +/// \brief Grow boundary conditions sets of element boundary surfaces +/// +/// \param Number of boundary sets +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_Dynamic_Elasticity::grow_boundary_sets(int num_sets) { int num_dim = simparam->num_dims; @@ -536,27 +563,40 @@ void FEA_Module_Dynamic_Elasticity::grow_boundary_sets(int num_sets) } } -/* ---------------------------------------------------------------------- - Assign sets of element boundary surfaces corresponding to user BCs -------------------------------------------------------------------------- */ - +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn generate_bcs +/// +/// \brief Assign sets of element boundary surfaces corresponding to user BCs +/// +/// Unused in this context +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_Dynamic_Elasticity::generate_bcs() { } // end generate_bcs -/* ---------------------------------------------------------------------- - Loop through applied boundary conditions and tag node ids to remove - necessary rows and columns from the assembled linear system -------------------------------------------------------------------------- */ - +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn Displacement_Boundary_Conditions +/// +/// \brief Loop through applied boundary conditions and tag node ids to remove +/// necessary rows and columns from the assembled linear system +/// +/// Unused in this context +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_Dynamic_Elasticity::Displacement_Boundary_Conditions() { } -/* ---------------------------------------------------------------------------- - Initialize output data structures -------------------------------------------------------------------------------- */ - +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn init_output +/// +/// \brief Initialize output data structures +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_Dynamic_Elasticity::init_output() { // check user parameters for output @@ -660,18 +700,32 @@ void FEA_Module_Dynamic_Elasticity::init_output() } } -/* ------------------------------------------------------------------------------------------- - Prompts sorting for elastic response output data. For now, nodal strains. ----------------------------------------------------------------------------------------------- */ - +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn sort_output +/// +/// \brief Prompts sorting for elastic response output data. For now, nodal strains. +/// +/// Unused in this context +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_Dynamic_Elasticity::sort_output(Teuchos::RCP> sorted_map) { } -/* ------------------------------------------------------------------------------------------- - populate requests this module makes for output data ----------------------------------------------------------------------------------------------- */ - +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn write_data +/// +/// \brief Populate requests this module makes for output data +/// +/// \param Map of scalar point data +/// \param Map of vector point data +/// \param Map of scalar cell data +/// \param Map of int cell data +/// \param Map of cell field data +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_Dynamic_Elasticity::write_data(std::map& point_data_scalars_double, std::map& point_data_vectors_double, std::map& cell_data_scalars_double, @@ -750,10 +804,15 @@ void FEA_Module_Dynamic_Elasticity::write_data(std::mapelem_stress.host(rk_level,0,0,0), 9); } -/* ------------------------------------------------------------------------------------------- - Prompts sorting for elastic response output data. For now, nodal strains. ----------------------------------------------------------------------------------------------- */ - +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn sort_element_output +/// +/// \brief Prompts sorting for elastic response output data. For now, element densities +/// +/// \param Sorted element data map +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_Dynamic_Elasticity::sort_element_output(Teuchos::RCP> sorted_map) { // interface element density data @@ -767,26 +826,39 @@ void FEA_Module_Dynamic_Elasticity::sort_element_output(Teuchos::RCP> global_reduce_map) { } -/* ------------------------------------------------------------------------------------------- - Prompts computation of elastic response output data. For now, nodal strains. ----------------------------------------------------------------------------------------------- */ - +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn compute_output +/// +/// \brief Prompts computation of elastic response output data. +/// +/// Unused in this context +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_Dynamic_Elasticity::compute_output() { } -/* ---------------------------------------------------------------------- - Communicate updated nodal velocities to ghost nodes -------------------------------------------------------------------------- */ - +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn comm_node_masses +/// +/// \brief Communicate updated nodal velocities to ghost nodes +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_Dynamic_Elasticity::comm_node_masses() { // debug print of design vector @@ -814,10 +886,15 @@ void FEA_Module_Dynamic_Elasticity::comm_node_masses() // } } -/* ------------------------------------------------------------------------------------------- - Communicate ghosts using the current optimization design data ----------------------------------------------------------------------------------------------- */ - +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn comm_variables +/// +/// \brief Communicate ghosts using the current optimization design data +/// +/// \param Design variables +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_Dynamic_Elasticity::comm_variables(Teuchos::RCP zp) { if (simparam->topology_optimization_on) @@ -847,10 +924,15 @@ void FEA_Module_Dynamic_Elasticity::comm_variables(Teuchos::RCP zp) } } -/* ------------------------------------------------------------------------------------------- - enforce constraints on nodes due to BCS ----------------------------------------------------------------------------------------------- */ - +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn node_density_constraints +/// +/// \brief Enforce constraints on nodes due to BCS +/// +/// \param Lower bound of nodal densities +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_Dynamic_Elasticity::node_density_constraints(host_vec_array& node_densities_lower_bound) { const_vec_array all_initial_node_coords = all_initial_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); @@ -884,10 +966,13 @@ void FEA_Module_Dynamic_Elasticity::node_density_constraints(host_vec_array& nod }); // end for parallel for over nodes } -/* ---------------------------------------------------------------------------- - Setup elastic solver data -------------------------------------------------------------------------------- */ - +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn setup +/// +/// \brief Setup elastic solver data +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_Dynamic_Elasticity::setup() { Dynamic_Options dynamic_options = simparam->dynamic_options; @@ -905,41 +990,6 @@ void FEA_Module_Dynamic_Elasticity::setup() // --------------------------------------------------------------------- elastic_interface_setup(node_interface, elem_interface, corner_interface); mesh->build_corner_connectivity(); - // debug print of corner ids - /* - if(myrank==1){ - for(int i = 0; i < mesh.num_nodes; i++){ - - // loop over all corners around the node and calculate the nodal force - for (size_t corner_lid=0; corner_lidgetGlobalElement(i) << " " << i << " " << all_node_map->getLocalElement(all_node_map->getGlobalElement(i)) << " " << corner_gid << " " << std::endl; - - } // end for corner_lid - //std::cout << explicit_solver_pointer->all_node_map->getGlobalElement(i) << " " << node_force[0] << " " << node_force[1] << " " << node_force[2] << std::endl; - //std::cout << explicit_solver_pointer->all_node_map->getGlobalElement(i) << " " << node_mass(i) << std::endl; - } - } - */ - /* - if(myrank==1){ - for(int i = 0; i < mesh.num_elems; i++){ - - // loop over all corners around the node and calculate the nodal force - for (size_t corner_lid=0; corner_lidgetGlobalElement(mesh.nodes_in_elem(i, corner_lid)) <<" " << corner_gid << " " << std::endl; - - } // end for corner_lid - //std::cout << explicit_solver_pointer->all_node_map->getGlobalElement(i) << " " << node_force[0] << " " << node_force[1] << " " << node_force[2] << std::endl; - //std::cout << explicit_solver_pointer->all_node_map->getGlobalElement(i) << " " << node_mass(i) << std::endl; - } - } - */ mesh->build_elem_elem_connectivity(); mesh->num_bdy_patches = nboundary_patches; if (num_dim == 2) @@ -977,14 +1027,9 @@ void FEA_Module_Dynamic_Elasticity::setup() // create Dual Views of the individual elem struct variables elem_den = DViewCArrayKokkos(&elem_interface.den(0), num_elems); elem_pres = DViewCArrayKokkos(&elem_interface.pres(0), num_elems); - elem_stress = DViewCArrayKokkos(&elem_interface.stress(0, 0, 0, 0), - rk_num_bins, - num_elems, - 3, 3); // always 3D even in 2D-RZ - elem_sspd = DViewCArrayKokkos(&elem_interface.sspd(0), num_elems); - elem_sie = DViewCArrayKokkos(&elem_interface.sie(0, 0), - rk_num_bins, - num_elems); + elem_stress = DViewCArrayKokkos(&elem_interface.stress(0, 0, 0, 0), rk_num_bins, num_elems, 3, 3); // always 3D even in 2D-RZ + elem_sspd = DViewCArrayKokkos(&elem_interface.sspd(0), num_elems); + elem_sie = DViewCArrayKokkos(&elem_interface.sie(0, 0), rk_num_bins, num_elems); elem_vol = DViewCArrayKokkos(&elem_interface.vol(0), num_elems); elem_div = DViewCArrayKokkos(&elem_interface.div(0), num_elems); elem_mass = DViewCArrayKokkos(&elem_interface.mass(0), num_elems); @@ -1374,7 +1419,7 @@ void FEA_Module_Dynamic_Elasticity::setup() size_t corner_gid = corners_in_elem(elem_gid, corner_lid); corner_mass(corner_gid) = corner_areas(corner_lid) * elem_den(elem_gid); // node radius is added later } // end for over corners - }); + }); // end of FOR_ALL_CLASS } // end of // calculate the nodal mass @@ -1448,19 +1493,25 @@ void FEA_Module_Dynamic_Elasticity::setup() return; } // end of setup -/* ---------------------------------------------------------------------------- - solve function called by solver -------------------------------------------------------------------------------- */ - +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn module_cleanup +/// +/// \brief Deallocate memory associated with the module +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_Dynamic_Elasticity::module_cleanup() { cleanup_material_models(); } -/* ---------------------------------------------------------------------------- - Deallocate memory used for material models -------------------------------------------------------------------------------- */ - +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn cleanup_material_models +/// +/// \brief Deallocate memory used for material models +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_Dynamic_Elasticity::cleanup_material_models() { const DCArrayKokkos material = simparam->material; @@ -1489,11 +1540,18 @@ void FEA_Module_Dynamic_Elasticity::cleanup_material_models() return; } // end cleanup_user_strength_model; -/** - * Determines which of the boundary patches are associated with which boundary. - * - * Modifies: bdy_patches_in_set -*/ +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn tag_bdys +/// +/// \brief Determines which of the boundary patches are associated with which boundary. +/// Modifies: bdy_patches_in_set +/// +/// \param Array of boundaries +/// \param The simulation mesh +/// \param Nodal coordinates +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_Dynamic_Elasticity::tag_bdys(const DCArrayKokkos& boundary, mesh_t& mesh, const DViewCArrayKokkos& node_coords) @@ -1534,16 +1592,7 @@ void FEA_Module_Dynamic_Elasticity::tag_bdys(const DCArrayKokkos& bo bc_type, val, node_coords, - rk_level); // no=0, yes=1 - - // debug check - /* - for (size_t patch_node_lid=0; patch_node_lid& bo return; } // end tag +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn check_bdy +/// +/// \brief Determines which of the boundary patches are associated with which boundary. +/// +/// \param Global index of the patch +/// \param Number of spatial dimensions +/// \param Number of the nodes in the patch +/// \param Type of boundary condition +/// \param Boundary value +/// \param Nodal coordinates +/// \param Runge Kutta integration level +/// +///////////////////////////////////////////////////////////////////////////// KOKKOS_INLINE_FUNCTION bool FEA_Module_Dynamic_Elasticity::check_bdy(const size_t patch_gid, const int num_dim, @@ -1639,10 +1703,15 @@ bool FEA_Module_Dynamic_Elasticity::check_bdy(const size_t patch_gid, return is_on_bdy == num_nodes_in_patch; } // end method to check bdy -/* ---------------------------------------------------------------------------- - Build set of nodes assigned to each boundary condition -------------------------------------------------------------------------------- */ - +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn build_boundry_node_sets +/// +/// \brief Build set of nodes assigned to each boundary condition +/// +/// \param The simulation mesh +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_Dynamic_Elasticity::build_boundry_node_sets(mesh_t& mesh) { // build boundary nodes in each boundary set @@ -1722,10 +1791,17 @@ void FEA_Module_Dynamic_Elasticity::build_boundry_node_sets(mesh_t& mesh) return; } // end method to build boundary nodes -/* ---------------------------------------------------------------------------- - solve function called by solver -------------------------------------------------------------------------------- */ - +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn solve +/// +/// \brief Solve function called by solver +/// +/// Calls elastic_solve +/// +/// \return 0 if the solver finishes +/// +///////////////////////////////////////////////////////////////////////////// int FEA_Module_Dynamic_Elasticity::solve() { elastic_solve(); @@ -1733,10 +1809,13 @@ int FEA_Module_Dynamic_Elasticity::solve() return 0; } -/* ---------------------------------------------------------------------------- - SGH solver loop -------------------------------------------------------------------------------- */ - +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn elastic_solve +/// +/// \brief Elastic solver loop +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_Dynamic_Elasticity::elastic_solve() { Dynamic_Options dynamic_options = simparam->dynamic_options; @@ -2050,41 +2129,6 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve() node_vel, elem_vol); - /* - debug block - if(myrank==1){ - std::cout << rk_alpha << " " << dt << std::endl; - for(int i = 0; i < nall_nodes; i++){ - double node_force[3]; - for (size_t dim = 0; dim < num_dim; dim++){ - node_force[dim] = 0.0; - } // end for dim - - // loop over all corners around the node and calculate the nodal force - for (size_t corner_lid=0; corner_lidall_node_map->getGlobalElement(i) << " " << corner_gid << " " << corner_force(corner_gid, 0) << " " << corner_force(corner_gid, 1) << " " << corner_force(corner_gid, 2) << std::endl; - // loop over dimension - for (size_t dim = 0; dim < num_dim; dim++){ - node_force[dim] += corner_force(corner_gid, dim); - } // end for dim - - } // end for corner_lid - //std::cout << Explicit_Solver_Pointer_->all_node_map->getGlobalElement(i) << " " << node_force[0] << " " << node_force[1] << " " << node_force[2] << std::endl; - //std::cout << Explicit_Solver_Pointer_->all_node_map->getGlobalElement(i) << " " << node_mass(i) << std::endl; - } - } - /* - //debug print vector values on a rank - /* - if(myrank==0) - for(int i = 0; i < nall_nodes; i++){ - std::cout << Explicit_Solver_Pointer_->all_node_map->getGlobalElement(i) << " " << node_vel(rk_level,i,0) << " " << node_vel(rk_level,i,1) << " " << node_vel(rk_level,i,2) << std::endl; - } - */ - // ---- Update nodal velocities ---- // get_force_elastic(material, *mesh, @@ -2365,7 +2409,7 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve() node_velocities_interface(node_gid, idim) = node_vel(rk_level, node_gid, idim); node_coords_interface(node_gid, idim) = node_coords(rk_level, node_gid, idim); } - }); + }); } // end view scope Kokkos::fence(); @@ -2484,7 +2528,7 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve() { node_coords_interface(node_gid, idim) = node_coords(rk_level, node_gid, idim); } - }); // end parallel for + }); // end parallel for } // end view scope if (simparam->output_options.output_file_format == OUTPUT_FORMAT::vtk) { diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.h b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.h index ac1c0e6a2..c9be1bb39 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.h +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.h @@ -11,14 +11,11 @@ This program is open source under the BSD-3 License. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. @@ -58,545 +55,532 @@ struct material_t; struct boundary_t; -class FEA_Module_Dynamic_Elasticity: public FEA_Module { - +class FEA_Module_Dynamic_Elasticity : public FEA_Module +{ public: - - FEA_Module_Dynamic_Elasticity( Dynamic_Elasticity_Parameters& params, Solver *Solver_Pointer, std::shared_ptr mesh_in, const int my_fea_module_index = 0); - ~FEA_Module_Dynamic_Elasticity(); - - //initialize data for boundaries of the model and storage for boundary conditions and applied loads - void elastic_interface_setup(node_t &node, elem_t &elem, corner_t &corner); - - void setup(); - - void cleanup_material_models(); - - int solve(); - - void module_cleanup(); - - void elastic_solve(); - - void get_force_vgradient_elastic(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &elem_mat_id, - const double rk_alpha, - const size_t cycle); - - void get_force_ugradient_elastic(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &elem_mat_id, - const double rk_alpha, - const size_t cycle); - - void get_force_dgradient_elastic(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &elem_mat_id, - const double rk_alpha, - const size_t cycle); - - void force_design_gradient_term(const_vec_array design_variables, vec_array design_gradients); - - void update_position_elastic(double rk_alpha, - const size_t num_nodes, - DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel); - - void get_vol(); - - void init_assembly(); - - KOKKOS_INLINE_FUNCTION - void get_vol_hex(const DViewCArrayKokkos &elem_vol, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const; - - - KOKKOS_INLINE_FUNCTION - void get_vol_quad(const DViewCArrayKokkos &elem_vol, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const; - - - KOKKOS_FUNCTION - double get_area_quad(const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const; - - - KOKKOS_FUNCTION - void get_bmatrix(const ViewCArrayKokkos &B_matrix, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const; - - KOKKOS_FUNCTION - void get_bmatrix2D(const ViewCArrayKokkos &B_matrix, + FEA_Module_Dynamic_Elasticity(Dynamic_Elasticity_Parameters& params, Solver* Solver_Pointer, std::shared_ptr mesh_in, const int my_fea_module_index = 0); + ~FEA_Module_Dynamic_Elasticity(); + + // initialize data for boundaries of the model and storage for boundary conditions and applied loads + void elastic_interface_setup(node_t& node, elem_t& elem, corner_t& corner); + + void setup(); + + void cleanup_material_models(); + + int solve(); + + void module_cleanup(); + + void elastic_solve(); + + void get_force_vgradient_elastic(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_pres, + const DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + const double rk_alpha, + const size_t cycle); + + void get_force_ugradient_elastic(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_pres, + const DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + const double rk_alpha, + const size_t cycle); + + void get_force_dgradient_elastic(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_pres, + const DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + const double rk_alpha, + const size_t cycle); + + void force_design_gradient_term(const_vec_array design_variables, vec_array design_gradients); + + void update_position_elastic(double rk_alpha, + const size_t num_nodes, + DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel); + + void get_vol(); + + void init_assembly(); + + KOKKOS_INLINE_FUNCTION + void get_vol_hex(const DViewCArrayKokkos& elem_vol, const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, const size_t rk_level) const; - KOKKOS_FUNCTION - void get_area_weights2D(const ViewCArrayKokkos &corner_areas, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const ViewCArrayKokkos &elem_node_gids, - const size_t rk_level) const; - + KOKKOS_INLINE_FUNCTION + void get_vol_quad(const DViewCArrayKokkos& elem_vol, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const; + + KOKKOS_FUNCTION + double get_area_quad(const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const; + + KOKKOS_FUNCTION + void get_bmatrix(const ViewCArrayKokkos& B_matrix, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const; - KOKKOS_INLINE_FUNCTION - double heron(const double x1, - const double y1, - const double x2, - const double y2, - const double x3, - const double y3) const; + KOKKOS_FUNCTION + void get_bmatrix2D(const ViewCArrayKokkos& B_matrix, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const; - double average_element_density(const int nodes_per_elem, const CArray current_element_densities) const; + KOKKOS_FUNCTION + void get_area_weights2D(const ViewCArrayKokkos& corner_areas, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const ViewCArrayKokkos& elem_node_gids, + const size_t rk_level) const; - void get_divergence(DViewCArrayKokkos &elem_div, - const mesh_t mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_vol); + KOKKOS_INLINE_FUNCTION + double heron(const double x1, + const double y1, + const double x2, + const double y2, + const double x3, + const double y3) const; + double average_element_density(const int nodes_per_elem, const CArray current_element_densities) const; - void get_divergence2D(DViewCArrayKokkos &elem_div, + void get_divergence(DViewCArrayKokkos& elem_div, const mesh_t mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &elem_vol); - - - KOKKOS_FUNCTION - void get_velgrad(ViewCArrayKokkos &vel_grad, - const ViewCArrayKokkos &elem_node_gids, - const DViewCArrayKokkos &node_vel, - const ViewCArrayKokkos &b_matrix, - const double elem_vol, - const size_t elem_gid, - const size_t rk_level) const; - - - KOKKOS_FUNCTION - void get_velgrad2D(ViewCArrayKokkos &vel_grad, - const ViewCArrayKokkos &elem_node_gids, - const DViewCArrayKokkos &node_vel, - const ViewCArrayKokkos &b_matrix, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_vol); + + void get_divergence2D(DViewCArrayKokkos& elem_div, + const mesh_t mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& elem_vol); + + KOKKOS_FUNCTION + void get_velgrad(ViewCArrayKokkos& vel_grad, + const ViewCArrayKokkos& elem_node_gids, + const DViewCArrayKokkos& node_vel, + const ViewCArrayKokkos& b_matrix, const double elem_vol, - const double elem_area, const size_t elem_gid, const size_t rk_level) const; - KOKKOS_INLINE_FUNCTION - void decompose_vel_grad(ViewCArrayKokkos &D_tensor, - ViewCArrayKokkos &W_tensor, - const ViewCArrayKokkos &vel_grad, - const ViewCArrayKokkos &elem_node_gids, - const size_t elem_gid, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const double vol) const; - - void boundary_velocity(const mesh_t &mesh, - const DCArrayKokkos &boundary, - DViewCArrayKokkos &node_vel); - - void boundary_adjoint(const mesh_t &mesh, - const DCArrayKokkos &boundary, - vec_array &node_adjoint, - vec_array &node_phi_adjoint); - - void tag_bdys(const DCArrayKokkos &boundary, - mesh_t &mesh, - const DViewCArrayKokkos &node_coords); - - KOKKOS_INLINE_FUNCTION - bool check_bdy(const size_t patch_gid, - const int num_dim, - const int num_nodes_in_patch, + KOKKOS_FUNCTION + void get_velgrad2D(ViewCArrayKokkos& vel_grad, + const ViewCArrayKokkos& elem_node_gids, + const DViewCArrayKokkos& node_vel, + const ViewCArrayKokkos& b_matrix, + const double elem_vol, + const double elem_area, + const size_t elem_gid, + const size_t rk_level) const; + + KOKKOS_INLINE_FUNCTION + void decompose_vel_grad(ViewCArrayKokkos& D_tensor, + ViewCArrayKokkos& W_tensor, + const ViewCArrayKokkos& vel_grad, + const ViewCArrayKokkos& elem_node_gids, + const size_t elem_gid, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const double vol) const; + + void boundary_velocity(const mesh_t& mesh, + const DCArrayKokkos& boundary, + DViewCArrayKokkos& node_vel); + + void boundary_adjoint(const mesh_t& mesh, + const DCArrayKokkos& boundary, + vec_array& node_adjoint, + vec_array& node_phi_adjoint); + + void tag_bdys(const DCArrayKokkos& boundary, + mesh_t& mesh, + const DViewCArrayKokkos& node_coords); + + KOKKOS_INLINE_FUNCTION + bool check_bdy(const size_t patch_gid, + const int num_dim, + const int num_nodes_in_patch, const BOUNDARY_TYPE this_bc_tag, const double val, - const DViewCArrayKokkos &node_coords, + const DViewCArrayKokkos& node_coords, const size_t rk_level) const; - void rk_init(DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &elem_sie, - DViewCArrayKokkos &elem_stress, - const size_t num_elems, - const size_t num_nodes); - - - void get_timestep(mesh_t &mesh, - DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &elem_sspd, - DViewCArrayKokkos &elem_vol); - - - void get_timestep2D(mesh_t &mesh, - DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &elem_sspd, - DViewCArrayKokkos &elem_vol); - - void update_state(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &elem_den, - DViewCArrayKokkos &elem_pres, - DViewCArrayKokkos &elem_stress, - DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_mass, - const DViewCArrayKokkos &elem_mat_id, - const double rk_alpha, - const size_t cycle); - - - void update_state2D(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &elem_den, - DViewCArrayKokkos &elem_pres, - DViewCArrayKokkos &elem_stress, - DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_mass, - const DViewCArrayKokkos &elem_mat_id, + void rk_init(DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_sie, + DViewCArrayKokkos& elem_stress, + const size_t num_elems, + const size_t num_nodes); + + void get_timestep(mesh_t& mesh, + DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_sspd, + DViewCArrayKokkos& elem_vol); + + void get_timestep2D(mesh_t& mesh, + DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_sspd, + DViewCArrayKokkos& elem_vol); + + void update_state(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_den, + DViewCArrayKokkos& elem_pres, + DViewCArrayKokkos& elem_stress, + DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& elem_mat_id, const double rk_alpha, const size_t cycle); - void build_boundry_node_sets(mesh_t &mesh); - - void init_boundaries(); - - //initializes memory for arrays used in the global stiffness matrix assembly - void init_boundary_sets(int num_boundary_sets); - - void grow_boundary_sets(int num_boundary_sets); - - virtual void update_forward_solve(Teuchos::RCP zp); - - void comm_node_masses(); - - void comm_adjoint_vectors(int cycle); - - void comm_variables(Teuchos::RCP zp); - - void read_conditions_ansys_dat(std::ifstream *in, std::streampos before_condition_header); - - //interfaces between user input and creating data structures for bcs - void generate_bcs(); - - void Displacement_Boundary_Conditions(); - - void init_output(); - - void compute_output(); - - void sort_output(Teuchos::RCP > sorted_map); - - void sort_element_output(Teuchos::RCP > sorted_map); - - void collect_output(Teuchos::RCP > global_reduce_map); - - void write_data(std::map &point_data_scalars_double, - std::map &point_data_vectors_double, - std::map &cell_data_scalars_double, - std::map &cell_data_scalars_int, - std::map > &cell_data_fields_double); - - void write_outputs (const mesh_t &mesh, - DViewCArrayKokkos &node_coords, - DViewCArrayKokkos &node_vel, - DViewCArrayKokkos &node_mass, - DViewCArrayKokkos &elem_den, - DViewCArrayKokkos &elem_pres, - DViewCArrayKokkos &elem_stress, - DViewCArrayKokkos &elem_sspd, - DViewCArrayKokkos &elem_sie, - DViewCArrayKokkos &elem_vol, - DViewCArrayKokkos &elem_mass, - DViewCArrayKokkos &elem_mat_id); - - - void ensight(const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &node_mass, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_mass, - const DViewCArrayKokkos &elem_mat_id); - - - void state_file(const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &node_mass, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_pres, - const DViewCArrayKokkos &elem_stress, - const DViewCArrayKokkos &elem_sspd, - const DViewCArrayKokkos &elem_sie, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_mass, - const DViewCArrayKokkos &elem_mat_id); - - void node_density_constraints(host_vec_array &node_densities_lower_bound); - - void compute_topology_optimization_adjoint(); //Force does not depend on node coords and velocity - - void compute_topology_optimization_adjoint_full(); //Force depends on node coords and velocity - - void compute_topology_optimization_gradient(const_vec_array design_densities, vec_array gradients); - - void compute_topology_optimization_gradient_full(Teuchos::RCP design_densities_distributed, Teuchos::RCP design_gradients_distributed); - - //elastic TO stuff - void get_force_elastic(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &node_mass, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &elem_mat_id, - DViewCArrayKokkos &corner_force, - const double rk_alpha, - const size_t cycle); - - void applied_forces(const DCArrayKokkos &material, - const mesh_t &mesh, - const DViewCArrayKokkos &node_coords, - const DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &node_mass, - const DViewCArrayKokkos &elem_den, - const DViewCArrayKokkos &elem_vol, - const DViewCArrayKokkos &elem_div, - const DViewCArrayKokkos &elem_mat_id, - DViewCArrayKokkos &corner_force, - const double rk_alpha, - const size_t cycle); - - void Element_Material_Properties(size_t ielem, real_t &Element_Modulus, real_t &Poisson_Ratio, real_t density); - - void compute_stiffness_gradients(const_host_vec_array &design_densities, host_vec_array &gradients); - - void Gradient_Element_Material_Properties(size_t ielem, real_t &Element_Modulus, real_t &Poisson_Ratio, real_t density); - - void local_matrix_multiply(int ielem, CArrayKokkos &Local_Matrix); - - void assemble_matrix(); - - bool nodal_density_flag; - real_t penalty_power; - Teuchos::RCP Global_Stiffness_Matrix; - RaggedRightArrayKokkos Stiffness_Matrix; - DCArrayKokkos Stiffness_Matrix_Strides; - DCArrayKokkos Global_Stiffness_Matrix_Assembly_Map; - //end elastic TO data - - Dynamic_Elasticity_Parameters *module_params; - Simulation_Parameters_Explicit *simparam; - Explicit_Solver *Explicit_Solver_Pointer_; - - elements::ref_element *ref_elem; - - std::shared_ptr mesh; - //shallow copies of mesh class views - size_t num_nodes_in_elem; - // corner ids in node - RaggedRightArrayKokkos corners_in_node; - CArrayKokkos num_corners_in_node; - - // elem ids in node - RaggedRightArrayKokkos elems_in_node; - - // node ids in node - RaggedRightArrayKokkos nodes_in_node; - CArrayKokkos num_nodes_in_node; - - // node ids in elem - DCArrayKokkos nodes_in_elem; - - // corner ids in elem - CArrayKokkos corners_in_elem; - - // elem ids in elem - RaggedRightArrayKokkos elems_in_elem; - CArrayKokkos num_elems_in_elem; - - // patch ids in elem - CArrayKokkos patches_in_elem; - - // node ids in a patch - CArrayKokkos nodes_in_patch; - - // element ids in a patch - CArrayKokkos elems_in_patch; - - // bdy nodes - CArrayKokkos bdy_nodes; - - //Topology optimization filter variable - DCArrayKokkos relative_element_densities; - - //Local FEA data - host_elem_conn_array interface_nodes_in_elem; //host view of element connectivity to nodes - - //Global FEA data - Teuchos::RCP node_velocities_distributed; - Teuchos::RCP initial_node_coords_distributed; - Teuchos::RCP all_initial_node_coords_distributed; - Teuchos::RCP initial_node_velocities_distributed; - Teuchos::RCP all_node_velocities_distributed; - Teuchos::RCP all_cached_node_velocities_distributed; - Teuchos::RCP node_masses_distributed; - Teuchos::RCP ghost_node_masses_distributed; - Teuchos::RCP adjoint_vector_distributed; - Teuchos::RCP phi_adjoint_vector_distributed; - Teuchos::RCP>> forward_solve_velocity_data; - Teuchos::RCP>> forward_solve_coordinate_data; - Teuchos::RCP>> adjoint_vector_data; - Teuchos::RCP>> phi_adjoint_vector_data; - Teuchos::RCP force_gradient_design; - Teuchos::RCP force_gradient_position; - Teuchos::RCP force_gradient_velocity; - - //Local FEA data - DCArrayKokkos Global_Gradient_Matrix_Assembly_Map; - RaggedRightArrayKokkos Graph_Matrix; //stores global indices - RaggedRightArrayKokkos DOF_Graph_Matrix; //stores global indices - RaggedRightArrayKokkos Force_Gradient_Positions; - RaggedRightArrayKokkos Force_Gradient_Velocities; - DCArrayKokkos Gradient_Matrix_Strides; - DCArrayKokkos Graph_Matrix_Strides; - RaggedRightArrayKokkos Original_Gradient_Entries; - RaggedRightArrayKokkos Original_Gradient_Entry_Indices; - DCArrayKokkos Original_Gradient_Entries_Strides; - - //distributed matrices - Teuchos::RCP distributed_force_gradient_positions; - Teuchos::RCP distributed_force_gradient_velocities; - - std::vector time_data; - unsigned long max_time_steps, last_time_step; - - // --------------------------------------------------------------------- - // state data type declarations (must stay in scope for output after run) - // --------------------------------------------------------------------- - node_t node_interface; - elem_t elem_interface; - corner_t corner_interface; - - //Dual View wrappers - // Dual Views of the individual node struct variables - DViewCArrayKokkos node_coords; - DViewCArrayKokkos node_vel; - DViewCArrayKokkos node_mass; - - // Dual Views of the individual elem struct variables - DViewCArrayKokkos elem_den; - DViewCArrayKokkos elem_pres; - DViewCArrayKokkos elem_stress; // always 3D even in 2D-RZ - DViewCArrayKokkos elem_sspd; - DViewCArrayKokkos elem_sie; - DViewCArrayKokkos elem_vol; - DViewCArrayKokkos elem_div; - DViewCArrayKokkos elem_mass; - DViewCArrayKokkos elem_mat_id; - - // Element velocity gradient - DCArrayKokkos elem_vel_grad; - - // for storing global variables used in user material model - DCArrayKokkos eos_global_vars; - DCArrayKokkos strength_global_vars; - - // for storing state variables used in user material model - DCArrayKokkos eos_state_vars; - DCArrayKokkos strength_state_vars; - - //elem_user_output_vars allow users to output variables of interest per element - DCArrayKokkos elem_user_output_vars; - - //material models - DCArrayKokkos elem_eos; - DCArrayKokkos elem_strength; - - // Dual Views of the corner struct variables - DViewCArrayKokkos corner_force; - DViewCArrayKokkos corner_mass; - - //Boundary Conditions Data - DCArrayKokkos Local_Index_Boundary_Patches; - //CArray Patch_Nodes; - enum bc_type {NONE, POINT_LOADING_CONDITION, LINE_LOADING_CONDITION, SURFACE_LOADING_CONDITION}; - - //Boundary Conditions Data - int max_boundary_sets; - - //output dof data - //Global arrays with collected data used to print - int output_velocity_index, output_strain_index, output_stress_index; - - //parameters - double time_value, time_final, dt, dt_max, dt_min, dt_cfl, graphics_time, graphics_dt_ival; - size_t graphics_cyc_ival, cycle_stop, rk_num_stages, graphics_id; - double fuzz, tiny, small; - CArray graphics_times; - int rk_num_bins; - - //optimization flags - bool kinetic_energy_objective; + void update_state2D(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& elem_den, + DViewCArrayKokkos& elem_pres, + DViewCArrayKokkos& elem_stress, + DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& elem_mat_id, + const double rk_alpha, + const size_t cycle); + + void build_boundry_node_sets(mesh_t& mesh); + + void init_boundaries(); + + // initializes memory for arrays used in the global stiffness matrix assembly + void init_boundary_sets(int num_boundary_sets); + + void grow_boundary_sets(int num_boundary_sets); + + virtual void update_forward_solve(Teuchos::RCP zp); + + void comm_node_masses(); + + void comm_adjoint_vectors(int cycle); + + void comm_variables(Teuchos::RCP zp); + + void read_conditions_ansys_dat(std::ifstream* in, std::streampos before_condition_header); + + // interfaces between user input and creating data structures for bcs + void generate_bcs(); + + void Displacement_Boundary_Conditions(); + + void init_output(); + + void compute_output(); + + void sort_output(Teuchos::RCP> sorted_map); + + void sort_element_output(Teuchos::RCP> sorted_map); + + void collect_output(Teuchos::RCP> global_reduce_map); + + void write_data(std::map& point_data_scalars_double, + std::map& point_data_vectors_double, + std::map& cell_data_scalars_double, + std::map& cell_data_scalars_int, + std::map>& cell_data_fields_double); + + void write_outputs(const mesh_t& mesh, + DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& node_vel, + DViewCArrayKokkos& node_mass, + DViewCArrayKokkos& elem_den, + DViewCArrayKokkos& elem_pres, + DViewCArrayKokkos& elem_stress, + DViewCArrayKokkos& elem_sspd, + DViewCArrayKokkos& elem_sie, + DViewCArrayKokkos& elem_vol, + DViewCArrayKokkos& elem_mass, + DViewCArrayKokkos& elem_mat_id); + + void ensight(const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_mass, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_pres, + const DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& elem_mat_id); + + void state_file(const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_mass, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_pres, + const DViewCArrayKokkos& elem_stress, + const DViewCArrayKokkos& elem_sspd, + const DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& elem_mat_id); + + void node_density_constraints(host_vec_array& node_densities_lower_bound); + + void compute_topology_optimization_adjoint(); // Force does not depend on node coords and velocity + + void compute_topology_optimization_adjoint_full(); // Force depends on node coords and velocity + + void compute_topology_optimization_gradient(const_vec_array design_densities, vec_array gradients); + + void compute_topology_optimization_gradient_full(Teuchos::RCP design_densities_distributed, Teuchos::RCP design_gradients_distributed); + + // elastic TO stuff + void get_force_elastic(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_mass, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + DViewCArrayKokkos& corner_force, + const double rk_alpha, + const size_t cycle); + + void applied_forces(const DCArrayKokkos& material, + const mesh_t& mesh, + const DViewCArrayKokkos& node_coords, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_mass, + const DViewCArrayKokkos& elem_den, + const DViewCArrayKokkos& elem_vol, + const DViewCArrayKokkos& elem_div, + const DViewCArrayKokkos& elem_mat_id, + DViewCArrayKokkos& corner_force, + const double rk_alpha, + const size_t cycle); + + void Element_Material_Properties(size_t ielem, real_t& Element_Modulus, real_t& Poisson_Ratio, real_t density); + + void compute_stiffness_gradients(const_host_vec_array& design_densities, host_vec_array& gradients); + + void Gradient_Element_Material_Properties(size_t ielem, real_t& Element_Modulus, real_t& Poisson_Ratio, real_t density); + + void local_matrix_multiply(int ielem, CArrayKokkos& Local_Matrix); + + void assemble_matrix(); + + bool nodal_density_flag; + real_t penalty_power; + Teuchos::RCP Global_Stiffness_Matrix; + RaggedRightArrayKokkos Stiffness_Matrix; + DCArrayKokkos Stiffness_Matrix_Strides; + DCArrayKokkos Global_Stiffness_Matrix_Assembly_Map; + // end elastic TO data + + Dynamic_Elasticity_Parameters* module_params; + Simulation_Parameters_Explicit* simparam; + Explicit_Solver* Explicit_Solver_Pointer_; + + elements::ref_element* ref_elem; + + std::shared_ptr mesh; + // shallow copies of mesh class views + size_t num_nodes_in_elem; + // corner ids in node + RaggedRightArrayKokkos corners_in_node; + CArrayKokkos num_corners_in_node; + + // elem ids in node + RaggedRightArrayKokkos elems_in_node; + + // node ids in node + RaggedRightArrayKokkos nodes_in_node; + CArrayKokkos num_nodes_in_node; + + // node ids in elem + DCArrayKokkos nodes_in_elem; + + // corner ids in elem + CArrayKokkos corners_in_elem; + + // elem ids in elem + RaggedRightArrayKokkos elems_in_elem; + CArrayKokkos num_elems_in_elem; + + // patch ids in elem + CArrayKokkos patches_in_elem; + + // node ids in a patch + CArrayKokkos nodes_in_patch; + + // element ids in a patch + CArrayKokkos elems_in_patch; + + // bdy nodes + CArrayKokkos bdy_nodes; + + // Topology optimization filter variable + DCArrayKokkos relative_element_densities; + + // Local FEA data + host_elem_conn_array interface_nodes_in_elem; // host view of element connectivity to nodes + + // Global FEA data + Teuchos::RCP node_velocities_distributed; + Teuchos::RCP initial_node_coords_distributed; + Teuchos::RCP all_initial_node_coords_distributed; + Teuchos::RCP initial_node_velocities_distributed; + Teuchos::RCP all_node_velocities_distributed; + Teuchos::RCP all_cached_node_velocities_distributed; + Teuchos::RCP node_masses_distributed; + Teuchos::RCP ghost_node_masses_distributed; + Teuchos::RCP adjoint_vector_distributed; + Teuchos::RCP phi_adjoint_vector_distributed; + Teuchos::RCP>> forward_solve_velocity_data; + Teuchos::RCP>> forward_solve_coordinate_data; + Teuchos::RCP>> adjoint_vector_data; + Teuchos::RCP>> phi_adjoint_vector_data; + Teuchos::RCP force_gradient_design; + Teuchos::RCP force_gradient_position; + Teuchos::RCP force_gradient_velocity; + + // Local FEA data + DCArrayKokkos Global_Gradient_Matrix_Assembly_Map; + RaggedRightArrayKokkos Graph_Matrix; // stores global indices + RaggedRightArrayKokkos DOF_Graph_Matrix; // stores global indices + RaggedRightArrayKokkos Force_Gradient_Positions; + RaggedRightArrayKokkos Force_Gradient_Velocities; + DCArrayKokkos Gradient_Matrix_Strides; + DCArrayKokkos Graph_Matrix_Strides; + RaggedRightArrayKokkos Original_Gradient_Entries; + RaggedRightArrayKokkos Original_Gradient_Entry_Indices; + DCArrayKokkos Original_Gradient_Entries_Strides; + + // distributed matrices + Teuchos::RCP distributed_force_gradient_positions; + Teuchos::RCP distributed_force_gradient_velocities; + + std::vector time_data; + unsigned long max_time_steps, last_time_step; + + // --------------------------------------------------------------------- + // state data type declarations (must stay in scope for output after run) + // --------------------------------------------------------------------- + node_t node_interface; + elem_t elem_interface; + corner_t corner_interface; + + // Dual View wrappers + // Dual Views of the individual node struct variables + DViewCArrayKokkos node_coords; + DViewCArrayKokkos node_vel; + DViewCArrayKokkos node_mass; + + // Dual Views of the individual elem struct variables + DViewCArrayKokkos elem_den; + DViewCArrayKokkos elem_pres; + DViewCArrayKokkos elem_stress; // always 3D even in 2D-RZ + DViewCArrayKokkos elem_sspd; + DViewCArrayKokkos elem_sie; + DViewCArrayKokkos elem_vol; + DViewCArrayKokkos elem_div; + DViewCArrayKokkos elem_mass; + DViewCArrayKokkos elem_mat_id; + + // Element velocity gradient + DCArrayKokkos elem_vel_grad; + + // for storing global variables used in user material model + DCArrayKokkos eos_global_vars; + DCArrayKokkos strength_global_vars; + + // for storing state variables used in user material model + DCArrayKokkos eos_state_vars; + DCArrayKokkos strength_state_vars; + + // elem_user_output_vars allow users to output variables of interest per element + DCArrayKokkos elem_user_output_vars; + + // material models + DCArrayKokkos elem_eos; + DCArrayKokkos elem_strength; + + // Dual Views of the corner struct variables + DViewCArrayKokkos corner_force; + DViewCArrayKokkos corner_mass; + + // Boundary Conditions Data + DCArrayKokkos Local_Index_Boundary_Patches; + // CArray Patch_Nodes; + enum bc_type { NONE, POINT_LOADING_CONDITION, LINE_LOADING_CONDITION, SURFACE_LOADING_CONDITION }; + + // Boundary Conditions Data + int max_boundary_sets; + + // output dof data + // Global arrays with collected data used to print + int output_velocity_index, output_strain_index, output_stress_index; + + // parameters + double time_value, time_final, dt, dt_max, dt_min, dt_cfl, graphics_time, graphics_dt_ival; + size_t graphics_cyc_ival, cycle_stop, rk_num_stages, graphics_id; + double fuzz, tiny, small; + CArray graphics_times; + int rk_num_bins; + + // optimization flags + bool kinetic_energy_objective; }; #endif // end HEADER_H diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/boundary.cpp b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/boundary.cpp index f45d6d532..4360f04f8 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/boundary.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/boundary.cpp @@ -31,16 +31,24 @@ OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. **********************************************************************************************/ -// ------------------------------------------------------- -// This function applys the boundary condition -// to points on a list of patches created at setup -// -------------------------------------------------------- #include "mesh.h" #include "state.h" #include "FEA_Module_Dynamic_Elasticity.h" #include "Simulation_Parameters/FEA_Module/Boundary_Conditions.h" +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn boundary_velocity +/// +/// \brief This function applys the velocity boundary condition +/// to points on a list of patches created at setup +/// +/// \param The simulation mesh +/// \param Array of boundary sets +/// \param View of the nodal velocity data +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_Dynamic_Elasticity::boundary_velocity(const mesh_t& mesh, const DCArrayKokkos& boundary, DViewCArrayKokkos& node_vel) diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/elastic_optimization.cpp b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/elastic_optimization.cpp index 6addef615..40d73a8b3 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/elastic_optimization.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/elastic_optimization.cpp @@ -1,11 +1,43 @@ - +/********************************************************************************************** + 2020. Triad National Security, LLC. All rights reserved. + This program was produced under U.S. Government contract 89233218CNA000001 for Los Alamos + National Laboratory (LANL), which is operated by Triad National Security, LLC for the U.S. + Department of Energy/National Nuclear Security Administration. All rights in the program are + reserved by Triad National Security, LLC, and the U.S. Department of Energy/National Nuclear + Security Administration. The Government is granted for itself and others acting on its behalf a + nonexclusive, paid-up, irrevocable worldwide license in this material to reproduce, prepare + derivative works, distribute copies to the public, perform publicly and display publicly, and + to permit others to do so. + This program is open source under the BSD-3 License. + Redistribution and use in source and binary forms, with or without modification, are permitted + provided that the following conditions are met: + 1. Redistributions of source code must retain the above copyright notice, this list of + conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright notice, this list of + conditions and the following disclaimer in the documentation and/or other materials + provided with the distribution. + 3. Neither the name of the copyright holder nor the names of its contributors may be used + to endorse or promote products derived from this software without specific prior + written permission. + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS + IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **********************************************************************************************/ #include "mesh.h" #include "state.h" #include #include #include #include -#include +#include #include // fmin, fmax, abs note: fminl is long #include #include @@ -34,218 +66,237 @@ #include "FEA_Module_Dynamic_Elasticity.h" #include "Explicit_Solver.h" -//optimization +// optimization #include "ROL_Solver.hpp" #include "Kinetic_Energy_Minimize.h" -/* ---------------------------------------------------------------------- - Compute new system response due to the design variable update -------------------------------------------------------------------------- */ - -void FEA_Module_Dynamic_Elasticity::update_forward_solve(Teuchos::RCP zp){ - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - //local variable for host view in the dual view - int num_dim = simparam->num_dims; - int nodes_per_elem = max_nodes_per_element; - int local_node_index, current_row, current_column; - int max_stride = 0; - int current_module_index; - size_t access_index, row_access_index, row_counter; - GO global_index, global_dof_index; - LO local_dof_index; - const size_t num_fills = simparam->regions.size(); - const size_t rk_num_bins = simparam->dynamic_options.rk_num_bins; - const size_t num_bcs = module_params->boundary_conditions.size(); - const size_t num_materials = simparam->materials.size(); - real_t objective_accumulation; - - // --- Read in the nodes in the mesh --- - int myrank = Explicit_Solver_Pointer_->myrank; - int nranks = Explicit_Solver_Pointer_->nranks; - - const DCArrayKokkos mat_fill = simparam->mat_fill; - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos material = simparam->material; - CArray current_element_nodal_densities = CArray(num_nodes_in_elem); - - std::vector> FEA_Module_My_TO_Modules = simparam->FEA_Module_My_TO_Modules; - problem = Explicit_Solver_Pointer_->problem; //Pointer to ROL optimization problem object - ROL::Ptr> obj_pointer; - - //compute element averaged density ratios corresponding to nodal density design variables - {//view scope - const_host_vec_array all_node_densities = all_node_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); - //debug print - //std::cout << "NODE DENSITY TEST " << all_node_densities(0,0) << std::endl; - for(int elem_id = 0; elem_id < rnum_elem; elem_id++){ - for(int inode = 0; inode < num_nodes_in_elem; inode++){ - current_element_nodal_densities(inode) = all_node_densities(nodes_in_elem(elem_id,inode),0); - } - relative_element_densities.host(elem_id) = average_element_density(num_nodes_in_elem, current_element_nodal_densities); - }//for - } //view scope - //debug print - //std::cout << "ELEMENT RELATIVE DENSITY TEST " << relative_element_densities.host(0) << std::endl; - relative_element_densities.update_device(); - - //set density vector to the current value chosen by the optimizer - test_node_densities_distributed = zp; - - //reset nodal coordinates to initial values - node_coords_distributed->assign(*initial_node_coords_distributed); - - //comms for ghosts - Explicit_Solver_Pointer_->comm_coordinates(); - - //view scope - { - const_vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array ghost_node_coords_interface = Explicit_Solver_Pointer_->ghost_node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); - vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - all_node_coords_interface(node_gid,idim) = node_coords_interface(node_gid,idim); - } +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn update_forward_solve +/// +/// \brief Compute new system response due to the design variable update +/// +/// \param Design variables +/// +///////////////////////////////////////////////////////////////////////////// +void FEA_Module_Dynamic_Elasticity::update_forward_solve(Teuchos::RCP zp) +{ + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + // local variable for host view in the dual view + int num_dim = simparam->num_dims; + int nodes_per_elem = max_nodes_per_element; + int local_node_index, current_row, current_column; + int max_stride = 0; + int current_module_index; + size_t access_index, row_access_index, row_counter; + GO global_index, global_dof_index; + LO local_dof_index; + const size_t num_fills = simparam->regions.size(); + const size_t rk_num_bins = simparam->dynamic_options.rk_num_bins; + const size_t num_bcs = module_params->boundary_conditions.size(); + const size_t num_materials = simparam->materials.size(); + real_t objective_accumulation; + + // --- Read in the nodes in the mesh --- + int myrank = Explicit_Solver_Pointer_->myrank; + int nranks = Explicit_Solver_Pointer_->nranks; + + const DCArrayKokkos mat_fill = simparam->mat_fill; + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos material = simparam->material; + CArray current_element_nodal_densities = CArray(num_nodes_in_elem); + + std::vector> FEA_Module_My_TO_Modules = simparam->FEA_Module_My_TO_Modules; + problem = Explicit_Solver_Pointer_->problem; // Pointer to ROL optimization problem object + ROL::Ptr> obj_pointer; + + // compute element averaged density ratios corresponding to nodal density design variables + { // view scope + const_host_vec_array all_node_densities = all_node_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); + // debug print + // std::cout << "NODE DENSITY TEST " << all_node_densities(0,0) << std::endl; + for (int elem_id = 0; elem_id < rnum_elem; elem_id++) + { + for (int inode = 0; inode < num_nodes_in_elem; inode++) + { + current_element_nodal_densities(inode) = all_node_densities(nodes_in_elem(elem_id, inode), 0); + } + relative_element_densities.host(elem_id) = average_element_density(num_nodes_in_elem, current_element_nodal_densities); + } // for + } // view scope + // debug print + // std::cout << "ELEMENT RELATIVE DENSITY TEST " << relative_element_densities.host(0) << std::endl; + relative_element_densities.update_device(); + + // set density vector to the current value chosen by the optimizer + test_node_densities_distributed = zp; + + // reset nodal coordinates to initial values + node_coords_distributed->assign(*initial_node_coords_distributed); + + // comms for ghosts + Explicit_Solver_Pointer_->comm_coordinates(); + + // view scope + { + const_vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array ghost_node_coords_interface = Explicit_Solver_Pointer_->ghost_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + all_node_coords_interface(node_gid, idim) = node_coords_interface(node_gid, idim); + } }); // end parallel for - Kokkos::fence(); + Kokkos::fence(); - FOR_ALL_CLASS(node_gid, nlocal_nodes, nlocal_nodes+nghost_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - all_node_coords_interface(node_gid,idim) = ghost_node_coords_interface(node_gid-nlocal_nodes,idim); - } + FOR_ALL_CLASS(node_gid, nlocal_nodes, nlocal_nodes + nghost_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + all_node_coords_interface(node_gid, idim) = ghost_node_coords_interface(node_gid - nlocal_nodes, idim); + } }); // end parallel for - Kokkos::fence(); - } //end view scope - - //reset velocities to initial conditions - node_velocities_distributed->assign(*initial_node_velocities_distributed); - - //reset time accumulating objective and constraints - /* - for(int imodule = 0 ; imodule < FEA_Module_My_TO_Modules[my_fea_module_index_].size(); imodule++){ - current_module_index = FEA_Module_My_TO_Modules[my_fea_module_index_][imodule]; - //test if module needs reset - if(){ - - } - } - */ - //simple setup to just request KE for now; above loop to be expanded and used later for scanning modules - obj_pointer = problem->getObjective(); - KineticEnergyMinimize_TopOpt& kinetic_energy_minimize_function = dynamic_cast(*obj_pointer); - kinetic_energy_minimize_function.objective_accumulation = 0; - - //interface trial density vector - - //interfacing of vectors(should be removed later once made compatible) - //view scope - { - host_vec_array interface_node_coords = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); - for(size_t ibin = 0; ibin < rk_num_bins; ibin++){ - //save node data to node.coords - //std::cout << "NODE DATA ON RANK " << myrank << std::endl; - if(num_dim==2){ - for(int inode = 0; inode < nall_nodes; inode++){ - //std::cout << "Node index " << inode+1 << " "; - node_coords.host(ibin,inode,0) = interface_node_coords(inode,0); - //std::cout << host_node_coords_state(0,inode,0)+1<< " "; - node_coords.host(ibin,inode,1) = interface_node_coords(inode,1); - //std::cout << host_node_coords_state(0,inode,1)+1<< " "; - } - } - else if(num_dim==3){ - for(int inode = 0; inode < nall_nodes; inode++){ - //std::cout << "Node index " << inode+1 << " "; - node_coords.host(ibin,inode,0) = interface_node_coords(inode,0); - //std::cout << host_node_coords_state(0,inode,0)+1<< " "; - node_coords.host(ibin,inode,1) = interface_node_coords(inode,1); - //std::cout << host_node_coords_state(0,inode,1)+1<< " "; - - node_coords.host(ibin,inode,2) = interface_node_coords(inode,2); - //std::cout << host_node_coords_state(0,inode,2)+1<< std::endl; - } + Kokkos::fence(); + } // end view scope + + // reset velocities to initial conditions + node_velocities_distributed->assign(*initial_node_velocities_distributed); + + // reset time accumulating objective and constraints + /* + for(int imodule = 0 ; imodule < FEA_Module_My_TO_Modules[my_fea_module_index_].size(); imodule++){ + current_module_index = FEA_Module_My_TO_Modules[my_fea_module_index_][imodule]; + //test if module needs reset + if(){ + } } - } //end view scope + */ + // simple setup to just request KE for now; above loop to be expanded and used later for scanning modules + obj_pointer = problem->getObjective(); + KineticEnergyMinimize_TopOpt& kinetic_energy_minimize_function = dynamic_cast(*obj_pointer); + kinetic_energy_minimize_function.objective_accumulation = 0; + + // interface trial density vector + + // interfacing of vectors(should be removed later once made compatible) + // view scope + { + host_vec_array interface_node_coords = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); + for (size_t ibin = 0; ibin < rk_num_bins; ibin++) + { + // save node data to node.coords + // std::cout << "NODE DATA ON RANK " << myrank << std::endl; + if (num_dim == 2) + { + for (int inode = 0; inode < nall_nodes; inode++) + { + // std::cout << "Node index " << inode+1 << " "; + node_coords.host(ibin, inode, 0) = interface_node_coords(inode, 0); + // std::cout << host_node_coords_state(0,inode,0)+1<< " "; + node_coords.host(ibin, inode, 1) = interface_node_coords(inode, 1); + // std::cout << host_node_coords_state(0,inode,1)+1<< " "; + } + } + else if (num_dim == 3) + { + for (int inode = 0; inode < nall_nodes; inode++) + { + // std::cout << "Node index " << inode+1 << " "; + node_coords.host(ibin, inode, 0) = interface_node_coords(inode, 0); + // std::cout << host_node_coords_state(0,inode,0)+1<< " "; + node_coords.host(ibin, inode, 1) = interface_node_coords(inode, 1); + // std::cout << host_node_coords_state(0,inode,1)+1<< " "; + + node_coords.host(ibin, inode, 2) = interface_node_coords(inode, 2); + // std::cout << host_node_coords_state(0,inode,2)+1<< std::endl; + } + } + } + } // end view scope // save the node coords to the current RK value - for (size_t node_gid=0; node_gid < nall_nodes; node_gid++){ - - for(int rk=1; rk elem_user_output_vars, elem_sspd, elem_den(elem_gid), - elem_sie(rk_level,elem_gid)); + elem_sie(rk_level, elem_gid)); - // --- Sound speed --- - elem_eos(elem_gid).calc_sound_speed(elem_pres, + // --- Sound speed --- + elem_eos(elem_gid).calc_sound_speed(elem_pres, elem_stress, elem_gid, elem_mat_id(elem_gid), @@ -270,372 +321,403 @@ void FEA_Module_Dynamic_Elasticity::update_forward_solve(Teuchos::RCP elem_user_output_vars, elem_sspd, elem_den(elem_gid), - elem_sie(rk_level,elem_gid)); - - // loop over the nodes of this element and apply velocity - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++){ - - // get the mesh node index - size_t node_gid = nodes_in_elem(elem_gid, node_lid); + elem_sie(rk_level, elem_gid)); - - // --- Velocity --- - switch(mat_fill(f_id).velocity) + // loop over the nodes of this element and apply velocity + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { - case VELOCITY_TYPE::cartesian: - { - - node_vel(rk_level, node_gid, 0) = mat_fill(f_id).u; - node_vel(rk_level, node_gid, 1) = mat_fill(f_id).v; - if (num_dim == 3) node_vel(rk_level, node_gid, 2) = mat_fill(f_id).w; - - - break; - } - case VELOCITY_TYPE::radial: + // get the mesh node index + size_t node_gid = nodes_in_elem(elem_gid, node_lid); + + // --- Velocity --- + switch (mat_fill(f_id).velocity) { - // Setting up cylindrical - double dir[2]; - dir[0] = 0.0; - dir[1] = 0.0; - double radius_val = 0.0; - - for(int dim=0; dim<2; dim++){ - dir[dim] = node_coords(rk_level, node_gid, dim); - radius_val += node_coords(rk_level, node_gid, dim)*node_coords(rk_level, node_gid, dim); - } // end for - radius_val = sqrt(radius_val); - - for(int dim=0; dim<2; dim++){ - if (radius_val > 1.0e-14){ - dir[dim] /= (radius_val); + case VELOCITY_TYPE::cartesian: + { + node_vel(rk_level, node_gid, 0) = mat_fill(f_id).u; + node_vel(rk_level, node_gid, 1) = mat_fill(f_id).v; + if (num_dim == 3) + { + node_vel(rk_level, node_gid, 2) = mat_fill(f_id).w; + } + + break; } - else{ - dir[dim] = 0.0; + case VELOCITY_TYPE::radial: + { + // Setting up cylindrical + double dir[2]; + dir[0] = 0.0; + dir[1] = 0.0; + double radius_val = 0.0; + + for (int dim = 0; dim < 2; dim++) + { + dir[dim] = node_coords(rk_level, node_gid, dim); + radius_val += node_coords(rk_level, node_gid, dim) * node_coords(rk_level, node_gid, dim); + } // end for + radius_val = sqrt(radius_val); + + for (int dim = 0; dim < 2; dim++) + { + if (radius_val > 1.0e-14) + { + dir[dim] /= (radius_val); + } + else + { + dir[dim] = 0.0; + } + } // end for + + node_vel(rk_level, node_gid, 0) = mat_fill(f_id).speed * dir[0]; + node_vel(rk_level, node_gid, 1) = mat_fill(f_id).speed * dir[1]; + if (num_dim == 3) + { + node_vel(rk_level, node_gid, 2) = 0.0; + } + + break; } - } // end for - - - node_vel(rk_level, node_gid, 0) = mat_fill(f_id).speed*dir[0]; - node_vel(rk_level, node_gid, 1) = mat_fill(f_id).speed*dir[1]; - if (num_dim == 3) node_vel(rk_level, node_gid, 2) = 0.0; - - break; - } - case VELOCITY_TYPE::spherical: - { - - // Setting up spherical - double dir[3]; - dir[0] = 0.0; - dir[1] = 0.0; - dir[2] = 0.0; - double radius_val = 0.0; - - for(int dim=0; dim<3; dim++){ - dir[dim] = node_coords(rk_level, node_gid, dim); - radius_val += node_coords(rk_level, node_gid, dim)*node_coords(rk_level, node_gid, dim); - } // end for - radius_val = sqrt(radius_val); - - for(int dim=0; dim<3; dim++){ - if (radius_val > 1.0e-14){ - dir[dim] /= (radius_val); + case VELOCITY_TYPE::spherical: + { + // Setting up spherical + double dir[3]; + dir[0] = 0.0; + dir[1] = 0.0; + dir[2] = 0.0; + double radius_val = 0.0; + + for (int dim = 0; dim < 3; dim++) + { + dir[dim] = node_coords(rk_level, node_gid, dim); + radius_val += node_coords(rk_level, node_gid, dim) * node_coords(rk_level, node_gid, dim); + } // end for + radius_val = sqrt(radius_val); + + for (int dim = 0; dim < 3; dim++) + { + if (radius_val > 1.0e-14) + { + dir[dim] /= (radius_val); + } + else + { + dir[dim] = 0.0; + } + } // end for + + node_vel(rk_level, node_gid, 0) = mat_fill(f_id).speed * dir[0]; + node_vel(rk_level, node_gid, 1) = mat_fill(f_id).speed * dir[1]; + if (num_dim == 3) + { + node_vel(rk_level, node_gid, 2) = mat_fill(f_id).speed * dir[2]; + } + + break; } - else{ - dir[dim] = 0.0; + case VELOCITY_TYPE::radial_linear: + { + break; } - } // end for - - node_vel(rk_level, node_gid, 0) = mat_fill(f_id).speed*dir[0]; - node_vel(rk_level, node_gid, 1) = mat_fill(f_id).speed*dir[1]; - if (num_dim == 3) node_vel(rk_level, node_gid, 2) = mat_fill(f_id).speed*dir[2]; - - break; - } - case VELOCITY_TYPE::radial_linear: - { - - break; - } - case VELOCITY_TYPE::spherical_linear: - { - - break; - } - case VELOCITY_TYPE::tg_vortex: - { - - node_vel(rk_level, node_gid, 0) = sin(PI * node_coords(rk_level,node_gid, 0)) * cos(PI * node_coords(rk_level,node_gid, 1)); - node_vel(rk_level, node_gid, 1) = -1.0*cos(PI * node_coords(rk_level,node_gid, 0)) * sin(PI * node_coords(rk_level,node_gid, 1)); - if (num_dim == 3) node_vel(rk_level, node_gid, 2) = 0.0; - - break; - } - } // end of switch - - }// end loop over nodes of element - - - if(mat_fill(f_id).velocity == VELOCITY_TYPE::tg_vortex) - { - elem_pres(elem_gid) = 0.25*( cos(2.0*PI*elem_coords[0]) + cos(2.0*PI*elem_coords[1]) ) + 1.0; - - // p = rho*ie*(gamma - 1) - size_t mat_id = f_id; - double gamma = eos_global_vars(mat_id,0); // gamma value - elem_sie(rk_level, elem_gid) = - elem_pres(elem_gid)/(mat_fill(f_id).den*(gamma - 1.0)); - } // end if + case VELOCITY_TYPE::spherical_linear: + { + break; + } + case VELOCITY_TYPE::tg_vortex: + { + node_vel(rk_level, node_gid, 0) = sin(PI * node_coords(rk_level, node_gid, 0)) * cos(PI * node_coords(rk_level, node_gid, 1)); + node_vel(rk_level, node_gid, 1) = -1.0 * cos(PI * node_coords(rk_level, node_gid, 0)) * sin(PI * node_coords(rk_level, node_gid, 1)); + if (num_dim == 3) + { + node_vel(rk_level, node_gid, 2) = 0.0; + } + + break; + } + } // end of switch + } // end loop over nodes of element - } // end if fill - + if (mat_fill(f_id).velocity == VELOCITY_TYPE::tg_vortex) + { + elem_pres(elem_gid) = 0.25 * (cos(2.0 * PI * elem_coords[0]) + cos(2.0 * PI * elem_coords[1]) ) + 1.0; + + // p = rho*ie*(gamma - 1) + size_t mat_id = f_id; + double gamma = eos_global_vars(mat_id, 0); // gamma value + elem_sie(rk_level, elem_gid) = + elem_pres(elem_gid) / (mat_fill(f_id).den * (gamma - 1.0)); + } // end if + } // end if fill }); // end FOR_ALL_CLASS element loop - Kokkos::fence(); - - - } // end for loop over fills - }//end view scope - - - + Kokkos::fence(); + } // end for loop over fills + } // end view scope + // apply BC's to velocity FEA_Module_Dynamic_Elasticity::boundary_velocity(*mesh, boundary, node_vel); - - + // calculate the corner massess if 2D - if(num_dim==2){ - + if (num_dim == 2) + { FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { - // facial area of the corners double corner_areas_array[4]; - - ViewCArrayKokkos corner_areas(&corner_areas_array[0],4); - ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 4); - + + ViewCArrayKokkos corner_areas(&corner_areas_array[0], 4); + ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 4); + get_area_weights2D(corner_areas, elem_gid, node_coords, elem_node_gids, rk_level); - + // loop over the corners of the element and calculate the mass - for (size_t corner_lid=0; corner_lid<4; corner_lid++){ - + for (size_t corner_lid = 0; corner_lid < 4; corner_lid++) + { size_t corner_gid = corners_in_elem(elem_gid, corner_lid); - corner_mass(corner_gid) = corner_areas(corner_lid)*elem_den(elem_gid); // node radius is added later - + corner_mass(corner_gid) = corner_areas(corner_lid) * elem_den(elem_gid); // node radius is added later } // end for over corners }); - } // end of - - + // calculate the nodal mass FOR_ALL_CLASS(node_gid, 0, nall_nodes, { - node_mass(node_gid) = 0.0; - - if(num_dim==3){ - - for(size_t elem_lid=0; elem_lid < num_corners_in_node(node_gid); elem_lid++){ - size_t elem_gid = elems_in_node(node_gid,elem_lid); - node_mass(node_gid) += 1.0/8.0*elem_mass(elem_gid); + + if (num_dim == 3) + { + for (size_t elem_lid = 0; elem_lid < num_corners_in_node(node_gid); elem_lid++) + { + size_t elem_gid = elems_in_node(node_gid, elem_lid); + node_mass(node_gid) += 1.0 / 8.0 * elem_mass(elem_gid); } // end for elem_lid - - }// end if dims=3 - else { - + } // end if dims=3 + else + { // 2D-RZ - for(size_t corner_lid=0; corner_lid < num_corners_in_node(node_gid); corner_lid++){ - - size_t corner_gid = corners_in_node(node_gid, corner_lid); + for (size_t corner_lid = 0; corner_lid < num_corners_in_node(node_gid); corner_lid++) + { + size_t corner_gid = corners_in_node(node_gid, corner_lid); node_mass(node_gid) += corner_mass(corner_gid); // sans the radius so it is areal node mass - - corner_mass(corner_gid) *= node_coords(rk_level,node_gid,1); // true corner mass now + + corner_mass(corner_gid) *= node_coords(rk_level, node_gid, 1); // true corner mass now } // end for elem_lid - } // end else - }); // end FOR_ALL_CLASS Kokkos::fence(); - - //current interface has differing mass arrays; this equates them until we unify memory - //view scope + // current interface has differing mass arrays; this equates them until we unify memory + // view scope { - vec_array node_mass_interface = node_masses_distributed->getLocalView (Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - node_mass_interface(node_gid,0) = node_mass(node_gid); + vec_array node_mass_interface = node_masses_distributed->getLocalView(Tpetra::Access::ReadWrite); + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + node_mass_interface(node_gid, 0) = node_mass(node_gid); }); // end parallel for - } //end view scope + } // end view scope Kokkos::fence(); - //communicate ghost densities + // communicate ghost densities comm_node_masses(); - //this is forcing a copy to the device - //view scope + // this is forcing a copy to the device + // view scope { - vec_array ghost_node_mass_interface = ghost_node_masses_distributed->getLocalView (Tpetra::Access::ReadWrite); + vec_array ghost_node_mass_interface = ghost_node_masses_distributed->getLocalView(Tpetra::Access::ReadWrite); - FOR_ALL_CLASS(node_gid, nlocal_nodes, nall_nodes, { - node_mass(node_gid) = ghost_node_mass_interface(node_gid-nlocal_nodes,0); + FOR_ALL_CLASS(node_gid, nlocal_nodes, nall_nodes, { + node_mass(node_gid) = ghost_node_mass_interface(node_gid - nlocal_nodes, 0); }); // end parallel for - } //end view scope + } // end view scope Kokkos::fence(); - //update stiffness matrix - if(simparam->topology_optimization_on || simparam->shape_optimization_on){ - assemble_matrix(); + // update stiffness matrix + if (simparam->topology_optimization_on || simparam->shape_optimization_on) + { + assemble_matrix(); } - - //execute solve - elastic_solve(); + // execute solve + elastic_solve(); } -/* ------------------------------------------------------------------------------------------- - Compute average density of an element from nodal densities ----------------------------------------------------------------------------------------------- */ - +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn average_element_density +/// +/// \brief Compute average density of an element from nodal densities +/// +/// \param Noder per element +/// \param Current element densities +/// +/// \return Element average densities +/// +///////////////////////////////////////////////////////////////////////////// double FEA_Module_Dynamic_Elasticity::average_element_density(const int nodes_per_elem, const CArray current_element_densities) const { - double result = 0; - for(int i=0; i < nodes_per_elem; i++){ - result += current_element_densities(i)/nodes_per_elem; - } + double result = 0; + for (int i = 0; i < nodes_per_elem; i++) + { + result += current_element_densities(i) / nodes_per_elem; + } - return result; + return result; } -/* --------------------------------------------------------------------------------------------------------------- - Simpler adjoint vector solve for the kinetic energy minimization problem - when force does not depend on u and v. ------------------------------------------------------------------------------------------------------------------- */ - -void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_adjoint(){ - - size_t num_bdy_nodes = mesh->num_bdy_nodes; - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos material = simparam->material; - const int num_dim = simparam->num_dims; - real_t global_dt; - size_t current_data_index, next_data_index; - Teuchos::RCP previous_adjoint_vector_distributed, current_adjoint_vector_distributed, previous_velocity_vector_distributed, current_velocity_vector_distributed; - //initialize first adjoint vector at last_time_step to 0 as the terminal value - (*adjoint_vector_data)[last_time_step+1]->putScalar(0); - - //solve terminal value problem, proceeds in time backward. For simplicity, we use the same timestep data from the forward solve. - //A linear interpolant is assumed between velocity data points; velocity midpoint is used to update the adjoint. - if(myrank==0) - std::cout << "Computing adjoint vector " << time_data.size() << std::endl; - - for (int cycle = last_time_step; cycle >= 0; cycle--) { - //compute timestep from time data - global_dt = time_data[cycle+1] - time_data[cycle]; - - //print - if(simparam->dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ - if (cycle==last_time_step){ - if(myrank==0) - printf("cycle = %d, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } - // print time step every 10 cycles - else if (cycle%20==0){ - if(myrank==0) - printf("cycle = %d, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } // end if +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn compute_topology_optimization_adjoint +/// +/// \brief Simpler adjoint vector solve for the kinetic energy minimization problem +/// when force does not depend on u and v. +/// +///////////////////////////////////////////////////////////////////////////// +void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_adjoint() +{ + size_t num_bdy_nodes = mesh->num_bdy_nodes; + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos material = simparam->material; + const int num_dim = simparam->num_dims; + real_t global_dt; + size_t current_data_index, next_data_index; + Teuchos::RCP previous_adjoint_vector_distributed, current_adjoint_vector_distributed, previous_velocity_vector_distributed, current_velocity_vector_distributed; + // initialize first adjoint vector at last_time_step to 0 as the terminal value + (*adjoint_vector_data)[last_time_step + 1]->putScalar(0); + + // solve terminal value problem, proceeds in time backward. For simplicity, we use the same timestep data from the forward solve. + // A linear interpolant is assumed between velocity data points; velocity midpoint is used to update the adjoint. + if (myrank == 0) + { + std::cout << "Computing adjoint vector " << time_data.size() << std::endl; } - //else if (cycle==1){ - //if(myrank==0) - //printf("cycle = %lu, time = %f, time step = %f \n", cycle-1, time_data[cycle-1], global_dt); - //} // end if - //compute adjoint vector for this data point; use velocity midpoint - //view scope + for (int cycle = last_time_step; cycle >= 0; cycle--) { - const_vec_array previous_velocity_vector = (*forward_solve_velocity_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - - const_vec_array previous_adjoint_vector = (*adjoint_vector_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView (Tpetra::Access::ReadWrite); - - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes + nghost_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - //cancellation of half from midpoint and 2 from adjoint equation already done - current_adjoint_vector(node_gid,idim) = -0.5*(current_velocity_vector(node_gid,idim)+previous_velocity_vector(node_gid,idim))*global_dt + previous_adjoint_vector(node_gid,idim); - } + // compute timestep from time data + global_dt = time_data[cycle + 1] - time_data[cycle]; + + // print + if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) + { + if (cycle == last_time_step) + { + if (myrank == 0) + { + printf("cycle = %d, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } + // print time step every 10 cycles + else if (cycle % 20 == 0) + { + if (myrank == 0) + { + printf("cycle = %d, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } // end if + } + // else if (cycle==1){ + // if(myrank==0) + // printf("cycle = %lu, time = %f, time step = %f \n", cycle-1, time_data[cycle-1], global_dt); + // } // end if + + // compute adjoint vector for this data point; use velocity midpoint + // view scope + { + const_vec_array previous_velocity_vector = (*forward_solve_velocity_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + + const_vec_array previous_adjoint_vector = (*adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadWrite); + + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes + nghost_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + // cancellation of half from midpoint and 2 from adjoint equation already done + current_adjoint_vector(node_gid, idim) = -0.5 * (current_velocity_vector(node_gid, idim) + previous_velocity_vector(node_gid, idim)) * global_dt + previous_adjoint_vector(node_gid, + idim); + } }); // end parallel for - Kokkos::fence(); - } //end view scope - - } + Kokkos::fence(); + } // end view scope + } } - -/* ------------------------------------------------------------------------------ - Coupled adjoint problem for the kinetic energy minimization problem ---------------------------------------------------------------------------------- */ - -void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_adjoint_full(){ - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - size_t num_bdy_nodes = mesh->num_bdy_nodes; - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos material = simparam->material; - const int num_dim = simparam->num_dims; - const real_t damping_constant = module_params->damping_constant; - real_t global_dt; - size_t current_data_index, next_data_index; - Teuchos::RCP previous_adjoint_vector_distributed, current_adjoint_vector_distributed, previous_velocity_vector_distributed, current_velocity_vector_distributed; - Teuchos::RCP previous_phi_adjoint_vector_distributed, current_phi_adjoint_vector_distributed; - //initialize first adjoint vector at last_time_step to 0 as the terminal value - (*adjoint_vector_data)[last_time_step+1]->putScalar(0); - (*phi_adjoint_vector_data)[last_time_step+1]->putScalar(0); - - //solve terminal value problem, proceeds in time backward. For simplicity, we use the same timestep data from the forward solve. - //A linear interpolant is assumed between velocity data points; velocity midpoint is used to update the adjoint. - if(myrank==0) - std::cout << "Computing adjoint vector " << time_data.size() << std::endl; - - for (int cycle = last_time_step; cycle >= 0; cycle--) { - //compute timestep from time data - global_dt = time_data[cycle+1] - time_data[cycle]; - - //print - if(simparam->dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ - if (cycle==last_time_step){ - if(myrank==0) - printf("cycle = %d, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } - // print time step every 10 cycles - else if (cycle%20==0){ - if(myrank==0) - printf("cycle = %d, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } // end if +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn compute_topology_optimization_adjoint_full +/// +/// \brief Coupled adjoint problem for the kinetic energy minimization problem +/// +///////////////////////////////////////////////////////////////////////////// +void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_adjoint_full() +{ + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + size_t num_bdy_nodes = mesh->num_bdy_nodes; + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos material = simparam->material; + const int num_dim = simparam->num_dims; + const real_t damping_constant = module_params->damping_constant; + real_t global_dt; + size_t current_data_index, next_data_index; + Teuchos::RCP previous_adjoint_vector_distributed, current_adjoint_vector_distributed, previous_velocity_vector_distributed, current_velocity_vector_distributed; + Teuchos::RCP previous_phi_adjoint_vector_distributed, current_phi_adjoint_vector_distributed; + // initialize first adjoint vector at last_time_step to 0 as the terminal value + (*adjoint_vector_data)[last_time_step + 1]->putScalar(0); + (*phi_adjoint_vector_data)[last_time_step + 1]->putScalar(0); + + // solve terminal value problem, proceeds in time backward. For simplicity, we use the same timestep data from the forward solve. + // A linear interpolant is assumed between velocity data points; velocity midpoint is used to update the adjoint. + if (myrank == 0) + { + std::cout << "Computing adjoint vector " << time_data.size() << std::endl; } - - //view scope + + for (int cycle = last_time_step; cycle >= 0; cycle--) { - - //set velocity and position for this timestep - const_vec_array previous_velocity_vector = (*forward_solve_velocity_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - - const_vec_array previous_coordinate_vector = (*forward_solve_coordinate_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_coordinate_vector = (*forward_solve_coordinate_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - - //interface of arrays for current implementation of force calculation - - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes+nghost_nodes, { - for (int idim = 0; idim < num_dim; idim++){ - node_vel(rk_level,node_gid,idim) = previous_velocity_vector(node_gid,idim); - node_coords(rk_level,node_gid,idim) = previous_coordinate_vector(node_gid,idim); + // compute timestep from time data + global_dt = time_data[cycle + 1] - time_data[cycle]; + + // print + if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) + { + if (cycle == last_time_step) + { + if (myrank == 0) + { + printf("cycle = %d, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } + // print time step every 10 cycles + else if (cycle % 20 == 0) + { + if (myrank == 0) + { + printf("cycle = %d, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } // end if } - }); - Kokkos::fence(); - get_force_vgradient_elastic(material, + // view scope + { + // set velocity and position for this timestep + const_vec_array previous_velocity_vector = (*forward_solve_velocity_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + + const_vec_array previous_coordinate_vector = (*forward_solve_coordinate_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array current_coordinate_vector = (*forward_solve_coordinate_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + + // interface of arrays for current implementation of force calculation + + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes + nghost_nodes, { + for (int idim = 0; idim < num_dim; idim++) + { + node_vel(rk_level, node_gid, idim) = previous_velocity_vector(node_gid, idim); + node_coords(rk_level, node_gid, idim) = previous_coordinate_vector(node_gid, idim); + } + }); + Kokkos::fence(); + + get_force_vgradient_elastic(material, *mesh, node_coords, node_vel, @@ -650,996 +732,1110 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_adjoint_full() 1, cycle); - //force_gradient_velocity->describe(*fos,Teuchos::VERB_EXTREME); - const_vec_array previous_force_gradient_position = force_gradient_position->getLocalView (Tpetra::Access::ReadOnly); - //const_vec_array current_force_gradient_position = force_gradient_position->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array previous_force_gradient_velocity = force_gradient_velocity->getLocalView (Tpetra::Access::ReadOnly); - //const_vec_array current_force_gradient_velocity = force_gradient_velocity->getLocalView (Tpetra::Access::ReadOnly); - //compute gradient of force with respect to velocity - - const_vec_array previous_adjoint_vector = (*adjoint_vector_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array phi_previous_adjoint_vector = (*phi_adjoint_vector_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - vec_array midpoint_adjoint_vector = adjoint_vector_distributed->getLocalView (Tpetra::Access::ReadWrite); - vec_array phi_midpoint_adjoint_vector = phi_adjoint_vector_distributed->getLocalView (Tpetra::Access::ReadWrite); - - //half step update for RK2 scheme - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - real_t rate_of_change; - real_t matrix_contribution; - size_t dof_id; - for (int idim = 0; idim < num_dim; idim++){ - matrix_contribution = 0; - //compute resulting row of force velocity gradient matrix transpose right multiplied by adjoint vector - /* - for(int idof = 0; idof < Gradient_Matrix_Strides(node_gid*num_dim+idim); idof++){ - dof_id = DOF_Graph_Matrix(node_gid*num_dim+idim,idof); - matrix_contribution += previous_adjoint_vector(dof_id/num_dim,dof_id%num_dim)*Force_Gradient_Velocities(node_gid*num_dim+idim,idof); - } - */ - matrix_contribution = -damping_constant*previous_adjoint_vector(node_gid,idim); - rate_of_change = previous_velocity_vector(node_gid,idim)- - matrix_contribution/node_mass(node_gid)- - phi_previous_adjoint_vector(node_gid,idim)/node_mass(node_gid); - midpoint_adjoint_vector(node_gid,idim) = -rate_of_change*global_dt/2 + previous_adjoint_vector(node_gid,idim); - matrix_contribution = 0; - //compute resulting row of force displacement gradient matrix transpose right multiplied by adjoint vector - for(int idof = 0; idof < Gradient_Matrix_Strides(node_gid*num_dim+idim); idof++){ - dof_id = DOF_Graph_Matrix(node_gid*num_dim+idim,idof); - matrix_contribution += -previous_adjoint_vector(dof_id/num_dim,dof_id%num_dim)*Force_Gradient_Positions(node_gid*num_dim+idim,idof); - } - rate_of_change = -matrix_contribution; - //rate_of_change = -0.0000001*previous_adjoint_vector(node_gid,idim); - phi_midpoint_adjoint_vector(node_gid,idim) = -rate_of_change*global_dt/2 + phi_previous_adjoint_vector(node_gid,idim); - } - }); // end parallel for - Kokkos::fence(); - - boundary_adjoint(*mesh, boundary,midpoint_adjoint_vector, phi_midpoint_adjoint_vector); - comm_adjoint_vectors(cycle); - //swap names to get ghost nodes for the midpoint vectors - vec_array current_adjoint_vector = adjoint_vector_distributed->getLocalView (Tpetra::Access::ReadWrite); - vec_array phi_current_adjoint_vector = phi_adjoint_vector_distributed->getLocalView (Tpetra::Access::ReadWrite); - midpoint_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView (Tpetra::Access::ReadWrite); - phi_midpoint_adjoint_vector = (*phi_adjoint_vector_data)[cycle]->getLocalView (Tpetra::Access::ReadWrite); - - //full step update with midpoint gradient for RK2 scheme - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - real_t rate_of_change; - real_t matrix_contribution; - size_t dof_id; - for (int idim = 0; idim < num_dim; idim++){ - matrix_contribution = 0; - //compute resulting row of force velocity gradient matrix transpose right multiplied by adjoint vector - /* - for(int idof = 0; idof < Gradient_Matrix_Strides(node_gid*num_dim+idim); idof++){ - dof_id = DOF_Graph_Matrix(node_gid*num_dim+idim,idof); - matrix_contribution += midpoint_adjoint_vector(dof_id/num_dim,dof_id%num_dim)*Force_Gradient_Velocities(node_gid*num_dim+idim,idof); - } - */ - matrix_contribution = -damping_constant*midpoint_adjoint_vector(node_gid,idim); - rate_of_change = (previous_velocity_vector(node_gid,idim) + current_velocity_vector(node_gid,idim))/2- - matrix_contribution/node_mass(node_gid)- - phi_midpoint_adjoint_vector(node_gid,idim)/node_mass(node_gid); - current_adjoint_vector(node_gid,idim) = -rate_of_change*global_dt + previous_adjoint_vector(node_gid,idim); - matrix_contribution = 0; - //compute resulting row of force displacement gradient matrix transpose right multiplied by adjoint vector - for(int idof = 0; idof < Gradient_Matrix_Strides(node_gid*num_dim+idim); idof++){ - dof_id = DOF_Graph_Matrix(node_gid*num_dim+idim,idof); - matrix_contribution += -midpoint_adjoint_vector(dof_id/num_dim,dof_id%num_dim)*Force_Gradient_Positions(node_gid*num_dim+idim,idof); - } - rate_of_change = -matrix_contribution; - //rate_of_change = -0.0000001*midpoint_adjoint_vector(node_gid,idim); - phi_current_adjoint_vector(node_gid,idim) = -rate_of_change*global_dt + phi_previous_adjoint_vector(node_gid,idim); - } - }); // end parallel for - Kokkos::fence(); - - - boundary_adjoint(*mesh, boundary,current_adjoint_vector, phi_current_adjoint_vector); + // force_gradient_velocity->describe(*fos,Teuchos::VERB_EXTREME); + const_vec_array previous_force_gradient_position = force_gradient_position->getLocalView(Tpetra::Access::ReadOnly); + // const_vec_array current_force_gradient_position = force_gradient_position->getLocalView (Tpetra::Access::ReadOnly); + const_vec_array previous_force_gradient_velocity = force_gradient_velocity->getLocalView(Tpetra::Access::ReadOnly); + // const_vec_array current_force_gradient_velocity = force_gradient_velocity->getLocalView (Tpetra::Access::ReadOnly); + // compute gradient of force with respect to velocity + + const_vec_array previous_adjoint_vector = (*adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array phi_previous_adjoint_vector = (*phi_adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + vec_array midpoint_adjoint_vector = adjoint_vector_distributed->getLocalView(Tpetra::Access::ReadWrite); + vec_array phi_midpoint_adjoint_vector = phi_adjoint_vector_distributed->getLocalView(Tpetra::Access::ReadWrite); + + // half step update for RK2 scheme + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + real_t rate_of_change; + real_t matrix_contribution; + size_t dof_id; + for (int idim = 0; idim < num_dim; idim++) + { + matrix_contribution = 0; + // compute resulting row of force velocity gradient matrix transpose right multiplied by adjoint vector + /* + for(int idof = 0; idof < Gradient_Matrix_Strides(node_gid*num_dim+idim); idof++){ + dof_id = DOF_Graph_Matrix(node_gid*num_dim+idim,idof); + matrix_contribution += previous_adjoint_vector(dof_id/num_dim,dof_id%num_dim)*Force_Gradient_Velocities(node_gid*num_dim+idim,idof); + } + */ + matrix_contribution = -damping_constant * previous_adjoint_vector(node_gid, idim); + rate_of_change = previous_velocity_vector(node_gid, idim) - + matrix_contribution / node_mass(node_gid) - + phi_previous_adjoint_vector(node_gid, idim) / node_mass(node_gid); + midpoint_adjoint_vector(node_gid, idim) = -rate_of_change * global_dt / 2 + previous_adjoint_vector(node_gid, idim); + matrix_contribution = 0; + // compute resulting row of force displacement gradient matrix transpose right multiplied by adjoint vector + for (int idof = 0; idof < Gradient_Matrix_Strides(node_gid * num_dim + idim); idof++) + { + dof_id = DOF_Graph_Matrix(node_gid * num_dim + idim, idof); + matrix_contribution += -previous_adjoint_vector(dof_id / num_dim, dof_id % num_dim) * Force_Gradient_Positions(node_gid * num_dim + idim, idof); + } + rate_of_change = -matrix_contribution; + // rate_of_change = -0.0000001*previous_adjoint_vector(node_gid,idim); + phi_midpoint_adjoint_vector(node_gid, idim) = -rate_of_change * global_dt / 2 + phi_previous_adjoint_vector(node_gid, idim); + } + }); // end parallel for + Kokkos::fence(); + + boundary_adjoint(*mesh, boundary, midpoint_adjoint_vector, phi_midpoint_adjoint_vector); + comm_adjoint_vectors(cycle); + // swap names to get ghost nodes for the midpoint vectors + vec_array current_adjoint_vector = adjoint_vector_distributed->getLocalView(Tpetra::Access::ReadWrite); + vec_array phi_current_adjoint_vector = phi_adjoint_vector_distributed->getLocalView(Tpetra::Access::ReadWrite); + midpoint_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadWrite); + phi_midpoint_adjoint_vector = (*phi_adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadWrite); + + // full step update with midpoint gradient for RK2 scheme + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + real_t rate_of_change; + real_t matrix_contribution; + size_t dof_id; + for (int idim = 0; idim < num_dim; idim++) + { + matrix_contribution = 0; + // compute resulting row of force velocity gradient matrix transpose right multiplied by adjoint vector + /* + for(int idof = 0; idof < Gradient_Matrix_Strides(node_gid*num_dim+idim); idof++){ + dof_id = DOF_Graph_Matrix(node_gid*num_dim+idim,idof); + matrix_contribution += midpoint_adjoint_vector(dof_id/num_dim,dof_id%num_dim)*Force_Gradient_Velocities(node_gid*num_dim+idim,idof); + } + */ + matrix_contribution = -damping_constant * midpoint_adjoint_vector(node_gid, idim); + rate_of_change = (previous_velocity_vector(node_gid, idim) + current_velocity_vector(node_gid, idim)) / 2 - + matrix_contribution / node_mass(node_gid) - + phi_midpoint_adjoint_vector(node_gid, idim) / node_mass(node_gid); + current_adjoint_vector(node_gid, idim) = -rate_of_change * global_dt + previous_adjoint_vector(node_gid, idim); + matrix_contribution = 0; + // compute resulting row of force displacement gradient matrix transpose right multiplied by adjoint vector + for (int idof = 0; idof < Gradient_Matrix_Strides(node_gid * num_dim + idim); idof++) + { + dof_id = DOF_Graph_Matrix(node_gid * num_dim + idim, idof); + matrix_contribution += -midpoint_adjoint_vector(dof_id / num_dim, dof_id % num_dim) * Force_Gradient_Positions(node_gid * num_dim + idim, idof); + } + rate_of_change = -matrix_contribution; + // rate_of_change = -0.0000001*midpoint_adjoint_vector(node_gid,idim); + phi_current_adjoint_vector(node_gid, idim) = -rate_of_change * global_dt + phi_previous_adjoint_vector(node_gid, idim); + } + }); // end parallel for + Kokkos::fence(); - } //end view scope + boundary_adjoint(*mesh, boundary, current_adjoint_vector, phi_current_adjoint_vector); + } // end view scope - comm_adjoint_vectors(cycle); - //phi_adjoint_vector_distributed->describe(*fos,Teuchos::VERB_EXTREME); - } + comm_adjoint_vectors(cycle); + // phi_adjoint_vector_distributed->describe(*fos,Teuchos::VERB_EXTREME); + } } +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn compute_topology_optimization_gradient +/// +/// \brief Gradient calculation for the kinetic energy minimization problem +/// +/// \param Vector of design variables +/// \param Vector of design gradients +/// +///////////////////////////////////////////////////////////////////////////// +void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient(const_vec_array design_variables, vec_array design_gradients) +{ + size_t num_bdy_nodes = mesh->num_bdy_nodes; + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos material = simparam->material; + const int num_dim = simparam->num_dims; + int num_corners = rnum_elem * num_nodes_in_elem; + real_t global_dt; + size_t current_data_index, next_data_index; + CArrayKokkos current_element_velocities = CArrayKokkos(num_nodes_in_elem, num_dim); + CArrayKokkos current_element_adjoint = CArrayKokkos(num_nodes_in_elem, num_dim); + + if (myrank == 0) + { + std::cout << "Computing accumulated kinetic energy gradient" << std::endl; + } -/* ---------------------------------------------------------------------------- - Gradient calculation for the kinetic energy minimization problem -------------------------------------------------------------------------------- */ + compute_topology_optimization_adjoint(); -void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient(const_vec_array design_variables, vec_array design_gradients){ + // compute design gradients + FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { + design_gradients(node_id, 0) = 0; + }); // end parallel for + Kokkos::fence(); - size_t num_bdy_nodes = mesh->num_bdy_nodes; - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos material = simparam->material; - const int num_dim = simparam->num_dims; - int num_corners = rnum_elem*num_nodes_in_elem; - real_t global_dt; - size_t current_data_index, next_data_index; - CArrayKokkos current_element_velocities = CArrayKokkos(num_nodes_in_elem,num_dim); - CArrayKokkos current_element_adjoint = CArrayKokkos(num_nodes_in_elem,num_dim); + // gradient contribution from kinetic energy vMv product. + for (unsigned long cycle = 0; cycle < last_time_step + 1; cycle++) + { + // compute timestep from time data + global_dt = time_data[cycle + 1] - time_data[cycle]; - if(myrank==0) - std::cout << "Computing accumulated kinetic energy gradient" << std::endl; + // print + if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) + { + if (cycle == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } + // print time step every 10 cycles + else if (cycle % 20 == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } // end if + } - compute_topology_optimization_adjoint(); + // compute adjoint vector for this data point; use velocity midpoint + // view scope + { + const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_velocity_vector = (*forward_solve_velocity_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_adjoint_vector = (*adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + + FOR_ALL_CLASS(elem_id, 0, rnum_elem, { + size_t node_id; + size_t corner_id; + real_t inner_product; + // std::cout << elem_mass(elem_id) <dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ - if (cycle==0){ - if(myrank==0) - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } - // print time step every 10 cycles - else if (cycle%20==0){ - if(myrank==0) - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } // end if - } + inner_product = 0; + for (int ifill = 0; ifill < num_nodes_in_elem; ifill++) + { + node_id = nodes_in_elem(elem_id, ifill); + for (int idim = 0; idim < num_dim; idim++) + { + inner_product += elem_mass(elem_id) * current_element_velocities(ifill, idim) * current_element_velocities(ifill, idim); + } + } - //compute adjoint vector for this data point; use velocity midpoint - //view scope - { - const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array next_velocity_vector = (*forward_solve_velocity_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array next_adjoint_vector = (*adjoint_vector_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - - FOR_ALL_CLASS(elem_id, 0, rnum_elem, { - size_t node_id; - size_t corner_id; - real_t inner_product; - //std::cout << elem_mass(elem_id) <dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ - if (cycle==0){ - if(myrank==0) - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } - // print time step every 10 cycles - else if (cycle%20==0){ - if(myrank==0) - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } // end if - } + Kokkos::fence(); - //compute adjoint vector for this data point; use velocity midpoint - //view scope - { - //const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array next_adjoint_vector = (*adjoint_vector_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - //const_vec_array current_coord_vector = forward_solve_coordinate_data[cycle]->getLocalView (Tpetra::Access::ReadOnly); - //const_vec_array final_coordinates = forward_solve_coordinate_data[last_time_step+1]->getLocalView (Tpetra::Access::ReadOnly); - - FOR_ALL_CLASS(elem_id, 0, rnum_elem, { - size_t node_id; - size_t corner_id; - real_t inner_product; - //std::cout << elem_mass(elem_id) < design_densities_distributed, Teuchos::RCP design_gradients_distributed){ + // multiply by Hex8 constants (the diagonlization here only works for Hex8 anyway) + FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { + design_gradients(node_id, 0) *= -0.5 / (double)num_nodes_in_elem / (double)num_nodes_in_elem; + // design_gradients(node_id,0) =0.00001; + }); // end parallel for + Kokkos::fence(); - size_t num_bdy_nodes = mesh->num_bdy_nodes; - const DCArrayKokkos boundary = module_params->boundary; - const DCArrayKokkos material = simparam->material; - const int num_dim = simparam->num_dims; - int num_corners = rnum_elem*num_nodes_in_elem; - real_t global_dt; - bool element_constant_density = true; - size_t current_data_index, next_data_index; - CArrayKokkos current_element_velocities = CArrayKokkos(num_nodes_in_elem,num_dim); - CArrayKokkos current_element_adjoint = CArrayKokkos(num_nodes_in_elem,num_dim); + // gradient contribution from Force vector. + for (unsigned long cycle = 0; cycle < last_time_step + 1; cycle++) + { + // compute timestep from time data + global_dt = time_data[cycle + 1] - time_data[cycle]; - if(myrank==0) - std::cout << "Computing accumulated kinetic energy gradient" << std::endl; + // print + if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) + { + if (cycle == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } + // print time step every 10 cycles + else if (cycle % 20 == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } // end if + } - compute_topology_optimization_adjoint_full(); + // compute adjoint vector for this data point; use velocity midpoint + // view scope + { + // const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); + const_vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_adjoint_vector = (*adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + // const_vec_array current_coord_vector = forward_solve_coordinate_data[cycle]->getLocalView (Tpetra::Access::ReadOnly); + // const_vec_array final_coordinates = forward_solve_coordinate_data[last_time_step+1]->getLocalView (Tpetra::Access::ReadOnly); + + FOR_ALL_CLASS(elem_id, 0, rnum_elem, { + size_t node_id; + size_t corner_id; + real_t inner_product; + // std::cout << elem_mass(elem_id) <getLocalView (Tpetra::Access::ReadWrite); - const_vec_array design_densities = design_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); - //compute design gradients - FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { - design_gradients(node_id,0) = 0; - }); // end parallel for - Kokkos::fence(); + inner_product = 0; + for (int ifill = 0; ifill < num_nodes_in_elem; ifill++) + { + node_id = nodes_in_elem(elem_id, ifill); + for (int idim = 0; idim < num_dim; idim++) + { + inner_product += 0.00001 * current_element_adjoint(ifill, idim); + // inner_product += 0.0001; + } + } - //gradient contribution from kinetic energy v(dM/drho)v product. - if(simparam->dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ - if(myrank==0){ - std::cout << "v*dM/drho*v term" << std::endl; - } + for (int inode = 0; inode < num_nodes_in_elem; inode++) + { + // compute gradient of local element contribution to v^t*M*v product + corner_id = elem_id * num_nodes_in_elem + inode; + corner_value_storage(corner_id) = -inner_product * global_dt / (double)num_nodes_in_elem; + } + }); // end parallel for + Kokkos::fence(); + + // accumulate node values from corner storage + // multiply + FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { + size_t corner_id; + for (int icorner = 0; icorner < num_corners_in_node(node_id); icorner++) + { + corner_id = corners_in_node(node_id, icorner); + design_gradients(node_id, 0) += corner_value_storage(corner_id); + } + }); // end parallel for + Kokkos::fence(); + } // end view scope } +} - for (unsigned long cycle = 0; cycle < last_time_step+1; cycle++) { - //compute timestep from time data - global_dt = time_data[cycle+1] - time_data[cycle]; - - //print - if(simparam->dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn compute_topology_optimization_gradient_full +/// +/// \brief Gradient for the (unsimplified) kinetic energy minimization problem +/// +/// \param Distributed design densities +/// \param Distributed design gradients +/// +///////////////////////////////////////////////////////////////////////////// +void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient_full(Teuchos::RCP design_densities_distributed, Teuchos::RCP design_gradients_distributed) +{ + size_t num_bdy_nodes = mesh->num_bdy_nodes; + const DCArrayKokkos boundary = module_params->boundary; + const DCArrayKokkos material = simparam->material; + const int num_dim = simparam->num_dims; + int num_corners = rnum_elem * num_nodes_in_elem; + real_t global_dt; + bool element_constant_density = true; + size_t current_data_index, next_data_index; + CArrayKokkos current_element_velocities = CArrayKokkos(num_nodes_in_elem, num_dim); + CArrayKokkos current_element_adjoint = CArrayKokkos(num_nodes_in_elem, num_dim); + + if (myrank == 0) + { + std::cout << "Computing accumulated kinetic energy gradient" << std::endl; + } - if (cycle==0){ - if(myrank==0) - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } - // print time step every 10 cycles - else if (cycle%20==0){ - if(myrank==0) - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } // end if - } + compute_topology_optimization_adjoint_full(); - //compute adjoint vector for this data point; use velocity midpoint - //view scope + { // view scope + vec_array design_gradients = design_gradients_distributed->getLocalView(Tpetra::Access::ReadWrite); + const_vec_array design_densities = design_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); + // compute design gradients + FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { + design_gradients(node_id, 0) = 0; + }); // end parallel for + Kokkos::fence(); + + // gradient contribution from kinetic energy v(dM/drho)v product. + if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) { - const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array next_velocity_vector = (*forward_solve_velocity_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array next_adjoint_vector = (*adjoint_vector_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - - FOR_ALL_CLASS(elem_id, 0, rnum_elem, { - size_t node_id; - size_t corner_id; - real_t inner_product; - //std::cout << elem_mass(elem_id) <dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) + { + if (cycle == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } + // print time step every 10 cycles + else if (cycle % 20 == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } // end if } - }); // end parallel for - Kokkos::fence(); - } //end view scope - - } - + // compute adjoint vector for this data point; use velocity midpoint + // view scope + { + const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_velocity_vector = (*forward_solve_velocity_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_adjoint_vector = (*adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + + FOR_ALL_CLASS(elem_id, 0, rnum_elem, { + size_t node_id; + size_t corner_id; + real_t inner_product; + // std::cout << elem_mass(elem_id) <dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ - if(myrank==0){ - std::cout << "gradient term involving adjoint derivative" << std::endl; + for (int inode = 0; inode < num_nodes_in_elem; inode++) + { + // compute gradient of local element contribution to v^t*M*v product + corner_id = elem_id * num_nodes_in_elem + inode; + // division by design ratio recovers nominal element mass used in the gradient operator + corner_value_storage(corner_id) = inner_product * global_dt / relative_element_densities(elem_id); + } + }); // end parallel for + Kokkos::fence(); + + // accumulate node values from corner storage + // multiply + FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { + size_t corner_id; + for (int icorner = 0; icorner < num_corners_in_node(node_id); icorner++) + { + corner_id = corners_in_node(node_id, icorner); + design_gradients(node_id, 0) += corner_value_storage(corner_id); + } + }); // end parallel for + Kokkos::fence(); + } // end view scope } - } - for (unsigned long cycle = 0; cycle < last_time_step+1; cycle++) { - //compute timestep from time data - global_dt = time_data[cycle+1] - time_data[cycle]; - //print - if(simparam->dynamic_options.output_time_sequence_level==TIME_OUTPUT_LEVEL::extreme){ + // multiply by Hex8 constants (the diagonlization here only works for Hex8 anyway) + FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { + design_gradients(node_id, 0) *= 0.5 / (double)num_nodes_in_elem / (double)num_nodes_in_elem; + // design_gradients(node_id,0) =0.00001; + }); // end parallel for + Kokkos::fence(); - if (cycle==0){ - if(myrank==0) - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + // gradient contribution from time derivative of adjoint \dot{lambda}(dM/drho)v product. + if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) + { + if (myrank == 0) + { + std::cout << "gradient term involving adjoint derivative" << std::endl; + } } - // print time step every 10 cycles - else if (cycle%20==0){ - if(myrank==0) - printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); - } // end if - } - //compute adjoint vector for this data point; use velocity midpoint - //view scope + for (unsigned long cycle = 0; cycle < last_time_step + 1; cycle++) { - const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_phi_adjoint_vector = (*phi_adjoint_vector_data)[cycle]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array next_velocity_vector = (*forward_solve_velocity_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array next_adjoint_vector = (*adjoint_vector_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - const_vec_array next_phi_adjoint_vector = (*phi_adjoint_vector_data)[cycle+1]->getLocalView (Tpetra::Access::ReadOnly); - - const real_t damping_constant = module_params->damping_constant; - FOR_ALL_CLASS(elem_id, 0, rnum_elem, { - real_t lambda_dot_current; - real_t lambda_dot_next; - size_t node_id; - size_t corner_id; - real_t inner_product; - //std::cout << elem_mass(elem_id) <dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) + { + if (cycle == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } + // print time step every 10 cycles + else if (cycle % 20 == 0) + { + if (myrank == 0) + { + printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); + } + } // end if } - inner_product = 0; - for(int ifill=0; ifill < num_nodes_in_elem; ifill++){ - node_id = nodes_in_elem(elem_id, ifill); - for(int idim=0; idim < num_dim; idim++){ - //lambda_dot_current = lambda_dot_next = (next_adjoint_vector(node_id,idim)-current_adjoint_vector(node_id,idim))/global_dt; - lambda_dot_current = current_velocity_vector(node_id,idim) + damping_constant*current_adjoint_vector(node_id,idim)/node_mass(node_id) - current_phi_adjoint_vector(node_id,idim)/node_mass(node_id); - lambda_dot_next = next_velocity_vector(node_id,idim) + damping_constant*next_adjoint_vector(node_id,idim)/node_mass(node_id) - next_phi_adjoint_vector(node_id,idim)/node_mass(node_id); - inner_product += elem_mass(elem_id)*(lambda_dot_current+lambda_dot_next)*current_element_velocities(ifill,idim)/2; - } - } + // compute adjoint vector for this data point; use velocity midpoint + // view scope + { + const_vec_array current_velocity_vector = (*forward_solve_velocity_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array current_phi_adjoint_vector = (*phi_adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_velocity_vector = (*forward_solve_velocity_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_adjoint_vector = (*adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + const_vec_array next_phi_adjoint_vector = (*phi_adjoint_vector_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); + + const real_t damping_constant = module_params->damping_constant; + FOR_ALL_CLASS(elem_id, 0, rnum_elem, { + real_t lambda_dot_current; + real_t lambda_dot_next; + size_t node_id; + size_t corner_id; + real_t inner_product; + // std::cout << elem_mass(elem_id) <getLocalView (Tpetra::Access::ReadOnly); - const_vec_array current_adjoint_vector = (*adjoint_vector_data)[0]->getLocalView (Tpetra::Access::ReadOnly); - - FOR_ALL_CLASS(elem_id, 0, rnum_elem, { - real_t lambda_dot; - size_t node_id; - size_t corner_id; - real_t inner_product; - //std::cout << elem_mass(elem_id) <getLocalView(Tpetra::Access::ReadOnly); + const_vec_array current_adjoint_vector = (*adjoint_vector_data)[0]->getLocalView(Tpetra::Access::ReadOnly); + + FOR_ALL_CLASS(elem_id, 0, rnum_elem, { + real_t lambda_dot; + size_t node_id; + size_t corner_id; + real_t inner_product; + // std::cout << elem_mass(elem_id) <getLocalView (Tpetra::Access::ReadWrite); - const_host_vec_array host_design_variables = design_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); - compute_stiffness_gradients(host_design_variables, host_design_gradients); - }//end view scope + inner_product = 0; + for (int ifill = 0; ifill < num_nodes_in_elem; ifill++) + { + node_id = nodes_in_elem(elem_id, ifill); + for (int idim = 0; idim < num_dim; idim++) + { + inner_product += elem_mass(elem_id) * current_adjoint_vector(node_id, idim) * current_element_velocities(ifill, idim); + } + } + for (int inode = 0; inode < num_nodes_in_elem; inode++) + { + // compute gradient of local element contribution to v^t*M*v product + corner_id = elem_id * num_nodes_in_elem + inode; + // division by design ratio recovers nominal element mass used in the gradient operator + corner_value_storage(corner_id) = inner_product / relative_element_densities(elem_id); + } + }); // end parallel for + Kokkos::fence(); + + // accumulate node values from corner storage + // multiply + FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { + size_t corner_id; + for (int icorner = 0; icorner < num_corners_in_node(node_id); icorner++) + { + corner_id = corners_in_node(node_id, icorner); + design_gradients(node_id, 0) += -corner_value_storage(corner_id) / (double)num_nodes_in_elem / (double)num_nodes_in_elem; + } + }); // end parallel for + Kokkos::fence(); + } // end view scope + } // end view scope + + // force_design_gradient_term(design_variables, design_gradients); + // view scope + { + host_vec_array host_design_gradients = design_gradients_distributed->getLocalView(Tpetra::Access::ReadWrite); + const_host_vec_array host_design_variables = design_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); + compute_stiffness_gradients(host_design_variables, host_design_gradients); + } // end view scope } -/* ---------------------------------------------------------------------- - Initialize global vectors and array maps needed for matrix assembly -------------------------------------------------------------------------- */ -void FEA_Module_Dynamic_Elasticity::init_assembly(){ - int num_dim = simparam->num_dims; - //const_host_elem_conn_array nodes_in_elem = global_nodes_in_elem_distributed->getLocalView (Tpetra::Access::ReadOnly); - Gradient_Matrix_Strides = DCArrayKokkos (nlocal_nodes*num_dim, "Gradient_Matrix_Strides"); - CArrayKokkos Graph_Fill(nall_nodes, "nall_nodes"); - CArrayKokkos current_row_nodes_scanned; - int local_node_index, current_column_index; - size_t max_stride = 0; - size_t nodes_per_element; - nodal_density_flag = simparam->nodal_density_flag; - penalty_power = simparam->optimization_options.simp_penalty_power; - - //allocate stride arrays - CArrayKokkos Graph_Matrix_Strides_initial(nlocal_nodes, "Graph_Matrix_Strides_initial"); - DCArrayKokkos Dual_Graph_Matrix_Strides_initial(nlocal_nodes, "Host_Graph_Matrix_Strides_initial"); - Graph_Matrix_Strides = DCArrayKokkos(nlocal_nodes, "Graph_Matrix_Strides"); - - //allocate storage for the sparse stiffness matrix map used in the assembly process - Global_Stiffness_Matrix_Assembly_Map = Global_Gradient_Matrix_Assembly_Map = DCArrayKokkos(rnum_elem, - max_nodes_per_element,max_nodes_per_element, "Global_Gradient_Matrix_Assembly_Map"); - - //allocate array used to determine global node repeats in the sparse graph later - DCArrayKokkos node_indices_used(nall_nodes, "node_indices_used"); - - /*allocate array that stores which column the node index occured on for the current row - when removing repeats*/ - DCArrayKokkos column_index(nall_nodes, "column_index"); - - //initialize nlocal arrays - FOR_ALL_CLASS(inode, 0, nlocal_nodes, { - Graph_Matrix_Strides_initial(inode) = 0; - Graph_Matrix_Strides(inode) = 0; - Graph_Fill(inode) = 0; - }); // end parallel for - Kokkos::fence(); +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn init_assembly +/// +/// \brief Initialize global vectors and array maps needed for matrix assembly +/// +///////////////////////////////////////////////////////////////////////////// +void FEA_Module_Dynamic_Elasticity::init_assembly() +{ + int num_dim = simparam->num_dims; + // const_host_elem_conn_array nodes_in_elem = global_nodes_in_elem_distributed->getLocalView (Tpetra::Access::ReadOnly); + Gradient_Matrix_Strides = DCArrayKokkos(nlocal_nodes * num_dim, "Gradient_Matrix_Strides"); + CArrayKokkos Graph_Fill(nall_nodes, "nall_nodes"); + CArrayKokkos current_row_nodes_scanned; + int local_node_index, current_column_index; + size_t max_stride = 0; + size_t nodes_per_element; + nodal_density_flag = simparam->nodal_density_flag; + penalty_power = simparam->optimization_options.simp_penalty_power; + + // allocate stride arrays + CArrayKokkos Graph_Matrix_Strides_initial(nlocal_nodes, "Graph_Matrix_Strides_initial"); + DCArrayKokkos Dual_Graph_Matrix_Strides_initial(nlocal_nodes, "Host_Graph_Matrix_Strides_initial"); + Graph_Matrix_Strides = DCArrayKokkos(nlocal_nodes, "Graph_Matrix_Strides"); + + // allocate storage for the sparse stiffness matrix map used in the assembly process + Global_Stiffness_Matrix_Assembly_Map = Global_Gradient_Matrix_Assembly_Map = DCArrayKokkos(rnum_elem, + max_nodes_per_element, max_nodes_per_element, "Global_Gradient_Matrix_Assembly_Map"); + + // allocate array used to determine global node repeats in the sparse graph later + DCArrayKokkos node_indices_used(nall_nodes, "node_indices_used"); + + /*allocate array that stores which column the node index occured on for the current row + when removing repeats*/ + DCArrayKokkos column_index(nall_nodes, "column_index"); + + // initialize nlocal arrays + FOR_ALL_CLASS(inode, 0, nlocal_nodes, { + Graph_Matrix_Strides_initial(inode) = 0; + Graph_Matrix_Strides(inode) = 0; + Graph_Fill(inode) = 0; + }); // end parallel for + Kokkos::fence(); - //initialize nall arrays - //initialize nlocal arrays - FOR_ALL_CLASS(inode, 0, nall_nodes, { - node_indices_used(inode) = 0; - column_index(inode) = 0; - }); // end parallel for - Kokkos::fence(); - - //count upper bound of strides for Sparse Pattern Graph by allowing repeats due to connectivity - if(num_dim == 2) - for (int ielem = 0; ielem < rnum_elem; ielem++){ - element_select->choose_2Delem_type(Element_Types(ielem), elem2D); - nodes_per_element = elem2D->num_nodes(); - for (int lnode = 0; lnode < nodes_per_element; lnode++){ - local_node_index = nodes_in_elem(ielem, lnode); - if(local_node_index < nlocal_nodes){ - Dual_Graph_Matrix_Strides_initial.host(local_node_index) += nodes_per_element; + // initialize nall arrays + // initialize nlocal arrays + FOR_ALL_CLASS(inode, 0, nall_nodes, { + node_indices_used(inode) = 0; + column_index(inode) = 0; + }); // end parallel for + Kokkos::fence(); + + // count upper bound of strides for Sparse Pattern Graph by allowing repeats due to connectivity + if (num_dim == 2) + { + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + element_select->choose_2Delem_type(Element_Types(ielem), elem2D); + nodes_per_element = elem2D->num_nodes(); + for (int lnode = 0; lnode < nodes_per_element; lnode++) + { + local_node_index = nodes_in_elem(ielem, lnode); + if (local_node_index < nlocal_nodes) + { + Dual_Graph_Matrix_Strides_initial.host(local_node_index) += nodes_per_element; + } + } } - } } - if(num_dim == 3) - for (int ielem = 0; ielem < rnum_elem; ielem++){ - element_select->choose_3Delem_type(Element_Types(ielem), elem); - nodes_per_element = elem->num_nodes(); - for (int lnode = 0; lnode < nodes_per_element; lnode++){ - local_node_index = nodes_in_elem(ielem, lnode); - if(local_node_index < nlocal_nodes){ - Dual_Graph_Matrix_Strides_initial.host(local_node_index) += nodes_per_element; + if (num_dim == 3) + { + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + element_select->choose_3Delem_type(Element_Types(ielem), elem); + nodes_per_element = elem->num_nodes(); + for (int lnode = 0; lnode < nodes_per_element; lnode++) + { + local_node_index = nodes_in_elem(ielem, lnode); + if (local_node_index < nlocal_nodes) + { + Dual_Graph_Matrix_Strides_initial.host(local_node_index) += nodes_per_element; + } + } } - } } - - Dual_Graph_Matrix_Strides_initial.update_device(); - //equate strides for later - FOR_ALL_CLASS(inode, 0, nlocal_nodes, { - Graph_Matrix_Strides(inode) = Graph_Matrix_Strides_initial(inode) = Dual_Graph_Matrix_Strides_initial(inode); - }); // end parallel for - - //for (int inode = 0; inode < nlocal_nodes; inode++) - //std::cout << Graph_Matrix_Strides_initial(inode) << std::endl; - - //compute maximum stride - size_t update = 0; - REDUCE_MAX_CLASS(inode, 0, nlocal_nodes, update, { - if(update < Graph_Matrix_Strides_initial(inode)) - update = Graph_Matrix_Strides_initial(inode); - }, max_stride); - - //std::cout << "THE MAX STRIDE" << max_stride << std::endl; - //allocate array used in the repeat removal process - current_row_nodes_scanned = CArrayKokkos(max_stride, "current_row_nodes_scanned"); - - //allocate sparse graph with node repeats - RaggedRightArrayKokkos Repeat_Graph_Matrix(Graph_Matrix_Strides_initial); - RaggedRightArrayofVectorsKokkos Element_local_indices(Graph_Matrix_Strides_initial,num_dim); - - //Fill the initial Graph with repeats - if(num_dim == 2){ - for (int ielem = 0; ielem < rnum_elem; ielem++){ - element_select->choose_2Delem_type(Element_Types(ielem), elem2D); - nodes_per_element = elem2D->num_nodes(); - for (int lnode = 0; lnode < nodes_per_element; lnode++){ - local_node_index = nodes_in_elem(ielem, lnode); - if(local_node_index < nlocal_nodes){ - for (int jnode = 0; jnode < nodes_per_element; jnode++){ - current_column_index = Graph_Fill(local_node_index)+jnode; - Repeat_Graph_Matrix(local_node_index, current_column_index) = nodes_in_elem(ielem,jnode); - - //fill inverse map - Element_local_indices(local_node_index,current_column_index,0) = ielem; - Element_local_indices(local_node_index,current_column_index,1) = lnode; - Element_local_indices(local_node_index,current_column_index,2) = jnode; - - //fill forward map - Global_Gradient_Matrix_Assembly_Map(ielem,lnode,jnode) = current_column_index; - } - Graph_Fill(local_node_index) += nodes_per_element; + Dual_Graph_Matrix_Strides_initial.update_device(); + + // equate strides for later + FOR_ALL_CLASS(inode, 0, nlocal_nodes, { + Graph_Matrix_Strides(inode) = Graph_Matrix_Strides_initial(inode) = Dual_Graph_Matrix_Strides_initial(inode); + }); // end parallel for + + // for (int inode = 0; inode < nlocal_nodes; inode++) + // std::cout << Graph_Matrix_Strides_initial(inode) << std::endl; + + // compute maximum stride + size_t update = 0; + REDUCE_MAX_CLASS(inode, 0, nlocal_nodes, update, { + if (update < Graph_Matrix_Strides_initial(inode)) + { + update = Graph_Matrix_Strides_initial(inode); + } + }, max_stride); + + // std::cout << "THE MAX STRIDE" << max_stride << std::endl; + // allocate array used in the repeat removal process + current_row_nodes_scanned = CArrayKokkos(max_stride, "current_row_nodes_scanned"); + + // allocate sparse graph with node repeats + RaggedRightArrayKokkos Repeat_Graph_Matrix(Graph_Matrix_Strides_initial); + RaggedRightArrayofVectorsKokkos Element_local_indices(Graph_Matrix_Strides_initial, num_dim); + + // Fill the initial Graph with repeats + if (num_dim == 2) + { + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + element_select->choose_2Delem_type(Element_Types(ielem), elem2D); + nodes_per_element = elem2D->num_nodes(); + for (int lnode = 0; lnode < nodes_per_element; lnode++) + { + local_node_index = nodes_in_elem(ielem, lnode); + if (local_node_index < nlocal_nodes) + { + for (int jnode = 0; jnode < nodes_per_element; jnode++) + { + current_column_index = Graph_Fill(local_node_index) + jnode; + Repeat_Graph_Matrix(local_node_index, current_column_index) = nodes_in_elem(ielem, jnode); + + // fill inverse map + Element_local_indices(local_node_index, current_column_index, 0) = ielem; + Element_local_indices(local_node_index, current_column_index, 1) = lnode; + Element_local_indices(local_node_index, current_column_index, 2) = jnode; + + // fill forward map + Global_Gradient_Matrix_Assembly_Map(ielem, lnode, jnode) = current_column_index; + } + Graph_Fill(local_node_index) += nodes_per_element; + } + } } - } } - } - - if(num_dim == 3){ - for (int ielem = 0; ielem < rnum_elem; ielem++){ - element_select->choose_3Delem_type(Element_Types(ielem), elem); - nodes_per_element = elem->num_nodes(); - for (int lnode = 0; lnode < nodes_per_element; lnode++){ - local_node_index = nodes_in_elem(ielem, lnode); - if(local_node_index < nlocal_nodes){ - for (int jnode = 0; jnode < nodes_per_element; jnode++){ - current_column_index = Graph_Fill(local_node_index)+jnode; - Repeat_Graph_Matrix(local_node_index, current_column_index) = nodes_in_elem(ielem,jnode); - - //fill inverse map - Element_local_indices(local_node_index,current_column_index,0) = ielem; - Element_local_indices(local_node_index,current_column_index,1) = lnode; - Element_local_indices(local_node_index,current_column_index,2) = jnode; - - //fill forward map - Global_Gradient_Matrix_Assembly_Map(ielem,lnode,jnode) = current_column_index; - } - Graph_Fill(local_node_index) += nodes_per_element; + + if (num_dim == 3) + { + for (int ielem = 0; ielem < rnum_elem; ielem++) + { + element_select->choose_3Delem_type(Element_Types(ielem), elem); + nodes_per_element = elem->num_nodes(); + for (int lnode = 0; lnode < nodes_per_element; lnode++) + { + local_node_index = nodes_in_elem(ielem, lnode); + if (local_node_index < nlocal_nodes) + { + for (int jnode = 0; jnode < nodes_per_element; jnode++) + { + current_column_index = Graph_Fill(local_node_index) + jnode; + Repeat_Graph_Matrix(local_node_index, current_column_index) = nodes_in_elem(ielem, jnode); + + // fill inverse map + Element_local_indices(local_node_index, current_column_index, 0) = ielem; + Element_local_indices(local_node_index, current_column_index, 1) = lnode; + Element_local_indices(local_node_index, current_column_index, 2) = jnode; + + // fill forward map + Global_Gradient_Matrix_Assembly_Map(ielem, lnode, jnode) = current_column_index; + } + Graph_Fill(local_node_index) += nodes_per_element; + } + } } - } } - } - - //debug statement - //std::cout << "started run" << std::endl; - //std::cout << "Graph Matrix Strides Repeat on task " << myrank << std::endl; - //for (int inode = 0; inode < nlocal_nodes; inode++) - //std::cout << Graph_Matrix_Strides(inode) << std::endl; - RUN_CLASS({ - //remove repeats from the inital graph setup - int current_node; - //remove repeats from the inital graph setup - int current_element_index; - int element_row_index; - int element_column_index; - int current_stride; - int current_row_n_nodes_scanned; - for (int inode = 0; inode < nlocal_nodes; inode++){ - current_row_n_nodes_scanned = 0; - for (int istride = 0; istride < Graph_Matrix_Strides(inode); istride++){ - //convert global index in graph to its local index for the flagging array - current_node = Repeat_Graph_Matrix(inode,istride); - //debug - //if(current_node==-1) - //std::cout << "Graph Matrix node access on task " << myrank << std::endl; - //std::cout << Repeat_Graph_Matrix(inode,istride) << std::endl; - if(node_indices_used(current_node)){ - //set global assembly map index to the location in the graph matrix where this global node was first found - current_element_index = Element_local_indices(inode,istride,0); - element_row_index = Element_local_indices(inode,istride,1); - element_column_index = Element_local_indices(inode,istride,2); - Global_Gradient_Matrix_Assembly_Map(current_element_index,element_row_index, element_column_index) - = column_index(current_node); - - - //swap current node with the end of the current row and shorten the stride of the row - //first swap information about the inverse and forward maps - - current_stride = Graph_Matrix_Strides(inode); - if(istride!=current_stride-1){ - Element_local_indices(inode,istride,0) = Element_local_indices(inode,current_stride-1,0); - Element_local_indices(inode,istride,1) = Element_local_indices(inode,current_stride-1,1); - Element_local_indices(inode,istride,2) = Element_local_indices(inode,current_stride-1,2); - current_element_index = Element_local_indices(inode,istride,0); - element_row_index = Element_local_indices(inode,istride,1); - element_column_index = Element_local_indices(inode,istride,2); - - Global_Gradient_Matrix_Assembly_Map(current_element_index,element_row_index, element_column_index) - = istride; - - //now that the element map information has been copied, copy the global node index and delete the last index - - Repeat_Graph_Matrix(inode,istride) = Repeat_Graph_Matrix(inode,current_stride-1); - } - istride--; - Graph_Matrix_Strides(inode)--; + + // debug statement + // std::cout << "started run" << std::endl; + // std::cout << "Graph Matrix Strides Repeat on task " << myrank << std::endl; + // for (int inode = 0; inode < nlocal_nodes; inode++) + // std::cout << Graph_Matrix_Strides(inode) << std::endl; + RUN_CLASS({ + // remove repeats from the inital graph setup + int current_node; + // remove repeats from the inital graph setup + int current_element_index; + int element_row_index; + int element_column_index; + int current_stride; + int current_row_n_nodes_scanned; + for (int inode = 0; inode < nlocal_nodes; inode++) + { + current_row_n_nodes_scanned = 0; + for (int istride = 0; istride < Graph_Matrix_Strides(inode); istride++) + { + // convert global index in graph to its local index for the flagging array + current_node = Repeat_Graph_Matrix(inode, istride); + // debug + // if(current_node==-1) + // std::cout << "Graph Matrix node access on task " << myrank << std::endl; + // std::cout << Repeat_Graph_Matrix(inode,istride) << std::endl; + if (node_indices_used(current_node)) + { + // set global assembly map index to the location in the graph matrix where this global node was first found + current_element_index = Element_local_indices(inode, istride, 0); + element_row_index = Element_local_indices(inode, istride, 1); + element_column_index = Element_local_indices(inode, istride, 2); + Global_Gradient_Matrix_Assembly_Map(current_element_index, element_row_index, element_column_index) + = column_index(current_node); + + // swap current node with the end of the current row and shorten the stride of the row + // first swap information about the inverse and forward maps + + current_stride = Graph_Matrix_Strides(inode); + if (istride != current_stride - 1) + { + Element_local_indices(inode, istride, 0) = Element_local_indices(inode, current_stride - 1, 0); + Element_local_indices(inode, istride, 1) = Element_local_indices(inode, current_stride - 1, 1); + Element_local_indices(inode, istride, 2) = Element_local_indices(inode, current_stride - 1, 2); + current_element_index = Element_local_indices(inode, istride, 0); + element_row_index = Element_local_indices(inode, istride, 1); + element_column_index = Element_local_indices(inode, istride, 2); + + Global_Gradient_Matrix_Assembly_Map(current_element_index, element_row_index, element_column_index) + = istride; + + // now that the element map information has been copied, copy the global node index and delete the last index + + Repeat_Graph_Matrix(inode, istride) = Repeat_Graph_Matrix(inode, current_stride - 1); + } + istride--; + Graph_Matrix_Strides(inode)--; + } + else + { + /*this node hasn't shown up in the row before; add it to the list of nodes + that have been scanned uniquely. Use this list to reset the flag array + afterwards without having to loop over all the nodes in the system*/ + node_indices_used(current_node) = 1; + column_index(current_node) = istride; + current_row_nodes_scanned(current_row_n_nodes_scanned) = current_node; + current_row_n_nodes_scanned++; + } + } + // reset nodes used list for the next row of the sparse list + for (int node_reset = 0; node_reset < current_row_n_nodes_scanned; node_reset++) + { + node_indices_used(current_row_nodes_scanned(node_reset)) = 0; + } } - else{ - /*this node hasn't shown up in the row before; add it to the list of nodes - that have been scanned uniquely. Use this list to reset the flag array - afterwards without having to loop over all the nodes in the system*/ - node_indices_used(current_node) = 1; - column_index(current_node) = istride; - current_row_nodes_scanned(current_row_n_nodes_scanned) = current_node; - current_row_n_nodes_scanned++; + }); + Kokkos::fence(); + + Graph_Matrix_Strides.update_host(); + // copy reduced content to non_repeat storage + Graph_Matrix = RaggedRightArrayKokkos(Graph_Matrix_Strides); + + FOR_ALL_CLASS(inode, 0, nlocal_nodes, { + for (int istride = 0; istride < Graph_Matrix_Strides(inode); istride++) + { + Graph_Matrix(inode, istride) = Repeat_Graph_Matrix(inode, istride); } - } - //reset nodes used list for the next row of the sparse list - for(int node_reset = 0; node_reset < current_row_n_nodes_scanned; node_reset++) - node_indices_used(current_row_nodes_scanned(node_reset)) = 0; + }); // end parallel for - } - }); - Kokkos::fence(); - - Graph_Matrix_Strides.update_host(); - //copy reduced content to non_repeat storage - Graph_Matrix = RaggedRightArrayKokkos(Graph_Matrix_Strides); - - FOR_ALL_CLASS(inode, 0, nlocal_nodes, { - for(int istride = 0; istride < Graph_Matrix_Strides(inode); istride++){ - Graph_Matrix(inode,istride) = Repeat_Graph_Matrix(inode,istride); - } - }); // end parallel for + // deallocate repeat matrix - //deallocate repeat matrix - - /*At this stage the sparse graph should have unique global indices on each row. - The constructed Assembly map (to the global sparse matrix) - is used to loop over each element's local stiffness matrix in the assembly process.*/ - - //expand strides for stiffness matrix by multipling by dim - FOR_ALL_CLASS(idof, 0, num_dim*nlocal_nodes, { - Gradient_Matrix_Strides(idof) = num_dim*Graph_Matrix_Strides(idof/num_dim); - }); // end parallel for + /*At this stage the sparse graph should have unique global indices on each row. + The constructed Assembly map (to the global sparse matrix) + is used to loop over each element's local stiffness matrix in the assembly process.*/ + + // expand strides for stiffness matrix by multipling by dim + FOR_ALL_CLASS(idof, 0, num_dim * nlocal_nodes, { + Gradient_Matrix_Strides(idof) = num_dim * Graph_Matrix_Strides(idof / num_dim); + }); // end parallel for + + Stiffness_Matrix = Force_Gradient_Positions = RaggedRightArrayKokkos(Gradient_Matrix_Strides); + Force_Gradient_Velocities = RaggedRightArrayKokkos(Gradient_Matrix_Strides); + DOF_Graph_Matrix = RaggedRightArrayKokkos(Gradient_Matrix_Strides); + + // set stiffness Matrix Graph + // debug print + // std::cout << "DOF GRAPH MATRIX ENTRIES ON TASK " << myrank << std::endl; + FOR_ALL_CLASS(idof, 0, num_dim * nlocal_nodes, { + for (int istride = 0; istride < Gradient_Matrix_Strides(idof); istride++) + { + DOF_Graph_Matrix(idof, istride) = Graph_Matrix(idof / num_dim, istride / num_dim) * num_dim + istride % num_dim; + } + }); // end parallel for + + Stiffness_Matrix_Strides = Gradient_Matrix_Strides; + + /* + //construct distributed gradient matrix from local kokkos data + //build column map for the global gradient matrix + Teuchos::RCP > colmap; + const Teuchos::RCP > dommap = local_dof_map; - Stiffness_Matrix = Force_Gradient_Positions = RaggedRightArrayKokkos(Gradient_Matrix_Strides); - Force_Gradient_Velocities = RaggedRightArrayKokkos(Gradient_Matrix_Strides); - DOF_Graph_Matrix = RaggedRightArrayKokkos (Gradient_Matrix_Strides); + Tpetra::Details::makeColMap(colmap,dommap,DOF_Graph_Matrix.get_kokkos_view(), nullptr); + + size_t nnz = DOF_Graph_Matrix.size(); + + //debug print + //std::cout << "DOF GRAPH SIZE ON RANK " << myrank << " IS " << nnz << std::endl; - //set stiffness Matrix Graph - //debug print - //std::cout << "DOF GRAPH MATRIX ENTRIES ON TASK " << myrank << std::endl; - FOR_ALL_CLASS(idof, 0, num_dim*nlocal_nodes, { - for (int istride = 0; istride < Gradient_Matrix_Strides(idof); istride++){ - DOF_Graph_Matrix(idof,istride) = Graph_Matrix(idof/num_dim,istride/num_dim)*num_dim + istride%num_dim; + //local indices in the graph using the constructed column map + CArrayKokkos gradient_local_indices(nnz, "gradient_local_indices"); + + //row offsets with compatible template arguments + Kokkos::View row_offsets = DOF_Graph_Matrix.start_index_; + row_pointers row_offsets_pass("row_offsets", nlocal_nodes*num_dim+1); + for(int ipass = 0; ipass < nlocal_nodes*num_dim + 1; ipass++){ + row_offsets_pass(ipass) = row_offsets(ipass); } - }); // end parallel for - Stiffness_Matrix_Strides = Gradient_Matrix_Strides; - - /* - //construct distributed gradient matrix from local kokkos data - //build column map for the global gradient matrix - Teuchos::RCP > colmap; - const Teuchos::RCP > dommap = local_dof_map; - - Tpetra::Details::makeColMap(colmap,dommap,DOF_Graph_Matrix.get_kokkos_view(), nullptr); - - size_t nnz = DOF_Graph_Matrix.size(); - - //debug print - //std::cout << "DOF GRAPH SIZE ON RANK " << myrank << " IS " << nnz << std::endl; - - //local indices in the graph using the constructed column map - CArrayKokkos gradient_local_indices(nnz, "gradient_local_indices"); - - //row offsets with compatible template arguments - Kokkos::View row_offsets = DOF_Graph_Matrix.start_index_; - row_pointers row_offsets_pass("row_offsets", nlocal_nodes*num_dim+1); - for(int ipass = 0; ipass < nlocal_nodes*num_dim + 1; ipass++){ - row_offsets_pass(ipass) = row_offsets(ipass); - } - - size_t entrycount = 0; - for(int irow = 0; irow < nlocal_nodes*num_dim; irow++){ - for(int istride = 0; istride < Gradient_Matrix_Strides(irow); istride++){ - gradient_local_indices(entrycount) = colmap->getLocalElement(DOF_Graph_Matrix(irow,istride)); - entrycount++; + size_t entrycount = 0; + for(int irow = 0; irow < nlocal_nodes*num_dim; irow++){ + for(int istride = 0; istride < Gradient_Matrix_Strides(irow); istride++){ + gradient_local_indices(entrycount) = colmap->getLocalElement(DOF_Graph_Matrix(irow,istride)); + entrycount++; + } } - } - - - //sort values and indices - Tpetra::Import_Util::sortCrsEntries(row_offsets_pass, gradient_local_indices.get_kokkos_view(), Force_Gradient_Positions.get_kokkos_view()); - Tpetra::Import_Util::sortCrsEntries(row_offsets_pass, gradient_local_indices.get_kokkos_view(), Force_Gradient_Velocities.get_kokkos_view()); - - //Teuchos::RCP crs_matrix_params = Teuchos::rcp(new Teuchos::ParameterList("crsmatrix")); - //crs_matrix_params->set("sorted", false); - distributed_force_gradient_positions = Teuchos::rcp(new MAT(local_dof_map, colmap, row_offsets_pass, gradient_local_indices.get_kokkos_view(), Force_Gradient_Positions.get_kokkos_view())); - distributed_force_gradient_positions->fillComplete(); - distributed_force_gradient_velocities = Teuchos::rcp(new MAT(local_dof_map, colmap, row_offsets_pass, gradient_local_indices.get_kokkos_view(), Force_Gradient_Velocities.get_kokkos_view())); - distributed_force_gradient_velocities->fillComplete(); - */ - //distributed_force_gradient_positions->describe(*fos,Teuchos::VERB_EXTREME); - //distributed_force_gradient_velocities->describe(*fos,Teuchos::VERB_EXTREME); + + + //sort values and indices + Tpetra::Import_Util::sortCrsEntries(row_offsets_pass, gradient_local_indices.get_kokkos_view(), Force_Gradient_Positions.get_kokkos_view()); + Tpetra::Import_Util::sortCrsEntries(row_offsets_pass, gradient_local_indices.get_kokkos_view(), Force_Gradient_Velocities.get_kokkos_view()); + + //Teuchos::RCP crs_matrix_params = Teuchos::rcp(new Teuchos::ParameterList("crsmatrix")); + //crs_matrix_params->set("sorted", false); + distributed_force_gradient_positions = Teuchos::rcp(new MAT(local_dof_map, colmap, row_offsets_pass, gradient_local_indices.get_kokkos_view(), Force_Gradient_Positions.get_kokkos_view())); + distributed_force_gradient_positions->fillComplete(); + distributed_force_gradient_velocities = Teuchos::rcp(new MAT(local_dof_map, colmap, row_offsets_pass, gradient_local_indices.get_kokkos_view(), Force_Gradient_Velocities.get_kokkos_view())); + distributed_force_gradient_velocities->fillComplete(); + */ + // distributed_force_gradient_positions->describe(*fos,Teuchos::VERB_EXTREME); + // distributed_force_gradient_velocities->describe(*fos,Teuchos::VERB_EXTREME); } -/* ---------------------------------------------------------------------- - Enforce boundary conditions on the adjoint vectors -------------------------------------------------------------------------- */ - -void FEA_Module_Dynamic_Elasticity::boundary_adjoint(const mesh_t &mesh, - const DCArrayKokkos &boundary, - vec_array &node_adjoint, - vec_array &node_phi_adjoint){ - - //error and debug flag - //DCArrayKokkos print_flag(1, "print_flag"); - //print_flag.host(0) = false; - //print_flag.update_device(); - - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn boundary_adjoint +/// +/// \brief Enforce boundary conditions on the adjoint vectors +/// +/// \param The simulation mesh +/// \param Array of boundary conditions +/// \param Nodal adjoing +/// \param Nodal adjoing phi value +/// +///////////////////////////////////////////////////////////////////////////// +void FEA_Module_Dynamic_Elasticity::boundary_adjoint(const mesh_t& mesh, + const DCArrayKokkos& boundary, + vec_array& node_adjoint, + vec_array& node_phi_adjoint) +{ + // error and debug flag + // DCArrayKokkos print_flag(1, "print_flag"); + // print_flag.host(0) = false; + // print_flag.update_device(); + + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; int num_dims = simparam->num_dims; // Loop over boundary sets - for (size_t bdy_set=0; bdy_set fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - //if(myrank==0) - //*fos << "Density data :" << std::endl; - //node_densities_distributed->describe(*fos,Teuchos::VERB_EXTREME); - //*fos << std::endl; - //std::fflush(stdout); - - //communicate design densities - //create import object using local node indices map and all indices map - //Tpetra::Import importer(map, all_node_map); - - //comms to get ghosts - (*adjoint_vector_data)[cycle]->doImport(*adjoint_vector_distributed, *importer, Tpetra::INSERT); - (*phi_adjoint_vector_data)[cycle]->doImport(*phi_adjoint_vector_distributed, *importer, Tpetra::INSERT); - //all_node_map->describe(*fos,Teuchos::VERB_EXTREME); - //all_node_velocities_distributed->describe(*fos,Teuchos::VERB_EXTREME); - - //update_count++; - //if(update_count==1){ - //MPI_Barrier(world); - //MPI_Abort(world,4); - //} +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn comm_adjoint_vectors +/// +/// \brief Communicate updated nodal adjoint vectors to ghost nodes +/// +/// \param Optimization cycle +/// +///////////////////////////////////////////////////////////////////////////// +void FEA_Module_Dynamic_Elasticity::comm_adjoint_vectors(int cycle) +{ + // debug print of design vector + // std::ostream &out = std::cout; + // Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); + // if(myrank==0) + // *fos << "Density data :" << std::endl; + // node_densities_distributed->describe(*fos,Teuchos::VERB_EXTREME); + // *fos << std::endl; + // std::fflush(stdout); + + // communicate design densities + // create import object using local node indices map and all indices map + // Tpetra::Import importer(map, all_node_map); + + // comms to get ghosts + (*adjoint_vector_data)[cycle]->doImport(*adjoint_vector_distributed, *importer, Tpetra::INSERT); + (*phi_adjoint_vector_data)[cycle]->doImport(*phi_adjoint_vector_distributed, *importer, Tpetra::INSERT); + // all_node_map->describe(*fos,Teuchos::VERB_EXTREME); + // all_node_velocities_distributed->describe(*fos,Teuchos::VERB_EXTREME); + + // update_count++; + // if(update_count==1){ + // MPI_Barrier(world); + // MPI_Abort(world,4); + // } } diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/force_elastic.cpp b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/force_elastic.cpp index 256efaa35..4757098c3 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/force_elastic.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/force_elastic.cpp @@ -39,10 +39,28 @@ #include "FEA_Module_Dynamic_Elasticity.h" #include "Explicit_Solver.h" -// ----------------------------------------------------------------------------- -// This function calculates the corner forces and the evolves stress (hypo) -// ------------------------------------------------------------------------------ -void FEA_Module_Dynamic_Elasticity::get_force_elastic(const DCArrayKokkos& material, +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn get_force_elastic +/// +/// \brief This function calculates the corner forces and the evolves stress +/// +/// \param An array of material_t that contains material specific data +/// \param The simulation mesh +/// \param A view into the nodal position array +/// \param A view into the nodal velocity array +/// \param A view into the nodal mass array +/// \param A view into the element density array +/// \param A view into the element volume array +/// \param A view into the element divergence of velocity array +/// \param A view into the element element material ID +/// \param A view into the corner force array +/// \param The current Runge Kutta integration alpha value +/// \param The current cycle index +/// +///////////////////////////////////////////////////////////////////////////// +void FEA_Module_Dynamic_Elasticity::get_force_elastic( + const DCArrayKokkos& material, const mesh_t& mesh, const DViewCArrayKokkos& node_coords, const DViewCArrayKokkos& node_vel, @@ -108,10 +126,28 @@ void FEA_Module_Dynamic_Elasticity::get_force_elastic(const DCArrayKokkos& material, +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn applied_forces +/// +/// \brief This function applies force to the nodes as a boundary condition +/// +/// \param An array of material_t that contains material specific data +/// \param The simulation mesh +/// \param A view into the nodal position array +/// \param A view into the nodal velocity array +/// \param A view into the nodal mass array +/// \param A view into the element density array +/// \param A view into the element volume array +/// \param A view into the element divergence of velocity array +/// \param A view into the element element material ID +/// \param A view into the corner force array +/// \param The current Runge Kutta integration alpha value +/// \param The current cycle index +/// +///////////////////////////////////////////////////////////////////////////// +void FEA_Module_Dynamic_Elasticity::applied_forces( + const DCArrayKokkos& material, const mesh_t& mesh, const DViewCArrayKokkos& node_coords, const DViewCArrayKokkos& node_vel, @@ -194,10 +230,13 @@ void FEA_Module_Dynamic_Elasticity::applied_forces(const DCArrayKokkosnum_dims; @@ -307,31 +346,20 @@ void FEA_Module_Dynamic_Elasticity::assemble_matrix() } } } - - // debug print of A matrix - // *fos << "Global Stiffness Matrix :" << std::endl; - // Global_Stiffness_Matrix->describe(*fos,Teuchos::VERB_EXTREME); - // *fos << std::endl; - - // filter small negative numbers (that should have been 0 from cancellation) from floating point error - /* - for (int idof = 0; idof < num_dim*nlocal_nodes; idof++){ - for (int istride = 0; istride < Stiffness_Matrix_Strides(idof); istride++){ - if(Stiffness_Matrix(idof,istride)<0.000000001*simparam->Elastic_Modulus*density_epsilon||Stiffness_Matrix(idof,istride)>-0.000000001*simparam->Elastic_Modulus*density_epsilon) - Stiffness_Matrix(idof,istride) = 0; - //debug print - //std::cout << "{" <get_unit_scaling(); @@ -351,10 +379,18 @@ void FEA_Module_Dynamic_Elasticity::Element_Material_Properties(size_t ielem, re Poisson_Ratio = module_params->material.poisson_ratio; } -/* ---------------------------------------------------------------------- - Retrieve derivative of material properties with respect to local density -------------------------------------------------------------------------- */ - +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn Gradient_Element_Material_Properties +/// +/// \brief Retrieve derivative of material properties with respect to local density +/// +/// \param Element ID +/// \param Element youngs modulus derivative +/// \param Element poisson ration +/// \param Element density +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_Dynamic_Elasticity::Gradient_Element_Material_Properties(size_t ielem, real_t& Element_Modulus_Derivative, real_t& Poisson_Ratio, real_t density) { real_t unit_scaling = simparam->get_unit_scaling(); @@ -375,10 +411,16 @@ void FEA_Module_Dynamic_Elasticity::Gradient_Element_Material_Properties(size_t Poisson_Ratio = module_params->material.poisson_ratio; } -/* ---------------------------------------------------------------------- - Construct the local stiffness matrix -------------------------------------------------------------------------- */ - +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn local_matrix_multiply +/// +/// \brief Construct the local stiffness matrix +/// +/// \param Element ID +/// \param Local matrix +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_Dynamic_Elasticity::local_matrix_multiply(int ielem, CArrayKokkos& Local_Matrix) { // local variable for host view in the dual view @@ -866,10 +908,16 @@ void FEA_Module_Dynamic_Elasticity::local_matrix_multiply(int ielem, CArrayKokko */ } -/* ---------------------------------------------------------------------- - Compute the gradient of strain energy with respect to nodal densities -------------------------------------------------------------------------- */ - +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn compute_stiffness_gradients +/// +/// \brief Compute the gradient of strain energy with respect to nodal densities +/// +/// \param Vector of design variables +/// \param Vector of design gradients +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_Dynamic_Elasticity::compute_stiffness_gradients(const_host_vec_array& design_variables, host_vec_array& design_gradients) { // local variable for host view in the dual view diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/force_gradients_elastic.cpp b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/force_gradients_elastic.cpp index fc33b514f..2b907fe30 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/force_gradients_elastic.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/force_gradients_elastic.cpp @@ -54,10 +54,30 @@ #include "Simulation_Parameters/Simulation_Parameters_Explicit.h" #include "FEA_Module_Dynamic_Elasticity.h" -// ----------------------------------------------------------------------------- -// This function calculates the corner forces and the evolves stress (hypo) -// ------------------------------------------------------------------------------ -void FEA_Module_Dynamic_Elasticity::get_force_vgradient_elastic(const DCArrayKokkos& material, +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn get_force_vgradient_elastic +/// +/// \brief Computes corner contribution of gradient of force with respect to velocity +/// +/// \param An array of material_t that contains material specific data +/// \param The simulation mesh +/// \param A view into the nodal position array +/// \param A view into the nodal velocity array +/// \param A view into the element density array +/// \param A view into the element specific internal energy +/// \param A view into the element pressure +/// \param A view into the element stress +/// \param A view into the element sound speed +/// \param A view into the element volume array +/// \param A view into the element divergence of velocity array +/// \param A view into the element element material ID +/// \param The current Runge Kutta integration alpha value +/// \param The current cycle index +/// +///////////////////////////////////////////////////////////////////////////// +void FEA_Module_Dynamic_Elasticity::get_force_vgradient_elastic( + const DCArrayKokkos& material, const mesh_t& mesh, const DViewCArrayKokkos& node_coords, const DViewCArrayKokkos& node_vel, @@ -429,9 +449,28 @@ void FEA_Module_Dynamic_Elasticity::get_force_vgradient_elastic(const DCArrayKok return; } // end of routine -// ----------------------------------------------------------------------------- -// This function calculates the corner forces and the evolves stress (hypo) -// ------------------------------------------------------------------------------ +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn get_force_ugradient_elastic +/// +/// \brief Computes corner contribution of gradient of force with respect to position +/// +/// \param An array of material_t that contains material specific data +/// \param The simulation mesh +/// \param A view into the nodal position array +/// \param A view into the nodal velocity array +/// \param A view into the element density array +/// \param A view into the element specific internal energy +/// \param A view into the element pressure +/// \param A view into the element stress +/// \param A view into the element sound speed +/// \param A view into the element volume array +/// \param A view into the element divergence of velocity array +/// \param A view into the element element material ID +/// \param The current Runge Kutta integration alpha value +/// \param The current cycle index +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_Dynamic_Elasticity::get_force_ugradient_elastic(const DCArrayKokkos& material, const mesh_t& mesh, const DViewCArrayKokkos& node_coords, @@ -794,10 +833,17 @@ void FEA_Module_Dynamic_Elasticity::get_force_ugradient_elastic(const DCArrayKok return; } // end of routine -// -------------------------------------------------------------------------------------- -// Computes corner contribution of gradient of force with respect to the design variable -// --------------------------------------------------------------------------------------- - +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn force_design_gradient_term +/// +/// \brief Computes corner contribution of gradient of force with respect +/// to the design variable +/// +/// \param Vector of design variables +/// \param Vector of design gradients +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_Dynamic_Elasticity::force_design_gradient_term(const_vec_array design_variables, vec_array design_gradients) { bool element_constant_density = true; @@ -906,7 +952,7 @@ void FEA_Module_Dynamic_Elasticity::force_design_gradient_term(const_vec_array d corner_value_storage(corner_id) = -inner_products(inode) * global_dt; } } - }); // end parallel for + }); // end parallel for Kokkos::fence(); // accumulate node values from corner storage @@ -918,16 +964,37 @@ void FEA_Module_Dynamic_Elasticity::force_design_gradient_term(const_vec_array d corner_id = corners_in_node(node_id, icorner); design_gradients(node_id, 0) += corner_value_storage(corner_id); } - }); // end parallel for + }); // end parallel for Kokkos::fence(); } // end view scope } } -// ----------------------------------------------------------------------------- -// This function calculates the corner forces and the evolves stress (hypo) -// ------------------------------------------------------------------------------ -void FEA_Module_Dynamic_Elasticity::get_force_dgradient_elastic(const DCArrayKokkos& material, +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn get_force_dgradient_elastic +/// +/// \brief Computes corner contribution of gradient of force with respect to +/// the design gradient +/// +/// \param An array of material_t that contains material specific data +/// \param The simulation mesh +/// \param A view into the nodal position array +/// \param A view into the nodal velocity array +/// \param A view into the element density array +/// \param A view into the element specific internal energy +/// \param A view into the element pressure +/// \param A view into the element stress +/// \param A view into the element sound speed +/// \param A view into the element volume array +/// \param A view into the element divergence of velocity array +/// \param A view into the element element material ID +/// \param The current Runge Kutta integration alpha value +/// \param The current cycle index +/// +///////////////////////////////////////////////////////////////////////////// +void FEA_Module_Dynamic_Elasticity::get_force_dgradient_elastic( + const DCArrayKokkos& material, const mesh_t& mesh, const DViewCArrayKokkos& node_coords, const DViewCArrayKokkos& node_vel, diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/geometry.cpp b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/geometry.cpp index a79f8fe56..95ca3639c 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/geometry.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/geometry.cpp @@ -31,14 +31,23 @@ OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. **********************************************************************************************/ - -// ----------------------------------------------------------------------------- -// This code handles the geometric information for the mesh for the SHG solver -// ------------------------------------------------------------------------------ #include "matar.h" #include "state.h" #include "FEA_Module_Dynamic_Elasticity.h" +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn update_position_elastic +/// +/// \brief Updates the nodal potitions for the dynamic elastic solver +/// +/// +/// \param Runge Kutta time integration alpha value +/// \param Number of nodes +/// \param Array of nodal potitions +/// \param Array of nodal velocities +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_Dynamic_Elasticity::update_position_elastic(double rk_alpha, const size_t num_nodes, DViewCArrayKokkos& node_coords, @@ -57,17 +66,26 @@ void FEA_Module_Dynamic_Elasticity::update_position_elastic(double rk_alpha, }); // end parallel for over nodes } // end subroutine -// ----------------------------------------------------------------------------- -// This function claculates -// B_p = J^{-T} \cdot (\nabla_{xi} \phi_p w -// where -// \phi_p is the basis function for vertex p -// w is the 1 gauss point for the cell (everything is evaluted at this point) -// J^{-T} is the inverse transpose of the Jacobi matrix -// \nabla_{xi} is the gradient opperator in the reference coordinates -// -// B_p is the OUTWARD corner area normal at node p -// ------------------------------------------------------------------------------ +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn get_bmatrix +/// +/// \brief Theis function calculate the finite element B matrix: +/// +/// B_p = J^{-T} \cdot (\nabla_{xi} \phi_p w, where: +/// \phi_p is the basis function for vertex p +/// w is the 1 gauss point for the cell (everything is evaluted at this point) +/// J^{-T} is the inverse transpose of the Jacobi matrix +/// \nabla_{xi} is the gradient opperator in the reference coordinates +/// B_p is the OUTWARD corner area normal at node p +/// +/// \param B matrix +/// \param Global index of the element +/// \param View of nodal position data +/// \param View of the elements node ids +/// \param Runge Kutta time integration level +/// +///////////////////////////////////////////////////////////////////////////// KOKKOS_FUNCTION void FEA_Module_Dynamic_Elasticity::get_bmatrix(const ViewCArrayKokkos& B_matrix, const size_t elem_gid, @@ -265,10 +283,13 @@ void FEA_Module_Dynamic_Elasticity::get_bmatrix(const ViewCArrayKokkos& + x(6) * (+y(2) + y(3) - y(4) - y(5) ) ) * twelth; } // end subroutine -/* ---------------------------------------------------------------------------- - Compute Volume of each finite element -------------------------------------------------------------------------------- */ - +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn get_vol +/// +/// \brief Compute Volume of each finite element +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_Dynamic_Elasticity::get_vol() { const size_t rk_level = rk_num_bins - 1; @@ -296,7 +317,19 @@ void FEA_Module_Dynamic_Elasticity::get_vol() return; } // end subroutine -// Exact volume for a hex element +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn get_vol_hex +/// +/// \brief Exact volume for a hex element +/// +/// \param View of element volume data +/// \param Global element index +/// \param View into nodal position data +/// \param Runge Kutta time integration level +/// WARNING: How does this work with const elem_vol? +/// +///////////////////////////////////////////////////////////////////////////// KOKKOS_INLINE_FUNCTION void FEA_Module_Dynamic_Elasticity::get_vol_hex(const DViewCArrayKokkos& elem_vol, const size_t elem_gid, @@ -341,6 +374,22 @@ void FEA_Module_Dynamic_Elasticity::get_vol_hex(const DViewCArrayKokkos& return; } // end subroutine +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn get_bmatrix2D +/// +/// \brief Calculate the 2D finite element B matrix +/// +/// +/// +/// \param B Matrix +/// \param Global index of the element +/// \param Nodal coordinates +/// \param Global indices of the nodes of this element +/// \param Runge Kutta time integration step +/// +///////////////////////////////////////////////////////////////////////////// KOKKOS_FUNCTION void FEA_Module_Dynamic_Elasticity::get_bmatrix2D(const ViewCArrayKokkos& B_matrix, const size_t elem_gid, @@ -398,7 +447,19 @@ void FEA_Module_Dynamic_Elasticity::get_bmatrix2D(const ViewCArrayKokkos return; } // end subroutine -// true volume of a quad in RZ coords +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn get_vol_quad +/// +/// \brief True volume of a quad in RZ coords +/// +/// \param Element volume +/// \param Global index of the element +/// \param Nodal coordinates +/// \param Global ids of the nodes in this element +/// \param Runge Kutta time integration level +/// +///////////////////////////////////////////////////////////////////////////// KOKKOS_INLINE_FUNCTION void FEA_Module_Dynamic_Elasticity::get_vol_quad(const DViewCArrayKokkos& elem_vol, const size_t elem_gid, @@ -457,7 +518,20 @@ void FEA_Module_Dynamic_Elasticity::get_vol_quad(const DViewCArrayKokkos return; } // end subroutine -// element facial area +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn get_area_quad +/// +/// \brief Calculate the area of a elements face +/// +/// \param Global index of the element +/// \param Nodal coordinates +/// \param Global ids of the nodes in this element +/// \param Runge Kutta time integration level +/// +/// \return Elements face area (double) +/// +///////////////////////////////////////////////////////////////////////////// KOKKOS_FUNCTION double FEA_Module_Dynamic_Elasticity::get_area_quad(const size_t elem_gid, const DViewCArrayKokkos& node_coords, @@ -492,6 +566,23 @@ double FEA_Module_Dynamic_Elasticity::get_area_quad(const size_t elem_gid, return elem_area; } // end subroutine +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn heron +/// +/// \brief Calculate the area of a triangle using the heron algorithm +/// +/// +/// \param Node 1 X coordinate +/// \param Node 1 Y coordinate +/// \param Node 2 X coordinate +/// \param Node 2 Y coordinate +/// \param Node 3 X coordinate +/// \param Node 3 Y coordinate +/// +/// \return Triangle area +/// +///////////////////////////////////////////////////////////////////////////// KOKKOS_INLINE_FUNCTION double FEA_Module_Dynamic_Elasticity::heron(const double x1, const double y1, @@ -516,6 +607,13 @@ double FEA_Module_Dynamic_Elasticity::heron(const double x1, return area; } +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn get_area_weights2D +/// +/// \brief Calculate the corner weighted area +/// +///////////////////////////////////////////////////////////////////////////// KOKKOS_FUNCTION void FEA_Module_Dynamic_Elasticity::get_area_weights2D(const ViewCArrayKokkos& corner_areas, const size_t elem_gid, diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/mesh.h b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/mesh.h index 4fac52d37..94aa204b7 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/mesh.h +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/mesh.h @@ -167,7 +167,15 @@ struct mesh_t RaggedRightArrayKokkos bdy_nodes_in_set; DCArrayKokkos num_bdy_nodes_in_set; - // initialization methods + ///////////////////////////////////////////////////////////////////////////// + /// + /// \fn initialize_nodes + /// + /// \brief Initialize the number of nodes in the mesh + /// + /// \param The number of nodes in the mesh + /// + ///////////////////////////////////////////////////////////////////////////// void initialize_nodes(const size_t num_nodes_inp) { num_nodes = num_nodes_inp; @@ -175,7 +183,16 @@ struct mesh_t return; }; // end method - // initialization methods + ///////////////////////////////////////////////////////////////////////////// + /// + /// \fn initialize_local_nodes + /// + /// \brief Initialize the number of nodes in the mesh owned by a particular + // MPI rank + /// + /// \param The number of nodes in the mesh on this MPI rank + /// + ///////////////////////////////////////////////////////////////////////////// void initialize_local_nodes(const size_t num_nodes_inp) { num_local_nodes = num_nodes_inp; @@ -183,7 +200,20 @@ struct mesh_t return; }; // end method - // initialization methods + ///////////////////////////////////////////////////////////////////////////// + /// + /// \fn initialize_elems + /// + /// \brief Initialize the elements and some associated connectivity memory + /// + /// Saves the number of elements and dimensions, as well as creates + /// storage for the data structures for nodes-element connectivity and + /// corner-element connectivity + /// + /// \param Number of elements in the mesh + /// \param The number of spacial dimensions the mesh containts (3D, 2D) + /// + ///////////////////////////////////////////////////////////////////////////// void initialize_elems(const size_t num_elems_inp, const size_t num_dims_inp) { num_dims = num_dims_inp; @@ -199,7 +229,15 @@ struct mesh_t return; }; // end method - // initialization methods + ///////////////////////////////////////////////////////////////////////////// + /// + /// \fn initialize_corners + /// + /// \brief Initialize num_corners data + /// + /// \param Number of corners read in + /// + ///////////////////////////////////////////////////////////////////////////// void initialize_corners(const size_t num_corners_inp) { num_corners = num_corners_inp; @@ -207,7 +245,16 @@ struct mesh_t return; }; // end method - // build the corner mesh connectivity arrays + ///////////////////////////////////////////////////////////////////////////// + /// + /// \fn build_corner_connectivity + /// + /// \brief This class builds the corners-node, element-node, and + /// corner-element data structures. + /// + /// A corner is defined as a cell-node pair. + /// + ///////////////////////////////////////////////////////////////////////////// void build_corner_connectivity() { num_corners_in_node = CArrayKokkos(num_nodes); // stride sizes @@ -269,7 +316,13 @@ struct mesh_t return; } // end of build_corner_connectivity - // build elem connectivity arrays + ///////////////////////////////////////////////////////////////////////////// + /// + /// \fn build_elem_elem_connectivity + /// + /// \brief This class builds the element-element connectivity structure + /// + ///////////////////////////////////////////////////////////////////////////// void build_elem_elem_connectivity() { // find the max number of elems around a node @@ -357,7 +410,17 @@ struct mesh_t return; } // end of build_elem_elem_connectivity - // build the patches + ///////////////////////////////////////////////////////////////////////////// + /// + /// \fn build_patch_connectivity + /// + /// \brief Builds the patch-element, node-patch, and element-patch as well as + /// tags all boundary nodes. + /// + /// A patch is defined a the surface of a linear element, for high order elements + /// a patch is the surface of a zone that is on the surface of an element. + /// + ///////////////////////////////////////////////////////////////////////////// void build_patch_connectivity() { // building patches @@ -642,7 +705,13 @@ struct mesh_t return; } // end patch connectivity method - // build the patches + ///////////////////////////////////////////////////////////////////////////// + /// + /// \fn build_node_node_connectivity + /// + /// \brief Builds the nodes-node connectivity structure + /// + ///////////////////////////////////////////////////////////////////////////// void build_node_node_connectivity() { // find the max number of elems around a node @@ -782,6 +851,15 @@ struct mesh_t }); // end parallel for over nodes } // end of node node connectivity + ///////////////////////////////////////////////////////////////////////////// + /// + /// \fn init_bdy_sets + /// + /// \brief Initialize boundary sets as dynamic ragged right arrays + /// + /// \param Number of boundary conditions + /// + ///////////////////////////////////////////////////////////////////////////// void init_bdy_sets(size_t num_bcs) { if (num_bcs == 0) diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/momentum.cpp b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/momentum.cpp index dff8ef928..771f265a3 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/momentum.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/momentum.cpp @@ -35,57 +35,21 @@ #include "state.h" #include "FEA_Module_Dynamic_Elasticity.h" -// ----------------------------------------------------------------------------- -// This function evolves the velocity at the nodes of the mesh -// ------------------------------------------------------------------------------ -/* -void FEA_Module_Dynamic_Elasticity::update_velocity_elastic(double rk_alpha, - const mesh_t &mesh, - DViewCArrayKokkos &node_vel, - const DViewCArrayKokkos &node_mass, - const DViewCArrayKokkos &corner_force - ){ - - const size_t rk_level = simparam->rk_num_bins - 1; - const size_t num_dims = mesh.num_dims; - - // walk over the nodes to update the velocity - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - - double node_force[3]; - for (size_t dim = 0; dim < num_dims; dim++){ - node_force[dim] = 0.0; - } // end for dim - - // loop over all corners around the node and calculate the nodal force - for (size_t corner_lid=0; corner_lid& vel_grad, const ViewCArrayKokkos& elem_node_gids, @@ -170,9 +134,22 @@ void FEA_Module_Dynamic_Elasticity::get_velgrad(ViewCArrayKokkos& vel_gr return; } // end function -// ----------------------------------------------------------------------------- -// This function calculates the velocity gradient -// ------------------------------------------------------------------------------ +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn get_velgrad2D +/// +/// \brief This function calculates the velocity gradient for a 2D element +/// +/// \param Velocity gradient +/// \param Global ids of the nodes in this element +/// \param View of the nodal velocity data +/// \param The finite element B matrix +/// \param The volume of the particular element +/// \param The elements surface area +/// \param The global id of this particular element +/// \param The Runge Kutta time integration level +/// +///////////////////////////////////////////////////////////////////////////// KOKKOS_FUNCTION void FEA_Module_Dynamic_Elasticity::get_velgrad2D(ViewCArrayKokkos& vel_grad, const ViewCArrayKokkos& elem_node_gids, @@ -236,9 +213,18 @@ void FEA_Module_Dynamic_Elasticity::get_velgrad2D(ViewCArrayKokkos& vel_ return; } // end function -// ----------------------------------------------------------------------------- -// This subroutine to calculate the velocity divergence in all elements -// ------------------------------------------------------------------------------ +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn get_divergence +/// +/// \brief This function calculates the divergence of velocity for all elements +/// +/// \param Divergence of velocity for all elements +/// \param iew of the nodal position data +/// \param View of the nodal velocity data +/// \param View of the volumes of each element +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_Dynamic_Elasticity::get_divergence(DViewCArrayKokkos& elem_div, const mesh_t mesh, const DViewCArrayKokkos& node_coords, @@ -311,9 +297,18 @@ void FEA_Module_Dynamic_Elasticity::get_divergence(DViewCArrayKokkos& el return; } // end subroutine -// ----------------------------------------------------------------------------- -// This subroutine to calculate the velocity divergence in all elements -// ------------------------------------------------------------------------------ +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn get_divergence +/// +/// \brief This function calculates the divergence of velocity for all 2D elements +/// +/// \param Divergence of velocity for all elements +/// \param iew of the nodal position data +/// \param View of the nodal velocity data +/// \param View of the volumes of each element +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_Dynamic_Elasticity::get_divergence2D(DViewCArrayKokkos& elem_div, const mesh_t mesh, const DViewCArrayKokkos& node_coords, @@ -391,10 +386,25 @@ void FEA_Module_Dynamic_Elasticity::get_divergence2D(DViewCArrayKokkos& return; } // end subroutine -// The velocity gradient can be decomposed into symmetric and antisymmetric tensors -// L = vel_grad -// D = sym(L) -// W = antisym(L) +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn decompose_vel_grad +/// +/// \brief Decomposes the velocity gradient into symmetric and antisymmetric tensors +/// +/// L = D*W, where L = vel_grad, D = sym(L), W = antisym(L) +/// can span multiple lines if needed> +/// +/// \param Symmetric decomposition of velocity gradient +/// \param Antisymmetric decomposition of velocity gradient +/// \param Gradient of velocity +/// \param Global ids of the nodes associated with this element +/// \param Global id of a specific element +/// \param View of the nodal coordinate data +/// \param View of the nodal velocity data +/// \param Volume of the element +/// +///////////////////////////////////////////////////////////////////////////// KOKKOS_INLINE_FUNCTION void FEA_Module_Dynamic_Elasticity::decompose_vel_grad(ViewCArrayKokkos& D_tensor, ViewCArrayKokkos& W_tensor, diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/properties.cpp b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/properties.cpp index f0c6f14ff..2f3d16377 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/properties.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/properties.cpp @@ -31,13 +31,32 @@ OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. **********************************************************************************************/ -// ----------------------------------------------------------------------------- -// This calls the models to update state -// ------------------------------------------------------------------------------ #include "state.h" #include "mesh.h" #include "FEA_Module_Dynamic_Elasticity.h" +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn update_state +/// +/// \brief This calls the models to update state +/// +/// \param An array of material_t that contains material specific data +/// \param The simulation mesh +/// \param A view into the nodal position array +/// \param A view into the nodal velocity array +/// \param A view into the element density array +/// \param A view into the element specific internal energy array +/// \param A view into the element pressure array +/// \param A view into the element stress array +/// \param A view into the element sound speed array +/// \param A view into the element volume array +/// \param A view into the element divergence of velocity array +/// \param A view into the element material identifier array +/// \param The current Runge Kutta integration alpha value +/// \param The current cycle index +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_Dynamic_Elasticity::update_state(const DCArrayKokkos& material, const mesh_t& mesh, const DViewCArrayKokkos& node_coords, @@ -174,6 +193,28 @@ void FEA_Module_Dynamic_Elasticity::update_state(const DCArrayKokkos return; } // end method to update state +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn update_state2D +/// +/// \brief Updates the state for 2D elements +/// +/// \param An array of material_t that contains material specific data +/// \param The simulation mesh +/// \param A view into the nodal position array +/// \param A view into the nodal velocity array +/// \param A view into the element density array +/// \param A view into the element specific internal energy array +/// \param A view into the element pressure array +/// \param A view into the element stress array +/// \param A view into the element sound speed array +/// \param A view into the element volume array +/// \param A view into the element divergence of velocity array +/// \param A view into the element material identifier array +/// \param The current Runge Kutta integration alpha value +/// \param The current cycle index +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_Dynamic_Elasticity::update_state2D(const DCArrayKokkos& material, const mesh_t& mesh, const DViewCArrayKokkos& node_coords, diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/time_integration.cpp b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/time_integration.cpp index e77705940..15bc24f49 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/time_integration.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/time_integration.cpp @@ -35,9 +35,20 @@ #include "state.h" #include "FEA_Module_Dynamic_Elasticity.h" -// ----------------------------------------------------------------------------- -// This function saves the variables at rk_stage = 0, which is t_n -// ------------------------------------------------------------------------------ +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn rk_init +/// +/// \brief This function saves the variables at rk_stage = 0, which is t_n +/// +/// \param View of nodal position data +/// \param View of nodal velocity data +/// \param View of element specific internal energy data +/// \param View of element stress +/// \param Number of elements +/// \param Number of nodes +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_Dynamic_Elasticity::rk_init(DViewCArrayKokkos& node_coords, DViewCArrayKokkos& node_vel, DViewCArrayKokkos& elem_sie, @@ -74,10 +85,22 @@ void FEA_Module_Dynamic_Elasticity::rk_init(DViewCArrayKokkos& node_coor return; } // end rk_init -// ----------------------------------------------------------------------------- -// This function calculates the time step by finding the shortest distance -// between any two nodes in the mesh -// ------------------------------------------------------------------------------ +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn get_timestep +/// +/// \brief This function calculates the time step by finding the shortest distance +/// between any two nodes in the mesh. +/// +/// WARNING WARNING : Only works for 3D, 8 node elements +/// +/// \param Simulation mesh +/// \param View of nodal position data +/// \param View of nodal velocity data +/// \param View of element sound speed +/// \param View of element volume +/// +///////////////////////////////////////////////////////////////////////////// // WARNING WARNING : Only works for 3D, 8 node elements void FEA_Module_Dynamic_Elasticity::get_timestep(mesh_t& mesh, DViewCArrayKokkos& node_coords, @@ -175,11 +198,22 @@ void FEA_Module_Dynamic_Elasticity::get_timestep(mesh_t& mesh, return; } // end get_timestep -// ----------------------------------------------------------------------------- -// This function calculates the time step by finding the shortest distance -// between any two nodes in the mesh -// ------------------------------------------------------------------------------ -// WARNING WARNING : Only works for 3D, 8 node elements +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn get_timestep2D +/// +/// \brief This function calculates the time step by finding the shortest distance +/// between any two nodes in the mesh. +/// +/// WARNING WARNING : Only works for 2D, 4 node elements +/// +/// \param Simulation mesh +/// \param View of nodal position data +/// \param View of nodal velocity data +/// \param View of element sound speed +/// \param View of element volume +/// +///////////////////////////////////////////////////////////////////////////// void FEA_Module_Dynamic_Elasticity::get_timestep2D(mesh_t& mesh, DViewCArrayKokkos& node_coords, DViewCArrayKokkos& node_vel, diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/momentum.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/momentum.cpp index e0f3d4220..a37caca93 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/momentum.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/momentum.cpp @@ -101,8 +101,6 @@ void FEA_Module_SGH::update_velocity_sgh(double rk_alpha, /// \param The global id of this particular element /// \param The Runge Kutta time integration level /// -/// \return -/// ///////////////////////////////////////////////////////////////////////////// KOKKOS_FUNCTION void FEA_Module_SGH::get_velgrad(ViewCArrayKokkos& vel_grad, @@ -190,7 +188,7 @@ void FEA_Module_SGH::get_velgrad(ViewCArrayKokkos& vel_grad, ///////////////////////////////////////////////////////////////////////////// /// -/// \fn get_velgrad +/// \fn get_velgrad2D /// /// \brief This function calculates the velocity gradient for a 2D element /// @@ -203,8 +201,6 @@ void FEA_Module_SGH::get_velgrad(ViewCArrayKokkos& vel_grad, /// \param The global id of this particular element /// \param The Runge Kutta time integration level /// -/// \return -/// ///////////////////////////////////////////////////////////////////////////// KOKKOS_FUNCTION void FEA_Module_SGH::get_velgrad2D(ViewCArrayKokkos& vel_grad, From 60a1775461d71c296588a216c60a08077a01788d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CAdrian-Diaz=E2=80=9D?= <“adriandiaz1117@gmail.com”> Date: Mon, 26 Feb 2024 13:31:57 -0700 Subject: [PATCH 37/64] BUG: FIxing data initialization between trials --- .../FEA_Module_Dynamic_Elasticity.cpp | 2 -- .../FEA_Module_Dynamic_Elasticity.h | 2 -- .../Parallel-Explicit/Explicit_Solver.cpp | 12 ++++----- .../Parallel-Explicit/Explicit_Solver.h | 2 -- .../SGH_Solver/FEA_Module_SGH.cpp | 25 +++++++++++++++---- .../SGH_Solver/FEA_Module_SGH.h | 2 -- .../Kinetic_Energy_Minimize.h | 24 +++++++++++------- .../Simulation_Parameters/Region.h | 10 +++++--- 8 files changed, 48 insertions(+), 31 deletions(-) diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.cpp b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.cpp index e09c67741..d852d2dca 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.cpp @@ -102,8 +102,6 @@ FEA_Module_Dynamic_Elasticity::FEA_Module_Dynamic_Elasticity( //set Tpetra vector pointers initial_node_velocities_distributed = Explicit_Solver_Pointer_->initial_node_velocities_distributed; - initial_node_coords_distributed = Explicit_Solver_Pointer_->initial_node_coords_distributed; - all_initial_node_coords_distributed = Explicit_Solver_Pointer_->all_initial_node_coords_distributed; node_coords_distributed = Explicit_Solver_Pointer_->node_coords_distributed; node_velocities_distributed = Explicit_Solver_Pointer_->node_velocities_distributed; all_node_velocities_distributed = Explicit_Solver_Pointer_->all_node_velocities_distributed; diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.h b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.h index fc8ac9d3a..30ec01296 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.h +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.h @@ -494,8 +494,6 @@ class FEA_Module_Dynamic_Elasticity: public FEA_Module { //Global FEA data Teuchos::RCP node_velocities_distributed; - Teuchos::RCP initial_node_coords_distributed; - Teuchos::RCP all_initial_node_coords_distributed; Teuchos::RCP initial_node_velocities_distributed; Teuchos::RCP all_node_velocities_distributed; Teuchos::RCP all_cached_node_velocities_distributed; diff --git a/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp b/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp index 056c13c34..cd0548c97 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp @@ -189,8 +189,6 @@ void Explicit_Solver::run() { //set initial saved coordinates //initial_node_coords_distributed->assign(*node_coords_distributed); - all_initial_node_coords_distributed->assign(*all_node_coords_distributed); - initial_node_coords_distributed = Teuchos::rcp(new MV(*all_initial_node_coords_distributed, map)); std::cout << "Num elements on process " << myrank << " = " << rnum_elem << std::endl; @@ -1038,8 +1036,9 @@ void Explicit_Solver::init_state_vectors(){ int num_dim = simparam.num_dims; //allocate node_velocities //node_velocities_distributed = Teuchos::rcp(new MV(map, num_dim)); - initial_node_coords_distributed = Teuchos::rcp(new MV(map, num_dim)); all_initial_node_coords_distributed = Teuchos::rcp(new MV(all_node_map, num_dim)); + initial_node_coords_distributed = Teuchos::rcp(new MV(*all_initial_node_coords_distributed, map)); + all_initial_node_coords_distributed->assign(*all_node_coords_distributed); initial_node_velocities_distributed = Teuchos::rcp(new MV(map, num_dim)); all_node_velocities_distributed = Teuchos::rcp(new MV(all_node_map, num_dim)); node_velocities_distributed = Teuchos::rcp(new MV(*all_node_velocities_distributed, map)); @@ -1443,7 +1442,7 @@ void Explicit_Solver::setup_optimization_problem(){ //directions(4,0) = -0.3; ROL::Ptr> rol_d = ROL::makePtr>(directions_distributed); - obj->checkGradient(*rol_x, *rol_d); + //obj->checkGradient(*rol_x, *rol_d); //obj->checkHessVec(*rol_x, *rol_d); //directions_distributed->putScalar(-0.000001); //obj->checkGradient(*rol_x, *rol_d); @@ -1456,7 +1455,7 @@ void Explicit_Solver::setup_optimization_problem(){ // Solve optimization problem. //std::ostream outStream; - //solver.solve(*fos); + solver.solve(*fos); //print final constraint satisfaction //fea_elasticity->compute_element_masses(design_densities,false); @@ -3310,7 +3309,8 @@ void Explicit_Solver::init_design(){ //communicate ghost information to the all vector //create import object using local node indices map and all indices map Tpetra::Import importer(map, all_node_map); - + + //design_node_densities_distributed->randomize(0.1,1); //comms to get ghosts all_node_densities_distributed->doImport(*design_node_densities_distributed, importer, Tpetra::INSERT); diff --git a/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.h b/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.h index 6a78e7a2e..4957d7e2e 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.h +++ b/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.h @@ -176,8 +176,6 @@ class Explicit_Solver: public Solver { //debug flags int gradient_print_sync; - - Teuchos::RCP initial_node_coords_distributed; }; #endif // end Header Guard diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/FEA_Module_SGH.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/FEA_Module_SGH.cpp index 648cfb3d1..1a42717d4 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/FEA_Module_SGH.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/FEA_Module_SGH.cpp @@ -107,9 +107,6 @@ FEA_Module_SGH::FEA_Module_SGH( //set Tpetra vector pointers initial_node_velocities_distributed = Explicit_Solver_Pointer_->initial_node_velocities_distributed; - initial_node_coords_distributed = Explicit_Solver_Pointer_->initial_node_coords_distributed; - all_initial_node_coords_distributed = Explicit_Solver_Pointer_->all_initial_node_coords_distributed; - node_coords_distributed = Explicit_Solver_Pointer_->node_coords_distributed; node_velocities_distributed = Explicit_Solver_Pointer_->node_velocities_distributed; all_node_velocities_distributed = Explicit_Solver_Pointer_->all_node_velocities_distributed; if(simparam->topology_optimization_on||simparam->shape_optimization_on){ @@ -1117,12 +1114,30 @@ void FEA_Module_SGH::setup(){ //initialize if topology optimization is used if(simparam->topology_optimization_on){ + //compute element averaged density ratios corresponding to nodal density design variables + CArray current_element_nodal_densities = CArray(num_nodes_in_elem); + {//view scope + const_host_vec_array all_node_densities = all_node_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); + //debug print + //std::cout << "NODE DENSITY TEST " << all_node_densities(0,0) << std::endl; + for(int elem_id = 0; elem_id < rnum_elem; elem_id++){ + for(int inode = 0; inode < num_nodes_in_elem; inode++){ + current_element_nodal_densities(inode) = all_node_densities(nodes_in_elem(elem_id,inode),0); + } + relative_element_densities.host(elem_id) = average_element_density(num_nodes_in_elem, current_element_nodal_densities); + }//for + } //view scope + //debug print + //std::cout << "ELEMENT RELATIVE DENSITY TEST " << relative_element_densities.host(0) << std::endl; + } + else{ for(int elem_id = 0; elem_id < rnum_elem; elem_id++){ relative_element_densities.host(elem_id) = 1; }//for - relative_element_densities.update_device(); } + relative_element_densities.update_device(); + // loop over the fill instructures for (int f_id = 0; f_id < num_fills; f_id++){ @@ -1158,7 +1173,7 @@ void FEA_Module_SGH::setup(){ if (fill_this){ // density - elem_den(elem_gid) = mat_fill(f_id).den; + elem_den(elem_gid) = mat_fill(f_id).den*relative_element_densities(elem_gid); // mass elem_mass(elem_gid) = elem_den(elem_gid)*elem_vol(elem_gid); diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/FEA_Module_SGH.h b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/FEA_Module_SGH.h index 50ea65e52..725e3fcd1 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/FEA_Module_SGH.h +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/FEA_Module_SGH.h @@ -580,8 +580,6 @@ class FEA_Module_SGH: public FEA_Module{ //Global FEA data Teuchos::RCP node_velocities_distributed; - Teuchos::RCP initial_node_coords_distributed; - Teuchos::RCP all_initial_node_coords_distributed; Teuchos::RCP initial_node_velocities_distributed; Teuchos::RCP all_node_velocities_distributed; Teuchos::RCP all_cached_node_velocities_distributed; diff --git a/src/Parallel-Solvers/Parallel-Explicit/Topology_Optimization/Kinetic_Energy_Minimize.h b/src/Parallel-Solvers/Parallel-Explicit/Topology_Optimization/Kinetic_Energy_Minimize.h index ff255f1f0..86ac74cb6 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Topology_Optimization/Kinetic_Energy_Minimize.h +++ b/src/Parallel-Solvers/Parallel-Explicit/Topology_Optimization/Kinetic_Energy_Minimize.h @@ -176,7 +176,9 @@ class KineticEnergyMinimize_TopOpt : public ROL::Objective { if (type == ROL::UpdateType::Initial) { // This is the first call to update //first linear solve was done in FEA class run function already - FEM_Dynamic_Elasticity_->elastic_solve(); + FEM_Dynamic_Elasticity_->comm_variables(zp); + //update deformation variables + FEM_Dynamic_Elasticity_->update_forward_solve(zp); //initial design density data was already communicated for ghost nodes in init_design() //decide to output current optimization state FEM_Dynamic_Elasticity_->Explicit_Solver_Pointer_->write_outputs(); @@ -230,10 +232,14 @@ class KineticEnergyMinimize_TopOpt : public ROL::Objective { if (type == ROL::UpdateType::Initial) { // This is the first call to update //first linear solve was done in FEA class run function already - FEM_SGH_->sgh_solve(); + if(Explicit_Solver_Pointer_->myrank==0) + *fos << "called SGH Initial" << std::endl; + + FEM_SGH_->comm_variables(zp); + FEM_SGH_->update_forward_solve(zp); //initial design density data was already communicated for ghost nodes in init_design() //decide to output current optimization state - FEM_SGH_->Explicit_Solver_Pointer_->write_outputs(); + //FEM_SGH_->Explicit_Solver_Pointer_->write_outputs(); } else if (type == ROL::UpdateType::Accept) { @@ -244,29 +250,29 @@ class KineticEnergyMinimize_TopOpt : public ROL::Objective { // Revert to cached value // This is a new value of x //communicate density variables for ghosts + if(Explicit_Solver_Pointer_->myrank==0) + *fos << "called SGH Revert" << std::endl; FEM_SGH_->comm_variables(zp); //update deformation variables FEM_SGH_->update_forward_solve(zp); - if(Explicit_Solver_Pointer_->myrank==0) - *fos << "called Revert" << std::endl; } else if (type == ROL::UpdateType::Trial) { // This is a new value of x //communicate density variables for ghosts + if(Explicit_Solver_Pointer_->myrank==0) + *fos << "called SGH Trial" << std::endl; FEM_SGH_->comm_variables(zp); //update deformation variables FEM_SGH_->update_forward_solve(zp); - if(Explicit_Solver_Pointer_->myrank==0) - *fos << "called Trial" << std::endl; //decide to output current optimization state - FEM_SGH_->Explicit_Solver_Pointer_->write_outputs(); + //FEM_SGH_->Explicit_Solver_Pointer_->write_outputs(); } else { // ROL::UpdateType::Temp // This is a new value of x used for, // e.g., finite-difference checks if(Explicit_Solver_Pointer_->myrank==0) - *fos << "called Temp" << std::endl; + *fos << "called SGH Temp" << std::endl; FEM_SGH_->comm_variables(zp); FEM_SGH_->update_forward_solve(zp); } diff --git a/src/Parallel-Solvers/Simulation_Parameters/Region.h b/src/Parallel-Solvers/Simulation_Parameters/Region.h index b7674ab07..8322c098c 100644 --- a/src/Parallel-Solvers/Simulation_Parameters/Region.h +++ b/src/Parallel-Solvers/Simulation_Parameters/Region.h @@ -25,6 +25,7 @@ struct mat_fill_t { double speed; double sie; double den; + bool extensive_energy_setting = false; }; struct Region : Yaml::DerivedFields, Yaml::ValidatedYaml, mat_fill_t { @@ -43,15 +44,18 @@ struct Region : Yaml::DerivedFields, Yaml::ValidatedYaml, mat_fill_t { } void derive() { - if (sie.has_value() == ie.has_value()) + if (sie.has_value() == ie.has_value()){ throw Yaml::ConfigurationException("Specify values for exactly one of: energy (ie) or specific energy (sie)."); - + } + mat_fill_t::u = u.value_or(0); mat_fill_t::v = v.value_or(0); mat_fill_t::w = w.value_or(0); - if (ie.has_value()) + if (ie.has_value()){ + mat_fill_t::extensive_energy_setting = true; sie = ie.value() / (den * volume.get_volume()); + } mat_fill_t::sie = sie.value_or(0); // Remove ie now that we have set sie. From 37d88065bd6852ffd59d96abef6140b6b5afd5db Mon Sep 17 00:00:00 2001 From: Adrian-Diaz <46537160+Adrian-Diaz@users.noreply.github.com> Date: Mon, 26 Feb 2024 15:55:32 -0700 Subject: [PATCH 38/64] correct pointers --- src/Parallel-Solvers/FEA_Module.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Parallel-Solvers/FEA_Module.cpp b/src/Parallel-Solvers/FEA_Module.cpp index 6544cf279..a1f50e1ef 100644 --- a/src/Parallel-Solvers/FEA_Module.cpp +++ b/src/Parallel-Solvers/FEA_Module.cpp @@ -99,8 +99,8 @@ FEA_Module::FEA_Module(Solver *Solver_Pointer) { node_nconn_distributed = Solver_Pointer->node_nconn_distributed; //how many elements a node is connected to node_coords_distributed = Solver_Pointer->node_coords_distributed; all_node_coords_distributed = Solver_Pointer->all_node_coords_distributed; - initial_node_coords_distributed = Solver_Pointer->node_coords_distributed; - all_initial_node_coords_distributed = Solver_Pointer->all_node_coords_distributed; + initial_node_coords_distributed = Solver_Pointer->initial_node_coords_distributed; + all_initial_node_coords_distributed = Solver_Pointer->all_initial_node_coords_distributed; design_node_densities_distributed = Solver_Pointer->design_node_densities_distributed; filtered_node_densities_distributed = Solver_Pointer->filtered_node_densities_distributed; test_node_densities_distributed = Solver_Pointer->test_node_densities_distributed; From 8b8e5ca04c4eede887c9753d7e2c0ea948ccefe4 Mon Sep 17 00:00:00 2001 From: Jacob Moore Date: Tue, 27 Feb 2024 08:34:54 -0600 Subject: [PATCH 39/64] BUG: Fix nodal positions in vtu outputs --- src/Parallel-Solvers/Parallel-Explicit/outputs.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Parallel-Solvers/Parallel-Explicit/outputs.cpp b/src/Parallel-Solvers/Parallel-Explicit/outputs.cpp index 2fb43dc98..38a014815 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/outputs.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/outputs.cpp @@ -176,6 +176,8 @@ Explicit_Solver::parallel_vtu_writer_new() int num_nodes_in_elem = 8; int num_points = nnonoverlap_elem_nodes; int num_cells = nlocal_elem_non_overlapping; + + all_node_coords_distributed->doImport(*node_coords_distributed, *importer, Tpetra::INSERT); host_vec_array node_coords = all_node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); CArray nodes_in_elem (rnum_elem, max_nodes_per_element); From 6b77bb8376cf999576bd352b31ecc685bbb183f0 Mon Sep 17 00:00:00 2001 From: Adrian-Diaz <46537160+Adrian-Diaz@users.noreply.github.com> Date: Tue, 27 Feb 2024 12:03:21 -0700 Subject: [PATCH 40/64] BUG: capture initial condition variation correctly --- .../SGH_Solver/FEA_Module_SGH.cpp | 22 ++++++++++++++++--- .../SGH_Solver/FEA_Module_SGH.h | 5 ++++- .../SGH_Solver/sgh_optimization.cpp | 10 ++++++++- 3 files changed, 32 insertions(+), 5 deletions(-) diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/FEA_Module_SGH.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/FEA_Module_SGH.cpp index 1a42717d4..58ecff8a2 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/FEA_Module_SGH.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/FEA_Module_SGH.cpp @@ -135,7 +135,6 @@ FEA_Module_SGH::FEA_Module_SGH( //optimization flags kinetic_energy_objective = false; - //set parameters Dynamic_Options dynamic_options = simparam->dynamic_options; time_value = dynamic_options.time_value; @@ -1000,7 +999,12 @@ void FEA_Module_SGH::setup(){ // allocate material models elem_eos = DCArrayKokkos (num_elems); - elem_strength = DCArrayKokkos (num_elems); + elem_strength = DCArrayKokkos (num_elems); + + //optimization flags + if(simparam->topology_optimization_on){ + elem_extensive_initial_energy_condition = DCArrayKokkos (num_elems); + } // --------------------------------------------------------------------- // calculate geometry @@ -1180,6 +1184,14 @@ void FEA_Module_SGH::setup(){ // specific internal energy elem_sie(rk_level, elem_gid) = mat_fill(f_id).sie; + if(simparam->topology_optimization_on&&mat_fill(f_id).extensive_energy_setting){ + elem_sie(rk_level, elem_gid) = elem_sie(rk_level, elem_gid)/relative_element_densities(elem_gid); + elem_extensive_initial_energy_condition(elem_gid) = true; + } + else if(simparam->topology_optimization_on&&!mat_fill(f_id).extensive_energy_setting){ + elem_extensive_initial_energy_condition(elem_gid) = false; + } + size_t mat_id = elem_mat_id(elem_gid); // short name @@ -1443,7 +1455,11 @@ void FEA_Module_SGH::setup(){ elem_sie.update_host(); elem_stress.update_host(); elem_pres.update_host(); - elem_sspd.update_host(); + elem_sspd.update_host(); + + if(simparam->topology_optimization_on){ + elem_extensive_initial_energy_condition.update_host(); + } return; diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/FEA_Module_SGH.h b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/FEA_Module_SGH.h index 725e3fcd1..95094a09d 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/FEA_Module_SGH.h +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/FEA_Module_SGH.h @@ -645,7 +645,7 @@ class FEA_Module_SGH: public FEA_Module{ DViewCArrayKokkos elem_sspd; DViewCArrayKokkos elem_sie; DViewCArrayKokkos elem_vol; - DViewCArrayKokkos elem_div; + DViewCArrayKokkos elem_div; DViewCArrayKokkos elem_mass; DViewCArrayKokkos elem_mat_id; @@ -665,6 +665,9 @@ class FEA_Module_SGH: public FEA_Module{ DCArrayKokkos elem_eos; DCArrayKokkos elem_strength; + //per element optimization flags + DCArrayKokkos elem_extensive_initial_energy_condition; + // Dual Views of the corner struct variables DViewCArrayKokkos corner_force; DViewCArrayKokkos corner_mass; diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/sgh_optimization.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/sgh_optimization.cpp index 960341b18..875cd5130 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/sgh_optimization.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/sgh_optimization.cpp @@ -232,6 +232,9 @@ void FEA_Module_SGH::update_forward_solve(Teuchos::RCP zp){ // specific internal energy elem_sie(rk_level, elem_gid) = mat_fill(f_id).sie; + if(simparam->topology_optimization_on&&mat_fill(f_id).extensive_energy_setting){ + elem_sie(rk_level, elem_gid) = elem_sie(rk_level, elem_gid)/relative_element_densities(elem_gid); + } elem_mat_id(elem_gid) = mat_fill(f_id).material_id; size_t mat_id = elem_mat_id(elem_gid); // short name @@ -1452,7 +1455,12 @@ void FEA_Module_SGH::compute_topology_optimization_gradient_full(Teuchos::RCP Date: Tue, 27 Feb 2024 13:46:50 -0700 Subject: [PATCH 41/64] ENH: eigensolver user options --- .../FEA_Physics_Modules/FEA_Module_Elasticity.cpp | 10 +++++++--- .../FEA_Module/Elasticity_Parameters.h | 6 +++++- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/Parallel-Solvers/Implicit-Lagrange/FEA_Physics_Modules/FEA_Module_Elasticity.cpp b/src/Parallel-Solvers/Implicit-Lagrange/FEA_Physics_Modules/FEA_Module_Elasticity.cpp index 0ef9315b0..b1c414ec9 100644 --- a/src/Parallel-Solvers/Implicit-Lagrange/FEA_Physics_Modules/FEA_Module_Elasticity.cpp +++ b/src/Parallel-Solvers/Implicit-Lagrange/FEA_Physics_Modules/FEA_Module_Elasticity.cpp @@ -5941,7 +5941,7 @@ void FEA_Module_Elasticity::node_density_constraints(host_vec_array node_densiti ------------------------------------------------------------------------- */ int FEA_Module_Elasticity::eigensolve(){ - int nev = 9; + int nev = module_params->num_modes; int blocksize = 1.5*nev; int num_dim = simparam->num_dims; GO global_index, global_dof_index; @@ -6192,7 +6192,11 @@ int FEA_Module_Elasticity::eigensolve(){ "SR" - smallest real part "SI" - smallest imaginary part */ - std::string which("SM"); + std::string which; + if(module_params->smallest_modes) + which = "SM"; + if(module_params->largest_modes) + which = "LM"; int NumImages = nranks; int numBlocks = 3 * NumImages; int maxRestarts = 50; @@ -6203,7 +6207,7 @@ int FEA_Module_Elasticity::eigensolve(){ // real_t mat_norm = std::max(Global_Stiffness_Matrix->getFrobeniusNorm(),Global_Mass_Matrix->getFrobeniusNorm()); - real_t tol = 1.0e-18*mat_norm; + real_t tol = module_params->convergence_tolerance*mat_norm; //tol = tol*mat_norm; // Create parameter list to pass into the solver manager Teuchos::ParameterList MyPL; diff --git a/src/Parallel-Solvers/Simulation_Parameters/FEA_Module/Elasticity_Parameters.h b/src/Parallel-Solvers/Simulation_Parameters/FEA_Module/Elasticity_Parameters.h index 4b9d84105..81ecf7ff3 100644 --- a/src/Parallel-Solvers/Simulation_Parameters/FEA_Module/Elasticity_Parameters.h +++ b/src/Parallel-Solvers/Simulation_Parameters/FEA_Module/Elasticity_Parameters.h @@ -9,6 +9,10 @@ struct Elasticity_Parameters bool strain_max_flag = false; bool modal_analysis = false; bool anisotropic_lattice = false; + int num_modes = 10; + bool smallest_modes = true; + bool largest_modes = false; + real_t convergence_tolerance = 1.0e-18; Elasticity_Parameters() : FEA_Module_Parameters({ FIELD::displacement, @@ -17,5 +21,5 @@ struct Elasticity_Parameters }) { } }; IMPL_YAML_SERIALIZABLE_WITH_BASE(Elasticity_Parameters, ImplicitModule, - strain_max_flag, modal_analysis, anisotropic_lattice + strain_max_flag, modal_analysis, anisotropic_lattice, num_modes, smallest_modes, largest_modes, convergence_tolerance ) From cc6ff5414b434f20d056924ef4b2a33dfd935e2a Mon Sep 17 00:00:00 2001 From: Adrian-Diaz <46537160+Adrian-Diaz@users.noreply.github.com> Date: Tue, 27 Feb 2024 15:06:24 -0700 Subject: [PATCH 42/64] ENH: add example yaml file for modal solver --- .../example_eigenvalues.yaml | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 src/Parallel-Solvers/Implicit-Lagrange/example_eigenvalues.yaml diff --git a/src/Parallel-Solvers/Implicit-Lagrange/example_eigenvalues.yaml b/src/Parallel-Solvers/Implicit-Lagrange/example_eigenvalues.yaml new file mode 100644 index 000000000..01eb80ed3 --- /dev/null +++ b/src/Parallel-Solvers/Implicit-Lagrange/example_eigenvalues.yaml @@ -0,0 +1,46 @@ +# input file +num_dims: 3 + +mesh_generation_options: + type: Box + origin: [0, 0, 0] + length: [10, 10, 100] + num_elems: [10, 10, 100] + +materials: + - id: 0 + elastic_modulus: 200000000000 + poisson_ratio: 0.3 + density: 7850 + thermal_conductivity: 10 + specific_internal_energy_rate: 1.0 + initial_temperature: 293 + +fea_module_parameters: + - type: Inertial + material_id: 0 + + - type: Elasticity + material_id: 0 + modal_analysis: true + smallest_modes: true + num_modes: 10 + convergence_tolerance: 1.0e-18 + # Dirichlet conditions + boundary_conditions: + - surface: + type: z_plane + plane_position: 0.0 + type: displacement + value: 0.0 + + # Loading/Neumann Conditions + loading_conditions: + - surface: + type: z_plane + plane_position: 100.0 + type: surface_traction + component_x: 500 + component_y: 0 + component_z: 0 + From f1091e8cc117453ca4f42f198ccff1ded0227dee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CAdrian-Diaz=E2=80=9D?= <“adriandiaz1117@gmail.com”> Date: Tue, 27 Feb 2024 23:21:38 -0700 Subject: [PATCH 43/64] BUG: missing optimization setting on boundaries --- .../Parallel-Explicit/Explicit_Solver.cpp | 114 +++++++++--------- 1 file changed, 58 insertions(+), 56 deletions(-) diff --git a/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp b/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp index cd0548c97..77f9c92b2 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp @@ -1290,64 +1290,66 @@ void Explicit_Solver::setup_optimization_problem(){ } //set lower bounds for nodes on surfaces with boundary and loading conditions - for(int imodule = 0; imodule < nfea_modules; imodule++){ - num_boundary_sets = fea_modules[imodule]->num_boundary_conditions; - for(int iboundary = 0; iboundary < num_boundary_sets; iboundary++){ - - num_bdy_patches_in_set = fea_modules[imodule]->bdy_patches_in_set.stride(iboundary); - - //loop over boundary patches for this boundary set - if(simparam.optimization_options.thick_condition_boundary){ - for (int bdy_patch_gid = 0; bdy_patch_gid < num_bdy_patches_in_set; bdy_patch_gid++){ - - // get the global id for this boundary patch - patch_id = fea_modules[imodule]->bdy_patches_in_set(iboundary, bdy_patch_gid); - Surface_Nodes = Boundary_Patches(patch_id).node_set; - current_element_index = Boundary_Patches(patch_id).element_id; - //debug print of local surface ids - //std::cout << " LOCAL SURFACE IDS " << std::endl; - //std::cout << local_surface_id << std::endl; - //acquire set of nodes for this face - for(int node_loop=0; node_loop < max_nodes_per_element; node_loop++){ - current_node_index = nodes_in_elem(current_element_index,node_loop); - if(map->isNodeGlobalElement(current_node_index)){ - local_node_index = map->getLocalElement(current_node_index); - node_densities_lower_bound(local_node_index,0) = 1; - } - }// node loop for - - //if - /* - else{ - Surface_Nodes = Boundary_Patches(patch_id).node_set; - local_surface_id = Boundary_Patches(patch_id).local_patch_id; - //debug print of local surface ids - //std::cout << " LOCAL SURFACE IDS " << std::endl; - //std::cout << local_surface_id << std::endl; - //acquire set of nodes for this face - for(int node_loop=0; node_loop < Surface_Nodes.size(); node_loop++){ - current_node_index = Surface_Nodes(node_loop); - if(map->isNodeGlobalElement(current_node_index)){ - local_node_index = map->getLocalElement(current_node_index); - node_densities_lower_bound(local_node_index,0) = 1; - } - }// node loop for - }//if - */ - }//boundary patch for - } - else{ - for(int node_loop=0; node_loop < fea_modules[imodule]->num_bdy_nodes_in_set(iboundary); node_loop++){ - local_node_index = fea_modules[imodule]->bdy_nodes_in_set(iboundary, node_loop); - node_densities_lower_bound(local_node_index,0) = 1; + if(!simparam.optimization_options.variable_outer_shell){ + for(int imodule = 0; imodule < nfea_modules; imodule++){ + num_boundary_sets = fea_modules[imodule]->num_boundary_conditions; + for(int iboundary = 0; iboundary < num_boundary_sets; iboundary++){ + + num_bdy_patches_in_set = fea_modules[imodule]->bdy_patches_in_set.stride(iboundary); + + //loop over boundary patches for this boundary set + if(simparam.optimization_options.thick_condition_boundary){ + for (int bdy_patch_gid = 0; bdy_patch_gid < num_bdy_patches_in_set; bdy_patch_gid++){ + + // get the global id for this boundary patch + patch_id = fea_modules[imodule]->bdy_patches_in_set(iboundary, bdy_patch_gid); + Surface_Nodes = Boundary_Patches(patch_id).node_set; + current_element_index = Boundary_Patches(patch_id).element_id; + //debug print of local surface ids + //std::cout << " LOCAL SURFACE IDS " << std::endl; + //std::cout << local_surface_id << std::endl; + //acquire set of nodes for this face + for(int node_loop=0; node_loop < max_nodes_per_element; node_loop++){ + current_node_index = nodes_in_elem(current_element_index,node_loop); + if(map->isNodeGlobalElement(current_node_index)){ + local_node_index = map->getLocalElement(current_node_index); + node_densities_lower_bound(local_node_index,0) = 1; + } + }// node loop for + + //if + /* + else{ + Surface_Nodes = Boundary_Patches(patch_id).node_set; + local_surface_id = Boundary_Patches(patch_id).local_patch_id; + //debug print of local surface ids + //std::cout << " LOCAL SURFACE IDS " << std::endl; + //std::cout << local_surface_id << std::endl; + //acquire set of nodes for this face + for(int node_loop=0; node_loop < Surface_Nodes.size(); node_loop++){ + current_node_index = Surface_Nodes(node_loop); + if(map->isNodeGlobalElement(current_node_index)){ + local_node_index = map->getLocalElement(current_node_index); + node_densities_lower_bound(local_node_index,0) = 1; + } + }// node loop for + }//if + */ + }//boundary patch for } - } - }//boundary set for + else{ + for(int node_loop=0; node_loop < fea_modules[imodule]->num_bdy_nodes_in_set(iboundary); node_loop++){ + local_node_index = fea_modules[imodule]->bdy_nodes_in_set(iboundary, node_loop); + node_densities_lower_bound(local_node_index,0) = 1; + } + } + }//boundary set for - //set node conditions due to point BCS that might not show up in boundary sets - //possible to have overlap in which nodes are set with the previous loop - fea_modules[imodule]->node_density_constraints(node_densities_lower_bound); - }//module for + //set node conditions due to point BCS that might not show up in boundary sets + //possible to have overlap in which nodes are set with the previous loop + fea_modules[imodule]->node_density_constraints(node_densities_lower_bound); + }//module for + } //sync device view dual_node_densities_upper_bound.sync_device(); From fb90b9a1dfe935212048a6d95a5786db0a519f24 Mon Sep 17 00:00:00 2001 From: Jacob Moore Date: Wed, 28 Feb 2024 11:50:54 -0600 Subject: [PATCH 44/64] STYLE: extend uncrustify.sh --- scripts/uncrustify.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/scripts/uncrustify.sh b/scripts/uncrustify.sh index 47ed2b0fb..becdde2c2 100755 --- a/scripts/uncrustify.sh +++ b/scripts/uncrustify.sh @@ -62,14 +62,14 @@ echo "Uncrusting: $sourceDir/Parallel-Solvers/Parallel-Explicit/Topology_Optimiz cd "$sourceDir/Parallel-Solvers/Parallel-Explicit/Topology_Optimization" || exit treeProcess -# # Uncrustify Eulerian Solver -# echo "Uncrusting: $sourceDir/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver" -# cd "$sourceDir/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver" || exit -# treeProcess - -# # Uncrustify Dynamic Elastic -# echo "Uncrusting: $sourceDir/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver" -# cd "$sourceDir/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver" || exit +# Uncrustify Eulerian Solver +echo "Uncrusting: $sourceDir/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver" +cd "$sourceDir/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver" || exit +treeProcess + +# Uncrustify Dynamic Elastic +echo "Uncrusting: $sourceDir/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver" +cd "$sourceDir/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver" || exit treeProcess exit From c5bcb4f2c4d9e0ec08f57fe2a71869c11181c259 Mon Sep 17 00:00:00 2001 From: Jacob Moore Date: Wed, 28 Feb 2024 13:26:41 -0600 Subject: [PATCH 45/64] STYLE: Uncrustify --- .../FEA_Module_Dynamic_Elasticity.cpp | 1 - .../FEA_Module_Dynamic_Elasticity.h | 1 - .../SGH_Solver/include/FEA_Module_SGH.h | 4 +- .../SGH_Solver/src/setup_sgh.cpp | 64 +++++++++--------- .../SGH_Solver/src/sgh_optimization.cpp | 65 +++++++++---------- .../Kinetic_Energy_Minimize.h | 16 ++--- 6 files changed, 76 insertions(+), 75 deletions(-) diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.cpp b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.cpp index 538851194..dc878f98f 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.cpp @@ -92,7 +92,6 @@ FEA_Module_Dynamic_Elasticity::FEA_Module_Dynamic_Elasticity( mesh = mesh_in; - // boundary condition data max_boundary_sets = 0; Local_Index_Boundary_Patches = Explicit_Solver_Pointer_->Local_Index_Boundary_Patches; diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.h b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.h index fc4f190e1..53833344a 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.h +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.h @@ -416,7 +416,6 @@ class FEA_Module_Dynamic_Elasticity : public FEA_Module void Gradient_Element_Material_Properties(size_t ielem, real_t& Element_Modulus, real_t& Poisson_Ratio, real_t density); - void local_matrix_multiply(int ielem, CArrayKokkos& Local_Matrix); void assemble_matrix(); diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/FEA_Module_SGH.h b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/FEA_Module_SGH.h index fe64ab3ce..e42e22d97 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/FEA_Module_SGH.h +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/FEA_Module_SGH.h @@ -658,8 +658,8 @@ class FEA_Module_SGH : public FEA_Module DCArrayKokkos elem_eos; DCArrayKokkos elem_strength; - //per element optimization flags - DCArrayKokkos elem_extensive_initial_energy_condition; + // per element optimization flags + DCArrayKokkos elem_extensive_initial_energy_condition; // Dual Views of the corner struct variables DViewCArrayKokkos corner_force; diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/setup_sgh.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/setup_sgh.cpp index 56abfc0a3..079f402a3 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/setup_sgh.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/setup_sgh.cpp @@ -122,9 +122,10 @@ void FEA_Module_SGH::setup() elem_eos = DCArrayKokkos(num_elems); elem_strength = DCArrayKokkos(num_elems); - //optimization flags - if(simparam->topology_optimization_on){ - elem_extensive_initial_energy_condition = DCArrayKokkos (num_elems); + // optimization flags + if (simparam->topology_optimization_on) + { + elem_extensive_initial_energy_condition = DCArrayKokkos(num_elems); } // --------------------------------------------------------------------- @@ -250,32 +251,35 @@ void FEA_Module_SGH::setup() // --- apply the fill instructions over each of the Elements---// - //initialize if topology optimization is used - if(simparam->topology_optimization_on){ - //compute element averaged density ratios corresponding to nodal density design variables + // initialize if topology optimization is used + if (simparam->topology_optimization_on) + { + // compute element averaged density ratios corresponding to nodal density design variables CArray current_element_nodal_densities = CArray(num_nodes_in_elem); - {//view scope - const_host_vec_array all_node_densities = all_node_densities_distributed->getLocalView (Tpetra::Access::ReadOnly); - //debug print - //std::cout << "NODE DENSITY TEST " << all_node_densities(0,0) << std::endl; - for(int elem_id = 0; elem_id < rnum_elem; elem_id++) + { // view scope + const_host_vec_array all_node_densities = all_node_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); + // debug print + // std::cout << "NODE DENSITY TEST " << all_node_densities(0,0) << std::endl; + for (int elem_id = 0; elem_id < rnum_elem; elem_id++) { - for(int inode = 0; inode < num_nodes_in_elem; inode++) + for (int inode = 0; inode < num_nodes_in_elem; inode++) { - current_element_nodal_densities(inode) = all_node_densities(nodes_in_elem(elem_id,inode),0); + current_element_nodal_densities(inode) = all_node_densities(nodes_in_elem(elem_id, inode), 0); } - relative_element_densities.host(elem_id) = average_element_density(num_nodes_in_elem, current_element_nodal_densities); - }//for - } //view scope - //debug print - //std::cout << "ELEMENT RELATIVE DENSITY TEST " << relative_element_densities.host(0) << std::endl; + relative_element_densities.host(elem_id) = average_element_density(num_nodes_in_elem, current_element_nodal_densities); + } // for + } // view scope + // debug print + // std::cout << "ELEMENT RELATIVE DENSITY TEST " << relative_element_densities.host(0) << std::endl; } - else{ - for(int elem_id = 0; elem_id < rnum_elem; elem_id++){ + else + { + for (int elem_id = 0; elem_id < rnum_elem; elem_id++) + { relative_element_densities.host(elem_id) = 1; - }//for + } // for } - + relative_element_densities.update_device(); // loop over the fill instructures @@ -315,7 +319,7 @@ void FEA_Module_SGH::setup() if (fill_this) { // density - elem_den(elem_gid) = mat_fill(f_id).den*relative_element_densities(elem_gid); + elem_den(elem_gid) = mat_fill(f_id).den * relative_element_densities(elem_gid); // mass elem_mass(elem_gid) = elem_den(elem_gid) * elem_vol(elem_gid); @@ -323,17 +327,16 @@ void FEA_Module_SGH::setup() // specific internal energy elem_sie(rk_level, elem_gid) = mat_fill(f_id).sie; - if(simparam->topology_optimization_on&&mat_fill(f_id).extensive_energy_setting) + if (simparam->topology_optimization_on && mat_fill(f_id).extensive_energy_setting) { - elem_sie(rk_level, elem_gid) = elem_sie(rk_level, elem_gid)/relative_element_densities(elem_gid); - elem_extensive_initial_energy_condition(elem_gid) = true; + elem_sie(rk_level, elem_gid) = elem_sie(rk_level, elem_gid) / relative_element_densities(elem_gid); + elem_extensive_initial_energy_condition(elem_gid) = true; } - else if(simparam->topology_optimization_on&&!mat_fill(f_id).extensive_energy_setting) + else if (simparam->topology_optimization_on && !mat_fill(f_id).extensive_energy_setting) { - elem_extensive_initial_energy_condition(elem_gid) = false; + elem_extensive_initial_energy_condition(elem_gid) = false; } - size_t mat_id = elem_mat_id(elem_gid); // short name // --- stress tensor --- @@ -600,7 +603,8 @@ void FEA_Module_SGH::setup() elem_pres.update_host(); elem_sspd.update_host(); - if(simparam->topology_optimization_on){ + if (simparam->topology_optimization_on) + { elem_extensive_initial_energy_condition.update_host(); } diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/sgh_optimization.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/sgh_optimization.cpp index 7d4039066..fd4822ca3 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/sgh_optimization.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/sgh_optimization.cpp @@ -281,10 +281,10 @@ void FEA_Module_SGH::update_forward_solve(Teuchos::RCP zp) // specific internal energy elem_sie(rk_level, elem_gid) = mat_fill(f_id).sie; - - if(simparam->topology_optimization_on && mat_fill(f_id).extensive_energy_setting) + + if (simparam->topology_optimization_on && mat_fill(f_id).extensive_energy_setting) { - elem_sie(rk_level, elem_gid) = elem_sie(rk_level, elem_gid)/relative_element_densities(elem_gid); + elem_sie(rk_level, elem_gid) = elem_sie(rk_level, elem_gid) / relative_element_densities(elem_gid); } elem_mat_id(elem_gid) = mat_fill(f_id).material_id; @@ -1251,7 +1251,7 @@ void FEA_Module_SGH::compute_topology_optimization_gradient_full(Teuchos::RCP(rnum_elem, num_nodes_in_elem * num_dim); Power_Gradient_Energies = CArrayKokkos(rnum_elem); @@ -1965,7 +1964,7 @@ void FEA_Module_SGH::init_assembly() { DOF_Graph_Matrix(idof, istride) = Graph_Matrix(idof / num_dim, istride / num_dim) * num_dim + istride % num_dim; } - }); // end parallel for + }); // end parallel for /* //construct distributed gradient matrix from local kokkos data diff --git a/src/Parallel-Solvers/Parallel-Explicit/Topology_Optimization/Kinetic_Energy_Minimize.h b/src/Parallel-Solvers/Parallel-Explicit/Topology_Optimization/Kinetic_Energy_Minimize.h index 63287ecdd..da79ecbf4 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Topology_Optimization/Kinetic_Energy_Minimize.h +++ b/src/Parallel-Solvers/Parallel-Explicit/Topology_Optimization/Kinetic_Energy_Minimize.h @@ -230,7 +230,7 @@ typedef MV::dual_view_type dual_vec_array; // This is the first call to update // first linear solve was done in FEA class run function already FEM_Dynamic_Elasticity_->comm_variables(zp); - //update deformation variables + // update deformation variables FEM_Dynamic_Elasticity_->update_forward_solve(zp); // initial design density data was already communicated for ghost nodes in init_design() // decide to output current optimization state @@ -306,16 +306,16 @@ typedef MV::dual_view_type dual_vec_array; if (type == ROL::UpdateType::Initial) { // This is the first call to update - if(Explicit_Solver_Pointer_->myrank==0) + if (Explicit_Solver_Pointer_->myrank == 0) { *fos << "called SGH Initial" << std::endl; } - + FEM_SGH_->comm_variables(zp); FEM_SGH_->update_forward_solve(zp); - //initial design density data was already communicated for ghost nodes in init_design() - //decide to output current optimization state - //FEM_SGH_->Explicit_Solver_Pointer_->write_outputs(); + // initial design density data was already communicated for ghost nodes in init_design() + // decide to output current optimization state + // FEM_SGH_->Explicit_Solver_Pointer_->write_outputs(); } else if (type == ROL::UpdateType::Accept) { @@ -327,8 +327,8 @@ typedef MV::dual_view_type dual_vec_array; // Revert to cached value // This is a new value of x // communicate density variables for ghosts - if(Explicit_Solver_Pointer_->myrank == 0) *fos << "called SGH Revert" << std::endl; - + if (Explicit_Solver_Pointer_->myrank == 0) { *fos << "called SGH Revert" << std::endl; } + FEM_SGH_->comm_variables(zp); // update deformation variables FEM_SGH_->update_forward_solve(zp); From 8ede33d121a1f1d4bc11fc23e0a150042edceaa6 Mon Sep 17 00:00:00 2001 From: Jacob Moore Date: Thu, 29 Feb 2024 12:05:10 -0600 Subject: [PATCH 46/64] STYLE: Tweaking sgh example input yaml --- src/Parallel-Solvers/Parallel-Explicit/example_simple.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Parallel-Solvers/Parallel-Explicit/example_simple.yaml b/src/Parallel-Solvers/Parallel-Explicit/example_simple.yaml index 2b4f86bfa..362b4568a 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/example_simple.yaml +++ b/src/Parallel-Solvers/Parallel-Explicit/example_simple.yaml @@ -10,7 +10,7 @@ mesh_generation_options: type: Box origin: [0, 0, 0] length: [1.2, 1.2, 1.2] - num_elems: [40, 40, 40] + num_elems: [32, 32, 32] output_options: timer_output_level: thorough @@ -87,7 +87,7 @@ regions: - volume: type: sphere radius1: 0.0 - radius2: 0.03 + radius2: 0.0375 material_id: 0 den: 1.0 ie: 0.25833839995946534 From 906bee33eb9e6f39c14077fe016ab50f9fe24596 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CAdrian-Diaz=E2=80=9D?= <“adriandiaz1117@gmail.com”> Date: Thu, 29 Feb 2024 18:38:41 -0700 Subject: [PATCH 47/64] BUG: prevent memory usage of TO structure --- .../SGH_Solver/src/setup_sgh.cpp | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/setup_sgh.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/setup_sgh.cpp index 079f402a3..d71b1e6a9 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/setup_sgh.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/setup_sgh.cpp @@ -1,5 +1,5 @@ /********************************************************************************************** - 2020. Triad National Security, LLC. All rights reserved. + � 2020. Triad National Security, LLC. All rights reserved. This program was produced under U.S. Government contract 89233218CNA000001 for Los Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC for the U.S. Department of Energy/National Nuclear Security Administration. All rights in the program are @@ -271,16 +271,10 @@ void FEA_Module_SGH::setup() } // view scope // debug print // std::cout << "ELEMENT RELATIVE DENSITY TEST " << relative_element_densities.host(0) << std::endl; - } - else - { - for (int elem_id = 0; elem_id < rnum_elem; elem_id++) - { - relative_element_densities.host(elem_id) = 1; - } // for + + relative_element_densities.update_device(); } - relative_element_densities.update_device(); // loop over the fill instructures for (int f_id = 0; f_id < num_fills; f_id++) @@ -319,7 +313,12 @@ void FEA_Module_SGH::setup() if (fill_this) { // density - elem_den(elem_gid) = mat_fill(f_id).den * relative_element_densities(elem_gid); + if (simparam->topology_optimization_on){ + elem_den(elem_gid) = mat_fill(f_id).den * relative_element_densities(elem_gid); + } + else{ + elem_den(elem_gid) = mat_fill(f_id).den; + } // mass elem_mass(elem_gid) = elem_den(elem_gid) * elem_vol(elem_gid); From 20e5f32cd8d54a837dfb69b9d5ba1ef0bccf8857 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CAdrian-Diaz=E2=80=9D?= <“adriandiaz1117@gmail.com”> Date: Thu, 29 Feb 2024 23:54:50 -0700 Subject: [PATCH 48/64] ENH: TO regional bound constraints --- .../Implicit-Lagrange/Implicit_Solver.cpp | 24 ++++++++++++- .../Parallel-Explicit/Explicit_Solver.cpp | 23 ++++++++++++ .../Optimization_Bound_Constraint_Region.h | 36 +++++++++++++++++++ .../Optimization_Options.h | 13 +++++-- 4 files changed, 93 insertions(+), 3 deletions(-) create mode 100644 src/Parallel-Solvers/Simulation_Parameters/Optimization_Bound_Constraint_Region.h diff --git a/src/Parallel-Solvers/Implicit-Lagrange/Implicit_Solver.cpp b/src/Parallel-Solvers/Implicit-Lagrange/Implicit_Solver.cpp index 4bf472a58..451b9adae 100644 --- a/src/Parallel-Solvers/Implicit-Lagrange/Implicit_Solver.cpp +++ b/src/Parallel-Solvers/Implicit-Lagrange/Implicit_Solver.cpp @@ -1205,7 +1205,29 @@ void Implicit_Solver::setup_optimization_problem(){ }//boundary patch for } }//if to check if shell should have any constraints - + + //constraints due to specified user regions + const size_t num_fills = simparam.optimization_options.volume_bound_constraints.size(); + const_host_vec_array node_coords_view = node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); + const DCArrayKokkos mat_fill = simparam.optimization_options.optimization_bound_constraint_volumes; + + for(int ifill = 0; ifill < num_fills; ifill++){ + for(int inode = 0; inode < nlocal_nodes; inode++){ + real_t node_coords[3]; + node_coords[0] = node_coords_view(inode,0); + node_coords[1] = node_coords_view(inode,1); + node_coords[2] = node_coords_view(inode,2); + bool fill_this = mat_fill(ifill).volume.contains(node_coords); + if(fill_this){ + node_densities_lower_bound(inode,0) = mat_fill(ifill).set_lower_density_bound; + //make sure 0 setting is increased to the epsilon value setting + if(node_densities_lower_bound(inode,0) < simparam.optimization_options.density_epsilon){ + node_densities_lower_bound(inode,0) = simparam.optimization_options.density_epsilon; + } + node_densities_upper_bound(inode,0) = mat_fill(ifill).set_upper_density_bound; + } + }//node for + }//fill region for } else{ //initialize memory for volume storage diff --git a/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp b/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp index 34f2f1970..25d07fe4c 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp @@ -1350,6 +1350,29 @@ void Explicit_Solver::setup_optimization_problem(){ fea_modules[imodule]->node_density_constraints(node_densities_lower_bound); }//module for } + + //constraints due to specified user regions + const size_t num_fills = simparam.optimization_options.volume_bound_constraints.size(); + const_host_vec_array node_coords_view = node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); + const DCArrayKokkos mat_fill = simparam.optimization_options.optimization_bound_constraint_volumes; + + for(int ifill = 0; ifill < num_fills; ifill++){ + for(int inode = 0; inode < nlocal_nodes; inode++){ + real_t node_coords[3]; + node_coords[0] = node_coords_view(inode,0); + node_coords[1] = node_coords_view(inode,1); + node_coords[2] = node_coords_view(inode,2); + bool fill_this = mat_fill(ifill).volume.contains(node_coords); + if(fill_this){ + node_densities_lower_bound(inode,0) = mat_fill(ifill).set_lower_density_bound; + //make sure 0 setting is increased to the epsilon value setting + if(node_densities_lower_bound(inode,0) < simparam.optimization_options.density_epsilon){ + node_densities_lower_bound(inode,0) = simparam.optimization_options.density_epsilon; + } + node_densities_upper_bound(inode,0) = mat_fill(ifill).set_upper_density_bound; + } + }//node for + }//fill region for //sync device view dual_node_densities_upper_bound.sync_device(); diff --git a/src/Parallel-Solvers/Simulation_Parameters/Optimization_Bound_Constraint_Region.h b/src/Parallel-Solvers/Simulation_Parameters/Optimization_Bound_Constraint_Region.h new file mode 100644 index 000000000..34fe794d8 --- /dev/null +++ b/src/Parallel-Solvers/Simulation_Parameters/Optimization_Bound_Constraint_Region.h @@ -0,0 +1,36 @@ +#pragma once + +#include "yaml-serializable.h" +#include +#include +#include "Simulation_Parameters/Geometry.h" + +struct Optimization_Bound_Constraint_Region : Yaml::DerivedFields, Yaml::ValidatedYaml { + size_t id; + Volume volume; + VELOCITY_TYPE velocity = VELOCITY_TYPE::cartesian; + std::optional lower_density_bound; + std::optional upper_density_bound; + double set_lower_density_bound; + double set_upper_density_bound; + + void validate() { + if (lower_density_bound.has_value()){ + if(lower_density_bound.value()<0) + throw Yaml::ConfigurationException("Specify values for lower relative density greater than or equal to 0 but less than or equal to 1"); + } + + if (upper_density_bound.has_value()){ + if(upper_density_bound.value()<0) + throw Yaml::ConfigurationException("Specify values for upper relative density greater than 0 but less than or equal to 1"); + } + } + + void derive() { + set_lower_density_bound = lower_density_bound.value_or(0); + set_upper_density_bound = upper_density_bound.value_or(1); + } +}; +IMPL_YAML_SERIALIZABLE_FOR(Optimization_Bound_Constraint_Region, + id, volume, lower_density_bound, upper_density_bound +) \ No newline at end of file diff --git a/src/Parallel-Solvers/Simulation_Parameters/Optimization_Options.h b/src/Parallel-Solvers/Simulation_Parameters/Optimization_Options.h index a033efcbe..fedf07644 100644 --- a/src/Parallel-Solvers/Simulation_Parameters/Optimization_Options.h +++ b/src/Parallel-Solvers/Simulation_Parameters/Optimization_Options.h @@ -1,5 +1,6 @@ #pragma once #include "yaml-serializable.h" +#include "Optimization_Bound_Constraint_Region.h" SERIALIZABLE_ENUM(FUNCTION_TYPE, @@ -92,10 +93,12 @@ struct MultiObjectiveModule { YAML_ADD_REQUIRED_FIELDS_FOR(MultiObjectiveModule, type, weight_coefficient) IMPL_YAML_SERIALIZABLE_FOR(MultiObjectiveModule, type, weight_coefficient) -struct Optimization_Options { +struct Optimization_Options: Yaml::DerivedFields { OPTIMIZATION_PROCESS optimization_process = OPTIMIZATION_PROCESS::none; OPTIMIZATION_OBJECTIVE optimization_objective = OPTIMIZATION_OBJECTIVE::none; std::vector constraints; + std::vector volume_bound_constraints; + DCArrayKokkos optimization_bound_constraint_volumes; bool method_of_moving_asymptotes = false; double simp_penalty_power = 3.0; bool thick_condition_boundary = true; @@ -110,13 +113,19 @@ struct Optimization_Options { MULTI_OBJECTIVE_STRUCTURE multi_objective_structure = MULTI_OBJECTIVE_STRUCTURE::linear; std::vector multi_objective_modules; + //derive function requires inheritance from Yaml::DerivedFields + void derive() { + if(volume_bound_constraints.size()>=1){ + mtr::from_vector(optimization_bound_constraint_volumes, volume_bound_constraints); + } + } }; YAML_ADD_REQUIRED_FIELDS_FOR(Optimization_Options, optimization_objective ) IMPL_YAML_SERIALIZABLE_FOR(Optimization_Options, optimization_process, optimization_objective, - constraints, method_of_moving_asymptotes, + constraints, method_of_moving_asymptotes, volume_bound_constraints, simp_penalty_power, density_epsilon, thick_condition_boundary, optimization_output_freq, density_filter, minimum_density, maximum_density, multi_objective_modules, multi_objective_structure, density_filter, retain_outer_shell, From a6d8c2c2d0a785afc9085c4ab6745951b7b8d6f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CAdrian-Diaz=E2=80=9D?= <“adriandiaz1117@gmail.com”> Date: Fri, 1 Mar 2024 00:09:21 -0700 Subject: [PATCH 49/64] ENH: add yaml file example for sgh topology optimization --- .../example_sgh_optimization.yaml | 120 ++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 Parallel-Solvers/Parallel-Explicit/example_sgh_optimization.yaml diff --git a/Parallel-Solvers/Parallel-Explicit/example_sgh_optimization.yaml b/Parallel-Solvers/Parallel-Explicit/example_sgh_optimization.yaml new file mode 100644 index 000000000..35b64abc2 --- /dev/null +++ b/Parallel-Solvers/Parallel-Explicit/example_sgh_optimization.yaml @@ -0,0 +1,120 @@ +num_dims: 3 +dynamic_options: + time_final: 0.5 + dt_min: 1.e-8 + dt_max: 1.e-3 + dt_start: 1.e-5 + cycle_stop: 2000000 + output_time_sequence_level: extreme + +mesh_generation_options: + type: Box + origin: [0, 0, 0] + length: [1.2, 1.2, 1.2] + num_elems: [32, 32, 32] + +output_options: + timer_output_level: thorough + output_file_format: vtk + graphics_step: 0.5 + write_initial: true + write_final: false + +fea_module_parameters: + - type: SGH + material_id: 0 + boundary_conditions: + # Tag X plane + - surface: + type: x_plane + plane_position: 0.0 + type: reflected + + # Tag Y plane + - surface: + type: y_plane + plane_position: 0.0 + type: reflected + + # Tag Z plane + - surface: + type: z_plane + plane_position: 0.0 + type: reflected + + # Tag X plane + - surface: + type: x_plane + plane_position: 1.2 + type: reflected + + # Tag Y plane + - surface: + type: y_plane + plane_position: 1.2 + type: reflected + + # Tag Z plane + - surface: + type: z_plane + plane_position: 1.2 + type: reflected + +materials: + - id: 0 + eos_model: ideal_gas + strength_model: none + elastic_modulus: 10 + poisson_ratio: 0.3 + maximum_limiter: true + q1: 1.0 + q2: 0 + q1ex: 1.0 + q2ex: 0.0 + eos_global_vars: + - 1.666666666666667 + - 1.0E-14 + - 1.0 + +regions: + - volume: + type: global + material_id: 0 + den: 1.0 + sie: 1.0e-10 + + velocity: cartesian + u: 0 + v: 0 + w: 0 + # energy source initial conditions + - volume: + type: sphere + radius1: 0.0 + radius2: 0.0375 + material_id: 0 + den: 1.0 + ie: 0.25833839995946534 + + velocity: cartesian + u: 0.0 + v: 0.0 + w: 0.0 + + +optimization_options: + optimization_process: topology_optimization + optimization_objective: minimize_kinetic_energy + density_epsilon: 0.1 + variable_outer_shell: true + constraints: + - type: mass + relation: equality + value: 0.40 + volume_bound_constraints: + - volume: + type: sphere + radius1: 0.0 + radius2: 0.225 + lower_density_bound: 1.0 + upper_density_bound: 1.0 From 44f433eb5d2924e38cf8d65d35fd4b98cc62268a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CAdrian-Diaz=E2=80=9D?= <“adriandiaz1117@gmail.com”> Date: Fri, 1 Mar 2024 13:17:00 -0700 Subject: [PATCH 50/64] BUG: change vumat.h name to avoid case sensitivity --- src/Parallel-Solvers/Material-Models/strength/VUMAT/VUMAT.cpp | 2 +- .../strength/VUMAT/{vumat.h => vumat_function.h} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename src/Parallel-Solvers/Material-Models/strength/VUMAT/{vumat.h => vumat_function.h} (100%) diff --git a/src/Parallel-Solvers/Material-Models/strength/VUMAT/VUMAT.cpp b/src/Parallel-Solvers/Material-Models/strength/VUMAT/VUMAT.cpp index b7f418cbf..159eb6aae 100644 --- a/src/Parallel-Solvers/Material-Models/strength/VUMAT/VUMAT.cpp +++ b/src/Parallel-Solvers/Material-Models/strength/VUMAT/VUMAT.cpp @@ -1,5 +1,5 @@ #include "VUMAT.h" -#include "vumat.h" +#include "vumat_function.h" // Below are the list of variables calculated or that have values. diff --git a/src/Parallel-Solvers/Material-Models/strength/VUMAT/vumat.h b/src/Parallel-Solvers/Material-Models/strength/VUMAT/vumat_function.h similarity index 100% rename from src/Parallel-Solvers/Material-Models/strength/VUMAT/vumat.h rename to src/Parallel-Solvers/Material-Models/strength/VUMAT/vumat_function.h From 9ab69f369c488a33bc7be24e54d6af385c28be8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CAdrian-Diaz=E2=80=9D?= <“adriandiaz1117@gmail.com”> Date: Mon, 4 Mar 2024 01:22:39 -0700 Subject: [PATCH 51/64] WIP: abaqus inp reader --- .../Simulation_Parameters/Input_Options.h | 6 +- src/Parallel-Solvers/Solver.cpp | 650 ++++++++++++++++++ src/Parallel-Solvers/Solver.h | 4 +- 3 files changed, 658 insertions(+), 2 deletions(-) diff --git a/src/Parallel-Solvers/Simulation_Parameters/Input_Options.h b/src/Parallel-Solvers/Simulation_Parameters/Input_Options.h index 710d23547..396ec88f2 100644 --- a/src/Parallel-Solvers/Simulation_Parameters/Input_Options.h +++ b/src/Parallel-Solvers/Simulation_Parameters/Input_Options.h @@ -7,7 +7,8 @@ SERIALIZABLE_ENUM(MESH_FORMAT, ensight, tecplot, vtk, - ansys_dat + ansys_dat, + abaqus_inp ) SERIALIZABLE_ENUM(ELEMENT_TYPE, @@ -40,6 +41,9 @@ struct Input_Options : Yaml::ValidatedYaml, Yaml::DerivedFields { case MESH_FORMAT::ensight: words_per_line = 1; break; + case MESH_FORMAT::abaqus_inp: + words_per_line = 3; + break; case MESH_FORMAT::vtk: case MESH_FORMAT::tecplot: words_per_line = 3; diff --git a/src/Parallel-Solvers/Solver.cpp b/src/Parallel-Solvers/Solver.cpp index aedf41d40..9591fd22a 100644 --- a/src/Parallel-Solvers/Solver.cpp +++ b/src/Parallel-Solvers/Solver.cpp @@ -2236,6 +2236,656 @@ void Solver::read_mesh_tecplot(const char* MESH) // } } // end read_mesh +/* ---------------------------------------------------------------------- + Read ANSYS dat format mesh file +------------------------------------------------------------------------- */ +void Solver::read_mesh_abaqus_inp(const char *MESH){ + Input_Options input_options = simparam.input_options.value(); + char ch; + int num_dim = simparam.num_dims; + int p_order = input_options.p_order; + real_t unit_scaling = input_options.unit_scaling; + bool restart_file = simparam.restart_file; + int local_node_index, current_column_index; + size_t strain_count; + std::string skip_line, read_line, substring, token; + std::stringstream line_parse; + CArrayKokkos read_buffer; + int buffer_loop, buffer_iteration, buffer_iterations, dof_limit, scan_loop, nodes_per_element; + size_t read_index_start, node_rid, elem_gid; + GO node_gid; + real_t dof_value; + host_vec_array node_densities; + bool zero_index_base = input_options.zero_index_base; + int negative_index_found = 0; + int global_negative_index_found = 0; + + //Nodes_Per_Element_Type = elements::elem_types::Nodes_Per_Element_Type; + + //read the mesh + //PLACEHOLDER: ensight_format(MESH); + // abaqus_format(MESH); + // vtk_format(MESH) + + //task 0 reads file + if(myrank==0){ + in = new std::ifstream(); + in->open(MESH); + } + + //Abaqus inp file doesn't specify total number of nodes, which is needed for the node map. + //First pass reads in node section to determine the maximum number of nodes, second pass distributes node data + //The elements section header does specify element count + num_nodes = 0; + if(myrank==0){ + bool searching_for_nodes = true; + //skip lines at the top with nonessential info; stop skipping when "*Nodes" string is reached + while (searching_for_nodes&&in->good()) { + getline(*in, skip_line); + //std::cout << skip_line << std::endl; + line_parse.clear(); + line_parse.str(skip_line); + //stop when the NODES= string is reached + while (!line_parse.eof()){ + line_parse >> substring; + //std::cout << substring << std::endl; + if(!substring.compare("*Node")){ + searching_for_nodes = false; + break; + } + } //while + + } + if(searching_for_nodes){ + std::cout << "FILE FORMAT ERROR" << std::endl; + } + + //tally node count (bug in dat files seems to print wrong node count so read is done in two passes) + //stop when apparent "-1" zone delimiter is reacher + searching_for_nodes = true; + int node_tally = 0; + while (searching_for_nodes) { + getline(*in, read_line); + //std::cout << read_line << std::endl; + line_parse.clear(); + line_parse.str(read_line); + line_parse >> substring; + + //std::cout << substring << std::endl; + if(substring == "*Element,"){ + searching_for_nodes = false; + break; + } + else{ + node_tally++; + } + + } + num_nodes = node_tally; + std::cout << "declared node count: " << num_nodes << std::endl; + } + + //broadcast number of nodes + MPI_Bcast(&num_nodes,1,MPI_LONG_LONG_INT,0,world); + + //construct contiguous parallel row map now that we know the number of nodes + map = Teuchos::rcp( new Tpetra::Map(num_nodes,0,comm)); + + //close and reopen file for second pass now that global node count is known + if(myrank==0){ + in->close(); + //in = new std::ifstream(); + in->open(MESH); + } + + // set the vertices in the mesh read in + global_size_t local_nrows = map->getLocalNumElements(); + nlocal_nodes = local_nrows; + //populate local row offset data from global data + global_size_t min_gid = map->getMinGlobalIndex(); + global_size_t max_gid = map->getMaxGlobalIndex(); + global_size_t index_base = map->getIndexBase(); + //debug print + //std::cout << "local node count on task: " << " " << nlocal_nodes << std::endl; + + //allocate node storage with dual view + //dual_node_coords = dual_vec_array("dual_node_coords", nlocal_nodes,num_dim); + //if(restart_file) + //dual_node_densities = dual_vec_array("dual_node_densities", nlocal_nodes,1); + + //local variable for host view in the dual view + node_coords_distributed = Teuchos::rcp(new MV(map, num_dim)); + //view scope + { + host_vec_array node_coords = node_coords_distributed->getLocalView (Tpetra::Access::ReadWrite); + //host_vec_array node_coords = dual_node_coords.view_host(); + if(restart_file){ + design_node_densities_distributed = Teuchos::rcp(new MV(map, 1)); + node_densities = design_node_densities_distributed->getLocalView (Tpetra::Access::ReadWrite); + } + //notify that the host view is going to be modified in the file readin + //dual_node_coords.modify_host(); + //if(restart_file) + //dual_node_densities.modify_host(); + + //old swage method + //mesh->init_nodes(local_nrows); // add 1 for index starting at 1 + + std::cout << "Num nodes assigned to task " << myrank << " = " << nlocal_nodes << std::endl; + + // read the initial mesh coordinates + // x-coords + /*only task 0 reads in nodes and elements from the input file + stores node data in a buffer and communicates once the buffer cap is reached + or the data ends*/ + + words_per_line = input_options.words_per_line; + //if(restart_file) words_per_line++; + elem_words_per_line = input_options.elem_words_per_line; + + //allocate read buffer + read_buffer = CArrayKokkos(BUFFER_LINES,words_per_line,MAX_WORD); + + dof_limit = num_nodes; + buffer_iterations = dof_limit/BUFFER_LINES; + if(dof_limit%BUFFER_LINES!=0) buffer_iterations++; + + //second pass to now read node coords with global node map defines + if(myrank==0){ + bool searching_for_nodes = true; + //skip lines at the top with nonessential info; stop skipping when "Nodes for the whole assembly" string is reached + while (searching_for_nodes&&in->good()) { + getline(*in, skip_line); + //std::cout << skip_line << std::endl; + line_parse.clear(); + line_parse.str(skip_line); + //stop when the NODES= string is reached + while (!line_parse.eof()){ + line_parse >> substring; + //std::cout << substring << std::endl; + if(!substring.compare("*Node")){ + searching_for_nodes = false; + break; + } + } //while + + } + if(searching_for_nodes){ + std::cout << "FILE FORMAT ERROR" << std::endl; + } + } + + //read coords, also density if restarting + read_index_start = 0; + for(buffer_iteration = 0; buffer_iteration < buffer_iterations; buffer_iteration++){ + //pack buffer on rank 0 + if(myrank==0&&buffer_iteration> substring; //skip node index column since coding for sorted inp + + for(int iword = 0; iword < words_per_line; iword++){ + //read portions of the line into the substring variable + line_parse >> substring; + //debug print + //std::cout<<" "<< substring <> substring; //skip node index column since coding for sorted inp + + for(int iword = 0; iword < words_per_line; iword++){ + //read portions of the line into the substring variable + line_parse >> substring; + //assign the substring variable as a word of the read buffer + strcpy(&read_buffer(buffer_loop,iword,0),substring.c_str()); + } + buffer_loop++; + } + + } + + //broadcast buffer to all ranks; each rank will determine which nodes in the buffer belong + MPI_Bcast(read_buffer.pointer(),BUFFER_LINES*words_per_line*MAX_WORD,MPI_CHAR,0,world); + //broadcast how many nodes were read into this buffer iteration + MPI_Bcast(&buffer_loop,1,MPI_INT,0,world); + + //debug_print + //std::cout << "NODE BUFFER LOOP IS: " << buffer_loop << std::endl; + //for(int iprint=0; iprint < buffer_loop; iprint++) + //std::cout<<"buffer packing: " << std::string(&read_buffer(iprint,0,0)) << std::endl; + //return; + + //determine which data to store in the swage mesh members (the local node data) + //loop through read buffer + for(scan_loop = 0; scan_loop < buffer_loop; scan_loop++){ + //set global node id (ensight specific order) + node_gid = read_index_start + scan_loop; + //let map decide if this node id belongs locally; if yes store data + if(map->isNodeGlobalElement(node_gid)){ + //set local node index in this mpi rank + node_rid = map->getLocalElement(node_gid); + //extract nodal position from the read buffer + //for tecplot format this is the three coords in the same line + dof_value = atof(&read_buffer(scan_loop,1,0)); + node_coords(node_rid, 0) = dof_value * unit_scaling; + dof_value = atof(&read_buffer(scan_loop,2,0)); + node_coords(node_rid, 1) = dof_value * unit_scaling; + dof_value = atof(&read_buffer(scan_loop,3,0)); + node_coords(node_rid, 2) = dof_value * unit_scaling; + //extract density if restarting + } + } + read_index_start+=BUFFER_LINES; + } + } //end view scope + //repartition node distribution + repartition_nodes(); + + //synchronize device data + //dual_node_coords.sync_device(); + //dual_node_coords.modify_device(); + //if(restart_file){ + //dual_node_densities.sync_device(); + //dual_node_densities.modify_device(); + //} + + //debug print of nodal data + + //debug print nodal positions and indices + /* + std::cout << " ------------NODAL POSITIONS ON TASK " << myrank << " --------------"<getGlobalElement(inode) + 1 << " { "; + for (int istride = 0; istride < num_dim; istride++){ + std::cout << node_coords(inode,istride) << " , "; + } + //std::cout << node_densities(inode,0); + std::cout << " }"<< std::endl; + } + */ + + //check that local assignments match global total + + + //read in element info + //seek element connectivity zone + int etype_index = 0; + if(myrank==0){ + bool searching_for_elements = true; + //skip lines at the top with nonessential info; stop skipping when "Nodes for the whole assembly" string is reached + while (searching_for_elements&&in->good()) { + getline(*in, skip_line); + //std::cout << skip_line << std::endl; + line_parse.clear(); + line_parse.str(skip_line); + //stop when the NODES= string is reached + while (!line_parse.eof()){ + line_parse >> substring; + //std::cout << substring << std::endl; + if(!substring.compare("*Element,")){ + searching_for_elements = false; + break; + } + } //while + + } + if(searching_for_elements){ + std::cout << "FILE FORMAT ERROR" << std::endl; + } + + if(in->good()) + first_elem_line_streampos = in->tellg(); + + //tally node count (bug in dat files seems to print wrong node count so read is done in two passes) + //stop when apparent "-1" zone delimiter is reacher + searching_for_elements = true; + int elem_tally = 0; + while (searching_for_elements&&in->good()) { + getline(*in, read_line); + //std::cout << read_line << std::endl; + line_parse.clear(); + line_parse.str(read_line); + line_parse >> substring; + + //std::cout << substring << std::endl; + if(substring == "*End Part"){ + searching_for_elements = false; + break; + } + else{ + elem_tally++; + } + + } + num_elem = elem_tally; + + in->seekg(first_elem_line_streampos); + } + + //broadcast element type + etype_index = 1; + MPI_Bcast(&etype_index,1,MPI_INT,0,world); + + elements::elem_types::elem_type mesh_element_type; + int elem_words_per_line_no_nodes = 0; + if(etype_index==1){ + mesh_element_type = elements::elem_types::Hex8; + nodes_per_element = 8; + //elem_words_per_line += 8; + max_nodes_per_patch = 4; + } + else if(etype_index==2){ + mesh_element_type = elements::elem_types::Hex20; + nodes_per_element = 20; + //elem_words_per_line += 20; + max_nodes_per_patch = 8; + } + else if(etype_index==3){ + mesh_element_type = elements::elem_types::Hex32; + nodes_per_element = 32; + //elem_words_per_line += 32; + max_nodes_per_patch = 12; + } + else{ + *fos << "ERROR: ABAQUS ELEMENT TYPE NOT FOUND OR RECOGNIZED" << std::endl; + exit_solver(0); + } + + //broadcast number of elements + MPI_Bcast(&num_elem,1,MPI_LONG_LONG_INT,0,world); + + *fos << "declared element count: " << num_elem << std::endl; + //std::cout<<"before initial mesh initialization"<(BUFFER_LINES,elem_words_per_line,MAX_WORD); + CArrayKokkos node_store(nodes_per_element); + + //calculate buffer iterations to read number of lines + buffer_iterations = num_elem/BUFFER_LINES; + int assign_flag; + + //dynamic buffer used to store elements before we know how many this rank needs + std::vector element_temp(BUFFER_LINES*elem_words_per_line); + std::vector global_indices_temp(BUFFER_LINES); + size_t buffer_max = BUFFER_LINES*elem_words_per_line; + size_t indices_buffer_max = BUFFER_LINES; + + if(num_elem%BUFFER_LINES!=0) buffer_iterations++; + read_index_start = 0; + //std::cout << "ELEMENT BUFFER ITERATIONS: " << buffer_iterations << std::endl; + rnum_elem = 0; + //std::cout << "BUFFER ITERATIONS IS: " << buffer_iterations << std::endl; + for(buffer_iteration = 0; buffer_iteration < buffer_iterations; buffer_iteration++){ + //pack buffer on rank 0 + if(myrank==0&&buffer_iteration> substring; //skip elem gid since coding for sorted inp + for(int iword = 0; iword < elem_words_per_line; iword++){ + //read portions of the line into the substring variable + line_parse >> substring; + //assign the substring variable as a word of the read buffer + strcpy(&read_buffer(buffer_loop,iword,0),substring.c_str()); + } + } + } + else if(myrank==0){ + buffer_loop=0; + while(buffer_iteration*BUFFER_LINES+buffer_loop < num_elem) { + getline(*in,read_line); + line_parse.clear(); + line_parse.str(read_line); + line_parse >> substring; //skip elem gid since coding for sorted inp + for(int iword = 0; iword < elem_words_per_line; iword++){ + //read portions of the line into the substring variable + line_parse >> substring; + //assign the substring variable as a word of the read buffer + strcpy(&read_buffer(buffer_loop,iword,0),substring.c_str()); + } + buffer_loop++; + //std::cout<<" "<< node_coords(node_gid, 0)<isNodeGlobalElement(node_gid)&&!assign_flag){ + assign_flag = 1; + rnum_elem++; + } + } + else{ + if(map->isNodeGlobalElement(node_gid-1)&&!assign_flag){ + assign_flag = 1; + rnum_elem++; + } + } + } + + if(assign_flag){ + for(int inode = 0; inode < nodes_per_element; inode++){ + if((rnum_elem-1)*nodes_per_element + inode>=buffer_max){ + element_temp.resize((rnum_elem-1)*nodes_per_element + inode + BUFFER_LINES*nodes_per_element); + buffer_max = (rnum_elem-1)*nodes_per_element + inode + BUFFER_LINES*nodes_per_element; + } + element_temp[(rnum_elem-1)*nodes_per_element + inode] = node_store(inode); + //std::cout << "VECTOR STORAGE FOR ELEM " << rnum_elem << " ON TASK " << myrank << " NODE " << inode+1 << " IS " << node_store(inode) + 1 << std::endl; + } + //assign global element id to temporary list + if(rnum_elem-1>=indices_buffer_max){ + global_indices_temp.resize(rnum_elem-1 + BUFFER_LINES); + indices_buffer_max = rnum_elem-1 + BUFFER_LINES; + } + global_indices_temp[rnum_elem-1] = elem_gid; + } + } + read_index_start+=BUFFER_LINES; + } + + //check if ABAQUS file has boundary and loading condition zones + bool No_Conditions = true; + if(myrank==0){ + if(in->good()) + before_condition_header = in->tellg(); + bool searching_for_conditions = true; + //skip lines at the top with nonessential info; stop skipping when "Fixed Supports or Pressure" string is reached + while (searching_for_conditions&&in->good()) { + getline(*in, skip_line); + //std::cout << skip_line << std::endl; + line_parse.clear(); + line_parse.str(skip_line); + //stop when the NODES= string is reached + while (!line_parse.eof()){ + line_parse >> substring; + //std::cout << substring << std::endl; + if(!substring.compare("Supports")||!substring.compare("Pressure")){ + No_Conditions = searching_for_conditions = false; + break; + } + } //while + + } //while + } + + //broadcast search condition + MPI_Bcast(&No_Conditions,1,MPI_CXX_BOOL,0,world); + + //flag elasticity fea module for boundary/loading conditions readin that remains + if(!No_Conditions){ + // check that the input file has configured some kind of acceptable module + simparam.validate_module_is_specified(FEA_MODULE_TYPE::Elasticity); + simparam.fea_module_must_read.insert(FEA_MODULE_TYPE::Elasticity); + } + + // Close mesh input file if no further readin is done by FEA modules for conditions + if(myrank==0&&No_Conditions){ + in->close(); + } + + std::cout << "RNUM ELEMENTS IS: " << rnum_elem << std::endl; + //copy temporary element storage to multivector storage + Element_Types = CArrayKokkos(rnum_elem); + + //set element object pointer + if(simparam.num_dims==2){ + element_select->choose_2Delem_type(mesh_element_type, elem2D); + max_nodes_per_element = elem2D->num_nodes(); + } + else if(simparam.num_dims==3){ + element_select->choose_3Delem_type(mesh_element_type, elem); + max_nodes_per_element = elem->num_nodes(); + } + + //1 type per mesh for now + for(int ielem = 0; ielem < rnum_elem; ielem++) + Element_Types(ielem) = mesh_element_type; + + dual_nodes_in_elem = dual_elem_conn_array("dual_nodes_in_elem", rnum_elem, max_nodes_per_element); + host_elem_conn_array nodes_in_elem = dual_nodes_in_elem.view_host(); + dual_nodes_in_elem.modify_host(); + + for(int ielem = 0; ielem < rnum_elem; ielem++) + for(int inode = 0; inode < nodes_per_element; inode++){ + nodes_in_elem(ielem, inode) = element_temp[ielem*nodes_per_element + inode]; + } + + //view storage for all local elements connected to local nodes on this rank + Kokkos::DualView All_Element_Global_Indices("All_Element_Global_Indices",rnum_elem); + + //copy temporary global indices storage to view storage + for(int ielem = 0; ielem < rnum_elem; ielem++){ + All_Element_Global_Indices.h_view(ielem) = global_indices_temp[ielem]; + if(global_indices_temp[ielem]<0){ + negative_index_found = 1; + } + } + + MPI_Allreduce(&negative_index_found,&global_negative_index_found,1,MPI_INT,MPI_MAX,MPI_COMM_WORLD); + if(global_negative_index_found){ + if(myrank==0){ + std::cout << "Node index less than or equal to zero detected; set \"zero_index_base: true\" under \"input_options\" in your yaml file if indices start at 0" << std::endl; + } + exit_solver(0); + } + + //debug print element edof + /* + std::cout << " ------------ELEMENT EDOF ON TASK " << myrank << " --------------"<().swap(element_temp); + std::vector().swap(global_indices_temp); + + All_Element_Global_Indices.modify_host(); + All_Element_Global_Indices.sync_device(); + + //construct overlapping element map (since different ranks can own the same elements due to the local node map) + all_element_map = Teuchos::rcp( new Tpetra::Map(Teuchos::OrdinalTraits::invalid(),All_Element_Global_Indices.d_view,0,comm)); + + + //element type selection (subject to change) + // ---- Set Element Type ---- // + // allocate element type memory + //elements::elem_type_t* elem_choice; + + int NE = 1; // number of element types in problem + + // Convert ijk index system to the finite element numbering convention + // for vertices in cell + CArrayKokkos convert_ensight_to_ijk(max_nodes_per_element); + CArrayKokkos tmp_ijk_indx(max_nodes_per_element); + convert_ensight_to_ijk(0) = 0; + convert_ensight_to_ijk(1) = 1; + convert_ensight_to_ijk(2) = 3; + convert_ensight_to_ijk(3) = 2; + convert_ensight_to_ijk(4) = 4; + convert_ensight_to_ijk(5) = 5; + convert_ensight_to_ijk(6) = 7; + convert_ensight_to_ijk(7) = 6; + + if(num_dim==2) + for (int cell_rid = 0; cell_rid < rnum_elem; cell_rid++) { + //set nodes per element + element_select->choose_2Delem_type(Element_Types(cell_rid), elem2D); + nodes_per_element = elem2D->num_nodes(); + for (int node_lid = 0; node_lid < nodes_per_element; node_lid++){ + tmp_ijk_indx(node_lid) = nodes_in_elem(cell_rid, convert_ensight_to_ijk(node_lid)); + } + + for (int node_lid = 0; node_lid < nodes_per_element; node_lid++){ + nodes_in_elem(cell_rid, node_lid) = tmp_ijk_indx(node_lid); + } + } + + if(num_dim==3) + for (int cell_rid = 0; cell_rid < rnum_elem; cell_rid++) { + //set nodes per element + element_select->choose_3Delem_type(Element_Types(cell_rid), elem); + nodes_per_element = elem->num_nodes(); + for (int node_lid = 0; node_lid < nodes_per_element; node_lid++){ + tmp_ijk_indx(node_lid) = nodes_in_elem(cell_rid, convert_ensight_to_ijk(node_lid)); + } + + for (int node_lid = 0; node_lid < nodes_per_element; node_lid++){ + nodes_in_elem(cell_rid, node_lid) = tmp_ijk_indx(node_lid); + } + } + +} // end read_mesh + /* ---------------------------------------------------------------------- Rebalance the initial node decomposition with Zoltan2 ------------------------------------------------------------------------- */ diff --git a/src/Parallel-Solvers/Solver.h b/src/Parallel-Solvers/Solver.h index c638108e7..be1eed83a 100644 --- a/src/Parallel-Solvers/Solver.h +++ b/src/Parallel-Solvers/Solver.h @@ -138,6 +138,8 @@ class Solver virtual void read_mesh_vtk(const char* MESH); + virtual void read_mesh_abaqus_inp(const char* MESH); //abaqus inp format reader + virtual void repartition_nodes(); virtual void comm_importer_setup(); @@ -262,7 +264,7 @@ class Solver // file readin variables std::ifstream* in = NULL; - std::streampos before_condition_header; + std::streampos before_condition_header, first_elem_line_streampos; std::string filename; int words_per_line, elem_words_per_line; enum node_ordering_convention { IJK, ENSIGHT }; From d12cfc3c6beb8607683ab6f6ce9b92c692bef6b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CAdrian-Diaz=E2=80=9D?= <“adriandiaz1117@gmail.com”> Date: Mon, 4 Mar 2024 16:45:53 -0700 Subject: [PATCH 52/64] WIP: Abaqus inp reader --- .../Implicit-Lagrange/Implicit_Solver.cpp | 3 ++ src/Parallel-Solvers/Solver.cpp | 37 ++++++++++++------- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/src/Parallel-Solvers/Implicit-Lagrange/Implicit_Solver.cpp b/src/Parallel-Solvers/Implicit-Lagrange/Implicit_Solver.cpp index 451b9adae..5469bba14 100644 --- a/src/Parallel-Solvers/Implicit-Lagrange/Implicit_Solver.cpp +++ b/src/Parallel-Solvers/Implicit-Lagrange/Implicit_Solver.cpp @@ -166,6 +166,9 @@ void Implicit_Solver::run(){ case MESH_FORMAT::ensight: read_mesh_ensight(mesh_file_name); break; + case MESH_FORMAT::abaqus_inp: + read_mesh_abaqus_inp(mesh_file_name); + break; } } else { generate_mesh(simparam.mesh_generation_options.value()); diff --git a/src/Parallel-Solvers/Solver.cpp b/src/Parallel-Solvers/Solver.cpp index 9591fd22a..452875821 100644 --- a/src/Parallel-Solvers/Solver.cpp +++ b/src/Parallel-Solvers/Solver.cpp @@ -2479,12 +2479,14 @@ void Solver::read_mesh_abaqus_inp(const char *MESH){ node_rid = map->getLocalElement(node_gid); //extract nodal position from the read buffer //for tecplot format this is the three coords in the same line - dof_value = atof(&read_buffer(scan_loop,1,0)); + dof_value = atof(&read_buffer(scan_loop,0,0)); node_coords(node_rid, 0) = dof_value * unit_scaling; - dof_value = atof(&read_buffer(scan_loop,2,0)); + dof_value = atof(&read_buffer(scan_loop,1,0)); node_coords(node_rid, 1) = dof_value * unit_scaling; - dof_value = atof(&read_buffer(scan_loop,3,0)); - node_coords(node_rid, 2) = dof_value * unit_scaling; + if(num_dim==3){ + dof_value = atof(&read_buffer(scan_loop,2,0)); + node_coords(node_rid, 2) = dof_value * unit_scaling; + } //extract density if restarting } } @@ -2555,13 +2557,14 @@ void Solver::read_mesh_abaqus_inp(const char *MESH){ int elem_tally = 0; while (searching_for_elements&&in->good()) { getline(*in, read_line); - //std::cout << read_line << std::endl; + // if(elem_tally < 1000) + // std::cout << read_line << std::endl; line_parse.clear(); line_parse.str(read_line); line_parse >> substring; //std::cout << substring << std::endl; - if(substring == "*End Part"){ + if(substring == "*End"){ searching_for_elements = false; break; } @@ -2584,19 +2587,19 @@ void Solver::read_mesh_abaqus_inp(const char *MESH){ if(etype_index==1){ mesh_element_type = elements::elem_types::Hex8; nodes_per_element = 8; - //elem_words_per_line += 8; + elem_words_per_line = 8; max_nodes_per_patch = 4; } else if(etype_index==2){ mesh_element_type = elements::elem_types::Hex20; nodes_per_element = 20; - //elem_words_per_line += 20; + elem_words_per_line = 20; max_nodes_per_patch = 8; } else if(etype_index==3){ mesh_element_type = elements::elem_types::Hex32; nodes_per_element = 32; - //elem_words_per_line += 32; + elem_words_per_line = 32; max_nodes_per_patch = 12; } else{ @@ -2618,7 +2621,6 @@ void Solver::read_mesh_abaqus_inp(const char *MESH){ //calculate buffer iterations to read number of lines buffer_iterations = num_elem/BUFFER_LINES; int assign_flag; - //dynamic buffer used to store elements before we know how many this rank needs std::vector element_temp(BUFFER_LINES*elem_words_per_line); std::vector global_indices_temp(BUFFER_LINES); @@ -2635,15 +2637,21 @@ void Solver::read_mesh_abaqus_inp(const char *MESH){ if(myrank==0&&buffer_iteration> substring; //skip elem gid since coding for sorted inp + std::getline(line_parse, substring, ',');; //skip elem gid since coding for sorted inp for(int iword = 0; iword < elem_words_per_line; iword++){ //read portions of the line into the substring variable - line_parse >> substring; + std::getline(line_parse, substring, ','); + //line_parse >> substring; //assign the substring variable as a word of the read buffer strcpy(&read_buffer(buffer_loop,iword,0),substring.c_str()); + std::cout<< substring << " "; } + std::cout<< std::endl; } } else if(myrank==0){ @@ -2652,10 +2660,11 @@ void Solver::read_mesh_abaqus_inp(const char *MESH){ getline(*in,read_line); line_parse.clear(); line_parse.str(read_line); - line_parse >> substring; //skip elem gid since coding for sorted inp + std::getline(line_parse, substring, ','); //skip elem gid since coding for sorted inp for(int iword = 0; iword < elem_words_per_line; iword++){ //read portions of the line into the substring variable - line_parse >> substring; + std::getline(line_parse, substring, ','); + //line_parse >> substring; //assign the substring variable as a word of the read buffer strcpy(&read_buffer(buffer_loop,iword,0),substring.c_str()); } From 19d37c8e5c81ad691bd1de2ee96b4d1610dc5505 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CAdrian-Diaz=E2=80=9D?= <“adriandiaz1117@gmail.com”> Date: Mon, 4 Mar 2024 22:14:58 -0700 Subject: [PATCH 53/64] BUG: remove print, add default case for mesh read --- .../Implicit-Lagrange/Implicit_Solver.cpp | 3 +++ .../Parallel-Explicit/Explicit_Solver.cpp | 9 ++++++++- src/Parallel-Solvers/Solver.cpp | 2 -- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/Parallel-Solvers/Implicit-Lagrange/Implicit_Solver.cpp b/src/Parallel-Solvers/Implicit-Lagrange/Implicit_Solver.cpp index 5469bba14..f6b04873f 100644 --- a/src/Parallel-Solvers/Implicit-Lagrange/Implicit_Solver.cpp +++ b/src/Parallel-Solvers/Implicit-Lagrange/Implicit_Solver.cpp @@ -169,6 +169,9 @@ void Implicit_Solver::run(){ case MESH_FORMAT::abaqus_inp: read_mesh_abaqus_inp(mesh_file_name); break; + default: + *fos << "ERROR: MESH FILE FORMAT NOT SUPPORTED BY IMPLICIT SOLVER" << std::endl; + exit_solver(0); } } else { generate_mesh(simparam.mesh_generation_options.value()); diff --git a/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp b/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp index 25d07fe4c..4b55492c0 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp @@ -176,6 +176,13 @@ void Explicit_Solver::run() { case MESH_FORMAT::ensight: read_mesh_ensight(mesh_file_name); break; + case MESH_FORMAT::abaqus_inp: + read_mesh_abaqus_inp(mesh_file_name); + break; + default: + *fos << "ERROR: MESH FILE FORMAT NOT SUPPORTED BY EXPLICIT SOLVER" << std::endl; + exit_solver(0); + } } else { generate_mesh(simparam.mesh_generation_options.value()); @@ -1355,7 +1362,7 @@ void Explicit_Solver::setup_optimization_problem(){ const size_t num_fills = simparam.optimization_options.volume_bound_constraints.size(); const_host_vec_array node_coords_view = node_coords_distributed->getLocalView (Tpetra::Access::ReadOnly); const DCArrayKokkos mat_fill = simparam.optimization_options.optimization_bound_constraint_volumes; - + for(int ifill = 0; ifill < num_fills; ifill++){ for(int inode = 0; inode < nlocal_nodes; inode++){ real_t node_coords[3]; diff --git a/src/Parallel-Solvers/Solver.cpp b/src/Parallel-Solvers/Solver.cpp index 452875821..b72d5eea6 100644 --- a/src/Parallel-Solvers/Solver.cpp +++ b/src/Parallel-Solvers/Solver.cpp @@ -2649,9 +2649,7 @@ void Solver::read_mesh_abaqus_inp(const char *MESH){ //line_parse >> substring; //assign the substring variable as a word of the read buffer strcpy(&read_buffer(buffer_loop,iword,0),substring.c_str()); - std::cout<< substring << " "; } - std::cout<< std::endl; } } else if(myrank==0){ From 2696011e4007053053558ff5154325f4a93a8575 Mon Sep 17 00:00:00 2001 From: Adrian-Diaz <46537160+Adrian-Diaz@users.noreply.github.com> Date: Tue, 5 Mar 2024 11:18:59 -0700 Subject: [PATCH 54/64] WIP: add q2 term to gradients --- .../SGH_Solver/src/force_gradients_sgh.cpp | 94 ++++++++++--------- .../SGH_Solver/src/power_gradients_sgh.cpp | 23 +++-- 2 files changed, 63 insertions(+), 54 deletions(-) diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp index e8defe6cd..27fd53b12 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp @@ -1,5 +1,5 @@ /********************************************************************************************** - 2020. Triad National Security, LLC. All rights reserved. + � 2020. Triad National Security, LLC. All rights reserved. This program was produced under U.S. Government contract 89233218CNA000001 for Los Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC for the U.S. Department of Energy/National Nuclear Security Administration. All rights in the program are @@ -107,22 +107,6 @@ void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& ma }); // end parallel for loop over nodes Kokkos::fence(); - // initialize buffer storage; not all components are explicitly set for this routine - FOR_ALL_CLASS(corner_gid, 0, num_corners, { - for (int dim = 0; dim < num_dims; dim++) - { - // assign gradient of corner contribution of force to relevant matrix entries with non-zero node velocity gradient - for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) - { - for (int jdim = 0; jdim < num_dims; jdim++) - { - corner_gradient_storage(corner_gid, dim, igradient, jdim) = 0; - } - } - } - }); // end parallel for loop over nodes - Kokkos::fence(); - // --- calculate the forces acting on the nodes from the element --- for (size_t elem_gid = 0; elem_gid < rnum_elem; elem_gid++) { @@ -142,14 +126,15 @@ void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& ma // the sums in the Riemann solver double sum_array[4]; + double sum_gradient_array[4 * max_nodes_per_element * num_dims]; // corner shock impeadance x |corner area normal dot shock_dir| double muc_array[8]; - double muc_gradient_array[8]; + double muc_gradient_array[8 * max_nodes_per_element * num_dims]; // Riemann velocity double vel_star_array[3]; - double vel_star_gradient_array[3 * num_nodes_in_elem]; + double vel_star_gradient_array[num_dims * max_nodes_per_element * num_dims]; // velocity gradient double vel_grad_array[9]; @@ -161,10 +146,11 @@ void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& ma ViewCArrayKokkos area_normal(area_normal_array, num_nodes_in_elem, num_dims); ViewCArrayKokkos shock_dir(shock_dir_array, num_dims); ViewCArrayKokkos sum(sum_array, 4); + ViewCArrayKokkos sum_gradient(sum_gradient_array, 4, max_nodes_per_element, num_dims); ViewCArrayKokkos muc(muc_array, num_nodes_in_elem); - ViewCArrayKokkos muc_gradient(muc_gradient_array, num_nodes_in_elem); + ViewCArrayKokkos muc_gradient(muc_gradient_array, num_nodes_in_elem, num_nodes_in_elem, num_dims); ViewCArrayKokkos vel_star(vel_star_array, num_dims); - ViewCArrayKokkos vel_star_gradient(vel_star_gradient_array, num_nodes_in_elem, num_dims); + ViewCArrayKokkos vel_star_gradient(vel_star_gradient_array, num_dims, num_nodes_in_elem, num_dims); ViewCArrayKokkos vel_grad(vel_grad_array, num_dims, num_dims); // --- abviatations of variables --- @@ -263,6 +249,16 @@ void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& ma { sum(i) = 0.0; } + for (int i = 0; i < 4; i++) + { + for (size_t igradient = 0; igradient < num_nodes_in_elem; igradient++) + { + for (size_t jdim = 0; jdim < num_dims; jdim++) + { + sum_gradient(i, igradient, jdim) = 0.0; + } + } + } double mag; // magnitude of the area normal double mag_vel; // magnitude of velocity @@ -365,24 +361,33 @@ void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& ma } } // end if - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) + // vel star gradients + if (sum(3) > fuzz) { - // The Riemann velocity, called vel_star - if (sum(3) > fuzz) + for (size_t i = 0; i < num_dims; i++) { - for (size_t i = 0; i < num_dims; i++) + for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) { - vel_star_gradient(node_lid, i) = muc(node_lid) / sum(3); + for (int jdim = 0; jdim < num_dims; jdim++) + { + vel_star_gradient(i, igradient, jdim) = sum_gradient(i, igradient, jdim) / sum(3) - sum(i) / (sum(3) * sum(3)) * sum_gradient(3, igradient, jdim); + } } } - else + } + else + { + for (int i = 0; i < num_dims; i++) { - for (int i = 0; i < num_dims; i++) + for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) { - vel_star_gradient(node_lid, i) = 0; + for (int jdim = 0; jdim < num_dims; jdim++) + { + vel_star_gradient(i, igradient, jdim) = 0; + } } - } // end if - } + } + } // end if // ---- Calculate the shock detector for the Riemann-solver ---- // @@ -463,23 +468,25 @@ void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& ma for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) { size_t gradient_node_gid = nodes_in_elem(elem_gid, igradient); - // if(!map->isNodeLocalElement(gradient_node_gid)) continue; column_index = num_dims * Global_Gradient_Matrix_Assembly_Map(elem_gid, igradient, node_lid); - if (node_lid == igradient) + for (int jdim = 0; jdim < num_dims; jdim++) { - if (map->isNodeLocalElement(gradient_node_gid)) + if (node_lid == igradient && jdim == dim) { - Force_Gradient_Velocities(gradient_node_gid * num_dims + dim, column_index + dim) += phi * muc(node_lid) * (vel_star_gradient(igradient, dim) - 1); + if (map->isNodeLocalElement(gradient_node_gid)) + { + Force_Gradient_Velocities(gradient_node_gid * num_dims + jdim, column_index + dim) += phi * muc(node_lid) * (vel_star_gradient(igradient, jdim) - 1); + } + corner_gradient_storage(corner_gid, dim, igradient, jdim) = phi * muc(node_lid) * (vel_star_gradient(igradient, jdim) - 1); } - corner_gradient_storage(corner_gid, dim, igradient, dim) = phi * muc(node_lid) * (vel_star_gradient(igradient, dim) - 1); - } - else - { - if (map->isNodeLocalElement(gradient_node_gid)) + else { - Force_Gradient_Velocities(gradient_node_gid * num_dims + dim, column_index + dim) += phi * muc(node_lid) * (vel_star_gradient(igradient, dim)); + if (map->isNodeLocalElement(gradient_node_gid)) + { + Force_Gradient_Velocities(gradient_node_gid * num_dims + jdim, column_index + dim) += phi * muc(node_lid) * (vel_star_gradient(igradient, jdim)); + } + corner_gradient_storage(corner_gid, dim, igradient, jdim) = phi * muc(node_lid) * (vel_star_gradient(igradient, jdim)); } - corner_gradient_storage(corner_gid, dim, igradient, dim) = phi * muc(node_lid) * (vel_star_gradient(igradient, dim)); } } } // end loop over dimension @@ -1514,8 +1521,7 @@ void FEA_Module_SGH::get_force_ugradient_sgh(const DCArrayKokkos& ma + area_normal_gradients(node_lid, 0, igradient, jdim) * tau(0, dim) + area_normal_gradients(node_lid, 1, igradient, jdim) * tau(1, dim) + area_normal_gradients(node_lid, 2, igradient, jdim) * tau(2, dim) - + phi * muc_gradient(node_lid, igradient, - jdim) * (vel_star(dim) - node_vel(rk_level, node_gid, dim)) + + phi * muc_gradient(node_lid, igradient, jdim) * (vel_star(dim) - node_vel(rk_level, node_gid, dim)) + phi * muc(node_lid) * (vel_star_gradient(dim, igradient, jdim)); } corner_gradient_storage(corner_gid, dim, igradient, jdim) = area_normal(node_lid, 0) * tau_gradient(0, dim, igradient, jdim) diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/power_gradients_sgh.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/power_gradients_sgh.cpp index 21d5483d0..a914741fb 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/power_gradients_sgh.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/power_gradients_sgh.cpp @@ -1,5 +1,5 @@ /********************************************************************************************** - 2020. Triad National Security, LLC. All rights reserved. + � 2020. Triad National Security, LLC. All rights reserved. This program was produced under U.S. Government contract 89233218CNA000001 for Los Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC for the U.S. Department of Energy/National Nuclear Security Administration. All rights in the program are @@ -489,16 +489,19 @@ void FEA_Module_SGH::get_power_vgradient_sgh(double rk_alpha, { continue; } - if (node_lid == igradient) - { - Power_Gradient_Velocities(gradient_node_id * num_dims + dim, column_id) -= - corner_gradient_storage(corner_gid, dim, igradient, dim) * node_vel(rk_level, node_gid, dim) - * node_radius + corner_force(corner_gid, dim) * node_radius; - } - else + for (size_t jdim = 0; jdim < num_dims; jdim++) { - Power_Gradient_Velocities(gradient_node_id * num_dims + dim, column_id) -= - corner_gradient_storage(corner_gid, dim, igradient, dim) * node_vel(rk_level, node_gid, dim) * node_radius; + if (node_lid == igradient&&jdim==dim) + { + Power_Gradient_Velocities(gradient_node_id * num_dims + jdim, column_id) -= + corner_gradient_storage(corner_gid, dim, igradient, jdim) * node_vel(rk_level, node_gid, dim) + * node_radius + corner_force(corner_gid, dim) * node_radius; + } + else + { + Power_Gradient_Velocities(gradient_node_id * num_dims + jdim, column_id) -= + corner_gradient_storage(corner_gid, dim, igradient, jdim) * node_vel(rk_level, node_gid, dim) * node_radius; + } } } } // end for dim From 52ee547e50ba83e871332114fafa63d013e17587 Mon Sep 17 00:00:00 2001 From: Adrian-Diaz <46537160+Adrian-Diaz@users.noreply.github.com> Date: Tue, 5 Mar 2024 15:24:19 -0700 Subject: [PATCH 55/64] ENH: add q2 to gradient terms --- .../SGH_Solver/src/force_gradients_sgh.cpp | 99 ++++++++++++++++--- 1 file changed, 88 insertions(+), 11 deletions(-) diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp index 27fd53b12..39f2e272b 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp @@ -136,6 +136,9 @@ void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& ma double vel_star_array[3]; double vel_star_gradient_array[num_dims * max_nodes_per_element * num_dims]; + //mag_vel gradient + double mag_vel_gradient_array[max_nodes_per_element * num_dims]; + // velocity gradient double vel_grad_array[9]; @@ -146,12 +149,13 @@ void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& ma ViewCArrayKokkos area_normal(area_normal_array, num_nodes_in_elem, num_dims); ViewCArrayKokkos shock_dir(shock_dir_array, num_dims); ViewCArrayKokkos sum(sum_array, 4); - ViewCArrayKokkos sum_gradient(sum_gradient_array, 4, max_nodes_per_element, num_dims); + ViewCArrayKokkos sum_gradient(sum_gradient_array, 4, num_nodes_in_elem, num_dims); ViewCArrayKokkos muc(muc_array, num_nodes_in_elem); ViewCArrayKokkos muc_gradient(muc_gradient_array, num_nodes_in_elem, num_nodes_in_elem, num_dims); ViewCArrayKokkos vel_star(vel_star_array, num_dims); ViewCArrayKokkos vel_star_gradient(vel_star_gradient_array, num_dims, num_nodes_in_elem, num_dims); ViewCArrayKokkos vel_grad(vel_grad_array, num_dims, num_dims); + ViewCArrayKokkos mag_vel_gradient(mag_vel_gradient_array, num_nodes_in_elem, num_dims); // --- abviatations of variables --- @@ -237,9 +241,9 @@ void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& ma ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); - vel_star(0) += 0.125 * vel(0); - vel_star(1) += 0.125 * vel(1); - vel_star(2) += 0.125 * vel(2); + vel_star(0) += vel(0)/num_nodes_in_elem; + vel_star(1) += vel(1)/num_nodes_in_elem; + vel_star(2) += vel(2)/num_nodes_in_elem; } // end for loop over nodes // find shock direction and shock impedance associated with each node @@ -277,6 +281,27 @@ void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& ma + (vel(1) - vel_star(1) ) * (vel(1) - vel_star(1) ) + (vel(2) - vel_star(2) ) * (vel(2) - vel_star(2) ) ); + for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) + { + for(int jdim = 0; jdim < num_dim; jdim++) + { + if(igradient == node_lid) + { + mag_vel_gradient(igradient,jdim) = (1 - 1/num_nodes_in_elem)*((vel(jdim) - vel_star(jdim) ))/ + sqrt( (vel(0) - vel_star(0) ) * (vel(0) - vel_star(0) ) + + (vel(1) - vel_star(1) ) * (vel(1) - vel_star(1) ) + + (vel(2) - vel_star(2) ) * (vel(2) - vel_star(2) ) ); + } + else + { + mag_vel_gradient(igradient,jdim) = -1/num_nodes_in_elem*( (vel(jdim) - vel_star(jdim) ))/ + sqrt( (vel(0) - vel_star(0) ) * (vel(0) - vel_star(0) ) + + (vel(1) - vel_star(1) ) * (vel(1) - vel_star(1) ) + + (vel(2) - vel_star(2) ) * (vel(2) - vel_star(2) ) ); + } + } + } + if (mag_vel > small) { // estimate of the shock direction, a unit normal @@ -306,11 +331,26 @@ void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& ma { muc(node_lid) = elem_den(elem_gid) * (material(mat_id).q1 * elem_sspd(elem_gid) + material(mat_id).q2 * mag_vel); + + for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) + { + for(int jdim = 0; jdim < num_dim; jdim++) + { + muc_gradient(node_lid, igradient, jdim) = material(mat_id).q2 * mag_vel_gradient(igradient,jdim); + } + } } else // element in expansion { muc(node_lid) = elem_den(elem_gid) * (material(mat_id).q1ex * elem_sspd(elem_gid) + material(mat_id).q2ex * mag_vel); + for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) + { + for(int jdim = 0; jdim < num_dim; jdim++) + { + muc_gradient(node_lid, igradient, jdim) = material(mat_id).q2ex * mag_vel_gradient(igradient,jdim); + } + } } // end if on divergence sign size_t use_shock_dir = 0; @@ -333,8 +373,19 @@ void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& ma // Using a full tensoral Riemann jump relation mu_term = muc(node_lid) * sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) - + area_normal(node_lid, 1) * area_normal(node_lid, 1) - + area_normal(node_lid, 2) * area_normal(node_lid, 2) ); + + area_normal(node_lid, 1) * area_normal(node_lid, 1) + + area_normal(node_lid, 2) * area_normal(node_lid, 2) ); + + for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) + { + for (int jdim = 0; jdim < num_dims; jdim++) + { + muc_gradient(node_lid, igradient, jdim) = muc_gradient(node_lid, igradient, jdim) + * sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) + + area_normal(node_lid, 1) * area_normal(node_lid, 1) + + area_normal(node_lid, 2) * area_normal(node_lid, 2) ); + } + } } sum(0) += mu_term * vel(0); @@ -342,6 +393,28 @@ void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& ma sum(2) += mu_term * vel(2); sum(3) += mu_term; + // sum gradients + + for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) + { + for (int jdim = 0; jdim < num_dims; jdim++) + { + if(igradient==node_lid&&jdim==0) + sum_gradient(0, igradient, jdim) += muc(node_lid); + + if(igradient==node_lid&&jdim==1) + sum_gradient(1, igradient, jdim) += muc(node_lid); + + if(igradient==node_lid&&jdim==2) + sum_gradient(2, igradient, jdim) += muc(node_lid); + + sum_gradient(0, igradient, jdim) += muc_gradient(node_lid, igradient, jdim) * vel(0); + sum_gradient(1, igradient, jdim) += muc_gradient(node_lid, igradient, jdim) * vel(1); + sum_gradient(2, igradient, jdim) += muc_gradient(node_lid, igradient, jdim) * vel(2); + sum_gradient(3, igradient, jdim) += muc_gradient(node_lid, igradient, jdim); + } + } + muc(node_lid) = mu_term; // the impeadance time surface area is stored here } // end for node_lid loop over nodes of the elem @@ -474,18 +547,22 @@ void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& ma if (node_lid == igradient && jdim == dim) { if (map->isNodeLocalElement(gradient_node_gid)) - { - Force_Gradient_Velocities(gradient_node_gid * num_dims + jdim, column_index + dim) += phi * muc(node_lid) * (vel_star_gradient(igradient, jdim) - 1); + { //node_vel(rk_level, node_gid, dim) + Force_Gradient_Velocities(gradient_node_gid * num_dims + jdim, column_index + dim) += phi * (muc(node_lid) * (vel_star_gradient(dim, igradient, jdim) - 1) + + muc_gradient(node_lid, igradient, jdim) * (vel_star(dim) - node_vel(rk_level, node_gid, dim))); } - corner_gradient_storage(corner_gid, dim, igradient, jdim) = phi * muc(node_lid) * (vel_star_gradient(igradient, jdim) - 1); + corner_gradient_storage(corner_gid, dim, igradient, jdim) = phi * (muc(node_lid) * (vel_star_gradient(dim, igradient, jdim) - 1) + + muc_gradient(node_lid, igradient, jdim) * (vel_star(dim) - node_vel(rk_level, node_gid, dim))); } else { if (map->isNodeLocalElement(gradient_node_gid)) { - Force_Gradient_Velocities(gradient_node_gid * num_dims + jdim, column_index + dim) += phi * muc(node_lid) * (vel_star_gradient(igradient, jdim)); + Force_Gradient_Velocities(gradient_node_gid * num_dims + jdim, column_index + dim) += phi * (muc(node_lid) * (vel_star_gradient(dim, igradient, jdim))+ + muc_gradient(node_lid, igradient, jdim) * (vel_star(dim) - node_vel(rk_level, node_gid, dim))); } - corner_gradient_storage(corner_gid, dim, igradient, jdim) = phi * muc(node_lid) * (vel_star_gradient(igradient, jdim)); + corner_gradient_storage(corner_gid, dim, igradient, jdim) = phi * (muc(node_lid) * (vel_star_gradient(dim, igradient, jdim))+ + muc_gradient(node_lid, igradient, jdim) * (vel_star(dim) - node_vel(rk_level, node_gid, dim))); } } } From 3122478444aab161d44abc0f436b732c969fbca5 Mon Sep 17 00:00:00 2001 From: Jacob Moore Date: Wed, 6 Mar 2024 12:18:27 -0600 Subject: [PATCH 56/64] BUG: Fix missed geometry merge and some style stuff in optimization --- .../SGH_Solver/src/geometry.cpp | 2 +- .../SGH_Solver/src/sgh_optimization.cpp | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/geometry.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/geometry.cpp index d098a8d89..65e1e0085 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/geometry.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/geometry.cpp @@ -1684,7 +1684,7 @@ void FEA_Module_SGH::get_bmatrix_gradients(const ViewCArrayKokkos& B_mat gradient_terms(0, 7) = y(0) + y(3) - y(5) - y(6); // y derivative - gradient_terms(1, 0) = -x(1) + x(3) - x(4) + x(7); + gradient_terms(1, 0) = -x(1) + x(3) - x(5) + x(7); gradient_terms(1, 1) = x(0) - x(5); gradient_terms(1, 2) = 0; gradient_terms(1, 3) = -x(0) + x(7); diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/sgh_optimization.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/sgh_optimization.cpp index fd4822ca3..c0982d635 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/sgh_optimization.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/sgh_optimization.cpp @@ -869,7 +869,7 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full() phi_previous_adjoint_vector(node_gid, idim) / node_mass(node_gid); midpoint_adjoint_vector(node_gid, idim) = -rate_of_change * global_dt / 2 + previous_adjoint_vector(node_gid, idim); } - }); // end parallel for + }); // end parallel for Kokkos::fence(); // half step update for RK2 scheme; EQUATION 2 @@ -900,7 +900,7 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full() // rate_of_change = -0.0000001*previous_adjoint_vector(node_gid,idim); phi_midpoint_adjoint_vector(node_gid, idim) = -rate_of_change * global_dt / 2 + phi_previous_adjoint_vector(node_gid, idim); } - }); // end parallel for + }); // end parallel for Kokkos::fence(); // phi_adjoint_vector_distributed->describe(*fos,Teuchos::VERB_EXTREME); @@ -922,7 +922,7 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full() rate_of_change = -(matrix_contribution + psi_previous_adjoint_vector(elem_gid, 0) * Power_Gradient_Energies(elem_gid)) / elem_mass(elem_gid); // rate_of_change = -0.0000001*previous_adjoint_vector(node_gid,idim); psi_midpoint_adjoint_vector(elem_gid, 0) = -rate_of_change * global_dt / 2 + psi_previous_adjoint_vector(elem_gid, 0); - }); // end parallel for + }); // end parallel for Kokkos::fence(); // apply BCs to adjoint vector, only matters for the momentum adjoint if using strictly velocity boundary conditions @@ -948,12 +948,12 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full() node_vel(rk_level, node_gid, idim) = 0.5 * (previous_velocity_vector(node_gid, idim) + current_velocity_vector(node_gid, idim)); node_coords(rk_level, node_gid, idim) = 0.5 * (previous_coordinate_vector(node_gid, idim) + current_coordinate_vector(node_gid, idim)); } - }); + }); Kokkos::fence(); FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { elem_sie(rk_level, elem_gid) = 0.5 * (previous_element_internal_energy(elem_gid, 0) + current_element_internal_energy(elem_gid, 0)); - }); + }); Kokkos::fence(); // set state according to phase data at this timestep @@ -1149,7 +1149,7 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full() phi_midpoint_adjoint_vector(node_gid, idim) / node_mass(node_gid); current_adjoint_vector(node_gid, idim) = -rate_of_change * global_dt + previous_adjoint_vector(node_gid, idim); } - }); // end parallel for + }); // end parallel for Kokkos::fence(); // full step update with midpoint gradient for RK2 scheme; EQUATION 2 @@ -1180,7 +1180,7 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full() // rate_of_change = -0.0000001*midpoint_adjoint_vector(node_gid,idim); phi_current_adjoint_vector(node_gid, idim) = -rate_of_change * global_dt + phi_previous_adjoint_vector(node_gid, idim); } - }); // end parallel for + }); // end parallel for Kokkos::fence(); // full step update for RK2 scheme; EQUATION 3 @@ -1201,7 +1201,7 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full() // debug // std::cout << "PSI RATE OF CHANGE " << rate_of_change << std::endl; psi_current_adjoint_vector(elem_gid, 0) = -rate_of_change * global_dt + psi_previous_adjoint_vector(elem_gid, 0); - }); // end parallel for + }); // end parallel for Kokkos::fence(); boundary_adjoint(*mesh, boundary, current_adjoint_vector, phi_current_adjoint_vector, psi_current_adjoint_vector); From cdcdddc831129f2b748c4260daad0fd9935182fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CAdrian-Diaz=E2=80=9D?= <“adriandiaz1117@gmail.com”> Date: Wed, 6 Mar 2024 22:56:49 -0700 Subject: [PATCH 57/64] BUG: missing merge + q2 gradient bugs --- .../FEA_Module_Dynamic_Elasticity.cpp | 1 - .../SGH_Solver/src/FEA_Module_SGH.cpp | 1 - .../SGH_Solver/src/force_gradients_sgh.cpp | 11 +++++++++-- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.cpp b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.cpp index dc878f98f..0a7e68098 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.cpp @@ -98,7 +98,6 @@ FEA_Module_Dynamic_Elasticity::FEA_Module_Dynamic_Elasticity( // set Tpetra vector pointers initial_node_velocities_distributed = Explicit_Solver_Pointer_->initial_node_velocities_distributed; - node_coords_distributed = Explicit_Solver_Pointer_->node_coords_distributed; node_velocities_distributed = Explicit_Solver_Pointer_->node_velocities_distributed; all_node_velocities_distributed = Explicit_Solver_Pointer_->all_node_velocities_distributed; diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/FEA_Module_SGH.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/FEA_Module_SGH.cpp index 9bbe75b9a..c4525370d 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/FEA_Module_SGH.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/FEA_Module_SGH.cpp @@ -106,7 +106,6 @@ FEA_Module_SGH::FEA_Module_SGH( // set Tpetra vector pointers initial_node_velocities_distributed = Explicit_Solver_Pointer_->initial_node_velocities_distributed; - node_coords_distributed = Explicit_Solver_Pointer_->node_coords_distributed; node_velocities_distributed = Explicit_Solver_Pointer_->node_velocities_distributed; all_node_velocities_distributed = Explicit_Solver_Pointer_->all_node_velocities_distributed; diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp index 39f2e272b..4762c5ad2 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp @@ -291,6 +291,9 @@ void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& ma sqrt( (vel(0) - vel_star(0) ) * (vel(0) - vel_star(0) ) + (vel(1) - vel_star(1) ) * (vel(1) - vel_star(1) ) + (vel(2) - vel_star(2) ) * (vel(2) - vel_star(2) ) ); + if(mag_vel==0){ + mag_vel_gradient(igradient,jdim) = (1 - 1/num_nodes_in_elem); + } } else { @@ -298,6 +301,9 @@ void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& ma sqrt( (vel(0) - vel_star(0) ) * (vel(0) - vel_star(0) ) + (vel(1) - vel_star(1) ) * (vel(1) - vel_star(1) ) + (vel(2) - vel_star(2) ) * (vel(2) - vel_star(2) ) ); + if(mag_vel==0){ + mag_vel_gradient(igradient,jdim) = -1/num_nodes_in_elem; + } } } } @@ -337,6 +343,7 @@ void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& ma for(int jdim = 0; jdim < num_dim; jdim++) { muc_gradient(node_lid, igradient, jdim) = material(mat_id).q2 * mag_vel_gradient(igradient,jdim); + //if(muc_gradient(node_lid, igradient, jdim)!=0) std::cout << " NON ZERO " << muc_gradient(node_lid, igradient, jdim) << std::endl; } } } @@ -349,6 +356,7 @@ void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& ma for(int jdim = 0; jdim < num_dim; jdim++) { muc_gradient(node_lid, igradient, jdim) = material(mat_id).q2ex * mag_vel_gradient(igradient,jdim); + //if(muc_gradient(node_lid, igradient, jdim)!=0) std::cout << " NON ZERO " << muc_gradient(node_lid, igradient, jdim) << std::endl; } } } // end if on divergence sign @@ -394,7 +402,7 @@ void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& ma sum(3) += mu_term; // sum gradients - + muc(node_lid) = mu_term; // the impeadance time surface area is stored here for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) { for (int jdim = 0; jdim < num_dims; jdim++) @@ -415,7 +423,6 @@ void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& ma } } - muc(node_lid) = mu_term; // the impeadance time surface area is stored here } // end for node_lid loop over nodes of the elem // The Riemann velocity, called vel_star From 7bd0f5d075bca6f6e23a58f832145b998953938b Mon Sep 17 00:00:00 2001 From: Jacob Moore <133793250+jacob-moore22@users.noreply.github.com> Date: Thu, 7 Mar 2024 07:44:11 -0600 Subject: [PATCH 58/64] Integrated tests for SGH solver (#179) * ENH: Adding inputs for integrated testing * ENH: Improving integrated tests for SGH solver * ENH: Improving test scripts * ENH: Integrate tests into workflow * DOCS: Tweak comments in test script * BUG: Attempt to fix workflow to call python test scripts * BUG: Debugging workflow * BUG: removing build step to check script call * BUG: removing build step to check script call * BUG: Fixing path * BUG: Fixed path, added back in build step to workflow --- .github/workflows/ubuntu-build.yaml | 6 + .../Solver-Inputs/SGH_Noh_simple.yaml | 70 + .../Solver-Inputs/SGH_Sedov_simple.yaml | 29 +- .../Solver-Inputs/SGH_Sod_simple.yaml | 92 + .../SGH/Noh/vtk/data/VTK0.vtk | 10696 ++++++++++ .../SGH/Noh/vtk/outputs.vtk.series | 6 + .../SGH/Sedov}/vtk/data/VTK0.vtk | 16172 ++++++++-------- .../SGH/Sedov}/vtk/outputs.vtk.series | 0 .../SGH/Sod/vtk/data/VTK0.vtk | 1642 ++ .../SGH/Sod/vtk/outputs.vtk.series | 6 + .../Test-scripts/test_fierro.py | 66 +- .../Simulation_Parameters/Geometry.h | 2 +- 12 files changed, 20647 insertions(+), 8140 deletions(-) create mode 100644 integrated-tests/Test-scripts/Solver-Inputs/SGH_Noh_simple.yaml rename python/Test-scripts/Solver-Inputs/SGH_Sedov_12x12x12.yaml => integrated-tests/Test-scripts/Solver-Inputs/SGH_Sedov_simple.yaml (75%) create mode 100644 integrated-tests/Test-scripts/Solver-Inputs/SGH_Sod_simple.yaml create mode 100644 integrated-tests/Test-scripts/standard-results/SGH/Noh/vtk/data/VTK0.vtk create mode 100644 integrated-tests/Test-scripts/standard-results/SGH/Noh/vtk/outputs.vtk.series rename {python/Test-scripts/standard-results/SGH/Sedov_12x12x12 => integrated-tests/Test-scripts/standard-results/SGH/Sedov}/vtk/data/VTK0.vtk (64%) rename {python/Test-scripts/standard-results/SGH/Sedov_12x12x12 => integrated-tests/Test-scripts/standard-results/SGH/Sedov}/vtk/outputs.vtk.series (100%) create mode 100644 integrated-tests/Test-scripts/standard-results/SGH/Sod/vtk/data/VTK0.vtk create mode 100644 integrated-tests/Test-scripts/standard-results/SGH/Sod/vtk/outputs.vtk.series rename {python => integrated-tests}/Test-scripts/test_fierro.py (63%) diff --git a/.github/workflows/ubuntu-build.yaml b/.github/workflows/ubuntu-build.yaml index 2a712e011..63703ed0c 100644 --- a/.github/workflows/ubuntu-build.yaml +++ b/.github/workflows/ubuntu-build.yaml @@ -41,3 +41,9 @@ jobs: chmod +x ./scripts/build-fierro.sh ./scripts/build-fierro.sh --build_action=full-app --solver=all --kokkos_build_type=openmp --build_cores=2 shell: bash + + - name: Run tests + run: | + python3 test_fierro.py + working-directory: ./integrated-tests/Test-scripts + shell: bash \ No newline at end of file diff --git a/integrated-tests/Test-scripts/Solver-Inputs/SGH_Noh_simple.yaml b/integrated-tests/Test-scripts/Solver-Inputs/SGH_Noh_simple.yaml new file mode 100644 index 000000000..aeb01c5da --- /dev/null +++ b/integrated-tests/Test-scripts/Solver-Inputs/SGH_Noh_simple.yaml @@ -0,0 +1,70 @@ +num_dims: 3 +dynamic_options: + time_final: 0.6 + dt_min: 1.e-8 + dt_max: 1.e-2 + dt_start: 1.e-5 + cycle_stop: 2000000 + +mesh_generation_options: + type: Box + origin: [0, 0, 0] + length: [1.0, 1.0, 1.0] + num_elems: [10, 10, 10] + +output_options: + timer_output_level: thorough + output_file_format: vtk + graphics_step: 0.6 + write_initial: false + write_final: true + + +fea_module_parameters: + - type: SGH + material_id: 0 + boundary_conditions: + # Tag X plane + - surface: + type: x_plane + plane_position: 0.0 + type: reflected + + # Tag Y plane + - surface: + type: y_plane + plane_position: 0.0 + type: reflected + + # Tag Z plane + - surface: + type: z_plane + plane_position: 0.0 + type: reflected + + +materials: + - id: 0 + eos_model: ideal_gas + strength_model: none + elastic_modulus: 10 + poisson_ratio: 0.3 + q1: 0.06 + q2: 1.2 + q1ex: 0.06 + q2ex: 1.2 + eos_global_vars: + - 1.666666666666667 + - 1.0E-14 + - 1.0 + +regions: + - volume: + type: global + material_id: 0 + den: 1.0 + sie: 1.e-10 + + velocity: spherical + speed: -1.0 + \ No newline at end of file diff --git a/python/Test-scripts/Solver-Inputs/SGH_Sedov_12x12x12.yaml b/integrated-tests/Test-scripts/Solver-Inputs/SGH_Sedov_simple.yaml similarity index 75% rename from python/Test-scripts/Solver-Inputs/SGH_Sedov_12x12x12.yaml rename to integrated-tests/Test-scripts/Solver-Inputs/SGH_Sedov_simple.yaml index 2adbcdfa6..b546e4cf9 100644 --- a/python/Test-scripts/Solver-Inputs/SGH_Sedov_12x12x12.yaml +++ b/integrated-tests/Test-scripts/Solver-Inputs/SGH_Sedov_simple.yaml @@ -17,6 +17,7 @@ output_options: output_file_format: vtk graphics_step: 1.0 write_initial: false + write_final: true fea_module_parameters: - type: SGH @@ -40,23 +41,6 @@ fea_module_parameters: plane_position: 0.0 type: reflected - # Tag X plane - - surface: - type: x_plane - plane_position: 1.2 - type: reflected - - # Tag Y plane - - surface: - type: y_plane - plane_position: 1.2 - type: reflected - - # Tag Z plane - - surface: - type: z_plane - plane_position: 1.2 - type: reflected materials: - id: 0 @@ -64,10 +48,10 @@ materials: strength_model: none elastic_modulus: 10 poisson_ratio: 0.3 - q1: 1.0 - q2: 0 - q1ex: 1.0 - q2ex: 0.0 + q1: 0.06 + q2: 1.0 + q1ex: 0.06 + q2ex: 1.0 eos_global_vars: - 1.666666666666667 - 1.0E-14 @@ -91,8 +75,7 @@ regions: radius2: 0.1 material_id: 0 den: 1.0 - ie: 0.5 - + ie: 0.25833839995946534 velocity: cartesian u: 0.0 v: 0.0 diff --git a/integrated-tests/Test-scripts/Solver-Inputs/SGH_Sod_simple.yaml b/integrated-tests/Test-scripts/Solver-Inputs/SGH_Sod_simple.yaml new file mode 100644 index 000000000..dadd7f4e1 --- /dev/null +++ b/integrated-tests/Test-scripts/Solver-Inputs/SGH_Sod_simple.yaml @@ -0,0 +1,92 @@ +num_dims: 3 +dynamic_options: + time_final: 0.15 + dt_min: 1.e-8 + dt_max: 1.e-2 + dt_start: 1.e-5 + cycle_stop: 2000000 + +mesh_generation_options: + type: Box + origin: [0, 0, 0] + length: [1.0, 0.01, 0.01] + num_elems: [100, 1, 1] + +output_options: + timer_output_level: thorough + output_file_format: vtk + graphics_step: 0.15 + write_initial: false + write_final: true + + +fea_module_parameters: + - type: SGH + material_id: 0 + boundary_conditions: + # Tag X plane + - surface: + type: x_plane + plane_position: 0.0 + type: reflected + + # Tag Y plane + - surface: + type: y_plane + plane_position: 0.0 + type: reflected + + # Tag Z plane + - surface: + type: z_plane + plane_position: 0.0 + type: reflected + + - surface: + type: x_plane + plane_position: 1.0 + type: reflected + + # Tag Y plane + - surface: + type: y_plane + plane_position: 0.01 + type: reflected + + # Tag Z plane + - surface: + type: z_plane + plane_position: 0.01 + type: reflected + + +materials: + - id: 0 + eos_model: ideal_gas + # Artificial viscosity linear and quadratic terms + q1: 0.06 + q2: 1.2 + q1ex: 0.06 + q2ex: 1.2 + eos_global_vars: + - 1.4 + - 1.0E-14 + - 1.0 + +regions: + - volume: + type: global + material_id: 0 + den: 1.0 + sie: 2.5 + - volume: + type: box + x1: 0.5 + x2: 1.0 + y1: 0.0 + y2: 0.01 + z1: 0.0 + z2: 0.01 + material_id: 0 + den: 0.125 + sie: 2 diff --git a/integrated-tests/Test-scripts/standard-results/SGH/Noh/vtk/data/VTK0.vtk b/integrated-tests/Test-scripts/standard-results/SGH/Noh/vtk/data/VTK0.vtk new file mode 100644 index 000000000..e8a29fc53 --- /dev/null +++ b/integrated-tests/Test-scripts/standard-results/SGH/Noh/vtk/data/VTK0.vtk @@ -0,0 +1,10696 @@ +# vtk DataFile Version 2.0 +Mesh for Fierro +ASCII +DATASET UNSTRUCTURED_GRID + +POINTS 1331 float +0.00000000 0.00000000 0.00000000 +0.04137547 0.00000000 0.00000000 +0.07260394 0.00000000 0.00000000 +0.10383619 0.00000000 0.00000000 +0.12798169 0.00000000 0.00000000 +0.16603252 0.00000000 0.00000000 +0.19049360 0.00000000 0.00000000 +0.22842797 0.00000000 0.00000000 +0.25352756 0.00000000 0.00000000 +0.31648306 0.00000000 0.00000000 +0.41701632 0.00000000 0.00000000 +0.00000000 0.04137547 0.00000000 +0.04448523 0.04448523 0.00000000 +0.07689775 0.04505137 0.00000000 +0.10389678 0.03861281 0.00000000 +0.13180384 0.03606643 0.00000000 +0.16586061 0.03402650 0.00000000 +0.19180308 0.03240498 0.00000000 +0.22818222 0.03133843 0.00000000 +0.25414093 0.03146531 0.00000000 +0.31931763 0.03445700 0.00000000 +0.41991306 0.03744864 0.00000000 +0.00000000 0.07260394 0.00000000 +0.04505137 0.07689775 0.00000000 +0.08135558 0.08135558 0.00000000 +0.11063105 0.07687125 0.00000000 +0.13938899 0.07179674 0.00000000 +0.16504801 0.06701318 0.00000000 +0.19639034 0.06469918 0.00000000 +0.22619322 0.06244954 0.00000000 +0.25716336 0.06363097 0.00000000 +0.32787915 0.07078589 0.00000000 +0.42837367 0.07692147 0.00000000 +0.00000000 0.10383619 0.00000000 +0.03861281 0.10389678 0.00000000 +0.07687125 0.11063105 0.00000000 +0.11212122 0.11212122 0.00000000 +0.14131691 0.10604135 0.00000000 +0.16753827 0.10027287 0.00000000 +0.19966482 0.09673549 0.00000000 +0.22322793 0.09361682 0.00000000 +0.26508350 0.09831107 0.00000000 +0.34204778 0.11094795 0.00000000 +0.44172797 0.12019524 0.00000000 +0.00000000 0.12798169 0.00000000 +0.03606643 0.13180384 0.00000000 +0.07179674 0.13938899 0.00000000 +0.10604135 0.14131691 0.00000000 +0.13838951 0.13838951 0.00000000 +0.17056324 0.13462518 0.00000000 +0.19677580 0.12769220 0.00000000 +0.22431428 0.12676672 0.00000000 +0.27981366 0.13843104 0.00000000 +0.36101723 0.15665838 0.00000000 +0.45895609 0.16853142 0.00000000 +0.00000000 0.16603252 0.00000000 +0.03402650 0.16586061 0.00000000 +0.06701318 0.16504801 0.00000000 +0.10027287 0.16753827 0.00000000 +0.13462518 0.17056324 0.00000000 +0.16516682 0.16516682 0.00000000 +0.19271574 0.15927068 0.00000000 +0.23478059 0.16680279 0.00000000 +0.30111445 0.18659786 0.00000000 +0.38329928 0.20881802 0.00000000 +0.47888269 0.22253677 0.00000000 +0.00000000 0.19049360 0.00000000 +0.03240498 0.19180308 0.00000000 +0.06469918 0.19639034 0.00000000 +0.09673549 0.19966482 0.00000000 +0.12769220 0.19677580 0.00000000 +0.15927068 0.19271574 0.00000000 +0.19889841 0.19889841 0.00000000 +0.25507793 0.21810515 0.00000000 +0.32647921 0.24349064 0.00000000 +0.40730343 0.26741508 0.00000000 +0.50039208 0.28225364 0.00000000 +0.00000000 0.22842797 0.00000000 +0.03133843 0.22818222 0.00000000 +0.06244954 0.22619322 0.00000000 +0.09361682 0.22322793 0.00000000 +0.12676672 0.22431428 0.00000000 +0.16680279 0.23478059 0.00000000 +0.21810515 0.25507793 0.00000000 +0.28118265 0.28118265 0.00000000 +0.35297348 0.30795378 0.00000000 +0.43178842 0.33197640 0.00000000 +0.52253780 0.34738292 0.00000000 +0.00000000 0.25352756 0.00000000 +0.03146531 0.25414093 0.00000000 +0.06363097 0.25716336 0.00000000 +0.09831107 0.26508350 0.00000000 +0.13843104 0.27981366 0.00000000 +0.18659786 0.30111445 0.00000000 +0.24349064 0.32647921 0.00000000 +0.30795378 0.35297348 0.00000000 +0.37871006 0.37871006 0.00000000 +0.45578204 0.40202913 0.00000000 +0.54444638 0.41760426 0.00000000 +0.00000000 0.31648306 0.00000000 +0.03445700 0.31931763 0.00000000 +0.07078589 0.32787915 0.00000000 +0.11094795 0.34204778 0.00000000 +0.15665838 0.36101723 0.00000000 +0.20881802 0.38329928 0.00000000 +0.26741508 0.40730343 0.00000000 +0.33197640 0.43178842 0.00000000 +0.40202913 0.45578204 0.00000000 +0.47789395 0.47789395 0.00000000 +0.56461509 0.49343932 0.00000000 +0.00000000 0.41701632 0.00000000 +0.03744864 0.41991306 0.00000000 +0.07692147 0.42837367 0.00000000 +0.12019524 0.44172797 0.00000000 +0.16853142 0.45895609 0.00000000 +0.22253677 0.47888269 0.00000000 +0.28225364 0.50039208 0.00000000 +0.34738292 0.52253780 0.00000000 +0.41760426 0.54444638 0.00000000 +0.49343932 0.56461509 0.00000000 +0.57926020 0.57926020 0.00000000 +0.00000000 0.00000000 0.04137547 +0.04448523 0.00000000 0.04448523 +0.07689775 0.00000000 0.04505137 +0.10389678 0.00000000 0.03861281 +0.13180384 0.00000000 0.03606643 +0.16586061 0.00000000 0.03402650 +0.19180308 0.00000000 0.03240498 +0.22818222 0.00000000 0.03133843 +0.25414093 0.00000000 0.03146531 +0.31931763 0.00000000 0.03445700 +0.41991306 0.00000000 0.03744864 +0.00000000 0.04448523 0.04448523 +0.04767835 0.04767835 0.04767835 +0.08042483 0.04444714 0.04444714 +0.10635936 0.03888445 0.03888445 +0.13522141 0.03599520 0.03599520 +0.16527208 0.03379863 0.03379863 +0.19349959 0.03236427 0.03236427 +0.22758643 0.03130662 0.03130662 +0.25495890 0.03155693 0.03155693 +0.32216687 0.03475138 0.03475138 +0.42277459 0.03777591 0.03777591 +0.00000000 0.07689775 0.04505137 +0.04444714 0.08042483 0.04444714 +0.08123679 0.08123679 0.04160538 +0.11132992 0.07700321 0.03846915 +0.14017920 0.07159086 0.03553894 +0.16521065 0.06678295 0.03328419 +0.19761278 0.06466801 0.03230615 +0.22535774 0.06239823 0.03121506 +0.25847356 0.06392712 0.03194659 +0.33074076 0.07139306 0.03565600 +0.43112986 0.07755608 0.03873898 +0.00000000 0.10389678 0.03861281 +0.03888445 0.10635936 0.03888445 +0.07700321 0.11132992 0.03846915 +0.11165785 0.11165785 0.03736752 +0.14047017 0.10542590 0.03494505 +0.16805230 0.10023020 0.03320312 +0.19954128 0.09654602 0.03219966 +0.22281187 0.09363069 0.03120195 +0.26695352 0.09897136 0.03291478 +0.34484971 0.11186931 0.03719588 +0.44431290 0.12109196 0.04027401 +0.00000000 0.13180384 0.03606643 +0.03599520 0.13522141 0.03599520 +0.07159086 0.14017920 0.03553894 +0.10542590 0.14047017 0.03494505 +0.13796001 0.13796001 0.03423991 +0.17033432 0.13431492 0.03341978 +0.19595259 0.12736654 0.03183834 +0.22490839 0.12720318 0.03173661 +0.28210191 0.13954458 0.03472783 +0.36364362 0.15784005 0.03931624 +0.46131821 0.16962287 0.04226789 +0.00000000 0.16586061 0.03402650 +0.03379863 0.16527208 0.03379863 +0.06678295 0.16521065 0.03328419 +0.10023020 0.16805230 0.03320312 +0.13431492 0.17033432 0.03341978 +0.16440035 0.16440035 0.03255157 +0.19256451 0.15928646 0.03170748 +0.23631246 0.16790764 0.03337845 +0.30351728 0.18809532 0.03739827 +0.38565699 0.21016006 0.04185400 +0.48099415 0.22374901 0.04457805 +0.00000000 0.19180308 0.03240498 +0.03236427 0.19349959 0.03236427 +0.06466801 0.19761278 0.03230615 +0.09654602 0.19954128 0.03219966 +0.12736654 0.19595259 0.03183834 +0.15928646 0.19256451 0.03170748 +0.19999251 0.19999251 0.03304667 +0.25709571 0.21983239 0.03633313 +0.32870097 0.24516944 0.04061697 +0.40935990 0.26882249 0.04461220 +0.50224876 0.28352154 0.04706396 +0.00000000 0.22818222 0.03133843 +0.03130662 0.22758643 0.03130662 +0.06239823 0.22535774 0.03121506 +0.09363069 0.22281187 0.03120195 +0.12720318 0.22490839 0.03173661 +0.16790764 0.23631246 0.03337845 +0.21983239 0.25709571 0.03633313 +0.28316028 0.28316028 0.04012183 +0.35488548 0.30963700 0.04398732 +0.43355602 0.33338482 0.04743180 +0.52415271 0.34865649 0.04960690 +0.00000000 0.25414093 0.03146531 +0.03155693 0.25495890 0.03155693 +0.06392712 0.25847356 0.03194659 +0.09897136 0.26695352 0.03291478 +0.13954458 0.28210191 0.03472783 +0.18809532 0.30351728 0.03739827 +0.24516944 0.32870097 0.04061697 +0.30963700 0.35488548 0.04398732 +0.38032148 0.38032148 0.04725421 +0.45729296 0.40339766 0.05018874 +0.54584389 0.41884809 0.05210283 +0.00000000 0.31931763 0.03445700 +0.03475138 0.32216687 0.03475138 +0.07139306 0.33074076 0.03565600 +0.11186931 0.34484971 0.03719588 +0.15784005 0.36364362 0.03931624 +0.21016006 0.38565699 0.04185400 +0.26882249 0.40935990 0.04461220 +0.33338482 0.43355602 0.04743180 +0.40339766 0.45729296 0.05018874 +0.47919634 0.47919634 0.05270709 +0.56583363 0.49462935 0.05438489 +0.00000000 0.41991306 0.03744864 +0.03777591 0.42277459 0.03777591 +0.07755608 0.43112986 0.03873898 +0.12109196 0.44431290 0.04027401 +0.16962287 0.46131821 0.04226789 +0.22374901 0.48099415 0.04457805 +0.28352154 0.50224876 0.04706396 +0.34865649 0.52415271 0.04960690 +0.41884809 0.54584389 0.05210283 +0.49462935 0.56583363 0.05438489 +0.58038155 0.58038155 0.05587349 +0.00000000 0.00000000 0.07260394 +0.04505137 0.00000000 0.07689775 +0.08135558 0.00000000 0.08135558 +0.11063105 0.00000000 0.07687125 +0.13938899 0.00000000 0.07179674 +0.16504801 0.00000000 0.06701318 +0.19639034 0.00000000 0.06469918 +0.22619322 0.00000000 0.06244954 +0.25716336 0.00000000 0.06363097 +0.32787915 0.00000000 0.07078589 +0.42837367 0.00000000 0.07692147 +0.00000000 0.04505137 0.07689775 +0.04444714 0.04444714 0.08042483 +0.08123679 0.04160538 0.08123679 +0.11132992 0.03846915 0.07700321 +0.14017920 0.03553894 0.07159086 +0.16521065 0.03328419 0.06678295 +0.19761278 0.03230615 0.06466801 +0.22535774 0.03121506 0.06239823 +0.25847356 0.03194659 0.06392712 +0.33074076 0.03565600 0.07139306 +0.43112986 0.03873898 0.07755608 +0.00000000 0.08135558 0.08135558 +0.04160538 0.08123679 0.08123679 +0.07962017 0.07962017 0.07962017 +0.11240255 0.07607861 0.07607861 +0.14029611 0.07046778 0.07046778 +0.16651293 0.06644700 0.06644700 +0.19944218 0.06446913 0.06446913 +0.22331116 0.06230866 0.06230866 +0.26320175 0.06500921 0.06500921 +0.33927141 0.07323468 0.07323468 +0.43917213 0.07941781 0.07941781 +0.00000000 0.11063105 0.07687125 +0.03846915 0.11132992 0.07700321 +0.07607861 0.11240255 0.07607861 +0.10950584 0.10950584 0.07313521 +0.13857114 0.10380406 0.06901969 +0.16948445 0.10017786 0.06661625 +0.19816769 0.09592841 0.06397840 +0.22256406 0.09396525 0.06262806 +0.27306506 0.10115207 0.06733091 +0.35311435 0.11461909 0.07630501 +0.45185224 0.12371489 0.08237341 +0.00000000 0.13938899 0.07179674 +0.03553894 0.14017920 0.07159086 +0.07046778 0.14029611 0.07046778 +0.10380406 0.13857114 0.06901969 +0.13727098 0.13727098 0.06806319 +0.16898875 0.13317935 0.06635855 +0.19367661 0.12658456 0.06323355 +0.22754142 0.12889672 0.06430314 +0.28916383 0.14300846 0.07125668 +0.37133248 0.16132014 0.08044838 +0.46821069 0.17281107 0.08620263 +0.00000000 0.16504801 0.06701318 +0.03328419 0.16521065 0.06678295 +0.06644700 0.16651293 0.06644700 +0.10017786 0.16948445 0.06661625 +0.13317935 0.16898875 0.06635855 +0.16220314 0.16220314 0.06430756 +0.19295021 0.15993884 0.06371271 +0.24138302 0.17154129 0.06823837 +0.31067903 0.19257853 0.07667036 +0.39254160 0.21408775 0.08534772 +0.48716380 0.22729157 0.09063831 +0.00000000 0.19639034 0.06469918 +0.03230615 0.19761278 0.06466801 +0.06446913 0.19944218 0.06446913 +0.09592841 0.19816769 0.06397840 +0.12658456 0.19367661 0.06323355 +0.15993884 0.19295021 0.06371271 +0.20378373 0.20378373 0.06738878 +0.26325188 0.22510791 0.07450346 +0.33521416 0.25009682 0.08295066 +0.41537115 0.27293991 0.09065553 +0.50768477 0.28723258 0.09542002 +0.00000000 0.22619322 0.06244954 +0.03121506 0.22535774 0.06239823 +0.06230866 0.22331116 0.06230866 +0.09396525 0.22256406 0.06262806 +0.12889672 0.22754142 0.06430314 +0.17154129 0.24138302 0.06823837 +0.22510791 0.26325188 0.07450346 +0.28897821 0.28897821 0.08198667 +0.36047637 0.31455961 0.08944390 +0.43873462 0.33751260 0.09609004 +0.52889128 0.35239173 0.10032597 +0.00000000 0.25716336 0.06363097 +0.03194659 0.25847356 0.06392712 +0.06500921 0.26320175 0.06500921 +0.10115207 0.27306506 0.06733091 +0.14300846 0.28916383 0.07125668 +0.19257853 0.31067903 0.07667036 +0.25009682 0.33521416 0.08295066 +0.31455961 0.36047637 0.08944390 +0.38504333 0.38504333 0.09574059 +0.46173017 0.40741719 0.10142164 +0.54995341 0.42250372 0.10515830 +0.00000000 0.32787915 0.07078589 +0.03565600 0.33074076 0.07139306 +0.07323468 0.33927141 0.07323468 +0.11461909 0.35311435 0.07630501 +0.16132014 0.37133248 0.08044838 +0.21408775 0.39254160 0.08534772 +0.27293991 0.41537115 0.09065553 +0.33751260 0.43873462 0.09609004 +0.40741719 0.46173017 0.10142164 +0.48302877 0.48302877 0.10631257 +0.56942351 0.49813299 0.10959590 +0.00000000 0.42837367 0.07692147 +0.03873898 0.43112986 0.07755608 +0.07941781 0.43917213 0.07941781 +0.12371489 0.45185224 0.08237341 +0.17281107 0.46821069 0.08620263 +0.22729157 0.48716380 0.09063831 +0.28723258 0.50768477 0.09542002 +0.35239173 0.52889128 0.10032597 +0.42250372 0.54995341 0.10515830 +0.49813299 0.56942351 0.10959590 +0.58368653 0.58368653 0.11251454 +0.00000000 0.00000000 0.10383619 +0.03861281 0.00000000 0.10389678 +0.07687125 0.00000000 0.11063105 +0.11212122 0.00000000 0.11212122 +0.14131691 0.00000000 0.10604135 +0.16753827 0.00000000 0.10027287 +0.19966482 0.00000000 0.09673549 +0.22322793 0.00000000 0.09361682 +0.26508350 0.00000000 0.09831107 +0.34204778 0.00000000 0.11094795 +0.44172797 0.00000000 0.12019524 +0.00000000 0.03861281 0.10389678 +0.03888445 0.03888445 0.10635936 +0.07700321 0.03846915 0.11132992 +0.11165785 0.03736752 0.11165785 +0.14047017 0.03494505 0.10542590 +0.16805230 0.03320312 0.10023020 +0.19954128 0.03219966 0.09654602 +0.22281187 0.03120195 0.09363069 +0.26695352 0.03291478 0.09897136 +0.34484971 0.03719588 0.11186931 +0.44431290 0.04027401 0.12109196 +0.00000000 0.07687125 0.11063105 +0.03846915 0.07700321 0.11132992 +0.07607861 0.07607861 0.11240255 +0.10950584 0.07313521 0.10950584 +0.13857114 0.06901969 0.10380406 +0.16948445 0.06661625 0.10017786 +0.19816769 0.06397840 0.09592841 +0.22256406 0.06262806 0.09396525 +0.27306506 0.06733091 0.10115207 +0.35311435 0.07630501 0.11461909 +0.45185224 0.08237341 0.12371489 +0.00000000 0.11212122 0.11212122 +0.03736752 0.11165785 0.11165785 +0.07313521 0.10950584 0.10950584 +0.10564281 0.10564281 0.10564281 +0.13765100 0.10269507 0.10269507 +0.16978486 0.09977760 0.09977760 +0.19454616 0.09493464 0.09493464 +0.22529178 0.09551073 0.09551073 +0.28440177 0.10527908 0.10527908 +0.36632390 0.11909101 0.11909101 +0.46373897 0.12786445 0.12786445 +0.00000000 0.14131691 0.10604135 +0.03494505 0.14047017 0.10542590 +0.06901969 0.13857114 0.10380406 +0.10269507 0.13765100 0.10269507 +0.13635455 0.13635455 0.10162602 +0.16528926 0.13066155 0.09776851 +0.19173267 0.12630998 0.09466217 +0.23426131 0.13284197 0.09945729 +0.30122183 0.14900458 0.11155453 +0.38350356 0.16687541 0.12499506 +0.47909364 0.17784997 0.13323235 +0.00000000 0.16753827 0.10027287 +0.03320312 0.16805230 0.10023020 +0.06661625 0.16948445 0.10017786 +0.09977760 0.16978486 0.09977760 +0.13066155 0.16528926 0.09776851 +0.15967131 0.15967131 0.09524201 +0.19620559 0.16294672 0.09741760 +0.25097867 0.17839181 0.10659898 +0.32228866 0.19989328 0.11957028 +0.40341841 0.22031293 0.13190039 +0.49693438 0.23289980 0.13945861 +0.00000000 0.19966482 0.09673549 +0.03219966 0.19954128 0.09654602 +0.06397840 0.19816769 0.09592841 +0.09493464 0.19454616 0.09493464 +0.12630998 0.19173267 0.09466217 +0.16294672 0.19620559 0.09741760 +0.21141894 0.21141894 0.10502028 +0.27355822 0.23395679 0.11636684 +0.34554853 0.25792654 0.12850023 +0.42489635 0.27947225 0.13937310 +0.51632683 0.29312704 0.14619476 +0.00000000 0.22322793 0.09361682 +0.03120195 0.22281187 0.09363069 +0.06262806 0.22256406 0.09396525 +0.09551073 0.22529178 0.09551073 +0.13284197 0.23426131 0.09945729 +0.17839181 0.25097867 0.10659898 +0.23395679 0.27355822 0.11636684 +0.29824623 0.29824623 0.12713034 +0.36933765 0.32236306 0.13764421 +0.44697728 0.34408616 0.14705310 +0.53645608 0.35834844 0.15313986 +0.00000000 0.26508350 0.09831107 +0.03291478 0.26695352 0.09897136 +0.06733091 0.27306506 0.10115207 +0.10527908 0.28440177 0.10527908 +0.14900458 0.30122183 0.11155453 +0.19989328 0.32228866 0.11957028 +0.25792654 0.34554853 0.12850023 +0.32236306 0.36933765 0.13764421 +0.39256085 0.39256085 0.14654208 +0.46882478 0.41384505 0.15462937 +0.55654083 0.42835694 0.16001917 +0.00000000 0.34204778 0.11094795 +0.03719588 0.34484971 0.11186931 +0.07630501 0.35311435 0.11461909 +0.11909101 0.36632390 0.11909101 +0.16687541 0.38350356 0.12499506 +0.22031293 0.40341841 0.13190039 +0.27947225 0.42489635 0.13937310 +0.34408616 0.44697728 0.14705310 +0.41384505 0.46882478 0.15462937 +0.48917953 0.48917953 0.16162631 +0.57519802 0.50376179 0.16638009 +0.00000000 0.44172797 0.12019524 +0.04027401 0.44431290 0.12109196 +0.08237341 0.45185224 0.12371489 +0.12786445 0.46373897 0.12786445 +0.17784997 0.47909364 0.13323235 +0.23289980 0.49693438 0.13945861 +0.29312704 0.51632683 0.14619476 +0.35834844 0.53645608 0.15313986 +0.42835694 0.55654083 0.16001917 +0.50376179 0.57519802 0.16638009 +0.58900721 0.58900721 0.17061849 +0.00000000 0.00000000 0.12798169 +0.03606643 0.00000000 0.13180384 +0.07179674 0.00000000 0.13938899 +0.10604135 0.00000000 0.14131691 +0.13838951 0.00000000 0.13838951 +0.17056324 0.00000000 0.13462518 +0.19677580 0.00000000 0.12769220 +0.22431428 0.00000000 0.12676672 +0.27981366 0.00000000 0.13843104 +0.36101723 0.00000000 0.15665838 +0.45895609 0.00000000 0.16853142 +0.00000000 0.03606643 0.13180384 +0.03599520 0.03599520 0.13522141 +0.07159086 0.03553894 0.14017920 +0.10542590 0.03494505 0.14047017 +0.13796001 0.03423991 0.13796001 +0.17033432 0.03341978 0.13431492 +0.19595259 0.03183834 0.12736654 +0.22490839 0.03173661 0.12720318 +0.28210191 0.03472783 0.13954458 +0.36364362 0.03931624 0.15784005 +0.46131821 0.04226789 0.16962287 +0.00000000 0.07179674 0.13938899 +0.03553894 0.07159086 0.14017920 +0.07046778 0.07046778 0.14029611 +0.10380406 0.06901969 0.13857114 +0.13727098 0.06806319 0.13727098 +0.16898875 0.06635855 0.13317935 +0.19367661 0.06323355 0.12658456 +0.22754142 0.06430314 0.12889672 +0.28916383 0.07125668 0.14300846 +0.37133248 0.08044838 0.16132014 +0.46821069 0.08620263 0.17281107 +0.00000000 0.10604135 0.14131691 +0.03494505 0.10542590 0.14047017 +0.06901969 0.10380406 0.13857114 +0.10269507 0.10269507 0.13765100 +0.13635455 0.10162602 0.13635455 +0.16528926 0.09776851 0.13066155 +0.19173267 0.09466217 0.12630998 +0.23426131 0.09945729 0.13284197 +0.30122183 0.11155453 0.14900458 +0.38350356 0.12499506 0.16687541 +0.47909364 0.13323235 0.17784997 +0.00000000 0.13838951 0.13838951 +0.03423991 0.13796001 0.13796001 +0.06806319 0.13727098 0.13727098 +0.10162602 0.13635455 0.13635455 +0.13265668 0.13265668 0.13265668 +0.16041241 0.12753255 0.12753255 +0.19423207 0.12866912 0.12866912 +0.24693111 0.14008098 0.14008098 +0.31779711 0.15736460 0.15736460 +0.39924565 0.17411116 0.17411116 +0.49319434 0.18437541 0.18437541 +0.00000000 0.17056324 0.13462518 +0.03341978 0.17033432 0.13431492 +0.06635855 0.16898875 0.13317935 +0.09776851 0.16528926 0.13066155 +0.12753255 0.16041241 0.12753255 +0.16034696 0.16034696 0.12794419 +0.20495514 0.17034960 0.13597078 +0.26545039 0.18875057 0.15070348 +0.33754904 0.20955846 0.16741692 +0.41750832 0.22839938 0.18253990 +0.50965141 0.24018856 0.19197134 +0.00000000 0.19677580 0.12769220 +0.03183834 0.19595259 0.12736654 +0.06323355 0.19367661 0.12658456 +0.09466217 0.19173267 0.12630998 +0.12866912 0.19423207 0.12866912 +0.17034960 0.20495514 0.13597078 +0.22357807 0.22357807 0.14841778 +0.28743393 0.24589284 0.16341109 +0.35895976 0.26809653 0.17834732 +0.43730117 0.28798885 0.19168770 +0.52763621 0.30082696 0.20023419 +0.00000000 0.22431428 0.12676672 +0.03173661 0.22490839 0.12720318 +0.06430314 0.22754142 0.12889672 +0.09945729 0.23426131 0.13284197 +0.14008098 0.24693111 0.14008098 +0.18875057 0.26545039 0.15070348 +0.24589284 0.28743393 0.16341109 +0.31029684 0.31029684 0.17666347 +0.38087573 0.33252465 0.18952893 +0.45778021 0.35270538 0.20114656 +0.54641261 0.36617434 0.20880907 +0.00000000 0.27981366 0.13843104 +0.03472783 0.28210191 0.13954458 +0.07125668 0.28916383 0.14300846 +0.11155453 0.30122183 0.14900458 +0.15736460 0.31779711 0.15736460 +0.20955846 0.33754904 0.16741692 +0.26809653 0.35895976 0.17834732 +0.33252465 0.38087573 0.18952893 +0.40241550 0.40241550 0.20048751 +0.47818105 0.42232280 0.21054602 +0.56525956 0.43609040 0.21736420 +0.00000000 0.36101723 0.15665838 +0.03931624 0.36364362 0.15784005 +0.08044838 0.37133248 0.16132014 +0.12499506 0.38350356 0.16687541 +0.17411116 0.39924565 0.17411116 +0.22839938 0.41750832 0.18253990 +0.28798885 0.43730117 0.19168770 +0.35270538 0.45778021 0.20114656 +0.42232280 0.47818105 0.21054602 +0.49733313 0.49733313 0.21930252 +0.58287752 0.51123432 0.22534505 +0.00000000 0.45895609 0.16853142 +0.04226789 0.46131821 0.16962287 +0.08620263 0.46821069 0.17281107 +0.13323235 0.47909364 0.17784997 +0.18437541 0.49319434 0.18437541 +0.24018856 0.50965141 0.19197134 +0.30082696 0.52763621 0.20023419 +0.36617434 0.54641261 0.20880907 +0.43609040 0.56525956 0.21736420 +0.51123432 0.58287752 0.22534505 +0.59609173 0.59609173 0.23075305 +0.00000000 0.00000000 0.16603252 +0.03402650 0.00000000 0.16586061 +0.06701318 0.00000000 0.16504801 +0.10027287 0.00000000 0.16753827 +0.13462518 0.00000000 0.17056324 +0.16516682 0.00000000 0.16516682 +0.19271574 0.00000000 0.15927068 +0.23478059 0.00000000 0.16680279 +0.30111445 0.00000000 0.18659786 +0.38329928 0.00000000 0.20881802 +0.47888269 0.00000000 0.22253677 +0.00000000 0.03402650 0.16586061 +0.03379863 0.03379863 0.16527208 +0.06678295 0.03328419 0.16521065 +0.10023020 0.03320312 0.16805230 +0.13431492 0.03341978 0.17033432 +0.16440035 0.03255157 0.16440035 +0.19256451 0.03170748 0.15928646 +0.23631246 0.03337845 0.16790764 +0.30351728 0.03739827 0.18809532 +0.38565699 0.04185400 0.21016006 +0.48099415 0.04457805 0.22374901 +0.00000000 0.06701318 0.16504801 +0.03328419 0.06678295 0.16521065 +0.06644700 0.06644700 0.16651293 +0.10017786 0.06661625 0.16948445 +0.13317935 0.06635855 0.16898875 +0.16220314 0.06430756 0.16220314 +0.19295021 0.06371271 0.15993884 +0.24138302 0.06823837 0.17154129 +0.31067903 0.07667036 0.19257853 +0.39254160 0.08534772 0.21408775 +0.48716380 0.09063831 0.22729157 +0.00000000 0.10027287 0.16753827 +0.03320312 0.10023020 0.16805230 +0.06661625 0.10017786 0.16948445 +0.09977760 0.09977760 0.16978486 +0.13066155 0.09776851 0.16528926 +0.15967131 0.09524201 0.15967131 +0.19620559 0.09741760 0.16294672 +0.25097867 0.10659898 0.17839181 +0.32228866 0.11957028 0.19989328 +0.40341841 0.13190039 0.22031293 +0.49693438 0.13945861 0.23289980 +0.00000000 0.13462518 0.17056324 +0.03341978 0.13431492 0.17033432 +0.06635855 0.13317935 0.16898875 +0.09776851 0.13066155 0.16528926 +0.12753255 0.12753255 0.16041241 +0.16034696 0.12794419 0.16034696 +0.20495514 0.13597078 0.17034960 +0.26545039 0.15070348 0.18875057 +0.33754904 0.16741692 0.20955846 +0.41750832 0.18253990 0.22839938 +0.50965141 0.19197134 0.24018856 +0.00000000 0.16516682 0.16516682 +0.03255157 0.16440035 0.16440035 +0.06430756 0.16220314 0.16220314 +0.09524201 0.15967131 0.15967131 +0.12794419 0.16034696 0.16034696 +0.16792486 0.16792486 0.16792486 +0.22005873 0.18295978 0.18295978 +0.28366675 0.20185059 0.20185059 +0.35533991 0.22085396 0.22085396 +0.43393788 0.23784423 0.23784423 +0.52457919 0.24871955 0.24871955 +0.00000000 0.19271574 0.15927068 +0.03170748 0.19256451 0.15928646 +0.06371271 0.19295021 0.15993884 +0.09741760 0.19620559 0.16294672 +0.13597078 0.20495514 0.17034960 +0.18295978 0.22005873 0.18295978 +0.23939788 0.23939788 0.19912927 +0.30376634 0.25995660 0.21634250 +0.37458634 0.27994855 0.23307732 +0.45186924 0.29799690 0.24815724 +0.54099837 0.30989882 0.25806313 +0.00000000 0.23478059 0.16680279 +0.03337845 0.23631246 0.16790764 +0.06823837 0.24138302 0.17154129 +0.10659898 0.25097867 0.17839181 +0.15070348 0.26545039 0.18875057 +0.20185059 0.28366675 0.20185059 +0.25995660 0.30376634 0.21634250 +0.32433606 0.32433606 0.23117645 +0.39441737 0.34445024 0.24566064 +0.47056425 0.36290679 0.25890266 +0.55825660 0.37545976 0.26783386 +0.00000000 0.30111445 0.18659786 +0.03739827 0.30351728 0.18809532 +0.07667036 0.31067903 0.19257853 +0.11957028 0.32228866 0.19989328 +0.16741692 0.33754904 0.20955846 +0.22085396 0.35533991 0.22085396 +0.27994855 0.37458634 0.23307732 +0.34445024 0.39441737 0.24566064 +0.41407826 0.41407826 0.25811160 +0.48933516 0.43242884 0.26967147 +0.57570024 0.44532950 0.27766325 +0.00000000 0.38329928 0.20881802 +0.04185400 0.38565699 0.21016006 +0.08534772 0.39254160 0.21408775 +0.13190039 0.40341841 0.22031293 +0.18253990 0.41750832 0.22839938 +0.23784423 0.43393788 0.23784423 +0.29799690 0.45186924 0.24815724 +0.36290679 0.47056425 0.25890266 +0.43242884 0.48933516 0.26967147 +0.50711368 0.50711368 0.27980554 +0.59212645 0.52021444 0.28692737 +0.00000000 0.47888269 0.22253677 +0.04457805 0.48099415 0.22374901 +0.09063831 0.48716380 0.22729157 +0.13945861 0.49693438 0.23289980 +0.19197134 0.50965141 0.24018856 +0.24871955 0.52457919 0.24871955 +0.30989882 0.54099837 0.25806313 +0.37545976 0.55825660 0.26783386 +0.44532950 0.57570024 0.27766325 +0.52021444 0.59212645 0.28692737 +0.60463726 0.60463726 0.29332937 +0.00000000 0.00000000 0.19049360 +0.03240498 0.00000000 0.19180308 +0.06469918 0.00000000 0.19639034 +0.09673549 0.00000000 0.19966482 +0.12769220 0.00000000 0.19677580 +0.15927068 0.00000000 0.19271574 +0.19889841 0.00000000 0.19889841 +0.25507793 0.00000000 0.21810515 +0.32647921 0.00000000 0.24349064 +0.40730343 0.00000000 0.26741508 +0.50039208 0.00000000 0.28225364 +0.00000000 0.03240498 0.19180308 +0.03236427 0.03236427 0.19349959 +0.06466801 0.03230615 0.19761278 +0.09654602 0.03219966 0.19954128 +0.12736654 0.03183834 0.19595259 +0.15928646 0.03170748 0.19256451 +0.19999251 0.03304667 0.19999251 +0.25709571 0.03633313 0.21983239 +0.32870097 0.04061697 0.24516944 +0.40935990 0.04461220 0.26882249 +0.50224876 0.04706396 0.28352154 +0.00000000 0.06469918 0.19639034 +0.03230615 0.06466801 0.19761278 +0.06446913 0.06446913 0.19944218 +0.09592841 0.06397840 0.19816769 +0.12658456 0.06323355 0.19367661 +0.15993884 0.06371271 0.19295021 +0.20378373 0.06738878 0.20378373 +0.26325188 0.07450346 0.22510791 +0.33521416 0.08295066 0.25009682 +0.41537115 0.09065553 0.27293991 +0.50768477 0.09542002 0.28723258 +0.00000000 0.09673549 0.19966482 +0.03219966 0.09654602 0.19954128 +0.06397840 0.09592841 0.19816769 +0.09493464 0.09493464 0.19454616 +0.12630998 0.09466217 0.19173267 +0.16294672 0.09741760 0.19620559 +0.21141894 0.10502028 0.21141894 +0.27355822 0.11636684 0.23395679 +0.34554853 0.12850023 0.25792654 +0.42489635 0.13937310 0.27947225 +0.51632683 0.14619476 0.29312704 +0.00000000 0.12769220 0.19677580 +0.03183834 0.12736654 0.19595259 +0.06323355 0.12658456 0.19367661 +0.09466217 0.12630998 0.19173267 +0.12866912 0.12866912 0.19423207 +0.17034960 0.13597078 0.20495514 +0.22357807 0.14841778 0.22357807 +0.28743393 0.16341109 0.24589284 +0.35895976 0.17834732 0.26809653 +0.43730117 0.19168770 0.28798885 +0.52763621 0.20023419 0.30082696 +0.00000000 0.15927068 0.19271574 +0.03170748 0.15928646 0.19256451 +0.06371271 0.15993884 0.19295021 +0.09741760 0.16294672 0.19620559 +0.13597078 0.17034960 0.20495514 +0.18295978 0.18295978 0.22005873 +0.23939788 0.19912927 0.23939788 +0.30376634 0.21634250 0.25995660 +0.37458634 0.23307732 0.27994855 +0.45186924 0.24815724 0.29799690 +0.54099837 0.25806313 0.30989882 +0.00000000 0.19889841 0.19889841 +0.03304667 0.19999251 0.19999251 +0.06738878 0.20378373 0.20378373 +0.10502028 0.21141894 0.21141894 +0.14841778 0.22357807 0.22357807 +0.19912927 0.23939788 0.23939788 +0.25706688 0.25706688 0.25706688 +0.32144229 0.27518024 0.27518024 +0.39160548 0.29285291 0.29285291 +0.46789700 0.30900794 0.30900794 +0.55580057 0.31991075 0.31991075 +0.00000000 0.25507793 0.21810515 +0.03633313 0.25709571 0.21983239 +0.07450346 0.26325188 0.22510791 +0.11636684 0.27355822 0.23395679 +0.16341109 0.28743393 0.24589284 +0.21634250 0.30376634 0.25995660 +0.27518024 0.32144229 0.27518024 +0.33960820 0.33960820 0.29082113 +0.40930089 0.35755537 0.30626268 +0.48474690 0.37422168 0.32057710 +0.57147455 0.38578858 0.33046684 +0.00000000 0.32647921 0.24349064 +0.04061697 0.32870097 0.24516944 +0.08295066 0.33521416 0.25009682 +0.12850023 0.34554853 0.25792654 +0.17834732 0.35895976 0.26809653 +0.23307732 0.37458634 0.27994855 +0.29285291 0.39160548 0.29285291 +0.35755537 0.40930089 0.30626268 +0.42701530 0.42701530 0.31967235 +0.50180986 0.44372810 0.33227723 +0.58743713 0.45568627 0.34118086 +0.00000000 0.40730343 0.26741508 +0.04461220 0.40935990 0.26882249 +0.09065553 0.41537115 0.27293991 +0.13937310 0.42489635 0.27947225 +0.19168770 0.43730117 0.28798885 +0.24815724 0.45186924 0.29799690 +0.30900794 0.46789700 0.30900794 +0.37422168 0.48474690 0.32057710 +0.44372810 0.50180986 0.33227723 +0.51812697 0.51812697 0.34340937 +0.60258949 0.53034860 0.35139141 +0.00000000 0.50039208 0.28225364 +0.04706396 0.50224876 0.28352154 +0.09542002 0.50768477 0.28723258 +0.14619476 0.51632683 0.29312704 +0.20023419 0.52763621 0.30082696 +0.25806313 0.54099837 0.30989882 +0.31991075 0.55580057 0.31991075 +0.38578858 0.57147455 0.33046684 +0.45568627 0.58743713 0.34118086 +0.53034860 0.60258949 0.35139141 +0.61432251 0.61432251 0.35860075 +0.00000000 0.00000000 0.22842797 +0.03133843 0.00000000 0.22818222 +0.06244954 0.00000000 0.22619322 +0.09361682 0.00000000 0.22322793 +0.12676672 0.00000000 0.22431428 +0.16680279 0.00000000 0.23478059 +0.21810515 0.00000000 0.25507793 +0.28118265 0.00000000 0.28118265 +0.35297348 0.00000000 0.30795378 +0.43178842 0.00000000 0.33197640 +0.52253780 0.00000000 0.34738292 +0.00000000 0.03133843 0.22818222 +0.03130662 0.03130662 0.22758643 +0.06239823 0.03121506 0.22535774 +0.09363069 0.03120195 0.22281187 +0.12720318 0.03173661 0.22490839 +0.16790764 0.03337845 0.23631246 +0.21983239 0.03633313 0.25709571 +0.28316028 0.04012183 0.28316028 +0.35488548 0.04398732 0.30963700 +0.43355602 0.04743180 0.33338482 +0.52415271 0.04960690 0.34865649 +0.00000000 0.06244954 0.22619322 +0.03121506 0.06239823 0.22535774 +0.06230866 0.06230866 0.22331116 +0.09396525 0.06262806 0.22256406 +0.12889672 0.06430314 0.22754142 +0.17154129 0.06823837 0.24138302 +0.22510791 0.07450346 0.26325188 +0.28897821 0.08198667 0.28897821 +0.36047637 0.08944390 0.31455961 +0.43873462 0.09609004 0.33751260 +0.52889128 0.10032597 0.35239173 +0.00000000 0.09361682 0.22322793 +0.03120195 0.09363069 0.22281187 +0.06262806 0.09396525 0.22256406 +0.09551073 0.09551073 0.22529178 +0.13284197 0.09945729 0.23426131 +0.17839181 0.10659898 0.25097867 +0.23395679 0.11636684 0.27355822 +0.29824623 0.12713034 0.29824623 +0.36933765 0.13764421 0.32236306 +0.44697728 0.14705310 0.34408616 +0.53645608 0.15313986 0.35834844 +0.00000000 0.12676672 0.22431428 +0.03173661 0.12720318 0.22490839 +0.06430314 0.12889672 0.22754142 +0.09945729 0.13284197 0.23426131 +0.14008098 0.14008098 0.24693111 +0.18875057 0.15070348 0.26545039 +0.24589284 0.16341109 0.28743393 +0.31029684 0.17666347 0.31029684 +0.38087573 0.18952893 0.33252465 +0.45778021 0.20114656 0.35270538 +0.54641261 0.20880907 0.36617434 +0.00000000 0.16680279 0.23478059 +0.03337845 0.16790764 0.23631246 +0.06823837 0.17154129 0.24138302 +0.10659898 0.17839181 0.25097867 +0.15070348 0.18875057 0.26545039 +0.20185059 0.20185059 0.28366675 +0.25995660 0.21634250 0.30376634 +0.32433606 0.23117645 0.32433606 +0.39441737 0.24566064 0.34445024 +0.47056425 0.25890266 0.36290679 +0.55825660 0.26783386 0.37545976 +0.00000000 0.21810515 0.25507793 +0.03633313 0.21983239 0.25709571 +0.07450346 0.22510791 0.26325188 +0.11636684 0.23395679 0.27355822 +0.16341109 0.24589284 0.28743393 +0.21634250 0.25995660 0.30376634 +0.27518024 0.27518024 0.32144229 +0.33960820 0.29082113 0.33960820 +0.40930089 0.30626268 0.35755537 +0.48474690 0.32057710 0.37422168 +0.57147455 0.33046684 0.38578858 +0.00000000 0.28118265 0.28118265 +0.04012183 0.28316028 0.28316028 +0.08198667 0.28897821 0.28897821 +0.12713034 0.29824623 0.29824623 +0.17666347 0.31029684 0.31029684 +0.23117645 0.32433606 0.32433606 +0.29082113 0.33960820 0.33960820 +0.35546287 0.35546287 0.35546287 +0.42492694 0.37131094 0.37131094 +0.49978604 0.38621282 0.38621282 +0.58557890 0.39677065 0.39677065 +0.00000000 0.35297348 0.30795378 +0.04398732 0.35488548 0.30963700 +0.08944390 0.36047637 0.31455961 +0.13764421 0.36933765 0.32236306 +0.18952893 0.38087573 0.33252465 +0.24566064 0.39441737 0.34445024 +0.30626268 0.40930089 0.35755537 +0.37131094 0.42492694 0.37131094 +0.44072876 0.44072876 0.38521458 +0.51514987 0.45580517 0.39845220 +0.60005658 0.46678873 0.40801817 +0.00000000 0.43178842 0.33197640 +0.04743180 0.43355602 0.33338482 +0.09609004 0.43873462 0.33751260 +0.14705310 0.44697728 0.34408616 +0.20114656 0.45778021 0.35270538 +0.25890266 0.47056425 0.36290679 +0.32057710 0.48474690 0.37422168 +0.38621282 0.49978604 0.38621282 +0.45580517 0.51514987 0.39845220 +0.52998910 0.52998910 0.41023128 +0.61391429 0.54129155 0.41886090 +0.00000000 0.52253780 0.34738292 +0.04960690 0.52415271 0.34865649 +0.10032597 0.52889128 0.35239173 +0.15313986 0.53645608 0.35834844 +0.20880907 0.54641261 0.36617434 +0.26783386 0.55825660 0.37545976 +0.33046684 0.57147455 0.38578858 +0.39677065 0.58557890 0.39677065 +0.46678873 0.60005658 0.40801817 +0.54129155 0.61391429 0.41886090 +0.62482826 0.62482826 0.42669332 +0.00000000 0.00000000 0.25352756 +0.03146531 0.00000000 0.25414093 +0.06363097 0.00000000 0.25716336 +0.09831107 0.00000000 0.26508350 +0.13843104 0.00000000 0.27981366 +0.18659786 0.00000000 0.30111445 +0.24349064 0.00000000 0.32647921 +0.30795378 0.00000000 0.35297348 +0.37871006 0.00000000 0.37871006 +0.45578204 0.00000000 0.40202913 +0.54444638 0.00000000 0.41760426 +0.00000000 0.03146531 0.25414093 +0.03155693 0.03155693 0.25495890 +0.06392712 0.03194659 0.25847356 +0.09897136 0.03291478 0.26695352 +0.13954458 0.03472783 0.28210191 +0.18809532 0.03739827 0.30351728 +0.24516944 0.04061697 0.32870097 +0.30963700 0.04398732 0.35488548 +0.38032148 0.04725421 0.38032148 +0.45729296 0.05018874 0.40339766 +0.54584389 0.05210283 0.41884809 +0.00000000 0.06363097 0.25716336 +0.03194659 0.06392712 0.25847356 +0.06500921 0.06500921 0.26320175 +0.10115207 0.06733091 0.27306506 +0.14300846 0.07125668 0.28916383 +0.19257853 0.07667036 0.31067903 +0.25009682 0.08295066 0.33521416 +0.31455961 0.08944390 0.36047637 +0.38504333 0.09574059 0.38504333 +0.46173017 0.10142164 0.40741719 +0.54995341 0.10515830 0.42250372 +0.00000000 0.09831107 0.26508350 +0.03291478 0.09897136 0.26695352 +0.06733091 0.10115207 0.27306506 +0.10527908 0.10527908 0.28440177 +0.14900458 0.11155453 0.30122183 +0.19989328 0.11957028 0.32228866 +0.25792654 0.12850023 0.34554853 +0.32236306 0.13764421 0.36933765 +0.39256085 0.14654208 0.39256085 +0.46882478 0.15462937 0.41384505 +0.55654083 0.16001917 0.42835694 +0.00000000 0.13843104 0.27981366 +0.03472783 0.13954458 0.28210191 +0.07125668 0.14300846 0.28916383 +0.11155453 0.14900458 0.30122183 +0.15736460 0.15736460 0.31779711 +0.20955846 0.16741692 0.33754904 +0.26809653 0.17834732 0.35895976 +0.33252465 0.18952893 0.38087573 +0.40241550 0.20048751 0.40241550 +0.47818105 0.21054602 0.42232280 +0.56525956 0.21736420 0.43609040 +0.00000000 0.18659786 0.30111445 +0.03739827 0.18809532 0.30351728 +0.07667036 0.19257853 0.31067903 +0.11957028 0.19989328 0.32228866 +0.16741692 0.20955846 0.33754904 +0.22085396 0.22085396 0.35533991 +0.27994855 0.23307732 0.37458634 +0.34445024 0.24566064 0.39441737 +0.41407826 0.25811160 0.41407826 +0.48933516 0.26967147 0.43242884 +0.57570024 0.27766325 0.44532950 +0.00000000 0.24349064 0.32647921 +0.04061697 0.24516944 0.32870097 +0.08295066 0.25009682 0.33521416 +0.12850023 0.25792654 0.34554853 +0.17834732 0.26809653 0.35895976 +0.23307732 0.27994855 0.37458634 +0.29285291 0.29285291 0.39160548 +0.35755537 0.30626268 0.40930089 +0.42701530 0.31967235 0.42701530 +0.50180986 0.33227723 0.44372810 +0.58743713 0.34118086 0.45568627 +0.00000000 0.30795378 0.35297348 +0.04398732 0.30963700 0.35488548 +0.08944390 0.31455961 0.36047637 +0.13764421 0.32236306 0.36933765 +0.18952893 0.33252465 0.38087573 +0.24566064 0.34445024 0.39441737 +0.30626268 0.35755537 0.40930089 +0.37131094 0.37131094 0.42492694 +0.44072876 0.38521458 0.44072876 +0.51514987 0.39845220 0.45580517 +0.60005658 0.40801817 0.46678873 +0.00000000 0.37871006 0.37871006 +0.04725421 0.38032148 0.38032148 +0.09574059 0.38504333 0.38504333 +0.14654208 0.39256085 0.39256085 +0.20048751 0.40241550 0.40241550 +0.25811160 0.41407826 0.41407826 +0.31967235 0.42701530 0.42701530 +0.38521458 0.44072876 0.44072876 +0.45473307 0.45473307 0.45473307 +0.52889720 0.46824434 0.46824434 +0.61311215 0.47826050 0.47826050 +0.00000000 0.45578204 0.40202913 +0.05018874 0.45729296 0.40339766 +0.10142164 0.46173017 0.40741719 +0.15462937 0.46882478 0.41384505 +0.21054602 0.47818105 0.42232280 +0.26967147 0.48933516 0.43242884 +0.33227723 0.50180986 0.44372810 +0.39845220 0.51514987 0.45580517 +0.46824434 0.52889720 0.46824434 +0.54230549 0.54230549 0.48035448 +0.62571134 0.55268669 0.48944833 +0.00000000 0.54444638 0.41760426 +0.05210283 0.54584389 0.41884809 +0.10515830 0.54995341 0.42250372 +0.16001917 0.55654083 0.42835694 +0.21736420 0.56525956 0.43609040 +0.27766325 0.57570024 0.44532950 +0.34118086 0.58743713 0.45568627 +0.40801817 0.60005658 0.46678873 +0.47826050 0.61311215 0.47826050 +0.55268669 0.62571134 0.48944833 +0.63580020 0.63580020 0.49774197 +0.00000000 0.00000000 0.31648306 +0.03445700 0.00000000 0.31931763 +0.07078589 0.00000000 0.32787915 +0.11094795 0.00000000 0.34204778 +0.15665838 0.00000000 0.36101723 +0.20881802 0.00000000 0.38329928 +0.26741508 0.00000000 0.40730343 +0.33197640 0.00000000 0.43178842 +0.40202913 0.00000000 0.45578204 +0.47789395 0.00000000 0.47789395 +0.56461509 0.00000000 0.49343932 +0.00000000 0.03445700 0.31931763 +0.03475138 0.03475138 0.32216687 +0.07139306 0.03565600 0.33074076 +0.11186931 0.03719588 0.34484971 +0.15784005 0.03931624 0.36364362 +0.21016006 0.04185400 0.38565699 +0.26882249 0.04461220 0.40935990 +0.33338482 0.04743180 0.43355602 +0.40339766 0.05018874 0.45729296 +0.47919634 0.05270709 0.47919634 +0.56583363 0.05438489 0.49462935 +0.00000000 0.07078589 0.32787915 +0.03565600 0.07139306 0.33074076 +0.07323468 0.07323468 0.33927141 +0.11461909 0.07630501 0.35311435 +0.16132014 0.08044838 0.37133248 +0.21408775 0.08534772 0.39254160 +0.27293991 0.09065553 0.41537115 +0.33751260 0.09609004 0.43873462 +0.40741719 0.10142164 0.46173017 +0.48302877 0.10631257 0.48302877 +0.56942351 0.10959590 0.49813299 +0.00000000 0.11094795 0.34204778 +0.03719588 0.11186931 0.34484971 +0.07630501 0.11461909 0.35311435 +0.11909101 0.11909101 0.36632390 +0.16687541 0.12499506 0.38350356 +0.22031293 0.13190039 0.40341841 +0.27947225 0.13937310 0.42489635 +0.34408616 0.14705310 0.44697728 +0.41384505 0.15462937 0.46882478 +0.48917953 0.16162631 0.48917953 +0.57519802 0.16638009 0.50376179 +0.00000000 0.15665838 0.36101723 +0.03931624 0.15784005 0.36364362 +0.08044838 0.16132014 0.37133248 +0.12499506 0.16687541 0.38350356 +0.17411116 0.17411116 0.39924565 +0.22839938 0.18253990 0.41750832 +0.28798885 0.19168770 0.43730117 +0.35270538 0.20114656 0.45778021 +0.42232280 0.21054602 0.47818105 +0.49733313 0.21930252 0.49733313 +0.58287752 0.22534505 0.51123432 +0.00000000 0.20881802 0.38329928 +0.04185400 0.21016006 0.38565699 +0.08534772 0.21408775 0.39254160 +0.13190039 0.22031293 0.40341841 +0.18253990 0.22839938 0.41750832 +0.23784423 0.23784423 0.43393788 +0.29799690 0.24815724 0.45186924 +0.36290679 0.25890266 0.47056425 +0.43242884 0.26967147 0.48933516 +0.50711368 0.27980554 0.50711368 +0.59212645 0.28692737 0.52021444 +0.00000000 0.26741508 0.40730343 +0.04461220 0.26882249 0.40935990 +0.09065553 0.27293991 0.41537115 +0.13937310 0.27947225 0.42489635 +0.19168770 0.28798885 0.43730117 +0.24815724 0.29799690 0.45186924 +0.30900794 0.30900794 0.46789700 +0.37422168 0.32057710 0.48474690 +0.44372810 0.33227723 0.50180986 +0.51812697 0.34340937 0.51812697 +0.60258949 0.35139141 0.53034860 +0.00000000 0.33197640 0.43178842 +0.04743180 0.33338482 0.43355602 +0.09609004 0.33751260 0.43873462 +0.14705310 0.34408616 0.44697728 +0.20114656 0.35270538 0.45778021 +0.25890266 0.36290679 0.47056425 +0.32057710 0.37422168 0.48474690 +0.38621282 0.38621282 0.49978604 +0.45580517 0.39845220 0.51514987 +0.52998910 0.41023128 0.52998910 +0.61391429 0.41886090 0.54129155 +0.00000000 0.40202913 0.45578204 +0.05018874 0.40339766 0.45729296 +0.10142164 0.40741719 0.46173017 +0.15462937 0.41384505 0.46882478 +0.21054602 0.42232280 0.47818105 +0.26967147 0.43242884 0.48933516 +0.33227723 0.44372810 0.50180986 +0.39845220 0.45580517 0.51514987 +0.46824434 0.46824434 0.52889720 +0.54230549 0.48035448 0.54230549 +0.62571134 0.48944833 0.55268669 +0.00000000 0.47789395 0.47789395 +0.05270709 0.47919634 0.47919634 +0.10631257 0.48302877 0.48302877 +0.16162631 0.48917953 0.48917953 +0.21930252 0.49733313 0.49733313 +0.27980554 0.50711368 0.50711368 +0.34340937 0.51812697 0.51812697 +0.41023128 0.52998910 0.52998910 +0.48035448 0.54230549 0.54230549 +0.55441514 0.55441514 0.55441514 +0.63717571 0.56391570 0.56391570 +0.00000000 0.56461509 0.49343932 +0.05438489 0.56583363 0.49462935 +0.10959590 0.56942351 0.49813299 +0.16638009 0.57519802 0.50376179 +0.22534505 0.58287752 0.51123432 +0.28692737 0.59212645 0.52021444 +0.35139141 0.60258949 0.53034860 +0.41886090 0.61391429 0.54129155 +0.48944833 0.62571134 0.55268669 +0.56391570 0.63717571 0.56391570 +0.64649176 0.64649176 0.57258913 +0.00000000 0.00000000 0.41701632 +0.03744864 0.00000000 0.41991306 +0.07692147 0.00000000 0.42837367 +0.12019524 0.00000000 0.44172797 +0.16853142 0.00000000 0.45895609 +0.22253677 0.00000000 0.47888269 +0.28225364 0.00000000 0.50039208 +0.34738292 0.00000000 0.52253780 +0.41760426 0.00000000 0.54444638 +0.49343932 0.00000000 0.56461509 +0.57926020 0.00000000 0.57926020 +0.00000000 0.03744864 0.41991306 +0.03777591 0.03777591 0.42277459 +0.07755608 0.03873898 0.43112986 +0.12109196 0.04027401 0.44431290 +0.16962287 0.04226789 0.46131821 +0.22374901 0.04457805 0.48099415 +0.28352154 0.04706396 0.50224876 +0.34865649 0.04960690 0.52415271 +0.41884809 0.05210283 0.54584389 +0.49462935 0.05438489 0.56583363 +0.58038155 0.05587349 0.58038155 +0.00000000 0.07692147 0.42837367 +0.03873898 0.07755608 0.43112986 +0.07941781 0.07941781 0.43917213 +0.12371489 0.08237341 0.45185224 +0.17281107 0.08620263 0.46821069 +0.22729157 0.09063831 0.48716380 +0.28723258 0.09542002 0.50768477 +0.35239173 0.10032597 0.52889128 +0.42250372 0.10515830 0.54995341 +0.49813299 0.10959590 0.56942351 +0.58368653 0.11251454 0.58368653 +0.00000000 0.12019524 0.44172797 +0.04027401 0.12109196 0.44431290 +0.08237341 0.12371489 0.45185224 +0.12786445 0.12786445 0.46373897 +0.17784997 0.13323235 0.47909364 +0.23289980 0.13945861 0.49693438 +0.29312704 0.14619476 0.51632683 +0.35834844 0.15313986 0.53645608 +0.42835694 0.16001917 0.55654083 +0.50376179 0.16638009 0.57519802 +0.58900721 0.17061849 0.58900721 +0.00000000 0.16853142 0.45895609 +0.04226789 0.16962287 0.46131821 +0.08620263 0.17281107 0.46821069 +0.13323235 0.17784997 0.47909364 +0.18437541 0.18437541 0.49319434 +0.24018856 0.19197134 0.50965141 +0.30082696 0.20023419 0.52763621 +0.36617434 0.20880907 0.54641261 +0.43609040 0.21736420 0.56525956 +0.51123432 0.22534505 0.58287752 +0.59609173 0.23075305 0.59609173 +0.00000000 0.22253677 0.47888269 +0.04457805 0.22374901 0.48099415 +0.09063831 0.22729157 0.48716380 +0.13945861 0.23289980 0.49693438 +0.19197134 0.24018856 0.50965141 +0.24871955 0.24871955 0.52457919 +0.30989882 0.25806313 0.54099837 +0.37545976 0.26783386 0.55825660 +0.44532950 0.27766325 0.57570024 +0.52021444 0.28692737 0.59212645 +0.60463726 0.29332937 0.60463726 +0.00000000 0.28225364 0.50039208 +0.04706396 0.28352154 0.50224876 +0.09542002 0.28723258 0.50768477 +0.14619476 0.29312704 0.51632683 +0.20023419 0.30082696 0.52763621 +0.25806313 0.30989882 0.54099837 +0.31991075 0.31991075 0.55580057 +0.38578858 0.33046684 0.57147455 +0.45568627 0.34118086 0.58743713 +0.53034860 0.35139141 0.60258949 +0.61432251 0.35860075 0.61432251 +0.00000000 0.34738292 0.52253780 +0.04960690 0.34865649 0.52415271 +0.10032597 0.35239173 0.52889128 +0.15313986 0.35834844 0.53645608 +0.20880907 0.36617434 0.54641261 +0.26783386 0.37545976 0.55825660 +0.33046684 0.38578858 0.57147455 +0.39677065 0.39677065 0.58557890 +0.46678873 0.40801817 0.60005658 +0.54129155 0.41886090 0.61391429 +0.62482826 0.42669332 0.62482826 +0.00000000 0.41760426 0.54444638 +0.05210283 0.41884809 0.54584389 +0.10515830 0.42250372 0.54995341 +0.16001917 0.42835694 0.55654083 +0.21736420 0.43609040 0.56525956 +0.27766325 0.44532950 0.57570024 +0.34118086 0.45568627 0.58743713 +0.40801817 0.46678873 0.60005658 +0.47826050 0.47826050 0.61311215 +0.55268669 0.48944833 0.62571134 +0.63580020 0.49774197 0.63580020 +0.00000000 0.49343932 0.56461509 +0.05438489 0.49462935 0.56583363 +0.10959590 0.49813299 0.56942351 +0.16638009 0.50376179 0.57519802 +0.22534505 0.51123432 0.58287752 +0.28692737 0.52021444 0.59212645 +0.35139141 0.53034860 0.60258949 +0.41886090 0.54129155 0.61391429 +0.48944833 0.55268669 0.62571134 +0.56391570 0.56391570 0.63717571 +0.64649176 0.57258913 0.64649176 +0.00000000 0.57926020 0.57926020 +0.05587349 0.58038155 0.58038155 +0.11251454 0.58368653 0.58368653 +0.17061849 0.58900721 0.58900721 +0.23075305 0.59609173 0.59609173 +0.29332937 0.60463726 0.60463726 +0.35860075 0.61432251 0.61432251 +0.42669332 0.62482826 0.62482826 +0.49774197 0.63580020 0.63580020 +0.57258913 0.64649176 0.64649176 +0.65505751 0.65505751 0.65505751 + +CELLS 1000 9000 +8 0 1 12 11 121 122 133 132 +8 1 2 13 12 122 123 134 133 +8 2 3 14 13 123 124 135 134 +8 3 4 15 14 124 125 136 135 +8 4 5 16 15 125 126 137 136 +8 5 6 17 16 126 127 138 137 +8 6 7 18 17 127 128 139 138 +8 7 8 19 18 128 129 140 139 +8 8 9 20 19 129 130 141 140 +8 9 10 21 20 130 131 142 141 +8 11 12 23 22 132 133 144 143 +8 12 13 24 23 133 134 145 144 +8 13 14 25 24 134 135 146 145 +8 14 15 26 25 135 136 147 146 +8 15 16 27 26 136 137 148 147 +8 16 17 28 27 137 138 149 148 +8 17 18 29 28 138 139 150 149 +8 18 19 30 29 139 140 151 150 +8 19 20 31 30 140 141 152 151 +8 20 21 32 31 141 142 153 152 +8 22 23 34 33 143 144 155 154 +8 23 24 35 34 144 145 156 155 +8 24 25 36 35 145 146 157 156 +8 25 26 37 36 146 147 158 157 +8 26 27 38 37 147 148 159 158 +8 27 28 39 38 148 149 160 159 +8 28 29 40 39 149 150 161 160 +8 29 30 41 40 150 151 162 161 +8 30 31 42 41 151 152 163 162 +8 31 32 43 42 152 153 164 163 +8 33 34 45 44 154 155 166 165 +8 34 35 46 45 155 156 167 166 +8 35 36 47 46 156 157 168 167 +8 36 37 48 47 157 158 169 168 +8 37 38 49 48 158 159 170 169 +8 38 39 50 49 159 160 171 170 +8 39 40 51 50 160 161 172 171 +8 40 41 52 51 161 162 173 172 +8 41 42 53 52 162 163 174 173 +8 42 43 54 53 163 164 175 174 +8 44 45 56 55 165 166 177 176 +8 45 46 57 56 166 167 178 177 +8 46 47 58 57 167 168 179 178 +8 47 48 59 58 168 169 180 179 +8 48 49 60 59 169 170 181 180 +8 49 50 61 60 170 171 182 181 +8 50 51 62 61 171 172 183 182 +8 51 52 63 62 172 173 184 183 +8 52 53 64 63 173 174 185 184 +8 53 54 65 64 174 175 186 185 +8 55 56 67 66 176 177 188 187 +8 56 57 68 67 177 178 189 188 +8 57 58 69 68 178 179 190 189 +8 58 59 70 69 179 180 191 190 +8 59 60 71 70 180 181 192 191 +8 60 61 72 71 181 182 193 192 +8 61 62 73 72 182 183 194 193 +8 62 63 74 73 183 184 195 194 +8 63 64 75 74 184 185 196 195 +8 64 65 76 75 185 186 197 196 +8 66 67 78 77 187 188 199 198 +8 67 68 79 78 188 189 200 199 +8 68 69 80 79 189 190 201 200 +8 69 70 81 80 190 191 202 201 +8 70 71 82 81 191 192 203 202 +8 71 72 83 82 192 193 204 203 +8 72 73 84 83 193 194 205 204 +8 73 74 85 84 194 195 206 205 +8 74 75 86 85 195 196 207 206 +8 75 76 87 86 196 197 208 207 +8 77 78 89 88 198 199 210 209 +8 78 79 90 89 199 200 211 210 +8 79 80 91 90 200 201 212 211 +8 80 81 92 91 201 202 213 212 +8 81 82 93 92 202 203 214 213 +8 82 83 94 93 203 204 215 214 +8 83 84 95 94 204 205 216 215 +8 84 85 96 95 205 206 217 216 +8 85 86 97 96 206 207 218 217 +8 86 87 98 97 207 208 219 218 +8 88 89 100 99 209 210 221 220 +8 89 90 101 100 210 211 222 221 +8 90 91 102 101 211 212 223 222 +8 91 92 103 102 212 213 224 223 +8 92 93 104 103 213 214 225 224 +8 93 94 105 104 214 215 226 225 +8 94 95 106 105 215 216 227 226 +8 95 96 107 106 216 217 228 227 +8 96 97 108 107 217 218 229 228 +8 97 98 109 108 218 219 230 229 +8 99 100 111 110 220 221 232 231 +8 100 101 112 111 221 222 233 232 +8 101 102 113 112 222 223 234 233 +8 102 103 114 113 223 224 235 234 +8 103 104 115 114 224 225 236 235 +8 104 105 116 115 225 226 237 236 +8 105 106 117 116 226 227 238 237 +8 106 107 118 117 227 228 239 238 +8 107 108 119 118 228 229 240 239 +8 108 109 120 119 229 230 241 240 +8 121 122 133 132 242 243 254 253 +8 122 123 134 133 243 244 255 254 +8 123 124 135 134 244 245 256 255 +8 124 125 136 135 245 246 257 256 +8 125 126 137 136 246 247 258 257 +8 126 127 138 137 247 248 259 258 +8 127 128 139 138 248 249 260 259 +8 128 129 140 139 249 250 261 260 +8 129 130 141 140 250 251 262 261 +8 130 131 142 141 251 252 263 262 +8 132 133 144 143 253 254 265 264 +8 133 134 145 144 254 255 266 265 +8 134 135 146 145 255 256 267 266 +8 135 136 147 146 256 257 268 267 +8 136 137 148 147 257 258 269 268 +8 137 138 149 148 258 259 270 269 +8 138 139 150 149 259 260 271 270 +8 139 140 151 150 260 261 272 271 +8 140 141 152 151 261 262 273 272 +8 141 142 153 152 262 263 274 273 +8 143 144 155 154 264 265 276 275 +8 144 145 156 155 265 266 277 276 +8 145 146 157 156 266 267 278 277 +8 146 147 158 157 267 268 279 278 +8 147 148 159 158 268 269 280 279 +8 148 149 160 159 269 270 281 280 +8 149 150 161 160 270 271 282 281 +8 150 151 162 161 271 272 283 282 +8 151 152 163 162 272 273 284 283 +8 152 153 164 163 273 274 285 284 +8 154 155 166 165 275 276 287 286 +8 155 156 167 166 276 277 288 287 +8 156 157 168 167 277 278 289 288 +8 157 158 169 168 278 279 290 289 +8 158 159 170 169 279 280 291 290 +8 159 160 171 170 280 281 292 291 +8 160 161 172 171 281 282 293 292 +8 161 162 173 172 282 283 294 293 +8 162 163 174 173 283 284 295 294 +8 163 164 175 174 284 285 296 295 +8 165 166 177 176 286 287 298 297 +8 166 167 178 177 287 288 299 298 +8 167 168 179 178 288 289 300 299 +8 168 169 180 179 289 290 301 300 +8 169 170 181 180 290 291 302 301 +8 170 171 182 181 291 292 303 302 +8 171 172 183 182 292 293 304 303 +8 172 173 184 183 293 294 305 304 +8 173 174 185 184 294 295 306 305 +8 174 175 186 185 295 296 307 306 +8 176 177 188 187 297 298 309 308 +8 177 178 189 188 298 299 310 309 +8 178 179 190 189 299 300 311 310 +8 179 180 191 190 300 301 312 311 +8 180 181 192 191 301 302 313 312 +8 181 182 193 192 302 303 314 313 +8 182 183 194 193 303 304 315 314 +8 183 184 195 194 304 305 316 315 +8 184 185 196 195 305 306 317 316 +8 185 186 197 196 306 307 318 317 +8 187 188 199 198 308 309 320 319 +8 188 189 200 199 309 310 321 320 +8 189 190 201 200 310 311 322 321 +8 190 191 202 201 311 312 323 322 +8 191 192 203 202 312 313 324 323 +8 192 193 204 203 313 314 325 324 +8 193 194 205 204 314 315 326 325 +8 194 195 206 205 315 316 327 326 +8 195 196 207 206 316 317 328 327 +8 196 197 208 207 317 318 329 328 +8 198 199 210 209 319 320 331 330 +8 199 200 211 210 320 321 332 331 +8 200 201 212 211 321 322 333 332 +8 201 202 213 212 322 323 334 333 +8 202 203 214 213 323 324 335 334 +8 203 204 215 214 324 325 336 335 +8 204 205 216 215 325 326 337 336 +8 205 206 217 216 326 327 338 337 +8 206 207 218 217 327 328 339 338 +8 207 208 219 218 328 329 340 339 +8 209 210 221 220 330 331 342 341 +8 210 211 222 221 331 332 343 342 +8 211 212 223 222 332 333 344 343 +8 212 213 224 223 333 334 345 344 +8 213 214 225 224 334 335 346 345 +8 214 215 226 225 335 336 347 346 +8 215 216 227 226 336 337 348 347 +8 216 217 228 227 337 338 349 348 +8 217 218 229 228 338 339 350 349 +8 218 219 230 229 339 340 351 350 +8 220 221 232 231 341 342 353 352 +8 221 222 233 232 342 343 354 353 +8 222 223 234 233 343 344 355 354 +8 223 224 235 234 344 345 356 355 +8 224 225 236 235 345 346 357 356 +8 225 226 237 236 346 347 358 357 +8 226 227 238 237 347 348 359 358 +8 227 228 239 238 348 349 360 359 +8 228 229 240 239 349 350 361 360 +8 229 230 241 240 350 351 362 361 +8 242 243 254 253 363 364 375 374 +8 243 244 255 254 364 365 376 375 +8 244 245 256 255 365 366 377 376 +8 245 246 257 256 366 367 378 377 +8 246 247 258 257 367 368 379 378 +8 247 248 259 258 368 369 380 379 +8 248 249 260 259 369 370 381 380 +8 249 250 261 260 370 371 382 381 +8 250 251 262 261 371 372 383 382 +8 251 252 263 262 372 373 384 383 +8 253 254 265 264 374 375 386 385 +8 254 255 266 265 375 376 387 386 +8 255 256 267 266 376 377 388 387 +8 256 257 268 267 377 378 389 388 +8 257 258 269 268 378 379 390 389 +8 258 259 270 269 379 380 391 390 +8 259 260 271 270 380 381 392 391 +8 260 261 272 271 381 382 393 392 +8 261 262 273 272 382 383 394 393 +8 262 263 274 273 383 384 395 394 +8 264 265 276 275 385 386 397 396 +8 265 266 277 276 386 387 398 397 +8 266 267 278 277 387 388 399 398 +8 267 268 279 278 388 389 400 399 +8 268 269 280 279 389 390 401 400 +8 269 270 281 280 390 391 402 401 +8 270 271 282 281 391 392 403 402 +8 271 272 283 282 392 393 404 403 +8 272 273 284 283 393 394 405 404 +8 273 274 285 284 394 395 406 405 +8 275 276 287 286 396 397 408 407 +8 276 277 288 287 397 398 409 408 +8 277 278 289 288 398 399 410 409 +8 278 279 290 289 399 400 411 410 +8 279 280 291 290 400 401 412 411 +8 280 281 292 291 401 402 413 412 +8 281 282 293 292 402 403 414 413 +8 282 283 294 293 403 404 415 414 +8 283 284 295 294 404 405 416 415 +8 284 285 296 295 405 406 417 416 +8 286 287 298 297 407 408 419 418 +8 287 288 299 298 408 409 420 419 +8 288 289 300 299 409 410 421 420 +8 289 290 301 300 410 411 422 421 +8 290 291 302 301 411 412 423 422 +8 291 292 303 302 412 413 424 423 +8 292 293 304 303 413 414 425 424 +8 293 294 305 304 414 415 426 425 +8 294 295 306 305 415 416 427 426 +8 295 296 307 306 416 417 428 427 +8 297 298 309 308 418 419 430 429 +8 298 299 310 309 419 420 431 430 +8 299 300 311 310 420 421 432 431 +8 300 301 312 311 421 422 433 432 +8 301 302 313 312 422 423 434 433 +8 302 303 314 313 423 424 435 434 +8 303 304 315 314 424 425 436 435 +8 304 305 316 315 425 426 437 436 +8 305 306 317 316 426 427 438 437 +8 306 307 318 317 427 428 439 438 +8 308 309 320 319 429 430 441 440 +8 309 310 321 320 430 431 442 441 +8 310 311 322 321 431 432 443 442 +8 311 312 323 322 432 433 444 443 +8 312 313 324 323 433 434 445 444 +8 313 314 325 324 434 435 446 445 +8 314 315 326 325 435 436 447 446 +8 315 316 327 326 436 437 448 447 +8 316 317 328 327 437 438 449 448 +8 317 318 329 328 438 439 450 449 +8 319 320 331 330 440 441 452 451 +8 320 321 332 331 441 442 453 452 +8 321 322 333 332 442 443 454 453 +8 322 323 334 333 443 444 455 454 +8 323 324 335 334 444 445 456 455 +8 324 325 336 335 445 446 457 456 +8 325 326 337 336 446 447 458 457 +8 326 327 338 337 447 448 459 458 +8 327 328 339 338 448 449 460 459 +8 328 329 340 339 449 450 461 460 +8 330 331 342 341 451 452 463 462 +8 331 332 343 342 452 453 464 463 +8 332 333 344 343 453 454 465 464 +8 333 334 345 344 454 455 466 465 +8 334 335 346 345 455 456 467 466 +8 335 336 347 346 456 457 468 467 +8 336 337 348 347 457 458 469 468 +8 337 338 349 348 458 459 470 469 +8 338 339 350 349 459 460 471 470 +8 339 340 351 350 460 461 472 471 +8 341 342 353 352 462 463 474 473 +8 342 343 354 353 463 464 475 474 +8 343 344 355 354 464 465 476 475 +8 344 345 356 355 465 466 477 476 +8 345 346 357 356 466 467 478 477 +8 346 347 358 357 467 468 479 478 +8 347 348 359 358 468 469 480 479 +8 348 349 360 359 469 470 481 480 +8 349 350 361 360 470 471 482 481 +8 350 351 362 361 471 472 483 482 +8 363 364 375 374 484 485 496 495 +8 364 365 376 375 485 486 497 496 +8 365 366 377 376 486 487 498 497 +8 366 367 378 377 487 488 499 498 +8 367 368 379 378 488 489 500 499 +8 368 369 380 379 489 490 501 500 +8 369 370 381 380 490 491 502 501 +8 370 371 382 381 491 492 503 502 +8 371 372 383 382 492 493 504 503 +8 372 373 384 383 493 494 505 504 +8 374 375 386 385 495 496 507 506 +8 375 376 387 386 496 497 508 507 +8 376 377 388 387 497 498 509 508 +8 377 378 389 388 498 499 510 509 +8 378 379 390 389 499 500 511 510 +8 379 380 391 390 500 501 512 511 +8 380 381 392 391 501 502 513 512 +8 381 382 393 392 502 503 514 513 +8 382 383 394 393 503 504 515 514 +8 383 384 395 394 504 505 516 515 +8 385 386 397 396 506 507 518 517 +8 386 387 398 397 507 508 519 518 +8 387 388 399 398 508 509 520 519 +8 388 389 400 399 509 510 521 520 +8 389 390 401 400 510 511 522 521 +8 390 391 402 401 511 512 523 522 +8 391 392 403 402 512 513 524 523 +8 392 393 404 403 513 514 525 524 +8 393 394 405 404 514 515 526 525 +8 394 395 406 405 515 516 527 526 +8 396 397 408 407 517 518 529 528 +8 397 398 409 408 518 519 530 529 +8 398 399 410 409 519 520 531 530 +8 399 400 411 410 520 521 532 531 +8 400 401 412 411 521 522 533 532 +8 401 402 413 412 522 523 534 533 +8 402 403 414 413 523 524 535 534 +8 403 404 415 414 524 525 536 535 +8 404 405 416 415 525 526 537 536 +8 405 406 417 416 526 527 538 537 +8 407 408 419 418 528 529 540 539 +8 408 409 420 419 529 530 541 540 +8 409 410 421 420 530 531 542 541 +8 410 411 422 421 531 532 543 542 +8 411 412 423 422 532 533 544 543 +8 412 413 424 423 533 534 545 544 +8 413 414 425 424 534 535 546 545 +8 414 415 426 425 535 536 547 546 +8 415 416 427 426 536 537 548 547 +8 416 417 428 427 537 538 549 548 +8 418 419 430 429 539 540 551 550 +8 419 420 431 430 540 541 552 551 +8 420 421 432 431 541 542 553 552 +8 421 422 433 432 542 543 554 553 +8 422 423 434 433 543 544 555 554 +8 423 424 435 434 544 545 556 555 +8 424 425 436 435 545 546 557 556 +8 425 426 437 436 546 547 558 557 +8 426 427 438 437 547 548 559 558 +8 427 428 439 438 548 549 560 559 +8 429 430 441 440 550 551 562 561 +8 430 431 442 441 551 552 563 562 +8 431 432 443 442 552 553 564 563 +8 432 433 444 443 553 554 565 564 +8 433 434 445 444 554 555 566 565 +8 434 435 446 445 555 556 567 566 +8 435 436 447 446 556 557 568 567 +8 436 437 448 447 557 558 569 568 +8 437 438 449 448 558 559 570 569 +8 438 439 450 449 559 560 571 570 +8 440 441 452 451 561 562 573 572 +8 441 442 453 452 562 563 574 573 +8 442 443 454 453 563 564 575 574 +8 443 444 455 454 564 565 576 575 +8 444 445 456 455 565 566 577 576 +8 445 446 457 456 566 567 578 577 +8 446 447 458 457 567 568 579 578 +8 447 448 459 458 568 569 580 579 +8 448 449 460 459 569 570 581 580 +8 449 450 461 460 570 571 582 581 +8 451 452 463 462 572 573 584 583 +8 452 453 464 463 573 574 585 584 +8 453 454 465 464 574 575 586 585 +8 454 455 466 465 575 576 587 586 +8 455 456 467 466 576 577 588 587 +8 456 457 468 467 577 578 589 588 +8 457 458 469 468 578 579 590 589 +8 458 459 470 469 579 580 591 590 +8 459 460 471 470 580 581 592 591 +8 460 461 472 471 581 582 593 592 +8 462 463 474 473 583 584 595 594 +8 463 464 475 474 584 585 596 595 +8 464 465 476 475 585 586 597 596 +8 465 466 477 476 586 587 598 597 +8 466 467 478 477 587 588 599 598 +8 467 468 479 478 588 589 600 599 +8 468 469 480 479 589 590 601 600 +8 469 470 481 480 590 591 602 601 +8 470 471 482 481 591 592 603 602 +8 471 472 483 482 592 593 604 603 +8 484 485 496 495 605 606 617 616 +8 485 486 497 496 606 607 618 617 +8 486 487 498 497 607 608 619 618 +8 487 488 499 498 608 609 620 619 +8 488 489 500 499 609 610 621 620 +8 489 490 501 500 610 611 622 621 +8 490 491 502 501 611 612 623 622 +8 491 492 503 502 612 613 624 623 +8 492 493 504 503 613 614 625 624 +8 493 494 505 504 614 615 626 625 +8 495 496 507 506 616 617 628 627 +8 496 497 508 507 617 618 629 628 +8 497 498 509 508 618 619 630 629 +8 498 499 510 509 619 620 631 630 +8 499 500 511 510 620 621 632 631 +8 500 501 512 511 621 622 633 632 +8 501 502 513 512 622 623 634 633 +8 502 503 514 513 623 624 635 634 +8 503 504 515 514 624 625 636 635 +8 504 505 516 515 625 626 637 636 +8 506 507 518 517 627 628 639 638 +8 507 508 519 518 628 629 640 639 +8 508 509 520 519 629 630 641 640 +8 509 510 521 520 630 631 642 641 +8 510 511 522 521 631 632 643 642 +8 511 512 523 522 632 633 644 643 +8 512 513 524 523 633 634 645 644 +8 513 514 525 524 634 635 646 645 +8 514 515 526 525 635 636 647 646 +8 515 516 527 526 636 637 648 647 +8 517 518 529 528 638 639 650 649 +8 518 519 530 529 639 640 651 650 +8 519 520 531 530 640 641 652 651 +8 520 521 532 531 641 642 653 652 +8 521 522 533 532 642 643 654 653 +8 522 523 534 533 643 644 655 654 +8 523 524 535 534 644 645 656 655 +8 524 525 536 535 645 646 657 656 +8 525 526 537 536 646 647 658 657 +8 526 527 538 537 647 648 659 658 +8 528 529 540 539 649 650 661 660 +8 529 530 541 540 650 651 662 661 +8 530 531 542 541 651 652 663 662 +8 531 532 543 542 652 653 664 663 +8 532 533 544 543 653 654 665 664 +8 533 534 545 544 654 655 666 665 +8 534 535 546 545 655 656 667 666 +8 535 536 547 546 656 657 668 667 +8 536 537 548 547 657 658 669 668 +8 537 538 549 548 658 659 670 669 +8 539 540 551 550 660 661 672 671 +8 540 541 552 551 661 662 673 672 +8 541 542 553 552 662 663 674 673 +8 542 543 554 553 663 664 675 674 +8 543 544 555 554 664 665 676 675 +8 544 545 556 555 665 666 677 676 +8 545 546 557 556 666 667 678 677 +8 546 547 558 557 667 668 679 678 +8 547 548 559 558 668 669 680 679 +8 548 549 560 559 669 670 681 680 +8 550 551 562 561 671 672 683 682 +8 551 552 563 562 672 673 684 683 +8 552 553 564 563 673 674 685 684 +8 553 554 565 564 674 675 686 685 +8 554 555 566 565 675 676 687 686 +8 555 556 567 566 676 677 688 687 +8 556 557 568 567 677 678 689 688 +8 557 558 569 568 678 679 690 689 +8 558 559 570 569 679 680 691 690 +8 559 560 571 570 680 681 692 691 +8 561 562 573 572 682 683 694 693 +8 562 563 574 573 683 684 695 694 +8 563 564 575 574 684 685 696 695 +8 564 565 576 575 685 686 697 696 +8 565 566 577 576 686 687 698 697 +8 566 567 578 577 687 688 699 698 +8 567 568 579 578 688 689 700 699 +8 568 569 580 579 689 690 701 700 +8 569 570 581 580 690 691 702 701 +8 570 571 582 581 691 692 703 702 +8 572 573 584 583 693 694 705 704 +8 573 574 585 584 694 695 706 705 +8 574 575 586 585 695 696 707 706 +8 575 576 587 586 696 697 708 707 +8 576 577 588 587 697 698 709 708 +8 577 578 589 588 698 699 710 709 +8 578 579 590 589 699 700 711 710 +8 579 580 591 590 700 701 712 711 +8 580 581 592 591 701 702 713 712 +8 581 582 593 592 702 703 714 713 +8 583 584 595 594 704 705 716 715 +8 584 585 596 595 705 706 717 716 +8 585 586 597 596 706 707 718 717 +8 586 587 598 597 707 708 719 718 +8 587 588 599 598 708 709 720 719 +8 588 589 600 599 709 710 721 720 +8 589 590 601 600 710 711 722 721 +8 590 591 602 601 711 712 723 722 +8 591 592 603 602 712 713 724 723 +8 592 593 604 603 713 714 725 724 +8 605 606 617 616 726 727 738 737 +8 606 607 618 617 727 728 739 738 +8 607 608 619 618 728 729 740 739 +8 608 609 620 619 729 730 741 740 +8 609 610 621 620 730 731 742 741 +8 610 611 622 621 731 732 743 742 +8 611 612 623 622 732 733 744 743 +8 612 613 624 623 733 734 745 744 +8 613 614 625 624 734 735 746 745 +8 614 615 626 625 735 736 747 746 +8 616 617 628 627 737 738 749 748 +8 617 618 629 628 738 739 750 749 +8 618 619 630 629 739 740 751 750 +8 619 620 631 630 740 741 752 751 +8 620 621 632 631 741 742 753 752 +8 621 622 633 632 742 743 754 753 +8 622 623 634 633 743 744 755 754 +8 623 624 635 634 744 745 756 755 +8 624 625 636 635 745 746 757 756 +8 625 626 637 636 746 747 758 757 +8 627 628 639 638 748 749 760 759 +8 628 629 640 639 749 750 761 760 +8 629 630 641 640 750 751 762 761 +8 630 631 642 641 751 752 763 762 +8 631 632 643 642 752 753 764 763 +8 632 633 644 643 753 754 765 764 +8 633 634 645 644 754 755 766 765 +8 634 635 646 645 755 756 767 766 +8 635 636 647 646 756 757 768 767 +8 636 637 648 647 757 758 769 768 +8 638 639 650 649 759 760 771 770 +8 639 640 651 650 760 761 772 771 +8 640 641 652 651 761 762 773 772 +8 641 642 653 652 762 763 774 773 +8 642 643 654 653 763 764 775 774 +8 643 644 655 654 764 765 776 775 +8 644 645 656 655 765 766 777 776 +8 645 646 657 656 766 767 778 777 +8 646 647 658 657 767 768 779 778 +8 647 648 659 658 768 769 780 779 +8 649 650 661 660 770 771 782 781 +8 650 651 662 661 771 772 783 782 +8 651 652 663 662 772 773 784 783 +8 652 653 664 663 773 774 785 784 +8 653 654 665 664 774 775 786 785 +8 654 655 666 665 775 776 787 786 +8 655 656 667 666 776 777 788 787 +8 656 657 668 667 777 778 789 788 +8 657 658 669 668 778 779 790 789 +8 658 659 670 669 779 780 791 790 +8 660 661 672 671 781 782 793 792 +8 661 662 673 672 782 783 794 793 +8 662 663 674 673 783 784 795 794 +8 663 664 675 674 784 785 796 795 +8 664 665 676 675 785 786 797 796 +8 665 666 677 676 786 787 798 797 +8 666 667 678 677 787 788 799 798 +8 667 668 679 678 788 789 800 799 +8 668 669 680 679 789 790 801 800 +8 669 670 681 680 790 791 802 801 +8 671 672 683 682 792 793 804 803 +8 672 673 684 683 793 794 805 804 +8 673 674 685 684 794 795 806 805 +8 674 675 686 685 795 796 807 806 +8 675 676 687 686 796 797 808 807 +8 676 677 688 687 797 798 809 808 +8 677 678 689 688 798 799 810 809 +8 678 679 690 689 799 800 811 810 +8 679 680 691 690 800 801 812 811 +8 680 681 692 691 801 802 813 812 +8 682 683 694 693 803 804 815 814 +8 683 684 695 694 804 805 816 815 +8 684 685 696 695 805 806 817 816 +8 685 686 697 696 806 807 818 817 +8 686 687 698 697 807 808 819 818 +8 687 688 699 698 808 809 820 819 +8 688 689 700 699 809 810 821 820 +8 689 690 701 700 810 811 822 821 +8 690 691 702 701 811 812 823 822 +8 691 692 703 702 812 813 824 823 +8 693 694 705 704 814 815 826 825 +8 694 695 706 705 815 816 827 826 +8 695 696 707 706 816 817 828 827 +8 696 697 708 707 817 818 829 828 +8 697 698 709 708 818 819 830 829 +8 698 699 710 709 819 820 831 830 +8 699 700 711 710 820 821 832 831 +8 700 701 712 711 821 822 833 832 +8 701 702 713 712 822 823 834 833 +8 702 703 714 713 823 824 835 834 +8 704 705 716 715 825 826 837 836 +8 705 706 717 716 826 827 838 837 +8 706 707 718 717 827 828 839 838 +8 707 708 719 718 828 829 840 839 +8 708 709 720 719 829 830 841 840 +8 709 710 721 720 830 831 842 841 +8 710 711 722 721 831 832 843 842 +8 711 712 723 722 832 833 844 843 +8 712 713 724 723 833 834 845 844 +8 713 714 725 724 834 835 846 845 +8 726 727 738 737 847 848 859 858 +8 727 728 739 738 848 849 860 859 +8 728 729 740 739 849 850 861 860 +8 729 730 741 740 850 851 862 861 +8 730 731 742 741 851 852 863 862 +8 731 732 743 742 852 853 864 863 +8 732 733 744 743 853 854 865 864 +8 733 734 745 744 854 855 866 865 +8 734 735 746 745 855 856 867 866 +8 735 736 747 746 856 857 868 867 +8 737 738 749 748 858 859 870 869 +8 738 739 750 749 859 860 871 870 +8 739 740 751 750 860 861 872 871 +8 740 741 752 751 861 862 873 872 +8 741 742 753 752 862 863 874 873 +8 742 743 754 753 863 864 875 874 +8 743 744 755 754 864 865 876 875 +8 744 745 756 755 865 866 877 876 +8 745 746 757 756 866 867 878 877 +8 746 747 758 757 867 868 879 878 +8 748 749 760 759 869 870 881 880 +8 749 750 761 760 870 871 882 881 +8 750 751 762 761 871 872 883 882 +8 751 752 763 762 872 873 884 883 +8 752 753 764 763 873 874 885 884 +8 753 754 765 764 874 875 886 885 +8 754 755 766 765 875 876 887 886 +8 755 756 767 766 876 877 888 887 +8 756 757 768 767 877 878 889 888 +8 757 758 769 768 878 879 890 889 +8 759 760 771 770 880 881 892 891 +8 760 761 772 771 881 882 893 892 +8 761 762 773 772 882 883 894 893 +8 762 763 774 773 883 884 895 894 +8 763 764 775 774 884 885 896 895 +8 764 765 776 775 885 886 897 896 +8 765 766 777 776 886 887 898 897 +8 766 767 778 777 887 888 899 898 +8 767 768 779 778 888 889 900 899 +8 768 769 780 779 889 890 901 900 +8 770 771 782 781 891 892 903 902 +8 771 772 783 782 892 893 904 903 +8 772 773 784 783 893 894 905 904 +8 773 774 785 784 894 895 906 905 +8 774 775 786 785 895 896 907 906 +8 775 776 787 786 896 897 908 907 +8 776 777 788 787 897 898 909 908 +8 777 778 789 788 898 899 910 909 +8 778 779 790 789 899 900 911 910 +8 779 780 791 790 900 901 912 911 +8 781 782 793 792 902 903 914 913 +8 782 783 794 793 903 904 915 914 +8 783 784 795 794 904 905 916 915 +8 784 785 796 795 905 906 917 916 +8 785 786 797 796 906 907 918 917 +8 786 787 798 797 907 908 919 918 +8 787 788 799 798 908 909 920 919 +8 788 789 800 799 909 910 921 920 +8 789 790 801 800 910 911 922 921 +8 790 791 802 801 911 912 923 922 +8 792 793 804 803 913 914 925 924 +8 793 794 805 804 914 915 926 925 +8 794 795 806 805 915 916 927 926 +8 795 796 807 806 916 917 928 927 +8 796 797 808 807 917 918 929 928 +8 797 798 809 808 918 919 930 929 +8 798 799 810 809 919 920 931 930 +8 799 800 811 810 920 921 932 931 +8 800 801 812 811 921 922 933 932 +8 801 802 813 812 922 923 934 933 +8 803 804 815 814 924 925 936 935 +8 804 805 816 815 925 926 937 936 +8 805 806 817 816 926 927 938 937 +8 806 807 818 817 927 928 939 938 +8 807 808 819 818 928 929 940 939 +8 808 809 820 819 929 930 941 940 +8 809 810 821 820 930 931 942 941 +8 810 811 822 821 931 932 943 942 +8 811 812 823 822 932 933 944 943 +8 812 813 824 823 933 934 945 944 +8 814 815 826 825 935 936 947 946 +8 815 816 827 826 936 937 948 947 +8 816 817 828 827 937 938 949 948 +8 817 818 829 828 938 939 950 949 +8 818 819 830 829 939 940 951 950 +8 819 820 831 830 940 941 952 951 +8 820 821 832 831 941 942 953 952 +8 821 822 833 832 942 943 954 953 +8 822 823 834 833 943 944 955 954 +8 823 824 835 834 944 945 956 955 +8 825 826 837 836 946 947 958 957 +8 826 827 838 837 947 948 959 958 +8 827 828 839 838 948 949 960 959 +8 828 829 840 839 949 950 961 960 +8 829 830 841 840 950 951 962 961 +8 830 831 842 841 951 952 963 962 +8 831 832 843 842 952 953 964 963 +8 832 833 844 843 953 954 965 964 +8 833 834 845 844 954 955 966 965 +8 834 835 846 845 955 956 967 966 +8 847 848 859 858 968 969 980 979 +8 848 849 860 859 969 970 981 980 +8 849 850 861 860 970 971 982 981 +8 850 851 862 861 971 972 983 982 +8 851 852 863 862 972 973 984 983 +8 852 853 864 863 973 974 985 984 +8 853 854 865 864 974 975 986 985 +8 854 855 866 865 975 976 987 986 +8 855 856 867 866 976 977 988 987 +8 856 857 868 867 977 978 989 988 +8 858 859 870 869 979 980 991 990 +8 859 860 871 870 980 981 992 991 +8 860 861 872 871 981 982 993 992 +8 861 862 873 872 982 983 994 993 +8 862 863 874 873 983 984 995 994 +8 863 864 875 874 984 985 996 995 +8 864 865 876 875 985 986 997 996 +8 865 866 877 876 986 987 998 997 +8 866 867 878 877 987 988 999 998 +8 867 868 879 878 988 989 1000 999 +8 869 870 881 880 990 991 1002 1001 +8 870 871 882 881 991 992 1003 1002 +8 871 872 883 882 992 993 1004 1003 +8 872 873 884 883 993 994 1005 1004 +8 873 874 885 884 994 995 1006 1005 +8 874 875 886 885 995 996 1007 1006 +8 875 876 887 886 996 997 1008 1007 +8 876 877 888 887 997 998 1009 1008 +8 877 878 889 888 998 999 1010 1009 +8 878 879 890 889 999 1000 1011 1010 +8 880 881 892 891 1001 1002 1013 1012 +8 881 882 893 892 1002 1003 1014 1013 +8 882 883 894 893 1003 1004 1015 1014 +8 883 884 895 894 1004 1005 1016 1015 +8 884 885 896 895 1005 1006 1017 1016 +8 885 886 897 896 1006 1007 1018 1017 +8 886 887 898 897 1007 1008 1019 1018 +8 887 888 899 898 1008 1009 1020 1019 +8 888 889 900 899 1009 1010 1021 1020 +8 889 890 901 900 1010 1011 1022 1021 +8 891 892 903 902 1012 1013 1024 1023 +8 892 893 904 903 1013 1014 1025 1024 +8 893 894 905 904 1014 1015 1026 1025 +8 894 895 906 905 1015 1016 1027 1026 +8 895 896 907 906 1016 1017 1028 1027 +8 896 897 908 907 1017 1018 1029 1028 +8 897 898 909 908 1018 1019 1030 1029 +8 898 899 910 909 1019 1020 1031 1030 +8 899 900 911 910 1020 1021 1032 1031 +8 900 901 912 911 1021 1022 1033 1032 +8 902 903 914 913 1023 1024 1035 1034 +8 903 904 915 914 1024 1025 1036 1035 +8 904 905 916 915 1025 1026 1037 1036 +8 905 906 917 916 1026 1027 1038 1037 +8 906 907 918 917 1027 1028 1039 1038 +8 907 908 919 918 1028 1029 1040 1039 +8 908 909 920 919 1029 1030 1041 1040 +8 909 910 921 920 1030 1031 1042 1041 +8 910 911 922 921 1031 1032 1043 1042 +8 911 912 923 922 1032 1033 1044 1043 +8 913 914 925 924 1034 1035 1046 1045 +8 914 915 926 925 1035 1036 1047 1046 +8 915 916 927 926 1036 1037 1048 1047 +8 916 917 928 927 1037 1038 1049 1048 +8 917 918 929 928 1038 1039 1050 1049 +8 918 919 930 929 1039 1040 1051 1050 +8 919 920 931 930 1040 1041 1052 1051 +8 920 921 932 931 1041 1042 1053 1052 +8 921 922 933 932 1042 1043 1054 1053 +8 922 923 934 933 1043 1044 1055 1054 +8 924 925 936 935 1045 1046 1057 1056 +8 925 926 937 936 1046 1047 1058 1057 +8 926 927 938 937 1047 1048 1059 1058 +8 927 928 939 938 1048 1049 1060 1059 +8 928 929 940 939 1049 1050 1061 1060 +8 929 930 941 940 1050 1051 1062 1061 +8 930 931 942 941 1051 1052 1063 1062 +8 931 932 943 942 1052 1053 1064 1063 +8 932 933 944 943 1053 1054 1065 1064 +8 933 934 945 944 1054 1055 1066 1065 +8 935 936 947 946 1056 1057 1068 1067 +8 936 937 948 947 1057 1058 1069 1068 +8 937 938 949 948 1058 1059 1070 1069 +8 938 939 950 949 1059 1060 1071 1070 +8 939 940 951 950 1060 1061 1072 1071 +8 940 941 952 951 1061 1062 1073 1072 +8 941 942 953 952 1062 1063 1074 1073 +8 942 943 954 953 1063 1064 1075 1074 +8 943 944 955 954 1064 1065 1076 1075 +8 944 945 956 955 1065 1066 1077 1076 +8 946 947 958 957 1067 1068 1079 1078 +8 947 948 959 958 1068 1069 1080 1079 +8 948 949 960 959 1069 1070 1081 1080 +8 949 950 961 960 1070 1071 1082 1081 +8 950 951 962 961 1071 1072 1083 1082 +8 951 952 963 962 1072 1073 1084 1083 +8 952 953 964 963 1073 1074 1085 1084 +8 953 954 965 964 1074 1075 1086 1085 +8 954 955 966 965 1075 1076 1087 1086 +8 955 956 967 966 1076 1077 1088 1087 +8 968 969 980 979 1089 1090 1101 1100 +8 969 970 981 980 1090 1091 1102 1101 +8 970 971 982 981 1091 1092 1103 1102 +8 971 972 983 982 1092 1093 1104 1103 +8 972 973 984 983 1093 1094 1105 1104 +8 973 974 985 984 1094 1095 1106 1105 +8 974 975 986 985 1095 1096 1107 1106 +8 975 976 987 986 1096 1097 1108 1107 +8 976 977 988 987 1097 1098 1109 1108 +8 977 978 989 988 1098 1099 1110 1109 +8 979 980 991 990 1100 1101 1112 1111 +8 980 981 992 991 1101 1102 1113 1112 +8 981 982 993 992 1102 1103 1114 1113 +8 982 983 994 993 1103 1104 1115 1114 +8 983 984 995 994 1104 1105 1116 1115 +8 984 985 996 995 1105 1106 1117 1116 +8 985 986 997 996 1106 1107 1118 1117 +8 986 987 998 997 1107 1108 1119 1118 +8 987 988 999 998 1108 1109 1120 1119 +8 988 989 1000 999 1109 1110 1121 1120 +8 990 991 1002 1001 1111 1112 1123 1122 +8 991 992 1003 1002 1112 1113 1124 1123 +8 992 993 1004 1003 1113 1114 1125 1124 +8 993 994 1005 1004 1114 1115 1126 1125 +8 994 995 1006 1005 1115 1116 1127 1126 +8 995 996 1007 1006 1116 1117 1128 1127 +8 996 997 1008 1007 1117 1118 1129 1128 +8 997 998 1009 1008 1118 1119 1130 1129 +8 998 999 1010 1009 1119 1120 1131 1130 +8 999 1000 1011 1010 1120 1121 1132 1131 +8 1001 1002 1013 1012 1122 1123 1134 1133 +8 1002 1003 1014 1013 1123 1124 1135 1134 +8 1003 1004 1015 1014 1124 1125 1136 1135 +8 1004 1005 1016 1015 1125 1126 1137 1136 +8 1005 1006 1017 1016 1126 1127 1138 1137 +8 1006 1007 1018 1017 1127 1128 1139 1138 +8 1007 1008 1019 1018 1128 1129 1140 1139 +8 1008 1009 1020 1019 1129 1130 1141 1140 +8 1009 1010 1021 1020 1130 1131 1142 1141 +8 1010 1011 1022 1021 1131 1132 1143 1142 +8 1012 1013 1024 1023 1133 1134 1145 1144 +8 1013 1014 1025 1024 1134 1135 1146 1145 +8 1014 1015 1026 1025 1135 1136 1147 1146 +8 1015 1016 1027 1026 1136 1137 1148 1147 +8 1016 1017 1028 1027 1137 1138 1149 1148 +8 1017 1018 1029 1028 1138 1139 1150 1149 +8 1018 1019 1030 1029 1139 1140 1151 1150 +8 1019 1020 1031 1030 1140 1141 1152 1151 +8 1020 1021 1032 1031 1141 1142 1153 1152 +8 1021 1022 1033 1032 1142 1143 1154 1153 +8 1023 1024 1035 1034 1144 1145 1156 1155 +8 1024 1025 1036 1035 1145 1146 1157 1156 +8 1025 1026 1037 1036 1146 1147 1158 1157 +8 1026 1027 1038 1037 1147 1148 1159 1158 +8 1027 1028 1039 1038 1148 1149 1160 1159 +8 1028 1029 1040 1039 1149 1150 1161 1160 +8 1029 1030 1041 1040 1150 1151 1162 1161 +8 1030 1031 1042 1041 1151 1152 1163 1162 +8 1031 1032 1043 1042 1152 1153 1164 1163 +8 1032 1033 1044 1043 1153 1154 1165 1164 +8 1034 1035 1046 1045 1155 1156 1167 1166 +8 1035 1036 1047 1046 1156 1157 1168 1167 +8 1036 1037 1048 1047 1157 1158 1169 1168 +8 1037 1038 1049 1048 1158 1159 1170 1169 +8 1038 1039 1050 1049 1159 1160 1171 1170 +8 1039 1040 1051 1050 1160 1161 1172 1171 +8 1040 1041 1052 1051 1161 1162 1173 1172 +8 1041 1042 1053 1052 1162 1163 1174 1173 +8 1042 1043 1054 1053 1163 1164 1175 1174 +8 1043 1044 1055 1054 1164 1165 1176 1175 +8 1045 1046 1057 1056 1166 1167 1178 1177 +8 1046 1047 1058 1057 1167 1168 1179 1178 +8 1047 1048 1059 1058 1168 1169 1180 1179 +8 1048 1049 1060 1059 1169 1170 1181 1180 +8 1049 1050 1061 1060 1170 1171 1182 1181 +8 1050 1051 1062 1061 1171 1172 1183 1182 +8 1051 1052 1063 1062 1172 1173 1184 1183 +8 1052 1053 1064 1063 1173 1174 1185 1184 +8 1053 1054 1065 1064 1174 1175 1186 1185 +8 1054 1055 1066 1065 1175 1176 1187 1186 +8 1056 1057 1068 1067 1177 1178 1189 1188 +8 1057 1058 1069 1068 1178 1179 1190 1189 +8 1058 1059 1070 1069 1179 1180 1191 1190 +8 1059 1060 1071 1070 1180 1181 1192 1191 +8 1060 1061 1072 1071 1181 1182 1193 1192 +8 1061 1062 1073 1072 1182 1183 1194 1193 +8 1062 1063 1074 1073 1183 1184 1195 1194 +8 1063 1064 1075 1074 1184 1185 1196 1195 +8 1064 1065 1076 1075 1185 1186 1197 1196 +8 1065 1066 1077 1076 1186 1187 1198 1197 +8 1067 1068 1079 1078 1188 1189 1200 1199 +8 1068 1069 1080 1079 1189 1190 1201 1200 +8 1069 1070 1081 1080 1190 1191 1202 1201 +8 1070 1071 1082 1081 1191 1192 1203 1202 +8 1071 1072 1083 1082 1192 1193 1204 1203 +8 1072 1073 1084 1083 1193 1194 1205 1204 +8 1073 1074 1085 1084 1194 1195 1206 1205 +8 1074 1075 1086 1085 1195 1196 1207 1206 +8 1075 1076 1087 1086 1196 1197 1208 1207 +8 1076 1077 1088 1087 1197 1198 1209 1208 +8 1089 1090 1101 1100 1210 1211 1222 1221 +8 1090 1091 1102 1101 1211 1212 1223 1222 +8 1091 1092 1103 1102 1212 1213 1224 1223 +8 1092 1093 1104 1103 1213 1214 1225 1224 +8 1093 1094 1105 1104 1214 1215 1226 1225 +8 1094 1095 1106 1105 1215 1216 1227 1226 +8 1095 1096 1107 1106 1216 1217 1228 1227 +8 1096 1097 1108 1107 1217 1218 1229 1228 +8 1097 1098 1109 1108 1218 1219 1230 1229 +8 1098 1099 1110 1109 1219 1220 1231 1230 +8 1100 1101 1112 1111 1221 1222 1233 1232 +8 1101 1102 1113 1112 1222 1223 1234 1233 +8 1102 1103 1114 1113 1223 1224 1235 1234 +8 1103 1104 1115 1114 1224 1225 1236 1235 +8 1104 1105 1116 1115 1225 1226 1237 1236 +8 1105 1106 1117 1116 1226 1227 1238 1237 +8 1106 1107 1118 1117 1227 1228 1239 1238 +8 1107 1108 1119 1118 1228 1229 1240 1239 +8 1108 1109 1120 1119 1229 1230 1241 1240 +8 1109 1110 1121 1120 1230 1231 1242 1241 +8 1111 1112 1123 1122 1232 1233 1244 1243 +8 1112 1113 1124 1123 1233 1234 1245 1244 +8 1113 1114 1125 1124 1234 1235 1246 1245 +8 1114 1115 1126 1125 1235 1236 1247 1246 +8 1115 1116 1127 1126 1236 1237 1248 1247 +8 1116 1117 1128 1127 1237 1238 1249 1248 +8 1117 1118 1129 1128 1238 1239 1250 1249 +8 1118 1119 1130 1129 1239 1240 1251 1250 +8 1119 1120 1131 1130 1240 1241 1252 1251 +8 1120 1121 1132 1131 1241 1242 1253 1252 +8 1122 1123 1134 1133 1243 1244 1255 1254 +8 1123 1124 1135 1134 1244 1245 1256 1255 +8 1124 1125 1136 1135 1245 1246 1257 1256 +8 1125 1126 1137 1136 1246 1247 1258 1257 +8 1126 1127 1138 1137 1247 1248 1259 1258 +8 1127 1128 1139 1138 1248 1249 1260 1259 +8 1128 1129 1140 1139 1249 1250 1261 1260 +8 1129 1130 1141 1140 1250 1251 1262 1261 +8 1130 1131 1142 1141 1251 1252 1263 1262 +8 1131 1132 1143 1142 1252 1253 1264 1263 +8 1133 1134 1145 1144 1254 1255 1266 1265 +8 1134 1135 1146 1145 1255 1256 1267 1266 +8 1135 1136 1147 1146 1256 1257 1268 1267 +8 1136 1137 1148 1147 1257 1258 1269 1268 +8 1137 1138 1149 1148 1258 1259 1270 1269 +8 1138 1139 1150 1149 1259 1260 1271 1270 +8 1139 1140 1151 1150 1260 1261 1272 1271 +8 1140 1141 1152 1151 1261 1262 1273 1272 +8 1141 1142 1153 1152 1262 1263 1274 1273 +8 1142 1143 1154 1153 1263 1264 1275 1274 +8 1144 1145 1156 1155 1265 1266 1277 1276 +8 1145 1146 1157 1156 1266 1267 1278 1277 +8 1146 1147 1158 1157 1267 1268 1279 1278 +8 1147 1148 1159 1158 1268 1269 1280 1279 +8 1148 1149 1160 1159 1269 1270 1281 1280 +8 1149 1150 1161 1160 1270 1271 1282 1281 +8 1150 1151 1162 1161 1271 1272 1283 1282 +8 1151 1152 1163 1162 1272 1273 1284 1283 +8 1152 1153 1164 1163 1273 1274 1285 1284 +8 1153 1154 1165 1164 1274 1275 1286 1285 +8 1155 1156 1167 1166 1276 1277 1288 1287 +8 1156 1157 1168 1167 1277 1278 1289 1288 +8 1157 1158 1169 1168 1278 1279 1290 1289 +8 1158 1159 1170 1169 1279 1280 1291 1290 +8 1159 1160 1171 1170 1280 1281 1292 1291 +8 1160 1161 1172 1171 1281 1282 1293 1292 +8 1161 1162 1173 1172 1282 1283 1294 1293 +8 1162 1163 1174 1173 1283 1284 1295 1294 +8 1163 1164 1175 1174 1284 1285 1296 1295 +8 1164 1165 1176 1175 1285 1286 1297 1296 +8 1166 1167 1178 1177 1287 1288 1299 1298 +8 1167 1168 1179 1178 1288 1289 1300 1299 +8 1168 1169 1180 1179 1289 1290 1301 1300 +8 1169 1170 1181 1180 1290 1291 1302 1301 +8 1170 1171 1182 1181 1291 1292 1303 1302 +8 1171 1172 1183 1182 1292 1293 1304 1303 +8 1172 1173 1184 1183 1293 1294 1305 1304 +8 1173 1174 1185 1184 1294 1295 1306 1305 +8 1174 1175 1186 1185 1295 1296 1307 1306 +8 1175 1176 1187 1186 1296 1297 1308 1307 +8 1177 1178 1189 1188 1298 1299 1310 1309 +8 1178 1179 1190 1189 1299 1300 1311 1310 +8 1179 1180 1191 1190 1300 1301 1312 1311 +8 1180 1181 1192 1191 1301 1302 1313 1312 +8 1181 1182 1193 1192 1302 1303 1314 1313 +8 1182 1183 1194 1193 1303 1304 1315 1314 +8 1183 1184 1195 1194 1304 1305 1316 1315 +8 1184 1185 1196 1195 1305 1306 1317 1316 +8 1185 1186 1197 1196 1306 1307 1318 1317 +8 1186 1187 1198 1197 1307 1308 1319 1318 +8 1188 1189 1200 1199 1309 1310 1321 1320 +8 1189 1190 1201 1200 1310 1311 1322 1321 +8 1190 1191 1202 1201 1311 1312 1323 1322 +8 1191 1192 1203 1202 1312 1313 1324 1323 +8 1192 1193 1204 1203 1313 1314 1325 1324 +8 1193 1194 1205 1204 1314 1315 1326 1325 +8 1194 1195 1206 1205 1315 1316 1327 1326 +8 1195 1196 1207 1206 1316 1317 1328 1327 +8 1196 1197 1208 1207 1317 1318 1329 1328 +8 1197 1198 1209 1208 1318 1319 1330 1329 + +CELL_TYPES 1000 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 + +POINT_DATA 1331 +VECTORS velocity float +0.00000000 0.00000000 0.00000000 +0.01255686 0.00000000 0.00000000 +-0.09483495 0.00000000 0.00000000 +0.01359320 0.00000000 0.00000000 +-0.06877149 0.00000000 0.00000000 +-0.09108924 0.00000000 0.00000000 +-0.06529717 0.00000000 0.00000000 +-0.06082157 0.00000000 0.00000000 +-0.17683359 0.00000000 0.00000000 +-0.76304465 0.00000000 0.00000000 +-0.89221786 0.00000000 0.00000000 +0.00000000 0.01255686 0.00000000 +-0.01790596 -0.01790596 0.00000000 +-0.01893837 0.01058892 0.00000000 +-0.01432075 0.01129048 0.00000000 +-0.09953064 -0.01718877 0.00000000 +-0.02979424 0.00414939 0.00000000 +-0.11979122 -0.00165618 0.00000000 +-0.01400056 -0.00600328 0.00000000 +-0.22372195 -0.03066312 0.00000000 +-0.77777523 -0.08521294 0.00000000 +-0.89069110 -0.10375184 0.00000000 +0.00000000 -0.09483495 0.00000000 +0.01058892 -0.01893837 0.00000000 +0.02868529 0.02868529 0.00000000 +-0.07048540 -0.00958673 0.00000000 +-0.03040053 0.00424549 0.00000000 +-0.00569182 -0.00107918 0.00000000 +-0.16991456 -0.01467785 0.00000000 +0.05557426 -0.01486519 0.00000000 +-0.34521204 -0.08704010 0.00000000 +-0.81164775 -0.18095308 0.00000000 +-0.88476837 -0.20667050 0.00000000 +0.00000000 0.01359320 0.00000000 +0.01129048 -0.01432075 0.00000000 +-0.00958673 -0.07048540 0.00000000 +-0.01871710 -0.01871710 0.00000000 +0.02987074 0.03066441 0.00000000 +-0.07864453 -0.02954805 0.00000000 +-0.04443228 0.00728286 0.00000000 +0.00990947 -0.04068166 0.00000000 +-0.50353474 -0.18667468 0.00000000 +-0.84080414 -0.28682303 0.00000000 +-0.87190638 -0.30540552 0.00000000 +0.00000000 -0.06877149 0.00000000 +-0.01718877 -0.09953064 0.00000000 +0.00424549 -0.03040053 0.00000000 +0.03066441 0.02987074 0.00000000 +-0.03470352 -0.03470352 0.00000000 +-0.04347352 -0.01318958 0.00000000 +0.06049588 0.01335262 0.00000000 +-0.20128544 -0.14103245 0.00000000 +-0.65198962 -0.32385182 0.00000000 +-0.84644775 -0.39066510 0.00000000 +-0.85119212 -0.39548766 0.00000000 +0.00000000 -0.09108924 0.00000000 +0.00414939 -0.02979424 0.00000000 +-0.00107918 -0.00569182 0.00000000 +-0.02954805 -0.07864453 0.00000000 +-0.01318958 -0.04347352 0.00000000 +0.05090066 0.05090066 0.00000000 +-0.05511400 -0.07321595 0.00000000 +-0.44895956 -0.32653470 0.00000000 +-0.73981762 -0.46473367 0.00000000 +-0.82791133 -0.47999560 0.00000000 +-0.82423162 -0.47445669 0.00000000 +0.00000000 -0.06529717 0.00000000 +-0.00165618 -0.11979122 0.00000000 +-0.01467785 -0.16991456 0.00000000 +0.00728286 -0.04443228 0.00000000 +0.01335262 0.06049588 0.00000000 +-0.07321595 -0.05511400 0.00000000 +-0.34141996 -0.34141996 0.00000000 +-0.61424514 -0.52709380 0.00000000 +-0.75390542 -0.57170560 0.00000000 +-0.79628425 -0.55261094 0.00000000 +-0.79352281 -0.54229063 0.00000000 +0.00000000 -0.06082157 0.00000000 +-0.00600328 -0.01400056 0.00000000 +-0.01486519 0.05557426 0.00000000 +-0.04068166 0.00990947 0.00000000 +-0.14103245 -0.20128544 0.00000000 +-0.32653470 -0.44895956 0.00000000 +-0.52709380 -0.61424514 0.00000000 +-0.66286491 -0.66286491 0.00000000 +-0.72717477 -0.64181314 0.00000000 +-0.76073727 -0.61239020 0.00000000 +-0.76125168 -0.59983936 0.00000000 +0.00000000 -0.17683359 0.00000000 +-0.03066312 -0.22372195 0.00000000 +-0.08704010 -0.34521204 0.00000000 +-0.18667468 -0.50353474 0.00000000 +-0.32385182 -0.65198962 0.00000000 +-0.46473367 -0.73981762 0.00000000 +-0.57170560 -0.75390542 0.00000000 +-0.64181314 -0.72717477 0.00000000 +-0.69184238 -0.69184238 0.00000000 +-0.72576389 -0.66113497 0.00000000 +-0.72980814 -0.64718057 0.00000000 +0.00000000 -0.76304465 0.00000000 +-0.08521294 -0.77777523 0.00000000 +-0.18095308 -0.81164775 0.00000000 +-0.28682303 -0.84080414 0.00000000 +-0.39066510 -0.84644775 0.00000000 +-0.47999560 -0.82791133 0.00000000 +-0.55261094 -0.79628425 0.00000000 +-0.61239020 -0.76073727 0.00000000 +-0.66113497 -0.72576389 0.00000000 +-0.69614090 -0.69614090 0.00000000 +-0.70399389 -0.68108181 0.00000000 +0.00000000 -0.89221786 0.00000000 +-0.10375184 -0.89069110 0.00000000 +-0.20667050 -0.88476837 0.00000000 +-0.30540552 -0.87190638 0.00000000 +-0.39548766 -0.85119212 0.00000000 +-0.47445669 -0.82423162 0.00000000 +-0.54229063 -0.79352281 0.00000000 +-0.59983936 -0.76125168 0.00000000 +-0.64718057 -0.72980814 0.00000000 +-0.68108181 -0.70399389 0.00000000 +-0.69023094 -0.69023094 0.00000000 +0.00000000 0.00000000 0.01255686 +-0.01790596 0.00000000 -0.01790596 +-0.01893837 0.00000000 0.01058892 +-0.01432075 0.00000000 0.01129048 +-0.09953064 0.00000000 -0.01718877 +-0.02979424 0.00000000 0.00414939 +-0.11979122 0.00000000 -0.00165618 +-0.01400056 0.00000000 -0.00600328 +-0.22372195 0.00000000 -0.03066312 +-0.77777523 0.00000000 -0.08521294 +-0.89069110 0.00000000 -0.10375184 +0.00000000 -0.01790596 -0.01790596 +-0.01689090 -0.01689090 -0.01689090 +0.01937232 0.00907652 0.00907652 +-0.04179363 0.00470707 0.00470707 +-0.08143856 -0.00880470 -0.00880470 +-0.00648873 0.00442025 0.00442025 +-0.15073143 -0.00508184 -0.00508184 +0.01917166 -0.00600589 -0.00600589 +-0.26730308 -0.03473961 -0.03473961 +-0.79077357 -0.08715612 -0.08715612 +-0.88892194 -0.10371347 -0.10371347 +0.00000000 -0.01893837 0.01058892 +0.00907652 0.01937232 0.00907652 +0.02201999 0.02201999 -0.00101736 +-0.07313340 -0.01161995 -0.00850555 +-0.00604054 0.00994791 0.00476665 +-0.02601812 -0.00662272 -0.00326580 +-0.15579641 -0.01196295 -0.00711509 +0.06178750 -0.01501634 -0.00619048 +-0.38143621 -0.09480113 -0.04648892 +-0.82008715 -0.18371120 -0.09183810 +-0.88242428 -0.20627793 -0.10325282 +0.00000000 -0.01432075 0.01129048 +0.00470707 -0.04179363 0.00470707 +-0.01161995 -0.07313340 -0.00850555 +-0.00178244 -0.00178244 0.00265408 +0.02786513 0.02624620 0.00927460 +-0.09436472 -0.03260943 -0.01123070 +-0.01468730 0.01065572 0.00124339 +-0.01613755 -0.04794334 -0.01537844 +-0.53080270 -0.19627845 -0.06401814 +-0.84380317 -0.28877463 -0.09639378 +-0.86901148 -0.30436733 -0.10174492 +0.00000000 -0.09953064 -0.01718877 +-0.00880470 -0.08143856 -0.00880470 +0.00994791 -0.00604054 0.00476665 +0.02624620 0.02786513 0.00927460 +-0.04683540 -0.04683540 -0.00776530 +-0.02775506 -0.00359523 -0.00212326 +0.06278737 0.01048493 0.00533018 +-0.23454127 -0.15572512 -0.03665819 +-0.66819177 -0.33221924 -0.08216745 +-0.84538373 -0.39076320 -0.09804449 +-0.84806123 -0.39381874 -0.09894888 +0.00000000 -0.02979424 0.00414939 +0.00442025 -0.00648873 0.00442025 +-0.00662272 -0.02601812 -0.00326580 +-0.03260943 -0.09436472 -0.01123070 +-0.00359523 -0.02775506 -0.00212326 +0.05302696 0.05302696 0.01163325 +-0.08159471 -0.09260859 -0.01794579 +-0.47200246 -0.34155947 -0.06583743 +-0.74500445 -0.46853732 -0.09377176 +-0.82509533 -0.47851295 -0.09627030 +-0.82118682 -0.47238472 -0.09512980 +0.00000000 -0.11979122 -0.00165618 +-0.00508184 -0.15073143 -0.00508184 +-0.01196295 -0.15579641 -0.00711509 +0.01065572 -0.01468730 0.00124339 +0.01048493 0.06278737 0.00533018 +-0.09260859 -0.08159471 -0.01794579 +-0.36268193 -0.36268193 -0.05820001 +-0.62416662 -0.53560841 -0.08889716 +-0.75275768 -0.57103333 -0.09578104 +-0.79327778 -0.55046753 -0.09250019 +-0.79073595 -0.54004136 -0.09074155 +0.00000000 -0.01400056 -0.00600328 +-0.00600589 0.01917166 -0.00600589 +-0.01501634 0.06178750 -0.00619048 +-0.04794334 -0.01613755 -0.01537844 +-0.15572512 -0.23454127 -0.03665819 +-0.34155947 -0.47200246 -0.06583743 +-0.53560841 -0.62416662 -0.08889716 +-0.66282743 -0.66282743 -0.09549542 +-0.72454631 -0.63948296 -0.09233753 +-0.75801348 -0.61010084 -0.08803594 +-0.75877302 -0.59756239 -0.08618160 +0.00000000 -0.22372195 -0.03066312 +-0.03473961 -0.26730308 -0.03473961 +-0.09480113 -0.38143621 -0.04648892 +-0.19627845 -0.53080270 -0.06401814 +-0.33221924 -0.66819177 -0.08216745 +-0.46853732 -0.74500445 -0.09377176 +-0.57103333 -0.75275768 -0.09578104 +-0.63948296 -0.72454631 -0.09233753 +-0.68933983 -0.68933983 -0.08766136 +-0.72336847 -0.65887538 -0.08360147 +-0.72762058 -0.64496630 -0.08180280 +0.00000000 -0.77777523 -0.08521294 +-0.08715612 -0.79077357 -0.08715612 +-0.18371120 -0.82008715 -0.09183810 +-0.28877463 -0.84380317 -0.09639378 +-0.39076320 -0.84538373 -0.09804449 +-0.47851295 -0.82509533 -0.09627030 +-0.55046753 -0.79327778 -0.09250019 +-0.61010084 -0.75801348 -0.08803594 +-0.65887538 -0.72336847 -0.08360147 +-0.69399038 -0.69399038 -0.07987246 +-0.70202393 -0.67898724 -0.07817257 +0.00000000 -0.89069110 -0.10375184 +-0.10371347 -0.88892194 -0.10371347 +-0.20627793 -0.88242428 -0.10325282 +-0.30436733 -0.86901148 -0.10174492 +-0.39381874 -0.84806123 -0.09894888 +-0.47238472 -0.82118682 -0.09512980 +-0.54004136 -0.79073595 -0.09074155 +-0.59756239 -0.75877302 -0.08618160 +-0.64496630 -0.72762058 -0.08180280 +-0.67898724 -0.70202393 -0.07817257 +-0.68831308 -0.68831308 -0.07658202 +0.00000000 0.00000000 -0.09483495 +0.01058892 0.00000000 -0.01893837 +0.02868529 0.00000000 0.02868529 +-0.07048540 0.00000000 -0.00958673 +-0.03040053 0.00000000 0.00424549 +-0.00569182 0.00000000 -0.00107918 +-0.16991456 0.00000000 -0.01467785 +0.05557426 0.00000000 -0.01486519 +-0.34521204 0.00000000 -0.08704010 +-0.81164775 0.00000000 -0.18095308 +-0.88476837 0.00000000 -0.20667050 +0.00000000 0.01058892 -0.01893837 +0.00907652 0.00907652 0.01937232 +0.02201999 -0.00101736 0.02201999 +-0.07313340 -0.00850555 -0.01161995 +-0.00604054 0.00476665 0.00994791 +-0.02601812 -0.00326580 -0.00662272 +-0.15579641 -0.00711509 -0.01196295 +0.06178750 -0.00619048 -0.01501634 +-0.38143621 -0.04648892 -0.09480113 +-0.82008715 -0.09183810 -0.18371120 +-0.88242428 -0.10325282 -0.20627793 +0.00000000 0.02868529 0.02868529 +-0.00101736 0.02201999 0.02201999 +-0.01370517 -0.01370517 -0.01370517 +-0.03789350 -0.00584291 -0.00584291 +0.03082386 0.01760618 0.01760618 +-0.08132750 -0.01954153 -0.01954153 +-0.06848098 -0.00144706 -0.00144706 +0.03144169 -0.02215810 -0.02215810 +-0.47768370 -0.11651843 -0.11651843 +-0.83771085 -0.19003333 -0.19003333 +-0.87460349 -0.20464185 -0.20464185 +0.00000000 -0.07048540 -0.00958673 +-0.00850555 -0.07313340 -0.01161995 +-0.00584291 -0.03789350 -0.00584291 +0.03019663 0.03019663 0.02128905 +-0.01100180 -0.00042391 -0.00040344 +-0.10198881 -0.03119256 -0.02282352 +0.04880167 0.01642565 0.01110309 +-0.11101744 -0.07548180 -0.04932061 +-0.60193794 -0.22231006 -0.14705935 +-0.84802854 -0.29254151 -0.19524254 +-0.86002000 -0.30095031 -0.20098431 +0.00000000 -0.03040053 0.00424549 +0.00476665 -0.00604054 0.00994791 +0.01760618 0.03082386 0.01760618 +-0.00042391 -0.01100180 -0.00040344 +-0.06363719 -0.06363719 -0.02583436 +0.02109410 0.02312962 0.00940285 +0.04234738 -0.00730841 -0.00090234 +-0.32825475 -0.19900305 -0.09563564 +-0.70725446 -0.35307454 -0.17583335 +-0.84018801 -0.38966560 -0.19540168 +-0.83870505 -0.38874589 -0.19511957 +0.00000000 -0.00569182 -0.00107918 +-0.00326580 -0.02601812 -0.00662272 +-0.01954153 -0.08132750 -0.01954153 +-0.03119256 -0.10198881 -0.02282352 +0.02312962 0.02109410 0.00940285 +0.04565360 0.04565360 0.01803601 +-0.16614848 -0.15512797 -0.05942580 +-0.53305683 -0.38233402 -0.14987638 +-0.75479693 -0.47616824 -0.19088408 +-0.81629552 -0.47364345 -0.19043942 +-0.81222959 -0.46625748 -0.18756221 +0.00000000 -0.16991456 -0.01467785 +-0.00711509 -0.15579641 -0.01196295 +-0.00144706 -0.06848098 -0.00144706 +0.01642565 0.04880167 0.01110309 +-0.00730841 0.04234738 -0.00090234 +-0.15512797 -0.16614848 -0.05942580 +-0.42131301 -0.42131301 -0.13694322 +-0.64646135 -0.55494181 -0.18497797 +-0.74766260 -0.56756738 -0.19036049 +-0.78439198 -0.54407867 -0.18267108 +-0.78257091 -0.53344201 -0.17905905 +0.00000000 0.05557426 -0.01486519 +-0.00619048 0.06178750 -0.01501634 +-0.02215810 0.03144169 -0.02215810 +-0.07548180 -0.11101744 -0.04932061 +-0.19900305 -0.32825475 -0.09563564 +-0.38233402 -0.53305683 -0.14987638 +-0.55494181 -0.64646135 -0.18497797 +-0.66040489 -0.66040489 -0.19036012 +-0.71662568 -0.63243377 -0.18249705 +-0.75002504 -0.60337079 -0.17394592 +-0.75150769 -0.59088829 -0.17026666 +0.00000000 -0.34521204 -0.08704010 +-0.04648892 -0.38143621 -0.09480113 +-0.11651843 -0.47768370 -0.11651843 +-0.22231006 -0.60193794 -0.14705935 +-0.35307454 -0.70725446 -0.17583335 +-0.47616824 -0.75479693 -0.19088408 +-0.56756738 -0.74766260 -0.19036049 +-0.63243377 -0.71662568 -0.18249705 +-0.68198358 -0.68198358 -0.17325302 +-0.71633890 -0.65223978 -0.16535510 +-0.72119706 -0.63846965 -0.16180977 +0.00000000 -0.81164775 -0.18095308 +-0.09183810 -0.82008715 -0.18371120 +-0.19003333 -0.83771085 -0.19003333 +-0.29254151 -0.84802854 -0.19524254 +-0.38966560 -0.84018801 -0.19540168 +-0.47364345 -0.81629552 -0.19043942 +-0.54407867 -0.78439198 -0.18267108 +-0.60337079 -0.75002504 -0.17394592 +-0.65223978 -0.71633890 -0.16535510 +-0.68767092 -0.68767092 -0.15811021 +-0.69622915 -0.67283536 -0.15476041 +0.00000000 -0.88476837 -0.20667050 +-0.10325282 -0.88242428 -0.20627793 +-0.20464185 -0.87460349 -0.20464185 +-0.30095031 -0.86002000 -0.20098431 +-0.38874589 -0.83870505 -0.19511957 +-0.46625748 -0.81222959 -0.18756221 +-0.53344201 -0.78257091 -0.17905905 +-0.59088829 -0.75150769 -0.17026666 +-0.63846965 -0.72119706 -0.16180977 +-0.67283536 -0.69622915 -0.15476041 +-0.68267326 -0.68267326 -0.15162007 +0.00000000 0.00000000 0.01359320 +0.01129048 0.00000000 -0.01432075 +-0.00958673 0.00000000 -0.07048540 +-0.01871710 0.00000000 -0.01871710 +0.02987074 0.00000000 0.03066441 +-0.07864453 0.00000000 -0.02954805 +-0.04443228 0.00000000 0.00728286 +0.00990947 0.00000000 -0.04068166 +-0.50353474 0.00000000 -0.18667468 +-0.84080414 0.00000000 -0.28682303 +-0.87190638 0.00000000 -0.30540552 +0.00000000 0.01129048 -0.01432075 +0.00470707 0.00470707 -0.04179363 +-0.01161995 -0.00850555 -0.07313340 +-0.00178244 0.00265408 -0.00178244 +0.02786513 0.00927460 0.02624620 +-0.09436472 -0.01123070 -0.03260943 +-0.01468730 0.00124339 0.01065572 +-0.01613755 -0.01537844 -0.04794334 +-0.53080270 -0.06401814 -0.19627845 +-0.84380317 -0.09639378 -0.28877463 +-0.86901148 -0.10174492 -0.30436733 +0.00000000 -0.00958673 -0.07048540 +-0.00850555 -0.01161995 -0.07313340 +-0.00584291 -0.00584291 -0.03789350 +0.03019663 0.02128905 0.03019663 +-0.01100180 -0.00040344 -0.00042391 +-0.10198881 -0.02282352 -0.03119256 +0.04880167 0.01110309 0.01642565 +-0.11101744 -0.04932061 -0.07548180 +-0.60193794 -0.14705935 -0.22231006 +-0.84802854 -0.19524254 -0.29254151 +-0.86002000 -0.20098431 -0.30095031 +0.00000000 -0.01871710 -0.01871710 +0.00265408 -0.00178244 -0.00178244 +0.02128905 0.03019663 0.03019663 +0.01757361 0.01757361 0.01757361 +-0.06667521 -0.03855220 -0.03855220 +-0.01528814 0.00206939 0.00206939 +0.06656776 0.00849144 0.00849144 +-0.27090843 -0.12624225 -0.12624225 +-0.68649813 -0.25551026 -0.25551026 +-0.84411757 -0.29360360 -0.29360360 +-0.84463243 -0.29472310 -0.29472310 +0.00000000 0.02987074 0.03066441 +0.00927460 0.02786513 0.02624620 +-0.00040344 -0.01100180 -0.00042391 +-0.03855220 -0.06667521 -0.03855220 +-0.02397566 -0.02397566 -0.01522644 +0.06814608 0.04735684 0.03639625 +-0.06365078 -0.06770219 -0.04919945 +-0.46026636 -0.26498463 -0.19625206 +-0.74503320 -0.37476401 -0.28106234 +-0.82755069 -0.38484575 -0.28913343 +-0.82345658 -0.38030448 -0.28581088 +0.00000000 -0.07864453 -0.02954805 +-0.01123070 -0.09436472 -0.03260943 +-0.02282352 -0.10198881 -0.03119256 +0.00206939 -0.01528814 0.00206939 +0.04735684 0.06814608 0.03639625 +-0.02178582 -0.02178582 -0.01703954 +-0.30045401 -0.25667240 -0.14959687 +-0.60694420 -0.43375502 -0.25884369 +-0.75712124 -0.47929336 -0.28831272 +-0.80125305 -0.46484536 -0.27997643 +-0.79791490 -0.45640487 -0.27490623 +0.00000000 -0.04443228 0.00728286 +0.00124339 -0.01468730 0.01065572 +0.01110309 0.04880167 0.01642565 +0.00849144 0.06656776 0.00849144 +-0.06770219 -0.06365078 -0.04919945 +-0.25667240 -0.30045401 -0.14959687 +-0.49786033 -0.49786033 -0.24663636 +-0.66337884 -0.57000970 -0.28580346 +-0.73608519 -0.55899630 -0.28104073 +-0.77011959 -0.53369803 -0.26834965 +-0.76957898 -0.52293056 -0.26286063 +0.00000000 0.00990947 -0.04068166 +-0.01537844 -0.01613755 -0.04794334 +-0.04932061 -0.11101744 -0.07548180 +-0.12624225 -0.27090843 -0.12624225 +-0.26498463 -0.46026636 -0.19625206 +-0.43375502 -0.60694420 -0.25884369 +-0.57000970 -0.66337884 -0.28580346 +-0.65177387 -0.65177387 -0.28176879 +-0.70361505 -0.62080095 -0.26831506 +-0.73729413 -0.59260795 -0.25586526 +-0.73992897 -0.58026107 -0.25044073 +0.00000000 -0.50353474 -0.18667468 +-0.06401814 -0.53080270 -0.19627845 +-0.14705935 -0.60193794 -0.22231006 +-0.25551026 -0.68649813 -0.25551026 +-0.37476401 -0.74503320 -0.28106234 +-0.47929336 -0.75712124 -0.28831272 +-0.55899630 -0.73608519 -0.28104073 +-0.62080095 -0.70361505 -0.26831506 +-0.67022067 -0.67022067 -0.25495048 +-0.70511284 -0.64163231 -0.24363838 +-0.71092520 -0.62810126 -0.23844692 +0.00000000 -0.84080414 -0.28682303 +-0.09639378 -0.84380317 -0.28877463 +-0.19524254 -0.84802854 -0.29254151 +-0.29360360 -0.84411757 -0.29360360 +-0.38484575 -0.82755069 -0.28913343 +-0.46484536 -0.80125305 -0.27997643 +-0.53369803 -0.77011959 -0.26834965 +-0.59260795 -0.73729413 -0.25586526 +-0.64163231 -0.70511284 -0.24363838 +-0.67755337 -0.67755337 -0.23326397 +-0.68693320 -0.66299604 -0.22835942 +0.00000000 -0.87190638 -0.30540552 +-0.10174492 -0.86901148 -0.30436733 +-0.20098431 -0.86002000 -0.30095031 +-0.29472310 -0.84463243 -0.29472310 +-0.38030448 -0.82345658 -0.28581088 +-0.45640487 -0.79791490 -0.27490623 +-0.52293056 -0.76957898 -0.26286063 +-0.58026107 -0.73992897 -0.25044073 +-0.62810126 -0.71092520 -0.23844692 +-0.66299604 -0.68693320 -0.22835942 +-0.67363148 -0.67363148 -0.22374881 +0.00000000 0.00000000 -0.06877149 +-0.01718877 0.00000000 -0.09953064 +0.00424549 0.00000000 -0.03040053 +0.03066441 0.00000000 0.02987074 +-0.03470352 0.00000000 -0.03470352 +-0.04347352 0.00000000 -0.01318958 +0.06049588 0.00000000 0.01335262 +-0.20128544 0.00000000 -0.14103245 +-0.65198962 0.00000000 -0.32385182 +-0.84644775 0.00000000 -0.39066510 +-0.85119212 0.00000000 -0.39548766 +0.00000000 -0.01718877 -0.09953064 +-0.00880470 -0.00880470 -0.08143856 +0.00994791 0.00476665 -0.00604054 +0.02624620 0.00927460 0.02786513 +-0.04683540 -0.00776530 -0.04683540 +-0.02775506 -0.00212326 -0.00359523 +0.06278737 0.00533018 0.01048493 +-0.23454127 -0.03665819 -0.15572512 +-0.66819177 -0.08216745 -0.33221924 +-0.84538373 -0.09804449 -0.39076320 +-0.84806123 -0.09894888 -0.39381874 +0.00000000 0.00424549 -0.03040053 +0.00476665 0.00994791 -0.00604054 +0.01760618 0.01760618 0.03082386 +-0.00042391 -0.00040344 -0.01100180 +-0.06363719 -0.02583436 -0.06363719 +0.02109410 0.00940285 0.02312962 +0.04234738 -0.00090234 -0.00730841 +-0.32825475 -0.09563564 -0.19900305 +-0.70725446 -0.17583335 -0.35307454 +-0.84018801 -0.19540168 -0.38966560 +-0.83870505 -0.19511957 -0.38874589 +0.00000000 0.03066441 0.02987074 +0.00927460 0.02624620 0.02786513 +-0.00040344 -0.00042391 -0.01100180 +-0.03855220 -0.03855220 -0.06667521 +-0.02397566 -0.01522644 -0.02397566 +0.06814608 0.03639625 0.04735684 +-0.06365078 -0.04919945 -0.06770219 +-0.46026636 -0.19625206 -0.26498463 +-0.74503320 -0.28106234 -0.37476401 +-0.82755069 -0.28913343 -0.38484575 +-0.82345658 -0.28581088 -0.38030448 +0.00000000 -0.03470352 -0.03470352 +-0.00776530 -0.04683540 -0.04683540 +-0.02583436 -0.06363719 -0.06363719 +-0.01522644 -0.02397566 -0.02397566 +0.04790421 0.04790421 0.04790421 +0.01757805 0.00532832 0.00532832 +-0.25190175 -0.17465694 -0.17465694 +-0.58679971 -0.33434658 -0.33434658 +-0.75865393 -0.38437484 -0.38437484 +-0.80697821 -0.37545423 -0.37545423 +-0.80318977 -0.36893723 -0.36893723 +0.00000000 -0.04347352 -0.01318958 +-0.00212326 -0.02775506 -0.00359523 +0.00940285 0.02109410 0.02312962 +0.03639625 0.06814608 0.04735684 +0.00532832 0.01757805 0.00532832 +-0.16992056 -0.16992056 -0.13654308 +-0.44713877 -0.37290409 -0.29637098 +-0.65700597 -0.47046771 -0.37641353 +-0.74540141 -0.47278057 -0.37888300 +-0.78074330 -0.45239341 -0.36262474 +-0.77913577 -0.44345012 -0.35542756 +0.00000000 0.06049588 0.01335262 +0.00533018 0.06278737 0.01048493 +-0.00090234 0.04234738 -0.00730841 +-0.04919945 -0.06365078 -0.06770219 +-0.17465694 -0.25190175 -0.17465694 +-0.37290409 -0.44713877 -0.29637098 +-0.55640613 -0.55640613 -0.37084894 +-0.66189001 -0.56923062 -0.38077705 +-0.71793826 -0.54506411 -0.36484376 +-0.75134898 -0.51991530 -0.34788886 +-0.75256075 -0.50918384 -0.34063560 +0.00000000 -0.20128544 -0.14103245 +-0.03665819 -0.23454127 -0.15572512 +-0.09563564 -0.32825475 -0.19900305 +-0.19625206 -0.46026636 -0.26498463 +-0.33434658 -0.58679971 -0.33434658 +-0.47046771 -0.65700597 -0.37641353 +-0.56923062 -0.66189001 -0.38077705 +-0.63622857 -0.63622857 -0.36626764 +-0.68625062 -0.60522656 -0.34808256 +-0.72058916 -0.57844219 -0.33239802 +-0.72471469 -0.56633672 -0.32536211 +0.00000000 -0.65198962 -0.32385182 +-0.08216745 -0.66819177 -0.33221924 +-0.17583335 -0.70725446 -0.35307454 +-0.28106234 -0.74503320 -0.37476401 +-0.38437484 -0.75865393 -0.38437484 +-0.47278057 -0.74540141 -0.37888300 +-0.54506411 -0.71793826 -0.36484376 +-0.60522656 -0.68625062 -0.34808256 +-0.65473449 -0.65473449 -0.33138094 +-0.69032513 -0.62764921 -0.31721070 +-0.69736623 -0.61446271 -0.31051397 +0.00000000 -0.84644775 -0.39066510 +-0.09804449 -0.84538373 -0.39076320 +-0.19540168 -0.84018801 -0.38966560 +-0.28913343 -0.82755069 -0.38484575 +-0.37545423 -0.80697821 -0.37545423 +-0.45239341 -0.78074330 -0.36262474 +-0.51991530 -0.75134898 -0.34788886 +-0.57844219 -0.72058916 -0.33239802 +-0.62764921 -0.69032513 -0.31721070 +-0.66418152 -0.66418152 -0.30420183 +-0.67461241 -0.65001050 -0.29787494 +0.00000000 -0.85119212 -0.39548766 +-0.09894888 -0.84806123 -0.39381874 +-0.19511957 -0.83870505 -0.38874589 +-0.28581088 -0.82345658 -0.38030448 +-0.36893723 -0.80318977 -0.36893723 +-0.44345012 -0.77913577 -0.35542756 +-0.50918384 -0.75256075 -0.34063560 +-0.56633672 -0.72471469 -0.32536211 +-0.61446271 -0.69736623 -0.31051397 +-0.65001050 -0.67461241 -0.29787494 +-0.66165857 -0.66165857 -0.29190835 +0.00000000 0.00000000 -0.09108924 +0.00414939 0.00000000 -0.02979424 +-0.00107918 0.00000000 -0.00569182 +-0.02954805 0.00000000 -0.07864453 +-0.01318958 0.00000000 -0.04347352 +0.05090066 0.00000000 0.05090066 +-0.05511400 0.00000000 -0.07321595 +-0.44895956 0.00000000 -0.32653470 +-0.73981762 0.00000000 -0.46473367 +-0.82791133 0.00000000 -0.47999560 +-0.82423162 0.00000000 -0.47445669 +0.00000000 0.00414939 -0.02979424 +0.00442025 0.00442025 -0.00648873 +-0.00662272 -0.00326580 -0.02601812 +-0.03260943 -0.01123070 -0.09436472 +-0.00359523 -0.00212326 -0.02775506 +0.05302696 0.01163325 0.05302696 +-0.08159471 -0.01794579 -0.09260859 +-0.47200246 -0.06583743 -0.34155947 +-0.74500445 -0.09377176 -0.46853732 +-0.82509533 -0.09627030 -0.47851295 +-0.82118682 -0.09512980 -0.47238472 +0.00000000 -0.00107918 -0.00569182 +-0.00326580 -0.00662272 -0.02601812 +-0.01954153 -0.01954153 -0.08132750 +-0.03119256 -0.02282352 -0.10198881 +0.02312962 0.00940285 0.02109410 +0.04565360 0.01803601 0.04565360 +-0.16614848 -0.05942580 -0.15512797 +-0.53305683 -0.14987638 -0.38233402 +-0.75479693 -0.19088408 -0.47616824 +-0.81629552 -0.19043942 -0.47364345 +-0.81222959 -0.18756221 -0.46625748 +0.00000000 -0.02954805 -0.07864453 +-0.01123070 -0.03260943 -0.09436472 +-0.02282352 -0.03119256 -0.10198881 +0.00206939 0.00206939 -0.01528814 +0.04735684 0.03639625 0.06814608 +-0.02178582 -0.01703954 -0.02178582 +-0.30045401 -0.14959687 -0.25667240 +-0.60694420 -0.25884369 -0.43375502 +-0.75712124 -0.28831272 -0.47929336 +-0.80125305 -0.27997643 -0.46484536 +-0.79791490 -0.27490623 -0.45640487 +0.00000000 -0.01318958 -0.04347352 +-0.00212326 -0.00359523 -0.02775506 +0.00940285 0.02312962 0.02109410 +0.03639625 0.04735684 0.06814608 +0.00532832 0.00532832 0.01757805 +-0.16992056 -0.13654308 -0.16992056 +-0.44713877 -0.29637098 -0.37290409 +-0.65700597 -0.37641353 -0.47046771 +-0.74540141 -0.37888300 -0.47278057 +-0.78074330 -0.36262474 -0.45239341 +-0.77913577 -0.35542756 -0.44345012 +0.00000000 0.05090066 0.05090066 +0.01163325 0.05302696 0.05302696 +0.01803601 0.04565360 0.04565360 +-0.01703954 -0.02178582 -0.02178582 +-0.13654308 -0.16992056 -0.16992056 +-0.34397167 -0.34397167 -0.34397167 +-0.54774863 -0.45638006 -0.45638006 +-0.66451228 -0.47704928 -0.47704928 +-0.72284742 -0.45845130 -0.45845130 +-0.75623518 -0.43722494 -0.43722494 +-0.75695117 -0.42820893 -0.42820893 +0.00000000 -0.05511400 -0.07321595 +-0.01794579 -0.08159471 -0.09260859 +-0.05942580 -0.16614848 -0.15512797 +-0.14959687 -0.30045401 -0.25667240 +-0.29637098 -0.44713877 -0.37290409 +-0.45638006 -0.54774863 -0.45638006 +-0.57230563 -0.57230563 -0.47769305 +-0.64449443 -0.55439364 -0.46301051 +-0.69512805 -0.52730570 -0.44030814 +-0.72916105 -0.50353553 -0.42035442 +-0.73242437 -0.49300861 -0.41152775 +0.00000000 -0.44895956 -0.32653470 +-0.06583743 -0.47200246 -0.34155947 +-0.14987638 -0.53305683 -0.38233402 +-0.25884369 -0.60694420 -0.43375502 +-0.37641353 -0.65700597 -0.47046771 +-0.47704928 -0.66451228 -0.47704928 +-0.55439364 -0.64449443 -0.46301051 +-0.61585124 -0.61585124 -0.44219004 +-0.66556701 -0.58665059 -0.42085685 +-0.70078890 -0.56162671 -0.40268668 +-0.70663428 -0.54987902 -0.39421472 +0.00000000 -0.73981762 -0.46473367 +-0.09377176 -0.74500445 -0.46853732 +-0.19088408 -0.75479693 -0.47616824 +-0.28831272 -0.75712124 -0.47929336 +-0.37888300 -0.74540141 -0.47278057 +-0.45845130 -0.72284742 -0.45845130 +-0.52730570 -0.69512805 -0.44030814 +-0.58665059 -0.66556701 -0.42085685 +-0.63632851 -0.63632851 -0.40169149 +-0.67270682 -0.61098924 -0.38526561 +-0.68116751 -0.59825305 -0.37723468 +0.00000000 -0.82791133 -0.47999560 +-0.09627030 -0.82509533 -0.47851295 +-0.19043942 -0.81629552 -0.47364345 +-0.27997643 -0.80125305 -0.46484536 +-0.36262474 -0.78074330 -0.45239341 +-0.43722494 -0.75623518 -0.43722494 +-0.50353553 -0.72916105 -0.42035442 +-0.56162671 -0.70078890 -0.40268668 +-0.61098924 -0.67270682 -0.38526561 +-0.64819069 -0.64819069 -0.37016115 +-0.65982690 -0.63450830 -0.36257220 +0.00000000 -0.82423162 -0.47445669 +-0.09512980 -0.82118682 -0.47238472 +-0.18756221 -0.81222959 -0.46625748 +-0.27490623 -0.79791490 -0.45640487 +-0.35542756 -0.77913577 -0.44345012 +-0.42820893 -0.75695117 -0.42820893 +-0.49300861 -0.73242437 -0.41152775 +-0.54987902 -0.70663428 -0.39421472 +-0.59825305 -0.68116751 -0.37723468 +-0.63450830 -0.65982690 -0.36257220 +-0.64730703 -0.64730703 -0.35539297 +0.00000000 0.00000000 -0.06529717 +-0.00165618 0.00000000 -0.11979122 +-0.01467785 0.00000000 -0.16991456 +0.00728286 0.00000000 -0.04443228 +0.01335262 0.00000000 0.06049588 +-0.07321595 0.00000000 -0.05511400 +-0.34141996 0.00000000 -0.34141996 +-0.61424514 0.00000000 -0.52709380 +-0.75390542 0.00000000 -0.57170560 +-0.79628425 0.00000000 -0.55261094 +-0.79352281 0.00000000 -0.54229063 +0.00000000 -0.00165618 -0.11979122 +-0.00508184 -0.00508184 -0.15073143 +-0.01196295 -0.00711509 -0.15579641 +0.01065572 0.00124339 -0.01468730 +0.01048493 0.00533018 0.06278737 +-0.09260859 -0.01794579 -0.08159471 +-0.36268193 -0.05820001 -0.36268193 +-0.62416662 -0.08889716 -0.53560841 +-0.75275768 -0.09578104 -0.57103333 +-0.79327778 -0.09250019 -0.55046753 +-0.79073595 -0.09074155 -0.54004136 +0.00000000 -0.01467785 -0.16991456 +-0.00711509 -0.01196295 -0.15579641 +-0.00144706 -0.00144706 -0.06848098 +0.01642565 0.01110309 0.04880167 +-0.00730841 -0.00090234 0.04234738 +-0.15512797 -0.05942580 -0.16614848 +-0.42131301 -0.13694322 -0.42131301 +-0.64646135 -0.18497797 -0.55494181 +-0.74766260 -0.19036049 -0.56756738 +-0.78439198 -0.18267108 -0.54407867 +-0.78257091 -0.17905905 -0.53344201 +0.00000000 0.00728286 -0.04443228 +0.00124339 0.01065572 -0.01468730 +0.01110309 0.01642565 0.04880167 +0.00849144 0.00849144 0.06656776 +-0.06770219 -0.04919945 -0.06365078 +-0.25667240 -0.14959687 -0.30045401 +-0.49786033 -0.24663636 -0.49786033 +-0.66337884 -0.28580346 -0.57000970 +-0.73608519 -0.28104073 -0.55899630 +-0.77011959 -0.26834965 -0.53369803 +-0.76957898 -0.26286063 -0.52293056 +0.00000000 0.01335262 0.06049588 +0.00533018 0.01048493 0.06278737 +-0.00090234 -0.00730841 0.04234738 +-0.04919945 -0.06770219 -0.06365078 +-0.17465694 -0.17465694 -0.25190175 +-0.37290409 -0.29637098 -0.44713877 +-0.55640613 -0.37084894 -0.55640613 +-0.66189001 -0.38077705 -0.56923062 +-0.71793826 -0.36484376 -0.54506411 +-0.75134898 -0.34788886 -0.51991530 +-0.75256075 -0.34063560 -0.50918384 +0.00000000 -0.07321595 -0.05511400 +-0.01794579 -0.09260859 -0.08159471 +-0.05942580 -0.15512797 -0.16614848 +-0.14959687 -0.25667240 -0.30045401 +-0.29637098 -0.37290409 -0.44713877 +-0.45638006 -0.45638006 -0.54774863 +-0.57230563 -0.47769305 -0.57230563 +-0.64449443 -0.46301051 -0.55439364 +-0.69512805 -0.44030814 -0.52730570 +-0.72916105 -0.42035442 -0.50353553 +-0.73242437 -0.41152775 -0.49300861 +0.00000000 -0.34141996 -0.34141996 +-0.05820001 -0.36268193 -0.36268193 +-0.13694322 -0.42131301 -0.42131301 +-0.24663636 -0.49786033 -0.49786033 +-0.37084894 -0.55640613 -0.55640613 +-0.47769305 -0.57230563 -0.57230563 +-0.55768140 -0.55768140 -0.55768140 +-0.61976735 -0.53296540 -0.53296540 +-0.66954661 -0.50732059 -0.50732059 +-0.70462293 -0.48539943 -0.48539943 +-0.71008085 -0.47522581 -0.47522581 +0.00000000 -0.61424514 -0.52709380 +-0.08889716 -0.62416662 -0.53560841 +-0.18497797 -0.64646135 -0.55494181 +-0.28580346 -0.66337884 -0.57000970 +-0.38077705 -0.66189001 -0.56923062 +-0.46301051 -0.64449443 -0.55439364 +-0.53296540 -0.61976735 -0.53296540 +-0.59277126 -0.59277126 -0.50950345 +-0.64259315 -0.56602006 -0.48628759 +-0.67877714 -0.54294234 -0.46633539 +-0.68646751 -0.53166473 -0.45663206 +0.00000000 -0.75390542 -0.57170560 +-0.09578104 -0.75275768 -0.57103333 +-0.19036049 -0.74766260 -0.56756738 +-0.28104073 -0.73608519 -0.55899630 +-0.36484376 -0.71793826 -0.54506411 +-0.44030814 -0.69512805 -0.52730570 +-0.50732059 -0.66954661 -0.50732059 +-0.56602006 -0.64259315 -0.48628759 +-0.61582664 -0.61582664 -0.46547077 +-0.65300609 -0.59237624 -0.44738236 +-0.66299628 -0.58018765 -0.43820883 +0.00000000 -0.79628425 -0.55261094 +-0.09250019 -0.79327778 -0.55046753 +-0.18267108 -0.78439198 -0.54407867 +-0.26834965 -0.77011959 -0.53369803 +-0.34788886 -0.75134898 -0.51991530 +-0.42035442 -0.72916105 -0.50353553 +-0.48539943 -0.70462293 -0.48539943 +-0.54294234 -0.67877714 -0.46633539 +-0.59237624 -0.65300609 -0.44738236 +-0.63023958 -0.63023958 -0.43071790 +-0.64316419 -0.61713656 -0.42204520 +0.00000000 -0.79352281 -0.54229063 +-0.09074155 -0.79073595 -0.54004136 +-0.17905905 -0.78257091 -0.53344201 +-0.26286063 -0.76957898 -0.52293056 +-0.34063560 -0.75256075 -0.50918384 +-0.41152775 -0.73242437 -0.49300861 +-0.47522581 -0.71008085 -0.47522581 +-0.53166473 -0.68646751 -0.45663206 +-0.58018765 -0.66299628 -0.43820883 +-0.61713656 -0.64316419 -0.42204520 +-0.63115275 -0.63115275 -0.41381762 +0.00000000 0.00000000 -0.06082157 +-0.00600328 0.00000000 -0.01400056 +-0.01486519 0.00000000 0.05557426 +-0.04068166 0.00000000 0.00990947 +-0.14103245 0.00000000 -0.20128544 +-0.32653470 0.00000000 -0.44895956 +-0.52709380 0.00000000 -0.61424514 +-0.66286491 0.00000000 -0.66286491 +-0.72717477 0.00000000 -0.64181314 +-0.76073727 0.00000000 -0.61239020 +-0.76125168 0.00000000 -0.59983936 +0.00000000 -0.00600328 -0.01400056 +-0.00600589 -0.00600589 0.01917166 +-0.01501634 -0.00619048 0.06178750 +-0.04794334 -0.01537844 -0.01613755 +-0.15572512 -0.03665819 -0.23454127 +-0.34155947 -0.06583743 -0.47200246 +-0.53560841 -0.08889716 -0.62416662 +-0.66282743 -0.09549542 -0.66282743 +-0.72454631 -0.09233753 -0.63948296 +-0.75801348 -0.08803594 -0.61010084 +-0.75877302 -0.08618160 -0.59756239 +0.00000000 -0.01486519 0.05557426 +-0.00619048 -0.01501634 0.06178750 +-0.02215810 -0.02215810 0.03144169 +-0.07548180 -0.04932061 -0.11101744 +-0.19900305 -0.09563564 -0.32825475 +-0.38233402 -0.14987638 -0.53305683 +-0.55494181 -0.18497797 -0.64646135 +-0.66040489 -0.19036012 -0.66040489 +-0.71662568 -0.18249705 -0.63243377 +-0.75002504 -0.17394592 -0.60337079 +-0.75150769 -0.17026666 -0.59088829 +0.00000000 -0.04068166 0.00990947 +-0.01537844 -0.04794334 -0.01613755 +-0.04932061 -0.07548180 -0.11101744 +-0.12624225 -0.12624225 -0.27090843 +-0.26498463 -0.19625206 -0.46026636 +-0.43375502 -0.25884369 -0.60694420 +-0.57000970 -0.28580346 -0.66337884 +-0.65177387 -0.28176879 -0.65177387 +-0.70361505 -0.26831506 -0.62080095 +-0.73729413 -0.25586526 -0.59260795 +-0.73992897 -0.25044073 -0.58026107 +0.00000000 -0.14103245 -0.20128544 +-0.03665819 -0.15572512 -0.23454127 +-0.09563564 -0.19900305 -0.32825475 +-0.19625206 -0.26498463 -0.46026636 +-0.33434658 -0.33434658 -0.58679971 +-0.47046771 -0.37641353 -0.65700597 +-0.56923062 -0.38077705 -0.66189001 +-0.63622857 -0.36626764 -0.63622857 +-0.68625062 -0.34808256 -0.60522656 +-0.72058916 -0.33239802 -0.57844219 +-0.72471469 -0.32536211 -0.56633672 +0.00000000 -0.32653470 -0.44895956 +-0.06583743 -0.34155947 -0.47200246 +-0.14987638 -0.38233402 -0.53305683 +-0.25884369 -0.43375502 -0.60694420 +-0.37641353 -0.47046771 -0.65700597 +-0.47704928 -0.47704928 -0.66451228 +-0.55439364 -0.46301051 -0.64449443 +-0.61585124 -0.44219004 -0.61585124 +-0.66556701 -0.42085685 -0.58665059 +-0.70078890 -0.40268668 -0.56162671 +-0.70663428 -0.39421472 -0.54987902 +0.00000000 -0.52709380 -0.61424514 +-0.08889716 -0.53560841 -0.62416662 +-0.18497797 -0.55494181 -0.64646135 +-0.28580346 -0.57000970 -0.66337884 +-0.38077705 -0.56923062 -0.66189001 +-0.46301051 -0.55439364 -0.64449443 +-0.53296540 -0.53296540 -0.61976735 +-0.59277126 -0.50950345 -0.59277126 +-0.64259315 -0.48628759 -0.56602006 +-0.67877714 -0.46633539 -0.54294234 +-0.68646751 -0.45663206 -0.53166473 +0.00000000 -0.66286491 -0.66286491 +-0.09549542 -0.66282743 -0.66282743 +-0.19036012 -0.66040489 -0.66040489 +-0.28176879 -0.65177387 -0.65177387 +-0.36626764 -0.63622857 -0.63622857 +-0.44219004 -0.61585124 -0.61585124 +-0.50950345 -0.59277126 -0.59277126 +-0.56841758 -0.56841758 -0.56841758 +-0.61832818 -0.54425197 -0.54425197 +-0.65544188 -0.52317582 -0.52317582 +-0.66502600 -0.51245790 -0.51245790 +0.00000000 -0.72717477 -0.64181314 +-0.09233753 -0.72454631 -0.63948296 +-0.18249705 -0.71662568 -0.63243377 +-0.26831506 -0.70361505 -0.62080095 +-0.34808256 -0.68625062 -0.60522656 +-0.42085685 -0.66556701 -0.58665059 +-0.48628759 -0.64259315 -0.56602006 +-0.54425197 -0.61832818 -0.54425197 +-0.59406396 -0.59406396 -0.52252053 +-0.63198461 -0.57254504 -0.50333929 +-0.64355572 -0.56097854 -0.49321946 +0.00000000 -0.76073727 -0.61239020 +-0.08803594 -0.75801348 -0.61010084 +-0.17394592 -0.75002504 -0.60337079 +-0.25586526 -0.73729413 -0.59260795 +-0.33239802 -0.72058916 -0.57844219 +-0.40268668 -0.70078890 -0.56162671 +-0.46633539 -0.67877714 -0.54294234 +-0.52317582 -0.65544188 -0.52317582 +-0.57254504 -0.63198461 -0.50333929 +-0.61100245 -0.61100245 -0.48562947 +-0.62525017 -0.59854565 -0.47605402 +0.00000000 -0.76125168 -0.59983936 +-0.08618160 -0.75877302 -0.59756239 +-0.17026666 -0.75150769 -0.59088829 +-0.25044073 -0.73992897 -0.58026107 +-0.32536211 -0.72471469 -0.56633672 +-0.39421472 -0.70663428 -0.54987902 +-0.45663206 -0.68646751 -0.53166473 +-0.51245790 -0.66502600 -0.51245790 +-0.56097854 -0.64355572 -0.49321946 +-0.59854565 -0.62525017 -0.47605402 +-0.61379836 -0.61379836 -0.46694972 +0.00000000 0.00000000 -0.17683359 +-0.03066312 0.00000000 -0.22372195 +-0.08704010 0.00000000 -0.34521204 +-0.18667468 0.00000000 -0.50353474 +-0.32385182 0.00000000 -0.65198962 +-0.46473367 0.00000000 -0.73981762 +-0.57170560 0.00000000 -0.75390542 +-0.64181314 0.00000000 -0.72717477 +-0.69184238 0.00000000 -0.69184238 +-0.72576389 0.00000000 -0.66113497 +-0.72980814 0.00000000 -0.64718057 +0.00000000 -0.03066312 -0.22372195 +-0.03473961 -0.03473961 -0.26730308 +-0.09480113 -0.04648892 -0.38143621 +-0.19627845 -0.06401814 -0.53080270 +-0.33221924 -0.08216745 -0.66819177 +-0.46853732 -0.09377176 -0.74500445 +-0.57103333 -0.09578104 -0.75275768 +-0.63948296 -0.09233753 -0.72454631 +-0.68933983 -0.08766136 -0.68933983 +-0.72336847 -0.08360147 -0.65887538 +-0.72762058 -0.08180280 -0.64496630 +0.00000000 -0.08704010 -0.34521204 +-0.04648892 -0.09480113 -0.38143621 +-0.11651843 -0.11651843 -0.47768370 +-0.22231006 -0.14705935 -0.60193794 +-0.35307454 -0.17583335 -0.70725446 +-0.47616824 -0.19088408 -0.75479693 +-0.56756738 -0.19036049 -0.74766260 +-0.63243377 -0.18249705 -0.71662568 +-0.68198358 -0.17325302 -0.68198358 +-0.71633890 -0.16535510 -0.65223978 +-0.72119706 -0.16180977 -0.63846965 +0.00000000 -0.18667468 -0.50353474 +-0.06401814 -0.19627845 -0.53080270 +-0.14705935 -0.22231006 -0.60193794 +-0.25551026 -0.25551026 -0.68649813 +-0.37476401 -0.28106234 -0.74503320 +-0.47929336 -0.28831272 -0.75712124 +-0.55899630 -0.28104073 -0.73608519 +-0.62080095 -0.26831506 -0.70361505 +-0.67022067 -0.25495048 -0.67022067 +-0.70511284 -0.24363838 -0.64163231 +-0.71092520 -0.23844692 -0.62810126 +0.00000000 -0.32385182 -0.65198962 +-0.08216745 -0.33221924 -0.66819177 +-0.17583335 -0.35307454 -0.70725446 +-0.28106234 -0.37476401 -0.74503320 +-0.38437484 -0.38437484 -0.75865393 +-0.47278057 -0.37888300 -0.74540141 +-0.54506411 -0.36484376 -0.71793826 +-0.60522656 -0.34808256 -0.68625062 +-0.65473449 -0.33138094 -0.65473449 +-0.69032513 -0.31721070 -0.62764921 +-0.69736623 -0.31051397 -0.61446271 +0.00000000 -0.46473367 -0.73981762 +-0.09377176 -0.46853732 -0.74500445 +-0.19088408 -0.47616824 -0.75479693 +-0.28831272 -0.47929336 -0.75712124 +-0.37888300 -0.47278057 -0.74540141 +-0.45845130 -0.45845130 -0.72284742 +-0.52730570 -0.44030814 -0.69512805 +-0.58665059 -0.42085685 -0.66556701 +-0.63632851 -0.40169149 -0.63632851 +-0.67270682 -0.38526561 -0.61098924 +-0.68116751 -0.37723468 -0.59825305 +0.00000000 -0.57170560 -0.75390542 +-0.09578104 -0.57103333 -0.75275768 +-0.19036049 -0.56756738 -0.74766260 +-0.28104073 -0.55899630 -0.73608519 +-0.36484376 -0.54506411 -0.71793826 +-0.44030814 -0.52730570 -0.69512805 +-0.50732059 -0.50732059 -0.66954661 +-0.56602006 -0.48628759 -0.64259315 +-0.61582664 -0.46547077 -0.61582664 +-0.65300609 -0.44738236 -0.59237624 +-0.66299628 -0.43820883 -0.58018765 +0.00000000 -0.64181314 -0.72717477 +-0.09233753 -0.63948296 -0.72454631 +-0.18249705 -0.63243377 -0.71662568 +-0.26831506 -0.62080095 -0.70361505 +-0.34808256 -0.60522656 -0.68625062 +-0.42085685 -0.58665059 -0.66556701 +-0.48628759 -0.56602006 -0.64259315 +-0.54425197 -0.54425197 -0.61832818 +-0.59406396 -0.52252053 -0.59406396 +-0.63198461 -0.50333929 -0.57254504 +-0.64355572 -0.49321946 -0.56097854 +0.00000000 -0.69184238 -0.69184238 +-0.08766136 -0.68933983 -0.68933983 +-0.17325302 -0.68198358 -0.68198358 +-0.25495048 -0.67022067 -0.67022067 +-0.33138094 -0.65473449 -0.65473449 +-0.40169149 -0.63632851 -0.63632851 +-0.46547077 -0.61582664 -0.61582664 +-0.52252053 -0.59406396 -0.59406396 +-0.57214078 -0.57214078 -0.57214078 +-0.61068108 -0.55245712 -0.55245712 +-0.62394011 -0.54153576 -0.54153576 +0.00000000 -0.72576389 -0.66113497 +-0.08360147 -0.72336847 -0.65887538 +-0.16535510 -0.71633890 -0.65223978 +-0.24363838 -0.70511284 -0.64163231 +-0.31721070 -0.69032513 -0.62764921 +-0.38526561 -0.67270682 -0.61098924 +-0.44738236 -0.65300609 -0.59237624 +-0.50333929 -0.63198461 -0.57254504 +-0.55245712 -0.61068108 -0.55245712 +-0.59139242 -0.59139242 -0.53424122 +-0.60706257 -0.57958865 -0.52389174 +0.00000000 -0.72980814 -0.64718057 +-0.08180280 -0.72762058 -0.64496630 +-0.16180977 -0.72119706 -0.63846965 +-0.23844692 -0.71092520 -0.62810126 +-0.31051397 -0.69736623 -0.61446271 +-0.37723468 -0.68116751 -0.59825305 +-0.43820883 -0.66299628 -0.58018765 +-0.49321946 -0.64355572 -0.56097854 +-0.54153576 -0.62394011 -0.54153576 +-0.57958865 -0.60706257 -0.52389174 +-0.59615655 -0.59615655 -0.51403691 +0.00000000 0.00000000 -0.76304465 +-0.08521294 0.00000000 -0.77777523 +-0.18095308 0.00000000 -0.81164775 +-0.28682303 0.00000000 -0.84080414 +-0.39066510 0.00000000 -0.84644775 +-0.47999560 0.00000000 -0.82791133 +-0.55261094 0.00000000 -0.79628425 +-0.61239020 0.00000000 -0.76073727 +-0.66113497 0.00000000 -0.72576389 +-0.69614090 0.00000000 -0.69614090 +-0.70399389 0.00000000 -0.68108181 +0.00000000 -0.08521294 -0.77777523 +-0.08715612 -0.08715612 -0.79077357 +-0.18371120 -0.09183810 -0.82008715 +-0.28877463 -0.09639378 -0.84380317 +-0.39076320 -0.09804449 -0.84538373 +-0.47851295 -0.09627030 -0.82509533 +-0.55046753 -0.09250019 -0.79327778 +-0.61010084 -0.08803594 -0.75801348 +-0.65887538 -0.08360147 -0.72336847 +-0.69399038 -0.07987246 -0.69399038 +-0.70202393 -0.07817257 -0.67898724 +0.00000000 -0.18095308 -0.81164775 +-0.09183810 -0.18371120 -0.82008715 +-0.19003333 -0.19003333 -0.83771085 +-0.29254151 -0.19524254 -0.84802854 +-0.38966560 -0.19540168 -0.84018801 +-0.47364345 -0.19043942 -0.81629552 +-0.54407867 -0.18267108 -0.78439198 +-0.60337079 -0.17394592 -0.75002504 +-0.65223978 -0.16535510 -0.71633890 +-0.68767092 -0.15811021 -0.68767092 +-0.69622915 -0.15476041 -0.67283536 +0.00000000 -0.28682303 -0.84080414 +-0.09639378 -0.28877463 -0.84380317 +-0.19524254 -0.29254151 -0.84802854 +-0.29360360 -0.29360360 -0.84411757 +-0.38484575 -0.28913343 -0.82755069 +-0.46484536 -0.27997643 -0.80125305 +-0.53369803 -0.26834965 -0.77011959 +-0.59260795 -0.25586526 -0.73729413 +-0.64163231 -0.24363838 -0.70511284 +-0.67755337 -0.23326397 -0.67755337 +-0.68693320 -0.22835942 -0.66299604 +0.00000000 -0.39066510 -0.84644775 +-0.09804449 -0.39076320 -0.84538373 +-0.19540168 -0.38966560 -0.84018801 +-0.28913343 -0.38484575 -0.82755069 +-0.37545423 -0.37545423 -0.80697821 +-0.45239341 -0.36262474 -0.78074330 +-0.51991530 -0.34788886 -0.75134898 +-0.57844219 -0.33239802 -0.72058916 +-0.62764921 -0.31721070 -0.69032513 +-0.66418152 -0.30420183 -0.66418152 +-0.67461241 -0.29787494 -0.65001050 +0.00000000 -0.47999560 -0.82791133 +-0.09627030 -0.47851295 -0.82509533 +-0.19043942 -0.47364345 -0.81629552 +-0.27997643 -0.46484536 -0.80125305 +-0.36262474 -0.45239341 -0.78074330 +-0.43722494 -0.43722494 -0.75623518 +-0.50353553 -0.42035442 -0.72916105 +-0.56162671 -0.40268668 -0.70078890 +-0.61098924 -0.38526561 -0.67270682 +-0.64819069 -0.37016115 -0.64819069 +-0.65982690 -0.36257220 -0.63450830 +0.00000000 -0.55261094 -0.79628425 +-0.09250019 -0.55046753 -0.79327778 +-0.18267108 -0.54407867 -0.78439198 +-0.26834965 -0.53369803 -0.77011959 +-0.34788886 -0.51991530 -0.75134898 +-0.42035442 -0.50353553 -0.72916105 +-0.48539943 -0.48539943 -0.70462293 +-0.54294234 -0.46633539 -0.67877714 +-0.59237624 -0.44738236 -0.65300609 +-0.63023958 -0.43071790 -0.63023958 +-0.64316419 -0.42204520 -0.61713656 +0.00000000 -0.61239020 -0.76073727 +-0.08803594 -0.61010084 -0.75801348 +-0.17394592 -0.60337079 -0.75002504 +-0.25586526 -0.59260795 -0.73729413 +-0.33239802 -0.57844219 -0.72058916 +-0.40268668 -0.56162671 -0.70078890 +-0.46633539 -0.54294234 -0.67877714 +-0.52317582 -0.52317582 -0.65544188 +-0.57254504 -0.50333929 -0.63198461 +-0.61100245 -0.48562947 -0.61100245 +-0.62525017 -0.47605402 -0.59854565 +0.00000000 -0.66113497 -0.72576389 +-0.08360147 -0.65887538 -0.72336847 +-0.16535510 -0.65223978 -0.71633890 +-0.24363838 -0.64163231 -0.70511284 +-0.31721070 -0.62764921 -0.69032513 +-0.38526561 -0.61098924 -0.67270682 +-0.44738236 -0.59237624 -0.65300609 +-0.50333929 -0.57254504 -0.63198461 +-0.55245712 -0.55245712 -0.61068108 +-0.59139242 -0.53424122 -0.59139242 +-0.60706257 -0.52389174 -0.57958865 +0.00000000 -0.69614090 -0.69614090 +-0.07987246 -0.69399038 -0.69399038 +-0.15811021 -0.68767092 -0.68767092 +-0.23326397 -0.67755337 -0.67755337 +-0.30420183 -0.66418152 -0.66418152 +-0.37016115 -0.64819069 -0.64819069 +-0.43071790 -0.63023958 -0.63023958 +-0.48562947 -0.61100245 -0.61100245 +-0.53424122 -0.59139242 -0.59139242 +-0.57346346 -0.57346346 -0.57346346 +-0.59126378 -0.56219263 -0.56219263 +0.00000000 -0.70399389 -0.68108181 +-0.07817257 -0.70202393 -0.67898724 +-0.15476041 -0.69622915 -0.67283536 +-0.22835942 -0.68693320 -0.66299604 +-0.29787494 -0.67461241 -0.65001050 +-0.36257220 -0.65982690 -0.63450830 +-0.42204520 -0.64316419 -0.61713656 +-0.47605402 -0.62525017 -0.59854565 +-0.52389174 -0.60706257 -0.57958865 +-0.56219263 -0.59126378 -0.56219263 +-0.58070957 -0.58070957 -0.55154419 +0.00000000 0.00000000 -0.89221786 +-0.10375184 0.00000000 -0.89069110 +-0.20667050 0.00000000 -0.88476837 +-0.30540552 0.00000000 -0.87190638 +-0.39548766 0.00000000 -0.85119212 +-0.47445669 0.00000000 -0.82423162 +-0.54229063 0.00000000 -0.79352281 +-0.59983936 0.00000000 -0.76125168 +-0.64718057 0.00000000 -0.72980814 +-0.68108181 0.00000000 -0.70399389 +-0.69023094 0.00000000 -0.69023094 +0.00000000 -0.10375184 -0.89069110 +-0.10371347 -0.10371347 -0.88892194 +-0.20627793 -0.10325282 -0.88242428 +-0.30436733 -0.10174492 -0.86901148 +-0.39381874 -0.09894888 -0.84806123 +-0.47238472 -0.09512980 -0.82118682 +-0.54004136 -0.09074155 -0.79073595 +-0.59756239 -0.08618160 -0.75877302 +-0.64496630 -0.08180280 -0.72762058 +-0.67898724 -0.07817257 -0.70202393 +-0.68831308 -0.07658202 -0.68831308 +0.00000000 -0.20667050 -0.88476837 +-0.10325282 -0.20627793 -0.88242428 +-0.20464185 -0.20464185 -0.87460349 +-0.30095031 -0.20098431 -0.86002000 +-0.38874589 -0.19511957 -0.83870505 +-0.46625748 -0.18756221 -0.81222959 +-0.53344201 -0.17905905 -0.78257091 +-0.59088829 -0.17026666 -0.75150769 +-0.63846965 -0.16180977 -0.72119706 +-0.67283536 -0.15476041 -0.69622915 +-0.68267326 -0.15162007 -0.68267326 +0.00000000 -0.30540552 -0.87190638 +-0.10174492 -0.30436733 -0.86901148 +-0.20098431 -0.30095031 -0.86002000 +-0.29472310 -0.29472310 -0.84463243 +-0.38030448 -0.28581088 -0.82345658 +-0.45640487 -0.27490623 -0.79791490 +-0.52293056 -0.26286063 -0.76957898 +-0.58026107 -0.25044073 -0.73992897 +-0.62810126 -0.23844692 -0.71092520 +-0.66299604 -0.22835942 -0.68693320 +-0.67363148 -0.22374881 -0.67363148 +0.00000000 -0.39548766 -0.85119212 +-0.09894888 -0.39381874 -0.84806123 +-0.19511957 -0.38874589 -0.83870505 +-0.28581088 -0.38030448 -0.82345658 +-0.36893723 -0.36893723 -0.80318977 +-0.44345012 -0.35542756 -0.77913577 +-0.50918384 -0.34063560 -0.75256075 +-0.56633672 -0.32536211 -0.72471469 +-0.61446271 -0.31051397 -0.69736623 +-0.65001050 -0.29787494 -0.67461241 +-0.66165857 -0.29190835 -0.66165857 +0.00000000 -0.47445669 -0.82423162 +-0.09512980 -0.47238472 -0.82118682 +-0.18756221 -0.46625748 -0.81222959 +-0.27490623 -0.45640487 -0.79791490 +-0.35542756 -0.44345012 -0.77913577 +-0.42820893 -0.42820893 -0.75695117 +-0.49300861 -0.41152775 -0.73242437 +-0.54987902 -0.39421472 -0.70663428 +-0.59825305 -0.37723468 -0.68116751 +-0.63450830 -0.36257220 -0.65982690 +-0.64730703 -0.35539297 -0.64730703 +0.00000000 -0.54229063 -0.79352281 +-0.09074155 -0.54004136 -0.79073595 +-0.17905905 -0.53344201 -0.78257091 +-0.26286063 -0.52293056 -0.76957898 +-0.34063560 -0.50918384 -0.75256075 +-0.41152775 -0.49300861 -0.73242437 +-0.47522581 -0.47522581 -0.71008085 +-0.53166473 -0.45663206 -0.68646751 +-0.58018765 -0.43820883 -0.66299628 +-0.61713656 -0.42204520 -0.64316419 +-0.63115275 -0.41381762 -0.63115275 +0.00000000 -0.59983936 -0.76125168 +-0.08618160 -0.59756239 -0.75877302 +-0.17026666 -0.59088829 -0.75150769 +-0.25044073 -0.58026107 -0.73992897 +-0.32536211 -0.56633672 -0.72471469 +-0.39421472 -0.54987902 -0.70663428 +-0.45663206 -0.53166473 -0.68646751 +-0.51245790 -0.51245790 -0.66502600 +-0.56097854 -0.49321946 -0.64355572 +-0.59854565 -0.47605402 -0.62525017 +-0.61379836 -0.46694972 -0.61379836 +0.00000000 -0.64718057 -0.72980814 +-0.08180280 -0.64496630 -0.72762058 +-0.16180977 -0.63846965 -0.72119706 +-0.23844692 -0.62810126 -0.71092520 +-0.31051397 -0.61446271 -0.69736623 +-0.37723468 -0.59825305 -0.68116751 +-0.43820883 -0.58018765 -0.66299628 +-0.49321946 -0.56097854 -0.64355572 +-0.54153576 -0.54153576 -0.62394011 +-0.57958865 -0.52389174 -0.60706257 +-0.59615655 -0.51403691 -0.59615655 +0.00000000 -0.68108181 -0.70399389 +-0.07817257 -0.67898724 -0.70202393 +-0.15476041 -0.67283536 -0.69622915 +-0.22835942 -0.66299604 -0.68693320 +-0.29787494 -0.65001050 -0.67461241 +-0.36257220 -0.63450830 -0.65982690 +-0.42204520 -0.61713656 -0.64316419 +-0.47605402 -0.59854565 -0.62525017 +-0.52389174 -0.57958865 -0.60706257 +-0.56219263 -0.56219263 -0.59126378 +-0.58070957 -0.55154419 -0.58070957 +0.00000000 -0.69023094 -0.69023094 +-0.07658202 -0.68831308 -0.68831308 +-0.15162007 -0.68267326 -0.68267326 +-0.22374881 -0.67363148 -0.67363148 +-0.29190835 -0.66165857 -0.66165857 +-0.35539297 -0.64730703 -0.64730703 +-0.41381762 -0.63115275 -0.63115275 +-0.46694972 -0.61379836 -0.61379836 +-0.51403691 -0.59615655 -0.59615655 +-0.55154419 -0.58070957 -0.58070957 +-0.57057459 -0.57057459 -0.57057459 + +CELL_DATA 1000 +SCALARS SIE float 1 +LOOKUP_TABLE default +1.91339396 +1.27306831 +0.98683557 +0.88455638 +0.68312175 +0.75008819 +0.55790672 +0.63041594 +0.14585148 +0.03606565 +1.27306831 +1.10761208 +0.91476972 +0.75779152 +0.70470543 +0.65299816 +0.57786082 +0.55235115 +0.11689185 +0.03324034 +0.98683557 +0.91476972 +0.79338512 +0.71038981 +0.69341457 +0.57099416 +0.62361115 +0.39943168 +0.07852243 +0.02857520 +0.88455638 +0.75779152 +0.71038981 +0.68760297 +0.60535667 +0.58061444 +0.58749868 +0.22856480 +0.04956275 +0.02337345 +0.68312175 +0.70470543 +0.69341457 +0.60535667 +0.57991426 +0.61696250 +0.37764957 +0.10339065 +0.03377135 +0.01858336 +0.75008819 +0.65299816 +0.57099416 +0.58061444 +0.61696250 +0.43017557 +0.15982690 +0.04784452 +0.02518039 +0.01460893 +0.55790672 +0.57786082 +0.62361115 +0.58749868 +0.37764957 +0.15982690 +0.05596153 +0.02943724 +0.01937662 +0.01146346 +0.63041594 +0.55235115 +0.39943168 +0.22856480 +0.10339065 +0.04784452 +0.02943724 +0.02102063 +0.01492892 +0.00897654 +0.14585148 +0.11689185 +0.07852243 +0.04956275 +0.03377135 +0.02518039 +0.01937662 +0.01492892 +0.01110260 +0.00682609 +0.03606565 +0.03324034 +0.02857520 +0.02337345 +0.01858336 +0.01460893 +0.01146346 +0.00897654 +0.00682609 +0.00440305 +1.27306831 +1.10761208 +0.91476972 +0.75779152 +0.70470543 +0.65299816 +0.57786082 +0.55235115 +0.11689185 +0.03324034 +1.10761208 +1.00486304 +0.83948024 +0.71697214 +0.70645293 +0.59598056 +0.60490814 +0.47180332 +0.09498356 +0.03076056 +0.91476972 +0.83948024 +0.75279615 +0.71138366 +0.65631435 +0.56313547 +0.63205381 +0.33212296 +0.06603671 +0.02663873 +0.75779152 +0.71697214 +0.71138366 +0.67005318 +0.58351857 +0.59877464 +0.54094504 +0.18571359 +0.04410057 +0.02199370 +0.70470543 +0.70645293 +0.65631435 +0.58351857 +0.59129723 +0.60141547 +0.32178040 +0.08563526 +0.03154254 +0.01765986 +0.65299816 +0.59598056 +0.56313547 +0.59877464 +0.60141547 +0.37563454 +0.13067803 +0.04291563 +0.02405041 +0.01400944 +0.57786082 +0.60490814 +0.63205381 +0.54094504 +0.32178040 +0.13067803 +0.04918619 +0.02793411 +0.01869152 +0.01107737 +0.55235115 +0.47180332 +0.33212296 +0.18571359 +0.08563526 +0.04291563 +0.02793411 +0.02027565 +0.01449087 +0.00872741 +0.11689185 +0.09498356 +0.06603671 +0.04410057 +0.03154254 +0.02405041 +0.01869152 +0.01449087 +0.01082533 +0.00666436 +0.03324034 +0.03076056 +0.02663873 +0.02199370 +0.01765986 +0.01400944 +0.01107737 +0.00872741 +0.00666436 +0.00429439 +0.98683557 +0.91476972 +0.79338512 +0.71038981 +0.69341457 +0.57099416 +0.62361115 +0.39943168 +0.07852243 +0.02857520 +0.91476972 +0.83948024 +0.75279615 +0.71138366 +0.65631435 +0.56313547 +0.63205381 +0.33212296 +0.06603671 +0.02663873 +0.79338512 +0.75279615 +0.73512453 +0.69612802 +0.59511125 +0.58292887 +0.58706481 +0.22502611 +0.04953379 +0.02337828 +0.71038981 +0.71138366 +0.69612802 +0.61460107 +0.57270678 +0.62598275 +0.42654209 +0.12342821 +0.03653030 +0.01963008 +0.69341457 +0.65631435 +0.59511125 +0.57270678 +0.62078799 +0.52521617 +0.22524553 +0.06161255 +0.02803517 +0.01604331 +0.57099416 +0.56313547 +0.58292887 +0.62598275 +0.52521617 +0.27147627 +0.08917821 +0.03601794 +0.02207562 +0.01293573 +0.62361115 +0.63205381 +0.58706481 +0.42654209 +0.22524553 +0.08917821 +0.03989788 +0.02543850 +0.01743862 +0.01037132 +0.39943168 +0.33212296 +0.22502611 +0.12342821 +0.06161255 +0.03601794 +0.02543850 +0.01891394 +0.01367414 +0.00826402 +0.07852243 +0.06603671 +0.04953379 +0.03653030 +0.02803517 +0.02207562 +0.01743862 +0.01367414 +0.01030325 +0.00636081 +0.02857520 +0.02663873 +0.02337828 +0.01963008 +0.01604331 +0.01293573 +0.01037132 +0.00826402 +0.00636081 +0.00409283 +0.88455638 +0.75779152 +0.71038981 +0.68760297 +0.60535667 +0.58061444 +0.58749868 +0.22856480 +0.04956275 +0.02337345 +0.75779152 +0.71697214 +0.71138366 +0.67005318 +0.58351857 +0.59877464 +0.54094504 +0.18571359 +0.04410057 +0.02199370 +0.71038981 +0.71138366 +0.69612802 +0.61460107 +0.57270678 +0.62598275 +0.42654209 +0.12342821 +0.03653030 +0.01963008 +0.68760297 +0.67005318 +0.61460107 +0.57044433 +0.61138427 +0.56869240 +0.26585365 +0.07151178 +0.02970265 +0.01683651 +0.60535667 +0.58351857 +0.57270678 +0.61138427 +0.60162184 +0.36270967 +0.12689326 +0.04276190 +0.02418997 +0.01406809 +0.58061444 +0.59877464 +0.62598275 +0.56869240 +0.36270967 +0.15511148 +0.05548097 +0.02976199 +0.01962579 +0.01157842 +0.58749868 +0.54094504 +0.42654209 +0.26585365 +0.12689326 +0.05548097 +0.03200732 +0.02247994 +0.01580279 +0.00945081 +0.22856480 +0.18571359 +0.12342821 +0.07151178 +0.04276190 +0.02976199 +0.02247994 +0.01713321 +0.01257917 +0.00764432 +0.04956275 +0.04410057 +0.03653030 +0.02970265 +0.02418997 +0.01962579 +0.01580279 +0.01257917 +0.00959254 +0.00594926 +0.02337345 +0.02199370 +0.01963008 +0.01683651 +0.01406809 +0.01157842 +0.00945081 +0.00764432 +0.00594926 +0.00382421 +0.68312175 +0.70470543 +0.69341457 +0.60535667 +0.57991426 +0.61696250 +0.37764957 +0.10339065 +0.03377135 +0.01858336 +0.70470543 +0.70645293 +0.65631435 +0.58351857 +0.59129723 +0.60141547 +0.32178040 +0.08563526 +0.03154254 +0.01765986 +0.69341457 +0.65631435 +0.59511125 +0.57270678 +0.62078799 +0.52521617 +0.22524553 +0.06161255 +0.02803517 +0.01604331 +0.60535667 +0.58351857 +0.57270678 +0.61138427 +0.60162184 +0.36270967 +0.12689326 +0.04276190 +0.02418997 +0.01406809 +0.57991426 +0.59129723 +0.62078799 +0.60162184 +0.41194977 +0.18463884 +0.06351941 +0.03156549 +0.02045234 +0.01203062 +0.61696250 +0.60141547 +0.52521617 +0.36270967 +0.18463884 +0.07481555 +0.03684485 +0.02461794 +0.01701505 +0.01012140 +0.37764957 +0.32178040 +0.22524553 +0.12689326 +0.06351941 +0.03684485 +0.02609172 +0.01939144 +0.01397854 +0.00842619 +0.10339065 +0.08563526 +0.06161255 +0.04276190 +0.03156549 +0.02461794 +0.01939144 +0.01514094 +0.01132084 +0.00693322 +0.03377135 +0.03154254 +0.02803517 +0.02418997 +0.02045234 +0.01701505 +0.01397854 +0.01132084 +0.00875976 +0.00546869 +0.01858336 +0.01765986 +0.01604331 +0.01406809 +0.01203062 +0.01012140 +0.00842619 +0.00693322 +0.00546869 +0.00351672 +0.75008819 +0.65299816 +0.57099416 +0.58061444 +0.61696250 +0.43017557 +0.15982690 +0.04784452 +0.02518039 +0.01460893 +0.65299816 +0.59598056 +0.56313547 +0.59877464 +0.60141547 +0.37563454 +0.13067803 +0.04291563 +0.02405041 +0.01400944 +0.57099416 +0.56313547 +0.58292887 +0.62598275 +0.52521617 +0.27147627 +0.08917821 +0.03601794 +0.02207562 +0.01293573 +0.58061444 +0.59877464 +0.62598275 +0.56869240 +0.36270967 +0.15511148 +0.05548097 +0.02976199 +0.01962579 +0.01157842 +0.61696250 +0.60141547 +0.52521617 +0.36270967 +0.18463884 +0.07481555 +0.03684485 +0.02461794 +0.01701505 +0.01012140 +0.43017557 +0.37563454 +0.27147627 +0.15511148 +0.07481555 +0.04014093 +0.02747378 +0.02020817 +0.01447156 +0.00869969 +0.15982690 +0.13067803 +0.08917821 +0.05548097 +0.03684485 +0.02747378 +0.02126778 +0.01640126 +0.01212935 +0.00738810 +0.04784452 +0.04291563 +0.03601794 +0.02976199 +0.02461794 +0.02020817 +0.01640126 +0.01311465 +0.01000463 +0.00618892 +0.02518039 +0.02405041 +0.02207562 +0.01962579 +0.01701505 +0.01447156 +0.01212935 +0.01000463 +0.00786892 +0.00495540 +0.01460893 +0.01400944 +0.01293573 +0.01157842 +0.01012140 +0.00869969 +0.00738810 +0.00618892 +0.00495540 +0.00319470 +0.55790672 +0.57786082 +0.62361115 +0.58749868 +0.37764957 +0.15982690 +0.05596153 +0.02943724 +0.01937662 +0.01146346 +0.57786082 +0.60490814 +0.63205381 +0.54094504 +0.32178040 +0.13067803 +0.04918619 +0.02793411 +0.01869152 +0.01107737 +0.62361115 +0.63205381 +0.58706481 +0.42654209 +0.22524553 +0.08917821 +0.03989788 +0.02543850 +0.01743862 +0.01037132 +0.58749868 +0.54094504 +0.42654209 +0.26585365 +0.12689326 +0.05548097 +0.03200732 +0.02247994 +0.01580279 +0.00945081 +0.37764957 +0.32178040 +0.22524553 +0.12689326 +0.06351941 +0.03684485 +0.02609172 +0.01939144 +0.01397854 +0.00842619 +0.15982690 +0.13067803 +0.08917821 +0.05548097 +0.03684485 +0.02747378 +0.02126778 +0.01640126 +0.01212935 +0.00738810 +0.05596153 +0.04918619 +0.03989788 +0.03200732 +0.02609172 +0.02126778 +0.01716827 +0.01365650 +0.01036472 +0.00639487 +0.02943724 +0.02793411 +0.02543850 +0.02247994 +0.01939144 +0.01640126 +0.01365650 +0.01117697 +0.00870834 +0.00545300 +0.01937662 +0.01869152 +0.01743862 +0.01580279 +0.01397854 +0.01212935 +0.01036472 +0.00870834 +0.00696971 +0.00443637 +0.01146346 +0.01107737 +0.01037132 +0.00945081 +0.00842619 +0.00738810 +0.00639487 +0.00545300 +0.00443637 +0.00287445 +0.63041594 +0.55235115 +0.39943168 +0.22856480 +0.10339065 +0.04784452 +0.02943724 +0.02102063 +0.01492892 +0.00897654 +0.55235115 +0.47180332 +0.33212296 +0.18571359 +0.08563526 +0.04291563 +0.02793411 +0.02027565 +0.01449087 +0.00872741 +0.39943168 +0.33212296 +0.22502611 +0.12342821 +0.06161255 +0.03601794 +0.02543850 +0.01891394 +0.01367414 +0.00826402 +0.22856480 +0.18571359 +0.12342821 +0.07151178 +0.04276190 +0.02976199 +0.02247994 +0.01713321 +0.01257917 +0.00764432 +0.10339065 +0.08563526 +0.06161255 +0.04276190 +0.03156549 +0.02461794 +0.01939144 +0.01514094 +0.01132084 +0.00693322 +0.04784452 +0.04291563 +0.03601794 +0.02976199 +0.02461794 +0.02020817 +0.01640126 +0.01311465 +0.01000463 +0.00618892 +0.02943724 +0.02793411 +0.02543850 +0.02247994 +0.01939144 +0.01640126 +0.01365650 +0.01117697 +0.00870834 +0.00545300 +0.02102063 +0.02027565 +0.01891394 +0.01713321 +0.01514094 +0.01311465 +0.01117697 +0.00935748 +0.00745166 +0.00473089 +0.01492892 +0.01449087 +0.01367414 +0.01257917 +0.01132084 +0.01000463 +0.00870834 +0.00745166 +0.00607330 +0.00391350 +0.00897654 +0.00872741 +0.00826402 +0.00764432 +0.00693322 +0.00618892 +0.00545300 +0.00473089 +0.00391350 +0.00255469 +0.14585148 +0.11689185 +0.07852243 +0.04956275 +0.03377135 +0.02518039 +0.01937662 +0.01492892 +0.01110260 +0.00682609 +0.11689185 +0.09498356 +0.06603671 +0.04410057 +0.03154254 +0.02405041 +0.01869152 +0.01449087 +0.01082533 +0.00666436 +0.07852243 +0.06603671 +0.04953379 +0.03653030 +0.02803517 +0.02207562 +0.01743862 +0.01367414 +0.01030325 +0.00636081 +0.04956275 +0.04410057 +0.03653030 +0.02970265 +0.02418997 +0.01962579 +0.01580279 +0.01257917 +0.00959254 +0.00594926 +0.03377135 +0.03154254 +0.02803517 +0.02418997 +0.02045234 +0.01701505 +0.01397854 +0.01132084 +0.00875976 +0.00546869 +0.02518039 +0.02405041 +0.02207562 +0.01962579 +0.01701505 +0.01447156 +0.01212935 +0.01000463 +0.00786892 +0.00495540 +0.01937662 +0.01869152 +0.01743862 +0.01580279 +0.01397854 +0.01212935 +0.01036472 +0.00870834 +0.00696971 +0.00443637 +0.01492892 +0.01449087 +0.01367414 +0.01257917 +0.01132084 +0.01000463 +0.00870834 +0.00745166 +0.00607330 +0.00391350 +0.01110260 +0.01082533 +0.01030325 +0.00959254 +0.00875976 +0.00786892 +0.00696971 +0.00607330 +0.00504806 +0.00329551 +0.00682609 +0.00666436 +0.00636081 +0.00594926 +0.00546869 +0.00495540 +0.00443637 +0.00391350 +0.00329551 +0.00217276 +0.03606565 +0.03324034 +0.02857520 +0.02337345 +0.01858336 +0.01460893 +0.01146346 +0.00897654 +0.00682609 +0.00440305 +0.03324034 +0.03076056 +0.02663873 +0.02199370 +0.01765986 +0.01400944 +0.01107737 +0.00872741 +0.00666436 +0.00429439 +0.02857520 +0.02663873 +0.02337828 +0.01963008 +0.01604331 +0.01293573 +0.01037132 +0.00826402 +0.00636081 +0.00409283 +0.02337345 +0.02199370 +0.01963008 +0.01683651 +0.01406809 +0.01157842 +0.00945081 +0.00764432 +0.00594926 +0.00382421 +0.01858336 +0.01765986 +0.01604331 +0.01406809 +0.01203062 +0.01012140 +0.00842619 +0.00693322 +0.00546869 +0.00351672 +0.01460893 +0.01400944 +0.01293573 +0.01157842 +0.01012140 +0.00869969 +0.00738810 +0.00618892 +0.00495540 +0.00319470 +0.01146346 +0.01107737 +0.01037132 +0.00945081 +0.00842619 +0.00738810 +0.00639487 +0.00545300 +0.00443637 +0.00287445 +0.00897654 +0.00872741 +0.00826402 +0.00764432 +0.00693322 +0.00618892 +0.00545300 +0.00473089 +0.00391350 +0.00255469 +0.00682609 +0.00666436 +0.00636081 +0.00594926 +0.00546869 +0.00495540 +0.00443637 +0.00391350 +0.00329551 +0.00217276 +0.00440305 +0.00429439 +0.00409283 +0.00382421 +0.00351672 +0.00319470 +0.00287445 +0.00255469 +0.00217276 +0.00144967 + +SCALARS element_density float 1 +LOOKUP_TABLE default +11.37187318 +15.01297188 +20.33348499 +26.11068669 +23.93155631 +34.77202595 +27.21281606 +38.82298982 +14.07577128 +7.63917636 +15.01297188 +18.78228268 +22.79406363 +24.80600823 +28.94248009 +31.42243051 +30.93019723 +34.13987445 +12.79375914 +7.19493636 +20.33348499 +22.79406363 +24.66773585 +27.25712093 +32.80774698 +29.14433610 +38.24848265 +25.52819812 +10.81428298 +6.45954861 +26.11068669 +24.80600823 +27.25712093 +30.64848336 +29.88816171 +32.62565261 +38.03471748 +17.39150013 +8.78908447 +5.64164861 +23.93155631 +28.94248009 +32.80774698 +29.88816171 +31.26992984 +38.39263359 +24.47135278 +11.78553303 +7.11248089 +4.89499574 +34.77202595 +31.42243051 +29.14433610 +32.62565261 +38.39263359 +26.80500827 +14.18066558 +8.37976063 +5.87745147 +4.27793575 +27.21281606 +30.93019723 +38.24848265 +38.03471748 +24.47135278 +14.18066558 +8.93889632 +6.39397914 +4.99467087 +3.78453846 +38.82298982 +34.13987445 +25.52819812 +17.39150013 +11.78553303 +8.37976063 +6.39397914 +5.20062803 +4.32844932 +3.38475341 +14.07577128 +12.79375914 +10.81428298 +8.78908447 +7.11248089 +5.87745147 +4.99467087 +4.32844932 +3.74928641 +3.02510299 +7.63917636 +7.19493636 +6.45954861 +5.64164861 +4.89499574 +4.27793575 +3.78453846 +3.38475341 +3.02510299 +2.55355247 +15.01297188 +18.78228268 +22.79406363 +24.80600823 +28.94248009 +31.42243051 +30.93019723 +34.13987445 +12.79375914 +7.19493636 +18.78228268 +21.40451372 +23.53509581 +25.56000569 +31.83971910 +29.61360503 +35.10791273 +29.49972915 +11.72122592 +6.80191598 +22.79406363 +23.53509581 +25.31842698 +29.30266855 +31.84035113 +29.75147932 +40.49326726 +22.22389751 +10.03203401 +6.14899107 +24.80600823 +25.56000569 +29.30266855 +31.02706568 +29.32466778 +35.18247959 +35.05633512 +15.56648805 +8.27423701 +5.41762291 +28.94248009 +31.83971910 +31.84035113 +29.32466778 +33.11049298 +37.92055089 +21.71583324 +10.86435293 +6.79658707 +4.74200020 +31.42243051 +29.61360503 +29.75147932 +35.18247959 +37.92055089 +24.04266690 +12.90755247 +7.91895837 +5.68841643 +4.17487736 +30.93019723 +35.10791273 +40.49326726 +35.05633512 +21.71583324 +12.90755247 +8.40202688 +6.16407388 +4.87738413 +3.71442410 +34.13987445 +29.49972915 +22.22389751 +15.56648805 +10.86435293 +7.91895837 +6.16407388 +5.07555310 +4.25131629 +3.33630039 +12.79375914 +11.72122592 +10.03203401 +8.27423701 +6.79658707 +5.68841643 +4.87738413 +4.25131629 +3.69732526 +2.99095449 +7.19493636 +6.80191598 +6.14899107 +5.41762291 +4.74200020 +4.17487736 +3.71442410 +3.33630039 +2.99095449 +2.52864207 +20.33348499 +22.79406363 +24.66773585 +27.25712093 +32.80774698 +29.14433610 +38.24848265 +25.52819812 +10.81428298 +6.45954861 +22.79406363 +23.53509581 +25.31842698 +29.30266855 +31.84035113 +29.75147932 +40.49326726 +22.22389751 +10.03203401 +6.14899107 +24.66773585 +25.31842698 +28.65679490 +31.59434169 +29.68435215 +33.35440384 +38.48745478 +17.34954702 +8.77250349 +5.62886359 +27.25712093 +29.30266855 +31.59434169 +29.65605931 +30.33981158 +39.43387906 +27.89660053 +12.79294647 +7.42875874 +5.03694085 +32.80774698 +31.84035113 +29.68435215 +30.33981158 +37.55270248 +33.42120549 +17.27079930 +9.39466885 +6.26643818 +4.47645117 +29.14433610 +29.75147932 +33.35440384 +39.43387906 +33.42120549 +19.20793867 +10.89058004 +7.16327224 +5.36124145 +3.99182719 +38.24848265 +40.49326726 +38.48745478 +27.89660053 +17.27079930 +10.89058004 +7.52932583 +5.77324703 +4.66721419 +3.58736778 +25.52819812 +22.22389751 +17.34954702 +12.79294647 +9.39466885 +7.16327224 +5.77324703 +4.85190145 +4.10986092 +3.24707707 +10.81428298 +10.03203401 +8.77250349 +7.42875874 +6.26643818 +5.36124145 +4.66721419 +4.10986092 +3.60080830 +2.92744223 +6.45954861 +6.14899107 +5.62886359 +5.03694085 +4.47645117 +3.99182719 +3.58736778 +3.24707707 +2.92744223 +2.48244864 +26.11068669 +24.80600823 +27.25712093 +30.64848336 +29.88816171 +32.62565261 +38.03471748 +17.39150013 +8.78908447 +5.64164861 +24.80600823 +25.56000569 +29.30266855 +31.02706568 +29.32466778 +35.18247959 +35.05633512 +15.56648805 +8.27423701 +5.41762291 +27.25712093 +29.30266855 +31.59434169 +29.65605931 +30.33981158 +39.43387906 +27.89660053 +12.79294647 +7.42875874 +5.03694085 +30.64848336 +31.02706568 +29.65605931 +29.32995253 +36.11102672 +36.81557482 +19.33452174 +10.03823853 +6.49923979 +4.59215739 +29.88816171 +29.32466778 +30.33981158 +36.11102672 +38.54251251 +24.16313934 +12.80178267 +7.85225168 +5.65982806 +4.15541897 +32.62565261 +35.18247959 +39.43387906 +36.81557482 +24.16313934 +14.04438604 +8.80965627 +6.33370141 +4.96616262 +3.76283542 +38.03471748 +35.05633512 +27.89660053 +19.33452174 +12.80178267 +8.80965627 +6.58682860 +5.31254865 +4.40030906 +3.42369317 +17.39150013 +15.56648805 +12.79294647 +10.03823853 +7.85225168 +6.33370141 +5.31254865 +4.56825187 +3.92434952 +3.12938913 +8.78908447 +8.27423701 +7.42875874 +6.49923979 +5.65982806 +4.96616262 +4.40030906 +3.92434952 +3.47183433 +2.84242280 +5.64164861 +5.41762291 +5.03694085 +4.59215739 +4.15541897 +3.76283542 +3.42369317 +3.12938913 +2.84242280 +2.42090865 +23.93155631 +28.94248009 +32.80774698 +29.88816171 +31.26992984 +38.39263359 +24.47135278 +11.78553303 +7.11248089 +4.89499574 +28.94248009 +31.83971910 +31.84035113 +29.32466778 +33.11049298 +37.92055089 +21.71583324 +10.86435293 +6.79658707 +4.74200020 +32.80774698 +31.84035113 +29.68435215 +30.33981158 +37.55270248 +33.42120549 +17.27079930 +9.39466885 +6.26643818 +4.47645117 +29.88816171 +29.32466778 +30.33981158 +36.11102672 +38.54251251 +24.16313934 +12.80178267 +7.85225168 +5.65982806 +4.15541897 +31.26992984 +33.11049298 +37.55270248 +38.54251251 +27.05202289 +15.46641071 +9.35753738 +6.55611830 +5.07734920 +3.82683188 +38.39263359 +37.92055089 +33.42120549 +24.16313934 +15.46641071 +10.01621647 +7.11788539 +5.58093193 +4.55962409 +3.51911109 +24.47135278 +21.71583324 +17.27079930 +12.80178267 +9.35753738 +7.11788539 +5.75247598 +4.84858754 +4.11047446 +3.24351181 +11.78553303 +10.86435293 +9.39466885 +7.85225168 +6.55611830 +5.58093193 +4.84858754 +4.26005592 +3.71570313 +2.99618395 +7.11248089 +6.79658707 +6.26643818 +5.65982806 +5.07734920 +4.55962409 +4.11047446 +3.71570313 +3.32354485 +2.74448209 +4.89499574 +4.74200020 +4.47645117 +4.15541897 +3.82683188 +3.51911109 +3.24351181 +2.99618395 +2.74448209 +2.35045973 +34.77202595 +31.42243051 +29.14433610 +32.62565261 +38.39263359 +26.80500827 +14.18066558 +8.37976063 +5.87745147 +4.27793575 +31.42243051 +29.61360503 +29.75147932 +35.18247959 +37.92055089 +24.04266690 +12.90755247 +7.91895837 +5.68841643 +4.17487736 +29.14433610 +29.75147932 +33.35440384 +39.43387906 +33.42120549 +19.20793867 +10.89058004 +7.16327224 +5.36124145 +3.99182719 +32.62565261 +35.18247959 +39.43387906 +36.81557482 +24.16313934 +14.04438604 +8.80965627 +6.33370141 +4.96616262 +3.76283542 +38.39263359 +37.92055089 +33.42120549 +24.16313934 +15.46641071 +10.01621647 +7.11788539 +5.58093193 +4.55962409 +3.51911109 +26.80500827 +24.04266690 +19.20793867 +14.04438604 +10.01621647 +7.43591735 +5.91562850 +4.94654060 +4.17380528 +3.28199890 +14.18066558 +12.90755247 +10.89058004 +8.80965627 +7.11788539 +5.91562850 +5.06792460 +4.41376394 +3.82212240 +3.06216435 +8.37976063 +7.91895837 +7.16327224 +6.33370141 +5.58093193 +4.94654060 +4.41376394 +3.95279670 +3.50092352 +2.85817820 +5.87745147 +5.68841643 +5.36124145 +4.96616262 +4.55962409 +4.17380528 +3.82212240 +3.50092352 +3.16729862 +2.64106823 +4.27793575 +4.17487736 +3.99182719 +3.76283542 +3.51911109 +3.28199890 +3.06216435 +2.85817820 +2.64106823 +2.27660509 +27.21281606 +30.93019723 +38.24848265 +38.03471748 +24.47135278 +14.18066558 +8.93889632 +6.39397914 +4.99467087 +3.78453846 +30.93019723 +35.10791273 +40.49326726 +35.05633512 +21.71583324 +12.90755247 +8.40202688 +6.16407388 +4.87738413 +3.71442410 +38.24848265 +40.49326726 +38.48745478 +27.89660053 +17.27079930 +10.89058004 +7.52932583 +5.77324703 +4.66721419 +3.58736778 +38.03471748 +35.05633512 +27.89660053 +19.33452174 +12.80178267 +8.80965627 +6.58682860 +5.31254865 +4.40030906 +3.42369317 +24.47135278 +21.71583324 +17.27079930 +12.80178267 +9.35753738 +7.11788539 +5.75247598 +4.84858754 +4.11047446 +3.24351181 +14.18066558 +12.90755247 +10.89058004 +8.80965627 +7.11788539 +5.91562850 +5.06792460 +4.41376394 +3.82212240 +3.06216435 +8.93889632 +8.40202688 +7.52932583 +6.58682860 +5.75247598 +5.06792460 +4.50349450 +4.01971821 +3.54897364 +2.88855310 +6.39397914 +6.16407388 +5.77324703 +5.31254865 +4.84858754 +4.41376394 +4.01971821 +3.66114702 +3.29087194 +2.72225465 +4.99467087 +4.87738413 +4.66721419 +4.40030906 +4.11047446 +3.82212240 +3.54897364 +3.29087194 +3.01083362 +2.53720463 +3.78453846 +3.71442410 +3.58736778 +3.42369317 +3.24351181 +3.06216435 +2.88855310 +2.72225465 +2.53720463 +2.20294634 +38.82298982 +34.13987445 +25.52819812 +17.39150013 +11.78553303 +8.37976063 +6.39397914 +5.20062803 +4.32844932 +3.38475341 +34.13987445 +29.49972915 +22.22389751 +15.56648805 +10.86435293 +7.91895837 +6.16407388 +5.07555310 +4.25131629 +3.33630039 +25.52819812 +22.22389751 +17.34954702 +12.79294647 +9.39466885 +7.16327224 +5.77324703 +4.85190145 +4.10986092 +3.24707707 +17.39150013 +15.56648805 +12.79294647 +10.03823853 +7.85225168 +6.33370141 +5.31254865 +4.56825187 +3.92434952 +3.12938913 +11.78553303 +10.86435293 +9.39466885 +7.85225168 +6.55611830 +5.58093193 +4.84858754 +4.26005592 +3.71570313 +2.99618395 +8.37976063 +7.91895837 +7.16327224 +6.33370141 +5.58093193 +4.94654060 +4.41376394 +3.95279670 +3.50092352 +2.85817820 +6.39397914 +6.16407388 +5.77324703 +5.31254865 +4.84858754 +4.41376394 +4.01971821 +3.66114702 +3.29087194 +2.72225465 +5.20062803 +5.07555310 +4.85190145 +4.56825187 +4.26005592 +3.95279670 +3.66114702 +3.38533673 +3.08616711 +2.58821615 +4.32844932 +4.25131629 +4.10986092 +3.92434952 +3.71570313 +3.50092352 +3.29087194 +3.08616711 +2.85441883 +2.43251993 +3.38475341 +3.33630039 +3.24707707 +3.12938913 +2.99618395 +2.85817820 +2.72225465 +2.58821615 +2.43251993 +2.12893827 +14.07577128 +12.79375914 +10.81428298 +8.78908447 +7.11248089 +5.87745147 +4.99467087 +4.32844932 +3.74928641 +3.02510299 +12.79375914 +11.72122592 +10.03203401 +8.27423701 +6.79658707 +5.68841643 +4.87738413 +4.25131629 +3.69732526 +2.99095449 +10.81428298 +10.03203401 +8.77250349 +7.42875874 +6.26643818 +5.36124145 +4.66721419 +4.10986092 +3.60080830 +2.92744223 +8.78908447 +8.27423701 +7.42875874 +6.49923979 +5.65982806 +4.96616262 +4.40030906 +3.92434952 +3.47183433 +2.84242280 +7.11248089 +6.79658707 +6.26643818 +5.65982806 +5.07734920 +4.55962409 +4.11047446 +3.71570313 +3.32354485 +2.74448209 +5.87745147 +5.68841643 +5.36124145 +4.96616262 +4.55962409 +4.17380528 +3.82212240 +3.50092352 +3.16729862 +2.64106823 +4.99467087 +4.87738413 +4.66721419 +4.40030906 +4.11047446 +3.82212240 +3.54897364 +3.29087194 +3.01083362 +2.53720463 +4.32844932 +4.25131629 +4.10986092 +3.92434952 +3.71570313 +3.50092352 +3.29087194 +3.08616711 +2.85441883 +2.43251993 +3.74928641 +3.69732526 +3.60080830 +3.47183433 +3.32354485 +3.16729862 +3.01083362 +2.85441883 +2.67045296 +2.30623411 +3.02510299 +2.99095449 +2.92744223 +2.84242280 +2.74448209 +2.64106823 +2.53720463 +2.43251993 +2.30623411 +2.03861926 +7.63917636 +7.19493636 +6.45954861 +5.64164861 +4.89499574 +4.27793575 +3.78453846 +3.38475341 +3.02510299 +2.55355247 +7.19493636 +6.80191598 +6.14899107 +5.41762291 +4.74200020 +4.17487736 +3.71442410 +3.33630039 +2.99095449 +2.52864207 +6.45954861 +6.14899107 +5.62886359 +5.03694085 +4.47645117 +3.99182719 +3.58736778 +3.24707707 +2.92744223 +2.48244864 +5.64164861 +5.41762291 +5.03694085 +4.59215739 +4.15541897 +3.76283542 +3.42369317 +3.12938913 +2.84242280 +2.42090865 +4.89499574 +4.74200020 +4.47645117 +4.15541897 +3.82683188 +3.51911109 +3.24351181 +2.99618395 +2.74448209 +2.35045973 +4.27793575 +4.17487736 +3.99182719 +3.76283542 +3.51911109 +3.28199890 +3.06216435 +2.85817820 +2.64106823 +2.27660509 +3.78453846 +3.71442410 +3.58736778 +3.42369317 +3.24351181 +3.06216435 +2.88855310 +2.72225465 +2.53720463 +2.20294634 +3.38475341 +3.33630039 +3.24707707 +3.12938913 +2.99618395 +2.85817820 +2.72225465 +2.58821615 +2.43251993 +2.12893827 +3.02510299 +2.99095449 +2.92744223 +2.84242280 +2.74448209 +2.64106823 +2.53720463 +2.43251993 +2.30623411 +2.03861926 +2.55355247 +2.52864207 +2.48244864 +2.42090865 +2.35045973 +2.27660509 +2.20294634 +2.12893827 +2.03861926 +1.83559305 + +SCALARS mass float 1 +LOOKUP_TABLE default +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 +0.00100000 + +SCALARS pressure float 1 +LOOKUP_TABLE default +14.50591564 +12.74169250 +13.37720415 +15.39758298 +10.89877768 +17.38805730 +10.12147534 +16.31642116 +1.36864807 +0.18367457 +12.74169250 +13.86898879 +13.90087944 +12.53185511 +13.59728201 +13.67919285 +11.91556612 +12.57146602 +0.99699081 +0.15944143 +13.37720415 +13.90087944 +13.04734298 +12.90878734 +15.16624660 +11.09416382 +15.90145346 +6.79784741 +0.56610917 +0.12305526 +15.39758298 +12.53185511 +12.90878734 +14.04932544 +12.06199866 +12.62861662 +14.89689757 +2.65005645 +0.29040747 +0.08790985 +10.89877768 +13.59728201 +15.16624660 +12.06199866 +12.08925216 +15.79121002 +6.16106384 +0.81234261 +0.16013205 +0.06064363 +17.38805730 +13.67919285 +11.09416382 +12.62861662 +15.79121002 +7.68723987 +1.51096788 +0.26728374 +0.09866437 +0.04166404 +10.12147534 +11.91556612 +15.90145346 +14.89689757 +6.16106384 +1.51096788 +0.33348955 +0.12548073 +0.06451988 +0.02892260 +16.31642116 +12.57146602 +6.79784741 +2.65005645 +0.81234261 +0.26728374 +0.12548073 +0.07288031 +0.04307939 +0.02025557 +1.36864807 +0.99699081 +0.56610917 +0.29040747 +0.16013205 +0.09866437 +0.06451988 +0.04307939 +0.02775122 +0.01376642 +0.18367457 +0.15944143 +0.12305526 +0.08790985 +0.06064363 +0.04166404 +0.02892260 +0.02025557 +0.01376642 +0.00749562 +12.74169250 +13.86898879 +13.90087944 +12.53185511 +13.59728201 +13.67919285 +11.91556612 +12.57146602 +0.99699081 +0.15944143 +13.86898879 +14.33906976 +13.17149862 +12.21720804 +14.99550859 +11.76608856 +14.15804142 +9.27871339 +0.74221586 +0.13948718 +13.90087944 +13.17149862 +12.70640950 +13.89695975 +13.93151953 +11.16940880 +17.06261598 +4.92071112 +0.44165500 +0.10920089 +12.53185511 +12.21720804 +13.89695975 +13.85985602 +11.40765884 +14.04425095 +12.64236714 +1.92727222 +0.24326571 +0.07943570 +13.59728201 +14.99550859 +13.93151953 +11.40765884 +13.05209509 +15.20400391 +4.65848628 +0.62024777 +0.14292107 +0.05582871 +13.67919285 +11.76608856 +11.16940880 +14.04425095 +15.20400391 +6.02083746 +1.12448903 +0.22656472 +0.09120582 +0.03899180 +11.91556612 +14.15804142 +17.06261598 +12.64236714 +4.65848628 +1.12448903 +0.27550911 +0.11479195 +0.06077715 +0.02743070 +12.57146602 +9.27871339 +4.92071112 +1.92727222 +0.62024777 +0.22656472 +0.11479195 +0.06860677 +0.04107019 +0.01941152 +0.99699081 +0.74221586 +0.44165500 +0.24326571 +0.14292107 +0.09120582 +0.06077715 +0.04107019 +0.02668318 +0.01328852 +0.15944143 +0.13948718 +0.10920089 +0.07943570 +0.05582871 +0.03899180 +0.02743070 +0.01941152 +0.01328852 +0.00723932 +13.37720415 +13.90087944 +13.04734298 +12.90878734 +15.16624660 +11.09416382 +15.90145346 +6.79784741 +0.56610917 +0.12305526 +13.90087944 +13.17149862 +12.70640950 +13.89695975 +13.93151953 +11.16940880 +17.06261598 +4.92071112 +0.44165500 +0.10920089 +13.04734298 +12.70640950 +14.04420868 +14.66247098 +11.77699466 +12.96216321 +15.06308698 +2.60273406 +0.28969025 +0.08772876 +12.90878734 +13.89695975 +14.66247098 +12.15109718 +11.58387727 +16.45661864 +7.93271612 +1.05267363 +0.18091653 +0.06591704 +15.16624660 +13.93151953 +11.77699466 +11.58387727 +15.54151118 +11.70223846 +2.59344691 +0.38588635 +0.11712044 +0.04787806 +11.09416382 +11.16940880 +12.96216321 +16.45661864 +11.70223846 +3.47633306 +0.64746829 +0.17200421 +0.07890180 +0.03442480 +15.90145346 +17.06261598 +15.06308698 +7.93271612 +2.59344691 +0.64746829 +0.20026942 +0.09790848 +0.05425984 +0.02480382 +6.79784741 +4.92071112 +2.60273406 +1.05267363 +0.38588635 +0.17200421 +0.09790848 +0.06117904 +0.03746588 +0.01788926 +0.56610917 +0.44165500 +0.28969025 +0.18091653 +0.11712044 +0.07890180 +0.05425984 +0.03746588 +0.02473336 +0.01241393 +0.12305526 +0.10920089 +0.08772876 +0.06591704 +0.04787806 +0.03442480 +0.02480382 +0.01788926 +0.01241393 +0.00677349 +15.39758298 +12.53185511 +12.90878734 +14.04932544 +12.06199866 +12.62861662 +14.89689757 +2.65005645 +0.29040747 +0.08790985 +12.53185511 +12.21720804 +13.89695975 +13.85985602 +11.40765884 +14.04425095 +12.64236714 +1.92727222 +0.24326571 +0.07943570 +12.90878734 +13.89695975 +14.66247098 +12.15109718 +11.58387727 +16.45661864 +7.93271612 +1.05267363 +0.18091653 +0.06591704 +14.04932544 +13.85985602 +12.15109718 +11.15407007 +14.71847572 +13.95782502 +3.42676880 +0.47856818 +0.12869641 +0.05154394 +12.06199866 +11.40765884 +11.58387727 +14.71847572 +15.45867807 +5.84280288 +1.08297332 +0.22385148 +0.09127405 +0.03897255 +12.62861662 +14.04425095 +16.45661864 +13.95782502 +5.84280288 +1.45229701 +0.32584550 +0.12566905 +0.06497658 +0.02904511 +14.89689757 +12.64236714 +7.93271612 +3.42676880 +1.08297332 +0.32584550 +0.14055116 +0.07961718 +0.04635811 +0.02157112 +2.65005645 +1.92727222 +1.05267363 +0.47856818 +0.22385148 +0.12566905 +0.07961718 +0.05217922 +0.03291005 +0.01594804 +0.29040747 +0.24326571 +0.18091653 +0.12869641 +0.09127405 +0.06497658 +0.04635811 +0.03291005 +0.02220248 +0.01127355 +0.08790985 +0.07943570 +0.06591704 +0.05154394 +0.03897255 +0.02904511 +0.02157112 +0.01594804 +0.01127355 +0.00617205 +10.89877768 +13.59728201 +15.16624660 +12.06199866 +12.08925216 +15.79121002 +6.16106384 +0.81234261 +0.16013205 +0.06064363 +13.59728201 +14.99550859 +13.93151953 +11.40765884 +13.05209509 +15.20400391 +4.65848628 +0.62024777 +0.14292107 +0.05582871 +15.16624660 +13.93151953 +11.77699466 +11.58387727 +15.54151118 +11.70223846 +2.59344691 +0.38588635 +0.11712044 +0.04787806 +12.06199866 +11.40765884 +11.58387727 +14.71847572 +15.45867807 +5.84280288 +1.08297332 +0.22385148 +0.09127405 +0.03897255 +12.08925216 +13.05209509 +15.54151118 +15.45867807 +7.42938303 +1.90380004 +0.39625683 +0.13796473 +0.06922913 +0.03069278 +15.79121002 +15.20400391 +11.70223846 +5.84280288 +1.90380004 +0.49957919 +0.17483829 +0.09159402 +0.05172147 +0.02374556 +6.16106384 +4.65848628 +2.59344691 +1.08297332 +0.39625683 +0.17483829 +0.10006133 +0.06268073 +0.03830562 +0.01822029 +0.81234261 +0.62024777 +0.38588635 +0.22385148 +0.13796473 +0.09159402 +0.06268073 +0.04300084 +0.02804325 +0.01384880 +0.16013205 +0.14292107 +0.11712044 +0.09127405 +0.06922913 +0.05172147 +0.03830562 +0.02804325 +0.01940896 +0.01000581 +0.06064363 +0.05582871 +0.04787806 +0.03897255 +0.03069278 +0.02374556 +0.01822029 +0.01384880 +0.01000581 +0.00551061 +17.38805730 +13.67919285 +11.09416382 +12.62861662 +15.79121002 +7.68723987 +1.51096788 +0.26728374 +0.09866437 +0.04166404 +13.67919285 +11.76608856 +11.16940880 +14.04425095 +15.20400391 +6.02083746 +1.12448903 +0.22656472 +0.09120582 +0.03899180 +11.09416382 +11.16940880 +12.96216321 +16.45661864 +11.70223846 +3.47633306 +0.64746829 +0.17200421 +0.07890180 +0.03442480 +12.62861662 +14.04425095 +16.45661864 +13.95782502 +5.84280288 +1.45229701 +0.32584550 +0.12566905 +0.06497658 +0.02904511 +15.79121002 +15.20400391 +11.70223846 +5.84280288 +1.90380004 +0.49957919 +0.17483829 +0.09159402 +0.05172147 +0.02374556 +7.68723987 +6.02083746 +3.47633306 +1.45229701 +0.49957919 +0.19898977 +0.10834978 +0.06664035 +0.04026766 +0.01903491 +1.51096788 +1.12448903 +0.64746829 +0.32584550 +0.17483829 +0.10834978 +0.07185568 +0.04826086 +0.03090658 +0.01508239 +0.26728374 +0.22656472 +0.17200421 +0.12566905 +0.09159402 +0.06664035 +0.04826086 +0.03455969 +0.02335030 +0.01179269 +0.09866437 +0.09120582 +0.07890180 +0.06497658 +0.05172147 +0.04026766 +0.03090658 +0.02335030 +0.01661548 +0.00872504 +0.04166404 +0.03899180 +0.03442480 +0.02904511 +0.02374556 +0.01903491 +0.01508239 +0.01179269 +0.00872504 +0.00484871 +10.12147534 +11.91556612 +15.90145346 +14.89689757 +6.16106384 +1.51096788 +0.33348955 +0.12548073 +0.06451988 +0.02892260 +11.91556612 +14.15804142 +17.06261598 +12.64236714 +4.65848628 +1.12448903 +0.27550911 +0.11479195 +0.06077715 +0.02743070 +15.90145346 +17.06261598 +15.06308698 +7.93271612 +2.59344691 +0.64746829 +0.20026942 +0.09790848 +0.05425984 +0.02480382 +14.89689757 +12.64236714 +7.93271612 +3.42676880 +1.08297332 +0.32584550 +0.14055116 +0.07961718 +0.04635811 +0.02157112 +6.16106384 +4.65848628 +2.59344691 +1.08297332 +0.39625683 +0.17483829 +0.10006133 +0.06268073 +0.03830562 +0.01822029 +1.51096788 +1.12448903 +0.64746829 +0.32584550 +0.17483829 +0.10834978 +0.07185568 +0.04826086 +0.03090658 +0.01508239 +0.33348955 +0.27550911 +0.20026942 +0.14055116 +0.10006133 +0.07185568 +0.05154482 +0.03659685 +0.02452273 +0.01231462 +0.12548073 +0.11479195 +0.09790848 +0.07961718 +0.06268073 +0.04826086 +0.03659685 +0.02728036 +0.01910536 +0.00989630 +0.06451988 +0.06077715 +0.05425984 +0.04635811 +0.03830562 +0.03090658 +0.02452273 +0.01910536 +0.01398976 +0.00750398 +0.02892260 +0.02743070 +0.02480382 +0.02157112 +0.01822029 +0.01508239 +0.01231462 +0.00989630 +0.00750398 +0.00422150 +16.31642116 +12.57146602 +6.79784741 +2.65005645 +0.81234261 +0.26728374 +0.12548073 +0.07288031 +0.04307939 +0.02025557 +12.57146602 +9.27871339 +4.92071112 +1.92727222 +0.62024777 +0.22656472 +0.11479195 +0.06860677 +0.04107019 +0.01941152 +6.79784741 +4.92071112 +2.60273406 +1.05267363 +0.38588635 +0.17200421 +0.09790848 +0.06117904 +0.03746588 +0.01788926 +2.65005645 +1.92727222 +1.05267363 +0.47856818 +0.22385148 +0.12566905 +0.07961718 +0.05217922 +0.03291005 +0.01594804 +0.81234261 +0.62024777 +0.38588635 +0.22385148 +0.13796473 +0.09159402 +0.06268073 +0.04300084 +0.02804325 +0.01384880 +0.26728374 +0.22656472 +0.17200421 +0.12566905 +0.09159402 +0.06664035 +0.04826086 +0.03455969 +0.02335030 +0.01179269 +0.12548073 +0.11479195 +0.09790848 +0.07961718 +0.06268073 +0.04826086 +0.03659685 +0.02728036 +0.01910536 +0.00989630 +0.07288031 +0.06860677 +0.06117904 +0.05217922 +0.04300084 +0.03455969 +0.02728036 +0.02111882 +0.01533137 +0.00816305 +0.04307939 +0.04107019 +0.03746588 +0.03291005 +0.02804325 +0.02335030 +0.01910536 +0.01533137 +0.01155716 +0.00634644 +0.02025557 +0.01941152 +0.01788926 +0.01594804 +0.01384880 +0.01179269 +0.00989630 +0.00816305 +0.00634644 +0.00362586 +1.36864807 +0.99699081 +0.56610917 +0.29040747 +0.16013205 +0.09866437 +0.06451988 +0.04307939 +0.02775122 +0.01376642 +0.99699081 +0.74221586 +0.44165500 +0.24326571 +0.14292107 +0.09120582 +0.06077715 +0.04107019 +0.02668318 +0.01328852 +0.56610917 +0.44165500 +0.28969025 +0.18091653 +0.11712044 +0.07890180 +0.05425984 +0.03746588 +0.02473336 +0.01241393 +0.29040747 +0.24326571 +0.18091653 +0.12869641 +0.09127405 +0.06497658 +0.04635811 +0.03291005 +0.02220248 +0.01127355 +0.16013205 +0.14292107 +0.11712044 +0.09127405 +0.06922913 +0.05172147 +0.03830562 +0.02804325 +0.01940896 +0.01000581 +0.09866437 +0.09120582 +0.07890180 +0.06497658 +0.05172147 +0.04026766 +0.03090658 +0.02335030 +0.01661548 +0.00872504 +0.06451988 +0.06077715 +0.05425984 +0.04635811 +0.03830562 +0.03090658 +0.02452273 +0.01910536 +0.01398976 +0.00750398 +0.04307939 +0.04107019 +0.03746588 +0.03291005 +0.02804325 +0.02335030 +0.01910536 +0.01533137 +0.01155716 +0.00634644 +0.02775122 +0.02668318 +0.02473336 +0.02220248 +0.01940896 +0.01661548 +0.01398976 +0.01155716 +0.00898707 +0.00506681 +0.01376642 +0.01328852 +0.01241393 +0.01127355 +0.01000581 +0.00872504 +0.00750398 +0.00634644 +0.00506681 +0.00295295 +0.18367457 +0.15944143 +0.12305526 +0.08790985 +0.06064363 +0.04166404 +0.02892260 +0.02025557 +0.01376642 +0.00749562 +0.15944143 +0.13948718 +0.10920089 +0.07943570 +0.05582871 +0.03899180 +0.02743070 +0.01941152 +0.01328852 +0.00723932 +0.12305526 +0.10920089 +0.08772876 +0.06591704 +0.04787806 +0.03442480 +0.02480382 +0.01788926 +0.01241393 +0.00677349 +0.08790985 +0.07943570 +0.06591704 +0.05154394 +0.03897255 +0.02904511 +0.02157112 +0.01594804 +0.01127355 +0.00617205 +0.06064363 +0.05582871 +0.04787806 +0.03897255 +0.03069278 +0.02374556 +0.01822029 +0.01384880 +0.01000581 +0.00551061 +0.04166404 +0.03899180 +0.03442480 +0.02904511 +0.02374556 +0.01903491 +0.01508239 +0.01179269 +0.00872504 +0.00484871 +0.02892260 +0.02743070 +0.02480382 +0.02157112 +0.01822029 +0.01508239 +0.01231462 +0.00989630 +0.00750398 +0.00422150 +0.02025557 +0.01941152 +0.01788926 +0.01594804 +0.01384880 +0.01179269 +0.00989630 +0.00816305 +0.00634644 +0.00362586 +0.01376642 +0.01328852 +0.01241393 +0.01127355 +0.01000581 +0.00872504 +0.00750398 +0.00634644 +0.00506681 +0.00295295 +0.00749562 +0.00723932 +0.00677349 +0.00617205 +0.00551061 +0.00484871 +0.00422150 +0.00362586 +0.00295295 +0.00177400 + +SCALARS sound_speed float 1 +LOOKUP_TABLE default +1.45807863 +1.18933609 +1.04713130 +0.99138308 +0.87121993 +0.91292460 +0.78733497 +0.83693617 +0.40256329 +0.20018228 +1.18933609 +1.10936022 +1.00817201 +0.91760044 +0.88487628 +0.85179429 +0.80129120 +0.78340507 +0.36038845 +0.19218146 +1.04713130 +1.00817201 +0.93890309 +0.88843796 +0.87775887 +0.79651614 +0.83240692 +0.66619290 +0.29537627 +0.17818592 +0.99138308 +0.91760044 +0.88843796 +0.87407282 +0.82013323 +0.80319808 +0.80794574 +0.50394532 +0.23466939 +0.16115364 +0.87121993 +0.88487628 +0.87775887 +0.82013323 +0.80271363 +0.82795766 +0.64777359 +0.33893731 +0.19371041 +0.14369472 +0.91292460 +0.85179429 +0.79651614 +0.80319808 +0.82795766 +0.69135581 +0.42140888 +0.23056577 +0.16726690 +0.12740544 +0.78733497 +0.80129120 +0.83240692 +0.80794574 +0.64777359 +0.42140888 +0.24935813 +0.18085365 +0.14672959 +0.11285910 +0.83693617 +0.78340507 +0.66619290 +0.50394532 +0.33893731 +0.23056577 +0.18085365 +0.15282753 +0.12879321 +0.09986956 +0.40256329 +0.36038845 +0.29537627 +0.23466939 +0.19371041 +0.16726690 +0.14672959 +0.12879321 +0.11106855 +0.08708931 +0.20018228 +0.19218146 +0.17818592 +0.16115364 +0.14369472 +0.12740544 +0.11285910 +0.09986956 +0.08708931 +0.06994484 +1.18933609 +1.10936022 +1.00817201 +0.91760044 +0.88487628 +0.85179429 +0.80129120 +0.78340507 +0.36038845 +0.19218146 +1.10936022 +1.05665249 +0.96579285 +0.89254452 +0.88597274 +0.81375710 +0.81982934 +0.72403447 +0.32486504 +0.18487402 +1.00817201 +0.96579285 +0.91457103 +0.88905922 +0.85395443 +0.79101585 +0.83802268 +0.60747470 +0.27087658 +0.17204241 +0.91760044 +0.89254452 +0.88905922 +0.86284618 +0.80520430 +0.81566240 +0.77527418 +0.45425591 +0.22136087 +0.15632479 +0.88487628 +0.88597274 +0.85395443 +0.80520430 +0.81055346 +0.81745912 +0.59794128 +0.30846440 +0.18720915 +0.14007879 +0.85179429 +0.81375710 +0.79101585 +0.81566240 +0.81745912 +0.64604312 +0.38104831 +0.21836674 +0.16347072 +0.12476396 +0.80129120 +0.81982934 +0.83802268 +0.77527418 +0.59794128 +0.38104831 +0.23377622 +0.17617577 +0.14411230 +0.11094228 +0.78340507 +0.72403447 +0.60747470 +0.45425591 +0.30846440 +0.21836674 +0.17617577 +0.15009498 +0.12688960 +0.09847399 +0.36038845 +0.32486504 +0.27087658 +0.22136087 +0.18720915 +0.16347072 +0.14411230 +0.12688960 +0.10967290 +0.08605137 +0.19218146 +0.18487402 +0.17204241 +0.15632479 +0.14007879 +0.12476396 +0.11094228 +0.09847399 +0.08605137 +0.06907639 +1.04713130 +1.00817201 +0.93890309 +0.88843796 +0.87775887 +0.79651614 +0.83240692 +0.66619290 +0.29537627 +0.17818592 +1.00817201 +0.96579285 +0.91457103 +0.88905922 +0.85395443 +0.79101585 +0.83802268 +0.60747470 +0.27087658 +0.17204241 +0.93890309 +0.91457103 +0.90377267 +0.87947460 +0.81316341 +0.80479733 +0.80764735 +0.50002901 +0.23460083 +0.16117029 +0.88843796 +0.88905922 +0.87947460 +0.82637163 +0.79770977 +0.83398824 +0.68842984 +0.37032749 +0.20146768 +0.14768616 +0.87775887 +0.85395443 +0.81316341 +0.79770977 +0.83052058 +0.76391984 +0.50027274 +0.26164555 +0.17649416 +0.13351366 +0.79651614 +0.79101585 +0.80479733 +0.83398824 +0.76391984 +0.54921790 +0.31478072 +0.20004983 +0.15661565 +0.11988758 +0.83240692 +0.83802268 +0.80764735 +0.68842984 +0.50027274 +0.31478072 +0.21054922 +0.16812196 +0.13919857 +0.10734844 +0.66619290 +0.60747470 +0.50002901 +0.37032749 +0.26164555 +0.20004983 +0.16812196 +0.14496719 +0.12326188 +0.09582400 +0.29537627 +0.27087658 +0.23460083 +0.20146768 +0.17649416 +0.15661565 +0.13919857 +0.12326188 +0.10699561 +0.08406881 +0.17818592 +0.17204241 +0.16117029 +0.14768616 +0.13351366 +0.11988758 +0.10734844 +0.09582400 +0.08406881 +0.06743581 +0.99138308 +0.91760044 +0.88843796 +0.87407282 +0.82013323 +0.80319808 +0.80794574 +0.50394532 +0.23466939 +0.16115364 +0.91760044 +0.89254452 +0.88905922 +0.86284618 +0.80520430 +0.81566240 +0.77527418 +0.45425591 +0.22136087 +0.15632479 +0.88843796 +0.88905922 +0.87947460 +0.82637163 +0.79770977 +0.83398824 +0.68842984 +0.37032749 +0.20146768 +0.14768616 +0.87407282 +0.86284618 +0.82637163 +0.79613255 +0.82420619 +0.79490908 +0.54350064 +0.28188212 +0.18166711 +0.13677439 +0.82013323 +0.80520430 +0.79770977 +0.82420619 +0.81759936 +0.63483127 +0.37548970 +0.21797529 +0.16394434 +0.12502486 +0.80319808 +0.81566240 +0.83398824 +0.79490908 +0.63483127 +0.41514587 +0.24828516 +0.18184851 +0.14767002 +0.11342357 +0.80794574 +0.77527418 +0.68842984 +0.54350064 +0.37548970 +0.24828516 +0.18858338 +0.15804338 +0.13250909 +0.10247391 +0.50394532 +0.45425591 +0.37032749 +0.28188212 +0.21797529 +0.18184851 +0.15804338 +0.13797428 +0.11822378 +0.09216124 +0.23466939 +0.22136087 +0.20146768 +0.18166711 +0.16394434 +0.14767002 +0.13250909 +0.11822378 +0.10323943 +0.08130370 +0.16115364 +0.15632479 +0.14768616 +0.13677439 +0.12502486 +0.11342357 +0.10247391 +0.09216124 +0.08130370 +0.06518531 +0.87121993 +0.88487628 +0.87775887 +0.82013323 +0.80271363 +0.82795766 +0.64777359 +0.33893731 +0.19371041 +0.14369472 +0.88487628 +0.88597274 +0.85395443 +0.80520430 +0.81055346 +0.81745912 +0.59794128 +0.30846440 +0.18720915 +0.14007879 +0.87775887 +0.85395443 +0.81316341 +0.79770977 +0.83052058 +0.76391984 +0.50027274 +0.26164555 +0.17649416 +0.13351366 +0.82013323 +0.80520430 +0.79770977 +0.82420619 +0.81759936 +0.63483127 +0.37548970 +0.21797529 +0.16394434 +0.12502486 +0.80271363 +0.81055346 +0.83052058 +0.81759936 +0.67655152 +0.45293958 +0.26566355 +0.18727725 +0.15074756 +0.11561730 +0.82795766 +0.81745912 +0.76391984 +0.63483127 +0.45293958 +0.28831995 +0.20233320 +0.16538822 +0.13749766 +0.10604718 +0.64777359 +0.59794128 +0.50027274 +0.37548970 +0.26566355 +0.20233320 +0.17026685 +0.14678571 +0.12462629 +0.09675965 +0.33893731 +0.30846440 +0.26164555 +0.21797529 +0.18727725 +0.16538822 +0.14678571 +0.12970455 +0.11215485 +0.08777002 +0.19371041 +0.18720915 +0.17649416 +0.16394434 +0.15074756 +0.13749766 +0.12462629 +0.11215485 +0.09865628 +0.07795076 +0.14369472 +0.14007879 +0.13351366 +0.12502486 +0.11561730 +0.10604718 +0.09675965 +0.08777002 +0.07795076 +0.06250974 +0.91292460 +0.85179429 +0.79651614 +0.80319808 +0.82795766 +0.69135581 +0.42140888 +0.23056577 +0.16726690 +0.12740544 +0.85179429 +0.81375710 +0.79101585 +0.81566240 +0.81745912 +0.64604312 +0.38104831 +0.21836674 +0.16347072 +0.12476396 +0.79651614 +0.79101585 +0.80479733 +0.83398824 +0.76391984 +0.54921790 +0.31478072 +0.20004983 +0.15661565 +0.11988758 +0.80319808 +0.81566240 +0.83398824 +0.79490908 +0.63483127 +0.41514587 +0.24828516 +0.18184851 +0.14767002 +0.11342357 +0.82795766 +0.81745912 +0.76391984 +0.63483127 +0.45293958 +0.28831995 +0.20233320 +0.16538822 +0.13749766 +0.10604718 +0.69135581 +0.64604312 +0.54921790 +0.41514587 +0.28831995 +0.21118958 +0.17471812 +0.14984499 +0.12680503 +0.09831745 +0.42140888 +0.38104831 +0.31478072 +0.24828516 +0.20233320 +0.17471812 +0.15372336 +0.13499490 +0.11609074 +0.09060356 +0.23056577 +0.21836674 +0.20004983 +0.18184851 +0.16538822 +0.14984499 +0.13499490 +0.12071384 +0.10543366 +0.08292515 +0.16726690 +0.16347072 +0.15661565 +0.14767002 +0.13749766 +0.12680503 +0.11609074 +0.10543366 +0.09350532 +0.07420245 +0.12740544 +0.12476396 +0.11988758 +0.11342357 +0.10604718 +0.09831745 +0.09060356 +0.08292515 +0.07420245 +0.05957907 +0.78733497 +0.80129120 +0.83240692 +0.80794574 +0.64777359 +0.42140888 +0.24935813 +0.18085365 +0.14672959 +0.11285910 +0.80129120 +0.81982934 +0.83802268 +0.77527418 +0.59794128 +0.38104831 +0.23377622 +0.17617577 +0.14411230 +0.11094228 +0.83240692 +0.83802268 +0.80764735 +0.68842984 +0.50027274 +0.31478072 +0.21054922 +0.16812196 +0.13919857 +0.10734844 +0.80794574 +0.77527418 +0.68842984 +0.54350064 +0.37548970 +0.24828516 +0.18858338 +0.15804338 +0.13250909 +0.10247391 +0.64777359 +0.59794128 +0.50027274 +0.37548970 +0.26566355 +0.20233320 +0.17026685 +0.14678571 +0.12462629 +0.09675965 +0.42140888 +0.38104831 +0.31478072 +0.24828516 +0.20233320 +0.17471812 +0.15372336 +0.13499490 +0.11609074 +0.09060356 +0.24935813 +0.23377622 +0.21054922 +0.18858338 +0.17026685 +0.15372336 +0.13811539 +0.12318234 +0.10731426 +0.08429362 +0.18085365 +0.17617577 +0.16812196 +0.15804338 +0.14678571 +0.13499490 +0.12318234 +0.11143992 +0.09836634 +0.07783885 +0.14672959 +0.14411230 +0.13919857 +0.13250909 +0.12462629 +0.11609074 +0.10731426 +0.09836634 +0.08800070 +0.07020895 +0.11285910 +0.11094228 +0.10734844 +0.10247391 +0.09675965 +0.09060356 +0.08429362 +0.07783885 +0.07020895 +0.05651399 +0.83693617 +0.78340507 +0.66619290 +0.50394532 +0.33893731 +0.23056577 +0.18085365 +0.15282753 +0.12879321 +0.09986956 +0.78340507 +0.72403447 +0.60747470 +0.45425591 +0.30846440 +0.21836674 +0.17617577 +0.15009498 +0.12688960 +0.09847399 +0.66619290 +0.60747470 +0.50002901 +0.37032749 +0.26164555 +0.20004983 +0.16812196 +0.14496719 +0.12326188 +0.09582400 +0.50394532 +0.45425591 +0.37032749 +0.28188212 +0.21797529 +0.18184851 +0.15804338 +0.13797428 +0.11822378 +0.09216124 +0.33893731 +0.30846440 +0.26164555 +0.21797529 +0.18727725 +0.16538822 +0.14678571 +0.12970455 +0.11215485 +0.08777002 +0.23056577 +0.21836674 +0.20004983 +0.18184851 +0.16538822 +0.14984499 +0.13499490 +0.12071384 +0.10543366 +0.08292515 +0.18085365 +0.17617577 +0.16812196 +0.15804338 +0.14678571 +0.13499490 +0.12318234 +0.11143992 +0.09836634 +0.07783885 +0.15282753 +0.15009498 +0.14496719 +0.13797428 +0.12970455 +0.12071384 +0.11143992 +0.10196669 +0.09099241 +0.07250205 +0.12879321 +0.12688960 +0.12326188 +0.11822378 +0.11215485 +0.10543366 +0.09836634 +0.09099241 +0.08214689 +0.06594186 +0.09986956 +0.09847399 +0.09582400 +0.09216124 +0.08777002 +0.08292515 +0.07783885 +0.07250205 +0.06594186 +0.05327802 +0.40256329 +0.36038845 +0.29537627 +0.23466939 +0.19371041 +0.16726690 +0.14672959 +0.12879321 +0.11106855 +0.08708931 +0.36038845 +0.32486504 +0.27087658 +0.22136087 +0.18720915 +0.16347072 +0.14411230 +0.12688960 +0.10967290 +0.08605137 +0.29537627 +0.27087658 +0.23460083 +0.20146768 +0.17649416 +0.15661565 +0.13919857 +0.12326188 +0.10699561 +0.08406881 +0.23466939 +0.22136087 +0.20146768 +0.18166711 +0.16394434 +0.14767002 +0.13250909 +0.11822378 +0.10323943 +0.08130370 +0.19371041 +0.18720915 +0.17649416 +0.16394434 +0.15074756 +0.13749766 +0.12462629 +0.11215485 +0.09865628 +0.07795076 +0.16726690 +0.16347072 +0.15661565 +0.14767002 +0.13749766 +0.12680503 +0.11609074 +0.10543366 +0.09350532 +0.07420245 +0.14672959 +0.14411230 +0.13919857 +0.13250909 +0.12462629 +0.11609074 +0.10731426 +0.09836634 +0.08800070 +0.07020895 +0.12879321 +0.12688960 +0.12326188 +0.11822378 +0.11215485 +0.10543366 +0.09836634 +0.09099241 +0.08214689 +0.06594186 +0.11106855 +0.10967290 +0.10699561 +0.10323943 +0.09865628 +0.09350532 +0.08800070 +0.08214689 +0.07489297 +0.06051181 +0.08708931 +0.08605137 +0.08406881 +0.08130370 +0.07795076 +0.07420245 +0.07020895 +0.06594186 +0.06051181 +0.04913429 +0.20018228 +0.19218146 +0.17818592 +0.16115364 +0.14369472 +0.12740544 +0.11285910 +0.09986956 +0.08708931 +0.06994484 +0.19218146 +0.18487402 +0.17204241 +0.15632479 +0.14007879 +0.12476396 +0.11094228 +0.09847399 +0.08605137 +0.06907639 +0.17818592 +0.17204241 +0.16117029 +0.14768616 +0.13351366 +0.11988758 +0.10734844 +0.09582400 +0.08406881 +0.06743581 +0.16115364 +0.15632479 +0.14768616 +0.13677439 +0.12502486 +0.11342357 +0.10247391 +0.09216124 +0.08130370 +0.06518531 +0.14369472 +0.14007879 +0.13351366 +0.12502486 +0.11561730 +0.10604718 +0.09675965 +0.08777002 +0.07795076 +0.06250974 +0.12740544 +0.12476396 +0.11988758 +0.11342357 +0.10604718 +0.09831745 +0.09060356 +0.08292515 +0.07420245 +0.05957907 +0.11285910 +0.11094228 +0.10734844 +0.10247391 +0.09675965 +0.09060356 +0.08429362 +0.07783885 +0.07020895 +0.05651399 +0.09986956 +0.09847399 +0.09582400 +0.09216124 +0.08777002 +0.08292515 +0.07783885 +0.07250205 +0.06594186 +0.05327802 +0.08708931 +0.08605137 +0.08406881 +0.08130370 +0.07795076 +0.07420245 +0.07020895 +0.06594186 +0.06051181 +0.04913429 +0.06994484 +0.06907639 +0.06743581 +0.06518531 +0.06250974 +0.05957907 +0.05651399 +0.05327802 +0.04913429 +0.04013403 + +SCALARS volume float 1 +LOOKUP_TABLE default +0.00008794 +0.00006661 +0.00004918 +0.00003830 +0.00004179 +0.00002876 +0.00003675 +0.00002576 +0.00007104 +0.00013090 +0.00006661 +0.00005324 +0.00004387 +0.00004031 +0.00003455 +0.00003182 +0.00003233 +0.00002929 +0.00007816 +0.00013899 +0.00004918 +0.00004387 +0.00004054 +0.00003669 +0.00003048 +0.00003431 +0.00002614 +0.00003917 +0.00009247 +0.00015481 +0.00003830 +0.00004031 +0.00003669 +0.00003263 +0.00003346 +0.00003065 +0.00002629 +0.00005750 +0.00011378 +0.00017725 +0.00004179 +0.00003455 +0.00003048 +0.00003346 +0.00003198 +0.00002605 +0.00004086 +0.00008485 +0.00014060 +0.00020429 +0.00002876 +0.00003182 +0.00003431 +0.00003065 +0.00002605 +0.00003731 +0.00007052 +0.00011934 +0.00017014 +0.00023376 +0.00003675 +0.00003233 +0.00002614 +0.00002629 +0.00004086 +0.00007052 +0.00011187 +0.00015640 +0.00020021 +0.00026423 +0.00002576 +0.00002929 +0.00003917 +0.00005750 +0.00008485 +0.00011934 +0.00015640 +0.00019228 +0.00023103 +0.00029544 +0.00007104 +0.00007816 +0.00009247 +0.00011378 +0.00014060 +0.00017014 +0.00020021 +0.00023103 +0.00026672 +0.00033057 +0.00013090 +0.00013899 +0.00015481 +0.00017725 +0.00020429 +0.00023376 +0.00026423 +0.00029544 +0.00033057 +0.00039161 +0.00006661 +0.00005324 +0.00004387 +0.00004031 +0.00003455 +0.00003182 +0.00003233 +0.00002929 +0.00007816 +0.00013899 +0.00005324 +0.00004672 +0.00004249 +0.00003912 +0.00003141 +0.00003377 +0.00002848 +0.00003390 +0.00008532 +0.00014702 +0.00004387 +0.00004249 +0.00003950 +0.00003413 +0.00003141 +0.00003361 +0.00002470 +0.00004500 +0.00009968 +0.00016263 +0.00004031 +0.00003912 +0.00003413 +0.00003223 +0.00003410 +0.00002842 +0.00002853 +0.00006424 +0.00012086 +0.00018458 +0.00003455 +0.00003141 +0.00003141 +0.00003410 +0.00003020 +0.00002637 +0.00004605 +0.00009204 +0.00014713 +0.00021088 +0.00003182 +0.00003377 +0.00003361 +0.00002842 +0.00002637 +0.00004159 +0.00007747 +0.00012628 +0.00017580 +0.00023953 +0.00003233 +0.00002848 +0.00002470 +0.00002853 +0.00004605 +0.00007747 +0.00011902 +0.00016223 +0.00020503 +0.00026922 +0.00002929 +0.00003390 +0.00004500 +0.00006424 +0.00009204 +0.00012628 +0.00016223 +0.00019702 +0.00023522 +0.00029973 +0.00007816 +0.00008532 +0.00009968 +0.00012086 +0.00014713 +0.00017580 +0.00020503 +0.00023522 +0.00027047 +0.00033434 +0.00013899 +0.00014702 +0.00016263 +0.00018458 +0.00021088 +0.00023953 +0.00026922 +0.00029973 +0.00033434 +0.00039547 +0.00004918 +0.00004387 +0.00004054 +0.00003669 +0.00003048 +0.00003431 +0.00002614 +0.00003917 +0.00009247 +0.00015481 +0.00004387 +0.00004249 +0.00003950 +0.00003413 +0.00003141 +0.00003361 +0.00002470 +0.00004500 +0.00009968 +0.00016263 +0.00004054 +0.00003950 +0.00003490 +0.00003165 +0.00003369 +0.00002998 +0.00002598 +0.00005764 +0.00011399 +0.00017766 +0.00003669 +0.00003413 +0.00003165 +0.00003372 +0.00003296 +0.00002536 +0.00003585 +0.00007817 +0.00013461 +0.00019853 +0.00003048 +0.00003141 +0.00003369 +0.00003296 +0.00002663 +0.00002992 +0.00005790 +0.00010644 +0.00015958 +0.00022339 +0.00003431 +0.00003361 +0.00002998 +0.00002536 +0.00002992 +0.00005206 +0.00009182 +0.00013960 +0.00018652 +0.00025051 +0.00002614 +0.00002470 +0.00002598 +0.00003585 +0.00005790 +0.00009182 +0.00013281 +0.00017321 +0.00021426 +0.00027876 +0.00003917 +0.00004500 +0.00005764 +0.00007817 +0.00010644 +0.00013960 +0.00017321 +0.00020610 +0.00024332 +0.00030797 +0.00009247 +0.00009968 +0.00011399 +0.00013461 +0.00015958 +0.00018652 +0.00021426 +0.00024332 +0.00027772 +0.00034160 +0.00015481 +0.00016263 +0.00017766 +0.00019853 +0.00022339 +0.00025051 +0.00027876 +0.00030797 +0.00034160 +0.00040283 +0.00003830 +0.00004031 +0.00003669 +0.00003263 +0.00003346 +0.00003065 +0.00002629 +0.00005750 +0.00011378 +0.00017725 +0.00004031 +0.00003912 +0.00003413 +0.00003223 +0.00003410 +0.00002842 +0.00002853 +0.00006424 +0.00012086 +0.00018458 +0.00003669 +0.00003413 +0.00003165 +0.00003372 +0.00003296 +0.00002536 +0.00003585 +0.00007817 +0.00013461 +0.00019853 +0.00003263 +0.00003223 +0.00003372 +0.00003409 +0.00002769 +0.00002716 +0.00005172 +0.00009962 +0.00015386 +0.00021776 +0.00003346 +0.00003410 +0.00003296 +0.00002769 +0.00002595 +0.00004139 +0.00007811 +0.00012735 +0.00017668 +0.00024065 +0.00003065 +0.00002842 +0.00002536 +0.00002716 +0.00004139 +0.00007120 +0.00011351 +0.00015789 +0.00020136 +0.00026576 +0.00002629 +0.00002853 +0.00003585 +0.00005172 +0.00007811 +0.00011351 +0.00015182 +0.00018823 +0.00022726 +0.00029208 +0.00005750 +0.00006424 +0.00007817 +0.00009962 +0.00012735 +0.00015789 +0.00018823 +0.00021890 +0.00025482 +0.00031955 +0.00011378 +0.00012086 +0.00013461 +0.00015386 +0.00017668 +0.00020136 +0.00022726 +0.00025482 +0.00028803 +0.00035181 +0.00017725 +0.00018458 +0.00019853 +0.00021776 +0.00024065 +0.00026576 +0.00029208 +0.00031955 +0.00035181 +0.00041307 +0.00004179 +0.00003455 +0.00003048 +0.00003346 +0.00003198 +0.00002605 +0.00004086 +0.00008485 +0.00014060 +0.00020429 +0.00003455 +0.00003141 +0.00003141 +0.00003410 +0.00003020 +0.00002637 +0.00004605 +0.00009204 +0.00014713 +0.00021088 +0.00003048 +0.00003141 +0.00003369 +0.00003296 +0.00002663 +0.00002992 +0.00005790 +0.00010644 +0.00015958 +0.00022339 +0.00003346 +0.00003410 +0.00003296 +0.00002769 +0.00002595 +0.00004139 +0.00007811 +0.00012735 +0.00017668 +0.00024065 +0.00003198 +0.00003020 +0.00002663 +0.00002595 +0.00003697 +0.00006466 +0.00010687 +0.00015253 +0.00019695 +0.00026131 +0.00002605 +0.00002637 +0.00002992 +0.00004139 +0.00006466 +0.00009984 +0.00014049 +0.00017918 +0.00021932 +0.00028416 +0.00004086 +0.00004605 +0.00005790 +0.00007811 +0.00010687 +0.00014049 +0.00017384 +0.00020625 +0.00024328 +0.00030831 +0.00008485 +0.00009204 +0.00010644 +0.00012735 +0.00015253 +0.00017918 +0.00020625 +0.00023474 +0.00026913 +0.00033376 +0.00014060 +0.00014713 +0.00015958 +0.00017668 +0.00019695 +0.00021932 +0.00024328 +0.00026913 +0.00030088 +0.00036437 +0.00020429 +0.00021088 +0.00022339 +0.00024065 +0.00026131 +0.00028416 +0.00030831 +0.00033376 +0.00036437 +0.00042545 +0.00002876 +0.00003182 +0.00003431 +0.00003065 +0.00002605 +0.00003731 +0.00007052 +0.00011934 +0.00017014 +0.00023376 +0.00003182 +0.00003377 +0.00003361 +0.00002842 +0.00002637 +0.00004159 +0.00007747 +0.00012628 +0.00017580 +0.00023953 +0.00003431 +0.00003361 +0.00002998 +0.00002536 +0.00002992 +0.00005206 +0.00009182 +0.00013960 +0.00018652 +0.00025051 +0.00003065 +0.00002842 +0.00002536 +0.00002716 +0.00004139 +0.00007120 +0.00011351 +0.00015789 +0.00020136 +0.00026576 +0.00002605 +0.00002637 +0.00002992 +0.00004139 +0.00006466 +0.00009984 +0.00014049 +0.00017918 +0.00021932 +0.00028416 +0.00003731 +0.00004159 +0.00005206 +0.00007120 +0.00009984 +0.00013448 +0.00016904 +0.00020216 +0.00023959 +0.00030469 +0.00007052 +0.00007747 +0.00009182 +0.00011351 +0.00014049 +0.00016904 +0.00019732 +0.00022656 +0.00026163 +0.00032657 +0.00011934 +0.00012628 +0.00013960 +0.00015789 +0.00017918 +0.00020216 +0.00022656 +0.00025299 +0.00028564 +0.00034987 +0.00017014 +0.00017580 +0.00018652 +0.00020136 +0.00021932 +0.00023959 +0.00026163 +0.00028564 +0.00031573 +0.00037863 +0.00023376 +0.00023953 +0.00025051 +0.00026576 +0.00028416 +0.00030469 +0.00032657 +0.00034987 +0.00037863 +0.00043925 +0.00003675 +0.00003233 +0.00002614 +0.00002629 +0.00004086 +0.00007052 +0.00011187 +0.00015640 +0.00020021 +0.00026423 +0.00003233 +0.00002848 +0.00002470 +0.00002853 +0.00004605 +0.00007747 +0.00011902 +0.00016223 +0.00020503 +0.00026922 +0.00002614 +0.00002470 +0.00002598 +0.00003585 +0.00005790 +0.00009182 +0.00013281 +0.00017321 +0.00021426 +0.00027876 +0.00002629 +0.00002853 +0.00003585 +0.00005172 +0.00007811 +0.00011351 +0.00015182 +0.00018823 +0.00022726 +0.00029208 +0.00004086 +0.00004605 +0.00005790 +0.00007811 +0.00010687 +0.00014049 +0.00017384 +0.00020625 +0.00024328 +0.00030831 +0.00007052 +0.00007747 +0.00009182 +0.00011351 +0.00014049 +0.00016904 +0.00019732 +0.00022656 +0.00026163 +0.00032657 +0.00011187 +0.00011902 +0.00013281 +0.00015182 +0.00017384 +0.00019732 +0.00022205 +0.00024877 +0.00028177 +0.00034619 +0.00015640 +0.00016223 +0.00017321 +0.00018823 +0.00020625 +0.00022656 +0.00024877 +0.00027314 +0.00030387 +0.00036734 +0.00020021 +0.00020503 +0.00021426 +0.00022726 +0.00024328 +0.00026163 +0.00028177 +0.00030387 +0.00033213 +0.00039413 +0.00026423 +0.00026922 +0.00027876 +0.00029208 +0.00030831 +0.00032657 +0.00034619 +0.00036734 +0.00039413 +0.00045394 +0.00002576 +0.00002929 +0.00003917 +0.00005750 +0.00008485 +0.00011934 +0.00015640 +0.00019228 +0.00023103 +0.00029544 +0.00002929 +0.00003390 +0.00004500 +0.00006424 +0.00009204 +0.00012628 +0.00016223 +0.00019702 +0.00023522 +0.00029973 +0.00003917 +0.00004500 +0.00005764 +0.00007817 +0.00010644 +0.00013960 +0.00017321 +0.00020610 +0.00024332 +0.00030797 +0.00005750 +0.00006424 +0.00007817 +0.00009962 +0.00012735 +0.00015789 +0.00018823 +0.00021890 +0.00025482 +0.00031955 +0.00008485 +0.00009204 +0.00010644 +0.00012735 +0.00015253 +0.00017918 +0.00020625 +0.00023474 +0.00026913 +0.00033376 +0.00011934 +0.00012628 +0.00013960 +0.00015789 +0.00017918 +0.00020216 +0.00022656 +0.00025299 +0.00028564 +0.00034987 +0.00015640 +0.00016223 +0.00017321 +0.00018823 +0.00020625 +0.00022656 +0.00024877 +0.00027314 +0.00030387 +0.00036734 +0.00019228 +0.00019702 +0.00020610 +0.00021890 +0.00023474 +0.00025299 +0.00027314 +0.00029539 +0.00032403 +0.00038637 +0.00023103 +0.00023522 +0.00024332 +0.00025482 +0.00026913 +0.00028564 +0.00030387 +0.00032403 +0.00035033 +0.00041110 +0.00029544 +0.00029973 +0.00030797 +0.00031955 +0.00033376 +0.00034987 +0.00036734 +0.00038637 +0.00041110 +0.00046972 +0.00007104 +0.00007816 +0.00009247 +0.00011378 +0.00014060 +0.00017014 +0.00020021 +0.00023103 +0.00026672 +0.00033057 +0.00007816 +0.00008532 +0.00009968 +0.00012086 +0.00014713 +0.00017580 +0.00020503 +0.00023522 +0.00027047 +0.00033434 +0.00009247 +0.00009968 +0.00011399 +0.00013461 +0.00015958 +0.00018652 +0.00021426 +0.00024332 +0.00027772 +0.00034160 +0.00011378 +0.00012086 +0.00013461 +0.00015386 +0.00017668 +0.00020136 +0.00022726 +0.00025482 +0.00028803 +0.00035181 +0.00014060 +0.00014713 +0.00015958 +0.00017668 +0.00019695 +0.00021932 +0.00024328 +0.00026913 +0.00030088 +0.00036437 +0.00017014 +0.00017580 +0.00018652 +0.00020136 +0.00021932 +0.00023959 +0.00026163 +0.00028564 +0.00031573 +0.00037863 +0.00020021 +0.00020503 +0.00021426 +0.00022726 +0.00024328 +0.00026163 +0.00028177 +0.00030387 +0.00033213 +0.00039413 +0.00023103 +0.00023522 +0.00024332 +0.00025482 +0.00026913 +0.00028564 +0.00030387 +0.00032403 +0.00035033 +0.00041110 +0.00026672 +0.00027047 +0.00027772 +0.00028803 +0.00030088 +0.00031573 +0.00033213 +0.00035033 +0.00037447 +0.00043361 +0.00033057 +0.00033434 +0.00034160 +0.00035181 +0.00036437 +0.00037863 +0.00039413 +0.00041110 +0.00043361 +0.00049053 +0.00013090 +0.00013899 +0.00015481 +0.00017725 +0.00020429 +0.00023376 +0.00026423 +0.00029544 +0.00033057 +0.00039161 +0.00013899 +0.00014702 +0.00016263 +0.00018458 +0.00021088 +0.00023953 +0.00026922 +0.00029973 +0.00033434 +0.00039547 +0.00015481 +0.00016263 +0.00017766 +0.00019853 +0.00022339 +0.00025051 +0.00027876 +0.00030797 +0.00034160 +0.00040283 +0.00017725 +0.00018458 +0.00019853 +0.00021776 +0.00024065 +0.00026576 +0.00029208 +0.00031955 +0.00035181 +0.00041307 +0.00020429 +0.00021088 +0.00022339 +0.00024065 +0.00026131 +0.00028416 +0.00030831 +0.00033376 +0.00036437 +0.00042545 +0.00023376 +0.00023953 +0.00025051 +0.00026576 +0.00028416 +0.00030469 +0.00032657 +0.00034987 +0.00037863 +0.00043925 +0.00026423 +0.00026922 +0.00027876 +0.00029208 +0.00030831 +0.00032657 +0.00034619 +0.00036734 +0.00039413 +0.00045394 +0.00029544 +0.00029973 +0.00030797 +0.00031955 +0.00033376 +0.00034987 +0.00036734 +0.00038637 +0.00041110 +0.00046972 +0.00033057 +0.00033434 +0.00034160 +0.00035181 +0.00036437 +0.00037863 +0.00039413 +0.00041110 +0.00043361 +0.00049053 +0.00039161 +0.00039547 +0.00040283 +0.00041307 +0.00042545 +0.00043925 +0.00045394 +0.00046972 +0.00049053 +0.00054478 + +FIELD FieldData 0 diff --git a/integrated-tests/Test-scripts/standard-results/SGH/Noh/vtk/outputs.vtk.series b/integrated-tests/Test-scripts/standard-results/SGH/Noh/vtk/outputs.vtk.series new file mode 100644 index 000000000..b77202e2c --- /dev/null +++ b/integrated-tests/Test-scripts/standard-results/SGH/Noh/vtk/outputs.vtk.series @@ -0,0 +1,6 @@ +{ + "file-series-version" : "1.0", + "files" : [ + { "name" : "data/VTK0.vtk", "time" : 6.00000e-01 }, + ] +} diff --git a/python/Test-scripts/standard-results/SGH/Sedov_12x12x12/vtk/data/VTK0.vtk b/integrated-tests/Test-scripts/standard-results/SGH/Sedov/vtk/data/VTK0.vtk similarity index 64% rename from python/Test-scripts/standard-results/SGH/Sedov_12x12x12/vtk/data/VTK0.vtk rename to integrated-tests/Test-scripts/standard-results/SGH/Sedov/vtk/data/VTK0.vtk index f5dc9b531..6214d319a 100644 --- a/python/Test-scripts/standard-results/SGH/Sedov_12x12x12/vtk/data/VTK0.vtk +++ b/integrated-tests/Test-scripts/standard-results/SGH/Sedov/vtk/data/VTK0.vtk @@ -5,154 +5,154 @@ DATASET UNSTRUCTURED_GRID POINTS 2197 float 0.00000000 0.00000000 0.00000000 -0.53776191 0.00000000 0.00000000 -0.60205274 0.00000000 0.00000000 -0.69594450 0.00000000 0.00000000 -0.77599848 0.00000000 0.00000000 -0.84746937 0.00000000 0.00000000 -0.91204792 0.00000000 0.00000000 -0.96962515 0.00000000 0.00000000 -1.01908611 0.00000000 0.00000000 -1.05975035 0.00000000 0.00000000 -1.10930635 0.00000000 0.00000000 -1.11637892 0.00000000 0.00000000 -1.20000000 0.00000000 0.00000000 -0.00000000 0.53776191 0.00000000 -0.44435254 0.44435254 0.00000000 -0.61155544 0.26939912 0.00000000 -0.69846681 0.22064358 0.00000000 -0.77688181 0.18976175 0.00000000 -0.84748313 0.16967335 0.00000000 -0.91018232 0.15517265 0.00000000 -0.96483786 0.14332892 0.00000000 -1.01357716 0.13287109 0.00000000 -1.05399342 0.12283560 0.00000000 -1.09851927 0.11378558 0.00000000 -1.11184016 0.10152648 0.00000000 -1.20000000 0.10000052 0.00000000 -0.00000000 0.60205274 0.00000000 -0.26939912 0.61155544 0.00000000 -0.54203263 0.54203263 0.00000000 -0.67307734 0.41180871 0.00000000 -0.75701522 0.35501801 0.00000000 -0.82700772 0.31714318 0.00000000 -0.88817242 0.29071456 0.00000000 -0.94191103 0.27030805 0.00000000 -0.98673239 0.25193669 0.00000000 -1.02993988 0.23617460 0.00000000 -1.06148154 0.21674437 0.00000000 -1.10311427 0.20081847 0.00000000 -1.20000000 0.20000021 0.00000000 -0.00000000 0.69594450 0.00000000 -0.22064358 0.69846681 0.00000000 -0.41180871 0.67307734 0.00000000 -0.60391464 0.60391464 0.00000000 -0.71080419 0.50149623 0.00000000 -0.78868751 0.44974512 0.00000000 -0.85270186 0.41230124 0.00000000 -0.90696809 0.38368634 0.00000000 -0.95438813 0.35904395 0.00000000 -0.99927356 0.33668601 0.00000000 -1.02383514 0.30867229 0.00000000 -1.10029021 0.30009933 0.00000000 -1.20000000 0.30000001 0.00000000 -0.00000000 0.77599848 0.00000000 -0.18976175 0.77688181 0.00000000 -0.35501801 0.75701522 0.00000000 -0.50149623 0.71080419 0.00000000 -0.64970821 0.64970821 0.00000000 -0.73928284 0.56858842 0.00000000 -0.81006029 0.52387144 0.00000000 -0.86770511 0.48767065 0.00000000 -0.92067108 0.45894538 0.00000000 -0.95881249 0.42696076 0.00000000 -1.00427903 0.40189866 0.00000000 -1.10000820 0.40000322 0.00000000 +0.49757855 0.00000000 0.00000000 +0.53107530 0.00000000 0.00000000 +0.61884298 0.00000000 0.00000000 +0.69317236 0.00000000 0.00000000 +0.76611578 0.00000000 0.00000000 +0.84297461 0.00000000 0.00000000 +0.89059622 0.00000000 0.00000000 +0.93342035 0.00000000 0.00000000 +0.98075140 0.00000000 0.00000000 +1.00853411 0.00000000 0.00000000 +1.10006517 0.00000000 0.00000000 +1.20000001 0.00000000 0.00000000 +0.00000000 0.49757855 0.00000000 +0.39595301 0.39595301 0.00000000 +0.55167805 0.27189577 0.00000000 +0.62894736 0.21693929 0.00000000 +0.70065996 0.17732266 0.00000000 +0.77112549 0.15482278 0.00000000 +0.84173053 0.14041137 0.00000000 +0.88520787 0.12735313 0.00000000 +0.93237960 0.11825295 0.00000000 +0.97137066 0.10846348 0.00000000 +1.00705897 0.10070084 0.00000000 +1.10005103 0.10000329 0.00000000 +1.20000001 0.10000000 0.00000000 +0.00000000 0.53107530 0.00000000 +0.27189577 0.55167805 0.00000000 +0.49481526 0.49481526 0.00000000 +0.61943218 0.39811627 0.00000000 +0.69528976 0.33499852 0.00000000 +0.76733124 0.29563371 0.00000000 +0.82443784 0.26937908 0.00000000 +0.86917115 0.24795578 0.00000000 +0.92090868 0.23188170 0.00000000 +0.94671483 0.21117043 0.00000000 +1.00382370 0.20075589 0.00000000 +1.10002348 0.20000311 0.00000000 +1.20000000 0.20000000 0.00000000 +0.00000000 0.61884298 0.00000000 +0.21693929 0.62894736 0.00000000 +0.39811627 0.61943218 0.00000000 +0.55444034 0.55444034 0.00000000 +0.66184942 0.47470537 0.00000000 +0.73656137 0.42358925 0.00000000 +0.79058389 0.38427682 0.00000000 +0.84610090 0.35875364 0.00000000 +0.89129645 0.33342199 0.00000000 +0.92259213 0.30763684 0.00000000 +1.00125619 0.30035799 0.00000000 +1.10000554 0.30000115 0.00000000 +1.20000000 0.30000000 0.00000000 +0.00000000 0.69317236 0.00000000 +0.17732266 0.70065996 0.00000000 +0.33499852 0.69528976 0.00000000 +0.47470537 0.66184942 0.00000000 +0.60527944 0.60527944 0.00000000 +0.69076838 0.53458256 0.00000000 +0.75510430 0.49155733 0.00000000 +0.81609925 0.45999413 0.00000000 +0.85278975 0.42453094 0.00000000 +0.90807622 0.40340321 0.00000000 +1.00024504 0.40008808 0.00000000 +1.10000051 0.40000015 0.00000000 1.20000000 0.40000000 0.00000000 -0.00000000 0.84746937 0.00000000 -0.16967335 0.84748313 0.00000000 -0.31714318 0.82700772 0.00000000 -0.44974512 0.78868751 0.00000000 -0.56858842 0.73928284 0.00000000 -0.68717236 0.68717236 0.00000000 -0.76182433 0.62063065 0.00000000 -0.82630414 0.58266570 0.00000000 -0.87929698 0.54716000 0.00000000 -0.91747243 0.50972929 0.00000000 -1.00030406 0.50015749 0.00000000 -1.10000004 0.50000002 0.00000000 +0.00000000 0.76611578 0.00000000 +0.15482278 0.77112549 0.00000000 +0.29563371 0.76733124 0.00000000 +0.42358925 0.73656137 0.00000000 +0.53458256 0.69076838 0.00000000 +0.63715933 0.63715933 0.00000000 +0.71794111 0.58791198 0.00000000 +0.77145885 0.54616444 0.00000000 +0.82018154 0.51151186 0.00000000 +0.90183755 0.50091570 0.00000000 +1.00002625 0.50001129 0.00000000 +1.10000002 0.50000001 0.00000000 1.20000000 0.50000000 0.00000000 -0.00000000 0.91204792 0.00000000 -0.15517265 0.91018232 0.00000000 -0.29071456 0.88817242 0.00000000 -0.41230124 0.85270186 0.00000000 -0.52387144 0.81006029 0.00000000 -0.62063065 0.76182433 0.00000000 -0.71717747 0.71717747 0.00000000 -0.78380908 0.66428258 0.00000000 -0.82986250 0.62120277 0.00000000 -0.90158256 0.60103535 0.00000000 -1.00000345 0.60000199 0.00000000 +0.00000000 0.84297461 0.00000000 +0.14041137 0.84173053 0.00000000 +0.26937908 0.82443784 0.00000000 +0.38427682 0.79058389 0.00000000 +0.49155733 0.75510430 0.00000000 +0.58791198 0.71794111 0.00000000 +0.67035303 0.67035303 0.00000000 +0.72785573 0.62173687 0.00000000 +0.80468955 0.60317337 0.00000000 +0.90020391 0.60011921 0.00000000 +1.00000113 0.60000057 0.00000000 1.10000000 0.60000000 0.00000000 1.20000000 0.60000000 0.00000000 -0.00000000 0.96962515 0.00000000 -0.14332892 0.96483786 0.00000000 -0.27030805 0.94191103 0.00000000 -0.38368634 0.90696809 0.00000000 -0.48767065 0.86770511 0.00000000 -0.58266570 0.82630414 0.00000000 -0.66428258 0.78380908 0.00000000 -0.73913964 0.73913964 0.00000000 -0.80329722 0.70269171 0.00000000 -0.90002831 0.70002079 0.00000000 -1.00000000 0.70000000 0.00000000 +0.00000000 0.89059622 0.00000000 +0.12735313 0.88520787 0.00000000 +0.24795578 0.86917115 0.00000000 +0.35875364 0.84610090 0.00000000 +0.45999413 0.81609925 0.00000000 +0.54616444 0.77145885 0.00000000 +0.62173687 0.72785573 0.00000000 +0.70606151 0.70606151 0.00000000 +0.80048441 0.70038645 0.00000000 +0.90000869 0.70000592 0.00000000 +1.00000001 0.70000001 0.00000000 1.10000000 0.70000000 0.00000000 1.20000000 0.70000000 0.00000000 -0.00000000 1.01908611 0.00000000 -0.13287109 1.01357716 0.00000000 -0.25193669 0.98673239 0.00000000 -0.35904395 0.95438813 0.00000000 -0.45894538 0.92067108 0.00000000 -0.54716000 0.87929698 0.00000000 -0.62120277 0.82986250 0.00000000 -0.70269171 0.80329722 0.00000000 -0.80011294 0.80011294 0.00000000 -0.90000003 0.80000002 0.00000000 +0.00000000 0.93342035 0.00000000 +0.11825295 0.93237960 0.00000000 +0.23188170 0.92090868 0.00000000 +0.33342199 0.89129645 0.00000000 +0.42453094 0.85278975 0.00000000 +0.51151186 0.82018154 0.00000000 +0.60317337 0.80468955 0.00000000 +0.70038645 0.80048441 0.00000000 +0.80001728 0.80001728 0.00000000 +0.90000008 0.80000006 0.00000000 1.00000000 0.80000000 0.00000000 1.10000000 0.80000000 0.00000000 1.20000000 0.80000000 0.00000000 -0.00000000 1.05975035 0.00000000 -0.12283560 1.05399342 0.00000000 -0.23617460 1.02993988 0.00000000 -0.33668601 0.99927356 0.00000000 -0.42696076 0.95881249 0.00000000 -0.50972929 0.91747243 0.00000000 -0.60103535 0.90158256 0.00000000 -0.70002079 0.90002831 0.00000000 -0.80000002 0.90000003 0.00000000 +0.00000000 0.98075140 0.00000000 +0.10846348 0.97137066 0.00000000 +0.21117043 0.94671483 0.00000000 +0.30763684 0.92259213 0.00000000 +0.40340321 0.90807622 0.00000000 +0.50091570 0.90183755 0.00000000 +0.60011921 0.90020391 0.00000000 +0.70000592 0.90000869 0.00000000 +0.80000006 0.90000008 0.00000000 0.90000000 0.90000000 0.00000000 1.00000000 0.90000000 0.00000000 1.10000000 0.90000000 0.00000000 1.20000000 0.90000000 0.00000000 -0.00000000 1.10930635 0.00000000 -0.11378558 1.09851927 0.00000000 -0.21674437 1.06148154 0.00000000 -0.30867229 1.02383514 0.00000000 -0.40189866 1.00427903 0.00000000 -0.50015749 1.00030406 0.00000000 -0.60000199 1.00000345 0.00000000 -0.70000000 1.00000000 0.00000000 +0.00000000 1.00853411 0.00000000 +0.10070084 1.00705897 0.00000000 +0.20075589 1.00382370 0.00000000 +0.30035799 1.00125619 0.00000000 +0.40008808 1.00024504 0.00000000 +0.50001129 1.00002625 0.00000000 +0.60000057 1.00000113 0.00000000 +0.70000001 1.00000001 0.00000000 0.80000000 1.00000000 0.00000000 0.90000000 1.00000000 0.00000000 1.00000000 1.00000000 0.00000000 1.10000000 1.00000000 0.00000000 1.20000000 1.00000000 0.00000000 -0.00000000 1.11637892 0.00000000 -0.10152648 1.11184016 0.00000000 -0.20081847 1.10311427 0.00000000 -0.30009933 1.10029021 0.00000000 -0.40000322 1.10000820 0.00000000 -0.50000002 1.10000004 0.00000000 +0.00000000 1.10006517 0.00000000 +0.10000329 1.10005103 0.00000000 +0.20000311 1.10002348 0.00000000 +0.30000115 1.10000554 0.00000000 +0.40000015 1.10000051 0.00000000 +0.50000001 1.10000002 0.00000000 0.60000000 1.10000000 0.00000000 0.70000000 1.10000000 0.00000000 0.80000000 1.10000000 0.00000000 @@ -160,10 +160,10 @@ POINTS 2197 float 1.00000000 1.10000000 0.00000000 1.10000000 1.10000000 0.00000000 1.20000000 1.10000000 0.00000000 -0.00000000 1.20000000 0.00000000 -0.10000052 1.20000000 0.00000000 -0.20000021 1.20000000 0.00000000 -0.30000001 1.20000000 0.00000000 +0.00000000 1.20000001 0.00000000 +0.10000000 1.20000001 0.00000000 +0.20000000 1.20000000 0.00000000 +0.30000000 1.20000000 0.00000000 0.40000000 1.20000000 0.00000000 0.50000000 1.20000000 0.00000000 0.60000000 1.20000000 0.00000000 @@ -173,155 +173,155 @@ POINTS 2197 float 1.00000000 1.20000000 0.00000000 1.10000000 1.20000000 0.00000000 1.20000000 1.20000000 0.00000000 -0.00000000 0.00000000 0.53776191 -0.44435254 0.00000000 0.44435254 -0.61155544 0.00000000 0.26939912 -0.69846681 0.00000000 0.22064358 -0.77688181 0.00000000 0.18976175 -0.84748313 0.00000000 0.16967335 -0.91018232 0.00000000 0.15517265 -0.96483786 0.00000000 0.14332892 -1.01357716 0.00000000 0.13287109 -1.05399342 0.00000000 0.12283560 -1.09851927 0.00000000 0.11378558 -1.11184016 0.00000000 0.10152648 -1.20000000 0.00000000 0.10000052 -0.00000000 0.44435254 0.44435254 -0.40158516 0.40158516 0.40158516 -0.60529265 0.27963723 0.27963723 -0.69636507 0.21528261 0.21528261 -0.77402488 0.18567451 0.18567451 -0.84371201 0.16592356 0.16592356 -0.90540812 0.15186355 0.15186355 -0.95897612 0.14058988 0.14058988 -1.00656530 0.13066508 0.13066508 -1.04745861 0.12143171 0.12143171 -1.08708267 0.11202892 0.11202892 -1.10834589 0.10110022 0.10110022 -1.20000000 0.10000032 0.10000032 -0.00000000 0.61155544 0.26939912 -0.27963723 0.60529265 0.27963723 -0.52500961 0.52500961 0.25920378 -0.66381197 0.41240417 0.21097409 -0.74994845 0.35105276 0.17683110 -0.82057257 0.31369245 0.15782758 -0.88178372 0.28735387 0.14416070 -0.93524877 0.26714900 0.13370188 -0.98023176 0.24916134 0.12455660 -1.02428904 0.23383545 0.11668098 -1.05283153 0.21405368 0.10675696 -1.10216182 0.20056038 0.10028399 -1.20000000 0.20000013 0.10000006 -0.00000000 0.69846681 0.22064358 -0.21528261 0.69636507 0.21528261 -0.41240417 0.66381197 0.21097409 -0.59221847 0.59221847 0.19712823 -0.70343685 0.50080222 0.17093147 -0.78235981 0.44678136 0.15018738 -0.84712657 0.40975168 0.13725219 -0.90155981 0.38108807 0.12715273 -0.94953048 0.35679227 0.11879272 -0.99386503 0.33427106 0.11124799 -1.01968452 0.30706878 0.10223985 -1.10020126 0.30006789 0.10002269 +0.00000000 0.00000000 0.49757855 +0.39595301 0.00000000 0.39595301 +0.55167805 0.00000000 0.27189577 +0.62894736 0.00000000 0.21693929 +0.70065996 0.00000000 0.17732266 +0.77112549 0.00000000 0.15482278 +0.84173053 0.00000000 0.14041137 +0.88520787 0.00000000 0.12735313 +0.93237960 0.00000000 0.11825295 +0.97137066 0.00000000 0.10846348 +1.00705897 0.00000000 0.10070084 +1.10005103 0.00000000 0.10000329 +1.20000001 0.00000000 0.10000000 +0.00000000 0.39595301 0.39595301 +0.36204725 0.36204725 0.36204725 +0.54757854 0.27020745 0.27020745 +0.63256292 0.20971718 0.20971718 +0.70344694 0.17371925 0.17371925 +0.77310595 0.15224945 0.15224945 +0.83795977 0.13856589 0.13856589 +0.88017760 0.12624382 0.12624382 +0.93016718 0.11763216 0.11763216 +0.96253456 0.10740242 0.10740242 +1.00581860 0.10057397 0.10057397 +1.10003983 0.10000256 0.10000256 +1.20000000 0.10000000 0.10000000 +0.00000000 0.55167805 0.27189577 +0.27020745 0.54757854 0.27020745 +0.48185028 0.48185028 0.24243851 +0.61142471 0.39371379 0.19828294 +0.69136043 0.33046720 0.16465710 +0.76292351 0.29267929 0.14608580 +0.81793874 0.26639851 0.13302817 +0.86468341 0.24601063 0.12275667 +0.91558385 0.22990962 0.11471946 +0.94064722 0.20960508 0.10468196 +1.00312991 0.20061325 0.10029978 +1.10001815 0.20000240 0.10000116 +1.20000000 0.20000000 0.10000000 +0.00000000 0.62894736 0.21693929 +0.20971718 0.63256292 0.20971718 +0.39371379 0.61142471 0.19828294 +0.54733782 0.54733782 0.18047772 +0.65596314 0.47231988 0.15704777 +0.73097090 0.42005042 0.13907776 +0.78536930 0.38158277 0.12674756 +0.84241872 0.35677352 0.11845901 +0.88558560 0.33096079 0.11001986 +0.91972682 0.30659998 0.10213886 +1.00102408 0.30028977 0.10009212 +1.10000422 0.30000088 0.10000026 1.20000000 0.30000000 0.10000000 -0.00000000 0.77688181 0.18976175 -0.18567451 0.77402488 0.18567451 -0.35105276 0.74994845 0.17683110 -0.50080222 0.70343685 0.17093147 -0.64119052 0.64119052 0.16128471 -0.73369463 0.56719040 0.14458578 -0.80467853 0.52099842 0.13108583 -0.86286900 0.48524650 0.12165362 -0.91652877 0.45674548 0.11417539 -0.95313141 0.42424467 0.10595209 -1.00335672 0.40148402 0.10037558 -1.10000555 0.40000217 0.10000050 +0.00000000 0.70065996 0.17732266 +0.17371925 0.70344694 0.17371925 +0.33046720 0.69136043 0.16465710 +0.47231988 0.65596314 0.15704777 +0.59909700 0.59909700 0.14710687 +0.68503398 0.53184308 0.13232192 +0.75150883 0.48917949 0.12139215 +0.81160479 0.45739004 0.11379863 +0.84822886 0.42230988 0.10530144 +0.90700443 0.40293226 0.10070420 +1.00019883 0.40007112 0.10001649 +1.10000039 0.40000011 0.10000002 1.20000000 0.40000000 0.10000000 -0.00000000 0.84748313 0.16967335 -0.16592356 0.84371201 0.16592356 -0.31369245 0.82057257 0.15782758 -0.44678136 0.78235981 0.15018738 -0.56719040 0.73369463 0.14458578 -0.68053012 0.68053012 0.13738574 -0.75739272 0.61883699 0.12555538 -0.82242886 0.58018583 0.11646131 -0.87441298 0.54424669 0.10890351 -0.91472437 0.50818514 0.10161500 -1.00022657 0.50011671 0.10002438 -1.10000003 0.50000001 0.10000000 +0.00000000 0.77112549 0.15482278 +0.15224945 0.77310595 0.15224945 +0.29267929 0.76292351 0.14608580 +0.42005042 0.73097090 0.13907776 +0.53184308 0.68503398 0.13232192 +0.63335093 0.63335093 0.12512370 +0.71422441 0.58590873 0.11651969 +0.76715562 0.54323430 0.10805615 +0.81808075 0.51027912 0.10195059 +0.90155996 0.50077499 0.10014558 +1.00002108 0.50000903 0.10000158 +1.10000001 0.50000000 0.10000000 1.20000000 0.50000000 0.10000000 -0.00000000 0.91018232 0.15517265 -0.15186355 0.90540812 0.15186355 -0.28735387 0.88178372 0.14416070 -0.40975168 0.84712657 0.13725219 -0.52099842 0.80467853 0.13108583 -0.61883699 0.75739272 0.12555538 -0.71254499 0.71254499 0.11987896 -0.78003954 0.66232491 0.11125308 -0.82642640 0.61873750 0.10310482 -0.90125052 0.60081694 0.10014389 -1.00000241 0.60000138 0.10000021 +0.00000000 0.84173053 0.14041137 +0.13856589 0.83795977 0.13856589 +0.26639851 0.81793874 0.13302817 +0.38158277 0.78536930 0.12674756 +0.48917949 0.75150883 0.12139215 +0.58590873 0.71422441 0.11651969 +0.66597252 0.66597252 0.11012756 +0.72544089 0.61996228 0.10309869 +0.80410674 0.60277408 0.10043095 +0.90016967 0.60009895 0.10001488 +1.00000089 0.60000045 0.10000006 1.10000000 0.60000000 0.10000000 1.20000000 0.60000000 0.10000000 -0.00000000 0.96483786 0.14332892 -0.14058988 0.95897612 0.14058988 -0.26714900 0.93524877 0.13370188 -0.38108807 0.90155981 0.12715273 -0.48524650 0.86286900 0.12165362 -0.58018583 0.82242886 0.11646131 -0.66232491 0.78003954 0.11125308 -0.73398435 0.73398435 0.10513418 -0.80271483 0.70223247 0.10034158 -0.90002082 0.70001527 0.10000221 -1.00000000 0.70000000 0.10000000 +0.00000000 0.88520787 0.12735313 +0.12624382 0.88017760 0.12624382 +0.24601063 0.86468341 0.12275667 +0.35677352 0.84241872 0.11845901 +0.45739004 0.81160479 0.11379863 +0.54323430 0.76715562 0.10805615 +0.61996228 0.72544089 0.10309869 +0.70529997 0.70529997 0.10069657 +0.80041163 0.70032903 0.10004229 +0.90000706 0.70000480 0.10000056 +1.00000001 0.70000001 0.10000000 1.10000000 0.70000000 0.10000000 1.20000000 0.70000000 0.10000000 -0.00000000 1.01357716 0.13287109 -0.13066508 1.00656530 0.13066508 -0.24916134 0.98023176 0.12455660 -0.35679227 0.94953048 0.11879272 -0.45674548 0.91652877 0.11417539 -0.54424669 0.87441298 0.10890351 -0.61873750 0.82642640 0.10310482 -0.70223247 0.80271483 0.10034158 -0.80007988 0.80007988 0.10001129 -0.90000002 0.80000002 0.10000000 +0.00000000 0.93237960 0.11825295 +0.11763216 0.93016718 0.11763216 +0.22990962 0.91558385 0.11471946 +0.33096079 0.88558560 0.11001986 +0.42230988 0.84822886 0.10530144 +0.51027912 0.81808075 0.10195059 +0.60277408 0.80410674 0.10043095 +0.70032903 0.80041163 0.10004229 +0.80001404 0.80001404 0.10000141 +0.90000006 0.80000005 0.10000000 1.00000000 0.80000000 0.10000000 1.10000000 0.80000000 0.10000000 1.20000000 0.80000000 0.10000000 -0.00000000 1.05399342 0.12283560 -0.12143171 1.04745861 0.12143171 -0.23383545 1.02428904 0.11668098 -0.33427106 0.99386503 0.11124799 -0.42424467 0.95313141 0.10595209 -0.50818514 0.91472437 0.10161500 -0.60081694 0.90125052 0.10014389 -0.70001527 0.90002082 0.10000221 -0.80000002 0.90000002 0.10000000 +0.00000000 0.97137066 0.10846348 +0.10740242 0.96253456 0.10740242 +0.20960508 0.94064722 0.10468196 +0.30659998 0.91972682 0.10213886 +0.40293226 0.90700443 0.10070420 +0.50077499 0.90155996 0.10014558 +0.60009895 0.90016967 0.10001488 +0.70000480 0.90000706 0.10000056 +0.80000005 0.90000006 0.10000000 0.90000000 0.90000000 0.10000000 1.00000000 0.90000000 0.10000000 1.10000000 0.90000000 0.10000000 1.20000000 0.90000000 0.10000000 -0.00000000 1.09851927 0.11378558 -0.11202892 1.08708267 0.11202892 -0.21405368 1.05283153 0.10675696 -0.30706878 1.01968452 0.10223985 -0.40148402 1.00335672 0.10037558 -0.50011671 1.00022657 0.10002438 -0.60000138 1.00000241 0.10000021 -0.70000000 1.00000000 0.10000000 +0.00000000 1.00705897 0.10070084 +0.10057397 1.00581860 0.10057397 +0.20061325 1.00312991 0.10029978 +0.30028977 1.00102408 0.10009212 +0.40007112 1.00019883 0.10001649 +0.50000903 1.00002108 0.10000158 +0.60000045 1.00000089 0.10000006 +0.70000001 1.00000001 0.10000000 0.80000000 1.00000000 0.10000000 0.90000000 1.00000000 0.10000000 1.00000000 1.00000000 0.10000000 1.10000000 1.00000000 0.10000000 1.20000000 1.00000000 0.10000000 -0.00000000 1.11184016 0.10152648 -0.10110022 1.10834589 0.10110022 -0.20056038 1.10216182 0.10028399 -0.30006789 1.10020126 0.10002269 -0.40000217 1.10000555 0.10000050 -0.50000001 1.10000003 0.10000000 +0.00000000 1.10005103 0.10000329 +0.10000256 1.10003983 0.10000256 +0.20000240 1.10001815 0.10000116 +0.30000088 1.10000422 0.10000026 +0.40000011 1.10000039 0.10000002 +0.50000000 1.10000001 0.10000000 0.60000000 1.10000000 0.10000000 0.70000000 1.10000000 0.10000000 0.80000000 1.10000000 0.10000000 @@ -329,9 +329,9 @@ POINTS 2197 float 1.00000000 1.10000000 0.10000000 1.10000000 1.10000000 0.10000000 1.20000000 1.10000000 0.10000000 -0.00000000 1.20000000 0.10000052 -0.10000032 1.20000000 0.10000032 -0.20000013 1.20000000 0.10000006 +0.00000000 1.20000001 0.10000000 +0.10000000 1.20000000 0.10000000 +0.20000000 1.20000000 0.10000000 0.30000000 1.20000000 0.10000000 0.40000000 1.20000000 0.10000000 0.50000000 1.20000000 0.10000000 @@ -342,154 +342,154 @@ POINTS 2197 float 1.00000000 1.20000000 0.10000000 1.10000000 1.20000000 0.10000000 1.20000000 1.20000000 0.10000000 -0.00000000 0.00000000 0.60205274 -0.26939912 0.00000000 0.61155544 -0.54203263 0.00000000 0.54203263 -0.67307734 0.00000000 0.41180871 -0.75701522 0.00000000 0.35501801 -0.82700772 0.00000000 0.31714318 -0.88817242 0.00000000 0.29071456 -0.94191103 0.00000000 0.27030805 -0.98673239 0.00000000 0.25193669 -1.02993988 0.00000000 0.23617460 -1.06148154 0.00000000 0.21674437 -1.10311427 0.00000000 0.20081847 -1.20000000 0.00000000 0.20000021 -0.00000000 0.26939912 0.61155544 -0.27963723 0.27963723 0.60529265 -0.52500961 0.25920378 0.52500961 -0.66381197 0.21097409 0.41240417 -0.74994845 0.17683110 0.35105276 -0.82057257 0.15782758 0.31369245 -0.88178372 0.14416070 0.28735387 -0.93524877 0.13370188 0.26714900 -0.98023176 0.12455660 0.24916134 -1.02428904 0.11668098 0.23383545 -1.05283153 0.10675696 0.21405368 -1.10216182 0.10028399 0.20056038 -1.20000000 0.10000006 0.20000013 -0.00000000 0.54203263 0.54203263 -0.25920378 0.52500961 0.52500961 -0.46801317 0.46801317 0.46801317 -0.62471435 0.40259738 0.40259738 -0.72359708 0.34252623 0.34252623 -0.79895918 0.30473274 0.30473274 -0.86136115 0.27879357 0.27879357 -0.91526121 0.25903270 0.25903270 -0.96161127 0.24190248 0.24190248 -1.00671014 0.22701033 0.22701033 -1.03110077 0.20766246 0.20766246 -1.10062140 0.20015072 0.20015072 -1.20000000 0.20000002 0.20000002 -0.00000000 0.67307734 0.41180871 -0.21097409 0.66381197 0.41240417 -0.40259738 0.62471435 0.40259738 -0.55777243 0.55777243 0.37130158 -0.67799702 0.49245949 0.33036339 -0.76188336 0.43867714 0.29400223 -0.82950195 0.40204041 0.26861035 -0.88510792 0.37365783 0.24924309 -0.93512570 0.35040098 0.23356272 -0.97657491 0.32704436 0.21784235 -1.01034139 0.30359058 0.20235762 -1.10006116 0.30001958 0.20001291 +0.00000000 0.00000000 0.53107530 +0.27189577 0.00000000 0.55167805 +0.49481526 0.00000000 0.49481526 +0.61943218 0.00000000 0.39811627 +0.69528976 0.00000000 0.33499852 +0.76733124 0.00000000 0.29563371 +0.82443784 0.00000000 0.26937908 +0.86917115 0.00000000 0.24795578 +0.92090868 0.00000000 0.23188170 +0.94671483 0.00000000 0.21117043 +1.00382370 0.00000000 0.20075589 +1.10002348 0.00000000 0.20000311 +1.20000000 0.00000000 0.20000000 +0.00000000 0.27189577 0.55167805 +0.27020745 0.27020745 0.54757854 +0.48185028 0.24243851 0.48185028 +0.61142471 0.19828294 0.39371379 +0.69136043 0.16465710 0.33046720 +0.76292351 0.14608580 0.29267929 +0.81793874 0.13302817 0.26639851 +0.86468341 0.12275667 0.24601063 +0.91558385 0.11471946 0.22990962 +0.94064722 0.10468196 0.20960508 +1.00312991 0.10029978 0.20061325 +1.10001815 0.10000116 0.20000240 +1.20000000 0.10000000 0.20000000 +0.00000000 0.49481526 0.49481526 +0.24243851 0.48185028 0.48185028 +0.44155850 0.44155850 0.44155850 +0.57760136 0.37749005 0.37749005 +0.67259986 0.32015320 0.32015320 +0.74546064 0.28439047 0.28439047 +0.79825205 0.25841182 0.25841182 +0.85129881 0.24060773 0.24060773 +0.89766595 0.22397871 0.22397871 +0.92664749 0.20605740 0.20605740 +1.00166449 0.20031639 0.20031639 +1.10000805 0.20000106 0.20000106 +1.20000000 0.20000000 0.20000000 +0.00000000 0.61943218 0.39811627 +0.19828294 0.61142471 0.39371379 +0.37749005 0.57760136 0.37749005 +0.52393740 0.52393740 0.34700724 +0.63610900 0.46257597 0.30713688 +0.71271826 0.41022707 0.27273859 +0.77084406 0.37440838 0.24904879 +0.83067969 0.35085869 0.23341483 +0.86874877 0.32412620 0.21581865 +0.91300074 0.30421784 0.20277513 +1.00054285 0.30015002 0.20009743 +1.10000181 0.30000037 0.20000023 1.20000000 0.30000000 0.20000000 -0.00000000 0.75701522 0.35501801 -0.17683110 0.74994845 0.35105276 -0.34252623 0.72359708 0.34252623 -0.49245949 0.67799702 0.33036339 -0.61746997 0.61746997 0.30925925 -0.71543727 0.55957630 0.28223638 -0.78816087 0.51279397 0.25768821 -0.84899828 0.47825791 0.23944822 -0.90311681 0.44979284 0.22492296 -0.93777007 0.41704032 0.20834384 -1.00154830 0.40067510 0.20034203 -1.10000159 0.40000060 0.20000028 +0.00000000 0.69528976 0.33499852 +0.16465710 0.69136043 0.33046720 +0.32015320 0.67259986 0.32015320 +0.46257597 0.63610900 0.30713688 +0.58084527 0.58084527 0.28632340 +0.66864949 0.52310454 0.26001908 +0.74058418 0.48258494 0.24011698 +0.79756131 0.44954991 0.22403142 +0.83612377 0.41650187 0.20796403 +0.90449165 0.40184411 0.20089456 +1.00010437 0.40003672 0.20001732 +1.10000017 0.40000005 0.20000002 1.20000000 0.40000000 0.20000000 -0.00000000 0.82700772 0.31714318 -0.15782758 0.82057257 0.31369245 -0.30473274 0.79895918 0.30473274 -0.43867714 0.76188336 0.29400223 -0.55957630 0.71543727 0.28223638 -0.66254081 0.66254081 0.26630201 -0.74412487 0.61203591 0.24659868 -0.81100840 0.57321991 0.22993157 -0.85996942 0.53560066 0.21419343 -0.90841516 0.50466014 0.20186981 -1.00008969 0.50004540 0.20001849 +0.00000000 0.76733124 0.29563371 +0.14608580 0.76292351 0.29267929 +0.28439047 0.74546064 0.28439047 +0.41022707 0.71271826 0.27273859 +0.52310454 0.66864949 0.26001908 +0.62258026 0.62258026 0.24621977 +0.70243248 0.57852225 0.22996354 +0.75465658 0.53498985 0.21320089 +0.81284737 0.50723678 0.20277440 +0.90093920 0.50046210 0.20017523 +1.00001074 0.50000454 0.20000162 1.10000001 0.50000000 0.20000000 1.20000000 0.50000000 0.20000000 -0.00000000 0.88817242 0.29071456 -0.14416070 0.88178372 0.28735387 -0.27879357 0.86136115 0.27879357 -0.40204041 0.82950195 0.26861035 -0.51279397 0.78816087 0.25768821 -0.61203591 0.74412487 0.24659868 -0.69988798 0.69988798 0.23467848 -0.76772981 0.65451792 0.21891426 -0.81767238 0.61254377 0.20417332 -0.90059787 0.60038862 0.20013549 -1.00000077 0.60000044 0.20000014 +0.00000000 0.82443784 0.26937908 +0.13302817 0.81793874 0.26639851 +0.25841182 0.79825205 0.25841182 +0.37440838 0.77084406 0.24904879 +0.48258494 0.74058418 0.24011698 +0.57852225 0.70243248 0.22996354 +0.65392762 0.65392762 0.21648949 +0.71907392 0.61515505 0.20473370 +0.80271309 0.60182509 0.20057061 +0.90009639 0.60005581 0.20001690 +1.00000043 0.60000021 0.20000006 1.10000000 0.60000000 0.20000000 1.20000000 0.60000000 0.20000000 -0.00000000 0.94191103 0.27030805 -0.13370188 0.93524877 0.26714900 -0.25903270 0.91526121 0.25903270 -0.37365783 0.88510792 0.24924309 -0.47825791 0.84899828 0.23944822 -0.57321991 0.81100840 0.22993157 -0.65451792 0.76772981 0.21891426 -0.72157041 0.72157041 0.20635303 -0.80144842 0.70121158 0.20037105 -0.90000795 0.70000581 0.20000164 +0.00000000 0.86917115 0.24795578 +0.12275667 0.86468341 0.24601063 +0.24060773 0.85129881 0.24060773 +0.35085869 0.83067969 0.23341483 +0.44954991 0.79756131 0.22403142 +0.53498985 0.75465658 0.21320089 +0.61515505 0.71907392 0.20473370 +0.70350656 0.70350656 0.20091722 +0.80024881 0.70019990 0.20005147 +0.90000372 0.70000252 0.20000060 1.00000000 0.70000000 0.20000000 1.10000000 0.70000000 0.20000000 1.20000000 0.70000000 0.20000000 -0.00000000 0.98673239 0.25193669 -0.12455660 0.98023176 0.24916134 -0.24190248 0.96161127 0.24190248 -0.35040098 0.93512570 0.23356272 -0.44979284 0.90311681 0.22492296 -0.53560066 0.85996942 0.21419343 -0.61254377 0.81767238 0.20417332 -0.70121158 0.80144842 0.20037105 -0.80002681 0.80002681 0.20000706 -0.90000001 0.80000001 0.20000000 +0.00000000 0.92090868 0.23188170 +0.11471946 0.91558385 0.22990962 +0.22397871 0.89766595 0.22397871 +0.32412620 0.86874877 0.21581865 +0.41650187 0.83612377 0.20796403 +0.50723678 0.81284737 0.20277440 +0.60182509 0.80271309 0.20057061 +0.70019990 0.80024881 0.20005147 +0.80000742 0.80000742 0.20000150 +0.90000003 0.80000002 0.20000000 1.00000000 0.80000000 0.20000000 1.10000000 0.80000000 0.20000000 1.20000000 0.80000000 0.20000000 -0.00000000 1.02993988 0.23617460 -0.11668098 1.02428904 0.23383545 -0.22701033 1.00671014 0.22701033 -0.32704436 0.97657491 0.21784235 -0.41704032 0.93777007 0.20834384 -0.50466014 0.90841516 0.20186981 -0.60038862 0.90059787 0.20013549 -0.70000581 0.90000795 0.20000164 -0.80000001 0.90000001 0.20000000 +0.00000000 0.94671483 0.21117043 +0.10468196 0.94064722 0.20960508 +0.20605740 0.92664749 0.20605740 +0.30421784 0.91300074 0.20277513 +0.40184411 0.90449165 0.20089456 +0.50046210 0.90093920 0.20017523 +0.60005581 0.90009639 0.20001690 +0.70000252 0.90000372 0.20000060 +0.80000002 0.90000003 0.20000000 0.90000000 0.90000000 0.20000000 1.00000000 0.90000000 0.20000000 1.10000000 0.90000000 0.20000000 1.20000000 0.90000000 0.20000000 -0.00000000 1.06148154 0.21674437 -0.10675696 1.05283153 0.21405368 -0.20766246 1.03110077 0.20766246 -0.30359058 1.01034139 0.20235762 -0.40067510 1.00154830 0.20034203 -0.50004540 1.00008969 0.20001849 -0.60000044 1.00000077 0.20000014 +0.00000000 1.00382370 0.20075589 +0.10029978 1.00312991 0.20061325 +0.20031639 1.00166449 0.20031639 +0.30015002 1.00054285 0.20009743 +0.40003672 1.00010437 0.20001732 +0.50000454 1.00001074 0.20000162 +0.60000021 1.00000043 0.20000006 0.70000000 1.00000000 0.20000000 0.80000000 1.00000000 0.20000000 0.90000000 1.00000000 0.20000000 1.00000000 1.00000000 0.20000000 1.10000000 1.00000000 0.20000000 1.20000000 1.00000000 0.20000000 -0.00000000 1.10311427 0.20081847 -0.10028399 1.10216182 0.20056038 -0.20015072 1.10062140 0.20015072 -0.30001958 1.10006116 0.20001291 -0.40000060 1.10000159 0.20000028 +0.00000000 1.10002348 0.20000311 +0.10000116 1.10001815 0.20000240 +0.20000106 1.10000805 0.20000106 +0.30000037 1.10000181 0.20000023 +0.40000005 1.10000017 0.20000002 0.50000000 1.10000001 0.20000000 0.60000000 1.10000000 0.20000000 0.70000000 1.10000000 0.20000000 @@ -498,9 +498,9 @@ POINTS 2197 float 1.00000000 1.10000000 0.20000000 1.10000000 1.10000000 0.20000000 1.20000000 1.10000000 0.20000000 -0.00000000 1.20000000 0.20000021 -0.10000006 1.20000000 0.20000013 -0.20000002 1.20000000 0.20000002 +0.00000000 1.20000000 0.20000000 +0.10000000 1.20000000 0.20000000 +0.20000000 1.20000000 0.20000000 0.30000000 1.20000000 0.20000000 0.40000000 1.20000000 0.20000000 0.50000000 1.20000000 0.20000000 @@ -511,154 +511,154 @@ POINTS 2197 float 1.00000000 1.20000000 0.20000000 1.10000000 1.20000000 0.20000000 1.20000000 1.20000000 0.20000000 -0.00000000 0.00000000 0.69594450 -0.22064358 0.00000000 0.69846681 -0.41180871 0.00000000 0.67307734 -0.60391464 0.00000000 0.60391464 -0.71080419 0.00000000 0.50149623 -0.78868751 0.00000000 0.44974512 -0.85270186 0.00000000 0.41230124 -0.90696809 0.00000000 0.38368634 -0.95438813 0.00000000 0.35904395 -0.99927356 0.00000000 0.33668601 -1.02383514 0.00000000 0.30867229 -1.10029021 0.00000000 0.30009933 -1.20000000 0.00000000 0.30000001 -0.00000000 0.22064358 0.69846681 -0.21528261 0.21528261 0.69636507 -0.41240417 0.21097409 0.66381197 -0.59221847 0.19712823 0.59221847 -0.70343685 0.17093147 0.50080222 -0.78235981 0.15018738 0.44678136 -0.84712657 0.13725219 0.40975168 -0.90155981 0.12715273 0.38108807 -0.94953048 0.11879272 0.35679227 -0.99386503 0.11124799 0.33427106 -1.01968452 0.10223985 0.30706878 -1.10020126 0.10002269 0.30006789 +0.00000000 0.00000000 0.61884298 +0.21693929 0.00000000 0.62894736 +0.39811627 0.00000000 0.61943218 +0.55444034 0.00000000 0.55444034 +0.66184942 0.00000000 0.47470537 +0.73656137 0.00000000 0.42358925 +0.79058389 0.00000000 0.38427682 +0.84610090 0.00000000 0.35875364 +0.89129645 0.00000000 0.33342199 +0.92259213 0.00000000 0.30763684 +1.00125619 0.00000000 0.30035799 +1.10000554 0.00000000 0.30000115 +1.20000000 0.00000000 0.30000000 +0.00000000 0.21693929 0.62894736 +0.20971718 0.20971718 0.63256292 +0.39371379 0.19828294 0.61142471 +0.54733782 0.18047772 0.54733782 +0.65596314 0.15704777 0.47231988 +0.73097090 0.13907776 0.42005042 +0.78536930 0.12674756 0.38158277 +0.84241872 0.11845901 0.35677352 +0.88558560 0.11001986 0.33096079 +0.91972682 0.10213886 0.30659998 +1.00102408 0.10009212 0.30028977 +1.10000422 0.10000026 0.30000088 1.20000000 0.10000000 0.30000000 -0.00000000 0.41180871 0.67307734 -0.21097409 0.41240417 0.66381197 -0.40259738 0.40259738 0.62471435 -0.55777243 0.37130158 0.55777243 -0.67799702 0.33036339 0.49245949 -0.76188336 0.29400223 0.43867714 -0.82950195 0.26861035 0.40204041 -0.88510792 0.24924309 0.37365783 -0.93512570 0.23356272 0.35040098 -0.97657491 0.21784235 0.32704436 -1.01034139 0.20235762 0.30359058 -1.10006116 0.20001291 0.30001958 +0.00000000 0.39811627 0.61943218 +0.19828294 0.39371379 0.61142471 +0.37749005 0.37749005 0.57760136 +0.52393740 0.34700724 0.52393740 +0.63610900 0.30713688 0.46257597 +0.71271826 0.27273859 0.41022707 +0.77084406 0.24904879 0.37440838 +0.83067969 0.23341483 0.35085869 +0.86874877 0.21581865 0.32412620 +0.91300074 0.20277513 0.30421784 +1.00054285 0.20009743 0.30015002 +1.10000181 0.20000023 0.30000037 1.20000000 0.20000000 0.30000000 -0.00000000 0.60391464 0.60391464 -0.19712823 0.59221847 0.59221847 -0.37130158 0.55777243 0.55777243 -0.51573703 0.51573703 0.51573703 -0.63468021 0.47026829 0.47026829 -0.72882359 0.42634679 0.42634679 -0.80031590 0.39037523 0.39037523 -0.85896581 0.36286548 0.36286548 -0.91301195 0.34124155 0.34124155 -0.94805255 0.31627253 0.31627253 -1.00287541 0.30096251 0.30096251 -1.10000642 0.30000187 0.30000187 +0.00000000 0.55444034 0.55444034 +0.18047772 0.54733782 0.54733782 +0.34700724 0.52393740 0.52393740 +0.48807962 0.48807962 0.48807962 +0.59982368 0.44196286 0.44196286 +0.68209364 0.39637314 0.39637314 +0.74983063 0.36483085 0.36483085 +0.80804636 0.34076624 0.34076624 +0.84436725 0.31505303 0.31505303 +0.90624407 0.30193390 0.30193390 +1.00017773 0.30004694 0.30004694 +1.10000040 0.30000008 0.30000008 1.20000000 0.30000000 0.30000000 -0.00000000 0.71080419 0.50149623 -0.17093147 0.70343685 0.50080222 -0.33036339 0.67799702 0.49245949 -0.47026829 0.63468021 0.47026829 -0.58727758 0.58727758 0.43999065 -0.68544422 0.54231789 0.40765797 -0.76328972 0.50064654 0.37623684 -0.82731919 0.46741977 0.35081514 -0.87928885 0.43790516 0.32839894 -0.91846729 0.40823181 0.30610938 -1.00037561 0.40015870 0.30011969 -1.10000015 0.40000005 0.30000004 +0.00000000 0.66184942 0.47470537 +0.15704777 0.65596314 0.47231988 +0.30713688 0.63610900 0.46257597 +0.44196286 0.59982368 0.44196286 +0.55326743 0.55326743 0.41149920 +0.64472303 0.50838937 0.37966213 +0.72156117 0.47193382 0.35303863 +0.77379510 0.43700725 0.32722084 +0.82134888 0.40958221 0.30706461 +0.90202252 0.40080598 0.30059446 +1.00003371 0.40001148 0.30000831 +1.10000004 0.40000001 0.30000001 1.20000000 0.40000000 0.30000000 -0.00000000 0.78868751 0.44974512 -0.15018738 0.78235981 0.44678136 -0.29400223 0.76188336 0.43867714 -0.42634679 0.72882359 0.42634679 -0.54231789 0.68544422 0.40765797 -0.63909979 0.63909979 0.38385260 -0.72252282 0.59824110 0.36006378 -0.79135631 0.56136548 0.33729954 -0.83815065 0.52260788 0.31343113 -0.90288377 0.50158998 0.30096622 -1.00001657 0.50000815 0.30000484 +0.00000000 0.73656137 0.42358925 +0.13907776 0.73097090 0.42005042 +0.27273859 0.71271826 0.41022707 +0.39637314 0.68209364 0.39637314 +0.50838937 0.64472303 0.37966213 +0.60602462 0.60602462 0.36013753 +0.68164548 0.56364685 0.33650109 +0.73654305 0.52340799 0.31347542 +0.80694211 0.50386270 0.30224793 +0.90038347 0.50018544 0.30010681 +1.00000329 0.50000136 0.30000075 1.10000000 0.50000000 0.30000000 1.20000000 0.50000000 0.30000000 -0.00000000 0.85270186 0.41230124 -0.13725219 0.84712657 0.40975168 -0.26861035 0.82950195 0.40204041 -0.39037523 0.80031590 0.39037523 -0.50064654 0.76328972 0.37623684 -0.59824110 0.72252282 0.36006378 -0.68192803 0.68192803 0.34163275 -0.74630639 0.63839074 0.31941918 -0.80785710 0.60564149 0.30285894 -0.90015817 0.60010177 0.30005217 -1.00000009 0.60000005 0.30000003 +0.00000000 0.79058389 0.38427682 +0.12674756 0.78536930 0.38158277 +0.24904879 0.77084406 0.37440838 +0.36483085 0.74983063 0.36483085 +0.47193382 0.72156117 0.35303863 +0.56364685 0.68164548 0.33650109 +0.63703231 0.63703231 0.31703047 +0.71120711 0.60901345 0.30426238 +0.80128826 0.60086304 0.30040857 +0.90003585 0.60002051 0.30000941 +1.00000012 0.60000006 0.30000002 1.10000000 0.60000000 0.30000000 1.20000000 0.60000000 0.30000000 -0.00000000 0.90696809 0.38368634 -0.12715273 0.90155981 0.38108807 -0.24924309 0.88510792 0.37365783 -0.36286548 0.85896581 0.36286548 -0.46741977 0.82731919 0.35081514 -0.56136548 0.79135631 0.33729954 -0.63839074 0.74630639 0.31941918 -0.70939520 0.70939520 0.30409247 -0.80044005 0.70037406 0.30016913 -0.90000138 0.70000101 0.30000042 +0.00000000 0.84610090 0.35875364 +0.11845901 0.84241872 0.35677352 +0.23341483 0.83067969 0.35085869 +0.34076624 0.80804636 0.34076624 +0.43700725 0.77379510 0.32722084 +0.52340799 0.73654305 0.31347542 +0.60901345 0.71120711 0.30426238 +0.70169821 0.70169821 0.30066473 +0.80010234 0.70008274 0.30003203 +0.90000122 0.70000082 0.30000030 1.00000000 0.70000000 0.30000000 1.10000000 0.70000000 0.30000000 1.20000000 0.70000000 0.30000000 -0.00000000 0.95438813 0.35904395 -0.11879272 0.94953048 0.35679227 -0.23356272 0.93512570 0.35040098 -0.34124155 0.91301195 0.34124155 -0.43790516 0.87928885 0.32839894 -0.52260788 0.83815065 0.31343113 -0.60564149 0.80785710 0.30285894 -0.70037406 0.80044005 0.30016913 -0.80000384 0.80000384 0.30000142 -0.90000000 0.80000000 0.30000000 +0.00000000 0.89129645 0.33342199 +0.11001986 0.88558560 0.33096079 +0.21581865 0.86874877 0.32412620 +0.31505303 0.84436725 0.31505303 +0.40958221 0.82134888 0.30706461 +0.50386270 0.80694211 0.30224793 +0.60086304 0.80128826 0.30040857 +0.70008274 0.80010234 0.30003203 +0.80000246 0.80000246 0.30000075 +0.90000001 0.80000001 0.30000000 1.00000000 0.80000000 0.30000000 1.10000000 0.80000000 0.30000000 1.20000000 0.80000000 0.30000000 -0.00000000 0.99927356 0.33668601 -0.11124799 0.99386503 0.33427106 -0.21784235 0.97657491 0.32704436 -0.31627253 0.94805255 0.31627253 -0.40823181 0.91846729 0.30610938 -0.50158998 0.90288377 0.30096622 -0.60010177 0.90015817 0.30005217 -0.70000101 0.90000138 0.30000042 -0.80000000 0.90000000 0.30000000 +0.00000000 0.92259213 0.30763684 +0.10213886 0.91972682 0.30659998 +0.20277513 0.91300074 0.30421784 +0.30193390 0.90624407 0.30193390 +0.40080598 0.90202252 0.30059446 +0.50018544 0.90038347 0.30010681 +0.60002051 0.90003585 0.30000941 +0.70000082 0.90000122 0.30000030 +0.80000001 0.90000001 0.30000000 0.90000000 0.90000000 0.30000000 1.00000000 0.90000000 0.30000000 1.10000000 0.90000000 0.30000000 1.20000000 0.90000000 0.30000000 -0.00000000 1.02383514 0.30867229 -0.10223985 1.01968452 0.30706878 -0.20235762 1.01034139 0.30359058 -0.30096251 1.00287541 0.30096251 -0.40015870 1.00037561 0.30011969 -0.50000815 1.00001657 0.30000484 -0.60000005 1.00000009 0.30000003 +0.00000000 1.00125619 0.30035799 +0.10009212 1.00102408 0.30028977 +0.20009743 1.00054285 0.30015002 +0.30004694 1.00017773 0.30004694 +0.40001148 1.00003371 0.30000831 +0.50000136 1.00000329 0.30000075 +0.60000006 1.00000012 0.30000002 0.70000000 1.00000000 0.30000000 0.80000000 1.00000000 0.30000000 0.90000000 1.00000000 0.30000000 1.00000000 1.00000000 0.30000000 1.10000000 1.00000000 0.30000000 1.20000000 1.00000000 0.30000000 -0.00000000 1.10029021 0.30009933 -0.10002269 1.10020126 0.30006789 -0.20001291 1.10006116 0.30001958 -0.30000187 1.10000642 0.30000187 -0.40000005 1.10000015 0.30000004 +0.00000000 1.10000554 0.30000115 +0.10000026 1.10000422 0.30000088 +0.20000023 1.10000181 0.30000037 +0.30000008 1.10000040 0.30000008 +0.40000001 1.10000004 0.30000001 0.50000000 1.10000000 0.30000000 0.60000000 1.10000000 0.30000000 0.70000000 1.10000000 0.30000000 @@ -667,7 +667,7 @@ POINTS 2197 float 1.00000000 1.10000000 0.30000000 1.10000000 1.10000000 0.30000000 1.20000000 1.10000000 0.30000000 -0.00000000 1.20000000 0.30000001 +0.00000000 1.20000000 0.30000000 0.10000000 1.20000000 0.30000000 0.20000000 1.20000000 0.30000000 0.30000000 1.20000000 0.30000000 @@ -680,153 +680,153 @@ POINTS 2197 float 1.00000000 1.20000000 0.30000000 1.10000000 1.20000000 0.30000000 1.20000000 1.20000000 0.30000000 -0.00000000 0.00000000 0.77599848 -0.18976175 0.00000000 0.77688181 -0.35501801 0.00000000 0.75701522 -0.50149623 0.00000000 0.71080419 -0.64970821 0.00000000 0.64970821 -0.73928284 0.00000000 0.56858842 -0.81006029 0.00000000 0.52387144 -0.86770511 0.00000000 0.48767065 -0.92067108 0.00000000 0.45894538 -0.95881249 0.00000000 0.42696076 -1.00427903 0.00000000 0.40189866 -1.10000820 0.00000000 0.40000322 +0.00000000 0.00000000 0.69317236 +0.17732266 0.00000000 0.70065996 +0.33499852 0.00000000 0.69528976 +0.47470537 0.00000000 0.66184942 +0.60527944 0.00000000 0.60527944 +0.69076838 0.00000000 0.53458256 +0.75510430 0.00000000 0.49155733 +0.81609925 0.00000000 0.45999413 +0.85278975 0.00000000 0.42453094 +0.90807622 0.00000000 0.40340321 +1.00024504 0.00000000 0.40008808 +1.10000051 0.00000000 0.40000015 1.20000000 0.00000000 0.40000000 -0.00000000 0.18976175 0.77688181 -0.18567451 0.18567451 0.77402488 -0.35105276 0.17683110 0.74994845 -0.50080222 0.17093147 0.70343685 -0.64119052 0.16128471 0.64119052 -0.73369463 0.14458578 0.56719040 -0.80467853 0.13108583 0.52099842 -0.86286900 0.12165362 0.48524650 -0.91652877 0.11417539 0.45674548 -0.95313141 0.10595209 0.42424467 -1.00335672 0.10037558 0.40148402 -1.10000555 0.10000050 0.40000217 +0.00000000 0.17732266 0.70065996 +0.17371925 0.17371925 0.70344694 +0.33046720 0.16465710 0.69136043 +0.47231988 0.15704777 0.65596314 +0.59909700 0.14710687 0.59909700 +0.68503398 0.13232192 0.53184308 +0.75150883 0.12139215 0.48917949 +0.81160479 0.11379863 0.45739004 +0.84822886 0.10530144 0.42230988 +0.90700443 0.10070420 0.40293226 +1.00019883 0.10001649 0.40007112 +1.10000039 0.10000002 0.40000011 1.20000000 0.10000000 0.40000000 -0.00000000 0.35501801 0.75701522 -0.17683110 0.35105276 0.74994845 -0.34252623 0.34252623 0.72359708 -0.49245949 0.33036339 0.67799702 -0.61746997 0.30925925 0.61746997 -0.71543727 0.28223638 0.55957630 -0.78816087 0.25768821 0.51279397 -0.84899828 0.23944822 0.47825791 -0.90311681 0.22492296 0.44979284 -0.93777007 0.20834384 0.41704032 -1.00154830 0.20034203 0.40067510 -1.10000159 0.20000028 0.40000060 +0.00000000 0.33499852 0.69528976 +0.16465710 0.33046720 0.69136043 +0.32015320 0.32015320 0.67259986 +0.46257597 0.30713688 0.63610900 +0.58084527 0.28632340 0.58084527 +0.66864949 0.26001908 0.52310454 +0.74058418 0.24011698 0.48258494 +0.79756131 0.22403142 0.44954991 +0.83612377 0.20796403 0.41650187 +0.90449165 0.20089456 0.40184411 +1.00010437 0.20001732 0.40003672 +1.10000017 0.20000002 0.40000005 1.20000000 0.20000000 0.40000000 -0.00000000 0.50149623 0.71080419 -0.17093147 0.50080222 0.70343685 -0.33036339 0.49245949 0.67799702 -0.47026829 0.47026829 0.63468021 -0.58727758 0.43999065 0.58727758 -0.68544422 0.40765797 0.54231789 -0.76328972 0.37623684 0.50064654 -0.82731919 0.35081514 0.46741977 -0.87928885 0.32839894 0.43790516 -0.91846729 0.30610938 0.40823181 -1.00037561 0.30011969 0.40015870 -1.10000015 0.30000004 0.40000005 +0.00000000 0.47470537 0.66184942 +0.15704777 0.47231988 0.65596314 +0.30713688 0.46257597 0.63610900 +0.44196286 0.44196286 0.59982368 +0.55326743 0.41149920 0.55326743 +0.64472303 0.37966213 0.50838937 +0.72156117 0.35303863 0.47193382 +0.77379510 0.32722084 0.43700725 +0.82134888 0.30706461 0.40958221 +0.90202252 0.30059446 0.40080598 +1.00003371 0.30000831 0.40001148 +1.10000004 0.30000001 0.40000001 1.20000000 0.30000000 0.40000000 -0.00000000 0.64970821 0.64970821 -0.16128471 0.64119052 0.64119052 -0.30925925 0.61746997 0.61746997 -0.43999065 0.58727758 0.58727758 -0.55432823 0.55432823 0.55432823 -0.64933117 0.51735904 0.51735904 -0.73125472 0.48386966 0.48386966 -0.80015564 0.45385923 0.45385923 -0.84633911 0.42186536 0.42186536 -0.90474355 0.40210769 0.40210769 -1.00003915 0.40001565 0.40001565 +0.00000000 0.60527944 0.60527944 +0.14710687 0.59909700 0.59909700 +0.28632340 0.58084527 0.58084527 +0.41149920 0.55326743 0.55326743 +0.52108297 0.52108297 0.52108297 +0.61658161 0.48887167 0.48887167 +0.69175588 0.45510921 0.45510921 +0.74419341 0.42207741 0.42207741 +0.80925855 0.40407424 0.40407424 +0.90058837 0.40022599 0.40022599 +1.00000616 0.40000200 0.40000200 1.10000000 0.40000000 0.40000000 1.20000000 0.40000000 0.40000000 -0.00000000 0.73928284 0.56858842 -0.14458578 0.73369463 0.56719040 -0.28223638 0.71543727 0.55957630 -0.40765797 0.68544422 0.54231789 -0.51735904 0.64933117 0.51735904 -0.61430584 0.61430584 0.49101416 -0.69689697 0.57911331 0.46343371 -0.76247824 0.54287421 0.43423088 -0.81599699 0.50953910 0.40759358 -0.90052141 0.50028459 0.40022936 -1.00000107 0.50000051 0.40000040 +0.00000000 0.69076838 0.53458256 +0.13232192 0.68503398 0.53184308 +0.26001908 0.66864949 0.52310454 +0.37966213 0.64472303 0.50838937 +0.48887167 0.61658161 0.48887167 +0.58225082 0.58225082 0.46329622 +0.65347289 0.54209737 0.43265961 +0.71865973 0.51205918 0.40944584 +0.80263150 0.50144764 0.40113819 +0.90009900 0.50004666 0.40003640 +1.00000055 0.50000022 0.40000017 1.10000000 0.50000000 0.40000000 1.20000000 0.50000000 0.40000000 -0.00000000 0.81006029 0.52387144 -0.13108583 0.80467853 0.52099842 -0.25768821 0.78816087 0.51279397 -0.37623684 0.76328972 0.50064654 -0.48386966 0.73125472 0.48386966 -0.57911331 0.69689697 0.46343371 -0.65872595 0.65872595 0.43876962 -0.72246813 0.61888951 0.41248886 -0.80192020 0.60140266 0.40095016 -0.90001954 0.60001237 0.40000825 -1.00000000 0.60000000 0.40000000 +0.00000000 0.75510430 0.49155733 +0.12139215 0.75150883 0.48917949 +0.24011698 0.74058418 0.48258494 +0.35303863 0.72156117 0.47193382 +0.45510921 0.69175588 0.45510921 +0.54209737 0.65347289 0.43265961 +0.61998642 0.61998642 0.41247798 +0.70476919 0.60386720 0.40246743 +0.80040738 0.60027207 0.40017365 +0.90000809 0.60000456 0.40000283 +1.00000002 0.60000001 0.40000000 1.10000000 0.60000000 0.40000000 1.20000000 0.60000000 0.40000000 -0.00000000 0.86770511 0.48767065 -0.12165362 0.86286900 0.48524650 -0.23944822 0.84899828 0.47825791 -0.35081514 0.82731919 0.46741977 -0.45385923 0.80015564 0.45385923 -0.54287421 0.76247824 0.43423088 -0.61888951 0.72246813 0.41248886 -0.70244509 0.70244509 0.40141285 -0.80006445 0.70005521 0.40003218 -0.90000008 0.70000006 0.40000003 +0.00000000 0.81609925 0.45999413 +0.11379863 0.81160479 0.45739004 +0.22403142 0.79756131 0.44954991 +0.32722084 0.77379510 0.43700725 +0.42207741 0.74419341 0.42207741 +0.51205918 0.71865973 0.40944584 +0.60386720 0.70476919 0.40246743 +0.70056455 0.70056455 0.40029574 +0.80002657 0.70002159 0.40001118 +0.90000022 0.70000015 0.40000007 1.00000000 0.70000000 0.40000000 1.10000000 0.70000000 0.40000000 1.20000000 0.70000000 0.40000000 -0.00000000 0.92067108 0.45894538 -0.11417539 0.91652877 0.45674548 -0.22492296 0.90311681 0.44979284 -0.32839894 0.87928885 0.43790516 -0.42186536 0.84633911 0.42186536 -0.50953910 0.81599699 0.40759358 -0.60140266 0.80192020 0.40095016 -0.70005521 0.80006445 0.40003218 -0.80000021 0.80000021 0.40000010 +0.00000000 0.85278975 0.42453094 +0.10530144 0.84822886 0.42230988 +0.20796403 0.83612377 0.41650187 +0.30706461 0.82134888 0.40958221 +0.40407424 0.80925855 0.40407424 +0.50144764 0.80263150 0.40113819 +0.60027207 0.80040738 0.40017365 +0.70002159 0.80002657 0.40001118 +0.80000047 0.80000047 0.40000019 0.90000000 0.80000000 0.40000000 1.00000000 0.80000000 0.40000000 1.10000000 0.80000000 0.40000000 1.20000000 0.80000000 0.40000000 -0.00000000 0.95881249 0.42696076 -0.10595209 0.95313141 0.42424467 -0.20834384 0.93777007 0.41704032 -0.30610938 0.91846729 0.40823181 -0.40210769 0.90474355 0.40210769 -0.50028459 0.90052141 0.40022936 -0.60001237 0.90001954 0.40000825 -0.70000006 0.90000008 0.40000003 +0.00000000 0.90807622 0.40340321 +0.10070420 0.90700443 0.40293226 +0.20089456 0.90449165 0.40184411 +0.30059446 0.90202252 0.40080598 +0.40022599 0.90058837 0.40022599 +0.50004666 0.90009900 0.40003640 +0.60000456 0.90000809 0.40000283 +0.70000015 0.90000022 0.40000007 0.80000000 0.90000000 0.40000000 0.90000000 0.90000000 0.40000000 1.00000000 0.90000000 0.40000000 1.10000000 0.90000000 0.40000000 1.20000000 0.90000000 0.40000000 -0.00000000 1.00427903 0.40189866 -0.10037558 1.00335672 0.40148402 -0.20034203 1.00154830 0.40067510 -0.30011969 1.00037561 0.40015870 -0.40001565 1.00003915 0.40001565 -0.50000051 1.00000107 0.40000040 -0.60000000 1.00000000 0.40000000 +0.00000000 1.00024504 0.40008808 +0.10001649 1.00019883 0.40007112 +0.20001732 1.00010437 0.40003672 +0.30000831 1.00003371 0.40001148 +0.40000200 1.00000616 0.40000200 +0.50000022 1.00000055 0.40000017 +0.60000001 1.00000002 0.40000000 0.70000000 1.00000000 0.40000000 0.80000000 1.00000000 0.40000000 0.90000000 1.00000000 0.40000000 1.00000000 1.00000000 0.40000000 1.10000000 1.00000000 0.40000000 1.20000000 1.00000000 0.40000000 -0.00000000 1.10000820 0.40000322 -0.10000050 1.10000555 0.40000217 -0.20000028 1.10000159 0.40000060 -0.30000004 1.10000015 0.40000005 +0.00000000 1.10000051 0.40000015 +0.10000002 1.10000039 0.40000011 +0.20000002 1.10000017 0.40000005 +0.30000001 1.10000004 0.40000001 0.40000000 1.10000000 0.40000000 0.50000000 1.10000000 0.40000000 0.60000000 1.10000000 0.40000000 @@ -849,142 +849,142 @@ POINTS 2197 float 1.00000000 1.20000000 0.40000000 1.10000000 1.20000000 0.40000000 1.20000000 1.20000000 0.40000000 -0.00000000 0.00000000 0.84746937 -0.16967335 0.00000000 0.84748313 -0.31714318 0.00000000 0.82700772 -0.44974512 0.00000000 0.78868751 -0.56858842 0.00000000 0.73928284 -0.68717236 0.00000000 0.68717236 -0.76182433 0.00000000 0.62063065 -0.82630414 0.00000000 0.58266570 -0.87929698 0.00000000 0.54716000 -0.91747243 0.00000000 0.50972929 -1.00030406 0.00000000 0.50015749 -1.10000004 0.00000000 0.50000002 +0.00000000 0.00000000 0.76611578 +0.15482278 0.00000000 0.77112549 +0.29563371 0.00000000 0.76733124 +0.42358925 0.00000000 0.73656137 +0.53458256 0.00000000 0.69076838 +0.63715933 0.00000000 0.63715933 +0.71794111 0.00000000 0.58791198 +0.77145885 0.00000000 0.54616444 +0.82018154 0.00000000 0.51151186 +0.90183755 0.00000000 0.50091570 +1.00002625 0.00000000 0.50001129 +1.10000002 0.00000000 0.50000001 1.20000000 0.00000000 0.50000000 -0.00000000 0.16967335 0.84748313 -0.16592356 0.16592356 0.84371201 -0.31369245 0.15782758 0.82057257 -0.44678136 0.15018738 0.78235981 -0.56719040 0.14458578 0.73369463 -0.68053012 0.13738574 0.68053012 -0.75739272 0.12555538 0.61883699 -0.82242886 0.11646131 0.58018583 -0.87441298 0.10890351 0.54424669 -0.91472437 0.10161500 0.50818514 -1.00022657 0.10002438 0.50011671 -1.10000003 0.10000000 0.50000001 +0.00000000 0.15482278 0.77112549 +0.15224945 0.15224945 0.77310595 +0.29267929 0.14608580 0.76292351 +0.42005042 0.13907776 0.73097090 +0.53184308 0.13232192 0.68503398 +0.63335093 0.12512370 0.63335093 +0.71422441 0.11651969 0.58590873 +0.76715562 0.10805615 0.54323430 +0.81808075 0.10195059 0.51027912 +0.90155996 0.10014558 0.50077499 +1.00002108 0.10000158 0.50000903 +1.10000001 0.10000000 0.50000000 1.20000000 0.10000000 0.50000000 -0.00000000 0.31714318 0.82700772 -0.15782758 0.31369245 0.82057257 -0.30473274 0.30473274 0.79895918 -0.43867714 0.29400223 0.76188336 -0.55957630 0.28223638 0.71543727 -0.66254081 0.26630201 0.66254081 -0.74412487 0.24659868 0.61203591 -0.81100840 0.22993157 0.57321991 -0.85996942 0.21419343 0.53560066 -0.90841516 0.20186981 0.50466014 -1.00008969 0.20001849 0.50004540 +0.00000000 0.29563371 0.76733124 +0.14608580 0.29267929 0.76292351 +0.28439047 0.28439047 0.74546064 +0.41022707 0.27273859 0.71271826 +0.52310454 0.26001908 0.66864949 +0.62258026 0.24621977 0.62258026 +0.70243248 0.22996354 0.57852225 +0.75465658 0.21320089 0.53498985 +0.81284737 0.20277440 0.50723678 +0.90093920 0.20017523 0.50046210 +1.00001074 0.20000162 0.50000454 1.10000001 0.20000000 0.50000000 1.20000000 0.20000000 0.50000000 -0.00000000 0.44974512 0.78868751 -0.15018738 0.44678136 0.78235981 -0.29400223 0.43867714 0.76188336 -0.42634679 0.42634679 0.72882359 -0.54231789 0.40765797 0.68544422 -0.63909979 0.38385260 0.63909979 -0.72252282 0.36006378 0.59824110 -0.79135631 0.33729954 0.56136548 -0.83815065 0.31343113 0.52260788 -0.90288377 0.30096622 0.50158998 -1.00001657 0.30000484 0.50000815 +0.00000000 0.42358925 0.73656137 +0.13907776 0.42005042 0.73097090 +0.27273859 0.41022707 0.71271826 +0.39637314 0.39637314 0.68209364 +0.50838937 0.37966213 0.64472303 +0.60602462 0.36013753 0.60602462 +0.68164548 0.33650109 0.56364685 +0.73654305 0.31347542 0.52340799 +0.80694211 0.30224793 0.50386270 +0.90038347 0.30010681 0.50018544 +1.00000329 0.30000075 0.50000136 1.10000000 0.30000000 0.50000000 1.20000000 0.30000000 0.50000000 -0.00000000 0.56858842 0.73928284 -0.14458578 0.56719040 0.73369463 -0.28223638 0.55957630 0.71543727 -0.40765797 0.54231789 0.68544422 -0.51735904 0.51735904 0.64933117 -0.61430584 0.49101416 0.61430584 -0.69689697 0.46343371 0.57911331 -0.76247824 0.43423088 0.54287421 -0.81599699 0.40759358 0.50953910 -0.90052141 0.40022936 0.50028459 -1.00000107 0.40000040 0.50000051 +0.00000000 0.53458256 0.69076838 +0.13232192 0.53184308 0.68503398 +0.26001908 0.52310454 0.66864949 +0.37966213 0.50838937 0.64472303 +0.48887167 0.48887167 0.61658161 +0.58225082 0.46329622 0.58225082 +0.65347289 0.43265961 0.54209737 +0.71865973 0.40944584 0.51205918 +0.80263150 0.40113819 0.50144764 +0.90009900 0.40003640 0.50004666 +1.00000055 0.40000017 0.50000022 1.10000000 0.40000000 0.50000000 1.20000000 0.40000000 0.50000000 -0.00000000 0.68717236 0.68717236 -0.13738574 0.68053012 0.68053012 -0.26630201 0.66254081 0.66254081 -0.38385260 0.63909979 0.63909979 -0.49101416 0.61430584 0.61430584 -0.58559764 0.58559764 0.58559764 -0.66643254 0.55462614 0.55462614 -0.72846227 0.51973761 0.51973761 -0.80311288 0.50189877 0.50189877 -0.90004039 0.50002147 0.50002147 -1.00000001 0.50000001 0.50000001 +0.00000000 0.63715933 0.63715933 +0.12512370 0.63335093 0.63335093 +0.24621977 0.62258026 0.62258026 +0.36013753 0.60602462 0.60602462 +0.46329622 0.58225082 0.58225082 +0.55046109 0.55046109 0.55046109 +0.62553297 0.52029475 0.52029475 +0.70651067 0.50425711 0.50425711 +0.80062010 0.50033768 0.50033768 +0.90001447 0.50000661 0.50000661 +1.00000004 0.50000002 0.50000002 1.10000000 0.50000000 0.50000000 1.20000000 0.50000000 0.50000000 -0.00000000 0.76182433 0.62063065 -0.12555538 0.75739272 0.61883699 -0.24659868 0.74412487 0.61203591 -0.36006378 0.72252282 0.59824110 -0.46343371 0.69689697 0.57911331 -0.55462614 0.66643254 0.55462614 -0.62980716 0.62980716 0.52456904 -0.70574649 0.60486862 0.50404594 -0.80022001 0.60016189 0.50013560 -0.90000077 0.60000048 0.50000040 +0.00000000 0.71794111 0.58791198 +0.11651969 0.71422441 0.58590873 +0.22996354 0.70243248 0.57852225 +0.33650109 0.68164548 0.56364685 +0.43265961 0.65347289 0.54209737 +0.52029475 0.62553297 0.52029475 +0.60749959 0.60749959 0.50603142 +0.70130761 0.60106493 0.50086380 +0.80007731 0.60005148 0.50004167 +0.90000097 0.60000054 0.50000043 1.00000000 0.60000000 0.50000000 1.10000000 0.60000000 0.50000000 1.20000000 0.60000000 0.50000000 -0.00000000 0.82630414 0.58266570 -0.11646131 0.82242886 0.58018583 -0.22993157 0.81100840 0.57321991 -0.33729954 0.79135631 0.56136548 -0.43423088 0.76247824 0.54287421 -0.51973761 0.72846227 0.51973761 -0.60486862 0.70574649 0.50404594 -0.70032266 0.70032266 0.50023117 -0.80000353 0.70000302 0.50000213 -0.90000000 0.70000000 0.50000000 +0.00000000 0.77145885 0.54616444 +0.10805615 0.76715562 0.54323430 +0.21320089 0.75465658 0.53498985 +0.31347542 0.73654305 0.52340799 +0.40944584 0.71865973 0.51205918 +0.50425711 0.70651067 0.50425711 +0.60106493 0.70130761 0.50086380 +0.70011636 0.70011636 0.50007695 +0.80000386 0.70000315 0.50000206 +0.90000002 0.70000001 0.50000001 1.00000000 0.70000000 0.50000000 1.10000000 0.70000000 0.50000000 1.20000000 0.70000000 0.50000000 -0.00000000 0.87929698 0.54716000 -0.10890351 0.87441298 0.54424669 -0.21419343 0.85996942 0.53560066 -0.31343113 0.83815065 0.52260788 -0.40759358 0.81599699 0.50953910 -0.50189877 0.80311288 0.50189877 -0.60016189 0.80022001 0.50013560 -0.70000302 0.80000353 0.50000213 -0.80000000 0.80000000 0.50000000 +0.00000000 0.82018154 0.51151186 +0.10195059 0.81808075 0.51027912 +0.20277440 0.81284737 0.50723678 +0.30224793 0.80694211 0.50386270 +0.40113819 0.80263150 0.50144764 +0.50033768 0.80062010 0.50033768 +0.60005148 0.80007731 0.50004167 +0.70000315 0.80000386 0.50000206 +0.80000005 0.80000005 0.50000002 0.90000000 0.80000000 0.50000000 1.00000000 0.80000000 0.50000000 1.10000000 0.80000000 0.50000000 1.20000000 0.80000000 0.50000000 -0.00000000 0.91747243 0.50972929 -0.10161500 0.91472437 0.50818514 -0.20186981 0.90841516 0.50466014 -0.30096622 0.90288377 0.50158998 -0.40022936 0.90052141 0.50028459 -0.50002147 0.90004039 0.50002147 -0.60000048 0.90000077 0.50000040 -0.70000000 0.90000000 0.50000000 +0.00000000 0.90183755 0.50091570 +0.10014558 0.90155996 0.50077499 +0.20017523 0.90093920 0.50046210 +0.30010681 0.90038347 0.50018544 +0.40003640 0.90009900 0.50004666 +0.50000661 0.90001447 0.50000661 +0.60000054 0.90000097 0.50000043 +0.70000001 0.90000002 0.50000001 0.80000000 0.90000000 0.50000000 0.90000000 0.90000000 0.50000000 1.00000000 0.90000000 0.50000000 1.10000000 0.90000000 0.50000000 1.20000000 0.90000000 0.50000000 -0.00000000 1.00030406 0.50015749 -0.10002438 1.00022657 0.50011671 -0.20001849 1.00008969 0.50004540 -0.30000484 1.00001657 0.50000815 -0.40000040 1.00000107 0.50000051 -0.50000001 1.00000001 0.50000001 +0.00000000 1.00002625 0.50001129 +0.10000158 1.00002108 0.50000903 +0.20000162 1.00001074 0.50000454 +0.30000075 1.00000329 0.50000136 +0.40000017 1.00000055 0.50000022 +0.50000002 1.00000004 0.50000002 0.60000000 1.00000000 0.50000000 0.70000000 1.00000000 0.50000000 0.80000000 1.00000000 0.50000000 @@ -992,8 +992,8 @@ POINTS 2197 float 1.00000000 1.00000000 0.50000000 1.10000000 1.00000000 0.50000000 1.20000000 1.00000000 0.50000000 -0.00000000 1.10000004 0.50000002 -0.10000000 1.10000003 0.50000001 +0.00000000 1.10000002 0.50000001 +0.10000000 1.10000001 0.50000000 0.20000000 1.10000001 0.50000000 0.30000000 1.10000000 0.50000000 0.40000000 1.10000000 0.50000000 @@ -1018,141 +1018,141 @@ POINTS 2197 float 1.00000000 1.20000000 0.50000000 1.10000000 1.20000000 0.50000000 1.20000000 1.20000000 0.50000000 -0.00000000 0.00000000 0.91204792 -0.15517265 0.00000000 0.91018232 -0.29071456 0.00000000 0.88817242 -0.41230124 0.00000000 0.85270186 -0.52387144 0.00000000 0.81006029 -0.62063065 0.00000000 0.76182433 -0.71717747 0.00000000 0.71717747 -0.78380908 0.00000000 0.66428258 -0.82986250 0.00000000 0.62120277 -0.90158256 0.00000000 0.60103535 -1.00000345 0.00000000 0.60000199 +0.00000000 0.00000000 0.84297461 +0.14041137 0.00000000 0.84173053 +0.26937908 0.00000000 0.82443784 +0.38427682 0.00000000 0.79058389 +0.49155733 0.00000000 0.75510430 +0.58791198 0.00000000 0.71794111 +0.67035303 0.00000000 0.67035303 +0.72785573 0.00000000 0.62173687 +0.80468955 0.00000000 0.60317337 +0.90020391 0.00000000 0.60011921 +1.00000113 0.00000000 0.60000057 1.10000000 0.00000000 0.60000000 1.20000000 0.00000000 0.60000000 -0.00000000 0.15517265 0.91018232 -0.15186355 0.15186355 0.90540812 -0.28735387 0.14416070 0.88178372 -0.40975168 0.13725219 0.84712657 -0.52099842 0.13108583 0.80467853 -0.61883699 0.12555538 0.75739272 -0.71254499 0.11987896 0.71254499 -0.78003954 0.11125308 0.66232491 -0.82642640 0.10310482 0.61873750 -0.90125052 0.10014389 0.60081694 -1.00000241 0.10000021 0.60000138 +0.00000000 0.14041137 0.84173053 +0.13856589 0.13856589 0.83795977 +0.26639851 0.13302817 0.81793874 +0.38158277 0.12674756 0.78536930 +0.48917949 0.12139215 0.75150883 +0.58590873 0.11651969 0.71422441 +0.66597252 0.11012756 0.66597252 +0.72544089 0.10309869 0.61996228 +0.80410674 0.10043095 0.60277408 +0.90016967 0.10001488 0.60009895 +1.00000089 0.10000006 0.60000045 1.10000000 0.10000000 0.60000000 1.20000000 0.10000000 0.60000000 -0.00000000 0.29071456 0.88817242 -0.14416070 0.28735387 0.88178372 -0.27879357 0.27879357 0.86136115 -0.40204041 0.26861035 0.82950195 -0.51279397 0.25768821 0.78816087 -0.61203591 0.24659868 0.74412487 -0.69988798 0.23467848 0.69988798 -0.76772981 0.21891426 0.65451792 -0.81767238 0.20417332 0.61254377 -0.90059787 0.20013549 0.60038862 -1.00000077 0.20000014 0.60000044 +0.00000000 0.26937908 0.82443784 +0.13302817 0.26639851 0.81793874 +0.25841182 0.25841182 0.79825205 +0.37440838 0.24904879 0.77084406 +0.48258494 0.24011698 0.74058418 +0.57852225 0.22996354 0.70243248 +0.65392762 0.21648949 0.65392762 +0.71907392 0.20473370 0.61515505 +0.80271309 0.20057061 0.60182509 +0.90009639 0.20001690 0.60005581 +1.00000043 0.20000006 0.60000021 1.10000000 0.20000000 0.60000000 1.20000000 0.20000000 0.60000000 -0.00000000 0.41230124 0.85270186 -0.13725219 0.40975168 0.84712657 -0.26861035 0.40204041 0.82950195 -0.39037523 0.39037523 0.80031590 -0.50064654 0.37623684 0.76328972 -0.59824110 0.36006378 0.72252282 -0.68192803 0.34163275 0.68192803 -0.74630639 0.31941918 0.63839074 -0.80785710 0.30285894 0.60564149 -0.90015817 0.30005217 0.60010177 -1.00000009 0.30000003 0.60000005 +0.00000000 0.38427682 0.79058389 +0.12674756 0.38158277 0.78536930 +0.24904879 0.37440838 0.77084406 +0.36483085 0.36483085 0.74983063 +0.47193382 0.35303863 0.72156117 +0.56364685 0.33650109 0.68164548 +0.63703231 0.31703047 0.63703231 +0.71120711 0.30426238 0.60901345 +0.80128826 0.30040857 0.60086304 +0.90003585 0.30000941 0.60002051 +1.00000012 0.30000002 0.60000006 1.10000000 0.30000000 0.60000000 1.20000000 0.30000000 0.60000000 -0.00000000 0.52387144 0.81006029 -0.13108583 0.52099842 0.80467853 -0.25768821 0.51279397 0.78816087 -0.37623684 0.50064654 0.76328972 -0.48386966 0.48386966 0.73125472 -0.57911331 0.46343371 0.69689697 -0.65872595 0.43876962 0.65872595 -0.72246813 0.41248886 0.61888951 -0.80192020 0.40095016 0.60140266 -0.90001954 0.40000825 0.60001237 -1.00000000 0.40000000 0.60000000 +0.00000000 0.49155733 0.75510430 +0.12139215 0.48917949 0.75150883 +0.24011698 0.48258494 0.74058418 +0.35303863 0.47193382 0.72156117 +0.45510921 0.45510921 0.69175588 +0.54209737 0.43265961 0.65347289 +0.61998642 0.41247798 0.61998642 +0.70476919 0.40246743 0.60386720 +0.80040738 0.40017365 0.60027207 +0.90000809 0.40000283 0.60000456 +1.00000002 0.40000000 0.60000001 1.10000000 0.40000000 0.60000000 1.20000000 0.40000000 0.60000000 -0.00000000 0.62063065 0.76182433 -0.12555538 0.61883699 0.75739272 -0.24659868 0.61203591 0.74412487 -0.36006378 0.59824110 0.72252282 -0.46343371 0.57911331 0.69689697 -0.55462614 0.55462614 0.66643254 -0.62980716 0.52456904 0.62980716 -0.70574649 0.50404594 0.60486862 -0.80022001 0.50013560 0.60016189 -0.90000077 0.50000040 0.60000048 +0.00000000 0.58791198 0.71794111 +0.11651969 0.58590873 0.71422441 +0.22996354 0.57852225 0.70243248 +0.33650109 0.56364685 0.68164548 +0.43265961 0.54209737 0.65347289 +0.52029475 0.52029475 0.62553297 +0.60749959 0.50603142 0.60749959 +0.70130761 0.50086380 0.60106493 +0.80007731 0.50004167 0.60005148 +0.90000097 0.50000043 0.60000054 1.00000000 0.50000000 0.60000000 1.10000000 0.50000000 0.60000000 1.20000000 0.50000000 0.60000000 -0.00000000 0.71717747 0.71717747 -0.11987896 0.71254499 0.71254499 -0.23467848 0.69988798 0.69988798 -0.34163275 0.68192803 0.68192803 -0.43876962 0.65872595 0.65872595 -0.52456904 0.62980716 0.62980716 -0.60700341 0.60700341 0.60700341 -0.70056154 0.60047943 0.60047943 -0.80000815 0.60000594 0.60000594 -0.90000001 0.60000000 0.60000000 +0.00000000 0.67035303 0.67035303 +0.11012756 0.66597252 0.66597252 +0.21648949 0.65392762 0.65392762 +0.31703047 0.63703231 0.63703231 +0.41247798 0.61998642 0.61998642 +0.50603142 0.60749959 0.60749959 +0.60163401 0.60163401 0.60163401 +0.70019725 0.60016074 0.60016074 +0.80000745 0.60000495 0.60000495 +0.90000005 0.60000003 0.60000003 1.00000000 0.60000000 0.60000000 1.10000000 0.60000000 0.60000000 1.20000000 0.60000000 0.60000000 -0.00000000 0.78380908 0.66428258 -0.11125308 0.78003954 0.66232491 -0.21891426 0.76772981 0.65451792 -0.31941918 0.74630639 0.63839074 -0.41248886 0.72246813 0.61888951 -0.50404594 0.70574649 0.60486862 -0.60047943 0.70056154 0.60047943 -0.70001539 0.70001539 0.60001308 -0.80000004 0.70000003 0.60000003 +0.00000000 0.72785573 0.62173687 +0.10309869 0.72544089 0.61996228 +0.20473370 0.71907392 0.61515505 +0.30426238 0.71120711 0.60901345 +0.40246743 0.70476919 0.60386720 +0.50086380 0.70130761 0.60106493 +0.60016074 0.70019725 0.60016074 +0.70001248 0.70001248 0.60001016 +0.80000025 0.70000021 0.60000017 0.90000000 0.70000000 0.60000000 1.00000000 0.70000000 0.60000000 1.10000000 0.70000000 0.60000000 1.20000000 0.70000000 0.60000000 -0.00000000 0.82986250 0.62120277 -0.10310482 0.82642640 0.61873750 -0.20417332 0.81767238 0.61254377 -0.30285894 0.80785710 0.60564149 -0.40095016 0.80192020 0.60140266 -0.50013560 0.80022001 0.60016189 -0.60000594 0.80000815 0.60000594 -0.70000003 0.80000004 0.60000003 +0.00000000 0.80468955 0.60317337 +0.10043095 0.80410674 0.60277408 +0.20057061 0.80271309 0.60182509 +0.30040857 0.80128826 0.60086304 +0.40017365 0.80040738 0.60027207 +0.50004167 0.80007731 0.60005148 +0.60000495 0.80000745 0.60000495 +0.70000021 0.80000025 0.60000017 0.80000000 0.80000000 0.60000000 0.90000000 0.80000000 0.60000000 1.00000000 0.80000000 0.60000000 1.10000000 0.80000000 0.60000000 1.20000000 0.80000000 0.60000000 -0.00000000 0.90158256 0.60103535 -0.10014389 0.90125052 0.60081694 -0.20013549 0.90059787 0.60038862 -0.30005217 0.90015817 0.60010177 -0.40000825 0.90001954 0.60001237 -0.50000040 0.90000077 0.60000048 -0.60000000 0.90000001 0.60000000 +0.00000000 0.90020391 0.60011921 +0.10001488 0.90016967 0.60009895 +0.20001690 0.90009639 0.60005581 +0.30000941 0.90003585 0.60002051 +0.40000283 0.90000809 0.60000456 +0.50000043 0.90000097 0.60000054 +0.60000003 0.90000005 0.60000003 0.70000000 0.90000000 0.60000000 0.80000000 0.90000000 0.60000000 0.90000000 0.90000000 0.60000000 1.00000000 0.90000000 0.60000000 1.10000000 0.90000000 0.60000000 1.20000000 0.90000000 0.60000000 -0.00000000 1.00000345 0.60000199 -0.10000021 1.00000241 0.60000138 -0.20000014 1.00000077 0.60000044 -0.30000003 1.00000009 0.60000005 -0.40000000 1.00000000 0.60000000 +0.00000000 1.00000113 0.60000057 +0.10000006 1.00000089 0.60000045 +0.20000006 1.00000043 0.60000021 +0.30000002 1.00000012 0.60000006 +0.40000000 1.00000002 0.60000001 0.50000000 1.00000000 0.60000000 0.60000000 1.00000000 0.60000000 0.70000000 1.00000000 0.60000000 @@ -1187,129 +1187,129 @@ POINTS 2197 float 1.00000000 1.20000000 0.60000000 1.10000000 1.20000000 0.60000000 1.20000000 1.20000000 0.60000000 -0.00000000 0.00000000 0.96962515 -0.14332892 0.00000000 0.96483786 -0.27030805 0.00000000 0.94191103 -0.38368634 0.00000000 0.90696809 -0.48767065 0.00000000 0.86770511 -0.58266570 0.00000000 0.82630414 -0.66428258 0.00000000 0.78380908 -0.73913964 0.00000000 0.73913964 -0.80329722 0.00000000 0.70269171 -0.90002831 0.00000000 0.70002079 -1.00000000 0.00000000 0.70000000 +0.00000000 0.00000000 0.89059622 +0.12735313 0.00000000 0.88520787 +0.24795578 0.00000000 0.86917115 +0.35875364 0.00000000 0.84610090 +0.45999413 0.00000000 0.81609925 +0.54616444 0.00000000 0.77145885 +0.62173687 0.00000000 0.72785573 +0.70606151 0.00000000 0.70606151 +0.80048441 0.00000000 0.70038645 +0.90000869 0.00000000 0.70000592 +1.00000001 0.00000000 0.70000001 1.10000000 0.00000000 0.70000000 1.20000000 0.00000000 0.70000000 -0.00000000 0.14332892 0.96483786 -0.14058988 0.14058988 0.95897612 -0.26714900 0.13370188 0.93524877 -0.38108807 0.12715273 0.90155981 -0.48524650 0.12165362 0.86286900 -0.58018583 0.11646131 0.82242886 -0.66232491 0.11125308 0.78003954 -0.73398435 0.10513418 0.73398435 -0.80271483 0.10034158 0.70223247 -0.90002082 0.10000221 0.70001527 -1.00000000 0.10000000 0.70000000 +0.00000000 0.12735313 0.88520787 +0.12624382 0.12624382 0.88017760 +0.24601063 0.12275667 0.86468341 +0.35677352 0.11845901 0.84241872 +0.45739004 0.11379863 0.81160479 +0.54323430 0.10805615 0.76715562 +0.61996228 0.10309869 0.72544089 +0.70529997 0.10069657 0.70529997 +0.80041163 0.10004229 0.70032903 +0.90000706 0.10000056 0.70000480 +1.00000001 0.10000000 0.70000001 1.10000000 0.10000000 0.70000000 1.20000000 0.10000000 0.70000000 -0.00000000 0.27030805 0.94191103 -0.13370188 0.26714900 0.93524877 -0.25903270 0.25903270 0.91526121 -0.37365783 0.24924309 0.88510792 -0.47825791 0.23944822 0.84899828 -0.57321991 0.22993157 0.81100840 -0.65451792 0.21891426 0.76772981 -0.72157041 0.20635303 0.72157041 -0.80144842 0.20037105 0.70121158 -0.90000795 0.20000164 0.70000581 +0.00000000 0.24795578 0.86917115 +0.12275667 0.24601063 0.86468341 +0.24060773 0.24060773 0.85129881 +0.35085869 0.23341483 0.83067969 +0.44954991 0.22403142 0.79756131 +0.53498985 0.21320089 0.75465658 +0.61515505 0.20473370 0.71907392 +0.70350656 0.20091722 0.70350656 +0.80024881 0.20005147 0.70019990 +0.90000372 0.20000060 0.70000252 1.00000000 0.20000000 0.70000000 1.10000000 0.20000000 0.70000000 1.20000000 0.20000000 0.70000000 -0.00000000 0.38368634 0.90696809 -0.12715273 0.38108807 0.90155981 -0.24924309 0.37365783 0.88510792 -0.36286548 0.36286548 0.85896581 -0.46741977 0.35081514 0.82731919 -0.56136548 0.33729954 0.79135631 -0.63839074 0.31941918 0.74630639 -0.70939520 0.30409247 0.70939520 -0.80044005 0.30016913 0.70037406 -0.90000138 0.30000042 0.70000101 +0.00000000 0.35875364 0.84610090 +0.11845901 0.35677352 0.84241872 +0.23341483 0.35085869 0.83067969 +0.34076624 0.34076624 0.80804636 +0.43700725 0.32722084 0.77379510 +0.52340799 0.31347542 0.73654305 +0.60901345 0.30426238 0.71120711 +0.70169821 0.30066473 0.70169821 +0.80010234 0.30003203 0.70008274 +0.90000122 0.30000030 0.70000082 1.00000000 0.30000000 0.70000000 1.10000000 0.30000000 0.70000000 1.20000000 0.30000000 0.70000000 -0.00000000 0.48767065 0.86770511 -0.12165362 0.48524650 0.86286900 -0.23944822 0.47825791 0.84899828 -0.35081514 0.46741977 0.82731919 -0.45385923 0.45385923 0.80015564 -0.54287421 0.43423088 0.76247824 -0.61888951 0.41248886 0.72246813 -0.70244509 0.40141285 0.70244509 -0.80006445 0.40003218 0.70005521 -0.90000008 0.40000003 0.70000006 +0.00000000 0.45999413 0.81609925 +0.11379863 0.45739004 0.81160479 +0.22403142 0.44954991 0.79756131 +0.32722084 0.43700725 0.77379510 +0.42207741 0.42207741 0.74419341 +0.51205918 0.40944584 0.71865973 +0.60386720 0.40246743 0.70476919 +0.70056455 0.40029574 0.70056455 +0.80002657 0.40001118 0.70002159 +0.90000022 0.40000007 0.70000015 1.00000000 0.40000000 0.70000000 1.10000000 0.40000000 0.70000000 1.20000000 0.40000000 0.70000000 -0.00000000 0.58266570 0.82630414 -0.11646131 0.58018583 0.82242886 -0.22993157 0.57321991 0.81100840 -0.33729954 0.56136548 0.79135631 -0.43423088 0.54287421 0.76247824 -0.51973761 0.51973761 0.72846227 -0.60486862 0.50404594 0.70574649 -0.70032266 0.50023117 0.70032266 -0.80000353 0.50000213 0.70000302 -0.90000000 0.50000000 0.70000000 +0.00000000 0.54616444 0.77145885 +0.10805615 0.54323430 0.76715562 +0.21320089 0.53498985 0.75465658 +0.31347542 0.52340799 0.73654305 +0.40944584 0.51205918 0.71865973 +0.50425711 0.50425711 0.70651067 +0.60106493 0.50086380 0.70130761 +0.70011636 0.50007695 0.70011636 +0.80000386 0.50000206 0.70000315 +0.90000002 0.50000001 0.70000001 1.00000000 0.50000000 0.70000000 1.10000000 0.50000000 0.70000000 1.20000000 0.50000000 0.70000000 -0.00000000 0.66428258 0.78380908 -0.11125308 0.66232491 0.78003954 -0.21891426 0.65451792 0.76772981 -0.31941918 0.63839074 0.74630639 -0.41248886 0.61888951 0.72246813 -0.50404594 0.60486862 0.70574649 -0.60047943 0.60047943 0.70056154 -0.70001539 0.60001308 0.70001539 -0.80000004 0.60000003 0.70000003 +0.00000000 0.62173687 0.72785573 +0.10309869 0.61996228 0.72544089 +0.20473370 0.61515505 0.71907392 +0.30426238 0.60901345 0.71120711 +0.40246743 0.60386720 0.70476919 +0.50086380 0.60106493 0.70130761 +0.60016074 0.60016074 0.70019725 +0.70001248 0.60001016 0.70001248 +0.80000025 0.60000017 0.70000021 0.90000000 0.60000000 0.70000000 1.00000000 0.60000000 0.70000000 1.10000000 0.60000000 0.70000000 1.20000000 0.60000000 0.70000000 -0.00000000 0.73913964 0.73913964 -0.10513418 0.73398435 0.73398435 -0.20635303 0.72157041 0.72157041 -0.30409247 0.70939520 0.70939520 -0.40141285 0.70244509 0.70244509 -0.50023117 0.70032266 0.70032266 -0.60001308 0.70001539 0.70001539 -0.70000010 0.70000010 0.70000010 -0.80000000 0.70000000 0.70000000 +0.00000000 0.70606151 0.70606151 +0.10069657 0.70529997 0.70529997 +0.20091722 0.70350656 0.70350656 +0.30066473 0.70169821 0.70169821 +0.40029574 0.70056455 0.70056455 +0.50007695 0.70011636 0.70011636 +0.60001016 0.70001248 0.70001248 +0.70000048 0.70000048 0.70000048 +0.80000001 0.70000000 0.70000000 0.90000000 0.70000000 0.70000000 1.00000000 0.70000000 0.70000000 1.10000000 0.70000000 0.70000000 1.20000000 0.70000000 0.70000000 -0.00000000 0.80329722 0.70269171 -0.10034158 0.80271483 0.70223247 -0.20037105 0.80144842 0.70121158 -0.30016913 0.80044005 0.70037406 -0.40003218 0.80006445 0.70005521 -0.50000213 0.80000353 0.70000302 -0.60000003 0.80000004 0.70000003 -0.70000000 0.80000000 0.70000000 +0.00000000 0.80048441 0.70038645 +0.10004229 0.80041163 0.70032903 +0.20005147 0.80024881 0.70019990 +0.30003203 0.80010234 0.70008274 +0.40001118 0.80002657 0.70002159 +0.50000206 0.80000386 0.70000315 +0.60000017 0.80000025 0.70000021 +0.70000000 0.80000001 0.70000000 0.80000000 0.80000000 0.70000000 0.90000000 0.80000000 0.70000000 1.00000000 0.80000000 0.70000000 1.10000000 0.80000000 0.70000000 1.20000000 0.80000000 0.70000000 -0.00000000 0.90002831 0.70002079 -0.10000221 0.90002082 0.70001527 -0.20000164 0.90000795 0.70000581 -0.30000042 0.90000138 0.70000101 -0.40000003 0.90000008 0.70000006 -0.50000000 0.90000000 0.70000000 +0.00000000 0.90000869 0.70000592 +0.10000056 0.90000706 0.70000480 +0.20000060 0.90000372 0.70000252 +0.30000030 0.90000122 0.70000082 +0.40000007 0.90000022 0.70000015 +0.50000001 0.90000002 0.70000001 0.60000000 0.90000000 0.70000000 0.70000000 0.90000000 0.70000000 0.80000000 0.90000000 0.70000000 @@ -1317,8 +1317,8 @@ POINTS 2197 float 1.00000000 0.90000000 0.70000000 1.10000000 0.90000000 0.70000000 1.20000000 0.90000000 0.70000000 -0.00000000 1.00000000 0.70000000 -0.10000000 1.00000000 0.70000000 +0.00000000 1.00000001 0.70000001 +0.10000000 1.00000001 0.70000001 0.20000000 1.00000000 0.70000000 0.30000000 1.00000000 0.70000000 0.40000000 1.00000000 0.70000000 @@ -1356,116 +1356,116 @@ POINTS 2197 float 1.00000000 1.20000000 0.70000000 1.10000000 1.20000000 0.70000000 1.20000000 1.20000000 0.70000000 -0.00000000 0.00000000 1.01908611 -0.13287109 0.00000000 1.01357716 -0.25193669 0.00000000 0.98673239 -0.35904395 0.00000000 0.95438813 -0.45894538 0.00000000 0.92067108 -0.54716000 0.00000000 0.87929698 -0.62120277 0.00000000 0.82986250 -0.70269171 0.00000000 0.80329722 -0.80011294 0.00000000 0.80011294 -0.90000003 0.00000000 0.80000002 +0.00000000 0.00000000 0.93342035 +0.11825295 0.00000000 0.93237960 +0.23188170 0.00000000 0.92090868 +0.33342199 0.00000000 0.89129645 +0.42453094 0.00000000 0.85278975 +0.51151186 0.00000000 0.82018154 +0.60317337 0.00000000 0.80468955 +0.70038645 0.00000000 0.80048441 +0.80001728 0.00000000 0.80001728 +0.90000008 0.00000000 0.80000006 1.00000000 0.00000000 0.80000000 1.10000000 0.00000000 0.80000000 1.20000000 0.00000000 0.80000000 -0.00000000 0.13287109 1.01357716 -0.13066508 0.13066508 1.00656530 -0.24916134 0.12455660 0.98023176 -0.35679227 0.11879272 0.94953048 -0.45674548 0.11417539 0.91652877 -0.54424669 0.10890351 0.87441298 -0.61873750 0.10310482 0.82642640 -0.70223247 0.10034158 0.80271483 -0.80007988 0.10001129 0.80007988 -0.90000002 0.10000000 0.80000002 +0.00000000 0.11825295 0.93237960 +0.11763216 0.11763216 0.93016718 +0.22990962 0.11471946 0.91558385 +0.33096079 0.11001986 0.88558560 +0.42230988 0.10530144 0.84822886 +0.51027912 0.10195059 0.81808075 +0.60277408 0.10043095 0.80410674 +0.70032903 0.10004229 0.80041163 +0.80001404 0.10000141 0.80001404 +0.90000006 0.10000000 0.80000005 1.00000000 0.10000000 0.80000000 1.10000000 0.10000000 0.80000000 1.20000000 0.10000000 0.80000000 -0.00000000 0.25193669 0.98673239 -0.12455660 0.24916134 0.98023176 -0.24190248 0.24190248 0.96161127 -0.35040098 0.23356272 0.93512570 -0.44979284 0.22492296 0.90311681 -0.53560066 0.21419343 0.85996942 -0.61254377 0.20417332 0.81767238 -0.70121158 0.20037105 0.80144842 -0.80002681 0.20000706 0.80002681 -0.90000001 0.20000000 0.80000001 +0.00000000 0.23188170 0.92090868 +0.11471946 0.22990962 0.91558385 +0.22397871 0.22397871 0.89766595 +0.32412620 0.21581865 0.86874877 +0.41650187 0.20796403 0.83612377 +0.50723678 0.20277440 0.81284737 +0.60182509 0.20057061 0.80271309 +0.70019990 0.20005147 0.80024881 +0.80000742 0.20000150 0.80000742 +0.90000003 0.20000000 0.80000002 1.00000000 0.20000000 0.80000000 1.10000000 0.20000000 0.80000000 1.20000000 0.20000000 0.80000000 -0.00000000 0.35904395 0.95438813 -0.11879272 0.35679227 0.94953048 -0.23356272 0.35040098 0.93512570 -0.34124155 0.34124155 0.91301195 -0.43790516 0.32839894 0.87928885 -0.52260788 0.31343113 0.83815065 -0.60564149 0.30285894 0.80785710 -0.70037406 0.30016913 0.80044005 -0.80000384 0.30000142 0.80000384 -0.90000000 0.30000000 0.80000000 +0.00000000 0.33342199 0.89129645 +0.11001986 0.33096079 0.88558560 +0.21581865 0.32412620 0.86874877 +0.31505303 0.31505303 0.84436725 +0.40958221 0.30706461 0.82134888 +0.50386270 0.30224793 0.80694211 +0.60086304 0.30040857 0.80128826 +0.70008274 0.30003203 0.80010234 +0.80000246 0.30000075 0.80000246 +0.90000001 0.30000000 0.80000001 1.00000000 0.30000000 0.80000000 1.10000000 0.30000000 0.80000000 1.20000000 0.30000000 0.80000000 -0.00000000 0.45894538 0.92067108 -0.11417539 0.45674548 0.91652877 -0.22492296 0.44979284 0.90311681 -0.32839894 0.43790516 0.87928885 -0.42186536 0.42186536 0.84633911 -0.50953910 0.40759358 0.81599699 -0.60140266 0.40095016 0.80192020 -0.70005521 0.40003218 0.80006445 -0.80000021 0.40000010 0.80000021 +0.00000000 0.42453094 0.85278975 +0.10530144 0.42230988 0.84822886 +0.20796403 0.41650187 0.83612377 +0.30706461 0.40958221 0.82134888 +0.40407424 0.40407424 0.80925855 +0.50144764 0.40113819 0.80263150 +0.60027207 0.40017365 0.80040738 +0.70002159 0.40001118 0.80002657 +0.80000047 0.40000019 0.80000047 0.90000000 0.40000000 0.80000000 1.00000000 0.40000000 0.80000000 1.10000000 0.40000000 0.80000000 1.20000000 0.40000000 0.80000000 -0.00000000 0.54716000 0.87929698 -0.10890351 0.54424669 0.87441298 -0.21419343 0.53560066 0.85996942 -0.31343113 0.52260788 0.83815065 -0.40759358 0.50953910 0.81599699 -0.50189877 0.50189877 0.80311288 -0.60016189 0.50013560 0.80022001 -0.70000302 0.50000213 0.80000353 -0.80000000 0.50000000 0.80000000 +0.00000000 0.51151186 0.82018154 +0.10195059 0.51027912 0.81808075 +0.20277440 0.50723678 0.81284737 +0.30224793 0.50386270 0.80694211 +0.40113819 0.50144764 0.80263150 +0.50033768 0.50033768 0.80062010 +0.60005148 0.50004167 0.80007731 +0.70000315 0.50000206 0.80000386 +0.80000005 0.50000002 0.80000005 0.90000000 0.50000000 0.80000000 1.00000000 0.50000000 0.80000000 1.10000000 0.50000000 0.80000000 1.20000000 0.50000000 0.80000000 -0.00000000 0.62120277 0.82986250 -0.10310482 0.61873750 0.82642640 -0.20417332 0.61254377 0.81767238 -0.30285894 0.60564149 0.80785710 -0.40095016 0.60140266 0.80192020 -0.50013560 0.60016189 0.80022001 -0.60000594 0.60000594 0.80000815 -0.70000003 0.60000003 0.80000004 +0.00000000 0.60317337 0.80468955 +0.10043095 0.60277408 0.80410674 +0.20057061 0.60182509 0.80271309 +0.30040857 0.60086304 0.80128826 +0.40017365 0.60027207 0.80040738 +0.50004167 0.60005148 0.80007731 +0.60000495 0.60000495 0.80000745 +0.70000021 0.60000017 0.80000025 0.80000000 0.60000000 0.80000000 0.90000000 0.60000000 0.80000000 1.00000000 0.60000000 0.80000000 1.10000000 0.60000000 0.80000000 1.20000000 0.60000000 0.80000000 -0.00000000 0.70269171 0.80329722 -0.10034158 0.70223247 0.80271483 -0.20037105 0.70121158 0.80144842 -0.30016913 0.70037406 0.80044005 -0.40003218 0.70005521 0.80006445 -0.50000213 0.70000302 0.80000353 -0.60000003 0.70000003 0.80000004 -0.70000000 0.70000000 0.80000000 +0.00000000 0.70038645 0.80048441 +0.10004229 0.70032903 0.80041163 +0.20005147 0.70019990 0.80024881 +0.30003203 0.70008274 0.80010234 +0.40001118 0.70002159 0.80002657 +0.50000206 0.70000315 0.80000386 +0.60000017 0.70000021 0.80000025 +0.70000000 0.70000000 0.80000001 0.80000000 0.70000000 0.80000000 0.90000000 0.70000000 0.80000000 1.00000000 0.70000000 0.80000000 1.10000000 0.70000000 0.80000000 1.20000000 0.70000000 0.80000000 -0.00000000 0.80011294 0.80011294 -0.10001129 0.80007988 0.80007988 -0.20000706 0.80002681 0.80002681 -0.30000142 0.80000384 0.80000384 -0.40000010 0.80000021 0.80000021 -0.50000000 0.80000000 0.80000000 +0.00000000 0.80001728 0.80001728 +0.10000141 0.80001404 0.80001404 +0.20000150 0.80000742 0.80000742 +0.30000075 0.80000246 0.80000246 +0.40000019 0.80000047 0.80000047 +0.50000002 0.80000005 0.80000005 0.60000000 0.80000000 0.80000000 0.70000000 0.80000000 0.80000000 0.80000000 0.80000000 0.80000000 @@ -1473,10 +1473,10 @@ POINTS 2197 float 1.00000000 0.80000000 0.80000000 1.10000000 0.80000000 0.80000000 1.20000000 0.80000000 0.80000000 -0.00000000 0.90000003 0.80000002 -0.10000000 0.90000002 0.80000002 -0.20000000 0.90000001 0.80000001 -0.30000000 0.90000000 0.80000000 +0.00000000 0.90000008 0.80000006 +0.10000000 0.90000006 0.80000005 +0.20000000 0.90000003 0.80000002 +0.30000000 0.90000001 0.80000001 0.40000000 0.90000000 0.80000000 0.50000000 0.90000000 0.80000000 0.60000000 0.90000000 0.80000000 @@ -1525,103 +1525,103 @@ POINTS 2197 float 1.00000000 1.20000000 0.80000000 1.10000000 1.20000000 0.80000000 1.20000000 1.20000000 0.80000000 -0.00000000 0.00000000 1.05975035 -0.12283560 0.00000000 1.05399342 -0.23617460 0.00000000 1.02993988 -0.33668601 0.00000000 0.99927356 -0.42696076 0.00000000 0.95881249 -0.50972929 0.00000000 0.91747243 -0.60103535 0.00000000 0.90158256 -0.70002079 0.00000000 0.90002831 -0.80000002 0.00000000 0.90000003 +0.00000000 0.00000000 0.98075140 +0.10846348 0.00000000 0.97137066 +0.21117043 0.00000000 0.94671483 +0.30763684 0.00000000 0.92259213 +0.40340321 0.00000000 0.90807622 +0.50091570 0.00000000 0.90183755 +0.60011921 0.00000000 0.90020391 +0.70000592 0.00000000 0.90000869 +0.80000006 0.00000000 0.90000008 0.90000000 0.00000000 0.90000000 1.00000000 0.00000000 0.90000000 1.10000000 0.00000000 0.90000000 1.20000000 0.00000000 0.90000000 -0.00000000 0.12283560 1.05399342 -0.12143171 0.12143171 1.04745861 -0.23383545 0.11668098 1.02428904 -0.33427106 0.11124799 0.99386503 -0.42424467 0.10595209 0.95313141 -0.50818514 0.10161500 0.91472437 -0.60081694 0.10014389 0.90125052 -0.70001527 0.10000221 0.90002082 -0.80000002 0.10000000 0.90000002 +0.00000000 0.10846348 0.97137066 +0.10740242 0.10740242 0.96253456 +0.20960508 0.10468196 0.94064722 +0.30659998 0.10213886 0.91972682 +0.40293226 0.10070420 0.90700443 +0.50077499 0.10014558 0.90155996 +0.60009895 0.10001488 0.90016967 +0.70000480 0.10000056 0.90000706 +0.80000005 0.10000000 0.90000006 0.90000000 0.10000000 0.90000000 1.00000000 0.10000000 0.90000000 1.10000000 0.10000000 0.90000000 1.20000000 0.10000000 0.90000000 -0.00000000 0.23617460 1.02993988 -0.11668098 0.23383545 1.02428904 -0.22701033 0.22701033 1.00671014 -0.32704436 0.21784235 0.97657491 -0.41704032 0.20834384 0.93777007 -0.50466014 0.20186981 0.90841516 -0.60038862 0.20013549 0.90059787 -0.70000581 0.20000164 0.90000795 -0.80000001 0.20000000 0.90000001 +0.00000000 0.21117043 0.94671483 +0.10468196 0.20960508 0.94064722 +0.20605740 0.20605740 0.92664749 +0.30421784 0.20277513 0.91300074 +0.40184411 0.20089456 0.90449165 +0.50046210 0.20017523 0.90093920 +0.60005581 0.20001690 0.90009639 +0.70000252 0.20000060 0.90000372 +0.80000002 0.20000000 0.90000003 0.90000000 0.20000000 0.90000000 1.00000000 0.20000000 0.90000000 1.10000000 0.20000000 0.90000000 1.20000000 0.20000000 0.90000000 -0.00000000 0.33668601 0.99927356 -0.11124799 0.33427106 0.99386503 -0.21784235 0.32704436 0.97657491 -0.31627253 0.31627253 0.94805255 -0.40823181 0.30610938 0.91846729 -0.50158998 0.30096622 0.90288377 -0.60010177 0.30005217 0.90015817 -0.70000101 0.30000042 0.90000138 -0.80000000 0.30000000 0.90000000 +0.00000000 0.30763684 0.92259213 +0.10213886 0.30659998 0.91972682 +0.20277513 0.30421784 0.91300074 +0.30193390 0.30193390 0.90624407 +0.40080598 0.30059446 0.90202252 +0.50018544 0.30010681 0.90038347 +0.60002051 0.30000941 0.90003585 +0.70000082 0.30000030 0.90000122 +0.80000001 0.30000000 0.90000001 0.90000000 0.30000000 0.90000000 1.00000000 0.30000000 0.90000000 1.10000000 0.30000000 0.90000000 1.20000000 0.30000000 0.90000000 -0.00000000 0.42696076 0.95881249 -0.10595209 0.42424467 0.95313141 -0.20834384 0.41704032 0.93777007 -0.30610938 0.40823181 0.91846729 -0.40210769 0.40210769 0.90474355 -0.50028459 0.40022936 0.90052141 -0.60001237 0.40000825 0.90001954 -0.70000006 0.40000003 0.90000008 +0.00000000 0.40340321 0.90807622 +0.10070420 0.40293226 0.90700443 +0.20089456 0.40184411 0.90449165 +0.30059446 0.40080598 0.90202252 +0.40022599 0.40022599 0.90058837 +0.50004666 0.40003640 0.90009900 +0.60000456 0.40000283 0.90000809 +0.70000015 0.40000007 0.90000022 0.80000000 0.40000000 0.90000000 0.90000000 0.40000000 0.90000000 1.00000000 0.40000000 0.90000000 1.10000000 0.40000000 0.90000000 1.20000000 0.40000000 0.90000000 -0.00000000 0.50972929 0.91747243 -0.10161500 0.50818514 0.91472437 -0.20186981 0.50466014 0.90841516 -0.30096622 0.50158998 0.90288377 -0.40022936 0.50028459 0.90052141 -0.50002147 0.50002147 0.90004039 -0.60000048 0.50000040 0.90000077 -0.70000000 0.50000000 0.90000000 +0.00000000 0.50091570 0.90183755 +0.10014558 0.50077499 0.90155996 +0.20017523 0.50046210 0.90093920 +0.30010681 0.50018544 0.90038347 +0.40003640 0.50004666 0.90009900 +0.50000661 0.50000661 0.90001447 +0.60000054 0.50000043 0.90000097 +0.70000001 0.50000001 0.90000002 0.80000000 0.50000000 0.90000000 0.90000000 0.50000000 0.90000000 1.00000000 0.50000000 0.90000000 1.10000000 0.50000000 0.90000000 1.20000000 0.50000000 0.90000000 -0.00000000 0.60103535 0.90158256 -0.10014389 0.60081694 0.90125052 -0.20013549 0.60038862 0.90059787 -0.30005217 0.60010177 0.90015817 -0.40000825 0.60001237 0.90001954 -0.50000040 0.60000048 0.90000077 -0.60000000 0.60000000 0.90000001 +0.00000000 0.60011921 0.90020391 +0.10001488 0.60009895 0.90016967 +0.20001690 0.60005581 0.90009639 +0.30000941 0.60002051 0.90003585 +0.40000283 0.60000456 0.90000809 +0.50000043 0.60000054 0.90000097 +0.60000003 0.60000003 0.90000005 0.70000000 0.60000000 0.90000000 0.80000000 0.60000000 0.90000000 0.90000000 0.60000000 0.90000000 1.00000000 0.60000000 0.90000000 1.10000000 0.60000000 0.90000000 1.20000000 0.60000000 0.90000000 -0.00000000 0.70002079 0.90002831 -0.10000221 0.70001527 0.90002082 -0.20000164 0.70000581 0.90000795 -0.30000042 0.70000101 0.90000138 -0.40000003 0.70000006 0.90000008 -0.50000000 0.70000000 0.90000000 +0.00000000 0.70000592 0.90000869 +0.10000056 0.70000480 0.90000706 +0.20000060 0.70000252 0.90000372 +0.30000030 0.70000082 0.90000122 +0.40000007 0.70000015 0.90000022 +0.50000001 0.70000001 0.90000002 0.60000000 0.70000000 0.90000000 0.70000000 0.70000000 0.90000000 0.80000000 0.70000000 0.90000000 @@ -1629,10 +1629,10 @@ POINTS 2197 float 1.00000000 0.70000000 0.90000000 1.10000000 0.70000000 0.90000000 1.20000000 0.70000000 0.90000000 -0.00000000 0.80000002 0.90000003 -0.10000000 0.80000002 0.90000002 -0.20000000 0.80000001 0.90000001 -0.30000000 0.80000000 0.90000000 +0.00000000 0.80000006 0.90000008 +0.10000000 0.80000005 0.90000006 +0.20000000 0.80000002 0.90000003 +0.30000000 0.80000001 0.90000001 0.40000000 0.80000000 0.90000000 0.50000000 0.80000000 0.90000000 0.60000000 0.80000000 0.90000000 @@ -1694,77 +1694,77 @@ POINTS 2197 float 1.00000000 1.20000000 0.90000000 1.10000000 1.20000000 0.90000000 1.20000000 1.20000000 0.90000000 -0.00000000 0.00000000 1.10930635 -0.11378558 0.00000000 1.09851927 -0.21674437 0.00000000 1.06148154 -0.30867229 0.00000000 1.02383514 -0.40189866 0.00000000 1.00427903 -0.50015749 0.00000000 1.00030406 -0.60000199 0.00000000 1.00000345 -0.70000000 0.00000000 1.00000000 +0.00000000 0.00000000 1.00853411 +0.10070084 0.00000000 1.00705897 +0.20075589 0.00000000 1.00382370 +0.30035799 0.00000000 1.00125619 +0.40008808 0.00000000 1.00024504 +0.50001129 0.00000000 1.00002625 +0.60000057 0.00000000 1.00000113 +0.70000001 0.00000000 1.00000001 0.80000000 0.00000000 1.00000000 0.90000000 0.00000000 1.00000000 1.00000000 0.00000000 1.00000000 1.10000000 0.00000000 1.00000000 1.20000000 0.00000000 1.00000000 -0.00000000 0.11378558 1.09851927 -0.11202892 0.11202892 1.08708267 -0.21405368 0.10675696 1.05283153 -0.30706878 0.10223985 1.01968452 -0.40148402 0.10037558 1.00335672 -0.50011671 0.10002438 1.00022657 -0.60000138 0.10000021 1.00000241 -0.70000000 0.10000000 1.00000000 +0.00000000 0.10070084 1.00705897 +0.10057397 0.10057397 1.00581860 +0.20061325 0.10029978 1.00312991 +0.30028977 0.10009212 1.00102408 +0.40007112 0.10001649 1.00019883 +0.50000903 0.10000158 1.00002108 +0.60000045 0.10000006 1.00000089 +0.70000001 0.10000000 1.00000001 0.80000000 0.10000000 1.00000000 0.90000000 0.10000000 1.00000000 1.00000000 0.10000000 1.00000000 1.10000000 0.10000000 1.00000000 1.20000000 0.10000000 1.00000000 -0.00000000 0.21674437 1.06148154 -0.10675696 0.21405368 1.05283153 -0.20766246 0.20766246 1.03110077 -0.30359058 0.20235762 1.01034139 -0.40067510 0.20034203 1.00154830 -0.50004540 0.20001849 1.00008969 -0.60000044 0.20000014 1.00000077 +0.00000000 0.20075589 1.00382370 +0.10029978 0.20061325 1.00312991 +0.20031639 0.20031639 1.00166449 +0.30015002 0.20009743 1.00054285 +0.40003672 0.20001732 1.00010437 +0.50000454 0.20000162 1.00001074 +0.60000021 0.20000006 1.00000043 0.70000000 0.20000000 1.00000000 0.80000000 0.20000000 1.00000000 0.90000000 0.20000000 1.00000000 1.00000000 0.20000000 1.00000000 1.10000000 0.20000000 1.00000000 1.20000000 0.20000000 1.00000000 -0.00000000 0.30867229 1.02383514 -0.10223985 0.30706878 1.01968452 -0.20235762 0.30359058 1.01034139 -0.30096251 0.30096251 1.00287541 -0.40015870 0.30011969 1.00037561 -0.50000815 0.30000484 1.00001657 -0.60000005 0.30000003 1.00000009 +0.00000000 0.30035799 1.00125619 +0.10009212 0.30028977 1.00102408 +0.20009743 0.30015002 1.00054285 +0.30004694 0.30004694 1.00017773 +0.40001148 0.30000831 1.00003371 +0.50000136 0.30000075 1.00000329 +0.60000006 0.30000002 1.00000012 0.70000000 0.30000000 1.00000000 0.80000000 0.30000000 1.00000000 0.90000000 0.30000000 1.00000000 1.00000000 0.30000000 1.00000000 1.10000000 0.30000000 1.00000000 1.20000000 0.30000000 1.00000000 -0.00000000 0.40189866 1.00427903 -0.10037558 0.40148402 1.00335672 -0.20034203 0.40067510 1.00154830 -0.30011969 0.40015870 1.00037561 -0.40001565 0.40001565 1.00003915 -0.50000051 0.40000040 1.00000107 -0.60000000 0.40000000 1.00000000 +0.00000000 0.40008808 1.00024504 +0.10001649 0.40007112 1.00019883 +0.20001732 0.40003672 1.00010437 +0.30000831 0.40001148 1.00003371 +0.40000200 0.40000200 1.00000616 +0.50000022 0.40000017 1.00000055 +0.60000001 0.40000000 1.00000002 0.70000000 0.40000000 1.00000000 0.80000000 0.40000000 1.00000000 0.90000000 0.40000000 1.00000000 1.00000000 0.40000000 1.00000000 1.10000000 0.40000000 1.00000000 1.20000000 0.40000000 1.00000000 -0.00000000 0.50015749 1.00030406 -0.10002438 0.50011671 1.00022657 -0.20001849 0.50004540 1.00008969 -0.30000484 0.50000815 1.00001657 -0.40000040 0.50000051 1.00000107 -0.50000001 0.50000001 1.00000001 +0.00000000 0.50001129 1.00002625 +0.10000158 0.50000903 1.00002108 +0.20000162 0.50000454 1.00001074 +0.30000075 0.50000136 1.00000329 +0.40000017 0.50000022 1.00000055 +0.50000002 0.50000002 1.00000004 0.60000000 0.50000000 1.00000000 0.70000000 0.50000000 1.00000000 0.80000000 0.50000000 1.00000000 @@ -1772,11 +1772,11 @@ POINTS 2197 float 1.00000000 0.50000000 1.00000000 1.10000000 0.50000000 1.00000000 1.20000000 0.50000000 1.00000000 -0.00000000 0.60000199 1.00000345 -0.10000021 0.60000138 1.00000241 -0.20000014 0.60000044 1.00000077 -0.30000003 0.60000005 1.00000009 -0.40000000 0.60000000 1.00000000 +0.00000000 0.60000057 1.00000113 +0.10000006 0.60000045 1.00000089 +0.20000006 0.60000021 1.00000043 +0.30000002 0.60000006 1.00000012 +0.40000000 0.60000001 1.00000002 0.50000000 0.60000000 1.00000000 0.60000000 0.60000000 1.00000000 0.70000000 0.60000000 1.00000000 @@ -1785,8 +1785,8 @@ POINTS 2197 float 1.00000000 0.60000000 1.00000000 1.10000000 0.60000000 1.00000000 1.20000000 0.60000000 1.00000000 -0.00000000 0.70000000 1.00000000 -0.10000000 0.70000000 1.00000000 +0.00000000 0.70000001 1.00000001 +0.10000000 0.70000001 1.00000001 0.20000000 0.70000000 1.00000000 0.30000000 0.70000000 1.00000000 0.40000000 0.70000000 1.00000000 @@ -1863,12 +1863,12 @@ POINTS 2197 float 1.00000000 1.20000000 1.00000000 1.10000000 1.20000000 1.00000000 1.20000000 1.20000000 1.00000000 -0.00000000 0.00000000 1.11637892 -0.10152648 0.00000000 1.11184016 -0.20081847 0.00000000 1.10311427 -0.30009933 0.00000000 1.10029021 -0.40000322 0.00000000 1.10000820 -0.50000002 0.00000000 1.10000004 +0.00000000 0.00000000 1.10006517 +0.10000329 0.00000000 1.10005103 +0.20000311 0.00000000 1.10002348 +0.30000115 0.00000000 1.10000554 +0.40000015 0.00000000 1.10000051 +0.50000001 0.00000000 1.10000002 0.60000000 0.00000000 1.10000000 0.70000000 0.00000000 1.10000000 0.80000000 0.00000000 1.10000000 @@ -1876,12 +1876,12 @@ POINTS 2197 float 1.00000000 0.00000000 1.10000000 1.10000000 0.00000000 1.10000000 1.20000000 0.00000000 1.10000000 -0.00000000 0.10152648 1.11184016 -0.10110022 0.10110022 1.10834589 -0.20056038 0.10028399 1.10216182 -0.30006789 0.10002269 1.10020126 -0.40000217 0.10000050 1.10000555 -0.50000001 0.10000000 1.10000003 +0.00000000 0.10000329 1.10005103 +0.10000256 0.10000256 1.10003983 +0.20000240 0.10000116 1.10001815 +0.30000088 0.10000026 1.10000422 +0.40000011 0.10000002 1.10000039 +0.50000000 0.10000000 1.10000001 0.60000000 0.10000000 1.10000000 0.70000000 0.10000000 1.10000000 0.80000000 0.10000000 1.10000000 @@ -1889,11 +1889,11 @@ POINTS 2197 float 1.00000000 0.10000000 1.10000000 1.10000000 0.10000000 1.10000000 1.20000000 0.10000000 1.10000000 -0.00000000 0.20081847 1.10311427 -0.10028399 0.20056038 1.10216182 -0.20015072 0.20015072 1.10062140 -0.30001958 0.20001291 1.10006116 -0.40000060 0.20000028 1.10000159 +0.00000000 0.20000311 1.10002348 +0.10000116 0.20000240 1.10001815 +0.20000106 0.20000106 1.10000805 +0.30000037 0.20000023 1.10000181 +0.40000005 0.20000002 1.10000017 0.50000000 0.20000000 1.10000001 0.60000000 0.20000000 1.10000000 0.70000000 0.20000000 1.10000000 @@ -1902,11 +1902,11 @@ POINTS 2197 float 1.00000000 0.20000000 1.10000000 1.10000000 0.20000000 1.10000000 1.20000000 0.20000000 1.10000000 -0.00000000 0.30009933 1.10029021 -0.10002269 0.30006789 1.10020126 -0.20001291 0.30001958 1.10006116 -0.30000187 0.30000187 1.10000642 -0.40000005 0.30000004 1.10000015 +0.00000000 0.30000115 1.10000554 +0.10000026 0.30000088 1.10000422 +0.20000023 0.30000037 1.10000181 +0.30000008 0.30000008 1.10000040 +0.40000001 0.30000001 1.10000004 0.50000000 0.30000000 1.10000000 0.60000000 0.30000000 1.10000000 0.70000000 0.30000000 1.10000000 @@ -1915,10 +1915,10 @@ POINTS 2197 float 1.00000000 0.30000000 1.10000000 1.10000000 0.30000000 1.10000000 1.20000000 0.30000000 1.10000000 -0.00000000 0.40000322 1.10000820 -0.10000050 0.40000217 1.10000555 -0.20000028 0.40000060 1.10000159 -0.30000004 0.40000005 1.10000015 +0.00000000 0.40000015 1.10000051 +0.10000002 0.40000011 1.10000039 +0.20000002 0.40000005 1.10000017 +0.30000001 0.40000001 1.10000004 0.40000000 0.40000000 1.10000000 0.50000000 0.40000000 1.10000000 0.60000000 0.40000000 1.10000000 @@ -1928,8 +1928,8 @@ POINTS 2197 float 1.00000000 0.40000000 1.10000000 1.10000000 0.40000000 1.10000000 1.20000000 0.40000000 1.10000000 -0.00000000 0.50000002 1.10000004 -0.10000000 0.50000001 1.10000003 +0.00000000 0.50000001 1.10000002 +0.10000000 0.50000000 1.10000001 0.20000000 0.50000000 1.10000001 0.30000000 0.50000000 1.10000000 0.40000000 0.50000000 1.10000000 @@ -2032,10 +2032,10 @@ POINTS 2197 float 1.00000000 1.20000000 1.10000000 1.10000000 1.20000000 1.10000000 1.20000000 1.20000000 1.10000000 -0.00000000 0.00000000 1.20000000 -0.10000052 0.00000000 1.20000000 -0.20000021 0.00000000 1.20000000 -0.30000001 0.00000000 1.20000000 +0.00000000 0.00000000 1.20000001 +0.10000000 0.00000000 1.20000001 +0.20000000 0.00000000 1.20000000 +0.30000000 0.00000000 1.20000000 0.40000000 0.00000000 1.20000000 0.50000000 0.00000000 1.20000000 0.60000000 0.00000000 1.20000000 @@ -2045,9 +2045,9 @@ POINTS 2197 float 1.00000000 0.00000000 1.20000000 1.10000000 0.00000000 1.20000000 1.20000000 0.00000000 1.20000000 -0.00000000 0.10000052 1.20000000 -0.10000032 0.10000032 1.20000000 -0.20000013 0.10000006 1.20000000 +0.00000000 0.10000000 1.20000001 +0.10000000 0.10000000 1.20000000 +0.20000000 0.10000000 1.20000000 0.30000000 0.10000000 1.20000000 0.40000000 0.10000000 1.20000000 0.50000000 0.10000000 1.20000000 @@ -2058,9 +2058,9 @@ POINTS 2197 float 1.00000000 0.10000000 1.20000000 1.10000000 0.10000000 1.20000000 1.20000000 0.10000000 1.20000000 -0.00000000 0.20000021 1.20000000 -0.10000006 0.20000013 1.20000000 -0.20000002 0.20000002 1.20000000 +0.00000000 0.20000000 1.20000000 +0.10000000 0.20000000 1.20000000 +0.20000000 0.20000000 1.20000000 0.30000000 0.20000000 1.20000000 0.40000000 0.20000000 1.20000000 0.50000000 0.20000000 1.20000000 @@ -2071,7 +2071,7 @@ POINTS 2197 float 1.00000000 0.20000000 1.20000000 1.10000000 0.20000000 1.20000000 1.20000000 0.20000000 1.20000000 -0.00000000 0.30000001 1.20000000 +0.00000000 0.30000000 1.20000000 0.10000000 0.30000000 1.20000000 0.20000000 0.30000000 1.20000000 0.30000000 0.30000000 1.20000000 @@ -5665,1010 +5665,1165 @@ CELL_TYPES 1728 POINT_DATA 2197 VECTORS velocity float 0.00000000 0.00000000 0.00000000 -0.15129286 0.00000000 0.00000000 -0.15323531 0.00000000 0.00000000 -0.18052624 0.00000000 0.00000000 -0.19922659 0.00000000 0.00000000 -0.21962195 0.00000000 0.00000000 -0.24196835 0.00000000 0.00000000 -0.26345823 0.00000000 0.00000000 -0.27808604 0.00000000 0.00000000 -0.35128500 0.00000000 0.00000000 -0.25113715 0.00000000 0.00000000 -0.38181582 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.15129286 0.00000000 -0.12109740 0.12109740 0.00000000 -0.16706916 0.08253341 0.00000000 -0.18507825 0.07255161 0.00000000 -0.20510401 0.06409840 0.00000000 -0.22557832 0.05784992 0.00000000 -0.24812410 0.05315190 0.00000000 -0.26935568 0.05031494 0.00000000 -0.29184205 0.05226268 0.00000000 -0.32796197 0.04779386 0.00000000 -0.34155637 0.05854756 0.00000000 -0.29245201 0.03622370 0.00000000 -0.00000000 0.00004101 0.00000000 -0.00000000 0.15323531 0.00000000 -0.08253341 0.16706916 0.00000000 -0.15129872 0.15129872 0.00000000 -0.18661483 0.12387973 0.00000000 -0.20744356 0.11091403 0.00000000 -0.23059254 0.09877480 0.00000000 -0.25292777 0.08951714 0.00000000 -0.27422715 0.08372217 0.00000000 -0.30646293 0.08576194 0.00000000 -0.28483792 0.07437695 0.00000000 -0.43878482 0.12180165 0.00000000 -0.08015231 0.02176584 0.00000000 -0.00000000 0.00001641 0.00000000 -0.00000000 0.18052624 0.00000000 -0.07255161 0.18507825 0.00000000 -0.12387973 0.18661483 0.00000000 -0.17156448 0.17156448 0.00000000 -0.20161696 0.14816375 0.00000000 -0.22568042 0.13415464 0.00000000 -0.24815988 0.12082245 0.00000000 -0.27809616 0.11779382 0.00000000 -0.28558967 0.10768783 0.00000000 -0.34061828 0.12922569 0.00000000 -0.32820362 0.11853629 0.00000000 -0.00927076 0.00327665 0.00000000 -0.00000000 0.00000048 0.00000000 -0.00000000 0.19922659 0.00000000 -0.06409840 0.20510401 0.00000000 -0.11091403 0.20744356 0.00000000 -0.14816375 0.20161696 0.00000000 -0.18839642 0.18839642 0.00000000 -0.21255306 0.16465196 0.00000000 -0.24407471 0.15550298 0.00000000 -0.26798333 0.14539135 0.00000000 -0.26576622 0.13184502 0.00000000 -0.39499686 0.18176438 0.00000000 -0.09442587 0.04187089 0.00000000 -0.00039903 0.00015905 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.21962195 0.00000000 -0.05784992 0.22557832 0.00000000 -0.09877480 0.23059254 0.00000000 -0.13415464 0.22568042 0.00000000 -0.16465196 0.21255306 0.00000000 -0.19805544 0.19805544 0.00000000 -0.23584271 0.18550201 0.00000000 -0.22778616 0.15266555 0.00000000 -0.33580943 0.20439457 0.00000000 -0.24988166 0.13969963 0.00000000 -0.00901370 0.00475284 0.00000000 -0.00000239 0.00000109 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.24196835 0.00000000 -0.05315190 0.24812410 0.00000000 -0.08951714 0.25292777 0.00000000 -0.12082245 0.24815988 0.00000000 -0.15550298 0.24407471 0.00000000 -0.18550201 0.23584271 0.00000000 -0.19670448 0.19670448 0.00000000 -0.27828749 0.22263260 0.00000000 -0.29186871 0.20940311 0.00000000 -0.03569997 0.02352921 0.00000000 -0.00016711 0.00009720 0.00000000 +0.09766305 0.00000000 0.00000000 +0.16929662 0.00000000 0.00000000 +0.09816013 0.00000000 0.00000000 +0.18530258 0.00000000 0.00000000 +0.12907870 0.00000000 0.00000000 +0.15222460 0.00000000 0.00000000 +0.33608041 0.00000000 0.00000000 +0.15474041 0.00000000 0.00000000 +0.39190663 0.00000000 0.00000000 +0.11954782 0.00000000 0.00000000 +0.00188256 0.00000000 0.00000000 +0.00000038 0.00000000 0.00000000 +0.00000000 0.09766305 0.00000000 +0.09112542 0.09112542 0.00000000 +0.16029073 0.08810659 0.00000000 +0.12315027 0.08115370 0.00000000 +0.18891806 0.06735718 0.00000000 +0.14402780 0.05007529 0.00000000 +0.19510935 0.05207875 0.00000000 +0.31051616 0.04944178 0.00000000 +0.16486838 0.02848046 0.00000000 +0.38895062 0.04797836 0.00000000 +0.10084103 0.01107822 0.00000000 +0.00148809 0.00010373 0.00000000 +0.00000029 0.00000001 0.00000000 +0.00000000 0.16929662 0.00000000 +0.08810659 0.16029073 0.00000000 +0.12833447 0.12833447 0.00000000 +0.16789467 0.13307679 0.00000000 +0.17640997 0.10686954 0.00000000 +0.17975567 0.08405257 0.00000000 +0.25516702 0.09285334 0.00000000 +0.23606788 0.06959312 0.00000000 +0.24805991 0.07512958 0.00000000 +0.33411183 0.08172305 0.00000000 +0.05834002 0.01251291 0.00000000 +0.00070577 0.00010027 0.00000000 +0.00000012 0.00000001 0.00000000 +0.00000000 0.09816013 0.00000000 +0.08115370 0.12315027 0.00000000 +0.13307679 0.16789467 0.00000000 +0.16677680 0.16677680 0.00000000 +0.15632530 0.12968357 0.00000000 +0.22165644 0.13275711 0.00000000 +0.24409979 0.11502818 0.00000000 +0.19458299 0.08207251 0.00000000 +0.32929456 0.12600050 0.00000000 +0.20869991 0.07246411 0.00000000 +0.02175687 0.00652072 0.00000000 +0.00017693 0.00003859 0.00000000 +0.00000002 0.00000000 0.00000000 +0.00000000 0.18530258 0.00000000 +0.06735718 0.18891806 0.00000000 +0.10686954 0.17640997 0.00000000 +0.12968357 0.15632530 0.00000000 +0.16386492 0.16386492 0.00000000 +0.23048447 0.17164479 0.00000000 +0.19042150 0.11897298 0.00000000 +0.25411718 0.14138649 0.00000000 +0.31252545 0.14747692 0.00000000 +0.09327119 0.04045245 0.00000000 +0.00500877 0.00185769 0.00000000 +0.00001847 0.00000548 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.12907870 0.00000000 +0.05007529 0.14402780 0.00000000 +0.08405257 0.17975567 0.00000000 +0.13275711 0.22165644 0.00000000 +0.17164479 0.23048447 0.00000000 +0.17445287 0.17445287 0.00000000 +0.20178487 0.16081472 0.00000000 +0.28953384 0.19352975 0.00000000 +0.17194500 0.10055272 0.00000000 +0.02753161 0.01406926 0.00000000 +0.00065474 0.00028711 0.00000000 +0.00000073 0.00000027 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.15222460 0.00000000 +0.05207875 0.19510935 0.00000000 +0.09285334 0.25516702 0.00000000 +0.11502818 0.24409979 0.00000000 +0.11897298 0.19042150 0.00000000 +0.16081472 0.20178487 0.00000000 +0.23656506 0.23656506 0.00000000 +0.19815450 0.15572209 0.00000000 +0.05450403 0.03771792 0.00000000 +0.00390964 0.00232750 0.00000000 +0.00003538 0.00001797 0.00000000 0.00000001 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.26345823 0.00000000 -0.05031494 0.26935568 0.00000000 -0.08372217 0.27422715 0.00000000 -0.11779382 0.27809616 0.00000000 -0.14539135 0.26798333 0.00000000 -0.15266555 0.22778616 0.00000000 -0.22263260 0.27828749 0.00000000 -0.28020504 0.28020504 0.00000000 -0.06343861 0.05159365 0.00000000 -0.00103623 0.00076839 0.00000000 -0.00000017 0.00000011 0.00000000 +0.00000000 0.33608041 0.00000000 +0.04944178 0.31051616 0.00000000 +0.06959312 0.23606788 0.00000000 +0.08207251 0.19458299 0.00000000 +0.14138649 0.25411718 0.00000000 +0.19352975 0.28953384 0.00000000 +0.15572209 0.19815450 0.00000000 +0.06303014 0.06303014 0.00000000 +0.00803571 0.00645709 0.00000000 +0.00021725 0.00014962 0.00000000 +0.00000049 0.00000029 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.27808604 0.00000000 -0.05226268 0.29184205 0.00000000 -0.08576194 0.30646293 0.00000000 -0.10768783 0.28558967 0.00000000 -0.13184502 0.26576622 0.00000000 -0.20439457 0.33580943 0.00000000 -0.20940311 0.29186871 0.00000000 -0.05159365 0.06343861 0.00000000 -0.00335127 0.00335127 0.00000000 -0.00000143 0.00000119 0.00000000 +0.00000000 0.15474041 0.00000000 +0.02848046 0.16486838 0.00000000 +0.07512958 0.24805991 0.00000000 +0.12600050 0.32929456 0.00000000 +0.14747692 0.31252545 0.00000000 +0.10055272 0.17194500 0.00000000 +0.03771792 0.05450403 0.00000000 +0.00645709 0.00803571 0.00000000 +0.00038992 0.00038992 0.00000000 +0.00000273 0.00000219 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.35128500 0.00000000 -0.04779386 0.32796197 0.00000000 -0.07437695 0.28483792 0.00000000 -0.12922569 0.34061828 0.00000000 -0.18176438 0.39499686 0.00000000 -0.13969963 0.24988166 0.00000000 -0.02352921 0.03569997 0.00000000 -0.00076839 0.00103623 0.00000000 -0.00000119 0.00000143 0.00000000 +0.00000000 0.39190663 0.00000000 +0.04797836 0.38895062 0.00000000 +0.08172305 0.33411183 0.00000000 +0.07246411 0.20869991 0.00000000 +0.04045245 0.09327119 0.00000000 +0.01406926 0.02753161 0.00000000 +0.00232750 0.00390964 0.00000000 +0.00014962 0.00021725 0.00000000 +0.00000219 0.00000273 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.25113715 0.00000000 -0.05854756 0.34155637 0.00000000 -0.12180165 0.43878482 0.00000000 -0.11853629 0.32820362 0.00000000 -0.04187089 0.09442587 0.00000000 -0.00475284 0.00901370 0.00000000 -0.00009720 0.00016711 0.00000000 -0.00000011 0.00000017 0.00000000 +0.00000000 0.11954782 0.00000000 +0.01107822 0.10084103 0.00000000 +0.01251291 0.05834002 0.00000000 +0.00652072 0.02175687 0.00000000 +0.00185769 0.00500877 0.00000000 +0.00028711 0.00065474 0.00000000 +0.00001797 0.00003538 0.00000000 +0.00000029 0.00000049 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.38181582 0.00000000 -0.03622370 0.29245201 0.00000000 -0.02176584 0.08015231 0.00000000 -0.00327665 0.00927076 0.00000000 -0.00015905 0.00039903 0.00000000 -0.00000109 0.00000239 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00188256 0.00000000 +0.00010373 0.00148809 0.00000000 +0.00010027 0.00070577 0.00000000 +0.00003859 0.00017693 0.00000000 +0.00000548 0.00001847 0.00000000 +0.00000027 0.00000073 0.00000000 0.00000000 0.00000001 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00004101 0.00000000 0.00000000 -0.00001641 0.00000000 0.00000000 -0.00000048 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.15129286 -0.12109740 0.00000000 0.12109740 -0.16706916 0.00000000 0.08253341 -0.18507825 0.00000000 0.07255161 -0.20510401 0.00000000 0.06409840 -0.22557832 0.00000000 0.05784992 -0.24812410 0.00000000 0.05315190 -0.26935568 0.00000000 0.05031494 -0.29184205 0.00000000 0.05226268 -0.32796197 0.00000000 0.04779386 -0.34155637 0.00000000 0.05854756 -0.29245201 0.00000000 0.03622370 -0.00000000 0.00000000 0.00004101 -0.00000000 0.12109740 0.12109740 -0.11194433 0.11194433 0.11194433 -0.16763493 0.08447348 0.08447348 -0.18818094 0.06881720 0.06881720 -0.20793413 0.06125917 0.06125917 -0.22912333 0.05484865 0.05484865 -0.25170547 0.05018152 0.05018152 -0.27224313 0.04698754 0.04698754 -0.30077655 0.04897950 0.04897950 -0.30800102 0.04250931 0.04250931 -0.40165640 0.06440826 0.06440826 -0.21450057 0.02766431 0.02766431 -0.00000000 0.00002538 0.00002538 -0.00000000 0.16706916 0.08253341 -0.08447348 0.16763493 0.08447348 -0.14671206 0.14671206 0.07661738 -0.18541418 0.12388847 0.06505718 -0.20769496 0.10849415 0.05558558 -0.23095378 0.09698467 0.04975907 -0.25306158 0.08754438 0.04486165 -0.27552492 0.08255333 0.04216991 -0.30537290 0.08302718 0.04225252 -0.28730540 0.07481661 0.03779717 -0.42864392 0.11485281 0.05668998 -0.05712571 0.01532577 0.00770589 -0.00000000 0.00001000 0.00000477 -0.00000000 0.18507825 0.07255161 -0.06881720 0.18818094 0.06881720 -0.12388847 0.18541418 0.06505718 -0.16862699 0.16862699 0.05971093 -0.20078594 0.14850555 0.05284890 -0.22513003 0.13297281 0.04626450 -0.24786893 0.12034648 0.04185713 -0.27864928 0.11728339 0.04033039 -0.27970791 0.10485952 0.03571615 -0.35622202 0.13373729 0.04466998 -0.29339533 0.10428998 0.03244443 -0.00661588 0.00230056 0.00077655 -0.00000000 0.00000030 0.00000008 -0.00000000 0.20510401 0.06409840 -0.06125917 0.20793413 0.06125917 -0.10849415 0.20769496 0.05558558 -0.14850555 0.20078594 0.05284890 -0.18594880 0.18594880 0.04921526 -0.21173705 0.16466959 0.04398982 -0.24448037 0.15544591 0.04078769 -0.26429520 0.14306836 0.03713853 -0.27336582 0.13597825 0.03482393 -0.38848962 0.17779283 0.04530383 -0.07612365 0.03370150 0.00840840 -0.00027532 0.00010882 0.00002590 -0.00000000 0.00000000 0.00000000 -0.00000000 0.22557832 0.05784992 -0.05484865 0.22912333 0.05484865 -0.09698467 0.23095378 0.04975907 -0.13297281 0.22513003 0.04626450 -0.16466959 0.21173705 0.04398982 -0.19891678 0.19891678 0.04227551 -0.23331275 0.18492540 0.03964287 -0.22853000 0.15384842 0.03262551 -0.34080524 0.20669146 0.04322289 -0.22244465 0.12399210 0.02378979 -0.00688265 0.00360923 0.00076146 -0.00000159 0.00000072 0.00000012 -0.00000000 0.00000000 0.00000000 -0.00000000 0.24812410 0.05315190 -0.05018152 0.25170547 0.05018152 -0.08754438 0.25306158 0.04486165 -0.12034648 0.24786893 0.04185713 -0.15544591 0.24448037 0.04078769 -0.18492540 0.23331275 0.03964287 -0.19466515 0.19466515 0.03375700 -0.28234193 0.22783320 0.04143619 -0.27650887 0.19774171 0.03245509 -0.02890315 0.01903327 0.00332244 -0.00011812 0.00006852 0.00001094 +0.00000000 0.00000038 0.00000000 +0.00000001 0.00000029 0.00000000 +0.00000001 0.00000012 0.00000000 +0.00000000 0.00000002 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.09766305 +0.09112542 0.00000000 0.09112542 +0.16029073 0.00000000 0.08810659 +0.12315027 0.00000000 0.08115370 +0.18891806 0.00000000 0.06735718 +0.14402780 0.00000000 0.05007529 +0.19510935 0.00000000 0.05207875 +0.31051616 0.00000000 0.04944178 +0.16486838 0.00000000 0.02848046 +0.38895062 0.00000000 0.04797836 +0.10084103 0.00000000 0.01107822 +0.00148809 0.00000000 0.00010373 +0.00000029 0.00000000 0.00000001 +0.00000000 0.09112542 0.09112542 +0.08970232 0.08970232 0.08970232 +0.14645742 0.08130178 0.08130178 +0.14175961 0.07534683 0.07534683 +0.18773534 0.06235476 0.06235476 +0.15683152 0.04600447 0.04600447 +0.22600556 0.05048551 0.05048551 +0.28270907 0.04417079 0.04417079 +0.18655396 0.03120305 0.03120305 +0.37790712 0.04625603 0.04625603 +0.08487877 0.00922784 0.00922784 +0.00117268 0.00008150 0.00008150 +0.00000021 0.00000001 0.00000001 +0.00000000 0.16029073 0.08810659 +0.08130178 0.14645742 0.08130178 +0.12687382 0.12687382 0.07093652 +0.17057609 0.12957711 0.06707446 +0.17210325 0.10132626 0.05103551 +0.18996871 0.08432837 0.04197592 +0.25964667 0.08992684 0.04501419 +0.22154538 0.06438320 0.03252494 +0.27489647 0.07935441 0.03913675 +0.30773307 0.07426184 0.03607248 +0.04883690 0.01033149 0.00506779 +0.00055085 0.00007797 0.00003780 +0.00000009 0.00000001 0.00000000 +0.00000000 0.12315027 0.08115370 +0.07534683 0.14175961 0.07534683 +0.12957711 0.17057609 0.06707446 +0.16019208 0.16019208 0.05634907 +0.15698780 0.12827413 0.04324489 +0.22782799 0.13365711 0.04355002 +0.23608830 0.11054067 0.03654545 +0.19702870 0.08317080 0.02773144 +0.33613598 0.12593078 0.04161993 +0.18754893 0.06436454 0.02068905 +0.01808061 0.00536641 0.00172001 +0.00013599 0.00002960 0.00000901 +0.00000002 0.00000000 0.00000000 +0.00000000 0.18891806 0.06735718 +0.06235476 0.18773534 0.06235476 +0.10132626 0.17210325 0.05103551 +0.12827413 0.15698780 0.04324489 +0.17130488 0.17130488 0.04344084 +0.22617172 0.16922187 0.04196924 +0.18638612 0.11632299 0.02876894 +0.26381562 0.14556127 0.03501913 +0.30081631 0.14125001 0.03369521 +0.08300776 0.03575188 0.00866162 +0.00412986 0.00152279 0.00035821 +0.00001417 0.00000419 0.00000087 +0.00000000 0.00000000 0.00000000 +0.00000000 0.14402780 0.05007529 +0.04600447 0.15683152 0.04600447 +0.08432837 0.18996871 0.04197592 +0.13365711 0.22782799 0.04355002 +0.16922187 0.22617172 0.04196924 +0.16754034 0.16754034 0.03238238 +0.20826869 0.16642781 0.03203612 +0.28882446 0.19152604 0.03615617 +0.15780911 0.09195549 0.01721894 +0.02384494 0.01213698 0.00230687 +0.00053206 0.00023233 0.00004180 +0.00000056 0.00000020 0.00000003 +0.00000000 0.00000000 0.00000000 +0.00000000 0.19510935 0.05207875 +0.05048551 0.22600556 0.05048551 +0.08992684 0.25964667 0.04501419 +0.11054067 0.23608830 0.03654545 +0.11632299 0.18638612 0.02876894 +0.16642781 0.20826869 0.03203612 +0.23761789 0.23761789 0.03731024 +0.18607966 0.14725666 0.02254232 +0.04893382 0.03375988 0.00529222 +0.00329647 0.00195719 0.00029944 +0.00002812 0.00001425 0.00000194 0.00000001 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.26935568 0.05031494 -0.04698754 0.27224313 0.04698754 -0.08255333 0.27552492 0.04216991 -0.11728339 0.27864928 0.04033039 -0.14306836 0.26429520 0.03713853 -0.15384842 0.22853000 0.03262551 -0.22783320 0.28234193 0.04143619 -0.26686098 0.26686098 0.04015244 -0.05361946 0.04401921 0.00666538 -0.00077884 0.00057648 0.00008618 -0.00000012 0.00000008 0.00000001 +0.00000000 0.31051616 0.04944178 +0.04417079 0.28270907 0.04417079 +0.06438320 0.22154538 0.03252494 +0.08317080 0.19702870 0.02773144 +0.14556127 0.26381562 0.03501913 +0.19152604 0.28882446 0.03615617 +0.14725666 0.18607966 0.02254232 +0.05656347 0.05656347 0.00756846 +0.00692960 0.00558060 0.00072667 +0.00017834 0.00012266 0.00001482 +0.00000038 0.00000022 0.00000002 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.29184205 0.05226268 -0.04897950 0.30077655 0.04897950 -0.08302718 0.30537290 0.04225252 -0.10485952 0.27970791 0.03571615 -0.13597825 0.27336582 0.03482393 -0.20669146 0.34080524 0.04322289 -0.19774171 0.27650887 0.03245509 -0.04401921 0.05361946 0.00666538 -0.00242573 0.00242573 0.00035678 -0.00000100 0.00000083 0.00000009 +0.00000000 0.16486838 0.02848046 +0.03120305 0.18655396 0.03120305 +0.07935441 0.27489647 0.03913675 +0.12593078 0.33613598 0.04161993 +0.14125001 0.30081631 0.03369521 +0.09195549 0.15780911 0.01721894 +0.03375988 0.04893382 0.00529222 +0.00558060 0.00692960 0.00072667 +0.00032059 0.00032059 0.00003335 +0.00000217 0.00000174 0.00000016 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.32796197 0.04779386 -0.04250931 0.30800102 0.04250931 -0.07481661 0.28730540 0.03779717 -0.13373729 0.35622202 0.04466998 -0.17779283 0.38848962 0.04530383 -0.12399210 0.22244465 0.02378979 -0.01903327 0.02890315 0.00332244 -0.00057648 0.00077884 0.00008618 -0.00000083 0.00000100 0.00000009 +0.00000000 0.38895062 0.04797836 +0.04625603 0.37790712 0.04625603 +0.07426184 0.30773307 0.03607248 +0.06436454 0.18754893 0.02068905 +0.03575188 0.08300776 0.00866162 +0.01213698 0.02384494 0.00230687 +0.00195719 0.00329647 0.00029944 +0.00012266 0.00017834 0.00001482 +0.00000174 0.00000217 0.00000016 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 -0.00000000 0.34155637 0.05854756 -0.06440826 0.40165640 0.06440826 -0.11485281 0.42864392 0.05668998 -0.10428998 0.29339533 0.03244443 -0.03370150 0.07612365 0.00840840 -0.00360923 0.00688265 0.00076146 -0.00006852 0.00011812 0.00001094 -0.00000008 0.00000012 0.00000001 -0.00000000 0.00000000 0.00000000 +0.00000000 0.10084103 0.01107822 +0.00922784 0.08487877 0.00922784 +0.01033149 0.04883690 0.00506779 +0.00536641 0.01808061 0.00172001 +0.00152279 0.00412986 0.00035821 +0.00023233 0.00053206 0.00004180 +0.00001425 0.00002812 0.00000194 +0.00000022 0.00000038 0.00000002 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.29245201 0.03622370 -0.02766431 0.21450057 0.02766431 -0.01532577 0.05712571 0.00770589 -0.00230056 0.00661588 0.00077655 -0.00010882 0.00027532 0.00002590 -0.00000072 0.00000159 0.00000012 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00148809 0.00010373 +0.00008150 0.00117268 0.00008150 +0.00007797 0.00055085 0.00003780 +0.00002960 0.00013599 0.00000901 +0.00000419 0.00001417 0.00000087 +0.00000020 0.00000056 0.00000003 0.00000000 0.00000001 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 -0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 0.00004101 -0.00002538 0.00000000 0.00002538 -0.00001000 0.00000000 0.00000477 -0.00000030 0.00000000 0.00000008 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000029 0.00000001 +0.00000001 0.00000021 0.00000001 +0.00000001 0.00000009 0.00000000 +0.00000000 0.00000002 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 0.15323531 -0.08253341 0.00000000 0.16706916 -0.15129872 0.00000000 0.15129872 -0.18661483 0.00000000 0.12387973 -0.20744356 0.00000000 0.11091403 -0.23059254 0.00000000 0.09877480 -0.25292777 0.00000000 0.08951714 -0.27422715 0.00000000 0.08372217 -0.30646293 0.00000000 0.08576194 -0.28483792 0.00000000 0.07437695 -0.43878482 0.00000000 0.12180165 -0.08015231 0.00000000 0.02176584 -0.00000000 0.00000000 0.00001641 -0.00000000 0.08253341 0.16706916 -0.08447348 0.08447348 0.16763493 -0.14671206 0.07661738 0.14671206 -0.18541418 0.06505718 0.12388847 -0.20769496 0.05558558 0.10849415 -0.23095378 0.04975907 0.09698467 -0.25306158 0.04486165 0.08754438 -0.27552492 0.04216991 0.08255333 -0.30537290 0.04225252 0.08302718 -0.28730540 0.03779717 0.07481661 -0.42864392 0.05668998 0.11485281 -0.05712571 0.00770589 0.01532577 -0.00000000 0.00000477 0.00001000 -0.00000000 0.15129872 0.15129872 -0.07661738 0.14671206 0.14671206 -0.13524621 0.13524621 0.13524621 -0.17594823 0.11939755 0.11939755 -0.20477920 0.10411901 0.10411901 -0.22897201 0.09274519 0.09274519 -0.25098717 0.08373253 0.08373253 -0.27829336 0.08042743 0.08042743 -0.29232812 0.07512978 0.07512978 -0.32324693 0.08311482 0.08311482 -0.35807989 0.08667964 0.08667964 -0.01778447 0.00447407 0.00447407 -0.00000000 0.00000178 0.00000178 -0.00000000 0.18661483 0.12387973 -0.06505718 0.18541418 0.12388847 -0.11939755 0.17594823 0.11939755 -0.16106826 0.16106826 0.11029892 -0.19508786 0.14601426 0.09944481 -0.22159286 0.13001978 0.08845597 -0.24729509 0.11936413 0.08111633 -0.27596054 0.11453460 0.07738642 -0.26793238 0.09981428 0.06718600 -0.39079255 0.14162973 0.09486673 -0.18972336 0.06513445 0.04209815 -0.00221287 0.00072722 0.00048335 -0.00000000 0.00000006 0.00000004 -0.00000000 0.20744356 0.11091403 -0.05558558 0.20769496 0.10849415 -0.10411901 0.20477920 0.10411901 -0.14601426 0.19508786 0.09944481 -0.17985915 0.17985915 0.09249949 -0.20934788 0.16423396 0.08491019 -0.24378950 0.15508359 0.08008805 -0.25175385 0.13583805 0.06946288 -0.29942689 0.14900888 0.07502505 -0.35117617 0.15836951 0.07863162 -0.03790964 0.01665534 0.00841665 -0.00008359 0.00003212 0.00001515 -0.00000000 0.00000000 0.00000000 -0.00000000 0.23059254 0.09877480 -0.04975907 0.23095378 0.09698467 -0.09274519 0.22897201 0.09274519 -0.13001978 0.22159286 0.08845597 -0.16423396 0.20934788 0.08491019 -0.19957410 0.19957410 0.08257578 -0.22329079 0.17940160 0.07444742 -0.23871448 0.16256273 0.06735030 -0.34498469 0.20762137 0.08568348 -0.14694051 0.08145713 0.03193184 -0.00293995 0.00151442 0.00062559 -0.00000044 0.00000020 0.00000007 -0.00000000 0.00000000 0.00000000 -0.00000000 0.25292777 0.08951714 -0.04486165 0.25306158 0.08754438 -0.08373253 0.25098717 0.08373253 -0.11936413 0.24729509 0.08111633 -0.15508359 0.24378950 0.08008805 -0.17940160 0.22329079 0.07444742 -0.19769699 0.19769699 0.06773175 -0.29264331 0.24083547 0.08576193 -0.22308379 0.15910566 0.05178263 -0.01485261 0.00974796 0.00339455 -0.00003945 0.00002269 0.00000711 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.27422715 0.08372217 -0.04216991 0.27552492 0.08255333 -0.08042743 0.27829336 0.08042743 -0.11453460 0.27596054 0.07738642 -0.13583805 0.25175385 0.06946288 -0.16256273 0.23871448 0.06735030 -0.24083547 0.29264331 0.08576193 -0.21988464 0.21988464 0.06371968 -0.03079889 0.02582956 0.00786198 -0.00031769 0.00023404 0.00006772 -0.00000004 0.00000003 0.00000001 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.30646293 0.08576194 -0.04225252 0.30537290 0.08302718 -0.07512978 0.29232812 0.07512978 -0.09981428 0.26793238 0.06718600 -0.14900888 0.29942689 0.07502505 -0.20762137 0.34498469 0.08568348 -0.15910566 0.22308379 0.05178263 -0.02582956 0.03079889 0.00786198 -0.00088028 0.00088028 0.00023977 -0.00000033 0.00000028 0.00000006 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.28483792 0.07437695 -0.03779717 0.28730540 0.07481661 -0.08311482 0.32324693 0.08311482 -0.14162973 0.39079255 0.09486673 -0.15836951 0.35117617 0.07863162 -0.08145713 0.14694051 0.03193184 -0.00974796 0.01485261 0.00339455 -0.00023404 0.00031769 0.00006772 -0.00000028 0.00000033 0.00000006 +0.00000000 0.00000000 0.16929662 +0.08810659 0.00000000 0.16029073 +0.12833447 0.00000000 0.12833447 +0.16789467 0.00000000 0.13307679 +0.17640997 0.00000000 0.10686954 +0.17975567 0.00000000 0.08405257 +0.25516702 0.00000000 0.09285334 +0.23606788 0.00000000 0.06959312 +0.24805991 0.00000000 0.07512958 +0.33411183 0.00000000 0.08172305 +0.05834002 0.00000000 0.01251291 +0.00070577 0.00000000 0.00010027 +0.00000012 0.00000000 0.00000001 +0.00000000 0.08810659 0.16029073 +0.08130178 0.08130178 0.14645742 +0.12687382 0.07093652 0.12687382 +0.17057609 0.06707446 0.12957711 +0.17210325 0.05103551 0.10132626 +0.18996871 0.04197592 0.08432837 +0.25964667 0.04501419 0.08992684 +0.22154538 0.03252494 0.06438320 +0.27489647 0.03913675 0.07935441 +0.30773307 0.03607248 0.07426184 +0.04883690 0.00506779 0.01033149 +0.00055085 0.00003780 0.00007797 +0.00000009 0.00000000 0.00000001 +0.00000000 0.12833447 0.12833447 +0.07093652 0.12687382 0.12687382 +0.12926406 0.12926406 0.12926406 +0.16354286 0.11719485 0.11719485 +0.16255243 0.09048414 0.09048414 +0.21693543 0.08720754 0.08720754 +0.25084352 0.07998512 0.07998512 +0.19835542 0.05619866 0.05619866 +0.32344678 0.08428874 0.08428874 +0.23024166 0.05321315 0.05321315 +0.02777758 0.00562052 0.00562052 +0.00025190 0.00003519 0.00003519 +0.00000003 0.00000000 0.00000000 +0.00000000 0.16789467 0.13307679 +0.06707446 0.17057609 0.12957711 +0.11719485 0.16354286 0.11719485 +0.14212360 0.14212360 0.09733606 +0.16674817 0.12988866 0.08652678 +0.23569721 0.13299970 0.08804522 +0.21187125 0.09806676 0.06505441 +0.22131051 0.09291444 0.06153255 +0.33696239 0.12071737 0.07994172 +0.13512990 0.04494632 0.02959288 +0.01015218 0.00292381 0.00190872 +0.00006001 0.00001293 0.00000813 +0.00000001 0.00000000 0.00000000 +0.00000000 0.17640997 0.10686954 +0.05103551 0.17210325 0.10132626 +0.09048414 0.16255243 0.09048414 +0.12988866 0.16674817 0.08652678 +0.18469152 0.18469152 0.09123081 +0.20610324 0.15635195 0.07681118 +0.18262799 0.11428578 0.05659935 +0.28586482 0.15333917 0.07474571 +0.25517121 0.11814635 0.05677162 +0.05749328 0.02424793 0.01183427 +0.00227204 0.00082206 0.00039243 +0.00000623 0.00000182 0.00000079 +0.00000000 0.00000000 0.00000000 +0.00000000 0.17975567 0.08405257 +0.04197592 0.18996871 0.08432837 +0.08720754 0.21693543 0.08720754 +0.13299970 0.23569721 0.08804522 +0.15635195 0.20610324 0.07681118 +0.15600927 0.15600927 0.06055819 +0.22703826 0.18192436 0.06957122 +0.27709881 0.18099661 0.06868772 +0.12085626 0.06969914 0.02657193 +0.01519361 0.00764467 0.00292605 +0.00028077 0.00012100 0.00004401 +0.00000024 0.00000009 0.00000003 +0.00000000 0.00000000 0.00000000 +0.00000000 0.25516702 0.09285334 +0.04501419 0.25964667 0.08992684 +0.07998512 0.25084352 0.07998512 +0.09806676 0.21187125 0.06505441 +0.11428578 0.18262799 0.05659935 +0.18192436 0.22703826 0.06957122 +0.23470776 0.23470776 0.07303303 +0.15093494 0.12121474 0.03733035 +0.03471153 0.02378104 0.00749465 +0.00194774 0.00114711 0.00035330 +0.00001388 0.00000699 0.00000193 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.23606788 0.06959312 +0.03252494 0.22154538 0.06438320 +0.05619866 0.19835542 0.05619866 +0.09291444 0.22131051 0.06153255 +0.15333917 0.28586482 0.07474571 +0.18099661 0.27709881 0.06868772 +0.12121474 0.15093494 0.03733035 +0.04043014 0.04043014 0.01073056 +0.00437189 0.00354037 0.00092362 +0.00009722 0.00006662 0.00001618 +0.00000018 0.00000010 0.00000002 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.24805991 0.07512958 +0.03913675 0.27489647 0.07935441 +0.08428874 0.32344678 0.08428874 +0.12071737 0.33696239 0.07994172 +0.11814635 0.25517121 0.05677162 +0.06969914 0.12085626 0.02657193 +0.02378104 0.03471153 0.00749465 +0.00354037 0.00437189 0.00092362 +0.00017594 0.00017594 0.00003660 +0.00000107 0.00000087 0.00000016 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.33411183 0.08172305 +0.03607248 0.30773307 0.07426184 +0.05321315 0.23024166 0.05321315 +0.04494632 0.13512990 0.02959288 +0.02424793 0.05749328 0.01183427 +0.00764467 0.01519361 0.00292605 +0.00114711 0.00194774 0.00035330 +0.00006662 0.00009722 0.00001618 +0.00000087 0.00000107 0.00000016 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 -0.00000000 0.43878482 0.12180165 -0.05668998 0.42864392 0.11485281 -0.08667964 0.35807989 0.08667964 -0.06513445 0.18972336 0.04209815 -0.01665534 0.03790964 0.00841665 -0.00151442 0.00293995 0.00062559 -0.00002269 0.00003945 0.00000711 -0.00000003 0.00000004 0.00000001 +0.00000000 0.05834002 0.01251291 +0.00506779 0.04883690 0.01033149 +0.00562052 0.02777758 0.00562052 +0.00292381 0.01015218 0.00190872 +0.00082206 0.00227204 0.00039243 +0.00012100 0.00028077 0.00004401 +0.00000699 0.00001388 0.00000193 +0.00000010 0.00000018 0.00000002 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00070577 0.00010027 +0.00003780 0.00055085 0.00007797 +0.00003519 0.00025190 0.00003519 +0.00001293 0.00006001 0.00000813 +0.00000182 0.00000623 0.00000079 +0.00000009 0.00000024 0.00000003 0.00000000 0.00000000 0.00000000 -0.00000000 0.08015231 0.02176584 -0.00770589 0.05712571 0.01532577 -0.00447407 0.01778447 0.00447407 -0.00072722 0.00221287 0.00048335 -0.00003212 0.00008359 0.00001515 -0.00000020 0.00000044 0.00000007 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000012 0.00000001 +0.00000000 0.00000009 0.00000001 +0.00000000 0.00000003 0.00000000 +0.00000000 0.00000001 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 -0.00000000 -0.00000000 -0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 0.00001641 -0.00000477 0.00000000 0.00001000 -0.00000178 0.00000000 0.00000178 -0.00000006 0.00000000 0.00000004 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.09816013 +0.08115370 0.00000000 0.12315027 +0.13307679 0.00000000 0.16789467 +0.16677680 0.00000000 0.16677680 +0.15632530 0.00000000 0.12968357 +0.22165644 0.00000000 0.13275711 +0.24409979 0.00000000 0.11502818 +0.19458299 0.00000000 0.08207251 +0.32929456 0.00000000 0.12600050 +0.20869991 0.00000000 0.07246411 +0.02175687 0.00000000 0.00652072 +0.00017693 0.00000000 0.00003859 +0.00000002 0.00000000 0.00000000 +0.00000000 0.08115370 0.12315027 +0.07534683 0.07534683 0.14175961 +0.12957711 0.06707446 0.17057609 +0.16019208 0.05634907 0.16019208 +0.15698780 0.04324489 0.12827413 +0.22782799 0.04355002 0.13365711 +0.23608830 0.03654545 0.11054067 +0.19702870 0.02773144 0.08317080 +0.33613598 0.04161993 0.12593078 +0.18754893 0.02068905 0.06436454 +0.01808061 0.00172001 0.00536641 +0.00013599 0.00000901 0.00002960 +0.00000002 0.00000000 0.00000000 +0.00000000 0.13307679 0.16789467 +0.06707446 0.12957711 0.17057609 +0.11719485 0.11719485 0.16354286 +0.14212360 0.09733606 0.14212360 +0.16674817 0.08652678 0.12988866 +0.23569721 0.08804522 0.13299970 +0.21187125 0.06505441 0.09806676 +0.22131051 0.06153255 0.09291444 +0.33696239 0.07994172 0.12071737 +0.13512990 0.02959288 0.04494632 +0.01015218 0.00190872 0.00292381 +0.00006001 0.00000813 0.00001293 +0.00000001 0.00000000 0.00000000 +0.00000000 0.16677680 0.16677680 +0.05634907 0.16019208 0.16019208 +0.09733606 0.14212360 0.14212360 +0.13301109 0.13301109 0.13301109 +0.18643842 0.13882752 0.13882752 +0.21860942 0.12197232 0.12197232 +0.18645333 0.08631737 0.08631737 +0.27399969 0.11015028 0.11015028 +0.28559968 0.09795618 0.09795618 +0.07496920 0.02384313 0.02384313 +0.00366065 0.00100137 0.00100137 +0.00001389 0.00000290 0.00000290 +0.00000000 0.00000000 0.00000000 +0.00000000 0.15632530 0.12968357 +0.04324489 0.15698780 0.12827413 +0.08652678 0.16674817 0.12988866 +0.13882752 0.18643842 0.13882752 +0.18022718 0.18022718 0.13249758 +0.17291370 0.13343434 0.09895724 +0.20485048 0.12881347 0.09551668 +0.29759815 0.15435960 0.11419372 +0.17816782 0.08124154 0.05962196 +0.02921873 0.01193606 0.00883079 +0.00079270 0.00027692 0.00020207 +0.00000144 0.00000041 0.00000028 +0.00000000 0.00000000 0.00000000 +0.00000000 0.22165644 0.13275711 +0.04355002 0.22782799 0.13365711 +0.08804522 0.23569721 0.13299970 +0.12197232 0.21860942 0.12197232 +0.13343434 0.17291370 0.09895724 +0.16266754 0.16266754 0.09459653 +0.24782533 0.19776329 0.11426984 +0.22917330 0.14846805 0.08505992 +0.07423723 0.04217799 0.02454575 +0.00675810 0.00333772 0.00193640 +0.00009135 0.00003848 0.00002137 +0.00000005 0.00000002 0.00000001 0.00000000 0.00000000 0.00000000 +0.00000000 0.24409979 0.11502818 +0.03654545 0.23608830 0.11054067 +0.06505441 0.21187125 0.09806676 +0.08631737 0.18645333 0.08631737 +0.12881347 0.20485048 0.09551668 +0.19776329 0.24782533 0.11426984 +0.20690601 0.20690601 0.09509358 +0.10076552 0.08189777 0.03848447 +0.01832969 0.01247414 0.00594445 +0.00077333 0.00044943 0.00020925 +0.00000403 0.00000201 0.00000086 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.19458299 0.08207251 +0.02773144 0.19702870 0.08317080 +0.06153255 0.22131051 0.09291444 +0.11015028 0.27399969 0.11015028 +0.15435960 0.29759815 0.11419372 +0.14846805 0.22917330 0.08505992 +0.08189777 0.10076552 0.03848447 +0.02208377 0.02208377 0.00874837 +0.00192564 0.00156904 0.00061546 +0.00003358 0.00002289 0.00000842 +0.00000005 0.00000003 0.00000001 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.32929456 0.12600050 +0.04161993 0.33613598 0.12593078 +0.07994172 0.33696239 0.12071737 +0.09795618 0.28559968 0.09795618 +0.08124154 0.17816782 0.05962196 +0.04217799 0.07423723 0.02454575 +0.01247414 0.01832969 0.00594445 +0.00156904 0.00192564 0.00061546 +0.00006200 0.00006200 0.00001934 +0.00000031 0.00000025 0.00000007 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.20869991 0.07246411 +0.02068905 0.18754893 0.06436454 +0.02959288 0.13512990 0.04494632 +0.02384313 0.07496920 0.02384313 +0.01193606 0.02921873 0.00883079 +0.00333772 0.00675810 0.00193640 +0.00044943 0.00077333 0.00020925 +0.00002289 0.00003358 0.00000842 +0.00000025 0.00000031 0.00000007 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 0.18052624 -0.07255161 0.00000000 0.18507825 -0.12387973 0.00000000 0.18661483 -0.17156448 0.00000000 0.17156448 -0.20161696 0.00000000 0.14816375 -0.22568042 0.00000000 0.13415464 -0.24815988 0.00000000 0.12082245 -0.27809616 0.00000000 0.11779382 -0.28558967 0.00000000 0.10768783 -0.34061828 0.00000000 0.12922569 -0.32820362 0.00000000 0.11853629 -0.00927076 0.00000000 0.00327665 -0.00000000 0.00000000 0.00000048 -0.00000000 0.07255161 0.18507825 -0.06881720 0.06881720 0.18818094 -0.12388847 0.06505718 0.18541418 -0.16862699 0.05971093 0.16862699 -0.20078594 0.05284890 0.14850555 -0.22513003 0.04626450 0.13297281 -0.24786893 0.04185713 0.12034648 -0.27864928 0.04033039 0.11728339 -0.27970791 0.03571615 0.10485952 -0.35622202 0.04466998 0.13373729 -0.29339533 0.03244443 0.10428998 -0.00661588 0.00077655 0.00230056 -0.00000000 0.00000008 0.00000030 -0.00000000 0.12387973 0.18661483 -0.06505718 0.12388847 0.18541418 -0.11939755 0.11939755 0.17594823 -0.16106826 0.11029892 0.16106826 -0.19508786 0.09944481 0.14601426 -0.22159286 0.08845597 0.13001978 -0.24729509 0.08111633 0.11936413 -0.27596054 0.07738642 0.11453460 -0.26793238 0.06718600 0.09981428 -0.39079255 0.09486673 0.14162973 -0.18972336 0.04209815 0.06513445 -0.00221287 0.00048335 0.00072722 -0.00000000 0.00000004 0.00000006 -0.00000000 0.17156448 0.17156448 -0.05971093 0.16862699 0.16862699 -0.11029892 0.16106826 0.16106826 -0.15140484 0.15140484 0.15140484 -0.18473471 0.13903198 0.13903198 -0.21415373 0.12654261 0.12654261 -0.24571981 0.11846918 0.11846918 -0.26056065 0.10657151 0.10657151 -0.28272496 0.10587090 0.10587090 -0.37633366 0.12899042 0.12899042 -0.06562444 0.02195203 0.02195203 -0.00028331 0.00008430 0.00008430 -0.00000000 0.00000000 0.00000000 -0.00000000 0.20161696 0.14816375 -0.05284890 0.20078594 0.14850555 -0.09944481 0.19508786 0.14601426 -0.13903198 0.18473471 0.13903198 -0.17280630 0.17280630 0.13037117 -0.20593067 0.16267392 0.12321924 -0.23536902 0.15087424 0.11447137 -0.23783199 0.12929001 0.09792657 -0.34029210 0.16756172 0.12658670 -0.24698436 0.10955503 0.08071337 -0.01040427 0.00446799 0.00337684 -0.00000867 0.00000318 0.00000229 -0.00000000 0.00000000 0.00000000 -0.00000000 0.22568042 0.13415464 -0.04626450 0.22513003 0.13297281 -0.08845597 0.22159286 0.13001978 -0.12654261 0.21415373 0.12654261 -0.16267392 0.20593067 0.12321924 -0.19476779 0.19476779 0.11779403 -0.20768059 0.16858893 0.10276335 -0.26752042 0.18489086 0.11274378 -0.31261553 0.18628836 0.11184057 -0.05968429 0.03299667 0.01991438 -0.00062314 0.00031089 0.00018728 -0.00000005 0.00000002 0.00000001 +0.00000000 0.02175687 0.00652072 +0.00172001 0.01808061 0.00536641 +0.00190872 0.01015218 0.00292381 +0.00100137 0.00366065 0.00100137 +0.00027692 0.00079270 0.00020207 +0.00003848 0.00009135 0.00002137 +0.00000201 0.00000403 0.00000086 +0.00000003 0.00000005 0.00000001 0.00000000 0.00000000 0.00000000 -0.00000000 0.24815988 0.12082245 -0.04185713 0.24786893 0.12034648 -0.08111633 0.24729509 0.11936413 -0.11846918 0.24571981 0.11846918 -0.15087424 0.23536902 0.11447137 -0.16858893 0.20768059 0.10276335 -0.21943567 0.21943567 0.11144995 -0.28722793 0.24046303 0.12382638 -0.12780462 0.09195493 0.04575108 -0.00443936 0.00288967 0.00149160 -0.00000523 0.00000297 0.00000139 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.27809616 0.11779382 -0.04033039 0.27864928 0.11728339 -0.07738642 0.27596054 0.11453460 -0.10657151 0.26056065 0.10657151 -0.12929001 0.23783199 0.09792657 -0.18489086 0.26752042 0.11274378 -0.24046303 0.28722793 0.12382638 -0.13479603 0.13479603 0.05758260 -0.01050494 0.00897675 0.00406497 -0.00006176 0.00004529 0.00001896 -0.00000001 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.28558967 0.10768783 -0.03571615 0.27970791 0.10485952 -0.06718600 0.26793238 0.09981428 -0.10587090 0.28272496 0.10587090 -0.16756172 0.34029210 0.12658670 -0.18628836 0.31261553 0.11184057 -0.09195493 0.12780462 0.04575108 -0.00897675 0.01050494 0.00406497 -0.00014707 0.00014707 0.00005548 -0.00000005 0.00000004 0.00000001 0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00017693 0.00003859 +0.00000901 0.00013599 0.00002960 +0.00000813 0.00006001 0.00001293 +0.00000290 0.00001389 0.00000290 +0.00000041 0.00000144 0.00000028 +0.00000002 0.00000005 0.00000001 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.34061828 0.12922569 -0.04466998 0.35622202 0.13373729 -0.09486673 0.39079255 0.14162973 -0.12899042 0.37633366 0.12899042 -0.10955503 0.24698436 0.08071337 -0.03299667 0.05968429 0.01991438 -0.00288967 0.00443936 0.00149160 -0.00004529 0.00006176 0.00001896 -0.00000004 0.00000005 0.00000001 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 -0.00000000 0.32820362 0.11853629 -0.03244443 0.29339533 0.10428998 -0.04209815 0.18972336 0.06513445 -0.02195203 0.06562444 0.02195203 -0.00446799 0.01040427 0.00337684 -0.00031089 0.00062314 0.00018728 -0.00000297 0.00000523 0.00000139 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000002 0.00000000 +0.00000000 0.00000002 0.00000000 0.00000000 0.00000001 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.00927076 0.00327665 -0.00077655 0.00661588 0.00230056 -0.00048335 0.00221287 0.00072722 -0.00008430 0.00028331 0.00008430 -0.00000318 0.00000867 0.00000229 -0.00000002 0.00000005 0.00000001 0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.18530258 +0.06735718 0.00000000 0.18891806 +0.10686954 0.00000000 0.17640997 +0.12968357 0.00000000 0.15632530 +0.16386492 0.00000000 0.16386492 +0.23048447 0.00000000 0.17164479 +0.19042150 0.00000000 0.11897298 +0.25411718 0.00000000 0.14138649 +0.31252545 0.00000000 0.14747692 +0.09327119 0.00000000 0.04045245 +0.00500877 0.00000000 0.00185769 +0.00001847 0.00000000 0.00000548 +0.00000000 0.00000000 0.00000000 +0.00000000 0.06735718 0.18891806 +0.06235476 0.06235476 0.18773534 +0.10132626 0.05103551 0.17210325 +0.12827413 0.04324489 0.15698780 +0.17130488 0.04344084 0.17130488 +0.22617172 0.04196924 0.16922187 +0.18638612 0.02876894 0.11632299 +0.26381562 0.03501913 0.14556127 +0.30081631 0.03369521 0.14125001 +0.08300776 0.00866162 0.03575188 +0.00412986 0.00035821 0.00152279 +0.00001417 0.00000087 0.00000419 +0.00000000 0.00000000 0.00000000 +0.00000000 0.10686954 0.17640997 +0.05103551 0.10132626 0.17210325 +0.09048414 0.09048414 0.16255243 +0.12988866 0.08652678 0.16674817 +0.18469152 0.09123081 0.18469152 +0.20610324 0.07681118 0.15635195 +0.18262799 0.05659935 0.11428578 +0.28586482 0.07474571 0.15333917 +0.25517121 0.05677162 0.11814635 +0.05749328 0.01183427 0.02424793 +0.00227204 0.00039243 0.00082206 +0.00000623 0.00000079 0.00000182 +0.00000000 0.00000000 0.00000000 +0.00000000 0.12968357 0.15632530 +0.04324489 0.12827413 0.15698780 +0.08652678 0.12988866 0.16674817 +0.13882752 0.13882752 0.18643842 +0.18022718 0.13249758 0.18022718 +0.17291370 0.09895724 0.13343434 +0.20485048 0.09551668 0.12881347 +0.29759815 0.11419372 0.15435960 +0.17816782 0.05962196 0.08124154 +0.02921873 0.00883079 0.01193606 +0.00079270 0.00020207 0.00027692 +0.00000144 0.00000028 0.00000041 +0.00000000 0.00000000 0.00000000 +0.00000000 0.16386492 0.16386492 +0.04344084 0.17130488 0.17130488 +0.09123081 0.18469152 0.18469152 +0.13249758 0.18022718 0.18022718 +0.14571376 0.14571376 0.14571376 +0.15984644 0.12490388 0.12490388 +0.24517307 0.15248368 0.15248368 +0.25258667 0.12778924 0.12778924 +0.09342942 0.04189178 0.04189178 +0.00985203 0.00387683 0.00387683 +0.00016179 0.00005365 0.00005365 +0.00000015 0.00000004 0.00000004 +0.00000000 0.00000000 0.00000000 +0.00000000 0.23048447 0.17164479 +0.04196924 0.22617172 0.16922187 +0.07681118 0.20610324 0.15635195 +0.09895724 0.17291370 0.13343434 +0.12490388 0.15984644 0.12490388 +0.19360522 0.19360522 0.15053598 +0.24071090 0.19094782 0.14864098 +0.14775479 0.09636086 0.07520711 +0.03335373 0.01869459 0.01471628 +0.00194571 0.00093620 0.00073327 +0.00001663 0.00000678 0.00000518 +0.00000001 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +0.00000000 0.19042150 0.11897298 +0.02876894 0.18638612 0.11632299 +0.05659935 0.18262799 0.11428578 +0.09551668 0.20485048 0.12881347 +0.15248368 0.24517307 0.15248368 +0.19094782 0.24071090 0.14864098 +0.14401719 0.14401719 0.08947020 +0.05124945 0.04187907 0.02670504 +0.00658284 0.00445794 0.00286224 +0.00019184 0.00010956 0.00006870 +0.00000062 0.00000030 0.00000018 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.25411718 0.14138649 +0.03501913 0.26381562 0.14556127 +0.07474571 0.28586482 0.15333917 +0.11419372 0.29759815 0.15435960 +0.12778924 0.25258667 0.12778924 +0.09636086 0.14775479 0.07520711 +0.04187907 0.05124945 0.02670504 +0.00848543 0.00848543 0.00449253 +0.00054959 0.00044968 0.00023565 +0.00000673 0.00000456 0.00000228 +0.00000001 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +0.00000000 0.31252545 0.14747692 +0.03369521 0.30081631 0.14125001 +0.05677162 0.25517121 0.11814635 +0.05962196 0.17816782 0.08124154 +0.04189178 0.09342942 0.04189178 +0.01869459 0.03335373 0.01471628 +0.00445794 0.00658284 0.00286224 +0.00044968 0.00054959 0.00023565 +0.00001305 0.00001305 0.00000547 +0.00000005 0.00000004 0.00000002 0.00000000 0.00000000 0.00000000 -0.00000000 -0.00000000 -0.00000000 -0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 0.00000048 -0.00000008 0.00000000 0.00000030 -0.00000004 0.00000000 0.00000006 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +0.00000000 0.09327119 0.04045245 +0.00866162 0.08300776 0.03575188 +0.01183427 0.05749328 0.02424793 +0.00883079 0.02921873 0.01193606 +0.00387683 0.00985203 0.00387683 +0.00093620 0.00194571 0.00073327 +0.00010956 0.00019184 0.00006870 +0.00000456 0.00000673 0.00000228 +0.00000004 0.00000005 0.00000002 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 -0.00000000 +0.00000000 0.00500877 0.00185769 +0.00035821 0.00412986 0.00152279 +0.00039243 0.00227204 0.00082206 +0.00020207 0.00079270 0.00027692 +0.00005365 0.00016179 0.00005365 +0.00000678 0.00001663 0.00000518 +0.00000030 0.00000062 0.00000018 +0.00000000 0.00000001 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 0.19922659 -0.06409840 0.00000000 0.20510401 -0.11091403 0.00000000 0.20744356 -0.14816375 0.00000000 0.20161696 -0.18839642 0.00000000 0.18839642 -0.21255306 0.00000000 0.16465196 -0.24407471 0.00000000 0.15550298 -0.26798333 0.00000000 0.14539135 -0.26576622 0.00000000 0.13184502 -0.39499686 0.00000000 0.18176438 -0.09442587 0.00000000 0.04187089 -0.00039903 0.00000000 0.00015905 -0.00000000 0.00000000 0.00000000 -0.00000000 0.06409840 0.20510401 -0.06125917 0.06125917 0.20793413 -0.10849415 0.05558558 0.20769496 -0.14850555 0.05284890 0.20078594 -0.18594880 0.04921526 0.18594880 -0.21173705 0.04398982 0.16466959 -0.24448037 0.04078769 0.15544591 -0.26429520 0.03713853 0.14306836 -0.27336582 0.03482393 0.13597825 -0.38848962 0.04530383 0.17779283 -0.07612365 0.00840840 0.03370150 -0.00027532 0.00002590 0.00010882 -0.00000000 0.00000000 0.00000000 -0.00000000 0.11091403 0.20744356 -0.05558558 0.10849415 0.20769496 -0.10411901 0.10411901 0.20477920 -0.14601426 0.09944481 0.19508786 -0.17985915 0.09249949 0.17985915 -0.20934788 0.08491019 0.16423396 -0.24378950 0.08008805 0.15508359 -0.25175385 0.06946288 0.13583805 -0.29942689 0.07502505 0.14900888 -0.35117617 0.07863162 0.15836951 -0.03790964 0.00841665 0.01665534 -0.00008359 0.00001515 0.00003212 -0.00000000 0.00000000 0.00000000 -0.00000000 0.14816375 0.20161696 -0.05284890 0.14850555 0.20078594 -0.09944481 0.14601426 0.19508786 -0.13903198 0.13903198 0.18473471 -0.17280630 0.13037117 0.17280630 -0.20593067 0.12321924 0.16267392 -0.23536902 0.11447137 0.15087424 -0.23783199 0.09792657 0.12929001 -0.34029210 0.12658670 0.16756172 -0.24698436 0.08071337 0.10955503 -0.01040427 0.00337684 0.00446799 -0.00000867 0.00000229 0.00000318 -0.00000000 0.00000000 0.00000000 -0.00000000 0.18839642 0.18839642 -0.04921526 0.18594880 0.18594880 -0.09249949 0.17985915 0.17985915 -0.13037117 0.17280630 0.17280630 -0.16611896 0.16611896 0.16611896 -0.19930738 0.15831015 0.15831015 -0.21424615 0.13877265 0.13877265 -0.25662287 0.14200517 0.14200517 -0.32983805 0.15821767 0.15821767 -0.09124718 0.04040967 0.04040967 -0.00132483 0.00054062 0.00054062 -0.00000020 0.00000007 0.00000007 -0.00000000 0.00000000 0.00000000 -0.00000000 0.21255306 0.16465196 -0.04398982 0.21173705 0.16466959 -0.08491019 0.20934788 0.16423396 -0.12321924 0.20593067 0.16267392 -0.15831015 0.19930738 0.15831015 -0.18007570 0.18007570 0.14379405 -0.21044762 0.17305007 0.13879825 -0.29860258 0.20935749 0.16849324 -0.20675213 0.12296526 0.09726934 -0.01280627 0.00705675 0.00569046 -0.00004980 0.00002382 0.00001877 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.24407471 0.15550298 -0.04078769 0.24448037 0.15544591 -0.08008805 0.24378950 0.15508359 -0.11447137 0.23536902 0.15087424 -0.13877265 0.21424615 0.13877265 -0.17305007 0.21044762 0.13879825 -0.25132266 0.25132266 0.16847290 -0.22475615 0.18922115 0.12425884 -0.03895527 0.02854692 0.01923163 -0.00066014 0.00042264 0.00028438 -0.00000021 0.00000012 0.00000008 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.26798333 0.14539135 -0.03713853 0.26429520 0.14306836 -0.06946288 0.25175385 0.13583805 -0.09792657 0.23783199 0.12929001 -0.14200517 0.25662287 0.14200517 -0.20935749 0.29860258 0.16849324 -0.18922115 0.22475615 0.12425884 -0.04653578 0.04653578 0.02666223 -0.00183246 0.00157772 0.00092690 -0.00000432 0.00000317 0.00000174 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.26576622 0.13184502 -0.03482393 0.27336582 0.13597825 -0.07502505 0.29942689 0.14900888 -0.12658670 0.34029210 0.16756172 -0.15821767 0.32983805 0.15821767 -0.12296526 0.20675213 0.09726934 -0.02854692 0.03895527 0.01923163 -0.00157772 0.00183246 0.00092690 -0.00000966 0.00000966 0.00000463 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.39499686 0.18176438 -0.04530383 0.38848962 0.17779283 -0.07863162 0.35117617 0.15836951 -0.08071337 0.24698436 0.10955503 -0.04040967 0.09124718 0.04040967 -0.00705675 0.01280627 0.00569046 -0.00042264 0.00066014 0.00028438 -0.00000317 0.00000432 0.00000174 +0.00000000 -0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00001847 0.00000548 +0.00000087 0.00001417 0.00000419 +0.00000079 0.00000623 0.00000182 +0.00000028 0.00000144 0.00000041 +0.00000004 0.00000015 0.00000004 +0.00000000 0.00000001 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 -0.00000000 -0.00000000 -0.00000000 0.09442587 0.04187089 -0.00840840 0.07612365 0.03370150 -0.00841665 0.03790964 0.01665534 -0.00337684 0.01040427 0.00446799 -0.00054062 0.00132483 0.00054062 -0.00002382 0.00004980 0.00001877 -0.00000012 0.00000021 0.00000008 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 -0.00000000 0.00039903 0.00015905 -0.00002590 0.00027532 0.00010882 -0.00001515 0.00008359 0.00003212 -0.00000229 0.00000867 0.00000318 -0.00000007 0.00000020 0.00000007 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.12907870 +0.05007529 0.00000000 0.14402780 +0.08405257 0.00000000 0.17975567 +0.13275711 0.00000000 0.22165644 +0.17164479 0.00000000 0.23048447 +0.17445287 0.00000000 0.17445287 +0.20178487 0.00000000 0.16081472 +0.28953384 0.00000000 0.19352975 +0.17194500 0.00000000 0.10055272 +0.02753161 0.00000000 0.01406926 +0.00065474 0.00000000 0.00028711 +0.00000073 0.00000000 0.00000027 +0.00000000 0.00000000 0.00000000 +0.00000000 0.05007529 0.14402780 +0.04600447 0.04600447 0.15683152 +0.08432837 0.04197592 0.18996871 +0.13365711 0.04355002 0.22782799 +0.16922187 0.04196924 0.22617172 +0.16754034 0.03238238 0.16754034 +0.20826869 0.03203612 0.16642781 +0.28882446 0.03615617 0.19152604 +0.15780911 0.01721894 0.09195549 +0.02384494 0.00230687 0.01213698 +0.00053206 0.00004180 0.00023233 +0.00000056 0.00000003 0.00000020 +0.00000000 0.00000000 0.00000000 +0.00000000 0.08405257 0.17975567 +0.04197592 0.08432837 0.18996871 +0.08720754 0.08720754 0.21693543 +0.13299970 0.08804522 0.23569721 +0.15635195 0.07681118 0.20610324 +0.15600927 0.06055819 0.15600927 +0.22703826 0.06957122 0.18192436 +0.27709881 0.06868772 0.18099661 +0.12085626 0.02657193 0.06969914 +0.01519361 0.00292605 0.00764467 +0.00028077 0.00004401 0.00012100 +0.00000024 0.00000003 0.00000009 +0.00000000 0.00000000 0.00000000 +0.00000000 0.13275711 0.22165644 +0.04355002 0.13365711 0.22782799 +0.08804522 0.13299970 0.23569721 +0.12197232 0.12197232 0.21860942 +0.13343434 0.09895724 0.17291370 +0.16266754 0.09459653 0.16266754 +0.24782533 0.11426984 0.19776329 +0.22917330 0.08505992 0.14846805 +0.07423723 0.02454575 0.04217799 +0.00675810 0.00193640 0.00333772 +0.00009135 0.00002137 0.00003848 +0.00000005 0.00000001 0.00000002 0.00000000 0.00000000 0.00000000 -0.00000000 -0.00000000 -0.00000000 -0.00000000 -0.00000000 -0.00000000 +0.00000000 0.17164479 0.23048447 +0.04196924 0.16922187 0.22617172 +0.07681118 0.15635195 0.20610324 +0.09895724 0.13343434 0.17291370 +0.12490388 0.12490388 0.15984644 +0.19360522 0.15053598 0.19360522 +0.24071090 0.14864098 0.19094782 +0.14775479 0.07520711 0.09636086 +0.03335373 0.01471628 0.01869459 +0.00194571 0.00073327 0.00093620 +0.00001663 0.00000518 0.00000678 +0.00000001 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +0.00000000 0.17445287 0.17445287 +0.03238238 0.16754034 0.16754034 +0.06055819 0.15600927 0.15600927 +0.09459653 0.16266754 0.16266754 +0.15053598 0.19360522 0.19360522 +0.20173051 0.20173051 0.20173051 +0.16777818 0.13322525 0.13322525 +0.06579591 0.04333248 0.04333248 +0.00949145 0.00526119 0.00526119 +0.00032491 0.00015114 0.00015114 +0.00000142 0.00000056 0.00000056 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +0.00000000 0.20178487 0.16081472 +0.03203612 0.20826869 0.16642781 +0.06957122 0.22703826 0.18192436 +0.11426984 0.24782533 0.19776329 +0.14864098 0.24071090 0.19094782 +0.13322525 0.16777818 0.13322525 +0.07015659 0.07015659 0.05637279 +0.01729732 0.01418164 0.01152414 +0.00144206 0.00097194 0.00078994 +0.00002613 0.00001465 0.00001173 +0.00000004 0.00000002 0.00000002 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +0.00000000 0.28953384 0.19352975 +0.03615617 0.28882446 0.19152604 +0.06868772 0.27709881 0.18099661 +0.08505992 0.22917330 0.14846805 +0.07520711 0.14775479 0.09636086 +0.04333248 0.06579591 0.04333248 +0.01418164 0.01729732 0.01152414 +0.00204469 0.00204469 0.00136438 +0.00009066 0.00007428 0.00004907 +0.00000067 0.00000045 0.00000029 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +0.00000000 0.17194500 0.10055272 +0.01721894 0.15780911 0.09195549 +0.02657193 0.12085626 0.06969914 +0.02454575 0.07423723 0.04217799 +0.01471628 0.03335373 0.01869459 +0.00526119 0.00949145 0.00526119 +0.00097194 0.00144206 0.00078994 +0.00007428 0.00009066 0.00004907 +0.00000144 0.00000144 0.00000077 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 0.21962195 -0.05784992 0.00000000 0.22557832 -0.09877480 0.00000000 0.23059254 -0.13415464 0.00000000 0.22568042 -0.16465196 0.00000000 0.21255306 -0.19805544 0.00000000 0.19805544 -0.23584271 0.00000000 0.18550201 -0.22778616 0.00000000 0.15266555 -0.33580943 0.00000000 0.20439457 -0.24988166 0.00000000 0.13969963 -0.00901370 0.00000000 0.00475284 -0.00000239 0.00000000 0.00000109 -0.00000000 0.00000000 0.00000000 -0.00000000 0.05784992 0.22557832 -0.05484865 0.05484865 0.22912333 -0.09698467 0.04975907 0.23095378 -0.13297281 0.04626450 0.22513003 -0.16466959 0.04398982 0.21173705 -0.19891678 0.04227551 0.19891678 -0.23331275 0.03964287 0.18492540 -0.22853000 0.03262551 0.15384842 -0.34080524 0.04322289 0.20669146 -0.22244465 0.02378979 0.12399210 -0.00688265 0.00076146 0.00360923 -0.00000159 0.00000012 0.00000072 -0.00000000 0.00000000 0.00000000 -0.00000000 0.09877480 0.23059254 -0.04975907 0.09698467 0.23095378 -0.09274519 0.09274519 0.22897201 -0.13001978 0.08845597 0.22159286 -0.16423396 0.08491019 0.20934788 -0.19957410 0.08257578 0.19957410 -0.22329079 0.07444742 0.17940160 -0.23871448 0.06735030 0.16256273 -0.34498469 0.08568348 0.20762137 -0.14694051 0.03193184 0.08145713 -0.00293995 0.00062559 0.00151442 -0.00000044 0.00000007 0.00000020 -0.00000000 0.00000000 0.00000000 -0.00000000 0.13415464 0.22568042 -0.04626450 0.13297281 0.22513003 -0.08845597 0.13001978 0.22159286 -0.12654261 0.12654261 0.21415373 -0.16267392 0.12321924 0.20593067 -0.19476779 0.11779403 0.19476779 -0.20768059 0.10276335 0.16858893 -0.26752042 0.11274378 0.18489086 -0.31261553 0.11184057 0.18628836 -0.05968429 0.01991438 0.03299667 -0.00062314 0.00018728 0.00031089 -0.00000005 0.00000001 0.00000002 0.00000000 0.00000000 0.00000000 -0.00000000 0.16465196 0.21255306 -0.04398982 0.16466959 0.21173705 -0.08491019 0.16423396 0.20934788 -0.12321924 0.16267392 0.20593067 -0.15831015 0.15831015 0.19930738 -0.18007570 0.14379405 0.18007570 -0.21044762 0.13879825 0.17305007 -0.29860258 0.16849324 0.20935749 -0.20675213 0.09726934 0.12296526 -0.01280627 0.00569046 0.00705675 -0.00004980 0.00001877 0.00002382 +0.00000000 0.02753161 0.01406926 +0.00230687 0.02384494 0.01213698 +0.00292605 0.01519361 0.00764467 +0.00193640 0.00675810 0.00333772 +0.00073327 0.00194571 0.00093620 +0.00015114 0.00032491 0.00015114 +0.00001465 0.00002613 0.00001173 +0.00000045 0.00000067 0.00000029 +0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.19805544 0.19805544 -0.04227551 0.19891678 0.19891678 -0.08257578 0.19957410 0.19957410 -0.11779403 0.19476779 0.19476779 -0.14379405 0.18007570 0.18007570 -0.17328577 0.17328577 0.17328577 -0.24761344 0.20555657 0.20555657 -0.24707821 0.17159958 0.17159958 -0.05937251 0.03616259 0.03616259 -0.00125378 0.00067638 0.00067638 -0.00000080 0.00000037 0.00000037 0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 -0.00000000 +0.00000000 0.00065474 0.00028711 +0.00004180 0.00053206 0.00023233 +0.00004401 0.00028077 0.00012100 +0.00002137 0.00009135 0.00003848 +0.00000518 0.00001663 0.00000678 +0.00000056 0.00000142 0.00000056 +0.00000002 0.00000004 0.00000002 0.00000000 0.00000000 0.00000000 -0.00000000 0.23584271 0.18550201 -0.03964287 0.23331275 0.18492540 -0.07444742 0.22329079 0.17940160 -0.10276335 0.20768059 0.16858893 -0.13879825 0.21044762 0.17305007 -0.20555657 0.24761344 0.20555657 -0.22620541 0.22620541 0.18680260 -0.09448152 0.07989574 0.06604124 -0.00548834 0.00406603 0.00340836 -0.00003419 0.00002144 0.00001768 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.22778616 0.15266555 -0.03262551 0.22853000 0.15384842 -0.06735030 0.23871448 0.16256273 -0.11274378 0.26752042 0.18489086 -0.16849324 0.29860258 0.20935749 -0.17159958 0.24707821 0.17159958 -0.07989574 0.09448152 0.06604124 -0.00763276 0.00763276 0.00547505 -0.00013148 0.00011296 0.00008002 -0.00000008 0.00000006 0.00000004 +0.00000000 -0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000073 0.00000027 +0.00000003 0.00000056 0.00000020 +0.00000003 0.00000024 0.00000009 +0.00000001 0.00000005 0.00000002 +0.00000000 0.00000001 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.33580943 0.20439457 -0.04322289 0.34080524 0.20669146 -0.08568348 0.34498469 0.20762137 -0.11184057 0.31261553 0.18628836 -0.09726934 0.20675213 0.12296526 -0.03616259 0.05937251 0.03616259 -0.00406603 0.00548834 0.00340836 -0.00011296 0.00013148 0.00008002 -0.00000018 0.00000018 0.00000011 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.24988166 0.13969963 -0.02378979 0.22244465 0.12399210 -0.03193184 0.14694051 0.08145713 -0.01991438 0.05968429 0.03299667 -0.00569046 0.01280627 0.00705675 -0.00067638 0.00125378 0.00067638 -0.00002144 0.00003419 0.00001768 -0.00000006 0.00000008 0.00000004 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 -0.00000000 -0.00000000 0.00901370 0.00475284 -0.00076146 0.00688265 0.00360923 -0.00062559 0.00293995 0.00151442 -0.00018728 0.00062314 0.00031089 -0.00001877 0.00004980 0.00002382 -0.00000037 0.00000080 0.00000037 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.00000239 0.00000109 -0.00000012 0.00000159 0.00000072 -0.00000007 0.00000044 0.00000020 -0.00000001 0.00000005 0.00000002 +0.00000000 0.00000000 0.15222460 +0.05207875 0.00000000 0.19510935 +0.09285334 0.00000000 0.25516702 +0.11502818 0.00000000 0.24409979 +0.11897298 0.00000000 0.19042150 +0.16081472 0.00000000 0.20178487 +0.23656506 0.00000000 0.23656506 +0.19815450 0.00000000 0.15572209 +0.05450403 0.00000000 0.03771792 +0.00390964 0.00000000 0.00232750 +0.00003538 0.00000000 0.00001797 +0.00000001 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +0.00000000 0.05207875 0.19510935 +0.05048551 0.05048551 0.22600556 +0.08992684 0.04501419 0.25964667 +0.11054067 0.03654545 0.23608830 +0.11632299 0.02876894 0.18638612 +0.16642781 0.03203612 0.20826869 +0.23761789 0.03731024 0.23761789 +0.18607966 0.02254232 0.14725666 +0.04893382 0.00529222 0.03375988 +0.00329647 0.00029944 0.00195719 +0.00002812 0.00000194 0.00001425 +0.00000001 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +0.00000000 0.09285334 0.25516702 +0.04501419 0.08992684 0.25964667 +0.07998512 0.07998512 0.25084352 +0.09806676 0.06505441 0.21187125 +0.11428578 0.05659935 0.18262799 +0.18192436 0.06957122 0.22703826 +0.23470776 0.07303303 0.23470776 +0.15093494 0.03733035 0.12121474 +0.03471153 0.00749465 0.02378104 +0.00194774 0.00035330 0.00114711 +0.00001388 0.00000193 0.00000699 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.11502818 0.24409979 +0.03654545 0.11054067 0.23608830 +0.06505441 0.09806676 0.21187125 +0.08631737 0.08631737 0.18645333 +0.12881347 0.09551668 0.20485048 +0.19776329 0.11426984 0.24782533 +0.20690601 0.09509358 0.20690601 +0.10076552 0.03848447 0.08189777 +0.01832969 0.00594445 0.01247414 +0.00077333 0.00020925 0.00044943 +0.00000403 0.00000086 0.00000201 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.11897298 0.19042150 +0.02876894 0.11632299 0.18638612 +0.05659935 0.11428578 0.18262799 +0.09551668 0.12881347 0.20485048 +0.15248368 0.15248368 0.24517307 +0.19094782 0.14864098 0.24071090 +0.14401719 0.08947020 0.14401719 +0.05124945 0.02670504 0.04187907 +0.00658284 0.00286224 0.00445794 +0.00019184 0.00006870 0.00010956 +0.00000062 0.00000018 0.00000030 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.16081472 0.20178487 +0.03203612 0.16642781 0.20826869 +0.06957122 0.18192436 0.22703826 +0.11426984 0.19776329 0.24782533 +0.14864098 0.19094782 0.24071090 +0.13322525 0.13322525 0.16777818 +0.07015659 0.05637279 0.07015659 +0.01729732 0.01152414 0.01418164 +0.00144206 0.00078994 0.00097194 +0.00002613 0.00001173 0.00001465 +0.00000004 0.00000002 0.00000002 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.23656506 0.23656506 +0.03731024 0.23761789 0.23761789 +0.07303303 0.23470776 0.23470776 +0.09509358 0.20690601 0.20690601 +0.08947020 0.14401719 0.14401719 +0.05637279 0.07015659 0.07015659 +0.02022042 0.02022042 0.02022042 +0.00323464 0.00265224 0.00265224 +0.00016500 0.00011052 0.00011052 +0.00000151 0.00000084 0.00000084 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.19815450 0.15572209 +0.02254232 0.18607966 0.14725666 +0.03733035 0.15093494 0.12121474 +0.03848447 0.10076552 0.08189777 +0.02670504 0.05124945 0.04187907 +0.01152414 0.01729732 0.01418164 +0.00265224 0.00323464 0.00265224 +0.00026143 0.00026143 0.00021364 +0.00000703 0.00000576 0.00000470 +0.00000003 0.00000002 0.00000001 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.05450403 0.03771792 +0.00529222 0.04893382 0.03375988 +0.00749465 0.03471153 0.02378104 +0.00594445 0.01832969 0.01247414 +0.00286224 0.00658284 0.00445794 +0.00078994 0.00144206 0.00097194 +0.00011052 0.00016500 0.00011052 +0.00000576 0.00000703 0.00000470 +0.00000007 0.00000007 0.00000004 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00390964 0.00232750 +0.00029944 0.00329647 0.00195719 +0.00035330 0.00194774 0.00114711 +0.00020925 0.00077333 0.00044943 +0.00006870 0.00019184 0.00010956 +0.00001173 0.00002613 0.00001465 +0.00000084 0.00000151 0.00000084 +0.00000002 0.00000003 0.00000001 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 -0.00000000 0.00000000 -0.00000000 -0.00000000 -0.00000000 0.00000000 -0.00000000 +0.00000000 0.00003538 0.00001797 +0.00000194 0.00002812 0.00001425 +0.00000193 0.00001388 0.00000699 +0.00000086 0.00000403 0.00000201 +0.00000018 0.00000062 0.00000030 +0.00000002 0.00000004 0.00000002 +0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000001 0.00000000 +0.00000000 0.00000001 0.00000000 +0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 @@ -6678,141 +6833,155 @@ VECTORS velocity float 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 0.24196835 -0.05315190 0.00000000 0.24812410 -0.08951714 0.00000000 0.25292777 -0.12082245 0.00000000 0.24815988 -0.15550298 0.00000000 0.24407471 -0.18550201 0.00000000 0.23584271 -0.19670448 0.00000000 0.19670448 -0.27828749 0.00000000 0.22263260 -0.29186871 0.00000000 0.20940311 -0.03569997 0.00000000 0.02352921 -0.00016711 0.00000000 0.00009720 -0.00000001 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.05315190 0.24812410 -0.05018152 0.05018152 0.25170547 -0.08754438 0.04486165 0.25306158 -0.12034648 0.04185713 0.24786893 -0.15544591 0.04078769 0.24448037 -0.18492540 0.03964287 0.23331275 -0.19466515 0.03375700 0.19466515 -0.28234193 0.04143619 0.22783320 -0.27650887 0.03245509 0.19774171 -0.02890315 0.00332244 0.01903327 -0.00011812 0.00001094 0.00006852 -0.00000001 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.08951714 0.25292777 -0.04486165 0.08754438 0.25306158 -0.08373253 0.08373253 0.25098717 -0.11936413 0.08111633 0.24729509 -0.15508359 0.08008805 0.24378950 -0.17940160 0.07444742 0.22329079 -0.19769699 0.06773175 0.19769699 -0.29264331 0.08576193 0.24083547 -0.22308379 0.05178263 0.15910566 -0.01485261 0.00339455 0.00974796 -0.00003945 0.00000711 0.00002269 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.12082245 0.24815988 -0.04185713 0.12034648 0.24786893 -0.08111633 0.11936413 0.24729509 -0.11846918 0.11846918 0.24571981 -0.15087424 0.11447137 0.23536902 -0.16858893 0.10276335 0.20768059 -0.21943567 0.11144995 0.21943567 -0.28722793 0.12382638 0.24046303 -0.12780462 0.04575108 0.09195493 -0.00443936 0.00149160 0.00288967 -0.00000523 0.00000139 0.00000297 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.15550298 0.24407471 -0.04078769 0.15544591 0.24448037 -0.08008805 0.15508359 0.24378950 -0.11447137 0.15087424 0.23536902 -0.13877265 0.13877265 0.21424615 -0.17305007 0.13879825 0.21044762 -0.25132266 0.16847290 0.25132266 -0.22475615 0.12425884 0.18922115 -0.03895527 0.01923163 0.02854692 -0.00066014 0.00028438 0.00042264 -0.00000021 0.00000008 0.00000012 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.18550201 0.23584271 -0.03964287 0.18492540 0.23331275 -0.07444742 0.17940160 0.22329079 -0.10276335 0.16858893 0.20768059 -0.13879825 0.17305007 0.21044762 -0.20555657 0.20555657 0.24761344 -0.22620541 0.18680260 0.22620541 -0.09448152 0.06604124 0.07989574 -0.00548834 0.00340836 0.00406603 -0.00003419 0.00001768 0.00002144 +-0.00000000 0.00000000 -0.00000000 +-0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.33608041 +0.04944178 0.00000000 0.31051616 +0.06959312 0.00000000 0.23606788 +0.08207251 0.00000000 0.19458299 +0.14138649 0.00000000 0.25411718 +0.19352975 0.00000000 0.28953384 +0.15572209 0.00000000 0.19815450 +0.06303014 0.00000000 0.06303014 +0.00803571 0.00000000 0.00645709 +0.00021725 0.00000000 0.00014962 +0.00000049 0.00000000 0.00000029 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.04944178 0.31051616 +0.04417079 0.04417079 0.28270907 +0.06438320 0.03252494 0.22154538 +0.08317080 0.02773144 0.19702870 +0.14556127 0.03501913 0.26381562 +0.19152604 0.03615617 0.28882446 +0.14725666 0.02254232 0.18607966 +0.05656347 0.00756846 0.05656347 +0.00692960 0.00072667 0.00558060 +0.00017834 0.00001482 0.00012266 +0.00000038 0.00000002 0.00000022 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.06959312 0.23606788 +0.03252494 0.06438320 0.22154538 +0.05619866 0.05619866 0.19835542 +0.09291444 0.06153255 0.22131051 +0.15333917 0.07474571 0.28586482 +0.18099661 0.06868772 0.27709881 +0.12121474 0.03733035 0.15093494 +0.04043014 0.01073056 0.04043014 +0.00437189 0.00092362 0.00354037 +0.00009722 0.00001618 0.00006662 +0.00000018 0.00000002 0.00000010 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.08207251 0.19458299 +0.02773144 0.08317080 0.19702870 +0.06153255 0.09291444 0.22131051 +0.11015028 0.11015028 0.27399969 +0.15435960 0.11419372 0.29759815 +0.14846805 0.08505992 0.22917330 +0.08189777 0.03848447 0.10076552 +0.02208377 0.00874837 0.02208377 +0.00192564 0.00061546 0.00156904 +0.00003358 0.00000842 0.00002289 +0.00000005 0.00000001 0.00000003 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.14138649 0.25411718 +0.03501913 0.14556127 0.26381562 +0.07474571 0.15333917 0.28586482 +0.11419372 0.15435960 0.29759815 +0.12778924 0.12778924 0.25258667 +0.09636086 0.07520711 0.14775479 +0.04187907 0.02670504 0.05124945 +0.00848543 0.00449253 0.00848543 +0.00054959 0.00023565 0.00044968 +0.00000673 0.00000228 0.00000456 +0.00000001 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +0.00000000 0.19352975 0.28953384 +0.03615617 0.19152604 0.28882446 +0.06868772 0.18099661 0.27709881 +0.08505992 0.14846805 0.22917330 +0.07520711 0.09636086 0.14775479 +0.04333248 0.04333248 0.06579591 +0.01418164 0.01152414 0.01729732 +0.00204469 0.00136438 0.00204469 +0.00009066 0.00004907 0.00007428 +0.00000067 0.00000029 0.00000045 0.00000000 0.00000000 0.00000000 -0.00000000 0.19670448 0.19670448 -0.03375700 0.19466515 0.19466515 -0.06773175 0.19769699 0.19769699 -0.11144995 0.21943567 0.21943567 -0.16847290 0.25132266 0.25132266 -0.18680260 0.22620541 0.22620541 -0.10426029 0.10426029 0.10426029 -0.01253251 0.01070927 0.01070927 -0.00027520 0.00020192 0.00020192 -0.00000027 0.00000017 0.00000017 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +0.00000000 0.15572209 0.19815450 +0.02254232 0.14725666 0.18607966 +0.03733035 0.12121474 0.15093494 +0.03848447 0.08189777 0.10076552 +0.02670504 0.04187907 0.05124945 +0.01152414 0.01418164 0.01729732 +0.00265224 0.00265224 0.00323464 +0.00026143 0.00021364 0.00026143 +0.00000703 0.00000470 0.00000576 +0.00000003 0.00000001 0.00000002 0.00000000 0.00000000 0.00000000 -0.00000000 0.27828749 0.22263260 -0.04143619 0.28234193 0.22783320 -0.08576193 0.29264331 0.24083547 -0.12382638 0.28722793 0.24046303 -0.12425884 0.22475615 0.18922115 -0.06604124 0.09448152 0.07989574 -0.01070927 0.01253251 0.01070927 -0.00047962 0.00047962 0.00040935 -0.00000196 0.00000168 0.00000141 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +0.00000000 0.06303014 0.06303014 +0.00756846 0.05656347 0.05656347 +0.01073056 0.04043014 0.04043014 +0.00874837 0.02208377 0.02208377 +0.00449253 0.00848543 0.00848543 +0.00136438 0.00204469 0.00204469 +0.00021364 0.00026143 0.00026143 +0.00001275 0.00001275 0.00001275 +0.00000017 0.00000014 0.00000014 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.29186871 0.20940311 -0.03245509 0.27650887 0.19774171 -0.05178263 0.22308379 0.15910566 -0.04575108 0.12780462 0.09195493 -0.01923163 0.03895527 0.02854692 -0.00340836 0.00548834 0.00406603 -0.00020192 0.00027520 0.00020192 -0.00000168 0.00000196 0.00000141 0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00803571 0.00645709 +0.00072667 0.00692960 0.00558060 +0.00092362 0.00437189 0.00354037 +0.00061546 0.00192564 0.00156904 +0.00023565 0.00054959 0.00044968 +0.00004907 0.00009066 0.00007428 +0.00000470 0.00000703 0.00000576 +0.00000014 0.00000017 0.00000014 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.03569997 0.02352921 -0.00332244 0.02890315 0.01903327 -0.00339455 0.01485261 0.00974796 -0.00149160 0.00443936 0.00288967 -0.00028438 0.00066014 0.00042264 -0.00001768 0.00003419 0.00002144 -0.00000017 0.00000027 0.00000017 0.00000000 0.00000000 0.00000000 +0.00000000 0.00021725 0.00014962 +0.00001482 0.00017834 0.00012266 +0.00001618 0.00009722 0.00006662 +0.00000842 0.00003358 0.00002289 +0.00000228 0.00000673 0.00000456 +0.00000029 0.00000067 0.00000045 +0.00000001 0.00000003 0.00000002 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 -0.00000000 -0.00000000 0.00016711 0.00009720 -0.00001094 0.00011812 0.00006852 -0.00000711 0.00003945 0.00002269 -0.00000139 0.00000523 0.00000297 -0.00000008 0.00000021 0.00000012 +0.00000000 -0.00000000 -0.00000000 +0.00000000 -0.00000000 -0.00000000 +0.00000000 0.00000049 0.00000029 +0.00000002 0.00000038 0.00000022 +0.00000002 0.00000018 0.00000010 +0.00000001 0.00000005 0.00000003 +0.00000000 0.00000001 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 @@ -6820,9 +6989,9 @@ VECTORS velocity float 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.00000001 0.00000000 -0.00000000 0.00000001 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 @@ -6832,7 +7001,7 @@ VECTORS velocity float 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 -0.00000000 -0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 @@ -6842,310 +7011,289 @@ VECTORS velocity float 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 0.26345823 -0.05031494 0.00000000 0.26935568 -0.08372217 0.00000000 0.27422715 -0.11779382 0.00000000 0.27809616 -0.14539135 0.00000000 0.26798333 -0.15266555 0.00000000 0.22778616 -0.22263260 0.00000000 0.27828749 -0.28020504 0.00000000 0.28020504 -0.06343861 0.00000000 0.05159365 -0.00103623 0.00000000 0.00076839 -0.00000017 0.00000000 0.00000011 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.05031494 0.26935568 -0.04698754 0.04698754 0.27224313 -0.08255333 0.04216991 0.27552492 -0.11728339 0.04033039 0.27864928 -0.14306836 0.03713853 0.26429520 -0.15384842 0.03262551 0.22853000 -0.22783320 0.04143619 0.28234193 -0.26686098 0.04015244 0.26686098 -0.05361946 0.00666538 0.04401921 -0.00077884 0.00008618 0.00057648 -0.00000012 0.00000001 0.00000008 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.08372217 0.27422715 -0.04216991 0.08255333 0.27552492 -0.08042743 0.08042743 0.27829336 -0.11453460 0.07738642 0.27596054 -0.13583805 0.06946288 0.25175385 -0.16256273 0.06735030 0.23871448 -0.24083547 0.08576193 0.29264331 -0.21988464 0.06371968 0.21988464 -0.03079889 0.00786198 0.02582956 -0.00031769 0.00006772 0.00023404 -0.00000004 0.00000001 0.00000003 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.11779382 0.27809616 -0.04033039 0.11728339 0.27864928 -0.07738642 0.11453460 0.27596054 -0.10657151 0.10657151 0.26056065 -0.12929001 0.09792657 0.23783199 -0.18489086 0.11274378 0.26752042 -0.24046303 0.12382638 0.28722793 -0.13479603 0.05758260 0.13479603 -0.01050494 0.00406497 0.00897675 -0.00006176 0.00001896 0.00004529 -0.00000001 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.15474041 +0.02848046 0.00000000 0.16486838 +0.07512958 0.00000000 0.24805991 +0.12600050 0.00000000 0.32929456 +0.14747692 0.00000000 0.31252545 +0.10055272 0.00000000 0.17194500 +0.03771792 0.00000000 0.05450403 +0.00645709 0.00000000 0.00803571 +0.00038992 0.00000000 0.00038992 +0.00000273 0.00000000 0.00000219 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.14539135 0.26798333 -0.03713853 0.14306836 0.26429520 -0.06946288 0.13583805 0.25175385 -0.09792657 0.12929001 0.23783199 -0.14200517 0.14200517 0.25662287 -0.20935749 0.16849324 0.29860258 -0.18922115 0.12425884 0.22475615 -0.04653578 0.02666223 0.04653578 -0.00183246 0.00092690 0.00157772 -0.00000432 0.00000174 0.00000317 0.00000000 0.00000000 0.00000000 +0.00000000 0.02848046 0.16486838 +0.03120305 0.03120305 0.18655396 +0.07935441 0.03913675 0.27489647 +0.12593078 0.04161993 0.33613598 +0.14125001 0.03369521 0.30081631 +0.09195549 0.01721894 0.15780911 +0.03375988 0.00529222 0.04893382 +0.00558060 0.00072667 0.00692960 +0.00032059 0.00003335 0.00032059 +0.00000217 0.00000016 0.00000174 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.15266555 0.22778616 -0.03262551 0.15384842 0.22853000 -0.06735030 0.16256273 0.23871448 -0.11274378 0.18489086 0.26752042 -0.16849324 0.20935749 0.29860258 -0.17159958 0.17159958 0.24707821 -0.07989574 0.06604124 0.09448152 -0.00763276 0.00547505 0.00763276 -0.00013148 0.00008002 0.00011296 -0.00000008 0.00000004 0.00000006 0.00000000 0.00000000 0.00000000 +0.00000000 0.07512958 0.24805991 +0.03913675 0.07935441 0.27489647 +0.08428874 0.08428874 0.32344678 +0.12071737 0.07994172 0.33696239 +0.11814635 0.05677162 0.25517121 +0.06969914 0.02657193 0.12085626 +0.02378104 0.00749465 0.03471153 +0.00354037 0.00092362 0.00437189 +0.00017594 0.00003660 0.00017594 +0.00000107 0.00000016 0.00000087 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.22263260 0.27828749 -0.04143619 0.22783320 0.28234193 -0.08576193 0.24083547 0.29264331 -0.12382638 0.24046303 0.28722793 -0.12425884 0.18922115 0.22475615 -0.06604124 0.07989574 0.09448152 -0.01070927 0.01070927 0.01253251 -0.00047962 0.00040935 0.00047962 -0.00000196 0.00000141 0.00000168 0.00000000 0.00000000 0.00000000 +0.00000000 0.12600050 0.32929456 +0.04161993 0.12593078 0.33613598 +0.07994172 0.12071737 0.33696239 +0.09795618 0.09795618 0.28559968 +0.08124154 0.05962196 0.17816782 +0.04217799 0.02454575 0.07423723 +0.01247414 0.00594445 0.01832969 +0.00156904 0.00061546 0.00192564 +0.00006200 0.00001934 0.00006200 +0.00000031 0.00000007 0.00000025 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.28020504 0.28020504 -0.04015244 0.26686098 0.26686098 -0.06371968 0.21988464 0.21988464 -0.05758260 0.13479603 0.13479603 -0.02666223 0.04653578 0.04653578 -0.00547505 0.00763276 0.00763276 -0.00040935 0.00047962 0.00047962 -0.00000479 0.00000479 0.00000479 -0.00000001 0.00000000 0.00000000 +0.00000000 0.14747692 0.31252545 +0.03369521 0.14125001 0.30081631 +0.05677162 0.11814635 0.25517121 +0.05962196 0.08124154 0.17816782 +0.04189178 0.04189178 0.09342942 +0.01869459 0.01471628 0.03335373 +0.00445794 0.00286224 0.00658284 +0.00044968 0.00023565 0.00054959 +0.00001305 0.00000547 0.00001305 +0.00000005 0.00000002 0.00000004 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 -0.00000000 -0.00000000 0.06343861 0.05159365 -0.00666538 0.05361946 0.04401921 -0.00786198 0.03079889 0.02582956 -0.00406497 0.01050494 0.00897675 -0.00092690 0.00183246 0.00157772 -0.00008002 0.00013148 0.00011296 -0.00000141 0.00000196 0.00000168 -0.00000000 0.00000001 0.00000000 +0.00000000 0.10055272 0.17194500 +0.01721894 0.09195549 0.15780911 +0.02657193 0.06969914 0.12085626 +0.02454575 0.04217799 0.07423723 +0.01471628 0.01869459 0.03335373 +0.00526119 0.00526119 0.00949145 +0.00097194 0.00078994 0.00144206 +0.00007428 0.00004907 0.00009066 +0.00000144 0.00000077 0.00000144 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +0.00000000 0.03771792 0.05450403 +0.00529222 0.03375988 0.04893382 +0.00749465 0.02378104 0.03471153 +0.00594445 0.01247414 0.01832969 +0.00286224 0.00445794 0.00658284 +0.00078994 0.00097194 0.00144206 +0.00011052 0.00011052 0.00016500 +0.00000576 0.00000470 0.00000703 +0.00000007 0.00000004 0.00000007 0.00000000 0.00000000 0.00000000 -0.00000000 0.00103623 0.00076839 -0.00008618 0.00077884 0.00057648 -0.00006772 0.00031769 0.00023404 -0.00001896 0.00006176 0.00004529 -0.00000174 0.00000432 0.00000317 -0.00000004 0.00000008 0.00000006 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00645709 0.00803571 +0.00072667 0.00558060 0.00692960 +0.00092362 0.00354037 0.00437189 +0.00061546 0.00156904 0.00192564 +0.00023565 0.00044968 0.00054959 +0.00004907 0.00007428 0.00009066 +0.00000470 0.00000576 0.00000703 +0.00000014 0.00000014 0.00000017 +0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000017 0.00000011 -0.00000001 0.00000012 0.00000008 -0.00000001 0.00000004 0.00000003 -0.00000000 0.00000001 0.00000000 +0.00000000 0.00038992 0.00038992 +0.00003335 0.00032059 0.00032059 +0.00003660 0.00017594 0.00017594 +0.00001934 0.00006200 0.00006200 +0.00000547 0.00001305 0.00001305 +0.00000077 0.00000144 0.00000144 +0.00000004 0.00000007 0.00000007 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +0.00000000 0.00000273 0.00000219 +0.00000016 0.00000217 0.00000174 +0.00000016 0.00000107 0.00000087 +0.00000007 0.00000031 0.00000025 +0.00000002 0.00000005 0.00000004 0.00000000 0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 -0.00000000 +0.00000000 -0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 -0.00000000 0.00000000 -0.00000000 -0.00000000 -0.00000000 -0.00000000 -0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 0.27808604 -0.05226268 0.00000000 0.29184205 -0.08576194 0.00000000 0.30646293 -0.10768783 0.00000000 0.28558967 -0.13184502 0.00000000 0.26576622 -0.20439457 0.00000000 0.33580943 -0.20940311 0.00000000 0.29186871 -0.05159365 0.00000000 0.06343861 -0.00335127 0.00000000 0.00335127 -0.00000143 0.00000000 0.00000119 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.05226268 0.29184205 -0.04897950 0.04897950 0.30077655 -0.08302718 0.04225252 0.30537290 -0.10485952 0.03571615 0.27970791 -0.13597825 0.03482393 0.27336582 -0.20669146 0.04322289 0.34080524 -0.19774171 0.03245509 0.27650887 -0.04401921 0.00666538 0.05361946 -0.00242573 0.00035678 0.00242573 -0.00000100 0.00000009 0.00000083 +-0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.08576194 0.30646293 -0.04225252 0.08302718 0.30537290 -0.07512978 0.07512978 0.29232812 -0.09981428 0.06718600 0.26793238 -0.14900888 0.07502505 0.29942689 -0.20762137 0.08568348 0.34498469 -0.15910566 0.05178263 0.22308379 -0.02582956 0.00786198 0.03079889 -0.00088028 0.00023977 0.00088028 -0.00000033 0.00000006 0.00000028 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.10768783 0.28558967 -0.03571615 0.10485952 0.27970791 -0.06718600 0.09981428 0.26793238 -0.10587090 0.10587090 0.28272496 -0.16756172 0.12658670 0.34029210 -0.18628836 0.11184057 0.31261553 -0.09195493 0.04575108 0.12780462 -0.00897675 0.00406497 0.01050494 -0.00014707 0.00005548 0.00014707 -0.00000005 0.00000001 0.00000004 0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.13184502 0.26576622 -0.03482393 0.13597825 0.27336582 -0.07502505 0.14900888 0.29942689 -0.12658670 0.16756172 0.34029210 -0.15821767 0.15821767 0.32983805 -0.12296526 0.09726934 0.20675213 -0.02854692 0.01923163 0.03895527 -0.00157772 0.00092690 0.00183246 -0.00000966 0.00000463 0.00000966 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.39190663 +0.04797836 0.00000000 0.38895062 +0.08172305 0.00000000 0.33411183 +0.07246411 0.00000000 0.20869991 +0.04045245 0.00000000 0.09327119 +0.01406926 0.00000000 0.02753161 +0.00232750 0.00000000 0.00390964 +0.00014962 0.00000000 0.00021725 +0.00000219 0.00000000 0.00000273 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.04797836 0.38895062 +0.04625603 0.04625603 0.37790712 +0.07426184 0.03607248 0.30773307 +0.06436454 0.02068905 0.18754893 +0.03575188 0.00866162 0.08300776 +0.01213698 0.00230687 0.02384494 +0.00195719 0.00029944 0.00329647 +0.00012266 0.00001482 0.00017834 +0.00000174 0.00000016 0.00000217 0.00000000 0.00000000 0.00000000 -0.00000000 0.20439457 0.33580943 -0.04322289 0.20669146 0.34080524 -0.08568348 0.20762137 0.34498469 -0.11184057 0.18628836 0.31261553 -0.09726934 0.12296526 0.20675213 -0.03616259 0.03616259 0.05937251 -0.00406603 0.00340836 0.00548834 -0.00011296 0.00008002 0.00013148 -0.00000018 0.00000011 0.00000018 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.08172305 0.33411183 +0.03607248 0.07426184 0.30773307 +0.05321315 0.05321315 0.23024166 +0.04494632 0.02959288 0.13512990 +0.02424793 0.01183427 0.05749328 +0.00764467 0.00292605 0.01519361 +0.00114711 0.00035330 0.00194774 +0.00006662 0.00001618 0.00009722 +0.00000087 0.00000016 0.00000107 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.20940311 0.29186871 -0.03245509 0.19774171 0.27650887 -0.05178263 0.15910566 0.22308379 -0.04575108 0.09195493 0.12780462 -0.01923163 0.02854692 0.03895527 -0.00340836 0.00406603 0.00548834 -0.00020192 0.00020192 0.00027520 -0.00000168 0.00000141 0.00000196 0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.07246411 0.20869991 +0.02068905 0.06436454 0.18754893 +0.02959288 0.04494632 0.13512990 +0.02384313 0.02384313 0.07496920 +0.01193606 0.00883079 0.02921873 +0.00333772 0.00193640 0.00675810 +0.00044943 0.00020925 0.00077333 +0.00002289 0.00000842 0.00003358 +0.00000025 0.00000007 0.00000031 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 -0.00000000 0.05159365 0.06343861 -0.00666538 0.04401921 0.05361946 -0.00786198 0.02582956 0.03079889 -0.00406497 0.00897675 0.01050494 -0.00092690 0.00157772 0.00183246 -0.00008002 0.00011296 0.00013148 -0.00000141 0.00000168 0.00000196 -0.00000000 0.00000000 0.00000001 -0.00000000 0.00000000 0.00000000 +0.00000000 0.04045245 0.09327119 +0.00866162 0.03575188 0.08300776 +0.01183427 0.02424793 0.05749328 +0.00883079 0.01193606 0.02921873 +0.00387683 0.00387683 0.00985203 +0.00093620 0.00073327 0.00194571 +0.00010956 0.00006870 0.00019184 +0.00000456 0.00000228 0.00000673 +0.00000004 0.00000002 0.00000005 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 -0.00000000 -0.00000000 0.00335127 0.00335127 -0.00035678 0.00242573 0.00242573 -0.00023977 0.00088028 0.00088028 -0.00005548 0.00014707 0.00014707 -0.00000463 0.00000966 0.00000966 -0.00000011 0.00000018 0.00000018 -0.00000000 0.00000000 0.00000000 +0.00000000 0.01406926 0.02753161 +0.00230687 0.01213698 0.02384494 +0.00292605 0.00764467 0.01519361 +0.00193640 0.00333772 0.00675810 +0.00073327 0.00093620 0.00194571 +0.00015114 0.00015114 0.00032491 +0.00001465 0.00001173 0.00002613 +0.00000045 0.00000029 0.00000067 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 -0.00000000 0.00000143 0.00000119 -0.00000009 0.00000100 0.00000083 -0.00000006 0.00000033 0.00000028 -0.00000001 0.00000005 0.00000004 +0.00000000 0.00232750 0.00390964 +0.00029944 0.00195719 0.00329647 +0.00035330 0.00114711 0.00194774 +0.00020925 0.00044943 0.00077333 +0.00006870 0.00010956 0.00019184 +0.00001173 0.00001465 0.00002613 +0.00000084 0.00000084 0.00000151 +0.00000002 0.00000001 0.00000003 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00014962 0.00021725 +0.00001482 0.00012266 0.00017834 +0.00001618 0.00006662 0.00009722 +0.00000842 0.00002289 0.00003358 +0.00000228 0.00000456 0.00000673 +0.00000029 0.00000045 0.00000067 +0.00000001 0.00000002 0.00000003 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 -0.00000000 +0.00000000 -0.00000000 -0.00000000 +0.00000000 0.00000219 0.00000273 +0.00000016 0.00000174 0.00000217 +0.00000016 0.00000087 0.00000107 +0.00000007 0.00000025 0.00000031 +0.00000002 0.00000004 0.00000005 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 @@ -7156,9 +7304,7 @@ VECTORS velocity float 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 @@ -7167,10 +7313,7 @@ VECTORS velocity float 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 -0.00000000 -0.00000000 0.00000000 -0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 @@ -7178,143 +7321,149 @@ VECTORS velocity float 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 0.35128500 -0.04779386 0.00000000 0.32796197 -0.07437695 0.00000000 0.28483792 -0.12922569 0.00000000 0.34061828 -0.18176438 0.00000000 0.39499686 -0.13969963 0.00000000 0.24988166 -0.02352921 0.00000000 0.03569997 -0.00076839 0.00000000 0.00103623 -0.00000119 0.00000000 0.00000143 +0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.04779386 0.32796197 -0.04250931 0.04250931 0.30800102 -0.07481661 0.03779717 0.28730540 -0.13373729 0.04466998 0.35622202 -0.17779283 0.04530383 0.38848962 -0.12399210 0.02378979 0.22244465 -0.01903327 0.00332244 0.02890315 -0.00057648 0.00008618 0.00077884 -0.00000083 0.00000009 0.00000100 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 -0.00000000 -0.00000000 0.07437695 0.28483792 -0.03779717 0.07481661 0.28730540 -0.08311482 0.08311482 0.32324693 -0.14162973 0.09486673 0.39079255 -0.15836951 0.07863162 0.35117617 -0.08145713 0.03193184 0.14694051 -0.00974796 0.00339455 0.01485261 -0.00023404 0.00006772 0.00031769 -0.00000028 0.00000006 0.00000033 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 -0.00000000 0.12922569 0.34061828 -0.04466998 0.13373729 0.35622202 -0.09486673 0.14162973 0.39079255 -0.12899042 0.12899042 0.37633366 -0.10955503 0.08071337 0.24698436 -0.03299667 0.01991438 0.05968429 -0.00288967 0.00149160 0.00443936 -0.00004529 0.00001896 0.00006176 -0.00000004 0.00000001 0.00000005 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.18176438 0.39499686 -0.04530383 0.17779283 0.38848962 -0.07863162 0.15836951 0.35117617 -0.08071337 0.10955503 0.24698436 -0.04040967 0.04040967 0.09124718 -0.00705675 0.00569046 0.01280627 -0.00042264 0.00028438 0.00066014 -0.00000317 0.00000174 0.00000432 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.11954782 +0.01107822 0.00000000 0.10084103 +0.01251291 0.00000000 0.05834002 +0.00652072 0.00000000 0.02175687 +0.00185769 0.00000000 0.00500877 +0.00028711 0.00000000 0.00065474 +0.00001797 0.00000000 0.00003538 +0.00000029 0.00000000 0.00000049 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 -0.00000000 0.13969963 0.24988166 -0.02378979 0.12399210 0.22244465 -0.03193184 0.08145713 0.14694051 -0.01991438 0.03299667 0.05968429 -0.00569046 0.00705675 0.01280627 -0.00067638 0.00067638 0.00125378 -0.00002144 0.00001768 0.00003419 -0.00000006 0.00000004 0.00000008 +0.00000000 0.01107822 0.10084103 +0.00922784 0.00922784 0.08487877 +0.01033149 0.00506779 0.04883690 +0.00536641 0.00172001 0.01808061 +0.00152279 0.00035821 0.00412986 +0.00023233 0.00004180 0.00053206 +0.00001425 0.00000194 0.00002812 +0.00000022 0.00000002 0.00000038 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.02352921 0.03569997 -0.00332244 0.01903327 0.02890315 -0.00339455 0.00974796 0.01485261 -0.00149160 0.00288967 0.00443936 -0.00028438 0.00042264 0.00066014 -0.00001768 0.00002144 0.00003419 -0.00000017 0.00000017 0.00000027 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.01251291 0.05834002 +0.00506779 0.01033149 0.04883690 +0.00562052 0.00562052 0.02777758 +0.00292381 0.00190872 0.01015218 +0.00082206 0.00039243 0.00227204 +0.00012100 0.00004401 0.00028077 +0.00000699 0.00000193 0.00001388 +0.00000010 0.00000002 0.00000018 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +0.00000000 0.00652072 0.02175687 +0.00172001 0.00536641 0.01808061 +0.00190872 0.00292381 0.01015218 +0.00100137 0.00100137 0.00366065 +0.00027692 0.00020207 0.00079270 +0.00003848 0.00002137 0.00009135 +0.00000201 0.00000086 0.00000403 +0.00000003 0.00000001 0.00000005 0.00000000 0.00000000 0.00000000 -0.00000000 0.00076839 0.00103623 -0.00008618 0.00057648 0.00077884 -0.00006772 0.00023404 0.00031769 -0.00001896 0.00004529 0.00006176 -0.00000174 0.00000317 0.00000432 -0.00000004 0.00000006 0.00000008 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00185769 0.00500877 +0.00035821 0.00152279 0.00412986 +0.00039243 0.00082206 0.00227204 +0.00020207 0.00027692 0.00079270 +0.00005365 0.00005365 0.00016179 +0.00000678 0.00000518 0.00001663 +0.00000030 0.00000018 0.00000062 +0.00000000 0.00000000 0.00000001 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000119 0.00000143 -0.00000009 0.00000083 0.00000100 -0.00000006 0.00000028 0.00000033 -0.00000001 0.00000004 0.00000005 +0.00000000 0.00028711 0.00065474 +0.00004180 0.00023233 0.00053206 +0.00004401 0.00012100 0.00028077 +0.00002137 0.00003848 0.00009135 +0.00000518 0.00000678 0.00001663 +0.00000056 0.00000056 0.00000142 +0.00000002 0.00000002 0.00000004 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 +0.00000000 0.00001797 0.00003538 +0.00000194 0.00001425 0.00002812 +0.00000193 0.00000699 0.00001388 +0.00000086 0.00000201 0.00000403 +0.00000018 0.00000030 0.00000062 +0.00000002 0.00000002 0.00000004 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 -0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000029 0.00000049 +0.00000002 0.00000022 0.00000038 +0.00000002 0.00000010 0.00000018 +0.00000001 0.00000003 0.00000005 +0.00000000 0.00000000 0.00000001 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 -0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 @@ -7324,9 +7473,7 @@ VECTORS velocity float 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 @@ -7337,144 +7484,136 @@ VECTORS velocity float 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 -0.00000000 -0.00000000 0.00000000 -0.00000000 0.00000000 -0.00000000 -0.00000000 -0.00000000 -0.00000000 -0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 0.25113715 -0.05854756 0.00000000 0.34155637 -0.12180165 0.00000000 0.43878482 -0.11853629 0.00000000 0.32820362 -0.04187089 0.00000000 0.09442587 -0.00475284 0.00000000 0.00901370 -0.00009720 0.00000000 0.00016711 -0.00000011 0.00000000 0.00000017 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 -0.00000000 -0.00000000 0.05854756 0.34155637 -0.06440826 0.06440826 0.40165640 -0.11485281 0.05668998 0.42864392 -0.10428998 0.03244443 0.29339533 -0.03370150 0.00840840 0.07612365 -0.00360923 0.00076146 0.00688265 -0.00006852 0.00001094 0.00011812 -0.00000008 0.00000001 0.00000012 0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 -0.00000000 +-0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 -0.00000000 0.12180165 0.43878482 -0.05668998 0.11485281 0.42864392 -0.08667964 0.08667964 0.35807989 -0.06513445 0.04209815 0.18972336 -0.01665534 0.00841665 0.03790964 -0.00151442 0.00062559 0.00293995 -0.00002269 0.00000711 0.00003945 -0.00000003 0.00000001 0.00000004 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 -0.00000000 0.11853629 0.32820362 -0.03244443 0.10428998 0.29339533 -0.04209815 0.06513445 0.18972336 -0.02195203 0.02195203 0.06562444 -0.00446799 0.00337684 0.01040427 -0.00031089 0.00018728 0.00062314 -0.00000297 0.00000139 0.00000523 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 -0.00000000 +0.00000000 0.00000000 0.00188256 +0.00010373 0.00000000 0.00148809 +0.00010027 0.00000000 0.00070577 +0.00003859 0.00000000 0.00017693 +0.00000548 0.00000000 0.00001847 +0.00000027 0.00000000 0.00000073 0.00000000 0.00000000 0.00000001 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.04187089 0.09442587 -0.00840840 0.03370150 0.07612365 -0.00841665 0.01665534 0.03790964 -0.00337684 0.00446799 0.01040427 -0.00054062 0.00054062 0.00132483 -0.00002382 0.00001877 0.00004980 -0.00000012 0.00000008 0.00000021 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +0.00000000 0.00010373 0.00148809 +0.00008150 0.00008150 0.00117268 +0.00007797 0.00003780 0.00055085 +0.00002960 0.00000901 0.00013599 +0.00000419 0.00000087 0.00001417 +0.00000020 0.00000003 0.00000056 +0.00000000 0.00000000 0.00000001 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 -0.00000000 -0.00000000 0.00475284 0.00901370 -0.00076146 0.00360923 0.00688265 -0.00062559 0.00151442 0.00293995 -0.00018728 0.00031089 0.00062314 -0.00001877 0.00002382 0.00004980 -0.00000037 0.00000037 0.00000080 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00010027 0.00070577 +0.00003780 0.00007797 0.00055085 +0.00003519 0.00003519 0.00025190 +0.00001293 0.00000813 0.00006001 +0.00000182 0.00000079 0.00000623 +0.00000009 0.00000003 0.00000024 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.00009720 0.00016711 -0.00001094 0.00006852 0.00011812 -0.00000711 0.00002269 0.00003945 -0.00000139 0.00000297 0.00000523 -0.00000008 0.00000012 0.00000021 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +0.00000000 0.00003859 0.00017693 +0.00000901 0.00002960 0.00013599 +0.00000813 0.00001293 0.00006001 +0.00000290 0.00000290 0.00001389 +0.00000041 0.00000028 0.00000144 +0.00000002 0.00000001 0.00000005 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.00000011 0.00000017 -0.00000001 0.00000008 0.00000012 -0.00000001 0.00000003 0.00000004 +0.00000000 -0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000548 0.00001847 +0.00000087 0.00000419 0.00001417 +0.00000079 0.00000182 0.00000623 +0.00000028 0.00000041 0.00000144 +0.00000004 0.00000004 0.00000015 0.00000000 0.00000000 0.00000001 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +0.00000000 0.00000027 0.00000073 +0.00000003 0.00000020 0.00000056 +0.00000003 0.00000009 0.00000024 +0.00000001 0.00000002 0.00000005 +0.00000000 0.00000000 0.00000001 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000001 +0.00000000 0.00000000 0.00000001 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 @@ -7483,7 +7622,6 @@ VECTORS velocity float 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 @@ -7497,77 +7635,81 @@ VECTORS velocity float 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +-0.00000000 -0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 -0.00000000 -0.00000000 -0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 -0.00000000 +-0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 -0.00000000 +-0.00000000 0.00000000 0.00000000 +-0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +-0.00000000 -0.00000000 0.00000000 +-0.00000000 -0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 0.38181582 -0.03622370 0.00000000 0.29245201 -0.02176584 0.00000000 0.08015231 -0.00327665 0.00000000 0.00927076 -0.00015905 0.00000000 0.00039903 -0.00000109 0.00000000 0.00000239 -0.00000000 0.00000000 0.00000001 0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000038 +0.00000001 0.00000000 0.00000029 +0.00000001 0.00000000 0.00000012 +0.00000000 0.00000000 0.00000002 0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 -0.00000000 -0.00000000 0.03622370 0.29245201 -0.02766431 0.02766431 0.21450057 -0.01532577 0.00770589 0.05712571 -0.00230056 0.00077655 0.00661588 -0.00010882 0.00002590 0.00027532 -0.00000072 0.00000012 0.00000159 -0.00000000 0.00000000 0.00000001 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 -0.00000000 0.00000000 -0.00000000 -0.00000000 0.02176584 0.08015231 -0.00770589 0.01532577 0.05712571 -0.00447407 0.00447407 0.01778447 -0.00072722 0.00048335 0.00221287 -0.00003212 0.00001515 0.00008359 -0.00000020 0.00000007 0.00000044 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +0.00000000 0.00000001 0.00000029 +0.00000001 0.00000001 0.00000021 +0.00000001 0.00000000 0.00000009 +0.00000000 0.00000000 0.00000002 0.00000000 0.00000000 0.00000000 -0.00000000 -0.00000000 -0.00000000 -0.00000000 0.00000000 -0.00000000 -0.00000000 0.00327665 0.00927076 -0.00077655 0.00230056 0.00661588 -0.00048335 0.00072722 0.00221287 -0.00008430 0.00008430 0.00028331 -0.00000318 0.00000229 0.00000867 -0.00000002 0.00000001 0.00000005 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 @@ -7575,35 +7717,33 @@ VECTORS velocity float 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.00015905 0.00039903 -0.00002590 0.00010882 0.00027532 -0.00001515 0.00003212 0.00008359 -0.00000229 0.00000318 0.00000867 -0.00000007 0.00000007 0.00000020 +0.00000000 -0.00000000 0.00000000 +0.00000000 0.00000001 0.00000012 +0.00000000 0.00000001 0.00000009 +0.00000000 0.00000000 0.00000003 +0.00000000 0.00000000 0.00000001 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 -0.00000000 -0.00000000 -0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000109 0.00000239 -0.00000012 0.00000072 0.00000159 -0.00000007 0.00000020 0.00000044 -0.00000001 0.00000002 0.00000005 0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000002 +0.00000000 0.00000000 0.00000002 +0.00000000 0.00000000 0.00000001 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000001 -0.00000000 0.00000000 0.00000001 -0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 @@ -7612,8 +7752,9 @@ VECTORS velocity float 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 -0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 +-0.00000000 -0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 @@ -7625,9 +7766,8 @@ VECTORS velocity float 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 -0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 @@ -7635,12 +7775,14 @@ VECTORS velocity float 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 -0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 +-0.00000000 -0.00000000 0.00000000 +-0.00000000 -0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 @@ -7649,10 +7791,9 @@ VECTORS velocity float 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 -0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 @@ -7660,336 +7801,195 @@ VECTORS velocity float 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 -0.00000000 -0.00000000 -0.00000000 -0.00000000 -0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 -0.00000000 -0.00000000 -0.00000000 -0.00000000 -0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 0.00000000 -0.00004101 0.00000000 0.00000000 -0.00001641 0.00000000 0.00000000 -0.00000048 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00004101 0.00000000 -0.00002538 0.00002538 0.00000000 -0.00001000 0.00000477 0.00000000 -0.00000030 0.00000008 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00001641 0.00000000 -0.00000477 0.00001000 0.00000000 -0.00000178 0.00000178 0.00000000 -0.00000006 0.00000004 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000048 0.00000000 -0.00000008 0.00000030 0.00000000 -0.00000004 0.00000006 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 +-0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 +-0.00000000 -0.00000000 0.00000000 +-0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 +-0.00000000 -0.00000000 0.00000000 +-0.00000000 -0.00000000 0.00000000 +-0.00000000 -0.00000000 0.00000000 +-0.00000000 -0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 +-0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 -0.00000000 -0.00000000 0.00000000 +-0.00000000 -0.00000000 0.00000000 +-0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 +-0.00000000 -0.00000000 0.00000000 +-0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 -0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 +-0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 CELL_DATA 1728 SCALARS SIE float 1 LOOKUP_TABLE default -11.96970310 -1.95000345 -0.68059159 -0.42679413 -0.31956721 -0.25884397 -0.21675030 -0.18791042 -0.17026011 -0.12799510 -0.15550975 -0.00010527 -1.95000345 -1.28520793 -0.55813255 -0.32283128 -0.23218801 -0.18530510 -0.15376736 -0.13124374 -0.11335403 -0.09892888 -0.04855330 -0.00002110 -0.68059159 -0.55813255 -0.42481935 -0.24743849 -0.17598073 -0.13956042 -0.11737831 -0.10329029 -0.08427341 -0.09175639 -0.00607623 -0.00000013 -0.42679413 -0.32283128 -0.24743849 -0.21151579 -0.14386465 -0.11536829 -0.10075016 -0.08506177 -0.08084794 -0.04458657 -0.00024090 -0.00000000 -0.31956721 -0.23218801 -0.17598073 -0.14386465 -0.12803331 -0.09721032 -0.08233922 -0.07372138 -0.07780606 -0.00588790 -0.00000049 -0.00000000 -0.25884397 -0.18530510 -0.13956042 -0.11536829 -0.09721032 -0.08964409 -0.06747199 -0.07294920 -0.01873712 -0.00009796 -0.00000000 -0.00000000 -0.21675030 -0.15376736 -0.11737831 -0.10075016 -0.08233922 -0.06747199 -0.07085256 -0.02962047 -0.00074974 -0.00000002 +4.79172403 +0.90441844 +0.33103068 +0.21822972 +0.15845275 +0.11959187 +0.12302833 +0.08951028 +0.07192393 +0.04396204 +0.00014597 +0.00000000 +0.90441844 +0.58140976 +0.28656607 +0.18285667 +0.12834488 +0.10471470 +0.10028512 +0.06878338 +0.07161517 +0.02489338 +0.00007284 +0.00000000 +0.33103068 +0.28656607 +0.21810275 +0.14066805 +0.10008825 +0.09123382 +0.07160368 +0.05761364 +0.06452384 +0.00770773 +0.00001287 +0.00000000 +0.21822972 +0.18285667 +0.14066805 +0.11122566 +0.08929286 +0.07722383 +0.05637075 +0.06205366 +0.03469847 +0.00122204 +0.00000058 +0.00000000 +0.15845275 +0.12834488 +0.10008825 +0.08929286 +0.08365169 +0.05708493 +0.05447213 +0.05497264 +0.01000724 +0.00008800 +0.00000001 0.00000000 +0.11959187 +0.10471470 +0.09123382 +0.07722383 +0.05708493 +0.05236315 +0.05615373 +0.02069746 +0.00094225 +0.00000181 +0.00000000 +0.00000000 +0.12302833 +0.10028512 +0.07160368 +0.05637075 +0.05447213 +0.05615373 +0.02773138 +0.00240347 +0.00002449 +0.00000000 +0.00000000 +0.00000000 +0.08951028 +0.06878338 +0.05761364 +0.06205366 +0.05497264 +0.02069746 +0.00240347 +0.00006884 +0.00000006 0.00000000 -0.18791042 -0.13124374 -0.10329029 -0.08506177 -0.07372138 -0.07294920 -0.02962047 -0.00298788 -0.00000029 0.00000000 0.00000000 +0.07192393 +0.07161517 +0.06452384 +0.03469847 +0.01000724 +0.00094225 +0.00002449 +0.00000006 0.00000000 -0.17026011 -0.11335403 -0.08427341 -0.08084794 -0.07780606 -0.01873712 -0.00074974 -0.00000029 0.00000000 0.00000000 0.00000000 +0.04396204 +0.02489338 +0.00770773 +0.00122204 +0.00008800 +0.00000181 0.00000000 -0.12799510 -0.09892888 -0.09175639 -0.04458657 -0.00588790 -0.00009796 -0.00000002 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.15550975 -0.04855330 -0.00607623 -0.00024090 -0.00000049 +0.00014597 +0.00007284 +0.00001287 +0.00000058 +0.00000001 0.00000000 0.00000000 0.00000000 @@ -7997,9 +7997,6 @@ LOOKUP_TABLE default 0.00000000 0.00000000 0.00000000 -0.00010527 -0.00002110 -0.00000013 0.00000000 0.00000000 0.00000000 @@ -8009,141 +8006,133 @@ LOOKUP_TABLE default 0.00000000 0.00000000 0.00000000 -1.95000345 -1.28520793 -0.55813255 -0.32283128 -0.23218801 -0.18530510 -0.15376736 -0.13124374 -0.11335403 -0.09892888 -0.04855330 -0.00002110 -1.28520793 -0.79807819 -0.46034581 -0.26637332 -0.19434385 -0.15332891 -0.12794963 -0.11231419 -0.09231118 -0.09251083 -0.01764984 -0.00000355 -0.55813255 -0.46034581 -0.31839076 -0.22030975 -0.15722323 -0.12704809 -0.10856596 -0.09405160 -0.08071688 -0.08188588 -0.00236939 -0.00000002 -0.32283128 -0.26637332 -0.22030975 -0.17199862 -0.13291264 -0.10597756 -0.09412617 -0.07907574 -0.08021050 -0.02804596 -0.00007821 0.00000000 -0.23218801 -0.19434385 -0.15722323 -0.13291264 -0.11010145 -0.09350691 -0.07665995 -0.07183175 -0.06574932 -0.00305855 -0.00000013 0.00000000 -0.18530510 -0.15332891 -0.12704809 -0.10597756 -0.09350691 -0.07775026 -0.06706071 -0.07144920 -0.01193449 -0.00003456 0.00000000 +0.90441844 +0.58140976 +0.28656607 +0.18285667 +0.12834488 +0.10471470 +0.10028512 +0.06878338 +0.07161517 +0.02489338 +0.00007284 0.00000000 -0.15376736 -0.12794963 -0.10856596 -0.09412617 -0.07665995 -0.06706071 -0.06813314 -0.02153258 -0.00035406 +0.58140976 +0.40878558 +0.24649672 +0.15799787 +0.11119937 +0.09725178 +0.08373850 +0.06064298 +0.07038526 +0.01440309 +0.00003526 0.00000000 +0.28656607 +0.24649672 +0.18678494 +0.12622749 +0.09485047 +0.08690181 +0.06412080 +0.05814278 +0.05421074 +0.00450693 +0.00000611 0.00000000 +0.18285667 +0.15799787 +0.12622749 +0.10215709 +0.08829797 +0.07103838 +0.05454037 +0.06357591 +0.02638426 +0.00071990 +0.00000027 0.00000000 -0.13124374 -0.11231419 -0.09405160 -0.07907574 -0.07183175 -0.07144920 -0.02153258 -0.00124650 -0.00000009 +0.12834488 +0.11119937 +0.09485047 +0.08829797 +0.07525913 +0.05493980 +0.05580676 +0.04761767 +0.00690121 +0.00005030 0.00000000 0.00000000 +0.10471470 +0.09725178 +0.08690181 +0.07103838 +0.05493980 +0.05306338 +0.05379563 +0.01603776 +0.00060703 +0.00000095 0.00000000 -0.11335403 -0.09231118 -0.08071688 -0.08021050 -0.06574932 -0.01193449 -0.00035406 -0.00000009 0.00000000 +0.10028512 +0.08373850 +0.06412080 +0.05454037 +0.05580676 +0.05379563 +0.02141048 +0.00168820 +0.00001437 0.00000000 0.00000000 0.00000000 -0.09892888 -0.09251083 -0.08188588 -0.02804596 -0.00305855 -0.00003456 +0.06878338 +0.06064298 +0.05814278 +0.06357591 +0.04761767 +0.01603776 +0.00168820 +0.00004052 +0.00000003 0.00000000 0.00000000 0.00000000 +0.07161517 +0.07038526 +0.05421074 +0.02638426 +0.00690121 +0.00060703 +0.00001437 +0.00000003 0.00000000 0.00000000 0.00000000 -0.04855330 -0.01764984 -0.00236939 -0.00007821 -0.00000013 0.00000000 +0.02489338 +0.01440309 +0.00450693 +0.00071990 +0.00005030 +0.00000095 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00002110 -0.00000355 -0.00000002 +0.00007284 +0.00003526 +0.00000611 +0.00000027 0.00000000 0.00000000 0.00000000 @@ -8153,131 +8142,141 @@ LOOKUP_TABLE default 0.00000000 0.00000000 0.00000000 -0.68059159 -0.55813255 -0.42481935 -0.24743849 -0.17598073 -0.13956042 -0.11737831 -0.10329029 -0.08427341 -0.09175639 -0.00607623 -0.00000013 -0.55813255 -0.46034581 -0.31839076 -0.22030975 -0.15722323 -0.12704809 -0.10856596 -0.09405160 -0.08071688 -0.08188588 -0.00236939 -0.00000002 -0.42481935 -0.31839076 -0.22944566 -0.17527137 -0.13735527 -0.11084416 -0.09659286 -0.08275941 -0.07860557 -0.04355621 -0.00030393 -0.00000000 -0.24743849 -0.22030975 -0.17527137 -0.13789615 -0.11555518 -0.09723650 -0.08277858 -0.07262965 -0.07833357 -0.00983444 -0.00000628 -0.00000000 -0.17598073 -0.15722323 -0.13735527 -0.11555518 -0.09525889 -0.08377748 -0.07027104 -0.07119741 -0.03692349 -0.00072355 -0.00000001 0.00000000 -0.13956042 -0.12704809 -0.11084416 -0.09723650 -0.08377748 -0.06818122 -0.06616563 -0.05853347 -0.00451546 -0.00000354 -0.00000000 -0.00000000 -0.11737831 -0.10856596 -0.09659286 -0.08277858 -0.07027104 -0.06616563 -0.06025065 -0.00972799 -0.00006690 -0.00000000 -0.00000000 -0.00000000 -0.10329029 -0.09405160 -0.08275941 -0.07262965 -0.07119741 -0.05853347 -0.00972799 -0.00020339 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.33103068 +0.28656607 +0.21810275 +0.14066805 +0.10008825 +0.09123382 +0.07160368 +0.05761364 +0.06452384 +0.00770773 +0.00001287 +0.00000000 +0.28656607 +0.24649672 +0.18678494 +0.12622749 +0.09485047 +0.08690181 +0.06412080 +0.05814278 +0.05421074 +0.00450693 +0.00000611 +0.00000000 +0.21810275 +0.18678494 +0.14011222 +0.10707286 +0.08950113 +0.07634635 +0.05616756 +0.06201354 +0.03392279 +0.00144642 +0.00000103 +0.00000000 +0.14066805 +0.12622749 +0.10707286 +0.09329520 +0.08271807 +0.06121225 +0.05394796 +0.06011152 +0.01441496 +0.00023064 +0.00000004 +0.00000000 +0.10008825 +0.09485047 +0.08950113 +0.08271807 +0.06352424 +0.05277152 +0.05861087 +0.03297638 +0.00312296 +0.00001463 +0.00000000 +0.00000000 +0.09123382 +0.08690181 +0.07634635 +0.06121225 +0.05277152 +0.05592119 +0.04373284 +0.00917166 +0.00023684 +0.00000022 +0.00000000 +0.00000000 +0.07160368 +0.06412080 +0.05616756 +0.05394796 +0.05861087 +0.04373284 +0.01259127 +0.00078413 +0.00000447 +0.00000000 +0.00000000 +0.00000000 +0.05761364 +0.05814278 +0.06201354 +0.06011152 +0.03297638 +0.00917166 +0.00078413 +0.00001290 0.00000001 0.00000000 0.00000000 0.00000000 -0.08427341 -0.08071688 -0.07860557 -0.07833357 -0.03692349 -0.00451546 -0.00006690 +0.06452384 +0.05421074 +0.03392279 +0.01441496 +0.00312296 +0.00023684 +0.00000447 0.00000001 0.00000000 0.00000000 0.00000000 0.00000000 -0.09175639 -0.08188588 -0.04355621 -0.00983444 -0.00072355 -0.00000354 +0.00770773 +0.00450693 +0.00144642 +0.00023064 +0.00001463 +0.00000022 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00607623 -0.00236939 -0.00030393 -0.00000628 -0.00000001 +0.00001287 +0.00000611 +0.00000103 +0.00000004 0.00000000 0.00000000 0.00000000 @@ -8285,8 +8284,6 @@ LOOKUP_TABLE default 0.00000000 0.00000000 0.00000000 -0.00000013 -0.00000002 0.00000000 0.00000000 0.00000000 @@ -8297,132 +8294,133 @@ LOOKUP_TABLE default 0.00000000 0.00000000 0.00000000 -0.42679413 -0.32283128 -0.24743849 -0.21151579 -0.14386465 -0.11536829 -0.10075016 -0.08506177 -0.08084794 -0.04458657 -0.00024090 -0.00000000 -0.32283128 -0.26637332 -0.22030975 -0.17199862 -0.13291264 -0.10597756 -0.09412617 -0.07907574 -0.08021050 -0.02804596 -0.00007821 -0.00000000 -0.24743849 -0.22030975 -0.17527137 -0.13789615 -0.11555518 -0.09723650 -0.08277858 -0.07262965 -0.07833357 -0.00983444 -0.00000628 -0.00000000 -0.21151579 -0.17199862 -0.13789615 -0.11829215 -0.09811864 -0.08655123 -0.07309248 -0.07001552 -0.04925510 -0.00151216 -0.00000006 0.00000000 -0.14386465 -0.13291264 -0.11555518 -0.09811864 -0.08753088 -0.07339695 -0.06683084 -0.07014299 -0.01118626 -0.00005207 -0.00000000 -0.00000000 -0.11536829 -0.10597756 -0.09723650 -0.08655123 -0.07339695 -0.06558138 -0.06847082 -0.02693209 -0.00080374 -0.00000007 0.00000000 0.00000000 -0.10075016 -0.09412617 -0.08277858 -0.07309248 -0.06683084 -0.06847082 -0.03245148 -0.00225449 -0.00000332 +0.21822972 +0.18285667 +0.14066805 +0.11122566 +0.08929286 +0.07722383 +0.05637075 +0.06205366 +0.03469847 +0.00122204 +0.00000058 +0.00000000 +0.18285667 +0.15799787 +0.12622749 +0.10215709 +0.08829797 +0.07103838 +0.05454037 +0.06357591 +0.02638426 +0.00071990 +0.00000027 +0.00000000 +0.14066805 +0.12622749 +0.10707286 +0.09329520 +0.08271807 +0.06121225 +0.05394796 +0.06011152 +0.01441496 +0.00023064 +0.00000004 +0.00000000 +0.11122566 +0.10215709 +0.09329520 +0.08609913 +0.06865393 +0.05377424 +0.05790228 +0.03989484 +0.00492826 +0.00003411 +0.00000000 +0.00000000 +0.08929286 +0.08829797 +0.08271807 +0.06865393 +0.05521576 +0.05438278 +0.05433309 +0.01667608 +0.00082738 +0.00000170 +0.00000000 +0.00000000 +0.07722383 +0.07103838 +0.06121225 +0.05377424 +0.05438278 +0.05632702 +0.02629611 +0.00346620 +0.00004804 +0.00000002 0.00000000 0.00000000 +0.05637075 +0.05454037 +0.05394796 +0.05790228 +0.05433309 +0.02629611 +0.00523048 +0.00020808 +0.00000059 0.00000000 -0.08506177 -0.07907574 -0.07262965 -0.07001552 -0.07014299 -0.02693209 -0.00225449 -0.00001003 0.00000000 0.00000000 +0.06205366 +0.06357591 +0.06011152 +0.03989484 +0.01667608 +0.00346620 +0.00020808 +0.00000188 0.00000000 0.00000000 -0.08084794 -0.08021050 -0.07833357 -0.04925510 -0.01118626 -0.00080374 -0.00000332 0.00000000 0.00000000 +0.03469847 +0.02638426 +0.01441496 +0.00492826 +0.00082738 +0.00004804 +0.00000059 0.00000000 0.00000000 0.00000000 -0.04458657 -0.02804596 -0.00983444 -0.00151216 -0.00005207 -0.00000007 0.00000000 0.00000000 +0.00122204 +0.00071990 +0.00023064 +0.00003411 +0.00000170 +0.00000002 0.00000000 0.00000000 0.00000000 0.00000000 -0.00024090 -0.00007821 -0.00000628 -0.00000006 0.00000000 0.00000000 +0.00000058 +0.00000027 +0.00000004 +0.00000000 0.00000000 0.00000000 0.00000000 @@ -8441,119 +8439,121 @@ LOOKUP_TABLE default 0.00000000 0.00000000 0.00000000 -0.31956721 -0.23218801 -0.17598073 -0.14386465 -0.12803331 -0.09721032 -0.08233922 -0.07372138 -0.07780606 -0.00588790 -0.00000049 0.00000000 -0.23218801 -0.19434385 -0.15722323 -0.13291264 -0.11010145 -0.09350691 -0.07665995 -0.07183175 -0.06574932 -0.00305855 -0.00000013 0.00000000 -0.17598073 -0.15722323 -0.13735527 -0.11555518 -0.09525889 -0.08377748 -0.07027104 -0.07119741 -0.03692349 -0.00072355 +0.15845275 +0.12834488 +0.10008825 +0.08929286 +0.08365169 +0.05708493 +0.05447213 +0.05497264 +0.01000724 +0.00008800 0.00000001 0.00000000 -0.14386465 -0.13291264 -0.11555518 -0.09811864 -0.08753088 -0.07339695 -0.06683084 -0.07014299 -0.01118626 -0.00005207 -0.00000000 -0.00000000 -0.12803331 -0.11010145 -0.09525889 -0.08753088 -0.07550642 -0.06670611 -0.06800883 -0.03625927 -0.00145810 -0.00000040 +0.12834488 +0.11119937 +0.09485047 +0.08829797 +0.07525913 +0.05493980 +0.05580676 +0.04761767 +0.00690121 +0.00005030 +0.00000000 +0.00000000 +0.10008825 +0.09485047 +0.08950113 +0.08271807 +0.06352424 +0.05277152 +0.05861087 +0.03297638 +0.00312296 +0.00001463 +0.00000000 +0.00000000 +0.08929286 +0.08829797 +0.08271807 +0.06865393 +0.05521576 +0.05438278 +0.05433309 +0.01667608 +0.00082738 +0.00000170 +0.00000000 +0.00000000 +0.08365169 +0.07525913 +0.06352424 +0.05521576 +0.05398433 +0.05810975 +0.03180041 +0.00520726 +0.00009987 +0.00000006 0.00000000 0.00000000 -0.09721032 -0.09350691 -0.08377748 -0.07339695 -0.06670611 -0.06780607 -0.05087100 -0.00582269 -0.00003846 +0.05708493 +0.05493980 +0.05277152 +0.05438278 +0.05810975 +0.03807936 +0.01060421 +0.00070147 +0.00000376 0.00000000 0.00000000 0.00000000 -0.08233922 -0.07665995 -0.07027104 -0.06683084 -0.06800883 -0.05087100 -0.00845280 -0.00018220 +0.05447213 +0.05580676 +0.05861087 +0.05433309 +0.03180041 +0.01060421 +0.00125575 +0.00002404 0.00000002 0.00000000 0.00000000 0.00000000 -0.07372138 -0.07183175 -0.07119741 -0.07014299 -0.03625927 -0.00582269 -0.00018220 -0.00000008 +0.05497264 +0.04761767 +0.03297638 +0.01667608 +0.00520726 +0.00070147 +0.00002404 +0.00000010 0.00000000 0.00000000 0.00000000 0.00000000 -0.07780606 -0.06574932 -0.03692349 -0.01118626 -0.00145810 -0.00003846 +0.01000724 +0.00690121 +0.00312296 +0.00082738 +0.00009987 +0.00000376 0.00000002 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00588790 -0.00305855 -0.00072355 -0.00005207 -0.00000040 +0.00008800 +0.00005030 +0.00001463 +0.00000170 +0.00000006 0.00000000 0.00000000 0.00000000 @@ -8561,8 +8561,6 @@ LOOKUP_TABLE default 0.00000000 0.00000000 0.00000000 -0.00000049 -0.00000013 0.00000001 0.00000000 0.00000000 @@ -8585,118 +8583,120 @@ LOOKUP_TABLE default 0.00000000 0.00000000 0.00000000 -0.25884397 -0.18530510 -0.13956042 -0.11536829 -0.09721032 -0.08964409 -0.06747199 -0.07294920 -0.01873712 -0.00009796 0.00000000 0.00000000 -0.18530510 -0.15332891 -0.12704809 -0.10597756 -0.09350691 -0.07775026 -0.06706071 -0.07144920 -0.01193449 -0.00003456 +0.11959187 +0.10471470 +0.09123382 +0.07722383 +0.05708493 +0.05236315 +0.05615373 +0.02069746 +0.00094225 +0.00000181 0.00000000 0.00000000 -0.13956042 -0.12704809 -0.11084416 -0.09723650 -0.08377748 -0.06818122 -0.06616563 -0.05853347 -0.00451546 -0.00000354 +0.10471470 +0.09725178 +0.08690181 +0.07103838 +0.05493980 +0.05306338 +0.05379563 +0.01603776 +0.00060703 +0.00000095 0.00000000 0.00000000 -0.11536829 -0.10597756 -0.09723650 -0.08655123 -0.07339695 -0.06558138 -0.06847082 -0.02693209 -0.00080374 -0.00000007 +0.09123382 +0.08690181 +0.07634635 +0.06121225 +0.05277152 +0.05592119 +0.04373284 +0.00917166 +0.00023684 +0.00000022 +0.00000000 +0.00000000 +0.07722383 +0.07103838 +0.06121225 +0.05377424 +0.05438278 +0.05632702 +0.02629611 +0.00346620 +0.00004804 +0.00000002 0.00000000 0.00000000 -0.09721032 -0.09350691 -0.08377748 -0.07339695 -0.06670611 -0.06780607 -0.05087100 -0.00582269 -0.00003846 +0.05708493 +0.05493980 +0.05277152 +0.05438278 +0.05810975 +0.03807936 +0.01060421 +0.00070147 +0.00000376 0.00000000 0.00000000 0.00000000 -0.08964409 -0.07775026 -0.06818122 -0.06558138 -0.06780607 -0.05775727 -0.01246553 -0.00037585 -0.00000012 +0.05236315 +0.05306338 +0.05592119 +0.05632702 +0.03807936 +0.01443167 +0.00213090 +0.00005135 +0.00000007 0.00000000 0.00000000 0.00000000 -0.06747199 -0.06706071 -0.06616563 -0.06847082 -0.05087100 -0.01246553 -0.00074932 -0.00000165 +0.05615373 +0.05379563 +0.04373284 +0.02629611 +0.01060421 +0.00213090 +0.00011434 +0.00000079 0.00000000 0.00000000 0.00000000 0.00000000 -0.07294920 -0.07144920 -0.05853347 -0.02693209 -0.00582269 -0.00037585 -0.00000165 +0.02069746 +0.01603776 +0.00917166 +0.00346620 +0.00070147 +0.00005135 +0.00000079 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.01873712 -0.01193449 -0.00451546 -0.00080374 -0.00003846 -0.00000012 +0.00094225 +0.00060703 +0.00023684 +0.00004804 +0.00000376 +0.00000007 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00009796 -0.00003456 -0.00000354 -0.00000007 +0.00000181 +0.00000095 +0.00000022 +0.00000002 0.00000000 0.00000000 0.00000000 @@ -8729,106 +8729,106 @@ LOOKUP_TABLE default 0.00000000 0.00000000 0.00000000 -0.21675030 -0.15376736 -0.11737831 -0.10075016 -0.08233922 -0.06747199 -0.07085256 -0.02962047 -0.00074974 -0.00000002 +0.12302833 +0.10028512 +0.07160368 +0.05637075 +0.05447213 +0.05615373 +0.02773138 +0.00240347 +0.00002449 0.00000000 0.00000000 -0.15376736 -0.12794963 -0.10856596 -0.09412617 -0.07665995 -0.06706071 -0.06813314 -0.02153258 -0.00035406 0.00000000 +0.10028512 +0.08373850 +0.06412080 +0.05454037 +0.05580676 +0.05379563 +0.02141048 +0.00168820 +0.00001437 0.00000000 0.00000000 -0.11737831 -0.10856596 -0.09659286 -0.08277858 -0.07027104 -0.06616563 -0.06025065 -0.00972799 -0.00006690 0.00000000 +0.07160368 +0.06412080 +0.05616756 +0.05394796 +0.05861087 +0.04373284 +0.01259127 +0.00078413 +0.00000447 0.00000000 0.00000000 -0.10075016 -0.09412617 -0.08277858 -0.07309248 -0.06683084 -0.06847082 -0.03245148 -0.00225449 -0.00000332 0.00000000 +0.05637075 +0.05454037 +0.05394796 +0.05790228 +0.05433309 +0.02629611 +0.00523048 +0.00020808 +0.00000059 0.00000000 0.00000000 -0.08233922 -0.07665995 -0.07027104 -0.06683084 -0.06800883 -0.05087100 -0.00845280 -0.00018220 +0.00000000 +0.05447213 +0.05580676 +0.05861087 +0.05433309 +0.03180041 +0.01060421 +0.00125575 +0.00002404 0.00000002 0.00000000 0.00000000 0.00000000 -0.06747199 -0.06706071 -0.06616563 -0.06847082 -0.05087100 -0.01246553 -0.00074932 -0.00000165 +0.05615373 +0.05379563 +0.04373284 +0.02629611 +0.01060421 +0.00213090 +0.00011434 +0.00000079 0.00000000 0.00000000 0.00000000 0.00000000 -0.07085256 -0.06813314 -0.06025065 -0.03245148 -0.00845280 -0.00074932 -0.00000578 +0.02773138 +0.02141048 +0.01259127 +0.00523048 +0.00125575 +0.00011434 +0.00000227 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.02962047 -0.02153258 -0.00972799 -0.00225449 -0.00018220 -0.00000165 +0.00240347 +0.00168820 +0.00078413 +0.00020808 +0.00002404 +0.00000079 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00074974 -0.00035406 -0.00006690 -0.00000332 +0.00002449 +0.00001437 +0.00000447 +0.00000059 0.00000002 0.00000000 0.00000000 @@ -8837,7 +8837,6 @@ LOOKUP_TABLE default 0.00000000 0.00000000 0.00000000 -0.00000002 0.00000000 0.00000000 0.00000000 @@ -8873,95 +8872,96 @@ LOOKUP_TABLE default 0.00000000 0.00000000 0.00000000 -0.18791042 -0.13124374 -0.10329029 -0.08506177 -0.07372138 -0.07294920 -0.02962047 -0.00298788 -0.00000029 +0.00000000 +0.08951028 +0.06878338 +0.05761364 +0.06205366 +0.05497264 +0.02069746 +0.00240347 +0.00006884 +0.00000006 0.00000000 0.00000000 0.00000000 -0.13124374 -0.11231419 -0.09405160 -0.07907574 -0.07183175 -0.07144920 -0.02153258 -0.00124650 -0.00000009 +0.06878338 +0.06064298 +0.05814278 +0.06357591 +0.04761767 +0.01603776 +0.00168820 +0.00004052 +0.00000003 0.00000000 0.00000000 0.00000000 -0.10329029 -0.09405160 -0.08275941 -0.07262965 -0.07119741 -0.05853347 -0.00972799 -0.00020339 +0.05761364 +0.05814278 +0.06201354 +0.06011152 +0.03297638 +0.00917166 +0.00078413 +0.00001290 0.00000001 0.00000000 0.00000000 0.00000000 -0.08506177 -0.07907574 -0.07262965 -0.07001552 -0.07014299 -0.02693209 -0.00225449 -0.00001003 +0.06205366 +0.06357591 +0.06011152 +0.03989484 +0.01667608 +0.00346620 +0.00020808 +0.00000188 0.00000000 0.00000000 0.00000000 0.00000000 -0.07372138 -0.07183175 -0.07119741 -0.07014299 -0.03625927 -0.00582269 -0.00018220 -0.00000008 +0.05497264 +0.04761767 +0.03297638 +0.01667608 +0.00520726 +0.00070147 +0.00002404 +0.00000010 0.00000000 0.00000000 0.00000000 0.00000000 -0.07294920 -0.07144920 -0.05853347 -0.02693209 -0.00582269 -0.00037585 -0.00000165 +0.02069746 +0.01603776 +0.00917166 +0.00346620 +0.00070147 +0.00005135 +0.00000079 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.02962047 -0.02153258 -0.00972799 -0.00225449 -0.00018220 -0.00000165 +0.00240347 +0.00168820 +0.00078413 +0.00020808 +0.00002404 +0.00000079 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00298788 -0.00124650 -0.00020339 -0.00001003 -0.00000008 +0.00006884 +0.00004052 +0.00001290 +0.00000188 +0.00000010 0.00000000 0.00000000 0.00000000 @@ -8969,8 +8969,8 @@ LOOKUP_TABLE default 0.00000000 0.00000000 0.00000000 -0.00000029 -0.00000009 +0.00000006 +0.00000003 0.00000001 0.00000000 0.00000000 @@ -9017,82 +9017,82 @@ LOOKUP_TABLE default 0.00000000 0.00000000 0.00000000 -0.17026011 -0.11335403 -0.08427341 -0.08084794 -0.07780606 -0.01873712 -0.00074974 -0.00000029 +0.07192393 +0.07161517 +0.06452384 +0.03469847 +0.01000724 +0.00094225 +0.00002449 +0.00000006 0.00000000 0.00000000 0.00000000 0.00000000 -0.11335403 -0.09231118 -0.08071688 -0.08021050 -0.06574932 -0.01193449 -0.00035406 -0.00000009 +0.07161517 +0.07038526 +0.05421074 +0.02638426 +0.00690121 +0.00060703 +0.00001437 +0.00000003 0.00000000 0.00000000 0.00000000 0.00000000 -0.08427341 -0.08071688 -0.07860557 -0.07833357 -0.03692349 -0.00451546 -0.00006690 +0.06452384 +0.05421074 +0.03392279 +0.01441496 +0.00312296 +0.00023684 +0.00000447 0.00000001 0.00000000 0.00000000 0.00000000 0.00000000 -0.08084794 -0.08021050 -0.07833357 -0.04925510 -0.01118626 -0.00080374 -0.00000332 +0.03469847 +0.02638426 +0.01441496 +0.00492826 +0.00082738 +0.00004804 +0.00000059 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.07780606 -0.06574932 -0.03692349 -0.01118626 -0.00145810 -0.00003846 +0.01000724 +0.00690121 +0.00312296 +0.00082738 +0.00009987 +0.00000376 0.00000002 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.01873712 -0.01193449 -0.00451546 -0.00080374 -0.00003846 -0.00000012 +0.00094225 +0.00060703 +0.00023684 +0.00004804 +0.00000376 +0.00000007 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00074974 -0.00035406 -0.00006690 -0.00000332 +0.00002449 +0.00001437 +0.00000447 +0.00000059 0.00000002 0.00000000 0.00000000 @@ -9101,8 +9101,8 @@ LOOKUP_TABLE default 0.00000000 0.00000000 0.00000000 -0.00000029 -0.00000009 +0.00000006 +0.00000003 0.00000001 0.00000000 0.00000000 @@ -9161,70 +9161,71 @@ LOOKUP_TABLE default 0.00000000 0.00000000 0.00000000 -0.12799510 -0.09892888 -0.09175639 -0.04458657 -0.00588790 -0.00009796 -0.00000002 +0.04396204 +0.02489338 +0.00770773 +0.00122204 +0.00008800 +0.00000181 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.09892888 -0.09251083 -0.08188588 -0.02804596 -0.00305855 -0.00003456 0.00000000 +0.02489338 +0.01440309 +0.00450693 +0.00071990 +0.00005030 +0.00000095 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.09175639 -0.08188588 -0.04355621 -0.00983444 -0.00072355 -0.00000354 0.00000000 +0.00770773 +0.00450693 +0.00144642 +0.00023064 +0.00001463 +0.00000022 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.04458657 -0.02804596 -0.00983444 -0.00151216 -0.00005207 -0.00000007 0.00000000 +0.00122204 +0.00071990 +0.00023064 +0.00003411 +0.00000170 +0.00000002 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00588790 -0.00305855 -0.00072355 -0.00005207 -0.00000040 0.00000000 +0.00008800 +0.00005030 +0.00001463 +0.00000170 +0.00000006 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00009796 -0.00003456 -0.00000354 -0.00000007 +0.00000000 +0.00000181 +0.00000095 +0.00000022 +0.00000002 +0.00000000 0.00000000 0.00000000 0.00000000 @@ -9233,7 +9234,6 @@ LOOKUP_TABLE default 0.00000000 0.00000000 0.00000000 -0.00000002 0.00000000 0.00000000 0.00000000 @@ -9305,23 +9305,23 @@ LOOKUP_TABLE default 0.00000000 0.00000000 0.00000000 -0.15550975 -0.04855330 -0.00607623 -0.00024090 -0.00000049 +0.00014597 +0.00007284 +0.00001287 +0.00000058 +0.00000001 +0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +0.00007284 +0.00003526 +0.00000611 +0.00000027 0.00000000 -0.04855330 -0.01764984 -0.00236939 -0.00007821 -0.00000013 0.00000000 0.00000000 0.00000000 @@ -9329,11 +9329,10 @@ LOOKUP_TABLE default 0.00000000 0.00000000 0.00000000 -0.00607623 -0.00236939 -0.00030393 -0.00000628 -0.00000001 +0.00001287 +0.00000611 +0.00000103 +0.00000004 0.00000000 0.00000000 0.00000000 @@ -9341,10 +9340,11 @@ LOOKUP_TABLE default 0.00000000 0.00000000 0.00000000 -0.00024090 -0.00007821 -0.00000628 -0.00000006 +0.00000000 +0.00000058 +0.00000027 +0.00000004 +0.00000000 0.00000000 0.00000000 0.00000000 @@ -9353,8 +9353,6 @@ LOOKUP_TABLE default 0.00000000 0.00000000 0.00000000 -0.00000049 -0.00000013 0.00000001 0.00000000 0.00000000 @@ -9449,9 +9447,6 @@ LOOKUP_TABLE default 0.00000000 0.00000000 0.00000000 -0.00010527 -0.00002110 -0.00000013 0.00000000 0.00000000 0.00000000 @@ -9461,9 +9456,6 @@ LOOKUP_TABLE default 0.00000000 0.00000000 0.00000000 -0.00002110 -0.00000355 -0.00000002 0.00000000 0.00000000 0.00000000 @@ -9473,8 +9465,6 @@ LOOKUP_TABLE default 0.00000000 0.00000000 0.00000000 -0.00000013 -0.00000002 0.00000000 0.00000000 0.00000000 @@ -9593,1533 +9583,1149 @@ LOOKUP_TABLE default 0.00000000 0.00000000 0.00000000 - -SCALARS element_density float 1 -LOOKUP_TABLE default -0.01059896 -0.06028598 -0.18275629 -0.30874663 -0.44891561 -0.61765647 -0.83598131 -1.10711539 -1.54571030 -1.64889211 -6.74836785 -1.12321467 -0.06028598 -0.09454520 -0.20220073 -0.37708219 -0.56522550 -0.78305577 -1.05302032 -1.41794008 -1.78819213 -2.55789130 -3.34525938 -1.06814764 -0.18275629 -0.20220073 -0.28684617 -0.45987870 -0.72089895 -0.99970957 -1.34702602 -1.80964877 -2.13732651 -4.92673785 -1.71008559 -1.01698812 -0.30874663 -0.37708219 -0.45987870 -0.60254824 -0.85104435 -1.20884303 -1.64579808 -1.99289978 -3.09972265 -3.69851764 -1.18071033 -1.00161247 -0.44891561 -0.56522550 -0.72089895 -0.85104435 -1.07338231 -1.47642710 -1.91607685 -2.48729715 -5.02047567 -1.74250470 -1.02797014 -1.00004667 -0.61765647 -0.78305577 -0.99970957 -1.20884303 -1.47642710 -1.84572537 -2.19363884 -4.46617088 -2.56923277 -1.14090797 -1.00196071 -1.00000022 -0.83598131 -1.05302032 -1.34702602 -1.64579808 -1.91607685 -2.19363884 -3.52467473 -3.18444579 -1.30733317 -1.01151158 -1.00002255 -1.00000000 -1.10711539 -1.41794008 -1.80964877 -1.99289978 -2.48729715 -4.46617088 -3.18444579 -1.54735240 -1.02717675 -1.00020750 -1.00000002 -1.00000000 -1.54571030 -1.78819213 -2.13732651 -3.09972265 -5.02047567 -2.56923277 -1.30733317 -1.02717675 -1.00093673 -1.00000020 -1.00000000 -1.00000000 -1.64889211 -2.55789130 -4.92673785 -3.69851764 -1.74250470 -1.14090797 -1.01151158 -1.00020750 -1.00000020 -1.00000000 -1.00000000 -1.00000000 -6.74836785 -3.34525938 -1.71008559 -1.18071033 -1.02797014 -1.00196071 -1.00002255 -1.00000002 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.12321467 -1.06814764 -1.01698812 -1.00161247 -1.00004667 -1.00000022 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -0.06028598 -0.09454520 -0.20220073 -0.37708219 -0.56522550 -0.78305577 -1.05302032 -1.41794008 -1.78819213 -2.55789130 -3.34525938 -1.06814764 -0.09454520 -0.13707712 -0.25363317 -0.43653918 -0.65601811 -0.91535002 -1.22495668 -1.65556195 -1.96413896 -3.79775217 -2.24473568 -1.03810236 -0.20220073 -0.25363317 -0.35047190 -0.53841255 -0.78764869 -1.08616431 -1.46680475 -1.91035799 -2.37151402 -5.21745587 -1.45588968 -1.00962730 -0.37708219 -0.43653918 -0.53841255 -0.68630633 -0.95276250 -1.30058326 -1.74284788 -2.08683938 -3.68913863 -2.96038837 -1.12195513 -1.00091153 -0.56522550 -0.65601811 -0.78764869 -0.95276250 -1.18653545 -1.60397360 -1.96337513 -2.79495075 -4.78716176 -1.54063711 -1.01848518 -1.00002536 -0.78305577 -0.91535002 -1.08616431 -1.30058326 -1.60397360 -1.90461577 -2.40758071 -4.82522598 -2.20193738 -1.10012297 -1.00121571 -1.00000012 -1.05302032 -1.22495668 -1.46680475 -1.74284788 -1.96337513 -2.40758071 -4.16408894 -2.80143960 -1.23126187 -1.00773513 -1.00001270 -1.00000000 -1.41794008 -1.65556195 -1.91035799 -2.08683938 -2.79495075 -4.82522598 -2.80143960 -1.37979954 -1.01932031 -1.00012613 -1.00000001 -1.00000000 -1.78819213 -1.96413896 -2.37151402 -3.68913863 -4.78716176 -2.20193738 -1.23126187 -1.01932031 -1.00054429 -1.00000011 -1.00000000 -1.00000000 -2.55789130 -3.79775217 -5.21745587 -2.96038837 -1.54063711 -1.10012297 -1.00773513 -1.00012613 -1.00000011 -1.00000000 -1.00000000 -1.00000000 -3.34525938 -2.24473568 -1.45588968 -1.12195513 -1.01848518 -1.00121571 -1.00001270 -1.00000001 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.06814764 -1.03810236 -1.00962730 -1.00091153 -1.00002536 -1.00000012 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -0.18275629 -0.20220073 -0.28684617 -0.45987870 -0.72089895 -0.99970957 -1.34702602 -1.80964877 -2.13732651 -4.92673785 -1.71008559 -1.01698812 -0.20220073 -0.25363317 -0.35047190 -0.53841255 -0.78764869 -1.08616431 -1.46680475 -1.91035799 -2.37151402 -5.21745587 -1.45588968 -1.00962730 -0.28684617 -0.35047190 -0.48503526 -0.67607895 -0.92000432 -1.23670716 -1.67408651 -2.03637404 -3.17013000 -3.74258616 -1.19708047 -1.00259320 -0.45987870 -0.53841255 -0.67607895 -0.85444168 -1.11636939 -1.47792318 -1.89455570 -2.36028267 -4.85497505 -2.02148892 -1.05398342 -1.00025353 -0.72089895 -0.78764869 -0.92000432 -1.11636939 -1.39929604 -1.78482881 -2.11214051 -3.62383286 -3.59078365 -1.29006240 -1.00767405 -1.00000659 -0.99970957 -1.08616431 -1.23670716 -1.47792318 -1.78482881 -2.05001846 -3.03428814 -4.69553640 -1.70798913 -1.04930793 -1.00043503 -1.00000003 -1.34702602 -1.46680475 -1.67408651 -1.89455570 -2.11214051 -3.03428814 -4.69121168 -2.11845737 -1.12863248 -1.00334449 -1.00000367 -1.00000000 -1.80964877 -1.91035799 -2.03637404 -2.36028267 -3.62383286 -4.69553640 -2.11845737 -1.19206755 -1.00928909 -1.00004347 -1.00000000 -1.00000000 -2.13732651 -2.37151402 -3.17013000 -4.85497505 -3.59078365 -1.70798913 -1.12863248 -1.00928909 -1.00016737 -1.00000004 -1.00000000 -1.00000000 -4.92673785 -5.21745587 -3.74258616 -2.02148892 -1.29006240 -1.04930793 -1.00334449 -1.00004347 -1.00000004 -1.00000000 -1.00000000 -1.00000000 -1.71008559 -1.45588968 -1.19708047 -1.05398342 -1.00767405 -1.00043503 -1.00000367 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.01698812 -1.00962730 -1.00259320 -1.00025353 -1.00000659 -1.00000003 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -0.30874663 -0.37708219 -0.45987870 -0.60254824 -0.85104435 -1.20884303 -1.64579808 -1.99289978 -3.09972265 -3.69851764 -1.18071033 -1.00161247 -0.37708219 -0.43653918 -0.53841255 -0.68630633 -0.95276250 -1.30058326 -1.74284788 -2.08683938 -3.68913863 -2.96038837 -1.12195513 -1.00091153 -0.45987870 -0.53841255 -0.67607895 -0.85444168 -1.11636939 -1.47792318 -1.89455570 -2.36028267 -4.85497505 -2.02148892 -1.05398342 -1.00025353 -0.60254824 -0.68630633 -0.85444168 -1.06799788 -1.34884632 -1.72646359 -2.05676383 -3.23815309 -4.23041524 -1.39968435 -1.01389135 -1.00002607 -0.85104435 -0.95276250 -1.11636939 -1.34884632 -1.66271014 -1.95104280 -2.56601603 -4.89786332 -2.19501261 -1.11057793 -1.00176949 -1.00000061 -1.20884303 -1.30058326 -1.47792318 -1.72646359 -1.95104280 -2.43868827 -4.31104219 -3.18754429 -1.32061019 -1.01554183 -1.00007703 -1.00000000 -1.64579808 -1.74284788 -1.89455570 -2.05676383 -2.56601603 -4.31104219 -3.53444834 -1.52323853 -1.04972809 -1.00084205 -1.00000045 -1.00000000 -1.99289978 -2.08683938 -2.36028267 -3.23815309 -4.89786332 -3.18754429 -1.52323853 -1.07104992 -1.00268739 -1.00000730 -1.00000000 -1.00000000 -3.09972265 -3.68913863 -4.85497505 -4.23041524 -2.19501261 -1.32061019 -1.04972809 -1.00268739 -1.00002353 -1.00000001 -1.00000000 -1.00000000 -3.69851764 -2.96038837 -2.02148892 -1.39968435 -1.11057793 -1.01554183 -1.00084205 -1.00000730 -1.00000001 -1.00000000 -1.00000000 -1.00000000 -1.18071033 -1.12195513 -1.05398342 -1.01389135 -1.00176949 -1.00007703 -1.00000045 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00161247 -1.00091153 -1.00025353 -1.00002607 -1.00000061 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -0.44891561 -0.56522550 -0.72089895 -0.85104435 -1.07338231 -1.47642710 -1.91607685 -2.48729715 -5.02047567 -1.74250470 -1.02797014 -1.00004667 -0.56522550 -0.65601811 -0.78764869 -0.95276250 -1.18653545 -1.60397360 -1.96337513 -2.79495075 -4.78716176 -1.54063711 -1.01848518 -1.00002536 -0.72089895 -0.78764869 -0.92000432 -1.11636939 -1.39929604 -1.78482881 -2.11214051 -3.62383286 -3.59078365 -1.29006240 -1.00767405 -1.00000659 -0.85104435 -0.95276250 -1.11636939 -1.34884632 -1.66271014 -1.95104280 -2.56601603 -4.89786332 -2.19501261 -1.11057793 -1.00176949 -1.00000061 -1.07338231 -1.18653545 -1.39929604 -1.66271014 -1.90393974 -2.29248911 -3.94722752 -3.71408451 -1.41865170 -1.02533557 -1.00018095 -1.00000001 -1.47642710 -1.60397360 -1.78482881 -1.95104280 -2.29248911 -3.58481503 -4.46756253 -1.85074347 -1.10400191 -1.00273473 -1.00000498 -1.00000000 -1.91607685 -1.96337513 -2.11214051 -2.56601603 -3.94722752 -4.46756253 -2.05894465 -1.18392652 -1.01153195 -1.00010268 -1.00000002 -1.00000000 -2.48729715 -2.79495075 -3.62383286 -4.89786332 -3.71408451 -1.85074347 -1.18392652 -1.01718902 -1.00039108 -1.00000045 -1.00000000 -1.00000000 -5.02047567 -4.78716176 -3.59078365 -2.19501261 -1.41865170 -1.10400191 -1.01153195 -1.00039108 -1.00000128 -1.00000000 -1.00000000 -1.00000000 -1.74250470 -1.54063711 -1.29006240 -1.11057793 -1.02533557 -1.00273473 -1.00010268 -1.00000045 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.02797014 -1.01848518 -1.00767405 -1.00176949 -1.00018095 -1.00000498 -1.00000002 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00004667 -1.00002536 -1.00000659 -1.00000061 -1.00000001 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -0.61765647 -0.78305577 -0.99970957 -1.20884303 -1.47642710 -1.84572537 -2.19363884 -4.46617088 -2.56923277 -1.14090797 -1.00196071 -1.00000022 -0.78305577 -0.91535002 -1.08616431 -1.30058326 -1.60397360 -1.90461577 -2.40758071 -4.82522598 -2.20193738 -1.10012297 -1.00121571 -1.00000012 -0.99970957 -1.08616431 -1.23670716 -1.47792318 -1.78482881 -2.05001846 -3.03428814 -4.69553640 -1.70798913 -1.04930793 -1.00043503 -1.00000003 -1.20884303 -1.30058326 -1.47792318 -1.72646359 -1.95104280 -2.43868827 -4.31104219 -3.18754429 -1.32061019 -1.01554183 -1.00007703 -1.00000000 -1.47642710 -1.60397360 -1.78482881 -1.95104280 -2.29248911 -3.58481503 -4.46756253 -1.85074347 -1.10400191 -1.00273473 -1.00000498 -1.00000000 -1.84572537 -1.90461577 -2.05001846 -2.43868827 -3.58481503 -4.69561911 -2.34453450 -1.24335023 -1.01858146 -1.00021178 -1.00000007 -1.00000000 -2.19363884 -2.40758071 -3.03428814 -4.31104219 -4.46756253 -2.34453450 -1.32099117 -1.04035687 -1.00132267 -1.00000413 -1.00000000 -1.00000000 -4.46617088 -4.82522598 -4.69553640 -3.18754429 -1.85074347 -1.24335023 -1.04035687 -1.00224842 -1.00002182 -1.00000001 -1.00000000 -1.00000000 -2.56923277 -2.20193738 -1.70798913 -1.32061019 -1.10400191 -1.01858146 -1.00132267 -1.00002182 -1.00000002 -1.00000000 -1.00000000 -1.00000000 -1.14090797 -1.10012297 -1.04930793 -1.01554183 -1.00273473 -1.00021178 -1.00000413 -1.00000001 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00196071 -1.00121571 -1.00043503 -1.00007703 -1.00000498 -1.00000007 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000022 -1.00000012 -1.00000003 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -0.83598131 -1.05302032 -1.34702602 -1.64579808 -1.91607685 -2.19363884 -3.52467473 -3.18444579 -1.30733317 -1.01151158 -1.00002255 -1.00000000 -1.05302032 -1.22495668 -1.46680475 -1.74284788 -1.96337513 -2.40758071 -4.16408894 -2.80143960 -1.23126187 -1.00773513 -1.00001270 -1.00000000 -1.34702602 -1.46680475 -1.67408651 -1.89455570 -2.11214051 -3.03428814 -4.69121168 -2.11845737 -1.12863248 -1.00334449 -1.00000367 -1.00000000 -1.64579808 -1.74284788 -1.89455570 -2.05676383 -2.56601603 -4.31104219 -3.53444834 -1.52323853 -1.04972809 -1.00084205 -1.00000045 -1.00000000 -1.91607685 -1.96337513 -2.11214051 -2.56601603 -3.94722752 -4.46756253 -2.05894465 -1.18392652 -1.01153195 -1.00010268 -1.00000002 -1.00000000 -2.19363884 -2.40758071 -3.03428814 -4.31104219 -4.46756253 -2.34453450 -1.32099117 -1.04035687 -1.00132267 -1.00000413 -1.00000000 -1.00000000 -3.52467473 -4.16408894 -4.69121168 -3.53444834 -2.05894465 -1.32099117 -1.05868539 -1.00389046 -1.00005026 -1.00000003 -1.00000000 -1.00000000 -3.18444579 -2.80143960 -2.11845737 -1.52323853 -1.18392652 -1.04035687 -1.00389046 -1.00011001 -1.00000025 -1.00000000 -1.00000000 -1.00000000 -1.30733317 -1.23126187 -1.12863248 -1.04972809 -1.01153195 -1.00132267 -1.00005026 -1.00000025 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.01151158 -1.00773513 -1.00334449 -1.00084205 -1.00010268 -1.00000413 -1.00000003 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00002255 -1.00001270 -1.00000367 -1.00000045 -1.00000002 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.10711539 -1.41794008 -1.80964877 -1.99289978 -2.48729715 -4.46617088 -3.18444579 -1.54735240 -1.02717675 -1.00020750 -1.00000002 -1.00000000 -1.41794008 -1.65556195 -1.91035799 -2.08683938 -2.79495075 -4.82522598 -2.80143960 -1.37979954 -1.01932031 -1.00012613 -1.00000001 -1.00000000 -1.80964877 -1.91035799 -2.03637404 -2.36028267 -3.62383286 -4.69553640 -2.11845737 -1.19206755 -1.00928909 -1.00004347 -1.00000000 -1.00000000 -1.99289978 -2.08683938 -2.36028267 -3.23815309 -4.89786332 -3.18754429 -1.52323853 -1.07104992 -1.00268739 -1.00000730 -1.00000000 -1.00000000 -2.48729715 -2.79495075 -3.62383286 -4.89786332 -3.71408451 -1.85074347 -1.18392652 -1.01718902 -1.00039108 -1.00000045 -1.00000000 -1.00000000 -4.46617088 -4.82522598 -4.69553640 -3.18754429 -1.85074347 -1.24335023 -1.04035687 -1.00224842 -1.00002182 -1.00000001 -1.00000000 -1.00000000 -3.18444579 -2.80143960 -2.11845737 -1.52323853 -1.18392652 -1.04035687 -1.00389046 -1.00011001 -1.00000025 -1.00000000 -1.00000000 -1.00000000 -1.54735240 -1.37979954 -1.19206755 -1.07104992 -1.01718902 -1.00224842 -1.00011001 -1.00000075 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.02717675 -1.01932031 -1.00928909 -1.00268739 -1.00039108 -1.00002182 -1.00000025 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00020750 -1.00012613 -1.00004347 -1.00000730 -1.00000045 -1.00000001 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000002 -1.00000001 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.54571030 -1.78819213 -2.13732651 -3.09972265 -5.02047567 -2.56923277 -1.30733317 -1.02717675 -1.00093673 -1.00000020 -1.00000000 -1.00000000 -1.78819213 -1.96413896 -2.37151402 -3.68913863 -4.78716176 -2.20193738 -1.23126187 -1.01932031 -1.00054429 -1.00000011 -1.00000000 -1.00000000 -2.13732651 -2.37151402 -3.17013000 -4.85497505 -3.59078365 -1.70798913 -1.12863248 -1.00928909 -1.00016737 -1.00000004 -1.00000000 -1.00000000 -3.09972265 -3.68913863 -4.85497505 -4.23041524 -2.19501261 -1.32061019 -1.04972809 -1.00268739 -1.00002353 -1.00000001 -1.00000000 -1.00000000 -5.02047567 -4.78716176 -3.59078365 -2.19501261 -1.41865170 -1.10400191 -1.01153195 -1.00039108 -1.00000128 -1.00000000 -1.00000000 -1.00000000 -2.56923277 -2.20193738 -1.70798913 -1.32061019 -1.10400191 -1.01858146 -1.00132267 -1.00002182 -1.00000002 -1.00000000 -1.00000000 -1.00000000 -1.30733317 -1.23126187 -1.12863248 -1.04972809 -1.01153195 -1.00132267 -1.00005026 -1.00000025 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.02717675 -1.01932031 -1.00928909 -1.00268739 -1.00039108 -1.00002182 -1.00000025 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00093673 -1.00054429 -1.00016737 -1.00002353 -1.00000128 -1.00000002 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000020 -1.00000011 -1.00000004 -1.00000001 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.64889211 -2.55789130 -4.92673785 -3.69851764 -1.74250470 -1.14090797 -1.01151158 -1.00020750 -1.00000020 -1.00000000 -1.00000000 -1.00000000 -2.55789130 -3.79775217 -5.21745587 -2.96038837 -1.54063711 -1.10012297 -1.00773513 -1.00012613 -1.00000011 -1.00000000 -1.00000000 -1.00000000 -4.92673785 -5.21745587 -3.74258616 -2.02148892 -1.29006240 -1.04930793 -1.00334449 -1.00004347 -1.00000004 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 + +SCALARS element_density float 1 +LOOKUP_TABLE default +0.01441802 +0.07758891 +0.20298445 +0.35958342 +0.51614999 +0.65753200 +1.28588318 +1.43441824 +2.01118713 +2.60873408 +1.06922233 +1.00048861 +0.07758891 +0.11974443 +0.24563729 +0.44481420 +0.60869869 +0.87129803 +1.48134254 +1.47735628 +3.00928592 +1.97717324 +1.04937399 +1.00032282 +0.20298445 +0.24563729 +0.36706316 +0.56490288 +0.79331597 +1.30986445 +1.57055745 +1.90771399 +4.10972809 +1.45494084 +1.02441433 +1.00013360 +0.35958342 +0.44481420 +0.56490288 +0.66752505 +1.07288519 +1.58617401 +1.62026572 +3.16137743 +2.66129669 +1.17980118 +1.00780148 +1.00003034 +0.51614999 +0.60869869 +0.79331597 +1.07288519 +1.48408792 +1.56107477 +2.37191377 +3.86315596 +1.58901521 +1.05816501 +1.00153070 +1.00000291 +0.65753200 +0.87129803 +1.30986445 +1.58617401 +1.56107477 +2.02335183 +3.89632228 +2.10174470 +1.17220796 +1.01279910 +1.00016751 +1.00000010 +1.28588318 +1.48134254 +1.57055745 +1.62026572 +2.37191377 +3.89632228 +2.43683732 +1.28347852 +1.03703270 +1.00145279 +1.00000746 +0.99999999 +1.43441824 +1.47735628 +1.90771399 +3.16137743 +3.86315596 +2.10174470 +1.28347852 +1.05707146 +1.00392093 +1.00006479 +1.00000009 +0.99999999 +2.01118713 +3.00928592 +4.10972809 +2.66129669 +1.58901521 +1.17220796 +1.03703270 +1.00392093 +1.00015292 +1.00000064 +1.00000000 +0.99999999 +2.60873408 +1.97717324 +1.45494084 +1.17980118 +1.05816501 +1.01279910 +1.00145279 +1.00006479 +1.00000064 +1.00000000 +1.00000000 +0.99999999 +1.06922233 +1.04937399 +1.02441433 +1.00780148 +1.00153070 +1.00016751 +1.00000746 +1.00000009 +1.00000000 +1.00000000 +1.00000000 +0.99999999 +1.00048861 +1.00032282 +1.00013360 +1.00003034 +1.00000291 +1.00000010 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.07758891 +0.11974443 +0.24563729 +0.44481420 +0.60869869 +0.87129803 +1.48134254 +1.47735628 +3.00928592 +1.97717324 +1.04937399 +1.00032282 +0.11974443 +0.17101487 +0.30465556 +0.51447327 +0.70139914 +1.09626503 +1.55756841 +1.64732788 +3.85123276 +1.65198806 +1.03517340 +1.00021179 +0.24563729 +0.30465556 +0.43401471 +0.61136881 +0.88709174 +1.45835153 +1.56296818 +2.26261683 +3.69015970 +1.33180253 +1.01731496 +1.00008642 +0.44481420 +0.51447327 +0.61136881 +0.76183352 +1.21033615 +1.57723232 +1.72889934 +3.66123433 +2.27243175 +1.13762028 +1.00550813 +1.00001926 +0.60869869 +0.70139914 +0.88709174 +1.21033615 +1.50617429 +1.60232330 +2.70070403 +3.49475584 +1.46244414 +1.04471470 +1.00107121 +1.00000183 +0.87129803 +1.09626503 +1.45835153 +1.57723232 +1.60232330 +2.33044596 +3.86941749 +1.89080986 +1.13778105 +1.00958451 +1.00011503 +1.00000006 +1.48134254 +1.55756841 +1.56296818 +1.72889934 +2.70070403 +3.86941749 +2.14309826 +1.23323550 +1.02903969 +1.00105502 +1.00000497 +0.99999999 +1.47735628 +1.64732788 +2.26261683 +3.66123433 +3.49475584 +1.89080986 +1.23323550 +1.04469782 +1.00295012 +1.00004519 +1.00000006 +0.99999999 +3.00928592 +3.85123276 +3.69015970 +2.27243175 +1.46244414 +1.13778105 +1.02903969 +1.00295012 +1.00010698 +1.00000043 +1.00000000 +0.99999999 +1.97717324 +1.65198806 +1.33180253 +1.13762028 +1.04471470 +1.00958451 +1.00105502 +1.00004519 +1.00000043 +1.00000000 +1.00000000 +0.99999999 +1.04937399 +1.03517340 +1.01731496 +1.00550813 +1.00107121 +1.00011503 +1.00000497 +1.00000006 +1.00000000 +1.00000000 +1.00000000 +0.99999999 +1.00032282 +1.00021179 +1.00008642 +1.00001926 +1.00000183 +1.00000006 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.20298445 +0.24563729 +0.36706316 +0.56490288 +0.79331597 +1.30986445 +1.57055745 +1.90771399 +4.10972809 +1.45494084 +1.02441433 +1.00013360 +0.24563729 +0.30465556 +0.43401471 +0.61136881 +0.88709174 +1.45835153 +1.56296818 +2.26261683 +3.69015970 +1.33180253 +1.01731496 +1.00008642 +0.36706316 +0.43401471 +0.56531501 +0.72279781 +1.10958113 +1.57685207 +1.63582702 +3.21357683 +2.65395418 +1.18568149 +1.00845322 +1.00003434 +0.56490288 +0.61136881 +0.72279781 +0.99202799 +1.41487533 +1.55478524 +2.10391019 +4.06033401 +1.76132517 +1.08082311 +1.00267008 +1.00000742 +0.79331597 +0.88709174 +1.10958113 +1.41487533 +1.51837903 +1.79372179 +3.45105143 +2.71377595 +1.29055254 +1.02596216 +1.00051057 +1.00000070 +1.30986445 +1.45835153 +1.57685207 +1.55478524 +1.79372179 +3.05293536 +3.35167765 +1.58608615 +1.08772601 +1.00526373 +1.00005277 +1.00000002 +1.57055745 +1.56296818 +1.63582702 +2.10391019 +3.45105143 +3.35167765 +1.74618611 +1.15623300 +1.01752405 +1.00054455 +1.00000214 +0.99999999 +1.90771399 +2.26261683 +3.21357683 +4.06033401 +2.71377595 +1.58608615 +1.15623300 +1.02711578 +1.00163431 +1.00002147 +1.00000002 +0.99999999 +4.10972809 +3.69015970 +2.65395418 +1.76132517 +1.29055254 +1.08772601 +1.01752405 +1.00163431 +1.00005123 +1.00000019 +1.00000000 +0.99999999 +1.45494084 +1.33180253 +1.18568149 +1.08082311 +1.02596216 +1.00526373 +1.00054455 +1.00002147 +1.00000019 +1.00000000 +1.00000000 +0.99999999 +1.02441433 +1.01731496 +1.00845322 +1.00267008 +1.00051057 +1.00005277 +1.00000214 +1.00000002 1.00000000 1.00000000 1.00000000 -3.69851764 -2.96038837 -2.02148892 -1.39968435 -1.11057793 -1.01554183 -1.00084205 -1.00000730 +0.99999999 +1.00013360 +1.00008642 +1.00003434 +1.00000742 +1.00000070 +1.00000002 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.35958342 +0.44481420 +0.56490288 +0.66752505 +1.07288519 +1.58617401 +1.62026572 +3.16137743 +2.66129669 +1.17980118 +1.00780148 +1.00003034 +0.44481420 +0.51447327 +0.61136881 +0.76183352 +1.21033615 +1.57723232 +1.72889934 +3.66123433 +2.27243175 +1.13762028 +1.00550813 +1.00001926 +0.56490288 +0.61136881 +0.72279781 +0.99202799 +1.41487533 +1.55478524 +2.10391019 +4.06033401 +1.76132517 +1.08082311 +1.00267008 +1.00000742 +0.66752505 +0.76183352 +0.99202799 +1.34762469 +1.50434935 +1.68621049 +3.09529179 +3.09807358 +1.37087050 +1.03554321 +1.00083707 +1.00000156 +1.07288519 +1.21033615 +1.41487533 +1.50434935 +1.60783216 +2.42429904 +3.85636763 +1.91261102 +1.14702177 +1.01089024 +1.00015620 +1.00000014 +1.58617401 +1.57723232 +1.55478524 +1.68621049 +2.42429904 +3.81326427 +2.39567485 +1.31689078 +1.04325608 +1.00202564 +1.00001520 +1.00000000 +1.62026572 +1.72889934 +2.10391019 +3.09529179 +3.85636763 +2.39567485 +1.40822128 +1.08298153 +1.00780428 +1.00019060 +1.00000056 +0.99999999 +3.16137743 +3.66123433 +4.06033401 +3.09807358 +1.91261102 +1.31689078 +1.08298153 +1.01236162 +1.00063636 +1.00000659 1.00000001 +0.99999999 +2.66129669 +2.27243175 +1.76132517 +1.37087050 +1.14702177 +1.04325608 +1.00780428 +1.00063636 +1.00001602 +1.00000005 +1.00000000 +0.99999999 +1.17980118 +1.13762028 +1.08082311 +1.03554321 +1.01089024 +1.00202564 +1.00019060 +1.00000659 +1.00000005 1.00000000 1.00000000 -1.00000000 -1.74250470 -1.54063711 -1.29006240 -1.11057793 -1.02533557 -1.00273473 -1.00010268 -1.00000045 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.14090797 -1.10012297 -1.04930793 -1.01554183 -1.00273473 -1.00021178 -1.00000413 +0.99999999 +1.00780148 +1.00550813 +1.00267008 +1.00083707 +1.00015620 +1.00001520 +1.00000056 1.00000001 1.00000000 1.00000000 1.00000000 -1.00000000 -1.01151158 -1.00773513 -1.00334449 -1.00084205 -1.00010268 -1.00000413 -1.00000003 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00020750 -1.00012613 -1.00004347 -1.00000730 -1.00000045 +0.99999999 +1.00003034 +1.00001926 +1.00000742 +1.00000156 +1.00000014 +1.00000000 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.51614999 +0.60869869 +0.79331597 +1.07288519 +1.48408792 +1.56107477 +2.37191377 +3.86315596 +1.58901521 +1.05816501 +1.00153070 +1.00000291 +0.60869869 +0.70139914 +0.88709174 +1.21033615 +1.50617429 +1.60232330 +2.70070403 +3.49475584 +1.46244414 +1.04471470 +1.00107121 +1.00000183 +0.79331597 +0.88709174 +1.10958113 +1.41487533 +1.51837903 +1.79372179 +3.45105143 +2.71377595 +1.29055254 +1.02596216 +1.00051057 +1.00000070 +1.07288519 +1.21033615 +1.41487533 +1.50434935 +1.60783216 +2.42429904 +3.85636763 +1.91261102 +1.14702177 +1.01089024 +1.00015620 +1.00000014 +1.48408792 +1.50617429 +1.51837903 +1.60783216 +2.16021301 +3.64214635 +2.69128886 +1.39739424 +1.05731126 +1.00304866 +1.00002787 1.00000001 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000020 -1.00000011 -1.00000004 +1.56107477 +1.60232330 +1.79372179 +2.42429904 +3.64214635 +3.00961461 +1.64058827 +1.13817215 +1.01546613 +1.00050696 +1.00000247 +0.99999999 +2.37191377 +2.70070403 +3.45105143 +3.85636763 +2.69128886 +1.64058827 +1.18372723 +1.03299726 +1.00238864 +1.00004175 +1.00000008 +0.99999999 +3.86315596 +3.49475584 +2.71377595 +1.91261102 +1.39739424 +1.13817215 +1.03299726 +1.00398275 +1.00016114 +1.00000119 +1.00000000 +0.99999999 +1.58901521 +1.46244414 +1.29055254 +1.14702177 +1.05731126 +1.01546613 +1.00238864 +1.00016114 +1.00000302 1.00000001 1.00000000 +0.99999999 +1.05816501 +1.04471470 +1.02596216 +1.01089024 +1.00304866 +1.00050696 +1.00004175 +1.00000119 +1.00000001 1.00000000 1.00000000 +0.99999999 +1.00153070 +1.00107121 +1.00051057 +1.00015620 +1.00002787 +1.00000247 +1.00000008 1.00000000 1.00000000 1.00000000 1.00000000 +0.99999999 +1.00000291 +1.00000183 +1.00000070 +1.00000014 +1.00000001 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.65753200 +0.87129803 +1.30986445 +1.58617401 +1.56107477 +2.02335183 +3.89632228 +2.10174470 +1.17220796 +1.01279910 +1.00016751 +1.00000010 +0.87129803 +1.09626503 +1.45835153 +1.57723232 +1.60232330 +2.33044596 +3.86941749 +1.89080986 +1.13778105 +1.00958451 +1.00011503 +1.00000006 +1.30986445 +1.45835153 +1.57685207 +1.55478524 +1.79372179 +3.05293536 +3.35167765 +1.58608615 +1.08772601 +1.00526373 +1.00005277 +1.00000002 +1.58617401 +1.57723232 +1.55478524 +1.68621049 +2.42429904 +3.81326427 +2.39567485 +1.31689078 +1.04325608 +1.00202564 +1.00001520 +1.00000000 +1.56107477 +1.60232330 +1.79372179 +2.42429904 +3.64214635 +3.00961461 +1.64058827 +1.13817215 +1.01546613 +1.00050696 +1.00000247 +0.99999999 +2.02335183 +2.33044596 +3.05293536 +3.81326427 +3.00961461 +1.80410519 +1.23651823 +1.04443467 +1.00357442 +1.00007347 +1.00000019 +0.99999999 +3.89632228 +3.86941749 +3.35167765 +2.39567485 +1.64058827 +1.23651823 +1.06215368 +1.00883805 +1.00045144 +1.00000499 +1.00000000 +0.99999999 +2.10174470 +1.89080986 +1.58608615 +1.31689078 +1.13817215 +1.04443467 +1.00883805 +1.00081734 +1.00002346 +1.00000011 1.00000000 +0.99999999 +1.17220796 +1.13778105 +1.08772601 +1.04325608 +1.01546613 +1.00357442 +1.00045144 +1.00002346 +1.00000030 +1.00000000 +1.00000000 +0.99999999 +1.01279910 +1.00958451 +1.00526373 +1.00202564 +1.00050696 +1.00007347 +1.00000499 +1.00000011 1.00000000 1.00000000 1.00000000 +0.99999999 +1.00016751 +1.00011503 +1.00005277 +1.00001520 +1.00000247 +1.00000019 1.00000000 1.00000000 1.00000000 1.00000000 1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -6.74836785 -3.34525938 -1.71008559 -1.18071033 -1.02797014 -1.00196071 -1.00002255 +0.99999999 +1.00000010 +1.00000006 1.00000002 1.00000000 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +1.28588318 +1.48134254 +1.57055745 +1.62026572 +2.37191377 +3.89632228 +2.43683732 +1.28347852 +1.03703270 +1.00145279 +1.00000746 +0.99999999 +1.48134254 +1.55756841 +1.56296818 +1.72889934 +2.70070403 +3.86941749 +2.14309826 +1.23323550 +1.02903969 +1.00105502 +1.00000497 +0.99999999 +1.57055745 +1.56296818 +1.63582702 +2.10391019 +3.45105143 +3.35167765 +1.74618611 +1.15623300 +1.01752405 +1.00054455 +1.00000214 +0.99999999 +1.62026572 +1.72889934 +2.10391019 +3.09529179 +3.85636763 +2.39567485 +1.40822128 +1.08298153 +1.00780428 +1.00019060 +1.00000056 +0.99999999 +2.37191377 +2.70070403 +3.45105143 +3.85636763 +2.69128886 +1.64058827 +1.18372723 +1.03299726 +1.00238864 +1.00004175 +1.00000008 +0.99999999 +3.89632228 +3.86941749 +3.35167765 +2.39567485 +1.64058827 +1.23651823 +1.06215368 +1.00883805 +1.00045144 +1.00000499 +1.00000000 +0.99999999 +2.43683732 +2.14309826 +1.74618611 +1.40822128 +1.18372723 +1.06215368 +1.01325198 +1.00135289 +1.00004444 +1.00000025 1.00000000 -1.00000000 -1.00000000 -3.34525938 -2.24473568 -1.45588968 -1.12195513 -1.01848518 -1.00121571 -1.00001270 +0.99999999 +1.28347852 +1.23323550 +1.15623300 +1.08298153 +1.03299726 +1.00883805 +1.00135289 +1.00008958 +1.00000158 +1.00000000 +1.00000000 +0.99999999 +1.03703270 +1.02903969 +1.01752405 +1.00780428 +1.00238864 +1.00045144 +1.00004444 +1.00000158 1.00000001 1.00000000 1.00000000 -1.00000000 -1.00000000 -1.71008559 -1.45588968 -1.19708047 -1.05398342 -1.00767405 -1.00043503 -1.00000367 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.18071033 -1.12195513 -1.05398342 -1.01389135 -1.00176949 -1.00007703 -1.00000045 -1.00000000 +0.99999999 +1.00145279 +1.00105502 +1.00054455 +1.00019060 +1.00004175 +1.00000499 +1.00000025 1.00000000 1.00000000 1.00000000 1.00000000 -1.02797014 -1.01848518 -1.00767405 -1.00176949 -1.00018095 -1.00000498 +0.99999999 +1.00000746 +1.00000497 +1.00000214 +1.00000056 +1.00000008 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +1.43441824 +1.47735628 +1.90771399 +3.16137743 +3.86315596 +2.10174470 +1.28347852 +1.05707146 +1.00392093 +1.00006479 +1.00000009 +0.99999999 +1.47735628 +1.64732788 +2.26261683 +3.66123433 +3.49475584 +1.89080986 +1.23323550 +1.04469782 +1.00295012 +1.00004519 +1.00000006 +0.99999999 +1.90771399 +2.26261683 +3.21357683 +4.06033401 +2.71377595 +1.58608615 +1.15623300 +1.02711578 +1.00163431 +1.00002147 1.00000002 +0.99999999 +3.16137743 +3.66123433 +4.06033401 +3.09807358 +1.91261102 +1.31689078 +1.08298153 +1.01236162 +1.00063636 +1.00000659 +1.00000001 +0.99999999 +3.86315596 +3.49475584 +2.71377595 +1.91261102 +1.39739424 +1.13817215 +1.03299726 +1.00398275 +1.00016114 +1.00000119 +1.00000000 +0.99999999 +2.10174470 +1.89080986 +1.58608615 +1.31689078 +1.13817215 +1.04443467 +1.00883805 +1.00081734 +1.00002346 +1.00000011 1.00000000 +0.99999999 +1.28347852 +1.23323550 +1.15623300 +1.08298153 +1.03299726 +1.00883805 +1.00135289 +1.00008958 +1.00000158 +1.00000000 +1.00000000 +0.99999999 +1.05707146 +1.04469782 +1.02711578 +1.01236162 +1.00398275 +1.00081734 +1.00008958 +1.00000366 +1.00000003 1.00000000 1.00000000 +0.99999999 +1.00392093 +1.00295012 +1.00163431 +1.00063636 +1.00016114 +1.00002346 +1.00000158 +1.00000003 1.00000000 1.00000000 -1.00196071 -1.00121571 -1.00043503 -1.00007703 -1.00000498 -1.00000007 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00002255 -1.00001270 -1.00000367 -1.00000045 -1.00000002 -1.00000000 1.00000000 +0.99999999 +1.00006479 +1.00004519 +1.00002147 +1.00000659 +1.00000119 +1.00000011 1.00000000 1.00000000 1.00000000 1.00000000 1.00000000 +0.99999999 +1.00000009 +1.00000006 1.00000002 1.00000001 1.00000000 @@ -11129,22 +10735,139 @@ LOOKUP_TABLE default 1.00000000 1.00000000 1.00000000 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +2.01118713 +3.00928592 +4.10972809 +2.66129669 +1.58901521 +1.17220796 +1.03703270 +1.00392093 +1.00015292 +1.00000064 +1.00000000 +0.99999999 +3.00928592 +3.85123276 +3.69015970 +2.27243175 +1.46244414 +1.13778105 +1.02903969 +1.00295012 +1.00010698 +1.00000043 +1.00000000 +0.99999999 +4.10972809 +3.69015970 +2.65395418 +1.76132517 +1.29055254 +1.08772601 +1.01752405 +1.00163431 +1.00005123 +1.00000019 +1.00000000 +0.99999999 +2.66129669 +2.27243175 +1.76132517 +1.37087050 +1.14702177 +1.04325608 +1.00780428 +1.00063636 +1.00001602 +1.00000005 +1.00000000 +0.99999999 +1.58901521 +1.46244414 +1.29055254 +1.14702177 +1.05731126 +1.01546613 +1.00238864 +1.00016114 +1.00000302 +1.00000001 1.00000000 +0.99999999 +1.17220796 +1.13778105 +1.08772601 +1.04325608 +1.01546613 +1.00357442 +1.00045144 +1.00002346 +1.00000030 +1.00000000 +1.00000000 +0.99999999 +1.03703270 +1.02903969 +1.01752405 +1.00780428 +1.00238864 +1.00045144 +1.00004444 +1.00000158 +1.00000001 1.00000000 1.00000000 +0.99999999 +1.00392093 +1.00295012 +1.00163431 +1.00063636 +1.00016114 +1.00002346 +1.00000158 +1.00000003 1.00000000 1.00000000 1.00000000 +0.99999999 +1.00015292 +1.00010698 +1.00005123 +1.00001602 +1.00000302 +1.00000030 +1.00000001 1.00000000 1.00000000 1.00000000 1.00000000 +0.99999999 +1.00000064 +1.00000043 +1.00000019 +1.00000005 +1.00000001 1.00000000 1.00000000 1.00000000 1.00000000 1.00000000 1.00000000 +0.99999999 1.00000000 1.00000000 1.00000000 @@ -11156,83 +10879,129 @@ LOOKUP_TABLE default 1.00000000 1.00000000 1.00000000 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +2.60873408 +1.97717324 +1.45494084 +1.17980118 +1.05816501 +1.01279910 +1.00145279 +1.00006479 +1.00000064 1.00000000 1.00000000 +0.99999999 +1.97717324 +1.65198806 +1.33180253 +1.13762028 +1.04471470 +1.00958451 +1.00105502 +1.00004519 +1.00000043 1.00000000 1.00000000 +0.99999999 +1.45494084 +1.33180253 +1.18568149 +1.08082311 +1.02596216 +1.00526373 +1.00054455 +1.00002147 +1.00000019 1.00000000 1.00000000 +0.99999999 +1.17980118 +1.13762028 +1.08082311 +1.03554321 +1.01089024 +1.00202564 +1.00019060 +1.00000659 +1.00000005 +1.00000000 +1.00000000 +0.99999999 +1.05816501 +1.04471470 +1.02596216 +1.01089024 +1.00304866 +1.00050696 +1.00004175 +1.00000119 +1.00000001 1.00000000 1.00000000 +0.99999999 +1.01279910 +1.00958451 +1.00526373 +1.00202564 +1.00050696 +1.00007347 +1.00000499 +1.00000011 1.00000000 1.00000000 1.00000000 +0.99999999 +1.00145279 +1.00105502 +1.00054455 +1.00019060 +1.00004175 +1.00000499 +1.00000025 1.00000000 1.00000000 1.00000000 1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.12321467 -1.06814764 -1.01698812 -1.00161247 -1.00004667 -1.00000022 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.06814764 -1.03810236 -1.00962730 -1.00091153 -1.00002536 -1.00000012 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.00000000 -1.01698812 -1.00962730 -1.00259320 -1.00025353 +0.99999999 +1.00006479 +1.00004519 +1.00002147 1.00000659 -1.00000003 +1.00000119 +1.00000011 1.00000000 1.00000000 1.00000000 1.00000000 1.00000000 +0.99999999 +1.00000064 +1.00000043 +1.00000019 +1.00000005 +1.00000001 1.00000000 -1.00161247 -1.00091153 -1.00025353 -1.00002607 -1.00000061 1.00000000 1.00000000 1.00000000 1.00000000 1.00000000 +0.99999999 1.00000000 1.00000000 -1.00004667 -1.00002536 -1.00000659 -1.00000061 -1.00000001 1.00000000 1.00000000 1.00000000 @@ -11240,11 +11009,9 @@ LOOKUP_TABLE default 1.00000000 1.00000000 1.00000000 -1.00000022 -1.00000012 -1.00000003 1.00000000 1.00000000 +0.99999999 1.00000000 1.00000000 1.00000000 @@ -11256,33 +11023,107 @@ LOOKUP_TABLE default 1.00000000 1.00000000 1.00000000 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +1.06922233 +1.04937399 +1.02441433 +1.00780148 +1.00153070 +1.00016751 +1.00000746 +1.00000009 1.00000000 1.00000000 1.00000000 +0.99999999 +1.04937399 +1.03517340 +1.01731496 +1.00550813 +1.00107121 +1.00011503 +1.00000497 +1.00000006 1.00000000 1.00000000 1.00000000 +0.99999999 +1.02441433 +1.01731496 +1.00845322 +1.00267008 +1.00051057 +1.00005277 +1.00000214 +1.00000002 1.00000000 1.00000000 1.00000000 +0.99999999 +1.00780148 +1.00550813 +1.00267008 +1.00083707 +1.00015620 +1.00001520 +1.00000056 +1.00000001 1.00000000 1.00000000 1.00000000 +0.99999999 +1.00153070 +1.00107121 +1.00051057 +1.00015620 +1.00002787 +1.00000247 +1.00000008 1.00000000 1.00000000 1.00000000 1.00000000 +0.99999999 +1.00016751 +1.00011503 +1.00005277 +1.00001520 +1.00000247 +1.00000019 1.00000000 1.00000000 1.00000000 1.00000000 1.00000000 +0.99999999 +1.00000746 +1.00000497 +1.00000214 +1.00000056 +1.00000008 1.00000000 1.00000000 1.00000000 1.00000000 1.00000000 1.00000000 +0.99999999 +1.00000009 +1.00000006 +1.00000002 +1.00000001 1.00000000 1.00000000 1.00000000 @@ -11290,6 +11131,7 @@ LOOKUP_TABLE default 1.00000000 1.00000000 1.00000000 +0.99999999 1.00000000 1.00000000 1.00000000 @@ -11301,6 +11143,7 @@ LOOKUP_TABLE default 1.00000000 1.00000000 1.00000000 +0.99999999 1.00000000 1.00000000 1.00000000 @@ -11312,6 +11155,7 @@ LOOKUP_TABLE default 1.00000000 1.00000000 1.00000000 +0.99999999 1.00000000 1.00000000 1.00000000 @@ -11323,7 +11167,163 @@ LOOKUP_TABLE default 1.00000000 1.00000000 1.00000000 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +1.00048861 +1.00032282 +1.00013360 +1.00003034 +1.00000291 +1.00000010 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +1.00032282 +1.00021179 +1.00008642 +1.00001926 +1.00000183 +1.00000006 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +1.00013360 +1.00008642 +1.00003434 +1.00000742 +1.00000070 +1.00000002 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +1.00003034 +1.00001926 +1.00000742 +1.00000156 +1.00000014 +1.00000000 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +1.00000291 +1.00000183 +1.00000070 +1.00000014 +1.00000001 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +1.00000010 +1.00000006 +1.00000002 1.00000000 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999999 +0.99999998 SCALARS mass float 1 LOOKUP_TABLE default @@ -13058,131 +13058,131 @@ LOOKUP_TABLE default SCALARS pressure float 1 LOOKUP_TABLE default -0.08457759 -0.07837191 -0.08292159 -0.08784750 -0.09563914 -0.10658444 -0.12079947 -0.13869234 -0.17544854 -0.14070008 -0.69962468 -0.00007882 -0.07837191 -0.08100683 -0.07523654 -0.08115595 -0.08749239 -0.09673615 -0.10794677 -0.12406384 -0.13513252 -0.16869955 -0.10828226 -0.00001502 -0.08292159 -0.07523654 -0.08123854 -0.07586113 -0.08457621 -0.09301326 -0.10540776 -0.12461277 -0.12007987 -0.30137312 -0.00692725 -0.00000009 -0.08784750 -0.08115595 -0.07586113 -0.08496564 -0.08162346 -0.09297477 -0.11054294 -0.11301305 -0.16707078 -0.10993614 -0.00018963 -0.00000000 -0.09563914 -0.08749239 -0.08457621 -0.08162346 -0.09161912 -0.09568263 -0.10517885 -0.12224465 -0.26041562 -0.00683980 -0.00000034 -0.00000000 -0.10658444 -0.09673615 -0.09301326 -0.09297477 -0.09568263 -0.11030559 -0.09867278 -0.21720238 -0.03209334 -0.00007451 -0.00000000 -0.00000000 -0.12079947 -0.10794677 -0.10540776 -0.11054294 -0.10517885 -0.09867278 -0.16648816 -0.06288320 -0.00065344 -0.00000001 +0.04605813 +0.04678190 +0.04479605 +0.05231453 +0.05452359 +0.05242366 +0.10546671 +0.08559678 +0.09643499 +0.07645685 +0.00010405 +0.00000000 +0.04678190 +0.04641372 +0.04692754 +0.05422483 +0.05208224 +0.06082514 +0.09903774 +0.06774504 +0.14367368 +0.03281234 +0.00005095 +0.00000000 +0.04479605 +0.04692754 +0.05337166 +0.05297586 +0.05293440 +0.07966929 +0.07497180 +0.07327357 +0.17678363 +0.00747620 +0.00000879 +0.00000000 +0.05231453 +0.05422483 +0.05297586 +0.04949728 +0.06386733 +0.08166028 +0.06089039 +0.13078337 +0.06156196 +0.00096118 +0.00000039 +0.00000000 +0.05452359 +0.05208224 +0.05293440 +0.06386733 +0.08276431 +0.05940923 +0.08613546 +0.14157858 +0.01060110 +0.00006208 +0.00000000 +0.00000000 +0.05242366 +0.06082514 +0.07966929 +0.08166028 +0.05940923 +0.07063272 +0.14586201 +0.02900052 +0.00073634 +0.00000122 +0.00000000 +0.00000000 +0.10546671 +0.09903774 +0.07497180 +0.06089039 +0.08613546 +0.14586201 +0.04505124 +0.00205654 +0.00001693 +0.00000000 +0.00000000 +0.00000000 +0.08559678 +0.06774504 +0.07327357 +0.13078337 +0.14157858 +0.02900052 +0.00205654 +0.00004851 +0.00000004 0.00000000 0.00000000 -0.13869234 -0.12406384 -0.12461277 -0.11301305 -0.12224465 -0.21720238 -0.06288320 -0.00308220 -0.00000020 0.00000000 +0.09643499 +0.14367368 +0.17678363 +0.06156196 +0.01060110 +0.00073634 +0.00001693 +0.00000004 0.00000000 0.00000000 -0.17544854 -0.13513252 -0.12007987 -0.16707078 -0.26041562 -0.03209334 -0.00065344 -0.00000020 0.00000000 0.00000000 +0.07645685 +0.03281234 +0.00747620 +0.00096118 +0.00006208 +0.00000122 0.00000000 0.00000000 -0.14070008 -0.16869955 -0.30137312 -0.10993614 -0.00683980 -0.00007451 -0.00000001 0.00000000 0.00000000 0.00000000 0.00000000 +0.00010405 +0.00005095 +0.00000879 +0.00000039 0.00000000 -0.69962468 -0.10828226 -0.00692725 -0.00018963 -0.00000034 0.00000000 0.00000000 0.00000000 @@ -13190,9 +13190,6 @@ LOOKUP_TABLE default 0.00000000 0.00000000 0.00000000 -0.00007882 -0.00001502 -0.00000009 0.00000000 0.00000000 0.00000000 @@ -13202,274 +13199,277 @@ LOOKUP_TABLE default 0.00000000 0.00000000 0.00000000 -0.07837191 -0.08100683 -0.07523654 -0.08115595 -0.08749239 -0.09673615 -0.10794677 -0.12406384 -0.13513252 -0.16869955 -0.10828226 -0.00001502 -0.08100683 -0.07293218 -0.07783931 -0.07752159 -0.08499539 -0.09356641 -0.10448850 -0.12396206 -0.12087465 -0.23422214 -0.02641282 -0.00000246 -0.07523654 -0.07783931 -0.07439134 -0.07907836 -0.08255778 -0.09199674 -0.10616337 -0.11978148 -0.12761413 -0.28482399 -0.00229972 -0.00000001 -0.08115595 -0.07752159 -0.07907836 -0.07869583 -0.08442278 -0.09188843 -0.10936506 -0.11001225 -0.19727178 -0.05535130 -0.00005850 -0.00000000 -0.08749239 -0.08499539 -0.08255778 -0.08442278 -0.08709285 -0.09998841 -0.10034150 -0.13384413 -0.20983509 -0.00314141 -0.00000009 -0.00000000 -0.09673615 -0.09356641 -0.09199674 -0.09188843 -0.09998841 -0.09872292 -0.10763605 -0.22983901 -0.01751933 -0.00002535 -0.00000000 -0.00000000 -0.10794677 -0.10448850 -0.10616337 -0.10936506 -0.10034150 -0.10763605 -0.18914162 -0.04021482 -0.00029063 -0.00000000 -0.00000000 -0.00000000 -0.12406384 -0.12396206 -0.11978148 -0.11001225 -0.13384413 -0.22983901 -0.04021482 -0.00114662 -0.00000006 0.00000000 0.00000000 0.00000000 -0.13513252 -0.12087465 -0.12761413 -0.19727178 -0.20983509 -0.01751933 -0.00029063 -0.00000006 +0.04678190 +0.04641372 +0.04692754 +0.05422483 +0.05208224 +0.06082514 +0.09903774 +0.06774504 +0.14367368 +0.03281234 +0.00005095 0.00000000 +0.04641372 +0.04660561 +0.05006440 +0.05419045 +0.05199676 +0.07107582 +0.08695230 +0.06659925 +0.18071334 +0.01586248 +0.00002434 0.00000000 +0.04692754 +0.05006440 +0.05404494 +0.05144770 +0.05609404 +0.08448892 +0.06681251 +0.08770322 +0.13336420 +0.00400156 +0.00000414 0.00000000 +0.05422483 +0.05419045 +0.05144770 +0.05188447 +0.07124681 +0.07469602 +0.06286320 +0.15517753 +0.03997096 +0.00054598 +0.00000018 0.00000000 -0.16869955 -0.23422214 -0.28482399 -0.05535130 -0.00314141 -0.00002535 +0.05208224 +0.05199676 +0.05609404 +0.07124681 +0.07556891 +0.05868754 +0.10047836 +0.11094142 +0.00672842 +0.00003503 0.00000000 0.00000000 +0.06082514 +0.07107582 +0.08448892 +0.07469602 +0.05868754 +0.08244089 +0.13877183 +0.02021623 +0.00046044 +0.00000064 0.00000000 0.00000000 +0.09903774 +0.08695230 +0.06681251 +0.06286320 +0.10047836 +0.13877183 +0.03058984 +0.00138796 +0.00000986 0.00000000 0.00000000 -0.10828226 -0.02641282 -0.00229972 -0.00005850 -0.00000009 0.00000000 +0.06774504 +0.06659925 +0.08770322 +0.15517753 +0.11094142 +0.02021623 +0.00138796 +0.00002822 +0.00000002 0.00000000 0.00000000 0.00000000 +0.14367368 +0.18071334 +0.13336420 +0.03997096 +0.00672842 +0.00046044 +0.00000986 +0.00000002 0.00000000 0.00000000 0.00000000 -0.00001502 -0.00000246 -0.00000001 0.00000000 +0.03281234 +0.01586248 +0.00400156 +0.00054598 +0.00003503 +0.00000064 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +0.00005095 +0.00002434 +0.00000414 +0.00000018 0.00000000 0.00000000 -0.08292159 -0.07523654 -0.08123854 -0.07586113 -0.08457621 -0.09301326 -0.10540776 -0.12461277 -0.12007987 -0.30137312 -0.00692725 -0.00000009 -0.07523654 -0.07783931 -0.07439134 -0.07907836 -0.08255778 -0.09199674 -0.10616337 -0.11978148 -0.12761413 -0.28482399 -0.00229972 -0.00000001 -0.08123854 -0.07439134 -0.07419282 -0.07899819 -0.08424496 -0.09138784 -0.10780320 -0.11235274 -0.16612659 -0.10867524 -0.00024256 -0.00000000 -0.07586113 -0.07907836 -0.07899819 -0.07854948 -0.08600151 -0.09580539 -0.10455243 -0.11428434 -0.25353834 -0.01325347 -0.00000441 -0.00000000 -0.08457621 -0.08255778 -0.08424496 -0.08600151 -0.08886359 -0.09968564 -0.09894821 -0.17200500 -0.08838950 -0.00062229 -0.00000000 -0.00000000 -0.09301326 -0.09199674 -0.09138784 -0.09580539 -0.09968564 -0.09318184 -0.13384373 -0.18323069 -0.00514157 -0.00000248 -0.00000000 -0.00000000 -0.10540776 -0.10616337 -0.10780320 -0.10455243 -0.09894821 -0.13384373 -0.18843237 -0.01373889 -0.00005033 -0.00000000 -0.00000000 -0.00000000 -0.12461277 -0.11978148 -0.11235274 -0.11428434 -0.17200500 -0.18323069 -0.01373889 -0.00016164 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.04479605 +0.04692754 +0.05337166 +0.05297586 +0.05293440 +0.07966929 +0.07497180 +0.07327357 +0.17678363 +0.00747620 +0.00000879 +0.00000000 +0.04692754 +0.05006440 +0.05404494 +0.05144770 +0.05609404 +0.08448892 +0.06681251 +0.08770322 +0.13336420 +0.00400156 +0.00000414 +0.00000000 +0.05337166 +0.05404494 +0.05280503 +0.05159469 +0.06620585 +0.08025793 +0.06125361 +0.13285684 +0.06001969 +0.00114333 +0.00000069 +0.00000000 +0.05297586 +0.05144770 +0.05159469 +0.06170097 +0.07802383 +0.06344794 +0.07566777 +0.16271524 +0.01692629 +0.00016619 +0.00000003 +0.00000000 +0.05293440 +0.05609404 +0.06620585 +0.07802383 +0.06430259 +0.06310495 +0.13484607 +0.05966034 +0.00268690 +0.00001001 +0.00000000 +0.00000000 +0.07966929 +0.08448892 +0.08025793 +0.06344794 +0.06310495 +0.11381586 +0.09771893 +0.00969803 +0.00017174 +0.00000015 +0.00000000 +0.00000000 +0.07497180 +0.06681251 +0.06125361 +0.07566777 +0.13484607 +0.09771893 +0.01465780 +0.00060443 +0.00000303 +0.00000000 +0.00000000 +0.00000000 +0.07327357 +0.08770322 +0.13285684 +0.16271524 +0.05966034 +0.00969803 +0.00060443 +0.00000883 0.00000001 0.00000000 0.00000000 0.00000000 -0.12007987 -0.12761413 -0.16612659 -0.25353834 -0.08838950 -0.00514157 -0.00005033 +0.17678363 +0.13336420 +0.06001969 +0.01692629 +0.00268690 +0.00017174 +0.00000303 0.00000001 0.00000000 0.00000000 0.00000000 0.00000000 -0.30137312 -0.28482399 -0.10867524 -0.01325347 -0.00062229 -0.00000248 +0.00747620 +0.00400156 +0.00114333 +0.00016619 +0.00001001 +0.00000015 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00692725 -0.00229972 -0.00024256 -0.00000441 +0.00000879 +0.00000414 +0.00000069 +0.00000003 0.00000000 0.00000000 0.00000000 @@ -13478,8 +13478,6 @@ LOOKUP_TABLE default 0.00000000 0.00000000 0.00000000 -0.00000009 -0.00000001 0.00000000 0.00000000 0.00000000 @@ -13490,138 +13488,131 @@ LOOKUP_TABLE default 0.00000000 0.00000000 0.00000000 -0.08784750 -0.08115595 -0.07586113 -0.08496564 -0.08162346 -0.09297477 -0.11054294 -0.11301305 -0.16707078 -0.10993614 -0.00018963 -0.00000000 -0.08115595 -0.07752159 -0.07907836 -0.07869583 -0.08442278 -0.09188843 -0.10936506 -0.11001225 -0.19727178 -0.05535130 -0.00005850 -0.00000000 -0.07586113 -0.07907836 -0.07899819 -0.07854948 -0.08600151 -0.09580539 -0.10455243 -0.11428434 -0.25353834 -0.01325347 -0.00000441 -0.00000000 -0.08496564 -0.07869583 -0.07854948 -0.08422384 -0.08823131 -0.09961837 -0.10022264 -0.15114731 -0.13891303 -0.00141103 -0.00000004 0.00000000 -0.08162346 -0.08442278 -0.08600151 -0.08823131 -0.09702565 -0.09546706 -0.11432600 -0.22903386 -0.01636932 -0.00003855 -0.00000000 -0.00000000 -0.09297477 -0.09188843 -0.09580539 -0.09961837 -0.09546706 -0.10662170 -0.19678706 -0.05723148 -0.00070762 -0.00000005 0.00000000 +0.05231453 +0.05422483 +0.05297586 +0.04949728 +0.06386733 +0.08166028 +0.06089039 +0.13078337 +0.06156196 +0.00096118 +0.00000039 0.00000000 -0.11054294 -0.10936506 -0.10455243 -0.10022264 -0.11432600 -0.19678706 -0.07646540 -0.00228942 -0.00000232 +0.05422483 +0.05419045 +0.05144770 +0.05188447 +0.07124681 +0.07469602 +0.06286320 +0.15517753 +0.03997096 +0.00054598 +0.00000018 0.00000000 +0.05297586 +0.05144770 +0.05159469 +0.06170097 +0.07802383 +0.06344794 +0.07566777 +0.16271524 +0.01692629 +0.00016619 +0.00000003 0.00000000 +0.04949728 +0.05188447 +0.06170097 +0.07735288 +0.06885300 +0.06044980 +0.11948297 +0.08239811 +0.00450400 +0.00002355 0.00000000 -0.11301305 -0.11001225 -0.11428434 -0.15114731 -0.22903386 -0.05723148 -0.00228942 -0.00000716 0.00000000 +0.06386733 +0.07124681 +0.07802383 +0.06885300 +0.05918511 +0.08789342 +0.13968559 +0.02126324 +0.00063268 +0.00000115 0.00000000 0.00000000 +0.08166028 +0.07469602 +0.06344794 +0.06044980 +0.08789342 +0.14319322 +0.04199796 +0.00304307 +0.00003341 +0.00000001 0.00000000 -0.16707078 -0.19727178 -0.25353834 -0.13891303 -0.01636932 -0.00070762 -0.00000232 0.00000000 +0.06089039 +0.06286320 +0.07566777 +0.11948297 +0.13968559 +0.04199796 +0.00491045 +0.00015023 +0.00000040 0.00000000 0.00000000 0.00000000 +0.13078337 +0.15517753 +0.16271524 +0.08239811 +0.02126324 +0.00304307 +0.00015023 +0.00000127 0.00000000 -0.10993614 -0.05535130 -0.01325347 -0.00141103 -0.00003855 -0.00000005 0.00000000 0.00000000 0.00000000 +0.06156196 +0.03997096 +0.01692629 +0.00450400 +0.00063268 +0.00003341 +0.00000040 0.00000000 0.00000000 0.00000000 -0.00018963 -0.00005850 -0.00000441 -0.00000004 0.00000000 0.00000000 +0.00096118 +0.00054598 +0.00016619 +0.00002355 +0.00000115 +0.00000001 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +0.00000039 +0.00000018 +0.00000003 0.00000000 0.00000000 0.00000000 @@ -13634,130 +13625,128 @@ LOOKUP_TABLE default 0.00000000 0.00000000 0.00000000 -0.09563914 -0.08749239 -0.08457621 -0.08162346 -0.09161912 -0.09568263 -0.10517885 -0.12224465 -0.26041562 -0.00683980 -0.00000034 0.00000000 -0.08749239 -0.08499539 -0.08255778 -0.08442278 -0.08709285 -0.09998841 -0.10034150 -0.13384413 -0.20983509 -0.00314141 -0.00000009 0.00000000 -0.08457621 -0.08255778 -0.08424496 -0.08600151 -0.08886359 -0.09968564 -0.09894821 -0.17200500 -0.08838950 -0.00062229 0.00000000 0.00000000 -0.08162346 -0.08442278 -0.08600151 -0.08823131 -0.09702565 -0.09546706 -0.11432600 -0.22903386 -0.01636932 -0.00003855 0.00000000 0.00000000 -0.09161912 -0.08709285 -0.08886359 -0.09702565 -0.09583979 -0.10194868 -0.17896421 -0.08978000 -0.00137903 -0.00000027 0.00000000 0.00000000 -0.09568263 -0.09998841 -0.09968564 -0.09546706 -0.10194868 -0.16204816 -0.15151291 -0.00718420 -0.00002830 0.00000000 +0.05452359 +0.05208224 +0.05293440 +0.06386733 +0.08276431 +0.05940923 +0.08613546 +0.14157858 +0.01060110 +0.00006208 0.00000000 0.00000000 -0.10517885 -0.10034150 -0.09894821 -0.11432600 -0.17896421 -0.15151291 -0.01160257 -0.00014381 -0.00000001 +0.05208224 +0.05199676 +0.05609404 +0.07124681 +0.07556891 +0.05868754 +0.10047836 +0.11094142 +0.00672842 +0.00003503 0.00000000 0.00000000 +0.05293440 +0.05609404 +0.06620585 +0.07802383 +0.06430259 +0.06310495 +0.13484607 +0.05966034 +0.00268690 +0.00001001 0.00000000 -0.12224465 -0.13384413 -0.17200500 -0.22903386 -0.08978000 -0.00718420 -0.00014381 -0.00000005 0.00000000 +0.06386733 +0.07124681 +0.07802383 +0.06885300 +0.05918511 +0.08789342 +0.13968559 +0.02126324 +0.00063268 +0.00000115 0.00000000 0.00000000 +0.08276431 +0.07556891 +0.06430259 +0.05918511 +0.07774509 +0.14109614 +0.05705607 +0.00485106 +0.00007040 +0.00000004 0.00000000 -0.26041562 -0.20983509 -0.08838950 -0.01636932 -0.00137903 -0.00002830 -0.00000001 0.00000000 +0.05940923 +0.05868754 +0.06310495 +0.08789342 +0.14109614 +0.07640281 +0.01159809 +0.00053227 +0.00000254 0.00000000 0.00000000 0.00000000 +0.08613546 +0.10047836 +0.13484607 +0.13968559 +0.05705607 +0.01159809 +0.00099097 +0.00001655 +0.00000002 0.00000000 -0.00683980 -0.00314141 -0.00062229 -0.00003855 -0.00000027 +0.00000000 +0.00000000 +0.14157858 +0.11094142 +0.05966034 +0.02126324 +0.00485106 +0.00053227 +0.00001655 +0.00000006 0.00000000 0.00000000 0.00000000 0.00000000 +0.01060110 +0.00672842 +0.00268690 +0.00063268 +0.00007040 +0.00000254 +0.00000002 0.00000000 0.00000000 0.00000000 -0.00000034 -0.00000009 0.00000000 0.00000000 +0.00006208 +0.00003503 +0.00001001 +0.00000115 +0.00000004 0.00000000 0.00000000 0.00000000 @@ -13778,117 +13767,118 @@ LOOKUP_TABLE default 0.00000000 0.00000000 0.00000000 -0.10658444 -0.09673615 -0.09301326 -0.09297477 -0.09568263 -0.11030559 -0.09867278 -0.21720238 -0.03209334 -0.00007451 0.00000000 0.00000000 -0.09673615 -0.09356641 -0.09199674 -0.09188843 -0.09998841 -0.09872292 -0.10763605 -0.22983901 -0.01751933 -0.00002535 0.00000000 0.00000000 -0.09301326 -0.09199674 -0.09138784 -0.09580539 -0.09968564 -0.09318184 -0.13384373 -0.18323069 -0.00514157 -0.00000248 0.00000000 0.00000000 -0.09297477 -0.09188843 -0.09580539 -0.09961837 -0.09546706 -0.10662170 -0.19678706 -0.05723148 -0.00070762 -0.00000005 0.00000000 0.00000000 -0.09568263 -0.09998841 -0.09968564 -0.09546706 -0.10194868 -0.16204816 -0.15151291 -0.00718420 -0.00002830 0.00000000 0.00000000 0.00000000 -0.11030559 -0.09872292 -0.09318184 -0.10662170 -0.16204816 -0.18080410 -0.01948391 -0.00031154 -0.00000008 +0.05242366 +0.06082514 +0.07966929 +0.08166028 +0.05940923 +0.07063272 +0.14586201 +0.02900052 +0.00073634 +0.00000122 0.00000000 0.00000000 +0.06082514 +0.07107582 +0.08448892 +0.07469602 +0.05868754 +0.08244089 +0.13877183 +0.02021623 +0.00046044 +0.00000064 0.00000000 -0.09867278 -0.10763605 -0.13384373 -0.19678706 -0.15151291 -0.01948391 -0.00065989 -0.00000114 0.00000000 +0.07966929 +0.08448892 +0.08025793 +0.06344794 +0.06310495 +0.11381586 +0.09771893 +0.00969803 +0.00017174 +0.00000015 0.00000000 0.00000000 +0.08166028 +0.07469602 +0.06344794 +0.06044980 +0.08789342 +0.14319322 +0.04199796 +0.00304307 +0.00003341 +0.00000001 +0.00000000 +0.00000000 +0.05940923 +0.05868754 +0.06310495 +0.08789342 +0.14109614 +0.07640281 +0.01159809 +0.00053227 +0.00000254 +0.00000000 0.00000000 -0.21720238 -0.22983901 -0.18323069 -0.05723148 -0.00718420 -0.00031154 -0.00000114 +0.00000000 +0.07063272 +0.08244089 +0.11381586 +0.14319322 +0.07640281 +0.01735750 +0.00175660 +0.00003575 +0.00000005 0.00000000 0.00000000 0.00000000 +0.14586201 +0.13877183 +0.09771893 +0.04199796 +0.01159809 +0.00175660 +0.00008097 +0.00000053 0.00000000 0.00000000 -0.03209334 -0.01751933 -0.00514157 -0.00070762 -0.00002830 -0.00000008 0.00000000 0.00000000 +0.02900052 +0.02021623 +0.00969803 +0.00304307 +0.00053227 +0.00003575 +0.00000053 0.00000000 0.00000000 0.00000000 0.00000000 -0.00007451 -0.00002535 -0.00000248 +0.00000000 +0.00073634 +0.00046044 +0.00017174 +0.00003341 +0.00000254 0.00000005 0.00000000 0.00000000 @@ -13896,6 +13886,10 @@ LOOKUP_TABLE default 0.00000000 0.00000000 0.00000000 +0.00000122 +0.00000064 +0.00000015 +0.00000001 0.00000000 0.00000000 0.00000000 @@ -13922,115 +13916,113 @@ LOOKUP_TABLE default 0.00000000 0.00000000 0.00000000 -0.12079947 -0.10794677 -0.10540776 -0.11054294 -0.10517885 -0.09867278 -0.16648816 -0.06288320 -0.00065344 -0.00000001 0.00000000 0.00000000 -0.10794677 -0.10448850 -0.10616337 -0.10936506 -0.10034150 -0.10763605 -0.18914162 -0.04021482 -0.00029063 0.00000000 0.00000000 0.00000000 -0.10540776 -0.10616337 -0.10780320 -0.10455243 -0.09894821 -0.13384373 -0.18843237 -0.01373889 -0.00005033 0.00000000 +0.10546671 +0.09903774 +0.07497180 +0.06089039 +0.08613546 +0.14586201 +0.04505124 +0.00205654 +0.00001693 0.00000000 0.00000000 -0.11054294 -0.10936506 -0.10455243 -0.10022264 -0.11432600 -0.19678706 -0.07646540 -0.00228942 -0.00000232 0.00000000 +0.09903774 +0.08695230 +0.06681251 +0.06286320 +0.10047836 +0.13877183 +0.03058984 +0.00138796 +0.00000986 0.00000000 0.00000000 -0.10517885 -0.10034150 -0.09894821 -0.11432600 -0.17896421 -0.15151291 -0.01160257 -0.00014381 -0.00000001 0.00000000 +0.07497180 +0.06681251 +0.06125361 +0.07566777 +0.13484607 +0.09771893 +0.01465780 +0.00060443 +0.00000303 0.00000000 0.00000000 -0.09867278 -0.10763605 -0.13384373 -0.19678706 -0.15151291 -0.01948391 -0.00065989 -0.00000114 0.00000000 +0.06089039 +0.06286320 +0.07566777 +0.11948297 +0.13968559 +0.04199796 +0.00491045 +0.00015023 +0.00000040 0.00000000 0.00000000 0.00000000 -0.16648816 -0.18914162 -0.18843237 -0.07646540 -0.01160257 -0.00065989 -0.00000408 +0.08613546 +0.10047836 +0.13484607 +0.13968559 +0.05705607 +0.01159809 +0.00099097 +0.00001655 +0.00000002 0.00000000 0.00000000 0.00000000 +0.14586201 +0.13877183 +0.09771893 +0.04199796 +0.01159809 +0.00175660 +0.00008097 +0.00000053 0.00000000 0.00000000 -0.06288320 -0.04021482 -0.01373889 -0.00228942 -0.00014381 -0.00000114 0.00000000 0.00000000 +0.04505124 +0.03058984 +0.01465780 +0.00491045 +0.00099097 +0.00008097 +0.00000154 0.00000000 0.00000000 0.00000000 0.00000000 -0.00065344 -0.00029063 -0.00005033 -0.00000232 -0.00000001 0.00000000 +0.00205654 +0.00138796 +0.00060443 +0.00015023 +0.00001655 +0.00000053 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000001 +0.00001693 +0.00000986 +0.00000303 +0.00000040 +0.00000002 0.00000000 0.00000000 0.00000000 @@ -14066,104 +14058,112 @@ LOOKUP_TABLE default 0.00000000 0.00000000 0.00000000 -0.13869234 -0.12406384 -0.12461277 -0.11301305 -0.12224465 -0.21720238 -0.06288320 -0.00308220 -0.00000020 0.00000000 0.00000000 0.00000000 -0.12406384 -0.12396206 -0.11978148 -0.11001225 -0.13384413 -0.22983901 -0.04021482 -0.00114662 -0.00000006 0.00000000 0.00000000 0.00000000 -0.12461277 -0.11978148 -0.11235274 -0.11428434 -0.17200500 -0.18323069 -0.01373889 -0.00016164 -0.00000001 0.00000000 0.00000000 +0.08559678 +0.06774504 +0.07327357 +0.13078337 +0.14157858 +0.02900052 +0.00205654 +0.00004851 +0.00000004 0.00000000 -0.11301305 -0.11001225 -0.11428434 -0.15114731 -0.22903386 -0.05723148 -0.00228942 -0.00000716 0.00000000 0.00000000 +0.06774504 +0.06659925 +0.08770322 +0.15517753 +0.11094142 +0.02021623 +0.00138796 +0.00002822 +0.00000002 0.00000000 0.00000000 -0.12224465 -0.13384413 -0.17200500 -0.22903386 -0.08978000 -0.00718420 -0.00014381 -0.00000005 0.00000000 +0.07327357 +0.08770322 +0.13285684 +0.16271524 +0.05966034 +0.00969803 +0.00060443 +0.00000883 +0.00000001 0.00000000 0.00000000 0.00000000 -0.21720238 -0.22983901 -0.18323069 -0.05723148 -0.00718420 -0.00031154 -0.00000114 +0.13078337 +0.15517753 +0.16271524 +0.08239811 +0.02126324 +0.00304307 +0.00015023 +0.00000127 0.00000000 0.00000000 0.00000000 0.00000000 +0.14157858 +0.11094142 +0.05966034 +0.02126324 +0.00485106 +0.00053227 +0.00001655 +0.00000006 0.00000000 -0.06288320 -0.04021482 -0.01373889 -0.00228942 -0.00014381 -0.00000114 0.00000000 0.00000000 0.00000000 +0.02900052 +0.02021623 +0.00969803 +0.00304307 +0.00053227 +0.00003575 +0.00000053 0.00000000 0.00000000 0.00000000 -0.00308220 -0.00114662 -0.00016164 -0.00000716 -0.00000005 0.00000000 0.00000000 +0.00205654 +0.00138796 +0.00060443 +0.00015023 +0.00001655 +0.00000053 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00000020 +0.00000000 +0.00004851 +0.00002822 +0.00000883 +0.00000127 0.00000006 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000004 +0.00000002 0.00000001 0.00000000 0.00000000 @@ -14210,83 +14210,83 @@ LOOKUP_TABLE default 0.00000000 0.00000000 0.00000000 -0.17544854 -0.13513252 -0.12007987 -0.16707078 -0.26041562 -0.03209334 -0.00065344 -0.00000020 +0.09643499 +0.14367368 +0.17678363 +0.06156196 +0.01060110 +0.00073634 +0.00001693 +0.00000004 0.00000000 0.00000000 0.00000000 0.00000000 -0.13513252 -0.12087465 -0.12761413 -0.19727178 -0.20983509 -0.01751933 -0.00029063 -0.00000006 +0.14367368 +0.18071334 +0.13336420 +0.03997096 +0.00672842 +0.00046044 +0.00000986 +0.00000002 0.00000000 0.00000000 0.00000000 0.00000000 -0.12007987 -0.12761413 -0.16612659 -0.25353834 -0.08838950 -0.00514157 -0.00005033 +0.17678363 +0.13336420 +0.06001969 +0.01692629 +0.00268690 +0.00017174 +0.00000303 0.00000001 0.00000000 0.00000000 0.00000000 0.00000000 -0.16707078 -0.19727178 -0.25353834 -0.13891303 -0.01636932 -0.00070762 -0.00000232 +0.06156196 +0.03997096 +0.01692629 +0.00450400 +0.00063268 +0.00003341 +0.00000040 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.26041562 -0.20983509 -0.08838950 -0.01636932 -0.00137903 -0.00002830 -0.00000001 +0.01060110 +0.00672842 +0.00268690 +0.00063268 +0.00007040 +0.00000254 +0.00000002 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.03209334 -0.01751933 -0.00514157 -0.00070762 -0.00002830 -0.00000008 +0.00073634 +0.00046044 +0.00017174 +0.00003341 +0.00000254 +0.00000005 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00065344 -0.00029063 -0.00005033 -0.00000232 -0.00000001 +0.00001693 +0.00000986 +0.00000303 +0.00000040 +0.00000002 0.00000000 0.00000000 0.00000000 @@ -14294,8 +14294,8 @@ LOOKUP_TABLE default 0.00000000 0.00000000 0.00000000 -0.00000020 -0.00000006 +0.00000004 +0.00000002 0.00000001 0.00000000 0.00000000 @@ -14354,59 +14354,77 @@ LOOKUP_TABLE default 0.00000000 0.00000000 0.00000000 -0.14070008 -0.16869955 -0.30137312 -0.10993614 -0.00683980 -0.00007451 -0.00000001 +0.07645685 +0.03281234 +0.00747620 +0.00096118 +0.00006208 +0.00000122 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.16869955 -0.23422214 -0.28482399 -0.05535130 -0.00314141 -0.00002535 +0.00000000 +0.03281234 +0.01586248 +0.00400156 +0.00054598 +0.00003503 +0.00000064 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00747620 +0.00400156 +0.00114333 +0.00016619 +0.00001001 +0.00000015 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00000000 +0.00096118 +0.00054598 +0.00016619 +0.00002355 +0.00000115 +0.00000001 +0.00000000 +0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +0.00006208 +0.00003503 +0.00001001 +0.00000115 +0.00000004 0.00000000 0.00000000 -0.30137312 -0.28482399 -0.10867524 -0.01325347 -0.00062229 -0.00000248 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 +0.00000122 +0.00000064 +0.00000015 +0.00000001 0.00000000 -0.10993614 -0.05535130 -0.01325347 -0.00141103 -0.00003855 -0.00000005 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.00683980 -0.00314141 -0.00062229 -0.00003855 -0.00000027 0.00000000 0.00000000 0.00000000 @@ -14414,10 +14432,6 @@ LOOKUP_TABLE default 0.00000000 0.00000000 0.00000000 -0.00007451 -0.00002535 -0.00000248 -0.00000005 0.00000000 0.00000000 0.00000000 @@ -14426,7 +14440,6 @@ LOOKUP_TABLE default 0.00000000 0.00000000 0.00000000 -0.00000001 0.00000000 0.00000000 0.00000000 @@ -14485,6 +14498,10 @@ LOOKUP_TABLE default 0.00000000 0.00000000 0.00000000 +0.00010405 +0.00005095 +0.00000879 +0.00000039 0.00000000 0.00000000 0.00000000 @@ -14493,39 +14510,36 @@ LOOKUP_TABLE default 0.00000000 0.00000000 0.00000000 +0.00005095 +0.00002434 +0.00000414 +0.00000018 0.00000000 0.00000000 0.00000000 0.00000000 0.00000000 -0.69962468 -0.10828226 -0.00692725 -0.00018963 -0.00000034 0.00000000 0.00000000 0.00000000 +0.00000879 +0.00000414 +0.00000069 +0.00000003 0.00000000 0.00000000 0.00000000 0.00000000 -0.10828226 -0.02641282 -0.00229972 -0.00005850 -0.00000009 0.00000000 0.00000000 0.00000000 0.00000000 +0.00000039 +0.00000018 +0.00000003 0.00000000 0.00000000 0.00000000 -0.00692725 -0.00229972 -0.00024256 -0.00000441 0.00000000 0.00000000 0.00000000 @@ -14534,10 +14548,6 @@ LOOKUP_TABLE default 0.00000000 0.00000000 0.00000000 -0.00018963 -0.00005850 -0.00000441 -0.00000004 0.00000000 0.00000000 0.00000000 @@ -14546,8 +14556,6 @@ LOOKUP_TABLE default 0.00000000 0.00000000 0.00000000 -0.00000034 -0.00000009 0.00000000 0.00000000 0.00000000 @@ -14642,9 +14650,6 @@ LOOKUP_TABLE default 0.00000000 0.00000000 0.00000000 -0.00007882 -0.00001502 -0.00000009 0.00000000 0.00000000 0.00000000 @@ -14654,9 +14659,6 @@ LOOKUP_TABLE default 0.00000000 0.00000000 0.00000000 -0.00001502 -0.00000246 -0.00000001 0.00000000 0.00000000 0.00000000 @@ -14666,8 +14668,6 @@ LOOKUP_TABLE default 0.00000000 0.00000000 0.00000000 -0.00000009 -0.00000001 0.00000000 0.00000000 0.00000000 @@ -14789,142 +14789,141 @@ LOOKUP_TABLE default SCALARS sound_speed float 1 LOOKUP_TABLE default -3.64687128 -1.47196145 -0.86960501 -0.68863321 -0.59588143 -0.53628762 -0.49074807 -0.45693474 -0.43494586 -0.37711640 -0.41567850 -0.01081493 -1.47196145 -1.19499322 -0.78749431 -0.59891687 -0.50792389 -0.45375605 -0.41334323 -0.38187220 -0.35489283 -0.33154333 -0.23226734 -0.00484158 -0.86960501 -0.78749431 -0.68703821 -0.52433926 -0.44219243 -0.39378564 -0.36113757 -0.33877278 -0.30600184 -0.31929852 -0.08216671 -0.00037403 -0.68863321 -0.59891687 -0.52433926 -0.48478608 -0.39981197 -0.35803210 -0.33458126 -0.30742979 -0.29971827 -0.22257725 -0.01636066 -0.00001481 -0.59588143 -0.50792389 -0.44219243 -0.39981197 -0.37717268 -0.32865098 -0.30246987 -0.28620385 -0.29402581 -0.08088334 -0.00073983 +2.30740933 +1.00245168 +0.60647496 +0.49242001 +0.41959339 +0.36452690 +0.36972712 +0.31536624 +0.28269326 +0.22101292 +0.01273520 +0.00001872 +1.00245168 +0.80374800 +0.56427542 +0.45074835 +0.37763134 +0.34110067 +0.33380819 +0.27645248 +0.28208582 +0.16631087 +0.00899601 +0.00001459 +0.60647496 +0.56427542 +0.49227674 +0.39534521 +0.33348038 +0.31838799 +0.28206320 +0.25301217 +0.26775578 +0.09254269 +0.00378195 +0.00001102 +0.49242001 +0.45074835 +0.39534521 +0.35154526 +0.31498300 +0.29292363 +0.25026818 +0.26258049 +0.19635137 +0.03684870 +0.00080406 0.00001055 -0.53628762 -0.45375605 -0.39378564 -0.35803210 -0.32865098 -0.31560188 -0.27380444 -0.28470101 -0.14428797 -0.01043263 -0.00001580 -0.00001054 -0.49074807 -0.41334323 -0.36113757 -0.33458126 -0.30246987 -0.27380444 -0.28057988 -0.18141565 -0.02886257 -0.00013649 -0.00001054 -0.00001054 -0.45693474 -0.38187220 -0.33877278 -0.30742979 -0.28620385 -0.28470101 -0.18141565 -0.05761829 -0.00057044 -0.00001061 -0.00001054 -0.00001054 -0.43494586 -0.35489283 -0.30600184 -0.29971827 -0.29402581 -0.14428797 -0.02886257 -0.00057044 -0.00001166 -0.00001054 -0.00001054 -0.00001054 -0.37711640 -0.33154333 -0.31929852 -0.22257725 -0.08088334 -0.01043263 -0.00013649 -0.00001061 -0.00001054 -0.00001054 -0.00001054 -0.00001054 -0.41567850 -0.23226734 -0.08216671 -0.01636066 -0.00073983 -0.00001580 -0.00001054 -0.00001054 -0.00001054 -0.00001054 -0.00001054 -0.00001054 -0.01081493 -0.00484158 -0.00037403 -0.00001481 +0.41959339 +0.37763134 +0.33348038 +0.31498300 +0.30487100 +0.25184856 +0.24601745 +0.24714512 +0.10544738 +0.00988828 +0.00008421 +0.00001054 +0.36452690 +0.34110067 +0.31838799 +0.29292363 +0.25184856 +0.24120795 +0.24978597 +0.15164821 +0.03235650 +0.00141913 +0.00001118 +0.00001054 +0.36972712 +0.33380819 +0.28206320 +0.25026818 +0.24601745 +0.24978597 +0.17553530 +0.05167714 +0.00521690 +0.00007069 +0.00001054 +0.00001054 +0.31536624 +0.27645248 +0.25301217 +0.26258049 +0.24714512 +0.15164821 +0.05167714 +0.00874571 +0.00026026 +0.00001058 +0.00001054 +0.00001054 +0.28269326 +0.28208582 +0.26775578 +0.19635137 +0.10544738 +0.03235650 +0.00521690 +0.00026026 +0.00001089 +0.00001054 +0.00001054 +0.00001054 +0.22101292 +0.16631087 +0.09254269 +0.03684870 +0.00988828 +0.00141913 +0.00007069 +0.00001058 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.01273520 +0.00899601 +0.00378195 +0.00080406 +0.00008421 +0.00001118 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001872 +0.00001459 +0.00001102 0.00001055 0.00001054 0.00001054 @@ -14933,419 +14932,285 @@ LOOKUP_TABLE default 0.00001054 0.00001054 0.00001054 -1.47196145 -1.19499322 -0.78749431 -0.59891687 -0.50792389 -0.45375605 -0.41334323 -0.38187220 -0.35489283 -0.33154333 -0.23226734 -0.00484158 -1.19499322 -0.94167592 -0.71518902 -0.54403158 -0.46469088 -0.41275350 -0.37704941 -0.35326129 -0.32026235 -0.32060850 -0.14003905 -0.00198558 -0.78749431 -0.71518902 -0.59478358 -0.49476117 -0.41796229 -0.37571871 -0.34731663 -0.32326735 -0.29947524 -0.30163606 -0.05130944 -0.00015391 -0.59891687 -0.54403158 -0.49476117 -0.43716082 -0.38429248 -0.34315135 -0.32339547 -0.29641515 -0.29853439 -0.17652814 -0.00932219 -0.00001221 -0.50792389 -0.46469088 -0.41796229 -0.38429248 -0.34976413 -0.32232991 -0.29185223 -0.28251204 -0.27028652 -0.05829573 -0.00037442 -0.00001054 -0.45375605 -0.41275350 -0.37571871 -0.34315135 -0.32232991 -0.29392036 -0.27296868 -0.28175875 -0.11515441 -0.00619698 -0.00001289 -0.00001054 -0.41334323 -0.37704941 -0.34731663 -0.32339547 -0.29185223 -0.27296868 -0.27514266 -0.15467737 -0.01983436 -0.00007437 -0.00001054 -0.00001054 -0.38187220 -0.35326129 -0.32326735 -0.29641515 -0.28251204 -0.28175875 -0.15467737 -0.03721566 -0.00032433 -0.00001057 0.00001054 +1.00245168 +0.80374800 +0.56427542 +0.45074835 +0.37763134 +0.34110067 +0.33380819 +0.27645248 +0.28208582 +0.16631087 +0.00899601 +0.00001459 +0.80374800 +0.67394822 +0.52334047 +0.41899067 +0.35150371 +0.32872106 +0.30502915 +0.25957867 +0.27965307 +0.12650466 +0.00625943 +0.00001237 +0.56427542 +0.52334047 +0.45556429 +0.37450335 +0.32463735 +0.31073713 +0.26691820 +0.25417138 +0.24542649 +0.07076512 +0.00260542 +0.00001074 +0.45074835 +0.41899067 +0.37450335 +0.33690931 +0.31322332 +0.28094757 +0.24617150 +0.26578167 +0.17121871 +0.02828229 +0.00055041 +0.00001054 +0.37763134 +0.35150371 +0.32463735 +0.31322332 +0.28917340 +0.24707128 +0.24901307 +0.23001852 +0.08756719 +0.00747603 +0.00005751 +0.00001054 +0.34110067 +0.32872106 +0.31073713 +0.28094757 +0.24707128 +0.24281538 +0.24448501 +0.13349055 +0.02597061 +0.00102749 +0.00001083 +0.00001054 +0.33380819 +0.30502915 +0.26691820 +0.24617150 +0.24901307 +0.24448501 +0.15423818 +0.04331022 +0.00399553 +0.00004983 +0.00001054 +0.00001054 +0.27645248 +0.25957867 +0.25417138 +0.26578167 +0.23001852 +0.13349055 +0.04331022 +0.00670995 +0.00018803 +0.00001056 0.00001054 -0.35489283 -0.32026235 -0.29947524 -0.29853439 -0.27028652 -0.11515441 -0.01983436 -0.00032433 -0.00001099 0.00001054 +0.28208582 +0.27965307 +0.24542649 +0.17121871 +0.08756719 +0.02597061 +0.00399553 +0.00018803 +0.00001070 0.00001054 0.00001054 -0.33154333 -0.32060850 -0.30163606 -0.17652814 -0.05829573 -0.00619698 -0.00007437 -0.00001057 0.00001054 +0.16631087 +0.12650466 +0.07076512 +0.02828229 +0.00747603 +0.00102749 +0.00004983 +0.00001056 0.00001054 0.00001054 0.00001054 -0.23226734 -0.14003905 -0.05130944 -0.00932219 -0.00037442 -0.00001289 -0.00001054 -0.00001054 -0.00001054 -0.00001054 -0.00001054 -0.00001054 -0.00484158 -0.00198558 -0.00015391 -0.00001221 -0.00001054 -0.00001054 -0.00001054 -0.00001054 -0.00001054 -0.00001054 -0.00001054 -0.00001054 -0.86960501 -0.78749431 -0.68703821 -0.52433926 -0.44219243 -0.39378564 -0.36113757 -0.33877278 -0.30600184 -0.31929852 -0.08216671 -0.00037403 -0.78749431 -0.71518902 -0.59478358 -0.49476117 -0.41796229 -0.37571871 -0.34731663 -0.32326735 -0.29947524 -0.30163606 -0.05130944 -0.00015391 -0.68703821 -0.59478358 -0.50491546 -0.44130032 -0.39066221 -0.35094184 -0.32760556 -0.30324065 -0.29553262 -0.21999042 -0.01837672 -0.00002368 -0.52433926 -0.49476117 -0.44130032 -0.39143064 -0.35832198 -0.32869523 -0.30327579 -0.28407677 -0.29502084 -0.10453302 -0.00264075 -0.00001071 -0.44219243 -0.41796229 -0.39066221 -0.35832198 -0.32533553 -0.30510013 -0.27942608 -0.28126185 -0.20254900 -0.02835399 -0.00008876 -0.00001054 -0.39378564 -0.37571871 -0.35094184 -0.32869523 -0.30510013 -0.27523973 -0.27114086 -0.25502390 -0.07083202 -0.00198369 -0.00001092 -0.00001054 -0.36113757 -0.34731663 -0.32760556 -0.30327579 -0.27942608 -0.27114086 -0.25873764 -0.10396576 -0.00862150 -0.00002511 -0.00001054 -0.00001054 -0.33877278 -0.32326735 -0.30324065 -0.28407677 -0.28126185 -0.25502390 -0.10396576 -0.01503287 -0.00009890 -0.00001055 0.00001054 +0.00899601 +0.00625943 +0.00260542 +0.00055041 +0.00005751 +0.00001083 0.00001054 -0.30600184 -0.29947524 -0.29553262 -0.29502084 -0.20254900 -0.07083202 -0.00862150 -0.00009890 -0.00001059 0.00001054 0.00001054 0.00001054 -0.31929852 -0.30163606 -0.21999042 -0.10453302 -0.02835399 -0.00198369 -0.00002511 -0.00001055 0.00001054 0.00001054 +0.00001459 +0.00001237 +0.00001074 0.00001054 0.00001054 -0.08216671 -0.05130944 -0.01837672 -0.00264075 -0.00008876 -0.00001092 -0.00001054 -0.00001054 -0.00001054 -0.00001054 -0.00001054 -0.00001054 -0.00037403 -0.00015391 -0.00002368 -0.00001071 -0.00001054 -0.00001054 -0.00001054 -0.00001054 -0.00001054 -0.00001054 -0.00001054 -0.00001054 -0.68863321 -0.59891687 -0.52433926 -0.48478608 -0.39981197 -0.35803210 -0.33458126 -0.30742979 -0.29971827 -0.22257725 -0.01636066 -0.00001481 -0.59891687 -0.54403158 -0.49476117 -0.43716082 -0.38429248 -0.34315135 -0.32339547 -0.29641515 -0.29853439 -0.17652814 -0.00932219 -0.00001221 -0.52433926 -0.49476117 -0.44130032 -0.39143064 -0.35832198 -0.32869523 -0.30327579 -0.28407677 -0.29502084 -0.10453302 -0.00264075 -0.00001071 -0.48478608 -0.43716082 -0.39143064 -0.36254065 -0.33018285 -0.31010972 -0.28498046 -0.27891759 -0.23393994 -0.04098991 -0.00025368 -0.00001054 -0.39981197 -0.38429248 -0.35832198 -0.33018285 -0.31185979 -0.28557340 -0.27250044 -0.27917138 -0.11148623 -0.00760629 -0.00001593 -0.00001054 -0.35803210 -0.34315135 -0.32869523 -0.31010972 -0.28557340 -0.26994111 -0.27582365 -0.17298712 -0.02988380 -0.00027286 -0.00001056 0.00001054 -0.33458126 -0.32339547 -0.30327579 -0.28498046 -0.27250044 -0.27582365 -0.18988735 -0.05004989 -0.00191934 -0.00001171 0.00001054 0.00001054 -0.30742979 -0.29641515 -0.28407677 -0.27891759 -0.27917138 -0.17298712 -0.05004989 -0.00333814 -0.00001997 0.00001054 0.00001054 0.00001054 -0.29971827 -0.29853439 -0.29502084 -0.23393994 -0.11148623 -0.02988380 -0.00191934 -0.00001997 +0.00001054 +0.60647496 +0.56427542 +0.49227674 +0.39534521 +0.33348038 +0.31838799 +0.28206320 +0.25301217 +0.26775578 +0.09254269 +0.00378195 +0.00001102 +0.56427542 +0.52334047 +0.45556429 +0.37450335 +0.32463735 +0.31073713 +0.26691820 +0.25417138 +0.24542649 +0.07076512 +0.00260542 +0.00001074 +0.49227674 +0.45556429 +0.39456336 +0.34492006 +0.31535013 +0.29125466 +0.24981674 +0.26249558 +0.19414425 +0.04008900 +0.00106842 +0.00001056 +0.39534521 +0.37450335 +0.34492006 +0.32196480 +0.30316491 +0.26079420 +0.24483091 +0.25843873 +0.12655681 +0.01600838 +0.00022326 +0.00001054 +0.33348038 +0.32463735 +0.31535013 +0.30316491 +0.26567366 +0.24214669 +0.25519244 +0.19141688 +0.05890637 +0.00403193 +0.00002466 +0.00001054 +0.31838799 +0.31073713 +0.29125466 +0.26079420 +0.24214669 +0.24926825 +0.22043604 +0.10094918 +0.01622207 +0.00049653 +0.00001058 +0.00001054 +0.28206320 +0.26691820 +0.24981674 +0.24483091 +0.25519244 +0.22043604 +0.11828060 +0.02951712 +0.00222794 +0.00002377 +0.00001054 +0.00001054 +0.25301217 +0.25417138 +0.26249558 +0.25843873 +0.19141688 +0.10094918 +0.02951712 +0.00378549 +0.00009122 +0.00001054 +0.00001054 +0.00001054 +0.26775578 +0.24542649 +0.19414425 +0.12655681 +0.05890637 +0.01622207 +0.00222794 +0.00009122 +0.00001057 +0.00001054 +0.00001054 +0.00001054 +0.09254269 +0.07076512 +0.04008900 +0.01600838 +0.00403193 +0.00049653 +0.00002377 +0.00001054 +0.00001054 0.00001054 0.00001054 0.00001054 +0.00378195 +0.00260542 +0.00106842 +0.00022326 +0.00002466 +0.00001058 0.00001054 -0.22257725 -0.17652814 -0.10453302 -0.04098991 -0.00760629 -0.00027286 -0.00001171 0.00001054 0.00001054 0.00001054 0.00001054 0.00001054 -0.01636066 -0.00932219 -0.00264075 -0.00025368 -0.00001593 +0.00001102 +0.00001074 0.00001056 0.00001054 0.00001054 @@ -15353,150 +15218,277 @@ LOOKUP_TABLE default 0.00001054 0.00001054 0.00001054 -0.00001481 -0.00001221 -0.00001071 0.00001054 0.00001054 0.00001054 +0.49242001 +0.45074835 +0.39534521 +0.35154526 +0.31498300 +0.29292363 +0.25026818 +0.26258049 +0.19635137 +0.03684870 +0.00080406 +0.00001055 +0.45074835 +0.41899067 +0.37450335 +0.33690931 +0.31322332 +0.28094757 +0.24617150 +0.26578167 +0.17121871 +0.02828229 +0.00055041 +0.00001054 +0.39534521 +0.37450335 +0.34492006 +0.32196480 +0.30316491 +0.26079420 +0.24483091 +0.25843873 +0.12655681 +0.01600838 +0.00022326 +0.00001054 +0.35154526 +0.33690931 +0.32196480 +0.30929872 +0.27619223 +0.24443641 +0.25364517 +0.21054122 +0.07399895 +0.00615658 +0.00004727 +0.00001054 +0.31498300 +0.31322332 +0.30316491 +0.27619223 +0.24769102 +0.24581561 +0.24570328 +0.13612119 +0.03032017 +0.00137449 +0.00001143 +0.00001054 +0.29292363 +0.28094757 +0.26079420 +0.24443641 +0.24581561 +0.25017110 +0.17093245 +0.06205908 +0.00730628 +0.00014159 +0.00001054 +0.00001054 +0.25026818 +0.24617150 +0.24483091 +0.25364517 +0.24570328 +0.17093245 +0.07623417 +0.01520520 +0.00081172 +0.00001174 +0.00001054 +0.00001054 +0.26258049 +0.26578167 +0.25843873 +0.21054122 +0.13612119 +0.06205908 +0.01520520 +0.00144338 +0.00002799 +0.00001054 +0.00001054 +0.00001054 +0.19635137 +0.17121871 +0.12655681 +0.07399895 +0.03032017 +0.00730628 +0.00081172 +0.00002799 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.03684870 +0.02828229 +0.01600838 +0.00615658 +0.00137449 +0.00014159 +0.00001174 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.00080406 +0.00055041 +0.00022326 +0.00004727 +0.00001143 +0.00001054 0.00001054 0.00001054 0.00001054 0.00001054 0.00001054 0.00001054 -0.59588143 -0.50792389 -0.44219243 -0.39981197 -0.37717268 -0.32865098 -0.30246987 -0.28620385 -0.29402581 -0.08088334 -0.00073983 0.00001055 -0.50792389 -0.46469088 -0.41796229 -0.38429248 -0.34976413 -0.32232991 -0.29185223 -0.28251204 -0.27028652 -0.05829573 -0.00037442 -0.00001054 -0.44219243 -0.41796229 -0.39066221 -0.35832198 -0.32533553 -0.30510013 -0.27942608 -0.28126185 -0.20254900 -0.02835399 -0.00008876 -0.00001054 -0.39981197 -0.38429248 -0.35832198 -0.33018285 -0.31185979 -0.28557340 -0.27250044 -0.27917138 -0.11148623 -0.00760629 -0.00001593 -0.00001054 -0.37717268 -0.34976413 -0.32533553 -0.31185979 -0.28964811 -0.27224602 -0.27489155 -0.20071891 -0.04025066 -0.00066411 -0.00001062 -0.00001054 -0.32865098 -0.32232991 -0.30510013 -0.28557340 -0.27224602 -0.27448148 -0.23774636 -0.08043416 -0.00653668 -0.00002130 -0.00001054 -0.00001054 -0.30246987 -0.29185223 -0.27942608 -0.27250044 -0.27489155 -0.23774636 -0.09691234 -0.01422822 -0.00015379 -0.00001056 0.00001054 0.00001054 -0.28620385 -0.28251204 -0.28126185 -0.27917138 -0.20071891 -0.08043416 -0.01422822 -0.00029793 -0.00001080 0.00001054 0.00001054 0.00001054 -0.29402581 -0.27028652 -0.20254900 -0.11148623 -0.04025066 -0.00653668 -0.00015379 -0.00001080 0.00001054 0.00001054 0.00001054 0.00001054 -0.08088334 -0.05829573 -0.02835399 -0.00760629 -0.00066411 -0.00002130 +0.00001054 +0.00001054 +0.41959339 +0.37763134 +0.33348038 +0.31498300 +0.30487100 +0.25184856 +0.24601745 +0.24714512 +0.10544738 +0.00988828 +0.00008421 +0.00001054 +0.37763134 +0.35150371 +0.32463735 +0.31322332 +0.28917340 +0.24707128 +0.24901307 +0.23001852 +0.08756719 +0.00747603 +0.00005751 +0.00001054 +0.33348038 +0.32463735 +0.31535013 +0.30316491 +0.26567366 +0.24214669 +0.25519244 +0.19141688 +0.05890637 +0.00403193 +0.00002466 +0.00001054 +0.31498300 +0.31322332 +0.30316491 +0.27619223 +0.24769102 +0.24581561 +0.24570328 +0.13612119 +0.03032017 +0.00137449 +0.00001143 +0.00001054 +0.30487100 +0.28917340 +0.26567366 +0.24769102 +0.24491342 +0.25409917 +0.18797285 +0.07606475 +0.01053405 +0.00025000 +0.00001055 +0.00001054 +0.25184856 +0.24707128 +0.24214669 +0.24581561 +0.25409917 +0.20569493 +0.10854701 +0.02791802 +0.00204273 +0.00002319 +0.00001054 +0.00001054 +0.24601745 +0.24901307 +0.25519244 +0.24570328 +0.18797285 +0.10854701 +0.03735336 +0.00516818 +0.00016619 0.00001056 0.00001054 0.00001054 +0.24714512 +0.23001852 +0.19141688 +0.13612119 +0.07606475 +0.02791802 +0.00516818 +0.00032502 +0.00001121 +0.00001054 0.00001054 0.00001054 +0.10544738 +0.08756719 +0.05890637 +0.03032017 +0.01053405 +0.00204273 +0.00016619 +0.00001121 0.00001054 -0.00073983 -0.00037442 -0.00008876 -0.00001593 -0.00001062 0.00001054 0.00001054 0.00001054 +0.00988828 +0.00747603 +0.00403193 +0.00137449 +0.00025000 +0.00002319 +0.00001056 +0.00001054 0.00001054 0.00001054 0.00001054 0.00001054 +0.00008421 +0.00005751 +0.00002466 +0.00001143 0.00001055 0.00001054 0.00001054 @@ -15509,119 +15501,127 @@ LOOKUP_TABLE default 0.00001054 0.00001054 0.00001054 -0.53628762 -0.45375605 -0.39378564 -0.35803210 -0.32865098 -0.31560188 -0.27380444 -0.28470101 -0.14428797 -0.01043263 -0.00001580 0.00001054 -0.45375605 -0.41275350 -0.37571871 -0.34315135 -0.32232991 -0.29392036 -0.27296868 -0.28175875 -0.11515441 -0.00619698 -0.00001289 0.00001054 -0.39378564 -0.37571871 -0.35094184 -0.32869523 -0.30510013 -0.27523973 -0.27114086 -0.25502390 -0.07083202 -0.00198369 -0.00001092 0.00001054 -0.35803210 -0.34315135 -0.32869523 -0.31010972 -0.28557340 -0.26994111 -0.27582365 -0.17298712 -0.02988380 -0.00027286 -0.00001056 0.00001054 -0.32865098 -0.32232991 -0.30510013 -0.28557340 -0.27224602 -0.27448148 -0.23774636 -0.08043416 -0.00653668 -0.00002130 -0.00001054 -0.00001054 -0.31560188 -0.29392036 -0.27523973 -0.26994111 -0.27448148 -0.25332735 -0.11768853 -0.02043551 -0.00035919 +0.00001054 +0.00001054 +0.00001054 +0.00001054 +0.36452690 +0.34110067 +0.31838799 +0.29292363 +0.25184856 +0.24120795 +0.24978597 +0.15164821 +0.03235650 +0.00141913 +0.00001118 +0.00001054 +0.34110067 +0.32872106 +0.31073713 +0.28094757 +0.24707128 +0.24281538 +0.24448501 +0.13349055 +0.02597061 +0.00102749 +0.00001083 +0.00001054 +0.31838799 +0.31073713 +0.29125466 +0.26079420 +0.24214669 +0.24926825 +0.22043604 +0.10094918 +0.01622207 +0.00049653 +0.00001058 +0.00001054 +0.29292363 +0.28094757 +0.26079420 +0.24443641 +0.24581561 +0.25017110 +0.17093245 +0.06205908 +0.00730628 +0.00014159 +0.00001054 +0.00001054 +0.25184856 +0.24707128 +0.24214669 +0.24581561 +0.25409917 +0.20569493 +0.10854701 +0.02791802 +0.00204273 +0.00002319 +0.00001054 +0.00001054 +0.24120795 +0.24281538 +0.24926825 +0.25017110 +0.20569493 +0.12663013 +0.04865864 +0.00755335 +0.00028690 0.00001063 0.00001054 0.00001054 -0.27380444 -0.27296868 -0.27114086 -0.27582365 -0.23774636 -0.11768853 -0.02885434 -0.00135211 -0.00001308 +0.24978597 +0.24448501 +0.22043604 +0.17093245 +0.10854701 +0.04865864 +0.01127160 +0.00093906 +0.00001938 0.00001054 0.00001054 0.00001054 -0.28470101 -0.28175875 -0.25502390 -0.17298712 -0.08043416 -0.02043551 -0.00135211 -0.00001727 +0.15164821 +0.13349055 +0.10094918 +0.06205908 +0.02791802 +0.00755335 +0.00093906 +0.00003800 0.00001054 0.00001054 0.00001054 0.00001054 -0.14428797 -0.11515441 -0.07083202 -0.02988380 -0.00653668 -0.00035919 -0.00001308 +0.03235650 +0.02597061 +0.01622207 +0.00730628 +0.00204273 +0.00028690 +0.00001938 0.00001054 0.00001054 0.00001054 0.00001054 0.00001054 -0.01043263 -0.00619698 -0.00198369 -0.00027286 -0.00002130 +0.00141913 +0.00102749 +0.00049653 +0.00014159 +0.00002319 0.00001063 0.00001054 0.00001054 @@ -15629,10 +15629,10 @@ LOOKUP_TABLE default 0.00001054 0.00001054 0.00001054 -0.00001580 -0.00001289 -0.00001092 -0.00001056 +0.00001118 +0.00001083 +0.00001058 +0.00001054 0.00001054 0.00001054 0.00001054 @@ -15653,118 +15653,118 @@ LOOKUP_TABLE default 0.00001054 0.00001054 0.00001054 -0.49074807 -0.41334323 -0.36113757 -0.33458126 -0.30246987 -0.27380444 -0.28057988 -0.18141565 -0.02886257 -0.00013649 +0.36972712 +0.33380819 +0.28206320 +0.25026818 +0.24601745 +0.24978597 +0.17553530 +0.05167714 +0.00521690 +0.00007069 0.00001054 0.00001054 -0.41334323 -0.37704941 -0.34731663 -0.32339547 -0.29185223 -0.27296868 -0.27514266 -0.15467737 -0.01983436 -0.00007437 +0.33380819 +0.30502915 +0.26691820 +0.24617150 +0.24901307 +0.24448501 +0.15423818 +0.04331022 +0.00399553 +0.00004983 0.00001054 0.00001054 -0.36113757 -0.34731663 -0.32760556 -0.30327579 -0.27942608 -0.27114086 -0.25873764 -0.10396576 -0.00862150 -0.00002511 +0.28206320 +0.26691820 +0.24981674 +0.24483091 +0.25519244 +0.22043604 +0.11828060 +0.02951712 +0.00222794 +0.00002377 0.00001054 0.00001054 -0.33458126 -0.32339547 -0.30327579 -0.28498046 -0.27250044 -0.27582365 -0.18988735 -0.05004989 -0.00191934 -0.00001171 +0.25026818 +0.24617150 +0.24483091 +0.25364517 +0.24570328 +0.17093245 +0.07623417 +0.01520520 +0.00081172 +0.00001174 0.00001054 0.00001054 -0.30246987 -0.29185223 -0.27942608 -0.27250044 -0.27489155 -0.23774636 -0.09691234 -0.01422822 -0.00015379 +0.24601745 +0.24901307 +0.25519244 +0.24570328 +0.18797285 +0.10854701 +0.03735336 +0.00516818 +0.00016619 0.00001056 0.00001054 0.00001054 -0.27380444 -0.27296868 -0.27114086 -0.27582365 -0.23774636 -0.11768853 -0.02885434 -0.00135211 -0.00001308 +0.24978597 +0.24448501 +0.22043604 +0.17093245 +0.10854701 +0.04865864 +0.01127160 +0.00093906 +0.00001938 0.00001054 0.00001054 0.00001054 -0.28057988 -0.27514266 -0.25873764 -0.18988735 -0.09691234 -0.02885434 -0.00253416 -0.00002930 -0.00001055 +0.17553530 +0.15423818 +0.11828060 +0.07623417 +0.03735336 +0.01127160 +0.00158985 +0.00007351 +0.00001056 0.00001054 0.00001054 0.00001054 -0.18141565 -0.15467737 -0.10396576 -0.05004989 -0.01422822 -0.00135211 -0.00002930 -0.00001056 +0.05167714 +0.04331022 +0.02951712 +0.01520520 +0.00516818 +0.00093906 +0.00007351 +0.00001067 0.00001054 0.00001054 0.00001054 0.00001054 -0.02886257 -0.01983436 -0.00862150 -0.00191934 -0.00015379 -0.00001308 -0.00001055 +0.00521690 +0.00399553 +0.00222794 +0.00081172 +0.00016619 +0.00001938 +0.00001056 0.00001054 0.00001054 0.00001054 0.00001054 0.00001054 -0.00013649 -0.00007437 -0.00002511 -0.00001171 +0.00007069 +0.00004983 +0.00002377 +0.00001174 0.00001056 0.00001054 0.00001054 @@ -15797,117 +15797,116 @@ LOOKUP_TABLE default 0.00001054 0.00001054 0.00001054 -0.45693474 -0.38187220 -0.33877278 -0.30742979 -0.28620385 -0.28470101 -0.18141565 -0.05761829 -0.00057044 -0.00001061 +0.31536624 +0.27645248 +0.25301217 +0.26258049 +0.24714512 +0.15164821 +0.05167714 +0.00874571 +0.00026026 +0.00001058 0.00001054 0.00001054 -0.38187220 -0.35326129 -0.32326735 -0.29641515 -0.28251204 -0.28175875 -0.15467737 -0.03721566 -0.00032433 -0.00001057 +0.27645248 +0.25957867 +0.25417138 +0.26578167 +0.23001852 +0.13349055 +0.04331022 +0.00670995 +0.00018803 +0.00001056 0.00001054 0.00001054 -0.33877278 -0.32326735 -0.30324065 -0.28407677 -0.28126185 -0.25502390 -0.10396576 -0.01503287 -0.00009890 -0.00001055 +0.25301217 +0.25417138 +0.26249558 +0.25843873 +0.19141688 +0.10094918 +0.02951712 +0.00378549 +0.00009122 0.00001054 0.00001054 -0.30742979 -0.29641515 -0.28407677 -0.27891759 -0.27917138 -0.17298712 -0.05004989 -0.00333814 -0.00001997 0.00001054 +0.26258049 +0.26578167 +0.25843873 +0.21054122 +0.13612119 +0.06205908 +0.01520520 +0.00144338 +0.00002799 0.00001054 0.00001054 -0.28620385 -0.28251204 -0.28126185 -0.27917138 -0.20071891 -0.08043416 -0.01422822 -0.00029793 -0.00001080 0.00001054 +0.24714512 +0.23001852 +0.19141688 +0.13612119 +0.07606475 +0.02791802 +0.00516818 +0.00032502 +0.00001121 0.00001054 0.00001054 -0.28470101 -0.28175875 -0.25502390 -0.17298712 -0.08043416 -0.02043551 -0.00135211 -0.00001727 0.00001054 +0.15164821 +0.13349055 +0.10094918 +0.06205908 +0.02791802 +0.00755335 +0.00093906 +0.00003800 0.00001054 0.00001054 0.00001054 -0.18141565 -0.15467737 -0.10396576 -0.05004989 -0.01422822 -0.00135211 -0.00002930 -0.00001056 0.00001054 +0.05167714 +0.04331022 +0.02951712 +0.01520520 +0.00516818 +0.00093906 +0.00007351 +0.00001067 0.00001054 0.00001054 0.00001054 -0.05761829 -0.03721566 -0.01503287 -0.00333814 -0.00029793 -0.00001727 -0.00001056 0.00001054 +0.00874571 +0.00670995 +0.00378549 +0.00144338 +0.00032502 +0.00003800 +0.00001067 0.00001054 0.00001054 0.00001054 0.00001054 -0.00057044 -0.00032433 -0.00009890 -0.00001997 -0.00001080 0.00001054 +0.00026026 +0.00018803 +0.00009122 +0.00002799 +0.00001121 0.00001054 0.00001054 0.00001054 0.00001054 0.00001054 0.00001054 -0.00001061 -0.00001057 -0.00001055 +0.00001054 +0.00001058 +0.00001056 0.00001054 0.00001054 0.00001054 @@ -15941,95 +15940,96 @@ LOOKUP_TABLE default 0.00001054 0.00001054 0.00001054 -0.43494586 -0.35489283 -0.30600184 -0.29971827 -0.29402581 -0.14428797 -0.02886257 -0.00057044 -0.00001166 0.00001054 +0.28269326 +0.28208582 +0.26775578 +0.19635137 +0.10544738 +0.03235650 +0.00521690 +0.00026026 +0.00001089 0.00001054 0.00001054 -0.35489283 -0.32026235 -0.29947524 -0.29853439 -0.27028652 -0.11515441 -0.01983436 -0.00032433 -0.00001099 0.00001054 +0.28208582 +0.27965307 +0.24542649 +0.17121871 +0.08756719 +0.02597061 +0.00399553 +0.00018803 +0.00001070 0.00001054 0.00001054 -0.30600184 -0.29947524 -0.29553262 -0.29502084 -0.20254900 -0.07083202 -0.00862150 -0.00009890 -0.00001059 0.00001054 +0.26775578 +0.24542649 +0.19414425 +0.12655681 +0.05890637 +0.01622207 +0.00222794 +0.00009122 +0.00001057 0.00001054 0.00001054 -0.29971827 -0.29853439 -0.29502084 -0.23393994 -0.11148623 -0.02988380 -0.00191934 -0.00001997 0.00001054 +0.19635137 +0.17121871 +0.12655681 +0.07399895 +0.03032017 +0.00730628 +0.00081172 +0.00002799 0.00001054 0.00001054 0.00001054 -0.29402581 -0.27028652 -0.20254900 -0.11148623 -0.04025066 -0.00653668 -0.00015379 -0.00001080 0.00001054 +0.10544738 +0.08756719 +0.05890637 +0.03032017 +0.01053405 +0.00204273 +0.00016619 +0.00001121 0.00001054 0.00001054 0.00001054 -0.14428797 -0.11515441 -0.07083202 -0.02988380 -0.00653668 -0.00035919 -0.00001308 0.00001054 +0.03235650 +0.02597061 +0.01622207 +0.00730628 +0.00204273 +0.00028690 +0.00001938 0.00001054 0.00001054 0.00001054 0.00001054 -0.02886257 -0.01983436 -0.00862150 -0.00191934 -0.00015379 -0.00001308 -0.00001055 +0.00001054 +0.00521690 +0.00399553 +0.00222794 +0.00081172 +0.00016619 +0.00001938 +0.00001056 0.00001054 0.00001054 0.00001054 0.00001054 0.00001054 -0.00057044 -0.00032433 -0.00009890 -0.00001997 -0.00001080 +0.00026026 +0.00018803 +0.00009122 +0.00002799 +0.00001121 0.00001054 0.00001054 0.00001054 @@ -16037,9 +16037,9 @@ LOOKUP_TABLE default 0.00001054 0.00001054 0.00001054 -0.00001166 -0.00001099 -0.00001059 +0.00001089 +0.00001070 +0.00001057 0.00001054 0.00001054 0.00001054 @@ -16085,71 +16085,71 @@ LOOKUP_TABLE default 0.00001054 0.00001054 0.00001054 -0.37711640 -0.33154333 -0.31929852 -0.22257725 -0.08088334 -0.01043263 -0.00013649 -0.00001061 +0.22101292 +0.16631087 +0.09254269 +0.03684870 +0.00988828 +0.00141913 +0.00007069 +0.00001058 0.00001054 0.00001054 0.00001054 0.00001054 -0.33154333 -0.32060850 -0.30163606 -0.17652814 -0.05829573 -0.00619698 -0.00007437 -0.00001057 +0.16631087 +0.12650466 +0.07076512 +0.02828229 +0.00747603 +0.00102749 +0.00004983 +0.00001056 0.00001054 0.00001054 0.00001054 0.00001054 -0.31929852 -0.30163606 -0.21999042 -0.10453302 -0.02835399 -0.00198369 -0.00002511 -0.00001055 +0.09254269 +0.07076512 +0.04008900 +0.01600838 +0.00403193 +0.00049653 +0.00002377 0.00001054 0.00001054 0.00001054 0.00001054 -0.22257725 -0.17652814 -0.10453302 -0.04098991 -0.00760629 -0.00027286 -0.00001171 0.00001054 +0.03684870 +0.02828229 +0.01600838 +0.00615658 +0.00137449 +0.00014159 +0.00001174 0.00001054 0.00001054 0.00001054 0.00001054 -0.08088334 -0.05829573 -0.02835399 -0.00760629 -0.00066411 -0.00002130 +0.00001054 +0.00988828 +0.00747603 +0.00403193 +0.00137449 +0.00025000 +0.00002319 0.00001056 0.00001054 0.00001054 0.00001054 0.00001054 0.00001054 -0.01043263 -0.00619698 -0.00198369 -0.00027286 -0.00002130 +0.00141913 +0.00102749 +0.00049653 +0.00014159 +0.00002319 0.00001063 0.00001054 0.00001054 @@ -16157,10 +16157,10 @@ LOOKUP_TABLE default 0.00001054 0.00001054 0.00001054 -0.00013649 -0.00007437 -0.00002511 -0.00001171 +0.00007069 +0.00004983 +0.00002377 +0.00001174 0.00001056 0.00001054 0.00001054 @@ -16169,9 +16169,8 @@ LOOKUP_TABLE default 0.00001054 0.00001054 0.00001054 -0.00001061 -0.00001057 -0.00001055 +0.00001058 +0.00001056 0.00001054 0.00001054 0.00001054 @@ -16229,70 +16228,71 @@ LOOKUP_TABLE default 0.00001054 0.00001054 0.00001054 -0.41567850 -0.23226734 -0.08216671 -0.01636066 -0.00073983 -0.00001580 0.00001054 +0.01273520 +0.00899601 +0.00378195 +0.00080406 +0.00008421 +0.00001118 0.00001054 0.00001054 0.00001054 0.00001054 0.00001054 -0.23226734 -0.14003905 -0.05130944 -0.00932219 -0.00037442 -0.00001289 0.00001054 +0.00899601 +0.00625943 +0.00260542 +0.00055041 +0.00005751 +0.00001083 0.00001054 0.00001054 0.00001054 0.00001054 0.00001054 -0.08216671 -0.05130944 -0.01837672 -0.00264075 -0.00008876 -0.00001092 0.00001054 +0.00378195 +0.00260542 +0.00106842 +0.00022326 +0.00002466 +0.00001058 0.00001054 0.00001054 0.00001054 0.00001054 0.00001054 -0.01636066 -0.00932219 -0.00264075 -0.00025368 -0.00001593 -0.00001056 +0.00001054 +0.00080406 +0.00055041 +0.00022326 +0.00004727 +0.00001143 0.00001054 0.00001054 0.00001054 0.00001054 0.00001054 0.00001054 -0.00073983 -0.00037442 -0.00008876 -0.00001593 -0.00001062 0.00001054 +0.00008421 +0.00005751 +0.00002466 +0.00001143 +0.00001055 0.00001054 0.00001054 0.00001054 0.00001054 0.00001054 0.00001054 -0.00001580 -0.00001289 -0.00001092 -0.00001056 +0.00001054 +0.00001118 +0.00001083 +0.00001058 +0.00001054 0.00001054 0.00001054 0.00001054 @@ -16373,10 +16373,9 @@ LOOKUP_TABLE default 0.00001054 0.00001054 0.00001054 -0.01081493 -0.00484158 -0.00037403 -0.00001481 +0.00001872 +0.00001459 +0.00001102 0.00001055 0.00001054 0.00001054 @@ -16385,11 +16384,10 @@ LOOKUP_TABLE default 0.00001054 0.00001054 0.00001054 -0.00484158 -0.00198558 -0.00015391 -0.00001221 0.00001054 +0.00001459 +0.00001237 +0.00001074 0.00001054 0.00001054 0.00001054 @@ -16397,21 +16395,24 @@ LOOKUP_TABLE default 0.00001054 0.00001054 0.00001054 -0.00037403 -0.00015391 -0.00002368 -0.00001071 0.00001054 0.00001054 +0.00001102 +0.00001074 +0.00001056 +0.00001054 0.00001054 0.00001054 0.00001054 0.00001054 0.00001054 0.00001054 -0.00001481 -0.00001221 -0.00001071 +0.00001054 +0.00001054 +0.00001055 +0.00001054 +0.00001054 +0.00001054 0.00001054 0.00001054 0.00001054 @@ -16421,7 +16422,6 @@ LOOKUP_TABLE default 0.00001054 0.00001054 0.00001054 -0.00001055 0.00001054 0.00001054 0.00001054 @@ -16520,287 +16520,287 @@ LOOKUP_TABLE default SCALARS volume float 1 LOOKUP_TABLE default -0.09434889 -0.01658761 -0.00547177 -0.00323890 -0.00222759 -0.00161902 -0.00119620 -0.00090325 -0.00064695 -0.00060647 -0.00014818 -0.00089030 -0.01658761 -0.01057695 -0.00494558 -0.00265194 -0.00176921 -0.00127705 -0.00094965 -0.00070525 -0.00055922 -0.00039095 -0.00029893 -0.00093620 -0.00547177 -0.00494558 -0.00348619 -0.00217449 -0.00138716 -0.00100029 -0.00074238 -0.00055259 -0.00046787 -0.00020297 -0.00058477 -0.00098330 -0.00323890 -0.00265194 -0.00217449 -0.00165962 -0.00117503 -0.00082724 -0.00060761 -0.00050178 -0.00032261 -0.00027038 -0.00084695 -0.00099839 -0.00222759 -0.00176921 -0.00138716 -0.00117503 -0.00093163 -0.00067731 -0.00052190 -0.00040204 -0.00019918 -0.00057389 -0.00097279 -0.00099995 -0.00161902 -0.00127705 -0.00100029 -0.00082724 -0.00067731 -0.00054179 -0.00045586 -0.00022391 -0.00038922 -0.00087649 -0.00099804 -0.00100000 -0.00119620 -0.00094965 -0.00074238 -0.00060761 -0.00052190 -0.00045586 -0.00028371 -0.00031403 -0.00076492 -0.00098862 -0.00099998 +0.06935763 +0.01288844 +0.00492649 +0.00278100 +0.00193742 +0.00152084 +0.00077768 +0.00069715 +0.00049722 +0.00038333 +0.00093526 +0.00099951 +0.01288844 +0.00835112 +0.00407104 +0.00224813 +0.00164285 +0.00114771 +0.00067506 +0.00067688 +0.00033230 +0.00050577 +0.00095295 +0.00099968 +0.00492649 +0.00407104 +0.00272433 +0.00177022 +0.00126053 +0.00076344 +0.00063672 +0.00052419 +0.00024333 +0.00068731 +0.00097617 +0.00099987 +0.00278100 +0.00224813 +0.00177022 +0.00149807 +0.00093207 +0.00063045 +0.00061718 +0.00031632 +0.00037576 +0.00084760 +0.00099226 +0.00099997 +0.00193742 +0.00164285 +0.00126053 +0.00093207 +0.00067381 +0.00064058 +0.00042160 +0.00025886 +0.00062932 +0.00094503 +0.00099847 +0.00100000 +0.00152084 +0.00114771 +0.00076344 +0.00063045 +0.00064058 +0.00049423 +0.00025665 +0.00047580 +0.00085309 +0.00098736 +0.00099983 0.00100000 -0.00090325 -0.00070525 -0.00055259 -0.00050178 -0.00040204 -0.00022391 -0.00031403 -0.00064627 -0.00097354 -0.00099979 +0.00077768 +0.00067506 +0.00063672 +0.00061718 +0.00042160 +0.00025665 +0.00041037 +0.00077913 +0.00096429 +0.00099855 +0.00099999 0.00100000 +0.00069715 +0.00067688 +0.00052419 +0.00031632 +0.00025886 +0.00047580 +0.00077913 +0.00094601 +0.00099609 +0.00099994 0.00100000 -0.00064695 -0.00055922 -0.00046787 -0.00032261 -0.00019918 -0.00038922 -0.00076492 -0.00097354 -0.00099906 0.00100000 +0.00049722 +0.00033230 +0.00024333 +0.00037576 +0.00062932 +0.00085309 +0.00096429 +0.00099609 +0.00099985 0.00100000 0.00100000 -0.00060647 -0.00039095 -0.00020297 -0.00027038 -0.00057389 -0.00087649 -0.00098862 -0.00099979 0.00100000 +0.00038333 +0.00050577 +0.00068731 +0.00084760 +0.00094503 +0.00098736 +0.00099855 +0.00099994 0.00100000 0.00100000 0.00100000 -0.00014818 -0.00029893 -0.00058477 -0.00084695 -0.00097279 -0.00099804 -0.00099998 0.00100000 +0.00093526 +0.00095295 +0.00097617 +0.00099226 +0.00099847 +0.00099983 +0.00099999 0.00100000 0.00100000 0.00100000 0.00100000 -0.00089030 -0.00093620 -0.00098330 -0.00099839 -0.00099995 0.00100000 +0.00099951 +0.00099968 +0.00099987 +0.00099997 0.00100000 0.00100000 0.00100000 0.00100000 0.00100000 0.00100000 -0.01658761 -0.01057695 -0.00494558 -0.00265194 -0.00176921 -0.00127705 -0.00094965 -0.00070525 -0.00055922 -0.00039095 -0.00029893 -0.00093620 -0.01057695 -0.00729516 -0.00394270 -0.00229075 -0.00152435 -0.00109248 -0.00081636 -0.00060402 -0.00050913 -0.00026331 -0.00044549 -0.00096330 -0.00494558 -0.00394270 -0.00285330 -0.00185731 -0.00126960 -0.00092067 -0.00068175 -0.00052346 -0.00042167 -0.00019166 -0.00068687 -0.00099046 -0.00265194 -0.00229075 -0.00185731 -0.00145708 -0.00104958 -0.00076889 -0.00057377 -0.00047919 -0.00027107 -0.00033779 -0.00089130 -0.00099909 -0.00176921 -0.00152435 -0.00126960 -0.00104958 -0.00084279 -0.00062345 -0.00050933 -0.00035779 -0.00020889 -0.00064908 -0.00098185 -0.00099997 -0.00127705 -0.00109248 -0.00092067 -0.00076889 -0.00062345 -0.00052504 -0.00041535 -0.00020724 -0.00045415 -0.00090899 -0.00099879 -0.00100000 -0.00094965 -0.00081636 -0.00068175 -0.00057377 -0.00050933 -0.00041535 -0.00024015 -0.00035696 -0.00081217 -0.00099232 -0.00099999 0.00100000 -0.00070525 -0.00060402 -0.00052346 -0.00047919 -0.00035779 -0.00020724 -0.00035696 -0.00072474 -0.00098105 -0.00099987 0.00100000 +0.01288844 +0.00835112 +0.00407104 +0.00224813 +0.00164285 +0.00114771 +0.00067506 +0.00067688 +0.00033230 +0.00050577 +0.00095295 +0.00099968 +0.00835112 +0.00584744 +0.00328240 +0.00194374 +0.00142572 +0.00091219 +0.00064203 +0.00060704 +0.00025966 +0.00060533 +0.00096602 +0.00099979 +0.00407104 +0.00328240 +0.00230407 +0.00163567 +0.00112728 +0.00068571 +0.00063981 +0.00044197 +0.00027099 +0.00075086 +0.00098298 +0.00099991 +0.00224813 +0.00194374 +0.00163567 +0.00131262 +0.00082622 +0.00063402 +0.00057840 +0.00027313 +0.00044006 +0.00087903 +0.00099452 +0.00099998 +0.00164285 +0.00142572 +0.00112728 +0.00082622 +0.00066393 +0.00062409 +0.00037027 +0.00028614 +0.00068379 +0.00095720 +0.00099893 +0.00100000 +0.00114771 +0.00091219 +0.00068571 +0.00063402 +0.00062409 +0.00042910 +0.00025844 +0.00052887 +0.00087890 +0.00099051 +0.00099988 +0.00100000 +0.00067506 +0.00064203 +0.00063981 +0.00057840 +0.00037027 +0.00025844 +0.00046661 +0.00081088 +0.00097178 +0.00099895 +0.00100000 +0.00100000 +0.00067688 +0.00060704 +0.00044197 +0.00027313 +0.00028614 +0.00052887 +0.00081088 +0.00095721 +0.00099706 +0.00099995 0.00100000 -0.00055922 -0.00050913 -0.00042167 -0.00027107 -0.00020889 -0.00045415 -0.00081217 -0.00098105 -0.00099946 0.00100000 +0.00033230 +0.00025966 +0.00027099 +0.00044006 +0.00068379 +0.00087890 +0.00097178 +0.00099706 +0.00099989 0.00100000 0.00100000 -0.00039095 -0.00026331 -0.00019166 -0.00033779 -0.00064908 -0.00090899 -0.00099232 -0.00099987 0.00100000 +0.00050577 +0.00060533 +0.00075086 +0.00087903 +0.00095720 +0.00099051 +0.00099895 +0.00099995 0.00100000 0.00100000 0.00100000 -0.00029893 -0.00044549 -0.00068687 -0.00089130 -0.00098185 -0.00099879 -0.00099999 0.00100000 +0.00095295 +0.00096602 +0.00098298 +0.00099452 +0.00099893 +0.00099988 0.00100000 0.00100000 0.00100000 0.00100000 -0.00093620 -0.00096330 -0.00099046 -0.00099909 -0.00099997 +0.00100000 +0.00100000 +0.00099968 +0.00099979 +0.00099991 +0.00099998 +0.00100000 0.00100000 0.00100000 0.00100000 @@ -16808,142 +16808,141 @@ LOOKUP_TABLE default 0.00100000 0.00100000 0.00100000 -0.00547177 -0.00494558 -0.00348619 -0.00217449 -0.00138716 -0.00100029 -0.00074238 -0.00055259 -0.00046787 -0.00020297 -0.00058477 -0.00098330 -0.00494558 -0.00394270 -0.00285330 -0.00185731 -0.00126960 -0.00092067 -0.00068175 -0.00052346 -0.00042167 -0.00019166 -0.00068687 -0.00099046 -0.00348619 -0.00285330 -0.00206171 -0.00147912 -0.00108695 -0.00080860 -0.00059734 -0.00049107 -0.00031544 -0.00026719 -0.00083537 -0.00099741 -0.00217449 -0.00185731 -0.00147912 -0.00117035 -0.00089576 -0.00067663 -0.00052783 -0.00042368 -0.00020597 -0.00049468 -0.00094878 -0.00099975 -0.00138716 -0.00126960 -0.00108695 -0.00089576 -0.00071465 -0.00056028 -0.00047345 -0.00027595 -0.00027849 -0.00077516 -0.00099238 +0.00492649 +0.00407104 +0.00272433 +0.00177022 +0.00126053 +0.00076344 +0.00063672 +0.00052419 +0.00024333 +0.00068731 +0.00097617 +0.00099987 +0.00407104 +0.00328240 +0.00230407 +0.00163567 +0.00112728 +0.00068571 +0.00063981 +0.00044197 +0.00027099 +0.00075086 +0.00098298 +0.00099991 +0.00272433 +0.00230407 +0.00176893 +0.00138351 +0.00090124 +0.00063417 +0.00061131 +0.00031118 +0.00037680 +0.00084340 +0.00099162 +0.00099997 +0.00177022 +0.00163567 +0.00138351 +0.00100804 +0.00070678 +0.00064318 +0.00047531 +0.00024629 +0.00056775 +0.00092522 +0.00099734 0.00099999 -0.00100029 -0.00092067 -0.00080860 -0.00067663 -0.00056028 -0.00048780 -0.00032957 -0.00021297 -0.00058548 -0.00095301 -0.00099957 -0.00100000 -0.00074238 -0.00068175 -0.00059734 -0.00052783 -0.00047345 -0.00032957 -0.00021316 -0.00047204 -0.00088603 -0.00099667 -0.00100000 -0.00100000 -0.00055259 -0.00052346 -0.00049107 -0.00042368 -0.00027595 -0.00021297 -0.00047204 -0.00083888 -0.00099080 -0.00099996 +0.00126053 +0.00112728 +0.00090124 +0.00070678 +0.00065860 +0.00055750 +0.00028977 +0.00036849 +0.00077486 +0.00097469 +0.00099949 +0.00100000 +0.00076344 +0.00068571 +0.00063417 +0.00064318 +0.00055750 +0.00032755 +0.00029836 +0.00063048 +0.00091935 +0.00099476 +0.00099995 0.00100000 +0.00063672 +0.00063981 +0.00061131 +0.00047531 +0.00028977 +0.00029836 +0.00057268 +0.00086488 +0.00098278 +0.00099946 0.00100000 -0.00046787 -0.00042167 -0.00031544 -0.00020597 -0.00027849 -0.00058548 -0.00088603 -0.00099080 -0.00099983 0.00100000 +0.00052419 +0.00044197 +0.00031118 +0.00024629 +0.00036849 +0.00063048 +0.00086488 +0.00097360 +0.00099837 +0.00099998 0.00100000 0.00100000 -0.00020297 -0.00019166 -0.00026719 -0.00049468 -0.00077516 -0.00095301 -0.00099667 -0.00099996 +0.00024333 +0.00027099 +0.00037680 +0.00056775 +0.00077486 +0.00091935 +0.00098278 +0.00099837 +0.00099995 +0.00100000 +0.00100000 +0.00100000 +0.00068731 +0.00075086 +0.00084340 +0.00092522 +0.00097469 +0.00099476 +0.00099946 +0.00099998 0.00100000 0.00100000 0.00100000 0.00100000 -0.00058477 -0.00068687 -0.00083537 -0.00094878 -0.00099238 -0.00099957 +0.00097617 +0.00098298 +0.00099162 +0.00099734 +0.00099949 +0.00099995 0.00100000 0.00100000 0.00100000 0.00100000 0.00100000 0.00100000 -0.00098330 -0.00099046 -0.00099741 -0.00099975 +0.00099987 +0.00099991 +0.00099997 0.00099999 0.00100000 0.00100000 @@ -16952,142 +16951,142 @@ LOOKUP_TABLE default 0.00100000 0.00100000 0.00100000 -0.00323890 -0.00265194 -0.00217449 -0.00165962 -0.00117503 -0.00082724 -0.00060761 -0.00050178 -0.00032261 -0.00027038 -0.00084695 -0.00099839 -0.00265194 -0.00229075 -0.00185731 -0.00145708 -0.00104958 -0.00076889 -0.00057377 -0.00047919 -0.00027107 -0.00033779 -0.00089130 -0.00099909 -0.00217449 -0.00185731 -0.00147912 -0.00117035 -0.00089576 -0.00067663 -0.00052783 -0.00042368 -0.00020597 -0.00049468 -0.00094878 -0.00099975 -0.00165962 -0.00145708 -0.00117035 -0.00093633 -0.00074137 -0.00057922 -0.00048620 -0.00030882 -0.00023638 -0.00071445 -0.00098630 +0.00100000 +0.00278100 +0.00224813 +0.00177022 +0.00149807 +0.00093207 +0.00063045 +0.00061718 +0.00031632 +0.00037576 +0.00084760 +0.00099226 0.00099997 -0.00117503 -0.00104958 -0.00089576 -0.00074137 -0.00060143 -0.00051255 -0.00038971 -0.00020417 -0.00045558 -0.00090043 -0.00099823 -0.00100000 -0.00082724 -0.00076889 -0.00067663 -0.00057922 -0.00051255 -0.00041006 -0.00023196 -0.00031372 -0.00075723 -0.00098470 -0.00099992 -0.00100000 -0.00060761 -0.00057377 -0.00052783 -0.00048620 -0.00038971 -0.00023196 -0.00028293 -0.00065650 -0.00095263 +0.00224813 +0.00194374 +0.00163567 +0.00131262 +0.00082622 +0.00063402 +0.00057840 +0.00027313 +0.00044006 +0.00087903 +0.00099452 +0.00099998 +0.00177022 +0.00163567 +0.00138351 +0.00100804 +0.00070678 +0.00064318 +0.00047531 +0.00024629 +0.00056775 +0.00092522 +0.00099734 +0.00099999 +0.00149807 +0.00131262 +0.00100804 +0.00074205 +0.00066474 +0.00059305 +0.00032307 +0.00032278 +0.00072946 +0.00096568 0.00099916 0.00100000 +0.00093207 +0.00082622 +0.00070678 +0.00066474 +0.00062196 +0.00041249 +0.00025931 +0.00052285 +0.00087182 +0.00098923 +0.00099984 +0.00100000 +0.00063045 +0.00063402 +0.00064318 +0.00059305 +0.00041249 +0.00026224 +0.00041742 +0.00075936 +0.00095854 +0.00099798 +0.00099998 0.00100000 -0.00050178 -0.00047919 -0.00042368 -0.00030882 -0.00020417 -0.00031372 -0.00065650 -0.00093366 -0.00099732 +0.00061718 +0.00057840 +0.00047531 +0.00032307 +0.00025931 +0.00041742 +0.00071012 +0.00092338 +0.00099226 +0.00099981 +0.00100000 +0.00100000 +0.00031632 +0.00027313 +0.00024629 +0.00032278 +0.00052285 +0.00075936 +0.00092338 +0.00098779 +0.00099936 0.00099999 0.00100000 0.00100000 -0.00032261 -0.00027107 -0.00020597 -0.00023638 -0.00045558 -0.00075723 -0.00095263 -0.00099732 +0.00037576 +0.00044006 +0.00056775 +0.00072946 +0.00087182 +0.00095854 +0.00099226 +0.00099936 0.00099998 0.00100000 0.00100000 0.00100000 -0.00027038 -0.00033779 -0.00049468 -0.00071445 -0.00090043 -0.00098470 -0.00099916 +0.00084760 +0.00087903 +0.00092522 +0.00096568 +0.00098923 +0.00099798 +0.00099981 0.00099999 0.00100000 0.00100000 0.00100000 0.00100000 -0.00084695 -0.00089130 -0.00094878 -0.00098630 -0.00099823 -0.00099992 +0.00099226 +0.00099452 +0.00099734 +0.00099916 +0.00099984 +0.00099998 0.00100000 0.00100000 0.00100000 0.00100000 0.00100000 0.00100000 -0.00099839 -0.00099909 -0.00099975 0.00099997 +0.00099998 +0.00099999 0.00100000 0.00100000 0.00100000 @@ -17096,131 +17095,136 @@ LOOKUP_TABLE default 0.00100000 0.00100000 0.00100000 -0.00222759 -0.00176921 -0.00138716 -0.00117503 -0.00093163 -0.00067731 -0.00052190 -0.00040204 -0.00019918 -0.00057389 -0.00097279 -0.00099995 -0.00176921 -0.00152435 -0.00126960 -0.00104958 -0.00084279 -0.00062345 -0.00050933 -0.00035779 -0.00020889 -0.00064908 -0.00098185 +0.00100000 +0.00193742 +0.00164285 +0.00126053 +0.00093207 +0.00067381 +0.00064058 +0.00042160 +0.00025886 +0.00062932 +0.00094503 +0.00099847 +0.00100000 +0.00164285 +0.00142572 +0.00112728 +0.00082622 +0.00066393 +0.00062409 +0.00037027 +0.00028614 +0.00068379 +0.00095720 +0.00099893 +0.00100000 +0.00126053 +0.00112728 +0.00090124 +0.00070678 +0.00065860 +0.00055750 +0.00028977 +0.00036849 +0.00077486 +0.00097469 +0.00099949 +0.00100000 +0.00093207 +0.00082622 +0.00070678 +0.00066474 +0.00062196 +0.00041249 +0.00025931 +0.00052285 +0.00087182 +0.00098923 +0.00099984 +0.00100000 +0.00067381 +0.00066393 +0.00065860 +0.00062196 +0.00046292 +0.00027456 +0.00037157 +0.00071562 +0.00094580 +0.00099696 0.00099997 -0.00138716 -0.00126960 -0.00108695 -0.00089576 -0.00071465 -0.00056028 -0.00047345 -0.00027595 -0.00027849 -0.00077516 -0.00099238 -0.00099999 -0.00117503 -0.00104958 -0.00089576 -0.00074137 -0.00060143 -0.00051255 -0.00038971 -0.00020417 -0.00045558 -0.00090043 -0.00099823 0.00100000 -0.00093163 -0.00084279 -0.00071465 -0.00060143 -0.00052523 -0.00043621 -0.00025334 -0.00026925 -0.00070489 -0.00097529 -0.00099982 +0.00064058 +0.00062409 +0.00055750 +0.00041249 +0.00027456 +0.00033227 +0.00060954 +0.00087860 +0.00098477 +0.00099949 +0.00100000 +0.00100000 +0.00042160 +0.00037027 +0.00028977 +0.00025931 +0.00037157 +0.00060954 +0.00084479 +0.00096806 +0.00099762 +0.00099996 +0.00100000 0.00100000 -0.00067731 -0.00062345 -0.00056028 -0.00051255 -0.00043621 -0.00027895 -0.00022384 -0.00054032 -0.00090580 -0.00099727 +0.00025886 +0.00028614 +0.00036849 +0.00052285 +0.00071562 +0.00087860 +0.00096806 +0.00099603 +0.00099984 0.00100000 0.00100000 -0.00052190 -0.00050933 -0.00047345 -0.00038971 -0.00025334 -0.00022384 -0.00048569 -0.00084465 -0.00098860 -0.00099990 0.00100000 +0.00062932 +0.00068379 +0.00077486 +0.00087182 +0.00094580 +0.00098477 +0.00099762 +0.00099984 0.00100000 -0.00040204 -0.00035779 -0.00027595 -0.00020417 -0.00026925 -0.00054032 -0.00084465 -0.00098310 -0.00099961 0.00100000 0.00100000 0.00100000 -0.00019918 -0.00020889 -0.00027849 -0.00045558 -0.00070489 -0.00090580 -0.00098860 -0.00099961 +0.00094503 +0.00095720 +0.00097469 +0.00098923 +0.00099696 +0.00099949 +0.00099996 0.00100000 0.00100000 0.00100000 0.00100000 -0.00057389 -0.00064908 -0.00077516 -0.00090043 -0.00097529 -0.00099727 -0.00099990 0.00100000 +0.00099847 +0.00099893 +0.00099949 +0.00099984 +0.00099997 0.00100000 0.00100000 0.00100000 0.00100000 -0.00097279 -0.00098185 -0.00099238 -0.00099823 -0.00099982 0.00100000 0.00100000 0.00100000 @@ -17228,9 +17232,6 @@ LOOKUP_TABLE default 0.00100000 0.00100000 0.00100000 -0.00099995 -0.00099997 -0.00099999 0.00100000 0.00100000 0.00100000 @@ -17239,131 +17240,133 @@ LOOKUP_TABLE default 0.00100000 0.00100000 0.00100000 +0.00152084 +0.00114771 +0.00076344 +0.00063045 +0.00064058 +0.00049423 +0.00025665 +0.00047580 +0.00085309 +0.00098736 +0.00099983 0.00100000 -0.00161902 -0.00127705 -0.00100029 -0.00082724 -0.00067731 -0.00054179 -0.00045586 -0.00022391 -0.00038922 -0.00087649 -0.00099804 -0.00100000 -0.00127705 -0.00109248 -0.00092067 -0.00076889 -0.00062345 -0.00052504 -0.00041535 -0.00020724 -0.00045415 -0.00090899 -0.00099879 -0.00100000 -0.00100029 -0.00092067 -0.00080860 -0.00067663 -0.00056028 -0.00048780 -0.00032957 -0.00021297 -0.00058548 -0.00095301 -0.00099957 -0.00100000 -0.00082724 -0.00076889 -0.00067663 -0.00057922 -0.00051255 -0.00041006 -0.00023196 -0.00031372 -0.00075723 -0.00098470 -0.00099992 -0.00100000 -0.00067731 -0.00062345 -0.00056028 -0.00051255 -0.00043621 -0.00027895 -0.00022384 -0.00054032 -0.00090580 -0.00099727 -0.00100000 -0.00100000 -0.00054179 -0.00052504 -0.00048780 -0.00041006 -0.00027895 -0.00021296 -0.00042652 -0.00080428 -0.00098176 -0.00099979 +0.00114771 +0.00091219 +0.00068571 +0.00063402 +0.00062409 +0.00042910 +0.00025844 +0.00052887 +0.00087890 +0.00099051 +0.00099988 +0.00100000 +0.00076344 +0.00068571 +0.00063417 +0.00064318 +0.00055750 +0.00032755 +0.00029836 +0.00063048 +0.00091935 +0.00099476 +0.00099995 0.00100000 +0.00063045 +0.00063402 +0.00064318 +0.00059305 +0.00041249 +0.00026224 +0.00041742 +0.00075936 +0.00095854 +0.00099798 +0.00099998 0.00100000 -0.00045586 -0.00041535 -0.00032957 -0.00023196 -0.00022384 -0.00042652 -0.00075701 -0.00096121 -0.00099868 +0.00064058 +0.00062409 +0.00055750 +0.00041249 +0.00027456 +0.00033227 +0.00060954 +0.00087860 +0.00098477 +0.00099949 +0.00100000 +0.00100000 +0.00049423 +0.00042910 +0.00032755 +0.00026224 +0.00033227 +0.00055429 +0.00080872 +0.00095746 +0.00099644 +0.00099993 +0.00100000 +0.00100000 +0.00025665 +0.00025844 +0.00029836 +0.00041742 +0.00060954 +0.00080872 +0.00094148 +0.00099124 +0.00099955 +0.00100000 +0.00100000 +0.00100000 +0.00047580 +0.00052887 +0.00063048 +0.00075936 +0.00087860 +0.00095746 +0.00099124 +0.00099918 +0.00099998 0.00100000 0.00100000 0.00100000 -0.00022391 -0.00020724 -0.00021297 -0.00031372 -0.00054032 -0.00080428 -0.00096121 -0.00099776 +0.00085309 +0.00087890 +0.00091935 +0.00095854 +0.00098477 +0.00099644 +0.00099955 0.00099998 0.00100000 0.00100000 0.00100000 -0.00038922 -0.00045415 -0.00058548 -0.00075723 -0.00090580 -0.00098176 -0.00099868 -0.00099998 0.00100000 +0.00098736 +0.00099051 +0.00099476 +0.00099798 +0.00099949 +0.00099993 0.00100000 0.00100000 0.00100000 -0.00087649 -0.00090899 -0.00095301 -0.00098470 -0.00099727 -0.00099979 0.00100000 0.00100000 0.00100000 +0.00099983 +0.00099988 +0.00099995 +0.00099998 0.00100000 0.00100000 0.00100000 -0.00099804 -0.00099879 -0.00099957 -0.00099992 0.00100000 0.00100000 0.00100000 @@ -17381,122 +17384,119 @@ LOOKUP_TABLE default 0.00100000 0.00100000 0.00100000 +0.00077768 +0.00067506 +0.00063672 +0.00061718 +0.00042160 +0.00025665 +0.00041037 +0.00077913 +0.00096429 +0.00099855 +0.00099999 0.00100000 +0.00067506 +0.00064203 +0.00063981 +0.00057840 +0.00037027 +0.00025844 +0.00046661 +0.00081088 +0.00097178 +0.00099895 +0.00100000 +0.00100000 +0.00063672 +0.00063981 +0.00061131 +0.00047531 +0.00028977 +0.00029836 +0.00057268 +0.00086488 +0.00098278 +0.00099946 0.00100000 0.00100000 -0.00119620 -0.00094965 -0.00074238 -0.00060761 -0.00052190 -0.00045586 -0.00028371 -0.00031403 -0.00076492 -0.00098862 -0.00099998 +0.00061718 +0.00057840 +0.00047531 +0.00032307 +0.00025931 +0.00041742 +0.00071012 +0.00092338 +0.00099226 +0.00099981 +0.00100000 +0.00100000 +0.00042160 +0.00037027 +0.00028977 +0.00025931 +0.00037157 +0.00060954 +0.00084479 +0.00096806 +0.00099762 +0.00099996 0.00100000 -0.00094965 -0.00081636 -0.00068175 -0.00057377 -0.00050933 -0.00041535 -0.00024015 -0.00035696 -0.00081217 -0.00099232 -0.00099999 0.00100000 -0.00074238 -0.00068175 -0.00059734 -0.00052783 -0.00047345 -0.00032957 -0.00021316 -0.00047204 -0.00088603 -0.00099667 -0.00100000 -0.00100000 -0.00060761 -0.00057377 -0.00052783 -0.00048620 -0.00038971 -0.00023196 -0.00028293 -0.00065650 -0.00095263 -0.00099916 +0.00025665 +0.00025844 +0.00029836 +0.00041742 +0.00060954 +0.00080872 +0.00094148 +0.00099124 +0.00099955 0.00100000 0.00100000 -0.00052190 -0.00050933 -0.00047345 -0.00038971 -0.00025334 -0.00022384 -0.00048569 -0.00084465 -0.00098860 -0.00099990 -0.00100000 -0.00100000 -0.00045586 -0.00041535 -0.00032957 -0.00023196 -0.00022384 -0.00042652 -0.00075701 -0.00096121 -0.00099868 -0.00100000 -0.00100000 -0.00100000 -0.00028371 -0.00024015 -0.00021316 -0.00028293 -0.00048569 -0.00075701 -0.00094457 -0.00099612 -0.00099995 0.00100000 +0.00041037 +0.00046661 +0.00057268 +0.00071012 +0.00084479 +0.00094148 +0.00098692 +0.00099865 +0.00099996 0.00100000 0.00100000 -0.00031403 -0.00035696 -0.00047204 -0.00065650 -0.00084465 -0.00096121 -0.00099612 -0.00099989 0.00100000 +0.00077913 +0.00081088 +0.00086488 +0.00092338 +0.00096806 +0.00099124 +0.00099865 +0.00099991 0.00100000 0.00100000 0.00100000 -0.00076492 -0.00081217 -0.00088603 -0.00095263 -0.00098860 -0.00099868 -0.00099995 0.00100000 +0.00096429 +0.00097178 +0.00098278 +0.00099226 +0.00099762 +0.00099955 +0.00099996 0.00100000 0.00100000 0.00100000 0.00100000 -0.00098862 -0.00099232 -0.00099667 -0.00099916 -0.00099990 +0.00100000 +0.00099855 +0.00099895 +0.00099946 +0.00099981 +0.00099996 0.00100000 0.00100000 0.00100000 @@ -17504,7 +17504,6 @@ LOOKUP_TABLE default 0.00100000 0.00100000 0.00100000 -0.00099998 0.00099999 0.00100000 0.00100000 @@ -17528,107 +17527,108 @@ LOOKUP_TABLE default 0.00100000 0.00100000 0.00100000 -0.00090325 -0.00070525 -0.00055259 -0.00050178 -0.00040204 -0.00022391 -0.00031403 -0.00064627 -0.00097354 -0.00099979 0.00100000 +0.00069715 +0.00067688 +0.00052419 +0.00031632 +0.00025886 +0.00047580 +0.00077913 +0.00094601 +0.00099609 +0.00099994 0.00100000 -0.00070525 -0.00060402 -0.00052346 -0.00047919 -0.00035779 -0.00020724 -0.00035696 -0.00072474 -0.00098105 -0.00099987 0.00100000 +0.00067688 +0.00060704 +0.00044197 +0.00027313 +0.00028614 +0.00052887 +0.00081088 +0.00095721 +0.00099706 +0.00099995 0.00100000 -0.00055259 -0.00052346 -0.00049107 -0.00042368 -0.00027595 -0.00021297 -0.00047204 -0.00083888 -0.00099080 -0.00099996 +0.00100000 +0.00052419 +0.00044197 +0.00031118 +0.00024629 +0.00036849 +0.00063048 +0.00086488 +0.00097360 +0.00099837 +0.00099998 0.00100000 0.00100000 -0.00050178 -0.00047919 -0.00042368 -0.00030882 -0.00020417 -0.00031372 -0.00065650 -0.00093366 -0.00099732 +0.00031632 +0.00027313 +0.00024629 +0.00032278 +0.00052285 +0.00075936 +0.00092338 +0.00098779 +0.00099936 0.00099999 0.00100000 0.00100000 -0.00040204 -0.00035779 -0.00027595 -0.00020417 -0.00026925 -0.00054032 -0.00084465 -0.00098310 -0.00099961 +0.00025886 +0.00028614 +0.00036849 +0.00052285 +0.00071562 +0.00087860 +0.00096806 +0.00099603 +0.00099984 0.00100000 0.00100000 0.00100000 -0.00022391 -0.00020724 -0.00021297 -0.00031372 -0.00054032 -0.00080428 -0.00096121 -0.00099776 +0.00047580 +0.00052887 +0.00063048 +0.00075936 +0.00087860 +0.00095746 +0.00099124 +0.00099918 0.00099998 0.00100000 0.00100000 0.00100000 -0.00031403 -0.00035696 -0.00047204 -0.00065650 -0.00084465 -0.00096121 -0.00099612 -0.00099989 +0.00077913 +0.00081088 +0.00086488 +0.00092338 +0.00096806 +0.00099124 +0.00099865 +0.00099991 0.00100000 0.00100000 0.00100000 0.00100000 -0.00064627 -0.00072474 -0.00083888 -0.00093366 -0.00098310 -0.00099776 -0.00099989 +0.00094601 +0.00095721 +0.00097360 +0.00098779 +0.00099603 +0.00099918 +0.00099991 0.00100000 0.00100000 0.00100000 0.00100000 0.00100000 -0.00097354 -0.00098105 -0.00099080 -0.00099732 -0.00099961 +0.00099609 +0.00099706 +0.00099837 +0.00099936 +0.00099984 0.00099998 0.00100000 0.00100000 @@ -17636,9 +17636,9 @@ LOOKUP_TABLE default 0.00100000 0.00100000 0.00100000 -0.00099979 -0.00099987 -0.00099996 +0.00099994 +0.00099995 +0.00099998 0.00099999 0.00100000 0.00100000 @@ -17672,95 +17672,95 @@ LOOKUP_TABLE default 0.00100000 0.00100000 0.00100000 -0.00064695 -0.00055922 -0.00046787 -0.00032261 -0.00019918 -0.00038922 -0.00076492 -0.00097354 -0.00099906 +0.00049722 +0.00033230 +0.00024333 +0.00037576 +0.00062932 +0.00085309 +0.00096429 +0.00099609 +0.00099985 0.00100000 0.00100000 0.00100000 -0.00055922 -0.00050913 -0.00042167 -0.00027107 -0.00020889 -0.00045415 -0.00081217 -0.00098105 -0.00099946 +0.00033230 +0.00025966 +0.00027099 +0.00044006 +0.00068379 +0.00087890 +0.00097178 +0.00099706 +0.00099989 0.00100000 0.00100000 0.00100000 -0.00046787 -0.00042167 -0.00031544 -0.00020597 -0.00027849 -0.00058548 -0.00088603 -0.00099080 -0.00099983 +0.00024333 +0.00027099 +0.00037680 +0.00056775 +0.00077486 +0.00091935 +0.00098278 +0.00099837 +0.00099995 0.00100000 0.00100000 0.00100000 -0.00032261 -0.00027107 -0.00020597 -0.00023638 -0.00045558 -0.00075723 -0.00095263 -0.00099732 +0.00037576 +0.00044006 +0.00056775 +0.00072946 +0.00087182 +0.00095854 +0.00099226 +0.00099936 0.00099998 0.00100000 0.00100000 0.00100000 -0.00019918 -0.00020889 -0.00027849 -0.00045558 -0.00070489 -0.00090580 -0.00098860 -0.00099961 +0.00062932 +0.00068379 +0.00077486 +0.00087182 +0.00094580 +0.00098477 +0.00099762 +0.00099984 0.00100000 0.00100000 0.00100000 0.00100000 -0.00038922 -0.00045415 -0.00058548 -0.00075723 -0.00090580 -0.00098176 -0.00099868 +0.00085309 +0.00087890 +0.00091935 +0.00095854 +0.00098477 +0.00099644 +0.00099955 0.00099998 0.00100000 0.00100000 0.00100000 0.00100000 -0.00076492 -0.00081217 -0.00088603 -0.00095263 -0.00098860 -0.00099868 -0.00099995 +0.00096429 +0.00097178 +0.00098278 +0.00099226 +0.00099762 +0.00099955 +0.00099996 0.00100000 0.00100000 0.00100000 0.00100000 0.00100000 -0.00097354 -0.00098105 -0.00099080 -0.00099732 -0.00099961 +0.00099609 +0.00099706 +0.00099837 +0.00099936 +0.00099984 0.00099998 0.00100000 0.00100000 @@ -17768,9 +17768,9 @@ LOOKUP_TABLE default 0.00100000 0.00100000 0.00100000 -0.00099906 -0.00099946 -0.00099983 +0.00099985 +0.00099989 +0.00099995 0.00099998 0.00100000 0.00100000 @@ -17816,83 +17816,83 @@ LOOKUP_TABLE default 0.00100000 0.00100000 0.00100000 -0.00060647 -0.00039095 -0.00020297 -0.00027038 -0.00057389 -0.00087649 -0.00098862 -0.00099979 +0.00038333 +0.00050577 +0.00068731 +0.00084760 +0.00094503 +0.00098736 +0.00099855 +0.00099994 0.00100000 0.00100000 0.00100000 0.00100000 -0.00039095 -0.00026331 -0.00019166 -0.00033779 -0.00064908 -0.00090899 -0.00099232 -0.00099987 +0.00050577 +0.00060533 +0.00075086 +0.00087903 +0.00095720 +0.00099051 +0.00099895 +0.00099995 0.00100000 0.00100000 0.00100000 0.00100000 -0.00020297 -0.00019166 -0.00026719 -0.00049468 -0.00077516 -0.00095301 -0.00099667 -0.00099996 +0.00068731 +0.00075086 +0.00084340 +0.00092522 +0.00097469 +0.00099476 +0.00099946 +0.00099998 0.00100000 0.00100000 0.00100000 0.00100000 -0.00027038 -0.00033779 -0.00049468 -0.00071445 -0.00090043 -0.00098470 -0.00099916 +0.00084760 +0.00087903 +0.00092522 +0.00096568 +0.00098923 +0.00099798 +0.00099981 0.00099999 0.00100000 0.00100000 0.00100000 0.00100000 -0.00057389 -0.00064908 -0.00077516 -0.00090043 -0.00097529 -0.00099727 -0.00099990 +0.00094503 +0.00095720 +0.00097469 +0.00098923 +0.00099696 +0.00099949 +0.00099996 0.00100000 0.00100000 0.00100000 0.00100000 0.00100000 -0.00087649 -0.00090899 -0.00095301 -0.00098470 -0.00099727 -0.00099979 +0.00098736 +0.00099051 +0.00099476 +0.00099798 +0.00099949 +0.00099993 0.00100000 0.00100000 0.00100000 0.00100000 0.00100000 0.00100000 -0.00098862 -0.00099232 -0.00099667 -0.00099916 -0.00099990 +0.00099855 +0.00099895 +0.00099946 +0.00099981 +0.00099996 0.00100000 0.00100000 0.00100000 @@ -17900,9 +17900,9 @@ LOOKUP_TABLE default 0.00100000 0.00100000 0.00100000 -0.00099979 -0.00099987 -0.00099996 +0.00099994 +0.00099995 +0.00099998 0.00099999 0.00100000 0.00100000 @@ -17960,59 +17960,59 @@ LOOKUP_TABLE default 0.00100000 0.00100000 0.00100000 -0.00014818 -0.00029893 -0.00058477 -0.00084695 -0.00097279 -0.00099804 -0.00099998 +0.00093526 +0.00095295 +0.00097617 +0.00099226 +0.00099847 +0.00099983 +0.00099999 0.00100000 0.00100000 0.00100000 0.00100000 0.00100000 -0.00029893 -0.00044549 -0.00068687 -0.00089130 -0.00098185 -0.00099879 -0.00099999 +0.00095295 +0.00096602 +0.00098298 +0.00099452 +0.00099893 +0.00099988 0.00100000 0.00100000 0.00100000 0.00100000 0.00100000 -0.00058477 -0.00068687 -0.00083537 -0.00094878 -0.00099238 -0.00099957 +0.00100000 +0.00097617 +0.00098298 +0.00099162 +0.00099734 +0.00099949 +0.00099995 0.00100000 0.00100000 0.00100000 0.00100000 0.00100000 0.00100000 -0.00084695 -0.00089130 -0.00094878 -0.00098630 -0.00099823 -0.00099992 +0.00099226 +0.00099452 +0.00099734 +0.00099916 +0.00099984 +0.00099998 0.00100000 0.00100000 0.00100000 0.00100000 0.00100000 0.00100000 -0.00097279 -0.00098185 -0.00099238 -0.00099823 -0.00099982 +0.00099847 +0.00099893 +0.00099949 +0.00099984 +0.00099997 0.00100000 0.00100000 0.00100000 @@ -18020,10 +18020,10 @@ LOOKUP_TABLE default 0.00100000 0.00100000 0.00100000 -0.00099804 -0.00099879 -0.00099957 -0.00099992 +0.00099983 +0.00099988 +0.00099995 +0.00099998 0.00100000 0.00100000 0.00100000 @@ -18032,7 +18032,6 @@ LOOKUP_TABLE default 0.00100000 0.00100000 0.00100000 -0.00099998 0.00099999 0.00100000 0.00100000 @@ -18104,11 +18103,11 @@ LOOKUP_TABLE default 0.00100000 0.00100000 0.00100000 -0.00089030 -0.00093620 -0.00098330 -0.00099839 -0.00099995 +0.00100000 +0.00099951 +0.00099968 +0.00099987 +0.00099997 0.00100000 0.00100000 0.00100000 @@ -18116,11 +18115,12 @@ LOOKUP_TABLE default 0.00100000 0.00100000 0.00100000 -0.00093620 -0.00096330 -0.00099046 -0.00099909 -0.00099997 +0.00100000 +0.00099968 +0.00099979 +0.00099991 +0.00099998 +0.00100000 0.00100000 0.00100000 0.00100000 @@ -18128,10 +18128,9 @@ LOOKUP_TABLE default 0.00100000 0.00100000 0.00100000 -0.00098330 -0.00099046 -0.00099741 -0.00099975 +0.00099987 +0.00099991 +0.00099997 0.00099999 0.00100000 0.00100000 @@ -18140,10 +18139,14 @@ LOOKUP_TABLE default 0.00100000 0.00100000 0.00100000 -0.00099839 -0.00099909 -0.00099975 +0.00100000 0.00099997 +0.00099998 +0.00099999 +0.00100000 +0.00100000 +0.00100000 +0.00100000 0.00100000 0.00100000 0.00100000 @@ -18152,9 +18155,6 @@ LOOKUP_TABLE default 0.00100000 0.00100000 0.00100000 -0.00099995 -0.00099997 -0.00099999 0.00100000 0.00100000 0.00100000 diff --git a/python/Test-scripts/standard-results/SGH/Sedov_12x12x12/vtk/outputs.vtk.series b/integrated-tests/Test-scripts/standard-results/SGH/Sedov/vtk/outputs.vtk.series similarity index 100% rename from python/Test-scripts/standard-results/SGH/Sedov_12x12x12/vtk/outputs.vtk.series rename to integrated-tests/Test-scripts/standard-results/SGH/Sedov/vtk/outputs.vtk.series diff --git a/integrated-tests/Test-scripts/standard-results/SGH/Sod/vtk/data/VTK0.vtk b/integrated-tests/Test-scripts/standard-results/SGH/Sod/vtk/data/VTK0.vtk new file mode 100644 index 000000000..4d423a963 --- /dev/null +++ b/integrated-tests/Test-scripts/standard-results/SGH/Sod/vtk/data/VTK0.vtk @@ -0,0 +1,1642 @@ +# vtk DataFile Version 2.0 +Mesh for Fierro +ASCII +DATASET UNSTRUCTURED_GRID + +POINTS 404 float +0.00000000 0.00000000 0.00000000 +0.01000000 0.00000000 0.00000000 +0.02000000 0.00000000 0.00000000 +0.03000000 0.00000000 0.00000000 +0.04000000 0.00000000 0.00000000 +0.05000000 0.00000000 0.00000000 +0.06000000 0.00000000 0.00000000 +0.07000000 0.00000000 0.00000000 +0.08000000 0.00000000 0.00000000 +0.09000000 0.00000000 0.00000000 +0.10000000 0.00000000 0.00000000 +0.11000000 0.00000000 0.00000000 +0.12000000 0.00000000 0.00000000 +0.13000000 0.00000000 0.00000000 +0.14000000 0.00000000 0.00000000 +0.15000000 0.00000000 0.00000000 +0.16000000 0.00000000 0.00000000 +0.17000000 0.00000000 0.00000000 +0.18000000 0.00000000 0.00000000 +0.19000000 0.00000000 0.00000000 +0.20000000 0.00000000 0.00000000 +0.21000000 0.00000000 0.00000000 +0.22000000 0.00000000 0.00000000 +0.23000000 0.00000000 0.00000000 +0.24000001 0.00000000 0.00000000 +0.25000005 0.00000000 0.00000000 +0.26000028 0.00000000 0.00000000 +0.27000148 0.00000000 0.00000000 +0.28000694 0.00000000 0.00000000 +0.29002817 0.00000000 0.00000000 +0.30009731 0.00000000 0.00000000 +0.31028121 0.00000000 0.00000000 +0.32067809 0.00000000 0.00000000 +0.33138981 0.00000000 0.00000000 +0.34250219 0.00000000 0.00000000 +0.35409115 0.00000000 0.00000000 +0.36623592 0.00000000 0.00000000 +0.37902551 0.00000000 0.00000000 +0.39255812 0.00000000 0.00000000 +0.40693863 0.00000000 0.00000000 +0.42228583 0.00000000 0.00000000 +0.43873152 0.00000000 0.00000000 +0.45641823 0.00000000 0.00000000 +0.47548280 0.00000000 0.00000000 +0.49602306 0.00000000 0.00000000 +0.51804559 0.00000000 0.00000000 +0.54136068 0.00000000 0.00000000 +0.56533343 0.00000000 0.00000000 +0.58900785 0.00000000 0.00000000 +0.61288286 0.00000000 0.00000000 +0.63768836 0.00000000 0.00000000 +0.64235979 0.00000000 0.00000000 +0.64703536 0.00000000 0.00000000 +0.65174451 0.00000000 0.00000000 +0.65645829 0.00000000 0.00000000 +0.66119417 0.00000000 0.00000000 +0.66592157 0.00000000 0.00000000 +0.67065538 0.00000000 0.00000000 +0.67538547 0.00000000 0.00000000 +0.68010355 0.00000000 0.00000000 +0.68483436 0.00000000 0.00000000 +0.68953471 0.00000000 0.00000000 +0.69426825 0.00000000 0.00000000 +0.69895455 0.00000000 0.00000000 +0.70369641 0.00000000 0.00000000 +0.70837203 0.00000000 0.00000000 +0.71313263 0.00000000 0.00000000 +0.71779813 0.00000000 0.00000000 +0.72258668 0.00000000 0.00000000 +0.72723297 0.00000000 0.00000000 +0.73205718 0.00000000 0.00000000 +0.73666551 0.00000000 0.00000000 +0.74153354 0.00000000 0.00000000 +0.74606693 0.00000000 0.00000000 +0.75102093 0.00000000 0.00000000 +0.75538097 0.00000000 0.00000000 +0.76107536 0.00000000 0.00000000 +0.77007680 0.00000000 0.00000000 +0.78000247 0.00000000 0.00000000 +0.79000006 0.00000000 0.00000000 +0.80000000 0.00000000 0.00000000 +0.81000000 0.00000000 0.00000000 +0.82000000 0.00000000 0.00000000 +0.83000000 0.00000000 0.00000000 +0.84000000 0.00000000 0.00000000 +0.85000000 0.00000000 0.00000000 +0.86000000 0.00000000 0.00000000 +0.87000000 0.00000000 0.00000000 +0.88000000 0.00000000 0.00000000 +0.89000000 0.00000000 0.00000000 +0.90000000 0.00000000 0.00000000 +0.91000000 0.00000000 0.00000000 +0.92000000 0.00000000 0.00000000 +0.93000000 0.00000000 0.00000000 +0.94000000 0.00000000 0.00000000 +0.95000000 0.00000000 0.00000000 +0.96000000 0.00000000 0.00000000 +0.97000000 0.00000000 0.00000000 +0.98000000 0.00000000 0.00000000 +0.99000000 0.00000000 0.00000000 +1.00000000 0.00000000 0.00000000 +0.00000000 0.01000000 0.00000000 +0.01000000 0.01000000 0.00000000 +0.02000000 0.01000000 0.00000000 +0.03000000 0.01000000 0.00000000 +0.04000000 0.01000000 0.00000000 +0.05000000 0.01000000 0.00000000 +0.06000000 0.01000000 0.00000000 +0.07000000 0.01000000 0.00000000 +0.08000000 0.01000000 0.00000000 +0.09000000 0.01000000 0.00000000 +0.10000000 0.01000000 0.00000000 +0.11000000 0.01000000 0.00000000 +0.12000000 0.01000000 0.00000000 +0.13000000 0.01000000 0.00000000 +0.14000000 0.01000000 0.00000000 +0.15000000 0.01000000 0.00000000 +0.16000000 0.01000000 0.00000000 +0.17000000 0.01000000 0.00000000 +0.18000000 0.01000000 0.00000000 +0.19000000 0.01000000 0.00000000 +0.20000000 0.01000000 0.00000000 +0.21000000 0.01000000 0.00000000 +0.22000000 0.01000000 0.00000000 +0.23000000 0.01000000 0.00000000 +0.24000001 0.01000000 0.00000000 +0.25000005 0.01000000 0.00000000 +0.26000028 0.01000000 0.00000000 +0.27000148 0.01000000 0.00000000 +0.28000694 0.01000000 0.00000000 +0.29002817 0.01000000 0.00000000 +0.30009731 0.01000000 0.00000000 +0.31028121 0.01000000 0.00000000 +0.32067809 0.01000000 0.00000000 +0.33138981 0.01000000 0.00000000 +0.34250219 0.01000000 0.00000000 +0.35409115 0.01000000 0.00000000 +0.36623592 0.01000000 0.00000000 +0.37902551 0.01000000 0.00000000 +0.39255812 0.01000000 0.00000000 +0.40693863 0.01000000 0.00000000 +0.42228583 0.01000000 0.00000000 +0.43873152 0.01000000 0.00000000 +0.45641823 0.01000000 0.00000000 +0.47548280 0.01000000 0.00000000 +0.49602306 0.01000000 0.00000000 +0.51804559 0.01000000 0.00000000 +0.54136068 0.01000000 0.00000000 +0.56533343 0.01000000 0.00000000 +0.58900785 0.01000000 0.00000000 +0.61288286 0.01000000 0.00000000 +0.63768836 0.01000000 0.00000000 +0.64235979 0.01000000 0.00000000 +0.64703536 0.01000000 0.00000000 +0.65174451 0.01000000 0.00000000 +0.65645829 0.01000000 0.00000000 +0.66119417 0.01000000 0.00000000 +0.66592157 0.01000000 0.00000000 +0.67065538 0.01000000 0.00000000 +0.67538547 0.01000000 0.00000000 +0.68010355 0.01000000 0.00000000 +0.68483436 0.01000000 0.00000000 +0.68953471 0.01000000 0.00000000 +0.69426825 0.01000000 0.00000000 +0.69895455 0.01000000 0.00000000 +0.70369641 0.01000000 0.00000000 +0.70837203 0.01000000 0.00000000 +0.71313263 0.01000000 0.00000000 +0.71779813 0.01000000 0.00000000 +0.72258668 0.01000000 0.00000000 +0.72723297 0.01000000 0.00000000 +0.73205718 0.01000000 0.00000000 +0.73666551 0.01000000 0.00000000 +0.74153354 0.01000000 0.00000000 +0.74606693 0.01000000 0.00000000 +0.75102093 0.01000000 0.00000000 +0.75538097 0.01000000 0.00000000 +0.76107536 0.01000000 0.00000000 +0.77007680 0.01000000 0.00000000 +0.78000247 0.01000000 0.00000000 +0.79000006 0.01000000 0.00000000 +0.80000000 0.01000000 0.00000000 +0.81000000 0.01000000 0.00000000 +0.82000000 0.01000000 0.00000000 +0.83000000 0.01000000 0.00000000 +0.84000000 0.01000000 0.00000000 +0.85000000 0.01000000 0.00000000 +0.86000000 0.01000000 0.00000000 +0.87000000 0.01000000 0.00000000 +0.88000000 0.01000000 0.00000000 +0.89000000 0.01000000 0.00000000 +0.90000000 0.01000000 0.00000000 +0.91000000 0.01000000 0.00000000 +0.92000000 0.01000000 0.00000000 +0.93000000 0.01000000 0.00000000 +0.94000000 0.01000000 0.00000000 +0.95000000 0.01000000 0.00000000 +0.96000000 0.01000000 0.00000000 +0.97000000 0.01000000 0.00000000 +0.98000000 0.01000000 0.00000000 +0.99000000 0.01000000 0.00000000 +1.00000000 0.01000000 0.00000000 +0.00000000 0.00000000 0.01000000 +0.01000000 0.00000000 0.01000000 +0.02000000 0.00000000 0.01000000 +0.03000000 0.00000000 0.01000000 +0.04000000 0.00000000 0.01000000 +0.05000000 0.00000000 0.01000000 +0.06000000 0.00000000 0.01000000 +0.07000000 0.00000000 0.01000000 +0.08000000 0.00000000 0.01000000 +0.09000000 0.00000000 0.01000000 +0.10000000 0.00000000 0.01000000 +0.11000000 0.00000000 0.01000000 +0.12000000 0.00000000 0.01000000 +0.13000000 0.00000000 0.01000000 +0.14000000 0.00000000 0.01000000 +0.15000000 0.00000000 0.01000000 +0.16000000 0.00000000 0.01000000 +0.17000000 0.00000000 0.01000000 +0.18000000 0.00000000 0.01000000 +0.19000000 0.00000000 0.01000000 +0.20000000 0.00000000 0.01000000 +0.21000000 0.00000000 0.01000000 +0.22000000 0.00000000 0.01000000 +0.23000000 0.00000000 0.01000000 +0.24000001 0.00000000 0.01000000 +0.25000005 0.00000000 0.01000000 +0.26000028 0.00000000 0.01000000 +0.27000148 0.00000000 0.01000000 +0.28000694 0.00000000 0.01000000 +0.29002817 0.00000000 0.01000000 +0.30009731 0.00000000 0.01000000 +0.31028121 0.00000000 0.01000000 +0.32067809 0.00000000 0.01000000 +0.33138981 0.00000000 0.01000000 +0.34250219 0.00000000 0.01000000 +0.35409115 0.00000000 0.01000000 +0.36623592 0.00000000 0.01000000 +0.37902551 0.00000000 0.01000000 +0.39255812 0.00000000 0.01000000 +0.40693863 0.00000000 0.01000000 +0.42228583 0.00000000 0.01000000 +0.43873152 0.00000000 0.01000000 +0.45641823 0.00000000 0.01000000 +0.47548280 0.00000000 0.01000000 +0.49602306 0.00000000 0.01000000 +0.51804559 0.00000000 0.01000000 +0.54136068 0.00000000 0.01000000 +0.56533343 0.00000000 0.01000000 +0.58900785 0.00000000 0.01000000 +0.61288286 0.00000000 0.01000000 +0.63768836 0.00000000 0.01000000 +0.64235979 0.00000000 0.01000000 +0.64703536 0.00000000 0.01000000 +0.65174451 0.00000000 0.01000000 +0.65645829 0.00000000 0.01000000 +0.66119417 0.00000000 0.01000000 +0.66592157 0.00000000 0.01000000 +0.67065538 0.00000000 0.01000000 +0.67538547 0.00000000 0.01000000 +0.68010355 0.00000000 0.01000000 +0.68483436 0.00000000 0.01000000 +0.68953471 0.00000000 0.01000000 +0.69426825 0.00000000 0.01000000 +0.69895455 0.00000000 0.01000000 +0.70369641 0.00000000 0.01000000 +0.70837203 0.00000000 0.01000000 +0.71313263 0.00000000 0.01000000 +0.71779813 0.00000000 0.01000000 +0.72258668 0.00000000 0.01000000 +0.72723297 0.00000000 0.01000000 +0.73205718 0.00000000 0.01000000 +0.73666551 0.00000000 0.01000000 +0.74153354 0.00000000 0.01000000 +0.74606693 0.00000000 0.01000000 +0.75102093 0.00000000 0.01000000 +0.75538097 0.00000000 0.01000000 +0.76107536 0.00000000 0.01000000 +0.77007680 0.00000000 0.01000000 +0.78000247 0.00000000 0.01000000 +0.79000006 0.00000000 0.01000000 +0.80000000 0.00000000 0.01000000 +0.81000000 0.00000000 0.01000000 +0.82000000 0.00000000 0.01000000 +0.83000000 0.00000000 0.01000000 +0.84000000 0.00000000 0.01000000 +0.85000000 0.00000000 0.01000000 +0.86000000 0.00000000 0.01000000 +0.87000000 0.00000000 0.01000000 +0.88000000 0.00000000 0.01000000 +0.89000000 0.00000000 0.01000000 +0.90000000 0.00000000 0.01000000 +0.91000000 0.00000000 0.01000000 +0.92000000 0.00000000 0.01000000 +0.93000000 0.00000000 0.01000000 +0.94000000 0.00000000 0.01000000 +0.95000000 0.00000000 0.01000000 +0.96000000 0.00000000 0.01000000 +0.97000000 0.00000000 0.01000000 +0.98000000 0.00000000 0.01000000 +0.99000000 0.00000000 0.01000000 +1.00000000 0.00000000 0.01000000 +0.00000000 0.01000000 0.01000000 +0.01000000 0.01000000 0.01000000 +0.02000000 0.01000000 0.01000000 +0.03000000 0.01000000 0.01000000 +0.04000000 0.01000000 0.01000000 +0.05000000 0.01000000 0.01000000 +0.06000000 0.01000000 0.01000000 +0.07000000 0.01000000 0.01000000 +0.08000000 0.01000000 0.01000000 +0.09000000 0.01000000 0.01000000 +0.10000000 0.01000000 0.01000000 +0.11000000 0.01000000 0.01000000 +0.12000000 0.01000000 0.01000000 +0.13000000 0.01000000 0.01000000 +0.14000000 0.01000000 0.01000000 +0.15000000 0.01000000 0.01000000 +0.16000000 0.01000000 0.01000000 +0.17000000 0.01000000 0.01000000 +0.18000000 0.01000000 0.01000000 +0.19000000 0.01000000 0.01000000 +0.20000000 0.01000000 0.01000000 +0.21000000 0.01000000 0.01000000 +0.22000000 0.01000000 0.01000000 +0.23000000 0.01000000 0.01000000 +0.24000001 0.01000000 0.01000000 +0.25000005 0.01000000 0.01000000 +0.26000028 0.01000000 0.01000000 +0.27000148 0.01000000 0.01000000 +0.28000694 0.01000000 0.01000000 +0.29002817 0.01000000 0.01000000 +0.30009731 0.01000000 0.01000000 +0.31028121 0.01000000 0.01000000 +0.32067809 0.01000000 0.01000000 +0.33138981 0.01000000 0.01000000 +0.34250219 0.01000000 0.01000000 +0.35409115 0.01000000 0.01000000 +0.36623592 0.01000000 0.01000000 +0.37902551 0.01000000 0.01000000 +0.39255812 0.01000000 0.01000000 +0.40693863 0.01000000 0.01000000 +0.42228583 0.01000000 0.01000000 +0.43873152 0.01000000 0.01000000 +0.45641823 0.01000000 0.01000000 +0.47548280 0.01000000 0.01000000 +0.49602306 0.01000000 0.01000000 +0.51804559 0.01000000 0.01000000 +0.54136068 0.01000000 0.01000000 +0.56533343 0.01000000 0.01000000 +0.58900785 0.01000000 0.01000000 +0.61288286 0.01000000 0.01000000 +0.63768836 0.01000000 0.01000000 +0.64235979 0.01000000 0.01000000 +0.64703536 0.01000000 0.01000000 +0.65174451 0.01000000 0.01000000 +0.65645829 0.01000000 0.01000000 +0.66119417 0.01000000 0.01000000 +0.66592157 0.01000000 0.01000000 +0.67065538 0.01000000 0.01000000 +0.67538547 0.01000000 0.01000000 +0.68010355 0.01000000 0.01000000 +0.68483436 0.01000000 0.01000000 +0.68953471 0.01000000 0.01000000 +0.69426825 0.01000000 0.01000000 +0.69895455 0.01000000 0.01000000 +0.70369641 0.01000000 0.01000000 +0.70837203 0.01000000 0.01000000 +0.71313263 0.01000000 0.01000000 +0.71779813 0.01000000 0.01000000 +0.72258668 0.01000000 0.01000000 +0.72723297 0.01000000 0.01000000 +0.73205718 0.01000000 0.01000000 +0.73666551 0.01000000 0.01000000 +0.74153354 0.01000000 0.01000000 +0.74606693 0.01000000 0.01000000 +0.75102093 0.01000000 0.01000000 +0.75538097 0.01000000 0.01000000 +0.76107536 0.01000000 0.01000000 +0.77007680 0.01000000 0.01000000 +0.78000247 0.01000000 0.01000000 +0.79000006 0.01000000 0.01000000 +0.80000000 0.01000000 0.01000000 +0.81000000 0.01000000 0.01000000 +0.82000000 0.01000000 0.01000000 +0.83000000 0.01000000 0.01000000 +0.84000000 0.01000000 0.01000000 +0.85000000 0.01000000 0.01000000 +0.86000000 0.01000000 0.01000000 +0.87000000 0.01000000 0.01000000 +0.88000000 0.01000000 0.01000000 +0.89000000 0.01000000 0.01000000 +0.90000000 0.01000000 0.01000000 +0.91000000 0.01000000 0.01000000 +0.92000000 0.01000000 0.01000000 +0.93000000 0.01000000 0.01000000 +0.94000000 0.01000000 0.01000000 +0.95000000 0.01000000 0.01000000 +0.96000000 0.01000000 0.01000000 +0.97000000 0.01000000 0.01000000 +0.98000000 0.01000000 0.01000000 +0.99000000 0.01000000 0.01000000 +1.00000000 0.01000000 0.01000000 + +CELLS 100 900 +8 0 1 102 101 202 203 304 303 +8 1 2 103 102 203 204 305 304 +8 2 3 104 103 204 205 306 305 +8 3 4 105 104 205 206 307 306 +8 4 5 106 105 206 207 308 307 +8 5 6 107 106 207 208 309 308 +8 6 7 108 107 208 209 310 309 +8 7 8 109 108 209 210 311 310 +8 8 9 110 109 210 211 312 311 +8 9 10 111 110 211 212 313 312 +8 10 11 112 111 212 213 314 313 +8 11 12 113 112 213 214 315 314 +8 12 13 114 113 214 215 316 315 +8 13 14 115 114 215 216 317 316 +8 14 15 116 115 216 217 318 317 +8 15 16 117 116 217 218 319 318 +8 16 17 118 117 218 219 320 319 +8 17 18 119 118 219 220 321 320 +8 18 19 120 119 220 221 322 321 +8 19 20 121 120 221 222 323 322 +8 20 21 122 121 222 223 324 323 +8 21 22 123 122 223 224 325 324 +8 22 23 124 123 224 225 326 325 +8 23 24 125 124 225 226 327 326 +8 24 25 126 125 226 227 328 327 +8 25 26 127 126 227 228 329 328 +8 26 27 128 127 228 229 330 329 +8 27 28 129 128 229 230 331 330 +8 28 29 130 129 230 231 332 331 +8 29 30 131 130 231 232 333 332 +8 30 31 132 131 232 233 334 333 +8 31 32 133 132 233 234 335 334 +8 32 33 134 133 234 235 336 335 +8 33 34 135 134 235 236 337 336 +8 34 35 136 135 236 237 338 337 +8 35 36 137 136 237 238 339 338 +8 36 37 138 137 238 239 340 339 +8 37 38 139 138 239 240 341 340 +8 38 39 140 139 240 241 342 341 +8 39 40 141 140 241 242 343 342 +8 40 41 142 141 242 243 344 343 +8 41 42 143 142 243 244 345 344 +8 42 43 144 143 244 245 346 345 +8 43 44 145 144 245 246 347 346 +8 44 45 146 145 246 247 348 347 +8 45 46 147 146 247 248 349 348 +8 46 47 148 147 248 249 350 349 +8 47 48 149 148 249 250 351 350 +8 48 49 150 149 250 251 352 351 +8 49 50 151 150 251 252 353 352 +8 50 51 152 151 252 253 354 353 +8 51 52 153 152 253 254 355 354 +8 52 53 154 153 254 255 356 355 +8 53 54 155 154 255 256 357 356 +8 54 55 156 155 256 257 358 357 +8 55 56 157 156 257 258 359 358 +8 56 57 158 157 258 259 360 359 +8 57 58 159 158 259 260 361 360 +8 58 59 160 159 260 261 362 361 +8 59 60 161 160 261 262 363 362 +8 60 61 162 161 262 263 364 363 +8 61 62 163 162 263 264 365 364 +8 62 63 164 163 264 265 366 365 +8 63 64 165 164 265 266 367 366 +8 64 65 166 165 266 267 368 367 +8 65 66 167 166 267 268 369 368 +8 66 67 168 167 268 269 370 369 +8 67 68 169 168 269 270 371 370 +8 68 69 170 169 270 271 372 371 +8 69 70 171 170 271 272 373 372 +8 70 71 172 171 272 273 374 373 +8 71 72 173 172 273 274 375 374 +8 72 73 174 173 274 275 376 375 +8 73 74 175 174 275 276 377 376 +8 74 75 176 175 276 277 378 377 +8 75 76 177 176 277 278 379 378 +8 76 77 178 177 278 279 380 379 +8 77 78 179 178 279 280 381 380 +8 78 79 180 179 280 281 382 381 +8 79 80 181 180 281 282 383 382 +8 80 81 182 181 282 283 384 383 +8 81 82 183 182 283 284 385 384 +8 82 83 184 183 284 285 386 385 +8 83 84 185 184 285 286 387 386 +8 84 85 186 185 286 287 388 387 +8 85 86 187 186 287 288 389 388 +8 86 87 188 187 288 289 390 389 +8 87 88 189 188 289 290 391 390 +8 88 89 190 189 290 291 392 391 +8 89 90 191 190 291 292 393 392 +8 90 91 192 191 292 293 394 393 +8 91 92 193 192 293 294 395 394 +8 92 93 194 193 294 295 396 395 +8 93 94 195 194 295 296 397 396 +8 94 95 196 195 296 297 398 397 +8 95 96 197 196 297 298 399 398 +8 96 97 198 197 298 299 400 399 +8 97 98 199 198 299 300 401 400 +8 98 99 200 199 300 301 402 401 +8 99 100 201 200 301 302 403 402 + +CELL_TYPES 100 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 +12 + +POINT_DATA 404 +VECTORS velocity float +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000003 0.00000000 0.00000000 +0.00000027 0.00000000 0.00000000 +0.00000184 0.00000000 0.00000000 +0.00001141 0.00000000 0.00000000 +0.00006315 0.00000000 0.00000000 +0.00030757 0.00000000 0.00000000 +0.00129743 0.00000000 0.00000000 +0.00463770 0.00000000 0.00000000 +0.01367394 0.00000000 0.00000000 +0.03254734 0.00000000 0.00000000 +0.06284289 0.00000000 0.00000000 +0.10239912 0.00000000 0.00000000 +0.14848776 0.00000000 0.00000000 +0.19990818 0.00000000 0.00000000 +0.25644379 0.00000000 0.00000000 +0.31790650 0.00000000 0.00000000 +0.38372274 0.00000000 0.00000000 +0.45300435 0.00000000 0.00000000 +0.52611851 0.00000000 0.00000000 +0.60236434 0.00000000 0.00000000 +0.68060226 0.00000000 0.00000000 +0.75824845 0.00000000 0.00000000 +0.83141786 0.00000000 0.00000000 +0.89432986 0.00000000 0.00000000 +0.93644499 0.00000000 0.00000000 +0.93895647 0.00000000 0.00000000 +0.92279235 0.00000000 0.00000000 +0.93476422 0.00000000 0.00000000 +0.92806061 0.00000000 0.00000000 +0.92713162 0.00000000 0.00000000 +0.92917611 0.00000000 0.00000000 +0.92599741 0.00000000 0.00000000 +0.93187413 0.00000000 0.00000000 +0.92564743 0.00000000 0.00000000 +0.93583017 0.00000000 0.00000000 +0.92607237 0.00000000 0.00000000 +0.93965842 0.00000000 0.00000000 +0.92748587 0.00000000 0.00000000 +0.94185642 0.00000000 0.00000000 +0.92920728 0.00000000 0.00000000 +0.94183858 0.00000000 0.00000000 +0.92976860 0.00000000 0.00000000 +0.93970790 0.00000000 0.00000000 +0.92920706 0.00000000 0.00000000 +0.93484825 0.00000000 0.00000000 +0.92919766 0.00000000 0.00000000 +0.92728112 0.00000000 0.00000000 +0.93315341 0.00000000 0.00000000 +0.91607911 0.00000000 0.00000000 +0.94587905 0.00000000 0.00000000 +0.89849407 0.00000000 0.00000000 +0.97605390 0.00000000 0.00000000 +0.85631753 0.00000000 0.00000000 +1.05542725 0.00000000 0.00000000 +0.40174969 0.00000000 0.00000000 +0.04085501 0.00000000 0.00000000 +0.00152918 0.00000000 0.00000000 +0.00003915 0.00000000 0.00000000 +0.00000098 0.00000000 0.00000000 +0.00000002 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000003 0.00000000 0.00000000 +0.00000027 0.00000000 0.00000000 +0.00000184 0.00000000 0.00000000 +0.00001141 0.00000000 0.00000000 +0.00006315 0.00000000 0.00000000 +0.00030757 0.00000000 0.00000000 +0.00129743 0.00000000 0.00000000 +0.00463770 0.00000000 0.00000000 +0.01367394 0.00000000 0.00000000 +0.03254734 0.00000000 0.00000000 +0.06284289 0.00000000 0.00000000 +0.10239912 0.00000000 0.00000000 +0.14848776 0.00000000 0.00000000 +0.19990818 0.00000000 0.00000000 +0.25644379 0.00000000 0.00000000 +0.31790650 0.00000000 0.00000000 +0.38372274 0.00000000 0.00000000 +0.45300435 0.00000000 0.00000000 +0.52611851 0.00000000 0.00000000 +0.60236434 0.00000000 0.00000000 +0.68060226 0.00000000 0.00000000 +0.75824845 0.00000000 0.00000000 +0.83141786 0.00000000 0.00000000 +0.89432986 0.00000000 0.00000000 +0.93644499 0.00000000 0.00000000 +0.93895647 0.00000000 0.00000000 +0.92279235 0.00000000 0.00000000 +0.93476422 0.00000000 0.00000000 +0.92806061 0.00000000 0.00000000 +0.92713162 0.00000000 0.00000000 +0.92917611 0.00000000 0.00000000 +0.92599741 0.00000000 0.00000000 +0.93187413 0.00000000 0.00000000 +0.92564743 0.00000000 0.00000000 +0.93583017 0.00000000 0.00000000 +0.92607237 0.00000000 0.00000000 +0.93965842 0.00000000 0.00000000 +0.92748587 0.00000000 0.00000000 +0.94185642 0.00000000 0.00000000 +0.92920728 0.00000000 0.00000000 +0.94183858 0.00000000 0.00000000 +0.92976860 0.00000000 0.00000000 +0.93970790 0.00000000 0.00000000 +0.92920706 0.00000000 0.00000000 +0.93484825 0.00000000 0.00000000 +0.92919766 0.00000000 0.00000000 +0.92728112 0.00000000 0.00000000 +0.93315341 0.00000000 0.00000000 +0.91607911 0.00000000 0.00000000 +0.94587905 0.00000000 0.00000000 +0.89849407 0.00000000 0.00000000 +0.97605390 0.00000000 0.00000000 +0.85631753 0.00000000 0.00000000 +1.05542725 0.00000000 0.00000000 +0.40174969 0.00000000 0.00000000 +0.04085501 0.00000000 0.00000000 +0.00152918 0.00000000 0.00000000 +0.00003915 0.00000000 0.00000000 +0.00000098 0.00000000 0.00000000 +0.00000002 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000003 0.00000000 0.00000000 +0.00000027 0.00000000 0.00000000 +0.00000184 0.00000000 0.00000000 +0.00001141 0.00000000 0.00000000 +0.00006315 0.00000000 0.00000000 +0.00030757 0.00000000 0.00000000 +0.00129743 0.00000000 0.00000000 +0.00463770 0.00000000 0.00000000 +0.01367394 0.00000000 0.00000000 +0.03254734 0.00000000 0.00000000 +0.06284289 0.00000000 0.00000000 +0.10239912 0.00000000 0.00000000 +0.14848776 0.00000000 0.00000000 +0.19990818 0.00000000 0.00000000 +0.25644379 0.00000000 0.00000000 +0.31790650 0.00000000 0.00000000 +0.38372274 0.00000000 0.00000000 +0.45300435 0.00000000 0.00000000 +0.52611851 0.00000000 0.00000000 +0.60236434 0.00000000 0.00000000 +0.68060226 0.00000000 0.00000000 +0.75824845 0.00000000 0.00000000 +0.83141786 0.00000000 0.00000000 +0.89432986 0.00000000 0.00000000 +0.93644499 0.00000000 0.00000000 +0.93895647 0.00000000 0.00000000 +0.92279235 0.00000000 0.00000000 +0.93476422 0.00000000 0.00000000 +0.92806061 0.00000000 0.00000000 +0.92713162 0.00000000 0.00000000 +0.92917611 0.00000000 0.00000000 +0.92599741 0.00000000 0.00000000 +0.93187413 0.00000000 0.00000000 +0.92564743 0.00000000 0.00000000 +0.93583017 0.00000000 0.00000000 +0.92607237 0.00000000 0.00000000 +0.93965842 0.00000000 0.00000000 +0.92748587 0.00000000 0.00000000 +0.94185642 0.00000000 0.00000000 +0.92920728 0.00000000 0.00000000 +0.94183858 0.00000000 0.00000000 +0.92976860 0.00000000 0.00000000 +0.93970790 0.00000000 0.00000000 +0.92920706 0.00000000 0.00000000 +0.93484825 0.00000000 0.00000000 +0.92919766 0.00000000 0.00000000 +0.92728112 0.00000000 0.00000000 +0.93315341 0.00000000 0.00000000 +0.91607911 0.00000000 0.00000000 +0.94587905 0.00000000 0.00000000 +0.89849407 0.00000000 0.00000000 +0.97605390 0.00000000 0.00000000 +0.85631753 0.00000000 0.00000000 +1.05542725 0.00000000 0.00000000 +0.40174969 0.00000000 0.00000000 +0.04085501 0.00000000 0.00000000 +0.00152918 0.00000000 0.00000000 +0.00003915 0.00000000 0.00000000 +0.00000098 0.00000000 0.00000000 +0.00000002 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000003 0.00000000 0.00000000 +0.00000027 0.00000000 0.00000000 +0.00000184 0.00000000 0.00000000 +0.00001141 0.00000000 0.00000000 +0.00006315 0.00000000 0.00000000 +0.00030757 0.00000000 0.00000000 +0.00129743 0.00000000 0.00000000 +0.00463770 0.00000000 0.00000000 +0.01367394 0.00000000 0.00000000 +0.03254734 0.00000000 0.00000000 +0.06284289 0.00000000 0.00000000 +0.10239912 0.00000000 0.00000000 +0.14848776 0.00000000 0.00000000 +0.19990818 0.00000000 0.00000000 +0.25644379 0.00000000 0.00000000 +0.31790650 0.00000000 0.00000000 +0.38372274 0.00000000 0.00000000 +0.45300435 0.00000000 0.00000000 +0.52611851 0.00000000 0.00000000 +0.60236434 0.00000000 0.00000000 +0.68060226 0.00000000 0.00000000 +0.75824845 0.00000000 0.00000000 +0.83141786 0.00000000 0.00000000 +0.89432986 0.00000000 0.00000000 +0.93644499 0.00000000 0.00000000 +0.93895647 0.00000000 0.00000000 +0.92279235 0.00000000 0.00000000 +0.93476422 0.00000000 0.00000000 +0.92806061 0.00000000 0.00000000 +0.92713162 0.00000000 0.00000000 +0.92917611 0.00000000 0.00000000 +0.92599741 0.00000000 0.00000000 +0.93187413 0.00000000 0.00000000 +0.92564743 0.00000000 0.00000000 +0.93583017 0.00000000 0.00000000 +0.92607237 0.00000000 0.00000000 +0.93965842 0.00000000 0.00000000 +0.92748587 0.00000000 0.00000000 +0.94185642 0.00000000 0.00000000 +0.92920728 0.00000000 0.00000000 +0.94183858 0.00000000 0.00000000 +0.92976860 0.00000000 0.00000000 +0.93970790 0.00000000 0.00000000 +0.92920706 0.00000000 0.00000000 +0.93484825 0.00000000 0.00000000 +0.92919766 0.00000000 0.00000000 +0.92728112 0.00000000 0.00000000 +0.93315341 0.00000000 0.00000000 +0.91607911 0.00000000 0.00000000 +0.94587905 0.00000000 0.00000000 +0.89849407 0.00000000 0.00000000 +0.97605390 0.00000000 0.00000000 +0.85631753 0.00000000 0.00000000 +1.05542725 0.00000000 0.00000000 +0.40174969 0.00000000 0.00000000 +0.04085501 0.00000000 0.00000000 +0.00152918 0.00000000 0.00000000 +0.00003915 0.00000000 0.00000000 +0.00000098 0.00000000 0.00000000 +0.00000002 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 +0.00000000 0.00000000 0.00000000 + +CELL_DATA 100 +SCALARS SIE float 1 +LOOKUP_TABLE default +2.50000000 +2.50000000 +2.50000000 +2.50000000 +2.50000000 +2.50000000 +2.50000000 +2.50000000 +2.50000000 +2.50000000 +2.50000000 +2.50000000 +2.50000000 +2.50000000 +2.50000000 +2.50000000 +2.50000000 +2.50000000 +2.50000000 +2.50000000 +2.50000000 +2.49999999 +2.49999992 +2.49999940 +2.49999605 +2.49997684 +2.49987972 +2.49945434 +2.49787961 +2.49312031 +2.48184532 +2.46139990 +2.43225569 +2.39689867 +2.35714895 +2.31365539 +2.26666055 +2.21659448 +2.16405784 +2.10933574 +2.05291413 +1.99544678 +1.93823248 +1.88344636 +1.83426296 +1.79610576 +1.78089179 +1.79836597 +1.81293899 +1.87851394 +2.82500242 +2.83198230 +2.84483131 +2.86012865 +2.86486121 +2.87255453 +2.87408775 +2.87650412 +2.87986809 +2.87647574 +2.88315221 +2.87389571 +2.88388635 +2.86850030 +2.88281200 +2.86035711 +2.88182329 +2.85025511 +2.88306648 +2.83829114 +2.88901242 +2.82463569 +2.90413292 +2.79997443 +2.94027458 +2.60532725 +2.08943210 +2.00598004 +2.00019246 +2.00000487 +2.00000012 +2.00000000 +2.00000000 +2.00000000 +2.00000000 +2.00000000 +2.00000000 +2.00000000 +2.00000000 +2.00000000 +2.00000000 +2.00000000 +2.00000000 +2.00000000 +2.00000000 +2.00000000 +2.00000000 +2.00000000 +2.00000000 +2.00000000 + +SCALARS element_density float 1 +LOOKUP_TABLE default +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +0.99999999 +0.99999992 +0.99999940 +0.99999605 +0.99997685 +0.99987972 +0.99945443 +0.99788097 +0.99313439 +0.98194155 +0.96182701 +0.93355666 +0.89989728 +0.86289054 +0.82339949 +0.78188581 +0.73895613 +0.69538538 +0.65158453 +0.60806207 +0.56539654 +0.52453305 +0.48684879 +0.45408046 +0.42890668 +0.41714041 +0.42239687 +0.41884786 +0.40313643 +0.26758425 +0.26734703 +0.26544034 +0.26518034 +0.26394256 +0.26441583 +0.26405754 +0.26426562 +0.26493846 +0.26422528 +0.26593788 +0.26407291 +0.26673477 +0.26360975 +0.26734429 +0.26257195 +0.26792411 +0.26103913 +0.26903186 +0.25911001 +0.27124774 +0.25677746 +0.27573198 +0.25232138 +0.28669420 +0.21951448 +0.13886674 +0.12593605 +0.12503007 +0.12500076 +0.12500002 +0.12500000 +0.12500000 +0.12500000 +0.12500000 +0.12500000 +0.12500000 +0.12500000 +0.12500000 +0.12500000 +0.12500000 +0.12500000 +0.12500000 +0.12500000 +0.12500000 +0.12500000 +0.12500000 +0.12500000 +0.12500000 +0.12500000 + +SCALARS mass float 1 +LOOKUP_TABLE default +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000012 +0.00000013 +0.00000012 +0.00000013 +0.00000012 +0.00000013 +0.00000012 +0.00000012 +0.00000013 +0.00000012 +0.00000012 +0.00000013 +0.00000012 +0.00000013 +0.00000012 +0.00000012 +0.00000013 +0.00000012 +0.00000013 +0.00000012 +0.00000012 +0.00000013 +0.00000012 +0.00000013 +0.00000012 +0.00000012 +0.00000013 +0.00000012 +0.00000013 +0.00000012 +0.00000013 +0.00000012 +0.00000013 +0.00000012 +0.00000013 +0.00000013 +0.00000012 +0.00000013 +0.00000013 +0.00000013 +0.00000012 +0.00000013 +0.00000013 +0.00000012 +0.00000013 +0.00000012 +0.00000013 +0.00000013 +0.00000012 +0.00000013 + +SCALARS pressure float 1 +LOOKUP_TABLE default +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +1.00000000 +0.99999999 +0.99999988 +0.99999916 +0.99999447 +0.99996758 +0.99983162 +0.99923629 +0.99703461 +0.99040141 +0.97481081 +0.94697636 +0.90825940 +0.86278504 +0.81358461 +0.76202507 +0.70890789 +0.65518644 +0.60194167 +0.54976421 +0.49931968 +0.45128748 +0.40666680 +0.36678143 +0.33316119 +0.30814470 +0.29715277 +0.30384966 +0.30373825 +0.30291896 +0.30237047 +0.30284882 +0.30205320 +0.30337996 +0.30246352 +0.30381956 +0.30356982 +0.30406446 +0.30519512 +0.30401504 +0.30669575 +0.30356720 +0.30769310 +0.30246586 +0.30828134 +0.30041982 +0.30884397 +0.29761124 +0.31025470 +0.29417186 +0.31345524 +0.29012111 +0.32030493 +0.28259736 +0.33718387 +0.22876282 +0.11606105 +0.10105008 +0.10003369 +0.10000085 +0.10000002 +0.10000000 +0.10000000 +0.10000000 +0.10000000 +0.10000000 +0.10000000 +0.10000000 +0.10000000 +0.10000000 +0.10000000 +0.10000000 +0.10000000 +0.10000000 +0.10000000 +0.10000000 +0.10000000 +0.10000000 +0.10000000 +0.10000000 + +SCALARS sound_speed float 1 +LOOKUP_TABLE default +1.18321596 +1.18321596 +1.18321596 +1.18321596 +1.18321596 +1.18321596 +1.18321596 +1.18321596 +1.18321596 +1.18321596 +1.18321596 +1.18321596 +1.18321596 +1.18321596 +1.18321596 +1.18321596 +1.18321596 +1.18321596 +1.18321596 +1.18321596 +1.18321596 +1.18321595 +1.18321594 +1.18321581 +1.18321502 +1.18321048 +1.18318749 +1.18308682 +1.18271407 +1.18158680 +1.17891195 +1.17404597 +1.16707463 +1.15856085 +1.14891402 +1.13826492 +1.12664542 +1.11413325 +1.10085076 +1.08684314 +1.07220889 +1.05709517 +1.04183021 +1.02700047 +1.01350247 +1.00290539 +0.99864879 +1.00353622 +1.00759408 +1.02565482 +1.25777635 +1.25932922 +1.26218285 +1.26557183 +1.26661844 +1.26831799 +1.26865643 +1.26918963 +1.26993155 +1.26918336 +1.27065544 +1.26861405 +1.27081720 +1.26742265 +1.27058047 +1.26562237 +1.27036256 +1.26338548 +1.27063655 +1.26073115 +1.27194613 +1.25769471 +1.27527034 +1.25219235 +1.28318111 +1.20788379 +1.08170328 +1.05988151 +1.05835144 +1.05830181 +1.05830056 +1.05830053 +1.05830052 +1.05830052 +1.05830052 +1.05830052 +1.05830052 +1.05830052 +1.05830052 +1.05830052 +1.05830052 +1.05830052 +1.05830052 +1.05830052 +1.05830052 +1.05830052 +1.05830052 +1.05830052 +1.05830052 +1.05830052 + +SCALARS volume float 1 +LOOKUP_TABLE default +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000101 +0.00000102 +0.00000104 +0.00000107 +0.00000111 +0.00000116 +0.00000121 +0.00000128 +0.00000135 +0.00000144 +0.00000153 +0.00000164 +0.00000177 +0.00000191 +0.00000205 +0.00000220 +0.00000233 +0.00000240 +0.00000237 +0.00000239 +0.00000248 +0.00000047 +0.00000047 +0.00000047 +0.00000047 +0.00000047 +0.00000047 +0.00000047 +0.00000047 +0.00000047 +0.00000047 +0.00000047 +0.00000047 +0.00000047 +0.00000047 +0.00000047 +0.00000048 +0.00000047 +0.00000048 +0.00000046 +0.00000048 +0.00000046 +0.00000049 +0.00000045 +0.00000050 +0.00000044 +0.00000057 +0.00000090 +0.00000099 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 +0.00000100 + +FIELD FieldData 0 diff --git a/integrated-tests/Test-scripts/standard-results/SGH/Sod/vtk/outputs.vtk.series b/integrated-tests/Test-scripts/standard-results/SGH/Sod/vtk/outputs.vtk.series new file mode 100644 index 000000000..04e3b3502 --- /dev/null +++ b/integrated-tests/Test-scripts/standard-results/SGH/Sod/vtk/outputs.vtk.series @@ -0,0 +1,6 @@ +{ + "file-series-version" : "1.0", + "files" : [ + { "name" : "data/VTK0.vtk", "time" : 1.50000e-01 }, + ] +} diff --git a/python/Test-scripts/test_fierro.py b/integrated-tests/Test-scripts/test_fierro.py similarity index 63% rename from python/Test-scripts/test_fierro.py rename to integrated-tests/Test-scripts/test_fierro.py index fbee4121e..ed4a91b47 100644 --- a/python/Test-scripts/test_fierro.py +++ b/integrated-tests/Test-scripts/test_fierro.py @@ -6,13 +6,20 @@ import math executable = "./../../build-fierro-openmp/bin/fierro-parallel-explicit" -sim_input = "Solver-Inputs/SGH_Sedov_12x12x12.yaml" -# Run simulation -os.system(executable + ' ' + sim_input) +inputs = [] +standard_results = [] +tests = ["Noh", "Sedov", "Sod"] -# Functions for reading results from vtk file +position_keyword = "POINTS" + +for i in range(len(tests)): + inputs.append("Solver-Inputs/SGH_"+tests[i]+"_simple.yaml") + standard_results.append("standard-results/SGH/"+tests[i]+"/vtk/data/VTK0.vtk") + + +# Extract vector valued data from vtk output file def extract_vector_data(filename, keyword): data = [] found_keyword = False @@ -36,6 +43,7 @@ def extract_vector_data(filename, keyword): return data +# Extract scalar valued data from vtk output file def extract_scalar_data(filename, keyword): data = [] found_keyword = False @@ -55,8 +63,7 @@ def extract_scalar_data(filename, keyword): return data - - +# Calculate the percent difference between two arrays of scalars def percent_difference_scalars(array1, array2): if len(array1) != len(array2): raise ValueError("Arrays must have the same length") @@ -68,6 +75,7 @@ def percent_difference_scalars(array1, array2): return percent_diff +# Calculate the percent difference between two arrays of vectors def percent_difference_vectors(array1, array2): if len(array1) != len(array2): raise ValueError("Arrays must have the same length") @@ -80,38 +88,36 @@ def percent_difference_vectors(array1, array2): diff = [array2[i][j] - array1[i][j] for j in range(3)] percent_diff.append([(diff[j] / array1[i][j]) * 100 if array1[i][j] != 0 else 0 for j in range(3)]) - return percent_diff + percent_diff_mag = [] + for i in range(len(array1)): + percent_diff_mag.append(magnitude(percent_diff[i])) + + return percent_diff_mag +# Calculate the magnitude of a vector def magnitude(array): mag = math.sqrt(sum(x**2 for x in array)) return mag +# Run each test +for i in range(len(tests)): + + # Run simulation + print("Running "+tests[i]) + os.system(executable + ' ' + inputs[i]) -# Read ground truth results for sedov -GT_filename = "standard-results/SGH/Sedov_12x12x12/vtk/data/VTK0.vtk" - -velocity_keyword = "VECTORS velocity float" -position_keyword = "POINTS 2197 float" -SIE_keyword = "SCALARS SIE float 1" -density_keyword = "SCALARS element_density float 1" - -GT_positions = extract_vector_data(GT_filename, position_keyword) -GT_velocities = extract_vector_data(GT_filename, velocity_keyword) -GT_SIE = extract_scalar_data(GT_filename, SIE_keyword) -GT_densities = extract_scalar_data(GT_filename, density_keyword) - - -# Read simulation results -results_filename = "vtk/data/VTK0.vtk" + GT_positions = extract_vector_data(standard_results[i], position_keyword) -results_positions = extract_vector_data(results_filename, position_keyword) -results_velocities = extract_vector_data(results_filename, velocity_keyword) -results_SIE = extract_scalar_data(results_filename, SIE_keyword) -results_densities = extract_scalar_data(results_filename, density_keyword) + # Read simulation results + results_filename = "vtk/data/VTK0.vtk" + results_positions = extract_vector_data(results_filename, position_keyword) + position_diff = percent_difference_vectors(GT_positions, results_positions) -density_diff = percent_difference_scalars(GT_densities, results_densities) -print("Density difference: ") -print(magnitude(density_diff)) + for i in range(len(position_diff)): + if position_diff[i] >= 1.0e-6: + raise ValueError(" ****************** ERROR: Position difference out of range for "+tests[i]+" problem ****************** ") + print("Removing simulation outputs") + os.system('rm -rf vtk' ) \ No newline at end of file diff --git a/src/Parallel-Solvers/Simulation_Parameters/Geometry.h b/src/Parallel-Solvers/Simulation_Parameters/Geometry.h index 0ba7a945c..44ff4fb4a 100644 --- a/src/Parallel-Solvers/Simulation_Parameters/Geometry.h +++ b/src/Parallel-Solvers/Simulation_Parameters/Geometry.h @@ -90,4 +90,4 @@ struct Volume : Yaml::ValidatedYaml { } } }; -IMPL_YAML_SERIALIZABLE_FOR(Volume, type, radius1, radius2) \ No newline at end of file +IMPL_YAML_SERIALIZABLE_FOR(Volume, type, radius1, radius2, x1, x2, y1, y2, z1, z2) \ No newline at end of file From 492b8a1a5b8e46ca7863ab8d16a960c10eb4cdef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CAdrian-Diaz=E2=80=9D?= <“adriandiaz1117@gmail.com”> Date: Thu, 7 Mar 2024 13:33:34 -0700 Subject: [PATCH 59/64] BUG: q2 gradient term bug --- .../SGH_Solver/src/force_gradients_sgh.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp index 4762c5ad2..01de10bbf 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp @@ -287,22 +287,22 @@ void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& ma { if(igradient == node_lid) { - mag_vel_gradient(igradient,jdim) = (1 - 1/num_nodes_in_elem)*((vel(jdim) - vel_star(jdim) ))/ + mag_vel_gradient(igradient,jdim) = (1 - 1/((real_t) num_nodes_in_elem))*((vel(jdim) - vel_star(jdim) ))/ sqrt( (vel(0) - vel_star(0) ) * (vel(0) - vel_star(0) ) + (vel(1) - vel_star(1) ) * (vel(1) - vel_star(1) ) + (vel(2) - vel_star(2) ) * (vel(2) - vel_star(2) ) ); if(mag_vel==0){ - mag_vel_gradient(igradient,jdim) = (1 - 1/num_nodes_in_elem); + mag_vel_gradient(igradient,jdim) = (1 - 1/((real_t) num_nodes_in_elem)); } } else { - mag_vel_gradient(igradient,jdim) = -1/num_nodes_in_elem*( (vel(jdim) - vel_star(jdim) ))/ + mag_vel_gradient(igradient,jdim) = -1/((real_t) num_nodes_in_elem)*( (vel(jdim) - vel_star(jdim) ))/ sqrt( (vel(0) - vel_star(0) ) * (vel(0) - vel_star(0) ) + (vel(1) - vel_star(1) ) * (vel(1) - vel_star(1) ) + (vel(2) - vel_star(2) ) * (vel(2) - vel_star(2) ) ); if(mag_vel==0){ - mag_vel_gradient(igradient,jdim) = -1/num_nodes_in_elem; + mag_vel_gradient(igradient,jdim) = -1/((real_t) num_nodes_in_elem); } } } From bb3dac2cc253b83e2a5965a203337c81608931ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CAdrian-Diaz=E2=80=9D?= <“adriandiaz1117@gmail.com”> Date: Thu, 7 Mar 2024 19:27:00 -0700 Subject: [PATCH 60/64] BUG: q2 term gradient bug --- .../SGH_Solver/src/force_gradients_sgh.cpp | 24 +++++++------------ 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp index 01de10bbf..4ad9039f9 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp @@ -241,9 +241,9 @@ void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& ma ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); - vel_star(0) += vel(0)/num_nodes_in_elem; - vel_star(1) += vel(1)/num_nodes_in_elem; - vel_star(2) += vel(2)/num_nodes_in_elem; + vel_star(0) += vel(0)/((real_t) num_nodes_in_elem); + vel_star(1) += vel(1)/((real_t) num_nodes_in_elem); + vel_star(2) += vel(2)/((real_t) num_nodes_in_elem); } // end for loop over nodes // find shock direction and shock impedance associated with each node @@ -287,22 +287,16 @@ void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& ma { if(igradient == node_lid) { - mag_vel_gradient(igradient,jdim) = (1 - 1/((real_t) num_nodes_in_elem))*((vel(jdim) - vel_star(jdim) ))/ - sqrt( (vel(0) - vel_star(0) ) * (vel(0) - vel_star(0) ) - + (vel(1) - vel_star(1) ) * (vel(1) - vel_star(1) ) - + (vel(2) - vel_star(2) ) * (vel(2) - vel_star(2) ) ); - if(mag_vel==0){ - mag_vel_gradient(igradient,jdim) = (1 - 1/((real_t) num_nodes_in_elem)); + mag_vel_gradient(igradient,jdim) = (1.0 - 1.0/((real_t) num_nodes_in_elem))*((vel(jdim) - vel_star(jdim) ))/mag_vel; + if(mag_vel < fuzz){ + mag_vel_gradient(igradient,jdim) = (1.0 - 1.0/((real_t) num_nodes_in_elem)); } } else { - mag_vel_gradient(igradient,jdim) = -1/((real_t) num_nodes_in_elem)*( (vel(jdim) - vel_star(jdim) ))/ - sqrt( (vel(0) - vel_star(0) ) * (vel(0) - vel_star(0) ) - + (vel(1) - vel_star(1) ) * (vel(1) - vel_star(1) ) - + (vel(2) - vel_star(2) ) * (vel(2) - vel_star(2) ) ); - if(mag_vel==0){ - mag_vel_gradient(igradient,jdim) = -1/((real_t) num_nodes_in_elem); + mag_vel_gradient(igradient,jdim) = -1.0/((real_t) num_nodes_in_elem)*( (vel(jdim) - vel_star(jdim) ))/mag_vel; + if(mag_vel < fuzz){ + mag_vel_gradient(igradient,jdim) = 1.0/((real_t) num_nodes_in_elem); } } } From bdb6141ccb0a37f27836b5b9b34ea375eb098d60 Mon Sep 17 00:00:00 2001 From: Jacob Moore <133793250+jacob-moore22@users.noreply.github.com> Date: Fri, 8 Mar 2024 10:04:32 -0600 Subject: [PATCH 61/64] Uncrustify left bracket (#181) * STYLE: Modifying uncrustify config * STYLE: Uncrustify to move left bracket --- docs/formatting/uncrustify.cfg | 8 +- scripts/uncrustify.sh | 11 +- .../FEA_Module_Dynamic_Elasticity.cpp | 480 ++++++----------- .../Dynamic_Elastic_Solver/boundary.cpp | 21 +- .../elastic_optimization.cpp | 471 ++++++----------- .../Dynamic_Elastic_Solver/force_elastic.cpp | 387 +++++--------- .../force_gradients_elastic.cpp | 279 ++++------ .../Dynamic_Elastic_Solver/geometry.cpp | 27 +- .../Dynamic_Elastic_Solver/mesh.h | 156 ++---- .../Dynamic_Elastic_Solver/momentum.cpp | 30 +- .../Dynamic_Elastic_Solver/properties.cpp | 12 +- .../time_integration.cpp | 51 +- .../Eulerian_Solver/FEA_Module_Eulerian.cpp | 267 ++++------ .../Simulation_Parameters_Eulerian.h | 15 +- .../Eulerian_Solver/eulerian_geometry.cpp | 3 +- .../eulerian_time_integration.cpp | 51 +- .../SGH_Solver/include/mesh.h | 156 ++---- .../SGH_Solver/src/FEA_Module_SGH.cpp | 351 +++++-------- .../SGH_Solver/src/boundary.cpp | 18 +- .../SGH_Solver/src/energy_sgh.cpp | 9 +- .../SGH_Solver/src/force_gradients_sgh.cpp | 486 ++++++----------- .../SGH_Solver/src/force_sgh.cpp | 204 +++----- .../SGH_Solver/src/geometry.cpp | 120 ++--- .../SGH_Solver/src/momentum.cpp | 42 +- .../SGH_Solver/src/power_gradients_sgh.cpp | 120 ++--- .../SGH_Solver/src/properties.cpp | 6 +- .../SGH_Solver/src/setup_sgh.cpp | 239 +++------ .../SGH_Solver/src/sgh_optimization.cpp | 488 ++++++------------ .../SGH_Solver/src/time_integration.cpp | 51 +- .../Topology_Optimization/Area_Normals.h | 33 +- .../Kinetic_Energy_Minimize.h | 98 ++-- 31 files changed, 1573 insertions(+), 3117 deletions(-) diff --git a/docs/formatting/uncrustify.cfg b/docs/formatting/uncrustify.cfg index e12185b98..2d050b858 100755 --- a/docs/formatting/uncrustify.cfg +++ b/docs/formatting/uncrustify.cfg @@ -100,13 +100,13 @@ nl_struct_brace = force # ignore/add/remove/force nl_union_brace = force # ignore/add/remove/force # Add or remove newline between 'if' and '{' -nl_if_brace = force # ignore/add/remove/force +nl_if_brace = remove # ignore/add/remove/force # Add or remove newline between '}' and 'else' nl_brace_else = force # ignore/add/remove/force # Add or remove newline between 'else' and '{' -nl_else_brace = force # ignore/add/remove/force +nl_else_brace = remove # ignore/add/remove/force # Add or remove newline between 'else' and 'if' nl_else_if = remove # ignore/add/remove/force @@ -121,7 +121,7 @@ nl_finally_brace = force # ignore/add/remove/force nl_try_brace = force # ignore/add/remove/force # Add or remove newline between 'for' and '{' -nl_for_brace = force # ignore/add/remove/force +nl_for_brace = remove # ignore/add/remove/force # Add or remove newline between 'catch' and '{' nl_catch_brace = force # ignore/add/remove/force @@ -136,7 +136,7 @@ nl_do_brace = force # ignore/add/remove/force nl_brace_while = force # ignore/add/remove/force # Add or remove newline between 'switch' and '{' -nl_switch_brace = force # ignore/add/remove/force +nl_switch_brace = remove # ignore/add/remove/force # Add or remove newline between function signature and '{' nl_fdef_brace = force # ignore/add/remove/force diff --git a/scripts/uncrustify.sh b/scripts/uncrustify.sh index becdde2c2..1a43f47e9 100755 --- a/scripts/uncrustify.sh +++ b/scripts/uncrustify.sh @@ -4,7 +4,9 @@ # This script assumes uncrustify has been installed + uncrustifyConfigDir="docs/formatting" +uncrustify_ex_dir="dev-utils/uncrustify/build" # Save the directory the user had before calling startDir=$(pwd) thisScriptDir=$(dirname "$0") @@ -17,6 +19,11 @@ cd $uncrustifyConfigDir fullUncrustifyDir=$(pwd) cd $startDir +#Get the full path to the uncrustify executable +cd $uncrustify_ex_dir +uncrustify_exe=$(pwd)/uncrustify +cd $startDir + # Get the source directory cd src sourceDir=$(pwd) @@ -29,14 +36,14 @@ treeProcess() { for file in *.cpp; do if [ -f "$file" ]; then # echo "Using: $(pwd)/$f" - uncrustify -c "$fullUncrustifyDir"/uncrustify.cfg --no-backup "$(pwd)/$file" + "$uncrustify_exe" -c "$fullUncrustifyDir"/uncrustify.cfg --no-backup "$(pwd)/$file" fi done for file in *.h; do if [ -f "$file" ]; then # echo "Using: $(pwd)/$f" - uncrustify -c "$fullUncrustifyDir"/uncrustify.cfg --no-backup "$(pwd)/$file" + "$uncrustify_exe" -c "$fullUncrustifyDir"/uncrustify.cfg --no-backup "$(pwd)/$file" fi done diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.cpp b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.cpp index dc878f98f..921703588 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/FEA_Module_Dynamic_Elasticity.cpp @@ -102,8 +102,7 @@ FEA_Module_Dynamic_Elasticity::FEA_Module_Dynamic_Elasticity( node_velocities_distributed = Explicit_Solver_Pointer_->node_velocities_distributed; all_node_velocities_distributed = Explicit_Solver_Pointer_->all_node_velocities_distributed; - if (simparam->topology_optimization_on || simparam->shape_optimization_on) - { + if (simparam->topology_optimization_on || simparam->shape_optimization_on) { all_cached_node_velocities_distributed = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); force_gradient_velocity = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); force_gradient_position = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); @@ -113,8 +112,7 @@ FEA_Module_Dynamic_Elasticity::FEA_Module_Dynamic_Elasticity( relative_element_densities = DCArrayKokkos(rnum_elem, "relative_element_densities"); } - if (simparam->topology_optimization_on || simparam->shape_optimization_on || simparam->num_dims == 2) - { + if (simparam->topology_optimization_on || simparam->shape_optimization_on || simparam->num_dims == 2) { node_masses_distributed = Teuchos::rcp(new MV(map, 1)); ghost_node_masses_distributed = Teuchos::rcp(new MV(ghost_node_map, 1)); adjoint_vector_distributed = Teuchos::rcp(new MV(map, simparam->num_dims)); @@ -152,8 +150,7 @@ FEA_Module_Dynamic_Elasticity::FEA_Module_Dynamic_Elasticity( tiny = dynamic_options.tiny; small = dynamic_options.small; - if (simparam->topology_optimization_on) - { + if (simparam->topology_optimization_on) { max_time_steps = BUFFER_GROW; forward_solve_velocity_data = Teuchos::rcp(new std::vector>(max_time_steps + 1)); time_data.resize(max_time_steps + 1); @@ -161,8 +158,7 @@ FEA_Module_Dynamic_Elasticity::FEA_Module_Dynamic_Elasticity( adjoint_vector_data = Teuchos::rcp(new std::vector>(max_time_steps + 1)); phi_adjoint_vector_data = Teuchos::rcp(new std::vector>(max_time_steps + 1)); // assign a multivector of corresponding size to each new timestep in the buffer - for (int istep = 0; istep < max_time_steps + 1; istep++) - { + for (int istep = 0; istep < max_time_steps + 1; istep++) { (*forward_solve_velocity_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); (*forward_solve_coordinate_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); (*adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); @@ -235,8 +231,7 @@ void FEA_Module_Dynamic_Elasticity::elastic_interface_setup(node_t& node, const size_t rk_num_bins = simparam->dynamic_options.rk_num_bins; num_nodes_in_elem = 1; - for (int dim = 0; dim < num_dim; dim++) - { + for (int dim = 0; dim < num_dim; dim++) { num_nodes_in_elem *= 2; } @@ -258,10 +253,8 @@ void FEA_Module_Dynamic_Elasticity::elastic_interface_setup(node_t& node, host_vec_array interface_node_coords = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); // save node data to node.coords // std::cout << "NODE DATA ON RANK " << myrank << std::endl; - if (num_dim == 2) - { - for (int inode = 0; inode < nall_nodes; inode++) - { + if (num_dim == 2) { + for (int inode = 0; inode < nall_nodes; inode++) { // std::cout << "Node index " << inode+1 << " "; node.coords.host(0, inode, 0) = interface_node_coords(inode, 0); // std::cout << host_node_coords_state(0,inode,0)+1<< " "; @@ -269,10 +262,8 @@ void FEA_Module_Dynamic_Elasticity::elastic_interface_setup(node_t& node, // std::cout << host_node_coords_state(0,inode,1)+1<< " "; } } - else if (num_dim == 3) - { - for (int inode = 0; inode < nall_nodes; inode++) - { + else if (num_dim == 3) { + for (int inode = 0; inode < nall_nodes; inode++) { // std::cout << "Node index " << inode+1 << " "; node.coords.host(0, inode, 0) = interface_node_coords(inode, 0); // std::cout << host_node_coords_state(0,inode,0)+1<< " "; @@ -300,11 +291,9 @@ void FEA_Module_Dynamic_Elasticity::elastic_interface_setup(node_t& node, host_elem_conn_array interface_nodes_in_elem = Explicit_Solver_Pointer_->global_nodes_in_elem_distributed->getLocalView(Tpetra::Access::ReadWrite); // save node data to node.coords // std::cout << "ELEMENT CONNECTIVITY ON RANK " << myrank << std::endl; - for (int ielem = 0; ielem < rnum_elem; ielem++) - { + for (int ielem = 0; ielem < rnum_elem; ielem++) { // std::cout << "Element index " << ielem+1 << " "; - for (int inode = 0; inode < num_nodes_in_elem; inode++) - { + for (int inode = 0; inode < num_nodes_in_elem; inode++) { nodes_in_elem.host(ielem, inode) = Explicit_Solver_Pointer_->all_node_map->getLocalElement(interface_nodes_in_elem(ielem, inode)); // debug print // std::cout << nodes_in_elem.get_kokkos_dual_view().h_view(ielem*num_nodes_in_elem + inode)+1<< " "; @@ -378,12 +367,9 @@ void FEA_Module_Dynamic_Elasticity::elastic_interface_setup(node_t& node, */ // save the node coords to the current RK value - for (size_t node_gid = 0; node_gid < nall_nodes; node_gid++) - { - for (int rk = 1; rk < rk_num_bins; rk++) - { - for (int dim = 0; dim < num_dim; dim++) - { + for (size_t node_gid = 0; node_gid < nall_nodes; node_gid++) { + for (int rk = 1; rk < rk_num_bins; rk++) { + for (int dim = 0; dim < num_dim; dim++) { node.coords.host(rk, node_gid, dim) = node.coords.host(0, node_gid, dim); } // end for dim } // end for rk @@ -440,14 +426,12 @@ void FEA_Module_Dynamic_Elasticity::init_boundaries() int num_dim = simparam->num_dims; // set the number of boundary sets - if (myrank == 0) - { + if (myrank == 0) { std::cout << "building boundary sets " << std::endl; } // initialize to 1 since there must be at least 1 boundary set anyway; read in may occure later - if (max_boundary_sets == 0) - { + if (max_boundary_sets == 0) { max_boundary_sets = 1; } // std::cout << "NUM BOUNDARY CONDITIONS ON RANK " << myrank << " FOR INIT " << num_boundary_conditions <(nall_nodes * num_dim, "Node_DOF_Boundary_Condition_Type"); // initialize - for (int init = 0; init < nall_nodes * num_dim; init++) - { + for (int init = 0; init < nall_nodes * num_dim; init++) { Node_DOF_Boundary_Condition_Type(init) = NONE; } @@ -476,8 +459,7 @@ void FEA_Module_Dynamic_Elasticity::init_boundaries() ///////////////////////////////////////////////////////////////////////////// void FEA_Module_Dynamic_Elasticity::init_boundary_sets(int num_sets) { - if (num_sets == 0) - { + if (num_sets == 0) { std::cout << " Warning: number of boundary conditions = 0"; return; } @@ -490,14 +472,12 @@ void FEA_Module_Dynamic_Elasticity::init_boundary_sets(int num_sets) Boundary_Condition_Patches = CArrayKokkos(num_sets, nboundary_patches, "Boundary_Condition_Patches"); // initialize data - for (int iset = 0; iset < num_sets; iset++) - { + for (int iset = 0; iset < num_sets; iset++) { NBoundary_Condition_Patches(iset) = 0; } // initialize - for (int ibdy = 0; ibdy < num_sets; ibdy++) - { + for (int ibdy = 0; ibdy < num_sets; ibdy++) { Boundary_Condition_Type_List(ibdy) = NONE; } } @@ -515,15 +495,13 @@ void FEA_Module_Dynamic_Elasticity::grow_boundary_sets(int num_sets) { int num_dim = simparam->num_dims; - if (num_sets == 0) - { + if (num_sets == 0) { std::cout << " Warning: number of boundary conditions being set to 0"; return; } // std::cout << " DEBUG PRINT "< max_boundary_sets) - { + if (num_sets > max_boundary_sets) { // temporary storage for previous data CArrayKokkos Temp_Boundary_Condition_Type_List = Boundary_Condition_Type_List; CArrayKokkos Temp_NBoundary_Condition_Patches = NBoundary_Condition_Patches; @@ -537,25 +515,21 @@ void FEA_Module_Dynamic_Elasticity::grow_boundary_sets(int num_sets) // copy previous data back over // std::cout << "NUM BOUNDARY CONDITIONS ON RANK " << myrank << " FOR COPY " << max_boundary_sets <num_dims; int Brows; - if (num_dim == 3) - { + if (num_dim == 3) { Brows = 6; } - else - { + else{ Brows = 3; } - if (output_velocity_flag) - { + if (output_velocity_flag) { // displacement_index is accessed by writers at the solver level for deformed output output_velocity_index = noutput; noutput += 1; @@ -631,13 +602,11 @@ void FEA_Module_Dynamic_Elasticity::init_output() output_dof_names[noutput - 1].resize(num_dim); output_dof_names[noutput - 1][0] = "vx"; output_dof_names[noutput - 1][1] = "vy"; - if (num_dim == 3) - { + if (num_dim == 3) { output_dof_names[noutput - 1][2] = "vz"; } } - if (output_strain_flag) - { + if (output_strain_flag) { output_strain_index = noutput; noutput += 1; module_outputs.resize(noutput); @@ -650,14 +619,12 @@ void FEA_Module_Dynamic_Elasticity::init_output() output_dof_names.resize(noutput); output_dof_names[noutput - 1].resize(Brows); - if (num_dim == 2) - { + if (num_dim == 2) { output_dof_names[noutput - 1][0] = "strain_xx"; output_dof_names[noutput - 1][1] = "strain_yy"; output_dof_names[noutput - 1][2] = "strain_xy"; } - if (num_dim == 3) - { + if (num_dim == 3) { output_dof_names[noutput - 1][0] = "strain_xx"; output_dof_names[noutput - 1][1] = "strain_yy"; output_dof_names[noutput - 1][2] = "strain_zz"; @@ -666,8 +633,7 @@ void FEA_Module_Dynamic_Elasticity::init_output() output_dof_names[noutput - 1][5] = "strain_yz"; } } - if (output_stress_flag) - { + if (output_stress_flag) { output_stress_index = noutput; noutput += 1; module_outputs.resize(noutput); @@ -680,14 +646,12 @@ void FEA_Module_Dynamic_Elasticity::init_output() output_dof_names.resize(noutput); output_dof_names[noutput - 1].resize(Brows); - if (num_dim == 2) - { + if (num_dim == 2) { output_dof_names[noutput - 1][0] = "stress_xx"; output_dof_names[noutput - 1][1] = "stress_yy"; output_dof_names[noutput - 1][3] = "stress_xy"; } - if (num_dim == 3) - { + if (num_dim == 3) { output_dof_names[noutput - 1][0] = "stress_xx"; output_dof_names[noutput - 1][1] = "stress_yy"; output_dof_names[noutput - 1][2] = "stress_zz"; @@ -732,10 +696,8 @@ void FEA_Module_Dynamic_Elasticity::write_data(std::mapdynamic_options.rk_num_bins - 1; - for (const FIELD& field_name : simparam->output_options.output_fields) - { - switch (field_name) - { + for (const FIELD& field_name : simparam->output_options.output_fields) { + switch (field_name) { case FIELD::velocity: // node "velocity" node_vel.update_host(); @@ -817,8 +779,7 @@ void FEA_Module_Dynamic_Elasticity::sort_element_output(Teuchos::RCPgetLocalView(Tpetra::Access::ReadWrite); elem_den.update_host(); - for (int ielem = 0; ielem < rnum_elem; ielem++) - { + for (int ielem = 0; ielem < rnum_elem; ielem++) { Element_Densities(ielem, 0) = elem_den.host(ielem); } } @@ -895,8 +856,7 @@ void FEA_Module_Dynamic_Elasticity::comm_node_masses() ///////////////////////////////////////////////////////////////////////////// void FEA_Module_Dynamic_Elasticity::comm_variables(Teuchos::RCP zp) { - if (simparam->topology_optimization_on) - { + if (simparam->topology_optimization_on) { // set density vector to the current value chosen by the optimizer test_node_densities_distributed = zp; @@ -916,8 +876,7 @@ void FEA_Module_Dynamic_Elasticity::comm_variables(Teuchos::RCP zp) // comms to get ghosts all_node_densities_distributed->doImport(*test_node_densities_distributed, *importer, Tpetra::INSERT); } - else if (simparam->shape_optimization_on) - { + else if (simparam->shape_optimization_on) { // clause to communicate boundary node data if the boundary nodes are ghosts on this rank } } @@ -945,19 +904,16 @@ void FEA_Module_Dynamic_Elasticity::node_density_constraints(host_vec_array& nod FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { double current_node_coords[3]; double radius; - for (size_t dim = 0; dim < num_dim; dim++) - { + for (size_t dim = 0; dim < num_dim; dim++) { current_node_coords[dim] = all_initial_node_coords(node_gid, dim); } // end for dim radius = sqrt(current_node_coords[0] * current_node_coords[0] + current_node_coords[1] * current_node_coords[1] + current_node_coords[2] * current_node_coords[2]); - for (size_t ilc = 0; ilc < num_lcs; ilc++) - { + for (size_t ilc = 0; ilc < num_lcs; ilc++) { // debug check // std::cout << "LOADING CONDITION VOLUME TYPE: " << to_string(loading(ilc).volume) << std::endl; bool fill_this = loading(ilc).volume.contains(current_node_coords); - if (fill_this) - { + if (fill_this) { node_densities_lower_bound(node_gid, 0) = 1; } } @@ -990,8 +946,7 @@ void FEA_Module_Dynamic_Elasticity::setup() mesh->build_corner_connectivity(); mesh->build_elem_elem_connectivity(); mesh->num_bdy_patches = nboundary_patches; - if (num_dim == 2) - { + if (num_dim == 2) { mesh->build_patch_connectivity(); mesh->build_node_node_connectivity(); } @@ -1074,8 +1029,7 @@ void FEA_Module_Dynamic_Elasticity::setup() // patch ids in bdy set bdy_patches_in_set = mesh->bdy_patches_in_set; - if (num_dim == 2) - { + if (num_dim == 2) { bdy_nodes = mesh->bdy_nodes; } @@ -1101,8 +1055,7 @@ void FEA_Module_Dynamic_Elasticity::setup() // elem-node conn & node-node conn elems_in_node = mesh->elems_in_node; - if (num_dim == 2) - { + if (num_dim == 2) { nodes_in_node = mesh->nodes_in_node; num_nodes_in_node = mesh->num_nodes_in_node; // patch conn @@ -1113,8 +1066,7 @@ void FEA_Module_Dynamic_Elasticity::setup() } // loop over BCs - for (size_t this_bdy = 0; this_bdy < num_bcs; this_bdy++) - { + for (size_t this_bdy = 0; this_bdy < num_bcs; this_bdy++) { RUN_CLASS({ printf("Boundary Condition number %lu \n", this_bdy); printf(" Num bdy patches in this set = %lu \n", bdy_patches_in_set.stride(this_bdy)); @@ -1124,8 +1076,7 @@ void FEA_Module_Dynamic_Elasticity::setup() } // end for // elem_mat_id needs to be initialized before initialization of material models - for (int f_id = 0; f_id < num_fills; f_id++) - { + for (int f_id = 0; f_id < num_fills; f_id++) { FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { elem_mat_id(elem_gid) = mat_fill(f_id).material_id; }); @@ -1167,18 +1118,15 @@ void FEA_Module_Dynamic_Elasticity::setup() // --- apply the fill instructions over each of the Elements---// // initialize if topology optimization is used - if (simparam->topology_optimization_on) - { - for (int elem_id = 0; elem_id < rnum_elem; elem_id++) - { + if (simparam->topology_optimization_on) { + for (int elem_id = 0; elem_id < rnum_elem; elem_id++) { relative_element_densities.host(elem_id) = 1; } // for relative_element_densities.update_device(); } // loop over the fill instructures - for (int f_id = 0; f_id < num_fills; f_id++) - { + for (int f_id = 0; f_id < num_fills; f_id++) { // parallel loop over elements in mesh FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { // calculate the coordinates and radius of the element @@ -1188,16 +1136,13 @@ void FEA_Module_Dynamic_Elasticity::setup() elem_coords[2] = 0.0; // get the coordinates of the element center - for (int node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) - { + for (int node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { elem_coords[0] += node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), 0); elem_coords[1] += node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), 1); - if (num_dim == 3) - { + if (num_dim == 3) { elem_coords[2] += node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), 2); } - else - { + else{ elem_coords[2] = 0.0; } } // end loop over nodes in element @@ -1209,8 +1154,7 @@ void FEA_Module_Dynamic_Elasticity::setup() bool fill_this = mat_fill(f_id).volume.contains(elem_coords); // paint the material state on the element - if (fill_this) - { + if (fill_this) { // density elem_den(elem_gid) = mat_fill(f_id).den; @@ -1224,10 +1168,8 @@ void FEA_Module_Dynamic_Elasticity::setup() // --- stress tensor --- // always 3D even for 2D-RZ - for (size_t i = 0; i < 3; i++) - { - for (size_t j = 0; j < 3; j++) - { + for (size_t i = 0; i < 3; i++) { + for (size_t j = 0; j < 3; j++) { elem_stress(rk_level, elem_gid, i, j) = 0.0; } } // end for @@ -1261,20 +1203,17 @@ void FEA_Module_Dynamic_Elasticity::setup() elem_sie(rk_level, elem_gid)); // loop over the nodes of this element and apply velocity - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) - { + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { // get the mesh node index size_t node_gid = nodes_in_elem(elem_gid, node_lid); // --- Velocity --- - switch (mat_fill(f_id).velocity) - { + switch (mat_fill(f_id).velocity) { case VELOCITY_TYPE::cartesian: { node_vel(rk_level, node_gid, 0) = mat_fill(f_id).u; node_vel(rk_level, node_gid, 1) = mat_fill(f_id).v; - if (num_dim == 3) - { + if (num_dim == 3) { node_vel(rk_level, node_gid, 2) = mat_fill(f_id).w; } @@ -1288,29 +1227,24 @@ void FEA_Module_Dynamic_Elasticity::setup() dir[1] = 0.0; double radius_val = 0.0; - for (int dim = 0; dim < 2; dim++) - { + for (int dim = 0; dim < 2; dim++) { dir[dim] = node_coords(rk_level, node_gid, dim); radius_val += node_coords(rk_level, node_gid, dim) * node_coords(rk_level, node_gid, dim); } // end for radius_val = sqrt(radius_val); - for (int dim = 0; dim < 2; dim++) - { - if (radius_val > 1.0e-14) - { + for (int dim = 0; dim < 2; dim++) { + if (radius_val > 1.0e-14) { dir[dim] /= (radius_val); } - else - { + else{ dir[dim] = 0.0; } } // end for node_vel(rk_level, node_gid, 0) = mat_fill(f_id).speed * dir[0]; node_vel(rk_level, node_gid, 1) = mat_fill(f_id).speed * dir[1]; - if (num_dim == 3) - { + if (num_dim == 3) { node_vel(rk_level, node_gid, 2) = 0.0; } @@ -1325,29 +1259,24 @@ void FEA_Module_Dynamic_Elasticity::setup() dir[2] = 0.0; double radius_val = 0.0; - for (int dim = 0; dim < 3; dim++) - { + for (int dim = 0; dim < 3; dim++) { dir[dim] = node_coords(rk_level, node_gid, dim); radius_val += node_coords(rk_level, node_gid, dim) * node_coords(rk_level, node_gid, dim); } // end for radius_val = sqrt(radius_val); - for (int dim = 0; dim < 3; dim++) - { - if (radius_val > 1.0e-14) - { + for (int dim = 0; dim < 3; dim++) { + if (radius_val > 1.0e-14) { dir[dim] /= (radius_val); } - else - { + else{ dir[dim] = 0.0; } } // end for node_vel(rk_level, node_gid, 0) = mat_fill(f_id).speed * dir[0]; node_vel(rk_level, node_gid, 1) = mat_fill(f_id).speed * dir[1]; - if (num_dim == 3) - { + if (num_dim == 3) { node_vel(rk_level, node_gid, 2) = mat_fill(f_id).speed * dir[2]; } @@ -1365,8 +1294,7 @@ void FEA_Module_Dynamic_Elasticity::setup() { node_vel(rk_level, node_gid, 0) = sin(PI * node_coords(rk_level, node_gid, 0)) * cos(PI * node_coords(rk_level, node_gid, 1)); node_vel(rk_level, node_gid, 1) = -1.0 * cos(PI * node_coords(rk_level, node_gid, 0)) * sin(PI * node_coords(rk_level, node_gid, 1)); - if (num_dim == 3) - { + if (num_dim == 3) { node_vel(rk_level, node_gid, 2) = 0.0; } @@ -1375,8 +1303,7 @@ void FEA_Module_Dynamic_Elasticity::setup() } // end of switch } // end loop over nodes of element - if (mat_fill(f_id).velocity == VELOCITY_TYPE::tg_vortex) - { + if (mat_fill(f_id).velocity == VELOCITY_TYPE::tg_vortex) { elem_pres(elem_gid) = 0.25 * (cos(2.0 * PI * elem_coords[0]) + cos(2.0 * PI * elem_coords[1]) ) + 1.0; // p = rho*ie*(gamma - 1) @@ -1396,8 +1323,7 @@ void FEA_Module_Dynamic_Elasticity::setup() FEA_Module_Dynamic_Elasticity::boundary_velocity(*mesh, boundary, node_vel); // calculate the corner massess if 2D - if (num_dim == 2) - { + if (num_dim == 2) { FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { // facial area of the corners double corner_areas_array[4]; @@ -1412,8 +1338,7 @@ void FEA_Module_Dynamic_Elasticity::setup() rk_level); // loop over the corners of the element and calculate the mass - for (size_t corner_lid = 0; corner_lid < 4; corner_lid++) - { + for (size_t corner_lid = 0; corner_lid < 4; corner_lid++) { size_t corner_gid = corners_in_elem(elem_gid, corner_lid); corner_mass(corner_gid) = corner_areas(corner_lid) * elem_den(elem_gid); // node radius is added later } // end for over corners @@ -1424,19 +1349,15 @@ void FEA_Module_Dynamic_Elasticity::setup() FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { node_mass(node_gid) = 0.0; - if (num_dim == 3) - { - for (size_t elem_lid = 0; elem_lid < num_corners_in_node(node_gid); elem_lid++) - { + if (num_dim == 3) { + for (size_t elem_lid = 0; elem_lid < num_corners_in_node(node_gid); elem_lid++) { size_t elem_gid = elems_in_node(node_gid, elem_lid); node_mass(node_gid) += 1.0 / 8.0 * elem_mass(elem_gid); } // end for elem_lid } // end if dims=3 - else - { + else{ // 2D-RZ - for (size_t corner_lid = 0; corner_lid < num_corners_in_node(node_gid); corner_lid++) - { + for (size_t corner_lid = 0; corner_lid < num_corners_in_node(node_gid); corner_lid++) { size_t corner_gid = corners_in_node(node_gid, corner_lid); node_mass(node_gid) += corner_mass(corner_gid); // sans the radius so it is areal node mass @@ -1448,8 +1369,7 @@ void FEA_Module_Dynamic_Elasticity::setup() // current interface has differing mass arrays; this equates them until we unify memory // view scope - if (simparam->topology_optimization_on || simparam->shape_optimization_on || simparam->num_dims == 2) - { + if (simparam->topology_optimization_on || simparam->shape_optimization_on || simparam->num_dims == 2) { { vec_array node_mass_interface = node_masses_distributed->getLocalView(Tpetra::Access::ReadWrite); @@ -1474,8 +1394,7 @@ void FEA_Module_Dynamic_Elasticity::setup() } // endif // initialize if topology optimization is used - if (simparam->topology_optimization_on || simparam->shape_optimization_on) - { + if (simparam->topology_optimization_on || simparam->shape_optimization_on) { init_assembly(); assemble_matrix(); } @@ -1578,8 +1497,7 @@ void FEA_Module_Dynamic_Elasticity::tag_bdys(const DCArrayKokkos& bo double val = boundary(bdy_set).surface.plane_position; // save the boundary patches to this set that are on the plane, spheres, etc. - for (size_t bdy_patch_lid = 0; bdy_patch_lid < nboundary_patches; bdy_patch_lid++) - { + for (size_t bdy_patch_lid = 0; bdy_patch_lid < nboundary_patches; bdy_patch_lid++) { // save the patch index size_t bdy_patch_gid = bdy_patch_lid; @@ -1591,8 +1509,7 @@ void FEA_Module_Dynamic_Elasticity::tag_bdys(const DCArrayKokkos& bo val, node_coords, rk_level); // no=0, yes=1 WARNING: POSSIBLE BUG - if (is_on_bdy) - { + if (is_on_bdy) { size_t index = bdy_patches_in_set.stride(bdy_set); // increment the number of boundary patches saved @@ -1642,56 +1559,44 @@ bool FEA_Module_Dynamic_Elasticity::check_bdy(const size_t patch_gid, double these_patch_coords[3]; // Note: cannot allocated array with num_dim // loop over the nodes on the patch - for (size_t patch_node_lid = 0; patch_node_lid < num_nodes_in_patch; patch_node_lid++) - { + for (size_t patch_node_lid = 0; patch_node_lid < num_nodes_in_patch; patch_node_lid++) { // get the nodal_gid for this node in the patch // size_t node_gid = mesh.nodes_in_patch(patch_gid, patch_node_lid); size_t node_gid = Local_Index_Boundary_Patches(patch_gid, patch_node_lid); - for (size_t dim = 0; dim < num_dim; dim++) - { + for (size_t dim = 0; dim < num_dim; dim++) { these_patch_coords[dim] = node_coords(rk_level, node_gid, dim); // (rk, node_gid, dim) } - if (bc_type == BOUNDARY_TYPE::x_plane) - { - if (fabs(these_patch_coords[0] - val) <= 1.0e-7) - { + if (bc_type == BOUNDARY_TYPE::x_plane) { + if (fabs(these_patch_coords[0] - val) <= 1.0e-7) { is_on_bdy += 1; } } - else if (bc_type == BOUNDARY_TYPE::y_plane) - { - if (fabs(these_patch_coords[1] - val) <= 1.0e-7) - { + else if (bc_type == BOUNDARY_TYPE::y_plane) { + if (fabs(these_patch_coords[1] - val) <= 1.0e-7) { is_on_bdy += 1; } } - else if (bc_type == BOUNDARY_TYPE::z_plane) - { - if (fabs(these_patch_coords[2] - val) <= 1.0e-7) - { + else if (bc_type == BOUNDARY_TYPE::z_plane) { + if (fabs(these_patch_coords[2] - val) <= 1.0e-7) { is_on_bdy += 1; } } - else if (bc_type == BOUNDARY_TYPE::cylinder) - { + else if (bc_type == BOUNDARY_TYPE::cylinder) { real_t R = sqrt(these_patch_coords[0] * these_patch_coords[0] + these_patch_coords[1] * these_patch_coords[1]); - if (fabs(R - val) <= 1.0e-7) - { + if (fabs(R - val) <= 1.0e-7) { is_on_bdy += 1; } } - else if (bc_type == BOUNDARY_TYPE::sphere) - { + else if (bc_type == BOUNDARY_TYPE::sphere) { real_t R = sqrt(these_patch_coords[0] * these_patch_coords[0] + these_patch_coords[1] * these_patch_coords[1] + these_patch_coords[2] * these_patch_coords[2]); - if (fabs(R - val) <= 1.0e-7) - { + if (fabs(R - val) <= 1.0e-7) { is_on_bdy += 1; } } @@ -1728,14 +1633,12 @@ void FEA_Module_Dynamic_Elasticity::build_boundry_node_sets(mesh_t& mesh) num_bdy_nodes_in_set(bdy_set) = 0; // Loop over boundary patches in boundary set - for (size_t bdy_patch_gid = 0; bdy_patch_gid < num_bdy_patches_in_set; bdy_patch_gid++) - { + for (size_t bdy_patch_gid = 0; bdy_patch_gid < num_bdy_patches_in_set; bdy_patch_gid++) { // get the global id for this boundary patch size_t patch_gid = bdy_patches_in_set(bdy_set, bdy_patch_gid); // apply boundary condition at nodes on boundary - for (size_t node_lid = 0; node_lid < num_nodes_in_patch; node_lid++) - { + for (size_t node_lid = 0; node_lid < num_nodes_in_patch; node_lid++) { size_t node_gid = Local_Index_Boundary_Patches(patch_gid, node_lid); temp_count_num_bdy_nodes_in_set(bdy_set, node_gid) = -1; @@ -1743,18 +1646,15 @@ void FEA_Module_Dynamic_Elasticity::build_boundry_node_sets(mesh_t& mesh) } // end for bdy_patch_gid // Loop over boundary patches in boundary set - for (size_t bdy_patch_gid = 0; bdy_patch_gid < num_bdy_patches_in_set; bdy_patch_gid++) - { + for (size_t bdy_patch_gid = 0; bdy_patch_gid < num_bdy_patches_in_set; bdy_patch_gid++) { // get the global id for this boundary patch size_t patch_gid = bdy_patches_in_set(bdy_set, bdy_patch_gid); // apply boundary condition at nodes on boundary - for (size_t node_lid = 0; node_lid < num_nodes_in_patch; node_lid++) - { + for (size_t node_lid = 0; node_lid < num_nodes_in_patch; node_lid++) { size_t node_gid = Local_Index_Boundary_Patches(patch_gid, node_lid); - if (temp_count_num_bdy_nodes_in_set(bdy_set, node_gid) == -1) - { + if (temp_count_num_bdy_nodes_in_set(bdy_set, node_gid) == -1) { size_t num_saved = num_bdy_nodes_in_set(bdy_set); num_bdy_nodes_in_set(bdy_set)++; @@ -1776,8 +1676,7 @@ void FEA_Module_Dynamic_Elasticity::build_boundry_node_sets(mesh_t& mesh) FOR_ALL_CLASS(bdy_set, 0, num_bdy_sets, { // Loop over boundary patches in boundary set - for (size_t bdy_node_lid = 0; bdy_node_lid < num_bdy_nodes_in_set(bdy_set); bdy_node_lid++) - { + for (size_t bdy_node_lid = 0; bdy_node_lid < num_bdy_nodes_in_set(bdy_set); bdy_node_lid++) { // save the bdy_node_gid bdy_nodes_in_set(bdy_set, bdy_node_lid) = temp_nodes_in_set(bdy_set, bdy_node_lid); } // end for @@ -1863,15 +1762,13 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve() } */ // simple setup to just request KE for now; above loop to be expanded and used later for scanning modules - if (simparam->topology_optimization_on) - { + if (simparam->topology_optimization_on) { obj_pointer = problem->getObjective(); KineticEnergyMinimize_TopOpt& kinetic_energy_minimize_function = dynamic_cast(*obj_pointer); kinetic_energy_minimize_function.objective_accumulation = 0; global_objective_accumulation = objective_accumulation = 0; kinetic_energy_objective = true; - if (max_time_steps + 1 > forward_solve_velocity_data->size()) - { + if (max_time_steps + 1 > forward_solve_velocity_data->size()) { old_max_forward_buffer = forward_solve_velocity_data->size(); time_data.resize(max_time_steps + 1); forward_solve_velocity_data->resize(max_time_steps + 1); @@ -1879,8 +1776,7 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve() adjoint_vector_data->resize(max_time_steps + 1); phi_adjoint_vector_data->resize(max_time_steps + 1); // assign a multivector of corresponding size to each new timestep in the buffer - for (int istep = old_max_forward_buffer; istep < max_time_steps + 1; istep++) - { + for (int istep = old_max_forward_buffer; istep < max_time_steps + 1; istep++) { (*forward_solve_velocity_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); (*forward_solve_coordinate_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); (*adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); @@ -1889,16 +1785,13 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve() } } - if (simparam->topology_optimization_on) - { + if (simparam->topology_optimization_on) { nTO_modules = simparam->TO_Module_List.size(); } int myrank = Explicit_Solver_Pointer_->myrank; - if (simparam->output_options.output_file_format == OUTPUT_FORMAT::vtk && simparam->output_options.write_initial) - { - if (myrank == 0) - { + if (simparam->output_options.output_file_format == OUTPUT_FORMAT::vtk && simparam->output_options.write_initial) { + if (myrank == 0) { printf("Writing outputs to file at %f \n", time_value); } @@ -1948,17 +1841,14 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve() // extensive KE REDUCE_SUM_CLASS(node_gid, 0, nlocal_nodes, KE_loc_sum, { double ke = 0; - for (size_t dim = 0; dim < num_dim; dim++) - { + for (size_t dim = 0; dim < num_dim; dim++) { ke += node_vel(rk_level, node_gid, dim) * node_vel(rk_level, node_gid, dim); // 1/2 at end } // end for - if (num_dim == 2) - { + if (num_dim == 2) { KE_loc_sum += node_mass(node_gid) * node_coords(rk_level, node_gid, 1) * ke; } - else - { + else{ KE_loc_sum += node_mass(node_gid) * ke; } }, KE_sum); @@ -1976,8 +1866,7 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve() // save the nodal mass FOR_ALL_CLASS(node_gid, 0, nall_nodes, { double radius = 1.0; - if (num_dim == 2) - { + if (num_dim == 2) { radius = node_coords(rk_level, node_gid, 1); } node_extensive_mass(node_gid) = node_mass(node_gid) * radius; @@ -1989,8 +1878,7 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve() auto time_1 = std::chrono::high_resolution_clock::now(); // save initial data - if (simparam->topology_optimization_on || simparam->shape_optimization_on) - { + if (simparam->topology_optimization_on || simparam->shape_optimization_on) { time_data[0] = 0; // assign current velocity data to multivector // view scope @@ -1998,8 +1886,7 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve() vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (int idim = 0; idim < num_dim; idim++) - { + for (int idim = 0; idim < num_dim; idim++) { node_velocities_interface(node_gid, idim) = node_vel(rk_level, node_gid, idim); node_coords_interface(node_gid, idim) = node_coords(rk_level, node_gid, idim); } @@ -2033,8 +1920,7 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve() const_vec_array ghost_node_coords_interface = Explicit_Solver_Pointer_->ghost_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (int idim = 0; idim < num_dim; idim++) - { + for (int idim = 0; idim < num_dim; idim++) { all_node_velocities_interface(node_gid, idim) = node_velocities_interface(node_gid, idim); all_node_coords_interface(node_gid, idim) = node_coords_interface(node_gid, idim); } @@ -2042,8 +1928,7 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve() Kokkos::fence(); FOR_ALL_CLASS(node_gid, nlocal_nodes, nlocal_nodes + nghost_nodes, { - for (int idim = 0; idim < num_dim; idim++) - { + for (int idim = 0; idim < num_dim; idim++) { all_node_velocities_interface(node_gid, idim) = ghost_node_velocities_interface(node_gid - nlocal_nodes, idim); all_node_coords_interface(node_gid, idim) = ghost_node_coords_interface(node_gid - nlocal_nodes, idim); } @@ -2056,19 +1941,16 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve() } // loop over the max number of time integration cycles - for (cycle = 0; cycle < cycle_stop; cycle++) - { + for (cycle = 0; cycle < cycle_stop; cycle++) { // get the step - if (num_dim == 2) - { + if (num_dim == 2) { get_timestep2D(*mesh, node_coords, node_vel, elem_sspd, elem_vol); } - else - { + else{ get_timestep(*mesh, node_coords, node_vel, @@ -2083,20 +1965,15 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve() // stop calculation if flag // if (stop_calc == 1) break; - if (simparam->dynamic_options.output_time_sequence_level >= TIME_OUTPUT_LEVEL::high) - { - if (cycle == 0) - { - if (myrank == 0) - { + if (simparam->dynamic_options.output_time_sequence_level >= TIME_OUTPUT_LEVEL::high) { + if (cycle == 0) { + if (myrank == 0) { printf("cycle = %lu, time = %12.5e, time step = %12.5e \n", cycle, time_value, dt); } } // print time step every 10 cycles - else if (cycle % 20 == 0) - { - if (myrank == 0) - { + else if (cycle % 20 == 0) { + if (myrank == 0) { printf("cycle = %lu, time = %12.5e, time step = %12.5e \n", cycle, time_value, dt); } } // end if @@ -2115,8 +1992,7 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve() nall_nodes); // integrate solution forward in time - for (size_t rk_stage = 0; rk_stage < rk_num_stages; rk_stage++) - { + for (size_t rk_stage = 0; rk_stage < rk_num_stages; rk_stage++) { // ---- RK coefficient ---- double rk_alpha = 1.0 / ((double)rk_num_stages - (double)rk_stage); @@ -2163,8 +2039,7 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve() { vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (int idim = 0; idim < num_dim; idim++) - { + for (int idim = 0; idim < num_dim; idim++) { node_velocities_interface(node_gid, idim) = node_vel(rk_level, node_gid, idim); } }); // end parallel for @@ -2187,8 +2062,7 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve() vec_array ghost_node_velocities_interface = Explicit_Solver_Pointer_->ghost_node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); FOR_ALL_CLASS(node_gid, nlocal_nodes, nall_nodes, { - for (int idim = 0; idim < num_dim; idim++) - { + for (int idim = 0; idim < num_dim; idim++) { node_vel(rk_level, node_gid, idim) = ghost_node_velocities_interface(node_gid - nlocal_nodes, idim); } }); // end parallel for @@ -2257,14 +2131,12 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve() // 3) strength models must be added by the user in user_mat.cpp // calculate the new corner masses if 2D - if (num_dim == 2) - { + if (num_dim == 2) { // calculate the nodal areal mass FOR_ALL_CLASS(node_gid, 0, nall_nodes, { node_mass(node_gid) = 0.0; - if (node_coords(rk_level, node_gid, 1) > tiny) - { + if (node_coords(rk_level, node_gid, 1) > tiny) { node_mass(node_gid) = node_extensive_mass(node_gid) / node_coords(rk_level, node_gid, 1); } // if(cycle==0&&node_gid==1&&myrank==0) @@ -2347,17 +2219,14 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve() // FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { size_t node_gid = bdy_nodes(node_bdy_gid); - if (node_coords(rk_level, node_gid, 1) < tiny) - { + if (node_coords(rk_level, node_gid, 1) < tiny) { // node is on the axis - for (size_t node_lid = 0; node_lid < num_nodes_in_node(node_gid); node_lid++) - { + for (size_t node_lid = 0; node_lid < num_nodes_in_node(node_gid); node_lid++) { size_t node_neighbor_gid = nodes_in_node(node_gid, node_lid); // if the node is off the axis, use it's areal mass on the boundary - if (node_coords(rk_level, node_neighbor_gid, 1) > tiny) - { + if (node_coords(rk_level, node_neighbor_gid, 1) > tiny) { node_mass(node_gid) = fmax(node_mass(node_gid), node_mass(node_neighbor_gid) / 2.0); } } // end for over neighboring nodes @@ -2369,15 +2238,12 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve() // increment the time Explicit_Solver_Pointer_->time_value = simparam->dynamic_options.time_value = time_value += dt; - if (simparam->topology_optimization_on || simparam->shape_optimization_on) - { - if (cycle >= max_time_steps) - { + if (simparam->topology_optimization_on || simparam->shape_optimization_on) { + if (cycle >= max_time_steps) { max_time_steps = cycle + 1; } - if (max_time_steps + 1 > forward_solve_velocity_data->size()) - { + if (max_time_steps + 1 > forward_solve_velocity_data->size()) { old_max_forward_buffer = forward_solve_velocity_data->size(); time_data.resize(max_time_steps + BUFFER_GROW + 1); forward_solve_velocity_data->resize(max_time_steps + BUFFER_GROW + 1); @@ -2385,8 +2251,7 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve() adjoint_vector_data->resize(max_time_steps + BUFFER_GROW + 1); phi_adjoint_vector_data->resize(max_time_steps + BUFFER_GROW + 1); // assign a multivector of corresponding size to each new timestep in the buffer - for (int istep = old_max_forward_buffer; istep < max_time_steps + BUFFER_GROW + 1; istep++) - { + for (int istep = old_max_forward_buffer; istep < max_time_steps + BUFFER_GROW + 1; istep++) { (*forward_solve_velocity_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); (*forward_solve_coordinate_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); (*adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); @@ -2402,8 +2267,7 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve() vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (int idim = 0; idim < num_dim; idim++) - { + for (int idim = 0; idim < num_dim; idim++) { node_velocities_interface(node_gid, idim) = node_vel(rk_level, node_gid, idim); node_coords_interface(node_gid, idim) = node_coords(rk_level, node_gid, idim); } @@ -2439,8 +2303,7 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve() vec_array all_node_velocities_interface = Explicit_Solver_Pointer_->all_node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (int idim = 0; idim < num_dim; idim++) - { + for (int idim = 0; idim < num_dim; idim++) { all_node_velocities_interface(node_gid, idim) = node_velocities_interface(node_gid, idim); all_node_coords_interface(node_gid, idim) = node_coords_interface(node_gid, idim); } @@ -2448,8 +2311,7 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve() Kokkos::fence(); FOR_ALL_CLASS(node_gid, nlocal_nodes, nlocal_nodes + nghost_nodes, { - for (int idim = 0; idim < num_dim; idim++) - { + for (int idim = 0; idim < num_dim; idim++) { all_node_velocities_interface(node_gid, idim) = ghost_node_velocities_interface(node_gid - nlocal_nodes, idim); all_node_coords_interface(node_gid, idim) = ghost_node_coords_interface(node_gid - nlocal_nodes, idim); } @@ -2465,8 +2327,7 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve() (*forward_solve_coordinate_data)[cycle + 1]->assign(*Explicit_Solver_Pointer_->all_node_coords_distributed); // kinetic energy accumulation - if (kinetic_energy_objective) - { + if (kinetic_energy_objective) { const_vec_array node_velocities_interface = (*forward_solve_velocity_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); const_vec_array previous_node_velocities_interface = (*forward_solve_velocity_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); KE_loc_sum = 0.0; @@ -2474,19 +2335,16 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve() // extensive KE REDUCE_SUM_CLASS(node_gid, 0, nlocal_nodes, KE_loc_sum, { double ke = 0; - for (size_t dim = 0; dim < num_dim; dim++) - { + for (size_t dim = 0; dim < num_dim; dim++) { // midpoint integration approximation ke += (node_velocities_interface(node_gid, dim) + previous_node_velocities_interface(node_gid, dim)) * (node_velocities_interface(node_gid, dim) + previous_node_velocities_interface(node_gid, dim)) / 4; // 1/2 at end } // end for - if (num_dim == 2) - { + if (num_dim == 2) { KE_loc_sum += node_mass(node_gid) * node_coords(rk_level, node_gid, 1) * ke; } - else - { + else{ KE_loc_sum += node_mass(node_gid) * ke; } }, KE_sum); @@ -2497,41 +2355,33 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve() } size_t write = 0; - if ((cycle + 1) % graphics_cyc_ival == 0 && cycle > 0) - { + if ((cycle + 1) % graphics_cyc_ival == 0 && cycle > 0) { write = 1; } - else if (cycle == cycle_stop) - { + else if (cycle == cycle_stop) { write = 1; } - else if (time_value >= time_final && simparam->output_options.write_final) - { + else if (time_value >= time_final && simparam->output_options.write_final) { write = 1; } - else if (time_value >= graphics_time) - { + else if (time_value >= graphics_time) { write = 1; } // write outputs - if (write == 1) - { + if (write == 1) { // interface nodal coordinate data (note: this is not needed if using write_outputs()) // view scope { vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (int idim = 0; idim < num_dim; idim++) - { + for (int idim = 0; idim < num_dim; idim++) { node_coords_interface(node_gid, idim) = node_coords(rk_level, node_gid, idim); } }); // end parallel for } // end view scope - if (simparam->output_options.output_file_format == OUTPUT_FORMAT::vtk) - { - if (myrank == 0) - { + if (simparam->output_options.output_file_format == OUTPUT_FORMAT::vtk) { + if (myrank == 0) { printf("Writing outputs to file at %f \n", graphics_time); } @@ -2546,8 +2396,7 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve() } // end if // end of calculation - if (time_value >= time_final) - { + if (time_value >= time_final) { break; } } // end for cycle loop @@ -2555,16 +2404,14 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve() last_time_step = cycle; // simple setup to just calculate KE minimize objective for now - if (simparam->topology_optimization_on) - { + if (simparam->topology_optimization_on) { KineticEnergyMinimize_TopOpt& kinetic_energy_minimize_function = dynamic_cast(*obj_pointer); // collect local objective values MPI_Allreduce(&objective_accumulation, &global_objective_accumulation, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); kinetic_energy_minimize_function.objective_accumulation = global_objective_accumulation; - if (myrank == 0) - { + if (myrank == 0) { std::cout << "CURRENT TIME INTEGRAL OF KINETIC ENERGY " << global_objective_accumulation << std::endl; } } @@ -2573,8 +2420,7 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve() auto time_difference = time_2 - time_1; // double calc_time = std::chrono::duration_cast(diff).count(); double calc_time = std::chrono::duration_cast(time_difference).count(); - if (myrank == 0) - { + if (myrank == 0) { printf("\nCalculation time in seconds: %f \n", calc_time * 1e-09); } @@ -2595,17 +2441,14 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve() // extensive KE REDUCE_SUM_CLASS(node_gid, 0, nlocal_nodes, KE_loc_sum, { double ke = 0; - for (size_t dim = 0; dim < num_dim; dim++) - { + for (size_t dim = 0; dim < num_dim; dim++) { ke += node_vel(rk_level, node_gid, dim) * node_vel(rk_level, node_gid, dim); // 1/2 at end } // end for - if (num_dim == 2) - { + if (num_dim == 2) { KE_loc_sum += node_mass(node_gid) * node_coords(rk_level, node_gid, 1) * ke; } - else - { + else{ KE_loc_sum += node_mass(node_gid) * ke; } }, KE_sum); @@ -2625,16 +2468,13 @@ void FEA_Module_Dynamic_Elasticity::elastic_solve() // reduce over MPI ranks - if (myrank == 0) - { + if (myrank == 0) { printf("Time=0: KE = %20.15f, IE = %20.15f, TE = %20.15f \n", KE_t0, IE_t0, TE_t0); } - if (myrank == 0) - { + if (myrank == 0) { printf("Time=End: KE = %20.15f, IE = %20.15f, TE = %20.15f \n", KE_tend, IE_tend, TE_tend); } - if (myrank == 0) - { + if (myrank == 0) { printf("total energy conservation error = %e \n\n", 100 * (TE_tend - TE_t0) / TE_t0); } diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/boundary.cpp b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/boundary.cpp index 4360f04f8..1d83a16a7 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/boundary.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/boundary.cpp @@ -61,15 +61,12 @@ void FEA_Module_Dynamic_Elasticity::boundary_velocity(const mesh_t& mesh, const size_t rk_level = rk_num_bins - 1; int num_dims = num_dim; // Loop over boundary sets - for (size_t bdy_set = 0; bdy_set < num_bdy_sets; bdy_set++) - { + for (size_t bdy_set = 0; bdy_set < num_bdy_sets; bdy_set++) { // Loop over boundary nodes in a boundary set // FOR_ALL_CLASS(bdy_node_lid, 0, num_bdy_nodes_in_set.host(bdy_set), { - for (size_t bdy_node_lid = 0; bdy_node_lid < num_bdy_nodes_in_set.host(bdy_set); bdy_node_lid++) - { + for (size_t bdy_node_lid = 0; bdy_node_lid < num_bdy_nodes_in_set.host(bdy_set); bdy_node_lid++) { // reflected (boundary array is on the device) - if (boundary(bdy_set).type == BOUNDARY_CONDITION_TYPE::reflected) - { + if (boundary(bdy_set).type == BOUNDARY_CONDITION_TYPE::reflected) { // directions with hydro_bc: // x_plane = 0, // y_plane = 1, @@ -81,27 +78,23 @@ void FEA_Module_Dynamic_Elasticity::boundary_velocity(const mesh_t& mesh, // Set velocity to zero in that directdion node_vel(rk_level, bdy_node_gid, direction) = 0.0; } - else if (boundary(bdy_set).type == BOUNDARY_CONDITION_TYPE::fixed_position) - { + else if (boundary(bdy_set).type == BOUNDARY_CONDITION_TYPE::fixed_position) { size_t bdy_node_gid = bdy_nodes_in_set(bdy_set, bdy_node_lid); // debug clause // if(bdy_node_gid==549412) print_flag(0) = true; - for (size_t dim = 0; dim < num_dims; dim++) - { + for (size_t dim = 0; dim < num_dims; dim++) { // Set velocity to zero node_vel(rk_level, bdy_node_gid, dim) = 0.0; } } - else if (boundary(bdy_set).type == BOUNDARY_CONDITION_TYPE::velocity) - { + else if (boundary(bdy_set).type == BOUNDARY_CONDITION_TYPE::velocity) { size_t bdy_node_gid = mesh.bdy_nodes_in_set(bdy_set, bdy_node_lid); node_vel(rk_level, bdy_node_gid, 0) = boundary(bdy_set).u; node_vel(rk_level, bdy_node_gid, 1) = boundary(bdy_set).v; - if (num_dims == 3) - { + if (num_dims == 3) { node_vel(rk_level, bdy_node_gid, 2) = boundary(bdy_set).w; } // if (mesh.num_dims == 3) node_vel(rk_level, bdy_node_gid, 2) = boundary(bdy_set).w * node_coords(rk_level, bdy_node_gid, 2); diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/elastic_optimization.cpp b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/elastic_optimization.cpp index 40d73a8b3..6f45205dd 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/elastic_optimization.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/elastic_optimization.cpp @@ -115,10 +115,8 @@ void FEA_Module_Dynamic_Elasticity::update_forward_solve(Teuchos::RCP const_host_vec_array all_node_densities = all_node_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); // debug print // std::cout << "NODE DENSITY TEST " << all_node_densities(0,0) << std::endl; - for (int elem_id = 0; elem_id < rnum_elem; elem_id++) - { - for (int inode = 0; inode < num_nodes_in_elem; inode++) - { + for (int elem_id = 0; elem_id < rnum_elem; elem_id++) { + for (int inode = 0; inode < num_nodes_in_elem; inode++) { current_element_nodal_densities(inode) = all_node_densities(nodes_in_elem(elem_id, inode), 0); } relative_element_densities.host(elem_id) = average_element_density(num_nodes_in_elem, current_element_nodal_densities); @@ -143,19 +141,17 @@ void FEA_Module_Dynamic_Elasticity::update_forward_solve(Teuchos::RCP const_vec_array ghost_node_coords_interface = Explicit_Solver_Pointer_->ghost_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (int idim = 0; idim < num_dim; idim++) - { + for (int idim = 0; idim < num_dim; idim++) { all_node_coords_interface(node_gid, idim) = node_coords_interface(node_gid, idim); } - }); // end parallel for + }); // end parallel for Kokkos::fence(); FOR_ALL_CLASS(node_gid, nlocal_nodes, nlocal_nodes + nghost_nodes, { - for (int idim = 0; idim < num_dim; idim++) - { + for (int idim = 0; idim < num_dim; idim++) { all_node_coords_interface(node_gid, idim) = ghost_node_coords_interface(node_gid - nlocal_nodes, idim); } - }); // end parallel for + }); // end parallel for Kokkos::fence(); } // end view scope @@ -183,14 +179,11 @@ void FEA_Module_Dynamic_Elasticity::update_forward_solve(Teuchos::RCP // view scope { host_vec_array interface_node_coords = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); - for (size_t ibin = 0; ibin < rk_num_bins; ibin++) - { + for (size_t ibin = 0; ibin < rk_num_bins; ibin++) { // save node data to node.coords // std::cout << "NODE DATA ON RANK " << myrank << std::endl; - if (num_dim == 2) - { - for (int inode = 0; inode < nall_nodes; inode++) - { + if (num_dim == 2) { + for (int inode = 0; inode < nall_nodes; inode++) { // std::cout << "Node index " << inode+1 << " "; node_coords.host(ibin, inode, 0) = interface_node_coords(inode, 0); // std::cout << host_node_coords_state(0,inode,0)+1<< " "; @@ -198,10 +191,8 @@ void FEA_Module_Dynamic_Elasticity::update_forward_solve(Teuchos::RCP // std::cout << host_node_coords_state(0,inode,1)+1<< " "; } } - else if (num_dim == 3) - { - for (int inode = 0; inode < nall_nodes; inode++) - { + else if (num_dim == 3) { + for (int inode = 0; inode < nall_nodes; inode++) { // std::cout << "Node index " << inode+1 << " "; node_coords.host(ibin, inode, 0) = interface_node_coords(inode, 0); // std::cout << host_node_coords_state(0,inode,0)+1<< " "; @@ -216,12 +207,9 @@ void FEA_Module_Dynamic_Elasticity::update_forward_solve(Teuchos::RCP } // end view scope // save the node coords to the current RK value - for (size_t node_gid = 0; node_gid < nall_nodes; node_gid++) - { - for (int rk = 1; rk < rk_num_bins; rk++) - { - for (int dim = 0; dim < num_dim; dim++) - { + for (size_t node_gid = 0; node_gid < nall_nodes; node_gid++) { + for (int rk = 1; rk < rk_num_bins; rk++) { + for (int dim = 0; dim < num_dim; dim++) { node_coords.host(rk, node_gid, dim) = node_coords.host(0, node_gid, dim); } // end for dim } // end for rk @@ -236,8 +224,7 @@ void FEA_Module_Dynamic_Elasticity::update_forward_solve(Teuchos::RCP // loop over the fill instructures // view scope { - for (int f_id = 0; f_id < num_fills; f_id++) - { + for (int f_id = 0; f_id < num_fills; f_id++) { // parallel loop over elements in mesh FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { // calculate the coordinates and radius of the element @@ -247,16 +234,13 @@ void FEA_Module_Dynamic_Elasticity::update_forward_solve(Teuchos::RCP elem_coords[2] = 0.0; // get the coordinates of the element center - for (int node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) - { + for (int node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { elem_coords[0] += node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), 0); elem_coords[1] += node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), 1); - if (num_dim == 3) - { + if (num_dim == 3) { elem_coords[2] += node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), 2); } - else - { + else{ elem_coords[2] = 0.0; } } // end loop over nodes in element @@ -268,8 +252,7 @@ void FEA_Module_Dynamic_Elasticity::update_forward_solve(Teuchos::RCP bool fill_this = mat_fill(f_id).volume.contains(elem_coords); // paint the material state on the element - if (fill_this) - { + if (fill_this) { // density elem_den(elem_gid) = mat_fill(f_id).den; @@ -287,10 +270,8 @@ void FEA_Module_Dynamic_Elasticity::update_forward_solve(Teuchos::RCP // --- stress tensor --- // always 3D even for 2D-RZ - for (size_t i = 0; i < 3; i++) - { - for (size_t j = 0; j < 3; j++) - { + for (size_t i = 0; i < 3; i++) { + for (size_t j = 0; j < 3; j++) { elem_stress(rk_level, elem_gid, i, j) = 0.0; } } // end for @@ -324,20 +305,17 @@ void FEA_Module_Dynamic_Elasticity::update_forward_solve(Teuchos::RCP elem_sie(rk_level, elem_gid)); // loop over the nodes of this element and apply velocity - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) - { + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { // get the mesh node index size_t node_gid = nodes_in_elem(elem_gid, node_lid); // --- Velocity --- - switch (mat_fill(f_id).velocity) - { + switch (mat_fill(f_id).velocity) { case VELOCITY_TYPE::cartesian: { node_vel(rk_level, node_gid, 0) = mat_fill(f_id).u; node_vel(rk_level, node_gid, 1) = mat_fill(f_id).v; - if (num_dim == 3) - { + if (num_dim == 3) { node_vel(rk_level, node_gid, 2) = mat_fill(f_id).w; } @@ -351,29 +329,24 @@ void FEA_Module_Dynamic_Elasticity::update_forward_solve(Teuchos::RCP dir[1] = 0.0; double radius_val = 0.0; - for (int dim = 0; dim < 2; dim++) - { + for (int dim = 0; dim < 2; dim++) { dir[dim] = node_coords(rk_level, node_gid, dim); radius_val += node_coords(rk_level, node_gid, dim) * node_coords(rk_level, node_gid, dim); } // end for radius_val = sqrt(radius_val); - for (int dim = 0; dim < 2; dim++) - { - if (radius_val > 1.0e-14) - { + for (int dim = 0; dim < 2; dim++) { + if (radius_val > 1.0e-14) { dir[dim] /= (radius_val); } - else - { + else{ dir[dim] = 0.0; } } // end for node_vel(rk_level, node_gid, 0) = mat_fill(f_id).speed * dir[0]; node_vel(rk_level, node_gid, 1) = mat_fill(f_id).speed * dir[1]; - if (num_dim == 3) - { + if (num_dim == 3) { node_vel(rk_level, node_gid, 2) = 0.0; } @@ -388,29 +361,24 @@ void FEA_Module_Dynamic_Elasticity::update_forward_solve(Teuchos::RCP dir[2] = 0.0; double radius_val = 0.0; - for (int dim = 0; dim < 3; dim++) - { + for (int dim = 0; dim < 3; dim++) { dir[dim] = node_coords(rk_level, node_gid, dim); radius_val += node_coords(rk_level, node_gid, dim) * node_coords(rk_level, node_gid, dim); } // end for radius_val = sqrt(radius_val); - for (int dim = 0; dim < 3; dim++) - { - if (radius_val > 1.0e-14) - { + for (int dim = 0; dim < 3; dim++) { + if (radius_val > 1.0e-14) { dir[dim] /= (radius_val); } - else - { + else{ dir[dim] = 0.0; } } // end for node_vel(rk_level, node_gid, 0) = mat_fill(f_id).speed * dir[0]; node_vel(rk_level, node_gid, 1) = mat_fill(f_id).speed * dir[1]; - if (num_dim == 3) - { + if (num_dim == 3) { node_vel(rk_level, node_gid, 2) = mat_fill(f_id).speed * dir[2]; } @@ -428,8 +396,7 @@ void FEA_Module_Dynamic_Elasticity::update_forward_solve(Teuchos::RCP { node_vel(rk_level, node_gid, 0) = sin(PI * node_coords(rk_level, node_gid, 0)) * cos(PI * node_coords(rk_level, node_gid, 1)); node_vel(rk_level, node_gid, 1) = -1.0 * cos(PI * node_coords(rk_level, node_gid, 0)) * sin(PI * node_coords(rk_level, node_gid, 1)); - if (num_dim == 3) - { + if (num_dim == 3) { node_vel(rk_level, node_gid, 2) = 0.0; } @@ -438,8 +405,7 @@ void FEA_Module_Dynamic_Elasticity::update_forward_solve(Teuchos::RCP } // end of switch } // end loop over nodes of element - if (mat_fill(f_id).velocity == VELOCITY_TYPE::tg_vortex) - { + if (mat_fill(f_id).velocity == VELOCITY_TYPE::tg_vortex) { elem_pres(elem_gid) = 0.25 * (cos(2.0 * PI * elem_coords[0]) + cos(2.0 * PI * elem_coords[1]) ) + 1.0; // p = rho*ie*(gamma - 1) @@ -449,7 +415,7 @@ void FEA_Module_Dynamic_Elasticity::update_forward_solve(Teuchos::RCP elem_pres(elem_gid) / (mat_fill(f_id).den * (gamma - 1.0)); } // end if } // end if fill - }); // end FOR_ALL_CLASS element loop + }); // end FOR_ALL_CLASS element loop Kokkos::fence(); } // end for loop over fills } // end view scope @@ -458,8 +424,7 @@ void FEA_Module_Dynamic_Elasticity::update_forward_solve(Teuchos::RCP FEA_Module_Dynamic_Elasticity::boundary_velocity(*mesh, boundary, node_vel); // calculate the corner massess if 2D - if (num_dim == 2) - { + if (num_dim == 2) { FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { // facial area of the corners double corner_areas_array[4]; @@ -474,8 +439,7 @@ void FEA_Module_Dynamic_Elasticity::update_forward_solve(Teuchos::RCP rk_level); // loop over the corners of the element and calculate the mass - for (size_t corner_lid = 0; corner_lid < 4; corner_lid++) - { + for (size_t corner_lid = 0; corner_lid < 4; corner_lid++) { size_t corner_gid = corners_in_elem(elem_gid, corner_lid); corner_mass(corner_gid) = corner_areas(corner_lid) * elem_den(elem_gid); // node radius is added later } // end for over corners @@ -486,19 +450,15 @@ void FEA_Module_Dynamic_Elasticity::update_forward_solve(Teuchos::RCP FOR_ALL_CLASS(node_gid, 0, nall_nodes, { node_mass(node_gid) = 0.0; - if (num_dim == 3) - { - for (size_t elem_lid = 0; elem_lid < num_corners_in_node(node_gid); elem_lid++) - { + if (num_dim == 3) { + for (size_t elem_lid = 0; elem_lid < num_corners_in_node(node_gid); elem_lid++) { size_t elem_gid = elems_in_node(node_gid, elem_lid); node_mass(node_gid) += 1.0 / 8.0 * elem_mass(elem_gid); } // end for elem_lid } // end if dims=3 - else - { + else{ // 2D-RZ - for (size_t corner_lid = 0; corner_lid < num_corners_in_node(node_gid); corner_lid++) - { + for (size_t corner_lid = 0; corner_lid < num_corners_in_node(node_gid); corner_lid++) { size_t corner_gid = corners_in_node(node_gid, corner_lid); node_mass(node_gid) += corner_mass(corner_gid); // sans the radius so it is areal node mass @@ -532,8 +492,7 @@ void FEA_Module_Dynamic_Elasticity::update_forward_solve(Teuchos::RCP Kokkos::fence(); // update stiffness matrix - if (simparam->topology_optimization_on || simparam->shape_optimization_on) - { + if (simparam->topology_optimization_on || simparam->shape_optimization_on) { assemble_matrix(); } @@ -556,8 +515,7 @@ void FEA_Module_Dynamic_Elasticity::update_forward_solve(Teuchos::RCP double FEA_Module_Dynamic_Elasticity::average_element_density(const int nodes_per_elem, const CArray current_element_densities) const { double result = 0; - for (int i = 0; i < nodes_per_elem; i++) - { + for (int i = 0; i < nodes_per_elem; i++) { result += current_element_densities(i) / nodes_per_elem; } @@ -586,31 +544,24 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_adjoint() // solve terminal value problem, proceeds in time backward. For simplicity, we use the same timestep data from the forward solve. // A linear interpolant is assumed between velocity data points; velocity midpoint is used to update the adjoint. - if (myrank == 0) - { + if (myrank == 0) { std::cout << "Computing adjoint vector " << time_data.size() << std::endl; } - for (int cycle = last_time_step; cycle >= 0; cycle--) - { + for (int cycle = last_time_step; cycle >= 0; cycle--) { // compute timestep from time data global_dt = time_data[cycle + 1] - time_data[cycle]; // print - if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) - { - if (cycle == last_time_step) - { - if (myrank == 0) - { + if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) { + if (cycle == last_time_step) { + if (myrank == 0) { printf("cycle = %d, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); } } // print time step every 10 cycles - else if (cycle % 20 == 0) - { - if (myrank == 0) - { + else if (cycle % 20 == 0) { + if (myrank == 0) { printf("cycle = %d, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); } } // end if @@ -630,8 +581,7 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_adjoint() vec_array current_adjoint_vector = (*adjoint_vector_data)[cycle]->getLocalView(Tpetra::Access::ReadWrite); FOR_ALL_CLASS(node_gid, 0, nlocal_nodes + nghost_nodes, { - for (int idim = 0; idim < num_dim; idim++) - { + for (int idim = 0; idim < num_dim; idim++) { // cancellation of half from midpoint and 2 from adjoint equation already done current_adjoint_vector(node_gid, idim) = -0.5 * (current_velocity_vector(node_gid, idim) + previous_velocity_vector(node_gid, idim)) * global_dt + previous_adjoint_vector(node_gid, idim); @@ -667,31 +617,24 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_adjoint_full() // solve terminal value problem, proceeds in time backward. For simplicity, we use the same timestep data from the forward solve. // A linear interpolant is assumed between velocity data points; velocity midpoint is used to update the adjoint. - if (myrank == 0) - { + if (myrank == 0) { std::cout << "Computing adjoint vector " << time_data.size() << std::endl; } - for (int cycle = last_time_step; cycle >= 0; cycle--) - { + for (int cycle = last_time_step; cycle >= 0; cycle--) { // compute timestep from time data global_dt = time_data[cycle + 1] - time_data[cycle]; // print - if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) - { - if (cycle == last_time_step) - { - if (myrank == 0) - { + if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) { + if (cycle == last_time_step) { + if (myrank == 0) { printf("cycle = %d, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); } } // print time step every 10 cycles - else if (cycle % 20 == 0) - { - if (myrank == 0) - { + else if (cycle % 20 == 0) { + if (myrank == 0) { printf("cycle = %d, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); } } // end if @@ -709,8 +652,7 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_adjoint_full() // interface of arrays for current implementation of force calculation FOR_ALL_CLASS(node_gid, 0, nlocal_nodes + nghost_nodes, { - for (int idim = 0; idim < num_dim; idim++) - { + for (int idim = 0; idim < num_dim; idim++) { node_vel(rk_level, node_gid, idim) = previous_velocity_vector(node_gid, idim); node_coords(rk_level, node_gid, idim) = previous_coordinate_vector(node_gid, idim); } @@ -749,8 +691,7 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_adjoint_full() real_t rate_of_change; real_t matrix_contribution; size_t dof_id; - for (int idim = 0; idim < num_dim; idim++) - { + for (int idim = 0; idim < num_dim; idim++) { matrix_contribution = 0; // compute resulting row of force velocity gradient matrix transpose right multiplied by adjoint vector /* @@ -766,8 +707,7 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_adjoint_full() midpoint_adjoint_vector(node_gid, idim) = -rate_of_change * global_dt / 2 + previous_adjoint_vector(node_gid, idim); matrix_contribution = 0; // compute resulting row of force displacement gradient matrix transpose right multiplied by adjoint vector - for (int idof = 0; idof < Gradient_Matrix_Strides(node_gid * num_dim + idim); idof++) - { + for (int idof = 0; idof < Gradient_Matrix_Strides(node_gid * num_dim + idim); idof++) { dof_id = DOF_Graph_Matrix(node_gid * num_dim + idim, idof); matrix_contribution += -previous_adjoint_vector(dof_id / num_dim, dof_id % num_dim) * Force_Gradient_Positions(node_gid * num_dim + idim, idof); } @@ -791,8 +731,7 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_adjoint_full() real_t rate_of_change; real_t matrix_contribution; size_t dof_id; - for (int idim = 0; idim < num_dim; idim++) - { + for (int idim = 0; idim < num_dim; idim++) { matrix_contribution = 0; // compute resulting row of force velocity gradient matrix transpose right multiplied by adjoint vector /* @@ -808,8 +747,7 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_adjoint_full() current_adjoint_vector(node_gid, idim) = -rate_of_change * global_dt + previous_adjoint_vector(node_gid, idim); matrix_contribution = 0; // compute resulting row of force displacement gradient matrix transpose right multiplied by adjoint vector - for (int idof = 0; idof < Gradient_Matrix_Strides(node_gid * num_dim + idim); idof++) - { + for (int idof = 0; idof < Gradient_Matrix_Strides(node_gid * num_dim + idim); idof++) { dof_id = DOF_Graph_Matrix(node_gid * num_dim + idim, idof); matrix_contribution += -midpoint_adjoint_vector(dof_id / num_dim, dof_id % num_dim) * Force_Gradient_Positions(node_gid * num_dim + idim, idof); } @@ -850,8 +788,7 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient(const CArrayKokkos current_element_velocities = CArrayKokkos(num_nodes_in_elem, num_dim); CArrayKokkos current_element_adjoint = CArrayKokkos(num_nodes_in_elem, num_dim); - if (myrank == 0) - { + if (myrank == 0) { std::cout << "Computing accumulated kinetic energy gradient" << std::endl; } @@ -864,26 +801,20 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient(const Kokkos::fence(); // gradient contribution from kinetic energy vMv product. - for (unsigned long cycle = 0; cycle < last_time_step + 1; cycle++) - { + for (unsigned long cycle = 0; cycle < last_time_step + 1; cycle++) { // compute timestep from time data global_dt = time_data[cycle + 1] - time_data[cycle]; // print - if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) - { - if (cycle == 0) - { - if (myrank == 0) - { + if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) { + if (cycle == 0) { + if (myrank == 0) { printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); } } // print time step every 10 cycles - else if (cycle % 20 == 0) - { - if (myrank == 0) - { + else if (cycle % 20 == 0) { + if (myrank == 0) { printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); } } // end if @@ -903,30 +834,25 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient(const real_t inner_product; // std::cout << elem_mass(elem_id) <dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) - { - if (cycle == 0) - { - if (myrank == 0) - { + if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) { + if (cycle == 0) { + if (myrank == 0) { printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); } } // print time step every 10 cycles - else if (cycle % 20 == 0) - { - if (myrank == 0) - { + else if (cycle % 20 == 0) { + if (myrank == 0) { printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); } } // end if @@ -1029,8 +948,7 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient(const real_t inner_product; // std::cout << elem_mass(elem_id) < current_element_velocities = CArrayKokkos(num_nodes_in_elem, num_dim); CArrayKokkos current_element_adjoint = CArrayKokkos(num_nodes_in_elem, num_dim); - if (myrank == 0) - { + if (myrank == 0) { std::cout << "Computing accumulated kinetic energy gradient" << std::endl; } @@ -1122,34 +1034,26 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient_full( Kokkos::fence(); // gradient contribution from kinetic energy v(dM/drho)v product. - if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) - { - if (myrank == 0) - { + if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) { + if (myrank == 0) { std::cout << "v*dM/drho*v term" << std::endl; } } - for (unsigned long cycle = 0; cycle < last_time_step + 1; cycle++) - { + for (unsigned long cycle = 0; cycle < last_time_step + 1; cycle++) { // compute timestep from time data global_dt = time_data[cycle + 1] - time_data[cycle]; // print - if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) - { - if (cycle == 0) - { - if (myrank == 0) - { + if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) { + if (cycle == 0) { + if (myrank == 0) { printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); } } // print time step every 10 cycles - else if (cycle % 20 == 0) - { - if (myrank == 0) - { + else if (cycle % 20 == 0) { + if (myrank == 0) { printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); } } // end if @@ -1169,30 +1073,25 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient_full( real_t inner_product; // std::cout << elem_mass(elem_id) <dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) - { - if (myrank == 0) - { + if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) { + if (myrank == 0) { std::cout << "gradient term involving adjoint derivative" << std::endl; } } - for (unsigned long cycle = 0; cycle < last_time_step + 1; cycle++) - { + for (unsigned long cycle = 0; cycle < last_time_step + 1; cycle++) { // compute timestep from time data global_dt = time_data[cycle + 1] - time_data[cycle]; // print - if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) - { - if (cycle == 0) - { - if (myrank == 0) - { + if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) { + if (cycle == 0) { + if (myrank == 0) { printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); } } // print time step every 10 cycles - else if (cycle % 20 == 0) - { - if (myrank == 0) - { + else if (cycle % 20 == 0) { + if (myrank == 0) { printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); } } // end if @@ -1274,24 +1164,20 @@ void FEA_Module_Dynamic_Elasticity::compute_topology_optimization_gradient_full( real_t inner_product; // std::cout << elem_mass(elem_id) <choose_2Delem_type(Element_Types(ielem), elem2D); nodes_per_element = elem2D->num_nodes(); - for (int lnode = 0; lnode < nodes_per_element; lnode++) - { + for (int lnode = 0; lnode < nodes_per_element; lnode++) { local_node_index = nodes_in_elem(ielem, lnode); - if (local_node_index < nlocal_nodes) - { + if (local_node_index < nlocal_nodes) { Dual_Graph_Matrix_Strides_initial.host(local_node_index) += nodes_per_element; } } } } - if (num_dim == 3) - { - for (int ielem = 0; ielem < rnum_elem; ielem++) - { + if (num_dim == 3) { + for (int ielem = 0; ielem < rnum_elem; ielem++) { element_select->choose_3Delem_type(Element_Types(ielem), elem); nodes_per_element = elem->num_nodes(); - for (int lnode = 0; lnode < nodes_per_element; lnode++) - { + for (int lnode = 0; lnode < nodes_per_element; lnode++) { local_node_index = nodes_in_elem(ielem, lnode); - if (local_node_index < nlocal_nodes) - { + if (local_node_index < nlocal_nodes) { Dual_Graph_Matrix_Strides_initial.host(local_node_index) += nodes_per_element; } } @@ -1493,8 +1363,7 @@ void FEA_Module_Dynamic_Elasticity::init_assembly() // compute maximum stride size_t update = 0; REDUCE_MAX_CLASS(inode, 0, nlocal_nodes, update, { - if (update < Graph_Matrix_Strides_initial(inode)) - { + if (update < Graph_Matrix_Strides_initial(inode)) { update = Graph_Matrix_Strides_initial(inode); } }, max_stride); @@ -1508,19 +1377,14 @@ void FEA_Module_Dynamic_Elasticity::init_assembly() RaggedRightArrayofVectorsKokkos Element_local_indices(Graph_Matrix_Strides_initial, num_dim); // Fill the initial Graph with repeats - if (num_dim == 2) - { - for (int ielem = 0; ielem < rnum_elem; ielem++) - { + if (num_dim == 2) { + for (int ielem = 0; ielem < rnum_elem; ielem++) { element_select->choose_2Delem_type(Element_Types(ielem), elem2D); nodes_per_element = elem2D->num_nodes(); - for (int lnode = 0; lnode < nodes_per_element; lnode++) - { + for (int lnode = 0; lnode < nodes_per_element; lnode++) { local_node_index = nodes_in_elem(ielem, lnode); - if (local_node_index < nlocal_nodes) - { - for (int jnode = 0; jnode < nodes_per_element; jnode++) - { + if (local_node_index < nlocal_nodes) { + for (int jnode = 0; jnode < nodes_per_element; jnode++) { current_column_index = Graph_Fill(local_node_index) + jnode; Repeat_Graph_Matrix(local_node_index, current_column_index) = nodes_in_elem(ielem, jnode); @@ -1538,19 +1402,14 @@ void FEA_Module_Dynamic_Elasticity::init_assembly() } } - if (num_dim == 3) - { - for (int ielem = 0; ielem < rnum_elem; ielem++) - { + if (num_dim == 3) { + for (int ielem = 0; ielem < rnum_elem; ielem++) { element_select->choose_3Delem_type(Element_Types(ielem), elem); nodes_per_element = elem->num_nodes(); - for (int lnode = 0; lnode < nodes_per_element; lnode++) - { + for (int lnode = 0; lnode < nodes_per_element; lnode++) { local_node_index = nodes_in_elem(ielem, lnode); - if (local_node_index < nlocal_nodes) - { - for (int jnode = 0; jnode < nodes_per_element; jnode++) - { + if (local_node_index < nlocal_nodes) { + for (int jnode = 0; jnode < nodes_per_element; jnode++) { current_column_index = Graph_Fill(local_node_index) + jnode; Repeat_Graph_Matrix(local_node_index, current_column_index) = nodes_in_elem(ielem, jnode); @@ -1582,19 +1441,16 @@ void FEA_Module_Dynamic_Elasticity::init_assembly() int element_column_index; int current_stride; int current_row_n_nodes_scanned; - for (int inode = 0; inode < nlocal_nodes; inode++) - { + for (int inode = 0; inode < nlocal_nodes; inode++) { current_row_n_nodes_scanned = 0; - for (int istride = 0; istride < Graph_Matrix_Strides(inode); istride++) - { + for (int istride = 0; istride < Graph_Matrix_Strides(inode); istride++) { // convert global index in graph to its local index for the flagging array current_node = Repeat_Graph_Matrix(inode, istride); // debug // if(current_node==-1) // std::cout << "Graph Matrix node access on task " << myrank << std::endl; // std::cout << Repeat_Graph_Matrix(inode,istride) << std::endl; - if (node_indices_used(current_node)) - { + if (node_indices_used(current_node)) { // set global assembly map index to the location in the graph matrix where this global node was first found current_element_index = Element_local_indices(inode, istride, 0); element_row_index = Element_local_indices(inode, istride, 1); @@ -1606,8 +1462,7 @@ void FEA_Module_Dynamic_Elasticity::init_assembly() // first swap information about the inverse and forward maps current_stride = Graph_Matrix_Strides(inode); - if (istride != current_stride - 1) - { + if (istride != current_stride - 1) { Element_local_indices(inode, istride, 0) = Element_local_indices(inode, current_stride - 1, 0); Element_local_indices(inode, istride, 1) = Element_local_indices(inode, current_stride - 1, 1); Element_local_indices(inode, istride, 2) = Element_local_indices(inode, current_stride - 1, 2); @@ -1625,8 +1480,7 @@ void FEA_Module_Dynamic_Elasticity::init_assembly() istride--; Graph_Matrix_Strides(inode)--; } - else - { + else{ /*this node hasn't shown up in the row before; add it to the list of nodes that have been scanned uniquely. Use this list to reset the flag array afterwards without having to loop over all the nodes in the system*/ @@ -1637,8 +1491,7 @@ void FEA_Module_Dynamic_Elasticity::init_assembly() } } // reset nodes used list for the next row of the sparse list - for (int node_reset = 0; node_reset < current_row_n_nodes_scanned; node_reset++) - { + for (int node_reset = 0; node_reset < current_row_n_nodes_scanned; node_reset++) { node_indices_used(current_row_nodes_scanned(node_reset)) = 0; } } @@ -1650,8 +1503,7 @@ void FEA_Module_Dynamic_Elasticity::init_assembly() Graph_Matrix = RaggedRightArrayKokkos(Graph_Matrix_Strides); FOR_ALL_CLASS(inode, 0, nlocal_nodes, { - for (int istride = 0; istride < Graph_Matrix_Strides(inode); istride++) - { + for (int istride = 0; istride < Graph_Matrix_Strides(inode); istride++) { Graph_Matrix(inode, istride) = Repeat_Graph_Matrix(inode, istride); } }); // end parallel for @@ -1675,8 +1527,7 @@ void FEA_Module_Dynamic_Elasticity::init_assembly() // debug print // std::cout << "DOF GRAPH MATRIX ENTRIES ON TASK " << myrank << std::endl; FOR_ALL_CLASS(idof, 0, num_dim * nlocal_nodes, { - for (int istride = 0; istride < Gradient_Matrix_Strides(idof); istride++) - { + for (int istride = 0; istride < Gradient_Matrix_Strides(idof); istride++) { DOF_Graph_Matrix(idof, istride) = Graph_Matrix(idof / num_dim, istride / num_dim) * num_dim + istride % num_dim; } }); // end parallel for @@ -1755,13 +1606,11 @@ void FEA_Module_Dynamic_Elasticity::boundary_adjoint(const mesh_t& mesh, const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; int num_dims = simparam->num_dims; // Loop over boundary sets - for (size_t bdy_set = 0; bdy_set < num_bdy_sets; bdy_set++) - { + for (size_t bdy_set = 0; bdy_set < num_bdy_sets; bdy_set++) { // Loop over boundary nodes in a boundary set FOR_ALL_CLASS(bdy_node_lid, 0, num_bdy_nodes_in_set.host(bdy_set), { // reflected (boundary array is on the device) - if (boundary(bdy_set).type == BOUNDARY_CONDITION_TYPE::reflected) - { + if (boundary(bdy_set).type == BOUNDARY_CONDITION_TYPE::reflected) { // directions with hydro_bc: // x_plane = 0, // y_plane = 1, @@ -1771,24 +1620,20 @@ void FEA_Module_Dynamic_Elasticity::boundary_adjoint(const mesh_t& mesh, size_t bdy_node_gid = bdy_nodes_in_set(bdy_set, bdy_node_lid); // Set velocity to zero in that directdion - if (bdy_node_gid < nlocal_nodes) - { + if (bdy_node_gid < nlocal_nodes) { node_adjoint(bdy_node_gid, direction) = 0.0; } // node_phi_adjoint(bdy_node_gid, direction) = 0.0; } - else if (boundary(bdy_set).type == BOUNDARY_CONDITION_TYPE::fixed_position) - { + else if (boundary(bdy_set).type == BOUNDARY_CONDITION_TYPE::fixed_position) { size_t bdy_node_gid = bdy_nodes_in_set(bdy_set, bdy_node_lid); // debug clause // if(bdy_node_gid==549412) print_flag(0) = true; - for (size_t dim = 0; dim < num_dims; dim++) - { + for (size_t dim = 0; dim < num_dims; dim++) { // Set velocity to zero - if (bdy_node_gid < nlocal_nodes) - { + if (bdy_node_gid < nlocal_nodes) { node_adjoint(bdy_node_gid, dim) = 0.0; } // node_phi_adjoint(bdy_node_gid, dim) = 0.0; diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/force_elastic.cpp b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/force_elastic.cpp index 4757098c3..81602bf96 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/force_elastic.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/force_elastic.cpp @@ -83,30 +83,25 @@ void FEA_Module_Dynamic_Elasticity::get_force_elastic( FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { size_t dof_id; double node_force[3]; - for (size_t dim = 0; dim < num_dim; dim++) - { + for (size_t dim = 0; dim < num_dim; dim++) { node_force[dim] = 0.0; } // end for dim // loop over all corners around the node and calculate the nodal force - for (size_t corner_lid = 0; corner_lid < num_corners_in_node(node_gid); corner_lid++) - { + for (size_t corner_lid = 0; corner_lid < num_corners_in_node(node_gid); corner_lid++) { // Get corner gid size_t corner_gid = corners_in_node(node_gid, corner_lid); // loop over dimension - for (size_t dim = 0; dim < num_dim; dim++) - { + for (size_t dim = 0; dim < num_dim; dim++) { node_force[dim] += -damping_constant * node_vel(rk_level, node_gid, dim); } // end for dim } // end for corner_lid // loop over dimension - for (size_t idim = 0; idim < num_dim; idim++) - { - for (int idof = 0; idof < Stiffness_Matrix_Strides(node_gid * num_dim + idim % num_dim); idof++) - { + for (size_t idim = 0; idim < num_dim; idim++) { + for (int idof = 0; idof < Stiffness_Matrix_Strides(node_gid * num_dim + idim % num_dim); idof++) { dof_id = DOF_Graph_Matrix(node_gid * num_dim + idim % num_dim, idof); node_force[idim] += -(node_coords(rk_level, dof_id / num_dim, dof_id % num_dim) - all_initial_node_coords(dof_id / num_dim, dof_id % num_dim)) * Stiffness_Matrix(node_gid * num_dim + idim, idof); @@ -116,8 +111,7 @@ void FEA_Module_Dynamic_Elasticity::get_force_elastic( } // end for dim // update the velocity - for (int dim = 0; dim < num_dim; dim++) - { + for (int dim = 0; dim < num_dim; dim++) { node_vel(rk_level, node_gid, dim) = node_vel(0, node_gid, dim) + rk_alpha * dt * node_force[dim] / node_mass(node_gid); } // end for dim @@ -175,15 +169,13 @@ void FEA_Module_Dynamic_Elasticity::applied_forces( // walk over the nodes to update the velocity // FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (size_t node_gid = 0; node_gid <= nlocal_nodes; node_gid++) - { + for (size_t node_gid = 0; node_gid <= nlocal_nodes; node_gid++) { double current_node_coords[3]; size_t dof_id; double node_force[3]; double applied_force[3]; double radius; - for (size_t dim = 0; dim < num_dim; dim++) - { + for (size_t dim = 0; dim < num_dim; dim++) { node_force[dim] = 0.0; current_node_coords[dim] = all_initial_node_coords(node_gid, dim); } // end for dim @@ -191,25 +183,21 @@ void FEA_Module_Dynamic_Elasticity::applied_forces( + current_node_coords[1] * current_node_coords[1] + current_node_coords[2] * current_node_coords[2]); - for (size_t ilc = 0; ilc < num_lcs; ilc++) - { + for (size_t ilc = 0; ilc < num_lcs; ilc++) { // debug check // std::cout << "LOADING CONDITION VOLUME TYPE: " << to_string(loading(ilc).volume) << std::endl; bool fill_this = loading(ilc).contains(current_node_coords); - if (fill_this) - { + if (fill_this) { // loop over all corners around the node and calculate the nodal force - for (size_t corner_lid = 0; corner_lid < num_corners_in_node(node_gid); corner_lid++) - { + for (size_t corner_lid = 0; corner_lid < num_corners_in_node(node_gid); corner_lid++) { // Get corner gid size_t corner_gid = corners_in_node(node_gid, corner_lid); applied_force[0] = loading(ilc).x; applied_force[1] = loading(ilc).y; applied_force[2] = loading(ilc).z; // loop over dimension - for (size_t dim = 0; dim < num_dim; dim++) - { + for (size_t dim = 0; dim < num_dim; dim++) { node_force[dim] += applied_force[dim] * (all_initial_node_coords(node_gid, 0) + all_initial_node_coords(node_gid, 1) + all_initial_node_coords(node_gid, 2)) / radius; @@ -217,8 +205,7 @@ void FEA_Module_Dynamic_Elasticity::applied_forces( } // end for corner_lid // update the velocity - for (int dim = 0; dim < num_dim; dim++) - { + for (int dim = 0; dim < num_dim; dim++) { node_vel(rk_level, node_gid, dim) += rk_alpha * dt * node_force[dim] / node_mass(node_gid); } // end for dim @@ -250,10 +237,8 @@ void FEA_Module_Dynamic_Elasticity::assemble_matrix() // initialize stiffness Matrix entries to 0 // debug print // std::cout << "DOF GRAPH MATRIX ENTRIES ON TASK " << myrank << std::endl; - for (int idof = 0; idof < num_dim * nlocal_nodes; idof++) - { - for (int istride = 0; istride < Stiffness_Matrix_Strides(idof); istride++) - { + for (int idof = 0; idof < num_dim * nlocal_nodes; idof++) { + for (int istride = 0; istride < Stiffness_Matrix_Strides(idof); istride++) { Stiffness_Matrix(idof, istride) = 0; // debug print // std::cout << "{" <choose_2Delem_type(Element_Types(ielem), elem2D); nodes_per_elem = elem2D->num_nodes(); // construct local stiffness matrix for this element local_matrix_multiply(ielem, Local_Stiffness_Matrix); // assign entries of this local matrix to the sparse global matrix storage; - for (int inode = 0; inode < nodes_per_elem; inode++) - { + for (int inode = 0; inode < nodes_per_elem; inode++) { // see if this node is local local_node_index = nodes_in_elem.host(ielem, inode); - if (!map->isNodeLocalElement(local_node_index)) - { + if (!map->isNodeLocalElement(local_node_index)) { continue; } // set dof row start index current_row = num_dim * local_node_index; - for (int jnode = 0; jnode < nodes_per_elem; jnode++) - { + for (int jnode = 0; jnode < nodes_per_elem; jnode++) { current_column = num_dim * Global_Stiffness_Matrix_Assembly_Map(ielem, inode, jnode); - for (int idim = 0; idim < num_dim; idim++) - { - for (int jdim = 0; jdim < num_dim; jdim++) - { + for (int idim = 0; idim < num_dim; idim++) { + for (int jdim = 0; jdim < num_dim; jdim++) { Stiffness_Matrix(current_row + idim, current_column + jdim) += Local_Stiffness_Matrix(num_dim * inode + idim, num_dim * jnode + jdim); } @@ -312,32 +288,25 @@ void FEA_Module_Dynamic_Elasticity::assemble_matrix() } } - if (num_dim == 3) - { - for (int ielem = 0; ielem < rnum_elem; ielem++) - { + if (num_dim == 3) { + for (int ielem = 0; ielem < rnum_elem; ielem++) { element_select->choose_3Delem_type(Element_Types(ielem), elem); nodes_per_elem = elem->num_nodes(); // construct local stiffness matrix for this element local_matrix_multiply(ielem, Local_Stiffness_Matrix); // assign entries of this local matrix to the sparse global matrix storage; - for (int inode = 0; inode < nodes_per_elem; inode++) - { + for (int inode = 0; inode < nodes_per_elem; inode++) { // see if this node is local local_node_index = nodes_in_elem.host(ielem, inode); - if (!map->isNodeLocalElement(local_node_index)) - { + if (!map->isNodeLocalElement(local_node_index)) { continue; } // set dof row start index current_row = num_dim * local_node_index; - for (int jnode = 0; jnode < nodes_per_elem; jnode++) - { + for (int jnode = 0; jnode < nodes_per_elem; jnode++) { current_column = num_dim * Global_Stiffness_Matrix_Assembly_Map(ielem, inode, jnode); - for (int idim = 0; idim < num_dim; idim++) - { - for (int jdim = 0; jdim < num_dim; jdim++) - { + for (int idim = 0; idim < num_dim; idim++) { + for (int jdim = 0; jdim < num_dim; jdim++) { Stiffness_Matrix(current_row + idim, current_column + jdim) += Local_Stiffness_Matrix(num_dim * inode + idim, num_dim * jnode + jdim); } @@ -365,12 +334,10 @@ void FEA_Module_Dynamic_Elasticity::Element_Material_Properties(size_t ielem, re real_t unit_scaling = simparam->get_unit_scaling(); real_t penalty_product = 1; real_t density_epsilon = simparam->optimization_options.density_epsilon; - if (density < 0) - { + if (density < 0) { density = 0; } - for (int i = 0; i < penalty_power; i++) - { + for (int i = 0; i < penalty_power; i++) { penalty_product *= density; } // relationship between density and stiffness @@ -397,12 +364,10 @@ void FEA_Module_Dynamic_Elasticity::Gradient_Element_Material_Properties(size_t real_t penalty_product = 1; real_t density_epsilon = simparam->optimization_options.density_epsilon; Element_Modulus_Derivative = 0; - if (density < 0) - { + if (density < 0) { density = 0; } - for (int i = 0; i < penalty_power - 1; i++) - { + for (int i = 0; i < penalty_power - 1; i++) { penalty_product *= density; } // relationship between density and stiffness @@ -429,19 +394,15 @@ void FEA_Module_Dynamic_Elasticity::local_matrix_multiply(int ielem, CArrayKokko // local variable for host view of densities from the dual view const_host_vec_array all_node_densities; - if (nodal_density_flag) - { - if (simparam->optimization_options.density_filter == DENSITY_FILTER::helmholtz_filter) - { + if (nodal_density_flag) { + if (simparam->optimization_options.density_filter == DENSITY_FILTER::helmholtz_filter) { all_node_densities = all_filtered_node_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); } - else - { + else{ all_node_densities = all_node_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); } } - else - { + else{ Element_Densities = Global_Element_Densities->getLocalView(Tpetra::Access::ReadOnly); } int num_dim = simparam->num_dims; @@ -485,12 +446,10 @@ void FEA_Module_Dynamic_Elasticity::local_matrix_multiply(int ielem, CArrayKokko CArrayKokkos nodal_density(elem->num_basis()); size_t Brows; - if (num_dim == 2) - { + if (num_dim == 2) { Brows = 3; } - if (num_dim == 3) - { + if (num_dim == 3) { Brows = 6; } FArrayKokkos B_matrix_contribution(Brows, num_dim * elem->num_basis()); @@ -506,28 +465,24 @@ void FEA_Module_Dynamic_Elasticity::local_matrix_multiply(int ielem, CArrayKokko real_t current_density = 1; CArrayKokkos convert_node_order(max_nodes_per_element); - if ((active_node_ordering_convention == Solver::ENSIGHT && num_dim == 3) || (active_node_ordering_convention == Solver::IJK && num_dim == 2)) - { + if ((active_node_ordering_convention == Solver::ENSIGHT && num_dim == 3) || (active_node_ordering_convention == Solver::IJK && num_dim == 2)) { convert_node_order(0) = 0; convert_node_order(1) = 1; convert_node_order(2) = 3; convert_node_order(3) = 2; - if (num_dim == 3) - { + if (num_dim == 3) { convert_node_order(4) = 4; convert_node_order(5) = 5; convert_node_order(6) = 7; convert_node_order(7) = 6; } } - else if ((active_node_ordering_convention == Solver::IJK && num_dim == 3) || (active_node_ordering_convention == Solver::ENSIGHT && num_dim == 2)) - { + else if ((active_node_ordering_convention == Solver::IJK && num_dim == 3) || (active_node_ordering_convention == Solver::ENSIGHT && num_dim == 2)) { convert_node_order(0) = 0; convert_node_order(1) = 1; convert_node_order(2) = 2; convert_node_order(3) = 3; - if (num_dim == 3) - { + if (num_dim == 3) { convert_node_order(4) = 4; convert_node_order(5) = 5; convert_node_order(6) = 6; @@ -536,14 +491,12 @@ void FEA_Module_Dynamic_Elasticity::local_matrix_multiply(int ielem, CArrayKokko } // acquire set of nodes for this local element - for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) - { + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) { local_node_id = nodes_in_elem.host(ielem, convert_node_order(node_loop)); nodal_positions(node_loop, 0) = all_initial_node_coords(local_node_id, 0); nodal_positions(node_loop, 1) = all_initial_node_coords(local_node_id, 1); nodal_positions(node_loop, 2) = all_initial_node_coords(local_node_id, 2); - if (nodal_density_flag) - { + if (nodal_density_flag) { nodal_density(node_loop) = all_node_densities(local_node_id, 0); } /* @@ -557,58 +510,47 @@ void FEA_Module_Dynamic_Elasticity::local_matrix_multiply(int ielem, CArrayKokko } // initialize C matrix - for (int irow = 0; irow < Brows; irow++) - { - for (int icol = 0; icol < Brows; icol++) - { + for (int irow = 0; irow < Brows; irow++) { + for (int icol = 0; icol < Brows; icol++) { C_matrix(irow, icol) = 0; } } // initialize local stiffness matrix storage - for (int ifill = 0; ifill < num_dim * nodes_per_elem; ifill++) - { - for (int jfill = 0; jfill < num_dim * nodes_per_elem; jfill++) - { + for (int ifill = 0; ifill < num_dim * nodes_per_elem; ifill++) { + for (int jfill = 0; jfill < num_dim * nodes_per_elem; jfill++) { Local_Matrix(ifill, jfill) = 0; } } // B matrix initialization - for (int irow = 0; irow < Brows; irow++) - { - for (int icol = 0; icol < num_dim * nodes_per_elem; icol++) - { + for (int irow = 0; irow < Brows; irow++) { + for (int icol = 0; icol < num_dim * nodes_per_elem; icol++) { CB_matrix(irow, icol) = B_matrix(irow, icol) = 0; } } // loop over quadrature points - for (int iquad = 0; iquad < direct_product_count; iquad++) - { + for (int iquad = 0; iquad < direct_product_count; iquad++) { // set current quadrature point - if (num_dim == 3) - { + if (num_dim == 3) { z_quad = iquad / (num_gauss_points * num_gauss_points); } y_quad = (iquad % (num_gauss_points * num_gauss_points)) / num_gauss_points; x_quad = iquad % num_gauss_points; quad_coordinate(0) = legendre_nodes_1D(x_quad); quad_coordinate(1) = legendre_nodes_1D(y_quad); - if (num_dim == 3) - { + if (num_dim == 3) { quad_coordinate(2) = legendre_nodes_1D(z_quad); } // set current quadrature weight quad_coordinate_weight(0) = legendre_weights_1D(x_quad); quad_coordinate_weight(1) = legendre_weights_1D(y_quad); - if (num_dim == 3) - { + if (num_dim == 3) { quad_coordinate_weight(2) = legendre_weights_1D(z_quad); } - else - { + else{ quad_coordinate_weight(2) = 1; } weight_multiply = quad_coordinate_weight(0) * quad_coordinate_weight(1) * quad_coordinate_weight(2); @@ -632,16 +574,14 @@ void FEA_Module_Dynamic_Elasticity::local_matrix_multiply(int ielem, CArrayKokko // std::cout << "Element Material Params " << Elastic_Constant << std::endl; // compute Elastic (C) matrix - if (num_dim == 2) - { + if (num_dim == 2) { C_matrix(0, 0) = Pressure_Term; C_matrix(1, 1) = Pressure_Term; C_matrix(0, 1) = Poisson_Ratio; C_matrix(1, 0) = Poisson_Ratio; C_matrix(2, 2) = Shear_Term; } - if (num_dim == 3) - { + if (num_dim == 3) { C_matrix(0, 0) = Pressure_Term; C_matrix(1, 1) = Pressure_Term; C_matrix(2, 2) = Pressure_Term; @@ -680,8 +620,7 @@ void FEA_Module_Dynamic_Elasticity::local_matrix_multiply(int ielem, CArrayKokko JT_row1(0) = 0; JT_row1(1) = 0; JT_row1(2) = 0; - for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) - { + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) { JT_row1(0) += nodal_positions(node_loop, 0) * basis_derivative_s1(node_loop); JT_row1(1) += nodal_positions(node_loop, 1) * basis_derivative_s1(node_loop); JT_row1(2) += nodal_positions(node_loop, 2) * basis_derivative_s1(node_loop); @@ -691,8 +630,7 @@ void FEA_Module_Dynamic_Elasticity::local_matrix_multiply(int ielem, CArrayKokko JT_row2(0) = 0; JT_row2(1) = 0; JT_row2(2) = 0; - for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) - { + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) { JT_row2(0) += nodal_positions(node_loop, 0) * basis_derivative_s2(node_loop); JT_row2(1) += nodal_positions(node_loop, 1) * basis_derivative_s2(node_loop); JT_row2(2) += nodal_positions(node_loop, 2) * basis_derivative_s2(node_loop); @@ -702,8 +640,7 @@ void FEA_Module_Dynamic_Elasticity::local_matrix_multiply(int ielem, CArrayKokko JT_row3(0) = 0; JT_row3(1) = 0; JT_row3(2) = 0; - for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) - { + for (int node_loop = 0; node_loop < elem->num_basis(); node_loop++) { JT_row3(0) += nodal_positions(node_loop, 0) * basis_derivative_s3(node_loop); JT_row3(1) += nodal_positions(node_loop, 1) * basis_derivative_s3(node_loop); JT_row3(2) += nodal_positions(node_loop, 2) * basis_derivative_s3(node_loop); @@ -719,16 +656,13 @@ void FEA_Module_Dynamic_Elasticity::local_matrix_multiply(int ielem, CArrayKokko Jacobian = JT_row1(0) * (JT_row2(1) * JT_row3(2) - JT_row3(1) * JT_row2(2)) - JT_row1(1) * (JT_row2(0) * JT_row3(2) - JT_row3(0) * JT_row2(2)) + JT_row1(2) * (JT_row2(0) * JT_row3(1) - JT_row3(0) * JT_row2(1)); - if (Jacobian < 0) - { + if (Jacobian < 0) { Jacobian = -Jacobian; } invJacobian = 1 / Jacobian; // compute the contributions of this quadrature point to the B matrix - if (num_dim == 2) - { - for (int ishape = 0; ishape < nodes_per_elem; ishape++) - { + if (num_dim == 2) { + for (int ishape = 0; ishape < nodes_per_elem; ishape++) { B_matrix_contribution(0, ishape * num_dim) = (basis_derivative_s1(ishape) * (JT_row2(1) * JT_row3(2) - JT_row3(1) * JT_row2(2)) - basis_derivative_s2(ishape) * (JT_row1(1) * JT_row3(2) - JT_row3(1) * JT_row1(2)) + basis_derivative_s3(ishape) * (JT_row1(1) * JT_row2(2) - JT_row2(1) * JT_row1(2))); @@ -767,10 +701,8 @@ void FEA_Module_Dynamic_Elasticity::local_matrix_multiply(int ielem, CArrayKokko basis_derivative_s3(ishape) * (JT_row1(0) * JT_row2(2) - JT_row2(0) * JT_row1(2))); } } - if (num_dim == 3) - { - for (int ishape = 0; ishape < nodes_per_elem; ishape++) - { + if (num_dim == 3) { + for (int ishape = 0; ishape < nodes_per_elem; ishape++) { B_matrix_contribution(0, ishape * num_dim) = (basis_derivative_s1(ishape) * (JT_row2(1) * JT_row3(2) - JT_row3(1) * JT_row2(2)) - basis_derivative_s2(ishape) * (JT_row1(1) * JT_row3(2) - JT_row3(1) * JT_row1(2)) + basis_derivative_s3(ishape) * (JT_row1(1) * JT_row2(2) - JT_row2(1) * JT_row1(2))); @@ -822,49 +754,38 @@ void FEA_Module_Dynamic_Elasticity::local_matrix_multiply(int ielem, CArrayKokko //end debug block */ // accumulate B matrix - for (int irow = 0; irow < Brows; irow++) - { - for (int icol = 0; icol < num_dim * nodes_per_elem; icol++) - { + for (int irow = 0; irow < Brows; irow++) { + for (int icol = 0; icol < num_dim * nodes_per_elem; icol++) { B_matrix(irow, icol) += B_matrix_contribution(irow, icol); } } // compute the previous multiplied by the Elastic (C) Matrix - for (int irow = 0; irow < Brows; irow++) - { - for (int icol = 0; icol < num_dim * nodes_per_elem; icol++) - { + for (int irow = 0; irow < Brows; irow++) { + for (int icol = 0; icol < num_dim * nodes_per_elem; icol++) { CB_matrix_contribution(irow, icol) = 0; - for (int span = 0; span < Brows; span++) - { + for (int span = 0; span < Brows; span++) { CB_matrix_contribution(irow, icol) += C_matrix(irow, span) * B_matrix_contribution(span, icol); } } } // accumulate CB matrix - for (int irow = 0; irow < Brows; irow++) - { - for (int icol = 0; icol < num_dim * nodes_per_elem; icol++) - { + for (int irow = 0; irow < Brows; irow++) { + for (int icol = 0; icol < num_dim * nodes_per_elem; icol++) { CB_matrix(irow, icol) += CB_matrix_contribution(irow, icol); } } // compute the contributions of this quadrature point to all the local stiffness matrix elements - for (int ifill = 0; ifill < num_dim * nodes_per_elem; ifill++) - { - for (int jfill = ifill; jfill < num_dim * nodes_per_elem; jfill++) - { + for (int ifill = 0; ifill < num_dim * nodes_per_elem; ifill++) { + for (int jfill = ifill; jfill < num_dim * nodes_per_elem; jfill++) { matrix_term = 0; - for (int span = 0; span < Brows; span++) - { + for (int span = 0; span < Brows; span++) { matrix_term += B_matrix_contribution(span, ifill) * CB_matrix_contribution(span, jfill); } Local_Matrix(ifill, jfill) += Elastic_Constant * weight_multiply * matrix_term * invJacobian; - if (ifill != jfill) - { + if (ifill != jfill) { Local_Matrix(jfill, ifill) = Local_Matrix(ifill, jfill); } } @@ -927,12 +848,10 @@ void FEA_Module_Dynamic_Elasticity::compute_stiffness_gradients(const_host_vec_a const_host_vec_array Element_Densities; // local variable for host view of densities from the dual view const_host_vec_array all_node_densities; - if (nodal_density_flag) - { + if (nodal_density_flag) { all_node_densities = all_node_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); } - else - { + else{ Element_Densities = Global_Element_Densities->getLocalView(Tpetra::Access::ReadOnly); } int num_dim = simparam->num_dims; @@ -985,12 +904,10 @@ void FEA_Module_Dynamic_Elasticity::compute_stiffness_gradients(const_host_vec_a CArrayKokkos nodal_density(elem->num_basis()); size_t Brows; - if (num_dim == 2) - { + if (num_dim == 2) { Brows = 3; } - if (num_dim == 3) - { + if (num_dim == 3) { Brows = 6; } FArrayKokkos B_matrix_contribution(Brows, num_dim * elem->num_basis()); @@ -1008,28 +925,24 @@ void FEA_Module_Dynamic_Elasticity::compute_stiffness_gradients(const_host_vec_a real_t current_density = 1; CArrayKokkos convert_node_order(max_nodes_per_element); - if ((active_node_ordering_convention == Solver::ENSIGHT && num_dim == 3) || (active_node_ordering_convention == Solver::IJK && num_dim == 2)) - { + if ((active_node_ordering_convention == Solver::ENSIGHT && num_dim == 3) || (active_node_ordering_convention == Solver::IJK && num_dim == 2)) { convert_node_order(0) = 0; convert_node_order(1) = 1; convert_node_order(2) = 3; convert_node_order(3) = 2; - if (num_dim == 3) - { + if (num_dim == 3) { convert_node_order(4) = 4; convert_node_order(5) = 5; convert_node_order(6) = 7; convert_node_order(7) = 6; } } - else if ((active_node_ordering_convention == Solver::IJK && num_dim == 3) || (active_node_ordering_convention == Solver::ENSIGHT && num_dim == 2)) - { + else if ((active_node_ordering_convention == Solver::IJK && num_dim == 3) || (active_node_ordering_convention == Solver::ENSIGHT && num_dim == 2)) { convert_node_order(0) = 0; convert_node_order(1) = 1; convert_node_order(2) = 2; convert_node_order(3) = 3; - if (num_dim == 3) - { + if (num_dim == 3) { convert_node_order(4) = 4; convert_node_order(5) = 5; convert_node_order(6) = 6; @@ -1038,34 +951,26 @@ void FEA_Module_Dynamic_Elasticity::compute_stiffness_gradients(const_host_vec_a } // loop through each element and assign the contribution to compliance gradient for each of its local nodes - if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) - { - if (myrank == 0) - { + if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) { + if (myrank == 0) { std::cout << "gradient term derivative of force" << std::endl; } } - for (unsigned long cycle = 0; cycle < last_time_step + 1; cycle++) - { + for (unsigned long cycle = 0; cycle < last_time_step + 1; cycle++) { // compute timestep from time data global_dt = time_data[cycle + 1] - time_data[cycle]; // print - if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) - { - if (cycle == 0) - { - if (myrank == 0) - { + if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) { + if (cycle == 0) { + if (myrank == 0) { printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); } } // print time step every 10 cycles - else if (cycle % 20 == 0) - { - if (myrank == 0) - { + else if (cycle % 20 == 0) { + if (myrank == 0) { printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); } } // end if @@ -1081,31 +986,25 @@ void FEA_Module_Dynamic_Elasticity::compute_stiffness_gradients(const_host_vec_a const_host_vec_array current_coordinate_vector = (*forward_solve_coordinate_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); // interface of arrays for current implementation of force calculation - for (size_t ielem = 0; ielem < rnum_elem; ielem++) - { + for (size_t ielem = 0; ielem < rnum_elem; ielem++) { nodes_per_elem = elem->num_basis(); // initialize C matrix - for (int irow = 0; irow < Brows; irow++) - { - for (int icol = 0; icol < Brows; icol++) - { + for (int irow = 0; irow < Brows; irow++) { + for (int icol = 0; icol < Brows; icol++) { C_matrix(irow, icol) = 0; } } // B matrix initialization - for (int irow = 0; irow < Brows; irow++) - { - for (int icol = 0; icol < num_dim * nodes_per_elem; icol++) - { + for (int irow = 0; irow < Brows; irow++) { + for (int icol = 0; icol < num_dim * nodes_per_elem; icol++) { CB_matrix(irow, icol) = 0; } } // acquire set of nodes and nodal displacements for this local element - for (int node_loop = 0; node_loop < nodes_per_elem; node_loop++) - { + for (int node_loop = 0; node_loop < nodes_per_elem; node_loop++) { local_node_id = nodes_in_elem.host(ielem, convert_node_order(node_loop)); local_dof_idx = nodes_in_elem.host(ielem, convert_node_order(node_loop)) * num_dim; local_dof_idy = local_dof_idx + 1; @@ -1125,16 +1024,14 @@ void FEA_Module_Dynamic_Elasticity::compute_stiffness_gradients(const_host_vec_a current_element_adjoint(node_loop * num_dim) = 0.5 * (current_adjoint_vector(local_node_id, 0) + next_adjoint_vector(local_node_id, 0)); current_element_adjoint(node_loop * num_dim + 1) = 0.5 * (current_adjoint_vector(local_node_id, 1) + next_adjoint_vector(local_node_id, 1)); - if (num_dim == 3) - { + if (num_dim == 3) { nodal_positions(node_loop, 2) = all_initial_node_coords(local_node_id, 2); current_nodal_displacements(node_loop * num_dim + 2) = 0.5 * (next_coordinate_vector(local_node_id, 2) - all_initial_node_coords(local_node_id, 2) + current_coordinate_vector(local_node_id, 2) - all_initial_node_coords(local_node_id, 2)); current_element_adjoint(node_loop * num_dim + 2) = 0.5 * (current_adjoint_vector(local_node_id, 2) + next_adjoint_vector(local_node_id, 2)); } - if (nodal_density_flag) - { + if (nodal_density_flag) { nodal_density(node_loop) = all_node_densities(local_node_id, 0); } // debug print @@ -1157,31 +1054,26 @@ void FEA_Module_Dynamic_Elasticity::compute_stiffness_gradients(const_host_vec_a */ // loop over quadrature points - for (int iquad = 0; iquad < direct_product_count; iquad++) - { + for (int iquad = 0; iquad < direct_product_count; iquad++) { // set current quadrature point - if (num_dim == 3) - { + if (num_dim == 3) { z_quad = iquad / (num_gauss_points * num_gauss_points); } y_quad = (iquad % (num_gauss_points * num_gauss_points)) / num_gauss_points; x_quad = iquad % num_gauss_points; quad_coordinate(0) = legendre_nodes_1D(x_quad); quad_coordinate(1) = legendre_nodes_1D(y_quad); - if (num_dim == 3) - { + if (num_dim == 3) { quad_coordinate(2) = legendre_nodes_1D(z_quad); } // set current quadrature weight quad_coordinate_weight(0) = legendre_weights_1D(x_quad); quad_coordinate_weight(1) = legendre_weights_1D(y_quad); - if (num_dim == 3) - { + if (num_dim == 3) { quad_coordinate_weight(2) = legendre_weights_1D(z_quad); } - else - { + else{ quad_coordinate_weight(2) = 1; } weight_multiply = quad_coordinate_weight(0) * quad_coordinate_weight(1) * quad_coordinate_weight(2); @@ -1200,8 +1092,7 @@ void FEA_Module_Dynamic_Elasticity::compute_stiffness_gradients(const_host_vec_a JT_row1(0) = 0; JT_row1(1) = 0; JT_row1(2) = 0; - for (int node_loop = 0; node_loop < nodes_per_elem; node_loop++) - { + for (int node_loop = 0; node_loop < nodes_per_elem; node_loop++) { JT_row1(0) += nodal_positions(node_loop, 0) * basis_derivative_s1(node_loop); JT_row1(1) += nodal_positions(node_loop, 1) * basis_derivative_s1(node_loop); JT_row1(2) += nodal_positions(node_loop, 2) * basis_derivative_s1(node_loop); @@ -1211,8 +1102,7 @@ void FEA_Module_Dynamic_Elasticity::compute_stiffness_gradients(const_host_vec_a JT_row2(0) = 0; JT_row2(1) = 0; JT_row2(2) = 0; - for (int node_loop = 0; node_loop < nodes_per_elem; node_loop++) - { + for (int node_loop = 0; node_loop < nodes_per_elem; node_loop++) { JT_row2(0) += nodal_positions(node_loop, 0) * basis_derivative_s2(node_loop); JT_row2(1) += nodal_positions(node_loop, 1) * basis_derivative_s2(node_loop); JT_row2(2) += nodal_positions(node_loop, 2) * basis_derivative_s2(node_loop); @@ -1222,8 +1112,7 @@ void FEA_Module_Dynamic_Elasticity::compute_stiffness_gradients(const_host_vec_a JT_row3(0) = 0; JT_row3(1) = 0; JT_row3(2) = 0; - for (int node_loop = 0; node_loop < nodes_per_elem; node_loop++) - { + for (int node_loop = 0; node_loop < nodes_per_elem; node_loop++) { JT_row3(0) += nodal_positions(node_loop, 0) * basis_derivative_s3(node_loop); JT_row3(1) += nodal_positions(node_loop, 1) * basis_derivative_s3(node_loop); JT_row3(2) += nodal_positions(node_loop, 2) * basis_derivative_s3(node_loop); @@ -1233,8 +1122,7 @@ void FEA_Module_Dynamic_Elasticity::compute_stiffness_gradients(const_host_vec_a Jacobian = JT_row1(0) * (JT_row2(1) * JT_row3(2) - JT_row3(1) * JT_row2(2)) - JT_row1(1) * (JT_row2(0) * JT_row3(2) - JT_row3(0) * JT_row2(2)) + JT_row1(2) * (JT_row2(0) * JT_row3(1) - JT_row3(0) * JT_row2(1)); - if (Jacobian < 0) - { + if (Jacobian < 0) { Jacobian = -Jacobian; } invJacobian = 1 / Jacobian; @@ -1246,10 +1134,8 @@ void FEA_Module_Dynamic_Elasticity::compute_stiffness_gradients(const_host_vec_a // std::cout << "Current Density " << current_density << std::endl; // compute the contributions of this quadrature point to the B matrix - if (num_dim == 2) - { - for (int ishape = 0; ishape < nodes_per_elem; ishape++) - { + if (num_dim == 2) { + for (int ishape = 0; ishape < nodes_per_elem; ishape++) { B_matrix_contribution(0, ishape * num_dim) = (basis_derivative_s1(ishape) * (JT_row2(1) * JT_row3(2) - JT_row3(1) * JT_row2(2)) - basis_derivative_s2(ishape) * (JT_row1(1) * JT_row3(2) - JT_row3(1) * JT_row1(2)) + basis_derivative_s3(ishape) * (JT_row1(1) * JT_row2(2) - JT_row2(1) * JT_row1(2))); @@ -1288,10 +1174,8 @@ void FEA_Module_Dynamic_Elasticity::compute_stiffness_gradients(const_host_vec_a basis_derivative_s3(ishape) * (JT_row1(0) * JT_row2(2) - JT_row2(0) * JT_row1(2))); } } - if (num_dim == 3) - { - for (int ishape = 0; ishape < nodes_per_elem; ishape++) - { + if (num_dim == 3) { + for (int ishape = 0; ishape < nodes_per_elem; ishape++) { B_matrix_contribution(0, ishape * num_dim) = (basis_derivative_s1(ishape) * (JT_row2(1) * JT_row3(2) - JT_row3(1) * JT_row2(2)) - basis_derivative_s2(ishape) * (JT_row1(1) * JT_row3(2) - JT_row3(1) * JT_row1(2)) + basis_derivative_s3(ishape) * (JT_row1(1) * JT_row2(2) - JT_row2(1) * JT_row1(2))); @@ -1341,16 +1225,14 @@ void FEA_Module_Dynamic_Elasticity::compute_stiffness_gradients(const_host_vec_a // std::cout << "Element Material Params " << Elastic_Constant << std::endl; // compute Elastic (C) matrix - if (num_dim == 2) - { + if (num_dim == 2) { C_matrix(0, 0) = Pressure_Term; C_matrix(1, 1) = Pressure_Term; C_matrix(0, 1) = Poisson_Ratio; C_matrix(1, 0) = Poisson_Ratio; C_matrix(2, 2) = Shear_Term; } - if (num_dim == 3) - { + if (num_dim == 3) { C_matrix(0, 0) = Pressure_Term; C_matrix(1, 1) = Pressure_Term; C_matrix(2, 2) = Pressure_Term; @@ -1366,31 +1248,24 @@ void FEA_Module_Dynamic_Elasticity::compute_stiffness_gradients(const_host_vec_a } // compute the previous multiplied by the Elastic (C) Matrix - for (int irow = 0; irow < Brows; irow++) - { - for (int icol = 0; icol < num_dim * nodes_per_elem; icol++) - { + for (int irow = 0; irow < Brows; irow++) { + for (int icol = 0; icol < num_dim * nodes_per_elem; icol++) { CB_matrix_contribution(irow, icol) = 0; - for (int span = 0; span < Brows; span++) - { + for (int span = 0; span < Brows; span++) { CB_matrix_contribution(irow, icol) += C_matrix(irow, span) * B_matrix_contribution(span, icol); } } } // compute the contributions of this quadrature point to all the local stiffness matrix elements - for (int ifill = 0; ifill < num_dim * nodes_per_elem; ifill++) - { - for (int jfill = ifill; jfill < num_dim * nodes_per_elem; jfill++) - { + for (int ifill = 0; ifill < num_dim * nodes_per_elem; ifill++) { + for (int jfill = ifill; jfill < num_dim * nodes_per_elem; jfill++) { matrix_term = 0; - for (int span = 0; span < Brows; span++) - { + for (int span = 0; span < Brows; span++) { matrix_term += B_matrix_contribution(span, ifill) * CB_matrix_contribution(span, jfill); } Local_Matrix_Contribution(ifill, jfill) = matrix_term; - if (ifill != jfill) - { + if (ifill != jfill) { Local_Matrix_Contribution(jfill, ifill) = Local_Matrix_Contribution(ifill, jfill); } } @@ -1398,10 +1273,8 @@ void FEA_Module_Dynamic_Elasticity::compute_stiffness_gradients(const_host_vec_a // compute inner product for this quadrature point contribution inner_product = 0; - for (int ifill = 0; ifill < num_dim * nodes_per_elem; ifill++) - { - for (int jfill = 0; jfill < num_dim * nodes_per_elem; jfill++) - { + for (int ifill = 0; ifill < num_dim * nodes_per_elem; ifill++) { + for (int jfill = 0; jfill < num_dim * nodes_per_elem; jfill++) { inner_product += Local_Matrix_Contribution(ifill, jfill) * current_element_adjoint(ifill) * current_nodal_displacements(jfill); // debug // if(Local_Matrix_Contribution(ifill, jfill)<0) Local_Matrix_Contribution(ifill, jfill) = - Local_Matrix_Contribution(ifill, jfill); @@ -1410,10 +1283,8 @@ void FEA_Module_Dynamic_Elasticity::compute_stiffness_gradients(const_host_vec_a } // evaluate local stiffness matrix gradient with respect to igradient - for (int igradient = 0; igradient < nodes_per_elem; igradient++) - { - if (!map->isNodeLocalElement(nodes_in_elem.host(ielem, igradient))) - { + for (int igradient = 0; igradient < nodes_per_elem; igradient++) { + if (!map->isNodeLocalElement(nodes_in_elem.host(ielem, igradient))) { continue; } local_node_id = nodes_in_elem.host(ielem, igradient); diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/force_gradients_elastic.cpp b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/force_gradients_elastic.cpp index 2b907fe30..6f1ff18f6 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/force_gradients_elastic.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/force_gradients_elastic.cpp @@ -158,10 +158,8 @@ void FEA_Module_Dynamic_Elasticity::get_force_vgradient_elastic( rk_level); // the -1 is for the inward surface area normal, - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) - { - for (size_t dim = 0; dim < num_dims; dim++) - { + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { + for (size_t dim = 0; dim < num_dims; dim++) { area_normal(node_lid, dim) = (-1.0) * area_normal(node_lid, dim); } // end for } // end for @@ -180,18 +178,15 @@ void FEA_Module_Dynamic_Elasticity::get_force_vgradient_elastic( double mag_curl = sqrt(curl[0] * curl[0] + curl[1] * curl[1] + curl[2] * curl[2]); // --- Calculate the Cauchy stress --- - for (size_t i = 0; i < 3; i++) - { - for (size_t j = 0; j < 3; j++) - { + for (size_t i = 0; i < 3; i++) { + for (size_t j = 0; j < 3; j++) { tau(i, j) = stress(i, j); // artificial viscosity can be added here to tau } // end for } // end for // add the pressure - for (int i = 0; i < num_dims; i++) - { + for (int i = 0; i < num_dims; i++) { tau(i, i) -= elem_pres(elem_gid); } // end for @@ -200,15 +195,13 @@ void FEA_Module_Dynamic_Elasticity::get_force_vgradient_elastic( // estimate of the Riemann velocity // initialize to Riemann velocity to zero - for (size_t dim = 0; dim < num_dims; dim++) - { + for (size_t dim = 0; dim < num_dims; dim++) { vel_star(dim) = 0.0; } // loop over nodes and calculate an average velocity, which is // an estimate of Riemann velocity - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) - { + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { // Get node gloabl index and create view of nodal velocity int node_gid = nodes_in_elem(elem_gid, node_lid); @@ -222,8 +215,7 @@ void FEA_Module_Dynamic_Elasticity::get_force_vgradient_elastic( // find shock direction and shock impedance associated with each node // initialize sum term in MARS to zero - for (int i = 0; i < 4; i++) - { + for (int i = 0; i < 4; i++) { sum(i) = 0.0; } @@ -231,8 +223,7 @@ void FEA_Module_Dynamic_Elasticity::get_force_vgradient_elastic( double mag_vel; // magnitude of velocity // loop over the nodes of the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) - { + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { // Get global node id size_t node_gid = nodes_in_elem(elem_gid, node_lid); @@ -244,16 +235,13 @@ void FEA_Module_Dynamic_Elasticity::get_force_vgradient_elastic( + (vel(1) - vel_star(1) ) * (vel(1) - vel_star(1) ) + (vel(2) - vel_star(2) ) * (vel(2) - vel_star(2) ) ); - if (mag_vel > small) - { + if (mag_vel > small) { // estimate of the shock direction, a unit normal - for (int dim = 0; dim < num_dims; dim++) - { + for (int dim = 0; dim < num_dims; dim++) { shock_dir(dim) = (vel(dim) - vel_star(dim)) / mag_vel; } } - else - { + else{ // if there is no velocity change, then use the surface area // normal as the shock direction mag = sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) @@ -261,21 +249,18 @@ void FEA_Module_Dynamic_Elasticity::get_force_vgradient_elastic( + area_normal(node_lid, 2) * area_normal(node_lid, 2) ); // estimate of the shock direction - for (int dim = 0; dim < num_dims; dim++) - { + for (int dim = 0; dim < num_dims; dim++) { shock_dir(dim) = area_normal(node_lid, dim) / mag; } } // end if mag_vel // cell divergence indicates compression or expansions size_t mat_id = elem_mat_id(elem_gid); - if (div < 0) // element in compression - { + if (div < 0) { // element in compression muc(node_lid) = elem_den(elem_gid) * (material(mat_id).q1 * elem_sspd(elem_gid) + material(mat_id).q2 * mag_vel); } - else // element in expansion - { + else{ // element in expansion muc(node_lid) = elem_den(elem_gid) * (material(mat_id).q1ex * elem_sspd(elem_gid) + material(mat_id).q2ex * mag_vel); } // end if on divergence sign @@ -284,8 +269,7 @@ void FEA_Module_Dynamic_Elasticity::get_force_vgradient_elastic( double mu_term; // Coding to use shock direction - if (use_shock_dir == 1) - { + if (use_shock_dir == 1) { // this is denominator of the Riamann solver and the multiplier // on velocity in the numerator. It filters on the shock // direction @@ -294,8 +278,7 @@ void FEA_Module_Dynamic_Elasticity::get_force_vgradient_elastic( + shock_dir(1) * area_normal(node_lid, 1) + shock_dir(2) * area_normal(node_lid, 2) ); } - else - { + else{ // Using a full tensoral Riemann jump relation mu_term = muc(node_lid) * sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) @@ -312,17 +295,13 @@ void FEA_Module_Dynamic_Elasticity::get_force_vgradient_elastic( } // end for node_lid loop over nodes of the elem // The Riemann velocity, called vel_star - if (sum(3) > fuzz) - { - for (size_t i = 0; i < num_dims; i++) - { + if (sum(3) > fuzz) { + for (size_t i = 0; i < num_dims; i++) { vel_star(i) = sum(i) / sum(3); } } - else - { - for (int i = 0; i < num_dims; i++) - { + else{ + for (int i = 0; i < num_dims; i++) { vel_star(i) = 0.0; } } // end if @@ -349,8 +328,7 @@ void FEA_Module_Dynamic_Elasticity::get_force_vgradient_elastic( // (1=nominal, and n_coeff > 1 oscillatory) // loop over the nieghboring cells - for (size_t elem_lid = 0; elem_lid < num_elems_in_elem(elem_gid); elem_lid++) - { + for (size_t elem_lid = 0; elem_lid < num_elems_in_elem(elem_gid); elem_lid++) { // Get global index for neighboring cell size_t neighbor_gid = elems_in_elem(elem_gid, elem_lid); @@ -388,8 +366,7 @@ void FEA_Module_Dynamic_Elasticity::get_force_vgradient_elastic( // ---- Calculate the Riemann force on each node ---- // loop over the each node in the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) - { + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { size_t corner_lid = node_lid; // Get corner gid @@ -399,8 +376,7 @@ void FEA_Module_Dynamic_Elasticity::get_force_vgradient_elastic( size_t node_gid = nodes_in_elem(elem_gid, node_lid); // loop over dimension - for (int dim = 0; dim < num_dims; dim++) - { + for (int dim = 0; dim < num_dims; dim++) { corner_vector_storage(corner_gid, dim) = -0; // corner_vector_storage(corner_gid, dim) = 0; } // end loop over dimension @@ -417,8 +393,7 @@ void FEA_Module_Dynamic_Elasticity::get_force_vgradient_elastic( // force_gradient_position->putScalar(0); // set back to zero FOR_ALL_CLASS(idof, 0, nlocal_nodes * num_dims, { - for (int jdof = 0; jdof < Gradient_Matrix_Strides(idof); jdof++) - { + for (int jdof = 0; jdof < Gradient_Matrix_Strides(idof); jdof++) { Force_Gradient_Velocities(idof, jdof) = 0; } }); // end parallel for @@ -552,10 +527,8 @@ void FEA_Module_Dynamic_Elasticity::get_force_ugradient_elastic(const DCArrayKok rk_level); // the -1 is for the inward surface area normal, - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) - { - for (size_t dim = 0; dim < num_dims; dim++) - { + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { + for (size_t dim = 0; dim < num_dims; dim++) { area_normal(node_lid, dim) = (-1.0) * area_normal(node_lid, dim); } // end for } // end for @@ -574,18 +547,15 @@ void FEA_Module_Dynamic_Elasticity::get_force_ugradient_elastic(const DCArrayKok double mag_curl = sqrt(curl[0] * curl[0] + curl[1] * curl[1] + curl[2] * curl[2]); // --- Calculate the Cauchy stress --- - for (size_t i = 0; i < 3; i++) - { - for (size_t j = 0; j < 3; j++) - { + for (size_t i = 0; i < 3; i++) { + for (size_t j = 0; j < 3; j++) { tau(i, j) = stress(i, j); // artificial viscosity can be added here to tau } // end for } // end for // add the pressure - for (int i = 0; i < num_dims; i++) - { + for (int i = 0; i < num_dims; i++) { tau(i, i) -= elem_pres(elem_gid); } // end for @@ -594,15 +564,13 @@ void FEA_Module_Dynamic_Elasticity::get_force_ugradient_elastic(const DCArrayKok // estimate of the Riemann velocity // initialize to Riemann velocity to zero - for (size_t dim = 0; dim < num_dims; dim++) - { + for (size_t dim = 0; dim < num_dims; dim++) { vel_star(dim) = 0.0; } // loop over nodes and calculate an average velocity, which is // an estimate of Riemann velocity - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) - { + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { // Get node gloabl index and create view of nodal velocity int node_gid = nodes_in_elem(elem_gid, node_lid); @@ -616,8 +584,7 @@ void FEA_Module_Dynamic_Elasticity::get_force_ugradient_elastic(const DCArrayKok // find shock direction and shock impedance associated with each node // initialize sum term in MARS to zero - for (int i = 0; i < 4; i++) - { + for (int i = 0; i < 4; i++) { sum(i) = 0.0; } @@ -625,8 +592,7 @@ void FEA_Module_Dynamic_Elasticity::get_force_ugradient_elastic(const DCArrayKok double mag_vel; // magnitude of velocity // loop over the nodes of the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) - { + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { // Get global node id size_t node_gid = nodes_in_elem(elem_gid, node_lid); @@ -638,16 +604,13 @@ void FEA_Module_Dynamic_Elasticity::get_force_ugradient_elastic(const DCArrayKok + (vel(1) - vel_star(1) ) * (vel(1) - vel_star(1) ) + (vel(2) - vel_star(2) ) * (vel(2) - vel_star(2) ) ); - if (mag_vel > small) - { + if (mag_vel > small) { // estimate of the shock direction, a unit normal - for (int dim = 0; dim < num_dims; dim++) - { + for (int dim = 0; dim < num_dims; dim++) { shock_dir(dim) = (vel(dim) - vel_star(dim)) / mag_vel; } } - else - { + else{ // if there is no velocity change, then use the surface area // normal as the shock direction mag = sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) @@ -655,21 +618,18 @@ void FEA_Module_Dynamic_Elasticity::get_force_ugradient_elastic(const DCArrayKok + area_normal(node_lid, 2) * area_normal(node_lid, 2) ); // estimate of the shock direction - for (int dim = 0; dim < num_dims; dim++) - { + for (int dim = 0; dim < num_dims; dim++) { shock_dir(dim) = area_normal(node_lid, dim) / mag; } } // end if mag_vel // cell divergence indicates compression or expansions size_t mat_id = elem_mat_id(elem_gid); - if (div < 0) // element in compression - { + if (div < 0) { // element in compression muc(node_lid) = elem_den(elem_gid) * (material(mat_id).q1 * elem_sspd(elem_gid) + material(mat_id).q2 * mag_vel); } - else // element in expansion - { + else{ // element in expansion muc(node_lid) = elem_den(elem_gid) * (material(mat_id).q1ex * elem_sspd(elem_gid) + material(mat_id).q2ex * mag_vel); } // end if on divergence sign @@ -678,8 +638,7 @@ void FEA_Module_Dynamic_Elasticity::get_force_ugradient_elastic(const DCArrayKok double mu_term; // Coding to use shock direction - if (use_shock_dir == 1) - { + if (use_shock_dir == 1) { // this is denominator of the Riamann solver and the multiplier // on velocity in the numerator. It filters on the shock // direction @@ -688,8 +647,7 @@ void FEA_Module_Dynamic_Elasticity::get_force_ugradient_elastic(const DCArrayKok + shock_dir(1) * area_normal(node_lid, 1) + shock_dir(2) * area_normal(node_lid, 2) ); } - else - { + else{ // Using a full tensoral Riemann jump relation mu_term = muc(node_lid) * sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) @@ -706,17 +664,13 @@ void FEA_Module_Dynamic_Elasticity::get_force_ugradient_elastic(const DCArrayKok } // end for node_lid loop over nodes of the elem // The Riemann velocity, called vel_star - if (sum(3) > fuzz) - { - for (size_t i = 0; i < num_dims; i++) - { + if (sum(3) > fuzz) { + for (size_t i = 0; i < num_dims; i++) { vel_star(i) = sum(i) / sum(3); } } - else - { - for (int i = 0; i < num_dims; i++) - { + else{ + for (int i = 0; i < num_dims; i++) { vel_star(i) = 0.0; } } // end if @@ -743,8 +697,7 @@ void FEA_Module_Dynamic_Elasticity::get_force_ugradient_elastic(const DCArrayKok // (1=nominal, and n_coeff > 1 oscillatory) // loop over the nieghboring cells - for (size_t elem_lid = 0; elem_lid < num_elems_in_elem(elem_gid); elem_lid++) - { + for (size_t elem_lid = 0; elem_lid < num_elems_in_elem(elem_gid); elem_lid++) { // Get global index for neighboring cell size_t neighbor_gid = elems_in_elem(elem_gid, elem_lid); @@ -782,8 +735,7 @@ void FEA_Module_Dynamic_Elasticity::get_force_ugradient_elastic(const DCArrayKok // ---- Calculate the Riemann force on each node ---- // loop over the each node in the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) - { + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { size_t corner_lid = node_lid; // Get corner gid @@ -793,8 +745,7 @@ void FEA_Module_Dynamic_Elasticity::get_force_ugradient_elastic(const DCArrayKok size_t node_gid = nodes_in_elem(elem_gid, node_lid); // loop over dimension - for (int dim = 0; dim < num_dims; dim++) - { + for (int dim = 0; dim < num_dims; dim++) { corner_vector_storage(corner_gid, dim) = -0.0001; // corner_vector_storage(corner_gid, dim) = 0; } // end loop over dimension @@ -810,8 +761,7 @@ void FEA_Module_Dynamic_Elasticity::get_force_ugradient_elastic(const DCArrayKok // force_gradient_position->putScalar(0); // set back to zero FOR_ALL_CLASS(idof, 0, nlocal_nodes * num_dims, { - for (int jdof = 0; jdof < Gradient_Matrix_Strides(idof); jdof++) - { + for (int jdof = 0; jdof < Gradient_Matrix_Strides(idof); jdof++) { Force_Gradient_Positions(idof, jdof) = 0; } }); // end parallel for @@ -820,8 +770,7 @@ void FEA_Module_Dynamic_Elasticity::get_force_ugradient_elastic(const DCArrayKok // vec_array force_gradient_position_view = force_gradient_position->getLocalView (Tpetra::Access::ReadWrite); FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { size_t corner_id; - for (int icorner = 0; icorner < num_corners_in_node(node_id); icorner++) - { + for (int icorner = 0; icorner < num_corners_in_node(node_id); icorner++) { corner_id = corners_in_node(node_id, icorner); Force_Gradient_Positions(node_id * num_dims, 0) += -0.001; Force_Gradient_Positions(node_id * num_dims + 1, 0) += -0.001; @@ -859,24 +808,19 @@ void FEA_Module_Dynamic_Elasticity::force_design_gradient_term(const_vec_array d auto current_element_adjoint = CArrayKokkos(num_nodes_in_elem, num_dim); // gradient contribution from gradient of Force vector with respect to design variable. - for (unsigned long cycle = 0; cycle < last_time_step + 1; cycle++) - { + for (unsigned long cycle = 0; cycle < last_time_step + 1; cycle++) { // compute timestep from time data global_dt = time_data[cycle + 1] - time_data[cycle]; // print - if (cycle == 0) - { - if (myrank == 0) - { + if (cycle == 0) { + if (myrank == 0) { printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); } } // print time step every 10 cycles - else if (cycle % 20 == 0) - { - if (myrank == 0) - { + else if (cycle % 20 == 0) { + if (myrank == 0) { printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); } } // end if @@ -896,48 +840,38 @@ void FEA_Module_Dynamic_Elasticity::force_design_gradient_term(const_vec_array d real_t inner_product; // std::cout << elem_mass(elem_id) < small) - { + if (mag_vel > small) { // estimate of the shock direction, a unit normal - for (int dim = 0; dim < num_dims; dim++) - { + for (int dim = 0; dim < num_dims; dim++) { shock_dir(dim) = (vel(dim) - vel_star(dim)) / mag_vel; } } - else - { + else{ // if there is no velocity change, then use the surface area // normal as the shock direction mag = sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) @@ -1178,21 +1098,18 @@ void FEA_Module_Dynamic_Elasticity::get_force_dgradient_elastic( + area_normal(node_lid, 2) * area_normal(node_lid, 2) ); // estimate of the shock direction - for (int dim = 0; dim < num_dims; dim++) - { + for (int dim = 0; dim < num_dims; dim++) { shock_dir(dim) = area_normal(node_lid, dim) / mag; } } // end if mag_vel // cell divergence indicates compression or expansions size_t mat_id = elem_mat_id(elem_gid); - if (div < 0) // element in compression - { + if (div < 0) { // element in compression muc(node_lid) = elem_den(elem_gid) * (material(mat_id).q1 * elem_sspd(elem_gid) + material(mat_id).q2 * mag_vel); } - else // element in expansion - { + else{ // element in expansion muc(node_lid) = elem_den(elem_gid) * (material(mat_id).q1ex * elem_sspd(elem_gid) + material(mat_id).q2ex * mag_vel); } // end if on divergence sign @@ -1201,8 +1118,7 @@ void FEA_Module_Dynamic_Elasticity::get_force_dgradient_elastic( double mu_term; // Coding to use shock direction - if (use_shock_dir == 1) - { + if (use_shock_dir == 1) { // this is denominator of the Riamann solver and the multiplier // on velocity in the numerator. It filters on the shock // direction @@ -1211,8 +1127,7 @@ void FEA_Module_Dynamic_Elasticity::get_force_dgradient_elastic( + shock_dir(1) * area_normal(node_lid, 1) + shock_dir(2) * area_normal(node_lid, 2) ); } - else - { + else{ // Using a full tensoral Riemann jump relation mu_term = muc(node_lid) * sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) @@ -1229,17 +1144,13 @@ void FEA_Module_Dynamic_Elasticity::get_force_dgradient_elastic( } // end for node_lid loop over nodes of the elem // The Riemann velocity, called vel_star - if (sum(3) > fuzz) - { - for (size_t i = 0; i < num_dims; i++) - { + if (sum(3) > fuzz) { + for (size_t i = 0; i < num_dims; i++) { vel_star(i) = sum(i) / sum(3); } } - else - { - for (int i = 0; i < num_dims; i++) - { + else{ + for (int i = 0; i < num_dims; i++) { vel_star(i) = 0.0; } } // end if @@ -1266,8 +1177,7 @@ void FEA_Module_Dynamic_Elasticity::get_force_dgradient_elastic( // (1=nominal, and n_coeff > 1 oscillatory) // loop over the nieghboring cells - for (size_t elem_lid = 0; elem_lid < num_elems_in_elem(elem_gid); elem_lid++) - { + for (size_t elem_lid = 0; elem_lid < num_elems_in_elem(elem_gid); elem_lid++) { // Get global index for neighboring cell size_t neighbor_gid = elems_in_elem(elem_gid, elem_lid); @@ -1305,8 +1215,7 @@ void FEA_Module_Dynamic_Elasticity::get_force_dgradient_elastic( // ---- Calculate the Riemann force on each node ---- // loop over the each node in the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) - { + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { size_t corner_lid = node_lid; // Get corner gid @@ -1316,8 +1225,7 @@ void FEA_Module_Dynamic_Elasticity::get_force_dgradient_elastic( size_t node_gid = nodes_in_elem(elem_gid, node_lid); // loop over dimension - for (int dim = 0; dim < num_dims; dim++) - { + for (int dim = 0; dim < num_dims; dim++) { corner_vector_storage(corner_gid, dim) = 0.0001; } // end loop over dimension } // end for loop over nodes in elem @@ -1334,8 +1242,7 @@ void FEA_Module_Dynamic_Elasticity::get_force_dgradient_elastic( vec_array force_gradient_design_view = force_gradient_design->getLocalView(Tpetra::Access::ReadWrite); FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { size_t corner_id; - for (int icorner = 0; icorner < num_corners_in_node(node_id); icorner++) - { + for (int icorner = 0; icorner < num_corners_in_node(node_id); icorner++) { corner_id = corners_in_node(node_id, icorner); force_gradient_design_view(node_id, 0) += corner_vector_storage(corner_id, 0); force_gradient_design_view(node_id, 1) += corner_vector_storage(corner_id, 1); diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/geometry.cpp b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/geometry.cpp index 95ca3639c..5d5c67791 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/geometry.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/geometry.cpp @@ -58,8 +58,7 @@ void FEA_Module_Dynamic_Elasticity::update_position_elastic(double rk_alpha, // loop over all the nodes in the mesh FOR_ALL_CLASS(node_gid, 0, num_nodes, { - for (int dim = 0; dim < num_dims; dim++) - { + for (int dim = 0; dim < num_dims; dim++) { double half_vel = (node_vel(rk_level, node_gid, dim) + node_vel(0, node_gid, dim)) * 0.5; node_coords(rk_level, node_gid, dim) = node_coords(0, node_gid, dim) + rk_alpha * dt * half_vel; } @@ -105,8 +104,7 @@ void FEA_Module_Dynamic_Elasticity::get_bmatrix(const ViewCArrayKokkos& auto z = ViewCArrayKokkos(z_array, num_nodes); // get the coordinates of the nodes(rk,elem,node) in this element - for (int node_lid = 0; node_lid < num_nodes; node_lid++) - { + for (int node_lid = 0; node_lid < num_nodes; node_lid++) { x(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 0); y(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 1); z(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 2); @@ -295,8 +293,7 @@ void FEA_Module_Dynamic_Elasticity::get_vol() const size_t rk_level = rk_num_bins - 1; const size_t num_dims = num_dim; - if (num_dims == 2) - { + if (num_dims == 2) { FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { // cut out the node_gids for this element ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 4); @@ -304,8 +301,7 @@ void FEA_Module_Dynamic_Elasticity::get_vol() }); Kokkos::fence(); } - else - { + else{ FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { // cut out the node_gids for this element ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 8); @@ -349,8 +345,7 @@ void FEA_Module_Dynamic_Elasticity::get_vol_hex(const DViewCArrayKokkos& auto z = ViewCArrayKokkos(z_array, num_nodes); // get the coordinates of the nodes(rk,elem,node) in this element - for (int node_lid = 0; node_lid < num_nodes; node_lid++) - { + for (int node_lid = 0; node_lid < num_nodes; node_lid++) { x(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 0); y(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 1); z(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 2); @@ -407,8 +402,7 @@ void FEA_Module_Dynamic_Elasticity::get_bmatrix2D(const ViewCArrayKokkos auto y = ViewCArrayKokkos(y_array, num_nodes); // get the coordinates of the nodes(rk,elem,node) in this element - for (int node_lid = 0; node_lid < num_nodes; node_lid++) - { + for (int node_lid = 0; node_lid < num_nodes; node_lid++) { x(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 0); y(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 1); } // end for @@ -502,8 +496,7 @@ void FEA_Module_Dynamic_Elasticity::get_vol_quad(const DViewCArrayKokkos auto y = ViewCArrayKokkos(y_array, num_nodes); // get the coordinates of the nodes(rk,elem,node) in this element - for (int node_lid = 0; node_lid < num_nodes; node_lid++) - { + for (int node_lid = 0; node_lid < num_nodes; node_lid++) { x(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 0); y(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 1); } // end for @@ -550,8 +543,7 @@ double FEA_Module_Dynamic_Elasticity::get_area_quad(const size_t elem_gid, auto y = ViewCArrayKokkos(y_array, num_nodes); // get the coordinates of the nodes(rk,elem,node) in this element - for (int node_lid = 0; node_lid < num_nodes; node_lid++) - { + for (int node_lid = 0; node_lid < num_nodes; node_lid++) { x(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 0); y(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 1); } // end for @@ -635,8 +627,7 @@ void FEA_Module_Dynamic_Elasticity::get_area_weights2D(const ViewCArrayKokkos arr[j + 1]) - { + for (size_t i = 0; i < (num - 1); i++) { + for (size_t j = 0; j < (num - i - 1); j++) { + if (arr[j] > arr[j + 1]) { size_t temp = arr[j]; arr[j] = arr[j + 1]; arr[j + 1] = temp; @@ -218,8 +215,7 @@ struct mesh_t { num_dims = num_dims_inp; num_nodes_in_elem = 1; - for (int dim = 0; dim < num_dims; dim++) - { + for (int dim = 0; dim < num_dims; dim++) { num_nodes_in_elem *= 2; } num_elems = num_elems_inp; @@ -264,8 +260,7 @@ struct mesh_t num_corners_in_node(node_gid) = 0; }); - for (size_t elem_gid = 0; elem_gid < num_elems; elem_gid++) - { + for (size_t elem_gid = 0; elem_gid < num_elems; elem_gid++) { FOR_ALL_CLASS(node_lid, 0, num_nodes_in_elem, { // get the global_id of the node size_t node_gid = nodes_in_elem(elem_gid, node_lid); @@ -289,8 +284,7 @@ struct mesh_t elems_in_node = RaggedRightArrayKokkos(num_corners_in_node, "elems_in_node"); // populate the elems connected to a node list and corners in a node - for (size_t elem_gid = 0; elem_gid < num_elems; elem_gid++) - { + for (size_t elem_gid = 0; elem_gid < num_elems; elem_gid++) { FOR_ALL_CLASS(node_lid, 0, num_nodes_in_elem, { // get the global_id of the node size_t node_gid = nodes_in_elem(elem_gid, node_lid); @@ -332,8 +326,7 @@ struct mesh_t // num_corners_in_node = num_elems_in_node size_t max_num = num_corners_in_node(node_gid); - if (max_num > max_num_lcl) - { + if (max_num > max_num_lcl) { max_num_lcl = max_num; } }, max_num_elems_in_node); // end parallel reduction on max @@ -350,14 +343,12 @@ struct mesh_t // find and save neighboring elem_gids of an elem FOR_ALL_CLASS(elem_gid, 0, num_elems, { - for (int node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) - { + for (int node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { // get the gid for the node size_t node_id = nodes_in_elem(elem_gid, node_lid); // loop over all elems connected to node_gid - for (int elem_lid = 0; elem_lid < num_corners_in_node(node_id); elem_lid++) - { + for (int elem_lid = 0; elem_lid < num_corners_in_node(node_id); elem_lid++) { // get the global id for the neighboring elem size_t neighbor_elem_gid = elems_in_node(node_id, elem_lid); @@ -365,23 +356,19 @@ struct mesh_t size_t save = 1; // a true neighbor_elem_id is not equal to elem_gid - if (neighbor_elem_gid == elem_gid) - { + if (neighbor_elem_gid == elem_gid) { save = 0; // don't save } // end if // check to see if the neighbor_elem_gid has been saved already size_t num_saved = temp_elems_in_elem.stride(elem_gid); - for (size_t i = 0; i < num_saved; i++) - { - if (neighbor_elem_gid == temp_elems_in_elem(elem_gid, i)) - { + for (size_t i = 0; i < num_saved; i++) { + if (neighbor_elem_gid == temp_elems_in_elem(elem_gid, i)) { save = 0; // don't save, it has been saved already } // end if } // end for i - if (save == 1) - { + if (save == 1) { // increment the number of neighboring elements saved temp_elems_in_elem.stride(elem_gid)++; @@ -400,8 +387,7 @@ struct mesh_t elems_in_elem = RaggedRightArrayKokkos(num_elems_in_elem, "elems_in_elem"); FOR_ALL_CLASS(elem_gid, 0, num_elems, { - for (size_t i = 0; i < num_elems_in_elem(elem_gid); i++) - { + for (size_t i = 0; i < num_elems_in_elem(elem_gid); i++) { elems_in_elem(elem_gid, i) = temp_elems_in_elem(elem_gid, i); } // end for i }); // end FOR_ALL elems @@ -431,8 +417,7 @@ struct mesh_t size_t node_lids_in_patch_in_elem[24]; - if (num_dims == 3) - { + if (num_dims == 3) { size_t temp_node_lids[24] = { 0, 4, 7, 3, 1, 2, 6, 5, 0, 1, 5, 4, @@ -440,13 +425,11 @@ struct mesh_t 0, 3, 2, 1, 4, 5, 6, 7 }; - for (size_t i = 0; i < 24; i++) - { + for (size_t i = 0; i < 24; i++) { node_lids_in_patch_in_elem[i] = temp_node_lids[i]; } // end for i } - else - { + else{ // J // | // 3---2 @@ -459,8 +442,7 @@ struct mesh_t 0, 1, 3, 2 }; - for (size_t i = 0; i < 8; i++) - { + for (size_t i = 0; i < 8; i++) { node_lids_in_patch_in_elem[i] = temp_node_lids[i]; } // end for i } // end if on dims @@ -481,13 +463,11 @@ struct mesh_t // step 1) calculate the hash values for each patch in the element FOR_ALL_CLASS(elem_gid, 0, num_elems, { - for (size_t patch_lid = 0; patch_lid < num_patches_in_elem; patch_lid++) - { + for (size_t patch_lid = 0; patch_lid < num_patches_in_elem; patch_lid++) { size_t sorted_patch_nodes[4]; // note: cannot be allocated with num_nodes_in_patch // first save the patch nodes - for (size_t patch_node_lid = 0; patch_node_lid < num_nodes_in_patch; patch_node_lid++) - { + for (size_t patch_node_lid = 0; patch_node_lid < num_nodes_in_patch; patch_node_lid++) { // get the local node index of the element for this patch and node in patch size_t node_lid = node_ordering_in_elem(patch_lid, patch_node_lid); @@ -499,12 +479,10 @@ struct mesh_t bubble_sort(sorted_patch_nodes, num_nodes_in_patch); long long int hash_key; - if (num_dims == 2) - { + if (num_dims == 2) { hash_key = (sorted_patch_nodes[0] + num_nodes * sorted_patch_nodes[1]); } - else - { + else{ hash_key = (sorted_patch_nodes[1] + num_nodes * sorted_patch_nodes[2] + num_nodes * num_nodes * sorted_patch_nodes[3]); // 3 largest node values } // end if on dims @@ -522,29 +500,23 @@ struct mesh_t size_t patch_gid = 0; size_t bdy_patch_gid = 0; - for (size_t elem_gid = 0; elem_gid < num_elems; elem_gid++) - { - for (size_t patch_lid = 0; patch_lid < num_patches_in_elem; patch_lid++) - { + for (size_t elem_gid = 0; elem_gid < num_elems; elem_gid++) { + for (size_t patch_lid = 0; patch_lid < num_patches_in_elem; patch_lid++) { long long int hash_key = hash_keys_in_elem(elem_gid, patch_lid); size_t exit = 0; - if (hash_key < 0) - { + if (hash_key < 0) { // find the nighboring patch with the same hash_key - for (size_t neighbor_elem_lid = 0; neighbor_elem_lid < num_elems_in_elem(elem_gid); neighbor_elem_lid++) - { + for (size_t neighbor_elem_lid = 0; neighbor_elem_lid < num_elems_in_elem(elem_gid); neighbor_elem_lid++) { // get the neighboring element global index size_t neighbor_elem_gid = elems_in_elem(elem_gid, neighbor_elem_lid); - for (size_t neighbor_patch_lid = 0; neighbor_patch_lid < num_patches_in_elem; neighbor_patch_lid++) - { + for (size_t neighbor_patch_lid = 0; neighbor_patch_lid < num_patches_in_elem; neighbor_patch_lid++) { // this hash is from the nodes on the patch long long int neighbor_hash_key = hash_keys_in_elem(neighbor_elem_gid, neighbor_patch_lid); - if (neighbor_hash_key == hash_keys_in_elem(elem_gid, patch_lid)) - { + if (neighbor_hash_key == hash_keys_in_elem(elem_gid, patch_lid)) { // save the respective elem_gid's as they are patch neighbors hash_keys_in_elem(elem_gid, patch_lid) = neighbor_elem_gid; hash_keys_in_elem(neighbor_elem_gid, neighbor_patch_lid) = elem_gid; @@ -564,8 +536,7 @@ struct mesh_t } // end if } // end for loop over a neighbors patch set - if (exit == 1) - { + if (exit == 1) { break; } } // end for loop over elem neighbors @@ -573,10 +544,8 @@ struct mesh_t } // end for patch_lid // remaining negative hash key values are the boundary patches - for (size_t patch_lid = 0; patch_lid < num_patches_in_elem; patch_lid++) - { - if (hash_keys_in_elem(elem_gid, patch_lid) < 0) - { + for (size_t patch_lid = 0; patch_lid < num_patches_in_elem; patch_lid++) { + if (hash_keys_in_elem(elem_gid, patch_lid) < 0) { hash_keys_in_elem(elem_gid, patch_lid) = elem_gid; // neighboring_side_lids(elem_gid, patch_lid) = patch_lid; @@ -620,8 +589,7 @@ struct mesh_t num_elems_in_patch_saved(patch_gid) = 0; }); - for (size_t elem_gid = 0; elem_gid < num_elems; elem_gid++) - { + for (size_t elem_gid = 0; elem_gid < num_elems; elem_gid++) { FOR_ALL_CLASS(patch_lid, 0, num_patches_in_elem, { size_t patch_gid = patches_in_elem(elem_gid, patch_lid); @@ -633,8 +601,7 @@ struct mesh_t num_elems_in_patch_saved(patch_gid)++; // save the nodes on this patch - for (size_t patch_node_lid = 0; patch_node_lid < num_nodes_in_patch; patch_node_lid++) - { + for (size_t patch_node_lid = 0; patch_node_lid < num_nodes_in_patch; patch_node_lid++) { // get the local node index of the element for this patch and node in patch size_t node_lid = node_ordering_in_elem(patch_lid, patch_node_lid); @@ -664,18 +631,15 @@ struct mesh_t RUN_CLASS({ num_bdy_nodes_saved(0) = 0; - for (size_t bdy_patch_gid = 0; bdy_patch_gid < num_bdy_patches; bdy_patch_gid++) - { + for (size_t bdy_patch_gid = 0; bdy_patch_gid < num_bdy_patches; bdy_patch_gid++) { // get the global index of the patch that is on the boundary size_t patch_gid = bdy_patches(bdy_patch_gid); // tag the boundary nodes - for (size_t node_lid = 0; node_lid < num_nodes_in_patch; node_lid++) - { + for (size_t node_lid = 0; node_lid < num_nodes_in_patch; node_lid++) { size_t node_gid = nodes_in_patch(patch_gid, node_lid); - if (hash_bdy_nodes(node_gid) < 0) - { + if (hash_bdy_nodes(node_gid) < 0) { hash_bdy_nodes(node_gid) = node_gid; temp_bdy_nodes(num_bdy_nodes_saved(0)) = node_gid; @@ -721,8 +685,7 @@ struct mesh_t // num_corners_in_node = num_elems_in_node size_t max_num = num_corners_in_node(node_gid); - if (max_num > max_num_lcl) - { + if (max_num > max_num_lcl) { max_num_lcl = max_num; } }, max_num_elems_in_node); // end parallel reduction on max @@ -736,24 +699,19 @@ struct mesh_t // walk over the patches and save the node node connectivity RUN_CLASS({ - if (num_dims == 3) - { - for (size_t patch_gid = 0; patch_gid < num_patches; patch_gid++) - { - for (size_t node_lid = 0; node_lid < num_nodes_in_patch; node_lid++) - { + if (num_dims == 3) { + for (size_t patch_gid = 0; patch_gid < num_patches; patch_gid++) { + for (size_t node_lid = 0; node_lid < num_nodes_in_patch; node_lid++) { // the first node on the edge size_t node_gid_0 = nodes_in_patch(patch_gid, node_lid); // second node on this edge size_t node_gid_1; - if (node_lid == num_nodes_in_patch - 1) - { + if (node_lid == num_nodes_in_patch - 1) { node_gid_1 = nodes_in_patch(patch_gid, 0); } - else - { + else{ node_gid_1 = nodes_in_patch(patch_gid, node_lid + 1); } // end if @@ -764,25 +722,20 @@ struct mesh_t size_t save_1 = 1; // check to see if the node_gid_1 was already saved - for (size_t contents_lid = 0; contents_lid < num_saved_0; contents_lid++) - { - if (temp_nodes_in_nodes(node_gid_0, contents_lid) == node_gid_1) - { + for (size_t contents_lid = 0; contents_lid < num_saved_0; contents_lid++) { + if (temp_nodes_in_nodes(node_gid_0, contents_lid) == node_gid_1) { save_0 = 0; // don't save, it was already saved } } // check to see if the node_gid_0 was already saved - for (size_t contents_lid = 0; contents_lid < num_saved_1; contents_lid++) - { - if (temp_nodes_in_nodes(node_gid_1, contents_lid) == node_gid_0) - { + for (size_t contents_lid = 0; contents_lid < num_saved_1; contents_lid++) { + if (temp_nodes_in_nodes(node_gid_1, contents_lid) == node_gid_0) { save_1 = 0; // don't save, it was already saved } } - if (save_0 == 1) - { + if (save_0 == 1) { // increment the number of nodes in a node saved temp_nodes_in_nodes.stride(node_gid_0)++; @@ -790,8 +743,7 @@ struct mesh_t temp_nodes_in_nodes(node_gid_0, num_saved_0) = node_gid_1; } - if (save_1 == 1) - { + if (save_1 == 1) { // increment the number of nodes in a node saved temp_nodes_in_nodes.stride(node_gid_1)++; @@ -805,10 +757,8 @@ struct mesh_t } // end for node in patch } // end for patches } // end if 3D - else - { - for (size_t patch_gid = 0; patch_gid < num_patches; patch_gid++) - { + else{ + for (size_t patch_gid = 0; patch_gid < num_patches; patch_gid++) { // the first node on the edge size_t node_gid_0 = nodes_in_patch(patch_gid, 0); @@ -841,8 +791,7 @@ struct mesh_t // save the connectivity FOR_ALL_CLASS(node_gid, 0, num_nodes, { size_t num_saved = 0; - for (size_t node_lid = 0; node_lid < num_nodes_in_node(node_gid); node_lid++) - { + for (size_t node_lid = 0; node_lid < num_nodes_in_node(node_gid); node_lid++) { nodes_in_node(node_gid, num_saved) = temp_nodes_in_nodes(node_gid, num_saved); // increment the number of nodes in node saved @@ -862,8 +811,7 @@ struct mesh_t ///////////////////////////////////////////////////////////////////////////// void init_bdy_sets(size_t num_bcs) { - if (num_bcs == 0) - { + if (num_bcs == 0) { printf("ERROR: number of boundary sets = 0, set it = 1"); num_bcs = 1; } diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/momentum.cpp b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/momentum.cpp index 771f265a3..8efda3571 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/momentum.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/momentum.cpp @@ -71,8 +71,7 @@ void FEA_Module_Dynamic_Elasticity::get_velgrad(ViewCArrayKokkos& vel_gr ViewCArrayKokkos w(w_array, num_nodes_in_elem); // z-dir vel component // get the vertex velocities for the cell - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) - { + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { // Get node gid size_t node_gid = elem_node_gids(node_lid); @@ -170,8 +169,7 @@ void FEA_Module_Dynamic_Elasticity::get_velgrad2D(ViewCArrayKokkos& vel_ ViewCArrayKokkos v(v_array, num_nodes_in_elem); // y-dir vel component // get the vertex velocities for the cell - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) - { + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { // Get node gid size_t node_gid = elem_node_gids(node_lid); @@ -180,10 +178,8 @@ void FEA_Module_Dynamic_Elasticity::get_velgrad2D(ViewCArrayKokkos& vel_ } // end for // initialize to zero - for (size_t i = 0; i < 3; i++) - { - for (size_t j = 0; j < 3; j++) - { + for (size_t i = 0; i < 3; i++) { + for (size_t j = 0; j < 3; j++) { vel_grad(i, j) = 0.0; } } @@ -260,8 +256,7 @@ void FEA_Module_Dynamic_Elasticity::get_divergence(DViewCArrayKokkos& el rk_level); // get the vertex velocities for the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) - { + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { // Get node gid size_t node_gid = elem_node_gids(node_lid); @@ -350,8 +345,7 @@ void FEA_Module_Dynamic_Elasticity::get_divergence2D(DViewCArrayKokkos& // true volume uses the elem_vol // get the vertex velocities and node coordinate for the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) - { + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { // Get node gid size_t node_gid = elem_node_gids(node_lid); @@ -421,19 +415,15 @@ void FEA_Module_Dynamic_Elasticity::decompose_vel_grad(ViewCArrayKokkos& const size_t num_dims = 3; // initialize to zero - for (size_t i = 0; i < num_dims; i++) - { - for (size_t j = 0; j < num_dims; j++) - { + for (size_t i = 0; i < num_dims; i++) { + for (size_t j = 0; j < num_dims; j++) { D_tensor(i, j) = 0.0; W_tensor(i, j) = 0.0; } } // end for - for (size_t i = 0; i < num_dims; i++) - { - for (size_t j = 0; j < num_dims; j++) - { + for (size_t i = 0; i < num_dims; i++) { + for (size_t j = 0; j < num_dims; j++) { D_tensor(i, j) = 0.5 * (vel_grad(i, j) + vel_grad(j, i)); W_tensor(i, j) = 0.5 * (vel_grad(i, j) - vel_grad(j, i)); } diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/properties.cpp b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/properties.cpp index 2f3d16377..e630b4e0c 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/properties.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/properties.cpp @@ -94,8 +94,7 @@ void FEA_Module_Dynamic_Elasticity::update_state(const DCArrayKokkos // --- Stress --- // hyper elastic plastic model - if (material(mat_id).strength_type == STRENGTH_TYPE::hyper) - { + if (material(mat_id).strength_type == STRENGTH_TYPE::hyper) { // cut out the node_gids for this element ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), num_nodes_in_elem); @@ -127,10 +126,8 @@ void FEA_Module_Dynamic_Elasticity::update_state(const DCArrayKokkos rk_level); // reinit values - for (size_t i = 0; i < 3; i++) - { - for (size_t j = 0; j < 3; j++) - { + for (size_t i = 0; i < 3; i++) { + for (size_t j = 0; j < 3; j++) { elem_stress(rk_level, elem_gid, i, j) = 0.0; } } // end for @@ -252,8 +249,7 @@ void FEA_Module_Dynamic_Elasticity::update_state2D(const DCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), num_nodes_in_elem); diff --git a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/time_integration.cpp b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/time_integration.cpp index 15bc24f49..01f4ff8f4 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/time_integration.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Dynamic_Elastic_Solver/time_integration.cpp @@ -61,10 +61,8 @@ void FEA_Module_Dynamic_Elasticity::rk_init(DViewCArrayKokkos& node_coor // save elem quantities FOR_ALL_CLASS(elem_gid, 0, num_elems, { // stress is always 3D even with 2D-RZ - for (size_t i = 0; i < 3; i++) - { - for (size_t j = 0; j < 3; j++) - { + for (size_t i = 0; i < 3; i++) { + for (size_t j = 0; j < 3; j++) { elem_stress(0, elem_gid, i, j) = elem_stress(rk_level, elem_gid, i, j); } } // end for @@ -74,8 +72,7 @@ void FEA_Module_Dynamic_Elasticity::rk_init(DViewCArrayKokkos& node_coor // save nodal quantities FOR_ALL_CLASS(node_gid, 0, num_nodes, { - for (size_t i = 0; i < num_dims; i++) - { + for (size_t i = 0; i < num_dims; i++) { node_coords(0, node_gid, i) = node_coords(rk_level, node_gid, i); node_vel(0, node_gid, i) = node_vel(rk_level, node_gid, i); } @@ -123,10 +120,8 @@ void FEA_Module_Dynamic_Elasticity::get_timestep(mesh_t& mesh, ViewCArrayKokkos dist(distance0, 28); // Getting the coordinates of the element - for (size_t node_lid = 0; node_lid < 8; node_lid++) - { - for (size_t dim = 0; dim < num_dims; dim++) - { + for (size_t node_lid = 0; node_lid < 8; node_lid++) { + for (size_t dim = 0; dim < num_dims; dim++) { coords(node_lid, dim) = node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), dim); } // end for dim } // end for loop over node_lid @@ -140,8 +135,7 @@ void FEA_Module_Dynamic_Elasticity::get_timestep(mesh_t& mesh, // Only works for 3D // Solving for the magnitude of distance between each node - for (size_t i = 0; i < 28; i++) - { + for (size_t i = 0; i < 28; i++) { a = countA; b = countB; @@ -154,8 +148,7 @@ void FEA_Module_Dynamic_Elasticity::get_timestep(mesh_t& mesh, countA++; // tricky indexing - if (countB > 7) - { + if (countB > 7) { loop++; countB = 1 + loop; countA = 0; @@ -164,8 +157,7 @@ void FEA_Module_Dynamic_Elasticity::get_timestep(mesh_t& mesh, double dist_min = dist(0); - for (int i = 0; i < 28; ++i) - { + for (int i = 0; i < 28; ++i) { dist_min = fmin(dist(i), dist_min); } @@ -176,16 +168,14 @@ void FEA_Module_Dynamic_Elasticity::get_timestep(mesh_t& mesh, dt_lcl_ = fmin(dt_lcl_, dt_max); // make dt small than dt_max dt_lcl_ = fmax(dt_lcl_, dt_min); // make dt larger than dt_min - if (dt_lcl_ < dt_lcl) - { + if (dt_lcl_ < dt_lcl) { dt_lcl = dt_lcl_; } }, min_dt_calc); // end parallel reduction Kokkos::fence(); // save the min dt - if (min_dt_calc < dt) - { + if (min_dt_calc < dt) { dt = min_dt_calc; } @@ -236,10 +226,8 @@ void FEA_Module_Dynamic_Elasticity::get_timestep2D(mesh_t& mesh, ViewCArrayKokkos dist(distance0, 6); // Getting the coordinates of the nodes of the element - for (size_t node_lid = 0; node_lid < 4; node_lid++) - { - for (size_t dim = 0; dim < num_dims; dim++) - { + for (size_t node_lid = 0; node_lid < 4; node_lid++) { + for (size_t dim = 0; dim < num_dims; dim++) { coords(node_lid, dim) = node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), dim); } // end for dim } // end for loop over node_lid @@ -247,10 +235,8 @@ void FEA_Module_Dynamic_Elasticity::get_timestep2D(mesh_t& mesh, // Only works for 2D // Solving for the magnitude of distance between each node size_t count = 0; - for (size_t i = 0; i < 3; i++) - { - for (size_t j = i + 1; j <= 3; j++) - { + for (size_t i = 0; i < 3; i++) { + for (size_t j = i + 1; j <= 3; j++) { // returns magnitude of distance between each node, 6 total options dist(count) = fabs( sqrt(pow((coords(i, 0) - coords(j, 0)), 2.0) @@ -262,8 +248,7 @@ void FEA_Module_Dynamic_Elasticity::get_timestep2D(mesh_t& mesh, double dist_min = dist(0); - for (int i = 0; i < 6; ++i) - { + for (int i = 0; i < 6; ++i) { dist_min = fmin(dist(i), dist_min); } @@ -274,16 +259,14 @@ void FEA_Module_Dynamic_Elasticity::get_timestep2D(mesh_t& mesh, dt_lcl_ = fmin(dt_lcl_, dt_max); // make dt small than dt_max dt_lcl_ = fmax(dt_lcl_, dt_min); // make dt larger than dt_min - if (dt_lcl_ < dt_lcl) - { + if (dt_lcl_ < dt_lcl) { dt_lcl = dt_lcl_; } }, min_dt_calc); // end parallel reduction Kokkos::fence(); // save the min dt - if (min_dt_calc < dt) - { + if (min_dt_calc < dt) { dt = min_dt_calc; } diff --git a/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/FEA_Module_Eulerian.cpp b/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/FEA_Module_Eulerian.cpp index d7f57b7bf..164cd923f 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/FEA_Module_Eulerian.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/FEA_Module_Eulerian.cpp @@ -130,8 +130,7 @@ FEA_Module_Eulerian::FEA_Module_Eulerian(Solver* Solver_Pointer, mesh_t& mesh, c node_coords_distributed = Explicit_Solver_Pointer_->node_coords_distributed; node_velocities_distributed = Explicit_Solver_Pointer_->node_velocities_distributed; all_node_velocities_distributed = Explicit_Solver_Pointer_->all_node_velocities_distributed; - if (simparam_dynamic_opt.topology_optimization_on || simparam_dynamic_opt.shape_optimization_on) - { + if (simparam_dynamic_opt.topology_optimization_on || simparam_dynamic_opt.shape_optimization_on) { all_cached_node_velocities_distributed = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); force_gradient_velocity = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); force_gradient_position = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); @@ -141,8 +140,7 @@ FEA_Module_Eulerian::FEA_Module_Eulerian(Solver* Solver_Pointer, mesh_t& mesh, c relative_element_densities = DCArrayKokkos(rnum_elem, "relative_element_densities"); } - if (simparam_dynamic_opt.topology_optimization_on || simparam_dynamic_opt.shape_optimization_on || simparam->num_dims == 2) - { + if (simparam_dynamic_opt.topology_optimization_on || simparam_dynamic_opt.shape_optimization_on || simparam->num_dims == 2) { node_masses_distributed = Teuchos::rcp(new MV(map, 1)); ghost_node_masses_distributed = Teuchos::rcp(new MV(ghost_node_map, 1)); adjoint_vector_distributed = Teuchos::rcp(new MV(map, simparam->num_dims)); @@ -175,8 +173,7 @@ FEA_Module_Eulerian::FEA_Module_Eulerian(Solver* Solver_Pointer, mesh_t& mesh, c graphics_times = simparam->graphics_options.graphics_times; graphics_id = simparam->graphics_options.graphics_id; - if (simparam_dynamic_opt.topology_optimization_on) - { + if (simparam_dynamic_opt.topology_optimization_on) { max_time_steps = BUFFER_GROW; forward_solve_velocity_data = Teuchos::rcp(new std::vector>(max_time_steps + 1)); time_data.resize(max_time_steps + 1); @@ -184,8 +181,7 @@ FEA_Module_Eulerian::FEA_Module_Eulerian(Solver* Solver_Pointer, mesh_t& mesh, c adjoint_vector_data = Teuchos::rcp(new std::vector>(max_time_steps + 1)); phi_adjoint_vector_data = Teuchos::rcp(new std::vector>(max_time_steps + 1)); // assign a multivector of corresponding size to each new timestep in the buffer - for (int istep = 0; istep < max_time_steps + 1; istep++) - { + for (int istep = 0; istep < max_time_steps + 1; istep++) { (*forward_solve_velocity_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); (*forward_solve_coordinate_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); (*adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); @@ -249,14 +245,12 @@ void FEA_Module_Eulerian::init_boundaries() int num_dim = simparam->num_dims; // set the number of boundary sets - if (myrank == 0) - { + if (myrank == 0) { std::cout << "building boundary sets " << std::endl; } // initialize to 1 since there must be at least 1 boundary set anyway; read in may occure later - if (max_boundary_sets == 0) - { + if (max_boundary_sets == 0) { max_boundary_sets = 1; } // std::cout << "NUM BOUNDARY CONDITIONS ON RANK " << myrank << " FOR INIT " << num_boundary_conditions <(nall_nodes * num_dim, "Node_DOF_Boundary_Condition_Type"); // initialize - for (int init = 0; init < nall_nodes * num_dim; init++) - { + for (int init = 0; init < nall_nodes * num_dim; init++) { Node_DOF_Boundary_Condition_Type(init) = NONE; } @@ -286,8 +279,7 @@ void FEA_Module_Eulerian::init_boundaries() ///////////////////////////////////////////////////////////////////////////// void FEA_Module_Eulerian::init_boundary_sets(int num_sets) { - if (num_sets == 0) - { + if (num_sets == 0) { std::cout << " Warning: number of boundary conditions = 0"; return; } @@ -300,14 +292,12 @@ void FEA_Module_Eulerian::init_boundary_sets(int num_sets) Boundary_Condition_Patches = CArrayKokkos(num_sets, nboundary_patches, "Boundary_Condition_Patches"); // initialize data - for (int iset = 0; iset < num_sets; iset++) - { + for (int iset = 0; iset < num_sets; iset++) { NBoundary_Condition_Patches(iset) = 0; } // initialize - for (int ibdy = 0; ibdy < num_sets; ibdy++) - { + for (int ibdy = 0; ibdy < num_sets; ibdy++) { Boundary_Condition_Type_List(ibdy) = NONE; } } @@ -325,15 +315,13 @@ void FEA_Module_Eulerian::grow_boundary_sets(int num_sets) { int num_dim = simparam->num_dims; - if (num_sets == 0) - { + if (num_sets == 0) { std::cout << " Warning: number of boundary conditions being set to 0"; return; } // std::cout << " DEBUG PRINT "< max_boundary_sets) - { + if (num_sets > max_boundary_sets) { // temporary storage for previous data CArrayKokkos Temp_Boundary_Condition_Type_List = Boundary_Condition_Type_List; CArrayKokkos Temp_NBoundary_Condition_Patches = NBoundary_Condition_Patches; @@ -347,25 +335,21 @@ void FEA_Module_Eulerian::grow_boundary_sets(int num_sets) // copy previous data back over // std::cout << "NUM BOUNDARY CONDITIONS ON RANK " << myrank << " FOR COPY " << max_boundary_sets <output_options.output_stress; int num_dim = simparam->num_dims; int Brows; - if (num_dim == 3) - { + if (num_dim == 3) { Brows = 6; } - else - { + else{ Brows = 3; } - if (output_velocity_flag) - { + if (output_velocity_flag) { // displacement_index is accessed by writers at the solver level for deformed output output_velocity_index = noutput; noutput += 1; @@ -439,13 +420,11 @@ void FEA_Module_Eulerian::init_output() output_dof_names[noutput - 1].resize(num_dim); output_dof_names[noutput - 1][0] = "vx"; output_dof_names[noutput - 1][1] = "vy"; - if (num_dim == 3) - { + if (num_dim == 3) { output_dof_names[noutput - 1][2] = "vz"; } } - if (output_strain_flag) - { + if (output_strain_flag) { output_strain_index = noutput; noutput += 1; module_outputs.resize(noutput); @@ -458,14 +437,12 @@ void FEA_Module_Eulerian::init_output() output_dof_names.resize(noutput); output_dof_names[noutput - 1].resize(Brows); - if (num_dim == 2) - { + if (num_dim == 2) { output_dof_names[noutput - 1][0] = "strain_xx"; output_dof_names[noutput - 1][1] = "strain_yy"; output_dof_names[noutput - 1][2] = "strain_xy"; } - if (num_dim == 3) - { + if (num_dim == 3) { output_dof_names[noutput - 1][0] = "strain_xx"; output_dof_names[noutput - 1][1] = "strain_yy"; output_dof_names[noutput - 1][2] = "strain_zz"; @@ -474,8 +451,7 @@ void FEA_Module_Eulerian::init_output() output_dof_names[noutput - 1][5] = "strain_yz"; } } - if (output_stress_flag) - { + if (output_stress_flag) { output_stress_index = noutput; noutput += 1; module_outputs.resize(noutput); @@ -488,14 +464,12 @@ void FEA_Module_Eulerian::init_output() output_dof_names.resize(noutput); output_dof_names[noutput - 1].resize(Brows); - if (num_dim == 2) - { + if (num_dim == 2) { output_dof_names[noutput - 1][0] = "stress_xx"; output_dof_names[noutput - 1][1] = "stress_yy"; output_dof_names[noutput - 1][3] = "stress_xy"; } - if (num_dim == 3) - { + if (num_dim == 3) { output_dof_names[noutput - 1][0] = "stress_xx"; output_dof_names[noutput - 1][1] = "stress_yy"; output_dof_names[noutput - 1][2] = "stress_zz"; @@ -612,8 +586,7 @@ void FEA_Module_Eulerian::comm_adjoint_vectors(int cycle) ///////////////////////////////////////////////////////////////////////////// void FEA_Module_Eulerian::comm_variables(Teuchos::RCP zp) { - if (simparam_dynamic_opt.topology_optimization_on) - { + if (simparam_dynamic_opt.topology_optimization_on) { // set density vector to the current value chosen by the optimizer test_node_densities_distributed = zp; @@ -633,8 +606,7 @@ void FEA_Module_Eulerian::comm_variables(Teuchos::RCP zp) // comms to get ghosts all_node_densities_distributed->doImport(*test_node_densities_distributed, *importer, Tpetra::INSERT); } - else if (simparam_dynamic_opt.shape_optimization_on) - { + else if (simparam_dynamic_opt.shape_optimization_on) { // clause to communicate boundary node data if the boundary nodes are ghosts on this rank } } @@ -681,8 +653,7 @@ void FEA_Module_Eulerian::setup() // patch ids in bdy set bdy_patches_in_set = mesh.bdy_patches_in_set; - if (num_dim == 2) - { + if (num_dim == 2) { bdy_nodes = mesh.bdy_nodes; } @@ -703,8 +674,7 @@ void FEA_Module_Eulerian::setup() // elem-node conn & node-node conn elems_in_node = mesh.elems_in_node; - if (num_dim == 2) - { + if (num_dim == 2) { nodes_in_node = mesh.nodes_in_node; num_nodes_in_node = mesh.num_nodes_in_node; // patch conn @@ -715,8 +685,7 @@ void FEA_Module_Eulerian::setup() } // initialize if topology optimization is used - if (simparam_dynamic_opt.topology_optimization_on || simparam_dynamic_opt.shape_optimization_on) - { + if (simparam_dynamic_opt.topology_optimization_on || simparam_dynamic_opt.shape_optimization_on) { // create parameter object simparam_elasticity = Simulation_Parameters_Elasticity(); init_assembly(); @@ -777,15 +746,13 @@ void FEA_Module_Eulerian::euler_solve() } */ // simple setup to just request KE for now; above loop to be expanded and used later for scanning modules - if (simparam_dynamic_opt.topology_optimization_on) - { + if (simparam_dynamic_opt.topology_optimization_on) { obj_pointer = problem->getObjective(); KineticEnergyMinimize_TopOpt& kinetic_energy_minimize_function = dynamic_cast(*obj_pointer); kinetic_energy_minimize_function.objective_accumulation = 0; global_objective_accumulation = objective_accumulation = 0; kinetic_energy_objective = true; - if (max_time_steps + 1 > forward_solve_velocity_data->size()) - { + if (max_time_steps + 1 > forward_solve_velocity_data->size()) { old_max_forward_buffer = forward_solve_velocity_data->size(); time_data.resize(max_time_steps + 1); forward_solve_velocity_data->resize(max_time_steps + 1); @@ -793,8 +760,7 @@ void FEA_Module_Eulerian::euler_solve() adjoint_vector_data->resize(max_time_steps + 1); phi_adjoint_vector_data->resize(max_time_steps + 1); // assign a multivector of corresponding size to each new timestep in the buffer - for (int istep = old_max_forward_buffer; istep < max_time_steps + 1; istep++) - { + for (int istep = old_max_forward_buffer; istep < max_time_steps + 1; istep++) { (*forward_solve_velocity_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); (*forward_solve_coordinate_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); (*adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); @@ -803,16 +769,13 @@ void FEA_Module_Eulerian::euler_solve() } } - if (simparam_dynamic_opt.topology_optimization_on) - { + if (simparam_dynamic_opt.topology_optimization_on) { nTO_modules = simparam_dynamic_opt.TO_Module_List.size(); } int myrank = Explicit_Solver_Pointer_->myrank; - if (simparam->output_options.output_file_format == OUTPUT_FORMAT::vtk) - { - if (myrank == 0) - { + if (simparam->output_options.output_file_format == OUTPUT_FORMAT::vtk) { + if (myrank == 0) { printf("Writing outputs to file at %f \n", time_value); } @@ -880,17 +843,14 @@ void FEA_Module_Eulerian::euler_solve() // extensive KE REDUCE_SUM_CLASS(node_gid, 0, nlocal_nodes, KE_loc_sum, { double ke = 0; - for (size_t dim = 0; dim < num_dim; dim++) - { + for (size_t dim = 0; dim < num_dim; dim++) { ke += node_vel(rk_level, node_gid, dim) * node_vel(rk_level, node_gid, dim); // 1/2 at end } // end for - if (num_dim == 2) - { + if (num_dim == 2) { KE_loc_sum += node_mass(node_gid) * node_coords(rk_level, node_gid, 1) * ke; } - else - { + else{ KE_loc_sum += node_mass(node_gid) * ke; } }, KE_sum); @@ -908,8 +868,7 @@ void FEA_Module_Eulerian::euler_solve() // save the nodal mass FOR_ALL_CLASS(node_gid, 0, nall_nodes, { double radius = 1.0; - if (num_dim == 2) - { + if (num_dim == 2) { radius = node_coords(rk_level, node_gid, 1); } node_extensive_mass(node_gid) = node_mass(node_gid) * radius; @@ -921,8 +880,7 @@ void FEA_Module_Eulerian::euler_solve() auto time_1 = std::chrono::high_resolution_clock::now(); // save initial data - if (simparam_dynamic_opt.topology_optimization_on || simparam_dynamic_opt.shape_optimization_on) - { + if (simparam_dynamic_opt.topology_optimization_on || simparam_dynamic_opt.shape_optimization_on) { time_data[0] = 0; // assign current velocity data to multivector // view scope @@ -930,8 +888,7 @@ void FEA_Module_Eulerian::euler_solve() vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (int idim = 0; idim < num_dim; idim++) - { + for (int idim = 0; idim < num_dim; idim++) { node_velocities_interface(node_gid, idim) = node_vel(rk_level, node_gid, idim); node_coords_interface(node_gid, idim) = node_coords(rk_level, node_gid, idim); } @@ -967,8 +924,7 @@ void FEA_Module_Eulerian::euler_solve() vec_array all_node_velocities_interface = Explicit_Solver_Pointer_->all_node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (int idim = 0; idim < num_dim; idim++) - { + for (int idim = 0; idim < num_dim; idim++) { all_node_velocities_interface(node_gid, idim) = node_velocities_interface(node_gid, idim); all_node_coords_interface(node_gid, idim) = node_coords_interface(node_gid, idim); } @@ -976,8 +932,7 @@ void FEA_Module_Eulerian::euler_solve() Kokkos::fence(); FOR_ALL_CLASS(node_gid, nlocal_nodes, nlocal_nodes + nghost_nodes, { - for (int idim = 0; idim < num_dim; idim++) - { + for (int idim = 0; idim < num_dim; idim++) { all_node_velocities_interface(node_gid, idim) = ghost_node_velocities_interface(node_gid - nlocal_nodes, idim); all_node_coords_interface(node_gid, idim) = ghost_node_coords_interface(node_gid - nlocal_nodes, idim); } @@ -990,19 +945,16 @@ void FEA_Module_Eulerian::euler_solve() } // loop over the max number of time integration cycles - for (cycle = 0; cycle < cycle_stop; cycle++) - { + for (cycle = 0; cycle < cycle_stop; cycle++) { // get the step - if (num_dim == 2) - { + if (num_dim == 2) { get_timestep2D(mesh, node_coords, node_vel, elem_sspd, elem_vol); } - else - { + else{ get_timestep(mesh, node_coords, node_vel, @@ -1017,18 +969,14 @@ void FEA_Module_Eulerian::euler_solve() // stop calculation if flag // if (stop_calc == 1) break; - if (cycle == 0) - { - if (myrank == 0) - { + if (cycle == 0) { + if (myrank == 0) { printf("cycle = %lu, time = %12.5e, time step = %12.5e \n", cycle, time_value, dt); } } // print time step every 10 cycles - else if (cycle % 20 == 0) - { - if (myrank == 0) - { + else if (cycle % 20 == 0) { + if (myrank == 0) { printf("cycle = %lu, time = %12.5e, time step = %12.5e \n", cycle, time_value, dt); } } // end if @@ -1046,8 +994,7 @@ void FEA_Module_Eulerian::euler_solve() nall_nodes); // integrate solution forward in time - for (size_t rk_stage = 0; rk_stage < rk_num_stages; rk_stage++) - { + for (size_t rk_stage = 0; rk_stage < rk_num_stages; rk_stage++) { // ---- RK coefficient ---- double rk_alpha = 1.0 / ((double)rk_num_stages - (double)rk_stage); @@ -1058,8 +1005,7 @@ void FEA_Module_Eulerian::euler_solve() { vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (int idim = 0; idim < num_dim; idim++) - { + for (int idim = 0; idim < num_dim; idim++) { node_velocities_interface(node_gid, idim) = node_vel(rk_level, node_gid, idim); } }); // end parallel for @@ -1082,8 +1028,7 @@ void FEA_Module_Eulerian::euler_solve() vec_array ghost_node_velocities_interface = Explicit_Solver_Pointer_->ghost_node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); FOR_ALL_CLASS(node_gid, nlocal_nodes, nall_nodes, { - for (int idim = 0; idim < num_dim; idim++) - { + for (int idim = 0; idim < num_dim; idim++) { node_vel(rk_level, node_gid, idim) = ghost_node_velocities_interface(node_gid - nlocal_nodes, idim); } }); // end parallel for @@ -1102,14 +1047,12 @@ void FEA_Module_Eulerian::euler_solve() */ // calculate the new corner masses if 2D - if (num_dim == 2) - { + if (num_dim == 2) { // calculate the nodal areal mass FOR_ALL_CLASS(node_gid, 0, nall_nodes, { node_mass(node_gid) = 0.0; - if (node_coords(rk_level, node_gid, 1) > tiny) - { + if (node_coords(rk_level, node_gid, 1) > tiny) { node_mass(node_gid) = node_extensive_mass(node_gid) / node_coords(rk_level, node_gid, 1); } // if(cycle==0&&node_gid==1&&myrank==0) @@ -1192,17 +1135,14 @@ void FEA_Module_Eulerian::euler_solve() // FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { size_t node_gid = bdy_nodes(node_bdy_gid); - if (node_coords(rk_level, node_gid, 1) < tiny) - { + if (node_coords(rk_level, node_gid, 1) < tiny) { // node is on the axis - for (size_t node_lid = 0; node_lid < num_nodes_in_node(node_gid); node_lid++) - { + for (size_t node_lid = 0; node_lid < num_nodes_in_node(node_gid); node_lid++) { size_t node_neighbor_gid = nodes_in_node(node_gid, node_lid); // if the node is off the axis, use it's areal mass on the boundary - if (node_coords(rk_level, node_neighbor_gid, 1) > tiny) - { + if (node_coords(rk_level, node_neighbor_gid, 1) > tiny) { node_mass(node_gid) = fmax(node_mass(node_gid), node_mass(node_neighbor_gid) / 2.0); } } // end for over neighboring nodes @@ -1215,15 +1155,12 @@ void FEA_Module_Eulerian::euler_solve() time_value += dt; simparam->time_value = time_value; - if (simparam_dynamic_opt.topology_optimization_on || simparam_dynamic_opt.shape_optimization_on) - { - if (cycle >= max_time_steps) - { + if (simparam_dynamic_opt.topology_optimization_on || simparam_dynamic_opt.shape_optimization_on) { + if (cycle >= max_time_steps) { max_time_steps = cycle + 1; } - if (max_time_steps + 1 > forward_solve_velocity_data->size()) - { + if (max_time_steps + 1 > forward_solve_velocity_data->size()) { old_max_forward_buffer = forward_solve_velocity_data->size(); time_data.resize(max_time_steps + BUFFER_GROW + 1); forward_solve_velocity_data->resize(max_time_steps + BUFFER_GROW + 1); @@ -1231,8 +1168,7 @@ void FEA_Module_Eulerian::euler_solve() adjoint_vector_data->resize(max_time_steps + BUFFER_GROW + 1); phi_adjoint_vector_data->resize(max_time_steps + BUFFER_GROW + 1); // assign a multivector of corresponding size to each new timestep in the buffer - for (int istep = old_max_forward_buffer; istep < max_time_steps + BUFFER_GROW + 1; istep++) - { + for (int istep = old_max_forward_buffer; istep < max_time_steps + BUFFER_GROW + 1; istep++) { (*forward_solve_velocity_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); (*forward_solve_coordinate_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); (*adjoint_vector_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); @@ -1248,8 +1184,7 @@ void FEA_Module_Eulerian::euler_solve() vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (int idim = 0; idim < num_dim; idim++) - { + for (int idim = 0; idim < num_dim; idim++) { node_velocities_interface(node_gid, idim) = node_vel(rk_level, node_gid, idim); node_coords_interface(node_gid, idim) = node_coords(rk_level, node_gid, idim); } @@ -1283,8 +1218,7 @@ void FEA_Module_Eulerian::euler_solve() const_vec_array ghost_node_coords_interface = Explicit_Solver_Pointer_->ghost_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (int idim = 0; idim < num_dim; idim++) - { + for (int idim = 0; idim < num_dim; idim++) { all_node_velocities_interface(node_gid, idim) = node_velocities_interface(node_gid, idim); all_node_coords_interface(node_gid, idim) = node_coords_interface(node_gid, idim); } @@ -1292,8 +1226,7 @@ void FEA_Module_Eulerian::euler_solve() Kokkos::fence(); FOR_ALL_CLASS(node_gid, nlocal_nodes, nlocal_nodes + nghost_nodes, { - for (int idim = 0; idim < num_dim; idim++) - { + for (int idim = 0; idim < num_dim; idim++) { all_node_velocities_interface(node_gid, idim) = ghost_node_velocities_interface(node_gid - nlocal_nodes, idim); all_node_coords_interface(node_gid, idim) = ghost_node_coords_interface(node_gid - nlocal_nodes, idim); } @@ -1309,8 +1242,7 @@ void FEA_Module_Eulerian::euler_solve() (*forward_solve_coordinate_data)[cycle + 1]->assign(*Explicit_Solver_Pointer_->all_node_coords_distributed); // kinetic energy accumulation - if (kinetic_energy_objective) - { + if (kinetic_energy_objective) { const_vec_array node_velocities_interface = (*forward_solve_velocity_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); const_vec_array previous_node_velocities_interface = (*forward_solve_velocity_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); KE_loc_sum = 0.0; @@ -1318,19 +1250,16 @@ void FEA_Module_Eulerian::euler_solve() // extensive KE REDUCE_SUM_CLASS(node_gid, 0, nlocal_nodes, KE_loc_sum, { double ke = 0; - for (size_t dim = 0; dim < num_dim; dim++) - { + for (size_t dim = 0; dim < num_dim; dim++) { // midpoint integration approximation ke += (node_velocities_interface(node_gid, dim) + node_velocities_interface(node_gid, dim)) * (node_velocities_interface(node_gid, dim) + node_velocities_interface(node_gid, dim)) / 4; // 1/2 at end } // end for - if (num_dim == 2) - { + if (num_dim == 2) { KE_loc_sum += node_mass(node_gid) * node_coords(rk_level, node_gid, 1) * ke; } - else - { + else{ KE_loc_sum += node_mass(node_gid) * ke; } }, KE_sum); @@ -1341,41 +1270,33 @@ void FEA_Module_Eulerian::euler_solve() } // end topology optimization if check size_t write = 0; - if ((cycle + 1) % graphics_cyc_ival == 0 && cycle > 0) - { + if ((cycle + 1) % graphics_cyc_ival == 0 && cycle > 0) { write = 1; } - else if (cycle == cycle_stop) - { + else if (cycle == cycle_stop) { write = 1; } - else if (time_value >= time_final) - { + else if (time_value >= time_final) { write = 1; } - else if (time_value >= graphics_time) - { + else if (time_value >= graphics_time) { write = 1; } // write outputs - if (write == 1) - { + if (write == 1) { // interface nodal coordinate data (note: this is not needed if using write_outputs_new()) // view scope { vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (int idim = 0; idim < num_dim; idim++) - { + for (int idim = 0; idim < num_dim; idim++) { node_coords_interface(node_gid, idim) = node_coords(rk_level, node_gid, idim); } }); // end parallel for } // end view scope - if (simparam->output_options.output_file_format == OUTPUT_FORMAT::vtk) - { - if (myrank == 0) - { + if (simparam->output_options.output_file_format == OUTPUT_FORMAT::vtk) { + if (myrank == 0) { printf("Writing outputs to file at %f \n", graphics_time); } @@ -1410,8 +1331,7 @@ void FEA_Module_Eulerian::euler_solve() } // end if // end of calculation - if (time_value >= time_final) - { + if (time_value >= time_final) { break; } } // end for cycle loop @@ -1419,16 +1339,14 @@ void FEA_Module_Eulerian::euler_solve() last_time_step = cycle; // simple setup to just calculate KE minimize objective for now - if (simparam_dynamic_opt.topology_optimization_on) - { + if (simparam_dynamic_opt.topology_optimization_on) { KineticEnergyMinimize_TopOpt& kinetic_energy_minimize_function = dynamic_cast(*obj_pointer); // collect local objective values MPI_Allreduce(&objective_accumulation, &global_objective_accumulation, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); kinetic_energy_minimize_function.objective_accumulation = global_objective_accumulation; - if (myrank == 0) - { + if (myrank == 0) { std::cout << "CURRENT TIME INTEGRAL OF KINETIC ENERGY " << global_objective_accumulation << std::endl; } } @@ -1437,8 +1355,7 @@ void FEA_Module_Eulerian::euler_solve() auto time_difference = time_2 - time_1; // double calc_time = std::chrono::duration_cast(diff).count(); double calc_time = std::chrono::duration_cast(time_difference).count(); - if (myrank == 0) - { + if (myrank == 0) { printf("\nCalculation time in seconds: %f \n", calc_time * 1e-09); } @@ -1459,17 +1376,14 @@ void FEA_Module_Eulerian::euler_solve() // extensive KE REDUCE_SUM_CLASS(node_gid, 0, nlocal_nodes, KE_loc_sum, { double ke = 0; - for (size_t dim = 0; dim < num_dim; dim++) - { + for (size_t dim = 0; dim < num_dim; dim++) { ke += node_vel(rk_level, node_gid, dim) * node_vel(rk_level, node_gid, dim); // 1/2 at end } // end for - if (num_dim == 2) - { + if (num_dim == 2) { KE_loc_sum += node_mass(node_gid) * node_coords(rk_level, node_gid, 1) * ke; } - else - { + else{ KE_loc_sum += node_mass(node_gid) * ke; } }, KE_sum); @@ -1489,16 +1403,13 @@ void FEA_Module_Eulerian::euler_solve() // reduce over MPI ranks - if (myrank == 0) - { + if (myrank == 0) { printf("Time=0: KE = %20.15f, IE = %20.15f, TE = %20.15f \n", KE_t0, IE_t0, TE_t0); } - if (myrank == 0) - { + if (myrank == 0) { printf("Time=End: KE = %20.15f, IE = %20.15f, TE = %20.15f \n", KE_tend, IE_tend, TE_tend); } - if (myrank == 0) - { + if (myrank == 0) { printf("total energy conservation error %= %e \n\n", 100 * (TE_tend - TE_t0) / TE_t0); } diff --git a/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/Simulation_Parameters_Eulerian.h b/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/Simulation_Parameters_Eulerian.h index 9bb4ad56d..e7770e606 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/Simulation_Parameters_Eulerian.h +++ b/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/Simulation_Parameters_Eulerian.h @@ -94,17 +94,14 @@ struct Simulation_Parameters_Eulerian : Simulation_Parameters eos_global_vars = DCArrayKokkos(material_options.size(), num_eos_global_vars); strength_global_vars = DCArrayKokkos(material_options.size(), num_strength_global_vars); - for (size_t i = 0; i < material_options.size(); i++) - { + for (size_t i = 0; i < material_options.size(); i++) { auto mat = material_options[i]; - for (size_t j = 0; j < mat.eos_global_vars.size(); j++) - { + for (size_t j = 0; j < mat.eos_global_vars.size(); j++) { eos_global_vars.host(i, j) = mat.eos_global_vars[j]; } - for (size_t j = 0; j < mat.strength_global_vars.size(); j++) - { + for (size_t j = 0; j < mat.strength_global_vars.size(); j++) { strength_global_vars.host(i, j) = mat.strength_global_vars[j]; } } @@ -123,8 +120,7 @@ struct Simulation_Parameters_Eulerian : Simulation_Parameters template void from_vector(DCArrayKokkos& array, const std::vector& vec) { array = DCArrayKokkos(vec.size()); - for (size_t i = 0; i < vec.size(); i++) - { + for (size_t i = 0; i < vec.size(); i++) { array.host(i) = *(T*)&vec[i]; } } @@ -140,8 +136,7 @@ struct Simulation_Parameters_Eulerian : Simulation_Parameters { max_num_eos_global_vars = 0; max_num_strength_global_vars = 0; - for (auto mo : material_options) - { + for (auto mo : material_options) { max_num_eos_global_vars = std::max(max_num_eos_global_vars, mo.eos_global_vars.size()); max_num_strength_global_vars = std::max(max_num_strength_global_vars, mo.strength_global_vars.size()); } diff --git a/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/eulerian_geometry.cpp b/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/eulerian_geometry.cpp index c7522856a..9233e96be 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/eulerian_geometry.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/eulerian_geometry.cpp @@ -30,8 +30,7 @@ void FEA_Module_Eulerian::example_function(double rk_alpha, // example device loop FOR_ALL_CLASS(node_gid, 0, num_nodes, { - for (int dim = 0; dim < num_dims; dim++) - { + for (int dim = 0; dim < num_dims; dim++) { double half_vel = (node_vel(rk_level, node_gid, dim) + node_vel(0, node_gid, dim)) * 0.5; node_coords(rk_level, node_gid, dim) = node_coords(0, node_gid, dim) + rk_alpha * dt * half_vel; } diff --git a/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/eulerian_time_integration.cpp b/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/eulerian_time_integration.cpp index a65205ad8..d6bdb6e4c 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/eulerian_time_integration.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Eulerian_Solver/eulerian_time_integration.cpp @@ -61,10 +61,8 @@ void FEA_Module_Eulerian::rk_init(DViewCArrayKokkos& node_coords, // save elem quantities FOR_ALL_CLASS(elem_gid, 0, num_elems, { // stress is always 3D even with 2D-RZ - for (size_t i = 0; i < 3; i++) - { - for (size_t j = 0; j < 3; j++) - { + for (size_t i = 0; i < 3; i++) { + for (size_t j = 0; j < 3; j++) { elem_stress(0, elem_gid, i, j) = elem_stress(rk_level, elem_gid, i, j); } } // end for @@ -74,8 +72,7 @@ void FEA_Module_Eulerian::rk_init(DViewCArrayKokkos& node_coords, // save nodal quantities FOR_ALL_CLASS(node_gid, 0, num_nodes, { - for (size_t i = 0; i < num_dims; i++) - { + for (size_t i = 0; i < num_dims; i++) { node_coords(0, node_gid, i) = node_coords(rk_level, node_gid, i); node_vel(0, node_gid, i) = node_vel(rk_level, node_gid, i); } @@ -124,10 +121,8 @@ void FEA_Module_Eulerian::get_timestep(mesh_t& mesh, ViewCArrayKokkos dist(distance0, 28); // Getting the coordinates of the element - for (size_t node_lid = 0; node_lid < 8; node_lid++) - { - for (size_t dim = 0; dim < num_dims; dim++) - { + for (size_t node_lid = 0; node_lid < 8; node_lid++) { + for (size_t dim = 0; dim < num_dims; dim++) { coords(node_lid, dim) = node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), dim); } // end for dim } // end for loop over node_lid @@ -141,8 +136,7 @@ void FEA_Module_Eulerian::get_timestep(mesh_t& mesh, // Only works for 3D // Solving for the magnitude of distance between each node - for (size_t i = 0; i < 28; i++) - { + for (size_t i = 0; i < 28; i++) { a = countA; b = countB; @@ -155,8 +149,7 @@ void FEA_Module_Eulerian::get_timestep(mesh_t& mesh, countA++; // tricky indexing - if (countB > 7) - { + if (countB > 7) { loop++; countB = 1 + loop; countA = 0; @@ -165,8 +158,7 @@ void FEA_Module_Eulerian::get_timestep(mesh_t& mesh, double dist_min = dist(0); - for (int i = 0; i < 28; ++i) - { + for (int i = 0; i < 28; ++i) { dist_min = fmin(dist(i), dist_min); } @@ -177,16 +169,14 @@ void FEA_Module_Eulerian::get_timestep(mesh_t& mesh, dt_lcl_ = fmin(dt_lcl_, dt_max); // make dt small than dt_max dt_lcl_ = fmax(dt_lcl_, dt_min); // make dt larger than dt_min - if (dt_lcl_ < dt_lcl) - { + if (dt_lcl_ < dt_lcl) { dt_lcl = dt_lcl_; } }, min_dt_calc); // end parallel reduction Kokkos::fence(); // save the min dt - if (min_dt_calc < dt) - { + if (min_dt_calc < dt) { dt = min_dt_calc; } @@ -239,10 +229,8 @@ void FEA_Module_Eulerian::get_timestep2D(mesh_t& mesh, ViewCArrayKokkos dist(distance0, 6); // Getting the coordinates of the nodes of the element - for (size_t node_lid = 0; node_lid < 4; node_lid++) - { - for (size_t dim = 0; dim < num_dims; dim++) - { + for (size_t node_lid = 0; node_lid < 4; node_lid++) { + for (size_t dim = 0; dim < num_dims; dim++) { coords(node_lid, dim) = node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), dim); } // end for dim } // end for loop over node_lid @@ -250,10 +238,8 @@ void FEA_Module_Eulerian::get_timestep2D(mesh_t& mesh, // Only works for 2D // Solving for the magnitude of distance between each node size_t count = 0; - for (size_t i = 0; i < 3; i++) - { - for (size_t j = i + 1; j <= 3; j++) - { + for (size_t i = 0; i < 3; i++) { + for (size_t j = i + 1; j <= 3; j++) { // returns magnitude of distance between each node, 6 total options dist(count) = fabs( sqrt(pow((coords(i, 0) - coords(j, 0)), 2.0) @@ -265,8 +251,7 @@ void FEA_Module_Eulerian::get_timestep2D(mesh_t& mesh, double dist_min = dist(0); - for (int i = 0; i < 6; ++i) - { + for (int i = 0; i < 6; ++i) { dist_min = fmin(dist(i), dist_min); } @@ -277,16 +262,14 @@ void FEA_Module_Eulerian::get_timestep2D(mesh_t& mesh, dt_lcl_ = fmin(dt_lcl_, dt_max); // make dt small than dt_max dt_lcl_ = fmax(dt_lcl_, dt_min); // make dt larger than dt_min - if (dt_lcl_ < dt_lcl) - { + if (dt_lcl_ < dt_lcl) { dt_lcl = dt_lcl_; } }, min_dt_calc); // end parallel reduction Kokkos::fence(); // save the min dt - if (min_dt_calc < dt) - { + if (min_dt_calc < dt) { dt = min_dt_calc; } diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/mesh.h b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/mesh.h index 09fb070ff..883f4f156 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/mesh.h +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/mesh.h @@ -82,12 +82,9 @@ patch 6: [4,5,6,7] zeta-plus dir KOKKOS_INLINE_FUNCTION void bubble_sort(size_t arr[], const size_t num) { - for (size_t i = 0; i < (num - 1); i++) - { - for (size_t j = 0; j < (num - i - 1); j++) - { - if (arr[j] > arr[j + 1]) - { + for (size_t i = 0; i < (num - 1); i++) { + for (size_t j = 0; j < (num - i - 1); j++) { + if (arr[j] > arr[j + 1]) { size_t temp = arr[j]; arr[j] = arr[j + 1]; arr[j + 1] = temp; @@ -214,8 +211,7 @@ struct mesh_t { num_dims = num_dims_inp; num_nodes_in_elem = 1; - for (int dim = 0; dim < num_dims; dim++) - { + for (int dim = 0; dim < num_dims; dim++) { num_nodes_in_elem *= 2; } num_elems = num_elems_inp; @@ -260,8 +256,7 @@ struct mesh_t num_corners_in_node(node_gid) = 0; }); - for (size_t elem_gid = 0; elem_gid < num_elems; elem_gid++) - { + for (size_t elem_gid = 0; elem_gid < num_elems; elem_gid++) { FOR_ALL_CLASS(node_lid, 0, num_nodes_in_elem, { // get the global_id of the node size_t node_gid = nodes_in_elem(elem_gid, node_lid); @@ -285,8 +280,7 @@ struct mesh_t elems_in_node = RaggedRightArrayKokkos(num_corners_in_node, "elems_in_node"); // populate the elems connected to a node list and corners in a node - for (size_t elem_gid = 0; elem_gid < num_elems; elem_gid++) - { + for (size_t elem_gid = 0; elem_gid < num_elems; elem_gid++) { FOR_ALL_CLASS(node_lid, 0, num_nodes_in_elem, { // get the global_id of the node size_t node_gid = nodes_in_elem(elem_gid, node_lid); @@ -328,8 +322,7 @@ struct mesh_t // num_corners_in_node = num_elems_in_node size_t max_num = num_corners_in_node(node_gid); - if (max_num > max_num_lcl) - { + if (max_num > max_num_lcl) { max_num_lcl = max_num; } }, max_num_elems_in_node); // end parallel reduction on max @@ -346,14 +339,12 @@ struct mesh_t // find and save neighboring elem_gids of an elem FOR_ALL_CLASS(elem_gid, 0, num_elems, { - for (int node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) - { + for (int node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { // get the gid for the node size_t node_id = nodes_in_elem(elem_gid, node_lid); // loop over all elems connected to node_gid - for (int elem_lid = 0; elem_lid < num_corners_in_node(node_id); elem_lid++) - { + for (int elem_lid = 0; elem_lid < num_corners_in_node(node_id); elem_lid++) { // get the global id for the neighboring elem size_t neighbor_elem_gid = elems_in_node(node_id, elem_lid); @@ -361,23 +352,19 @@ struct mesh_t size_t save = 1; // a true neighbor_elem_id is not equal to elem_gid - if (neighbor_elem_gid == elem_gid) - { + if (neighbor_elem_gid == elem_gid) { save = 0; // don't save } // end if // check to see if the neighbor_elem_gid has been saved already size_t num_saved = temp_elems_in_elem.stride(elem_gid); - for (size_t i = 0; i < num_saved; i++) - { - if (neighbor_elem_gid == temp_elems_in_elem(elem_gid, i)) - { + for (size_t i = 0; i < num_saved; i++) { + if (neighbor_elem_gid == temp_elems_in_elem(elem_gid, i)) { save = 0; // don't save, it has been saved already } // end if } // end for i - if (save == 1) - { + if (save == 1) { // increment the number of neighboring elements saved temp_elems_in_elem.stride(elem_gid)++; @@ -396,8 +383,7 @@ struct mesh_t elems_in_elem = RaggedRightArrayKokkos(num_elems_in_elem, "elems_in_elem"); FOR_ALL_CLASS(elem_gid, 0, num_elems, { - for (size_t i = 0; i < num_elems_in_elem(elem_gid); i++) - { + for (size_t i = 0; i < num_elems_in_elem(elem_gid); i++) { elems_in_elem(elem_gid, i) = temp_elems_in_elem(elem_gid, i); } // end for i }); // end FOR_ALL elems @@ -427,8 +413,7 @@ struct mesh_t size_t node_lids_in_patch_in_elem[24]; - if (num_dims == 3) - { + if (num_dims == 3) { size_t temp_node_lids[24] = { 0, 4, 7, 3, 1, 2, 6, 5, 0, 1, 5, 4, @@ -436,13 +421,11 @@ struct mesh_t 0, 3, 2, 1, 4, 5, 6, 7 }; - for (size_t i = 0; i < 24; i++) - { + for (size_t i = 0; i < 24; i++) { node_lids_in_patch_in_elem[i] = temp_node_lids[i]; } // end for i } - else - { + else{ // J // | // 3---2 @@ -455,8 +438,7 @@ struct mesh_t 0, 1, 3, 2 }; - for (size_t i = 0; i < 8; i++) - { + for (size_t i = 0; i < 8; i++) { node_lids_in_patch_in_elem[i] = temp_node_lids[i]; } // end for i } // end if on dims @@ -477,13 +459,11 @@ struct mesh_t // step 1) calculate the hash values for each patch in the element FOR_ALL_CLASS(elem_gid, 0, num_elems, { - for (size_t patch_lid = 0; patch_lid < num_patches_in_elem; patch_lid++) - { + for (size_t patch_lid = 0; patch_lid < num_patches_in_elem; patch_lid++) { size_t sorted_patch_nodes[4]; // note: cannot be allocated with num_nodes_in_patch // first save the patch nodes - for (size_t patch_node_lid = 0; patch_node_lid < num_nodes_in_patch; patch_node_lid++) - { + for (size_t patch_node_lid = 0; patch_node_lid < num_nodes_in_patch; patch_node_lid++) { // get the local node index of the element for this patch and node in patch size_t node_lid = node_ordering_in_elem(patch_lid, patch_node_lid); @@ -495,12 +475,10 @@ struct mesh_t bubble_sort(sorted_patch_nodes, num_nodes_in_patch); long long int hash_key; - if (num_dims == 2) - { + if (num_dims == 2) { hash_key = (sorted_patch_nodes[0] + num_nodes * sorted_patch_nodes[1]); } - else - { + else{ hash_key = (sorted_patch_nodes[1] + num_nodes * sorted_patch_nodes[2] + num_nodes * num_nodes * sorted_patch_nodes[3]); // 3 largest node values } // end if on dims @@ -518,29 +496,23 @@ struct mesh_t size_t patch_gid = 0; size_t bdy_patch_gid = 0; - for (size_t elem_gid = 0; elem_gid < num_elems; elem_gid++) - { - for (size_t patch_lid = 0; patch_lid < num_patches_in_elem; patch_lid++) - { + for (size_t elem_gid = 0; elem_gid < num_elems; elem_gid++) { + for (size_t patch_lid = 0; patch_lid < num_patches_in_elem; patch_lid++) { long long int hash_key = hash_keys_in_elem(elem_gid, patch_lid); size_t exit = 0; - if (hash_key < 0) - { + if (hash_key < 0) { // find the nighboring patch with the same hash_key - for (size_t neighbor_elem_lid = 0; neighbor_elem_lid < num_elems_in_elem(elem_gid); neighbor_elem_lid++) - { + for (size_t neighbor_elem_lid = 0; neighbor_elem_lid < num_elems_in_elem(elem_gid); neighbor_elem_lid++) { // get the neighboring element global index size_t neighbor_elem_gid = elems_in_elem(elem_gid, neighbor_elem_lid); - for (size_t neighbor_patch_lid = 0; neighbor_patch_lid < num_patches_in_elem; neighbor_patch_lid++) - { + for (size_t neighbor_patch_lid = 0; neighbor_patch_lid < num_patches_in_elem; neighbor_patch_lid++) { // this hash is from the nodes on the patch long long int neighbor_hash_key = hash_keys_in_elem(neighbor_elem_gid, neighbor_patch_lid); - if (neighbor_hash_key == hash_keys_in_elem(elem_gid, patch_lid)) - { + if (neighbor_hash_key == hash_keys_in_elem(elem_gid, patch_lid)) { // save the respective elem_gid's as they are patch neighbors hash_keys_in_elem(elem_gid, patch_lid) = neighbor_elem_gid; hash_keys_in_elem(neighbor_elem_gid, neighbor_patch_lid) = elem_gid; @@ -560,8 +532,7 @@ struct mesh_t } // end if } // end for loop over a neighbors patch set - if (exit == 1) - { + if (exit == 1) { break; } } // end for loop over elem neighbors @@ -569,10 +540,8 @@ struct mesh_t } // end for patch_lid // remaining negative hash key values are the boundary patches - for (size_t patch_lid = 0; patch_lid < num_patches_in_elem; patch_lid++) - { - if (hash_keys_in_elem(elem_gid, patch_lid) < 0) - { + for (size_t patch_lid = 0; patch_lid < num_patches_in_elem; patch_lid++) { + if (hash_keys_in_elem(elem_gid, patch_lid) < 0) { hash_keys_in_elem(elem_gid, patch_lid) = elem_gid; // neighboring_side_lids(elem_gid, patch_lid) = patch_lid; @@ -616,8 +585,7 @@ struct mesh_t num_elems_in_patch_saved(patch_gid) = 0; }); - for (size_t elem_gid = 0; elem_gid < num_elems; elem_gid++) - { + for (size_t elem_gid = 0; elem_gid < num_elems; elem_gid++) { FOR_ALL_CLASS(patch_lid, 0, num_patches_in_elem, { size_t patch_gid = patches_in_elem(elem_gid, patch_lid); @@ -629,8 +597,7 @@ struct mesh_t num_elems_in_patch_saved(patch_gid)++; // save the nodes on this patch - for (size_t patch_node_lid = 0; patch_node_lid < num_nodes_in_patch; patch_node_lid++) - { + for (size_t patch_node_lid = 0; patch_node_lid < num_nodes_in_patch; patch_node_lid++) { // get the local node index of the element for this patch and node in patch size_t node_lid = node_ordering_in_elem(patch_lid, patch_node_lid); @@ -660,18 +627,15 @@ struct mesh_t RUN_CLASS({ num_bdy_nodes_saved(0) = 0; - for (size_t bdy_patch_gid = 0; bdy_patch_gid < num_bdy_patches; bdy_patch_gid++) - { + for (size_t bdy_patch_gid = 0; bdy_patch_gid < num_bdy_patches; bdy_patch_gid++) { // get the global index of the patch that is on the boundary size_t patch_gid = bdy_patches(bdy_patch_gid); // tag the boundary nodes - for (size_t node_lid = 0; node_lid < num_nodes_in_patch; node_lid++) - { + for (size_t node_lid = 0; node_lid < num_nodes_in_patch; node_lid++) { size_t node_gid = nodes_in_patch(patch_gid, node_lid); - if (hash_bdy_nodes(node_gid) < 0) - { + if (hash_bdy_nodes(node_gid) < 0) { hash_bdy_nodes(node_gid) = node_gid; temp_bdy_nodes(num_bdy_nodes_saved(0)) = node_gid; @@ -717,8 +681,7 @@ struct mesh_t // num_corners_in_node = num_elems_in_node size_t max_num = num_corners_in_node(node_gid); - if (max_num > max_num_lcl) - { + if (max_num > max_num_lcl) { max_num_lcl = max_num; } }, max_num_elems_in_node); // end parallel reduction on max @@ -732,24 +695,19 @@ struct mesh_t // walk over the patches and save the node node connectivity RUN_CLASS({ - if (num_dims == 3) - { - for (size_t patch_gid = 0; patch_gid < num_patches; patch_gid++) - { - for (size_t node_lid = 0; node_lid < num_nodes_in_patch; node_lid++) - { + if (num_dims == 3) { + for (size_t patch_gid = 0; patch_gid < num_patches; patch_gid++) { + for (size_t node_lid = 0; node_lid < num_nodes_in_patch; node_lid++) { // the first node on the edge size_t node_gid_0 = nodes_in_patch(patch_gid, node_lid); // second node on this edge size_t node_gid_1; - if (node_lid == num_nodes_in_patch - 1) - { + if (node_lid == num_nodes_in_patch - 1) { node_gid_1 = nodes_in_patch(patch_gid, 0); } - else - { + else{ node_gid_1 = nodes_in_patch(patch_gid, node_lid + 1); } // end if @@ -760,25 +718,20 @@ struct mesh_t size_t save_1 = 1; // check to see if the node_gid_1 was already saved - for (size_t contents_lid = 0; contents_lid < num_saved_0; contents_lid++) - { - if (temp_nodes_in_nodes(node_gid_0, contents_lid) == node_gid_1) - { + for (size_t contents_lid = 0; contents_lid < num_saved_0; contents_lid++) { + if (temp_nodes_in_nodes(node_gid_0, contents_lid) == node_gid_1) { save_0 = 0; // don't save, it was already saved } } // check to see if the node_gid_0 was already saved - for (size_t contents_lid = 0; contents_lid < num_saved_1; contents_lid++) - { - if (temp_nodes_in_nodes(node_gid_1, contents_lid) == node_gid_0) - { + for (size_t contents_lid = 0; contents_lid < num_saved_1; contents_lid++) { + if (temp_nodes_in_nodes(node_gid_1, contents_lid) == node_gid_0) { save_1 = 0; // don't save, it was already saved } } - if (save_0 == 1) - { + if (save_0 == 1) { // increment the number of nodes in a node saved temp_nodes_in_nodes.stride(node_gid_0)++; @@ -786,8 +739,7 @@ struct mesh_t temp_nodes_in_nodes(node_gid_0, num_saved_0) = node_gid_1; } - if (save_1 == 1) - { + if (save_1 == 1) { // increment the number of nodes in a node saved temp_nodes_in_nodes.stride(node_gid_1)++; @@ -801,10 +753,8 @@ struct mesh_t } // end for node in patch } // end for patches } // end if 3D - else - { - for (size_t patch_gid = 0; patch_gid < num_patches; patch_gid++) - { + else{ + for (size_t patch_gid = 0; patch_gid < num_patches; patch_gid++) { // the first node on the edge size_t node_gid_0 = nodes_in_patch(patch_gid, 0); @@ -837,8 +787,7 @@ struct mesh_t // save the connectivity FOR_ALL_CLASS(node_gid, 0, num_nodes, { size_t num_saved = 0; - for (size_t node_lid = 0; node_lid < num_nodes_in_node(node_gid); node_lid++) - { + for (size_t node_lid = 0; node_lid < num_nodes_in_node(node_gid); node_lid++) { nodes_in_node(node_gid, num_saved) = temp_nodes_in_nodes(node_gid, num_saved); // increment the number of nodes in node saved @@ -858,8 +807,7 @@ struct mesh_t ///////////////////////////////////////////////////////////////////////////// void init_bdy_sets(size_t num_bcs) { - if (num_bcs == 0) - { + if (num_bcs == 0) { printf("ERROR: number of boundary sets = 0, set it = 1"); num_bcs = 1; } diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/FEA_Module_SGH.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/FEA_Module_SGH.cpp index 9bbe75b9a..84c811a8d 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/FEA_Module_SGH.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/FEA_Module_SGH.cpp @@ -111,8 +111,7 @@ FEA_Module_SGH::FEA_Module_SGH( all_node_velocities_distributed = Explicit_Solver_Pointer_->all_node_velocities_distributed; // Switch for optimization solver - if (simparam->topology_optimization_on || simparam->shape_optimization_on) - { + if (simparam->topology_optimization_on || simparam->shape_optimization_on) { all_cached_node_velocities_distributed = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); force_gradient_velocity = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); force_gradient_position = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); @@ -126,8 +125,7 @@ FEA_Module_SGH::FEA_Module_SGH( psi_adjoint_vector_distributed = Teuchos::rcp(new MV(all_element_map, 1)); } - if (simparam->topology_optimization_on || simparam->shape_optimization_on || simparam->num_dims == 2) - { + if (simparam->topology_optimization_on || simparam->shape_optimization_on || simparam->num_dims == 2) { node_masses_distributed = Teuchos::rcp(new MV(map, 1)); ghost_node_masses_distributed = Teuchos::rcp(new MV(ghost_node_map, 1)); } @@ -159,8 +157,7 @@ FEA_Module_SGH::FEA_Module_SGH( graphics_id = simparam->output_options.graphics_id; rk_num_bins = simparam->dynamic_options.rk_num_bins; - if (simparam->topology_optimization_on) - { + if (simparam->topology_optimization_on) { max_time_steps = BUFFER_GROW; time_data.resize(max_time_steps + 1); element_internal_energy_distributed = Teuchos::rcp(new MV(all_element_map, 1)); @@ -172,8 +169,7 @@ FEA_Module_SGH::FEA_Module_SGH( psi_adjoint_vector_data = Teuchos::rcp(new std::vector>(max_time_steps + 1)); // assign a multivector of corresponding size to each new timestep in the buffer - for (int istep = 0; istep < max_time_steps + 1; istep++) - { + for (int istep = 0; istep < max_time_steps + 1; istep++) { (*forward_solve_velocity_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); (*forward_solve_coordinate_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); (*forward_solve_internal_energy_data)[istep] = Teuchos::rcp(new MV(all_element_map, 1)); @@ -276,18 +272,15 @@ void FEA_Module_SGH::init_output() bool output_stress_flag = simparam->output(FIELD::stress); int num_dim = simparam->num_dims; int Brows; - if (num_dim == 3) - { + if (num_dim == 3) { Brows = 6; } - else - { + else{ Brows = 3; } // Implicit compliant code - if (output_velocity_flag) - { + if (output_velocity_flag) { // displacement_index is accessed by writers at the solver level for deformed output output_velocity_index = noutput; noutput += 1; @@ -303,13 +296,11 @@ void FEA_Module_SGH::init_output() output_dof_names[noutput - 1].resize(num_dim); output_dof_names[noutput - 1][0] = "vx"; output_dof_names[noutput - 1][1] = "vy"; - if (num_dim == 3) - { + if (num_dim == 3) { output_dof_names[noutput - 1][2] = "vz"; } } - if (output_strain_flag) - { + if (output_strain_flag) { output_strain_index = noutput; noutput += 1; module_outputs.resize(noutput); @@ -322,14 +313,12 @@ void FEA_Module_SGH::init_output() output_dof_names.resize(noutput); output_dof_names[noutput - 1].resize(Brows); - if (num_dim == 2) - { + if (num_dim == 2) { output_dof_names[noutput - 1][0] = "strain_xx"; output_dof_names[noutput - 1][1] = "strain_yy"; output_dof_names[noutput - 1][2] = "strain_xy"; } - if (num_dim == 3) - { + if (num_dim == 3) { output_dof_names[noutput - 1][0] = "strain_xx"; output_dof_names[noutput - 1][1] = "strain_yy"; output_dof_names[noutput - 1][2] = "strain_zz"; @@ -338,8 +327,7 @@ void FEA_Module_SGH::init_output() output_dof_names[noutput - 1][5] = "strain_yz"; } } - if (output_stress_flag) - { + if (output_stress_flag) { output_stress_index = noutput; noutput += 1; module_outputs.resize(noutput); @@ -352,14 +340,12 @@ void FEA_Module_SGH::init_output() output_dof_names.resize(noutput); output_dof_names[noutput - 1].resize(Brows); - if (num_dim == 2) - { + if (num_dim == 2) { output_dof_names[noutput - 1][0] = "stress_xx"; output_dof_names[noutput - 1][1] = "stress_yy"; output_dof_names[noutput - 1][3] = "stress_xy"; } - if (num_dim == 3) - { + if (num_dim == 3) { output_dof_names[noutput - 1][0] = "stress_xx"; output_dof_names[noutput - 1][1] = "stress_yy"; output_dof_names[noutput - 1][2] = "stress_zz"; @@ -403,10 +389,8 @@ void FEA_Module_SGH::write_data(std::map& point_data { const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - for (const auto& field_name : simparam->output_options.output_fields) - { - switch (field_name) - { + for (const auto& field_name : simparam->output_options.output_fields) { + switch (field_name) { case FIELD::velocity: // node "velocity" node_vel.update_host(); @@ -498,8 +482,7 @@ void FEA_Module_SGH::sort_element_output(Teuchos::RCPgetLocalView(Tpetra::Access::ReadWrite); elem_den.update_host(); - for (int ielem = 0; ielem < rnum_elem; ielem++) - { + for (int ielem = 0; ielem < rnum_elem; ielem++) { Element_Densities(ielem, 0) = elem_den.host(ielem); } } @@ -540,8 +523,7 @@ void FEA_Module_SGH::comm_node_masses() #ifdef DEBUG std::ostream& out = std::cout; Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - if (myrank == 0) - { + if (myrank == 0) { *fos << "Density data :" << std::endl; } node_densities_distributed->describe(*fos, Teuchos::VERB_EXTREME); @@ -560,8 +542,7 @@ void FEA_Module_SGH::comm_node_masses() all_node_velocities_distributed->describe(*fos, Teuchos::VERB_EXTREME); update_count++; - if (update_count == 1) - { + if (update_count == 1) { MPI_Barrier(world); MPI_Abort(world, 4); } @@ -577,16 +558,14 @@ void FEA_Module_SGH::comm_node_masses() ///////////////////////////////////////////////////////////////////////////// void FEA_Module_SGH::comm_variables(Teuchos::RCP zp) { - if (simparam->topology_optimization_on) - { + if (simparam->topology_optimization_on) { // set density vector to the current value chosen by the optimizer test_node_densities_distributed = zp; #ifdef DEBUG // debug print of design vector std::ostream& out = std::cout; Teuchos::RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(out)); - if (myrank == 0) - { + if (myrank == 0) { *fos << "Density data :" << std::endl; } node_densities_distributed->describe(*fos, Teuchos::VERB_EXTREME); @@ -600,8 +579,7 @@ void FEA_Module_SGH::comm_variables(Teuchos::RCP zp) // comms to get ghosts all_node_densities_distributed->doImport(*test_node_densities_distributed, *importer, Tpetra::INSERT); } - else if (simparam->shape_optimization_on) - { + else if (simparam->shape_optimization_on) { // clause to communicate boundary node data if the boundary nodes are ghosts on this rank } } @@ -630,22 +608,19 @@ void FEA_Module_SGH::node_density_constraints(host_vec_array node_densities_lowe FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { double current_node_coords[3]; double radius; - for (size_t dim = 0; dim < num_dim; dim++) - { + for (size_t dim = 0; dim < num_dim; dim++) { current_node_coords[dim] = all_initial_node_coords(node_gid, dim); } // end for dim radius = sqrt(current_node_coords[0] * current_node_coords[0] + current_node_coords[1] * current_node_coords[1] + current_node_coords[2] * current_node_coords[2]); - for (size_t ilc = 0; ilc < num_lcs; ilc++) - { + for (size_t ilc = 0; ilc < num_lcs; ilc++) { // debug check #ifdef DEBUG std::cout << "LOADING CONDITION VOLUME TYPE: " << to_string(loading(ilc).volume) << std::endl; #endif bool fill_this = loading(ilc).volume.contains(current_node_coords); - if (fill_this) - { + if (fill_this) { node_densities_lower_bound(node_gid, 0) = 1; } } @@ -723,8 +698,7 @@ void FEA_Module_SGH::tag_bdys(const DCArrayKokkos& boundary, int nboundary_patches = Explicit_Solver_Pointer_->nboundary_patches; int num_nodes_in_patch = mesh.num_nodes_in_patch; #ifdef DEBUG - if (bdy_set == mesh.num_bdy_sets) - { + if (bdy_set == mesh.num_bdy_sets) { printf(" ERROR: number of boundary sets must be increased by %zu", bdy_set - mesh.num_bdy_sets + 1); exit(0); @@ -742,8 +716,7 @@ void FEA_Module_SGH::tag_bdys(const DCArrayKokkos& boundary, double val = boundary(bdy_set).surface.plane_position; // save the boundary patches to this set that are on the plane, spheres, etc. - for (size_t bdy_patch_lid = 0; bdy_patch_lid < nboundary_patches; bdy_patch_lid++) - { + for (size_t bdy_patch_lid = 0; bdy_patch_lid < nboundary_patches; bdy_patch_lid++) { // save the patch index size_t bdy_patch_gid = bdy_patch_lid; @@ -758,18 +731,15 @@ void FEA_Module_SGH::tag_bdys(const DCArrayKokkos& boundary, // debug check #ifdef DEBUG - for (size_t patch_node_lid = 0; patch_node_lid < mesh.num_nodes_in_patch; patch_node_lid++) - { + for (size_t patch_node_lid = 0; patch_node_lid < mesh.num_nodes_in_patch; patch_node_lid++) { size_t node_gid = mesh.nodes_in_patch(bdy_patch_gid, patch_node_lid); - if (bdy_node_gid == 549412) - { + if (bdy_node_gid == 549412) { print_flag(0) = true; } } #endif - if (is_on_bdy) - { + if (is_on_bdy) { size_t index = bdy_patches_in_set.stride(bdy_set); // increment the number of boundary patches saved @@ -782,8 +752,7 @@ void FEA_Module_SGH::tag_bdys(const DCArrayKokkos& boundary, #ifdef DEBUG print_flag.update_host(); - if (print_flag.host(0)) - { + if (print_flag.host(0)) { std::cout << "found boundary node with id 549412" << std::endl; } #endif @@ -826,56 +795,44 @@ bool FEA_Module_SGH::check_bdy(const size_t patch_gid, double these_patch_coords[3]; // Note: cannot allocated array with num_dim // loop over the nodes on the patch - for (size_t patch_node_lid = 0; patch_node_lid < num_nodes_in_patch; patch_node_lid++) - { + for (size_t patch_node_lid = 0; patch_node_lid < num_nodes_in_patch; patch_node_lid++) { // get the nodal_gid for this node in the patch // size_t node_gid = mesh.nodes_in_patch(patch_gid, patch_node_lid); size_t node_gid = Local_Index_Boundary_Patches(patch_gid, patch_node_lid); - for (size_t dim = 0; dim < num_dim; dim++) - { + for (size_t dim = 0; dim < num_dim; dim++) { these_patch_coords[dim] = node_coords(rk_level, node_gid, dim); // (rk, node_gid, dim) } - if (bc_type == BOUNDARY_TYPE::x_plane) - { - if (fabs(these_patch_coords[0] - val) <= 1.0e-7) - { + if (bc_type == BOUNDARY_TYPE::x_plane) { + if (fabs(these_patch_coords[0] - val) <= 1.0e-7) { is_on_bdy += 1; } } - else if (bc_type == BOUNDARY_TYPE::y_plane) - { - if (fabs(these_patch_coords[1] - val) <= 1.0e-7) - { + else if (bc_type == BOUNDARY_TYPE::y_plane) { + if (fabs(these_patch_coords[1] - val) <= 1.0e-7) { is_on_bdy += 1; } } - else if (bc_type == BOUNDARY_TYPE::z_plane) - { - if (fabs(these_patch_coords[2] - val) <= 1.0e-7) - { + else if (bc_type == BOUNDARY_TYPE::z_plane) { + if (fabs(these_patch_coords[2] - val) <= 1.0e-7) { is_on_bdy += 1; } } - else if (bc_type == BOUNDARY_TYPE::cylinder) - { + else if (bc_type == BOUNDARY_TYPE::cylinder) { real_t R = sqrt(these_patch_coords[0] * these_patch_coords[0] + these_patch_coords[1] * these_patch_coords[1]); - if (fabs(R - val) <= 1.0e-7) - { + if (fabs(R - val) <= 1.0e-7) { is_on_bdy += 1; } } - else if (bc_type == BOUNDARY_TYPE::sphere) - { + else if (bc_type == BOUNDARY_TYPE::sphere) { real_t R = sqrt(these_patch_coords[0] * these_patch_coords[0] + these_patch_coords[1] * these_patch_coords[1] + these_patch_coords[2] * these_patch_coords[2]); - if (fabs(R - val) <= 1.0e-7) - { + if (fabs(R - val) <= 1.0e-7) { is_on_bdy += 1; } } @@ -956,15 +913,13 @@ void FEA_Module_SGH::sgh_solve() } */ // simple setup to just request KE for now; above loop to be expanded and used later for scanning modules - if (simparam->topology_optimization_on) - { + if (simparam->topology_optimization_on) { obj_pointer = problem->getObjective(); KineticEnergyMinimize_TopOpt& kinetic_energy_minimize_function = dynamic_cast(*obj_pointer); kinetic_energy_minimize_function.objective_accumulation = 0; global_objective_accumulation = objective_accumulation = 0; kinetic_energy_objective = true; - if (max_time_steps + 1 > forward_solve_velocity_data->size()) - { + if (max_time_steps + 1 > forward_solve_velocity_data->size()) { old_max_forward_buffer = forward_solve_velocity_data->size(); time_data.resize(max_time_steps + 1); forward_solve_velocity_data->resize(max_time_steps + 1); @@ -974,8 +929,7 @@ void FEA_Module_SGH::sgh_solve() phi_adjoint_vector_data->resize(max_time_steps + 1); psi_adjoint_vector_data->resize(max_time_steps + 1); // assign a multivector of corresponding size to each new timestep in the buffer - for (int istep = old_max_forward_buffer; istep < max_time_steps + 1; istep++) - { + for (int istep = old_max_forward_buffer; istep < max_time_steps + 1; istep++) { (*forward_solve_velocity_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); (*forward_solve_coordinate_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); (*forward_solve_internal_energy_data)[istep] = Teuchos::rcp(new MV(all_element_map, 1)); @@ -986,16 +940,13 @@ void FEA_Module_SGH::sgh_solve() } } - if (simparam->topology_optimization_on) - { + if (simparam->topology_optimization_on) { nTO_modules = simparam->TO_Module_List.size(); } int myrank = Explicit_Solver_Pointer_->myrank; - if (simparam->output_options.write_initial) - { - if (myrank == 0) - { + if (simparam->output_options.write_initial) { + if (myrank == 0) { printf("Writing outputs to file at %f \n", time_value); } @@ -1045,17 +996,14 @@ void FEA_Module_SGH::sgh_solve() // extensive KE REDUCE_SUM_CLASS(node_gid, 0, nlocal_nodes, KE_loc_sum, { double ke = 0; - for (size_t dim = 0; dim < num_dim; dim++) - { + for (size_t dim = 0; dim < num_dim; dim++) { ke += node_vel(rk_level, node_gid, dim) * node_vel(rk_level, node_gid, dim); // 1/2 at end } // end for - if (num_dim == 2) - { + if (num_dim == 2) { KE_loc_sum += node_mass(node_gid) * node_coords(rk_level, node_gid, 1) * ke; } - else - { + else{ KE_loc_sum += node_mass(node_gid) * ke; } }, KE_sum); @@ -1073,8 +1021,7 @@ void FEA_Module_SGH::sgh_solve() // save the nodal mass FOR_ALL_CLASS(node_gid, 0, nall_nodes, { double radius = 1.0; - if (num_dim == 2) - { + if (num_dim == 2) { radius = node_coords(rk_level, node_gid, 1); } node_extensive_mass(node_gid) = node_mass(node_gid) * radius; @@ -1086,8 +1033,7 @@ void FEA_Module_SGH::sgh_solve() auto time_1 = std::chrono::high_resolution_clock::now(); // save initial data - if (simparam->topology_optimization_on || simparam->shape_optimization_on) - { + if (simparam->topology_optimization_on || simparam->shape_optimization_on) { time_data[0] = 0; // assign current velocity data to multivector // view scope @@ -1095,8 +1041,7 @@ void FEA_Module_SGH::sgh_solve() vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (int idim = 0; idim < num_dim; idim++) - { + for (int idim = 0; idim < num_dim; idim++) { node_velocities_interface(node_gid, idim) = node_vel(rk_level, node_gid, idim); node_coords_interface(node_gid, idim) = node_coords(rk_level, node_gid, idim); } @@ -1131,8 +1076,7 @@ void FEA_Module_SGH::sgh_solve() const_vec_array ghost_node_coords_interface = Explicit_Solver_Pointer_->ghost_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (int idim = 0; idim < num_dim; idim++) - { + for (int idim = 0; idim < num_dim; idim++) { all_node_velocities_interface(node_gid, idim) = node_velocities_interface(node_gid, idim); all_node_coords_interface(node_gid, idim) = node_coords_interface(node_gid, idim); } @@ -1140,8 +1084,7 @@ void FEA_Module_SGH::sgh_solve() Kokkos::fence(); FOR_ALL_CLASS(node_gid, nlocal_nodes, nlocal_nodes + nghost_nodes, { - for (int idim = 0; idim < num_dim; idim++) - { + for (int idim = 0; idim < num_dim; idim++) { all_node_velocities_interface(node_gid, idim) = ghost_node_velocities_interface(node_gid - nlocal_nodes, idim); all_node_coords_interface(node_gid, idim) = ghost_node_coords_interface(node_gid - nlocal_nodes, idim); } @@ -1161,19 +1104,16 @@ void FEA_Module_SGH::sgh_solve() } // loop over the max number of time integration cycles - for (cycle = 0; cycle < cycle_stop; cycle++) - { + for (cycle = 0; cycle < cycle_stop; cycle++) { // get the step - if (num_dim == 2) - { + if (num_dim == 2) { get_timestep2D(*mesh, node_coords, node_vel, elem_sspd, elem_vol); } - else - { + else{ get_timestep(*mesh, node_coords, node_vel, @@ -1188,20 +1128,15 @@ void FEA_Module_SGH::sgh_solve() // stop calculation if flag // if (stop_calc == 1) break; - if (simparam->dynamic_options.output_time_sequence_level >= TIME_OUTPUT_LEVEL::high) - { - if (cycle == 0) - { - if (myrank == 0) - { + if (simparam->dynamic_options.output_time_sequence_level >= TIME_OUTPUT_LEVEL::high) { + if (cycle == 0) { + if (myrank == 0) { printf("cycle = %lu, time = %12.5e, time step = %12.5e \n", cycle, time_value, dt); } } // print time step every 10 cycles - else if (cycle % 20 == 0) - { - if (myrank == 0) - { + else if (cycle % 20 == 0) { + if (myrank == 0) { printf("cycle = %lu, time = %12.5e, time step = %12.5e \n", cycle, time_value, dt); } } // end if @@ -1220,21 +1155,18 @@ void FEA_Module_SGH::sgh_solve() nall_nodes); // integrate solution forward in time - for (size_t rk_stage = 0; rk_stage < rk_num_stages; rk_stage++) - { + for (size_t rk_stage = 0; rk_stage < rk_num_stages; rk_stage++) { // ---- RK coefficient ---- double rk_alpha = 1.0 / ((double)rk_num_stages - (double)rk_stage); // ---- Calculate velocity diveregence for the element ---- - if (num_dim == 2) - { + if (num_dim == 2) { get_divergence2D(elem_div, node_coords, node_vel, elem_vol); } - else - { + else{ get_divergence(elem_div, node_coords, node_vel, @@ -1242,8 +1174,7 @@ void FEA_Module_SGH::sgh_solve() } // end if 2D // ---- calculate the forces on the vertices and evolve stress (hypo model) ---- - if (num_dim == 2) - { + if (num_dim == 2) { get_force_sgh2D(material, *mesh, node_coords, @@ -1260,8 +1191,7 @@ void FEA_Module_SGH::sgh_solve() rk_alpha, cycle); } - else - { + else{ get_force_sgh(material, *mesh, node_coords, @@ -1280,27 +1210,22 @@ void FEA_Module_SGH::sgh_solve() } #ifdef DEBUG - if (myrank == 1) - { + if (myrank == 1) { std::cout << "rk_alpha = " << rk_alpha << ", dt = " << dt << std::endl; - for (int i = 0; i < nall_nodes; i++) - { + for (int i = 0; i < nall_nodes; i++) { double node_force[3]; - for (size_t dim = 0; dim < num_dim; dim++) - { + for (size_t dim = 0; dim < num_dim; dim++) { node_force[dim] = 0.0; } // end for dim // loop over all corners around the node and calculate the nodal force - for (size_t corner_lid = 0; corner_lid < mesh.num_corners_in_node(i); corner_lid++) - { + for (size_t corner_lid = 0; corner_lid < mesh.num_corners_in_node(i); corner_lid++) { // Get corner gid size_t corner_gid = mesh.corners_in_node(i, corner_lid); std::cout << Explicit_Solver_Pointer_->all_node_map->getGlobalElement(i) << " " << corner_gid << " " << corner_force(corner_gid, 0) << " " << corner_force(corner_gid, 1) << " " << corner_force(corner_gid, 2) << std::endl; // loop over dimension - for (size_t dim = 0; dim < num_dim; dim++) - { + for (size_t dim = 0; dim < num_dim; dim++) { node_force[dim] += corner_force(corner_gid, dim); } // end for dim } // end for corner_lid @@ -1309,10 +1234,8 @@ void FEA_Module_SGH::sgh_solve() // debug print vector values on a rank - if (myrank == 0) - { - for (int i = 0; i < nall_nodes; i++) - { + if (myrank == 0) { + for (int i = 0; i < nall_nodes; i++) { std::cout << Explicit_Solver_Pointer_->all_node_map->getGlobalElement(i) << " " << node_vel(rk_level, i, 0) << " " << node_vel(rk_level, i, 1) << " " << node_vel(rk_level, i, 2) << std::endl; } @@ -1325,8 +1248,7 @@ void FEA_Module_SGH::sgh_solve() node_mass, corner_force); - if (have_loading_conditions) - { + if (have_loading_conditions) { applied_forces(material, *mesh, node_coords, @@ -1351,8 +1273,7 @@ void FEA_Module_SGH::sgh_solve() { vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (int idim = 0; idim < num_dim; idim++) - { + for (int idim = 0; idim < num_dim; idim++) { node_velocities_interface(node_gid, idim) = node_vel(rk_level, node_gid, idim); } }); // end parallel for @@ -1374,8 +1295,7 @@ void FEA_Module_SGH::sgh_solve() { vec_array ghost_node_velocities_interface = Explicit_Solver_Pointer_->ghost_node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); FOR_ALL_CLASS(node_gid, nlocal_nodes, nall_nodes, { - for (int idim = 0; idim < num_dim; idim++) - { + for (int idim = 0; idim < num_dim; idim++) { node_vel(rk_level, node_gid, idim) = ghost_node_velocities_interface(node_gid - nlocal_nodes, idim); } }); // end parallel for @@ -1388,10 +1308,8 @@ void FEA_Module_SGH::sgh_solve() #ifdef DEBUG // debug print vector values on a rank - if (myrank == 0) - { - for (int i = 0; i < nall_nodes; i++) - { + if (myrank == 0) { + for (int i = 0; i < nall_nodes; i++) { std::cout << Explicit_Solver_Pointer_->all_node_map->getGlobalElement(i) << " " << node_vel(rk_level, i, 0) << " " << node_vel(rk_level, i, 1) << " " << node_vel(rk_level, i, 2) << std::endl; } @@ -1416,8 +1334,7 @@ void FEA_Module_SGH::sgh_solve() get_vol(); // ---- Calculate elem state (den, pres, sound speed, stress) for next time step ---- - if (num_dim == 2) - { + if (num_dim == 2) { update_state2D(material, *mesh, node_coords, @@ -1433,8 +1350,7 @@ void FEA_Module_SGH::sgh_solve() rk_alpha, cycle); } - else - { + else{ update_state(material, *mesh, node_coords, @@ -1457,14 +1373,12 @@ void FEA_Module_SGH::sgh_solve() // 3) strength models must be added by the user in user_mat.cpp // calculate the new corner masses if 2D - if (num_dim == 2) - { + if (num_dim == 2) { // calculate the nodal areal mass FOR_ALL_CLASS(node_gid, 0, nall_nodes, { node_mass(node_gid) = 0.0; - if (node_coords(rk_level, node_gid, 1) > tiny) - { + if (node_coords(rk_level, node_gid, 1) > tiny) { node_mass(node_gid) = node_extensive_mass(node_gid) / node_coords(rk_level, node_gid, 1); } // if(cycle==0&&node_gid==1&&myrank==0) @@ -1547,17 +1461,14 @@ void FEA_Module_SGH::sgh_solve() // FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { size_t node_gid = bdy_nodes(node_bdy_gid); - if (node_coords(rk_level, node_gid, 1) < tiny) - { + if (node_coords(rk_level, node_gid, 1) < tiny) { // node is on the axis - for (size_t node_lid = 0; node_lid < num_nodes_in_node(node_gid); node_lid++) - { + for (size_t node_lid = 0; node_lid < num_nodes_in_node(node_gid); node_lid++) { size_t node_neighbor_gid = nodes_in_node(node_gid, node_lid); // if the node is off the axis, use it's areal mass on the boundary - if (node_coords(rk_level, node_neighbor_gid, 1) > tiny) - { + if (node_coords(rk_level, node_neighbor_gid, 1) > tiny) { node_mass(node_gid) = fmax(node_mass(node_gid), node_mass(node_neighbor_gid) / 2.0); } } // end for over neighboring nodes @@ -1569,15 +1480,12 @@ void FEA_Module_SGH::sgh_solve() // increment the time Explicit_Solver_Pointer_->time_value = simparam->dynamic_options.time_value = time_value += dt; - if (simparam->topology_optimization_on || simparam->shape_optimization_on) - { - if (cycle >= max_time_steps) - { + if (simparam->topology_optimization_on || simparam->shape_optimization_on) { + if (cycle >= max_time_steps) { max_time_steps = cycle + 1; } - if (max_time_steps + 1 > forward_solve_velocity_data->size()) - { + if (max_time_steps + 1 > forward_solve_velocity_data->size()) { old_max_forward_buffer = forward_solve_velocity_data->size(); time_data.resize(max_time_steps + BUFFER_GROW + 1); forward_solve_velocity_data->resize(max_time_steps + BUFFER_GROW + 1); @@ -1587,8 +1495,7 @@ void FEA_Module_SGH::sgh_solve() phi_adjoint_vector_data->resize(max_time_steps + BUFFER_GROW + 1); psi_adjoint_vector_data->resize(max_time_steps + BUFFER_GROW + 1); // assign a multivector of corresponding size to each new timestep in the buffer - for (int istep = old_max_forward_buffer; istep < max_time_steps + BUFFER_GROW + 1; istep++) - { + for (int istep = old_max_forward_buffer; istep < max_time_steps + BUFFER_GROW + 1; istep++) { (*forward_solve_velocity_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); (*forward_solve_coordinate_data)[istep] = Teuchos::rcp(new MV(all_node_map, simparam->num_dims)); (*forward_solve_internal_energy_data)[istep] = Teuchos::rcp(new MV(all_element_map, 1)); @@ -1606,8 +1513,7 @@ void FEA_Module_SGH::sgh_solve() vec_array node_velocities_interface = Explicit_Solver_Pointer_->node_velocities_distributed->getLocalView(Tpetra::Access::ReadWrite); vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (int idim = 0; idim < num_dim; idim++) - { + for (int idim = 0; idim < num_dim; idim++) { node_velocities_interface(node_gid, idim) = node_vel(rk_level, node_gid, idim); node_coords_interface(node_gid, idim) = node_coords(rk_level, node_gid, idim); } @@ -1642,8 +1548,7 @@ void FEA_Module_SGH::sgh_solve() const_vec_array ghost_node_coords_interface = Explicit_Solver_Pointer_->ghost_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (int idim = 0; idim < num_dim; idim++) - { + for (int idim = 0; idim < num_dim; idim++) { all_node_velocities_interface(node_gid, idim) = node_velocities_interface(node_gid, idim); all_node_coords_interface(node_gid, idim) = node_coords_interface(node_gid, idim); } @@ -1651,8 +1556,7 @@ void FEA_Module_SGH::sgh_solve() Kokkos::fence(); FOR_ALL_CLASS(node_gid, nlocal_nodes, nlocal_nodes + nghost_nodes, { - for (int idim = 0; idim < num_dim; idim++) - { + for (int idim = 0; idim < num_dim; idim++) { all_node_velocities_interface(node_gid, idim) = ghost_node_velocities_interface(node_gid - nlocal_nodes, idim); all_node_coords_interface(node_gid, idim) = ghost_node_coords_interface(node_gid - nlocal_nodes, idim); } @@ -1675,8 +1579,7 @@ void FEA_Module_SGH::sgh_solve() (*forward_solve_coordinate_data)[cycle + 1]->assign(*Explicit_Solver_Pointer_->all_node_coords_distributed); // kinetic energy accumulation - if (kinetic_energy_objective) - { + if (kinetic_energy_objective) { const_vec_array node_velocities_interface = (*forward_solve_velocity_data)[cycle + 1]->getLocalView(Tpetra::Access::ReadOnly); const_vec_array previous_node_velocities_interface = (*forward_solve_velocity_data)[cycle]->getLocalView(Tpetra::Access::ReadOnly); KE_loc_sum = 0.0; @@ -1684,19 +1587,16 @@ void FEA_Module_SGH::sgh_solve() // extensive KE REDUCE_SUM_CLASS(node_gid, 0, nlocal_nodes, KE_loc_sum, { double ke = 0; - for (size_t dim = 0; dim < num_dim; dim++) - { + for (size_t dim = 0; dim < num_dim; dim++) { // midpoint integration approximation ke += (node_velocities_interface(node_gid, dim) + previous_node_velocities_interface(node_gid, dim)) * (node_velocities_interface(node_gid, dim) + previous_node_velocities_interface(node_gid, dim)) / 4; // 1/2 at end } // end for - if (num_dim == 2) - { + if (num_dim == 2) { KE_loc_sum += node_mass(node_gid) * node_coords(rk_level, node_gid, 1) * ke; } - else - { + else{ KE_loc_sum += node_mass(node_gid) * ke; } }, KE_sum); @@ -1707,40 +1607,33 @@ void FEA_Module_SGH::sgh_solve() } size_t write = 0; - if ((cycle + 1) % graphics_cyc_ival == 0 && cycle > 0) - { + if ((cycle + 1) % graphics_cyc_ival == 0 && cycle > 0) { write = 1; } - else if (cycle == cycle_stop) - { + else if (cycle == cycle_stop) { write = 1; } - else if (time_value >= time_final && simparam->output_options.write_final) - { + else if (time_value >= time_final && simparam->output_options.write_final) { write = 1; } - else if (time_value >= graphics_time) - { + else if (time_value >= graphics_time) { write = 1; } // write outputs - if (write == 1) - { + if (write == 1) { // interface nodal coordinate data (note: this is not needed if using write_outputs()) // view scope { vec_array node_coords_interface = Explicit_Solver_Pointer_->node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (int idim = 0; idim < num_dim; idim++) - { + for (int idim = 0; idim < num_dim; idim++) { node_coords_interface(node_gid, idim) = node_coords(rk_level, node_gid, idim); } }); // end parallel for } // end view scope - if (myrank == 0) - { + if (myrank == 0) { printf("Writing outputs to file at %f \n", graphics_time); } @@ -1754,8 +1647,7 @@ void FEA_Module_SGH::sgh_solve() } // end if // end of calculation - if (time_value >= time_final) - { + if (time_value >= time_final) { break; } } // end for cycle loop @@ -1763,16 +1655,14 @@ void FEA_Module_SGH::sgh_solve() last_time_step = cycle; // simple setup to just calculate KE minimize objective for now - if (simparam->topology_optimization_on) - { + if (simparam->topology_optimization_on) { KineticEnergyMinimize_TopOpt& kinetic_energy_minimize_function = dynamic_cast(*obj_pointer); // collect local objective values MPI_Allreduce(&objective_accumulation, &global_objective_accumulation, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); kinetic_energy_minimize_function.objective_accumulation = global_objective_accumulation; - if (myrank == 0) - { + if (myrank == 0) { std::cout << "CURRENT TIME INTEGRAL OF KINETIC ENERGY " << global_objective_accumulation << std::endl; } } @@ -1781,8 +1671,7 @@ void FEA_Module_SGH::sgh_solve() auto time_difference = time_2 - time_1; // double calc_time = std::chrono::duration_cast(diff).count(); double calc_time = std::chrono::duration_cast(time_difference).count(); - if (myrank == 0) - { + if (myrank == 0) { printf("\nCalculation time in seconds: %f \n", calc_time * 1e-09); } @@ -1803,17 +1692,14 @@ void FEA_Module_SGH::sgh_solve() // extensive KE REDUCE_SUM_CLASS(node_gid, 0, nlocal_nodes, KE_loc_sum, { double ke = 0; - for (size_t dim = 0; dim < num_dim; dim++) - { + for (size_t dim = 0; dim < num_dim; dim++) { ke += node_vel(rk_level, node_gid, dim) * node_vel(rk_level, node_gid, dim); // 1/2 at end } // end for - if (num_dim == 2) - { + if (num_dim == 2) { KE_loc_sum += node_mass(node_gid) * node_coords(rk_level, node_gid, 1) * ke; } - else - { + else{ KE_loc_sum += node_mass(node_gid) * ke; } }, KE_sum); @@ -1833,16 +1719,13 @@ void FEA_Module_SGH::sgh_solve() // reduce over MPI ranks - if (myrank == 0) - { + if (myrank == 0) { printf("Time=0: KE = %20.15f, IE = %20.15f, TE = %20.15f \n", KE_t0, IE_t0, TE_t0); } - if (myrank == 0) - { + if (myrank == 0) { printf("Time=End: KE = %20.15f, IE = %20.15f, TE = %20.15f \n", KE_tend, IE_tend, TE_tend); } - if (myrank == 0) - { + if (myrank == 0) { printf("total energy conservation error = %e \n\n", 100 * (TE_tend - TE_t0) / TE_t0); } diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/boundary.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/boundary.cpp index 3c07262ae..628cdf4b8 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/boundary.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/boundary.cpp @@ -54,13 +54,11 @@ void FEA_Module_SGH::boundary_velocity(const mesh_t& mesh, const size_t rk_level = rk_num_bins - 1; int num_dims = num_dim; // Loop over boundary sets - for (size_t bdy_set = 0; bdy_set < num_bdy_sets; bdy_set++) - { + for (size_t bdy_set = 0; bdy_set < num_bdy_sets; bdy_set++) { // Loop over boundary nodes in a boundary set FOR_ALL_CLASS(bdy_node_lid, 0, num_bdy_nodes_in_set.host(bdy_set), { // reflected (boundary array is on the device) - if (boundary(bdy_set).type == BOUNDARY_CONDITION_TYPE::reflected) - { + if (boundary(bdy_set).type == BOUNDARY_CONDITION_TYPE::reflected) { // directions with hydro_bc: // x_plane = 0, // y_plane = 1, @@ -72,27 +70,23 @@ void FEA_Module_SGH::boundary_velocity(const mesh_t& mesh, // Set velocity to zero in that directdion node_vel(rk_level, bdy_node_gid, direction) = 0.0; } - else if (boundary(bdy_set).type == BOUNDARY_CONDITION_TYPE::fixed_position) - { + else if (boundary(bdy_set).type == BOUNDARY_CONDITION_TYPE::fixed_position) { size_t bdy_node_gid = bdy_nodes_in_set(bdy_set, bdy_node_lid); // debug clause // if(bdy_node_gid==549412) print_flag(0) = true; - for (size_t dim = 0; dim < num_dims; dim++) - { + for (size_t dim = 0; dim < num_dims; dim++) { // Set velocity to zero node_vel(rk_level, bdy_node_gid, dim) = 0.0; } } - else if (boundary(bdy_set).type == BOUNDARY_CONDITION_TYPE::velocity) - { + else if (boundary(bdy_set).type == BOUNDARY_CONDITION_TYPE::velocity) { size_t bdy_node_gid = mesh.bdy_nodes_in_set(bdy_set, bdy_node_lid); node_vel(rk_level, bdy_node_gid, 0) = boundary(bdy_set).u; node_vel(rk_level, bdy_node_gid, 1) = boundary(bdy_set).v; - if (mesh.num_dims == 3) - { + if (mesh.num_dims == 3) { node_vel(rk_level, bdy_node_gid, 2) = boundary(bdy_set).w; } } // end if diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/energy_sgh.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/energy_sgh.cpp index e08af3db2..936aa4c8e 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/energy_sgh.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/energy_sgh.cpp @@ -68,8 +68,7 @@ void FEA_Module_SGH::update_energy_sgh(double rk_alpha, // --- tally the contribution from each corner to the element --- // Loop over the nodes in the element - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) - { + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { size_t corner_lid = node_lid; // Get node global id for the local node id @@ -79,14 +78,12 @@ void FEA_Module_SGH::update_energy_sgh(double rk_alpha, size_t corner_gid = corners_in_elem(elem_gid, corner_lid); double node_radius = 1; - if (num_dims == 2) - { + if (num_dims == 2) { node_radius = node_coords(rk_level, node_gid, 1); } // calculate the Power=F dot V for this corner - for (size_t dim = 0; dim < num_dims; dim++) - { + for (size_t dim = 0; dim < num_dims; dim++) { double half_vel = (node_vel(rk_level, node_gid, dim) + node_vel(0, node_gid, dim)) * 0.5; elem_power += corner_force(corner_gid, dim) * node_radius * half_vel; } // end for dim diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp index e8defe6cd..a844634e6 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp @@ -100,8 +100,7 @@ void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& ma // initialize gradient matrix FOR_ALL_CLASS(dof_gid, 0, nlocal_nodes * num_dims, { - for (int idof = 0; idof < Gradient_Matrix_Strides(dof_gid); idof++) - { + for (int idof = 0; idof < Gradient_Matrix_Strides(dof_gid); idof++) { Force_Gradient_Velocities(dof_gid, idof) = 0; } }); // end parallel for loop over nodes @@ -109,13 +108,10 @@ void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& ma // initialize buffer storage; not all components are explicitly set for this routine FOR_ALL_CLASS(corner_gid, 0, num_corners, { - for (int dim = 0; dim < num_dims; dim++) - { + for (int dim = 0; dim < num_dims; dim++) { // assign gradient of corner contribution of force to relevant matrix entries with non-zero node velocity gradient - for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) - { - for (int jdim = 0; jdim < num_dims; jdim++) - { + for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) { + for (int jdim = 0; jdim < num_dims; jdim++) { corner_gradient_storage(corner_gid, dim, igradient, jdim) = 0; } } @@ -124,8 +120,7 @@ void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& ma Kokkos::fence(); // --- calculate the forces acting on the nodes from the element --- - for (size_t elem_gid = 0; elem_gid < rnum_elem; elem_gid++) - { + for (size_t elem_gid = 0; elem_gid < rnum_elem; elem_gid++) { // FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { const size_t num_nodes_in_elem = 8; @@ -195,10 +190,8 @@ void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& ma rk_level); // the -1 is for the inward surface area normal, - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) - { - for (size_t dim = 0; dim < num_dims; dim++) - { + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { + for (size_t dim = 0; dim < num_dims; dim++) { area_normal(node_lid, dim) = (-1.0) * area_normal(node_lid, dim); } // end for } // end for @@ -217,18 +210,15 @@ void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& ma double mag_curl = sqrt(curl[0] * curl[0] + curl[1] * curl[1] + curl[2] * curl[2]); // --- Calculate the Cauchy stress --- - for (size_t i = 0; i < 3; i++) - { - for (size_t j = 0; j < 3; j++) - { + for (size_t i = 0; i < 3; i++) { + for (size_t j = 0; j < 3; j++) { tau(i, j) = stress(i, j); // artificial viscosity can be added here to tau } // end for } // end for // add the pressure - for (int i = 0; i < num_dims; i++) - { + for (int i = 0; i < num_dims; i++) { tau(i, i) -= elem_pres(elem_gid); } // end for @@ -237,15 +227,13 @@ void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& ma // estimate of the Riemann velocity // initialize to Riemann velocity to zero - for (size_t dim = 0; dim < num_dims; dim++) - { + for (size_t dim = 0; dim < num_dims; dim++) { vel_star(dim) = 0.0; } // loop over nodes and calculate an average velocity, which is // an estimate of Riemann velocity - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) - { + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { // Get node gloabl index and create view of nodal velocity int node_gid = nodes_in_elem(elem_gid, node_lid); @@ -259,8 +247,7 @@ void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& ma // find shock direction and shock impedance associated with each node // initialize sum term in MARS to zero - for (int i = 0; i < 4; i++) - { + for (int i = 0; i < 4; i++) { sum(i) = 0.0; } @@ -268,8 +255,7 @@ void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& ma double mag_vel; // magnitude of velocity // loop over the nodes of the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) - { + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { // Get global node id size_t node_gid = nodes_in_elem(elem_gid, node_lid); @@ -281,16 +267,13 @@ void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& ma + (vel(1) - vel_star(1) ) * (vel(1) - vel_star(1) ) + (vel(2) - vel_star(2) ) * (vel(2) - vel_star(2) ) ); - if (mag_vel > small) - { + if (mag_vel > small) { // estimate of the shock direction, a unit normal - for (int dim = 0; dim < num_dims; dim++) - { + for (int dim = 0; dim < num_dims; dim++) { shock_dir(dim) = (vel(dim) - vel_star(dim)) / mag_vel; } } - else - { + else{ // if there is no velocity change, then use the surface area // normal as the shock direction mag = sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) @@ -298,21 +281,18 @@ void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& ma + area_normal(node_lid, 2) * area_normal(node_lid, 2) ); // estimate of the shock direction - for (int dim = 0; dim < num_dims; dim++) - { + for (int dim = 0; dim < num_dims; dim++) { shock_dir(dim) = area_normal(node_lid, dim) / mag; } } // end if mag_vel // cell divergence indicates compression or expansions size_t mat_id = elem_mat_id(elem_gid); - if (div < 0) // element in compression - { + if (div < 0) { // element in compression muc(node_lid) = elem_den(elem_gid) * (material(mat_id).q1 * elem_sspd(elem_gid) + material(mat_id).q2 * mag_vel); } - else // element in expansion - { + else{ // element in expansion muc(node_lid) = elem_den(elem_gid) * (material(mat_id).q1ex * elem_sspd(elem_gid) + material(mat_id).q2ex * mag_vel); } // end if on divergence sign @@ -322,8 +302,7 @@ void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& ma double mu_term_gradient; // Coding to use shock direction - if (use_shock_dir == 1) - { + if (use_shock_dir == 1) { // this is denominator of the Riamann solver and the multiplier // on velocity in the numerator. It filters on the shock // direction @@ -332,8 +311,7 @@ void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& ma + shock_dir(1) * area_normal(node_lid, 1) + shock_dir(2) * area_normal(node_lid, 2) ); // code gradient for shock dir w.r.t velocity if using shock_dir } - else - { + else{ // Using a full tensoral Riemann jump relation mu_term = muc(node_lid) * sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) @@ -350,35 +328,26 @@ void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& ma } // end for node_lid loop over nodes of the elem // The Riemann velocity, called vel_star - if (sum(3) > fuzz) - { - for (size_t i = 0; i < num_dims; i++) - { + if (sum(3) > fuzz) { + for (size_t i = 0; i < num_dims; i++) { vel_star(i) = sum(i) / sum(3); } } - else - { - for (int i = 0; i < num_dims; i++) - { + else{ + for (int i = 0; i < num_dims; i++) { vel_star(i) = 0.0; } } // end if - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) - { + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { // The Riemann velocity, called vel_star - if (sum(3) > fuzz) - { - for (size_t i = 0; i < num_dims; i++) - { + if (sum(3) > fuzz) { + for (size_t i = 0; i < num_dims; i++) { vel_star_gradient(node_lid, i) = muc(node_lid) / sum(3); } } - else - { - for (int i = 0; i < num_dims; i++) - { + else{ + for (int i = 0; i < num_dims; i++) { vel_star_gradient(node_lid, i) = 0; } } // end if @@ -406,8 +375,7 @@ void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& ma // (1=nominal, and n_coeff > 1 oscillatory) // loop over the nieghboring cells - for (size_t elem_lid = 0; elem_lid < num_elems_in_elem(elem_gid); elem_lid++) - { + for (size_t elem_lid = 0; elem_lid < num_elems_in_elem(elem_gid); elem_lid++) { // Get global index for neighboring cell size_t neighbor_gid = elems_in_elem(elem_gid, elem_lid); @@ -445,8 +413,7 @@ void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& ma // ---- Calculate the Riemann force on each node ---- // loop over the each node in the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) - { + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { size_t corner_lid = node_lid; size_t column_index; @@ -457,26 +424,20 @@ void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& ma size_t node_gid = nodes_in_elem(elem_gid, node_lid); // loop over dimension - for (int dim = 0; dim < num_dims; dim++) - { + for (int dim = 0; dim < num_dims; dim++) { // assign gradient of corner contribution of force to relevant matrix entries with non-zero node velocity gradient - for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) - { + for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) { size_t gradient_node_gid = nodes_in_elem(elem_gid, igradient); // if(!map->isNodeLocalElement(gradient_node_gid)) continue; column_index = num_dims * Global_Gradient_Matrix_Assembly_Map(elem_gid, igradient, node_lid); - if (node_lid == igradient) - { - if (map->isNodeLocalElement(gradient_node_gid)) - { + if (node_lid == igradient) { + if (map->isNodeLocalElement(gradient_node_gid)) { Force_Gradient_Velocities(gradient_node_gid * num_dims + dim, column_index + dim) += phi * muc(node_lid) * (vel_star_gradient(igradient, dim) - 1); } corner_gradient_storage(corner_gid, dim, igradient, dim) = phi * muc(node_lid) * (vel_star_gradient(igradient, dim) - 1); } - else - { - if (map->isNodeLocalElement(gradient_node_gid)) - { + else{ + if (map->isNodeLocalElement(gradient_node_gid)) { Force_Gradient_Velocities(gradient_node_gid * num_dims + dim, column_index + dim) += phi * muc(node_lid) * (vel_star_gradient(igradient, dim)); } corner_gradient_storage(corner_gid, dim, igradient, dim) = phi * muc(node_lid) * (vel_star_gradient(igradient, dim)); @@ -558,16 +519,14 @@ void FEA_Module_SGH::get_force_egradient_sgh(const DCArrayKokkos& ma // initialize gradient matrix FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { // compute resulting row of force displacement gradient matrix transpose right multiplied by adjoint vector - for (int idof = 0; idof < num_nodes_in_elem * num_dims; idof++) - { + for (int idof = 0; idof < num_nodes_in_elem * num_dims; idof++) { Force_Gradient_Energies(elem_gid, idof) = 0; } }); // end parallel for Kokkos::fence(); // --- calculate the forces acting on the nodes from the element --- - for (size_t elem_gid = 0; elem_gid < rnum_elem; elem_gid++) - { + for (size_t elem_gid = 0; elem_gid < rnum_elem; elem_gid++) { // FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { const size_t num_nodes_in_elem = 8; @@ -639,10 +598,8 @@ void FEA_Module_SGH::get_force_egradient_sgh(const DCArrayKokkos& ma rk_level); // the -1 is for the inward surface area normal, - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) - { - for (size_t dim = 0; dim < num_dims; dim++) - { + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { + for (size_t dim = 0; dim < num_dims; dim++) { area_normal(node_lid, dim) = (-1.0) * area_normal(node_lid, dim); } // end for } // end for @@ -661,10 +618,8 @@ void FEA_Module_SGH::get_force_egradient_sgh(const DCArrayKokkos& ma double mag_curl = sqrt(curl[0] * curl[0] + curl[1] * curl[1] + curl[2] * curl[2]); // --- Calculate the Cauchy stress --- - for (size_t i = 0; i < num_dims; i++) - { - for (size_t j = 0; j < num_dims; j++) - { + for (size_t i = 0; i < num_dims; i++) { + for (size_t j = 0; j < num_dims; j++) { tau(i, j) = stress(i, j); tau_gradient(i, j) = 0; // artificial viscosity can be added here to tau @@ -672,8 +627,7 @@ void FEA_Module_SGH::get_force_egradient_sgh(const DCArrayKokkos& ma } // end for // add the pressure - for (int i = 0; i < num_dims; i++) - { + for (int i = 0; i < num_dims; i++) { tau(i, i) -= elem_pres(elem_gid); tau_gradient(i, i) = -elem_eos(elem_gid).calc_pressure_gradient_internal_energy(elem_pres, elem_stress, @@ -694,15 +648,13 @@ void FEA_Module_SGH::get_force_egradient_sgh(const DCArrayKokkos& ma // estimate of the Riemann velocity // initialize to Riemann velocity to zero - for (size_t dim = 0; dim < num_dims; dim++) - { + for (size_t dim = 0; dim < num_dims; dim++) { vel_star(dim) = 0.0; } // loop over nodes and calculate an average velocity, which is // an estimate of Riemann velocity - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) - { + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { // Get node gloabl index and create view of nodal velocity int node_gid = nodes_in_elem(elem_gid, node_lid); @@ -716,8 +668,7 @@ void FEA_Module_SGH::get_force_egradient_sgh(const DCArrayKokkos& ma // find shock direction and shock impedance associated with each node // initialize sum term in MARS to zero - for (int i = 0; i < 4; i++) - { + for (int i = 0; i < 4; i++) { sum(i) = sum_gradient(i) = 0.0; } @@ -740,8 +691,7 @@ void FEA_Module_SGH::get_force_egradient_sgh(const DCArrayKokkos& ma elem_sie(rk_level, elem_gid)); // loop over the nodes of the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) - { + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { // Get global node id size_t node_gid = nodes_in_elem(elem_gid, node_lid); @@ -753,16 +703,13 @@ void FEA_Module_SGH::get_force_egradient_sgh(const DCArrayKokkos& ma + (vel(1) - vel_star(1) ) * (vel(1) - vel_star(1) ) + (vel(2) - vel_star(2) ) * (vel(2) - vel_star(2) ) ); - if (mag_vel > small) - { + if (mag_vel > small) { // estimate of the shock direction, a unit normal - for (int dim = 0; dim < num_dims; dim++) - { + for (int dim = 0; dim < num_dims; dim++) { shock_dir(dim) = (vel(dim) - vel_star(dim)) / mag_vel; } } - else - { + else{ // if there is no velocity change, then use the surface area // normal as the shock direction mag = sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) @@ -770,23 +717,20 @@ void FEA_Module_SGH::get_force_egradient_sgh(const DCArrayKokkos& ma + area_normal(node_lid, 2) * area_normal(node_lid, 2) ); // estimate of the shock direction - for (int dim = 0; dim < num_dims; dim++) - { + for (int dim = 0; dim < num_dims; dim++) { shock_dir(dim) = area_normal(node_lid, dim) / mag; } } // end if mag_vel // cell divergence indicates compression or expansions size_t mat_id = elem_mat_id(elem_gid); - if (div < 0) // element in compression - { + if (div < 0) { // element in compression muc(node_lid) = elem_den(elem_gid) * (material(mat_id).q1 * elem_sspd(elem_gid) + material(mat_id).q2 * mag_vel); muc_gradient(node_lid) = elem_den(elem_gid) * (material(mat_id).q1 * sound_speed_gradient_energy); } - else // element in expansion - { + else{ // element in expansion muc(node_lid) = elem_den(elem_gid) * (material(mat_id).q1ex * elem_sspd(elem_gid) + material(mat_id).q2ex * mag_vel); muc_gradient(node_lid) = elem_den(elem_gid) * @@ -798,8 +742,7 @@ void FEA_Module_SGH::get_force_egradient_sgh(const DCArrayKokkos& ma double mu_term_gradient; // Coding to use shock direction - if (use_shock_dir == 1) - { + if (use_shock_dir == 1) { // this is denominator of the Riamann solver and the multiplier // on velocity in the numerator. It filters on the shock // direction @@ -812,8 +755,7 @@ void FEA_Module_SGH::get_force_egradient_sgh(const DCArrayKokkos& ma + shock_dir(1) * area_normal(node_lid, 1) + shock_dir(2) * area_normal(node_lid, 2) ); } - else - { + else{ // Using a full tensoral Riemann jump relation mu_term = muc(node_lid) * sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) @@ -840,18 +782,14 @@ void FEA_Module_SGH::get_force_egradient_sgh(const DCArrayKokkos& ma } // end for node_lid loop over nodes of the elem // The Riemann velocity, called vel_star - if (sum(3) > fuzz) - { - for (size_t i = 0; i < num_dims; i++) - { + if (sum(3) > fuzz) { + for (size_t i = 0; i < num_dims; i++) { vel_star(i) = sum(i) / sum(3); vel_star_gradient(i) = sum_gradient(i) / sum(3) - sum_gradient(3) * sum(i) / sum(3) / sum(3); } } - else - { - for (int i = 0; i < num_dims; i++) - { + else{ + for (int i = 0; i < num_dims; i++) { vel_star(i) = 0.0; vel_star_gradient(i) = 0; } @@ -879,8 +817,7 @@ void FEA_Module_SGH::get_force_egradient_sgh(const DCArrayKokkos& ma // (1=nominal, and n_coeff > 1 oscillatory) // loop over the nieghboring cells - for (size_t elem_lid = 0; elem_lid < num_elems_in_elem(elem_gid); elem_lid++) - { + for (size_t elem_lid = 0; elem_lid < num_elems_in_elem(elem_gid); elem_lid++) { // Get global index for neighboring cell size_t neighbor_gid = elems_in_elem(elem_gid, elem_lid); @@ -918,8 +855,7 @@ void FEA_Module_SGH::get_force_egradient_sgh(const DCArrayKokkos& ma // ---- Calculate the Riemann force on each node ---- // loop over the each node in the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) - { + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { size_t corner_lid = node_lid; // Get corner gid @@ -929,8 +865,7 @@ void FEA_Module_SGH::get_force_egradient_sgh(const DCArrayKokkos& ma size_t node_gid = nodes_in_elem(elem_gid, node_lid); // loop over dimension - for (int dim = 0; dim < num_dims; dim++) - { + for (int dim = 0; dim < num_dims; dim++) { Force_Gradient_Energies(elem_gid, node_lid * num_dims + dim) = corner_vector_storage(corner_gid, dim) = area_normal(node_lid, 0) * tau_gradient(0, dim) + area_normal(node_lid, 1) * tau_gradient(1, dim) @@ -1012,15 +947,13 @@ void FEA_Module_SGH::get_force_ugradient_sgh(const DCArrayKokkos& ma // initialize gradient matrix FOR_ALL_CLASS(dof_gid, 0, nlocal_nodes * num_dims, { - for (int idof = 0; idof < Gradient_Matrix_Strides(dof_gid); idof++) - { + for (int idof = 0; idof < Gradient_Matrix_Strides(dof_gid); idof++) { Force_Gradient_Positions(dof_gid, idof) = 0; } }); // end parallel for loop over nodes Kokkos::fence(); // --- calculate the forces acting on the nodes from the element --- - for (size_t elem_gid = 0; elem_gid < rnum_elem; elem_gid++) - { + for (size_t elem_gid = 0; elem_gid < rnum_elem; elem_gid++) { // FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { const size_t num_nodes_in_elem = 8; @@ -1117,23 +1050,17 @@ void FEA_Module_SGH::get_force_ugradient_sgh(const DCArrayKokkos& ma rk_level); // the -1 is for the inward surface area normal, - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) - { - for (size_t dim = 0; dim < num_dims; dim++) - { + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { + for (size_t dim = 0; dim < num_dims; dim++) { area_normal(node_lid, dim) = (-1.0) * area_normal(node_lid, dim); } // end for } // end for // the -1 is for the inward surface area normal, - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) - { - for (size_t dim = 0; dim < num_dims; dim++) - { - for (size_t igradient = 0; igradient < num_nodes_in_elem; igradient++) - { - for (size_t jdim = 0; jdim < num_dims; jdim++) - { + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { + for (size_t dim = 0; dim < num_dims; dim++) { + for (size_t igradient = 0; igradient < num_nodes_in_elem; igradient++) { + for (size_t jdim = 0; jdim < num_dims; jdim++) { area_normal_gradients(node_lid, dim, igradient, jdim) = (-1.0) * area_normal_gradients(node_lid, dim, igradient, jdim); } } @@ -1154,26 +1081,20 @@ void FEA_Module_SGH::get_force_ugradient_sgh(const DCArrayKokkos& ma double mag_curl = sqrt(curl[0] * curl[0] + curl[1] * curl[1] + curl[2] * curl[2]); // --- Calculate the Cauchy stress --- - for (size_t i = 0; i < 3; i++) - { - for (size_t j = 0; j < 3; j++) - { + for (size_t i = 0; i < 3; i++) { + for (size_t j = 0; j < 3; j++) { tau(i, j) = stress(i, j); // artificial viscosity can be added here to tau } // end for } // end for // initialize gradient array - for (size_t i = 0; i < 3; i++) - { - for (size_t j = 0; j < 3; j++) - { - for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) - { + for (size_t i = 0; i < 3; i++) { + for (size_t j = 0; j < 3; j++) { + for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) { tau_gradient(i, j, igradient, 0) = 0; tau_gradient(i, j, igradient, 1) = 0; - if (num_dims == 3) - { + if (num_dims == 3) { tau_gradient(i, j, igradient, 2) = 0; } } @@ -1181,12 +1102,10 @@ void FEA_Module_SGH::get_force_ugradient_sgh(const DCArrayKokkos& ma } // end for // add the pressure - for (int i = 0; i < num_dims; i++) - { + for (int i = 0; i < num_dims; i++) { tau(i, i) -= elem_pres(elem_gid); // current expression only works for ideal gas! generalize. - for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) - { + for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) { tau_gradient(i, i, igradient, 0) = elem_pres(elem_gid) * volume_gradients(igradient, 0) / vol; tau_gradient(i, i, igradient, 1) = elem_pres(elem_gid) * volume_gradients(igradient, 1) / vol; tau_gradient(i, i, igradient, 2) = elem_pres(elem_gid) * volume_gradients(igradient, 2) / vol; @@ -1198,15 +1117,13 @@ void FEA_Module_SGH::get_force_ugradient_sgh(const DCArrayKokkos& ma // estimate of the Riemann velocity // initialize to Riemann velocity to zero - for (size_t dim = 0; dim < num_dims; dim++) - { + for (size_t dim = 0; dim < num_dims; dim++) { vel_star(dim) = 0.0; } // loop over nodes and calculate an average velocity, which is // an estimate of Riemann velocity - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) - { + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { // Get node gloabl index and create view of nodal velocity int node_gid = nodes_in_elem(elem_gid, node_lid); @@ -1220,16 +1137,12 @@ void FEA_Module_SGH::get_force_ugradient_sgh(const DCArrayKokkos& ma // find shock direction and shock impedance associated with each node // initialize sum term in MARS to zero - for (int i = 0; i < 4; i++) - { + for (int i = 0; i < 4; i++) { sum(i) = 0.0; } - for (int i = 0; i < 4; i++) - { - for (size_t igradient = 0; igradient < num_nodes_in_elem; igradient++) - { - for (size_t jdim = 0; jdim < num_dims; jdim++) - { + for (int i = 0; i < 4; i++) { + for (size_t igradient = 0; igradient < num_nodes_in_elem; igradient++) { + for (size_t jdim = 0; jdim < num_dims; jdim++) { sum_gradient(i, igradient, jdim) = 0.0; } } @@ -1239,8 +1152,7 @@ void FEA_Module_SGH::get_force_ugradient_sgh(const DCArrayKokkos& ma double mag_vel; // magnitude of velocity // loop over the nodes of the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) - { + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { // Get global node id size_t node_gid = nodes_in_elem(elem_gid, node_lid); @@ -1252,16 +1164,13 @@ void FEA_Module_SGH::get_force_ugradient_sgh(const DCArrayKokkos& ma + (vel(1) - vel_star(1) ) * (vel(1) - vel_star(1) ) + (vel(2) - vel_star(2) ) * (vel(2) - vel_star(2) ) ); - if (mag_vel > small) - { + if (mag_vel > small) { // estimate of the shock direction, a unit normal - for (int dim = 0; dim < num_dims; dim++) - { + for (int dim = 0; dim < num_dims; dim++) { shock_dir(dim) = (vel(dim) - vel_star(dim)) / mag_vel; } } - else - { + else{ // if there is no velocity change, then use the surface area // normal as the shock direction mag = sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) @@ -1269,21 +1178,18 @@ void FEA_Module_SGH::get_force_ugradient_sgh(const DCArrayKokkos& ma + area_normal(node_lid, 2) * area_normal(node_lid, 2) ); // estimate of the shock direction - for (int dim = 0; dim < num_dims; dim++) - { + for (int dim = 0; dim < num_dims; dim++) { shock_dir(dim) = area_normal(node_lid, dim) / mag; } } // end if mag_vel // cell divergence indicates compression or expansions size_t mat_id = elem_mat_id(elem_gid); - if (div < 0) // element in compression - { + if (div < 0) { // element in compression muc(node_lid) = elem_den(elem_gid) * (material(mat_id).q1 * elem_sspd(elem_gid) + material(mat_id).q2 * mag_vel); - for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) - { + for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) { muc_gradient(node_lid, igradient, 0) = -elem_den(elem_gid) * volume_gradients(igradient, 0) / vol * (material(mat_id).q1 * elem_sspd(elem_gid)); muc_gradient(node_lid, igradient, 1) = -elem_den(elem_gid) * volume_gradients(igradient, 1) / vol * @@ -1292,12 +1198,10 @@ void FEA_Module_SGH::get_force_ugradient_sgh(const DCArrayKokkos& ma (material(mat_id).q1 * elem_sspd(elem_gid)); } } - else // element in expansion - { + else{ // element in expansion muc(node_lid) = elem_den(elem_gid) * (material(mat_id).q1ex * elem_sspd(elem_gid) + material(mat_id).q2ex * mag_vel); - for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) - { + for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) { muc_gradient(node_lid, igradient, 0) = -elem_den(elem_gid) * volume_gradients(igradient, 0) / vol * (material(mat_id).q1ex * elem_sspd(elem_gid)); muc_gradient(node_lid, igradient, 1) = -elem_den(elem_gid) * volume_gradients(igradient, 1) / vol * @@ -1312,8 +1216,7 @@ void FEA_Module_SGH::get_force_ugradient_sgh(const DCArrayKokkos& ma double mu_term_gradient; // Coding to use shock direction - if (use_shock_dir == 1) - { + if (use_shock_dir == 1) { // this is denominator of the Riamann solver and the multiplier // on velocity in the numerator. It filters on the shock // direction @@ -1330,17 +1233,14 @@ void FEA_Module_SGH::get_force_ugradient_sgh(const DCArrayKokkos& ma // + shock_dir(1)*area_normal_gradients(node_lid,1) // + shock_dir(2)*area_normal_gradients(node_lid,2) ); } - else - { + else{ // Using a full tensoral Riemann jump relation mu_term = muc(node_lid) * sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) + area_normal(node_lid, 1) * area_normal(node_lid, 1) + area_normal(node_lid, 2) * area_normal(node_lid, 2) ); - for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) - { - for (int jdim = 0; jdim < num_dims; jdim++) - { + for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) { + for (int jdim = 0; jdim < num_dims; jdim++) { mu_term_gradient = muc_gradient(node_lid, igradient, jdim) * sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) + area_normal(node_lid, 1) * area_normal(node_lid, 1) @@ -1365,10 +1265,8 @@ void FEA_Module_SGH::get_force_ugradient_sgh(const DCArrayKokkos& ma // sum gradients - for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) - { - for (int jdim = 0; jdim < num_dims; jdim++) - { + for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) { + for (int jdim = 0; jdim < num_dims; jdim++) { sum_gradient(0, igradient, jdim) += muc_gradient(node_lid, igradient, jdim) * vel(0); sum_gradient(1, igradient, jdim) += muc_gradient(node_lid, igradient, jdim) * vel(1); sum_gradient(2, igradient, jdim) += muc_gradient(node_lid, igradient, jdim) * vel(2); @@ -1380,43 +1278,31 @@ void FEA_Module_SGH::get_force_ugradient_sgh(const DCArrayKokkos& ma } // end for node_lid loop over nodes of the elem // The Riemann velocity, called vel_star - if (sum(3) > fuzz) - { - for (size_t i = 0; i < num_dims; i++) - { + if (sum(3) > fuzz) { + for (size_t i = 0; i < num_dims; i++) { vel_star(i) = sum(i) / sum(3); } } - else - { - for (int i = 0; i < num_dims; i++) - { + else{ + for (int i = 0; i < num_dims; i++) { vel_star(i) = 0.0; } } // end if // vel star gradients - if (sum(3) > fuzz) - { - for (size_t i = 0; i < num_dims; i++) - { - for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) - { - for (int jdim = 0; jdim < num_dims; jdim++) - { + if (sum(3) > fuzz) { + for (size_t i = 0; i < num_dims; i++) { + for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) { + for (int jdim = 0; jdim < num_dims; jdim++) { vel_star_gradient(i, igradient, jdim) = sum_gradient(i, igradient, jdim) / sum(3) - sum(i) / (sum(3) * sum(3)) * sum_gradient(3, igradient, jdim); } } } } - else - { - for (int i = 0; i < num_dims; i++) - { - for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) - { - for (int jdim = 0; jdim < num_dims; jdim++) - { + else{ + for (int i = 0; i < num_dims; i++) { + for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) { + for (int jdim = 0; jdim < num_dims; jdim++) { vel_star_gradient(i, igradient, jdim) = 0; } } @@ -1445,8 +1331,7 @@ void FEA_Module_SGH::get_force_ugradient_sgh(const DCArrayKokkos& ma // (1=nominal, and n_coeff > 1 oscillatory) // loop over the nieghboring cells - for (size_t elem_lid = 0; elem_lid < num_elems_in_elem(elem_gid); elem_lid++) - { + for (size_t elem_lid = 0; elem_lid < num_elems_in_elem(elem_gid); elem_lid++) { // Get global index for neighboring cell size_t neighbor_gid = elems_in_elem(elem_gid, elem_lid); @@ -1484,8 +1369,7 @@ void FEA_Module_SGH::get_force_ugradient_sgh(const DCArrayKokkos& ma // ---- Calculate the Riemann force on each node ---- // loop over the each node in the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) - { + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { size_t corner_lid = node_lid; size_t column_index; @@ -1496,18 +1380,14 @@ void FEA_Module_SGH::get_force_ugradient_sgh(const DCArrayKokkos& ma size_t node_gid = nodes_in_elem(elem_gid, node_lid); // loop over dimension - for (int dim = 0; dim < num_dims; dim++) - { + for (int dim = 0; dim < num_dims; dim++) { // assign gradient of corner contribution of force to relevant matrix entries with non-zero node velocity gradient - for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) - { - for (int jdim = 0; jdim < num_dims; jdim++) - { + for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) { + for (int jdim = 0; jdim < num_dims; jdim++) { size_t gradient_node_gid = nodes_in_elem(elem_gid, igradient); // if(!map->isNodeLocalElement(gradient_node_gid)) continue; column_index = num_dims * Global_Gradient_Matrix_Assembly_Map(elem_gid, igradient, node_lid); - if (map->isNodeLocalElement(gradient_node_gid)) - { + if (map->isNodeLocalElement(gradient_node_gid)) { Force_Gradient_Positions(gradient_node_gid * num_dims + jdim, column_index + dim) += area_normal(node_lid, 0) * tau_gradient(0, dim, igradient, jdim) + area_normal(node_lid, 1) * tau_gradient(1, dim, igradient, jdim) + area_normal(node_lid, 2) * tau_gradient(2, dim, igradient, jdim) @@ -1593,33 +1473,25 @@ void FEA_Module_SGH::force_design_gradient_term(const_vec_array design_variables CArrayKokkos current_element_adjoint = CArrayKokkos(num_nodes_in_elem, num_dim); // gradient contribution from gradient of Force vector with respect to design variable. - if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) - { - if (myrank == 0) - { + if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) { + if (myrank == 0) { std::cout << "gradient term involving adjoint derivative" << std::endl; } } - for (unsigned long cycle = 0; cycle < last_time_step + 1; cycle++) - { + for (unsigned long cycle = 0; cycle < last_time_step + 1; cycle++) { // compute timestep from time data global_dt = time_data[cycle + 1] - time_data[cycle]; // print - if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) - { - if (cycle == 0) - { - if (myrank == 0) - { + if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) { + if (cycle == 0) { + if (myrank == 0) { printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); } } // print time step every 10 cycles - else if (cycle % 20 == 0) - { - if (myrank == 0) - { + else if (cycle % 20 == 0) { + if (myrank == 0) { printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); } } // end if @@ -1638,8 +1510,7 @@ void FEA_Module_SGH::force_design_gradient_term(const_vec_array design_variables // first half of integration step calculation FOR_ALL_CLASS(node_gid, 0, nall_nodes, { - for (int idim = 0; idim < num_dim; idim++) - { + for (int idim = 0; idim < num_dim; idim++) { node_vel(rk_level, node_gid, idim) = current_velocity_vector(node_gid, idim); node_coords(rk_level, node_gid, idim) = current_coord_vector(node_gid, idim); } @@ -1654,15 +1525,13 @@ void FEA_Module_SGH::force_design_gradient_term(const_vec_array design_variables get_vol(); // ---- Calculate velocity diveregence for the element ---- - if (num_dim == 2) - { + if (num_dim == 2) { get_divergence2D(elem_div, node_coords, node_vel, elem_vol); } - else - { + else{ get_divergence(elem_div, node_coords, node_vel, @@ -1670,8 +1539,7 @@ void FEA_Module_SGH::force_design_gradient_term(const_vec_array design_variables } // end if 2D // ---- Calculate elem state (den, pres, sound speed, stress) for next time step ---- - if (num_dim == 2) - { + if (num_dim == 2) { update_state2D(material, *mesh, node_coords, @@ -1687,8 +1555,7 @@ void FEA_Module_SGH::force_design_gradient_term(const_vec_array design_variables 1.0, cycle); } - else - { + else{ update_state(material, *mesh, node_coords, @@ -1727,18 +1594,15 @@ void FEA_Module_SGH::force_design_gradient_term(const_vec_array design_variables real_t inner_product; inner_product = 0; - for (int ifill = 0; ifill < num_nodes_in_elem; ifill++) - { + for (int ifill = 0; ifill < num_nodes_in_elem; ifill++) { node_id = nodes_in_elem(elem_id, ifill); corner_id = elem_id * num_nodes_in_elem + ifill; - for (int idim = 0; idim < num_dim; idim++) - { + for (int idim = 0; idim < num_dim; idim++) { inner_product += corner_vector_storage(corner_id, idim) * current_adjoint_vector(node_id, idim); } } - for (int inode = 0; inode < num_nodes_in_elem; inode++) - { + for (int inode = 0; inode < num_nodes_in_elem; inode++) { // compute gradient of local element contribution to v^t*M*v product corner_id = elem_id * num_nodes_in_elem + inode; corner_value_storage(corner_id) = inner_product; @@ -1750,8 +1614,7 @@ void FEA_Module_SGH::force_design_gradient_term(const_vec_array design_variables // multiply FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { size_t corner_id; - for (int icorner = 0; icorner < num_corners_in_node(node_id); icorner++) - { + for (int icorner = 0; icorner < num_corners_in_node(node_id); icorner++) { corner_id = corners_in_node(node_id, icorner); design_gradients(node_id, 0) += -corner_value_storage(corner_id) * global_dt; } @@ -1802,8 +1665,7 @@ void FEA_Module_SGH::get_force_dgradient_sgh(const DCArrayKokkos& ma const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; const size_t num_dims = simparam->num_dims; // --- calculate the forces acting on the nodes from the element --- - for (size_t elem_gid = 0; elem_gid < rnum_elem; elem_gid++) - { + for (size_t elem_gid = 0; elem_gid < rnum_elem; elem_gid++) { // FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { const size_t num_nodes_in_elem = 8; @@ -1875,10 +1737,8 @@ void FEA_Module_SGH::get_force_dgradient_sgh(const DCArrayKokkos& ma rk_level); // the -1 is for the inward surface area normal, - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) - { - for (size_t dim = 0; dim < num_dims; dim++) - { + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { + for (size_t dim = 0; dim < num_dims; dim++) { area_normal(node_lid, dim) = (-1.0) * area_normal(node_lid, dim); } // end for } // end for @@ -1897,18 +1757,15 @@ void FEA_Module_SGH::get_force_dgradient_sgh(const DCArrayKokkos& ma double mag_curl = sqrt(curl[0] * curl[0] + curl[1] * curl[1] + curl[2] * curl[2]); // --- Calculate the Cauchy stress --- - for (size_t i = 0; i < 3; i++) - { - for (size_t j = 0; j < 3; j++) - { + for (size_t i = 0; i < 3; i++) { + for (size_t j = 0; j < 3; j++) { tau(i, j) = stress(i, j); tau_gradient(i, j) = 0; } // end for } // end for // add the pressure - for (int i = 0; i < num_dims; i++) - { + for (int i = 0; i < num_dims; i++) { tau(i, i) -= elem_pres(elem_gid); tau_gradient(i, i) -= elem_pres(elem_gid) / num_nodes_in_elem / relative_element_densities(elem_gid); } // end for @@ -1918,15 +1775,13 @@ void FEA_Module_SGH::get_force_dgradient_sgh(const DCArrayKokkos& ma // estimate of the Riemann velocity // initialize to Riemann velocity to zero - for (size_t dim = 0; dim < num_dims; dim++) - { + for (size_t dim = 0; dim < num_dims; dim++) { vel_star(dim) = 0.0; } // loop over nodes and calculate an average velocity, which is // an estimate of Riemann velocity - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) - { + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { // Get node gloabl index and create view of nodal velocity int node_gid = nodes_in_elem(elem_gid, node_lid); @@ -1940,8 +1795,7 @@ void FEA_Module_SGH::get_force_dgradient_sgh(const DCArrayKokkos& ma // find shock direction and shock impedance associated with each node // initialize sum term in MARS to zero - for (int i = 0; i < 4; i++) - { + for (int i = 0; i < 4; i++) { sum(i) = sum_gradient(i) = 0.0; } @@ -1949,8 +1803,7 @@ void FEA_Module_SGH::get_force_dgradient_sgh(const DCArrayKokkos& ma double mag_vel; // magnitude of velocity // loop over the nodes of the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) - { + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { // Get global node id size_t node_gid = nodes_in_elem(elem_gid, node_lid); @@ -1962,16 +1815,13 @@ void FEA_Module_SGH::get_force_dgradient_sgh(const DCArrayKokkos& ma + (vel(1) - vel_star(1) ) * (vel(1) - vel_star(1) ) + (vel(2) - vel_star(2) ) * (vel(2) - vel_star(2) ) ); - if (mag_vel > small) - { + if (mag_vel > small) { // estimate of the shock direction, a unit normal - for (int dim = 0; dim < num_dims; dim++) - { + for (int dim = 0; dim < num_dims; dim++) { shock_dir(dim) = (vel(dim) - vel_star(dim)) / mag_vel; } } - else - { + else{ // if there is no velocity change, then use the surface area // normal as the shock direction mag = sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) @@ -1979,24 +1829,21 @@ void FEA_Module_SGH::get_force_dgradient_sgh(const DCArrayKokkos& ma + area_normal(node_lid, 2) * area_normal(node_lid, 2) ); // estimate of the shock direction - for (int dim = 0; dim < num_dims; dim++) - { + for (int dim = 0; dim < num_dims; dim++) { shock_dir(dim) = area_normal(node_lid, dim) / mag; } } // end if mag_vel // cell divergence indicates compression or expansions size_t mat_id = elem_mat_id(elem_gid); - if (div < 0) // element in compression - { + if (div < 0) { // element in compression muc(node_lid) = elem_den(elem_gid) * (material(mat_id).q1 * elem_sspd(elem_gid) + material(mat_id).q2 * mag_vel); muc_gradient(node_lid) = elem_den(elem_gid) / relative_element_densities(elem_gid) / num_nodes_in_elem * (material(mat_id).q1 * elem_sspd(elem_gid) + material(mat_id).q2 * mag_vel); } - else // element in expansion - { + else{ // element in expansion muc(node_lid) = elem_den(elem_gid) * (material(mat_id).q1ex * elem_sspd(elem_gid) + material(mat_id).q2ex * mag_vel); muc_gradient(node_lid) = elem_den(elem_gid) / relative_element_densities(elem_gid) / num_nodes_in_elem * @@ -2008,8 +1855,7 @@ void FEA_Module_SGH::get_force_dgradient_sgh(const DCArrayKokkos& ma double mu_term_gradient; // Coding to use shock direction - if (use_shock_dir == 1) - { + if (use_shock_dir == 1) { // this is denominator of the Riamann solver and the multiplier // on velocity in the numerator. It filters on the shock // direction @@ -2022,8 +1868,7 @@ void FEA_Module_SGH::get_force_dgradient_sgh(const DCArrayKokkos& ma + shock_dir(1) * area_normal(node_lid, 1) + shock_dir(2) * area_normal(node_lid, 2) ); } - else - { + else{ // Using a full tensoral Riemann jump relation mu_term = muc(node_lid) * sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) @@ -2050,18 +1895,14 @@ void FEA_Module_SGH::get_force_dgradient_sgh(const DCArrayKokkos& ma } // end for node_lid loop over nodes of the elem // The Riemann velocity, called vel_star - if (sum(3) > fuzz) - { - for (size_t i = 0; i < num_dims; i++) - { + if (sum(3) > fuzz) { + for (size_t i = 0; i < num_dims; i++) { vel_star(i) = sum(i) / sum(3); vel_star_gradient(i) = sum_gradient(i) / sum(3) - sum_gradient(3) * sum(i) / sum(3) / sum(3); } } - else - { - for (int i = 0; i < num_dims; i++) - { + else{ + for (int i = 0; i < num_dims; i++) { vel_star(i) = 0.0; vel_star_gradient(i) = 0.0; } @@ -2089,8 +1930,7 @@ void FEA_Module_SGH::get_force_dgradient_sgh(const DCArrayKokkos& ma // (1=nominal, and n_coeff > 1 oscillatory) // loop over the nieghboring cells - for (size_t elem_lid = 0; elem_lid < num_elems_in_elem(elem_gid); elem_lid++) - { + for (size_t elem_lid = 0; elem_lid < num_elems_in_elem(elem_gid); elem_lid++) { // Get global index for neighboring cell size_t neighbor_gid = elems_in_elem(elem_gid, elem_lid); @@ -2128,8 +1968,7 @@ void FEA_Module_SGH::get_force_dgradient_sgh(const DCArrayKokkos& ma // ---- Calculate the Riemann force on each node ---- // loop over the each node in the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) - { + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { size_t corner_lid = node_lid; // Get corner gid @@ -2139,8 +1978,7 @@ void FEA_Module_SGH::get_force_dgradient_sgh(const DCArrayKokkos& ma size_t node_gid = nodes_in_elem(elem_gid, node_lid); // loop over dimension - for (int dim = 0; dim < num_dims; dim++) - { + for (int dim = 0; dim < num_dims; dim++) { corner_vector_storage(corner_gid, dim) = area_normal(node_lid, 0) * tau_gradient(0, dim) + area_normal(node_lid, 1) * tau_gradient(1, dim) diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_sgh.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_sgh.cpp index 71d2adbc8..c3dee5cf9 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_sgh.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_sgh.cpp @@ -80,10 +80,8 @@ void FEA_Module_SGH::get_force_sgh(const DCArrayKokkos& material, // check to see if any material model will be run on the host bool any_host_material_model_run = false; - for (int imat = 0; imat < material.size(); imat++) - { - if (material.host(imat).strength_run_location == RUN_LOCATION::host) - { + for (int imat = 0; imat < material.size(); imat++) { + if (material.host(imat).strength_run_location == RUN_LOCATION::host) { any_host_material_model_run = true; } } @@ -154,19 +152,15 @@ void FEA_Module_SGH::get_force_sgh(const DCArrayKokkos& material, rk_level); // save vel_grad in elem_vel_grad - for (size_t i = 0; i < 3; i++) - { - for (size_t j = 0; j < 3; j++) - { + for (size_t i = 0; i < 3; i++) { + for (size_t j = 0; j < 3; j++) { elem_vel_grad(elem_gid, i, j) = vel_grad(i, j); } } // the -1 is for the inward surface area normal, - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) - { - for (size_t dim = 0; dim < num_dims; dim++) - { + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { + for (size_t dim = 0; dim < num_dims; dim++) { area_normal(node_lid, dim) = (-1.0) * area_normal(node_lid, dim); } // end for } // end for @@ -185,20 +179,16 @@ void FEA_Module_SGH::get_force_sgh(const DCArrayKokkos& material, double mag_curl = sqrt(curl[0] * curl[0] + curl[1] * curl[1] + curl[2] * curl[2]); // --- Calculate the Cauchy stress --- - for (size_t i = 0; i < 3; i++) - { - for (size_t j = 0; j < 3; j++) - { + for (size_t i = 0; i < 3; i++) { + for (size_t j = 0; j < 3; j++) { tau(i, j) = stress(i, j); // artificial viscosity can be added here to tau } // end for } // end for // add the pressure - if (elem_pres(elem_gid) != 0) - { - for (int i = 0; i < num_dims; i++) - { + if (elem_pres(elem_gid) != 0) { + for (int i = 0; i < num_dims; i++) { tau(i, i) -= elem_pres(elem_gid); } // end for } @@ -208,15 +198,13 @@ void FEA_Module_SGH::get_force_sgh(const DCArrayKokkos& material, // estimate of the Riemann velocity // initialize to Riemann velocity to zero - for (size_t dim = 0; dim < num_dims; dim++) - { + for (size_t dim = 0; dim < num_dims; dim++) { vel_star(dim) = 0.0; } // loop over nodes and calculate an average velocity, which is // an estimate of Riemann velocity - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) - { + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { // Get node gloabl index and create view of nodal velocity int node_gid = nodes_in_elem(elem_gid, node_lid); @@ -230,8 +218,7 @@ void FEA_Module_SGH::get_force_sgh(const DCArrayKokkos& material, // find shock direction and shock impedance associated with each node // initialize sum term in MARS to zero - for (int i = 0; i < 4; i++) - { + for (int i = 0; i < 4; i++) { sum(i) = 0.0; } @@ -240,8 +227,7 @@ void FEA_Module_SGH::get_force_sgh(const DCArrayKokkos& material, size_t mat_id = elem_mat_id(elem_gid); // loop over the nodes of the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) - { + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { // Get global node id size_t node_gid = nodes_in_elem(elem_gid, node_lid); @@ -253,16 +239,13 @@ void FEA_Module_SGH::get_force_sgh(const DCArrayKokkos& material, + (vel(1) - vel_star(1) ) * (vel(1) - vel_star(1) ) + (vel(2) - vel_star(2) ) * (vel(2) - vel_star(2) ) ); - if (mag_vel > small) - { + if (mag_vel > small) { // estimate of the shock direction, a unit normal - for (int dim = 0; dim < num_dims; dim++) - { + for (int dim = 0; dim < num_dims; dim++) { shock_dir(dim) = (vel(dim) - vel_star(dim)) / mag_vel; } } - else - { + else{ // if there is no velocity change, then use the surface area // normal as the shock direction mag = sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) @@ -270,20 +253,17 @@ void FEA_Module_SGH::get_force_sgh(const DCArrayKokkos& material, + area_normal(node_lid, 2) * area_normal(node_lid, 2) ); // estimate of the shock direction - for (int dim = 0; dim < num_dims; dim++) - { + for (int dim = 0; dim < num_dims; dim++) { shock_dir(dim) = area_normal(node_lid, dim) / mag; } } // end if mag_vel // cell divergence indicates compression or expansions - if (div < 0) // element in compression - { + if (div < 0) { // element in compression muc(node_lid) = elem_den(elem_gid) * (material(mat_id).q1 * elem_sspd(elem_gid) + material(mat_id).q2 * mag_vel); } - else // element in expansion - { + else{ // element in expansion muc(node_lid) = elem_den(elem_gid) * (material(mat_id).q1ex * elem_sspd(elem_gid) + material(mat_id).q2ex * mag_vel); } // end if on divergence sign @@ -292,8 +272,7 @@ void FEA_Module_SGH::get_force_sgh(const DCArrayKokkos& material, double mu_term; // Coding to use shock direction - if (use_shock_dir == 1) - { + if (use_shock_dir == 1) { // this is denominator of the Riamann solver and the multiplier // on velocity in the numerator. It filters on the shock // direction @@ -302,8 +281,7 @@ void FEA_Module_SGH::get_force_sgh(const DCArrayKokkos& material, + shock_dir(1) * area_normal(node_lid, 1) + shock_dir(2) * area_normal(node_lid, 2) ); } - else - { + else{ // Using a full tensoral Riemann jump relation mu_term = muc(node_lid) * sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) @@ -320,17 +298,13 @@ void FEA_Module_SGH::get_force_sgh(const DCArrayKokkos& material, } // end for node_lid loop over nodes of the elem // The Riemann velocity, called vel_star - if (sum(3) > fuzz) - { - for (size_t i = 0; i < num_dims; i++) - { + if (sum(3) > fuzz) { + for (size_t i = 0; i < num_dims; i++) { vel_star(i) = sum(i) / sum(3); } } - else - { - for (int i = 0; i < num_dims; i++) - { + else{ + for (int i = 0; i < num_dims; i++) { vel_star(i) = 0.0; } } // end if @@ -357,8 +331,7 @@ void FEA_Module_SGH::get_force_sgh(const DCArrayKokkos& material, // (1=nominal, and n_coeff > 1 oscillatory) // loop over the nieghboring cells - for (size_t elem_lid = 0; elem_lid < num_elems_in_elem(elem_gid); elem_lid++) - { + for (size_t elem_lid = 0; elem_lid < num_elems_in_elem(elem_gid); elem_lid++) { // Get global index for neighboring cell size_t neighbor_gid = elems_in_elem(elem_gid, elem_lid); @@ -389,16 +362,14 @@ void FEA_Module_SGH::get_force_sgh(const DCArrayKokkos& material, phi = fmin(phi_curl * phi, alpha * phi); // if noise arrises in simulation on really smooth flows, then try something like phi = fmax(phi, 0.001); // ensuring a very small amount of dissipation for stability and robustness - if (material(mat_id).maximum_limiter) - { + if (material(mat_id).maximum_limiter) { phi = 1; } // ---- Calculate the Riemann force on each node ---- // loop over the each node in the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) - { + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { size_t corner_lid = node_lid; // Get corner gid @@ -408,8 +379,7 @@ void FEA_Module_SGH::get_force_sgh(const DCArrayKokkos& material, size_t node_gid = nodes_in_elem(elem_gid, node_lid); // loop over dimension - for (int dim = 0; dim < num_dims; dim++) - { + for (int dim = 0; dim < num_dims; dim++) { corner_force(corner_gid, dim) = area_normal(node_lid, 0) * tau(0, dim) + area_normal(node_lid, 1) * tau(1, dim) @@ -425,10 +395,8 @@ void FEA_Module_SGH::get_force_sgh(const DCArrayKokkos& material, // calculate the new stress at the next rk level, if it is a hypo model // hypo elastic plastic model - if (material(mat_id).strength_type == STRENGTH_TYPE::hypo) - { - if (material(mat_id).strength_run_location == RUN_LOCATION::device) - { + if (material(mat_id).strength_type == STRENGTH_TYPE::hypo) { + if (material(mat_id).strength_run_location == RUN_LOCATION::device) { // cut out the node_gids for this element ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 8); @@ -459,8 +427,7 @@ void FEA_Module_SGH::get_force_sgh(const DCArrayKokkos& material, } // end logical on hypo strength model }); // end parallel for loop over elements - if (any_host_material_model_run == true) - { + if (any_host_material_model_run == true) { // update host elem_vel_grad.update_host(); // below host updates are commented out to save time because they are not used for @@ -473,16 +440,13 @@ void FEA_Module_SGH::get_force_sgh(const DCArrayKokkos& material, // elem_vol.update_host(); // calling user strength model on host - for (size_t elem_gid = 0; elem_gid < mesh.num_elems; elem_gid++) - { + for (size_t elem_gid = 0; elem_gid < mesh.num_elems; elem_gid++) { const size_t num_dims = 3; size_t mat_id = elem_mat_id.host(elem_gid); // hypo elastic plastic model - if (material.host(mat_id).strength_type == STRENGTH_TYPE::hypo) - { - if (material.host(mat_id).strength_run_location == RUN_LOCATION::host) - { + if (material.host(mat_id).strength_type == STRENGTH_TYPE::hypo) { + if (material.host(mat_id).strength_run_location == RUN_LOCATION::host) { // cut out the node_gids for this element ViewCArrayKokkos elem_node_gids(&nodes_in_elem.host(elem_gid, 0), 8); @@ -628,10 +592,8 @@ void FEA_Module_SGH::get_force_sgh2D(const DCArrayKokkos& material, rk_level); // the -1 is for the inward surface area normal, - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) - { - for (size_t dim = 0; dim < num_dims; dim++) - { + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { + for (size_t dim = 0; dim < num_dims; dim++) { area_normal(node_lid, dim) = (-1.0) * area_normal(node_lid, dim); } // end for } // end for @@ -647,18 +609,15 @@ void FEA_Module_SGH::get_force_sgh2D(const DCArrayKokkos& material, double mag_curl = curl; // --- Calculate the Cauchy stress --- - for (size_t i = 0; i < 3; i++) - { - for (size_t j = 0; j < 3; j++) - { + for (size_t i = 0; i < 3; i++) { + for (size_t j = 0; j < 3; j++) { tau(i, j) = stress(i, j); // artificial viscosity can be added here to tau } // end for } // end for // add the pressure - for (int i = 0; i < 3; i++) - { + for (int i = 0; i < 3; i++) { tau(i, i) -= elem_pres(elem_gid); } // end for @@ -667,15 +626,13 @@ void FEA_Module_SGH::get_force_sgh2D(const DCArrayKokkos& material, // estimate of the Riemann velocity // initialize to Riemann velocity to zero - for (size_t dim = 0; dim < num_dims; dim++) - { + for (size_t dim = 0; dim < num_dims; dim++) { vel_star(dim) = 0.0; } // loop over nodes and calculate an average velocity, which is // an estimate of Riemann velocity - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) - { + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { // Get node gloabl index and create view of nodal velocity int node_gid = nodes_in_elem(elem_gid, node_lid); @@ -688,8 +645,7 @@ void FEA_Module_SGH::get_force_sgh2D(const DCArrayKokkos& material, // find shock direction and shock impedance associated with each node // initialize sum term in MARS to zero - for (int i = 0; i < 4; i++) - { + for (int i = 0; i < 4; i++) { sum(i) = 0.0; } @@ -697,8 +653,7 @@ void FEA_Module_SGH::get_force_sgh2D(const DCArrayKokkos& material, double mag_vel; // magnitude of velocity // loop over the nodes of the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) - { + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { // Get global node id size_t node_gid = nodes_in_elem(elem_gid, node_lid); @@ -709,37 +664,31 @@ void FEA_Module_SGH::get_force_sgh2D(const DCArrayKokkos& material, mag_vel = sqrt( (vel(0) - vel_star(0) ) * (vel(0) - vel_star(0) ) + (vel(1) - vel_star(1) ) * (vel(1) - vel_star(1) ) ); - if (mag_vel > small) - { + if (mag_vel > small) { // estimate of the shock direction, a unit normal - for (int dim = 0; dim < num_dims; dim++) - { + for (int dim = 0; dim < num_dims; dim++) { shock_dir(dim) = (vel(dim) - vel_star(dim)) / mag_vel; } } - else - { + else{ // if there is no velocity change, then use the surface area // normal as the shock direction mag = sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) + area_normal(node_lid, 1) * area_normal(node_lid, 1) ); // estimate of the shock direction - for (int dim = 0; dim < num_dims; dim++) - { + for (int dim = 0; dim < num_dims; dim++) { shock_dir(dim) = area_normal(node_lid, dim) / mag; } } // end if mag_vel // cell divergence indicates compression or expansions size_t mat_id = elem_mat_id(elem_gid); - if (div < 0) // element in compression - { + if (div < 0) { // element in compression muc(node_lid) = elem_den(elem_gid) * (material(mat_id).q1 * elem_sspd(elem_gid) + material(mat_id).q2 * mag_vel); } - else // element in expansion - { + else{ // element in expansion muc(node_lid) = elem_den(elem_gid) * (material(mat_id).q1ex * elem_sspd(elem_gid) + material(mat_id).q2ex * mag_vel); } // end if on divergence sign @@ -748,8 +697,7 @@ void FEA_Module_SGH::get_force_sgh2D(const DCArrayKokkos& material, double mu_term; // Coding to use shock direction - if (use_shock_dir == 1) - { + if (use_shock_dir == 1) { // this is denominator of the Riamann solver and the multiplier // on velocity in the numerator. It filters on the shock // direction @@ -757,8 +705,7 @@ void FEA_Module_SGH::get_force_sgh2D(const DCArrayKokkos& material, fabs(shock_dir(0) * area_normal(0) + shock_dir(1) * area_normal(1) ); } - else - { + else{ // Using a full tensoral Riemann jump relation mu_term = muc(node_lid) * sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) @@ -773,17 +720,13 @@ void FEA_Module_SGH::get_force_sgh2D(const DCArrayKokkos& material, } // end for node_lid loop over nodes of the elem // The Riemann velocity, called vel_star - if (sum(3) > fuzz) - { - for (size_t i = 0; i < num_dims; i++) - { + if (sum(3) > fuzz) { + for (size_t i = 0; i < num_dims; i++) { vel_star(i) = sum(i) / sum(3); } } - else - { - for (int i = 0; i < num_dims; i++) - { + else{ + for (int i = 0; i < num_dims; i++) { vel_star(i) = 0.0; } } // end if @@ -810,8 +753,7 @@ void FEA_Module_SGH::get_force_sgh2D(const DCArrayKokkos& material, // (1=nominal, and n_coeff > 1 oscillatory) // loop over the nieghboring cells - for (size_t elem_lid = 0; elem_lid < num_elems_in_elem(elem_gid); elem_lid++) - { + for (size_t elem_lid = 0; elem_lid < num_elems_in_elem(elem_gid); elem_lid++) { // Get global index for neighboring cell size_t neighbor_gid = elems_in_elem(elem_gid, elem_lid); @@ -848,8 +790,7 @@ void FEA_Module_SGH::get_force_sgh2D(const DCArrayKokkos& material, // ---- Calculate the Riemann force on each node ---- // loop over the each node in the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) - { + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { size_t corner_lid = node_lid; // Get corner gid @@ -859,8 +800,7 @@ void FEA_Module_SGH::get_force_sgh2D(const DCArrayKokkos& material, size_t node_gid = nodes_in_elem(elem_gid, node_lid); // loop over dimension - for (int dim = 0; dim < num_dims; dim++) - { + for (int dim = 0; dim < num_dims; dim++) { corner_force(corner_gid, dim) = area_normal(node_lid, 0) * tau(0, dim) + area_normal(node_lid, 1) * tau(1, dim) @@ -871,8 +811,7 @@ void FEA_Module_SGH::get_force_sgh2D(const DCArrayKokkos& material, double node_radius = node_coords(rk_level, node_gid, 1); - if (node_radius > 1e-14) - { + if (node_radius > 1e-14) { // sigma_RZ / R_p corner_force(corner_gid, 0) += tau(1, 0) * elem_area * 0.25 / node_radius; @@ -887,8 +826,7 @@ void FEA_Module_SGH::get_force_sgh2D(const DCArrayKokkos& material, size_t mat_id = elem_mat_id(elem_gid); // hypo elastic plastic model - if (material(mat_id).strength_type == STRENGTH_TYPE::hypo) - { + if (material(mat_id).strength_type == STRENGTH_TYPE::hypo) { // cut out the node_gids for this element ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 4); @@ -975,38 +913,32 @@ void FEA_Module_SGH::applied_forces(const DCArrayKokkos& material, double node_force[3]; double applied_force[3]; double radius; - for (size_t dim = 0; dim < num_dim; dim++) - { + for (size_t dim = 0; dim < num_dim; dim++) { node_force[dim] = 0.0; current_node_coords[dim] = all_initial_node_coords(node_gid, dim); } // end for dim radius = sqrt(current_node_coords[0] * current_node_coords[0] + current_node_coords[1] * current_node_coords[1] + current_node_coords[2] * current_node_coords[2]); - for (size_t ilc = 0; ilc < num_lcs; ilc++) - { + for (size_t ilc = 0; ilc < num_lcs; ilc++) { // debug check // std::cout << "LOADING CONDITION VOLUME TYPE: " << to_string(loading(ilc).volume) << std::endl; bool fill_this = loading(ilc).volume.contains(current_node_coords); - if (fill_this) - { + if (fill_this) { // loop over all corners around the node and calculate the nodal force - for (size_t corner_lid = 0; corner_lid < num_corners_in_node(node_gid); corner_lid++) - { + for (size_t corner_lid = 0; corner_lid < num_corners_in_node(node_gid); corner_lid++) { // Get corner gid size_t corner_gid = corners_in_node(node_gid, corner_lid); applied_force[0] = loading(ilc).x; applied_force[1] = loading(ilc).y; applied_force[2] = loading(ilc).z; // loop over dimension - for (size_t dim = 0; dim < num_dim; dim++) - { + for (size_t dim = 0; dim < num_dim; dim++) { node_force[dim] += applied_force[dim] * (all_initial_node_coords(node_gid, 0) + all_initial_node_coords(node_gid, 1) + all_initial_node_coords(node_gid, 2)) / radius; } // end for dim } // end for corner_lid // update the velocity - for (int dim = 0; dim < num_dim; dim++) - { + for (int dim = 0; dim < num_dim; dim++) { node_vel(rk_level, node_gid, dim) += rk_alpha * dt * node_force[dim] / node_mass(node_gid); } // end for dim diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/geometry.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/geometry.cpp index 65e1e0085..4a7cfee6c 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/geometry.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/geometry.cpp @@ -61,8 +61,7 @@ void FEA_Module_SGH::update_position_sgh(double rk_alpha, // loop over all the nodes in the mesh FOR_ALL_CLASS(node_gid, 0, num_nodes, { - for (int dim = 0; dim < num_dims; dim++) - { + for (int dim = 0; dim < num_dims; dim++) { double half_vel = (node_vel(rk_level, node_gid, dim) + node_vel(0, node_gid, dim)) * 0.5; node_coords(rk_level, node_gid, dim) = node_coords(0, node_gid, dim) + rk_alpha * dt * half_vel; } @@ -108,8 +107,7 @@ void FEA_Module_SGH::get_bmatrix(const ViewCArrayKokkos& B_matrix, auto z = ViewCArrayKokkos(z_array, num_nodes); // get the coordinates of the nodes(rk,elem,node) in this element - for (int node_lid = 0; node_lid < num_nodes; node_lid++) - { + for (int node_lid = 0; node_lid < num_nodes; node_lid++) { x(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 0); y(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 1); z(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 2); @@ -298,8 +296,7 @@ void FEA_Module_SGH::get_vol() const size_t rk_level = rk_num_bins - 1; const size_t num_dims = num_dim; - if (num_dims == 2) - { + if (num_dims == 2) { FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { // cut out the node_gids for this element ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 4); @@ -307,8 +304,7 @@ void FEA_Module_SGH::get_vol() }); Kokkos::fence(); } - else - { + else{ FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { // cut out the node_gids for this element ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 8); @@ -352,8 +348,7 @@ void FEA_Module_SGH::get_vol_hex(const DViewCArrayKokkos& elem_vol, auto z = ViewCArrayKokkos(z_array, num_nodes); // get the coordinates of the nodes(rk,elem,node) in this element - for (int node_lid = 0; node_lid < num_nodes; node_lid++) - { + for (int node_lid = 0; node_lid < num_nodes; node_lid++) { x(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 0); y(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 1); z(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 2); @@ -410,8 +405,7 @@ void FEA_Module_SGH::get_bmatrix2D(const ViewCArrayKokkos& B_matrix, auto y = ViewCArrayKokkos(y_array, num_nodes); // get the coordinates of the nodes(rk,elem,node) in this element - for (int node_lid = 0; node_lid < num_nodes; node_lid++) - { + for (int node_lid = 0; node_lid < num_nodes; node_lid++) { x(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 0); y(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 1); } // end for @@ -505,8 +499,7 @@ void FEA_Module_SGH::get_vol_quad(const DViewCArrayKokkos& elem_vol, auto y = ViewCArrayKokkos(y_array, num_nodes); // get the coordinates of the nodes(rk,elem,node) in this element - for (int node_lid = 0; node_lid < num_nodes; node_lid++) - { + for (int node_lid = 0; node_lid < num_nodes; node_lid++) { x(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 0); y(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 1); } // end for @@ -553,8 +546,7 @@ double FEA_Module_SGH::get_area_quad(const size_t elem_gid, auto y = ViewCArrayKokkos(y_array, num_nodes); // get the coordinates of the nodes(rk,elem,node) in this element - for (int node_lid = 0; node_lid < num_nodes; node_lid++) - { + for (int node_lid = 0; node_lid < num_nodes; node_lid++) { x(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 0); y(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 1); } // end for @@ -638,8 +630,7 @@ void FEA_Module_SGH::get_area_weights2D(const ViewCArrayKokkos& corner_a // get the coordinates of the nodes(rk,elem,node) in this element rc = zc = 0.0; - for (int node_lid = 0; node_lid < num_nodes; node_lid++) - { + for (int node_lid = 0; node_lid < num_nodes; node_lid++) { x(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 0); y(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 1); rc += 0.25 * y(node_lid); @@ -679,8 +670,7 @@ void FEA_Module_SGH::get_vol_ugradient(const size_t gradient_node_id, const size const size_t rk_level = rk_num_bins - 1; const size_t num_dims = num_dim; - if (num_dims == 2) - { + if (num_dims == 2) { FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { // cut out the node_gids for this element ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 4); @@ -688,8 +678,7 @@ void FEA_Module_SGH::get_vol_ugradient(const size_t gradient_node_id, const size }); Kokkos::fence(); } - else - { + else{ FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { // cut out the node_gids for this element ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), 8); @@ -735,8 +724,7 @@ void FEA_Module_SGH::get_vol_hex_ugradient(const ViewCArrayKokkos& elem_ auto z = ViewCArrayKokkos(z_array, num_nodes); // get the coordinates of the nodes(rk,elem,node) in this element - for (int node_lid = 0; node_lid < num_nodes; node_lid++) - { + for (int node_lid = 0; node_lid < num_nodes; node_lid++) { x(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 0); y(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 1); z(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 2); @@ -745,12 +733,9 @@ void FEA_Module_SGH::get_vol_hex_ugradient(const ViewCArrayKokkos& elem_ double twelth = 1. / 12.; // element volume gradient - for (int inode = 0; inode < 8; inode++) - { - for (int idim = 0; idim < num_dims; idim++) - { - switch (num_dims * inode + idim) - { + for (int inode = 0; inode < 8; inode++) { + for (int idim = 0; idim < num_dims; idim++) { + switch (num_dims * inode + idim) { case 0: gradient_result = ((y(2) * (z(1) - z(3)) + y(7) * (z(3) - z(4)) + y(5) * (-z(1) + z(4)) + y(1) * (-z(2) - z(3) + z(4) + z(5)) + y(3) * (z(1) + z(2) - z(4) - z(7)) + y(4) * @@ -977,8 +962,7 @@ void FEA_Module_SGH::get_bmatrix_gradients(const ViewCArrayKokkos& B_mat auto gradient_terms = ViewCArrayKokkos(gradient_terms_array, 2, num_nodes); // get the coordinates of the nodes(rk,elem,node) in this element - for (int node_lid = 0; node_lid < num_nodes; node_lid++) - { + for (int node_lid = 0; node_lid < num_nodes; node_lid++) { x(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 0); y(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 1); z(node_lid) = node_coords(rk_level, elem_node_gids(node_lid), 2); @@ -1013,8 +997,7 @@ void FEA_Module_SGH::get_bmatrix_gradients(const ViewCArrayKokkos& B_mat gradient_terms(1, 6) = 0; gradient_terms(1, 7) = -y(3) + y(4); - for (int inode = 0; inode < 8; inode++) - { + for (int inode = 0; inode < 8; inode++) { B_matrix_gradients(0, 0, inode, 0) = 0; B_matrix_gradients(0, 0, inode, 1) = gradient_terms(0, inode) * twelth; B_matrix_gradients(0, 0, inode, 2) = gradient_terms(1, inode) * twelth; @@ -1047,8 +1030,7 @@ void FEA_Module_SGH::get_bmatrix_gradients(const ViewCArrayKokkos& B_mat gradient_terms(1, 6) = y(2) - y(5); gradient_terms(1, 7) = 0; - for (int inode = 0; inode < 8; inode++) - { + for (int inode = 0; inode < 8; inode++) { B_matrix_gradients(1, 0, inode, 0) = 0; B_matrix_gradients(1, 0, inode, 1) = gradient_terms(0, inode) * twelth; B_matrix_gradients(1, 0, inode, 2) = gradient_terms(1, inode) * twelth; @@ -1081,8 +1063,7 @@ void FEA_Module_SGH::get_bmatrix_gradients(const ViewCArrayKokkos& B_mat gradient_terms(1, 6) = -y(1) + y(3) - y(5) + y(7); gradient_terms(1, 7) = y(3) - y(6); - for (int inode = 0; inode < 8; inode++) - { + for (int inode = 0; inode < 8; inode++) { B_matrix_gradients(2, 0, inode, 0) = 0; B_matrix_gradients(2, 0, inode, 1) = gradient_terms(0, inode) * twelth; B_matrix_gradients(2, 0, inode, 2) = gradient_terms(1, inode) * twelth; @@ -1115,8 +1096,7 @@ void FEA_Module_SGH::get_bmatrix_gradients(const ViewCArrayKokkos& B_mat gradient_terms(1, 6) = -y(2) + y(7); gradient_terms(1, 7) = y(0) - y(2) + y(4) - y(6); - for (int inode = 0; inode < 8; inode++) - { + for (int inode = 0; inode < 8; inode++) { B_matrix_gradients(3, 0, inode, 0) = 0; B_matrix_gradients(3, 0, inode, 1) = gradient_terms(0, inode) * twelth; B_matrix_gradients(3, 0, inode, 2) = gradient_terms(1, inode) * twelth; @@ -1149,8 +1129,7 @@ void FEA_Module_SGH::get_bmatrix_gradients(const ViewCArrayKokkos& B_mat gradient_terms(1, 6) = y(5) - y(7); gradient_terms(1, 7) = -y(0) - y(3) + y(5) + y(6); - for (int inode = 0; inode < 8; inode++) - { + for (int inode = 0; inode < 8; inode++) { B_matrix_gradients(4, 0, inode, 0) = 0; B_matrix_gradients(4, 0, inode, 1) = gradient_terms(0, inode) * twelth; B_matrix_gradients(4, 0, inode, 2) = gradient_terms(1, inode) * twelth; @@ -1183,8 +1162,7 @@ void FEA_Module_SGH::get_bmatrix_gradients(const ViewCArrayKokkos& B_mat gradient_terms(1, 6) = y(1) + y(2) - y(4) - y(7); gradient_terms(1, 7) = -y(4) + y(6); - for (int inode = 0; inode < 8; inode++) - { + for (int inode = 0; inode < 8; inode++) { B_matrix_gradients(5, 0, inode, 0) = 0; B_matrix_gradients(5, 0, inode, 1) = gradient_terms(0, inode) * twelth; B_matrix_gradients(5, 0, inode, 2) = gradient_terms(1, inode) * twelth; @@ -1217,8 +1195,7 @@ void FEA_Module_SGH::get_bmatrix_gradients(const ViewCArrayKokkos& B_mat gradient_terms(1, 6) = 0; gradient_terms(1, 7) = y(2) + y(3) - y(4) - y(5); - for (int inode = 0; inode < 8; inode++) - { + for (int inode = 0; inode < 8; inode++) { B_matrix_gradients(6, 0, inode, 0) = 0; B_matrix_gradients(6, 0, inode, 1) = gradient_terms(0, inode) * twelth; B_matrix_gradients(6, 0, inode, 2) = gradient_terms(1, inode) * twelth; @@ -1251,8 +1228,7 @@ void FEA_Module_SGH::get_bmatrix_gradients(const ViewCArrayKokkos& B_mat gradient_terms(1, 6) = -y(2) - y(3) + y(4) + y(5); gradient_terms(1, 7) = 0; - for (int inode = 0; inode < 8; inode++) - { + for (int inode = 0; inode < 8; inode++) { B_matrix_gradients(7, 0, inode, 0) = 0; B_matrix_gradients(7, 0, inode, 1) = gradient_terms(0, inode) * twelth; B_matrix_gradients(7, 0, inode, 2) = gradient_terms(1, inode) * twelth; @@ -1285,8 +1261,7 @@ void FEA_Module_SGH::get_bmatrix_gradients(const ViewCArrayKokkos& B_mat gradient_terms(1, 6) = 0; gradient_terms(1, 7) = x(3) - x(4); - for (int inode = 0; inode < 8; inode++) - { + for (int inode = 0; inode < 8; inode++) { B_matrix_gradients(0, 1, inode, 0) = gradient_terms(0, inode) * twelth; B_matrix_gradients(0, 1, inode, 1) = 0; B_matrix_gradients(0, 1, inode, 2) = gradient_terms(1, inode) * twelth; @@ -1319,8 +1294,7 @@ void FEA_Module_SGH::get_bmatrix_gradients(const ViewCArrayKokkos& B_mat gradient_terms(1, 6) = -x(2) + x(5); gradient_terms(1, 7) = 0; - for (int inode = 0; inode < 8; inode++) - { + for (int inode = 0; inode < 8; inode++) { B_matrix_gradients(1, 1, inode, 0) = gradient_terms(0, inode) * twelth; B_matrix_gradients(1, 1, inode, 1) = 0; B_matrix_gradients(1, 1, inode, 2) = gradient_terms(1, inode) * twelth; @@ -1353,8 +1327,7 @@ void FEA_Module_SGH::get_bmatrix_gradients(const ViewCArrayKokkos& B_mat gradient_terms(1, 6) = x(1) - x(3) + x(5) - x(7); gradient_terms(1, 7) = -x(3) + x(6); - for (int inode = 0; inode < 8; inode++) - { + for (int inode = 0; inode < 8; inode++) { B_matrix_gradients(2, 1, inode, 0) = gradient_terms(0, inode) * twelth; B_matrix_gradients(2, 1, inode, 1) = 0; B_matrix_gradients(2, 1, inode, 2) = gradient_terms(1, inode) * twelth; @@ -1387,8 +1360,7 @@ void FEA_Module_SGH::get_bmatrix_gradients(const ViewCArrayKokkos& B_mat gradient_terms(1, 6) = x(2) - x(7); gradient_terms(1, 7) = -x(0) + x(2) - x(4) + x(6); - for (int inode = 0; inode < 8; inode++) - { + for (int inode = 0; inode < 8; inode++) { B_matrix_gradients(3, 1, inode, 0) = gradient_terms(0, inode) * twelth; B_matrix_gradients(3, 1, inode, 1) = 0; B_matrix_gradients(3, 1, inode, 2) = gradient_terms(1, inode) * twelth; @@ -1421,8 +1393,7 @@ void FEA_Module_SGH::get_bmatrix_gradients(const ViewCArrayKokkos& B_mat gradient_terms(1, 6) = -x(5) + x(7); gradient_terms(1, 7) = x(0) + x(3) - x(5) - x(6); - for (int inode = 0; inode < 8; inode++) - { + for (int inode = 0; inode < 8; inode++) { B_matrix_gradients(4, 1, inode, 0) = gradient_terms(0, inode) * twelth; B_matrix_gradients(4, 1, inode, 1) = 0; B_matrix_gradients(4, 1, inode, 2) = gradient_terms(1, inode) * twelth; @@ -1455,8 +1426,7 @@ void FEA_Module_SGH::get_bmatrix_gradients(const ViewCArrayKokkos& B_mat gradient_terms(1, 6) = -x(1) - x(2) + x(4) + x(7); gradient_terms(1, 7) = x(4) - x(6); - for (int inode = 0; inode < 8; inode++) - { + for (int inode = 0; inode < 8; inode++) { B_matrix_gradients(5, 1, inode, 0) = gradient_terms(0, inode) * twelth; B_matrix_gradients(5, 1, inode, 1) = 0; B_matrix_gradients(5, 1, inode, 2) = gradient_terms(1, inode) * twelth; @@ -1489,8 +1459,7 @@ void FEA_Module_SGH::get_bmatrix_gradients(const ViewCArrayKokkos& B_mat gradient_terms(1, 6) = 0; gradient_terms(1, 7) = -x(2) - x(3) + x(4) + x(5); - for (int inode = 0; inode < 8; inode++) - { + for (int inode = 0; inode < 8; inode++) { B_matrix_gradients(6, 1, inode, 0) = gradient_terms(0, inode) * twelth; B_matrix_gradients(6, 1, inode, 1) = 0; B_matrix_gradients(6, 1, inode, 2) = gradient_terms(1, inode) * twelth; @@ -1523,8 +1492,7 @@ void FEA_Module_SGH::get_bmatrix_gradients(const ViewCArrayKokkos& B_mat gradient_terms(1, 6) = x(2) + x(3) - x(4) - x(5); gradient_terms(1, 7) = 0; - for (int inode = 0; inode < 8; inode++) - { + for (int inode = 0; inode < 8; inode++) { B_matrix_gradients(7, 1, inode, 0) = gradient_terms(0, inode) * twelth; B_matrix_gradients(7, 1, inode, 1) = 0; B_matrix_gradients(7, 1, inode, 2) = gradient_terms(1, inode) * twelth; @@ -1557,8 +1525,7 @@ void FEA_Module_SGH::get_bmatrix_gradients(const ViewCArrayKokkos& B_mat gradient_terms(1, 6) = 0; gradient_terms(1, 7) = -x(3) + x(4); - for (int inode = 0; inode < 8; inode++) - { + for (int inode = 0; inode < 8; inode++) { B_matrix_gradients(0, 2, inode, 0) = gradient_terms(0, inode) * twelth; B_matrix_gradients(0, 2, inode, 1) = gradient_terms(1, inode) * twelth; B_matrix_gradients(0, 2, inode, 2) = 0; @@ -1591,8 +1558,7 @@ void FEA_Module_SGH::get_bmatrix_gradients(const ViewCArrayKokkos& B_mat gradient_terms(1, 6) = x(2) - x(5); gradient_terms(1, 7) = 0; - for (int inode = 0; inode < 8; inode++) - { + for (int inode = 0; inode < 8; inode++) { B_matrix_gradients(1, 2, inode, 0) = gradient_terms(0, inode) * twelth; B_matrix_gradients(1, 2, inode, 1) = gradient_terms(1, inode) * twelth; B_matrix_gradients(1, 2, inode, 2) = 0; @@ -1625,8 +1591,7 @@ void FEA_Module_SGH::get_bmatrix_gradients(const ViewCArrayKokkos& B_mat gradient_terms(1, 6) = -x(1) + x(3) - x(5) + x(7); gradient_terms(1, 7) = x(3) - x(6); - for (int inode = 0; inode < 8; inode++) - { + for (int inode = 0; inode < 8; inode++) { B_matrix_gradients(2, 2, inode, 0) = gradient_terms(0, inode) * twelth; B_matrix_gradients(2, 2, inode, 1) = gradient_terms(1, inode) * twelth; B_matrix_gradients(2, 2, inode, 2) = 0; @@ -1659,8 +1624,7 @@ void FEA_Module_SGH::get_bmatrix_gradients(const ViewCArrayKokkos& B_mat gradient_terms(1, 6) = -x(2) + x(7); gradient_terms(1, 7) = x(0) - x(2) + x(4) - x(6); - for (int inode = 0; inode < 8; inode++) - { + for (int inode = 0; inode < 8; inode++) { B_matrix_gradients(3, 2, inode, 0) = gradient_terms(0, inode) * twelth; B_matrix_gradients(3, 2, inode, 1) = gradient_terms(1, inode) * twelth; B_matrix_gradients(3, 2, inode, 2) = 0; @@ -1693,8 +1657,7 @@ void FEA_Module_SGH::get_bmatrix_gradients(const ViewCArrayKokkos& B_mat gradient_terms(1, 6) = x(5) - x(7); gradient_terms(1, 7) = -x(0) - x(3) + x(5) + x(6); - for (int inode = 0; inode < 8; inode++) - { + for (int inode = 0; inode < 8; inode++) { B_matrix_gradients(4, 2, inode, 0) = gradient_terms(0, inode) * twelth; B_matrix_gradients(4, 2, inode, 1) = gradient_terms(1, inode) * twelth; B_matrix_gradients(4, 2, inode, 2) = 0; @@ -1727,8 +1690,7 @@ void FEA_Module_SGH::get_bmatrix_gradients(const ViewCArrayKokkos& B_mat gradient_terms(1, 6) = x(1) + x(2) - x(4) - x(7); gradient_terms(1, 7) = -x(4) + x(6); - for (int inode = 0; inode < 8; inode++) - { + for (int inode = 0; inode < 8; inode++) { B_matrix_gradients(5, 2, inode, 0) = gradient_terms(0, inode) * twelth; B_matrix_gradients(5, 2, inode, 1) = gradient_terms(1, inode) * twelth; B_matrix_gradients(5, 2, inode, 2) = 0; @@ -1761,8 +1723,7 @@ void FEA_Module_SGH::get_bmatrix_gradients(const ViewCArrayKokkos& B_mat gradient_terms(1, 6) = 0; gradient_terms(1, 7) = x(2) + x(3) - x(4) - x(5); - for (int inode = 0; inode < 8; inode++) - { + for (int inode = 0; inode < 8; inode++) { B_matrix_gradients(6, 2, inode, 0) = gradient_terms(0, inode) * twelth; B_matrix_gradients(6, 2, inode, 1) = gradient_terms(1, inode) * twelth; B_matrix_gradients(6, 2, inode, 2) = 0; @@ -1795,8 +1756,7 @@ void FEA_Module_SGH::get_bmatrix_gradients(const ViewCArrayKokkos& B_mat gradient_terms(1, 6) = -x(2) - x(3) + x(4) + x(5); gradient_terms(1, 7) = 0; - for (int inode = 0; inode < 8; inode++) - { + for (int inode = 0; inode < 8; inode++) { B_matrix_gradients(7, 2, inode, 0) = gradient_terms(0, inode) * twelth; B_matrix_gradients(7, 2, inode, 1) = gradient_terms(1, inode) * twelth; B_matrix_gradients(7, 2, inode, 2) = 0; diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/momentum.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/momentum.cpp index a37caca93..9c03ecf85 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/momentum.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/momentum.cpp @@ -58,27 +58,23 @@ void FEA_Module_SGH::update_velocity_sgh(double rk_alpha, // walk over the nodes to update the velocity FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { double node_force[3]; - for (size_t dim = 0; dim < num_dims; dim++) - { + for (size_t dim = 0; dim < num_dims; dim++) { node_force[dim] = 0.0; } // end for dim // loop over all corners around the node and calculate the nodal force - for (size_t corner_lid = 0; corner_lid < num_corners_in_node(node_gid); corner_lid++) - { + for (size_t corner_lid = 0; corner_lid < num_corners_in_node(node_gid); corner_lid++) { // Get corner gid size_t corner_gid = corners_in_node(node_gid, corner_lid); // loop over dimension - for (size_t dim = 0; dim < num_dims; dim++) - { + for (size_t dim = 0; dim < num_dims; dim++) { node_force[dim] += corner_force(corner_gid, dim); } // end for dim } // end for corner_lid // update the velocity - for (int dim = 0; dim < num_dims; dim++) - { + for (int dim = 0; dim < num_dims; dim++) { node_vel(rk_level, node_gid, dim) = node_vel(0, node_gid, dim) + rk_alpha * dt * node_force[dim] / node_mass(node_gid); } // end for dim @@ -123,8 +119,7 @@ void FEA_Module_SGH::get_velgrad(ViewCArrayKokkos& vel_grad, ViewCArrayKokkos w(w_array, num_nodes_in_elem); // get the vertex velocities for the cell - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) - { + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { // Get node gid size_t node_gid = elem_node_gids(node_lid); @@ -222,8 +217,7 @@ void FEA_Module_SGH::get_velgrad2D(ViewCArrayKokkos& vel_grad, ViewCArrayKokkos v(v_array, num_nodes_in_elem); // y-dir vel component // get the vertex velocities for the cell - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) - { + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { // Get node gid size_t node_gid = elem_node_gids(node_lid); @@ -232,10 +226,8 @@ void FEA_Module_SGH::get_velgrad2D(ViewCArrayKokkos& vel_grad, } // end for // initialize to zero - for (size_t i = 0; i < 3; i++) - { - for (size_t j = 0; j < 3; j++) - { + for (size_t i = 0; i < 3; i++) { + for (size_t j = 0; j < 3; j++) { vel_grad(i, j) = 0.0; } } @@ -311,8 +303,7 @@ void FEA_Module_SGH::get_divergence(DViewCArrayKokkos& elem_div, rk_level); // get the vertex velocities for the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) - { + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { // Get node gid size_t node_gid = elem_node_gids(node_lid); @@ -399,8 +390,7 @@ void FEA_Module_SGH::get_divergence2D(DViewCArrayKokkos& elem_div, // true volume uses the elem_vol // get the vertex velocities and node coordinate for the elem - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) - { + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { // Get node gid size_t node_gid = elem_node_gids(node_lid); @@ -470,19 +460,15 @@ void FEA_Module_SGH::decompose_vel_grad(ViewCArrayKokkos& D_tensor, const size_t num_dims = 3; // initialize to zero - for (size_t i = 0; i < num_dims; i++) - { - for (size_t j = 0; j < num_dims; j++) - { + for (size_t i = 0; i < num_dims; i++) { + for (size_t j = 0; j < num_dims; j++) { D_tensor(i, j) = 0.0; W_tensor(i, j) = 0.0; } } // end for - for (size_t i = 0; i < num_dims; i++) - { - for (size_t j = 0; j < num_dims; j++) - { + for (size_t i = 0; i < num_dims; i++) { + for (size_t j = 0; j < num_dims; j++) { D_tensor(i, j) = 0.5 * (vel_grad(i, j) + vel_grad(j, i)); W_tensor(i, j) = 0.5 * (vel_grad(i, j) - vel_grad(j, i)); } diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/power_gradients_sgh.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/power_gradients_sgh.cpp index 21d5483d0..d799fec5d 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/power_gradients_sgh.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/power_gradients_sgh.cpp @@ -66,33 +66,25 @@ void FEA_Module_SGH::power_design_gradient_term(const_vec_array design_variables DCArrayKokkos elem_power_dgradients(rnum_elem); // gradient contribution from gradient of Force vector with respect to design variable. - if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) - { - if (myrank == 0) - { + if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) { + if (myrank == 0) { std::cout << "gradient term involving adjoint derivative" << std::endl; } } - for (unsigned long cycle = 0; cycle < last_time_step + 1; cycle++) - { + for (unsigned long cycle = 0; cycle < last_time_step + 1; cycle++) { // compute timestep from time data global_dt = time_data[cycle + 1] - time_data[cycle]; // print - if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) - { - if (cycle == 0) - { - if (myrank == 0) - { + if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) { + if (cycle == 0) { + if (myrank == 0) { printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); } } // print time step every 10 cycles - else if (cycle % 20 == 0) - { - if (myrank == 0) - { + else if (cycle % 20 == 0) { + if (myrank == 0) { printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); } } // end if @@ -111,8 +103,7 @@ void FEA_Module_SGH::power_design_gradient_term(const_vec_array design_variables // first half of integration step calculation FOR_ALL_CLASS(node_gid, 0, nall_nodes, { - for (int idim = 0; idim < num_dim; idim++) - { + for (int idim = 0; idim < num_dim; idim++) { node_vel(rk_level, node_gid, idim) = current_velocity_vector(node_gid, idim); node_coords(rk_level, node_gid, idim) = current_coord_vector(node_gid, idim); } @@ -127,18 +118,15 @@ void FEA_Module_SGH::power_design_gradient_term(const_vec_array design_variables get_vol(); // ---- Calculate velocity diveregence for the element ---- - if (num_dim == 2) - { + if (num_dim == 2) { get_divergence2D(elem_div, node_coords, node_vel, elem_vol); } - else - { + else{ get_divergence(elem_div, node_coords, node_vel, elem_vol); } // end if 2D // ---- Calculate elem state (den, pres, sound speed, stress) for next time step ---- - if (num_dim == 2) - { + if (num_dim == 2) { update_state2D(material, *mesh, node_coords, @@ -154,8 +142,7 @@ void FEA_Module_SGH::power_design_gradient_term(const_vec_array design_variables 1.0, cycle); } - else - { + else{ update_state(material, *mesh, node_coords, @@ -173,8 +160,7 @@ void FEA_Module_SGH::power_design_gradient_term(const_vec_array design_variables } // ---- calculate the forces on the vertices and evolve stress (hypo model) ---- - if (num_dim == 2) - { + if (num_dim == 2) { get_force_sgh2D(material, *mesh, node_coords, @@ -191,8 +177,7 @@ void FEA_Module_SGH::power_design_gradient_term(const_vec_array design_variables 1.0, cycle); } - else - { + else{ get_force_sgh(material, *mesh, node_coords, @@ -242,8 +227,7 @@ void FEA_Module_SGH::power_design_gradient_term(const_vec_array design_variables inner_product = current_psi_adjoint_vector(elem_id, 0) * elem_power_dgradients(elem_id); - for (int inode = 0; inode < num_nodes_in_elem; inode++) - { + for (int inode = 0; inode < num_nodes_in_elem; inode++) { // compute gradient of local element contribution to v^t*M*v product corner_id = elem_id * num_nodes_in_elem + inode; corner_value_storage(corner_id) = inner_product; @@ -255,8 +239,7 @@ void FEA_Module_SGH::power_design_gradient_term(const_vec_array design_variables // multiply FOR_ALL_CLASS(node_id, 0, nlocal_nodes, { size_t corner_id; - for (int icorner = 0; icorner < num_corners_in_node(node_id); icorner++) - { + for (int icorner = 0; icorner < num_corners_in_node(node_id); icorner++) { corner_id = corners_in_node(node_id, icorner); design_gradients(node_id, 0) += -corner_value_storage(corner_id) * global_dt; } @@ -303,8 +286,7 @@ void FEA_Module_SGH::get_power_dgradient_sgh(double rk_alpha, // --- tally the contribution from each corner to the element --- // Loop over the nodes in the element - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) - { + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { size_t corner_lid = node_lid; // Get node global id for the local node id @@ -314,14 +296,12 @@ void FEA_Module_SGH::get_power_dgradient_sgh(double rk_alpha, size_t corner_gid = corners_in_elem(elem_gid, corner_lid); double node_radius = 1; - if (num_dims == 2) - { + if (num_dims == 2) { node_radius = node_coords(rk_level, node_gid, 1); } // calculate the Power=F dot V for this corner - for (size_t dim = 0; dim < num_dims; dim++) - { + for (size_t dim = 0; dim < num_dims; dim++) { double half_vel = (node_vel(rk_level, node_gid, dim) + node_vel(0, node_gid, dim)) * 0.5; elem_power_dgradients(elem_gid) -= corner_vector_storage(corner_gid, dim) * node_radius * node_vel(rk_level, node_gid, dim); } // end for dim @@ -362,16 +342,14 @@ void FEA_Module_SGH::get_power_ugradient_sgh(double rk_alpha, // initialize gradient matrix FOR_ALL_CLASS(dof_gid, 0, nlocal_nodes * num_dims, { - for (int idof = 0; idof < DOF_to_Elem_Matrix_Strides(dof_gid); idof++) - { + for (int idof = 0; idof < DOF_to_Elem_Matrix_Strides(dof_gid); idof++) { Power_Gradient_Positions(dof_gid, idof) = 0; } }); // end parallel for loop over nodes Kokkos::fence(); // loop over all the elements in the mesh - for (size_t elem_gid = 0; elem_gid < rnum_elem; elem_gid++) - { + for (size_t elem_gid = 0; elem_gid < rnum_elem; elem_gid++) { // FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { double elem_power = 0.0; @@ -379,8 +357,7 @@ void FEA_Module_SGH::get_power_ugradient_sgh(double rk_alpha, // --- tally the contribution from each corner to the element --- // Loop over the nodes in the element - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) - { + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { size_t corner_lid = node_lid; size_t column_id; size_t gradient_node_id; @@ -391,22 +368,17 @@ void FEA_Module_SGH::get_power_ugradient_sgh(double rk_alpha, size_t corner_gid = corners_in_elem(elem_gid, corner_lid); double node_radius = 1; - if (num_dims == 2) - { + if (num_dims == 2) { node_radius = node_coords(rk_level, node_gid, 1); } // calculate the Power=F dot V for this corner - for (size_t dim = 0; dim < num_dims; dim++) - { - for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) - { - for (size_t jdim = 0; jdim < num_dims; jdim++) - { + for (size_t dim = 0; dim < num_dims; dim++) { + for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) { + for (size_t jdim = 0; jdim < num_dims; jdim++) { column_id = Element_Gradient_Matrix_Assembly_Map(elem_gid, igradient); gradient_node_id = nodes_in_elem(elem_gid, igradient); - if (!map->isNodeLocalElement(gradient_node_id)) - { + if (!map->isNodeLocalElement(gradient_node_id)) { continue; } Power_Gradient_Positions(gradient_node_id * num_dims + jdim, column_id) -= @@ -444,16 +416,14 @@ void FEA_Module_SGH::get_power_vgradient_sgh(double rk_alpha, // initialize gradient matrix FOR_ALL_CLASS(dof_gid, 0, nlocal_nodes * num_dims, { - for (int idof = 0; idof < DOF_to_Elem_Matrix_Strides(dof_gid); idof++) - { + for (int idof = 0; idof < DOF_to_Elem_Matrix_Strides(dof_gid); idof++) { Power_Gradient_Velocities(dof_gid, idof) = 0; } }); // end parallel for loop over nodes Kokkos::fence(); // loop over all the elements in the mesh - for (size_t elem_gid = 0; elem_gid < rnum_elem; elem_gid++) - { + for (size_t elem_gid = 0; elem_gid < rnum_elem; elem_gid++) { // FOR_ALL_CLASS (elem_gid, 0, rnum_elem, { double elem_power = 0.0; @@ -461,8 +431,7 @@ void FEA_Module_SGH::get_power_vgradient_sgh(double rk_alpha, // --- tally the contribution from each corner to the element --- // Loop over the nodes in the element - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) - { + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { size_t corner_lid = node_lid; size_t column_id; size_t gradient_node_id; @@ -473,30 +442,24 @@ void FEA_Module_SGH::get_power_vgradient_sgh(double rk_alpha, size_t corner_gid = corners_in_elem(elem_gid, corner_lid); double node_radius = 1; - if (num_dims == 2) - { + if (num_dims == 2) { node_radius = node_coords(rk_level, node_gid, 1); } // calculate the Power=F dot V for this corner - for (size_t dim = 0; dim < num_dims; dim++) - { - for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) - { + for (size_t dim = 0; dim < num_dims; dim++) { + for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) { column_id = Element_Gradient_Matrix_Assembly_Map(elem_gid, igradient); gradient_node_id = nodes_in_elem(elem_gid, igradient); - if (!map->isNodeLocalElement(gradient_node_id)) - { + if (!map->isNodeLocalElement(gradient_node_id)) { continue; } - if (node_lid == igradient) - { + if (node_lid == igradient) { Power_Gradient_Velocities(gradient_node_id * num_dims + dim, column_id) -= corner_gradient_storage(corner_gid, dim, igradient, dim) * node_vel(rk_level, node_gid, dim) * node_radius + corner_force(corner_gid, dim) * node_radius; } - else - { + else{ Power_Gradient_Velocities(gradient_node_id * num_dims + dim, column_id) -= corner_gradient_storage(corner_gid, dim, igradient, dim) * node_vel(rk_level, node_gid, dim) * node_radius; } @@ -549,8 +512,7 @@ void FEA_Module_SGH::get_power_egradient_sgh(double rk_alpha, // --- tally the contribution from each corner to the element --- // Loop over the nodes in the element - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) - { + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { size_t corner_lid = node_lid; // Get node global id for the local node id @@ -560,14 +522,12 @@ void FEA_Module_SGH::get_power_egradient_sgh(double rk_alpha, size_t corner_gid = corners_in_elem(elem_gid, corner_lid); double node_radius = 1; - if (num_dims == 2) - { + if (num_dims == 2) { node_radius = node_coords(rk_level, node_gid, 1); } // calculate the Power=F dot V for this corner - for (size_t dim = 0; dim < num_dims; dim++) - { + for (size_t dim = 0; dim < num_dims; dim++) { double half_vel = (node_vel(rk_level, node_gid, dim) + node_vel(0, node_gid, dim)) * 0.5; Power_Gradient_Energies(elem_gid) -= Force_Gradient_Energies(elem_gid, node_lid * num_dims + dim) * node_radius * node_vel(rk_level, node_gid, dim); } // end for dim diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/properties.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/properties.cpp index c06d53247..57f42e32c 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/properties.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/properties.cpp @@ -94,8 +94,7 @@ void FEA_Module_SGH::update_state(const DCArrayKokkos& material, // --- Stress --- // hyper elastic plastic model - if (material(mat_id).strength_type == STRENGTH_TYPE::hyper) - { + if (material(mat_id).strength_type == STRENGTH_TYPE::hyper) { // cut out the node_gids for this element ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), num_nodes_in_elem); @@ -243,8 +242,7 @@ void FEA_Module_SGH::update_state2D(const DCArrayKokkos& material, // --- Stress --- // hyper elastic plastic model - if (material(mat_id).strength_type == STRENGTH_TYPE::hyper) - { + if (material(mat_id).strength_type == STRENGTH_TYPE::hyper) { // cut out the node_gids for this element ViewCArrayKokkos elem_node_gids(&nodes_in_elem(elem_gid, 0), num_nodes_in_elem); diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/setup_sgh.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/setup_sgh.cpp index d71b1e6a9..bb115da40 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/setup_sgh.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/setup_sgh.cpp @@ -56,8 +56,7 @@ void FEA_Module_SGH::setup() const size_t num_materials = simparam->materials.size(); const int num_dim = simparam->num_dims; const size_t num_lcs = module_params->loading.size(); - if (num_lcs) - { + if (num_lcs) { have_loading_conditions = true; } @@ -68,8 +67,7 @@ void FEA_Module_SGH::setup() mesh->build_corner_connectivity(); mesh->build_elem_elem_connectivity(); mesh->num_bdy_patches = nboundary_patches; - if (num_dim == 2) - { + if (num_dim == 2) { mesh->build_patch_connectivity(); mesh->build_node_node_connectivity(); } @@ -123,8 +121,7 @@ void FEA_Module_SGH::setup() elem_strength = DCArrayKokkos(num_elems); // optimization flags - if (simparam->topology_optimization_on) - { + if (simparam->topology_optimization_on) { elem_extensive_initial_energy_condition = DCArrayKokkos(num_elems); } @@ -159,8 +156,7 @@ void FEA_Module_SGH::setup() // patch ids in bdy set bdy_patches_in_set = mesh->bdy_patches_in_set; - if (num_dim == 2) - { + if (num_dim == 2) { bdy_nodes = mesh->bdy_nodes; } @@ -186,8 +182,7 @@ void FEA_Module_SGH::setup() // elem-node conn & node-node conn elems_in_node = mesh->elems_in_node; - if (num_dim == 2) - { + if (num_dim == 2) { nodes_in_node = mesh->nodes_in_node; num_nodes_in_node = mesh->num_nodes_in_node; // patch conn @@ -198,8 +193,7 @@ void FEA_Module_SGH::setup() } // loop over BCs - for (size_t this_bdy = 0; this_bdy < num_bcs; this_bdy++) - { + for (size_t this_bdy = 0; this_bdy < num_bcs; this_bdy++) { RUN_CLASS({ printf("Boundary Condition number %lu \n", this_bdy); printf(" Num bdy patches in this set = %lu \n", bdy_patches_in_set.stride(this_bdy)); @@ -209,8 +203,7 @@ void FEA_Module_SGH::setup() } // end for // elem_mat_id needs to be initialized before initialization of material models - for (int f_id = 0; f_id < num_fills; f_id++) - { + for (int f_id = 0; f_id < num_fills; f_id++) { FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { elem_mat_id(elem_gid) = mat_fill(f_id).material_id; }); @@ -252,18 +245,15 @@ void FEA_Module_SGH::setup() // --- apply the fill instructions over each of the Elements---// // initialize if topology optimization is used - if (simparam->topology_optimization_on) - { + if (simparam->topology_optimization_on) { // compute element averaged density ratios corresponding to nodal density design variables CArray current_element_nodal_densities = CArray(num_nodes_in_elem); { // view scope const_host_vec_array all_node_densities = all_node_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); // debug print // std::cout << "NODE DENSITY TEST " << all_node_densities(0,0) << std::endl; - for (int elem_id = 0; elem_id < rnum_elem; elem_id++) - { - for (int inode = 0; inode < num_nodes_in_elem; inode++) - { + for (int elem_id = 0; elem_id < rnum_elem; elem_id++) { + for (int inode = 0; inode < num_nodes_in_elem; inode++) { current_element_nodal_densities(inode) = all_node_densities(nodes_in_elem(elem_id, inode), 0); } relative_element_densities.host(elem_id) = average_element_density(num_nodes_in_elem, current_element_nodal_densities); @@ -271,14 +261,12 @@ void FEA_Module_SGH::setup() } // view scope // debug print // std::cout << "ELEMENT RELATIVE DENSITY TEST " << relative_element_densities.host(0) << std::endl; - + relative_element_densities.update_device(); } - // loop over the fill instructures - for (int f_id = 0; f_id < num_fills; f_id++) - { + for (int f_id = 0; f_id < num_fills; f_id++) { // parallel loop over elements in mesh // for (size_t elem_gid = 0; elem_gid <= rnum_elem; elem_gid++) { FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { @@ -289,16 +277,13 @@ void FEA_Module_SGH::setup() elem_coords[2] = 0.0; // get the coordinates of the element center - for (int node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) - { + for (int node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { elem_coords[0] += node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), 0); elem_coords[1] += node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), 1); - if (num_dim == 3) - { + if (num_dim == 3) { elem_coords[2] += node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), 2); } - else - { + else{ elem_coords[2] = 0.0; } } // end loop over nodes in element @@ -310,10 +295,9 @@ void FEA_Module_SGH::setup() bool fill_this = mat_fill(f_id).volume.contains(elem_coords); // paint the material state on the element - if (fill_this) - { + if (fill_this) { // density - if (simparam->topology_optimization_on){ + if (simparam->topology_optimization_on) { elem_den(elem_gid) = mat_fill(f_id).den * relative_element_densities(elem_gid); } else{ @@ -326,13 +310,11 @@ void FEA_Module_SGH::setup() // specific internal energy elem_sie(rk_level, elem_gid) = mat_fill(f_id).sie; - if (simparam->topology_optimization_on && mat_fill(f_id).extensive_energy_setting) - { + if (simparam->topology_optimization_on && mat_fill(f_id).extensive_energy_setting) { elem_sie(rk_level, elem_gid) = elem_sie(rk_level, elem_gid) / relative_element_densities(elem_gid); elem_extensive_initial_energy_condition(elem_gid) = true; } - else if (simparam->topology_optimization_on && !mat_fill(f_id).extensive_energy_setting) - { + else if (simparam->topology_optimization_on && !mat_fill(f_id).extensive_energy_setting) { elem_extensive_initial_energy_condition(elem_gid) = false; } @@ -340,10 +322,8 @@ void FEA_Module_SGH::setup() // --- stress tensor --- // always 3D even for 2D-RZ - for (size_t i = 0; i < 3; i++) - { - for (size_t j = 0; j < 3; j++) - { + for (size_t i = 0; i < 3; i++) { + for (size_t j = 0; j < 3; j++) { elem_stress(rk_level, elem_gid, i, j) = 0.0; } } // end for @@ -377,20 +357,17 @@ void FEA_Module_SGH::setup() elem_sie(rk_level, elem_gid)); // loop over the nodes of this element and apply velocity - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) - { + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { // get the mesh node index size_t node_gid = nodes_in_elem(elem_gid, node_lid); // --- Velocity --- - switch (mat_fill(f_id).velocity) - { + switch (mat_fill(f_id).velocity) { case VELOCITY_TYPE::cartesian: { node_vel(rk_level, node_gid, 0) = mat_fill(f_id).u; node_vel(rk_level, node_gid, 1) = mat_fill(f_id).v; - if (num_dim == 3) - { + if (num_dim == 3) { node_vel(rk_level, node_gid, 2) = mat_fill(f_id).w; } @@ -404,29 +381,24 @@ void FEA_Module_SGH::setup() dir[1] = 0.0; double radius_val = 0.0; - for (int dim = 0; dim < 2; dim++) - { + for (int dim = 0; dim < 2; dim++) { dir[dim] = node_coords(rk_level, node_gid, dim); radius_val += node_coords(rk_level, node_gid, dim) * node_coords(rk_level, node_gid, dim); } // end for radius_val = sqrt(radius_val); - for (int dim = 0; dim < 2; dim++) - { - if (radius_val > 1.0e-14) - { + for (int dim = 0; dim < 2; dim++) { + if (radius_val > 1.0e-14) { dir[dim] /= (radius_val); } - else - { + else{ dir[dim] = 0.0; } } // end for node_vel(rk_level, node_gid, 0) = mat_fill(f_id).speed * dir[0]; node_vel(rk_level, node_gid, 1) = mat_fill(f_id).speed * dir[1]; - if (num_dim == 3) - { + if (num_dim == 3) { node_vel(rk_level, node_gid, 2) = 0.0; } @@ -441,29 +413,24 @@ void FEA_Module_SGH::setup() dir[2] = 0.0; double radius_val = 0.0; - for (int dim = 0; dim < 3; dim++) - { + for (int dim = 0; dim < 3; dim++) { dir[dim] = node_coords(rk_level, node_gid, dim); radius_val += node_coords(rk_level, node_gid, dim) * node_coords(rk_level, node_gid, dim); } // end for radius_val = sqrt(radius_val); - for (int dim = 0; dim < 3; dim++) - { - if (radius_val > 1.0e-14) - { + for (int dim = 0; dim < 3; dim++) { + if (radius_val > 1.0e-14) { dir[dim] /= (radius_val); } - else - { + else{ dir[dim] = 0.0; } } // end for node_vel(rk_level, node_gid, 0) = mat_fill(f_id).speed * dir[0]; node_vel(rk_level, node_gid, 1) = mat_fill(f_id).speed * dir[1]; - if (num_dim == 3) - { + if (num_dim == 3) { node_vel(rk_level, node_gid, 2) = mat_fill(f_id).speed * dir[2]; } @@ -481,8 +448,7 @@ void FEA_Module_SGH::setup() { node_vel(rk_level, node_gid, 0) = sin(PI * node_coords(rk_level, node_gid, 0)) * cos(PI * node_coords(rk_level, node_gid, 1)); node_vel(rk_level, node_gid, 1) = -1.0 * cos(PI * node_coords(rk_level, node_gid, 0)) * sin(PI * node_coords(rk_level, node_gid, 1)); - if (num_dim == 3) - { + if (num_dim == 3) { node_vel(rk_level, node_gid, 2) = 0.0; } @@ -491,8 +457,7 @@ void FEA_Module_SGH::setup() } // end of switch } // end loop over nodes of element - if (mat_fill(f_id).velocity == VELOCITY_TYPE::tg_vortex) - { + if (mat_fill(f_id).velocity == VELOCITY_TYPE::tg_vortex) { elem_pres(elem_gid) = 0.25 * (cos(2.0 * PI * elem_coords[0]) + cos(2.0 * PI * elem_coords[1]) ) + 1.0; // p = rho*ie*(gamma - 1) @@ -510,8 +475,7 @@ void FEA_Module_SGH::setup() FEA_Module_SGH::boundary_velocity(*mesh, boundary, node_vel); // calculate the corner massess if 2D - if (num_dim == 2) - { + if (num_dim == 2) { FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { // facial area of the corners double corner_areas_array[4]; @@ -526,8 +490,7 @@ void FEA_Module_SGH::setup() rk_level); // loop over the corners of the element and calculate the mass - for (size_t corner_lid = 0; corner_lid < 4; corner_lid++) - { + for (size_t corner_lid = 0; corner_lid < 4; corner_lid++) { size_t corner_gid = corners_in_elem(elem_gid, corner_lid); corner_mass(corner_gid) = corner_areas(corner_lid) * elem_den(elem_gid); // node radius is added later } // end for over corners @@ -538,19 +501,15 @@ void FEA_Module_SGH::setup() FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { node_mass(node_gid) = 0.0; - if (num_dim == 3) - { - for (size_t elem_lid = 0; elem_lid < num_corners_in_node(node_gid); elem_lid++) - { + if (num_dim == 3) { + for (size_t elem_lid = 0; elem_lid < num_corners_in_node(node_gid); elem_lid++) { size_t elem_gid = elems_in_node(node_gid, elem_lid); node_mass(node_gid) += 1.0 / 8.0 * elem_mass(elem_gid); } // end for elem_lid } // end if dims=3 - else - { + else{ // 2D-RZ - for (size_t corner_lid = 0; corner_lid < num_corners_in_node(node_gid); corner_lid++) - { + for (size_t corner_lid = 0; corner_lid < num_corners_in_node(node_gid); corner_lid++) { size_t corner_gid = corners_in_node(node_gid, corner_lid); node_mass(node_gid) += corner_mass(corner_gid); // sans the radius so it is areal node mass @@ -562,8 +521,7 @@ void FEA_Module_SGH::setup() // current interface has differing mass arrays; this equates them until we unify memory // view scope - if (simparam->topology_optimization_on || simparam->shape_optimization_on || simparam->num_dims == 2) - { + if (simparam->topology_optimization_on || simparam->shape_optimization_on || simparam->num_dims == 2) { { vec_array node_mass_interface = node_masses_distributed->getLocalView(Tpetra::Access::ReadWrite); @@ -588,8 +546,7 @@ void FEA_Module_SGH::setup() } // endif // initialize if topology optimization is used - if (simparam->topology_optimization_on || simparam->shape_optimization_on) - { + if (simparam->topology_optimization_on || simparam->shape_optimization_on) { init_assembly(); // assemble_matrix(); } @@ -602,8 +559,7 @@ void FEA_Module_SGH::setup() elem_pres.update_host(); elem_sspd.update_host(); - if (simparam->topology_optimization_on) - { + if (simparam->topology_optimization_on) { elem_extensive_initial_energy_condition.update_host(); } @@ -627,8 +583,7 @@ void FEA_Module_SGH::sgh_interface_setup(node_t& node, elem_t& elem, corner_t& c const size_t rk_num_bins = simparam->dynamic_options.rk_num_bins; num_nodes_in_elem = 1; - for (int dim = 0; dim < num_dim; dim++) - { + for (int dim = 0; dim < num_dim; dim++) { num_nodes_in_elem *= 2; } @@ -650,10 +605,8 @@ void FEA_Module_SGH::sgh_interface_setup(node_t& node, elem_t& elem, corner_t& c host_vec_array interface_node_coords = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); // save node data to node.coords // std::cout << "NODE DATA ON RANK " << myrank << std::endl; - if (num_dim == 2) - { - for (int inode = 0; inode < nall_nodes; inode++) - { + if (num_dim == 2) { + for (int inode = 0; inode < nall_nodes; inode++) { // std::cout << "Node index " << inode+1 << " "; node.coords.host(0, inode, 0) = interface_node_coords(inode, 0); // std::cout << host_node_coords_state(0,inode,0)+1<< " "; @@ -661,10 +614,8 @@ void FEA_Module_SGH::sgh_interface_setup(node_t& node, elem_t& elem, corner_t& c // std::cout << host_node_coords_state(0,inode,1)+1<< " "; } } - else if (num_dim == 3) - { - for (int inode = 0; inode < nall_nodes; inode++) - { + else if (num_dim == 3) { + for (int inode = 0; inode < nall_nodes; inode++) { // std::cout << "Node index " << inode+1 << " "; node.coords.host(0, inode, 0) = interface_node_coords(inode, 0); // std::cout << host_node_coords_state(0,inode,0)+1<< " "; @@ -692,11 +643,9 @@ void FEA_Module_SGH::sgh_interface_setup(node_t& node, elem_t& elem, corner_t& c host_elem_conn_array interface_nodes_in_elem = Explicit_Solver_Pointer_->global_nodes_in_elem_distributed->getLocalView(Tpetra::Access::ReadWrite); // save node data to node.coords // std::cout << "ELEMENT CONNECTIVITY ON RANK " << myrank << std::endl; - for (int ielem = 0; ielem < rnum_elem; ielem++) - { + for (int ielem = 0; ielem < rnum_elem; ielem++) { // std::cout << "Element index " << ielem+1 << " "; - for (int inode = 0; inode < num_nodes_in_elem; inode++) - { + for (int inode = 0; inode < num_nodes_in_elem; inode++) { nodes_in_elem.host(ielem, inode) = Explicit_Solver_Pointer_->all_node_map->getLocalElement(interface_nodes_in_elem(ielem, inode)); } } @@ -710,14 +659,11 @@ void FEA_Module_SGH::sgh_interface_setup(node_t& node, elem_t& elem, corner_t& c device_mesh_nodes_in_elem.get_kokkos_view() = nodes_in_elem.get_kokkos_dual_view().d_view; host_mesh_nodes_in_elem.get_kokkos_view() = nodes_in_elem.get_kokkos_dual_view().view_host(); - if (myrank == 1) - { + if (myrank == 1) { std::cout << "ELEMENT CONNECTIVITY ON RANK 1 in LOCAL INDICES" << myrank << std::endl; - for (int ielem = 0; ielem < rnum_elem; ielem++) - { + for (int ielem = 0; ielem < rnum_elem; ielem++) { std::cout << "Element index " << ielem + 1 << " "; - for (int inode = 0; inode < num_nodes_in_elem; inode++) - { + for (int inode = 0; inode < num_nodes_in_elem; inode++) { // debug print device_mesh_nodes_in_elem(ielem, inode) = Explicit_Solver_Pointer_->all_node_map->getLocalElement(interface_nodes_in_elem(ielem, inode)); std::cout << nodes_in_elem(ielem, inode) + 1 << " "; @@ -727,15 +673,12 @@ void FEA_Module_SGH::sgh_interface_setup(node_t& node, elem_t& elem, corner_t& c } std::cout.flush(); - if (myrank == 1) - { + if (myrank == 1) { std::cout << "ELEMENT CONNECTIVITY ON RANK 1 in GLOBAL INDICES" << myrank << std::endl; std::cout << "local node index of global index 275 on rank 1 " << Explicit_Solver_Pointer_->all_node_map->getLocalElement(275) << std::endl; - for (int ielem = 0; ielem < rnum_elem; ielem++) - { + for (int ielem = 0; ielem < rnum_elem; ielem++) { std::cout << ielem << " "; - for (int inode = 0; inode < num_nodes_in_elem; inode++) - { + for (int inode = 0; inode < num_nodes_in_elem; inode++) { // debug print device_mesh_nodes_in_elem(ielem, inode) = Explicit_Solver_Pointer_->all_node_map->getLocalElement(interface_nodes_in_elem(ielem, inode)); std::cout << Explicit_Solver_Pointer_->all_node_map->getGlobalElement(nodes_in_elem(ielem, inode)) << " "; @@ -759,8 +702,7 @@ void FEA_Module_SGH::sgh_interface_setup(node_t& node, elem_t& elem, corner_t& c // save node data to node.coords std::cout << "ALL NODE DATA ON RANK " << myrank << std::endl; - for (int inode = 0; inode < nall_nodes; inode++) - { + for (int inode = 0; inode < nall_nodes; inode++) { // std::cout << "Node index " << inode+1 << " "; node.all_coords.host(0, inode, 0) = interface_all_node_coords(inode, 0); // std::cout << host_all_node_coords_state(0,inode,0)+1<< " "; @@ -772,12 +714,9 @@ void FEA_Module_SGH::sgh_interface_setup(node_t& node, elem_t& elem, corner_t& c #endif // save the node coords to the current RK value - for (size_t node_gid = 0; node_gid < nall_nodes; node_gid++) - { - for (int rk = 1; rk < rk_num_bins; rk++) - { - for (int dim = 0; dim < num_dim; dim++) - { + for (size_t node_gid = 0; node_gid < nall_nodes; node_gid++) { + for (int rk = 1; rk < rk_num_bins; rk++) { + for (int dim = 0; dim < num_dim; dim++) { node.coords.host(rk, node_gid, dim) = node.coords.host(0, node_gid, dim); } // end for dim } // end for rk @@ -848,14 +787,12 @@ void FEA_Module_SGH::build_boundry_node_sets(mesh_t& mesh) num_bdy_nodes_in_set(bdy_set) = 0; // Loop over boundary patches in boundary set - for (size_t bdy_patch_gid = 0; bdy_patch_gid < num_bdy_patches_in_set; bdy_patch_gid++) - { + for (size_t bdy_patch_gid = 0; bdy_patch_gid < num_bdy_patches_in_set; bdy_patch_gid++) { // get the global id for this boundary patch size_t patch_gid = bdy_patches_in_set(bdy_set, bdy_patch_gid); // apply boundary condition at nodes on boundary - for (size_t node_lid = 0; node_lid < num_nodes_in_patch; node_lid++) - { + for (size_t node_lid = 0; node_lid < num_nodes_in_patch; node_lid++) { size_t node_gid = Local_Index_Boundary_Patches(patch_gid, node_lid); temp_count_num_bdy_nodes_in_set(bdy_set, node_gid) = -1; @@ -863,18 +800,15 @@ void FEA_Module_SGH::build_boundry_node_sets(mesh_t& mesh) } // end for bdy_patch_gid // Loop over boundary patches in boundary set - for (size_t bdy_patch_gid = 0; bdy_patch_gid < num_bdy_patches_in_set; bdy_patch_gid++) - { + for (size_t bdy_patch_gid = 0; bdy_patch_gid < num_bdy_patches_in_set; bdy_patch_gid++) { // get the global id for this boundary patch size_t patch_gid = bdy_patches_in_set(bdy_set, bdy_patch_gid); // apply boundary condition at nodes on boundary - for (size_t node_lid = 0; node_lid < num_nodes_in_patch; node_lid++) - { + for (size_t node_lid = 0; node_lid < num_nodes_in_patch; node_lid++) { size_t node_gid = Local_Index_Boundary_Patches(patch_gid, node_lid); - if (temp_count_num_bdy_nodes_in_set(bdy_set, node_gid) == -1) - { + if (temp_count_num_bdy_nodes_in_set(bdy_set, node_gid) == -1) { size_t num_saved = num_bdy_nodes_in_set(bdy_set); num_bdy_nodes_in_set(bdy_set)++; @@ -896,8 +830,7 @@ void FEA_Module_SGH::build_boundry_node_sets(mesh_t& mesh) FOR_ALL_CLASS(bdy_set, 0, num_bdy_sets, { // Loop over boundary patches in boundary set - for (size_t bdy_node_lid = 0; bdy_node_lid < num_bdy_nodes_in_set(bdy_set); bdy_node_lid++) - { + for (size_t bdy_node_lid = 0; bdy_node_lid < num_bdy_nodes_in_set(bdy_set); bdy_node_lid++) { // save the bdy_node_gid bdy_nodes_in_set(bdy_set, bdy_node_lid) = temp_nodes_in_set(bdy_set, bdy_node_lid); } // end for @@ -922,14 +855,12 @@ void FEA_Module_SGH::init_boundaries() int num_dim = simparam->num_dims; // set the number of boundary sets - if (myrank == 0) - { + if (myrank == 0) { std::cout << "building boundary sets " << std::endl; } // initialize to 1 since there must be at least 1 boundary set anyway; read in may occure later - if (max_boundary_sets == 0) - { + if (max_boundary_sets == 0) { max_boundary_sets = 1; } // std::cout << "NUM BOUNDARY CONDITIONS ON RANK " << myrank << " FOR INIT " << num_boundary_conditions <(nall_nodes * num_dim, "Node_DOF_Boundary_Condition_Type"); // initialize - for (int init = 0; init < nall_nodes * num_dim; init++) - { + for (int init = 0; init < nall_nodes * num_dim; init++) { Node_DOF_Boundary_Condition_Type(init) = NONE; } @@ -958,8 +888,7 @@ void FEA_Module_SGH::init_boundaries() ///////////////////////////////////////////////////////////////////////////// void FEA_Module_SGH::init_boundary_sets(int num_sets) { - if (num_sets == 0) - { + if (num_sets == 0) { std::cout << " Warning: number of boundary conditions = 0"; return; } @@ -972,14 +901,12 @@ void FEA_Module_SGH::init_boundary_sets(int num_sets) Boundary_Condition_Patches = CArrayKokkos(num_sets, nboundary_patches, "Boundary_Condition_Patches"); // initialize data - for (int iset = 0; iset < num_sets; iset++) - { + for (int iset = 0; iset < num_sets; iset++) { NBoundary_Condition_Patches(iset) = 0; } // initialize - for (int ibdy = 0; ibdy < num_sets; ibdy++) - { + for (int ibdy = 0; ibdy < num_sets; ibdy++) { Boundary_Condition_Type_List(ibdy) = NONE; } } @@ -997,15 +924,13 @@ void FEA_Module_SGH::grow_boundary_sets(int num_sets) { int num_dim = simparam->num_dims; - if (num_sets == 0) - { + if (num_sets == 0) { std::cout << " Warning: number of boundary conditions being set to 0"; return; } // std::cout << " DEBUG PRINT "< max_boundary_sets) - { + if (num_sets > max_boundary_sets) { // temporary storage for previous data CArrayKokkos Temp_Boundary_Condition_Type_List = Boundary_Condition_Type_List; CArrayKokkos Temp_NBoundary_Condition_Patches = NBoundary_Condition_Patches; @@ -1021,25 +946,21 @@ void FEA_Module_SGH::grow_boundary_sets(int num_sets) #ifdef DEBUG std::cout << "NUM BOUNDARY CONDITIONS ON RANK " << myrank << " FOR COPY " << max_boundary_sets << std::endl; #endif - for (int iset = 0; iset < num_boundary_conditions; iset++) - { + for (int iset = 0; iset < num_boundary_conditions; iset++) { Boundary_Condition_Type_List(iset) = Temp_Boundary_Condition_Type_List(iset); NBoundary_Condition_Patches(iset) = Temp_NBoundary_Condition_Patches(iset); - for (int ipatch = 0; ipatch < nboundary_patches; ipatch++) - { + for (int ipatch = 0; ipatch < nboundary_patches; ipatch++) { Boundary_Condition_Patches(iset, ipatch) = Temp_Boundary_Condition_Patches(iset, ipatch); } } // initialize data - for (int iset = num_boundary_conditions; iset < max_boundary_sets; iset++) - { + for (int iset = num_boundary_conditions; iset < max_boundary_sets; iset++) { NBoundary_Condition_Patches(iset) = 0; } // initialize - for (int ibdy = num_boundary_conditions; ibdy < max_boundary_sets; ibdy++) - { + for (int ibdy = num_boundary_conditions; ibdy < max_boundary_sets; ibdy++) { Boundary_Condition_Type_List(ibdy) = NONE; } } diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/sgh_optimization.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/sgh_optimization.cpp index c0982d635..3c1089c24 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/sgh_optimization.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/sgh_optimization.cpp @@ -115,10 +115,8 @@ void FEA_Module_SGH::update_forward_solve(Teuchos::RCP zp) const_host_vec_array all_node_densities = all_node_densities_distributed->getLocalView(Tpetra::Access::ReadOnly); // debug print // std::cout << "NODE DENSITY TEST " << all_node_densities(0,0) << std::endl; - for (int elem_id = 0; elem_id < rnum_elem; elem_id++) - { - for (int inode = 0; inode < num_nodes_in_elem; inode++) - { + for (int elem_id = 0; elem_id < rnum_elem; elem_id++) { + for (int inode = 0; inode < num_nodes_in_elem; inode++) { current_element_nodal_densities(inode) = all_node_densities(nodes_in_elem(elem_id, inode), 0); } relative_element_densities.host(elem_id) = average_element_density(num_nodes_in_elem, current_element_nodal_densities); @@ -143,16 +141,14 @@ void FEA_Module_SGH::update_forward_solve(Teuchos::RCP zp) const_vec_array ghost_node_coords_interface = Explicit_Solver_Pointer_->ghost_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); vec_array all_node_coords_interface = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - for (int idim = 0; idim < num_dim; idim++) - { + for (int idim = 0; idim < num_dim; idim++) { all_node_coords_interface(node_gid, idim) = node_coords_interface(node_gid, idim); } }); // end parallel for Kokkos::fence(); FOR_ALL_CLASS(node_gid, nlocal_nodes, nlocal_nodes + nghost_nodes, { - for (int idim = 0; idim < num_dim; idim++) - { + for (int idim = 0; idim < num_dim; idim++) { all_node_coords_interface(node_gid, idim) = ghost_node_coords_interface(node_gid - nlocal_nodes, idim); } }); // end parallel for @@ -183,14 +179,11 @@ void FEA_Module_SGH::update_forward_solve(Teuchos::RCP zp) // view scope { host_vec_array interface_node_coords = Explicit_Solver_Pointer_->all_node_coords_distributed->getLocalView(Tpetra::Access::ReadWrite); - for (size_t ibin = 0; ibin < rk_num_bins; ibin++) - { + for (size_t ibin = 0; ibin < rk_num_bins; ibin++) { // save node data to node.coords // std::cout << "NODE DATA ON RANK " << myrank << std::endl; - if (num_dim == 2) - { - for (int inode = 0; inode < nall_nodes; inode++) - { + if (num_dim == 2) { + for (int inode = 0; inode < nall_nodes; inode++) { // std::cout << "Node index " << inode+1 << " "; node_coords.host(ibin, inode, 0) = interface_node_coords(inode, 0); // std::cout << host_node_coords_state(0,inode,0)+1<< " "; @@ -198,10 +191,8 @@ void FEA_Module_SGH::update_forward_solve(Teuchos::RCP zp) // std::cout << host_node_coords_state(0,inode,1)+1<< " "; } } - else if (num_dim == 3) - { - for (int inode = 0; inode < nall_nodes; inode++) - { + else if (num_dim == 3) { + for (int inode = 0; inode < nall_nodes; inode++) { // std::cout << "Node index " << inode+1 << " "; node_coords.host(ibin, inode, 0) = interface_node_coords(inode, 0); // std::cout << host_node_coords_state(0,inode,0)+1<< " "; @@ -216,12 +207,9 @@ void FEA_Module_SGH::update_forward_solve(Teuchos::RCP zp) } // end view scope // save the node coords to the current RK value - for (size_t node_gid = 0; node_gid < nall_nodes; node_gid++) - { - for (int rk = 1; rk < rk_num_bins; rk++) - { - for (int dim = 0; dim < num_dim; dim++) - { + for (size_t node_gid = 0; node_gid < nall_nodes; node_gid++) { + for (int rk = 1; rk < rk_num_bins; rk++) { + for (int dim = 0; dim < num_dim; dim++) { node_coords.host(rk, node_gid, dim) = node_coords.host(0, node_gid, dim); } // end for dim } // end for rk @@ -236,8 +224,7 @@ void FEA_Module_SGH::update_forward_solve(Teuchos::RCP zp) // loop over the fill instructures // view scope { - for (int f_id = 0; f_id < num_fills; f_id++) - { + for (int f_id = 0; f_id < num_fills; f_id++) { // parallel loop over elements in mesh FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { // calculate the coordinates and radius of the element @@ -247,16 +234,13 @@ void FEA_Module_SGH::update_forward_solve(Teuchos::RCP zp) elem_coords[2] = 0.0; // get the coordinates of the element center - for (int node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) - { + for (int node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { elem_coords[0] += node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), 0); elem_coords[1] += node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), 1); - if (num_dim == 3) - { + if (num_dim == 3) { elem_coords[2] += node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), 2); } - else - { + else{ elem_coords[2] = 0.0; } } // end loop over nodes in element @@ -268,8 +252,7 @@ void FEA_Module_SGH::update_forward_solve(Teuchos::RCP zp) bool fill_this = mat_fill(f_id).volume.contains(elem_coords); // paint the material state on the element - if (fill_this) - { + if (fill_this) { // density elem_den(elem_gid) = mat_fill(f_id).den; @@ -282,8 +265,7 @@ void FEA_Module_SGH::update_forward_solve(Teuchos::RCP zp) // specific internal energy elem_sie(rk_level, elem_gid) = mat_fill(f_id).sie; - if (simparam->topology_optimization_on && mat_fill(f_id).extensive_energy_setting) - { + if (simparam->topology_optimization_on && mat_fill(f_id).extensive_energy_setting) { elem_sie(rk_level, elem_gid) = elem_sie(rk_level, elem_gid) / relative_element_densities(elem_gid); } @@ -292,10 +274,8 @@ void FEA_Module_SGH::update_forward_solve(Teuchos::RCP zp) // --- stress tensor --- // always 3D even for 2D-RZ - for (size_t i = 0; i < 3; i++) - { - for (size_t j = 0; j < 3; j++) - { + for (size_t i = 0; i < 3; i++) { + for (size_t j = 0; j < 3; j++) { elem_stress(rk_level, elem_gid, i, j) = 0.0; } } // end for @@ -329,20 +309,17 @@ void FEA_Module_SGH::update_forward_solve(Teuchos::RCP zp) elem_sie(rk_level, elem_gid)); // loop over the nodes of this element and apply velocity - for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) - { + for (size_t node_lid = 0; node_lid < num_nodes_in_elem; node_lid++) { // get the mesh node index size_t node_gid = nodes_in_elem(elem_gid, node_lid); // --- Velocity --- - switch (mat_fill(f_id).velocity) - { + switch (mat_fill(f_id).velocity) { case VELOCITY_TYPE::cartesian: { node_vel(rk_level, node_gid, 0) = mat_fill(f_id).u; node_vel(rk_level, node_gid, 1) = mat_fill(f_id).v; - if (num_dim == 3) - { + if (num_dim == 3) { node_vel(rk_level, node_gid, 2) = mat_fill(f_id).w; } @@ -356,29 +333,24 @@ void FEA_Module_SGH::update_forward_solve(Teuchos::RCP zp) dir[1] = 0.0; double radius_val = 0.0; - for (int dim = 0; dim < 2; dim++) - { + for (int dim = 0; dim < 2; dim++) { dir[dim] = node_coords(rk_level, node_gid, dim); radius_val += node_coords(rk_level, node_gid, dim) * node_coords(rk_level, node_gid, dim); } // end for radius_val = sqrt(radius_val); - for (int dim = 0; dim < 2; dim++) - { - if (radius_val > 1.0e-14) - { + for (int dim = 0; dim < 2; dim++) { + if (radius_val > 1.0e-14) { dir[dim] /= (radius_val); } - else - { + else{ dir[dim] = 0.0; } } // end for node_vel(rk_level, node_gid, 0) = mat_fill(f_id).speed * dir[0]; node_vel(rk_level, node_gid, 1) = mat_fill(f_id).speed * dir[1]; - if (num_dim == 3) - { + if (num_dim == 3) { node_vel(rk_level, node_gid, 2) = 0.0; } @@ -393,29 +365,24 @@ void FEA_Module_SGH::update_forward_solve(Teuchos::RCP zp) dir[2] = 0.0; double radius_val = 0.0; - for (int dim = 0; dim < 3; dim++) - { + for (int dim = 0; dim < 3; dim++) { dir[dim] = node_coords(rk_level, node_gid, dim); radius_val += node_coords(rk_level, node_gid, dim) * node_coords(rk_level, node_gid, dim); } // end for radius_val = sqrt(radius_val); - for (int dim = 0; dim < 3; dim++) - { - if (radius_val > 1.0e-14) - { + for (int dim = 0; dim < 3; dim++) { + if (radius_val > 1.0e-14) { dir[dim] /= (radius_val); } - else - { + else{ dir[dim] = 0.0; } } // end for node_vel(rk_level, node_gid, 0) = mat_fill(f_id).speed * dir[0]; node_vel(rk_level, node_gid, 1) = mat_fill(f_id).speed * dir[1]; - if (num_dim == 3) - { + if (num_dim == 3) { node_vel(rk_level, node_gid, 2) = mat_fill(f_id).speed * dir[2]; } @@ -433,8 +400,7 @@ void FEA_Module_SGH::update_forward_solve(Teuchos::RCP zp) { node_vel(rk_level, node_gid, 0) = sin(PI * node_coords(rk_level, node_gid, 0)) * cos(PI * node_coords(rk_level, node_gid, 1)); node_vel(rk_level, node_gid, 1) = -1.0 * cos(PI * node_coords(rk_level, node_gid, 0)) * sin(PI * node_coords(rk_level, node_gid, 1)); - if (num_dim == 3) - { + if (num_dim == 3) { node_vel(rk_level, node_gid, 2) = 0.0; } @@ -443,8 +409,7 @@ void FEA_Module_SGH::update_forward_solve(Teuchos::RCP zp) } // end of switch } // end loop over nodes of element - if (mat_fill(f_id).velocity == VELOCITY_TYPE::tg_vortex) - { + if (mat_fill(f_id).velocity == VELOCITY_TYPE::tg_vortex) { elem_pres(elem_gid) = 0.25 * (cos(2.0 * PI * elem_coords[0]) + cos(2.0 * PI * elem_coords[1]) ) + 1.0; // p = rho*ie*(gamma - 1) @@ -463,8 +428,7 @@ void FEA_Module_SGH::update_forward_solve(Teuchos::RCP zp) FEA_Module_SGH::boundary_velocity(*mesh, boundary, node_vel); // calculate the corner massess if 2D - if (num_dim == 2) - { + if (num_dim == 2) { FOR_ALL_CLASS(elem_gid, 0, rnum_elem, { // facial area of the corners double corner_areas_array[4]; @@ -479,8 +443,7 @@ void FEA_Module_SGH::update_forward_solve(Teuchos::RCP zp) rk_level); // loop over the corners of the element and calculate the mass - for (size_t corner_lid = 0; corner_lid < 4; corner_lid++) - { + for (size_t corner_lid = 0; corner_lid < 4; corner_lid++) { size_t corner_gid = corners_in_elem(elem_gid, corner_lid); corner_mass(corner_gid) = corner_areas(corner_lid) * elem_den(elem_gid); // node radius is added later } // end for over corners @@ -491,19 +454,15 @@ void FEA_Module_SGH::update_forward_solve(Teuchos::RCP zp) FOR_ALL_CLASS(node_gid, 0, nall_nodes, { node_mass(node_gid) = 0.0; - if (num_dim == 3) - { - for (size_t elem_lid = 0; elem_lid < num_corners_in_node(node_gid); elem_lid++) - { + if (num_dim == 3) { + for (size_t elem_lid = 0; elem_lid < num_corners_in_node(node_gid); elem_lid++) { size_t elem_gid = elems_in_node(node_gid, elem_lid); node_mass(node_gid) += 1.0 / 8.0 * elem_mass(elem_gid); } // end for elem_lid } // end if dims=3 - else - { + else{ // 2D-RZ - for (size_t corner_lid = 0; corner_lid < num_corners_in_node(node_gid); corner_lid++) - { + for (size_t corner_lid = 0; corner_lid < num_corners_in_node(node_gid); corner_lid++) { size_t corner_gid = corners_in_node(node_gid, corner_lid); node_mass(node_gid) += corner_mass(corner_gid); // sans the radius so it is areal node mass @@ -537,8 +496,7 @@ void FEA_Module_SGH::update_forward_solve(Teuchos::RCP zp) Kokkos::fence(); // update stiffness matrix - if (simparam->topology_optimization_on || simparam->shape_optimization_on) - { + if (simparam->topology_optimization_on || simparam->shape_optimization_on) { // assemble_matrix(); } @@ -569,8 +527,7 @@ void FEA_Module_SGH::update_forward_solve(Teuchos::RCP zp) double FEA_Module_SGH::average_element_density(const int nodes_per_elem, const CArray current_nodal_densities) const { double result = 0; - for (int i = 0; i < nodes_per_elem; i++) - { + for (int i = 0; i < nodes_per_elem; i++) { result += current_nodal_densities(i) / nodes_per_elem; } @@ -602,30 +559,23 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full() // solve terminal value problem, proceeds in time backward. For simplicity, we use the same timestep data from the forward solve. // A linear interpolant is assumed between velocity data points; velocity midpoint is used to update the adjoint. - if (myrank == 0) - { + if (myrank == 0) { std::cout << "Computing adjoint vector " << time_data.size() << std::endl; } - for (int cycle = last_time_step; cycle >= 0; cycle--) - { + for (int cycle = last_time_step; cycle >= 0; cycle--) { // compute timestep from time data global_dt = time_data[cycle + 1] - time_data[cycle]; // print - if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) - { - if (cycle == last_time_step) - { - if (myrank == 0) - { + if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) { + if (cycle == last_time_step) { + if (myrank == 0) { printf("cycle = %d, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); } } // print time step every 20 cycles - else if (cycle % 20 == 0) - { - if (myrank == 0) - { + else if (cycle % 20 == 0) { + if (myrank == 0) { printf("cycle = %d, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); } } // end if @@ -651,8 +601,7 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full() // interface of arrays for current implementation of force calculation FOR_ALL_CLASS(node_gid, 0, nlocal_nodes + nghost_nodes, { - for (int idim = 0; idim < num_dim; idim++) - { + for (int idim = 0; idim < num_dim; idim++) { node_vel(rk_level, node_gid, idim) = previous_velocity_vector(node_gid, idim); node_coords(rk_level, node_gid, idim) = previous_coordinate_vector(node_gid, idim); } @@ -669,15 +618,13 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full() get_vol(); // ---- Calculate velocity diveregence for the element ---- - if (num_dim == 2) - { + if (num_dim == 2) { get_divergence2D(elem_div, node_coords, node_vel, elem_vol); } - else - { + else{ get_divergence(elem_div, node_coords, node_vel, @@ -685,8 +632,7 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full() } // end if 2D // ---- Calculate elem state (den, pres, sound speed, stress) for next time step ---- - if (num_dim == 2) - { + if (num_dim == 2) { update_state2D(material, *mesh, node_coords, @@ -702,8 +648,7 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full() 1.0, cycle); } - else - { + else{ update_state(material, *mesh, node_coords, @@ -720,8 +665,7 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full() cycle); } - if (num_dim == 2) - { + if (num_dim == 2) { get_force_sgh2D(material, *mesh, node_coords, @@ -738,8 +682,7 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full() 1.0, cycle); } - else - { + else{ get_force_sgh(material, *mesh, node_coords, @@ -847,20 +790,17 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full() real_t matrix_contribution; size_t dof_id; size_t elem_id; - for (int idim = 0; idim < num_dim; idim++) - { + for (int idim = 0; idim < num_dim; idim++) { // EQUATION 1 matrix_contribution = 0; // compute resulting row of force velocity gradient matrix transpose right multiplied by adjoint vector - for (int idof = 0; idof < Gradient_Matrix_Strides(node_gid * num_dim + idim); idof++) - { + for (int idof = 0; idof < Gradient_Matrix_Strides(node_gid * num_dim + idim); idof++) { dof_id = DOF_Graph_Matrix(node_gid * num_dim + idim, idof); matrix_contribution += previous_adjoint_vector(dof_id / num_dim, dof_id % num_dim) * Force_Gradient_Velocities(node_gid * num_dim + idim, idof); } // compute resulting row of transpose of power gradient w.r.t velocity matrix right multiplied by psi adjoint vector - for (int ielem = 0; ielem < DOF_to_Elem_Matrix_Strides(node_gid * num_dim + idim); ielem++) - { + for (int ielem = 0; ielem < DOF_to_Elem_Matrix_Strides(node_gid * num_dim + idim); ielem++) { elem_id = elems_in_node(node_gid, ielem); matrix_contribution += psi_previous_adjoint_vector(elem_id, 0) * Power_Gradient_Velocities(node_gid * num_dim + idim, ielem); } @@ -878,20 +818,17 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full() real_t matrix_contribution; size_t dof_id; size_t elem_id; - for (int idim = 0; idim < num_dim; idim++) - { + for (int idim = 0; idim < num_dim; idim++) { // EQUATION 2 matrix_contribution = 0; // compute resulting row of force displacement gradient matrix transpose right multiplied by adjoint vector - for (int idof = 0; idof < Gradient_Matrix_Strides(node_gid * num_dim + idim); idof++) - { + for (int idof = 0; idof < Gradient_Matrix_Strides(node_gid * num_dim + idim); idof++) { dof_id = DOF_Graph_Matrix(node_gid * num_dim + idim, idof); matrix_contribution += previous_adjoint_vector(dof_id / num_dim, dof_id % num_dim) * Force_Gradient_Positions(node_gid * num_dim + idim, idof); } // compute resulting row of transpose of power gradient w.r.t displacement matrix right multiplied by psi adjoint vector - for (int ielem = 0; ielem < DOF_to_Elem_Matrix_Strides(node_gid * num_dim + idim); ielem++) - { + for (int ielem = 0; ielem < DOF_to_Elem_Matrix_Strides(node_gid * num_dim + idim); ielem++) { elem_id = elems_in_node(node_gid, ielem); matrix_contribution += psi_previous_adjoint_vector(elem_id, 0) * Power_Gradient_Positions(node_gid * num_dim + idim, ielem); } @@ -914,8 +851,7 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full() // EQUATION 3 matrix_contribution = 0; // compute resulting row of force displacement gradient matrix transpose right multiplied by adjoint vector - for (int idof = 0; idof < num_nodes_in_elem * num_dim; idof++) - { + for (int idof = 0; idof < num_nodes_in_elem * num_dim; idof++) { dof_id = nodes_in_elem(elem_gid, idof / num_dim) * num_dim + idof % num_dim; matrix_contribution += previous_adjoint_vector(dof_id / num_dim, dof_id % num_dim) * Force_Gradient_Energies(elem_gid, idof); } @@ -943,8 +879,7 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full() // compute gradients at midpoint FOR_ALL_CLASS(node_gid, 0, nlocal_nodes + nghost_nodes, { - for (int idim = 0; idim < num_dim; idim++) - { + for (int idim = 0; idim < num_dim; idim++) { node_vel(rk_level, node_gid, idim) = 0.5 * (previous_velocity_vector(node_gid, idim) + current_velocity_vector(node_gid, idim)); node_coords(rk_level, node_gid, idim) = 0.5 * (previous_coordinate_vector(node_gid, idim) + current_coordinate_vector(node_gid, idim)); } @@ -961,15 +896,13 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full() get_vol(); // ---- Calculate velocity diveregence for the element ---- - if (num_dim == 2) - { + if (num_dim == 2) { get_divergence2D(elem_div, node_coords, node_vel, elem_vol); } - else - { + else{ get_divergence(elem_div, node_coords, node_vel, @@ -977,8 +910,7 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full() } // end if 2D // ---- Calculate elem state (den, pres, sound speed, stress) for next time step ---- - if (num_dim == 2) - { + if (num_dim == 2) { update_state2D(material, *mesh, node_coords, @@ -994,8 +926,7 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full() 1.0, cycle); } - else - { + else{ update_state(material, *mesh, node_coords, @@ -1012,8 +943,7 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full() cycle); } - if (num_dim == 2) - { + if (num_dim == 2) { get_force_sgh2D(material, *mesh, node_coords, @@ -1030,8 +960,7 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full() 1.0, cycle); } - else - { + else{ get_force_sgh(material, *mesh, node_coords, @@ -1125,21 +1054,18 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full() real_t matrix_contribution; size_t dof_id; size_t elem_id; - for (int idim = 0; idim < num_dim; idim++) - { + for (int idim = 0; idim < num_dim; idim++) { // EQUATION 1 matrix_contribution = 0; // compute resulting row of force velocity gradient matrix transpose right multiplied by adjoint vector - for (int idof = 0; idof < Gradient_Matrix_Strides(node_gid * num_dim + idim); idof++) - { + for (int idof = 0; idof < Gradient_Matrix_Strides(node_gid * num_dim + idim); idof++) { dof_id = DOF_Graph_Matrix(node_gid * num_dim + idim, idof); matrix_contribution += midpoint_adjoint_vector(dof_id / num_dim, dof_id % num_dim) * Force_Gradient_Velocities(node_gid * num_dim + idim, idof); } // compute resulting row of transpose of power gradient w.r.t velocity matrix right multiplied by psi adjoint vector - for (int ielem = 0; ielem < DOF_to_Elem_Matrix_Strides(node_gid * num_dim + idim); ielem++) - { + for (int ielem = 0; ielem < DOF_to_Elem_Matrix_Strides(node_gid * num_dim + idim); ielem++) { elem_id = elems_in_node(node_gid, ielem); matrix_contribution += psi_midpoint_adjoint_vector(elem_id, 0) * Power_Gradient_Velocities(node_gid * num_dim + idim, ielem); } @@ -1158,20 +1084,17 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full() real_t matrix_contribution; size_t dof_id; size_t elem_id; - for (int idim = 0; idim < num_dim; idim++) - { + for (int idim = 0; idim < num_dim; idim++) { // EQUATION 2 matrix_contribution = 0; // compute resulting row of force displacement gradient matrix transpose right multiplied by adjoint vector - for (int idof = 0; idof < Gradient_Matrix_Strides(node_gid * num_dim + idim); idof++) - { + for (int idof = 0; idof < Gradient_Matrix_Strides(node_gid * num_dim + idim); idof++) { dof_id = DOF_Graph_Matrix(node_gid * num_dim + idim, idof); matrix_contribution += midpoint_adjoint_vector(dof_id / num_dim, dof_id % num_dim) * Force_Gradient_Positions(node_gid * num_dim + idim, idof); } // compute resulting row of transpose of power gradient w.r.t displacement matrix right multiplied by psi adjoint vector - for (int ielem = 0; ielem < DOF_to_Elem_Matrix_Strides(node_gid * num_dim + idim); ielem++) - { + for (int ielem = 0; ielem < DOF_to_Elem_Matrix_Strides(node_gid * num_dim + idim); ielem++) { elem_id = elems_in_node(node_gid, ielem); matrix_contribution += psi_midpoint_adjoint_vector(elem_id, 0) * Power_Gradient_Positions(node_gid * num_dim + idim, ielem); } @@ -1192,8 +1115,7 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full() // EQUATION 3 matrix_contribution = 0; // compute resulting row of force displacement gradient matrix transpose right multiplied by adjoint vector - for (int idof = 0; idof < num_nodes_in_elem * num_dim; idof++) - { + for (int idof = 0; idof < num_nodes_in_elem * num_dim; idof++) { dof_id = nodes_in_elem(elem_gid, idof / num_dim) * num_dim + idof % num_dim; matrix_contribution += midpoint_adjoint_vector(dof_id / num_dim, dof_id % num_dim) * Force_Gradient_Energies(elem_gid, idof); } @@ -1238,8 +1160,7 @@ void FEA_Module_SGH::compute_topology_optimization_gradient_full(Teuchos::RCP current_element_velocities = CArrayKokkos(num_nodes_in_elem, num_dim); CArrayKokkos current_element_adjoint = CArrayKokkos(num_nodes_in_elem, num_dim); - if (myrank == 0) - { + if (myrank == 0) { std::cout << "Computing accumulated kinetic energy gradient" << std::endl; } @@ -1255,34 +1176,26 @@ void FEA_Module_SGH::compute_topology_optimization_gradient_full(Teuchos::RCPdynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) - { - if (myrank == 0) - { + if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) { + if (myrank == 0) { std::cout << "v*dM/drho*v term" << std::endl; } } - for (unsigned long cycle = 0; cycle < last_time_step + 1; cycle++) - { + for (unsigned long cycle = 0; cycle < last_time_step + 1; cycle++) { // compute timestep from time data global_dt = time_data[cycle + 1] - time_data[cycle]; // print - if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) - { - if (cycle == 0) - { - if (myrank == 0) - { + if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) { + if (cycle == 0) { + if (myrank == 0) { printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); } } // print time step every 10 cycles - else if (cycle % 20 == 0) - { - if (myrank == 0) - { + else if (cycle % 20 == 0) { + if (myrank == 0) { printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); } } // end if @@ -1299,30 +1212,25 @@ void FEA_Module_SGH::compute_topology_optimization_gradient_full(Teuchos::RCPdynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) - { - if (myrank == 0) - { + if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) { + if (myrank == 0) { std::cout << "gradient term involving adjoint derivative" << std::endl; } } - for (unsigned long cycle = 0; cycle < last_time_step + 1; cycle++) - { + for (unsigned long cycle = 0; cycle < last_time_step + 1; cycle++) { // compute timestep from time data global_dt = time_data[cycle + 1] - time_data[cycle]; // print - if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) - { - if (cycle == 0) - { - if (myrank == 0) - { + if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) { + if (cycle == 0) { + if (myrank == 0) { printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); } } // print time step every 10 cycles - else if (cycle % 20 == 0) - { - if (myrank == 0) - { + else if (cycle % 20 == 0) { + if (myrank == 0) { printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); } } // end if @@ -1400,24 +1299,20 @@ void FEA_Module_SGH::compute_topology_optimization_gradient_full(Teuchos::RCPdynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) - { - if (myrank == 0) - { + if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) { + if (myrank == 0) { std::cout << "gradient term involving adjoint derivative" << std::endl; } } - for (unsigned long cycle = 0; cycle < last_time_step + 1; cycle++) - { + for (unsigned long cycle = 0; cycle < last_time_step + 1; cycle++) { // compute timestep from time data global_dt = time_data[cycle + 1] - time_data[cycle]; // print - if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) - { - if (cycle == 0) - { - if (myrank == 0) - { + if (simparam->dynamic_options.output_time_sequence_level == TIME_OUTPUT_LEVEL::extreme) { + if (cycle == 0) { + if (myrank == 0) { printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); } } // print time step every 10 cycles - else if (cycle % 20 == 0) - { - if (myrank == 0) - { + else if (cycle % 20 == 0) { + if (myrank == 0) { printf("cycle = %lu, time = %f, time step = %f \n", cycle, time_data[cycle], global_dt); } } // end if @@ -1499,8 +1384,7 @@ void FEA_Module_SGH::compute_topology_optimization_gradient_full(Teuchos::RCPchoose_2Delem_type(Element_Types(ielem), elem2D); nodes_per_element = elem2D->num_nodes(); - for (int lnode = 0; lnode < nodes_per_element; lnode++) - { + for (int lnode = 0; lnode < nodes_per_element; lnode++) { local_node_index = nodes_in_elem(ielem, lnode); - if (local_node_index < nlocal_nodes) - { + if (local_node_index < nlocal_nodes) { Dual_Graph_Matrix_Strides_initial.host(local_node_index) += nodes_per_element; } } } } - if (num_dim == 3) - { - for (int ielem = 0; ielem < rnum_elem; ielem++) - { + if (num_dim == 3) { + for (int ielem = 0; ielem < rnum_elem; ielem++) { element_select->choose_3Delem_type(Element_Types(ielem), elem); nodes_per_element = elem->num_nodes(); - for (int lnode = 0; lnode < nodes_per_element; lnode++) - { + for (int lnode = 0; lnode < nodes_per_element; lnode++) { local_node_index = nodes_in_elem(ielem, lnode); - if (local_node_index < nlocal_nodes) - { + if (local_node_index < nlocal_nodes) { Dual_Graph_Matrix_Strides_initial.host(local_node_index) += nodes_per_element; } } @@ -1744,8 +1609,7 @@ void FEA_Module_SGH::init_assembly() // compute maximum stride size_t update = 0; REDUCE_MAX_CLASS(inode, 0, nlocal_nodes, update, { - if (update < Graph_Matrix_Strides_initial(inode)) - { + if (update < Graph_Matrix_Strides_initial(inode)) { update = Graph_Matrix_Strides_initial(inode); } }, max_stride); @@ -1759,19 +1623,14 @@ void FEA_Module_SGH::init_assembly() RaggedRightArrayofVectorsKokkos Element_local_indices(Graph_Matrix_Strides_initial, 3); // Fill the initial Graph with repeats - if (num_dim == 2) - { - for (int ielem = 0; ielem < rnum_elem; ielem++) - { + if (num_dim == 2) { + for (int ielem = 0; ielem < rnum_elem; ielem++) { element_select->choose_2Delem_type(Element_Types(ielem), elem2D); nodes_per_element = elem2D->num_nodes(); - for (int lnode = 0; lnode < nodes_per_element; lnode++) - { + for (int lnode = 0; lnode < nodes_per_element; lnode++) { local_node_index = nodes_in_elem(ielem, lnode); - if (local_node_index < nlocal_nodes) - { - for (int jnode = 0; jnode < nodes_per_element; jnode++) - { + if (local_node_index < nlocal_nodes) { + for (int jnode = 0; jnode < nodes_per_element; jnode++) { current_column_index = Graph_Fill(local_node_index) + jnode; Repeat_Graph_Matrix(local_node_index, current_column_index) = nodes_in_elem(ielem, jnode); @@ -1789,19 +1648,14 @@ void FEA_Module_SGH::init_assembly() } } - if (num_dim == 3) - { - for (int ielem = 0; ielem < rnum_elem; ielem++) - { + if (num_dim == 3) { + for (int ielem = 0; ielem < rnum_elem; ielem++) { element_select->choose_3Delem_type(Element_Types(ielem), elem); nodes_per_element = elem->num_nodes(); - for (int lnode = 0; lnode < nodes_per_element; lnode++) - { + for (int lnode = 0; lnode < nodes_per_element; lnode++) { local_node_index = nodes_in_elem(ielem, lnode); - if (local_node_index < nlocal_nodes) - { - for (int jnode = 0; jnode < nodes_per_element; jnode++) - { + if (local_node_index < nlocal_nodes) { + for (int jnode = 0; jnode < nodes_per_element; jnode++) { current_column_index = Graph_Fill(local_node_index) + jnode; Repeat_Graph_Matrix(local_node_index, current_column_index) = nodes_in_elem(ielem, jnode); @@ -1833,19 +1687,16 @@ void FEA_Module_SGH::init_assembly() int element_column_index; int current_stride; int current_row_n_nodes_scanned; - for (int inode = 0; inode < nlocal_nodes; inode++) - { + for (int inode = 0; inode < nlocal_nodes; inode++) { current_row_n_nodes_scanned = 0; - for (int istride = 0; istride < Graph_Matrix_Strides(inode); istride++) - { + for (int istride = 0; istride < Graph_Matrix_Strides(inode); istride++) { // convert global index in graph to its local index for the flagging array current_node = Repeat_Graph_Matrix(inode, istride); // debug // if(current_node==-1) // std::cout << "Graph Matrix node access on task " << myrank << std::endl; // std::cout << Repeat_Graph_Matrix(inode,istride) << std::endl; - if (node_indices_used(current_node)) - { + if (node_indices_used(current_node)) { // set global assembly map index to the location in the graph matrix where this global node was first found current_element_index = Element_local_indices(inode, istride, 0); element_row_index = Element_local_indices(inode, istride, 1); @@ -1857,8 +1708,7 @@ void FEA_Module_SGH::init_assembly() // first swap information about the inverse and forward maps current_stride = Graph_Matrix_Strides(inode); - if (istride != current_stride - 1) - { + if (istride != current_stride - 1) { Element_local_indices(inode, istride, 0) = Element_local_indices(inode, current_stride - 1, 0); Element_local_indices(inode, istride, 1) = Element_local_indices(inode, current_stride - 1, 1); Element_local_indices(inode, istride, 2) = Element_local_indices(inode, current_stride - 1, 2); @@ -1876,8 +1726,7 @@ void FEA_Module_SGH::init_assembly() istride--; Graph_Matrix_Strides(inode)--; } - else - { + else{ /*this node hasn't shown up in the row before; add it to the list of nodes that have been scanned uniquely. Use this list to reset the flag array afterwards without having to loop over all the nodes in the system*/ @@ -1888,8 +1737,7 @@ void FEA_Module_SGH::init_assembly() } } // reset nodes used list for the next row of the sparse list - for (int node_reset = 0; node_reset < current_row_n_nodes_scanned; node_reset++) - { + for (int node_reset = 0; node_reset < current_row_n_nodes_scanned; node_reset++) { node_indices_used(current_row_nodes_scanned(node_reset)) = 0; } } @@ -1901,8 +1749,7 @@ void FEA_Module_SGH::init_assembly() Graph_Matrix = RaggedRightArrayKokkos(Graph_Matrix_Strides); FOR_ALL_CLASS(inode, 0, nlocal_nodes, { - for (int istride = 0; istride < Graph_Matrix_Strides(inode); istride++) - { + for (int istride = 0; istride < Graph_Matrix_Strides(inode); istride++) { Graph_Matrix(inode, istride) = Repeat_Graph_Matrix(inode, istride); } }); // end parallel for @@ -1921,8 +1768,7 @@ void FEA_Module_SGH::init_assembly() Gradient_Matrix_Strides.update_host(); // build inverse map for element gradient assembly - for (size_t elem_gid = 0; elem_gid < rnum_elem; elem_gid++) - { + for (size_t elem_gid = 0; elem_gid < rnum_elem; elem_gid++) { FOR_ALL_CLASS(node_lid, 0, num_nodes_in_elem, { // get the global_id of the node size_t node_gid = nodes_in_elem(elem_gid, node_lid); @@ -1945,8 +1791,7 @@ void FEA_Module_SGH::init_assembly() Kokkos::View node_to_elem_strides = elems_in_node.mystrides_; DOF_to_Elem_Matrix_Strides = DCArrayKokkos(nlocal_nodes * num_dim); FOR_ALL_CLASS(inode, 0, nlocal_nodes, { - for (int idim = 0; idim < num_dim; idim++) - { + for (int idim = 0; idim < num_dim; idim++) { DOF_to_Elem_Matrix_Strides(inode * num_dim + idim) = count_saved_corners_in_node(inode); } }); // end parallel for @@ -1960,8 +1805,7 @@ void FEA_Module_SGH::init_assembly() // debug print // std::cout << "DOF GRAPH MATRIX ENTRIES ON TASK " << myrank << std::endl; FOR_ALL_CLASS(idof, 0, num_dim * nlocal_nodes, { - for (int istride = 0; istride < Gradient_Matrix_Strides(idof); istride++) - { + for (int istride = 0; istride < Gradient_Matrix_Strides(idof); istride++) { DOF_Graph_Matrix(idof, istride) = Graph_Matrix(idof / num_dim, istride / num_dim) * num_dim + istride % num_dim; } }); // end parallel for @@ -2040,13 +1884,11 @@ void FEA_Module_SGH::boundary_adjoint(const mesh_t& mesh, const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; int num_dims = simparam->num_dims; // Loop over boundary sets - for (size_t bdy_set = 0; bdy_set < num_bdy_sets; bdy_set++) - { + for (size_t bdy_set = 0; bdy_set < num_bdy_sets; bdy_set++) { // Loop over boundary nodes in a boundary set FOR_ALL_CLASS(bdy_node_lid, 0, num_bdy_nodes_in_set.host(bdy_set), { // reflected (boundary array is on the device) - if (boundary(bdy_set).type == BOUNDARY_CONDITION_TYPE::reflected) - { + if (boundary(bdy_set).type == BOUNDARY_CONDITION_TYPE::reflected) { // directions with hydro_bc: // x_plane = 0, // y_plane = 1, @@ -2056,25 +1898,21 @@ void FEA_Module_SGH::boundary_adjoint(const mesh_t& mesh, size_t bdy_node_gid = bdy_nodes_in_set(bdy_set, bdy_node_lid); // Set velocity to zero in that directdion - if (bdy_node_gid < nlocal_nodes) - { + if (bdy_node_gid < nlocal_nodes) { node_adjoint(bdy_node_gid, direction) = 0.0; node_phi_adjoint(bdy_node_gid, direction) = 0.0; } // node_phi_adjoint(bdy_node_gid, direction) = 0.0; } - else if (boundary(bdy_set).type == BOUNDARY_CONDITION_TYPE::fixed_position) - { + else if (boundary(bdy_set).type == BOUNDARY_CONDITION_TYPE::fixed_position) { size_t bdy_node_gid = bdy_nodes_in_set(bdy_set, bdy_node_lid); // debug clause // if(bdy_node_gid==549412) print_flag(0) = true; - for (size_t dim = 0; dim < num_dims; dim++) - { + for (size_t dim = 0; dim < num_dims; dim++) { // Set velocity to zero - if (bdy_node_gid < nlocal_nodes) - { + if (bdy_node_gid < nlocal_nodes) { node_adjoint(bdy_node_gid, dim) = 0.0; node_phi_adjoint(bdy_node_gid, dim) = 0.0; } diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/time_integration.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/time_integration.cpp index dd1d74577..c07a05eb1 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/time_integration.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/time_integration.cpp @@ -61,10 +61,8 @@ void FEA_Module_SGH::rk_init(DViewCArrayKokkos& node_coords, // save elem quantities FOR_ALL_CLASS(elem_gid, 0, num_elems, { // stress is always 3D even with 2D-RZ - for (size_t i = 0; i < 3; i++) - { - for (size_t j = 0; j < 3; j++) - { + for (size_t i = 0; i < 3; i++) { + for (size_t j = 0; j < 3; j++) { elem_stress(0, elem_gid, i, j) = elem_stress(rk_level, elem_gid, i, j); } } // end for @@ -74,8 +72,7 @@ void FEA_Module_SGH::rk_init(DViewCArrayKokkos& node_coords, // save nodal quantities FOR_ALL_CLASS(node_gid, 0, num_nodes, { - for (size_t i = 0; i < num_dims; i++) - { + for (size_t i = 0; i < num_dims; i++) { node_coords(0, node_gid, i) = node_coords(rk_level, node_gid, i); node_vel(0, node_gid, i) = node_vel(rk_level, node_gid, i); } @@ -122,10 +119,8 @@ void FEA_Module_SGH::get_timestep(mesh_t& mesh, ViewCArrayKokkos dist(distance0, 28); // Getting the coordinates of the element - for (size_t node_lid = 0; node_lid < 8; node_lid++) - { - for (size_t dim = 0; dim < num_dims; dim++) - { + for (size_t node_lid = 0; node_lid < 8; node_lid++) { + for (size_t dim = 0; dim < num_dims; dim++) { coords(node_lid, dim) = node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), dim); } // end for dim } // end for loop over node_lid @@ -139,8 +134,7 @@ void FEA_Module_SGH::get_timestep(mesh_t& mesh, // Only works for 3D // Solving for the magnitude of distance between each node - for (size_t i = 0; i < 28; i++) - { + for (size_t i = 0; i < 28; i++) { a = countA; b = countB; @@ -153,8 +147,7 @@ void FEA_Module_SGH::get_timestep(mesh_t& mesh, countA++; // tricky indexing - if (countB > 7) - { + if (countB > 7) { loop++; countB = 1 + loop; countA = 0; @@ -163,8 +156,7 @@ void FEA_Module_SGH::get_timestep(mesh_t& mesh, double dist_min = dist(0); - for (int i = 0; i < 28; ++i) - { + for (int i = 0; i < 28; ++i) { dist_min = fmin(dist(i), dist_min); } @@ -175,16 +167,14 @@ void FEA_Module_SGH::get_timestep(mesh_t& mesh, dt_lcl_ = fmin(dt_lcl_, dt_max); // make dt small than dt_max dt_lcl_ = fmax(dt_lcl_, dt_min); // make dt larger than dt_min - if (dt_lcl_ < dt_lcl) - { + if (dt_lcl_ < dt_lcl) { dt_lcl = dt_lcl_; } }, min_dt_calc); // end parallel reduction Kokkos::fence(); // save the min dt - if (min_dt_calc < dt) - { + if (min_dt_calc < dt) { dt = min_dt_calc; } @@ -235,10 +225,8 @@ void FEA_Module_SGH::get_timestep2D(mesh_t& mesh, ViewCArrayKokkos dist(distance0, 6); // Getting the coordinates of the nodes of the element - for (size_t node_lid = 0; node_lid < 4; node_lid++) - { - for (size_t dim = 0; dim < num_dims; dim++) - { + for (size_t node_lid = 0; node_lid < 4; node_lid++) { + for (size_t dim = 0; dim < num_dims; dim++) { coords(node_lid, dim) = node_coords(rk_level, nodes_in_elem(elem_gid, node_lid), dim); } // end for dim } // end for loop over node_lid @@ -246,10 +234,8 @@ void FEA_Module_SGH::get_timestep2D(mesh_t& mesh, // Only works for 2D // Solving for the magnitude of distance between each node size_t count = 0; - for (size_t i = 0; i < 3; i++) - { - for (size_t j = i + 1; j <= 3; j++) - { + for (size_t i = 0; i < 3; i++) { + for (size_t j = i + 1; j <= 3; j++) { // returns magnitude of distance between each node, 6 total options dist(count) = fabs( sqrt(pow((coords(i, 0) - coords(j, 0)), 2.0) @@ -261,8 +247,7 @@ void FEA_Module_SGH::get_timestep2D(mesh_t& mesh, double dist_min = dist(0); - for (int i = 0; i < 6; ++i) - { + for (int i = 0; i < 6; ++i) { dist_min = fmin(dist(i), dist_min); } @@ -273,16 +258,14 @@ void FEA_Module_SGH::get_timestep2D(mesh_t& mesh, dt_lcl_ = fmin(dt_lcl_, dt_max); // make dt small than dt_max dt_lcl_ = fmax(dt_lcl_, dt_min); // make dt larger than dt_min - if (dt_lcl_ < dt_lcl) - { + if (dt_lcl_ < dt_lcl) { dt_lcl = dt_lcl_; } }, min_dt_calc); // end parallel reduction Kokkos::fence(); // save the min dt - if (min_dt_calc < dt) - { + if (min_dt_calc < dt) { dt = min_dt_calc; } diff --git a/src/Parallel-Solvers/Parallel-Explicit/Topology_Optimization/Area_Normals.h b/src/Parallel-Solvers/Parallel-Explicit/Topology_Optimization/Area_Normals.h index 4255258b5..318e2eaf4 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Topology_Optimization/Area_Normals.h +++ b/src/Parallel-Solvers/Parallel-Explicit/Topology_Optimization/Area_Normals.h @@ -145,17 +145,13 @@ typedef MV::dual_view_type dual_vec_array; nvalid_modules = valid_fea_modules.size(); const Simulation_Parameters& simparam = Explicit_Solver_Pointer_->simparam; - for (const auto& fea_module : Explicit_Solver_Pointer_->fea_modules) - { - for (int ivalid = 0; ivalid < nvalid_modules; ivalid++) - { - if (fea_module->Module_Type == FEA_MODULE_TYPE::SGH) - { + for (const auto& fea_module : Explicit_Solver_Pointer_->fea_modules) { + for (int ivalid = 0; ivalid < nvalid_modules; ivalid++) { + if (fea_module->Module_Type == FEA_MODULE_TYPE::SGH) { FEM_SGH_ = dynamic_cast(fea_module); set_module_type = FEA_MODULE_TYPE::SGH; } - if (fea_module->Module_Type == FEA_MODULE_TYPE::Dynamic_Elasticity) - { + if (fea_module->Module_Type == FEA_MODULE_TYPE::Dynamic_Elasticity) { FEM_Dynamic_Elasticity_ = dynamic_cast(fea_module); set_module_type = FEA_MODULE_TYPE::Dynamic_Elasticity; } @@ -208,8 +204,7 @@ typedef MV::dual_view_type dual_vec_array; size_t nodes_in_elem_array[8]; double node_coords_array[2 * 8 * 3]; ViewCArrayKokkos elem_node_gids(nodes_in_elem_array, 8); - for (int init = 0; init < 8; init++) - { + for (int init = 0; init < 8; init++) { elem_node_gids(init) = init; } ViewCArrayKokkos area_normal(area_normal_array, 8, 3); @@ -219,8 +214,7 @@ typedef MV::dual_view_type dual_vec_array; const_host_vec_array design_coordinates = zp->getLocalView(Tpetra::Access::ReadOnly); DViewCArrayKokkos node_coords(node_coords_array, 2, 8, 3); - for (int inode = 0; inode < 8; inode++) - { + for (int inode = 0; inode < 8; inode++) { size_t local_node_id = FEM_SGH_->nodes_in_elem(0, inode); node_coords(1, inode, 0) = design_coordinates(local_node_id, 0); node_coords(1, inode, 1) = design_coordinates(local_node_id, 1); @@ -260,8 +254,7 @@ typedef MV::dual_view_type dual_vec_array; size_t nodes_in_elem[8]; double node_coords_array[2 * 8 * 3]; ViewCArrayKokkos elem_node_gids(nodes_in_elem, 8); - for (int init = 0; init < 8; init++) - { + for (int init = 0; init < 8; init++) { elem_node_gids(init) = init; } @@ -272,16 +265,14 @@ typedef MV::dual_view_type dual_vec_array; const_host_vec_array design_coordinates = zp->getLocalView(Tpetra::Access::ReadOnly); host_vec_array design_gradients = gp->getLocalView(Tpetra::Access::ReadWrite); DViewCArrayKokkos node_coords(node_coords_array, 2, 8, 3); - for (int inode = 0; inode < 8; inode++) - { + for (int inode = 0; inode < 8; inode++) { size_t local_node_id = FEM_SGH_->nodes_in_elem(0, inode); node_coords(1, inode, 0) = design_coordinates(local_node_id, 0); node_coords(1, inode, 1) = design_coordinates(local_node_id, 1); node_coords(1, inode, 2) = design_coordinates(local_node_id, 2); } - for (int igradient = 0; igradient < FEM_SGH_->nlocal_nodes + FEM_SGH_->nghost_nodes; igradient++) - { + for (int igradient = 0; igradient < FEM_SGH_->nlocal_nodes + FEM_SGH_->nghost_nodes; igradient++) { design_gradients(igradient, 0) = 0; design_gradients(igradient, 1) = 0; design_gradients(igradient, 2) = 0; @@ -293,11 +284,9 @@ typedef MV::dual_view_type dual_vec_array; elem_node_gids, 1); - for (int inode = 0; inode < 8; inode++) - { + for (int inode = 0; inode < 8; inode++) { size_t local_node_id = FEM_SGH_->nodes_in_elem(0, inode); - for (int idim = 0; idim < 3; idim++) - { + for (int idim = 0; idim < 3; idim++) { design_gradients(local_node_id, idim) = -area_normal_gradients(set_node_, set_dim_, inode, idim); } } diff --git a/src/Parallel-Solvers/Parallel-Explicit/Topology_Optimization/Kinetic_Energy_Minimize.h b/src/Parallel-Solvers/Parallel-Explicit/Topology_Optimization/Kinetic_Energy_Minimize.h index da79ecbf4..5a96ac489 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Topology_Optimization/Kinetic_Energy_Minimize.h +++ b/src/Parallel-Solvers/Parallel-Explicit/Topology_Optimization/Kinetic_Energy_Minimize.h @@ -148,17 +148,13 @@ typedef MV::dual_view_type dual_vec_array; nvalid_modules = valid_fea_modules.size(); const Simulation_Parameters& simparam = Explicit_Solver_Pointer_->simparam; - for (const auto& fea_module : Explicit_Solver_Pointer_->fea_modules) - { - for (int ivalid = 0; ivalid < nvalid_modules; ivalid++) - { - if (fea_module->Module_Type == FEA_MODULE_TYPE::SGH) - { + for (const auto& fea_module : Explicit_Solver_Pointer_->fea_modules) { + for (int ivalid = 0; ivalid < nvalid_modules; ivalid++) { + if (fea_module->Module_Type == FEA_MODULE_TYPE::SGH) { FEM_SGH_ = dynamic_cast(fea_module); set_module_type = FEA_MODULE_TYPE::SGH; } - if (fea_module->Module_Type == FEA_MODULE_TYPE::Dynamic_Elasticity) - { + if (fea_module->Module_Type == FEA_MODULE_TYPE::Dynamic_Elasticity) { FEM_Dynamic_Elasticity_ = dynamic_cast(fea_module); set_module_type = FEA_MODULE_TYPE::Dynamic_Elasticity; } @@ -171,12 +167,10 @@ typedef MV::dual_view_type dual_vec_array; objective_accumulation = 0; // ROL_Force = ROL::makePtr(FEM_->Global_Nodal_Forces); - if (set_module_type == FEA_MODULE_TYPE::SGH) - { + if (set_module_type == FEA_MODULE_TYPE::SGH) { ROL_Velocities = ROL::makePtr(FEM_SGH_->node_velocities_distributed); } - if (set_module_type == FEA_MODULE_TYPE::Dynamic_Elasticity) - { + if (set_module_type == FEA_MODULE_TYPE::Dynamic_Elasticity) { ROL_Velocities = ROL::makePtr(FEM_Dynamic_Elasticity_->node_velocities_distributed); } } @@ -194,12 +188,10 @@ typedef MV::dual_view_type dual_vec_array; ///////////////////////////////////////////////////////////////////////////// void update(const ROL::Vector& z, ROL::UpdateType type, int iter = -1) { - if (set_module_type == FEA_MODULE_TYPE::SGH) - { + if (set_module_type == FEA_MODULE_TYPE::SGH) { update_sgh(z, type, iter); } - if (set_module_type == FEA_MODULE_TYPE::Dynamic_Elasticity) - { + if (set_module_type == FEA_MODULE_TYPE::Dynamic_Elasticity) { update_elasticity(z, type, iter); } } @@ -225,8 +217,7 @@ typedef MV::dual_view_type dual_vec_array; ROL::Ptr zp = getVector(z); const_host_vec_array design_densities = zp->getLocalView(Tpetra::Access::ReadOnly); - if (type == ROL::UpdateType::Initial) - { + if (type == ROL::UpdateType::Initial) { // This is the first call to update // first linear solve was done in FEA class run function already FEM_Dynamic_Elasticity_->comm_variables(zp); @@ -236,11 +227,9 @@ typedef MV::dual_view_type dual_vec_array; // decide to output current optimization state FEM_Dynamic_Elasticity_->Explicit_Solver_Pointer_->write_outputs(); } - else if (type == ROL::UpdateType::Accept) - { + else if (type == ROL::UpdateType::Accept) { } - else if (type == ROL::UpdateType::Revert) - { + else if (type == ROL::UpdateType::Revert) { // u_ was set to u=S(x) during a trial update // and has been rejected as the new iterate // Revert to cached value @@ -249,32 +238,27 @@ typedef MV::dual_view_type dual_vec_array; FEM_Dynamic_Elasticity_->comm_variables(zp); // update deformation variables FEM_Dynamic_Elasticity_->update_forward_solve(zp); - if (Explicit_Solver_Pointer_->myrank == 0) - { + if (Explicit_Solver_Pointer_->myrank == 0) { *fos << "called Revert" << std::endl; } } - else if (type == ROL::UpdateType::Trial) - { + else if (type == ROL::UpdateType::Trial) { // This is a new value of x // communicate density variables for ghosts FEM_Dynamic_Elasticity_->comm_variables(zp); // update deformation variables FEM_Dynamic_Elasticity_->update_forward_solve(zp); - if (Explicit_Solver_Pointer_->myrank == 0) - { + if (Explicit_Solver_Pointer_->myrank == 0) { *fos << "called Trial" << std::endl; } // decide to output current optimization state FEM_Dynamic_Elasticity_->Explicit_Solver_Pointer_->write_outputs(); } - else // ROL::UpdateType::Temp - // This is a new value of x used for, - // e.g., finite-difference checks - { - if (Explicit_Solver_Pointer_->myrank == 0) - { + else{ // ROL::UpdateType::Temp + // This is a new value of x used for, + // e.g., finite-difference checks + if (Explicit_Solver_Pointer_->myrank == 0) { *fos << "called Temp" << std::endl; } FEM_Dynamic_Elasticity_->comm_variables(zp); @@ -303,11 +287,9 @@ typedef MV::dual_view_type dual_vec_array; ROL::Ptr zp = getVector(z); const_host_vec_array design_densities = zp->getLocalView(Tpetra::Access::ReadOnly); - if (type == ROL::UpdateType::Initial) - { + if (type == ROL::UpdateType::Initial) { // This is the first call to update - if (Explicit_Solver_Pointer_->myrank == 0) - { + if (Explicit_Solver_Pointer_->myrank == 0) { *fos << "called SGH Initial" << std::endl; } @@ -317,11 +299,9 @@ typedef MV::dual_view_type dual_vec_array; // decide to output current optimization state // FEM_SGH_->Explicit_Solver_Pointer_->write_outputs(); } - else if (type == ROL::UpdateType::Accept) - { + else if (type == ROL::UpdateType::Accept) { } - else if (type == ROL::UpdateType::Revert) - { + else if (type == ROL::UpdateType::Revert) { // u_ was set to u=S(x) during a trial update // and has been rejected as the new iterate // Revert to cached value @@ -332,32 +312,27 @@ typedef MV::dual_view_type dual_vec_array; FEM_SGH_->comm_variables(zp); // update deformation variables FEM_SGH_->update_forward_solve(zp); - if (Explicit_Solver_Pointer_->myrank == 0) - { + if (Explicit_Solver_Pointer_->myrank == 0) { *fos << "called Revert" << std::endl; } } - else if (type == ROL::UpdateType::Trial) - { + else if (type == ROL::UpdateType::Trial) { // This is a new value of x // communicate density variables for ghosts FEM_SGH_->comm_variables(zp); // update deformation variables FEM_SGH_->update_forward_solve(zp); - if (Explicit_Solver_Pointer_->myrank == 0) - { + if (Explicit_Solver_Pointer_->myrank == 0) { *fos << "called Trial" << std::endl; } // decide to output current optimization state // FEM_SGH_->Explicit_Solver_Pointer_->write_outputs(); } - else // ROL::UpdateType::Temp - // This is a new value of x used for, - // e.g., finite-difference checks - { - if (Explicit_Solver_Pointer_->myrank == 0) - { + else{ // ROL::UpdateType::Temp + // This is a new value of x used for, + // e.g., finite-difference checks + if (Explicit_Solver_Pointer_->myrank == 0) { *fos << "called SGH Temp" << std::endl; } FEM_SGH_->comm_variables(zp); @@ -417,18 +392,15 @@ typedef MV::dual_view_type dual_vec_array; // std::fflush(stdout); // ROL_Force = ROL::makePtr(FEM_->Global_Nodal_Forces); - if (set_module_type == FEA_MODULE_TYPE::SGH) - { + if (set_module_type == FEA_MODULE_TYPE::SGH) { ROL_Velocities = ROL::makePtr(FEM_SGH_->node_velocities_distributed); } - if (set_module_type == FEA_MODULE_TYPE::Dynamic_Elasticity) - { + if (set_module_type == FEA_MODULE_TYPE::Dynamic_Elasticity) { ROL_Velocities = ROL::makePtr(FEM_Dynamic_Elasticity_->node_velocities_distributed); } std::cout.precision(10); - if (Explicit_Solver_Pointer_->myrank == 0) - { + if (Explicit_Solver_Pointer_->myrank == 0) { std::cout << "CURRENT TIME INTEGRAL OF KINETIC ENERGY " << objective_accumulation << std::endl; } @@ -459,12 +431,10 @@ typedef MV::dual_view_type dual_vec_array; // FEM_->gradient_print_sync=0; // get local view of the data - if (set_module_type == FEA_MODULE_TYPE::SGH) - { + if (set_module_type == FEA_MODULE_TYPE::SGH) { FEM_SGH_->compute_topology_optimization_gradient_full(zp, gp); } - if (set_module_type == FEA_MODULE_TYPE::Dynamic_Elasticity) - { + if (set_module_type == FEA_MODULE_TYPE::Dynamic_Elasticity) { FEM_Dynamic_Elasticity_->compute_topology_optimization_gradient_full(zp, gp); } // debug print of gradient From c718c5f19de90f490f68a65233a5a671ffad62bd Mon Sep 17 00:00:00 2001 From: Adrian-Diaz Date: Sat, 9 Mar 2024 13:16:21 -0700 Subject: [PATCH 62/64] STYLE: Uncrustify commits --- .../SGH_Solver/src/force_gradients_sgh.cpp | 107 ++++++++---------- .../SGH_Solver/src/power_gradients_sgh.cpp | 3 +- 2 files changed, 47 insertions(+), 63 deletions(-) diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp index 344e8a75b..23131fcb1 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp @@ -134,7 +134,7 @@ void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& ma double vel_star_array[3]; double vel_star_gradient_array[num_dims * max_nodes_per_element * num_dims]; - //mag_vel gradient + // mag_vel gradient double mag_vel_gradient_array[max_nodes_per_element * num_dims]; // velocity gradient @@ -232,9 +232,9 @@ void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& ma ViewCArrayKokkos vel(&node_vel(rk_level, node_gid, 0), num_dims); - vel_star(0) += vel(0)/((real_t) num_nodes_in_elem); - vel_star(1) += vel(1)/((real_t) num_nodes_in_elem); - vel_star(2) += vel(2)/((real_t) num_nodes_in_elem); + vel_star(0) += vel(0) / ((real_t) num_nodes_in_elem); + vel_star(1) += vel(1) / ((real_t) num_nodes_in_elem); + vel_star(2) += vel(2) / ((real_t) num_nodes_in_elem); } // end for loop over nodes // find shock direction and shock impedance associated with each node @@ -243,12 +243,9 @@ void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& ma for (int i = 0; i < 4; i++) { sum(i) = 0.0; } - for (int i = 0; i < 4; i++) - { - for (size_t igradient = 0; igradient < num_nodes_in_elem; igradient++) - { - for (size_t jdim = 0; jdim < num_dims; jdim++) - { + for (int i = 0; i < 4; i++) { + for (size_t igradient = 0; igradient < num_nodes_in_elem; igradient++) { + for (size_t jdim = 0; jdim < num_dims; jdim++) { sum_gradient(i, igradient, jdim) = 0.0; } } @@ -270,29 +267,24 @@ void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& ma + (vel(1) - vel_star(1) ) * (vel(1) - vel_star(1) ) + (vel(2) - vel_star(2) ) * (vel(2) - vel_star(2) ) ); - for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) - { - for(int jdim = 0; jdim < num_dim; jdim++) - { - if(igradient == node_lid) - { - mag_vel_gradient(igradient,jdim) = (1.0 - 1.0/((real_t) num_nodes_in_elem))*((vel(jdim) - vel_star(jdim) ))/mag_vel; - if(mag_vel < fuzz){ - mag_vel_gradient(igradient,jdim) = (1.0 - 1.0/((real_t) num_nodes_in_elem)); + for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) { + for (int jdim = 0; jdim < num_dim; jdim++) { + if (igradient == node_lid) { + mag_vel_gradient(igradient, jdim) = (1.0 - 1.0 / ((real_t) num_nodes_in_elem)) * ((vel(jdim) - vel_star(jdim) )) / mag_vel; + if (mag_vel < fuzz) { + mag_vel_gradient(igradient, jdim) = (1.0 - 1.0 / ((real_t) num_nodes_in_elem)); } } - else - { - mag_vel_gradient(igradient,jdim) = -1.0/((real_t) num_nodes_in_elem)*( (vel(jdim) - vel_star(jdim) ))/mag_vel; - if(mag_vel < fuzz){ - mag_vel_gradient(igradient,jdim) = 1.0/((real_t) num_nodes_in_elem); + else{ + mag_vel_gradient(igradient, jdim) = -1.0 / ((real_t) num_nodes_in_elem) * ( (vel(jdim) - vel_star(jdim) )) / mag_vel; + if (mag_vel < fuzz) { + mag_vel_gradient(igradient, jdim) = 1.0 / ((real_t) num_nodes_in_elem); } } } } if (mag_vel > small) { - // estimate of the shock direction, a unit normal for (int dim = 0; dim < num_dims; dim++) { shock_dir(dim) = (vel(dim) - vel_star(dim)) / mag_vel; @@ -317,24 +309,20 @@ void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& ma muc(node_lid) = elem_den(elem_gid) * (material(mat_id).q1 * elem_sspd(elem_gid) + material(mat_id).q2 * mag_vel); - for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) - { - for(int jdim = 0; jdim < num_dim; jdim++) - { - muc_gradient(node_lid, igradient, jdim) = material(mat_id).q2 * mag_vel_gradient(igradient,jdim); - //if(muc_gradient(node_lid, igradient, jdim)!=0) std::cout << " NON ZERO " << muc_gradient(node_lid, igradient, jdim) << std::endl; + for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) { + for (int jdim = 0; jdim < num_dim; jdim++) { + muc_gradient(node_lid, igradient, jdim) = material(mat_id).q2 * mag_vel_gradient(igradient, jdim); + // if(muc_gradient(node_lid, igradient, jdim)!=0) std::cout << " NON ZERO " << muc_gradient(node_lid, igradient, jdim) << std::endl; } } } else{ // element in expansion muc(node_lid) = elem_den(elem_gid) * (material(mat_id).q1ex * elem_sspd(elem_gid) + material(mat_id).q2ex * mag_vel); - for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) - { - for(int jdim = 0; jdim < num_dim; jdim++) - { - muc_gradient(node_lid, igradient, jdim) = material(mat_id).q2ex * mag_vel_gradient(igradient,jdim); - //if(muc_gradient(node_lid, igradient, jdim)!=0) std::cout << " NON ZERO " << muc_gradient(node_lid, igradient, jdim) << std::endl; + for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) { + for (int jdim = 0; jdim < num_dim; jdim++) { + muc_gradient(node_lid, igradient, jdim) = material(mat_id).q2ex * mag_vel_gradient(igradient, jdim); + // if(muc_gradient(node_lid, igradient, jdim)!=0) std::cout << " NON ZERO " << muc_gradient(node_lid, igradient, jdim) << std::endl; } } } // end if on divergence sign @@ -357,17 +345,15 @@ void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& ma // Using a full tensoral Riemann jump relation mu_term = muc(node_lid) * sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) - + area_normal(node_lid, 1) * area_normal(node_lid, 1) - + area_normal(node_lid, 2) * area_normal(node_lid, 2) ); + + area_normal(node_lid, 1) * area_normal(node_lid, 1) + + area_normal(node_lid, 2) * area_normal(node_lid, 2) ); - for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) - { - for (int jdim = 0; jdim < num_dims; jdim++) - { + for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) { + for (int jdim = 0; jdim < num_dims; jdim++) { muc_gradient(node_lid, igradient, jdim) = muc_gradient(node_lid, igradient, jdim) - * sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) - + area_normal(node_lid, 1) * area_normal(node_lid, 1) - + area_normal(node_lid, 2) * area_normal(node_lid, 2) ); + * sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) + + area_normal(node_lid, 1) * area_normal(node_lid, 1) + + area_normal(node_lid, 2) * area_normal(node_lid, 2) ); } } } @@ -379,26 +365,26 @@ void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& ma // sum gradients muc(node_lid) = mu_term; // the impeadance time surface area is stored here - for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) - { - for (int jdim = 0; jdim < num_dims; jdim++) - { - if(igradient==node_lid&&jdim==0) + for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) { + for (int jdim = 0; jdim < num_dims; jdim++) { + if (igradient == node_lid && jdim == 0) { sum_gradient(0, igradient, jdim) += muc(node_lid); + } - if(igradient==node_lid&&jdim==1) + if (igradient == node_lid && jdim == 1) { sum_gradient(1, igradient, jdim) += muc(node_lid); + } - if(igradient==node_lid&&jdim==2) + if (igradient == node_lid && jdim == 2) { sum_gradient(2, igradient, jdim) += muc(node_lid); - + } + sum_gradient(0, igradient, jdim) += muc_gradient(node_lid, igradient, jdim) * vel(0); sum_gradient(1, igradient, jdim) += muc_gradient(node_lid, igradient, jdim) * vel(1); sum_gradient(2, igradient, jdim) += muc_gradient(node_lid, igradient, jdim) * vel(2); sum_gradient(3, igradient, jdim) += muc_gradient(node_lid, igradient, jdim); } } - } // end for node_lid loop over nodes of the elem // The Riemann velocity, called vel_star @@ -429,7 +415,6 @@ void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& ma for (int jdim = 0; jdim < num_dims; jdim++) { vel_star_gradient(i, igradient, jdim) = 0; } - } } } // end if @@ -512,19 +497,19 @@ void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& ma column_index = num_dims * Global_Gradient_Matrix_Assembly_Map(elem_gid, igradient, node_lid); for (int jdim = 0; jdim < num_dims; jdim++) { if (node_lid == igradient && jdim == dim) { - if (map->isNodeLocalElement(gradient_node_gid)) { //node_vel(rk_level, node_gid, dim) - Force_Gradient_Velocities(gradient_node_gid * num_dims + jdim, column_index + dim) += phi * (muc(node_lid) * (vel_star_gradient(dim, igradient, jdim) - 1) + + if (map->isNodeLocalElement(gradient_node_gid)) { + Force_Gradient_Velocities(gradient_node_gid * num_dims + jdim, column_index + dim) += phi * (muc(node_lid) * (vel_star_gradient(dim, igradient, jdim) - 1) + muc_gradient(node_lid, igradient, jdim) * (vel_star(dim) - node_vel(rk_level, node_gid, dim))); } - corner_gradient_storage(corner_gid, dim, igradient, jdim) = phi * (muc(node_lid) * (vel_star_gradient(dim, igradient, jdim) - 1) + + corner_gradient_storage(corner_gid, dim, igradient, jdim) = phi * (muc(node_lid) * (vel_star_gradient(dim, igradient, jdim) - 1) + muc_gradient(node_lid, igradient, jdim) * (vel_star(dim) - node_vel(rk_level, node_gid, dim))); } else { if (map->isNodeLocalElement(gradient_node_gid)) { - Force_Gradient_Velocities(gradient_node_gid * num_dims + jdim, column_index + dim) += phi * (muc(node_lid) * (vel_star_gradient(dim, igradient, jdim))+ + Force_Gradient_Velocities(gradient_node_gid * num_dims + jdim, column_index + dim) += phi * (muc(node_lid) * (vel_star_gradient(dim, igradient, jdim)) + muc_gradient(node_lid, igradient, jdim) * (vel_star(dim) - node_vel(rk_level, node_gid, dim))); } - corner_gradient_storage(corner_gid, dim, igradient, jdim) = phi * (muc(node_lid) * (vel_star_gradient(dim, igradient, jdim))+ + corner_gradient_storage(corner_gid, dim, igradient, jdim) = phi * (muc(node_lid) * (vel_star_gradient(dim, igradient, jdim)) + muc_gradient(node_lid, igradient, jdim) * (vel_star(dim) - node_vel(rk_level, node_gid, dim))); } } diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/power_gradients_sgh.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/power_gradients_sgh.cpp index 7642d2723..2243f8004 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/power_gradients_sgh.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/power_gradients_sgh.cpp @@ -455,7 +455,7 @@ void FEA_Module_SGH::get_power_vgradient_sgh(double rk_alpha, continue; } for (size_t jdim = 0; jdim < num_dims; jdim++) { - if (node_lid == igradient&&jdim==dim) { + if (node_lid == igradient && jdim == dim) { Power_Gradient_Velocities(gradient_node_id * num_dims + jdim, column_id) -= corner_gradient_storage(corner_gid, dim, igradient, jdim) * node_vel(rk_level, node_gid, dim) * node_radius + corner_force(corner_gid, dim) * node_radius; @@ -464,7 +464,6 @@ void FEA_Module_SGH::get_power_vgradient_sgh(double rk_alpha, Power_Gradient_Velocities(gradient_node_id * num_dims + jdim, column_id) -= corner_gradient_storage(corner_gid, dim, igradient, jdim) * node_vel(rk_level, node_gid, dim) * node_radius; } - } } } // end for dim From 5fb3acca14fec2abbe434523a820637889e13f54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CAdrian-Diaz=E2=80=9D?= <“adriandiaz1117@gmail.com”> Date: Wed, 13 Mar 2024 11:45:44 -0600 Subject: [PATCH 63/64] BUG: q2 gradient term fix, remove spurious yaml directory --- .../SGH_Solver/src/force_gradients_sgh.cpp | 42 ++++++++++++++----- .../example_sgh_optimization.yaml | 2 +- .../Parallel-Explicit/example_simple.yaml | 2 +- 3 files changed, 33 insertions(+), 13 deletions(-) rename {Parallel-Solvers => src/Parallel-Solvers}/Parallel-Explicit/example_sgh_optimization.yaml (99%) diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp index 23131fcb1..bf5fa73c0 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp @@ -272,13 +272,33 @@ void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& ma if (igradient == node_lid) { mag_vel_gradient(igradient, jdim) = (1.0 - 1.0 / ((real_t) num_nodes_in_elem)) * ((vel(jdim) - vel_star(jdim) )) / mag_vel; if (mag_vel < fuzz) { - mag_vel_gradient(igradient, jdim) = (1.0 - 1.0 / ((real_t) num_nodes_in_elem)); + //if(mag_vel!=0) std::cout << " SMALL MAG VEL ENCOUNTERED " << std::endl; + if(vel(jdim)<0) { + mag_vel_gradient(igradient, jdim) = -(1.0 - 1.0 / ((real_t) num_nodes_in_elem)); + } + else if(vel(jdim)>0) { + mag_vel_gradient(igradient, jdim) = (1.0 - 1.0 / ((real_t) num_nodes_in_elem)); + } + else if(mag_vel==0) { + mag_vel_gradient(igradient, jdim) = 0; + } + } } - else{ + else { mag_vel_gradient(igradient, jdim) = -1.0 / ((real_t) num_nodes_in_elem) * ( (vel(jdim) - vel_star(jdim) )) / mag_vel; if (mag_vel < fuzz) { - mag_vel_gradient(igradient, jdim) = 1.0 / ((real_t) num_nodes_in_elem); + //if(mag_vel!=0) std::cout << " SMALL MAG VEL ENCOUNTERED " << std::endl; + if(vel(jdim)<0) { + mag_vel_gradient(igradient, jdim) = -1.0 / ((real_t) num_nodes_in_elem); + } + else if(vel(jdim)>0) { + mag_vel_gradient(igradient, jdim) = 1.0 / ((real_t) num_nodes_in_elem); + } + else if (mag_vel==0) { + mag_vel_gradient(igradient, jdim) = 0; + } + } } } @@ -311,7 +331,7 @@ void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& ma for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) { for (int jdim = 0; jdim < num_dim; jdim++) { - muc_gradient(node_lid, igradient, jdim) = material(mat_id).q2 * mag_vel_gradient(igradient, jdim); + muc_gradient(node_lid, igradient, jdim) = elem_den(elem_gid) * material(mat_id).q2 * mag_vel_gradient(igradient, jdim); // if(muc_gradient(node_lid, igradient, jdim)!=0) std::cout << " NON ZERO " << muc_gradient(node_lid, igradient, jdim) << std::endl; } } @@ -321,7 +341,7 @@ void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& ma (material(mat_id).q1ex * elem_sspd(elem_gid) + material(mat_id).q2ex * mag_vel); for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) { for (int jdim = 0; jdim < num_dim; jdim++) { - muc_gradient(node_lid, igradient, jdim) = material(mat_id).q2ex * mag_vel_gradient(igradient, jdim); + muc_gradient(node_lid, igradient, jdim) = elem_den(elem_gid) * material(mat_id).q2ex * mag_vel_gradient(igradient, jdim); // if(muc_gradient(node_lid, igradient, jdim)!=0) std::cout << " NON ZERO " << muc_gradient(node_lid, igradient, jdim) << std::endl; } } @@ -1262,11 +1282,11 @@ void FEA_Module_SGH::get_force_ugradient_sgh(const DCArrayKokkos& ma for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) { muc_gradient(node_lid, igradient, 0) = -elem_den(elem_gid) * volume_gradients(igradient, 0) / vol * - (material(mat_id).q1 * elem_sspd(elem_gid)); + (material(mat_id).q1 * elem_sspd(elem_gid)+ material(mat_id).q2 * mag_vel); muc_gradient(node_lid, igradient, 1) = -elem_den(elem_gid) * volume_gradients(igradient, 1) / vol * - (material(mat_id).q1 * elem_sspd(elem_gid)); + (material(mat_id).q1 * elem_sspd(elem_gid)+ material(mat_id).q2 * mag_vel); muc_gradient(node_lid, igradient, 2) = -elem_den(elem_gid) * volume_gradients(igradient, 2) / vol * - (material(mat_id).q1 * elem_sspd(elem_gid)); + (material(mat_id).q1 * elem_sspd(elem_gid)+ material(mat_id).q2 * mag_vel); } } else{ // element in expansion @@ -1274,11 +1294,11 @@ void FEA_Module_SGH::get_force_ugradient_sgh(const DCArrayKokkos& ma (material(mat_id).q1ex * elem_sspd(elem_gid) + material(mat_id).q2ex * mag_vel); for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) { muc_gradient(node_lid, igradient, 0) = -elem_den(elem_gid) * volume_gradients(igradient, 0) / vol * - (material(mat_id).q1ex * elem_sspd(elem_gid)); + (material(mat_id).q1ex * elem_sspd(elem_gid)+ material(mat_id).q2ex * mag_vel); muc_gradient(node_lid, igradient, 1) = -elem_den(elem_gid) * volume_gradients(igradient, 1) / vol * - (material(mat_id).q1ex * elem_sspd(elem_gid)); + (material(mat_id).q1ex * elem_sspd(elem_gid)+ material(mat_id).q2ex * mag_vel); muc_gradient(node_lid, igradient, 2) = -elem_den(elem_gid) * volume_gradients(igradient, 2) / vol * - (material(mat_id).q1ex * elem_sspd(elem_gid)); + (material(mat_id).q1ex * elem_sspd(elem_gid)+ material(mat_id).q2ex * mag_vel); } } // end if on divergence sign diff --git a/Parallel-Solvers/Parallel-Explicit/example_sgh_optimization.yaml b/src/Parallel-Solvers/Parallel-Explicit/example_sgh_optimization.yaml similarity index 99% rename from Parallel-Solvers/Parallel-Explicit/example_sgh_optimization.yaml rename to src/Parallel-Solvers/Parallel-Explicit/example_sgh_optimization.yaml index 35b64abc2..f8f0d76b9 100644 --- a/Parallel-Solvers/Parallel-Explicit/example_sgh_optimization.yaml +++ b/src/Parallel-Solvers/Parallel-Explicit/example_sgh_optimization.yaml @@ -68,7 +68,7 @@ materials: poisson_ratio: 0.3 maximum_limiter: true q1: 1.0 - q2: 0 + q2: 1.33 q1ex: 1.0 q2ex: 0.0 eos_global_vars: diff --git a/src/Parallel-Solvers/Parallel-Explicit/example_simple.yaml b/src/Parallel-Solvers/Parallel-Explicit/example_simple.yaml index 362b4568a..c5a48b9f2 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/example_simple.yaml +++ b/src/Parallel-Solvers/Parallel-Explicit/example_simple.yaml @@ -64,7 +64,7 @@ materials: elastic_modulus: 10 poisson_ratio: 0.3 q1: 1.0 - q2: 0 + q2: 1.33 q1ex: 1.0 q2ex: 0.0 eos_global_vars: From 23f28b1c1846c77dd49a6d2d257b6c4fd8add10d Mon Sep 17 00:00:00 2001 From: Adrian-Diaz <46537160+Adrian-Diaz@users.noreply.github.com> Date: Wed, 13 Mar 2024 15:06:44 -0600 Subject: [PATCH 64/64] STYLE: uncrustify changes --- .../SGH_Solver/src/force_gradients_sgh.cpp | 30 +++++++++---------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp index bf5fa73c0..567789239 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp @@ -272,33 +272,31 @@ void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& ma if (igradient == node_lid) { mag_vel_gradient(igradient, jdim) = (1.0 - 1.0 / ((real_t) num_nodes_in_elem)) * ((vel(jdim) - vel_star(jdim) )) / mag_vel; if (mag_vel < fuzz) { - //if(mag_vel!=0) std::cout << " SMALL MAG VEL ENCOUNTERED " << std::endl; - if(vel(jdim)<0) { + // if(mag_vel!=0) std::cout << " SMALL MAG VEL ENCOUNTERED " << std::endl; + if (vel(jdim) < 0) { mag_vel_gradient(igradient, jdim) = -(1.0 - 1.0 / ((real_t) num_nodes_in_elem)); } - else if(vel(jdim)>0) { + else if (vel(jdim) > 0) { mag_vel_gradient(igradient, jdim) = (1.0 - 1.0 / ((real_t) num_nodes_in_elem)); } - else if(mag_vel==0) { + else if (mag_vel == 0) { mag_vel_gradient(igradient, jdim) = 0; } - } } else { mag_vel_gradient(igradient, jdim) = -1.0 / ((real_t) num_nodes_in_elem) * ( (vel(jdim) - vel_star(jdim) )) / mag_vel; if (mag_vel < fuzz) { - //if(mag_vel!=0) std::cout << " SMALL MAG VEL ENCOUNTERED " << std::endl; - if(vel(jdim)<0) { + // if(mag_vel!=0) std::cout << " SMALL MAG VEL ENCOUNTERED " << std::endl; + if (vel(jdim) < 0) { mag_vel_gradient(igradient, jdim) = -1.0 / ((real_t) num_nodes_in_elem); } - else if(vel(jdim)>0) { + else if (vel(jdim) > 0) { mag_vel_gradient(igradient, jdim) = 1.0 / ((real_t) num_nodes_in_elem); } - else if (mag_vel==0) { + else if (mag_vel == 0) { mag_vel_gradient(igradient, jdim) = 0; } - } } } @@ -1282,11 +1280,11 @@ void FEA_Module_SGH::get_force_ugradient_sgh(const DCArrayKokkos& ma for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) { muc_gradient(node_lid, igradient, 0) = -elem_den(elem_gid) * volume_gradients(igradient, 0) / vol * - (material(mat_id).q1 * elem_sspd(elem_gid)+ material(mat_id).q2 * mag_vel); + (material(mat_id).q1 * elem_sspd(elem_gid) + material(mat_id).q2 * mag_vel); muc_gradient(node_lid, igradient, 1) = -elem_den(elem_gid) * volume_gradients(igradient, 1) / vol * - (material(mat_id).q1 * elem_sspd(elem_gid)+ material(mat_id).q2 * mag_vel); + (material(mat_id).q1 * elem_sspd(elem_gid) + material(mat_id).q2 * mag_vel); muc_gradient(node_lid, igradient, 2) = -elem_den(elem_gid) * volume_gradients(igradient, 2) / vol * - (material(mat_id).q1 * elem_sspd(elem_gid)+ material(mat_id).q2 * mag_vel); + (material(mat_id).q1 * elem_sspd(elem_gid) + material(mat_id).q2 * mag_vel); } } else{ // element in expansion @@ -1294,11 +1292,11 @@ void FEA_Module_SGH::get_force_ugradient_sgh(const DCArrayKokkos& ma (material(mat_id).q1ex * elem_sspd(elem_gid) + material(mat_id).q2ex * mag_vel); for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) { muc_gradient(node_lid, igradient, 0) = -elem_den(elem_gid) * volume_gradients(igradient, 0) / vol * - (material(mat_id).q1ex * elem_sspd(elem_gid)+ material(mat_id).q2ex * mag_vel); + (material(mat_id).q1ex * elem_sspd(elem_gid) + material(mat_id).q2ex * mag_vel); muc_gradient(node_lid, igradient, 1) = -elem_den(elem_gid) * volume_gradients(igradient, 1) / vol * - (material(mat_id).q1ex * elem_sspd(elem_gid)+ material(mat_id).q2ex * mag_vel); + (material(mat_id).q1ex * elem_sspd(elem_gid) + material(mat_id).q2ex * mag_vel); muc_gradient(node_lid, igradient, 2) = -elem_den(elem_gid) * volume_gradients(igradient, 2) / vol * - (material(mat_id).q1ex * elem_sspd(elem_gid)+ material(mat_id).q2ex * mag_vel); + (material(mat_id).q1ex * elem_sspd(elem_gid) + material(mat_id).q2ex * mag_vel); } } // end if on divergence sign