From 2d2985a0e10008341ae435c26bf06f4eaa882b0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CAdrian-Diaz=E2=80=9D?= <“adriandiaz1117@gmail.com”> Date: Mon, 24 Jun 2024 14:40:48 -0600 Subject: [PATCH 001/130] WIP: rework interface for Trilinos parameter list settings --- .../FEA_Physics_Modules/FEA_Module_Elasticity.cpp | 12 +++++++++--- .../FEA_Module/Elasticity_Parameters.h | 4 +++- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/Parallel-Solvers/Implicit-Lagrange/FEA_Physics_Modules/FEA_Module_Elasticity.cpp b/src/Parallel-Solvers/Implicit-Lagrange/FEA_Physics_Modules/FEA_Module_Elasticity.cpp index 046b5cb3b..3c8d856c0 100644 --- a/src/Parallel-Solvers/Implicit-Lagrange/FEA_Physics_Modules/FEA_Module_Elasticity.cpp +++ b/src/Parallel-Solvers/Implicit-Lagrange/FEA_Physics_Modules/FEA_Module_Elasticity.cpp @@ -6255,9 +6255,15 @@ void FEA_Module_Elasticity::linear_solver_parameters(){ } else{ Linear_Solve_Params = Teuchos::rcp(new Teuchos::ParameterList("MueLu")); - std::string xmlFileName = "elasticity3D.xml"; - //std::string xmlFileName = "simple_test.xml"; - Teuchos::updateParametersFromXmlFileAndBroadcast(xmlFileName, Teuchos::Ptr(&(*Linear_Solve_Params)), *comm); + if(module_params->muelu_parameters_xml_file){ + std::string xmlFileName = module_params->xml_parameters_file_name; + //std::string xmlFileName = "simple_test.xml"; + Teuchos::updateParametersFromXmlFileAndBroadcast(xmlFileName, Teuchos::Ptr(&(*Linear_Solve_Params)), *comm); + } + else{ + Linear_Solve_Params->sublist("smoother: params"); + Linear_Solve_Params->sublist("repartition: params"); + } } } diff --git a/src/Parallel-Solvers/Simulation_Parameters/FEA_Module/Elasticity_Parameters.h b/src/Parallel-Solvers/Simulation_Parameters/FEA_Module/Elasticity_Parameters.h index 026523dd9..449a21144 100644 --- a/src/Parallel-Solvers/Simulation_Parameters/FEA_Module/Elasticity_Parameters.h +++ b/src/Parallel-Solvers/Simulation_Parameters/FEA_Module/Elasticity_Parameters.h @@ -16,6 +16,8 @@ struct Elasticity_Parameters real_t constant_pressure = 0; real_t constant_stress[6]; bool constant_stress_flag = false; + bool muelu_parameters_xml_file = true; + std::string xml_parameters_file_name = "elasticity3D.xml"; Elasticity_Parameters() : FEA_Module_Parameters({ FIELD::displacement, @@ -33,5 +35,5 @@ struct Elasticity_Parameters }; IMPL_YAML_SERIALIZABLE_WITH_BASE(Elasticity_Parameters, ImplicitModule, strain_max_flag, modal_analysis, anisotropic_lattice, num_modes, smallest_modes, largest_modes, - convergence_tolerance, constant_pressure, constant_stress + convergence_tolerance, constant_pressure, constant_stress, muelu_parameters_xml_file, xml_parameters_file_name ) From 2708ccd4b8a27e6fb0bba687140ae569ae6a8deb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CAdrian-Diaz=E2=80=9D?= <“adriandiaz1117@gmail.com”> Date: Tue, 25 Jun 2024 00:03:55 -0600 Subject: [PATCH 002/130] ENH: specify xml parameter files --- .../Example_Yaml_Scripts/example_simple.yaml | 2 ++ .../FEA_Module_Heat_Conduction.cpp | 11 +++++++--- .../Implicit-Lagrange/Implicit_Solver.cpp | 17 ++++++++++++-- ...t_ex01.xml => optimization_parameters.xml} | 0 .../Parallel-Explicit/Explicit_Solver.cpp | 22 +++++++++++-------- .../FEA_Module/Heat_Conduction_Parameters.h | 4 +++- .../Optimization_Options.h | 5 ++++- 7 files changed, 45 insertions(+), 16 deletions(-) rename src/Parallel-Solvers/Implicit-Lagrange/{input_ex01.xml => optimization_parameters.xml} (100%) diff --git a/src/Parallel-Solvers/Implicit-Lagrange/Example_Yaml_Scripts/example_simple.yaml b/src/Parallel-Solvers/Implicit-Lagrange/Example_Yaml_Scripts/example_simple.yaml index e90ef6a2d..080c48d41 100644 --- a/src/Parallel-Solvers/Implicit-Lagrange/Example_Yaml_Scripts/example_simple.yaml +++ b/src/Parallel-Solvers/Implicit-Lagrange/Example_Yaml_Scripts/example_simple.yaml @@ -68,6 +68,8 @@ optimization_options: method_of_moving_asymptotes: false density_epsilon: 0.00001 simp_penalty_power: 4 + optimization_parameters_xml_file: true + xml_parameters_file_name: "optimization_parameters.xml" optimization_objective: multi_objective #Weight coefficients should add up to 1 multi_objective_structure: linear diff --git a/src/Parallel-Solvers/Implicit-Lagrange/FEA_Physics_Modules/FEA_Module_Heat_Conduction.cpp b/src/Parallel-Solvers/Implicit-Lagrange/FEA_Physics_Modules/FEA_Module_Heat_Conduction.cpp index a791ed963..efc7d91dd 100644 --- a/src/Parallel-Solvers/Implicit-Lagrange/FEA_Physics_Modules/FEA_Module_Heat_Conduction.cpp +++ b/src/Parallel-Solvers/Implicit-Lagrange/FEA_Physics_Modules/FEA_Module_Heat_Conduction.cpp @@ -3421,9 +3421,14 @@ void FEA_Module_Heat_Conduction::linear_solver_parameters(){ } else{ Linear_Solve_Params = Teuchos::rcp(new Teuchos::ParameterList("MueLu")); - std::string xmlFileName = "MueLu_Thermal_3D_Params.xml"; - //std::string xmlFileName = "simple_test.xml"; - Teuchos::updateParametersFromXmlFileAndBroadcast(xmlFileName, Teuchos::Ptr(&(*Linear_Solve_Params)), *comm); + if(module_params.muelu_parameters_xml_file){ + std::string xmlFileName = module_params.xml_parameters_file_name; + //std::string xmlFileName = "simple_test.xml"; + Teuchos::updateParametersFromXmlFileAndBroadcast(xmlFileName, Teuchos::Ptr(&(*Linear_Solve_Params)), *comm); + } + else{ + + } } } diff --git a/src/Parallel-Solvers/Implicit-Lagrange/Implicit_Solver.cpp b/src/Parallel-Solvers/Implicit-Lagrange/Implicit_Solver.cpp index dfff1b768..4adb2352c 100644 --- a/src/Parallel-Solvers/Implicit-Lagrange/Implicit_Solver.cpp +++ b/src/Parallel-Solvers/Implicit-Lagrange/Implicit_Solver.cpp @@ -1148,8 +1148,21 @@ void Implicit_Solver::setup_optimization_problem(){ // fill parameter list with desired algorithmic options or leave as default // Read optimization input parameter list. - std::string filename = "input_ex01.xml"; - auto parlist = ROL::getParametersFromXmlFile( filename ); + Teuchos::RCP parlist; + if(simparam.optimization_options.optimization_parameters_xml_file){ + std::string xmlFileName = simparam.optimization_options.xml_parameters_file_name; + + //check if parameter file exists + std::ifstream param_file_check(xmlFileName); + if (!param_file_check.is_open()) { + *fos << "Unable to find xml parameter file required for optimization with the ROL library" << std::endl; + exit_solver(0); + } + param_file_check.close(); + + parlist = ROL::getParametersFromXmlFile(xmlFileName); + } + //ROL::ParameterList parlist; //Design variables to optimize diff --git a/src/Parallel-Solvers/Implicit-Lagrange/input_ex01.xml b/src/Parallel-Solvers/Implicit-Lagrange/optimization_parameters.xml similarity index 100% rename from src/Parallel-Solvers/Implicit-Lagrange/input_ex01.xml rename to src/Parallel-Solvers/Implicit-Lagrange/optimization_parameters.xml diff --git a/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp b/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp index 446e28230..ed5bdb361 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp @@ -1132,17 +1132,21 @@ void Explicit_Solver::setup_optimization_problem(){ // fill parameter list with desired algorithmic options or leave as default // Read optimization input parameter list. - std::string filename = "optimization_parameters.xml"; - //check if parameter file exists - std::ifstream param_file_check("optimization_parameters.xml"); - if (!param_file_check.is_open()) { - *fos << "Unable to find xml parameter file required for optimization with the ROL library" << std::endl; - exit_solver(0); - } - param_file_check.close(); + Teuchos::RCP parlist; + if(simparam.optimization_options.optimization_parameters_xml_file){ + std::string xmlFileName = simparam.optimization_options.xml_parameters_file_name; + + //check if parameter file exists + std::ifstream param_file_check(xmlFileName); + if (!param_file_check.is_open()) { + *fos << "Unable to find xml parameter file required for optimization with the ROL library" << std::endl; + exit_solver(0); + } + param_file_check.close(); - auto parlist = ROL::getParametersFromXmlFile( filename ); + parlist = ROL::getParametersFromXmlFile(xmlFileName); + } //ROL::ParameterList parlist; ROL::Ptr > bnd, mma_bnd; diff --git a/src/Parallel-Solvers/Simulation_Parameters/FEA_Module/Heat_Conduction_Parameters.h b/src/Parallel-Solvers/Simulation_Parameters/FEA_Module/Heat_Conduction_Parameters.h index 55cf7e099..589007af7 100644 --- a/src/Parallel-Solvers/Simulation_Parameters/FEA_Module/Heat_Conduction_Parameters.h +++ b/src/Parallel-Solvers/Simulation_Parameters/FEA_Module/Heat_Conduction_Parameters.h @@ -8,6 +8,8 @@ struct Heat_Conduction_Parameters : virtual ImplicitModule, FEA_Module_Parameters::Register { bool thermal_flag = false; bool flux_max_flag = false; + bool muelu_parameters_xml_file = true; + std::string xml_parameters_file_name = "MueLu_Thermal_3D_Params.xml"; Heat_Conduction_Parameters() : FEA_Module_Parameters({ FIELD::temperature, @@ -16,5 +18,5 @@ struct Heat_Conduction_Parameters }; IMPL_YAML_SERIALIZABLE_WITH_BASE(Heat_Conduction_Parameters, ImplicitModule, thermal_flag, - flux_max_flag + flux_max_flag, muelu_parameters_xml_file, xml_parameters_file_name ) diff --git a/src/Parallel-Solvers/Simulation_Parameters/Optimization_Options.h b/src/Parallel-Solvers/Simulation_Parameters/Optimization_Options.h index 34b060838..c8d97cc6f 100644 --- a/src/Parallel-Solvers/Simulation_Parameters/Optimization_Options.h +++ b/src/Parallel-Solvers/Simulation_Parameters/Optimization_Options.h @@ -121,6 +121,8 @@ struct Optimization_Options: Yaml::DerivedFields { size_t num_solve_checkpoints = 10; //number of checkpoints to store explicit solve solutions for adjoint solves bool use_solve_checkpoints = false; //when false; all timesteps of explicit solves are stored for adjoint solves; expensive bool use_gradient_tally = false; //tallies gradient in tandem with the time sequence solving for the adjoint vectors + bool optimization_parameters_xml_file = true; + std::string xml_parameters_file_name = "optimization_parameters.xml"; MULTI_OBJECTIVE_STRUCTURE multi_objective_structure = MULTI_OBJECTIVE_STRUCTURE::linear; std::vector multi_objective_modules; @@ -147,5 +149,6 @@ IMPL_YAML_SERIALIZABLE_FOR(Optimization_Options, optimization_output_freq, density_filter, minimum_density, maximum_density, multi_objective_modules, multi_objective_structure, density_filter, retain_outer_shell, variable_outer_shell, shell_density, objective_normalization_constant, - num_solve_checkpoints, use_solve_checkpoints, use_gradient_tally, disable_forward_solve_output + num_solve_checkpoints, use_solve_checkpoints, use_gradient_tally, disable_forward_solve_output, + optimization_parameters_xml_file, xml_parameters_file_name ) From abcd4e4d771b41e27525a0e040863965479fe468 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CAdrian-Diaz=E2=80=9D?= <“adriandiaz1117@gmail.com”> Date: Tue, 25 Jun 2024 23:54:06 -0600 Subject: [PATCH 003/130] ENH: integrate muelu params into cpp to run without xml file --- .../FEA_Module_Elasticity.cpp | 40 +++++++++++++++- .../FEA_Module_Heat_Conduction.cpp | 39 ++++++++++++++- .../FEA_Module_Thermo_Elasticity.cpp | 48 +++++++++++++++++-- .../MueLu_Thermal_3D_Params.xml | 0 .../{ => MueLu_XML_Examples}/elasticity3D.xml | 4 +- .../{ => MueLu_XML_Examples}/simple_test.xml | 0 .../FEA_Module/Elasticity_Parameters.h | 2 +- .../FEA_Module/Heat_Conduction_Parameters.h | 2 +- .../FEA_Module/Thermo_Elasticity_Parameters.h | 4 +- 9 files changed, 128 insertions(+), 11 deletions(-) rename src/Parallel-Solvers/Implicit-Lagrange/{ => MueLu_XML_Examples}/MueLu_Thermal_3D_Params.xml (100%) rename src/Parallel-Solvers/Implicit-Lagrange/{ => MueLu_XML_Examples}/elasticity3D.xml (98%) rename src/Parallel-Solvers/Implicit-Lagrange/{ => MueLu_XML_Examples}/simple_test.xml (100%) diff --git a/src/Parallel-Solvers/Implicit-Lagrange/FEA_Physics_Modules/FEA_Module_Elasticity.cpp b/src/Parallel-Solvers/Implicit-Lagrange/FEA_Physics_Modules/FEA_Module_Elasticity.cpp index 3c8d856c0..24eed1913 100644 --- a/src/Parallel-Solvers/Implicit-Lagrange/FEA_Physics_Modules/FEA_Module_Elasticity.cpp +++ b/src/Parallel-Solvers/Implicit-Lagrange/FEA_Physics_Modules/FEA_Module_Elasticity.cpp @@ -6261,8 +6261,44 @@ void FEA_Module_Elasticity::linear_solver_parameters(){ Teuchos::updateParametersFromXmlFileAndBroadcast(xmlFileName, Teuchos::Ptr(&(*Linear_Solve_Params)), *comm); } else{ - Linear_Solve_Params->sublist("smoother: params"); - Linear_Solve_Params->sublist("repartition: params"); + //set default parameters for MueLu + Linear_Solve_Params->set("problem: type", "Elasticity-3D"); + Linear_Solve_Params->set("verbosity", "none"); + Linear_Solve_Params->set("coarse: max size", (int) 2000); + Linear_Solve_Params->set("multigrid algorithm", "sa"); + Linear_Solve_Params->set("coarse: type", "Klu2"); + Linear_Solve_Params->set("transpose: use implicit", true); + Linear_Solve_Params->set("max levels", (int) 10); + Linear_Solve_Params->set("number of equations", (int) 3); + Linear_Solve_Params->set("sa: use filtered matrix", true); + Linear_Solve_Params->set("aggregation: type", "uncoupled"); + Linear_Solve_Params->set("aggregation: drop scheme", "classical"); + Linear_Solve_Params->set("reuse: type", "S"); + //Linear_Solve_Params->set("aggregation: drop tol", (double) 0.02); + + //smoother options + Linear_Solve_Params->set("smoother: type", "CHEBYSHEV"); + Linear_Solve_Params->sublist("smoother: params").set("debug", false); + Linear_Solve_Params->sublist("smoother: params").set("chebyshev: degree", (int) 2); + Linear_Solve_Params->sublist("smoother: params").set("chebyshev: ratio eigenvalue", (double) 7.0); + Linear_Solve_Params->sublist("smoother: params").set("chebyshev: min eigenvalue", (double) 1.0); + Linear_Solve_Params->sublist("smoother: params").set("chebyshev: zero starting solution", true); + Linear_Solve_Params->sublist("smoother: params").set("chebyshev: eigenvalue max iterations", (int) 100); + + //repartition options + Linear_Solve_Params->set("repartition: enable", true); + Linear_Solve_Params->set("repartition: partitioner", "zoltan2"); + Linear_Solve_Params->set("repartition: start level", (int) 2); + Linear_Solve_Params->set("repartition: min rows per proc", (int) 2000); + Linear_Solve_Params->set("repartition: max imbalance", (double) 1.10); + Linear_Solve_Params->set("repartition: remap parts", true); + Linear_Solve_Params->set("repartition: rebalance P and R", true); + + Linear_Solve_Params->sublist("repartition: params").set("algorithm", "multijagged"); + Linear_Solve_Params->sublist("repartition: params").set("mj_premigration_option", (int) 1); + + //for device usage + Linear_Solve_Params->set("use kokkos refactor", false); } } } diff --git a/src/Parallel-Solvers/Implicit-Lagrange/FEA_Physics_Modules/FEA_Module_Heat_Conduction.cpp b/src/Parallel-Solvers/Implicit-Lagrange/FEA_Physics_Modules/FEA_Module_Heat_Conduction.cpp index efc7d91dd..0759a6a27 100644 --- a/src/Parallel-Solvers/Implicit-Lagrange/FEA_Physics_Modules/FEA_Module_Heat_Conduction.cpp +++ b/src/Parallel-Solvers/Implicit-Lagrange/FEA_Physics_Modules/FEA_Module_Heat_Conduction.cpp @@ -3427,7 +3427,44 @@ void FEA_Module_Heat_Conduction::linear_solver_parameters(){ Teuchos::updateParametersFromXmlFileAndBroadcast(xmlFileName, Teuchos::Ptr(&(*Linear_Solve_Params)), *comm); } else{ - + //set default parameters for MueLu + Linear_Solve_Params->set("problem: type", "Poisson-3D"); + Linear_Solve_Params->set("verbosity", "none"); + Linear_Solve_Params->set("coarse: max size", (int) 2000); + Linear_Solve_Params->set("multigrid algorithm", "sa"); + Linear_Solve_Params->set("coarse: type", "Klu2"); + Linear_Solve_Params->set("transpose: use implicit", true); + Linear_Solve_Params->set("max levels", (int) 10); + Linear_Solve_Params->set("number of equations", (int) 1); + Linear_Solve_Params->set("sa: use filtered matrix", true); + Linear_Solve_Params->set("aggregation: type", "uncoupled"); + Linear_Solve_Params->set("aggregation: drop scheme", "classical"); + Linear_Solve_Params->set("reuse: type", "S"); + //Linear_Solve_Params->set("aggregation: drop tol", (double) 0.02); + + //smoother options + Linear_Solve_Params->set("smoother: type", "CHEBYSHEV"); + Linear_Solve_Params->sublist("smoother: params").set("debug", false); + Linear_Solve_Params->sublist("smoother: params").set("chebyshev: degree", (int) 2); + Linear_Solve_Params->sublist("smoother: params").set("chebyshev: ratio eigenvalue", (double) 7.0); + Linear_Solve_Params->sublist("smoother: params").set("chebyshev: min eigenvalue", (double) 1.0); + Linear_Solve_Params->sublist("smoother: params").set("chebyshev: zero starting solution", true); + Linear_Solve_Params->sublist("smoother: params").set("chebyshev: eigenvalue max iterations", (int) 100); + + //repartition options + Linear_Solve_Params->set("repartition: enable", true); + Linear_Solve_Params->set("repartition: partitioner", "zoltan2"); + Linear_Solve_Params->set("repartition: start level", (int) 2); + Linear_Solve_Params->set("repartition: min rows per proc", (int) 2000); + Linear_Solve_Params->set("repartition: max imbalance", (double) 1.10); + Linear_Solve_Params->set("repartition: remap parts", true); + Linear_Solve_Params->set("repartition: rebalance P and R", true); + + Linear_Solve_Params->sublist("repartition: params").set("algorithm", "multijagged"); + Linear_Solve_Params->sublist("repartition: params").set("mj_premigration_option", (int) 1); + + //for device usage + Linear_Solve_Params->set("use kokkos refactor", false); } } } diff --git a/src/Parallel-Solvers/Implicit-Lagrange/FEA_Physics_Modules/FEA_Module_Thermo_Elasticity.cpp b/src/Parallel-Solvers/Implicit-Lagrange/FEA_Physics_Modules/FEA_Module_Thermo_Elasticity.cpp index 8fe688a4a..6bbde04c9 100644 --- a/src/Parallel-Solvers/Implicit-Lagrange/FEA_Physics_Modules/FEA_Module_Thermo_Elasticity.cpp +++ b/src/Parallel-Solvers/Implicit-Lagrange/FEA_Physics_Modules/FEA_Module_Thermo_Elasticity.cpp @@ -5355,9 +5355,51 @@ void FEA_Module_Thermo_Elasticity::linear_solver_parameters(){ } else{ Linear_Solve_Params = Teuchos::rcp(new Teuchos::ParameterList("MueLu")); - std::string xmlFileName = "elasticity3D.xml"; - //std::string xmlFileName = "simple_test.xml"; - Teuchos::updateParametersFromXmlFileAndBroadcast(xmlFileName, Teuchos::Ptr(&(*Linear_Solve_Params)), *comm); + if(module_params.muelu_parameters_xml_file){ + std::string xmlFileName = module_params.xml_parameters_file_name; + //std::string xmlFileName = "simple_test.xml"; + Teuchos::updateParametersFromXmlFileAndBroadcast(xmlFileName, Teuchos::Ptr(&(*Linear_Solve_Params)), *comm); + } + else{ + //set default parameters for MueLu + Linear_Solve_Params->set("problem: type", "Elasticity-3D"); + Linear_Solve_Params->set("verbosity", "none"); + Linear_Solve_Params->set("coarse: max size", (int) 2000); + Linear_Solve_Params->set("multigrid algorithm", "sa"); + Linear_Solve_Params->set("coarse: type", "Klu2"); + Linear_Solve_Params->set("transpose: use implicit", true); + Linear_Solve_Params->set("max levels", (int) 10); + Linear_Solve_Params->set("number of equations", (int) 3); + Linear_Solve_Params->set("sa: use filtered matrix", true); + Linear_Solve_Params->set("aggregation: type", "uncoupled"); + Linear_Solve_Params->set("aggregation: drop scheme", "classical"); + Linear_Solve_Params->set("reuse: type", "S"); + //Linear_Solve_Params->set("aggregation: drop tol", (double) 0.02); + + //smoother options + Linear_Solve_Params->set("smoother: type", "CHEBYSHEV"); + Linear_Solve_Params->sublist("smoother: params").set("debug", false); + Linear_Solve_Params->sublist("smoother: params").set("chebyshev: degree", (int) 2); + Linear_Solve_Params->sublist("smoother: params").set("chebyshev: ratio eigenvalue", (double) 7.0); + Linear_Solve_Params->sublist("smoother: params").set("chebyshev: min eigenvalue", (double) 1.0); + Linear_Solve_Params->sublist("smoother: params").set("chebyshev: zero starting solution", true); + Linear_Solve_Params->sublist("smoother: params").set("chebyshev: eigenvalue max iterations", (int) 100); + + //repartition options + Linear_Solve_Params->set("repartition: enable", true); + Linear_Solve_Params->set("repartition: partitioner", "zoltan2"); + Linear_Solve_Params->set("repartition: start level", (int) 2); + Linear_Solve_Params->set("repartition: min rows per proc", (int) 2000); + Linear_Solve_Params->set("repartition: max imbalance", (double) 1.10); + Linear_Solve_Params->set("repartition: remap parts", true); + Linear_Solve_Params->set("repartition: rebalance P and R", true); + + Linear_Solve_Params->sublist("repartition: params").set("algorithm", "multijagged"); + Linear_Solve_Params->sublist("repartition: params").set("mj_premigration_option", (int) 1); + + //for device usage + Linear_Solve_Params->set("use kokkos refactor", false); + } } } diff --git a/src/Parallel-Solvers/Implicit-Lagrange/MueLu_Thermal_3D_Params.xml b/src/Parallel-Solvers/Implicit-Lagrange/MueLu_XML_Examples/MueLu_Thermal_3D_Params.xml similarity index 100% rename from src/Parallel-Solvers/Implicit-Lagrange/MueLu_Thermal_3D_Params.xml rename to src/Parallel-Solvers/Implicit-Lagrange/MueLu_XML_Examples/MueLu_Thermal_3D_Params.xml diff --git a/src/Parallel-Solvers/Implicit-Lagrange/elasticity3D.xml b/src/Parallel-Solvers/Implicit-Lagrange/MueLu_XML_Examples/elasticity3D.xml similarity index 98% rename from src/Parallel-Solvers/Implicit-Lagrange/elasticity3D.xml rename to src/Parallel-Solvers/Implicit-Lagrange/MueLu_XML_Examples/elasticity3D.xml index 4c0d5fc11..8348c1c25 100644 --- a/src/Parallel-Solvers/Implicit-Lagrange/elasticity3D.xml +++ b/src/Parallel-Solvers/Implicit-Lagrange/MueLu_XML_Examples/elasticity3D.xml @@ -37,8 +37,8 @@ - > - > + + diff --git a/src/Parallel-Solvers/Implicit-Lagrange/simple_test.xml b/src/Parallel-Solvers/Implicit-Lagrange/MueLu_XML_Examples/simple_test.xml similarity index 100% rename from src/Parallel-Solvers/Implicit-Lagrange/simple_test.xml rename to src/Parallel-Solvers/Implicit-Lagrange/MueLu_XML_Examples/simple_test.xml diff --git a/src/Parallel-Solvers/Simulation_Parameters/FEA_Module/Elasticity_Parameters.h b/src/Parallel-Solvers/Simulation_Parameters/FEA_Module/Elasticity_Parameters.h index 449a21144..71eedd88a 100644 --- a/src/Parallel-Solvers/Simulation_Parameters/FEA_Module/Elasticity_Parameters.h +++ b/src/Parallel-Solvers/Simulation_Parameters/FEA_Module/Elasticity_Parameters.h @@ -16,7 +16,7 @@ struct Elasticity_Parameters real_t constant_pressure = 0; real_t constant_stress[6]; bool constant_stress_flag = false; - bool muelu_parameters_xml_file = true; + bool muelu_parameters_xml_file = false; std::string xml_parameters_file_name = "elasticity3D.xml"; Elasticity_Parameters() : FEA_Module_Parameters({ diff --git a/src/Parallel-Solvers/Simulation_Parameters/FEA_Module/Heat_Conduction_Parameters.h b/src/Parallel-Solvers/Simulation_Parameters/FEA_Module/Heat_Conduction_Parameters.h index 589007af7..b5e4e138a 100644 --- a/src/Parallel-Solvers/Simulation_Parameters/FEA_Module/Heat_Conduction_Parameters.h +++ b/src/Parallel-Solvers/Simulation_Parameters/FEA_Module/Heat_Conduction_Parameters.h @@ -8,7 +8,7 @@ struct Heat_Conduction_Parameters : virtual ImplicitModule, FEA_Module_Parameters::Register { bool thermal_flag = false; bool flux_max_flag = false; - bool muelu_parameters_xml_file = true; + bool muelu_parameters_xml_file = false; std::string xml_parameters_file_name = "MueLu_Thermal_3D_Params.xml"; Heat_Conduction_Parameters() : FEA_Module_Parameters({ diff --git a/src/Parallel-Solvers/Simulation_Parameters/FEA_Module/Thermo_Elasticity_Parameters.h b/src/Parallel-Solvers/Simulation_Parameters/FEA_Module/Thermo_Elasticity_Parameters.h index b0b4ad135..54c12cc17 100644 --- a/src/Parallel-Solvers/Simulation_Parameters/FEA_Module/Thermo_Elasticity_Parameters.h +++ b/src/Parallel-Solvers/Simulation_Parameters/FEA_Module/Thermo_Elasticity_Parameters.h @@ -16,6 +16,8 @@ struct Thermo_Elasticity_Parameters real_t constant_pressure = 0; real_t constant_stress[6]; bool constant_stress_flag = false; + bool muelu_parameters_xml_file = false; + std::string xml_parameters_file_name = "elasticity3D.xml"; std::set default_output_fields { FIELD::displacement, @@ -34,5 +36,5 @@ struct Thermo_Elasticity_Parameters }; IMPL_YAML_SERIALIZABLE_WITH_BASE(Thermo_Elasticity_Parameters, ImplicitModule, strain_max_flag, modal_analysis, anisotropic_lattice, num_modes, smallest_modes, largest_modes, - convergence_tolerance, constant_pressure, constant_stress + convergence_tolerance, constant_pressure, constant_stress, muelu_parameters_xml_file, xml_parameters_file_name ) From a3ebc2266db6043b50f6ac62ce5136fc6334e9bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CAdrian-Diaz=E2=80=9D?= <“adriandiaz1117@gmail.com”> Date: Wed, 26 Jun 2024 23:00:40 -0600 Subject: [PATCH 004/130] WIP: integrate optimization xml options --- .../Implicit-Lagrange/Implicit_Solver.cpp | 101 ++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/src/Parallel-Solvers/Implicit-Lagrange/Implicit_Solver.cpp b/src/Parallel-Solvers/Implicit-Lagrange/Implicit_Solver.cpp index 4adb2352c..529936c71 100644 --- a/src/Parallel-Solvers/Implicit-Lagrange/Implicit_Solver.cpp +++ b/src/Parallel-Solvers/Implicit-Lagrange/Implicit_Solver.cpp @@ -1162,6 +1162,107 @@ void Implicit_Solver::setup_optimization_problem(){ parlist = ROL::getParametersFromXmlFile(xmlFileName); } + else{ + //set defaults here + parlist->sublist("General").set("Variable Objective Function", false); + parlist->sublist("General").set("Scale for Epsilon Active Sets", (double) 1.0); + parlist->sublist("General").set("Output Level", (int) 1); + parlist->sublist("General").set("Inexact Objective Function", false); + parlist->sublist("General").set("Inexact Gradient", false); + parlist->sublist("General").set("Inexact Hessian-Times-A-Vector", false); + parlist->sublist("General").set("Projected Gradient Criticality Measure", false); + + // + // + // + // + // + // + // + // + // + // + + // + // + // + // + // + // + // + // + // + // + + // + // + // + // + // + // + // + + // + // + // + // + // + // + // + + parlist->sublist("Step").sublist("Line Search").set("Function Evaluation Limit", (int) 20); + parlist->sublist("Step").sublist("Line Search").set("Sufficient Decrease Tolerance", (double) 1.e-2); + parlist->sublist("Step").sublist("Line Search").set("Initial Step Size", (double) 5e0); + parlist->sublist("Step").sublist("Line Search").set("User Defined Initial Step Size", true); + parlist->sublist("Step").sublist("Line Search").set("Normalize Initial Step Size", false); + parlist->sublist("Step").sublist("Line Search").set("Accept Last Alpha", false); + parlist->sublist("Step").sublist("Line Search").set("Use Previous Step Length as Initial Guess", false); + parlist->sublist("Step").sublist("Line Search").set("Maximum Step Size", (double) 5e3); + parlist->sublist("Step").sublist("Line Search").set("Use Adaptive Step Size Selection", true); + + parlist->sublist("Step").sublist("Line Search").sublist("Descent Method").set("Type", "Quasi-Newton Method"); + parlist->sublist("Step").sublist("Line Search").sublist("Descent Method").set("Nonlinear CG Type", "Hestenes-Stiefel"); + + parlist->sublist("Step").sublist("Line Search").sublist("Curvature Condition").set("Type", "Strong Wolfe Conditions"); + parlist->sublist("Step").sublist("Line Search").sublist("Curvature Condition").set("General Parameter", (double) 0.9); + parlist->sublist("Step").sublist("Line Search").sublist("Curvature Condition").set("Generalized Wolfe Parameter", (double) 0.6); + + parlist->sublist("Step").sublist("Line Search").sublist("Line-Search Method").set("Type", "Cubic Interpolation"); + parlist->sublist("Step").sublist("Line Search").sublist("Line-Search Method").set("Increase Rate", (double) "5e0"); + parlist->sublist("Step").sublist("Line Search").sublist("Line-Search Method").set("Backtracking Rate" , (double) "0.5"); + parlist->sublist("Step").sublist("Line Search").sublist("Line-Search Method").set("Bracketing Tolerance" , (double) "1.e-8"); + + + // + // + // + // + // + + // + // + // + // + // + // + + // + // + // + // + // + // + + // + // + // + // + // + // + } //ROL::ParameterList parlist; From 019fa234ebb2abe21e1c1ba997b163e022fb70cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CAdrian-Diaz=E2=80=9D?= <“adriandiaz1117@gmail.com”> Date: Thu, 27 Jun 2024 13:11:44 -0600 Subject: [PATCH 005/130] WIP: integrate xml optimization options into Fierro --- .../Implicit-Lagrange/Implicit_Solver.cpp | 156 ++++++++++-------- 1 file changed, 84 insertions(+), 72 deletions(-) diff --git a/src/Parallel-Solvers/Implicit-Lagrange/Implicit_Solver.cpp b/src/Parallel-Solvers/Implicit-Lagrange/Implicit_Solver.cpp index 529936c71..e5a7f833c 100644 --- a/src/Parallel-Solvers/Implicit-Lagrange/Implicit_Solver.cpp +++ b/src/Parallel-Solvers/Implicit-Lagrange/Implicit_Solver.cpp @@ -1172,47 +1172,25 @@ void Implicit_Solver::setup_optimization_problem(){ parlist->sublist("General").set("Inexact Hessian-Times-A-Vector", false); parlist->sublist("General").set("Projected Gradient Criticality Measure", false); - // - // - // - // - // - // - // - // - // - // - - // - // - // - // - // - // - // - // - // - // - - // - // - // - // - // - // - // - - // - // - // - // - // - // - // - + parlist->sublist("General").sublist("Secant").set("Type", "Limited-Memory BFGS"); + parlist->sublist("General").sublist("Secant").set("Use as Preconditioner", false); + parlist->sublist("General").sublist("Secant").set("Use as Hessian", false); + parlist->sublist("General").sublist("Secant").set("Maximum Storage", (int) 5); + parlist->sublist("General").sublist("Secant").set("Use Default Scaling", false); + parlist->sublist("General").sublist("Secant").set("Initial Hessian Scale", (double) 1e-16); + parlist->sublist("General").sublist("Secant").set("Barzilai-Borwein Type", (int) 1); + + parlist->sublist("General").sublist("Krylov").set("Type", "Conjugate Gradients"); + parlist->sublist("General").sublist("Krylov").set("Absolute Tolerance", (double) 1.e-4); + parlist->sublist("General").sublist("Krylov").set("Relative Tolerance", (double) 1.e-2); + parlist->sublist("General").sublist("Krylov").set("Iteration Limit", (int) 50); + + parlist->sublist("General").sublist("Polyhedral Projection").set("Type", "Dai-Fletcher"); + parlist->sublist("General").sublist("Polyhedral Projection").set("Iteration Limit", (int) 1000); + parlist->sublist("General").sublist("Polyhedral Projection").set("Absolute Tolerance", (double) 1.e-4); + parlist->sublist("General").sublist("Polyhedral Projection").set("Relative Tolerance", (double) 1.e-2); + + //Line search settings parlist->sublist("Step").sublist("Line Search").set("Function Evaluation Limit", (int) 20); parlist->sublist("Step").sublist("Line Search").set("Sufficient Decrease Tolerance", (double) 1.e-2); parlist->sublist("Step").sublist("Line Search").set("Initial Step Size", (double) 5e0); @@ -1231,37 +1209,71 @@ void Implicit_Solver::setup_optimization_problem(){ parlist->sublist("Step").sublist("Line Search").sublist("Curvature Condition").set("Generalized Wolfe Parameter", (double) 0.6); parlist->sublist("Step").sublist("Line Search").sublist("Line-Search Method").set("Type", "Cubic Interpolation"); - parlist->sublist("Step").sublist("Line Search").sublist("Line-Search Method").set("Increase Rate", (double) "5e0"); - parlist->sublist("Step").sublist("Line Search").sublist("Line-Search Method").set("Backtracking Rate" , (double) "0.5"); - parlist->sublist("Step").sublist("Line Search").sublist("Line-Search Method").set("Bracketing Tolerance" , (double) "1.e-8"); - - - // - // - // - // - // - - // - // - // - // - // - // - - // - // - // - // - // - // - - // - // - // - // - // - // + parlist->sublist("Step").sublist("Line Search").sublist("Line-Search Method").set("Increase Rate", (double) 5e0); + parlist->sublist("Step").sublist("Line Search").sublist("Line-Search Method").set("Backtracking Rate" , (double) 0.5); + parlist->sublist("Step").sublist("Line Search").sublist("Line-Search Method").set("Bracketing Tolerance" , (double) 1.e-8); + + parlist->sublist("Step").sublist("Line Search").sublist("Line-Search Method").sublist("Path-Based Target Level").set("Target Relaxation Parameter" , (double) 1.0); + parlist->sublist("Step").sublist("Line Search").sublist("Line-Search Method").sublist("Path-Based Target Level").set("Upper Bound on Path Length" , (double) 1.0); + + //Trust region settings + parlist->sublist("Step").sublist("Trust Region").set("Subproblem Solver", "Truncated CG"); + parlist->sublist("Step").sublist("Trust Region").set("Subproblem Model", "SPG"); + parlist->sublist("Step").sublist("Trust Region").set("Initial Radius", (double) 2e1); + parlist->sublist("Step").sublist("Trust Region").set("Maximum Radius", (double) 5.e8); + parlist->sublist("Step").sublist("Trust Region").set("Step Acceptance Threshold", (double) 0.05); + parlist->sublist("Step").sublist("Trust Region").set("Radius Shrinking Threshold", (double) 0.05); + parlist->sublist("Step").sublist("Trust Region").set("Radius Growing Threshold", (double) 0.9); + parlist->sublist("Step").sublist("Trust Region").set("Radius Shrinking Rate (Negative rho)", (double) 0.0625); + parlist->sublist("Step").sublist("Trust Region").set("Radius Shrinking Rate (Positive rho)", (double) 0.25); + parlist->sublist("Step").sublist("Trust Region").set("Radius Growing Rate", (double) 2.5); + parlist->sublist("Step").sublist("Trust Region").set("Safeguard Size", (double) 1.e1); + +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// } //ROL::ParameterList parlist; From d2ebcc0674c4ce6eb38d94646e41571c994fdff8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CAdrian-Diaz=E2=80=9D?= <“adriandiaz1117@gmail.com”> Date: Thu, 27 Jun 2024 21:18:04 -0600 Subject: [PATCH 006/130] WIP: rol params cpp integration --- .../Implicit-Lagrange/Implicit_Solver.cpp | 112 +----------- .../Parallel-Explicit/Explicit_Solver.cpp | 3 + src/Parallel-Solvers/Solver.cpp | 172 ++++++++++++++++++ src/Parallel-Solvers/Solver.h | 2 + 4 files changed, 178 insertions(+), 111 deletions(-) diff --git a/src/Parallel-Solvers/Implicit-Lagrange/Implicit_Solver.cpp b/src/Parallel-Solvers/Implicit-Lagrange/Implicit_Solver.cpp index e5a7f833c..c979a6481 100644 --- a/src/Parallel-Solvers/Implicit-Lagrange/Implicit_Solver.cpp +++ b/src/Parallel-Solvers/Implicit-Lagrange/Implicit_Solver.cpp @@ -1163,117 +1163,7 @@ void Implicit_Solver::setup_optimization_problem(){ parlist = ROL::getParametersFromXmlFile(xmlFileName); } else{ - //set defaults here - parlist->sublist("General").set("Variable Objective Function", false); - parlist->sublist("General").set("Scale for Epsilon Active Sets", (double) 1.0); - parlist->sublist("General").set("Output Level", (int) 1); - parlist->sublist("General").set("Inexact Objective Function", false); - parlist->sublist("General").set("Inexact Gradient", false); - parlist->sublist("General").set("Inexact Hessian-Times-A-Vector", false); - parlist->sublist("General").set("Projected Gradient Criticality Measure", false); - - parlist->sublist("General").sublist("Secant").set("Type", "Limited-Memory BFGS"); - parlist->sublist("General").sublist("Secant").set("Use as Preconditioner", false); - parlist->sublist("General").sublist("Secant").set("Use as Hessian", false); - parlist->sublist("General").sublist("Secant").set("Maximum Storage", (int) 5); - parlist->sublist("General").sublist("Secant").set("Use Default Scaling", false); - parlist->sublist("General").sublist("Secant").set("Initial Hessian Scale", (double) 1e-16); - parlist->sublist("General").sublist("Secant").set("Barzilai-Borwein Type", (int) 1); - - parlist->sublist("General").sublist("Krylov").set("Type", "Conjugate Gradients"); - parlist->sublist("General").sublist("Krylov").set("Absolute Tolerance", (double) 1.e-4); - parlist->sublist("General").sublist("Krylov").set("Relative Tolerance", (double) 1.e-2); - parlist->sublist("General").sublist("Krylov").set("Iteration Limit", (int) 50); - - parlist->sublist("General").sublist("Polyhedral Projection").set("Type", "Dai-Fletcher"); - parlist->sublist("General").sublist("Polyhedral Projection").set("Iteration Limit", (int) 1000); - parlist->sublist("General").sublist("Polyhedral Projection").set("Absolute Tolerance", (double) 1.e-4); - parlist->sublist("General").sublist("Polyhedral Projection").set("Relative Tolerance", (double) 1.e-2); - - //Line search settings - parlist->sublist("Step").sublist("Line Search").set("Function Evaluation Limit", (int) 20); - parlist->sublist("Step").sublist("Line Search").set("Sufficient Decrease Tolerance", (double) 1.e-2); - parlist->sublist("Step").sublist("Line Search").set("Initial Step Size", (double) 5e0); - parlist->sublist("Step").sublist("Line Search").set("User Defined Initial Step Size", true); - parlist->sublist("Step").sublist("Line Search").set("Normalize Initial Step Size", false); - parlist->sublist("Step").sublist("Line Search").set("Accept Last Alpha", false); - parlist->sublist("Step").sublist("Line Search").set("Use Previous Step Length as Initial Guess", false); - parlist->sublist("Step").sublist("Line Search").set("Maximum Step Size", (double) 5e3); - parlist->sublist("Step").sublist("Line Search").set("Use Adaptive Step Size Selection", true); - - parlist->sublist("Step").sublist("Line Search").sublist("Descent Method").set("Type", "Quasi-Newton Method"); - parlist->sublist("Step").sublist("Line Search").sublist("Descent Method").set("Nonlinear CG Type", "Hestenes-Stiefel"); - - parlist->sublist("Step").sublist("Line Search").sublist("Curvature Condition").set("Type", "Strong Wolfe Conditions"); - parlist->sublist("Step").sublist("Line Search").sublist("Curvature Condition").set("General Parameter", (double) 0.9); - parlist->sublist("Step").sublist("Line Search").sublist("Curvature Condition").set("Generalized Wolfe Parameter", (double) 0.6); - - parlist->sublist("Step").sublist("Line Search").sublist("Line-Search Method").set("Type", "Cubic Interpolation"); - parlist->sublist("Step").sublist("Line Search").sublist("Line-Search Method").set("Increase Rate", (double) 5e0); - parlist->sublist("Step").sublist("Line Search").sublist("Line-Search Method").set("Backtracking Rate" , (double) 0.5); - parlist->sublist("Step").sublist("Line Search").sublist("Line-Search Method").set("Bracketing Tolerance" , (double) 1.e-8); - - parlist->sublist("Step").sublist("Line Search").sublist("Line-Search Method").sublist("Path-Based Target Level").set("Target Relaxation Parameter" , (double) 1.0); - parlist->sublist("Step").sublist("Line Search").sublist("Line-Search Method").sublist("Path-Based Target Level").set("Upper Bound on Path Length" , (double) 1.0); - - //Trust region settings - parlist->sublist("Step").sublist("Trust Region").set("Subproblem Solver", "Truncated CG"); - parlist->sublist("Step").sublist("Trust Region").set("Subproblem Model", "SPG"); - parlist->sublist("Step").sublist("Trust Region").set("Initial Radius", (double) 2e1); - parlist->sublist("Step").sublist("Trust Region").set("Maximum Radius", (double) 5.e8); - parlist->sublist("Step").sublist("Trust Region").set("Step Acceptance Threshold", (double) 0.05); - parlist->sublist("Step").sublist("Trust Region").set("Radius Shrinking Threshold", (double) 0.05); - parlist->sublist("Step").sublist("Trust Region").set("Radius Growing Threshold", (double) 0.9); - parlist->sublist("Step").sublist("Trust Region").set("Radius Shrinking Rate (Negative rho)", (double) 0.0625); - parlist->sublist("Step").sublist("Trust Region").set("Radius Shrinking Rate (Positive rho)", (double) 0.25); - parlist->sublist("Step").sublist("Trust Region").set("Radius Growing Rate", (double) 2.5); - parlist->sublist("Step").sublist("Trust Region").set("Safeguard Size", (double) 1.e1); - -// -// -// -// -// -// -// -// -// -// -// -// -// -// -// -// -// -// -// -// -// -// -// -// -// -// -// -// -// -// -// -// -// -// -// -// -// -// -// -// -// -// -// -// -// + set_rol_params(parlist); } //ROL::ParameterList parlist; diff --git a/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp b/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp index ed5bdb361..1b4cf4baf 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp @@ -1147,6 +1147,9 @@ void Explicit_Solver::setup_optimization_problem(){ parlist = ROL::getParametersFromXmlFile(xmlFileName); } + else{ + set_rol_params(parlist); + } //ROL::ParameterList parlist; ROL::Ptr > bnd, mma_bnd; diff --git a/src/Parallel-Solvers/Solver.cpp b/src/Parallel-Solvers/Solver.cpp index ecf12e746..799a8c481 100644 --- a/src/Parallel-Solvers/Solver.cpp +++ b/src/Parallel-Solvers/Solver.cpp @@ -3859,6 +3859,178 @@ void Solver::Get_Boundary_Patches() // std::fflush(stdout); } +/* ---------------------------------------------------------------------- + Parameters to feed ROL Trilinos package +------------------------------------------------------------------------- */ + +void Solver::set_rol_params(Teuchos::RCP parlist) +{ + //set defaults here + parlist->sublist("General").set("Variable Objective Function", false); + parlist->sublist("General").set("Scale for Epsilon Active Sets", (double) 1.0); + parlist->sublist("General").set("Output Level", (int) 1); + parlist->sublist("General").set("Inexact Objective Function", false); + parlist->sublist("General").set("Inexact Gradient", false); + parlist->sublist("General").set("Inexact Hessian-Times-A-Vector", false); + parlist->sublist("General").set("Projected Gradient Criticality Measure", false); + + parlist->sublist("General").sublist("Secant").set("Type", "Limited-Memory BFGS"); + parlist->sublist("General").sublist("Secant").set("Use as Preconditioner", false); + parlist->sublist("General").sublist("Secant").set("Use as Hessian", false); + parlist->sublist("General").sublist("Secant").set("Maximum Storage", (int) 5); + parlist->sublist("General").sublist("Secant").set("Use Default Scaling", false); + parlist->sublist("General").sublist("Secant").set("Initial Hessian Scale", (double) 1e-16); + parlist->sublist("General").sublist("Secant").set("Barzilai-Borwein Type", (int) 1); + + parlist->sublist("General").sublist("Krylov").set("Type", "Conjugate Gradients"); + parlist->sublist("General").sublist("Krylov").set("Absolute Tolerance", (double) 1e-4); + parlist->sublist("General").sublist("Krylov").set("Relative Tolerance", (double) 1e-2); + parlist->sublist("General").sublist("Krylov").set("Iteration Limit", (int) 50); + + parlist->sublist("General").sublist("Polyhedral Projection").set("Type", "Dai-Fletcher"); + parlist->sublist("General").sublist("Polyhedral Projection").set("Iteration Limit", (int) 1000); + parlist->sublist("General").sublist("Polyhedral Projection").set("Absolute Tolerance", (double) 1e-4); + parlist->sublist("General").sublist("Polyhedral Projection").set("Relative Tolerance", (double) 1e-2); + + //Line search settings + parlist->sublist("Step").sublist("Line Search").set("Function Evaluation Limit", (int) 20); + parlist->sublist("Step").sublist("Line Search").set("Sufficient Decrease Tolerance", (double) 1.e-2); + parlist->sublist("Step").sublist("Line Search").set("Initial Step Size", (double) 5e0); + parlist->sublist("Step").sublist("Line Search").set("User Defined Initial Step Size", true); + parlist->sublist("Step").sublist("Line Search").set("Normalize Initial Step Size", false); + parlist->sublist("Step").sublist("Line Search").set("Accept Last Alpha", false); + parlist->sublist("Step").sublist("Line Search").set("Use Previous Step Length as Initial Guess", false); + parlist->sublist("Step").sublist("Line Search").set("Maximum Step Size", (double) 5e3); + parlist->sublist("Step").sublist("Line Search").set("Use Adaptive Step Size Selection", true); + + parlist->sublist("Step").sublist("Line Search").sublist("Descent Method").set("Type", "Quasi-Newton Method"); + parlist->sublist("Step").sublist("Line Search").sublist("Descent Method").set("Nonlinear CG Type", "Hestenes-Stiefel"); + + parlist->sublist("Step").sublist("Line Search").sublist("Curvature Condition").set("Type", "Strong Wolfe Conditions"); + parlist->sublist("Step").sublist("Line Search").sublist("Curvature Condition").set("General Parameter", (double) 0.9); + parlist->sublist("Step").sublist("Line Search").sublist("Curvature Condition").set("Generalized Wolfe Parameter", (double) 0.6); + + parlist->sublist("Step").sublist("Line Search").sublist("Line-Search Method").set("Type", "Cubic Interpolation"); + parlist->sublist("Step").sublist("Line Search").sublist("Line-Search Method").set("Increase Rate", (double) 5e0); + parlist->sublist("Step").sublist("Line Search").sublist("Line-Search Method").set("Backtracking Rate" , (double) 0.5); + parlist->sublist("Step").sublist("Line Search").sublist("Line-Search Method").set("Bracketing Tolerance" , (double) 1e-8); + + parlist->sublist("Step").sublist("Line Search").sublist("Line-Search Method").sublist("Path-Based Target Level").set("Target Relaxation Parameter" , (double) 1.0); + parlist->sublist("Step").sublist("Line Search").sublist("Line-Search Method").sublist("Path-Based Target Level").set("Upper Bound on Path Length" , (double) 1.0); + + //Trust region settings + parlist->sublist("Step").sublist("Trust Region").set("Subproblem Solver", "Truncated CG"); + parlist->sublist("Step").sublist("Trust Region").set("Subproblem Model", "SPG"); + parlist->sublist("Step").sublist("Trust Region").set("Initial Radius", (double) 2e1); + parlist->sublist("Step").sublist("Trust Region").set("Maximum Radius", (double) 5e8); + parlist->sublist("Step").sublist("Trust Region").set("Step Acceptance Threshold", (double) 0.05); + parlist->sublist("Step").sublist("Trust Region").set("Radius Shrinking Threshold", (double) 0.05); + parlist->sublist("Step").sublist("Trust Region").set("Radius Growing Threshold", (double) 0.9); + parlist->sublist("Step").sublist("Trust Region").set("Radius Shrinking Rate (Negative rho)", (double) 0.0625); + parlist->sublist("Step").sublist("Trust Region").set("Radius Shrinking Rate (Positive rho)", (double) 0.25); + parlist->sublist("Step").sublist("Trust Region").set("Radius Growing Rate", (double) 2.5); + parlist->sublist("Step").sublist("Trust Region").set("Safeguard Size", (double) 1e1); + + //Trust region Lin-More + parlist->sublist("Step").sublist("Trust Region").sublist("Lin-More").set("Maximum Number of Minor Iterations", (int) 10); + parlist->sublist("Step").sublist("Trust Region").sublist("Lin-More").set("Sufficient Decrease Parameter", (double) 1e-2); + parlist->sublist("Step").sublist("Trust Region").sublist("Lin-More").set("Relative Tolerance Exponent", (double) 1.1); + + parlist->sublist("Step").sublist("Trust Region").sublist("Lin-More").sublist("Cauchy Point").set("Maximum Number of Reduction Steps", (int) 10); + parlist->sublist("Step").sublist("Trust Region").sublist("Lin-More").sublist("Cauchy Point").set("Maximum Number of Expansion Steps", (int) 10); + parlist->sublist("Step").sublist("Trust Region").sublist("Lin-More").sublist("Cauchy Point").set("Initial Step Size", (double) 1.0); + parlist->sublist("Step").sublist("Trust Region").sublist("Lin-More").sublist("Cauchy Point").set("Normalize Initial Step Size", true); + parlist->sublist("Step").sublist("Trust Region").sublist("Lin-More").sublist("Cauchy Point").set("Reduction Rate", (double) 0.1); + parlist->sublist("Step").sublist("Trust Region").sublist("Lin-More").sublist("Cauchy Point").set("Expansion Rate", (double) 5.0); + parlist->sublist("Step").sublist("Trust Region").sublist("Lin-More").sublist("Cauchy Point").set("Decrease Tolerance", (double) 1e-8); + + parlist->sublist("Step").sublist("Trust Region").sublist("Lin-More").sublist("Projected Search").set("Backtracking Rate", (double) 0.5); + parlist->sublist("Step").sublist("Trust Region").sublist("Lin-More").sublist("Projected Search").set("Maximum Number of Steps", (int) 20); + + //Trust region SPG (Spectral Projected Gradient) + parlist->sublist("Step").sublist("Trust Region").sublist("SPG").set("Use Nonmonotone Trust Region", false); + parlist->sublist("Step").sublist("Trust Region").sublist("SPG").set("Maximum Storage Size", (int) 10); + parlist->sublist("Step").sublist("Trust Region").sublist("SPG").sublist("Solver").set("Iteration Limit", (int) 25); + parlist->sublist("Step").sublist("Trust Region").sublist("SPG").sublist("Solver").set("Minimum Spectral Step Size", (double) 1e-12); + parlist->sublist("Step").sublist("Trust Region").sublist("SPG").sublist("Solver").set("Maximum Spectral Step Size", (double) 1e12); + parlist->sublist("Step").sublist("Trust Region").sublist("SPG").sublist("Solver").set("Use Smallest Model Iterate", false); + + //Controls for Inexactness + parlist->sublist("Step").sublist("Trust Region").sublist("Inexact").sublist("Value").set("Tolerance Scaling", (double) 1e-1); + parlist->sublist("Step").sublist("Trust Region").sublist("Inexact").sublist("Value").set("Exponent", (double) 0.9); + parlist->sublist("Step").sublist("Trust Region").sublist("Inexact").sublist("Value").set("Forcing Sequence Initial Value", (double) 1.0); + parlist->sublist("Step").sublist("Trust Region").sublist("Inexact").sublist("Value").set("Forcing Sequence Update Frequency", (int) 10); + parlist->sublist("Step").sublist("Trust Region").sublist("Inexact").sublist("Value").set("Forcing Sequence Reduction Factor", (double) 0.1); + + parlist->sublist("Step").sublist("Trust Region").sublist("Inexact").sublist("Gradient").set("Tolerance Scaling", (double) 1e-1); + parlist->sublist("Step").sublist("Trust Region").sublist("Inexact").sublist("Gradient").set("Relative Tolerance", (double) 2.0); + + //Spectral gradient options + parlist->sublist("Step").sublist("Spectral Gradient").set("Minimum Spectral Step Size", (double) 1e-12); + parlist->sublist("Step").sublist("Spectral Gradient").set("Maximum Spectral Step Size", (double) 1e12); + + //Primal dual active set options + parlist->sublist("Step").sublist("Primal Dual Active Set").set("Dual Scaling", (double) 1.0); + parlist->sublist("Step").sublist("Primal Dual Active Set").set("Iteration Limit", (int) 10); + parlist->sublist("Step").sublist("Primal Dual Active Set").set("Relative Step Tolerance", (double) 1e-8); + parlist->sublist("Step").sublist("Primal Dual Active Set").set("Relative Gradient Tolerance", (double) 1e-6); + + parlist->sublist("Step").sublist("Composite Step").set("Output Level", (int) 0); + parlist->sublist("Step").sublist("Composite Step").sublist("Optimality System Solver").set("Nominal Relative Tolerance", (double) 1e-10); + parlist->sublist("Step").sublist("Composite Step").sublist("Optimality System Solver").set("Fix Tolerance", true); + + parlist->sublist("Step").sublist("Composite Step").sublist("Tangential Subproblem Solver").set("Iteration Limit", (int) 20); + parlist->sublist("Step").sublist("Composite Step").sublist("Tangential Subproblem Solver").set("Relative Tolerance", (double) 1e-2); + + //Augmented Lagrangian Options + parlist->sublist("Step").sublist("Augmented Lagrangian").set("Level of Hessian Approximation", (int) 0); + parlist->sublist("Step").sublist("Augmented Lagrangian").set("Use Default Problem Scaling", true); + parlist->sublist("Step").sublist("Augmented Lagrangian").set("Objective Scaling", (double) 1e0); + parlist->sublist("Step").sublist("Augmented Lagrangian").set("Constraint Scaling", (double) 1e0); + parlist->sublist("Step").sublist("Augmented Lagrangian").set("Use Default Initial Penalty Parameter", false); + parlist->sublist("Step").sublist("Augmented Lagrangian").set("Initial Penalty Parameter", (double) 2e3); + parlist->sublist("Step").sublist("Augmented Lagrangian").set("Penalty Parameter Growth Factor", (double) 1e1); + parlist->sublist("Step").sublist("Augmented Lagrangian").set("Minimum Penalty Parameter Reciprocal", (double) 0.1); + parlist->sublist("Step").sublist("Augmented Lagrangian").set("Initial Optimality Tolerance", (double) 1.0); + parlist->sublist("Step").sublist("Augmented Lagrangian").set("Optimality Tolerance Update Exponent", (double) 1.0); + parlist->sublist("Step").sublist("Augmented Lagrangian").set("Optimality Tolerance Decrease Exponent", (double) 1.0); + + parlist->sublist("Step").sublist("Augmented Lagrangian").set("Initial Feasibility Tolerance", (double) 1000.0); + parlist->sublist("Step").sublist("Augmented Lagrangian").set("Feasibility Tolerance Update Exponent", (double) 0.1); + parlist->sublist("Step").sublist("Augmented Lagrangian").set("Feasibility Tolerance Decrease Exponent", (double) 0.9); + + parlist->sublist("Step").sublist("Augmented Lagrangian").set("Print Intermediate Optimization History", false); + parlist->sublist("Step").sublist("Augmented Lagrangian").set("Subproblem Step Type", "Trust Region"); + parlist->sublist("Step").sublist("Augmented Lagrangian").set("Subproblem Iteration Limit", (int) 20); + + parlist->sublist("Step").sublist("Moreau-Yosida Penalty").set("Initial Penalty Parameter", (double) 1e-9); + parlist->sublist("Step").sublist("Moreau-Yosida Penalty").set("Penalty Parameter Growth Factor", (double) 1.5); + parlist->sublist("Step").sublist("Moreau-Yosida Penalty").sublist("Subproblem").set("Optimality Tolerance", (double) 1e-12); + parlist->sublist("Step").sublist("Moreau-Yosida Penalty").sublist("Subproblem").set("Feasibility Tolerance", (double) 1e-12); + parlist->sublist("Step").sublist("Moreau-Yosida Penalty").sublist("Subproblem").set("Print History", true); + parlist->sublist("Step").sublist("Moreau-Yosida Penalty").sublist("Subproblem").set("Iteration Limit", (int) 200); + + parlist->sublist("Step").sublist("Bundle").set("Initial Trust-Region Parameter", (double) 1e1); + parlist->sublist("Step").sublist("Bundle").set("Maximum Trust-Region Parameter", (double) 1e8); + parlist->sublist("Step").sublist("Bundle").set("Tolerance for Trust-Region Parameter", (double) 1e-4); + parlist->sublist("Step").sublist("Bundle").set("Epsilon Solution Tolerance", (double) 1e-8); + parlist->sublist("Step").sublist("Bundle").set("Upper Threshold for Serious Step", (double) 1e-1); + parlist->sublist("Step").sublist("Bundle").set("Lower Threshold for Serious Step", (double) 2e-1); + parlist->sublist("Step").sublist("Bundle").set("Upper Threshold for Null Step", (double) 9e-1); + parlist->sublist("Step").sublist("Bundle").set("Distance Measure Coefficient", (double) 1e-6); + parlist->sublist("Step").sublist("Bundle").set("Maximum Bundle Size", (int) 50); + parlist->sublist("Step").sublist("Bundle").set("Removal Size for Bundle Update", (int) 2); + parlist->sublist("Step").sublist("Bundle").set("Cutting Plane Tolerance", (double) 1e-8); + parlist->sublist("Step").sublist("Bundle").set("Cutting Plane Iteration Limit", (int) 1000); + + + parlist->sublist("Status Test").set("Gradient Tolerance", (double) 1e-5); + parlist->sublist("Status Test").set("Constraint Tolerance", (double) 1e-5); + parlist->sublist("Status Test").set("Step Tolerance", (double) 1e-5); + parlist->sublist("Status Test").set("Iteration Limit", (int) 100); + parlist->sublist("Status Test").set("Use Relative Tolerances", true); +} + /* ---------------------------------------------------------------------- Setup Tpetra importers for comms ------------------------------------------------------------------------- */ diff --git a/src/Parallel-Solvers/Solver.h b/src/Parallel-Solvers/Solver.h index 1a139d48c..c3ced4c40 100644 --- a/src/Parallel-Solvers/Solver.h +++ b/src/Parallel-Solvers/Solver.h @@ -162,6 +162,8 @@ class Solver // finds the boundary element surfaces in this model virtual void Get_Boundary_Patches(); + virtual void set_rol_params(Teuchos::RCP parlist); + int setup_flag, finalize_flag; // MPI data From b3c7a01b2823cd5fee38d5f16de262a9d7236f1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CAdrian-Diaz=E2=80=9D?= <“adriandiaz1117@gmail.com”> Date: Fri, 28 Jun 2024 00:10:52 -0600 Subject: [PATCH 007/130] ENH: default ROL params integrated --- src/Parallel-Solvers/Implicit-Lagrange/Implicit_Solver.cpp | 1 + src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp | 1 + src/Parallel-Solvers/Solver.cpp | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Parallel-Solvers/Implicit-Lagrange/Implicit_Solver.cpp b/src/Parallel-Solvers/Implicit-Lagrange/Implicit_Solver.cpp index c979a6481..04634503a 100644 --- a/src/Parallel-Solvers/Implicit-Lagrange/Implicit_Solver.cpp +++ b/src/Parallel-Solvers/Implicit-Lagrange/Implicit_Solver.cpp @@ -1163,6 +1163,7 @@ void Implicit_Solver::setup_optimization_problem(){ parlist = ROL::getParametersFromXmlFile(xmlFileName); } else{ + parlist = Teuchos::rcp(new Teuchos::ParameterList("Inputs")); set_rol_params(parlist); } diff --git a/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp b/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp index 1b4cf4baf..36a7e3d47 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp @@ -1148,6 +1148,7 @@ void Explicit_Solver::setup_optimization_problem(){ parlist = ROL::getParametersFromXmlFile(xmlFileName); } else{ + parlist = Teuchos::rcp(new Teuchos::ParameterList("Inputs")); set_rol_params(parlist); } //ROL::ParameterList parlist; diff --git a/src/Parallel-Solvers/Solver.cpp b/src/Parallel-Solvers/Solver.cpp index 799a8c481..403da2f6f 100644 --- a/src/Parallel-Solvers/Solver.cpp +++ b/src/Parallel-Solvers/Solver.cpp @@ -3988,7 +3988,7 @@ void Solver::set_rol_params(Teuchos::RCP parlist) parlist->sublist("Step").sublist("Augmented Lagrangian").set("Objective Scaling", (double) 1e0); parlist->sublist("Step").sublist("Augmented Lagrangian").set("Constraint Scaling", (double) 1e0); parlist->sublist("Step").sublist("Augmented Lagrangian").set("Use Default Initial Penalty Parameter", false); - parlist->sublist("Step").sublist("Augmented Lagrangian").set("Initial Penalty Parameter", (double) 2e3); + parlist->sublist("Step").sublist("Augmented Lagrangian").set("Initial Penalty Parameter", (double) 1e1); parlist->sublist("Step").sublist("Augmented Lagrangian").set("Penalty Parameter Growth Factor", (double) 1e1); parlist->sublist("Step").sublist("Augmented Lagrangian").set("Minimum Penalty Parameter Reciprocal", (double) 0.1); parlist->sublist("Step").sublist("Augmented Lagrangian").set("Initial Optimality Tolerance", (double) 1.0); From 77d949f46692d1f817d26f8b06b49d3141e0c3b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CAdrian-Diaz=E2=80=9D?= <“adriandiaz1117@gmail.com”> Date: Fri, 28 Jun 2024 14:00:53 -0600 Subject: [PATCH 008/130] ENH: yaml readin for rol params --- .../example_sgh_opt_region.yaml | 7 + .../example_sgh_opt_restart.yaml | 144 ++++++++++++++++++ .../Optimization_Options.h | 44 +++++- src/Parallel-Solvers/Solver.cpp | 12 +- 4 files changed, 198 insertions(+), 9 deletions(-) create mode 100644 src/Parallel-Solvers/Parallel-Explicit/Example_Yaml_Scripts/example_sgh_opt_restart.yaml diff --git a/src/Parallel-Solvers/Parallel-Explicit/Example_Yaml_Scripts/example_sgh_opt_region.yaml b/src/Parallel-Solvers/Parallel-Explicit/Example_Yaml_Scripts/example_sgh_opt_region.yaml index e739caab6..c6c56ce20 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Example_Yaml_Scripts/example_sgh_opt_region.yaml +++ b/src/Parallel-Solvers/Parallel-Explicit/Example_Yaml_Scripts/example_sgh_opt_region.yaml @@ -106,6 +106,13 @@ optimization_options: optimization_process: topology_optimization optimization_objective: minimize_kinetic_energy use_solve_checkpoints: true + rol_params: + subproblem_algorithm: line_search + initial_constraint_penalty: 1.e-2 + step_tolerance: 1.e-3 + gradient_tolerance: 1.e-3 + constraint_tolerance: 1.e-3 + iteration_limit: 20 objective_regions: - type: box x1: 0.1 diff --git a/src/Parallel-Solvers/Parallel-Explicit/Example_Yaml_Scripts/example_sgh_opt_restart.yaml b/src/Parallel-Solvers/Parallel-Explicit/Example_Yaml_Scripts/example_sgh_opt_restart.yaml new file mode 100644 index 000000000..7d139f46d --- /dev/null +++ b/src/Parallel-Solvers/Parallel-Explicit/Example_Yaml_Scripts/example_sgh_opt_restart.yaml @@ -0,0 +1,144 @@ +num_dims: 3 +dynamic_options: + time_final: 0.1 + dt_min: 1.e-8 + dt_max: 1.e-3 + dt_start: 1.e-5 + cycle_stop: 2000000 + output_time_sequence_level: extreme + print_cycle: 20 + +num_dims: 3 +input_options: + mesh_file_format: vtk + mesh_file_name: VTK0.vtk + zero_index_base: true + topology_optimization_restart: true + +output_options: + timer_output_level: thorough + output_file_format: vtk + graphics_step: 0.25 + write_initial: true + write_final: false + optimization_restart_file: true + +fea_module_parameters: + - type: SGH + material_id: 0 + boundary_conditions: + # Tag X plane + - surface: + type: x_plane + plane_position: 0.0 + type: reflected + + # Tag Y plane + - surface: + type: y_plane + plane_position: 0.0 + type: reflected + + # Tag Z plane + - surface: + type: z_plane + plane_position: 0.0 + type: reflected + + # Tag X plane + - surface: + type: x_plane + plane_position: 1.2 + type: reflected + + # Tag Y plane + - surface: + type: y_plane + plane_position: 1.2 + type: reflected + + # Tag Z plane + - surface: + type: z_plane + plane_position: 1.2 + type: reflected + +materials: + - id: 0 + eos_model: ideal_gas + strength_model: none + elastic_modulus: 10 + poisson_ratio: 0.3 + maximum_limiter: true + q1: 1.0 + q2: 1.33 + q1ex: 1.0 + q2ex: 1.33 + eos_global_vars: + - 1.666666666666667 + - 1.0E-14 + - 1.0 + +regions: + - volume: + type: global + material_id: 0 + den: 1.0 + sie: 1.0e-10 + + velocity: cartesian + u: 0 + v: 0 + w: 0 + # energy source initial conditions + - volume: + type: sphere + radius1: 0.0 + radius2: 0.0375 + material_id: 0 + den: 1.0 + ie: 0.25833839995946534 + + velocity: cartesian + u: 0.0 + v: 0.0 + w: 0.0 + + +optimization_options: + optimization_process: topology_optimization + optimization_objective: minimize_kinetic_energy + use_gradient_tally: true + use_solve_checkpoints: true + disable_forward_solve_output: true + optimization_output_freq: 20 + optimization_parameters_xml_file: false + xml_parameters_file_name: "optimization_parameters.xml" + rol_params: + subproblem_algorithm: line_search + initial_constraint_penalty: 1.e-2 + step_tolerance: 1.e-3 + gradient_tolerance: 1.e-3 + constraint_tolerance: 1.e-3 + iteration_limit: 20 + objective_regions: + - type: box + x1: 0.1 + x2: 0.6 + y1: 0.1 + y2: 0.6 + z1: 0.1 + z2: 0.6 + density_epsilon: 0.1 + variable_outer_shell: true + constraints: + - type: mass + relation: equality + value: 0.40 + volume_bound_constraints: + - volume: + type: sphere + radius1: 0.0 + radius2: 0.225 + lower_density_bound: 1.0 + upper_density_bound: 1.0 diff --git a/src/Parallel-Solvers/Simulation_Parameters/Optimization_Options.h b/src/Parallel-Solvers/Simulation_Parameters/Optimization_Options.h index c8d97cc6f..3d65fbfb5 100644 --- a/src/Parallel-Solvers/Simulation_Parameters/Optimization_Options.h +++ b/src/Parallel-Solvers/Simulation_Parameters/Optimization_Options.h @@ -29,6 +29,7 @@ SERIALIZABLE_ENUM(TO_MODULE_TYPE, SERIALIZABLE_ENUM(OPTIMIZATION_PROCESS, none, topology_optimization, shape_optimization) +SERIALIZABLE_ENUM(ROL_SUBPROBLEM_ALGORITHM, trust_region, line_search) SERIALIZABLE_ENUM(OPTIMIZATION_OBJECTIVE, none, minimize_kinetic_energy, multi_objective, minimize_compliance, minimize_thermal_resistance, maximize_compliance, maximize_kinetic_energy, maximize_thermal_resistance) @@ -95,6 +96,42 @@ struct MultiObjectiveModule { YAML_ADD_REQUIRED_FIELDS_FOR(MultiObjectiveModule, type, weight_coefficient) IMPL_YAML_SERIALIZABLE_FOR(MultiObjectiveModule, type, weight_coefficient) +//ROL options read in struct +struct ROL_Params: Yaml::DerivedFields { + ROL_SUBPROBLEM_ALGORITHM subproblem_algorithm = ROL_SUBPROBLEM_ALGORITHM::trust_region; + double initial_constraint_penalty = 1e1; + double step_tolerance = 1e-5; + double gradient_tolerance = 1e-5; + double constraint_tolerance = 1e-5; + int iteration_limit = 100; + + std::string subproblem_algorithm_string; + + void validate() { + if (iteration_limit<=0) { + throw Yaml::ConfigurationException("iteration limit setting cannot be less than or equal to 0"); + } + } + + void derive() { + switch(subproblem_algorithm) { + case ROL_SUBPROBLEM_ALGORITHM::line_search: + subproblem_algorithm_string = "Line Search"; + break; + case ROL_SUBPROBLEM_ALGORITHM::trust_region: + subproblem_algorithm_string = "Trust Region"; + break; + default: + throw std::runtime_error("Unsupported rol subproblem algorithm through yaml; try xml input"); + } + } +}; + +IMPL_YAML_SERIALIZABLE_FOR(ROL_Params, + subproblem_algorithm, initial_constraint_penalty, step_tolerance, constraint_tolerance, + gradient_tolerance, iteration_limit +) + struct Optimization_Options: Yaml::DerivedFields { OPTIMIZATION_PROCESS optimization_process = OPTIMIZATION_PROCESS::none; OPTIMIZATION_OBJECTIVE optimization_objective = OPTIMIZATION_OBJECTIVE::none; @@ -102,6 +139,7 @@ struct Optimization_Options: Yaml::DerivedFields { std::vector volume_bound_constraints; DCArrayKokkos optimization_bound_constraint_volumes; std::vector objective_regions; + ROL_Params rol_params; DCArrayKokkos optimization_objective_regions; bool maximize_flag = false; bool normalized_objective = false; @@ -119,9 +157,9 @@ struct Optimization_Options: Yaml::DerivedFields { double shell_density = 1; //contraint value for outer shell of model real_t objective_normalization_constant = 0; //allows a user specified normalization of the objective; default is initial value size_t num_solve_checkpoints = 10; //number of checkpoints to store explicit solve solutions for adjoint solves - bool use_solve_checkpoints = false; //when false; all timesteps of explicit solves are stored for adjoint solves; expensive + bool use_solve_checkpoints = true; //when false; all timesteps of explicit solves are stored for adjoint solves; expensive bool use_gradient_tally = false; //tallies gradient in tandem with the time sequence solving for the adjoint vectors - bool optimization_parameters_xml_file = true; + bool optimization_parameters_xml_file = false; std::string xml_parameters_file_name = "optimization_parameters.xml"; MULTI_OBJECTIVE_STRUCTURE multi_objective_structure = MULTI_OBJECTIVE_STRUCTURE::linear; @@ -150,5 +188,5 @@ IMPL_YAML_SERIALIZABLE_FOR(Optimization_Options, multi_objective_modules, multi_objective_structure, density_filter, retain_outer_shell, variable_outer_shell, shell_density, objective_normalization_constant, num_solve_checkpoints, use_solve_checkpoints, use_gradient_tally, disable_forward_solve_output, - optimization_parameters_xml_file, xml_parameters_file_name + optimization_parameters_xml_file, xml_parameters_file_name, rol_params ) diff --git a/src/Parallel-Solvers/Solver.cpp b/src/Parallel-Solvers/Solver.cpp index 403da2f6f..5c11f2647 100644 --- a/src/Parallel-Solvers/Solver.cpp +++ b/src/Parallel-Solvers/Solver.cpp @@ -3988,7 +3988,7 @@ void Solver::set_rol_params(Teuchos::RCP parlist) parlist->sublist("Step").sublist("Augmented Lagrangian").set("Objective Scaling", (double) 1e0); parlist->sublist("Step").sublist("Augmented Lagrangian").set("Constraint Scaling", (double) 1e0); parlist->sublist("Step").sublist("Augmented Lagrangian").set("Use Default Initial Penalty Parameter", false); - parlist->sublist("Step").sublist("Augmented Lagrangian").set("Initial Penalty Parameter", (double) 1e1); + parlist->sublist("Step").sublist("Augmented Lagrangian").set("Initial Penalty Parameter", simparam.optimization_options.rol_params.initial_constraint_penalty); parlist->sublist("Step").sublist("Augmented Lagrangian").set("Penalty Parameter Growth Factor", (double) 1e1); parlist->sublist("Step").sublist("Augmented Lagrangian").set("Minimum Penalty Parameter Reciprocal", (double) 0.1); parlist->sublist("Step").sublist("Augmented Lagrangian").set("Initial Optimality Tolerance", (double) 1.0); @@ -4000,7 +4000,7 @@ void Solver::set_rol_params(Teuchos::RCP parlist) parlist->sublist("Step").sublist("Augmented Lagrangian").set("Feasibility Tolerance Decrease Exponent", (double) 0.9); parlist->sublist("Step").sublist("Augmented Lagrangian").set("Print Intermediate Optimization History", false); - parlist->sublist("Step").sublist("Augmented Lagrangian").set("Subproblem Step Type", "Trust Region"); + parlist->sublist("Step").sublist("Augmented Lagrangian").set("Subproblem Step Type", simparam.optimization_options.rol_params.subproblem_algorithm_string); parlist->sublist("Step").sublist("Augmented Lagrangian").set("Subproblem Iteration Limit", (int) 20); parlist->sublist("Step").sublist("Moreau-Yosida Penalty").set("Initial Penalty Parameter", (double) 1e-9); @@ -4024,10 +4024,10 @@ void Solver::set_rol_params(Teuchos::RCP parlist) parlist->sublist("Step").sublist("Bundle").set("Cutting Plane Iteration Limit", (int) 1000); - parlist->sublist("Status Test").set("Gradient Tolerance", (double) 1e-5); - parlist->sublist("Status Test").set("Constraint Tolerance", (double) 1e-5); - parlist->sublist("Status Test").set("Step Tolerance", (double) 1e-5); - parlist->sublist("Status Test").set("Iteration Limit", (int) 100); + parlist->sublist("Status Test").set("Gradient Tolerance", simparam.optimization_options.rol_params.gradient_tolerance); + parlist->sublist("Status Test").set("Constraint Tolerance", simparam.optimization_options.rol_params.constraint_tolerance); + parlist->sublist("Status Test").set("Step Tolerance", simparam.optimization_options.rol_params.step_tolerance); + parlist->sublist("Status Test").set("Iteration Limit", simparam.optimization_options.rol_params.iteration_limit); parlist->sublist("Status Test").set("Use Relative Tolerances", true); } From a37bf0fe8b7818e68859184909f40a34f0b2bcdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CAdrian-Diaz=E2=80=9D?= <“adriandiaz1117@gmail.com”> Date: Mon, 1 Jul 2024 18:21:54 -0600 Subject: [PATCH 009/130] STYLE: Contain and comment debug prints, add imbalance printout --- .../Implicit-Lagrange/Implicit_Solver.cpp | 17 ++++++++++++++--- .../Parallel-Explicit/Explicit_Solver.cpp | 17 ++++++++++++++--- .../SGH_Solver/src/setup_sgh.cpp | 9 +++++++-- src/Parallel-Solvers/Solver.cpp | 8 ++++---- 4 files changed, 39 insertions(+), 12 deletions(-) diff --git a/src/Parallel-Solvers/Implicit-Lagrange/Implicit_Solver.cpp b/src/Parallel-Solvers/Implicit-Lagrange/Implicit_Solver.cpp index 04634503a..e8fad7eea 100644 --- a/src/Parallel-Solvers/Implicit-Lagrange/Implicit_Solver.cpp +++ b/src/Parallel-Solvers/Implicit-Lagrange/Implicit_Solver.cpp @@ -184,8 +184,19 @@ void Implicit_Solver::run(){ //equate pointers for this solver initial_node_coords_distributed = node_coords_distributed; all_initial_node_coords_distributed = all_node_coords_distributed; + + //print element imbalance stats + int rnum_global_sum = 0; + MPI_Allreduce(&rnum_elem, &rnum_global_sum, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); + double local_imbalance, max_imbalance, avg_elem; + max_imbalance = 0; + avg_elem = rnum_global_sum/((double) nranks); + local_imbalance = rnum_elem/avg_elem; + MPI_Allreduce(&local_imbalance, &max_imbalance, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); + + *fos << "Element imbalance: " << max_imbalance << std::endl; - std::cout << "Num elements on process " << myrank << " = " << rnum_elem << std::endl; + //std::cout << "Num elements on process " << myrank << " = " << rnum_elem << std::endl; init_clock(); @@ -497,7 +508,7 @@ void Implicit_Solver::read_mesh_ansys_dat(const char *MESH){ //old swage method //mesh->init_nodes(local_nrows); // add 1 for index starting at 1 - std::cout << "Num nodes assigned to task " << myrank << " = " << nlocal_nodes << std::endl; + //std::cout << "Num nodes assigned to task " << myrank << " = " << nlocal_nodes << std::endl; // read the initial mesh coordinates // x-coords @@ -1677,7 +1688,7 @@ void Implicit_Solver::init_boundaries(){ std::cout << "Starting boundary patch setup" << std::endl <assign(*node_coords_distributed); - std::cout << "Num elements on process " << myrank << " = " << rnum_elem << std::endl; + //std::cout << "Num elements on process " << myrank << " = " << rnum_elem << std::endl; //initialize timing init_clock(); @@ -526,7 +537,7 @@ void Explicit_Solver::read_mesh_ansys_dat(const char *MESH){ //old swage method //mesh->init_nodes(local_nrows); // add 1 for index starting at 1 - std::cout << "Num nodes assigned to task " << myrank << " = " << nlocal_nodes << std::endl; + //std::cout << "Num nodes assigned to task " << myrank << " = " << nlocal_nodes << std::endl; // read the initial mesh coordinates // x-coords @@ -1572,7 +1583,7 @@ void Explicit_Solver::init_boundaries(){ Local_Index_Boundary_Patches.update_device(); //std::cout << "Done with boundary patch setup" << std::endl <myrank==0){ + printf("Num BC's = %lu\n", num_bcs); + } // patch ids in bdy set bdy_patches_in_set = mesh->bdy_patches_in_set; @@ -193,7 +196,8 @@ void FEA_Module_SGH::setup() nodes_in_patch = mesh->nodes_in_patch; elems_in_patch = mesh->elems_in_patch; } - + +#ifdef DEBUG // loop over BCs for (size_t this_bdy = 0; this_bdy < num_bcs; this_bdy++) { RUN_CLASS({ @@ -203,6 +207,7 @@ void FEA_Module_SGH::setup() }); Kokkos::fence(); } // end for +#endif // elem_mat_id needs to be initialized before initialization of material models for (int f_id = 0; f_id < num_fills; f_id++) { diff --git a/src/Parallel-Solvers/Solver.cpp b/src/Parallel-Solvers/Solver.cpp index 5c11f2647..e52ea4947 100644 --- a/src/Parallel-Solvers/Solver.cpp +++ b/src/Parallel-Solvers/Solver.cpp @@ -374,7 +374,7 @@ void Solver::read_mesh_ensight(const char* MESH) // old swage method // mesh->init_nodes(local_nrows); // add 1 for index starting at 1 - std::cout << "Num nodes assigned to task " << myrank << " = " << nlocal_nodes << std::endl; + //std::cout << "Num nodes assigned to task " << myrank << " = " << nlocal_nodes << std::endl; // read the initial mesh coordinates // x-coords @@ -1107,7 +1107,7 @@ void Solver::read_mesh_vtk(const char* MESH) // old swage method // mesh->init_nodes(local_nrows); // add 1 for index starting at 1 - std::cout << "Num nodes assigned to task " << myrank << " = " << nlocal_nodes << std::endl; + //std::cout << "Num nodes assigned to task " << myrank << " = " << nlocal_nodes << std::endl; // read the initial mesh coordinates // x-coords @@ -1900,7 +1900,7 @@ void Solver::read_mesh_tecplot(const char* MESH) // old swage method // mesh->init_nodes(local_nrows); // add 1 for index starting at 1 - std::cout << "Num nodes assigned to task " << myrank << " = " << nlocal_nodes << std::endl; + //std::cout << "Num nodes assigned to task " << myrank << " = " << nlocal_nodes << std::endl; // read the initial mesh coordinates // x-coords @@ -2541,7 +2541,7 @@ void Solver::read_mesh_abaqus_inp(const char *MESH){ //old swage method //mesh->init_nodes(local_nrows); // add 1 for index starting at 1 - std::cout << "Num nodes assigned to task " << myrank << " = " << nlocal_nodes << std::endl; + //std::cout << "Num nodes assigned to task " << myrank << " = " << nlocal_nodes << std::endl; // read the initial mesh coordinates // x-coords From 1df5216cc09247849261fb379a011f945103e64b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CAdrian-Diaz=E2=80=9D?= <“adriandiaz1117@gmail.com”> Date: Mon, 8 Jul 2024 22:07:14 -0600 Subject: [PATCH 010/130] DOC: update references to example yaml files and small fixes --- README.md | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 373200185..3dc76f04d 100644 --- a/README.md +++ b/README.md @@ -39,8 +39,16 @@ After installing the finite element solvers, it gives you access to `fierro-mesh ``` fierro-parallel-explicit input.yaml ``` -A sample input file for the explicit finite element solver is here: -`./src/Parallel-Solvers/Parallel-Explicit/example_simple.yaml` +Sample yaml input files for the explicit finite element solver can be found at: +`./src/Parallel-Solvers/Parallel-Explicit/Example_Yaml_Scripts` + +The implicit solver and topology optimization modules can be called using: +``` +fierro-parallel-implicit input.yaml +``` + +Sample yaml input files for the implicit finite element solver and topology optimization problem setups can be found at: +`./src/Parallel-Solvers/Implicit-Lagrange/Example_Yaml_Scripts` The GUI can be run in the anaconda enviroment by typing: ``` @@ -139,14 +147,15 @@ If you are new to Fierro development and don't have push access to the repositor 3. [Push](https://docs.github.com/en/get-started/using-git/pushing-commits-to-a-remote-repository) the branch to your GitHub fork. 4. Build and test your changes. 5. Update any necessary documentation. -6. Create a [Pull Request](https://github.com/lanl/Fierro/pulls). +6. Synchronize your branch with changes made to the upstream repository since the last merge/fork. +7. Create a [Pull Request](https://github.com/lanl/Fierro/pulls). This corresponds to the **Fork & Pull Model** described in the [GitHub collaborative development](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/getting-started/about-collaborative-development-models) documentation. ## Integrating a PR Integrating your contributions to Fierro is relatively straightforward; here is the checklist: -- All test pass +- All tests pass - The changes build with no new compiler warnings/errors - All feedback has been addressed - Consensus is reached. This usually means that at least two reviewers approved the changes (or added a `LGTM` comment) and at least one business day passed without anyone objecting. `LGTM` is an acronym for Looks Good to Me. From 87799897e191482854362944599dc3cec87bc926 Mon Sep 17 00:00:00 2001 From: 397740 <397740@win.lanl.gov> Date: Tue, 9 Jul 2024 11:18:51 -0600 Subject: [PATCH 011/130] ENH: Added yaml workflow files for testing installation of Fierro via anaconda and some example inputs --- .etc/conda_build_environment.yaml | 1 + .github/workflows/fierro-test-no-implicit.yml | 83 +++++++++++++++++ .github/workflows/fierro-test.yml | 91 +++++++++++++++++++ 3 files changed, 175 insertions(+) create mode 100644 .github/workflows/fierro-test-no-implicit.yml create mode 100644 .github/workflows/fierro-test.yml diff --git a/.etc/conda_build_environment.yaml b/.etc/conda_build_environment.yaml index a4bf88c4c..3db6dbf36 100644 --- a/.etc/conda_build_environment.yaml +++ b/.etc/conda_build_environment.yaml @@ -7,3 +7,4 @@ dependencies: - boa # For building - conda-build - anaconda-client # For uploading + - python=3.8 diff --git a/.github/workflows/fierro-test-no-implicit.yml b/.github/workflows/fierro-test-no-implicit.yml new file mode 100644 index 000000000..2751db443 --- /dev/null +++ b/.github/workflows/fierro-test-no-implicit.yml @@ -0,0 +1,83 @@ +name: fierro-test-no-implicit + +# Workflow runs on new push to main +on: + push: + branches: + - main + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - name: update the package list + run: + sudo apt-get update + shell: bash + - name: Install build tools + run: + sudo apt-get install build-essential + shell: bash + - name: Install LaPack + run: + sudo apt-get install libblas-dev liblapack-dev + shell: bash + - name: Install MPI + run: | + sudo apt-get install libopenmpi-dev openmpi-bin + echo "MPI_OPTS=--oversubscribe" >> $GITHUB_ENV + - name: Install cmake + run: + sudo apt-get -y install cmake + shell: bash + - name: Install python + run: + sudo apt-get install python3.6 + shell: bash + + - name: Set up Miniconda + uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: true + python-version: 3.8 + + - name: Create conda environment + run: conda env create -f ./.etc/conda_build_environment.yaml + + - name: Initialize conda + run: conda init bash + + - name: Activate conda environment, install Fierro anaconda packages + run: | + source ~/.bashrc + conda activate build-env + conda install --solver=classic conda-forge::conda-libmamba-solver conda-forge::libmamba conda-forge::libmambapy conda-forge::libarchive + conda install -c conda-forge -c fierromechanics fierro-cpu + conda install -c conda-forge -c fierromechanics evpfft + conda install -c conda-forge -c fierromechanics fierro_gui + + - name: Verify package dependencies + run: | + conda info --envs + conda list + + - name: Test fierro-parallel-explicit + run: | + source ~/.bashrc + conda activate build-env + fierro-parallel-explicit ./src/Parallel-Solvers/Parallel-Explicit/Example_Yaml_Scripts/example_simple.yaml + shell: bash + + - name: Test fierro-mesh-builder + run: | + source ~/.bashrc + conda activate build-env + fierro-mesh-builder ./src/Mesh-Builder/examples/mesh_Box_16_p3.yaml + shell: bash + + diff --git a/.github/workflows/fierro-test.yml b/.github/workflows/fierro-test.yml new file mode 100644 index 000000000..cd08820f3 --- /dev/null +++ b/.github/workflows/fierro-test.yml @@ -0,0 +1,91 @@ +name: fierro-test + +# Workflow runs on new push to main +on: + push: + branches: + - main + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - name: update the package list + run: + sudo apt-get update + shell: bash + - name: Install build tools + run: + sudo apt-get install build-essential + shell: bash + - name: Install LaPack + run: + sudo apt-get install libblas-dev liblapack-dev + shell: bash + - name: Install MPI + run: | + sudo apt-get install libopenmpi-dev openmpi-bin + echo "MPI_OPTS=--oversubscribe" >> $GITHUB_ENV + - name: Install cmake + run: + sudo apt-get -y install cmake + shell: bash + - name: Install python + run: + sudo apt-get install python3.6 + shell: bash + + - name: Set up Miniconda + uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: true + python-version: 3.8 + + - name: Create conda environment + run: conda env create -f ./.etc/conda_build_environment.yaml + + - name: Initialize conda + run: conda init bash + + - name: Activate conda environment, install Fierro anaconda packages + run: | + source ~/.bashrc + conda activate build-env + conda install --solver=classic conda-forge::conda-libmamba-solver conda-forge::libmamba conda-forge::libmambapy conda-forge::libarchive + conda install -c conda-forge -c fierromechanics fierro-cpu + conda install -c conda-forge -c fierromechanics evpfft + conda install -c conda-forge -c fierromechanics fierro_gui + + - name: Verify package dependencies + run: | + conda info --envs + conda list + + - name: Test fierro-parallel-explicit + run: | + source ~/.bashrc + conda activate build-env + fierro-parallel-explicit ./src/Parallel-Solvers/Parallel-Explicit/Example_Yaml_Scripts/example_simple.yaml + shell: bash + + - name: Test fierro-parallel-implicit + run: | + source ~/.bashrc + conda activate build-env + fierro-parallel-implicit ./Example_Yaml_Scripts/example_simple.yaml + working-directory: ./src/Parallel-Solvers/Implicit-Lagrange + shell: bash + + - name: Test fierro-mesh-builder + run: | + source ~/.bashrc + conda activate build-env + fierro-mesh-builder ./src/Mesh-Builder/examples/mesh_Box_16_p3.yaml + shell: bash + + From d82620bc2c4287b4267d0f707de552d58c760a80 Mon Sep 17 00:00:00 2001 From: 397740 <397740@win.lanl.gov> Date: Tue, 9 Jul 2024 11:43:03 -0600 Subject: [PATCH 012/130] ENH: Added yaml workflow files for testing installation of Fierro via anaconda and some example input --- .github/workflows/fierro-test.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/fierro-test.yml b/.github/workflows/fierro-test.yml index cd08820f3..0f3619f8b 100644 --- a/.github/workflows/fierro-test.yml +++ b/.github/workflows/fierro-test.yml @@ -73,19 +73,21 @@ jobs: fierro-parallel-explicit ./src/Parallel-Solvers/Parallel-Explicit/Example_Yaml_Scripts/example_simple.yaml shell: bash - - name: Test fierro-parallel-implicit + - name: Test fierro-mesh-builder run: | source ~/.bashrc conda activate build-env - fierro-parallel-implicit ./Example_Yaml_Scripts/example_simple.yaml - working-directory: ./src/Parallel-Solvers/Implicit-Lagrange + fierro-mesh-builder ./src/Mesh-Builder/examples/mesh_Box_16_p3.yaml shell: bash - - - name: Test fierro-mesh-builder + + - name: Test fierro-parallel-implicit run: | source ~/.bashrc conda activate build-env - fierro-mesh-builder ./src/Mesh-Builder/examples/mesh_Box_16_p3.yaml + fierro-parallel-implicit ./Example_Yaml_Scripts/example_simple.yaml + working-directory: ./src/Parallel-Solvers/Implicit-Lagrange shell: bash + + From f7996289bffa59d0eb55f063bbc26c5d745eb668 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CAdrian-Diaz=E2=80=9D?= <“adriandiaz1117@gmail.com”> Date: Tue, 9 Jul 2024 12:53:15 -0600 Subject: [PATCH 013/130] BUG: prevent optimization array segfault --- .../Parallel-Explicit/SGH_Solver/src/FEA_Module_SGH.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/FEA_Module_SGH.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/FEA_Module_SGH.cpp index 99d9866db..0d4ee757e 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/FEA_Module_SGH.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/FEA_Module_SGH.cpp @@ -938,6 +938,7 @@ void FEA_Module_SGH::sgh_solve() std::set::iterator current_checkpoint, last_raised_checkpoint, dispensable_checkpoint, search_end; int last_raised_level = 0; bool dispensable_found = false; + bool optimization_on = simparam->topology_optimization_on||simparam->shape_optimization_on; num_active_checkpoints = 0; @@ -1218,7 +1219,7 @@ void FEA_Module_SGH::sgh_solve() rnum_elem, nall_nodes); - if(use_solve_checkpoints){ + if(use_solve_checkpoints&&optimization_on){ previous_node_velocities_distributed->assign(*all_node_velocities_distributed); } From ac36aa84fa4e4e1c5ff582bc86f00a0bac5f4007 Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Wed, 10 Jul 2024 10:08:34 -0600 Subject: [PATCH 014/130] Create notification.yml --- .github/workflows/notification.yml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .github/workflows/notification.yml diff --git a/.github/workflows/notification.yml b/.github/workflows/notification.yml new file mode 100644 index 000000000..9de22002a --- /dev/null +++ b/.github/workflows/notification.yml @@ -0,0 +1,4 @@ +name: test notification workflow +on: workflow_dispatch + + From e19528b2e281a15dde1f29590dd560414b7d2ccc Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Wed, 10 Jul 2024 10:09:50 -0600 Subject: [PATCH 015/130] Update notification.yml --- .github/workflows/notification.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/notification.yml b/.github/workflows/notification.yml index 9de22002a..6599cf924 100644 --- a/.github/workflows/notification.yml +++ b/.github/workflows/notification.yml @@ -1,4 +1,18 @@ name: test notification workflow on: workflow_dispatch +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - name: update the package list + run: + sudo apt-get update + shell: bash + From 21711264ef58030635f2d2db39ee8360d99e5863 Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Wed, 10 Jul 2024 10:11:09 -0600 Subject: [PATCH 016/130] Update fierro-test.yml --- .github/workflows/fierro-test.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/fierro-test.yml b/.github/workflows/fierro-test.yml index 0f3619f8b..407962e1c 100644 --- a/.github/workflows/fierro-test.yml +++ b/.github/workflows/fierro-test.yml @@ -1,10 +1,11 @@ name: fierro-test -# Workflow runs on new push to main -on: - push: - branches: - - main +# Workflow runs following successful anaconda distribution +on: + workflow_run: + workflows: ["test notification workflow"] + types: + - completed jobs: build: From 4598a317692501695d5895b31d76cb15c23d129b Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Wed, 10 Jul 2024 10:12:24 -0600 Subject: [PATCH 017/130] Update fierro-test-no-implicit.yml --- .github/workflows/fierro-test-no-implicit.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/fierro-test-no-implicit.yml b/.github/workflows/fierro-test-no-implicit.yml index 2751db443..a8e16f1d4 100644 --- a/.github/workflows/fierro-test-no-implicit.yml +++ b/.github/workflows/fierro-test-no-implicit.yml @@ -1,10 +1,11 @@ name: fierro-test-no-implicit -# Workflow runs on new push to main -on: - push: - branches: - - main +# Workflow runs on following successful conda distribution +on: + workflow_run: + workflows: ["test notification workflow"] + types: + - completed jobs: build: From 5b3043575892832255b4dc713656636086a222ca Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Wed, 10 Jul 2024 10:19:10 -0600 Subject: [PATCH 018/130] Create test2.yml --- .github/workflows/test2.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 .github/workflows/test2.yml diff --git a/.github/workflows/test2.yml b/.github/workflows/test2.yml new file mode 100644 index 000000000..7e30b113d --- /dev/null +++ b/.github/workflows/test2.yml @@ -0,0 +1,16 @@ +name: test notification workflow2 +on: workflow_dispatch + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - name: update the package list + run: + sudo apt-get update + shell: bash From 0ebe7cd60594bcf6f3eac8fd3e15cb711b8aa9c2 Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Wed, 10 Jul 2024 10:19:35 -0600 Subject: [PATCH 019/130] Update fierro-test-no-implicit.yml --- .github/workflows/fierro-test-no-implicit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/fierro-test-no-implicit.yml b/.github/workflows/fierro-test-no-implicit.yml index a8e16f1d4..ee7255dc1 100644 --- a/.github/workflows/fierro-test-no-implicit.yml +++ b/.github/workflows/fierro-test-no-implicit.yml @@ -3,7 +3,7 @@ name: fierro-test-no-implicit # Workflow runs on following successful conda distribution on: workflow_run: - workflows: ["test notification workflow"] + workflows: ["test notification workflow, test notification workflow2"] types: - completed From 8f3f5e4d9b5768ddfdec01e4f55f27b7902341c7 Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Wed, 10 Jul 2024 10:21:20 -0600 Subject: [PATCH 020/130] Update fierro-test-no-implicit.yml --- .github/workflows/fierro-test-no-implicit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/fierro-test-no-implicit.yml b/.github/workflows/fierro-test-no-implicit.yml index ee7255dc1..a8e16f1d4 100644 --- a/.github/workflows/fierro-test-no-implicit.yml +++ b/.github/workflows/fierro-test-no-implicit.yml @@ -3,7 +3,7 @@ name: fierro-test-no-implicit # Workflow runs on following successful conda distribution on: workflow_run: - workflows: ["test notification workflow, test notification workflow2"] + workflows: ["test notification workflow"] types: - completed From c00e0f21e0a13ec0aa94a3eb0fe95bba8393f7b8 Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Wed, 10 Jul 2024 10:26:52 -0600 Subject: [PATCH 021/130] Update fierro-test-no-implicit.yml --- .github/workflows/fierro-test-no-implicit.yml | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/.github/workflows/fierro-test-no-implicit.yml b/.github/workflows/fierro-test-no-implicit.yml index a8e16f1d4..2f5b572f2 100644 --- a/.github/workflows/fierro-test-no-implicit.yml +++ b/.github/workflows/fierro-test-no-implicit.yml @@ -1,17 +1,24 @@ name: fierro-test-no-implicit # Workflow runs on following successful conda distribution -on: - workflow_run: - workflows: ["test notification workflow"] - types: - - completed +#on: + #workflow_run: + #workflows: ["test notification workflow"] + #types: + #- completed + +on: workflow_dispatch jobs: - build: + test1: + uses: Fierro/.github/workflows/notification.yml@main + test2: + uses: Fierro/.github/workflows/test.yml@main + + build: runs-on: ubuntu-latest - + needs: [test1, test2] steps: - uses: actions/checkout@v4 with: From 148bb1c8086daf73c2e054e8edbd16167ae6569f Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Wed, 10 Jul 2024 10:28:51 -0600 Subject: [PATCH 022/130] Update fierro-test-no-implicit.yml --- .github/workflows/fierro-test-no-implicit.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/fierro-test-no-implicit.yml b/.github/workflows/fierro-test-no-implicit.yml index 2f5b572f2..7c5058d39 100644 --- a/.github/workflows/fierro-test-no-implicit.yml +++ b/.github/workflows/fierro-test-no-implicit.yml @@ -11,10 +11,10 @@ on: workflow_dispatch jobs: test1: - uses: Fierro/.github/workflows/notification.yml@main + uses: ./.github/workflows/notification.yml test2: - uses: Fierro/.github/workflows/test.yml@main + uses: ./.github/workflows/test.yml build: runs-on: ubuntu-latest From bc1c25bb82380d29f1e9de77e2fdf0eb7b40a3f2 Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Wed, 10 Jul 2024 10:32:42 -0600 Subject: [PATCH 023/130] Update notification.yml --- .github/workflows/notification.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/notification.yml b/.github/workflows/notification.yml index 6599cf924..9f2841168 100644 --- a/.github/workflows/notification.yml +++ b/.github/workflows/notification.yml @@ -1,5 +1,6 @@ name: test notification workflow -on: workflow_dispatch +on: + workflow_call jobs: build: From 04bade51e783228d6760a9e386c598bbaa7dc459 Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Wed, 10 Jul 2024 10:33:39 -0600 Subject: [PATCH 024/130] Update and rename test2.yml to notification2.yml --- .github/workflows/{test2.yml => notification2.yml} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename .github/workflows/{test2.yml => notification2.yml} (92%) diff --git a/.github/workflows/test2.yml b/.github/workflows/notification2.yml similarity index 92% rename from .github/workflows/test2.yml rename to .github/workflows/notification2.yml index 7e30b113d..04454af51 100644 --- a/.github/workflows/test2.yml +++ b/.github/workflows/notification2.yml @@ -1,5 +1,5 @@ name: test notification workflow2 -on: workflow_dispatch +on: workflow_call jobs: build: From 212b7530a314caddb64fb93c77ba5150f2954ed2 Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Wed, 10 Jul 2024 10:34:05 -0600 Subject: [PATCH 025/130] Update fierro-test-no-implicit.yml --- .github/workflows/fierro-test-no-implicit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/fierro-test-no-implicit.yml b/.github/workflows/fierro-test-no-implicit.yml index 7c5058d39..68ff4d7dd 100644 --- a/.github/workflows/fierro-test-no-implicit.yml +++ b/.github/workflows/fierro-test-no-implicit.yml @@ -14,7 +14,7 @@ jobs: uses: ./.github/workflows/notification.yml test2: - uses: ./.github/workflows/test.yml + uses: ./.github/workflows/notification2.yml build: runs-on: ubuntu-latest From 5b12fcf26db136459aacc70ae8555fc7abeaf92e Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Wed, 10 Jul 2024 11:28:01 -0600 Subject: [PATCH 026/130] Create publish-all.yml --- .github/workflows/publish-all.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .github/workflows/publish-all.yml diff --git a/.github/workflows/publish-all.yml b/.github/workflows/publish-all.yml new file mode 100644 index 000000000..00c654b3c --- /dev/null +++ b/.github/workflows/publish-all.yml @@ -0,0 +1,10 @@ +name: Publish All + +on: workflow_dispatch + +jobs: + test1: + uses: ./.github/workflows/notification.yml + + test2: + uses: ./.github/workflows/notification2.yml From 874c45f9b5e166e7dd8b4e74423ca6a3411eb838 Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Wed, 10 Jul 2024 11:30:45 -0600 Subject: [PATCH 027/130] Update fierro-test-no-implicit.yml --- .github/workflows/fierro-test-no-implicit.yml | 21 ++++++------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/.github/workflows/fierro-test-no-implicit.yml b/.github/workflows/fierro-test-no-implicit.yml index 68ff4d7dd..88dbad84a 100644 --- a/.github/workflows/fierro-test-no-implicit.yml +++ b/.github/workflows/fierro-test-no-implicit.yml @@ -1,24 +1,15 @@ name: fierro-test-no-implicit # Workflow runs on following successful conda distribution -#on: - #workflow_run: - #workflows: ["test notification workflow"] - #types: - #- completed - -on: workflow_dispatch - +on: + workflow_run: + workflows: ["test notification workflow, test notification workflow2"] + types: + - completed + jobs: - test1: - uses: ./.github/workflows/notification.yml - - test2: - uses: ./.github/workflows/notification2.yml - build: runs-on: ubuntu-latest - needs: [test1, test2] steps: - uses: actions/checkout@v4 with: From 792d20bf8fce9c5c9b6c0b7ce948176e0eaf7376 Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Wed, 10 Jul 2024 11:43:00 -0600 Subject: [PATCH 028/130] Update publish-all.yml --- .github/workflows/publish-all.yml | 35 +++++++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/.github/workflows/publish-all.yml b/.github/workflows/publish-all.yml index 00c654b3c..dc98e0e29 100644 --- a/.github/workflows/publish-all.yml +++ b/.github/workflows/publish-all.yml @@ -3,8 +3,35 @@ name: Publish All on: workflow_dispatch jobs: - test1: - uses: ./.github/workflows/notification.yml + elements: + uses: ./.github/workflows/publish-elements.yml - test2: - uses: ./.github/workflows/notification2.yml + evpfft-cpu: + uses: ./.github/workflows/publish-evpfft-cpu.yml + + evpfft-dev: + uses: ./.github/workflows/publish-evpfft-dev.yml + + evpfft-gui: + uses: ./.github/workflows/publish-evpfft-gui.yml + + fierro-cpu: + uses: ./.github/workflows/publish-fierro-cpu.yml + + fierro-dev: + uses: ./.github/workflows/publish-fierro-dev.yml + + fierro-gui: + uses: ./.github/workflows/publish-fierro-gui.yml + + heffte-cpu: + uses: ./.github/workflows/publish-heffte-cpu.yml + + heffte-cuda: + uses: ./.github/workflows/publish-heffte-cuda.yml + + trilinos: + uses: ./.github/workflows/publish-trillinos.yml + + voxelizer: + uses: ./.github/workflows/publish-voxelizer.yml From 511bfe647614d5ac530f9acaf8972a8dd91be292 Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Wed, 10 Jul 2024 11:44:13 -0600 Subject: [PATCH 029/130] Update fierro-test-no-implicit.yml --- .github/workflows/fierro-test-no-implicit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/fierro-test-no-implicit.yml b/.github/workflows/fierro-test-no-implicit.yml index 88dbad84a..9012d922c 100644 --- a/.github/workflows/fierro-test-no-implicit.yml +++ b/.github/workflows/fierro-test-no-implicit.yml @@ -1,4 +1,4 @@ -name: fierro-test-no-implicit +name: Fierro Test (No Implicit) # Workflow runs on following successful conda distribution on: From e8b7fee5b22143ab71da5e5d6b2de86c1819efe3 Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Wed, 10 Jul 2024 11:44:38 -0600 Subject: [PATCH 030/130] Update notification.yml --- .github/workflows/notification.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/notification.yml b/.github/workflows/notification.yml index 9f2841168..e60c04e2e 100644 --- a/.github/workflows/notification.yml +++ b/.github/workflows/notification.yml @@ -1,6 +1,6 @@ name: test notification workflow on: - workflow_call + workflow_dispatch jobs: build: From 5d16b8489a457a4539ab840aa2106a5f0e2a59b7 Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Wed, 10 Jul 2024 11:48:44 -0600 Subject: [PATCH 031/130] Update fierro-test-no-implicit.yml --- .github/workflows/fierro-test-no-implicit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/fierro-test-no-implicit.yml b/.github/workflows/fierro-test-no-implicit.yml index 9012d922c..414dbb51a 100644 --- a/.github/workflows/fierro-test-no-implicit.yml +++ b/.github/workflows/fierro-test-no-implicit.yml @@ -3,7 +3,7 @@ name: Fierro Test (No Implicit) # Workflow runs on following successful conda distribution on: workflow_run: - workflows: ["test notification workflow, test notification workflow2"] + workflows: [test notification workflow, test notification workflow2] types: - completed From bb043651c9deb717c1947a57434693f34aa8765a Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Wed, 10 Jul 2024 11:49:40 -0600 Subject: [PATCH 032/130] Update notification2.yml --- .github/workflows/notification2.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/notification2.yml b/.github/workflows/notification2.yml index 04454af51..7e30b113d 100644 --- a/.github/workflows/notification2.yml +++ b/.github/workflows/notification2.yml @@ -1,5 +1,5 @@ name: test notification workflow2 -on: workflow_call +on: workflow_dispatch jobs: build: From 5081d50e0a5b0def61f7b1223fe5b0e407086731 Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Wed, 10 Jul 2024 11:54:01 -0600 Subject: [PATCH 033/130] Update fierro-test-no-implicit.yml --- .github/workflows/fierro-test-no-implicit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/fierro-test-no-implicit.yml b/.github/workflows/fierro-test-no-implicit.yml index 414dbb51a..da0aece2d 100644 --- a/.github/workflows/fierro-test-no-implicit.yml +++ b/.github/workflows/fierro-test-no-implicit.yml @@ -3,7 +3,7 @@ name: Fierro Test (No Implicit) # Workflow runs on following successful conda distribution on: workflow_run: - workflows: [test notification workflow, test notification workflow2] + workflows: [Publish Test, Publish All, Publish Elements, Publish EVPFFT-CPU, Publish EVPFFT-dev, Publish EVPFFT-GPU, Publish FIERRO GUI, Publish Fierro-CPU, Publish Fierro-dev, Publish Heffte, Publish Heffte CUDA, Publish Trilinos-CPU] types: - completed From 92f5e8b59adcbe3da7921e600aaf1e2e2c44bf84 Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Wed, 10 Jul 2024 11:54:46 -0600 Subject: [PATCH 034/130] Update and rename notification.yml to publish-test.yml --- .github/workflows/{notification.yml => publish-test.yml} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename .github/workflows/{notification.yml => publish-test.yml} (88%) diff --git a/.github/workflows/notification.yml b/.github/workflows/publish-test.yml similarity index 88% rename from .github/workflows/notification.yml rename to .github/workflows/publish-test.yml index e60c04e2e..8b5561645 100644 --- a/.github/workflows/notification.yml +++ b/.github/workflows/publish-test.yml @@ -1,4 +1,4 @@ -name: test notification workflow +name: Publish Test on: workflow_dispatch From 1282bfcfc6be59a9e0c0e4f1fcb88cefd925e271 Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Wed, 10 Jul 2024 12:00:42 -0600 Subject: [PATCH 035/130] Update fierro-test.yml --- .github/workflows/fierro-test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/fierro-test.yml b/.github/workflows/fierro-test.yml index 407962e1c..39c8fd373 100644 --- a/.github/workflows/fierro-test.yml +++ b/.github/workflows/fierro-test.yml @@ -1,9 +1,9 @@ -name: fierro-test +name: Fierro Test -# Workflow runs following successful anaconda distribution +# Workflow runs on following successful conda distribution on: workflow_run: - workflows: ["test notification workflow"] + workflows: [Publish Test, Publish All, Publish Elements, Publish EVPFFT-CPU, Publish EVPFFT-dev, Publish EVPFFT-GPU, Publish FIERRO GUI, Publish Fierro-CPU, Publish Fierro-dev, Publish Heffte, Publish Heffte CUDA, Publish Trilinos-CPU] types: - completed From 550fcd3acf1c691289a9b4c956ecaed21222c9b1 Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Wed, 10 Jul 2024 12:01:36 -0600 Subject: [PATCH 036/130] Delete .github/workflows/notification2.yml --- .github/workflows/notification2.yml | 16 ---------------- 1 file changed, 16 deletions(-) delete mode 100644 .github/workflows/notification2.yml diff --git a/.github/workflows/notification2.yml b/.github/workflows/notification2.yml deleted file mode 100644 index 7e30b113d..000000000 --- a/.github/workflows/notification2.yml +++ /dev/null @@ -1,16 +0,0 @@ -name: test notification workflow2 -on: workflow_dispatch - -jobs: - build: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - with: - submodules: recursive - - name: update the package list - run: - sudo apt-get update - shell: bash From 7db141c280e2cc8413e0ac9cbab95c19ab4f409c Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Wed, 10 Jul 2024 12:08:38 -0600 Subject: [PATCH 037/130] Update fierro-test-no-implicit.yml --- .github/workflows/fierro-test-no-implicit.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/fierro-test-no-implicit.yml b/.github/workflows/fierro-test-no-implicit.yml index da0aece2d..12d2bec12 100644 --- a/.github/workflows/fierro-test-no-implicit.yml +++ b/.github/workflows/fierro-test-no-implicit.yml @@ -14,6 +14,9 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive + - name: print trigger event + run: | + echo "::notice:: inputs ${{ github.event_name }}" - name: update the package list run: sudo apt-get update From f1ec00de31ca98a978ddd32afa7d890d8feedb55 Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Wed, 10 Jul 2024 12:20:06 -0600 Subject: [PATCH 038/130] Update fierro-test-no-implicit.yml --- .github/workflows/fierro-test-no-implicit.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/fierro-test-no-implicit.yml b/.github/workflows/fierro-test-no-implicit.yml index 12d2bec12..b8faa517d 100644 --- a/.github/workflows/fierro-test-no-implicit.yml +++ b/.github/workflows/fierro-test-no-implicit.yml @@ -16,7 +16,9 @@ jobs: submodules: recursive - name: print trigger event run: | - echo "::notice:: inputs ${{ github.event_name }}" + - run: echo "workflow triggering event name is:" ${{ github.event_name }} + - run: echo "workflow triggering event type is:" ${{ github.event.action }} + - run: echo "workflow triggering event user is:" ${{ github.event.actor }} - name: update the package list run: sudo apt-get update From a9695a202d17f61100580972b57bd4ac0ae00a0b Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Wed, 10 Jul 2024 12:25:40 -0600 Subject: [PATCH 039/130] Update fierro-test-no-implicit.yml --- .github/workflows/fierro-test-no-implicit.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/fierro-test-no-implicit.yml b/.github/workflows/fierro-test-no-implicit.yml index b8faa517d..e7a02a40d 100644 --- a/.github/workflows/fierro-test-no-implicit.yml +++ b/.github/workflows/fierro-test-no-implicit.yml @@ -17,8 +17,8 @@ jobs: - name: print trigger event run: | - run: echo "workflow triggering event name is:" ${{ github.event_name }} - - run: echo "workflow triggering event type is:" ${{ github.event.action }} - - run: echo "workflow triggering event user is:" ${{ github.event.actor }} + - run: echo "workflow triggering event type is:" ${{ github.event_action }} + - run: echo "workflow triggering event user is:" ${{ github.event_actor }} - name: update the package list run: sudo apt-get update From a46b2a4527d9838f6b72261c09a66939ce49b809 Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Wed, 10 Jul 2024 12:29:24 -0600 Subject: [PATCH 040/130] Update fierro-test-no-implicit.yml --- .github/workflows/fierro-test-no-implicit.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/fierro-test-no-implicit.yml b/.github/workflows/fierro-test-no-implicit.yml index e7a02a40d..c361f5ec3 100644 --- a/.github/workflows/fierro-test-no-implicit.yml +++ b/.github/workflows/fierro-test-no-implicit.yml @@ -17,8 +17,6 @@ jobs: - name: print trigger event run: | - run: echo "workflow triggering event name is:" ${{ github.event_name }} - - run: echo "workflow triggering event type is:" ${{ github.event_action }} - - run: echo "workflow triggering event user is:" ${{ github.event_actor }} - name: update the package list run: sudo apt-get update From 25ac4eab12ebed8d5de42bcfba956cde84eed84a Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Wed, 10 Jul 2024 12:41:55 -0600 Subject: [PATCH 041/130] Update fierro-test-no-implicit.yml --- .github/workflows/fierro-test-no-implicit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/fierro-test-no-implicit.yml b/.github/workflows/fierro-test-no-implicit.yml index c361f5ec3..4eb958f53 100644 --- a/.github/workflows/fierro-test-no-implicit.yml +++ b/.github/workflows/fierro-test-no-implicit.yml @@ -16,7 +16,7 @@ jobs: submodules: recursive - name: print trigger event run: | - - run: echo "workflow triggering event name is:" ${{ github.event_name }} + echo "::workflow triggering event name is::" ${{ github.event_name }} - name: update the package list run: sudo apt-get update From 76c56359ad9471fd78ca314475b9b9a9c34577b5 Mon Sep 17 00:00:00 2001 From: Nathaniel Morgan Date: Wed, 10 Jul 2024 15:18:00 -0600 Subject: [PATCH 042/130] fixed bug with gpu memory --- single-node-refactor/src/common/region.h | 25 +++++--- .../src/common/simulation_parameters.h | 4 +- single-node-refactor/src/driver.h | 15 +++-- single-node-refactor/src/input/parse_yaml.cpp | 59 +++++++++++-------- single-node-refactor/src/input/parse_yaml.h | 3 +- 5 files changed, 65 insertions(+), 41 deletions(-) diff --git a/single-node-refactor/src/common/region.h b/single-node-refactor/src/common/region.h index 93e62d4e7..7c8a83da7 100644 --- a/single-node-refactor/src/common/region.h +++ b/single-node-refactor/src/common/region.h @@ -80,8 +80,6 @@ struct reg_fill_t // type region::vol_tag volume; ///< Type of volume for this region eg. global, box, sphere, planes, etc. - std::string file_path = ""; ///< Absolute path of mesh file - // material id size_t material_id; ///< Material ID for this region @@ -97,11 +95,6 @@ struct reg_fill_t double radius1 = 0.0; ///< Inner radius to fill for sphere double radius2 = 0.0; ///< Outer radius to fill for sphere - // scale parameters for mesh files - double scale_x = 1.0; - double scale_y = 1.0; - double scale_z = 1.0; - // initial conditions init_conds::init_velocity_conds velocity; ///< Initial conditions for this region WARNING: Currently unimplemented @@ -119,6 +112,24 @@ struct reg_fill_t double origin[3] = {0.0, 0.0, 0.0}; ///< Origin for region }; + +///////////////////////////////////////////////////////////////////////////// +/// +/// \struct reg_fill_host_t +/// +/// \brief Geometry data, on the cpu only, for regions of materials/states +/// +///////////////////////////////////////////////////////////////////////////// +struct reg_fill_host_t +{ + std::string file_path; ///< path of mesh file + + // scale parameters for input mesh files + double scale_x = 1.0; + double scale_y = 1.0; + double scale_z = 1.0; +}; + // ---------------------------------- // valid inputs for a material fill // ---------------------------------- diff --git a/single-node-refactor/src/common/simulation_parameters.h b/single-node-refactor/src/common/simulation_parameters.h index a70dfc180..0287ba764 100644 --- a/single-node-refactor/src/common/simulation_parameters.h +++ b/single-node-refactor/src/common/simulation_parameters.h @@ -62,7 +62,9 @@ struct SimulationParameters_t std::vector solver_inputs; ///< Solvers to use during the simulation - DCArrayKokkos region_fills; ///< Region data for simulation mesh, set the initial conditions + CArrayKokkos region_fills; ///< Region data for simulation mesh, set the initial conditions + + CArray region_fills_host; ///< Region data on CPU, set the initial conditions }; // simulation_parameters_t diff --git a/single-node-refactor/src/driver.h b/single-node-refactor/src/driver.h index b41431fdf..6afe4115f 100644 --- a/single-node-refactor/src/driver.h +++ b/single-node-refactor/src/driver.h @@ -44,7 +44,8 @@ #include "state.h" -void fill_regions(DCArrayKokkos&, +void fill_regions(CArrayKokkos&, + CArray &, Material_t&, mesh_t&, node_t&, @@ -228,6 +229,7 @@ class Driver //fill_regions(); fill_regions(SimulationParamaters.region_fills, + SimulationParamaters.region_fills_host, Materials, mesh, node, @@ -328,7 +330,8 @@ class Driver /// \brief Fills mesh regions based on YAML input /// ///////////////////////////////////////////////////////////////////////////// - void fill_regions(DCArrayKokkos& region_fills, + void fill_regions(CArrayKokkos& region_fills, + CArray ®ion_fills_host, Material_t& Materials, mesh_t& mesh, node_t& node, @@ -376,10 +379,10 @@ class Driver voxel_dx, voxel_dy, voxel_dz, orig_x, orig_y, orig_z, voxel_num_i, voxel_num_j, voxel_num_k, - region_fills(f_id).scale_x, - region_fills(f_id).scale_y, - region_fills(f_id).scale_z, - region_fills(f_id).file_path); + region_fills_host(f_id).scale_x, + region_fills_host(f_id).scale_y, + region_fills_host(f_id).scale_z, + region_fills_host(f_id).file_path); // copy values read from file to device voxel_elem_mat_id.update_device(); diff --git a/single-node-refactor/src/input/parse_yaml.cpp b/single-node-refactor/src/input/parse_yaml.cpp index 454785722..2ccec85cf 100644 --- a/single-node-refactor/src/input/parse_yaml.cpp +++ b/single-node-refactor/src/input/parse_yaml.cpp @@ -313,7 +313,7 @@ void parse_yaml(Yaml::Node& root, SimulationParameters_t& SimulationParamaters, std::cout << "Parsing YAML regions:" << std::endl; } // parse the region yaml text into a vector of region_fills - parse_regions(root, SimulationParamaters.region_fills); + parse_regions(root, SimulationParamaters.region_fills, SimulationParamaters.region_fills_host); if (VERBOSE) { printf("\n"); @@ -898,13 +898,16 @@ void parse_output_options(Yaml::Node& root, output_options_t& output_options) // ================================================================================= // Parse Fill regions // ================================================================================= -void parse_regions(Yaml::Node& root, DCArrayKokkos& region_fills) +void parse_regions(Yaml::Node& root, + CArrayKokkos& region_fills, + CArray ®ion_fills_host) { Yaml::Node& region_yaml = root["regions"]; size_t num_regions = region_yaml.Size(); - region_fills = DCArrayKokkos(num_regions , "sim_param.region_fills"); + region_fills = CArrayKokkos(num_regions , "sim_param.region_fills"); + region_fills_host = CArray(num_regions); // loop over the fill regions specified @@ -1117,9 +1120,8 @@ void parse_regions(Yaml::Node& root, DCArrayKokkos& region_fills) std::cout << "\tscale_x = " << scale_x << std::endl; } - RUN({ - region_fills(reg_id).scale_x = scale_x; - }); + region_fills_host(reg_id).scale_x = scale_x; + } // scale_x else if (a_word.compare("scale_y") == 0) { // outer plane @@ -1129,9 +1131,8 @@ void parse_regions(Yaml::Node& root, DCArrayKokkos& region_fills) std::cout << "\tscale_y = " << scale_y << std::endl; } - RUN({ - region_fills(reg_id).scale_y = scale_y; - }); + region_fills_host(reg_id).scale_y = scale_y; + } // scale_y else if (a_word.compare("scale_z") == 0) { // outer plane @@ -1141,9 +1142,8 @@ void parse_regions(Yaml::Node& root, DCArrayKokkos& region_fills) std::cout << "\tscale_z = " << scale_z << std::endl; } - RUN({ - region_fills(reg_id).scale_z = scale_z; - }); + region_fills_host(reg_id).scale_z = scale_z; + } // scale_z else if (a_word.compare("velocity") == 0) { @@ -1325,7 +1325,7 @@ void parse_regions(Yaml::Node& root, DCArrayKokkos& region_fills) std::cout << "\tfile_path = " << path << std::endl; } - region_fills(reg_id).file_path = path; // saving the absolute file path + region_fills_host(reg_id).file_path = path; // saving the absolute file path } // end file path // @@ -1376,21 +1376,28 @@ void parse_regions(Yaml::Node& root, DCArrayKokkos& region_fills) // ----------------------------------------------- // check for consistency in input settings - - // if the following is true, stop simulation; must add all mesh read options - if (region_fills(reg_id).volume == region::readVoxelFile && region_fills(reg_id).file_path.empty()) { - std::cout << "ERROR: When using a file to initialize a region, a file_path must be set to point to the mesh file" << std::endl; - } - // add all mesh read options here - if (region_fills(reg_id).volume != region::readVoxelFile ) { - // this means it is a geometric definition of the region + // check to see if a file path is empty + if(region_fills_host(reg_id).file_path.empty()){ - // check to see if a file path was set - if(region_fills(reg_id).file_path.size()>0){ - std::cout << "ERROR: When a geometric entity defines the region, a mesh file cannot be passed to set the region" << std::endl; - exit(0); - } + RUN({ + // if the following is true, stop simulation; must add all mesh read options + if (region_fills(reg_id).volume == region::readVoxelFile) { + printf("ERROR: When using a file to initialize a region, a file_path must be set to point to the mesh file"); + exit(0); + } + }); + } // end if check + + // check to see if a file path was set + if(region_fills_host(reg_id).file_path.size()>0){ + RUN({ + if (region_fills(reg_id).volume != region::readVoxelFile){ + // this means it is a geometric definition of the region + printf("ERROR: When a geometric entity defines the region, a mesh file cannot be passed to set the region"); + exit(0); + } + }); } // ----------------------------------------------- diff --git a/single-node-refactor/src/input/parse_yaml.h b/single-node-refactor/src/input/parse_yaml.h index 23dbacef5..af11ef464 100644 --- a/single-node-refactor/src/input/parse_yaml.h +++ b/single-node-refactor/src/input/parse_yaml.h @@ -50,6 +50,7 @@ struct SimulationParameters_t; struct solver_input_t; struct mesh_input_t; struct reg_fill_t; +struct reg_fill_host_t; struct output_options_t; struct Material_t; @@ -105,7 +106,7 @@ void parse_mesh_input(Yaml::Node& root, mesh_input_t& mesh_input); void parse_output_options(Yaml::Node& root, output_options_t& output_options); // parse the region text -void parse_regions(Yaml::Node& root, DCArrayKokkos& region_fills); +void parse_regions(Yaml::Node& root, CArrayKokkos& region_fills, CArray ®ion_fills_host); // parse the region text void parse_materials(Yaml::Node& root, Material_t& Materials); From 7f6a395ba9b270cb29b48dceb2f2e1ad3584a86d Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Wed, 10 Jul 2024 15:49:48 -0600 Subject: [PATCH 043/130] Add files via upload --- .../example_implicit_basic.yaml | 99 +++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 src/Parallel-Solvers/Implicit-Lagrange/Example_Yaml_Scripts/example_implicit_basic.yaml diff --git a/src/Parallel-Solvers/Implicit-Lagrange/Example_Yaml_Scripts/example_implicit_basic.yaml b/src/Parallel-Solvers/Implicit-Lagrange/Example_Yaml_Scripts/example_implicit_basic.yaml new file mode 100644 index 000000000..6093d542f --- /dev/null +++ b/src/Parallel-Solvers/Implicit-Lagrange/Example_Yaml_Scripts/example_implicit_basic.yaml @@ -0,0 +1,99 @@ +# input file +num_dims: 3 + +mesh_generation_options: + type: Box + origin: [0, 0, 0] + length: [10, 10, 100] + num_elems: [5, 5, 50] + +materials: + - id: 0 + elastic_modulus: 200000000000 + poisson_ratio: 0.3 + density: 7850 + thermal_conductivity: 10 + specific_internal_energy_rate: 1.0 + initial_temperature: 293 + +fea_module_parameters: + - type: Inertial + material_id: 0 + inertia_center_x: 0 + inertia_center_y: 0 + inertia_center_z: 0 + + - type: Elasticity + material_id: 0 + # Dirichlet conditions + boundary_conditions: + - surface: + type: z_plane + plane_position: 0.0 + type: displacement + value: 0.0 + + # Loading/Neumann Conditions + loading_conditions: + - surface: + type: z_plane + plane_position: 100.0 + type: surface_traction + component_x: 500 + component_y: 0 + component_z: 0 + + - type: Heat_Conduction + material_id: 0 + # Dirichlet conditions + boundary_conditions: + - surface: + type: z_plane + plane_position: 0.0 + type: temperature + value: 293.0 + + # Loading/Neumann Conditions + loading_conditions: + - surface: + type: z_plane + plane_position: 100.0 + type: surface_heat_flux + #can be normal (q dot n) or coordinated (vector q) for curved surfaces + specification: normal + flux_value: -0.1 + +optimization_options: + optimization_process: topology_optimization + method_of_moving_asymptotes: false + density_epsilon: 0.001 + simp_penalty_power: 3 + rol_params: + subproblem_algorithm: trust_region + initial_constraint_penalty: 1.e1 + step_tolerance: 1.e-5 + gradient_tolerance: 1.e-5 + constraint_tolerance: 1.e-5 + iteration_limit: 5 + optimization_objective: multi_objective + #Weight coefficients should add up to 1 + multi_objective_structure: linear + multi_objective_modules: + - type: minimize_thermal_resistance + weight_coefficient: 0.25 + - type: minimize_compliance + weight_coefficient: 0.75 + + constraints: + - type: mass + relation: equality + value: 0.25 + - type: moment_of_inertia + relation: equality + component: yy + value: 0.35 + - type: moment_of_inertia + relation: equality + component: xy + value: 0.0 + From 014da3c48a5991d1622910c545173c673d0e728b Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Wed, 10 Jul 2024 15:50:52 -0600 Subject: [PATCH 044/130] Update fierro-test.yml --- .github/workflows/fierro-test.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/fierro-test.yml b/.github/workflows/fierro-test.yml index 39c8fd373..b6351e978 100644 --- a/.github/workflows/fierro-test.yml +++ b/.github/workflows/fierro-test.yml @@ -85,7 +85,11 @@ jobs: run: | source ~/.bashrc conda activate build-env - fierro-parallel-implicit ./Example_Yaml_Scripts/example_simple.yaml + export OMP_PROC_BIND=spread + export OMP_NUM_THREADS=1 + export OMP_PLACES=threads + mpirun -np 8 --bind-to core fierro-parallel-explicit example_simple.yaml + fierro-parallel-implicit ./Example_Yaml_Scripts/example_implicit_basic.yaml working-directory: ./src/Parallel-Solvers/Implicit-Lagrange shell: bash From c2709a5984209419c839a6b3210e93897da5fefc Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Wed, 10 Jul 2024 15:51:08 -0600 Subject: [PATCH 045/130] Update fierro-test.yml --- .github/workflows/fierro-test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/fierro-test.yml b/.github/workflows/fierro-test.yml index b6351e978..9cf4edb4a 100644 --- a/.github/workflows/fierro-test.yml +++ b/.github/workflows/fierro-test.yml @@ -6,6 +6,7 @@ on: workflows: [Publish Test, Publish All, Publish Elements, Publish EVPFFT-CPU, Publish EVPFFT-dev, Publish EVPFFT-GPU, Publish FIERRO GUI, Publish Fierro-CPU, Publish Fierro-dev, Publish Heffte, Publish Heffte CUDA, Publish Trilinos-CPU] types: - completed + workflow_dispatch: jobs: build: From 853bd9a10667509b740d8e753854cdf406fd5dd3 Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Wed, 10 Jul 2024 15:51:48 -0600 Subject: [PATCH 046/130] Update fierro-test-no-implicit.yml --- .github/workflows/fierro-test-no-implicit.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/fierro-test-no-implicit.yml b/.github/workflows/fierro-test-no-implicit.yml index 4eb958f53..b829b5e06 100644 --- a/.github/workflows/fierro-test-no-implicit.yml +++ b/.github/workflows/fierro-test-no-implicit.yml @@ -6,6 +6,7 @@ on: workflows: [Publish Test, Publish All, Publish Elements, Publish EVPFFT-CPU, Publish EVPFFT-dev, Publish EVPFFT-GPU, Publish FIERRO GUI, Publish Fierro-CPU, Publish Fierro-dev, Publish Heffte, Publish Heffte CUDA, Publish Trilinos-CPU] types: - completed + workflow_dispatch: jobs: build: From ffed4c76ed3e709ef8d7a18d88669b9828837eb2 Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Wed, 10 Jul 2024 15:58:49 -0600 Subject: [PATCH 047/130] Update fierro-test.yml --- .github/workflows/fierro-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/fierro-test.yml b/.github/workflows/fierro-test.yml index 9cf4edb4a..8837a5c18 100644 --- a/.github/workflows/fierro-test.yml +++ b/.github/workflows/fierro-test.yml @@ -89,7 +89,7 @@ jobs: export OMP_PROC_BIND=spread export OMP_NUM_THREADS=1 export OMP_PLACES=threads - mpirun -np 8 --bind-to core fierro-parallel-explicit example_simple.yaml + mpirun --oversubscribe -np 8 --bind-to core fierro-parallel-explicit example_simple.yaml fierro-parallel-implicit ./Example_Yaml_Scripts/example_implicit_basic.yaml working-directory: ./src/Parallel-Solvers/Implicit-Lagrange shell: bash From 4f9b9c3b2ea576ba5213542f89b32576f4cb66ce Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Wed, 10 Jul 2024 16:05:50 -0600 Subject: [PATCH 048/130] Update fierro-test.yml --- .github/workflows/fierro-test.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/fierro-test.yml b/.github/workflows/fierro-test.yml index 8837a5c18..a12a8d141 100644 --- a/.github/workflows/fierro-test.yml +++ b/.github/workflows/fierro-test.yml @@ -68,6 +68,12 @@ jobs: conda info --envs conda list + - name: Set OMP env variables + run: | + export OMP_PROC_BIND=spread + export OMP_NUM_THREADS=1 + export OMP_PLACES=threads + - name: Test fierro-parallel-explicit run: | source ~/.bashrc @@ -86,10 +92,6 @@ jobs: run: | source ~/.bashrc conda activate build-env - export OMP_PROC_BIND=spread - export OMP_NUM_THREADS=1 - export OMP_PLACES=threads - mpirun --oversubscribe -np 8 --bind-to core fierro-parallel-explicit example_simple.yaml fierro-parallel-implicit ./Example_Yaml_Scripts/example_implicit_basic.yaml working-directory: ./src/Parallel-Solvers/Implicit-Lagrange shell: bash From ce56057a4256c72fb0b25289d1bdea016479eafa Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Wed, 10 Jul 2024 16:11:15 -0600 Subject: [PATCH 049/130] Update fierro-test.yml --- .github/workflows/fierro-test.yml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/fierro-test.yml b/.github/workflows/fierro-test.yml index a12a8d141..51c4004e5 100644 --- a/.github/workflows/fierro-test.yml +++ b/.github/workflows/fierro-test.yml @@ -67,13 +67,7 @@ jobs: run: | conda info --envs conda list - - - name: Set OMP env variables - run: | - export OMP_PROC_BIND=spread - export OMP_NUM_THREADS=1 - export OMP_PLACES=threads - + - name: Test fierro-parallel-explicit run: | source ~/.bashrc @@ -92,6 +86,9 @@ jobs: run: | source ~/.bashrc conda activate build-env + export OMP_PROC_BIND=spread + export OMP_NUM_THREADS=1 + export OMP_PLACES=threads fierro-parallel-implicit ./Example_Yaml_Scripts/example_implicit_basic.yaml working-directory: ./src/Parallel-Solvers/Implicit-Lagrange shell: bash From 9ff0ba0adb815e7545e80c247a4053a77d28e179 Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Wed, 10 Jul 2024 16:20:59 -0600 Subject: [PATCH 050/130] Update fierro-test.yml --- .github/workflows/fierro-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/fierro-test.yml b/.github/workflows/fierro-test.yml index 51c4004e5..4140392ee 100644 --- a/.github/workflows/fierro-test.yml +++ b/.github/workflows/fierro-test.yml @@ -89,7 +89,7 @@ jobs: export OMP_PROC_BIND=spread export OMP_NUM_THREADS=1 export OMP_PLACES=threads - fierro-parallel-implicit ./Example_Yaml_Scripts/example_implicit_basic.yaml + mpirun -np 8 --bind-to core fierro-parallel-implicit ./Example_Yaml_Scripts/example_implicit_basic.yaml working-directory: ./src/Parallel-Solvers/Implicit-Lagrange shell: bash From 6fa90bc67cc17acaacd8104fa913d2172e265db6 Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Wed, 10 Jul 2024 16:26:50 -0600 Subject: [PATCH 051/130] Update fierro-test.yml --- .github/workflows/fierro-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/fierro-test.yml b/.github/workflows/fierro-test.yml index 4140392ee..dc7839ed0 100644 --- a/.github/workflows/fierro-test.yml +++ b/.github/workflows/fierro-test.yml @@ -89,7 +89,7 @@ jobs: export OMP_PROC_BIND=spread export OMP_NUM_THREADS=1 export OMP_PLACES=threads - mpirun -np 8 --bind-to core fierro-parallel-implicit ./Example_Yaml_Scripts/example_implicit_basic.yaml + mpirun -np 4 --bind-to core fierro-parallel-implicit ./Example_Yaml_Scripts/example_implicit_basic.yaml working-directory: ./src/Parallel-Solvers/Implicit-Lagrange shell: bash From 18bb632f3f69fa8964909a2f2203949ecad311c4 Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Wed, 10 Jul 2024 16:31:51 -0600 Subject: [PATCH 052/130] Update fierro-test.yml --- .github/workflows/fierro-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/fierro-test.yml b/.github/workflows/fierro-test.yml index dc7839ed0..8937ae6a5 100644 --- a/.github/workflows/fierro-test.yml +++ b/.github/workflows/fierro-test.yml @@ -89,7 +89,7 @@ jobs: export OMP_PROC_BIND=spread export OMP_NUM_THREADS=1 export OMP_PLACES=threads - mpirun -np 4 --bind-to core fierro-parallel-implicit ./Example_Yaml_Scripts/example_implicit_basic.yaml + mpirun -np 2 --bind-to core fierro-parallel-implicit ./Example_Yaml_Scripts/example_implicit_basic.yaml working-directory: ./src/Parallel-Solvers/Implicit-Lagrange shell: bash From 4c33dfc1b462aa44a01b8090242e394d6e881f3b Mon Sep 17 00:00:00 2001 From: Nathaniel Morgan Date: Wed, 10 Jul 2024 16:50:05 -0600 Subject: [PATCH 053/130] updated exit error to work with gpus --- single-node-refactor/src/input/parse_yaml.cpp | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/single-node-refactor/src/input/parse_yaml.cpp b/single-node-refactor/src/input/parse_yaml.cpp index 2ccec85cf..d3ba743b4 100644 --- a/single-node-refactor/src/input/parse_yaml.cpp +++ b/single-node-refactor/src/input/parse_yaml.cpp @@ -1376,28 +1376,41 @@ void parse_regions(Yaml::Node& root, // ----------------------------------------------- // check for consistency in input settings + DCArrayKokkos error_value(1); // check to see if a file path is empty if(region_fills_host(reg_id).file_path.empty()){ RUN({ + error_value(0) = 0; + // if the following is true, stop simulation; must add all mesh read options if (region_fills(reg_id).volume == region::readVoxelFile) { printf("ERROR: When using a file to initialize a region, a file_path must be set to point to the mesh file"); - exit(0); } }); + error_value.update_host(); + Kokkos::fence(); + + if(error_value(0)==1) exit(0); + } // end if check // check to see if a file path was set if(region_fills_host(reg_id).file_path.size()>0){ RUN({ + error_value(0) = 0; + if (region_fills(reg_id).volume != region::readVoxelFile){ // this means it is a geometric definition of the region printf("ERROR: When a geometric entity defines the region, a mesh file cannot be passed to set the region"); exit(0); } }); + error_value.update_host(); + Kokkos::fence(); + + if(error_value(0)==1) exit(0); } // ----------------------------------------------- From 3f7b96f6876dc5158e43f017cf275a0c0eca6f80 Mon Sep 17 00:00:00 2001 From: Nathaniel Morgan Date: Wed, 10 Jul 2024 17:00:14 -0600 Subject: [PATCH 054/130] updated exit error to work with gpus --- single-node-refactor/src/input/parse_yaml.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/single-node-refactor/src/input/parse_yaml.cpp b/single-node-refactor/src/input/parse_yaml.cpp index d3ba743b4..04ad9fd63 100644 --- a/single-node-refactor/src/input/parse_yaml.cpp +++ b/single-node-refactor/src/input/parse_yaml.cpp @@ -1404,7 +1404,6 @@ void parse_regions(Yaml::Node& root, if (region_fills(reg_id).volume != region::readVoxelFile){ // this means it is a geometric definition of the region printf("ERROR: When a geometric entity defines the region, a mesh file cannot be passed to set the region"); - exit(0); } }); error_value.update_host(); From c6a7f524de9e66dfde8aa51d6942376f34519a29 Mon Sep 17 00:00:00 2001 From: Jacob Moore Date: Thu, 11 Jul 2024 08:22:12 -0500 Subject: [PATCH 055/130] ENH: Testing Kokkos::abort --- single-node-refactor/src/input/parse_yaml.cpp | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/single-node-refactor/src/input/parse_yaml.cpp b/single-node-refactor/src/input/parse_yaml.cpp index 04ad9fd63..15cefd374 100644 --- a/single-node-refactor/src/input/parse_yaml.cpp +++ b/single-node-refactor/src/input/parse_yaml.cpp @@ -1387,12 +1387,17 @@ void parse_regions(Yaml::Node& root, // if the following is true, stop simulation; must add all mesh read options if (region_fills(reg_id).volume == region::readVoxelFile) { printf("ERROR: When using a file to initialize a region, a file_path must be set to point to the mesh file"); + + Kokkos::abort("********************************************************************************************\n + ERROR: \n + When using a file to initialize a region, a file_path must be set to point to the mesh file\n + ********************************************************************************************") } }); error_value.update_host(); Kokkos::fence(); - if(error_value(0)==1) exit(0); + //if(error_value.host(0)==1) exit(0); } // end if check @@ -1404,12 +1409,17 @@ void parse_regions(Yaml::Node& root, if (region_fills(reg_id).volume != region::readVoxelFile){ // this means it is a geometric definition of the region printf("ERROR: When a geometric entity defines the region, a mesh file cannot be passed to set the region"); + + Kokkos::abort("********************************************************************************************\n + ERROR: \n + hen a geometric entity defines the region, a mesh file cannot be passed to set the region\n + ********************************************************************************************") } }); error_value.update_host(); Kokkos::fence(); - if(error_value(0)==1) exit(0); + // if(error_value(0)==1) exit(0); } // ----------------------------------------------- From d0b27be2a51870fbc3bb41eedea4dbde3282a53b Mon Sep 17 00:00:00 2001 From: Jacob Moore Date: Thu, 11 Jul 2024 08:25:36 -0500 Subject: [PATCH 056/130] BUG: Tweak string in Kokkos::abort --- single-node-refactor/src/input/parse_yaml.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/single-node-refactor/src/input/parse_yaml.cpp b/single-node-refactor/src/input/parse_yaml.cpp index 15cefd374..a43f2f745 100644 --- a/single-node-refactor/src/input/parse_yaml.cpp +++ b/single-node-refactor/src/input/parse_yaml.cpp @@ -1388,10 +1388,10 @@ void parse_regions(Yaml::Node& root, if (region_fills(reg_id).volume == region::readVoxelFile) { printf("ERROR: When using a file to initialize a region, a file_path must be set to point to the mesh file"); - Kokkos::abort("********************************************************************************************\n - ERROR: \n - When using a file to initialize a region, a file_path must be set to point to the mesh file\n - ********************************************************************************************") + Kokkos::abort("********************************************************************************************\n" + "ERROR: \n" + "When using a file to initialize a region, a file_path must be set to point to the mesh file\n" + "********************************************************************************************") } }); error_value.update_host(); @@ -1410,10 +1410,10 @@ void parse_regions(Yaml::Node& root, // this means it is a geometric definition of the region printf("ERROR: When a geometric entity defines the region, a mesh file cannot be passed to set the region"); - Kokkos::abort("********************************************************************************************\n - ERROR: \n - hen a geometric entity defines the region, a mesh file cannot be passed to set the region\n - ********************************************************************************************") + Kokkos::abort("********************************************************************************************\n" + "ERROR: \n" + "When a geometric entity defines the region, a mesh file cannot be passed to set the region\n" + "********************************************************************************************") } }); error_value.update_host(); From dc5fb21011b66f650d65be203ea740f88cb42ca9 Mon Sep 17 00:00:00 2001 From: Jacob Moore Date: Thu, 11 Jul 2024 08:28:06 -0500 Subject: [PATCH 057/130] BUG: Add missing semicolon --- single-node-refactor/src/input/parse_yaml.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/single-node-refactor/src/input/parse_yaml.cpp b/single-node-refactor/src/input/parse_yaml.cpp index a43f2f745..b21f3d1bf 100644 --- a/single-node-refactor/src/input/parse_yaml.cpp +++ b/single-node-refactor/src/input/parse_yaml.cpp @@ -1391,7 +1391,7 @@ void parse_regions(Yaml::Node& root, Kokkos::abort("********************************************************************************************\n" "ERROR: \n" "When using a file to initialize a region, a file_path must be set to point to the mesh file\n" - "********************************************************************************************") + "********************************************************************************************"); } }); error_value.update_host(); @@ -1413,7 +1413,7 @@ void parse_regions(Yaml::Node& root, Kokkos::abort("********************************************************************************************\n" "ERROR: \n" "When a geometric entity defines the region, a mesh file cannot be passed to set the region\n" - "********************************************************************************************") + "********************************************************************************************"); } }); error_value.update_host(); From 75b7664790330c9931dcd27c272fbf7ad8a08430 Mon Sep 17 00:00:00 2001 From: Jacob Moore Date: Thu, 11 Jul 2024 08:36:03 -0500 Subject: [PATCH 058/130] BUG: Fix style of error messages --- single-node-refactor/src/input/parse_yaml.cpp | 29 +++---------------- 1 file changed, 4 insertions(+), 25 deletions(-) diff --git a/single-node-refactor/src/input/parse_yaml.cpp b/single-node-refactor/src/input/parse_yaml.cpp index b21f3d1bf..447886306 100644 --- a/single-node-refactor/src/input/parse_yaml.cpp +++ b/single-node-refactor/src/input/parse_yaml.cpp @@ -1376,54 +1376,33 @@ void parse_regions(Yaml::Node& root, // ----------------------------------------------- // check for consistency in input settings - DCArrayKokkos error_value(1); // check to see if a file path is empty if(region_fills_host(reg_id).file_path.empty()){ RUN({ - error_value(0) = 0; - // if the following is true, stop simulation; must add all mesh read options if (region_fills(reg_id).volume == region::readVoxelFile) { - printf("ERROR: When using a file to initialize a region, a file_path must be set to point to the mesh file"); - - Kokkos::abort("********************************************************************************************\n" + Kokkos::abort("\n********************************************************************************************\n" "ERROR: \n" "When using a file to initialize a region, a file_path must be set to point to the mesh file\n" - "********************************************************************************************"); + "********************************************************************************************\n"); } }); - error_value.update_host(); - Kokkos::fence(); - - //if(error_value.host(0)==1) exit(0); - } // end if check // check to see if a file path was set if(region_fills_host(reg_id).file_path.size()>0){ RUN({ - error_value(0) = 0; - if (region_fills(reg_id).volume != region::readVoxelFile){ // this means it is a geometric definition of the region - printf("ERROR: When a geometric entity defines the region, a mesh file cannot be passed to set the region"); - - Kokkos::abort("********************************************************************************************\n" + Kokkos::abort("\n********************************************************************************************\n" "ERROR: \n" "When a geometric entity defines the region, a mesh file cannot be passed to set the region\n" - "********************************************************************************************"); + "********************************************************************************************\n"); } }); - error_value.update_host(); - Kokkos::fence(); - - // if(error_value(0)==1) exit(0); } - // ----------------------------------------------- - - } // end loop over regions } // end of function to parse region From 270a5bb3fc66746d4af83dc01c62e3945ed68e17 Mon Sep 17 00:00:00 2001 From: Nathaniel Morgan Date: Thu, 11 Jul 2024 07:54:49 -0600 Subject: [PATCH 059/130] fixed mesh_t names to be elem --- single-node-refactor/src/common/mesh.h | 28 +++++++++++++------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/single-node-refactor/src/common/mesh.h b/single-node-refactor/src/common/mesh.h index 8e1660ea5..a491619ed 100644 --- a/single-node-refactor/src/common/mesh.h +++ b/single-node-refactor/src/common/mesh.h @@ -111,15 +111,15 @@ void bubble_sort(size_t arr[], const size_t num) } // end for i } // end function -struct zones_in_MaterialPoint_t +struct zones_in_elem_t { private: size_t num_zones_in_elem_; public: - zones_in_MaterialPoint_t() { + zones_in_elem_t() { }; - zones_in_MaterialPoint_t(const size_t num_zones_in_elem_inp) { + zones_in_elem_t(const size_t num_zones_in_elem_inp) { this->num_zones_in_elem_ = num_zones_in_elem_inp; }; @@ -138,15 +138,15 @@ struct zones_in_MaterialPoint_t }; // if material points are defined strictly internal to the element. -struct legendre_in_MaterialPoint_t +struct legendre_in_elem_t { private: size_t num_leg_gauss_in_elem_; public: - legendre_in_MaterialPoint_t() { + legendre_in_elem_t() { }; - legendre_in_MaterialPoint_t(const size_t num_leg_gauss_in_elem_inp) { + legendre_in_elem_t(const size_t num_leg_gauss_in_elem_inp) { this->num_leg_gauss_in_elem_ = num_leg_gauss_in_elem_inp; }; @@ -165,15 +165,15 @@ struct legendre_in_MaterialPoint_t }; /// if material points are defined at element interfaces -struct lobatto_in_MaterialPoint_t +struct lobatto_in_elem_t { private: size_t num_lob_gauss_in_elem_; public: - lobatto_in_MaterialPoint_t() { + lobatto_in_elem_t() { }; - lobatto_in_MaterialPoint_t(const size_t num_lob_gauss_in_elem_inp) { + lobatto_in_elem_t(const size_t num_lob_gauss_in_elem_inp) { this->num_lob_gauss_in_elem_ = num_lob_gauss_in_elem_inp; }; @@ -249,9 +249,9 @@ struct mesh_t CArrayKokkos surfs_in_elem; ///< Surfaces on an element // CArrayKokkos zones_in_elem; ///< Zones in an element - zones_in_MaterialPoint_t zones_in_elem; ///< Zones in an element - lobatto_in_MaterialPoint_t lobatto_in_elem; ///< Gauss Lobatto points in an element - legendre_in_MaterialPoint_t legendre_in_elem; ///< Gauss Legendre points in an element + zones_in_elem_t zones_in_elem; ///< Zones in an element + lobatto_in_elem_t lobatto_in_elem; ///< Gauss Lobatto points in an element + legendre_in_elem_t legendre_in_elem; ///< Gauss Legendre points in an element // ---- Node Data Definitions ---- // size_t num_nodes; ///< Number of nodes in the mesh @@ -348,10 +348,10 @@ struct mesh_t nodes_in_elem = DCArrayKokkos(num_elems, num_nodes_in_elem, "mesh.nodes_in_elem"); corners_in_elem = CArrayKokkos(num_elems, num_nodes_in_elem, "mesh.corners_in_elem"); - zones_in_elem = zones_in_MaterialPoint_t(num_zones_in_elem); + zones_in_elem = zones_in_elem_t(num_zones_in_elem); surfs_in_elem = CArrayKokkos(num_elems, num_surfs_in_elem, "mesh.surfs_in_zone"); nodes_in_zone = CArrayKokkos(num_zones, num_nodes_in_zone, "mesh.nodes_in_zone"); - legendre_in_elem = legendre_in_MaterialPoint_t(num_leg_gauss_in_elem); + legendre_in_elem = legendre_in_elem_t(num_leg_gauss_in_elem); return; }; // end method From c31717f754524b5f9c145b6c68644ceaf127f1c3 Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Thu, 11 Jul 2024 09:37:08 -0600 Subject: [PATCH 060/130] Update fierro-test.yml --- .github/workflows/fierro-test.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/fierro-test.yml b/.github/workflows/fierro-test.yml index 8937ae6a5..0c9428fb3 100644 --- a/.github/workflows/fierro-test.yml +++ b/.github/workflows/fierro-test.yml @@ -86,10 +86,7 @@ jobs: run: | source ~/.bashrc conda activate build-env - export OMP_PROC_BIND=spread - export OMP_NUM_THREADS=1 - export OMP_PLACES=threads - mpirun -np 2 --bind-to core fierro-parallel-implicit ./Example_Yaml_Scripts/example_implicit_basic.yaml + fierro-parallel-implicit ./Example_Yaml_Scripts/example_implicit_basic.yaml working-directory: ./src/Parallel-Solvers/Implicit-Lagrange shell: bash From b8f93900c51b8b414f0f87232984ca622fa89e18 Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Thu, 11 Jul 2024 09:46:17 -0600 Subject: [PATCH 061/130] Update fierro-test.yml --- .github/workflows/fierro-test.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/fierro-test.yml b/.github/workflows/fierro-test.yml index 0c9428fb3..51c4004e5 100644 --- a/.github/workflows/fierro-test.yml +++ b/.github/workflows/fierro-test.yml @@ -86,6 +86,9 @@ jobs: run: | source ~/.bashrc conda activate build-env + export OMP_PROC_BIND=spread + export OMP_NUM_THREADS=1 + export OMP_PLACES=threads fierro-parallel-implicit ./Example_Yaml_Scripts/example_implicit_basic.yaml working-directory: ./src/Parallel-Solvers/Implicit-Lagrange shell: bash From 465c14042dcf9acadd058a20eab2b51f6f2f4e42 Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Thu, 11 Jul 2024 09:51:50 -0600 Subject: [PATCH 062/130] Update fierro-test.yml --- .github/workflows/fierro-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/fierro-test.yml b/.github/workflows/fierro-test.yml index 51c4004e5..4140392ee 100644 --- a/.github/workflows/fierro-test.yml +++ b/.github/workflows/fierro-test.yml @@ -89,7 +89,7 @@ jobs: export OMP_PROC_BIND=spread export OMP_NUM_THREADS=1 export OMP_PLACES=threads - fierro-parallel-implicit ./Example_Yaml_Scripts/example_implicit_basic.yaml + mpirun -np 8 --bind-to core fierro-parallel-implicit ./Example_Yaml_Scripts/example_implicit_basic.yaml working-directory: ./src/Parallel-Solvers/Implicit-Lagrange shell: bash From 3e6e190e68ef894b8c6ba872822fe195e0711f74 Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Thu, 11 Jul 2024 10:10:50 -0600 Subject: [PATCH 063/130] Update fierro-test.yml --- .github/workflows/fierro-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/fierro-test.yml b/.github/workflows/fierro-test.yml index 4140392ee..91ae94eda 100644 --- a/.github/workflows/fierro-test.yml +++ b/.github/workflows/fierro-test.yml @@ -89,7 +89,7 @@ jobs: export OMP_PROC_BIND=spread export OMP_NUM_THREADS=1 export OMP_PLACES=threads - mpirun -np 8 --bind-to core fierro-parallel-implicit ./Example_Yaml_Scripts/example_implicit_basic.yaml + mpirun -np 3 --bind-to core fierro-parallel-implicit ./Example_Yaml_Scripts/example_implicit_basic.yaml working-directory: ./src/Parallel-Solvers/Implicit-Lagrange shell: bash From e688e61c695d82b3e9f4ac82cc375facccc59aea Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Thu, 11 Jul 2024 10:11:01 -0600 Subject: [PATCH 064/130] Update fierro-test.yml --- .github/workflows/fierro-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/fierro-test.yml b/.github/workflows/fierro-test.yml index 91ae94eda..dc7839ed0 100644 --- a/.github/workflows/fierro-test.yml +++ b/.github/workflows/fierro-test.yml @@ -89,7 +89,7 @@ jobs: export OMP_PROC_BIND=spread export OMP_NUM_THREADS=1 export OMP_PLACES=threads - mpirun -np 3 --bind-to core fierro-parallel-implicit ./Example_Yaml_Scripts/example_implicit_basic.yaml + mpirun -np 4 --bind-to core fierro-parallel-implicit ./Example_Yaml_Scripts/example_implicit_basic.yaml working-directory: ./src/Parallel-Solvers/Implicit-Lagrange shell: bash From b21a417621d86a161a21fd56561acfd266894347 Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Thu, 11 Jul 2024 10:43:33 -0600 Subject: [PATCH 065/130] Update fierro-test.yml --- .github/workflows/fierro-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/fierro-test.yml b/.github/workflows/fierro-test.yml index dc7839ed0..8937ae6a5 100644 --- a/.github/workflows/fierro-test.yml +++ b/.github/workflows/fierro-test.yml @@ -89,7 +89,7 @@ jobs: export OMP_PROC_BIND=spread export OMP_NUM_THREADS=1 export OMP_PLACES=threads - mpirun -np 4 --bind-to core fierro-parallel-implicit ./Example_Yaml_Scripts/example_implicit_basic.yaml + mpirun -np 2 --bind-to core fierro-parallel-implicit ./Example_Yaml_Scripts/example_implicit_basic.yaml working-directory: ./src/Parallel-Solvers/Implicit-Lagrange shell: bash From 34447fc94192716a49be53294f0c270e84d503ef Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Thu, 11 Jul 2024 11:12:07 -0600 Subject: [PATCH 066/130] Update fierro-test.yml --- .github/workflows/fierro-test.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/fierro-test.yml b/.github/workflows/fierro-test.yml index 8937ae6a5..897ca4b9a 100644 --- a/.github/workflows/fierro-test.yml +++ b/.github/workflows/fierro-test.yml @@ -72,7 +72,10 @@ jobs: run: | source ~/.bashrc conda activate build-env - fierro-parallel-explicit ./src/Parallel-Solvers/Parallel-Explicit/Example_Yaml_Scripts/example_simple.yaml + export OMP_PROC_BIND=spread + export OMP_NUM_THREADS=1 + export OMP_PLACES=threads + mpirun -np 8 --bind-to core fierro-parallel-explicit ./src/Parallel-Solvers/Parallel-Explicit/Example_Yaml_Scripts/example_simple.yaml shell: bash - name: Test fierro-mesh-builder From 7419f20a831ed01e1b4f1317b0fc256900701c65 Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Thu, 11 Jul 2024 11:15:43 -0600 Subject: [PATCH 067/130] Update fierro-test.yml --- .github/workflows/fierro-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/fierro-test.yml b/.github/workflows/fierro-test.yml index 897ca4b9a..edab0c94a 100644 --- a/.github/workflows/fierro-test.yml +++ b/.github/workflows/fierro-test.yml @@ -75,7 +75,7 @@ jobs: export OMP_PROC_BIND=spread export OMP_NUM_THREADS=1 export OMP_PLACES=threads - mpirun -np 8 --bind-to core fierro-parallel-explicit ./src/Parallel-Solvers/Parallel-Explicit/Example_Yaml_Scripts/example_simple.yaml + mpirun -np 2 --bind-to core fierro-parallel-explicit ./src/Parallel-Solvers/Parallel-Explicit/Example_Yaml_Scripts/example_simple.yaml shell: bash - name: Test fierro-mesh-builder From 91cd3a5ffa3d26bc45db0df6f229eb3839878bd3 Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Thu, 11 Jul 2024 11:34:15 -0600 Subject: [PATCH 068/130] Update fierro-test.yml --- .github/workflows/fierro-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/fierro-test.yml b/.github/workflows/fierro-test.yml index edab0c94a..abb4f3047 100644 --- a/.github/workflows/fierro-test.yml +++ b/.github/workflows/fierro-test.yml @@ -75,7 +75,7 @@ jobs: export OMP_PROC_BIND=spread export OMP_NUM_THREADS=1 export OMP_PLACES=threads - mpirun -np 2 --bind-to core fierro-parallel-explicit ./src/Parallel-Solvers/Parallel-Explicit/Example_Yaml_Scripts/example_simple.yaml + mpirun -np 1 --bind-to core fierro-parallel-explicit ./src/Parallel-Solvers/Parallel-Explicit/Example_Yaml_Scripts/example_simple.yaml shell: bash - name: Test fierro-mesh-builder @@ -92,7 +92,7 @@ jobs: export OMP_PROC_BIND=spread export OMP_NUM_THREADS=1 export OMP_PLACES=threads - mpirun -np 2 --bind-to core fierro-parallel-implicit ./Example_Yaml_Scripts/example_implicit_basic.yaml + mpirun -np 1 --bind-to core fierro-parallel-implicit ./Example_Yaml_Scripts/example_simple.yaml working-directory: ./src/Parallel-Solvers/Implicit-Lagrange shell: bash From 6025e390e392c57f628a4e72e86ee138fd738482 Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Thu, 11 Jul 2024 11:39:09 -0600 Subject: [PATCH 069/130] Update fierro-test.yml --- .github/workflows/fierro-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/fierro-test.yml b/.github/workflows/fierro-test.yml index abb4f3047..337d9836f 100644 --- a/.github/workflows/fierro-test.yml +++ b/.github/workflows/fierro-test.yml @@ -92,7 +92,7 @@ jobs: export OMP_PROC_BIND=spread export OMP_NUM_THREADS=1 export OMP_PLACES=threads - mpirun -np 1 --bind-to core fierro-parallel-implicit ./Example_Yaml_Scripts/example_simple.yaml + mpirun -np 1 --bind-to core fierro-parallel-implicit ./Example_Yaml_Scripts/example_implict_basic.yaml working-directory: ./src/Parallel-Solvers/Implicit-Lagrange shell: bash From 60b34e14e8f182f5a67350d62bec09419b7c2e0a Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Thu, 11 Jul 2024 11:40:19 -0600 Subject: [PATCH 070/130] Update fierro-test.yml --- .github/workflows/fierro-test.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/fierro-test.yml b/.github/workflows/fierro-test.yml index 337d9836f..84e0713a8 100644 --- a/.github/workflows/fierro-test.yml +++ b/.github/workflows/fierro-test.yml @@ -92,8 +92,7 @@ jobs: export OMP_PROC_BIND=spread export OMP_NUM_THREADS=1 export OMP_PLACES=threads - mpirun -np 1 --bind-to core fierro-parallel-implicit ./Example_Yaml_Scripts/example_implict_basic.yaml - working-directory: ./src/Parallel-Solvers/Implicit-Lagrange + mpirun -np 1 --bind-to core fierro-parallel-implicit ./src/Parallel-Solvers/Implicit-Lagrange./Example_Yaml_Scripts/example_implict_basic.yaml shell: bash From 515e4c65503c70e51c2889197b61674f21291956 Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Thu, 11 Jul 2024 11:45:22 -0600 Subject: [PATCH 071/130] Update fierro-test.yml --- .github/workflows/fierro-test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/fierro-test.yml b/.github/workflows/fierro-test.yml index 84e0713a8..54ee20629 100644 --- a/.github/workflows/fierro-test.yml +++ b/.github/workflows/fierro-test.yml @@ -92,7 +92,8 @@ jobs: export OMP_PROC_BIND=spread export OMP_NUM_THREADS=1 export OMP_PLACES=threads - mpirun -np 1 --bind-to core fierro-parallel-implicit ./src/Parallel-Solvers/Implicit-Lagrange./Example_Yaml_Scripts/example_implict_basic.yaml + mpirun -np 1 --bind-to core fierro-parallel-implicit ./Example_Yaml_Scripts/example_implicit_basic.yaml + working-directory: ./src/Parallel-Solvers/Implicit-Lagrange shell: bash From 936ddeb7baf36fcfd27c2362c1191d626ed5efb4 Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Thu, 11 Jul 2024 11:51:09 -0600 Subject: [PATCH 072/130] Update fierro-test.yml --- .github/workflows/fierro-test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/fierro-test.yml b/.github/workflows/fierro-test.yml index 54ee20629..425234bc3 100644 --- a/.github/workflows/fierro-test.yml +++ b/.github/workflows/fierro-test.yml @@ -89,6 +89,7 @@ jobs: run: | source ~/.bashrc conda activate build-env + dos2unix ./Example_Yaml_Scripts/example_implicit_basic.yaml export OMP_PROC_BIND=spread export OMP_NUM_THREADS=1 export OMP_PLACES=threads From 7a067ceb502f884005e9bb5dfaec184f08f1b484 Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Thu, 11 Jul 2024 11:56:44 -0600 Subject: [PATCH 073/130] Update fierro-test.yml --- .github/workflows/fierro-test.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/fierro-test.yml b/.github/workflows/fierro-test.yml index 425234bc3..55da1845e 100644 --- a/.github/workflows/fierro-test.yml +++ b/.github/workflows/fierro-test.yml @@ -41,6 +41,10 @@ jobs: run: sudo apt-get install python3.6 shell: bash + - name: Install dos2unix + run: + sudo apt-get install dos2unix + shell: bash - name: Set up Miniconda uses: conda-incubator/setup-miniconda@v2 From f81658e588739c47fef63c8d4b718a38135ab7ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CAdrian-Diaz=E2=80=9D?= <“adriandiaz1117@gmail.com”> Date: Thu, 11 Jul 2024 15:54:41 -0600 Subject: [PATCH 074/130] PERF: speed up adjoint loops --- .../SGH_Solver/src/force_gradients_sgh.cpp | 56 +++++++++---------- .../SGH_Solver/src/power_gradients_sgh.cpp | 38 ++++++------- 2 files changed, 46 insertions(+), 48 deletions(-) diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp index 567789239..830c062ec 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp @@ -509,26 +509,32 @@ void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& ma // loop over dimension for (int dim = 0; dim < num_dims; dim++) { + const double current_node_vel = node_vel(rk_level, node_gid, dim); // assign gradient of corner contribution of force to relevant matrix entries with non-zero node velocity gradient for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) { size_t gradient_node_gid = nodes_in_elem(elem_gid, igradient); column_index = num_dims * Global_Gradient_Matrix_Assembly_Map(elem_gid, igradient, node_lid); - for (int jdim = 0; jdim < num_dims; jdim++) { - if (node_lid == igradient && jdim == dim) { - if (map->isNodeLocalElement(gradient_node_gid)) { + + if (map->isNodeLocalElement(gradient_node_gid)) { + for (int jdim = 0; jdim < num_dims; jdim++) { + if (node_lid == igradient && jdim == dim) { Force_Gradient_Velocities(gradient_node_gid * num_dims + jdim, column_index + dim) += phi * (muc(node_lid) * (vel_star_gradient(dim, igradient, jdim) - 1) + - muc_gradient(node_lid, igradient, jdim) * (vel_star(dim) - node_vel(rk_level, node_gid, dim))); + muc_gradient(node_lid, igradient, jdim) * (vel_star(dim) - current_node_vel)); } + else { + Force_Gradient_Velocities(gradient_node_gid * num_dims + jdim, column_index + dim) += phi * (muc(node_lid) * (vel_star_gradient(dim, igradient, jdim)) + + muc_gradient(node_lid, igradient, jdim) * (vel_star(dim) - current_node_vel)); + } + } + } + for (int jdim = 0; jdim < num_dims; jdim++) { + if (node_lid == igradient && jdim == dim) { corner_gradient_storage(corner_gid, dim, igradient, jdim) = phi * (muc(node_lid) * (vel_star_gradient(dim, igradient, jdim) - 1) + - muc_gradient(node_lid, igradient, jdim) * (vel_star(dim) - node_vel(rk_level, node_gid, dim))); + muc_gradient(node_lid, igradient, jdim) * (vel_star(dim) - current_node_vel)); } else { - if (map->isNodeLocalElement(gradient_node_gid)) { - Force_Gradient_Velocities(gradient_node_gid * num_dims + jdim, column_index + dim) += phi * (muc(node_lid) * (vel_star_gradient(dim, igradient, jdim)) + - muc_gradient(node_lid, igradient, jdim) * (vel_star(dim) - node_vel(rk_level, node_gid, dim))); - } corner_gradient_storage(corner_gid, dim, igradient, jdim) = phi * (muc(node_lid) * (vel_star_gradient(dim, igradient, jdim)) + - muc_gradient(node_lid, igradient, jdim) * (vel_star(dim) - node_vel(rk_level, node_gid, dim))); + muc_gradient(node_lid, igradient, jdim) * (vel_star(dim) - current_node_vel)); } } } @@ -1093,8 +1099,6 @@ void FEA_Module_SGH::get_force_ugradient_sgh(const DCArrayKokkos& ma ViewCArrayKokkos vel_star_gradient(vel_star_gradient_array, num_dims, num_nodes_in_elem, num_dims); ViewCArrayKokkos vel_grad(vel_grad_array, num_dims, num_dims); - // --- abviatations of variables --- - // element volume double vol = elem_vol(elem_gid); @@ -1470,40 +1474,34 @@ void FEA_Module_SGH::get_force_ugradient_sgh(const DCArrayKokkos& ma // loop over dimension for (int dim = 0; dim < num_dims; dim++) { + const double current_node_vel = node_vel(rk_level, node_gid, dim); // assign gradient of corner contribution of force to relevant matrix entries with non-zero node velocity gradient for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) { - for (int jdim = 0; jdim < num_dims; jdim++) { - size_t gradient_node_gid = nodes_in_elem(elem_gid, igradient); - // if(!map->isNodeLocalElement(gradient_node_gid)) continue; - column_index = num_dims * Global_Gradient_Matrix_Assembly_Map(elem_gid, igradient, node_lid); - if (map->isNodeLocalElement(gradient_node_gid)) { + size_t gradient_node_gid = nodes_in_elem(elem_gid, igradient); + // if(!map->isNodeLocalElement(gradient_node_gid)) continue; + column_index = num_dims * Global_Gradient_Matrix_Assembly_Map(elem_gid, igradient, node_lid); + + if (map->isNodeLocalElement(gradient_node_gid)) { + for (int jdim = 0; jdim < num_dims; jdim++) { Force_Gradient_Positions(gradient_node_gid * num_dims + jdim, column_index + dim) += area_normal(node_lid, 0) * tau_gradient(0, dim, igradient, jdim) + area_normal(node_lid, 1) * tau_gradient(1, dim, igradient, jdim) + area_normal(node_lid, 2) * tau_gradient(2, dim, igradient, jdim) + area_normal_gradients(node_lid, 0, igradient, jdim) * tau(0, dim) + area_normal_gradients(node_lid, 1, igradient, jdim) * tau(1, dim) + area_normal_gradients(node_lid, 2, igradient, jdim) * tau(2, dim) - + phi * muc_gradient(node_lid, igradient, jdim) * (vel_star(dim) - node_vel(rk_level, node_gid, dim)) + + phi * muc_gradient(node_lid, igradient, jdim) * (vel_star(dim) - current_node_vel) + phi * muc(node_lid) * (vel_star_gradient(dim, igradient, jdim)); } + } + for (int jdim = 0; jdim < num_dims; jdim++) { corner_gradient_storage(corner_gid, dim, igradient, jdim) = area_normal(node_lid, 0) * tau_gradient(0, dim, igradient, jdim) + area_normal(node_lid, 1) * tau_gradient(1, dim, igradient, jdim) + area_normal(node_lid, 2) * tau_gradient(2, dim, igradient, jdim) + area_normal_gradients(node_lid, 0, igradient, jdim) * tau(0, dim) + area_normal_gradients(node_lid, 1, igradient, jdim) * tau(1, dim) + area_normal_gradients(node_lid, 2, igradient, jdim) * tau(2, dim) - + phi * muc_gradient(node_lid, igradient, jdim) * (vel_star(dim) - node_vel(rk_level, node_gid, dim)) + + phi * muc_gradient(node_lid, igradient, jdim) * (vel_star(dim) - current_node_vel) + phi * muc(node_lid) * (vel_star_gradient(dim, igradient, jdim)); - // if(map->isNodeLocalElement(gradient_node_gid)){ - // Force_Gradient_Positions(gradient_node_gid*num_dims+jdim, column_index+dim) += - // + area_normal_gradients(node_lid, 0, igradient, jdim)*tau(0, dim) - // + area_normal_gradients(node_lid, 1, igradient, jdim)*tau(1, dim) - // + area_normal_gradients(node_lid, 2, igradient, jdim)*tau(2, dim); - // } - // corner_gradient_storage(corner_gid,dim,igradient,jdim) = - // + area_normal_gradients(node_lid, 0, igradient, jdim)*tau(0, dim) - // + area_normal_gradients(node_lid, 1, igradient, jdim)*tau(1, dim) - // + area_normal_gradients(node_lid, 2, igradient, jdim)*tau(2, dim); } } } // end loop over dimension diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/power_gradients_sgh.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/power_gradients_sgh.cpp index f9068c2a8..1a2d76f9c 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/power_gradients_sgh.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/power_gradients_sgh.cpp @@ -372,15 +372,15 @@ void FEA_Module_SGH::get_power_ugradient_sgh(double rk_alpha, // calculate the Power=F dot V for this corner for (size_t dim = 0; dim < num_dims; dim++) { + const double current_node_vel = node_vel(rk_level, node_gid, dim); for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) { - for (size_t jdim = 0; jdim < num_dims; jdim++) { - column_id = Element_Gradient_Matrix_Assembly_Map(elem_gid, igradient); - gradient_node_id = nodes_in_elem(elem_gid, igradient); - if (!map->isNodeLocalElement(gradient_node_id)) { - continue; + column_id = Element_Gradient_Matrix_Assembly_Map(elem_gid, igradient); + gradient_node_id = nodes_in_elem(elem_gid, igradient); + if (map->isNodeLocalElement(gradient_node_id)) { + for (size_t jdim = 0; jdim < num_dims; jdim++) { + Power_Gradient_Positions(gradient_node_id * num_dims + jdim, column_id) -= + corner_gradient_storage(corner_gid, dim, igradient, jdim) * current_node_vel * node_radius; } - Power_Gradient_Positions(gradient_node_id * num_dims + jdim, column_id) -= - corner_gradient_storage(corner_gid, dim, igradient, jdim) * node_vel(rk_level, node_gid, dim) * node_radius; } } } // end for dim @@ -446,21 +446,21 @@ void FEA_Module_SGH::get_power_vgradient_sgh(double rk_alpha, // calculate the Power=F dot V for this corner for (size_t dim = 0; dim < num_dims; dim++) { + const double current_node_vel = node_vel(rk_level, node_gid, dim); for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) { column_id = Element_Gradient_Matrix_Assembly_Map(elem_gid, igradient); gradient_node_id = nodes_in_elem(elem_gid, igradient); - if (!map->isNodeLocalElement(gradient_node_id)) { - continue; - } - for (size_t jdim = 0; jdim < num_dims; jdim++) { - if (node_lid == igradient && jdim == dim) { - Power_Gradient_Velocities(gradient_node_id * num_dims + jdim, column_id) -= - corner_gradient_storage(corner_gid, dim, igradient, jdim) * node_vel(rk_level, node_gid, dim) - * node_radius + corner_force(corner_gid, dim) * node_radius; - } - else { - Power_Gradient_Velocities(gradient_node_id * num_dims + jdim, column_id) -= - corner_gradient_storage(corner_gid, dim, igradient, jdim) * node_vel(rk_level, node_gid, dim) * node_radius; + if (map->isNodeLocalElement(gradient_node_id)) { + for (size_t jdim = 0; jdim < num_dims; jdim++) { + if (node_lid == igradient && jdim == dim) { + Power_Gradient_Velocities(gradient_node_id * num_dims + jdim, column_id) -= + corner_gradient_storage(corner_gid, dim, igradient, jdim) * current_node_vel + * node_radius + corner_force(corner_gid, dim) * node_radius; + } + else { + Power_Gradient_Velocities(gradient_node_id * num_dims + jdim, column_id) -= + corner_gradient_storage(corner_gid, dim, igradient, jdim) * current_node_vel * node_radius; + } } } } From 87bbe71a50a5d1c34988af865ac81db67a408041 Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Fri, 12 Jul 2024 09:38:58 -0600 Subject: [PATCH 075/130] Update publish-fierro-cpu.yaml --- .github/workflows/publish-fierro-cpu.yaml | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/.github/workflows/publish-fierro-cpu.yaml b/.github/workflows/publish-fierro-cpu.yaml index 561447e42..c23c0faff 100644 --- a/.github/workflows/publish-fierro-cpu.yaml +++ b/.github/workflows/publish-fierro-cpu.yaml @@ -9,7 +9,16 @@ on: jobs: publish: - uses: ./.github/workflows/build-conda-package.yaml - with: - recipe_dir: .conda/fierro/cpu - secrets: inherit + runs-on: ubuntu-latest + + steps: + - uses: ./.github/workflows/build-conda-package.yaml + with: + recipe_dir: .conda/fierro/cpu + - name: Install MPI + run: | + sudo apt-get install libopenmpi-dev openmpi-bin + echo "MPI_OPTS=--oversubscribe" >> $GITHUB_ENV + + + From d99d4040f23979318a158671a0ec20de4fc1e5f4 Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Fri, 12 Jul 2024 09:40:00 -0600 Subject: [PATCH 076/130] Update publish-fierro-cpu.yaml --- .github/workflows/publish-fierro-cpu.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/publish-fierro-cpu.yaml b/.github/workflows/publish-fierro-cpu.yaml index c23c0faff..01c8afcaa 100644 --- a/.github/workflows/publish-fierro-cpu.yaml +++ b/.github/workflows/publish-fierro-cpu.yaml @@ -12,6 +12,9 @@ jobs: runs-on: ubuntu-latest steps: + - uses: actions/checkout@v4 + with: + submodules: recursive - uses: ./.github/workflows/build-conda-package.yaml with: recipe_dir: .conda/fierro/cpu From a66baf76cd997a116bb8a3eb916da1e0535a42d6 Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Fri, 12 Jul 2024 09:41:24 -0600 Subject: [PATCH 077/130] Update build-conda-package.yaml --- .github/workflows/build-conda-package.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-conda-package.yaml b/.github/workflows/build-conda-package.yaml index 84f6b2174..161f8c7bd 100644 --- a/.github/workflows/build-conda-package.yaml +++ b/.github/workflows/build-conda-package.yaml @@ -53,7 +53,7 @@ jobs: # conda-${{ runner.os }}--${{ runner.arch }}--${{ steps.get-date.outputs.today }}-${{ hashFiles('.etc/conda_build_environment.yaml') }}-${{ env.CACHE_NUMBER }} # env: # CACHE_NUMBER: 1 - # id: cache + # id: cached - name: Update environment run: From 6c500d77cd137daf06b56e010ae22157fb6955d3 Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Fri, 12 Jul 2024 09:44:55 -0600 Subject: [PATCH 078/130] Update publish-fierro-cpu.yaml --- .github/workflows/publish-fierro-cpu.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/publish-fierro-cpu.yaml b/.github/workflows/publish-fierro-cpu.yaml index 01c8afcaa..019746b16 100644 --- a/.github/workflows/publish-fierro-cpu.yaml +++ b/.github/workflows/publish-fierro-cpu.yaml @@ -15,13 +15,14 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive - - uses: ./.github/workflows/build-conda-package.yaml - with: - recipe_dir: .conda/fierro/cpu - name: Install MPI run: | sudo apt-get install libopenmpi-dev openmpi-bin echo "MPI_OPTS=--oversubscribe" >> $GITHUB_ENV + - uses: ./.github/workflows/build-conda-package.yaml + with: + recipe_dir: .conda/fierro/cpu + From b9ff7b4dbcbfd902c3315374c7923cde554282ff Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Fri, 12 Jul 2024 09:46:17 -0600 Subject: [PATCH 079/130] Update publish-fierro-cpu.yaml --- .github/workflows/publish-fierro-cpu.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/publish-fierro-cpu.yaml b/.github/workflows/publish-fierro-cpu.yaml index 019746b16..a125f5364 100644 --- a/.github/workflows/publish-fierro-cpu.yaml +++ b/.github/workflows/publish-fierro-cpu.yaml @@ -22,6 +22,7 @@ jobs: - uses: ./.github/workflows/build-conda-package.yaml with: recipe_dir: .conda/fierro/cpu + secrets: inherit From 7f09ddb69ac8de363d2808989aad85039c43351b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CAdrian-Diaz=E2=80=9D?= <“adriandiaz1117@gmail.com”> Date: Fri, 12 Jul 2024 14:14:33 -0600 Subject: [PATCH 080/130] WIP: cleanup adjoint loops --- .../SGH_Solver/src/sgh_optimization.cpp | 47 ++++++++++++------- 1 file changed, 31 insertions(+), 16 deletions(-) diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/sgh_optimization.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/sgh_optimization.cpp index 40d7c1169..669f0421f 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/sgh_optimization.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/sgh_optimization.cpp @@ -566,7 +566,11 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full(Teuchos::RCPdynamic_options.print_cycle; - + + double total_adjoint_time = Explicit_Solver_Pointer_->CPU_Time(); + double state_adjoint_time = 0; + double state_adjoint_time_start, state_adjoint_time_end; + auto optimization_objective_regions = simparam->optimization_options.optimization_objective_regions; //initialize tally storage of gradient vector cached_design_gradients_distributed->putScalar(0); // initialize first adjoint vector at last_time_step to 0 as the terminal value @@ -682,7 +686,6 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full(Teuchos::RCPCPU_Time(); // force_gradient_velocity->describe(*fos,Teuchos::VERB_EXTREME); const_vec_array previous_force_gradient_position = force_gradient_position->getLocalView(Tpetra::Access::ReadOnly); // const_vec_array current_force_gradient_position = force_gradient_position->getLocalView (Tpetra::Access::ReadOnly); @@ -862,10 +866,9 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full(Teuchos::RCPgetLocalView(Tpetra::Access::ReadWrite); vec_array phi_midpoint_adjoint_vector = phi_adjoint_vector_distributed->getLocalView(Tpetra::Access::ReadWrite); vec_array psi_midpoint_adjoint_vector = psi_adjoint_vector_distributed->getLocalView(Tpetra::Access::ReadWrite); - // half step update for RK2 scheme; EQUATION 1 - if(simparam->optimization_options.optimization_objective_regions.size()){ - int nobj_volumes = simparam->optimization_options.optimization_objective_regions.size(); + if(optimization_objective_regions.size()){ + int nobj_volumes = optimization_objective_regions.size(); const_vec_array all_initial_node_coords = all_initial_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { real_t rate_of_change; @@ -878,7 +881,7 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full(Teuchos::RCPoptimization_options.optimization_objective_regions(ivolume).contains(current_node_coords)){ + if(optimization_objective_regions(ivolume).contains(current_node_coords)){ contained = 1; } } @@ -1027,9 +1030,10 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full(Teuchos::RCPCPU_Time(); + // state_adjoint_time += state_adjoint_time_end-state_adjoint_time_start; // set state according to phase data at this timestep - get_vol(); // ---- Calculate velocity diveregence for the element ---- @@ -1184,10 +1188,11 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full(Teuchos::RCPCPU_Time(); // full step update with midpoint gradient for RK2 scheme; EQUATION 1 - if(simparam->optimization_options.optimization_objective_regions.size()){ - int nobj_volumes = simparam->optimization_options.optimization_objective_regions.size(); + if(optimization_objective_regions.size()){ + int nobj_volumes = optimization_objective_regions.size(); const_vec_array all_initial_node_coords = all_initial_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { real_t rate_of_change; @@ -1199,8 +1204,9 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full(Teuchos::RCPoptimization_options.optimization_objective_regions(ivolume).contains(current_node_coords)){ + if(optimization_objective_regions(ivolume).contains(current_node_coords)){ contained = 1; } } @@ -1316,6 +1322,9 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full(Teuchos::RCPassign(*all_adjoint_vector_distributed); (*psi_adjoint_vector_data)[cycle]->assign(*psi_adjoint_vector_distributed); } + + // state_adjoint_time_end = Explicit_Solver_Pointer_->CPU_Time(); + // state_adjoint_time += state_adjoint_time_end-state_adjoint_time_start; } // end view scope //tally contribution to the gradient vector @@ -1367,7 +1376,6 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full(Teuchos::RCPCPU_Time(); get_force_dgradient_sgh(material, *mesh, node_coords, @@ -1476,7 +1485,9 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full(Teuchos::RCPCPU_Time(); + //state_adjoint_time += state_adjoint_time_end-state_adjoint_time_start; compute_topology_optimization_gradient_tally(design_densities_distributed, cached_design_gradients_distributed, cycle, global_dt); } @@ -1489,6 +1500,10 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full(Teuchos::RCPdescribe(*fos,Teuchos::VERB_EXTREME); } + + double total_adjoint_time_end = Explicit_Solver_Pointer_->CPU_Time(); + std::cout << "ADJOINT CALCULATION TIME ON RANK " << myrank << " IS " << total_adjoint_time_end-total_adjoint_time << std::endl; + // std::cout << "ADJOINT STATE CALCULATION TIME ON RANK " << myrank << " IS " << state_adjoint_time << std::endl; } ///////////////////////////////////////////////////////////////////////////// From 8525b50a8fdbdaec1b4104b699174f6e91a15be3 Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Fri, 12 Jul 2024 14:29:34 -0600 Subject: [PATCH 081/130] Update build.sh --- .conda/fierro/cpu/build.sh | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/.conda/fierro/cpu/build.sh b/.conda/fierro/cpu/build.sh index 58a902ee3..f2113fd60 100755 --- a/.conda/fierro/cpu/build.sh +++ b/.conda/fierro/cpu/build.sh @@ -1,5 +1,15 @@ # These flag variables are set by anaconda. source "$RECIPE_DIR/../../cross-compile-setup.sh" +BUILD_PREFIX=${BUILD_PREFIX:-$CONDA_PREFIX} + +# Set the MPI compiler variables +export MPI_C_COMPILER="${BUILD_PREFIX}/bin/mpicc" +export MPI_CXX_COMPILER="${BUILD_PREFIX}/bin/mpicxx" + +export PATH="${BUILD_PREFIX}/bin:$PATH" +export LD_LIBRARY_PATH="${BUILD_PREFIX}/lib:$LD_LIBRARY_PATH" +export INCLUDE_PATH="${BUILD_PREFIX}/include:$INCLUDE_PATH" + mkdir build cd build @@ -17,8 +27,8 @@ cmake .. \ -D DISTRIBUTION=On \ $CMAKE_ARGS \ -D CMAKE_CXX_FLAGS="$PATCHED_CXXFLAGS -fopenmp -D_LIBCPP_DISABLE_AVAILABILITY" \ - -D MPI_C_COMPILER="$BUILD_PREFIX/bin/mpicc" \ - -D MPI_CXX_COMPILER="$BUILD_PREFIX/bin/mpicxx" \ + -D MPI_C_COMPILER="${MPI_C_COMPILER}" \ + -D MPI_CXX_COMPILER="${MPI_CXX_COMPILER}" \ -D VECTOR_ARCH_FLAGS="$VECTOR_ARCH_FLAGS" \ -make -j 10 install \ No newline at end of file +make -j 10 install From 0f580474eae3e20db54e8a7f689081b67c8c9d77 Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Fri, 12 Jul 2024 14:30:11 -0600 Subject: [PATCH 082/130] Update publish-fierro-cpu.yaml --- .github/workflows/publish-fierro-cpu.yaml | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/.github/workflows/publish-fierro-cpu.yaml b/.github/workflows/publish-fierro-cpu.yaml index a125f5364..a3aeff01b 100644 --- a/.github/workflows/publish-fierro-cpu.yaml +++ b/.github/workflows/publish-fierro-cpu.yaml @@ -9,21 +9,10 @@ on: jobs: publish: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - with: - submodules: recursive - - name: Install MPI - run: | - sudo apt-get install libopenmpi-dev openmpi-bin - echo "MPI_OPTS=--oversubscribe" >> $GITHUB_ENV - - uses: ./.github/workflows/build-conda-package.yaml - with: - recipe_dir: .conda/fierro/cpu - secrets: inherit - + uses: ./.github/workflows/build-conda-package.yaml + with: + recipe_dir: .conda/fierro/cpu + secrets: inherit From 970e8e9cd3fd627464d4700ab8b071a09e283125 Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Fri, 12 Jul 2024 14:58:47 -0600 Subject: [PATCH 083/130] Update build.sh --- .conda/fierro/cpu/build.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.conda/fierro/cpu/build.sh b/.conda/fierro/cpu/build.sh index f2113fd60..ee349aa07 100755 --- a/.conda/fierro/cpu/build.sh +++ b/.conda/fierro/cpu/build.sh @@ -29,6 +29,8 @@ cmake .. \ -D CMAKE_CXX_FLAGS="$PATCHED_CXXFLAGS -fopenmp -D_LIBCPP_DISABLE_AVAILABILITY" \ -D MPI_C_COMPILER="${MPI_C_COMPILER}" \ -D MPI_CXX_COMPILER="${MPI_CXX_COMPILER}" \ + -D MPI_C_INCLUDE_PATH="${BUILD_PREFIX}/include" \ + -D MPI_C_LIBRARIES="${BUILD_PREFIX}/lib" \ -D VECTOR_ARCH_FLAGS="$VECTOR_ARCH_FLAGS" \ make -j 10 install From efc895e5dee7bdc15ff829a5b4edb45891133792 Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Fri, 12 Jul 2024 15:33:54 -0600 Subject: [PATCH 084/130] Update build.sh --- .conda/fierro/cpu/build.sh | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/.conda/fierro/cpu/build.sh b/.conda/fierro/cpu/build.sh index ee349aa07..1203ff367 100755 --- a/.conda/fierro/cpu/build.sh +++ b/.conda/fierro/cpu/build.sh @@ -1,15 +1,5 @@ # These flag variables are set by anaconda. source "$RECIPE_DIR/../../cross-compile-setup.sh" -BUILD_PREFIX=${BUILD_PREFIX:-$CONDA_PREFIX} - -# Set the MPI compiler variables -export MPI_C_COMPILER="${BUILD_PREFIX}/bin/mpicc" -export MPI_CXX_COMPILER="${BUILD_PREFIX}/bin/mpicxx" - -export PATH="${BUILD_PREFIX}/bin:$PATH" -export LD_LIBRARY_PATH="${BUILD_PREFIX}/lib:$LD_LIBRARY_PATH" -export INCLUDE_PATH="${BUILD_PREFIX}/include:$INCLUDE_PATH" - mkdir build cd build @@ -18,6 +8,8 @@ cd build # see https://conda-forge.org/docs/maintainer/knowledge_base.html#newer-c-features-with-old-sdk cmake .. \ + -D CMAKE_C_COMPILER="$BUILD_PREFIX/bin/mpicc" \ + -D CMAKE_CXX_COMPILER="$BUILD_PREFIX/bin/mpicxx" \ -D CMAKE_BUILD_TYPE:STRING=RELEASE \ -D CMAKE_INSTALL_PREFIX:PATH=$PREFIX \ -D CMAKE_CXX_STANDARD:STRING=17 \ @@ -27,10 +19,6 @@ cmake .. \ -D DISTRIBUTION=On \ $CMAKE_ARGS \ -D CMAKE_CXX_FLAGS="$PATCHED_CXXFLAGS -fopenmp -D_LIBCPP_DISABLE_AVAILABILITY" \ - -D MPI_C_COMPILER="${MPI_C_COMPILER}" \ - -D MPI_CXX_COMPILER="${MPI_CXX_COMPILER}" \ - -D MPI_C_INCLUDE_PATH="${BUILD_PREFIX}/include" \ - -D MPI_C_LIBRARIES="${BUILD_PREFIX}/lib" \ -D VECTOR_ARCH_FLAGS="$VECTOR_ARCH_FLAGS" \ make -j 10 install From d20473d42f503790f77cc49525e22eb3df40ad0d Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Fri, 12 Jul 2024 15:52:17 -0600 Subject: [PATCH 085/130] Update build.sh --- .conda/fierro/cpu/build.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.conda/fierro/cpu/build.sh b/.conda/fierro/cpu/build.sh index 1203ff367..e37b9c280 100755 --- a/.conda/fierro/cpu/build.sh +++ b/.conda/fierro/cpu/build.sh @@ -1,5 +1,10 @@ # These flag variables are set by anaconda. source "$RECIPE_DIR/../../cross-compile-setup.sh" +BUILD_PREFIX=${BUILD_PREFIX:-$CONDA_PREFIX} + +export MPI_C_COMPILER="${BUILD_PREFIX}/bin/mpicc" +export MPI_CXX_COMPILER="${BUILD_PREFIX}/bin/mpicxx" + mkdir build cd build From 3fb36d6260e5245acfba39cdccf42e8f5cf75ac4 Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Fri, 12 Jul 2024 16:15:54 -0600 Subject: [PATCH 086/130] Update build.sh --- .conda/fierro/cpu/build.sh | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.conda/fierro/cpu/build.sh b/.conda/fierro/cpu/build.sh index e37b9c280..501ea78ac 100755 --- a/.conda/fierro/cpu/build.sh +++ b/.conda/fierro/cpu/build.sh @@ -1,9 +1,7 @@ # These flag variables are set by anaconda. source "$RECIPE_DIR/../../cross-compile-setup.sh" -BUILD_PREFIX=${BUILD_PREFIX:-$CONDA_PREFIX} - -export MPI_C_COMPILER="${BUILD_PREFIX}/bin/mpicc" -export MPI_CXX_COMPILER="${BUILD_PREFIX}/bin/mpicxx" +export PATH=$PATH:$BUILD_PREFIX/mpi/bin +export OPAL_PREFIX=/opt/hpcx/ompi mkdir build cd build @@ -13,8 +11,6 @@ cd build # see https://conda-forge.org/docs/maintainer/knowledge_base.html#newer-c-features-with-old-sdk cmake .. \ - -D CMAKE_C_COMPILER="$BUILD_PREFIX/bin/mpicc" \ - -D CMAKE_CXX_COMPILER="$BUILD_PREFIX/bin/mpicxx" \ -D CMAKE_BUILD_TYPE:STRING=RELEASE \ -D CMAKE_INSTALL_PREFIX:PATH=$PREFIX \ -D CMAKE_CXX_STANDARD:STRING=17 \ @@ -24,6 +20,8 @@ cmake .. \ -D DISTRIBUTION=On \ $CMAKE_ARGS \ -D CMAKE_CXX_FLAGS="$PATCHED_CXXFLAGS -fopenmp -D_LIBCPP_DISABLE_AVAILABILITY" \ + -D MPI_C_COMPILER="$BUILD_PREFIX/bin/mpicc" \ + -D MPI_CXX_COMPILER="$BUILD_PREFIX/bin/mpicxx" \ -D VECTOR_ARCH_FLAGS="$VECTOR_ARCH_FLAGS" \ make -j 10 install From 6e60a15f66d12b8db856d4934a86776fb080bab4 Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Fri, 12 Jul 2024 16:43:39 -0600 Subject: [PATCH 087/130] Update build.sh --- .conda/fierro/cpu/build.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.conda/fierro/cpu/build.sh b/.conda/fierro/cpu/build.sh index 501ea78ac..fa1d73ec3 100755 --- a/.conda/fierro/cpu/build.sh +++ b/.conda/fierro/cpu/build.sh @@ -1,7 +1,5 @@ # These flag variables are set by anaconda. source "$RECIPE_DIR/../../cross-compile-setup.sh" -export PATH=$PATH:$BUILD_PREFIX/mpi/bin -export OPAL_PREFIX=/opt/hpcx/ompi mkdir build cd build @@ -11,6 +9,7 @@ cd build # see https://conda-forge.org/docs/maintainer/knowledge_base.html#newer-c-features-with-old-sdk cmake .. \ + -D CMAKE_PREFIX_PATH=$BUILD_PREFIX/bin/mpiexec -D CMAKE_BUILD_TYPE:STRING=RELEASE \ -D CMAKE_INSTALL_PREFIX:PATH=$PREFIX \ -D CMAKE_CXX_STANDARD:STRING=17 \ From 2c08deb4f16eae13ce75c0cc7b8e4b6951bb02d5 Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Fri, 12 Jul 2024 16:53:14 -0600 Subject: [PATCH 088/130] Update build.sh --- .conda/fierro/cpu/build.sh | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/.conda/fierro/cpu/build.sh b/.conda/fierro/cpu/build.sh index fa1d73ec3..d0ad9a50c 100755 --- a/.conda/fierro/cpu/build.sh +++ b/.conda/fierro/cpu/build.sh @@ -1,7 +1,23 @@ # These flag variables are set by anaconda. source "$RECIPE_DIR/../../cross-compile-setup.sh" -mkdir build +export PATH=$PATH:/usr/local/mpi/bin +export OPAL_PREFIX=/opt/hpcx/ompi + +# Define the MPI root directory +export MPI_ROOT=/path/to/your/mpi + +# Set the MPI compiler variables +export MPI_C_COMPILER=${MPI_ROOT}/bin/mpicc +export MPI_CXX_COMPILER=${MPI_ROOT}/bin/mpicxx + +# Ensure the MPI libraries and includes are accessible +export PATH=${MPI_ROOT}/bin:$PATH +export LD_LIBRARY_PATH=${MPI_ROOT}/lib:$LD_LIBRARY_PATH +export INCLUDE_PATH=${MPI_ROOT}/include:$INCLUDE_PATH + +# Create a build directory +mkdir -p build cd build # -D _LIBCPP_DISABLE_AVAILABILITY @@ -9,7 +25,6 @@ cd build # see https://conda-forge.org/docs/maintainer/knowledge_base.html#newer-c-features-with-old-sdk cmake .. \ - -D CMAKE_PREFIX_PATH=$BUILD_PREFIX/bin/mpiexec -D CMAKE_BUILD_TYPE:STRING=RELEASE \ -D CMAKE_INSTALL_PREFIX:PATH=$PREFIX \ -D CMAKE_CXX_STANDARD:STRING=17 \ @@ -19,8 +34,8 @@ cmake .. \ -D DISTRIBUTION=On \ $CMAKE_ARGS \ -D CMAKE_CXX_FLAGS="$PATCHED_CXXFLAGS -fopenmp -D_LIBCPP_DISABLE_AVAILABILITY" \ - -D MPI_C_COMPILER="$BUILD_PREFIX/bin/mpicc" \ - -D MPI_CXX_COMPILER="$BUILD_PREFIX/bin/mpicxx" \ + -D MPI_C_COMPILER=${MPI_C_COMPILER} \ + -D MPI_CXX_COMPILER=${MPI_CXX_COMPILER} \ -D VECTOR_ARCH_FLAGS="$VECTOR_ARCH_FLAGS" \ make -j 10 install From 1140d7252089efec32f095a8dd05a8b3e82b0142 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CAdrian-Diaz=E2=80=9D?= <“adriandiaz1117@gmail.com”> Date: Sun, 14 Jul 2024 16:51:27 -0600 Subject: [PATCH 089/130] PERF: clean up gradient ops --- .../SGH_Solver/src/force_gradients_sgh.cpp | 31 +- .../SGH_Solver/src/geometry.cpp | 336 ++++++++---------- 2 files changed, 155 insertions(+), 212 deletions(-) diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp index 830c062ec..cd32db21e 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_gradients_sgh.cpp @@ -361,17 +361,14 @@ void FEA_Module_SGH::get_force_vgradient_sgh(const DCArrayKokkos& ma } else{ // Using a full tensoral Riemann jump relation - mu_term = muc(node_lid) - * sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) - + area_normal(node_lid, 1) * area_normal(node_lid, 1) - + area_normal(node_lid, 2) * area_normal(node_lid, 2) ); + real_t area_normal_norm = sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) + +area_normal(node_lid, 1) * area_normal(node_lid, 1) + +area_normal(node_lid, 2) * area_normal(node_lid, 2)); + mu_term = muc(node_lid)*area_normal_norm; for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) { for (int jdim = 0; jdim < num_dims; jdim++) { - muc_gradient(node_lid, igradient, jdim) = muc_gradient(node_lid, igradient, jdim) - * sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) - + area_normal(node_lid, 1) * area_normal(node_lid, 1) - + area_normal(node_lid, 2) * area_normal(node_lid, 2) ); + muc_gradient(node_lid, igradient, jdim) = muc_gradient(node_lid, igradient, jdim)*area_normal_norm; } } } @@ -1330,22 +1327,20 @@ void FEA_Module_SGH::get_force_ugradient_sgh(const DCArrayKokkos& ma // Using a full tensoral Riemann jump relation mu_term = muc(node_lid) * sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) - + area_normal(node_lid, 1) * area_normal(node_lid, 1) - + area_normal(node_lid, 2) * area_normal(node_lid, 2) ); + + area_normal(node_lid, 1) * area_normal(node_lid, 1) + + area_normal(node_lid, 2) * area_normal(node_lid, 2) ); + real_t area_normal_norm = sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) + +area_normal(node_lid, 1) * area_normal(node_lid, 1) + +area_normal(node_lid, 2) * area_normal(node_lid, 2)); for (int igradient = 0; igradient < num_nodes_in_elem; igradient++) { for (int jdim = 0; jdim < num_dims; jdim++) { mu_term_gradient = muc_gradient(node_lid, igradient, jdim) - * sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) - + area_normal(node_lid, 1) * area_normal(node_lid, 1) - + area_normal(node_lid, 2) * area_normal(node_lid, 2) ); + * area_normal_norm; mu_term_gradient += muc(node_lid) * (area_normal(node_lid, 0) * area_normal_gradients(node_lid, 0, igradient, jdim) + area_normal(node_lid, 1) * area_normal_gradients(node_lid, 1, igradient, jdim) - + area_normal(node_lid, 2) * area_normal_gradients(node_lid, 2, igradient, jdim)) / - sqrt(area_normal(node_lid, 0) * area_normal(node_lid, 0) - + area_normal(node_lid, 1) * area_normal(node_lid, 1) - + area_normal(node_lid, 2) * area_normal(node_lid, 2) ); - + + area_normal(node_lid, 2) * area_normal_gradients(node_lid, 2, igradient, jdim))/ area_normal_norm; + muc_gradient(node_lid, igradient, jdim) = mu_term_gradient; } } diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/geometry.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/geometry.cpp index 6e2b124ad..0357a1e76 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/geometry.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/geometry.cpp @@ -1,5 +1,5 @@ /********************************************************************************************** - 2020. Triad National Security, LLC. All rights reserved. + � 2020. Triad National Security, LLC. All rights reserved. This program was produced under U.S. Government contract 89233218CNA000001 for Los Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC for the U.S. Department of Energy/National Nuclear Security Administration. All rights in the program are @@ -732,199 +732,147 @@ void FEA_Module_SGH::get_vol_hex_ugradient(const ViewCArrayKokkos& elem_ double twelth = 1. / 12.; // element volume gradient - for (int inode = 0; inode < 8; inode++) { - for (int idim = 0; idim < num_dims; idim++) { - switch (num_dims * inode + idim) { - case 0: - gradient_result = - ((y(2) * (z(1) - z(3)) + y(7) * (z(3) - z(4)) + y(5) * (-z(1) + z(4)) + y(1) * (-z(2) - z(3) + z(4) + z(5)) + y(3) * (z(1) + z(2) - z(4) - z(7)) + y(4) * - (-z(1) + z(3) - z(5) + z(7)))) * twelth; - break; - case 3: - gradient_result = - ((y(3) * (-z(0) + z(2)) + y(4) * (z(0) - z(5)) + y(0) * (z(2) + z(3) - z(4) - z(5)) + y(6) * (-z(2) + z(5)) + y(5) * (z(0) - z(2) + z(4) - z(6)) + y(2) * - (-z(0) - z(3) + z(5) + z(6)))) * twelth; - break; - - case 6: - gradient_result = - ((y(0) * (-z(1) + z(3)) + y(5) * (z(1) - z(6)) + y(1) * (z(0) + z(3) - z(5) - z(6)) + y(7) * (-z(3) + z(6)) + y(6) * (z(1) - z(3) + z(5) - z(7)) + y(3) * - (-z(0) - z(1) + z(6) + z(7)))) * twelth; - break; - case 9: - gradient_result = - ((y(1) * (z(0) - z(2)) + y(7) * (-z(0) + z(2) - z(4) + z(6)) + y(6) * (z(2) - z(7)) + y(2) * (z(0) + z(1) - z(6) - z(7)) + y(4) * (-z(0) + z(7)) + y(0) * - (-z(1) - z(2) + z(4) + z(7)))) * twelth; - break; - case 12: - gradient_result = - ((y(1) * (-z(0) + z(5)) + y(7) * (z(0) + z(3) - z(5) - z(6)) + y(3) * (z(0) - z(7)) + y(0) * (z(1) - z(3) + z(5) - z(7)) + y(6) * (-z(5) + z(7)) + y(5) * - (-z(0) - z(1) + z(6) + z(7)))) * twelth; - break; - case 15: - gradient_result = - ((y(0) * (z(1) - z(4)) + y(7) * (z(4) - z(6)) + y(2) * (-z(1) + z(6)) + y(1) * (-z(0) + z(2) - z(4) + z(6)) + y(4) * (z(0) + z(1) - z(6) - z(7)) + y(6) * - (-z(1) - z(2) + z(4) + z(7)))) * twelth; - break; - case 18: - gradient_result = - ((y(1) * (z(2) - z(5)) + y(7) * (-z(2) - z(3) + z(4) + z(5)) + y(5) * (z(1) + z(2) - z(4) - z(7)) + y(4) * (z(5) - z(7)) + y(3) * (-z(2) + z(7)) + y(2) * - (-z(1) + z(3) - z(5) + z(7)))) * twelth; - break; - case 21: - gradient_result = - ((y(0) * (-z(3) + z(4)) + y(6) * (z(2) + z(3) - z(4) - z(5)) + y(2) * (z(3) - z(6)) + y(3) * (z(0) - z(2) + z(4) - z(6)) + y(5) * (-z(4) + z(6)) + y(4) * - (-z(0) - z(3) + z(5) + z(6)))) * twelth; - break; - case 1: - gradient_result = - (x(1) * ((z(2) + z(3) - z(4) - z(5))) + - x(7) * ((-z(3) + z(4))) + - x(3) * ((-z(1) - z(2) + z(4) + z(7))) + - x(5) * ((z(1) - z(4))) + - x(2) * ((-z(1) + z(3))) + - x(4) * ((z(1) - z(3) + z(5) - z(7)))) * twelth; - break; - case 4: - gradient_result = - (x(3) * ((z(0) - z(2))) + - x(5) * ((-z(0) + z(2) - z(4) + z(6))) + - x(6) * ((z(2) - z(5))) + - x(0) * ((-z(2) - z(3) + z(4) + z(5))) + - x(2) * ((z(0) + z(3) - z(5) - z(6))) + - x(4) * ((-z(0) + z(5)))) * twelth; - break; - case 7: - gradient_result = - (x(1) * ((-z(0) - z(3) + z(5) + z(6))) + - x(7) * ((z(3) - z(6))) + - x(3) * ((z(0) + z(1) - z(6) - z(7))) + - x(5) * ((-z(1) + z(6))) + - x(6) * ((-z(1) + z(3) - z(5) + z(7))) + - x(0) * ((z(1) - z(3)))) * twelth; - break; - case 10: - gradient_result = - (x(1) * ((-z(0) + z(2))) + - x(7) * ((z(0) - z(2) + z(4) - z(6))) + - x(6) * ((-z(2) + z(7))) + - x(0) * ((z(1) + z(2) - z(4) - z(7))) + - x(2) * ((-z(0) - z(1) + z(6) + z(7))) + - x(4) * ((z(0) - z(7)))) * twelth; - break; - case 13: - gradient_result = - (x(1) * ((z(0) - z(5))) + - x(7) * ((-z(0) - z(3) + z(5) + z(6))) + - x(3) * ((-z(0) + z(7))) + - x(5) * ((z(0) + z(1) - z(6) - z(7))) + - x(6) * ((z(5) - z(7))) + - x(0) * ((-z(1) + z(3) - z(5) + z(7)))) * twelth; - break; - case 16: - gradient_result = - (x(1) * ((z(0) - z(2) + z(4) - z(6))) + - x(7) * ((-z(4) + z(6))) + - x(6) * ((z(1) + z(2) - z(4) - z(7))) + - x(0) * ((-z(1) + z(4))) + - x(2) * ((z(1) - z(6))) + - x(4) * ((-z(0) - z(1) + z(6) + z(7)))) * twelth; - break; - case 19: - gradient_result = - (x(1) * ((-z(2) + z(5))) + - x(7) * ((z(2) + z(3) - z(4) - z(5))) + - x(3) * ((z(2) - z(7))) + - x(5) * ((-z(1) - z(2) + z(4) + z(7))) + - x(2) * ((z(1) - z(3) + z(5) - z(7))) + - x(4) * ((-z(5) + z(7)))) * twelth; - break; - case 22: - gradient_result = - (x(3) * ((-z(0) + z(2) - z(4) + z(6))) + - x(5) * ((z(4) - z(6))) + - x(6) * ((-z(2) - z(3) + z(4) + z(5))) + - x(0) * ((z(3) - z(4))) + - x(2) * ((-z(3) + z(6))) + - x(4) * ((z(0) + z(3) - z(5) - z(6)))) * twelth; - break; - case 2: - gradient_result = - (x(1) * (-y(3) + y(4) + y(5) - y(2)) + - x(7) * (y(3) - y(4)) + - x(3) * (y(1) - y(7) + y(2) - y(4)) + - x(5) * (-y(1) + y(4)) + - x(2) * (y(1) - y(3)) + - x(4) * (-y(1) + y(7) + y(3) - y(5))) * twelth; - break; - case 5: - gradient_result = - (x(3) * (y(2) - y(0)) + - x(5) * (y(0) - y(2) + y(4) - y(6)) + - x(6) * (y(5) - y(2)) + - x(0) * (y(2) - y(5) + y(3) - y(4)) + - x(2) * (-y(0) + y(5) + y(6) - y(3)) + - x(4) * (y(0) - y(5))) * twelth; - break; - case 8: - gradient_result = - (x(1) * (y(3) + y(0) - y(6) - y(5)) + - x(7) * (y(6) - y(3)) + - x(3) * (-y(1) + y(7) + y(6) - y(0)) + - x(5) * (y(1) - y(6)) + - x(6) * (y(1) - y(7) + y(5) - y(3)) + - x(0) * (-y(1) + y(3))) * twelth; - break; - case 11: - gradient_result = - (x(1) * (y(0) - y(2)) + - x(7) * (-y(0) + y(6) + y(2) - y(4)) + - x(6) * (-y(7) + y(2)) + - x(0) * (-y(2) + y(7) - y(1) + y(4)) + - x(2) * (y(0) + y(1) - y(7) - y(6)) + - x(4) * (y(7) - y(0))) * twelth; - break; - case 14: - gradient_result = - (x(1) * (-y(0) + y(5)) + - x(7) * (y(0) - y(6) + y(3) - y(5)) + - x(3) * (-y(7) + y(0)) + - x(5) * (-y(0) + y(7) - y(1) + y(6)) + - x(6) * (y(7) - y(5)) + - x(0) * (-y(7) + y(5) + y(1) - y(3))) * twelth; - break; - case 17: - gradient_result = - (x(1) * (-y(4) - y(0) + y(6) + y(2)) + - x(7) * (-y(6) + y(4)) + - x(6) * (-y(1) + y(7) + y(4) - y(2)) + - x(0) * (y(1) - y(4)) + - x(2) * (-y(1) + y(6)) + - x(4) * (y(1) - y(7) + y(0) - y(6))) * twelth; - break; - case 20: - gradient_result = - (x(1) * (-y(5) + y(2)) + - x(7) * (-y(2) - y(3) + y(5) + y(4)) + - x(3) * (y(7) - y(2)) + - x(5) * (-y(7) + y(2) + y(1) - y(4)) + - x(2) * (-y(5) - y(1) + y(7) + y(3)) + - x(4) * (-y(7) + y(5))) * twelth; - break; - case 23: - gradient_result = - (x(3) * (-y(6) - y(2) + y(4) + y(0)) + - x(5) * (-y(4) + y(6)) + - x(6) * (-y(5) - y(4) + y(3) + y(2)) + - x(0) * (-y(3) + y(4)) + - x(2) * (-y(6) + y(3)) + - x(4) * (-y(3) - y(0) + y(6) + y(5))) * twelth; - break; - } - elem_vol_gradients(inode, idim) = gradient_result; - } - } - return; + //x gradients + elem_vol_gradients(0, 0) = + ((y(2) * (z(1) - z(3)) + y(7) * (z(3) - z(4)) + y(5) * (-z(1) + z(4)) + y(1) * (-z(2) - z(3) + z(4) + z(5)) + y(3) * (z(1) + z(2) - z(4) - z(7)) + y(4) * + (-z(1) + z(3) - z(5) + z(7)))) * twelth; + elem_vol_gradients(1, 0) = + ((y(3) * (-z(0) + z(2)) + y(4) * (z(0) - z(5)) + y(0) * (z(2) + z(3) - z(4) - z(5)) + y(6) * (-z(2) + z(5)) + y(5) * (z(0) - z(2) + z(4) - z(6)) + y(2) * + (-z(0) - z(3) + z(5) + z(6)))) * twelth; + elem_vol_gradients(2, 0) = + ((y(0) * (-z(1) + z(3)) + y(5) * (z(1) - z(6)) + y(1) * (z(0) + z(3) - z(5) - z(6)) + y(7) * (-z(3) + z(6)) + y(6) * (z(1) - z(3) + z(5) - z(7)) + y(3) * + (-z(0) - z(1) + z(6) + z(7)))) * twelth; + elem_vol_gradients(3, 0) = + ((y(1) * (z(0) - z(2)) + y(7) * (-z(0) + z(2) - z(4) + z(6)) + y(6) * (z(2) - z(7)) + y(2) * (z(0) + z(1) - z(6) - z(7)) + y(4) * (-z(0) + z(7)) + y(0) * + (-z(1) - z(2) + z(4) + z(7)))) * twelth; + elem_vol_gradients(4, 0) = + ((y(1) * (-z(0) + z(5)) + y(7) * (z(0) + z(3) - z(5) - z(6)) + y(3) * (z(0) - z(7)) + y(0) * (z(1) - z(3) + z(5) - z(7)) + y(6) * (-z(5) + z(7)) + y(5) * + (-z(0) - z(1) + z(6) + z(7)))) * twelth; + elem_vol_gradients(5, 0) = + ((y(0) * (z(1) - z(4)) + y(7) * (z(4) - z(6)) + y(2) * (-z(1) + z(6)) + y(1) * (-z(0) + z(2) - z(4) + z(6)) + y(4) * (z(0) + z(1) - z(6) - z(7)) + y(6) * + (-z(1) - z(2) + z(4) + z(7)))) * twelth; + elem_vol_gradients(6, 0) = + ((y(1) * (z(2) - z(5)) + y(7) * (-z(2) - z(3) + z(4) + z(5)) + y(5) * (z(1) + z(2) - z(4) - z(7)) + y(4) * (z(5) - z(7)) + y(3) * (-z(2) + z(7)) + y(2) * + (-z(1) + z(3) - z(5) + z(7)))) * twelth; + elem_vol_gradients(7, 0) = + ((y(0) * (-z(3) + z(4)) + y(6) * (z(2) + z(3) - z(4) - z(5)) + y(2) * (z(3) - z(6)) + y(3) * (z(0) - z(2) + z(4) - z(6)) + y(5) * (-z(4) + z(6)) + y(4) * + (-z(0) - z(3) + z(5) + z(6)))) * twelth; + + //y gradients + elem_vol_gradients(0, 1) = + (x(1) * ((z(2) + z(3) - z(4) - z(5))) + + x(7) * ((-z(3) + z(4))) + + x(3) * ((-z(1) - z(2) + z(4) + z(7))) + + x(5) * ((z(1) - z(4))) + + x(2) * ((-z(1) + z(3))) + + x(4) * ((z(1) - z(3) + z(5) - z(7)))) * twelth; + elem_vol_gradients(1, 1) = + (x(3) * ((z(0) - z(2))) + + x(5) * ((-z(0) + z(2) - z(4) + z(6))) + + x(6) * ((z(2) - z(5))) + + x(0) * ((-z(2) - z(3) + z(4) + z(5))) + + x(2) * ((z(0) + z(3) - z(5) - z(6))) + + x(4) * ((-z(0) + z(5)))) * twelth; + elem_vol_gradients(2, 1) = + (x(1) * ((-z(0) - z(3) + z(5) + z(6))) + + x(7) * ((z(3) - z(6))) + + x(3) * ((z(0) + z(1) - z(6) - z(7))) + + x(5) * ((-z(1) + z(6))) + + x(6) * ((-z(1) + z(3) - z(5) + z(7))) + + x(0) * ((z(1) - z(3)))) * twelth; + elem_vol_gradients(3, 1) = + (x(1) * ((-z(0) + z(2))) + + x(7) * ((z(0) - z(2) + z(4) - z(6))) + + x(6) * ((-z(2) + z(7))) + + x(0) * ((z(1) + z(2) - z(4) - z(7))) + + x(2) * ((-z(0) - z(1) + z(6) + z(7))) + + x(4) * ((z(0) - z(7)))) * twelth; + elem_vol_gradients(4, 1) = + (x(1) * ((z(0) - z(5))) + + x(7) * ((-z(0) - z(3) + z(5) + z(6))) + + x(3) * ((-z(0) + z(7))) + + x(5) * ((z(0) + z(1) - z(6) - z(7))) + + x(6) * ((z(5) - z(7))) + + x(0) * ((-z(1) + z(3) - z(5) + z(7)))) * twelth; + elem_vol_gradients(5, 1) = + (x(1) * ((z(0) - z(2) + z(4) - z(6))) + + x(7) * ((-z(4) + z(6))) + + x(6) * ((z(1) + z(2) - z(4) - z(7))) + + x(0) * ((-z(1) + z(4))) + + x(2) * ((z(1) - z(6))) + + x(4) * ((-z(0) - z(1) + z(6) + z(7)))) * twelth; + elem_vol_gradients(6, 1) = + (x(1) * ((-z(2) + z(5))) + + x(7) * ((z(2) + z(3) - z(4) - z(5))) + + x(3) * ((z(2) - z(7))) + + x(5) * ((-z(1) - z(2) + z(4) + z(7))) + + x(2) * ((z(1) - z(3) + z(5) - z(7))) + + x(4) * ((-z(5) + z(7)))) * twelth; + elem_vol_gradients(7, 1) = + (x(3) * ((-z(0) + z(2) - z(4) + z(6))) + + x(5) * ((z(4) - z(6))) + + x(6) * ((-z(2) - z(3) + z(4) + z(5))) + + x(0) * ((z(3) - z(4))) + + x(2) * ((-z(3) + z(6))) + + x(4) * ((z(0) + z(3) - z(5) - z(6)))) * twelth; + + //z gradients + elem_vol_gradients(0, 2) = + (x(1) * (-y(3) + y(4) + y(5) - y(2)) + + x(7) * (y(3) - y(4)) + + x(3) * (y(1) - y(7) + y(2) - y(4)) + + x(5) * (-y(1) + y(4)) + + x(2) * (y(1) - y(3)) + + x(4) * (-y(1) + y(7) + y(3) - y(5))) * twelth; + elem_vol_gradients(1, 2) = + (x(3) * (y(2) - y(0)) + + x(5) * (y(0) - y(2) + y(4) - y(6)) + + x(6) * (y(5) - y(2)) + + x(0) * (y(2) - y(5) + y(3) - y(4)) + + x(2) * (-y(0) + y(5) + y(6) - y(3)) + + x(4) * (y(0) - y(5))) * twelth; + elem_vol_gradients(2, 2) = + (x(1) * (y(3) + y(0) - y(6) - y(5)) + + x(7) * (y(6) - y(3)) + + x(3) * (-y(1) + y(7) + y(6) - y(0)) + + x(5) * (y(1) - y(6)) + + x(6) * (y(1) - y(7) + y(5) - y(3)) + + x(0) * (-y(1) + y(3))) * twelth; + elem_vol_gradients(3, 2) = + (x(1) * (y(0) - y(2)) + + x(7) * (-y(0) + y(6) + y(2) - y(4)) + + x(6) * (-y(7) + y(2)) + + x(0) * (-y(2) + y(7) - y(1) + y(4)) + + x(2) * (y(0) + y(1) - y(7) - y(6)) + + x(4) * (y(7) - y(0))) * twelth; + elem_vol_gradients(4, 2) = + (x(1) * (-y(0) + y(5)) + + x(7) * (y(0) - y(6) + y(3) - y(5)) + + x(3) * (-y(7) + y(0)) + + x(5) * (-y(0) + y(7) - y(1) + y(6)) + + x(6) * (y(7) - y(5)) + + x(0) * (-y(7) + y(5) + y(1) - y(3))) * twelth; + elem_vol_gradients(5, 2) = + (x(1) * (-y(4) - y(0) + y(6) + y(2)) + + x(7) * (-y(6) + y(4)) + + x(6) * (-y(1) + y(7) + y(4) - y(2)) + + x(0) * (y(1) - y(4)) + + x(2) * (-y(1) + y(6)) + + x(4) * (y(1) - y(7) + y(0) - y(6))) * twelth; + elem_vol_gradients(6, 2) = + (x(1) * (-y(5) + y(2)) + + x(7) * (-y(2) - y(3) + y(5) + y(4)) + + x(3) * (y(7) - y(2)) + + x(5) * (-y(7) + y(2) + y(1) - y(4)) + + x(2) * (-y(5) - y(1) + y(7) + y(3)) + + x(4) * (-y(7) + y(5))) * twelth; + elem_vol_gradients(7, 2) = + (x(3) * (-y(6) - y(2) + y(4) + y(0)) + + x(5) * (-y(4) + y(6)) + + x(6) * (-y(5) - y(4) + y(3) + y(2)) + + x(0) * (-y(3) + y(4)) + + x(2) * (-y(6) + y(3)) + + x(4) * (-y(3) - y(0) + y(6) + y(5))) * twelth; } // end subroutine ///////////////////////////////////////////////////////////////////////////// From 7fdab4d5f511e46e62fe8b54ce9fb3336b6b8634 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CAdrian-Diaz=E2=80=9D?= <“adriandiaz1117@gmail.com”> Date: Sun, 14 Jul 2024 16:58:32 -0600 Subject: [PATCH 090/130] STYLE: adjoint time printout --- .../Parallel-Explicit/SGH_Solver/src/sgh_optimization.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/sgh_optimization.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/sgh_optimization.cpp index 669f0421f..b927054ae 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/sgh_optimization.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/sgh_optimization.cpp @@ -1502,7 +1502,7 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full(Teuchos::RCPCPU_Time(); - std::cout << "ADJOINT CALCULATION TIME ON RANK " << myrank << " IS " << total_adjoint_time_end-total_adjoint_time << std::endl; + *fos << "ADJOINT CALCULATION TIME ON RANK " << myrank << " IS " << total_adjoint_time_end-total_adjoint_time << std::endl; // std::cout << "ADJOINT STATE CALCULATION TIME ON RANK " << myrank << " IS " << state_adjoint_time << std::endl; } From c1da2cd3d53187824968d219662d27dbfe1c9306 Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Mon, 15 Jul 2024 15:12:30 -0600 Subject: [PATCH 091/130] Update build.sh --- .conda/fierro/cpu/build.sh | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/.conda/fierro/cpu/build.sh b/.conda/fierro/cpu/build.sh index d0ad9a50c..97ad998b0 100755 --- a/.conda/fierro/cpu/build.sh +++ b/.conda/fierro/cpu/build.sh @@ -1,20 +1,10 @@ # These flag variables are set by anaconda. source "$RECIPE_DIR/../../cross-compile-setup.sh" -export PATH=$PATH:/usr/local/mpi/bin -export OPAL_PREFIX=/opt/hpcx/ompi - -# Define the MPI root directory -export MPI_ROOT=/path/to/your/mpi - -# Set the MPI compiler variables -export MPI_C_COMPILER=${MPI_ROOT}/bin/mpicc -export MPI_CXX_COMPILER=${MPI_ROOT}/bin/mpicxx - -# Ensure the MPI libraries and includes are accessible -export PATH=${MPI_ROOT}/bin:$PATH -export LD_LIBRARY_PATH=${MPI_ROOT}/lib:$LD_LIBRARY_PATH -export INCLUDE_PATH=${MPI_ROOT}/include:$INCLUDE_PATH +export MPI_C_COMPILER=/usr/local/bin/mpicc +export MPI_CXX_COMPILER=/usr/local/bin/mpicxx +export MPI_C_INCLUDE_PATH=/usr/local/bin/mpi/include +export MPI_CXX_INCLUDE_PATH=/usr/local/bin/mpi/include # Create a build directory mkdir -p build From 7bc3866c92e281db655c03a0728ae1d7b78bd2ed Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Mon, 15 Jul 2024 15:22:05 -0600 Subject: [PATCH 092/130] Update CMakeLists.txt --- CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index cc39a5050..9649d2837 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,6 +2,8 @@ cmake_minimum_required(VERSION 3.17) project(Fierro NONE) set(CMAKE_CXX_STANDARD 17) +set(MPI_C_COMPILER /usr/local/bin/mpicc) +set(MPI_CXX_COMPILER /usr/local/bin/mpicxx) enable_language(CXX) # Disable Kokkos warning about not supporting C++ extensions set(CMAKE_CXX_EXTENSIONS OFF) From 3ee1b19d10c61530c984f44df3609c5c15af3a6a Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Mon, 15 Jul 2024 15:22:28 -0600 Subject: [PATCH 093/130] Update build.sh --- .conda/fierro/cpu/build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.conda/fierro/cpu/build.sh b/.conda/fierro/cpu/build.sh index 97ad998b0..7f9d2a45c 100755 --- a/.conda/fierro/cpu/build.sh +++ b/.conda/fierro/cpu/build.sh @@ -24,8 +24,8 @@ cmake .. \ -D DISTRIBUTION=On \ $CMAKE_ARGS \ -D CMAKE_CXX_FLAGS="$PATCHED_CXXFLAGS -fopenmp -D_LIBCPP_DISABLE_AVAILABILITY" \ - -D MPI_C_COMPILER=${MPI_C_COMPILER} \ - -D MPI_CXX_COMPILER=${MPI_CXX_COMPILER} \ + -D MPI_C_COMPILER="$BUILD_PREFIX/bin/mpicc" \ + -D MPI_CXX_COMPILER="$BUILD_PREFIX/bin/mpicxx" \ -D VECTOR_ARCH_FLAGS="$VECTOR_ARCH_FLAGS" \ make -j 10 install From 64fe65bb77b626d65d06699268bf3643a4401ad1 Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Mon, 15 Jul 2024 16:08:39 -0600 Subject: [PATCH 094/130] Update build.sh --- .conda/fierro/cpu/build.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.conda/fierro/cpu/build.sh b/.conda/fierro/cpu/build.sh index 7f9d2a45c..53a977b9c 100755 --- a/.conda/fierro/cpu/build.sh +++ b/.conda/fierro/cpu/build.sh @@ -1,10 +1,10 @@ # These flag variables are set by anaconda. source "$RECIPE_DIR/../../cross-compile-setup.sh" -export MPI_C_COMPILER=/usr/local/bin/mpicc -export MPI_CXX_COMPILER=/usr/local/bin/mpicxx -export MPI_C_INCLUDE_PATH=/usr/local/bin/mpi/include -export MPI_CXX_INCLUDE_PATH=/usr/local/bin/mpi/include +export MPI_C_COMPILER=$BUILD_PREFIX/bin/mpicc +export MPI_CXX_COMPILER=$BUILD_PREFIX/bin/mpicxx +export MPI_C_INCLUDE_PATH=$BUILD_PREFIX/bin/mpi/include +export MPI_CXX_INCLUDE_PATH=$BUILD_PREFIX/bin/mpi/include # Create a build directory mkdir -p build From b3e1c968ebd9897f51328cdc8693a7f1223fdaff Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Mon, 15 Jul 2024 16:17:00 -0600 Subject: [PATCH 095/130] Update CMakeLists.txt --- CMakeLists.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9649d2837..cc39a5050 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,8 +2,6 @@ cmake_minimum_required(VERSION 3.17) project(Fierro NONE) set(CMAKE_CXX_STANDARD 17) -set(MPI_C_COMPILER /usr/local/bin/mpicc) -set(MPI_CXX_COMPILER /usr/local/bin/mpicxx) enable_language(CXX) # Disable Kokkos warning about not supporting C++ extensions set(CMAKE_CXX_EXTENSIONS OFF) From 57e1d55f5ae7b2cf23a84c7f7a77e3e74597a5e8 Mon Sep 17 00:00:00 2001 From: Daniel Dunning <39738037+djdunning@users.noreply.github.com> Date: Tue, 16 Jul 2024 09:39:08 -0600 Subject: [PATCH 096/130] tried without specific mpi version --- .conda/fierro/cpu/meta.yaml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.conda/fierro/cpu/meta.yaml b/.conda/fierro/cpu/meta.yaml index d4953cf47..b312d74f0 100644 --- a/.conda/fierro/cpu/meta.yaml +++ b/.conda/fierro/cpu/meta.yaml @@ -26,16 +26,19 @@ requirements: - {{ compiler('cxx') }}={{ linux_compiler_version }} # [linux] - {{ compiler('cxx') }}={{ macos_compiler_version }} # [osx] - {{ compiler('fortran') }} - - openmpi={{ mpi_version }} + #DAN- openmpi={{ mpi_version }} + - openmpi host: - _openmp_mutex # [linux] - llvm-openmp # [osx] - - openmpi={{ mpi_version }} + #DAN- openmpi={{ mpi_version }} - fierro-trilinos-cpu - elements + - openmpi run: - fierro-trilinos-cpu - - mpi + - openmpi + #DAN- mpi about: home: https://github.com/lanl/Fierro From c064b921e1384322f3ca80bf1ebdafec0c9eb0a3 Mon Sep 17 00:00:00 2001 From: Daniel Dunning <39738037+djdunning@users.noreply.github.com> Date: Tue, 16 Jul 2024 10:28:34 -0600 Subject: [PATCH 097/130] continuing... --- .conda/fierro/cpu/meta.yaml | 8 +++++--- lib/.gitignore | 3 --- lib/Elements | 1 - scripts/build-fierro.sh | 4 ++-- scripts/cmake_build.sh | 28 ++++++++++++++-------------- 5 files changed, 21 insertions(+), 23 deletions(-) delete mode 100644 lib/.gitignore delete mode 160000 lib/Elements diff --git a/.conda/fierro/cpu/meta.yaml b/.conda/fierro/cpu/meta.yaml index b312d74f0..8ac4208f2 100644 --- a/.conda/fierro/cpu/meta.yaml +++ b/.conda/fierro/cpu/meta.yaml @@ -1,6 +1,6 @@ {% set version = "1.0.0" %} {% set linux_compiler_version = "10.4.0" %} -{% set macos_compiler_version = "12" %} +#{% set macos_compiler_version = "12" %} # We need the same MPI version in build + host. # So we have to specify it, unfortunately {% set mpi_version = "4.1" %} @@ -22,9 +22,11 @@ requirements: build: - cmake >=3.17.0 - {{ compiler('c') }}={{ linux_compiler_version }} # [linux] - - {{ compiler('c') }}={{ macos_compiler_version }} # [osx] + #- {{ compiler('c') }}={{ macos_compiler_version }} # [osx] + - compiler('c') + - compiler('cxx') - {{ compiler('cxx') }}={{ linux_compiler_version }} # [linux] - - {{ compiler('cxx') }}={{ macos_compiler_version }} # [osx] + #- {{ compiler('cxx') }}={{ macos_compiler_version }} # [osx] - {{ compiler('fortran') }} #DAN- openmpi={{ mpi_version }} - openmpi diff --git a/lib/.gitignore b/lib/.gitignore deleted file mode 100644 index c0bc38668..000000000 --- a/lib/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -Trilinos -hdf5 -heffte diff --git a/lib/Elements b/lib/Elements deleted file mode 160000 index 0de655cd2..000000000 --- a/lib/Elements +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 0de655cd260e60e4f037a78561dddfb9c8757fc7 diff --git a/scripts/build-fierro.sh b/scripts/build-fierro.sh index 42b60d177..65e30470d 100755 --- a/scripts/build-fierro.sh +++ b/scripts/build-fierro.sh @@ -183,8 +183,8 @@ source setup-env.sh ${machine} ${kokkos_build_type} ${build_cores} # Next, do action based on args if [ "$build_action" = "full-app" ]; then - source uncrustify-install.sh - source trilinos-install.sh ${kokkos_build_type} ${machine} + #source uncrustify-install.sh + #source trilinos-install.sh ${kokkos_build_type} ${machine} if [ "$solver" = "explicit-evpfft" ] || [ "${solver}" = "explicit-ls-evpfft" ]; then source hdf5-install.sh source heffte-install.sh ${heffte_build_type} ${machine} diff --git a/scripts/cmake_build.sh b/scripts/cmake_build.sh index 130584a95..3af34724f 100644 --- a/scripts/cmake_build.sh +++ b/scripts/cmake_build.sh @@ -12,7 +12,7 @@ kokkos_build_type="${3}" if { [ ! -d "${ELEMENTS_SOURCE_DIR}/elements" ] || [ ! -d "${ELEMENTS_SOURCE_DIR}/matar/src" ] ;} then echo "Missing submodules, downloading them...." - git submodule update "${ELEMENTS_SOURCE_DIR}" + #DANgit submodule update --recursive "${ELEMENTS_SOURCE_DIR}" fi if [ ! -d "${TRILINOS_INSTALL_DIR}/lib" ]; then @@ -24,22 +24,22 @@ fi # Install Elements if [ ! -d "${ELEMENTS_INSTALL_DIR}/lib" ]; then echo "Installing Elements..." - cmake -D CMAKE_INSTALL_PREFIX="$ELEMENTS_INSTALL_DIR" -D Trilinos_DIR="${Trilinos_DIR}" -D Matar_ENABLE_KOKKOS=ON -D Matar_KOKKOS_PACKAGE=Trilinos -B "${ELEMENTS_BUILD_DIR}" -S "${ELEMENTS_SOURCE_DIR}" - make -C "${ELEMENTS_BUILD_DIR}" -j${FIERRO_BUILD_CORES} - make -C "${ELEMENTS_BUILD_DIR}" install + #cmake -D CMAKE_INSTALL_PREFIX="$ELEMENTS_INSTALL_DIR" -D Trilinos_DIR="${Trilinos_DIR}" -D Matar_ENABLE_KOKKOS=ON -D Matar_KOKKOS_PACKAGE=Trilinos -B "${ELEMENTS_BUILD_DIR}" -S "${ELEMENTS_SOURCE_DIR}" + #make -C "${ELEMENTS_BUILD_DIR}" -j${FIERRO_BUILD_CORES} + #make -C "${ELEMENTS_BUILD_DIR}" install fi # Removing stale build directory -if [ -d "${FIERRO_BUILD_DIR}" ]; then - if make -C ${FIERRO_BUILD_DIR} distclean; then - echo ""; - else - echo "distclean failed. Removing build directory." - rm -r ${FIERRO_BUILD_DIR} - fi -else - mkdir -p ${FIERRO_BUILD_DIR} -fi +#if [ -d "${FIERRO_BUILD_DIR}" ]; then + #if make -C ${FIERRO_BUILD_DIR} distclean; then + # echo ""; + #else + # echo "distclean failed. Removing build directory." + # rm -r ${FIERRO_BUILD_DIR} + #fi +#else + #mkdir -p ${FIERRO_BUILD_DIR} +#fi # Configure EVPFFT using CMake cmake_options=( From 10706412c281c1c599dd3feb1601bdee1769f78e Mon Sep 17 00:00:00 2001 From: Daniel Dunning <39738037+djdunning@users.noreply.github.com> Date: Tue, 16 Jul 2024 10:33:42 -0600 Subject: [PATCH 098/130] continuing... --- .conda/fierro/cpu/meta.yaml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.conda/fierro/cpu/meta.yaml b/.conda/fierro/cpu/meta.yaml index 8ac4208f2..f97a657d0 100644 --- a/.conda/fierro/cpu/meta.yaml +++ b/.conda/fierro/cpu/meta.yaml @@ -1,6 +1,6 @@ {% set version = "1.0.0" %} {% set linux_compiler_version = "10.4.0" %} -#{% set macos_compiler_version = "12" %} +{% set macos_compiler_version = "16" %} # We need the same MPI version in build + host. # So we have to specify it, unfortunately {% set mpi_version = "4.1" %} @@ -22,11 +22,9 @@ requirements: build: - cmake >=3.17.0 - {{ compiler('c') }}={{ linux_compiler_version }} # [linux] - #- {{ compiler('c') }}={{ macos_compiler_version }} # [osx] - - compiler('c') - - compiler('cxx') + - {{ compiler('c') }}={{ macos_compiler_version }} # [osx] - {{ compiler('cxx') }}={{ linux_compiler_version }} # [linux] - #- {{ compiler('cxx') }}={{ macos_compiler_version }} # [osx] + - {{ compiler('cxx') }}={{ macos_compiler_version }} # [osx] - {{ compiler('fortran') }} #DAN- openmpi={{ mpi_version }} - openmpi From 08eaf5c1273b381a34a06ae8bad631e137c150fe Mon Sep 17 00:00:00 2001 From: Daniel Dunning <39738037+djdunning@users.noreply.github.com> Date: Tue, 16 Jul 2024 12:45:34 -0600 Subject: [PATCH 099/130] updated conda builds to us clang 16 on macs. Worked for Fierro, hopefully it works for others --- .conda/elements/meta.yaml | 2 +- .conda/evpfft-dev/cpu/meta.yaml | 2 +- .conda/evpfft/cpu/meta.yaml | 2 +- .conda/evpfft/cuda/meta.yaml | 2 +- .conda/fierro-dev/cpu/meta.yaml | 2 +- .conda/fierro/cpu/meta.yaml | 6 ++---- .conda/heffte/cpu/meta.yaml | 2 +- .conda/heffte/cuda/meta.yaml | 2 +- .conda/trilinos/cpu/meta.yaml | 2 +- .conda/voxelizer/meta.yaml | 2 +- 10 files changed, 11 insertions(+), 13 deletions(-) diff --git a/.conda/elements/meta.yaml b/.conda/elements/meta.yaml index 5be108cdf..a3b2cde6a 100644 --- a/.conda/elements/meta.yaml +++ b/.conda/elements/meta.yaml @@ -1,6 +1,6 @@ {% set version = "1.0.0" %} {% set linux_compiler_version = "10.4.0" %} -{% set macos_compiler_version = "12" %} +{% set macos_compiler_version = "16" %} package: name: elements diff --git a/.conda/evpfft-dev/cpu/meta.yaml b/.conda/evpfft-dev/cpu/meta.yaml index e59bb6636..31a431ba2 100644 --- a/.conda/evpfft-dev/cpu/meta.yaml +++ b/.conda/evpfft-dev/cpu/meta.yaml @@ -1,6 +1,6 @@ {% set version = "1.0.0" %} {% set linux_compiler_version = "10.4.0" %} -{% set macos_compiler_version = "12" %} +{% set macos_compiler_version = "16" %} package: name: evpfft-dev diff --git a/.conda/evpfft/cpu/meta.yaml b/.conda/evpfft/cpu/meta.yaml index 934140589..1fe23e817 100644 --- a/.conda/evpfft/cpu/meta.yaml +++ b/.conda/evpfft/cpu/meta.yaml @@ -1,6 +1,6 @@ {% set version = "1.0.0" %} {% set linux_compiler_version = "10.4.0" %} -{% set macos_compiler_version = "12" %} +{% set macos_compiler_version = "16" %} package: name: evpfft diff --git a/.conda/evpfft/cuda/meta.yaml b/.conda/evpfft/cuda/meta.yaml index 80d01d516..7327b24ac 100644 --- a/.conda/evpfft/cuda/meta.yaml +++ b/.conda/evpfft/cuda/meta.yaml @@ -1,6 +1,6 @@ {% set version = "1.0.0" %} {% set linux_compiler_version = "10.4.0" %} -{% set macos_compiler_version = "12" %} +{% set macos_compiler_version = "16" %} package: name: evpfft-cuda diff --git a/.conda/fierro-dev/cpu/meta.yaml b/.conda/fierro-dev/cpu/meta.yaml index f5346e1bf..3ef68df40 100644 --- a/.conda/fierro-dev/cpu/meta.yaml +++ b/.conda/fierro-dev/cpu/meta.yaml @@ -1,6 +1,6 @@ {% set version = "1.0.0" %} {% set linux_compiler_version = "10.4.0" %} -{% set macos_compiler_version = "12" %} +{% set macos_compiler_version = "16" %} package: name: fierro-dev diff --git a/.conda/fierro/cpu/meta.yaml b/.conda/fierro/cpu/meta.yaml index f97a657d0..2dad0321a 100644 --- a/.conda/fierro/cpu/meta.yaml +++ b/.conda/fierro/cpu/meta.yaml @@ -3,7 +3,8 @@ {% set macos_compiler_version = "16" %} # We need the same MPI version in build + host. # So we have to specify it, unfortunately -{% set mpi_version = "4.1" %} +# UPDATE: Not necessary right now, perhaps in the future +#{% set mpi_version = "4.1" %} package: name: fierro-cpu @@ -26,19 +27,16 @@ requirements: - {{ compiler('cxx') }}={{ linux_compiler_version }} # [linux] - {{ compiler('cxx') }}={{ macos_compiler_version }} # [osx] - {{ compiler('fortran') }} - #DAN- openmpi={{ mpi_version }} - openmpi host: - _openmp_mutex # [linux] - llvm-openmp # [osx] - #DAN- openmpi={{ mpi_version }} - fierro-trilinos-cpu - elements - openmpi run: - fierro-trilinos-cpu - openmpi - #DAN- mpi about: home: https://github.com/lanl/Fierro diff --git a/.conda/heffte/cpu/meta.yaml b/.conda/heffte/cpu/meta.yaml index 8bdca2aed..9b5c10033 100644 --- a/.conda/heffte/cpu/meta.yaml +++ b/.conda/heffte/cpu/meta.yaml @@ -1,6 +1,6 @@ {% set version = "1.0.0" %} {% set linux_compiler_version = "10.4.0" %} -{% set macos_compiler_version = "12" %} +{% set macos_compiler_version = "16" %} package: name: fierro-heffte diff --git a/.conda/heffte/cuda/meta.yaml b/.conda/heffte/cuda/meta.yaml index 92ec25398..bf78b85bc 100644 --- a/.conda/heffte/cuda/meta.yaml +++ b/.conda/heffte/cuda/meta.yaml @@ -1,6 +1,6 @@ {% set version = "1.0.0" %} {% set linux_compiler_version = "10.4.0" %} -{% set macos_compiler_version = "12" %} +{% set macos_compiler_version = "16" %} {% set cuda_version = "12.3" %} package: diff --git a/.conda/trilinos/cpu/meta.yaml b/.conda/trilinos/cpu/meta.yaml index 5bd236483..94d70de08 100644 --- a/.conda/trilinos/cpu/meta.yaml +++ b/.conda/trilinos/cpu/meta.yaml @@ -1,6 +1,6 @@ {% set version = "15.0.0" %} {% set linux_compiler_version = "10.4.0" %} -{% set macos_compiler_version = "12" %} +{% set macos_compiler_version = "16" %} package: name: fierro-trilinos-cpu diff --git a/.conda/voxelizer/meta.yaml b/.conda/voxelizer/meta.yaml index e027c6741..cfcbf40d6 100644 --- a/.conda/voxelizer/meta.yaml +++ b/.conda/voxelizer/meta.yaml @@ -1,6 +1,6 @@ {% set version = "1.0.0" %} {% set linux_compiler_version = "10.4.0" %} -{% set macos_compiler_version = "12" %} +{% set macos_compiler_version = "16" %} package: name: fierro-voxelizer From 45e529b328bb35940637b759edb83aaedf260cdd Mon Sep 17 00:00:00 2001 From: Daniel Dunning <39738037+djdunning@users.noreply.github.com> Date: Tue, 16 Jul 2024 13:06:48 -0600 Subject: [PATCH 100/130] trying to fix heffte cuda on linux?? --- .conda/heffte/cuda/meta.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.conda/heffte/cuda/meta.yaml b/.conda/heffte/cuda/meta.yaml index bf78b85bc..f20315ccf 100644 --- a/.conda/heffte/cuda/meta.yaml +++ b/.conda/heffte/cuda/meta.yaml @@ -28,6 +28,7 @@ requirements: - {{ compiler('cxx') }}={{ macos_compiler_version }} # [osx] - {{ compiler('fortran') }}={{ linux_compiler_version }} - openmpi + - cuda-compiler - cuda-toolkit={{ cuda_version }} # These host CUDA dependencies are definitely overspecified. # Not sure which ones are totally necessary. From f7324ad2b01696c3493c82d986f68f0f4d76eb0f Mon Sep 17 00:00:00 2001 From: Daniel Dunning <39738037+djdunning@users.noreply.github.com> Date: Tue, 16 Jul 2024 13:15:13 -0600 Subject: [PATCH 101/130] trying to fix heffte cuda on linux?? --- .conda/heffte/cuda/meta.yaml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.conda/heffte/cuda/meta.yaml b/.conda/heffte/cuda/meta.yaml index f20315ccf..0519a5297 100644 --- a/.conda/heffte/cuda/meta.yaml +++ b/.conda/heffte/cuda/meta.yaml @@ -1,7 +1,7 @@ {% set version = "1.0.0" %} {% set linux_compiler_version = "10.4.0" %} {% set macos_compiler_version = "16" %} -{% set cuda_version = "12.3" %} +#{% set cuda_version = "12.3" %} package: name: fierro-heffte-cuda @@ -29,11 +29,14 @@ requirements: - {{ compiler('fortran') }}={{ linux_compiler_version }} - openmpi - cuda-compiler - - cuda-toolkit={{ cuda_version }} + #- cuda-toolkit={{ cuda_version }} + - cuda-toolkit # These host CUDA dependencies are definitely overspecified. # Not sure which ones are totally necessary. host: - - cuda-cudart-static={{ cuda_version }} # Make sure your nvcc version and cudart versions are the same + #DAN- cuda-cudart-static={{ cuda_version }} # Make sure your nvcc version and cudart versions are the same + - cuda-cudart-static + - cuda-toolkit - libcusolver-dev - libcufft-dev - libcublas-dev From ffe679854108290e4f4baec095b8ecf87390abce Mon Sep 17 00:00:00 2001 From: Daniel Dunning <39738037+djdunning@users.noreply.github.com> Date: Tue, 16 Jul 2024 13:32:35 -0600 Subject: [PATCH 102/130] trying to fix heffte cuda on linux?? --- .conda/heffte/cuda/meta.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.conda/heffte/cuda/meta.yaml b/.conda/heffte/cuda/meta.yaml index 0519a5297..9595a5747 100644 --- a/.conda/heffte/cuda/meta.yaml +++ b/.conda/heffte/cuda/meta.yaml @@ -35,6 +35,7 @@ requirements: # Not sure which ones are totally necessary. host: #DAN- cuda-cudart-static={{ cuda_version }} # Make sure your nvcc version and cudart versions are the same + - cuda-compiler - cuda-cudart-static - cuda-toolkit - libcusolver-dev From 62a37f27c8fbadc80299ac15567742a5d5d317c0 Mon Sep 17 00:00:00 2001 From: Daniel Dunning <39738037+djdunning@users.noreply.github.com> Date: Tue, 16 Jul 2024 14:16:41 -0600 Subject: [PATCH 103/130] trying to fix heffte cuda on linux?? --- .conda/heffte/cuda/meta.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.conda/heffte/cuda/meta.yaml b/.conda/heffte/cuda/meta.yaml index 9595a5747..123c74ce9 100644 --- a/.conda/heffte/cuda/meta.yaml +++ b/.conda/heffte/cuda/meta.yaml @@ -1,7 +1,7 @@ {% set version = "1.0.0" %} -{% set linux_compiler_version = "10.4.0" %} +{% set linux_compiler_version = "12.4.0" %} {% set macos_compiler_version = "16" %} -#{% set cuda_version = "12.3" %} +{% set cuda_version = "12.4" %} package: name: fierro-heffte-cuda @@ -28,15 +28,15 @@ requirements: - {{ compiler('cxx') }}={{ macos_compiler_version }} # [osx] - {{ compiler('fortran') }}={{ linux_compiler_version }} - openmpi - - cuda-compiler - #- cuda-toolkit={{ cuda_version }} - - cuda-toolkit + # - cuda-compiler + - cuda-toolkit={{ cuda_version }} + #- cuda-toolkit # These host CUDA dependencies are definitely overspecified. # Not sure which ones are totally necessary. host: - #DAN- cuda-cudart-static={{ cuda_version }} # Make sure your nvcc version and cudart versions are the same + - cuda-cudart-static={{ cuda_version }} # Make sure your nvcc version and cudart versions are the same - cuda-compiler - - cuda-cudart-static + #- cuda-cudart-static - cuda-toolkit - libcusolver-dev - libcufft-dev From ffd01b70f755f663cbedcbe7da460589c890de44 Mon Sep 17 00:00:00 2001 From: Daniel Dunning <39738037+djdunning@users.noreply.github.com> Date: Tue, 16 Jul 2024 14:26:53 -0600 Subject: [PATCH 104/130] trying to fix heffte cuda on linux?? --- .conda/heffte/cuda/build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.conda/heffte/cuda/build.sh b/.conda/heffte/cuda/build.sh index 214794bc0..c39144d30 100755 --- a/.conda/heffte/cuda/build.sh +++ b/.conda/heffte/cuda/build.sh @@ -33,8 +33,8 @@ cmake -D CMAKE_BUILD_TYPE:STRING=RELEASE \ -D MPI_C_COMPILER="$BUILD_PREFIX/bin/mpicc" \ -D MPI_CXX_COMPILER="$BUILD_PREFIX/bin/mpicxx" \ -D CMAKE_CUDA_HOST_LINK_LAUNCHER=$CXX \ - -D CMAKE_CUDA_COMPILER=$(which nvcc) \ + -D CMAKE_CUDA_COMPILER='which nvcc' \ make -j 10 install -source "$RECIPE_DIR/../make-relocatable.sh" \ No newline at end of file +source "$RECIPE_DIR/../make-relocatable.sh" From 06825e83b03b07945a24b725d942027f1cae3f4a Mon Sep 17 00:00:00 2001 From: Daniel Dunning <39738037+djdunning@users.noreply.github.com> Date: Tue, 16 Jul 2024 14:39:25 -0600 Subject: [PATCH 105/130] trying to fix heffte cuda on linux?? --- .conda/heffte/cuda/build.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.conda/heffte/cuda/build.sh b/.conda/heffte/cuda/build.sh index c39144d30..d9cc3671e 100755 --- a/.conda/heffte/cuda/build.sh +++ b/.conda/heffte/cuda/build.sh @@ -7,6 +7,7 @@ export MPI_FLAGS="--allow-run-as-root" if [ $(uname) == Linux ]; then export MPI_FLAGS="$MPI_FLAGS;-mca;plm;isolated" + export CUDACXX='which nvcc' fi source "$RECIPE_DIR/../../cross-compile-setup.sh" @@ -33,7 +34,7 @@ cmake -D CMAKE_BUILD_TYPE:STRING=RELEASE \ -D MPI_C_COMPILER="$BUILD_PREFIX/bin/mpicc" \ -D MPI_CXX_COMPILER="$BUILD_PREFIX/bin/mpicxx" \ -D CMAKE_CUDA_HOST_LINK_LAUNCHER=$CXX \ - -D CMAKE_CUDA_COMPILER='which nvcc' \ +# -D CMAKE_CUDA_COMPILER='which nvcc' \ make -j 10 install From 2f26735016a92e11811847496741af3a84fcdea2 Mon Sep 17 00:00:00 2001 From: Daniel Dunning <39738037+djdunning@users.noreply.github.com> Date: Tue, 16 Jul 2024 14:44:37 -0600 Subject: [PATCH 106/130] trying to fix heffte cuda on linux?? --- .conda/heffte/cuda/build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.conda/heffte/cuda/build.sh b/.conda/heffte/cuda/build.sh index d9cc3671e..ed2ec4daf 100755 --- a/.conda/heffte/cuda/build.sh +++ b/.conda/heffte/cuda/build.sh @@ -7,12 +7,12 @@ export MPI_FLAGS="--allow-run-as-root" if [ $(uname) == Linux ]; then export MPI_FLAGS="$MPI_FLAGS;-mca;plm;isolated" - export CUDACXX='which nvcc' fi source "$RECIPE_DIR/../../cross-compile-setup.sh" export OMPI_CXX=nvcc +export CUDACXX='which nvcc' # Only do this for cross compiling if [ "$PLATFORM" != "linux-64" ] ; then export NVCC_PREPEND_FLAGS="-ccbin $CXX"; @@ -22,6 +22,7 @@ fi export LD_LIBRARY_PATH="$LD_LIBRARY_PATH $PREFIX/lib" export LIBRARIES="$LIBRARIES \"-L$PREFIX/lib\"" +# -D CMAKE_CUDA_COMPILER='which nvcc' \ cmake -D CMAKE_BUILD_TYPE:STRING=RELEASE \ -D BUILD_SHARED_LIBS=ON \ -D CMAKE_INSTALL_PREFIX:PATH=$PREFIX \ @@ -34,7 +35,6 @@ cmake -D CMAKE_BUILD_TYPE:STRING=RELEASE \ -D MPI_C_COMPILER="$BUILD_PREFIX/bin/mpicc" \ -D MPI_CXX_COMPILER="$BUILD_PREFIX/bin/mpicxx" \ -D CMAKE_CUDA_HOST_LINK_LAUNCHER=$CXX \ -# -D CMAKE_CUDA_COMPILER='which nvcc' \ make -j 10 install From 4841ce37dec053c2eb91472301ee4d45fb6bb61b Mon Sep 17 00:00:00 2001 From: Daniel Dunning <39738037+djdunning@users.noreply.github.com> Date: Tue, 16 Jul 2024 14:53:51 -0600 Subject: [PATCH 107/130] trying to fix heffte cuda on linux?? --- .conda/heffte/cuda/build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.conda/heffte/cuda/build.sh b/.conda/heffte/cuda/build.sh index ed2ec4daf..eb3205af8 100755 --- a/.conda/heffte/cuda/build.sh +++ b/.conda/heffte/cuda/build.sh @@ -12,7 +12,7 @@ fi source "$RECIPE_DIR/../../cross-compile-setup.sh" export OMPI_CXX=nvcc -export CUDACXX='which nvcc' +export CUDACXX=$(which nvcc) # Only do this for cross compiling if [ "$PLATFORM" != "linux-64" ] ; then export NVCC_PREPEND_FLAGS="-ccbin $CXX"; @@ -22,7 +22,6 @@ fi export LD_LIBRARY_PATH="$LD_LIBRARY_PATH $PREFIX/lib" export LIBRARIES="$LIBRARIES \"-L$PREFIX/lib\"" -# -D CMAKE_CUDA_COMPILER='which nvcc' \ cmake -D CMAKE_BUILD_TYPE:STRING=RELEASE \ -D BUILD_SHARED_LIBS=ON \ -D CMAKE_INSTALL_PREFIX:PATH=$PREFIX \ @@ -35,6 +34,7 @@ cmake -D CMAKE_BUILD_TYPE:STRING=RELEASE \ -D MPI_C_COMPILER="$BUILD_PREFIX/bin/mpicc" \ -D MPI_CXX_COMPILER="$BUILD_PREFIX/bin/mpicxx" \ -D CMAKE_CUDA_HOST_LINK_LAUNCHER=$CXX \ + -D CMAKE_CUDA_COMPILER=${CUDACXX} \ make -j 10 install From e156ed3eed5d9d2d3f58a4ffba995b795b59a2fa Mon Sep 17 00:00:00 2001 From: Daniel Dunning <39738037+djdunning@users.noreply.github.com> Date: Tue, 16 Jul 2024 15:03:07 -0600 Subject: [PATCH 108/130] trying to fix heffte cuda on linux?? --- .conda/heffte/cuda/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.conda/heffte/cuda/build.sh b/.conda/heffte/cuda/build.sh index eb3205af8..dda2d00a3 100755 --- a/.conda/heffte/cuda/build.sh +++ b/.conda/heffte/cuda/build.sh @@ -22,6 +22,7 @@ fi export LD_LIBRARY_PATH="$LD_LIBRARY_PATH $PREFIX/lib" export LIBRARIES="$LIBRARIES \"-L$PREFIX/lib\"" + #-D CMAKE_CUDA_COMPILER=${CUDACXX} \ cmake -D CMAKE_BUILD_TYPE:STRING=RELEASE \ -D BUILD_SHARED_LIBS=ON \ -D CMAKE_INSTALL_PREFIX:PATH=$PREFIX \ @@ -34,7 +35,6 @@ cmake -D CMAKE_BUILD_TYPE:STRING=RELEASE \ -D MPI_C_COMPILER="$BUILD_PREFIX/bin/mpicc" \ -D MPI_CXX_COMPILER="$BUILD_PREFIX/bin/mpicxx" \ -D CMAKE_CUDA_HOST_LINK_LAUNCHER=$CXX \ - -D CMAKE_CUDA_COMPILER=${CUDACXX} \ make -j 10 install From 62c40cecfdd836f17fc933dea9a3585abdaa2e8a Mon Sep 17 00:00:00 2001 From: Daniel Dunning <39738037+djdunning@users.noreply.github.com> Date: Tue, 16 Jul 2024 15:08:05 -0600 Subject: [PATCH 109/130] trying to fix heffte cuda on linux?? --- .conda/heffte/cuda/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.conda/heffte/cuda/build.sh b/.conda/heffte/cuda/build.sh index dda2d00a3..657e679d8 100755 --- a/.conda/heffte/cuda/build.sh +++ b/.conda/heffte/cuda/build.sh @@ -12,7 +12,7 @@ fi source "$RECIPE_DIR/../../cross-compile-setup.sh" export OMPI_CXX=nvcc -export CUDACXX=$(which nvcc) +#export CUDACXX=$(which nvcc) # Only do this for cross compiling if [ "$PLATFORM" != "linux-64" ] ; then export NVCC_PREPEND_FLAGS="-ccbin $CXX"; From 7b29c12f80e53b512d0740305dcfe0ff330fba4b Mon Sep 17 00:00:00 2001 From: Daniel Dunning <39738037+djdunning@users.noreply.github.com> Date: Tue, 16 Jul 2024 15:26:16 -0600 Subject: [PATCH 110/130] trying to fix heffte cuda on linux?? --- .conda/heffte/cuda/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.conda/heffte/cuda/build.sh b/.conda/heffte/cuda/build.sh index 657e679d8..de24a05a8 100755 --- a/.conda/heffte/cuda/build.sh +++ b/.conda/heffte/cuda/build.sh @@ -7,12 +7,12 @@ export MPI_FLAGS="--allow-run-as-root" if [ $(uname) == Linux ]; then export MPI_FLAGS="$MPI_FLAGS;-mca;plm;isolated" + export CUDACXX=$(which nvcc) fi source "$RECIPE_DIR/../../cross-compile-setup.sh" export OMPI_CXX=nvcc -#export CUDACXX=$(which nvcc) # Only do this for cross compiling if [ "$PLATFORM" != "linux-64" ] ; then export NVCC_PREPEND_FLAGS="-ccbin $CXX"; From 2e6774db54fd075d304736e166e49b734edd68ac Mon Sep 17 00:00:00 2001 From: Zachary Baker Date: Tue, 16 Jul 2024 15:36:52 -0600 Subject: [PATCH 111/130] Update build.sh --- .conda/fierro/cpu/build.sh | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/.conda/fierro/cpu/build.sh b/.conda/fierro/cpu/build.sh index 53a977b9c..4e6db1c0b 100755 --- a/.conda/fierro/cpu/build.sh +++ b/.conda/fierro/cpu/build.sh @@ -1,13 +1,6 @@ # These flag variables are set by anaconda. source "$RECIPE_DIR/../../cross-compile-setup.sh" - -export MPI_C_COMPILER=$BUILD_PREFIX/bin/mpicc -export MPI_CXX_COMPILER=$BUILD_PREFIX/bin/mpicxx -export MPI_C_INCLUDE_PATH=$BUILD_PREFIX/bin/mpi/include -export MPI_CXX_INCLUDE_PATH=$BUILD_PREFIX/bin/mpi/include - -# Create a build directory -mkdir -p build +mkdir build cd build # -D _LIBCPP_DISABLE_AVAILABILITY From 3afaaf1ba114ec6ad232326a2584d5b15769acfb Mon Sep 17 00:00:00 2001 From: Daniel Dunning <39738037+djdunning@users.noreply.github.com> Date: Wed, 17 Jul 2024 10:12:17 -0600 Subject: [PATCH 112/130] readding accidently deleted files --- lib/.gitignore | 3 +++ lib/Elements | 1 + scripts/build-fierro.sh | 4 ++-- scripts/cmake_build.sh | 28 ++++++++++++++-------------- 4 files changed, 20 insertions(+), 16 deletions(-) create mode 100644 lib/.gitignore create mode 160000 lib/Elements diff --git a/lib/.gitignore b/lib/.gitignore new file mode 100644 index 000000000..f4ee70f9a --- /dev/null +++ b/lib/.gitignore @@ -0,0 +1,3 @@ +- Trilinos +- hdf5 +- heffte diff --git a/lib/Elements b/lib/Elements new file mode 160000 index 000000000..0de655cd2 --- /dev/null +++ b/lib/Elements @@ -0,0 +1 @@ +Subproject commit 0de655cd260e60e4f037a78561dddfb9c8757fc7 diff --git a/scripts/build-fierro.sh b/scripts/build-fierro.sh index 65e30470d..42b60d177 100755 --- a/scripts/build-fierro.sh +++ b/scripts/build-fierro.sh @@ -183,8 +183,8 @@ source setup-env.sh ${machine} ${kokkos_build_type} ${build_cores} # Next, do action based on args if [ "$build_action" = "full-app" ]; then - #source uncrustify-install.sh - #source trilinos-install.sh ${kokkos_build_type} ${machine} + source uncrustify-install.sh + source trilinos-install.sh ${kokkos_build_type} ${machine} if [ "$solver" = "explicit-evpfft" ] || [ "${solver}" = "explicit-ls-evpfft" ]; then source hdf5-install.sh source heffte-install.sh ${heffte_build_type} ${machine} diff --git a/scripts/cmake_build.sh b/scripts/cmake_build.sh index 3af34724f..55c7f49c2 100644 --- a/scripts/cmake_build.sh +++ b/scripts/cmake_build.sh @@ -12,7 +12,7 @@ kokkos_build_type="${3}" if { [ ! -d "${ELEMENTS_SOURCE_DIR}/elements" ] || [ ! -d "${ELEMENTS_SOURCE_DIR}/matar/src" ] ;} then echo "Missing submodules, downloading them...." - #DANgit submodule update --recursive "${ELEMENTS_SOURCE_DIR}" + git submodule update --recursive "${ELEMENTS_SOURCE_DIR}" fi if [ ! -d "${TRILINOS_INSTALL_DIR}/lib" ]; then @@ -24,22 +24,22 @@ fi # Install Elements if [ ! -d "${ELEMENTS_INSTALL_DIR}/lib" ]; then echo "Installing Elements..." - #cmake -D CMAKE_INSTALL_PREFIX="$ELEMENTS_INSTALL_DIR" -D Trilinos_DIR="${Trilinos_DIR}" -D Matar_ENABLE_KOKKOS=ON -D Matar_KOKKOS_PACKAGE=Trilinos -B "${ELEMENTS_BUILD_DIR}" -S "${ELEMENTS_SOURCE_DIR}" - #make -C "${ELEMENTS_BUILD_DIR}" -j${FIERRO_BUILD_CORES} - #make -C "${ELEMENTS_BUILD_DIR}" install + cmake -D CMAKE_INSTALL_PREFIX="$ELEMENTS_INSTALL_DIR" -D Trilinos_DIR="${Trilinos_DIR}" -D Matar_ENABLE_KOKKOS=ON -D Matar_KOKKOS_PACKAGE=Trilinos -B "${ELEMENTS_BUILD_DIR}" -S "${ELEMENTS_SOURCE_DIR}" + make -C "${ELEMENTS_BUILD_DIR}" -j${FIERRO_BUILD_CORES} + make -C "${ELEMENTS_BUILD_DIR}" install fi # Removing stale build directory -#if [ -d "${FIERRO_BUILD_DIR}" ]; then - #if make -C ${FIERRO_BUILD_DIR} distclean; then - # echo ""; - #else - # echo "distclean failed. Removing build directory." - # rm -r ${FIERRO_BUILD_DIR} - #fi -#else - #mkdir -p ${FIERRO_BUILD_DIR} -#fi +if [ -d "${FIERRO_BUILD_DIR}" ]; then + if make -C ${FIERRO_BUILD_DIR} distclean; then + echo ""; + else + echo "distclean failed. Removing build directory." + rm -r ${FIERRO_BUILD_DIR} + fi +else + mkdir -p ${FIERRO_BUILD_DIR} +fi # Configure EVPFFT using CMake cmake_options=( From 7f2015451e57475361d00bb3ca09d9d1b5b5229e Mon Sep 17 00:00:00 2001 From: Daniel Dunning <39738037+djdunning@users.noreply.github.com> Date: Wed, 17 Jul 2024 10:14:04 -0600 Subject: [PATCH 113/130] formatting --- lib/.gitignore | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/.gitignore b/lib/.gitignore index f4ee70f9a..e4545aab8 100644 --- a/lib/.gitignore +++ b/lib/.gitignore @@ -1,3 +1,3 @@ -- Trilinos -- hdf5 -- heffte + - Trilinos + - hdf5 + - heffte From ddf22a6cfce67a9c2fe888100c81b15e10bac131 Mon Sep 17 00:00:00 2001 From: Daniel Dunning <39738037+djdunning@users.noreply.github.com> Date: Wed, 17 Jul 2024 10:29:56 -0600 Subject: [PATCH 114/130] continuing... --- .conda/heffte/cuda/build.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.conda/heffte/cuda/build.sh b/.conda/heffte/cuda/build.sh index de24a05a8..2f7288df1 100755 --- a/.conda/heffte/cuda/build.sh +++ b/.conda/heffte/cuda/build.sh @@ -7,7 +7,7 @@ export MPI_FLAGS="--allow-run-as-root" if [ $(uname) == Linux ]; then export MPI_FLAGS="$MPI_FLAGS;-mca;plm;isolated" - export CUDACXX=$(which nvcc) +# export CUDACXX=$(which nvcc) fi source "$RECIPE_DIR/../../cross-compile-setup.sh" @@ -32,9 +32,9 @@ cmake -D CMAKE_BUILD_TYPE:STRING=RELEASE \ $CMAKE_ARGS \ $SRC_DIR \ -D CMAKE_CXX_FLAGS="$PATCHED_CXXFLAGS" \ - -D MPI_C_COMPILER="$BUILD_PREFIX/bin/mpicc" \ - -D MPI_CXX_COMPILER="$BUILD_PREFIX/bin/mpicxx" \ - -D CMAKE_CUDA_HOST_LINK_LAUNCHER=$CXX \ +# -D MPI_C_COMPILER="$BUILD_PREFIX/bin/mpicc" \ +# -D MPI_CXX_COMPILER="$BUILD_PREFIX/bin/mpicxx" \ +# -D CMAKE_CUDA_HOST_LINK_LAUNCHER=$CXX \ make -j 10 install From 7d32dc3967aa3b4cf3dc9449613cad7222698d32 Mon Sep 17 00:00:00 2001 From: Daniel Dunning <39738037+djdunning@users.noreply.github.com> Date: Wed, 17 Jul 2024 10:36:50 -0600 Subject: [PATCH 115/130] continuing... --- .conda/heffte/cuda/build.sh | 4 ++-- .conda/heffte/cuda/meta.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.conda/heffte/cuda/build.sh b/.conda/heffte/cuda/build.sh index 2f7288df1..c3c233768 100755 --- a/.conda/heffte/cuda/build.sh +++ b/.conda/heffte/cuda/build.sh @@ -6,7 +6,7 @@ cd build export MPI_FLAGS="--allow-run-as-root" if [ $(uname) == Linux ]; then - export MPI_FLAGS="$MPI_FLAGS;-mca;plm;isolated" +# export MPI_FLAGS="$MPI_FLAGS;-mca;plm;isolated" # export CUDACXX=$(which nvcc) fi @@ -15,7 +15,7 @@ source "$RECIPE_DIR/../../cross-compile-setup.sh" export OMPI_CXX=nvcc # Only do this for cross compiling if [ "$PLATFORM" != "linux-64" ] ; then - export NVCC_PREPEND_FLAGS="-ccbin $CXX"; +# export NVCC_PREPEND_FLAGS="-ccbin $CXX"; fi # These things need to be set for certain cmake versions. # The cmake CUDA compiler tests need to see the host env libs diff --git a/.conda/heffte/cuda/meta.yaml b/.conda/heffte/cuda/meta.yaml index 123c74ce9..b327aff92 100644 --- a/.conda/heffte/cuda/meta.yaml +++ b/.conda/heffte/cuda/meta.yaml @@ -1,7 +1,7 @@ {% set version = "1.0.0" %} {% set linux_compiler_version = "12.4.0" %} {% set macos_compiler_version = "16" %} -{% set cuda_version = "12.4" %} +{% set cuda_version = "12.5" %} package: name: fierro-heffte-cuda From eebb692ea80f582aa7551634f2bd039271b449d0 Mon Sep 17 00:00:00 2001 From: Daniel Dunning <39738037+djdunning@users.noreply.github.com> Date: Wed, 17 Jul 2024 10:41:40 -0600 Subject: [PATCH 116/130] continuing... --- .conda/heffte/cuda/meta.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.conda/heffte/cuda/meta.yaml b/.conda/heffte/cuda/meta.yaml index b327aff92..cc7d1abad 100644 --- a/.conda/heffte/cuda/meta.yaml +++ b/.conda/heffte/cuda/meta.yaml @@ -1,7 +1,7 @@ {% set version = "1.0.0" %} {% set linux_compiler_version = "12.4.0" %} {% set macos_compiler_version = "16" %} -{% set cuda_version = "12.5" %} +#{% set cuda_version = "12.5" %} package: name: fierro-heffte-cuda @@ -29,14 +29,14 @@ requirements: - {{ compiler('fortran') }}={{ linux_compiler_version }} - openmpi # - cuda-compiler - - cuda-toolkit={{ cuda_version }} - #- cuda-toolkit + # - cuda-toolkit={{ cuda_version }} + - cuda-toolkit # These host CUDA dependencies are definitely overspecified. # Not sure which ones are totally necessary. host: - - cuda-cudart-static={{ cuda_version }} # Make sure your nvcc version and cudart versions are the same + #- cuda-cudart-static={{ cuda_version }} # Make sure your nvcc version and cudart versions are the same - cuda-compiler - #- cuda-cudart-static + - cuda-cudart-static - cuda-toolkit - libcusolver-dev - libcufft-dev From 0a0affefc14d6cbc32702c52d5892a7f4d8c0c9e Mon Sep 17 00:00:00 2001 From: Daniel Dunning <39738037+djdunning@users.noreply.github.com> Date: Wed, 17 Jul 2024 10:54:17 -0600 Subject: [PATCH 117/130] continuing... --- .conda/heffte/cuda/build.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.conda/heffte/cuda/build.sh b/.conda/heffte/cuda/build.sh index c3c233768..26a7b1d7c 100755 --- a/.conda/heffte/cuda/build.sh +++ b/.conda/heffte/cuda/build.sh @@ -6,7 +6,7 @@ cd build export MPI_FLAGS="--allow-run-as-root" if [ $(uname) == Linux ]; then -# export MPI_FLAGS="$MPI_FLAGS;-mca;plm;isolated" + export MPI_FLAGS="$MPI_FLAGS;-mca;plm;isolated" # export CUDACXX=$(which nvcc) fi @@ -14,9 +14,9 @@ source "$RECIPE_DIR/../../cross-compile-setup.sh" export OMPI_CXX=nvcc # Only do this for cross compiling -if [ "$PLATFORM" != "linux-64" ] ; then +#if [ "$PLATFORM" != "linux-64" ] ; then # export NVCC_PREPEND_FLAGS="-ccbin $CXX"; -fi +#fi # These things need to be set for certain cmake versions. # The cmake CUDA compiler tests need to see the host env libs export LD_LIBRARY_PATH="$LD_LIBRARY_PATH $PREFIX/lib" From ac479ae505444f932662e444bb0ab4544fd00970 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CAdrian-Diaz=E2=80=9D?= <“adriandiaz1117@gmail.com”> Date: Thu, 18 Jul 2024 13:37:21 -0600 Subject: [PATCH 118/130] ENH: implosion example work --- .../example_implosion.yaml | 84 +++++++++++++++++++ .../SGH_Solver/src/force_sgh.cpp | 4 +- 2 files changed, 86 insertions(+), 2 deletions(-) create mode 100644 src/Parallel-Solvers/Parallel-Explicit/Example_Yaml_Scripts/example_implosion.yaml diff --git a/src/Parallel-Solvers/Parallel-Explicit/Example_Yaml_Scripts/example_implosion.yaml b/src/Parallel-Solvers/Parallel-Explicit/Example_Yaml_Scripts/example_implosion.yaml new file mode 100644 index 000000000..ea43b3273 --- /dev/null +++ b/src/Parallel-Solvers/Parallel-Explicit/Example_Yaml_Scripts/example_implosion.yaml @@ -0,0 +1,84 @@ +num_dims: 3 +dynamic_options: + time_final: 20.0 + dt_min: 1.e-8 + dt_max: 1.e-2 + dt_start: 1.e-5 + cycle_stop: 2000000 + +mesh_generation_options: + type: Box + origin: [0, 0, 0] + length: [1.2, 1.2, 1.2] + num_elems: [32, 32, 32] + +output_options: + timer_output_level: thorough + output_file_format: vtk + graphics_step: 0.25 + +fea_module_parameters: + - type: SGH + material_id: 0 + boundary_conditions: + # Tag X plane + - surface: + type: x_plane + plane_position: 0.0 + type: reflected + + # Tag Y plane + - surface: + type: y_plane + plane_position: 0.0 + type: reflected + + # Tag Z plane + - surface: + type: z_plane + plane_position: 0.0 + type: reflected + + loading_conditions: + # Load radially around 0 0 0 corner + - volume: + type: sphere + radius1: 1.0 + radius2: 2.5 + type: body_force + component_x: -0.00000005 + component_y: -0.00000005 + component_z: -0.00000005 + +materials: + - id: 0 + eos_model: ideal_gas + strength_model: none + elastic_modulus: 10 + poisson_ratio: 0.3 + maximum_limiter: 1 + q1: 2.0 + q2: 4.0 + q1ex: 2.0 + q2ex: 4.0 + eos_global_vars: + - 1.666666666666667 + - 1.0E-14 + - 1.0 + +regions: + - volume: + type: global + material_id: 0 + den: 1.0 + sie: 1.e-10 + + velocity: cartesian + u: 0.0 + v: 0.0 + w: 0.0 + + velocity: cartesian + u: 0.0 + v: 0.0 + w: 0.0 diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_sgh.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_sgh.cpp index c3dee5cf9..1e70547f8 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_sgh.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/force_sgh.cpp @@ -1,5 +1,5 @@ /********************************************************************************************** - 2020. Triad National Security, LLC. All rights reserved. + � 2020. Triad National Security, LLC. All rights reserved. This program was produced under U.S. Government contract 89233218CNA000001 for Los Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC for the U.S. Department of Energy/National Nuclear Security Administration. All rights in the program are @@ -933,7 +933,7 @@ void FEA_Module_SGH::applied_forces(const DCArrayKokkos& material, applied_force[2] = loading(ilc).z; // loop over dimension for (size_t dim = 0; dim < num_dim; dim++) { - node_force[dim] += applied_force[dim] * (all_initial_node_coords(node_gid, 0) + all_initial_node_coords(node_gid, 1) + all_initial_node_coords(node_gid, 2)) / radius; + node_force[dim] += applied_force[dim] * current_node_coords[dim] / radius; } // end for dim } // end for corner_lid From dd59ac0331e7ecf4588c78737f2d5158e3a84dfc Mon Sep 17 00:00:00 2001 From: Adrian-Diaz <46537160+Adrian-Diaz@users.noreply.github.com> Date: Thu, 18 Jul 2024 15:22:22 -0600 Subject: [PATCH 119/130] ENH: intel mkl build support --- scripts/build-fierro.sh | 18 ++++++++++++++++-- scripts/trilinos-install.sh | 25 ++++++++++++++++++++++++- 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/scripts/build-fierro.sh b/scripts/build-fierro.sh index 42b60d177..83a555bf8 100755 --- a/scripts/build-fierro.sh +++ b/scripts/build-fierro.sh @@ -7,6 +7,7 @@ show_help() { echo " --build_action=. Default is 'full-app'" echo " --machine=. Default is 'linux'" echo " --build_cores=. Default is set 1" + echo " --intel_mkl=. Default is 'disabled'" echo " --heffte_build_type=. Default is set 'fftw'" echo " --help: Display this help message" echo " " @@ -63,6 +64,7 @@ machine="linux" kokkos_build_type="serial" heffte_build_type="fftw" build_cores="1" +intel_mkl="disabled" # Define arrays of valid options valid_build_action=("full-app" "set-env" "install-trilinos" "install-hdf5" "install-heffte" "install-uncrustify" "fierro") @@ -70,6 +72,7 @@ valid_solver=("all" "explicit" "explicit-evpfft" "explicit-ls-evpfft" "implicit" valid_kokkos_build_types=("serial" "openmp" "pthreads" "cuda" "hip") valid_heffte_build_types=("fftw" "cufft" "rocfft") valid_machines=("darwin" "chicoma" "linux" "mac" "msu") +valid_intel_mkl=("disabled" "enabled") # Parse command line arguments for arg in "$@"; do @@ -114,6 +117,16 @@ for arg in "$@"; do return 1 fi ;; + --intel_mkl=*) + option="${arg#*=}" + if [[ " ${valid_intel_mkl[*]} " == *" $option "* ]]; then + intel_mkl="$option" + else + echo "Error: Invalid --intel_mkl specified." + show_help + return 1 + fi + ;; --heffte_build_type=*) option="${arg#*=}" if [[ " ${valid_heffte_build_types[*]} " == *" $option "* ]]; then @@ -170,6 +183,7 @@ echo "Building based on these argument options:" echo "Build action - ${build_action}" echo "Solver - ${solver}" echo "Kokkos backend - ${kokkos_build_type}" +echo "Intel MKL library - ${intel_mkl}" echo "Machine - ${machine}" if [ "${solver}" = "explicit-evpfft" ] || [ "${solver}" = "explicit-ls-evpfft" ]; then echo "HEFFTE - ${heffte_build_type}" @@ -184,14 +198,14 @@ source setup-env.sh ${machine} ${kokkos_build_type} ${build_cores} # Next, do action based on args if [ "$build_action" = "full-app" ]; then source uncrustify-install.sh - source trilinos-install.sh ${kokkos_build_type} ${machine} + source trilinos-install.sh ${kokkos_build_type} ${machine} ${intel_mkl} if [ "$solver" = "explicit-evpfft" ] || [ "${solver}" = "explicit-ls-evpfft" ]; then source hdf5-install.sh source heffte-install.sh ${heffte_build_type} ${machine} fi source cmake_build.sh ${solver} ${heffte_build_type} ${kokkos_build_type} elif [ "$build_action" = "install-trilinos" ]; then - source trilinos-install.sh ${kokkos_build_type} ${machine} + source trilinos-install.sh ${kokkos_build_type} ${machine} ${intel_mkl} elif [ "$build_action" = "install-hdf5" ]; then source hdf5-install.sh elif [ "$build_action" = "install-heffte" ]; then diff --git a/scripts/trilinos-install.sh b/scripts/trilinos-install.sh index 9c7aad598..7bc7e6768 100644 --- a/scripts/trilinos-install.sh +++ b/scripts/trilinos-install.sh @@ -2,6 +2,7 @@ kokkos_build_type="${1}" machine="${2}" +intel_mkl="${3}" # If all arguments are valid, you can use them in your script as needed echo "Trilinos Kokkos Build Type: $kokkos_build_type" @@ -28,7 +29,9 @@ fi #check if Trilinos library files were installed, install them otherwise. [ -d "${TRILINOS_BUILD_DIR}/lib" ] && echo "Directory ${TRILINOS_BUILD_DIR}/lib exists, assuming successful installation; delete build folder and run build script again if there was an environment error that has been corrected." -if [ ! -d "${TRILINOS_BUILD_DIR}/lib" ] +[ -d "${TRILINOS_BUILD_DIR}/lib64" ] && echo "Directory ${TRILINOS_BUILD_DIR}/lib64 exists, assuming successful installation; delete build folder and run build script again if there was an environment error that has been corrected." + +if [ ! -d "${TRILINOS_BUILD_DIR}/lib" ] && [ ! -d "${TRILINOS_BUILD_DIR}/lib64" ] then echo "Directory Trilinos/build/lib does not exist, compiling Trilinos (this might take a while)...." @@ -113,6 +116,26 @@ ${ADDITIONS[@]} -D CMAKE_INSTALL_PREFIX=${TRILINOS_INSTALL_DIR} ) +# Flags for building with Intel MKL library +INTEL_MKL_ADDITIONS=( +-D TPL_ENABLE_MKL=ON +-D BLAS_LIBRARY_NAMES="libmkl_rt.so" +-D BLAS_LIBRARY_DIRS="$MKLROOT/lib/intel64" +-D LAPACK_LIBRARY_NAMES="libmkl_rt.so" +-D LAPACK_LIBRARY_DIRS="$MKLROOT/lib/intel64" +-D MKL_LIBRARY_DIRS="$MKLROOT/lib/intel64" +-D MKL_LIBRARY_NAMES="mkl_rt" +-D MKL_INCLUDE_DIRS="$MKLROOT/include" +) + +echo "**** Intel MKL = ${intel_mkl} ****" +if [ "$intel_mkl" = "enabled" ]; then + echo "**** assuming MKL installation at $MKLROOT ****" + cmake_options+=( + ${INTEL_MKL_ADDITIONS[@]} + ) +fi + if [ "$kokkos_build_type" = "openmp" ]; then cmake_options+=( ${OPENMP_ADDITIONS[@]} From 472ac0e23576a6d2710c858a5453512216a34ce2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CAdrian-Diaz=E2=80=9D?= <“adriandiaz1117@gmail.com”> Date: Sun, 21 Jul 2024 01:21:23 -0600 Subject: [PATCH 120/130] ENH: graphics dt reset and reduce debug output --- src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp | 6 +++--- .../Parallel-Explicit/SGH_Solver/src/FEA_Module_SGH.cpp | 7 +++++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp b/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp index 91c20bb05..99681e149 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp @@ -359,11 +359,11 @@ void Explicit_Solver::run() { } */ - std::cout << " RUNTIME OF CODE ON TASK " << myrank << " is "<< current_cpu-initial_CPU_time << " comms time " + *fos << " Runtime of code is "<< current_cpu-initial_CPU_time << " comms time " << communication_time << " host to dev time " << host2dev_time << " dev to host time " << dev2host_time << std::endl; if(simparam.output_options.timer_output_level == TIMER_VERBOSITY::thorough){ - std::cout << " OUTPUT TIME OF CODE ON TASK " << myrank << " is "<< output_time << std::endl; + *fos << " Output time of code is "<< output_time << std::endl; } //parallel_vtk_writer(); @@ -3384,7 +3384,7 @@ void Explicit_Solver::init_design(){ //create import object using local node indices map and all indices map Tpetra::Import importer(map, all_node_map); - //design_node_densities_distributed->randomize(0.1,1); + design_node_densities_distributed->randomize(0.1,1); //comms to get ghosts all_node_densities_distributed->doImport(*design_node_densities_distributed, importer, Tpetra::INSERT); diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/FEA_Module_SGH.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/FEA_Module_SGH.cpp index 0d4ee757e..c0492832a 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/FEA_Module_SGH.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/FEA_Module_SGH.cpp @@ -921,6 +921,8 @@ void FEA_Module_SGH::sgh_solve() tiny = dynamic_options.tiny; small = dynamic_options.small; + double cached_pregraphics_dt = fuzz; + size_t num_bdy_nodes = mesh->num_bdy_nodes; size_t cycle; real_t objective_accumulation, global_objective_accumulation; @@ -1170,6 +1172,10 @@ void FEA_Module_SGH::sgh_solve() // loop over the max number of time integration cycles for (cycle = 0; cycle < cycle_stop; cycle++) { + + //save timestep from before graphics output contraction + cached_pregraphics_dt = dt; + // get the step if (num_dim == 2) { get_timestep2D(*mesh, @@ -1825,6 +1831,7 @@ void FEA_Module_SGH::sgh_solve() Explicit_Solver_Pointer_->output_time += comm_time2 - comm_time1; graphics_time = time_value + graphics_dt_ival; + dt = cached_pregraphics_dt; } // end if // end of calculation From a9932ae4dbff3c9fcc6d5393c08633d7722fb17d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CAdrian-Diaz=E2=80=9D?= <“adriandiaz1117@gmail.com”> Date: Sun, 21 Jul 2024 16:11:50 -0600 Subject: [PATCH 121/130] BUG: remove debug statement --- src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp | 2 +- .../Parallel-Explicit/SGH_Solver/src/FEA_Module_SGH.cpp | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp b/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp index 99681e149..8074bd3ad 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp @@ -3384,7 +3384,7 @@ void Explicit_Solver::init_design(){ //create import object using local node indices map and all indices map Tpetra::Import importer(map, all_node_map); - design_node_densities_distributed->randomize(0.1,1); + //design_node_densities_distributed->randomize(0.1,1); //comms to get ghosts all_node_densities_distributed->doImport(*design_node_densities_distributed, importer, Tpetra::INSERT); diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/FEA_Module_SGH.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/FEA_Module_SGH.cpp index c0492832a..ee0c361a2 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/FEA_Module_SGH.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/FEA_Module_SGH.cpp @@ -600,10 +600,6 @@ void FEA_Module_SGH::comm_variables(Teuchos::RCP zp) node_densities_distributed->describe(*fos, Teuchos::VERB_EXTREME); *fos << std::endl; std::fflush(stdout); - - // communicate design densities - // create import object using local node indices map and all indices map - Tpetra::Import importer(map, all_node_map); #endif // comms to get ghosts all_node_densities_distributed->doImport(*test_node_densities_distributed, *importer, Tpetra::INSERT); From 640d2905db4e8357b41b79a27dce3dcdf514f4c8 Mon Sep 17 00:00:00 2001 From: Adrian-Diaz <46537160+Adrian-Diaz@users.noreply.github.com> Date: Wed, 24 Jul 2024 14:00:11 -0600 Subject: [PATCH 122/130] STYLE: update yaml example files and error handling --- .../example_sedov_sgh_opt.yaml | 19 +++++++++++-------- .../example_sgh_opt_region.yaml | 3 +++ .../example_sgh_opt_restart.yaml | 4 ++-- .../Optimization_Options.h | 4 ++++ 4 files changed, 20 insertions(+), 10 deletions(-) diff --git a/src/Parallel-Solvers/Parallel-Explicit/Example_Yaml_Scripts/example_sedov_sgh_opt.yaml b/src/Parallel-Solvers/Parallel-Explicit/Example_Yaml_Scripts/example_sedov_sgh_opt.yaml index 000c0b724..6ffd94127 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Example_Yaml_Scripts/example_sedov_sgh_opt.yaml +++ b/src/Parallel-Solvers/Parallel-Explicit/Example_Yaml_Scripts/example_sedov_sgh_opt.yaml @@ -105,14 +105,17 @@ regions: optimization_options: optimization_process: topology_optimization optimization_objective: minimize_kinetic_energy - objective_regions: - - type: box - x1: 0.4 - x2: 0.8 - y1: 0.4 - y2: 0.8 - z1: 0.4 - z2: 0.8 + use_solve_checkpoints: true + num_solve_checkpoints : 10 + disable_forward_solve_output: true + optimization_output_freq: 1 + rol_params: + subproblem_algorithm: line_search + initial_constraint_penalty: 1.e-2 + step_tolerance: 1.e-3 + gradient_tolerance: 1.e-3 + constraint_tolerance: 1.e-3 + iteration_limit: 20 density_epsilon: 0.1 variable_outer_shell: true constraints: diff --git a/src/Parallel-Solvers/Parallel-Explicit/Example_Yaml_Scripts/example_sgh_opt_region.yaml b/src/Parallel-Solvers/Parallel-Explicit/Example_Yaml_Scripts/example_sgh_opt_region.yaml index c6c56ce20..1f457dfdb 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Example_Yaml_Scripts/example_sgh_opt_region.yaml +++ b/src/Parallel-Solvers/Parallel-Explicit/Example_Yaml_Scripts/example_sgh_opt_region.yaml @@ -106,6 +106,9 @@ optimization_options: optimization_process: topology_optimization optimization_objective: minimize_kinetic_energy use_solve_checkpoints: true + num_solve_checkpoints : 10 + disable_forward_solve_output: true + optimization_output_freq: 1 rol_params: subproblem_algorithm: line_search initial_constraint_penalty: 1.e-2 diff --git a/src/Parallel-Solvers/Parallel-Explicit/Example_Yaml_Scripts/example_sgh_opt_restart.yaml b/src/Parallel-Solvers/Parallel-Explicit/Example_Yaml_Scripts/example_sgh_opt_restart.yaml index 7d139f46d..9f5b45796 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Example_Yaml_Scripts/example_sgh_opt_restart.yaml +++ b/src/Parallel-Solvers/Parallel-Explicit/Example_Yaml_Scripts/example_sgh_opt_restart.yaml @@ -108,10 +108,10 @@ regions: optimization_options: optimization_process: topology_optimization optimization_objective: minimize_kinetic_energy - use_gradient_tally: true use_solve_checkpoints: true + num_solve_checkpoints : 10 disable_forward_solve_output: true - optimization_output_freq: 20 + optimization_output_freq: 1 optimization_parameters_xml_file: false xml_parameters_file_name: "optimization_parameters.xml" rol_params: diff --git a/src/Parallel-Solvers/Simulation_Parameters/Optimization_Options.h b/src/Parallel-Solvers/Simulation_Parameters/Optimization_Options.h index 3d65fbfb5..fa4b259ea 100644 --- a/src/Parallel-Solvers/Simulation_Parameters/Optimization_Options.h +++ b/src/Parallel-Solvers/Simulation_Parameters/Optimization_Options.h @@ -175,6 +175,10 @@ struct Optimization_Options: Yaml::DerivedFields { if(use_solve_checkpoints){ use_gradient_tally = true; } + + if(retain_outer_shell&&variable_outer_shell){ + throw Yaml::ConfigurationException("Cannot specify both retain_outer_shell and variable_outer_shell as true"); + } } }; YAML_ADD_REQUIRED_FIELDS_FOR(Optimization_Options, From 34abb4a3eaffba373d73d2ab4ecf7475c268abf3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CAdrian-Diaz=E2=80=9D?= <“adriandiaz1117@gmail.com”> Date: Sat, 27 Jul 2024 22:50:01 -0600 Subject: [PATCH 123/130] ENH: more body force support --- .../SGH_Solver/include/FEA_Module_SGH.h | 8 ++ .../SGH_Solver/src/FEA_Module_SGH.cpp | 39 ++++--- .../SGH_Solver/src/energy_sgh.cpp | 101 +++++++++++++++++- .../SGH_Solver/src/force_sgh.cpp | 67 ++++++------ .../SGH_Solver/src/sgh_optimization.cpp | 46 ++++++++ .../Simulation_Parameters/Geometry.h | 20 ++-- 6 files changed, 221 insertions(+), 60 deletions(-) diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/FEA_Module_SGH.h b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/FEA_Module_SGH.h index fb294c7f6..941b9927f 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/FEA_Module_SGH.h +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/FEA_Module_SGH.h @@ -348,6 +348,14 @@ class FEA_Module_SGH : public FEA_Module const DViewCArrayKokkos& elem_mass, const DViewCArrayKokkos& corner_force); + void update_external_energy_sgh(double rk_alpha, + const mesh_t& mesh, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& corner_force); + void power_design_gradient_term(const_vec_array design_variables, vec_array design_gradients); void get_power_dgradient_sgh(double rk_alpha, diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/FEA_Module_SGH.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/FEA_Module_SGH.cpp index ee0c361a2..402f56fd4 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/FEA_Module_SGH.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/FEA_Module_SGH.cpp @@ -1280,6 +1280,21 @@ void FEA_Module_SGH::sgh_solve() cycle); } + if (have_loading_conditions) { + applied_forces(material, + *mesh, + node_coords, + node_vel, + node_mass, + elem_den, + elem_vol, + elem_div, + elem_mat_id, + corner_force, + rk_alpha, + cycle); + } + #ifdef DEBUG if (myrank == 1) { std::cout << "rk_alpha = " << rk_alpha << ", dt = " << dt << std::endl; @@ -1319,20 +1334,6 @@ void FEA_Module_SGH::sgh_solve() node_mass, corner_force); - if (have_loading_conditions) { - applied_forces(material, - *mesh, - node_coords, - node_vel, - node_mass, - elem_den, - elem_vol, - elem_div, - elem_mat_id, - corner_force, - rk_alpha, - cycle); - } // ---- apply force boundary conditions to the boundary patches---- boundary_velocity(*mesh, boundary, node_vel); @@ -1395,6 +1396,16 @@ void FEA_Module_SGH::sgh_solve() elem_mass, corner_force); + if (have_loading_conditions) { + update_external_energy_sgh(rk_alpha, + *mesh, + node_vel, + node_coords, + elem_sie, + elem_mass, + corner_force); + } + // ---- Update nodal positions ---- update_position_sgh(rk_alpha, nall_nodes, diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/energy_sgh.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/energy_sgh.cpp index 936aa4c8e..3adb2accc 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/energy_sgh.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/energy_sgh.cpp @@ -1,5 +1,5 @@ /********************************************************************************************** - 2020. Triad National Security, LLC. All rights reserved. + � 2020. Triad National Security, LLC. All rights reserved. This program was produced under U.S. Government contract 89233218CNA000001 for Los Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC for the U.S. Department of Energy/National Nuclear Security Administration. All rights in the program are @@ -34,6 +34,8 @@ #include "mesh.h" #include "state.h" #include "FEA_Module_SGH.h" +#include "Simulation_Parameters/Simulation_Parameters_Explicit.h" +#include "Simulation_Parameters/FEA_Module/SGH_Parameters.h" ///////////////////////////////////////////////////////////////////////////// /// @@ -96,3 +98,100 @@ void FEA_Module_SGH::update_energy_sgh(double rk_alpha, return; } // end subroutine + +///////////////////////////////////////////////////////////////////////////// +/// +/// \fn update_external_energy_sgh +/// +/// \brief Evolves the specific internal energy +/// +/// \param The current Runge Kutta alpha value +/// \param The simulation mesh +/// \param A view into the nodal velocity data +/// \param A view into the nodal position data +/// \param A view into the element specific internal energy data +/// \param A view into the corner force data +/// +///////////////////////////////////////////////////////////////////////////// +void FEA_Module_SGH::update_external_energy_sgh(double rk_alpha, + const mesh_t& mesh, + const DViewCArrayKokkos& node_vel, + const DViewCArrayKokkos& node_coords, + DViewCArrayKokkos& elem_sie, + const DViewCArrayKokkos& elem_mass, + const DViewCArrayKokkos& corner_force) +{ + + const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; + const size_t num_dim = mesh.num_dims; + const_vec_array all_initial_node_coords = all_initial_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); + const size_t num_lcs = module_params->loading.size(); + + const DCArrayKokkos mat_fill = simparam->mat_fill; + const DCArrayKokkos loading = module_params->loading; + + // debug check + // std::cout << "NUMBER OF LOADING CONDITIONS: " << num_lcs << std::endl; + + for (size_t ilc = 0; ilc < num_lcs; ilc++) { + + const Volume current_volume = loading(ilc).volume; + const double applied_force[] = {loading(ilc).x, loading(ilc).y, loading(ilc).z}; + + FOR_ALL_CLASS(elem_id, 0, rnum_elem, { + double current_node_coords[3]; + size_t dof_id; + double node_force[3]; + double radius; + size_t node_id; + size_t corner_id; + size_t corner_lid; + double node_radius; + double elem_power = 0.0; + double half_vel; + // std::cout << elem_mass(elem_id) <& material, // debug check // std::cout << "NUMBER OF LOADING CONDITIONS: " << num_lcs << std::endl; - // walk over the nodes to update the velocity - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - double current_node_coords[3]; - size_t dof_id; - double node_force[3]; - double applied_force[3]; - double radius; - for (size_t dim = 0; dim < num_dim; dim++) { - node_force[dim] = 0.0; - current_node_coords[dim] = all_initial_node_coords(node_gid, dim); - } // end for dim - radius = sqrt(current_node_coords[0] * current_node_coords[0] + current_node_coords[1] * current_node_coords[1] + current_node_coords[2] * current_node_coords[2]); - for (size_t ilc = 0; ilc < num_lcs; ilc++) { - // debug check - // std::cout << "LOADING CONDITION VOLUME TYPE: " << to_string(loading(ilc).volume) << std::endl; - - bool fill_this = loading(ilc).volume.contains(current_node_coords); - if (fill_this) { - // loop over all corners around the node and calculate the nodal force - for (size_t corner_lid = 0; corner_lid < num_corners_in_node(node_gid); corner_lid++) { - // Get corner gid - size_t corner_gid = corners_in_node(node_gid, corner_lid); - applied_force[0] = loading(ilc).x; - applied_force[1] = loading(ilc).y; - applied_force[2] = loading(ilc).z; + for (size_t ilc = 0; ilc < num_lcs; ilc++) { + + const Volume current_volume = loading(ilc).volume; + const double applied_force[] = {loading(ilc).x, loading(ilc).y, loading(ilc).z}; + + FOR_ALL_CLASS(elem_id, 0, rnum_elem, { + double current_node_coords[3]; + size_t dof_id; + double node_force[3]; + double radius; + size_t node_id; + size_t corner_id; + // std::cout << elem_mass(elem_id) <num_dims; bool use_solve_checkpoints = simparam->optimization_options.use_solve_checkpoints; bool use_gradient_tally = simparam->optimization_options.use_gradient_tally; + const size_t num_lcs = module_params->loading_conditions.size(); real_t global_dt, current_time; size_t current_data_index, next_data_index; int print_cycle = simparam->dynamic_options.print_cycle; @@ -771,6 +772,21 @@ void FEA_Module_SGH::compute_topology_optimization_adjoint_full(Teuchos::RCPCPU_Time(); get_force_dgradient_sgh(material, diff --git a/src/Parallel-Solvers/Simulation_Parameters/Geometry.h b/src/Parallel-Solvers/Simulation_Parameters/Geometry.h index dd36d171e..aa0e593ae 100644 --- a/src/Parallel-Solvers/Simulation_Parameters/Geometry.h +++ b/src/Parallel-Solvers/Simulation_Parameters/Geometry.h @@ -47,14 +47,6 @@ struct Volume : Yaml::ValidatedYaml { double length_x = 0; double length_y = 0; double length_z = 0; - double i0_real; - double j0_real; - double k0_real; - int i0; - int j0; - int k0; - int elem_id0; - bool fill_this; // Run voxelization scheme on stl file KOKKOS_FUNCTION @@ -68,9 +60,17 @@ struct Volume : Yaml::ValidatedYaml { } KOKKOS_FUNCTION - bool contains(const double* elem_coords) { + bool contains(const double* elem_coords) const { double radius; - + bool fill_this; + double i0_real; + double j0_real; + double k0_real; + int i0; + int j0; + int k0; + int elem_id0; + switch(type) { case VOLUME_TYPE::global: return true; From 253a0e577f4b816637f81ec264b250dde2fc2283 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CAdrian-Diaz=E2=80=9D?= <“adriandiaz1117@gmail.com”> Date: Mon, 29 Jul 2024 09:36:54 -0600 Subject: [PATCH 124/130] ENH: refactor external force contribution + change xml default --- .../SGH_Solver/include/FEA_Module_SGH.h | 9 +- .../SGH_Solver/src/FEA_Module_SGH.cpp | 10 -- .../SGH_Solver/src/energy_sgh.cpp | 97 ------------------- .../SGH_Solver/src/force_sgh.cpp | 11 ++- .../SGH_Solver/src/momentum.cpp | 70 +++++++++---- .../SGH_Solver/src/setup_sgh.cpp | 5 + .../optimization_parameters.xml | 2 +- 7 files changed, 66 insertions(+), 138 deletions(-) diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/FEA_Module_SGH.h b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/FEA_Module_SGH.h index 941b9927f..4d0f4ffac 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/FEA_Module_SGH.h +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/include/FEA_Module_SGH.h @@ -348,14 +348,6 @@ class FEA_Module_SGH : public FEA_Module const DViewCArrayKokkos& elem_mass, const DViewCArrayKokkos& corner_force); - void update_external_energy_sgh(double rk_alpha, - const mesh_t& mesh, - const DViewCArrayKokkos& node_vel, - const DViewCArrayKokkos& node_coords, - DViewCArrayKokkos& elem_sie, - const DViewCArrayKokkos& elem_mass, - const DViewCArrayKokkos& corner_force); - void power_design_gradient_term(const_vec_array design_variables, vec_array design_gradients); void get_power_dgradient_sgh(double rk_alpha, @@ -686,6 +678,7 @@ class FEA_Module_SGH : public FEA_Module // Dual Views of the corner struct variables DViewCArrayKokkos corner_force; + DCArrayKokkos corner_external_force; DViewCArrayKokkos corner_mass; // Boundary Conditions Data diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/FEA_Module_SGH.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/FEA_Module_SGH.cpp index 402f56fd4..2036ce050 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/FEA_Module_SGH.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/FEA_Module_SGH.cpp @@ -1396,16 +1396,6 @@ void FEA_Module_SGH::sgh_solve() elem_mass, corner_force); - if (have_loading_conditions) { - update_external_energy_sgh(rk_alpha, - *mesh, - node_vel, - node_coords, - elem_sie, - elem_mass, - corner_force); - } - // ---- Update nodal positions ---- update_position_sgh(rk_alpha, nall_nodes, diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/energy_sgh.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/energy_sgh.cpp index 3adb2accc..f5b925746 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/energy_sgh.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/energy_sgh.cpp @@ -98,100 +98,3 @@ void FEA_Module_SGH::update_energy_sgh(double rk_alpha, return; } // end subroutine - -///////////////////////////////////////////////////////////////////////////// -/// -/// \fn update_external_energy_sgh -/// -/// \brief Evolves the specific internal energy -/// -/// \param The current Runge Kutta alpha value -/// \param The simulation mesh -/// \param A view into the nodal velocity data -/// \param A view into the nodal position data -/// \param A view into the element specific internal energy data -/// \param A view into the corner force data -/// -///////////////////////////////////////////////////////////////////////////// -void FEA_Module_SGH::update_external_energy_sgh(double rk_alpha, - const mesh_t& mesh, - const DViewCArrayKokkos& node_vel, - const DViewCArrayKokkos& node_coords, - DViewCArrayKokkos& elem_sie, - const DViewCArrayKokkos& elem_mass, - const DViewCArrayKokkos& corner_force) -{ - - const size_t rk_level = simparam->dynamic_options.rk_num_bins - 1; - const size_t num_dim = mesh.num_dims; - const_vec_array all_initial_node_coords = all_initial_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); - const size_t num_lcs = module_params->loading.size(); - - const DCArrayKokkos mat_fill = simparam->mat_fill; - const DCArrayKokkos loading = module_params->loading; - - // debug check - // std::cout << "NUMBER OF LOADING CONDITIONS: " << num_lcs << std::endl; - - for (size_t ilc = 0; ilc < num_lcs; ilc++) { - - const Volume current_volume = loading(ilc).volume; - const double applied_force[] = {loading(ilc).x, loading(ilc).y, loading(ilc).z}; - - FOR_ALL_CLASS(elem_id, 0, rnum_elem, { - double current_node_coords[3]; - size_t dof_id; - double node_force[3]; - double radius; - size_t node_id; - size_t corner_id; - size_t corner_lid; - double node_radius; - double elem_power = 0.0; - double half_vel; - // std::cout << elem_mass(elem_id) <& material, const size_t num_dim = mesh.num_dims; const_vec_array all_initial_node_coords = all_initial_node_coords_distributed->getLocalView(Tpetra::Access::ReadOnly); const size_t num_lcs = module_params->loading.size(); + const size_t num_corners = mesh.num_corners; const DCArrayKokkos mat_fill = simparam->mat_fill; const DCArrayKokkos loading = module_params->loading; @@ -906,6 +907,14 @@ void FEA_Module_SGH::applied_forces(const DCArrayKokkos& material, // debug check // std::cout << "NUMBER OF LOADING CONDITIONS: " << num_lcs << std::endl; + //initialize + FOR_ALL_CLASS(corner_id, 0, num_corners, { + for (size_t dim = 0; dim < num_dim; dim++) { + corner_external_force(corner_id,dim) = 0; + } + }); // end parallel for + Kokkos::fence(); + for (size_t ilc = 0; ilc < num_lcs; ilc++) { const Volume current_volume = loading(ilc).volume; @@ -934,7 +943,7 @@ void FEA_Module_SGH::applied_forces(const DCArrayKokkos& material, if (fill_this) { // loop over dimension for (size_t dim = 0; dim < num_dim; dim++) { - corner_force(corner_id,dim) += applied_force[dim] * current_node_coords[dim] / radius / num_nodes_in_elem; + corner_external_force(corner_id,dim) += applied_force[dim] * current_node_coords[dim] / radius / num_nodes_in_elem; } // end for dim } } diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/momentum.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/momentum.cpp index 9c03ecf85..93b1a1d43 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/momentum.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/momentum.cpp @@ -1,5 +1,5 @@ /********************************************************************************************** - 2020. Triad National Security, LLC. All rights reserved. + � 2020. Triad National Security, LLC. All rights reserved. This program was produced under U.S. Government contract 89233218CNA000001 for Los Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC for the U.S. Department of Energy/National Nuclear Security Administration. All rights in the program are @@ -33,6 +33,7 @@ **********************************************************************************************/ #include "state.h" #include "FEA_Module_SGH.h" +#include "Simulation_Parameters/FEA_Module/SGH_Parameters.h" ///////////////////////////////////////////////////////////////////////////// /// @@ -54,32 +55,59 @@ void FEA_Module_SGH::update_velocity_sgh(double rk_alpha, { const size_t rk_level = rk_num_bins - 1; const size_t num_dims = num_dim; + const size_t num_lcs = module_params->loading.size(); // walk over the nodes to update the velocity - FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { - double node_force[3]; - for (size_t dim = 0; dim < num_dims; dim++) { - node_force[dim] = 0.0; - } // end for dim - - // loop over all corners around the node and calculate the nodal force - for (size_t corner_lid = 0; corner_lid < num_corners_in_node(node_gid); corner_lid++) { - // Get corner gid - size_t corner_gid = corners_in_node(node_gid, corner_lid); - - // loop over dimension + if(num_lcs){ + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + double node_force[3]; for (size_t dim = 0; dim < num_dims; dim++) { - node_force[dim] += corner_force(corner_gid, dim); + node_force[dim] = 0.0; } // end for dim - } // end for corner_lid - // update the velocity - for (int dim = 0; dim < num_dims; dim++) { - node_vel(rk_level, node_gid, dim) = node_vel(0, node_gid, dim) + - rk_alpha * dt * node_force[dim] / node_mass(node_gid); - } // end for dim - }); // end for parallel for over nodes + // loop over all corners around the node and calculate the nodal force + for (size_t corner_lid = 0; corner_lid < num_corners_in_node(node_gid); corner_lid++) { + // Get corner gid + size_t corner_gid = corners_in_node(node_gid, corner_lid); + + // loop over dimension + for (size_t dim = 0; dim < num_dims; dim++) { + node_force[dim] += corner_force(corner_gid, dim) + corner_external_force(corner_gid, dim); + } // end for dim + } // end for corner_lid + + // update the velocity + for (int dim = 0; dim < num_dims; dim++) { + node_vel(rk_level, node_gid, dim) = node_vel(0, node_gid, dim) + + rk_alpha * dt * node_force[dim] / node_mass(node_gid); + } // end for dim + }); // end for parallel for over nodes + } + else{ + FOR_ALL_CLASS(node_gid, 0, nlocal_nodes, { + double node_force[3]; + for (size_t dim = 0; dim < num_dims; dim++) { + node_force[dim] = 0.0; + } // end for dim + // loop over all corners around the node and calculate the nodal force + for (size_t corner_lid = 0; corner_lid < num_corners_in_node(node_gid); corner_lid++) { + // Get corner gid + size_t corner_gid = corners_in_node(node_gid, corner_lid); + + // loop over dimension + for (size_t dim = 0; dim < num_dims; dim++) { + node_force[dim] += corner_force(corner_gid, dim); + } // end for dim + } // end for corner_lid + + // update the velocity + for (int dim = 0; dim < num_dims; dim++) { + node_vel(rk_level, node_gid, dim) = node_vel(0, node_gid, dim) + + rk_alpha * dt * node_force[dim] / node_mass(node_gid); + } // end for dim + }); // end for parallel for over nodes + } return; } // end subroutine update_velocity diff --git a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/setup_sgh.cpp b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/setup_sgh.cpp index b45326d27..f86e6f1b2 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/setup_sgh.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/SGH_Solver/src/setup_sgh.cpp @@ -115,6 +115,11 @@ void FEA_Module_SGH::setup() corner_force = DViewCArrayKokkos(&corner_interface.force(0, 0), num_corners, num_dim); corner_mass = DViewCArrayKokkos(&corner_interface.mass(0), num_corners); + //external force storage + if(num_lcs){ + corner_external_force = DCArrayKokkos(num_corners, num_dim); + } + // allocate elem_vel_grad elem_vel_grad = DCArrayKokkos(num_elems, 3, 3); diff --git a/src/Parallel-Solvers/Parallel-Explicit/optimization_parameters.xml b/src/Parallel-Solvers/Parallel-Explicit/optimization_parameters.xml index 7f2862aa5..028470728 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/optimization_parameters.xml +++ b/src/Parallel-Solvers/Parallel-Explicit/optimization_parameters.xml @@ -62,7 +62,7 @@ - + From 12069277d4f308765bae362b478c4fd93c1c523e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CAdrian-Diaz=E2=80=9D?= <“adriandiaz1117@gmail.com”> Date: Tue, 30 Jul 2024 23:25:43 -0600 Subject: [PATCH 125/130] ENH: prevent double init call for AL --- .../Kinetic_Energy_Minimize.h | 51 +++++++++++-------- 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/src/Parallel-Solvers/Parallel-Explicit/Topology_Optimization/Kinetic_Energy_Minimize.h b/src/Parallel-Solvers/Parallel-Explicit/Topology_Optimization/Kinetic_Energy_Minimize.h index cb369efe1..ac146e35b 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Topology_Optimization/Kinetic_Energy_Minimize.h +++ b/src/Parallel-Solvers/Parallel-Explicit/Topology_Optimization/Kinetic_Energy_Minimize.h @@ -99,6 +99,7 @@ typedef MV::dual_view_type dual_vec_array; real_t previous_objective_accumulation, objective_sign; bool useLC_; // Use linear form of energy. Otherwise use quadratic form. + bool first_init; //prevents ROL from calling init computation twice at start for the AL algorithm ///////////////////////////////////////////////////////////////////////////// /// @@ -145,7 +146,7 @@ typedef MV::dual_view_type dual_vec_array; : useLC_(true) { Explicit_Solver_Pointer_ = Explicit_Solver_Pointer; - + first_init = false; valid_fea_modules.push_back(FEA_MODULE_TYPE::SGH); valid_fea_modules.push_back(FEA_MODULE_TYPE::Dynamic_Elasticity); nvalid_modules = valid_fea_modules.size(); @@ -211,14 +212,17 @@ typedef MV::dual_view_type dual_vec_array; const_host_vec_array design_densities = zp->getLocalView(Tpetra::Access::ReadOnly); if (type == ROL::UpdateType::Initial) { - // This is the first call to update - // first linear solve was done in FEA class run function already - FEM_Dynamic_Elasticity_->comm_variables(zp); - // update deformation variables - FEM_Dynamic_Elasticity_->update_forward_solve(zp); - // initial design density data was already communicated for ghost nodes in init_design() - // decide to output current optimization state - FEM_Dynamic_Elasticity_->Explicit_Solver_Pointer_->write_outputs(); + if(first_init){ + // This is the first call to update + // first linear solve was done in FEA class run function already + FEM_Dynamic_Elasticity_->comm_variables(zp); + // update deformation variables + FEM_Dynamic_Elasticity_->update_forward_solve(zp); + // initial design density data was already communicated for ghost nodes in init_design() + // decide to output current optimization state + FEM_Dynamic_Elasticity_->Explicit_Solver_Pointer_->write_outputs(); + } + first_init = true; } else if (type == ROL::UpdateType::Accept) { @@ -274,19 +278,22 @@ typedef MV::dual_view_type dual_vec_array; const_host_vec_array design_densities = zp->getLocalView(Tpetra::Access::ReadOnly); if (type == ROL::UpdateType::Initial) { - // This is the first call to update - if (Explicit_Solver_Pointer_->myrank == 0) { - *fos << "called SGH Initial" << std::endl; - } + if(first_init){ + // This is the first call to update + if (Explicit_Solver_Pointer_->myrank == 0) { + *fos << "called SGH Initial" << std::endl; + } - FEM_SGH_->comm_variables(zp); - FEM_SGH_->update_forward_solve(zp); - FEM_SGH_->compute_topology_optimization_adjoint_full(zp); - previous_objective_accumulation = objective_accumulation; - previous_gradients->assign(*(FEM_SGH_->cached_design_gradients_distributed)); - // initial design density data was already communicated for ghost nodes in init_design() - // decide to output current optimization state - // FEM_SGH_->Explicit_Solver_Pointer_->write_outputs(); + FEM_SGH_->comm_variables(zp); + FEM_SGH_->update_forward_solve(zp); + FEM_SGH_->compute_topology_optimization_adjoint_full(zp); + previous_objective_accumulation = objective_accumulation; + previous_gradients->assign(*(FEM_SGH_->cached_design_gradients_distributed)); + // initial design density data was already communicated for ghost nodes in init_design() + // decide to output current optimization state + // FEM_SGH_->Explicit_Solver_Pointer_->write_outputs(); + } + first_init = true; } else if (type == ROL::UpdateType::Accept) { if (Explicit_Solver_Pointer_->myrank == 0) { @@ -392,7 +399,7 @@ typedef MV::dual_view_type dual_vec_array; } std::cout.precision(10); - if (Explicit_Solver_Pointer_->myrank == 0) { + if (Explicit_Solver_Pointer_->myrank == 0&&first_init) { std::cout << "CURRENT TIME INTEGRAL OF KINETIC ENERGY " << objective_accumulation << std::endl; } From 201edbae2676b3eeb3b367a550db082512e351c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CAdrian-Diaz=E2=80=9D?= <“adriandiaz1117@gmail.com”> Date: Fri, 2 Aug 2024 16:56:44 -0600 Subject: [PATCH 126/130] BUG: align mpi data type for allreduce --- src/Parallel-Solvers/Implicit-Lagrange/Implicit_Solver.cpp | 5 +++-- src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/Parallel-Solvers/Implicit-Lagrange/Implicit_Solver.cpp b/src/Parallel-Solvers/Implicit-Lagrange/Implicit_Solver.cpp index e8fad7eea..72d7cad72 100644 --- a/src/Parallel-Solvers/Implicit-Lagrange/Implicit_Solver.cpp +++ b/src/Parallel-Solvers/Implicit-Lagrange/Implicit_Solver.cpp @@ -186,8 +186,9 @@ void Implicit_Solver::run(){ all_initial_node_coords_distributed = all_node_coords_distributed; //print element imbalance stats - int rnum_global_sum = 0; - MPI_Allreduce(&rnum_elem, &rnum_global_sum, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); + long long int rnum_global_sum = 0; + long long int temp_rnum_elem = rnum_elem; + MPI_Allreduce(&temp_rnum_elem, &rnum_global_sum, 1, MPI_LONG_LONG_INT, MPI_SUM, MPI_COMM_WORLD); double local_imbalance, max_imbalance, avg_elem; max_imbalance = 0; avg_elem = rnum_global_sum/((double) nranks); diff --git a/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp b/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp index 8074bd3ad..65cdb7d2a 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp +++ b/src/Parallel-Solvers/Parallel-Explicit/Explicit_Solver.cpp @@ -194,8 +194,9 @@ void Explicit_Solver::run() { init_maps(); //print element imbalance stats - int rnum_global_sum = 0; - MPI_Allreduce(&rnum_elem, &rnum_global_sum, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); + long long int rnum_global_sum = 0; + long long int temp_rnum_elem = rnum_elem; + MPI_Allreduce(&temp_rnum_elem, &rnum_global_sum, 1, MPI_LONG_LONG_INT, MPI_SUM, MPI_COMM_WORLD); double local_imbalance, max_imbalance, avg_elem; max_imbalance = 0; avg_elem = rnum_global_sum/((double) nranks); From ca60842002af6b30633425135532be0cdbbcb1ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CAdrian-Diaz=E2=80=9D?= <“adriandiaz1117@gmail.com”> Date: Mon, 5 Aug 2024 00:06:58 -0600 Subject: [PATCH 127/130] WIP: refactor dynamic optimization classes --- .../Fierro_Optimization_Objective.hpp | 129 ++++++++++++++++++ .../Kinetic_Energy_Minimize.h | 18 ++- 2 files changed, 141 insertions(+), 6 deletions(-) create mode 100644 src/Parallel-Solvers/Optimization_Modules/Fierro_Optimization_Objective.hpp diff --git a/src/Parallel-Solvers/Optimization_Modules/Fierro_Optimization_Objective.hpp b/src/Parallel-Solvers/Optimization_Modules/Fierro_Optimization_Objective.hpp new file mode 100644 index 000000000..5494ef17e --- /dev/null +++ b/src/Parallel-Solvers/Optimization_Modules/Fierro_Optimization_Objective.hpp @@ -0,0 +1,129 @@ +// @HEADER +// ************************************************************************ +// +// Rapid Optimization Library (ROL) Package +// Copyright (2014) Sandia Corporation +// +// Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive +// license for use of this work by or on behalf of the U.S. Government. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact lead developers: +// Drew Kouri (dpkouri@sandia.gov) and +// Denis Ridzal (dridzal@sandia.gov) +// +// ************************************************************************ +// @HEADER + +#ifndef FIERRO_OPTIMIZATION_OBJECTIVE_H +#define FIERRO_OPTIMIZATION_OBJECTIVE_H + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include "Tpetra_Details_DefaultTypes.hpp" + +#include "ROL_Types.hpp" +#include +#include "ROL_Elementwise_Reduce.hpp" +#include "ROL_Objective.hpp" +#include "ROL_BoundConstraint.hpp" + +/** @ingroup func_group + \class ROL::ObjectiveMMA + \brief Provides the interface to to Method of Moving Asymptotes + Objective function + + --- +*/ + +class FierroOptimizationObjective : public ROL::Objective { + + typedef Tpetra::Map<>::local_ordinal_type LO; + typedef Tpetra::Map<>::global_ordinal_type GO; + typedef Tpetra::Map<>::node_type Node; + typedef Tpetra::Map Map; + typedef Tpetra::MultiVector MV; + typedef ROL::Vector V; + typedef const ROL::Vector const_V; + typedef ROL::TpetraMultiVector ROL_MV; + + using traits = Kokkos::ViewTraits; + using array_layout = typename traits::array_layout; + using execution_space = typename traits::execution_space; + using device_type = typename traits::device_type; + using memory_traits = typename traits::memory_traits; + using global_size_t = Tpetra::global_size_t; + + typedef Kokkos::View values_array; + typedef Kokkos::View global_indices_array; + typedef Kokkos::View indices_array; + + //typedef Kokkos::DualView::t_dev vec_array; + typedef MV::dual_view_type::t_dev vec_array; + typedef MV::dual_view_type::t_host host_vec_array; + typedef Kokkos::View const_host_vec_array; + typedef MV::dual_view_type dual_vec_array; + typedef ROL::Objective OBJ; + typedef ROL::BoundConstraint BND; + int update_count = 0; + +private: + + real_t fval_; // Original objective value + + real_t tol_; + + ROL::Ptr getVector( const V& x ) { + return dynamic_cast(x).getVector(); + } + + ROL::Ptr getVector( V& x ) { + return dynamic_cast(x).getVector(); + } + +public: + + FierroOptimizationObjective(){ + + } + + +}; // class ObjectiveMMA + + +#endif // FIERRO_MMA_OBJECTIVE + diff --git a/src/Parallel-Solvers/Parallel-Explicit/Topology_Optimization/Kinetic_Energy_Minimize.h b/src/Parallel-Solvers/Parallel-Explicit/Topology_Optimization/Kinetic_Energy_Minimize.h index ac146e35b..4266d8564 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Topology_Optimization/Kinetic_Energy_Minimize.h +++ b/src/Parallel-Solvers/Parallel-Explicit/Topology_Optimization/Kinetic_Energy_Minimize.h @@ -53,12 +53,13 @@ #include "ROL_Types.hpp" #include #include "ROL_Objective.hpp" +#include "Fierro_Optimization_Objective.hpp" #include "ROL_Elementwise_Reduce.hpp" #include "FEA_Module_SGH.h" #include "FEA_Module_Dynamic_Elasticity.h" #include "Explicit_Solver.h" -class KineticEnergyMinimize_TopOpt : public ROL::Objective +class KineticEnergyMinimize_TopOpt : public FierroOptimizationObjective { typedef Tpetra::Map<>::local_ordinal_type LO; typedef Tpetra::Map<>::global_ordinal_type GO; @@ -286,6 +287,9 @@ typedef MV::dual_view_type dual_vec_array; FEM_SGH_->comm_variables(zp); FEM_SGH_->update_forward_solve(zp); + if(Explicit_Solver_Pointer_->myrank == 0){ + std::cout << "CURRENT TIME INTEGRAL OF KINETIC ENERGY " << objective_accumulation << std::endl; + } FEM_SGH_->compute_topology_optimization_adjoint_full(zp); previous_objective_accumulation = objective_accumulation; previous_gradients->assign(*(FEM_SGH_->cached_design_gradients_distributed)); @@ -312,7 +316,9 @@ typedef MV::dual_view_type dual_vec_array; if (Explicit_Solver_Pointer_->myrank == 0) { *fos << "called Revert" << std::endl; } objective_accumulation = previous_objective_accumulation; FEM_SGH_->cached_design_gradients_distributed->assign(*previous_gradients); - + if(Explicit_Solver_Pointer_->myrank == 0){ + std::cout << "CURRENT TIME INTEGRAL OF KINETIC ENERGY " << objective_accumulation << std::endl; + } // FEM_SGH_->comm_variables(zp); // // update deformation variables // FEM_SGH_->update_forward_solve(zp); @@ -331,8 +337,11 @@ typedef MV::dual_view_type dual_vec_array; FEM_SGH_->comm_variables(zp); // update deformation variables FEM_SGH_->update_forward_solve(zp, print_flag); + + if(Explicit_Solver_Pointer_->myrank == 0){ + std::cout << "CURRENT TIME INTEGRAL OF KINETIC ENERGY " << objective_accumulation << std::endl; + } FEM_SGH_->compute_topology_optimization_adjoint_full(zp); - // decide to output current optimization state // FEM_SGH_->Explicit_Solver_Pointer_->write_outputs(); } @@ -399,9 +408,6 @@ typedef MV::dual_view_type dual_vec_array; } std::cout.precision(10); - if (Explicit_Solver_Pointer_->myrank == 0&&first_init) { - std::cout << "CURRENT TIME INTEGRAL OF KINETIC ENERGY " << objective_accumulation << std::endl; - } // std::cout << "Ended obj value on task " <myrank << std::endl; return objective_sign*objective_accumulation; From 3cad43d12f43b98435d28de10ef0a27eeba216cf Mon Sep 17 00:00:00 2001 From: Adrian-Diaz <46537160+Adrian-Diaz@users.noreply.github.com> Date: Mon, 5 Aug 2024 11:27:44 -0600 Subject: [PATCH 128/130] WIP: dynamic objectives refactor --- .../Optimization_Modules/Fierro_Optimization_Objective.hpp | 6 +++++- .../Topology_Optimization/Kinetic_Energy_Minimize.h | 4 +--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/Parallel-Solvers/Optimization_Modules/Fierro_Optimization_Objective.hpp b/src/Parallel-Solvers/Optimization_Modules/Fierro_Optimization_Objective.hpp index 5494ef17e..d765f13c5 100644 --- a/src/Parallel-Solvers/Optimization_Modules/Fierro_Optimization_Objective.hpp +++ b/src/Parallel-Solvers/Optimization_Modules/Fierro_Optimization_Objective.hpp @@ -117,8 +117,12 @@ class FierroOptimizationObjective : public ROL::Objective { public: + bool time_accumulation; + real_t objective_accumulation; + FierroOptimizationObjective(){ - + objective_accumulation = 0; + time_accumulation = false; } diff --git a/src/Parallel-Solvers/Parallel-Explicit/Topology_Optimization/Kinetic_Energy_Minimize.h b/src/Parallel-Solvers/Parallel-Explicit/Topology_Optimization/Kinetic_Energy_Minimize.h index 4266d8564..fc7937718 100644 --- a/src/Parallel-Solvers/Parallel-Explicit/Topology_Optimization/Kinetic_Energy_Minimize.h +++ b/src/Parallel-Solvers/Parallel-Explicit/Topology_Optimization/Kinetic_Energy_Minimize.h @@ -135,13 +135,12 @@ typedef MV::dual_view_type dual_vec_array; } public: - bool nodal_density_flag_, time_accumulation; + bool nodal_density_flag_; int last_comm_step, last_solve_step, current_step; size_t nvalid_modules; std::vector valid_fea_modules; // modules that may interface with this objective function FEA_MODULE_TYPE set_module_type; // std::string my_fea_module = "SGH"; - real_t objective_accumulation; KineticEnergyMinimize_TopOpt(Explicit_Solver* Explicit_Solver_Pointer, bool nodal_density_flag) : useLC_(true) @@ -169,7 +168,6 @@ typedef MV::dual_view_type dual_vec_array; last_comm_step = last_solve_step = -1; current_step = 0; time_accumulation = true; - objective_accumulation = 0; previous_gradients = Teuchos::rcp(new MV(Explicit_Solver_Pointer_->map, 1)); if(Explicit_Solver_Pointer_->simparam.optimization_options.maximize_flag){ From 6ba229ff2705783e7c4b0ddeab2aface97465818 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CAdrian-Diaz=E2=80=9D?= <“adriandiaz1117@gmail.com”> Date: Tue, 6 Aug 2024 00:35:14 -0600 Subject: [PATCH 129/130] ENH: allow automatic uncrustify submodule download --- scripts/setup-env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/setup-env.sh b/scripts/setup-env.sh index 5f1895fc9..5ef15dfc9 100644 --- a/scripts/setup-env.sh +++ b/scripts/setup-env.sh @@ -20,7 +20,7 @@ export installdir=${basedir}/install export devutilsdir=${topdir}/dev-utils -export UNCRUSTIFY_SOURCE_DIR=${devutilsdir}/uncrustify +export UNCRUSTIFY_SOURCE_DIR=${devutilsdir}/uncrustify/src export UNCRUSTIFY_BUILD_DIR=${devutilsdir}/uncrustify/build export UNCRUSTIFY_INSTALL_DIR=${devutilsdir}/uncrustify/build From 25be3de976bf1ee48259f6dd68e4ce3248ffb021 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CAdrian-Diaz=E2=80=9D?= <“adriandiaz1117@gmail.com”> Date: Tue, 6 Aug 2024 09:30:58 -0600 Subject: [PATCH 130/130] ENH: allow automatic submodule downloads on clean build --- scripts/cmake_build.sh | 2 +- scripts/setup-env.sh | 2 +- scripts/uncrustify-install.sh | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/cmake_build.sh b/scripts/cmake_build.sh index 55c7f49c2..dd6cd480e 100644 --- a/scripts/cmake_build.sh +++ b/scripts/cmake_build.sh @@ -12,7 +12,7 @@ kokkos_build_type="${3}" if { [ ! -d "${ELEMENTS_SOURCE_DIR}/elements" ] || [ ! -d "${ELEMENTS_SOURCE_DIR}/matar/src" ] ;} then echo "Missing submodules, downloading them...." - git submodule update --recursive "${ELEMENTS_SOURCE_DIR}" + git submodule update --init --recursive "${ELEMENTS_SOURCE_DIR}" fi if [ ! -d "${TRILINOS_INSTALL_DIR}/lib" ]; then diff --git a/scripts/setup-env.sh b/scripts/setup-env.sh index 5ef15dfc9..5f1895fc9 100644 --- a/scripts/setup-env.sh +++ b/scripts/setup-env.sh @@ -20,7 +20,7 @@ export installdir=${basedir}/install export devutilsdir=${topdir}/dev-utils -export UNCRUSTIFY_SOURCE_DIR=${devutilsdir}/uncrustify/src +export UNCRUSTIFY_SOURCE_DIR=${devutilsdir}/uncrustify export UNCRUSTIFY_BUILD_DIR=${devutilsdir}/uncrustify/build export UNCRUSTIFY_INSTALL_DIR=${devutilsdir}/uncrustify/build diff --git a/scripts/uncrustify-install.sh b/scripts/uncrustify-install.sh index b0cf694b5..49312555d 100644 --- a/scripts/uncrustify-install.sh +++ b/scripts/uncrustify-install.sh @@ -2,10 +2,10 @@ # Check if the uncrustify build directory exists and is not empty in the parent directory; if not, clone it -if [ ! -d "${UNCRUSTIFY_SOURCE_DIR}" ]; +if [ ! -d "${UNCRUSTIFY_SOURCE_DIR}/src" ]; then echo "Missing Uncrustify submodule, downloading...." - git submodule update ${UNCRUSTIFY_SOURCE_DIR} + git submodule update --init ${UNCRUSTIFY_SOURCE_DIR} fi if [ ! -f "${UNCRUSTIFY_BUILD_DIR}/uncrustify" ]; then