From 3db7cd584e63467a65579ba2ba234272d7c09b81 Mon Sep 17 00:00:00 2001 From: Yoo Date: Wed, 11 Sep 2024 12:48:19 -0700 Subject: [PATCH 1/4] Visualization of multicut locations. 1) In the input file, you can add: eb2.plt_multiple_cuts = "true" to produce plot files to see the multicut cells. 2) For a 3D case, you would get up to three plot files: plt.multicut.x, plt.multicut.y, and plt.multicut.z. This depends on which face has multicut issues. -> See AMReX_EB2_3D_C.cpp 3) For a 2D case, you get plt.multicut -> See AMReX_EB2_2D_C.cpp 4) For any dimension, the cell value of 2 indicates regular, 0 (or very tiny) is covered, and some cell values > 2 means multicut cells. 5) If "plt_multiple_cuts" is true, we abort the code after creating the plot files; This is in the AMReX_EB2_Level.H. --- Src/EB/AMReX_EB2_2D_C.cpp | 12 ++++++++-- Src/EB/AMReX_EB2_3D_C.cpp | 28 ++++++++++++++++++---- Src/EB/AMReX_EB2_C.H | 8 ++++--- Src/EB/AMReX_EB2_Level.H | 49 ++++++++++++++++++++++++++++++++++++--- 4 files changed, 85 insertions(+), 12 deletions(-) diff --git a/Src/EB/AMReX_EB2_2D_C.cpp b/Src/EB/AMReX_EB2_2D_C.cpp index b99b5559c77..e4202655833 100644 --- a/Src/EB/AMReX_EB2_2D_C.cpp +++ b/Src/EB/AMReX_EB2_2D_C.cpp @@ -203,7 +203,8 @@ int build_faces (Box const& bx, Array4 const& cell, Array4 const& fcx, Array4 const& fcy, GpuArray const& dx, GpuArray const& problo, - bool cover_multiple_cuts, int& nsmallfaces) noexcept + bool cover_multiple_cuts, int& nsmallfaces, + bool plt_multiple_cuts, Array4 const& mcx) noexcept { #ifdef AMREX_USE_FLOAT constexpr Real small = 1.e-5_rt; @@ -311,6 +312,10 @@ int build_faces (Box const& bx, Array4 const& cell, if (ncuts > 2) { Gpu::Atomic::Add(dp,1); } + if (plt_multiple_cuts) + { + mcx(i,j,k) = ncuts; + } } } }); @@ -342,7 +347,10 @@ int build_faces (Box const& bx, Array4 const& cell, nsmallfaces += *(hp+1); if (*hp > 0 && !cover_multiple_cuts) { - amrex::Abort("amrex::EB2::build_faces: more than 2 cuts not supported"); + if (!plt_multiple_cuts) + { + amrex::Abort("amrex::EB2::build_faces: more than 2 cuts not supported"); + } } return *hp; diff --git a/Src/EB/AMReX_EB2_3D_C.cpp b/Src/EB/AMReX_EB2_3D_C.cpp index 2d02e53bdc7..353afe4160e 100644 --- a/Src/EB/AMReX_EB2_3D_C.cpp +++ b/Src/EB/AMReX_EB2_3D_C.cpp @@ -370,7 +370,8 @@ int build_faces (Box const& bx, Array4 const& cell, Array4 const& m2z, GpuArray const& dx, GpuArray const& problo, - bool cover_multiple_cuts) noexcept + bool cover_multiple_cuts, bool plt_multiple_cuts, + Array4 const& mcx, Array4 const& mcy, Array4 const& mcz) noexcept { Gpu::Buffer nmulticuts = {0}; int* hp = nmulticuts.hostData(); @@ -491,6 +492,11 @@ int build_faces (Box const& bx, Array4 const& cell, } else if (apx(i,j,k) == 1.0_rt) { fx(i,j,k) = Type::regular; } + + if (plt_multiple_cuts){ + mcx(i,j,k) = ncuts; + } + } }); @@ -599,6 +605,10 @@ int build_faces (Box const& bx, Array4 const& cell, } else if (apy(i,j,k) == 1.0_rt) { fy(i,j,k) = Type::regular; } + + if (plt_multiple_cuts){ + mcy(i,j,k) = ncuts; + } } }); @@ -707,6 +717,10 @@ int build_faces (Box const& bx, Array4 const& cell, } else if (apz(i,j,k) == 1.0_rt) { fz(i,j,k) = Type::regular; } + + if (plt_multiple_cuts){ + mcz(i,j,k) = ncuts; + } } }); @@ -768,7 +782,10 @@ int build_faces (Box const& bx, Array4 const& cell, } }); } else { - amrex::Abort("amrex::EB2::build_faces: more than 2 cuts not supported"); + if (!plt_multiple_cuts) + { + amrex::Abort("amrex::EB2::build_faces: more than 2 cuts not supported"); + } } } @@ -787,7 +804,7 @@ void build_cells (Box const& bx, Array4 const& cell, Array4 const& barea, Array4 const& bcent, Array4 const& bnorm, Array4 const& ctmp, Array4 const& levset, Real small_volfrac, Geometry const& geom, - bool extend_domain_face, bool cover_multiple_cuts, + bool extend_domain_face, bool cover_multiple_cuts, bool plt_multiple_cuts, int& nsmallcells, int& nmulticuts) noexcept { Gpu::Buffer n_smallcell_multicuts = {0,0}; @@ -932,7 +949,10 @@ void build_cells (Box const& bx, Array4 const& cell, if (nsmallcells > 0 || nmulticuts > 0) { if (!cover_multiple_cuts && nmulticuts > 0) { - amrex::Abort("amrex::EB2::build_cells: multi-cuts not supported"); + if (!plt_multiple_cuts) + { + amrex::Abort("amrex::EB2::build_cells: multi-cuts not supported"); + } } return; } else { diff --git a/Src/EB/AMReX_EB2_C.H b/Src/EB/AMReX_EB2_C.H index 49db3a270bd..74912ced149 100644 --- a/Src/EB/AMReX_EB2_C.H +++ b/Src/EB/AMReX_EB2_C.H @@ -25,7 +25,8 @@ int build_faces (Box const& bx, Array4 const& cell, Array4 const& fcx, Array4 const& fcy, GpuArray const& dx, GpuArray const& problo, - bool cover_multiple_cuts, int& nsmallfaces) noexcept; + bool cover_multiple_cuts, int& nsmallfaces, + bool plt_multiple_cuts, Array4 const& mcx) noexcept; void build_cells (Box const& bx, Array4 const& cell, Array4 const& fx, Array4 const& fy, @@ -55,7 +56,8 @@ int build_faces (Box const& bx, Array4 const& cell, Array4 const& m2z, GpuArray const& dx, GpuArray const& problo, - bool cover_multiple_cuts) noexcept; + bool cover_multiple_cuts, bool plt_multiple_cuts, + Array4 const& mcx, Array4 const& mcy, Array4 const& mcz) noexcept; void build_cells (Box const& bx, Array4 const& cell, Array4 const& fx, Array4 const& fy, @@ -69,7 +71,7 @@ void build_cells (Box const& bx, Array4 const& cell, Array4 const& barea, Array4 const& bcent, Array4 const& bnorm, Array4 const& ctmp, Array4 const& levset, Real small_volfrac, Geometry const& geom, - bool extend_domain_face, bool cover_multiple_cuts, + bool extend_domain_face, bool cover_multiple_cuts, bool plt_multiple_cuts, int& nsmallcells, int& nmulticuts) noexcept; void set_connection_flags(Box const& bx, Box const& bxg1, diff --git a/Src/EB/AMReX_EB2_Level.H b/Src/EB/AMReX_EB2_Level.H index 7e30e51a6ba..42b204d6980 100644 --- a/Src/EB/AMReX_EB2_Level.H +++ b/Src/EB/AMReX_EB2_Level.H @@ -14,6 +14,7 @@ #include #include #include +#include #ifdef AMREX_USE_OMP #include @@ -92,6 +93,7 @@ protected: Array m_areafrac; Array m_facecent; Array m_edgecent; + Array m_multicuts; iMultiFab m_cutcellmask; bool m_allregular = false; bool m_ok = false; @@ -167,12 +169,14 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, Real small_volfrac = 1.e-14; #endif bool cover_multiple_cuts = false; + bool plt_multiple_cuts = false; int maxiter = 32; { ParmParse pp("eb2"); pp.queryAdd("small_volfrac", small_volfrac); pp.queryAdd("cover_multiple_cuts", cover_multiple_cuts); pp.queryAdd("maxiter", maxiter); + pp.queryAdd("plt_multiple_cuts", plt_multiple_cuts); } maxiter = std::min(100000, maxiter); @@ -293,6 +297,8 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { m_areafrac[idim].define(amrex::convert(m_grids, IntVect::TheDimensionVector(idim)), m_dmap, 1, ng, mf_info); + m_multicuts[idim].define(amrex::convert(m_grids, IntVect::TheDimensionVector(idim)), + m_dmap, 1, ng, mf_info); m_facecent[idim].define(amrex::convert(m_grids, IntVect::TheDimensionVector(idim)), m_dmap, AMREX_SPACEDIM-1, ng, mf_info); IntVect edge_type{1}; edge_type[idim] = 0; @@ -413,9 +419,14 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, Array4 const& ym2 = M2[1].array(); Array4 const& zm2 = M2[2].array(); + AMREX_D_TERM(Array4 const& mcx = m_multicuts[0].array(mfi);, + Array4 const& mcy = m_multicuts[1].array(mfi);, + Array4 const& mcz = m_multicuts[2].array(mfi);); + nmc = build_faces(vbx, cfg, ftx, fty, ftz, xdg, ydg, zdg, lst, xip, yip, zip, apx, apy, apz, fcx, fcy, fcz, - xm2, ym2, zm2, dx, problo, cover_multiple_cuts); + xm2, ym2, zm2, dx, problo, cover_multiple_cuts, + plt_multiple_cuts, mcx, mcy, mcz); cellflagtmp.resize(m_cellflag[mfi].box()); Elixir cellflagtmp_eli = cellflagtmp.elixir(); @@ -424,7 +435,8 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, build_cells(vbx, cfg, ftx, fty, ftz, apx, apy, apz, fcx, fcy, fcz, xm2, ym2, zm2, dx, vfr, ctr, bar, bct, bnm, cfgtmp, lst, - small_volfrac, geom, extend_domain_face, cover_multiple_cuts, + small_volfrac, geom, extend_domain_face, + cover_multiple_cuts, plt_multiple_cuts, nsm, nmc); // Because it is used in a synchronous reduction kernel in @@ -466,8 +478,11 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, clst, geom); } + Array4 const& mcx = m_multicuts[0].array(mfi); + nmc = build_faces(vbx, cfg, ftx, fty, lst, xip, yip, apx, apy, fcx, fcy, - dx, problo, cover_multiple_cuts, nsm); + dx, problo, cover_multiple_cuts, nsm, + plt_multiple_cuts, mcx); build_cells(vbx, cfg, ftx, fty, apx, apy, dx, vfr, ctr, bar, bct, bnm, lst, small_volfrac, geom, extend_domain_face, @@ -479,6 +494,34 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, } ParallelAllReduce::Sum({nsmallcells,nmulticuts}, ParallelContext::CommunicatorSub()); + + if (plt_multiple_cuts && nmulticuts > 0) + { + amrex::Print() << "Total number of multicuts = " << nmulticuts << "\n"; + amrex::Print() << "Plotting multicut locations..." << "\n"; + + #if (AMREX_SPACEDIM == 3) + if (m_multicuts[0].max(0) > 2) + { + WriteSingleLevelPlotfile("plt.multicut.x", m_multicuts[0], {"multicut_fcx"}, geom, 0.0, 0); + } + if (m_multicuts[1].max(0) > 2) + { + WriteSingleLevelPlotfile("plt.multicut.y", m_multicuts[1], {"multicut_fcy"}, geom, 0.0, 0); + } + if (m_multicuts[2].max(0) > 2) + { + WriteSingleLevelPlotfile("plt.multicut.z", m_multicuts[2], {"multicut_fcz"}, geom, 0.0, 0); + } + #elif (AMREX_SPACEDIM == 2) + if (m_multicuts[0].max(0) > 2) + { + WriteSingleLevelPlotfile("plt.multicut", m_multicuts[0], {"multicut_fcx"}, geom, 0.0, 0); + } + #endif + amrex::Abort("amrex::EB2:: more than 2 cuts not supported"); + } + if (nsmallcells == 0 && nmulticuts == 0) { break; } else { From 1f267fe3452bfeda0e0ec3c72959a56328b07e53 Mon Sep 17 00:00:00 2001 From: Yoo Date: Wed, 11 Sep 2024 14:30:30 -0700 Subject: [PATCH 2/4] Fixing Trailing Whitespaces --- Src/EB/AMReX_EB2_3D_C.cpp | 2 +- Src/EB/AMReX_EB2_C.H | 2 +- Src/EB/AMReX_EB2_Level.H | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Src/EB/AMReX_EB2_3D_C.cpp b/Src/EB/AMReX_EB2_3D_C.cpp index 353afe4160e..ce6dd8ec5a6 100644 --- a/Src/EB/AMReX_EB2_3D_C.cpp +++ b/Src/EB/AMReX_EB2_3D_C.cpp @@ -370,7 +370,7 @@ int build_faces (Box const& bx, Array4 const& cell, Array4 const& m2z, GpuArray const& dx, GpuArray const& problo, - bool cover_multiple_cuts, bool plt_multiple_cuts, + bool cover_multiple_cuts, bool plt_multiple_cuts, Array4 const& mcx, Array4 const& mcy, Array4 const& mcz) noexcept { Gpu::Buffer nmulticuts = {0}; diff --git a/Src/EB/AMReX_EB2_C.H b/Src/EB/AMReX_EB2_C.H index 74912ced149..aeaac8d2318 100644 --- a/Src/EB/AMReX_EB2_C.H +++ b/Src/EB/AMReX_EB2_C.H @@ -56,7 +56,7 @@ int build_faces (Box const& bx, Array4 const& cell, Array4 const& m2z, GpuArray const& dx, GpuArray const& problo, - bool cover_multiple_cuts, bool plt_multiple_cuts, + bool cover_multiple_cuts, bool plt_multiple_cuts, Array4 const& mcx, Array4 const& mcy, Array4 const& mcz) noexcept; void build_cells (Box const& bx, Array4 const& cell, diff --git a/Src/EB/AMReX_EB2_Level.H b/Src/EB/AMReX_EB2_Level.H index 42b204d6980..50f7c6b5aa7 100644 --- a/Src/EB/AMReX_EB2_Level.H +++ b/Src/EB/AMReX_EB2_Level.H @@ -435,7 +435,7 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, build_cells(vbx, cfg, ftx, fty, ftz, apx, apy, apz, fcx, fcy, fcz, xm2, ym2, zm2, dx, vfr, ctr, bar, bct, bnm, cfgtmp, lst, - small_volfrac, geom, extend_domain_face, + small_volfrac, geom, extend_domain_face, cover_multiple_cuts, plt_multiple_cuts, nsm, nmc); From 6642d6193cfef721f15c2780eca4319a3a0bb6cc Mon Sep 17 00:00:00 2001 From: Yoo Date: Wed, 11 Sep 2024 15:36:01 -0700 Subject: [PATCH 3/4] Added the documentation for the new input parameter eb2.plt_multiple_cuts in RuntimeParameters.rst. --- Docs/sphinx_documentation/source/RuntimeParameters.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Docs/sphinx_documentation/source/RuntimeParameters.rst b/Docs/sphinx_documentation/source/RuntimeParameters.rst index 4e9f4196804..5840445624b 100644 --- a/Docs/sphinx_documentation/source/RuntimeParameters.rst +++ b/Docs/sphinx_documentation/source/RuntimeParameters.rst @@ -787,6 +787,15 @@ Embedded Boundary .. tip:: Because AMReX currently does not support multi-cut cells, it would be a runtime error if multi-cut cells are left unfixed. +.. py:data:: eb2.plt_multiple_cuts + :type: bool + :value: false + + If this parameter is set to true and multi-cut cells are present, + we produce plot files to see the location of the multi-cut issue. + In the plot file, a cell value of 2 indicates regular, 0 means covered, + and values greater than 2 indicate multi-cut cells. + .. py:data:: eb2.maxiter :type: int :value: 32 From 1656459b5ebf142e7f462118042f29eb58d5b29c Mon Sep 17 00:00:00 2001 From: Yoo Date: Fri, 20 Sep 2024 15:54:40 -0400 Subject: [PATCH 4/4] Changed to cell-centered MFs --- Src/EB/AMReX_EB2_Level.H | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/Src/EB/AMReX_EB2_Level.H b/Src/EB/AMReX_EB2_Level.H index 50f7c6b5aa7..2d9470bfa6f 100644 --- a/Src/EB/AMReX_EB2_Level.H +++ b/Src/EB/AMReX_EB2_Level.H @@ -93,7 +93,9 @@ protected: Array m_areafrac; Array m_facecent; Array m_edgecent; - Array m_multicuts; + MultiFab m_multicut_fcx; + MultiFab m_multicut_fcy; + MultiFab m_multicut_fcz; iMultiFab m_cutcellmask; bool m_allregular = false; bool m_ok = false; @@ -294,11 +296,12 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, m_bndryarea.define(m_grids, m_dmap, 1, ng, mf_info); m_bndrycent.define(m_grids, m_dmap, AMREX_SPACEDIM, ng, mf_info); m_bndrynorm.define(m_grids, m_dmap, AMREX_SPACEDIM, ng, mf_info); + m_multicut_fcx.define(m_grids, m_dmap, 1, ng, mf_info); + m_multicut_fcy.define(m_grids, m_dmap, 1, ng, mf_info); + m_multicut_fcz.define(m_grids, m_dmap, 1, ng, mf_info); for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { m_areafrac[idim].define(amrex::convert(m_grids, IntVect::TheDimensionVector(idim)), m_dmap, 1, ng, mf_info); - m_multicuts[idim].define(amrex::convert(m_grids, IntVect::TheDimensionVector(idim)), - m_dmap, 1, ng, mf_info); m_facecent[idim].define(amrex::convert(m_grids, IntVect::TheDimensionVector(idim)), m_dmap, AMREX_SPACEDIM-1, ng, mf_info); IntVect edge_type{1}; edge_type[idim] = 0; @@ -419,9 +422,9 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, Array4 const& ym2 = M2[1].array(); Array4 const& zm2 = M2[2].array(); - AMREX_D_TERM(Array4 const& mcx = m_multicuts[0].array(mfi);, - Array4 const& mcy = m_multicuts[1].array(mfi);, - Array4 const& mcz = m_multicuts[2].array(mfi);); + Array4 const& mcx = m_multicut_fcx.array(mfi); + Array4 const& mcy = m_multicut_fcy.array(mfi); + Array4 const& mcz = m_multicut_fcz.array(mfi); nmc = build_faces(vbx, cfg, ftx, fty, ftz, xdg, ydg, zdg, lst, xip, yip, zip, apx, apy, apz, fcx, fcy, fcz, @@ -478,7 +481,7 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, clst, geom); } - Array4 const& mcx = m_multicuts[0].array(mfi); + Array4 const& mcx = m_multicut_fcx.array(mfi); nmc = build_faces(vbx, cfg, ftx, fty, lst, xip, yip, apx, apy, fcx, fcy, dx, problo, cover_multiple_cuts, nsm, @@ -501,22 +504,22 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, amrex::Print() << "Plotting multicut locations..." << "\n"; #if (AMREX_SPACEDIM == 3) - if (m_multicuts[0].max(0) > 2) + if (m_multicut_fcx.max(0) > 2) { - WriteSingleLevelPlotfile("plt.multicut.x", m_multicuts[0], {"multicut_fcx"}, geom, 0.0, 0); + WriteSingleLevelPlotfile("plt.multicut.x", m_multicut_fcx, {"multicut_fcx"}, geom, 0.0, 0); } - if (m_multicuts[1].max(0) > 2) + if (m_multicut_fcy.max(0) > 2) { - WriteSingleLevelPlotfile("plt.multicut.y", m_multicuts[1], {"multicut_fcy"}, geom, 0.0, 0); + WriteSingleLevelPlotfile("plt.multicut.y", m_multicut_fcy, {"multicut_fcy"}, geom, 0.0, 0); } - if (m_multicuts[2].max(0) > 2) + if (m_multicut_fcz.max(0) > 2) { - WriteSingleLevelPlotfile("plt.multicut.z", m_multicuts[2], {"multicut_fcz"}, geom, 0.0, 0); + WriteSingleLevelPlotfile("plt.multicut.z", m_multicut_fcz, {"multicut_fcz"}, geom, 0.0, 0); } #elif (AMREX_SPACEDIM == 2) - if (m_multicuts[0].max(0) > 2) + if (m_multicut_fcx.max(0) > 2) { - WriteSingleLevelPlotfile("plt.multicut", m_multicuts[0], {"multicut_fcx"}, geom, 0.0, 0); + WriteSingleLevelPlotfile("plt.multicut", m_multicut_fcx, {"multicut_fcx"}, geom, 0.0, 0); } #endif amrex::Abort("amrex::EB2:: more than 2 cuts not supported");