Skip to content

Commit

Permalink
Pushing some code so LBNL can debug.
Browse files Browse the repository at this point in the history
  • Loading branch information
Bruce J Palmer committed Jan 11, 2024
1 parent 4108588 commit c2cb944
Show file tree
Hide file tree
Showing 2 changed files with 43 additions and 225 deletions.
1 change: 0 additions & 1 deletion src/des/bmx_pc_fusion.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -860,7 +860,6 @@ void BMXParticleContainer::CalculateFungalCM(const Vector<MultiFab*> cost,

amrex::Gpu::Device::synchronize();

pstruct = particles().dataPtr();
amrex::ParallelFor( nrp, [pstruct,cmx,cmy,cmz,ntotp]
AMREX_GPU_DEVICE (int pid) noexcept
{
Expand Down
267 changes: 43 additions & 224 deletions src/des/bmx_pc_write.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,14 @@
#include <bmx_chem_species_parms.H>

using namespace amrex;
using PinnedTile = typename ContainerLike<amrex::PinnedArenaAllocator>::ParticleTileType;

void BMXParticleContainer::WriteToAscii (const std::string& vtk_filename,int nstep, Real time)
{
int nsegments, nparticles;
countParticleTypes(nparticles, nsegments);
WriteSegmentsToAscii(vtk_filename, nstep, time, nsegments);
WriteParticlesToAscii(vtk_filename, nstep, time, nparticles);
if (nsegments > 0) WriteSegmentsToAscii(vtk_filename, nstep, time, nsegments);
if (nparticles > 0) WriteParticlesToAscii(vtk_filename, nstep, time, nparticles);
}

void BMXParticleContainer::WriteSegmentsToAscii (const std::string& vtk_filename,int nstep, Real time, const int nsegments)
Expand Down Expand Up @@ -96,7 +97,12 @@ void BMXParticleContainer::WriteSegmentsToAscii (const std::string& vtk_filename
const auto& plevel = GetParticles(lev);
for (const auto& kv : plevel)
{
const auto& particles = kv.second.GetArrayOfStructs();
PinnedTile pinned_tile;
pinned_tile.define(NumRuntimeRealComps(),NumRuntimeIntComps());
pinned_tile.resize(kv.second.numParticles());
amrex::copyParticles(pinned_tile, particle_tile, 0, 0,
kv.second.numParticles());
const auto& particles = pinned_tile.GetArrayOfStructs();

for (int i = 0; i < particles.numParticles(); ++i)
{
Expand Down Expand Up @@ -457,247 +463,60 @@ void BMXParticleContainer::WriteParticlesToAscii (const std::string& vtk_filenam

}

#if 0
void BMXParticleContainer::WriteParticlesToAscii (const std::string& vtk_filename,int nstep, Real time, const int nparticles)
void BMXParticleContainer::countParticleTypes (int &nparticles, int &nsegments)
{
BL_PROFILE("BMXParticleContainer::WriteParticlesToAscii");

std::string particlefile = vtk_filename;
particlefile.append("_part");
const std::string& filename = amrex::Concatenate(particlefile,nstep,6);

Real l_max_vol = SPECIES::max_vol;
Real particle_rad = pow((3.0*l_max_vol/(4.0*M_PI)),1.0/3.0);

amrex::Print() << " " << std::endl;
amrex::Print() << " Writing particle vtkfile " << filename << " at time " << time << std::endl;
amrex::Print() << " " << std::endl;

long total_number_of_particles = 0;

for (int lev = 0; lev <= finest_level; lev++)
{
total_number_of_particles += NumberOfParticlesAtLevel(lev);
}
BL_PROFILE("BMXParticleContainer::WriteSegmentsToAscii");

long total_number_of_vertices = nparticles;
nparticles = 0;
nsegments = 0;

// First write all the header information just once (from the I/O procesor)
if (ParallelDescriptor::IOProcessor())
{
//
// Have I/O processor open file and write out particle metadata.
//
std::ofstream File;
File.open(filename + ".vtk", std::ios::out | std::ios::trunc);

if (!File.good())
amrex::FileOpenFailed(filename+".vtk");

// vtkAscii File contents
File << "# vtk DataFile Version 3.0 " << std::endl;
File << "hexagon using polygon function vtk " << std::endl;
File << "ASCII" << std::endl;
File << "DATASET POLYDATA" << std::endl;
File << " " << std::endl;
File << "POINTS" << " " << nparticles << " " << "float" << std::endl;
File.flush();
File.close();
if (!File.good())
amrex::Abort("BMXarticleContainer::WriteVTKAscii(): problem writing file");
}

ParallelDescriptor::Barrier();

const int MyProc = ParallelDescriptor::MyProc();

for (int proc = 0; proc < ParallelDescriptor::NProcs(); proc++)
for (int lev = 0; lev <= finest_level; lev++)
{
if (MyProc == proc)
{
//
// Each CPU opens the file for appending and adds its particles.
//
VisMF::IO_Buffer io_buffer(VisMF::IO_Buffer_Size);

std::ofstream File;

File.rdbuf()->pubsetbuf(io_buffer.dataPtr(), io_buffer.size());
int n_at_lev = this->NumberOfParticlesAtLevel(lev);

File.open(filename+".vtk", std::ios::out|std::ios::app);

File.precision(8);

if (!File.good())
amrex::FileOpenFailed(filename+".vtk");

for (int lev = 0; lev <= finest_level; lev++)
{
const auto& plevel = GetParticles(lev);
for (const auto& kv : plevel)
{
const auto& particles = kv.second.GetArrayOfStructs();

for (int i = 0; i < particles.numParticles(); ++i)
{
const Real* par = &particles[i].rdata(0);


int type = particles[i].idata(intIdx::cell_type);
Real x, y, z;
if (type == cellType::YEAST) {
Real c_length = particle_rad;
x = particles[i].pos(0);
y = particles[i].pos(1);
z = particles[i].pos(2);
File << x << " " << y << " " << z << std::endl;
}

} // i
} // loop over particles at level "lev"
} // levels
File.flush();
File.close();
if (!File.good())
amrex::Abort("BMXarticleContainer::WriteVTKAscii(): problem writing file");
} // MyProc
ParallelDescriptor::Barrier();
} // loop over proc

// POLYGONS # of cells #, of vertices per cell
if (ParallelDescriptor::IOProcessor())
if (n_at_lev == 0) continue;
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
for (BMXParIter pti(*this, lev); pti.isValid(); ++pti)
{
VisMF::IO_Buffer io_buffer(VisMF::IO_Buffer_Size);
std::ofstream File;
File.rdbuf()->pubsetbuf(io_buffer.dataPtr(), io_buffer.size());
File.open(filename+".vtk", std::ios::out|std::ios::app);
if (!File.good())
amrex::FileOpenFailed(filename+".vtk");

File << "POLYGONS" << " " << nparticles << std::endl;

// Write polygon info
for (long i = 0; i < nparticles; ++i)
{
File << "1 ";
File << i << " ";
File << std::endl;
}
PairIndex index(pti.index(), pti.LocalTileIndex());

File << " " << std::endl;
File << "CELL_DATA" << " " << nparticles << std::endl;
File << "POINT_DATA" << " " << nparticles << std::endl;
File << "SCALARS " << "cell " << "int" << std::endl;
File << "LOOKUP_TABLE " << "default" << std::endl;
auto& plevel = GetParticles(lev);
auto& ptile = plevel[index];
auto& aos = ptile.GetArrayOfStructs();
ParticleType* pstruct = aos().dataPtr();

// Write lookup info - this will be used to color the vertices of one cell the same color
for (long i = 0; i < nparticles; ++i)
{
File << i << " ";
File << std::endl;
}
File << std::endl;

File << "SCALARS " << "conc_E " << "float" << std::endl;
File << "LOOKUP_TABLE " << "default" << std::endl;
Gpu::DeviceScalar<int> npart_gpu(0);
Gpu::DeviceScalar<int> nseg_gpu(0);
int* npart = npart_gpu.dataPtr();
int* nseg = nseg_gpu.dataPtr();

File.flush();
File.close();
if (!File.good())
amrex::Abort("BMXarticleContainer::WriteVTKAscii(): problem writing file");
} // I/O Proc
const int nrp = GetParticles(lev)[index].numRealParticles();

ParallelDescriptor::Barrier();

for (int proc = 0; proc < ParallelDescriptor::NProcs(); proc++)
{
if (MyProc == proc)
amrex::ParallelFor(nrp, [pstruct,npart,nseg]
AMREX_GPU_DEVICE(int i) noexcept
{
//
// Each CPU opens the file for appending and adds its particles.
//
VisMF::IO_Buffer io_buffer(VisMF::IO_Buffer_Size);

std::ofstream File;

File.rdbuf()->pubsetbuf(io_buffer.dataPtr(), io_buffer.size());
auto& particle = pstruct[i];

File.open(filename+".vtk", std::ios::out|std::ios::app);

File.precision(8);

if (!File.good())
amrex::FileOpenFailed(filename+".vtk");

for (int lev = 0; lev <= finest_level; lev++)
{
const auto& plevel = GetParticles(lev);
for (const auto& kv : plevel)
{
const auto& particles = kv.second.GetArrayOfStructs();

for (int i = 0; i < particles.numParticles(); ++i)
{
const Real* par = &particles[i].rdata(0);

Real concE = par[realIdx::first_data+4];

if (particles[i].idata(intIdx::cell_type) == cellType::YEAST) {
File << concE << std::endl;
}
} // i
} // loop over particles at level "lev"
} // levels
File.flush();
File.close();
if (!File.good())
amrex::Abort("BMXarticleContainer::WriteVTKAscii(): problem writing file");
} // MyProc
ParallelDescriptor::Barrier();
} // loop over proc

}
#endif

void BMXParticleContainer::countParticleTypes (int &nparticles, int &nsegments)
{
BL_PROFILE("BMXParticleContainer::WriteSegmentsToAscii");

nparticles = 0;
nsegments = 0;

const int MyProc = ParallelDescriptor::MyProc();

for (int proc = 0; proc < ParallelDescriptor::NProcs(); proc++)
{
if (MyProc == proc)
{
for (int lev = 0; lev <= finest_level; lev++)
{
const auto& plevel = GetParticles(lev);
for (const auto& kv : plevel)
{
const auto& particles = kv.second.GetArrayOfStructs();

for (int i = 0; i < particles.numParticles(); ++i)
{
int type = particles[i].idata(intIdx::cell_type);
if (type == cellType::FUNGI) {
nsegments++;
} else {
nparticles++;
}
} // i
} // loop over particles at level "lev"
} // levels
ParallelDescriptor::Barrier();
int type = particle.idata(intIdx::cell_type);
if (type == cellType::FUNGI) {
(*nseg)++;
} else {
(*npart)++;
}
}); // loop over particles at level "lev"
nparticles += npart_gpu.dataValue();
nsegments += nseg_gpu.dataValue();
}
} // loop over proc
} // levels
int buf[2];
buf[0] = nparticles;
buf[1] = nsegments;
ParallelDescriptor::ReduceIntSum(buf,2);
nparticles = buf[0];
nsegments = buf[1];

}

0 comments on commit c2cb944

Please sign in to comment.