From f4e84e7428cd43e5f8d87a2e6a40f47676eff1c0 Mon Sep 17 00:00:00 2001 From: "G. Dylan Dickerson" Date: Wed, 16 Oct 2024 10:55:23 -0600 Subject: [PATCH] WIP add routines to try to ensure data consistency Add routines to transfer to/from the GPU before/after calling atm_do_timestep. This is to try to ensure the GPU MPAS-A dycore has the data it needs based on fields in dyn_in and that the fields CAM needs in dyn_out are updated on the CPU. --- .../mpas/driver/cam_mpas_subdriver.F90 | 103 +++++++++++++++++- 1 file changed, 102 insertions(+), 1 deletion(-) diff --git a/src/dynamics/mpas/driver/cam_mpas_subdriver.F90 b/src/dynamics/mpas/driver/cam_mpas_subdriver.F90 index cc6ac75114..ede3df723d 100644 --- a/src/dynamics/mpas/driver/cam_mpas_subdriver.F90 +++ b/src/dynamics/mpas/driver/cam_mpas_subdriver.F90 @@ -2222,7 +2222,7 @@ subroutine cam_mpas_run(integrationLength) type (MPAS_Time_Type) :: currTime type (MPAS_Time_type) :: runUntilTime character(len=StrKIND) :: timeStamp - type (mpas_pool_type), pointer :: state, diag, mesh + type (mpas_pool_type), pointer :: state, diag, mesh, tend_physics integer, pointer :: index_qv integer, pointer :: nCellsSolve @@ -2238,6 +2238,11 @@ subroutine cam_mpas_run(integrationLength) call mpas_pool_get_subpool(domain_ptr % blocklist % structs, 'state', state) call mpas_pool_get_subpool(domain_ptr % blocklist % structs, 'diag', diag) call mpas_pool_get_subpool(domain_ptr % blocklist % structs, 'mesh', mesh) + call mpas_pool_get_subpool(domain_ptr % blocklist % structs, 'tend_physics', tend_physics) + +#ifdef MPAS_OPENACC + call cam_mpas_gpudata_host_to_device(state, diag, tend_physics) +#endif !MPAS_OPENACC ! During integration, time level 1 stores the model state at the beginning of the ! time step, and time level 2 stores the state advanced dt in time by timestep(...) @@ -2263,6 +2268,10 @@ subroutine cam_mpas_run(integrationLength) currTime = mpas_get_clock_time(clock, MPAS_NOW, ierr) end do +#ifdef MPAS_OPENACC + call cam_mpas_gpudata_device_to_host(state, diag, tend_physics) +#endif !MPAS_OPENACC + ! ! Compute diagnostic fields from the final prognostic state ! @@ -2281,6 +2290,98 @@ subroutine cam_mpas_run(integrationLength) end subroutine cam_mpas_run + !----------------------------------------------------------------------- + ! routine cam_mpas_gpudata_host_to_device + ! + !> \brief Transfer data from CPU to GPU before atm_do_timestep + !> \author G. Dylan Dickerson + !> \date 15 October 2024 + !> \details + !> SOME DETAIL HERE + ! + !----------------------------------------------------------------------- + subroutine cam_mpas_gpudata_host_to_device(state, diag, tend_physics) + + use mpas_derived_types, only : mpas_pool_type + use mpas_kind_types, only : RKIND + use mpas_pool_routines, only : mpas_pool_get_array_gpu + + type (mpas_pool_type), pointer :: state, diag, tend_physics + real(kind=RKIND), dimension(:,:), pointer :: u, w, theta_m, rho_zz + real(kind=RKIND), dimension(:,:,:), pointer :: scalars + real(kind=RKIND), dimension(:,:), pointer :: theta, exner, rho, uReconstructZonal, uReconstructMeridional + real(kind=RKIND), dimension(:,:), pointer :: tend_ru_physics, tend_rho_physics, tend_rtheta_physics + + ! state pool arrays, modify the "current" state in timeLevel=1 + call mpas_pool_get_array_gpu(state, 'u', u, timeLevel=1) + call mpas_pool_get_array_gpu(state, 'w', w, timeLevel=1) + call mpas_pool_get_array_gpu(state, 'theta_m', theta_m, timeLevel=1) + call mpas_pool_get_array_gpu(state, 'rho_zz', rho_zz, timeLevel=1) + call mpas_pool_get_array_gpu(state, 'scalars', scalars, timeLevel=1) + !$acc update device(u,w,theta_m,rho_zz,scalars) + + ! diag pool arrays + call mpas_pool_get_array_gpu(diag, 'theta', theta) + call mpas_pool_get_array_gpu(diag, 'exner', exner) + call mpas_pool_get_array_gpu(diag, 'rho', rho) + call mpas_pool_get_array_gpu(diag, 'uReconstructZonal', uReconstructZonal) + call mpas_pool_get_array_gpu(diag, 'uReconstructMeridional', uReconstructMeridional) + !$acc update device(theta,exner,rho,uReconstructZonal,uReconstructMeridional) + + ! tend pool arrays + call mpas_pool_get_array_gpu(tend_physics, 'tend_ru_physics', tend_ru_physics) + call mpas_pool_get_array_gpu(tend_physics, 'tend_rtheta_physics', tend_rtheta_physics) + call mpas_pool_get_array_gpu(tend_physics, 'tend_rho_physics', tend_rho_physics) + !$acc update device(tend_ru_physics,tend_rtheta_physics,tend_rho_physics) + + end subroutine cam_mpas_gpudata_host_to_device + + + !----------------------------------------------------------------------- + ! routine cam_mpas_gpudata_device_to_host + ! + !> \brief Transfer data from GPU to CPU after atm_do_timestep + !> \author G. Dylan Dickerson + !> \date 15 October 2024 + !> \details + !> SOME DETAIL HERE + ! + !----------------------------------------------------------------------- + subroutine cam_mpas_gpudata_device_to_host(state, diag, tend_physics) + + use mpas_derived_types, only : mpas_pool_type + use mpas_kind_types, only : RKIND + use mpas_pool_routines, only : mpas_pool_get_array_gpu + + type (mpas_pool_type), pointer :: state, diag, tend_physics + real(kind=RKIND), dimension(:,:), pointer :: u, w, theta_m, rho_zz + real(kind=RKIND), dimension(:,:,:), pointer :: scalars + real(kind=RKIND), dimension(:,:), pointer :: theta, exner, rho, uReconstructZonal, uReconstructMeridional, & + v, vorticity, divergence + + ! state pool arrays, modify the "current" state in timeLevel=1 + call mpas_pool_get_array_gpu(state, 'u', u, timeLevel=1) + call mpas_pool_get_array_gpu(state, 'w', w, timeLevel=1) + call mpas_pool_get_array_gpu(state, 'theta_m', theta_m, timeLevel=1) + call mpas_pool_get_array_gpu(state, 'rho_zz', rho_zz, timeLevel=1) + call mpas_pool_get_array_gpu(state, 'scalars', scalars, timeLevel=1) + !$acc update host(u,w,theta_m,rho_zz,scalars) + + ! diag pool arrays + call mpas_pool_get_array_gpu(diag, 'theta', theta) + call mpas_pool_get_array_gpu(diag, 'exner', exner) + call mpas_pool_get_array_gpu(diag, 'rho', rho) + call mpas_pool_get_array_gpu(diag, 'uReconstructZonal', uReconstructZonal) + call mpas_pool_get_array_gpu(diag, 'uReconstructMeridional', uReconstructMeridional) + call mpas_pool_get_array_gpu(diag, 'v', v) + call mpas_pool_get_array_gpu(diag, 'vorticity', vorticity) + call mpas_pool_get_array_gpu(diag, 'divergence', divergence) + !$acc update host(theta,exner,rho,uReconstructZonal,uReconstructMeridional, & + !$acc v,vorticity,divergence) + + end subroutine cam_mpas_gpudata_device_to_host + + !----------------------------------------------------------------------- ! routine cam_mpas_finalize !