diff --git a/atmos_cubed_sphere/driver/fvGFS/atmosphere.F90 b/atmos_cubed_sphere/driver/fvGFS/atmosphere.F90 index 0aef26450..753edc807 100644 --- a/atmos_cubed_sphere/driver/fvGFS/atmosphere.F90 +++ b/atmos_cubed_sphere/driver/fvGFS/atmosphere.F90 @@ -736,16 +736,21 @@ end subroutine set_atmosphere_pelist !! the "domain2d" variable associated with the coupling grid and the !! decomposition for the current cubed-sphere tile. !>@detail Coupling is done using the mass/temperature grid with no halos. - subroutine atmosphere_domain ( fv_domain, layout, regional ) + subroutine atmosphere_domain ( fv_domain, layout, regional, nested, pelist ) type(domain2d), intent(out) :: fv_domain integer, intent(out) :: layout(2) logical, intent(out) :: regional + logical, intent(out) :: nested + integer, pointer, intent(out) :: pelist(:) ! returns the domain2d variable associated with the coupling grid ! note: coupling is done using the mass/temperature grid with no halos fv_domain = Atm(mytile)%domain_for_coupler layout(1:2) = Atm(mytile)%layout(1:2) regional = Atm(mytile)%flagstruct%regional + nested = ngrids > 1 + call set_atmosphere_pelist() + pelist => Atm(mytile)%pelist end subroutine atmosphere_domain diff --git a/atmos_cubed_sphere/driver/fvGFS/fv_nggps_diag.F90 b/atmos_cubed_sphere/driver/fvGFS/fv_nggps_diag.F90 index e2a68dfb9..3f9ceb2a5 100644 --- a/atmos_cubed_sphere/driver/fvGFS/fv_nggps_diag.F90 +++ b/atmos_cubed_sphere/driver/fvGFS/fv_nggps_diag.F90 @@ -551,7 +551,7 @@ end subroutine store_data #ifdef use_WRTCOMP - subroutine fv_dyn_bundle_setup(axes, dyn_bundle, fcst_grid, quilting ) + subroutine fv_dyn_bundle_setup(axes, dyn_bundle, fcst_grid, quilting, rc) ! !------------------------------------------------------------- !*** set esmf bundle for dyn output fields @@ -564,10 +564,11 @@ subroutine fv_dyn_bundle_setup(axes, dyn_bundle, fcst_grid, quilting ) type(ESMF_FieldBundle),intent(inout) :: dyn_bundle type(ESMF_Grid),intent(inout) :: fcst_grid logical,intent(in) :: quilting + integer,intent(out) :: rc !*** local variables - integer i, j, k, n, rc + integer i, j, k, n integer num_axes, id, axis_length, direction, edges integer num_attributes, num_field_dyn, axis_typ character(255) :: units, long_name, cart_name,axis_direct,edgesS @@ -577,7 +578,7 @@ subroutine fv_dyn_bundle_setup(axes, dyn_bundle, fcst_grid, quilting ) type(domain1d) :: Domain type(domainuG) :: DomainU real,dimension(:),allocatable :: axis_data - type(diag_atttype),dimension(:),allocatable :: attributes + type(diag_atttype),dimension(:),allocatable :: attributes character(2) axis_id type(ESMF_Field) :: field @@ -589,6 +590,10 @@ subroutine fv_dyn_bundle_setup(axes, dyn_bundle, fcst_grid, quilting ) ! type(ESMF_Field),dimension(:),allocatable :: fieldlist ! !------------------------------------------------------------ + +! initialize RC + rc = ESMF_SUCCESS + !--- use wrte grid component for output use_wrtgridcomp_output = quilting diff --git a/atmos_cubed_sphere/model/fv_arrays.F90 b/atmos_cubed_sphere/model/fv_arrays.F90 index ba323060d..1e7c6a686 100644 --- a/atmos_cubed_sphere/model/fv_arrays.F90 +++ b/atmos_cubed_sphere/model/fv_arrays.F90 @@ -1176,7 +1176,7 @@ module fv_arrays_mod logical :: grid_active = .true. !Always active for now !This is kept here instead of in neststruct% simply for convenience - type(fv_atmos_type), pointer :: parent_grid _NULL + type(fv_atmos_type), pointer :: parent_grid => NULL() !----------------------------------------------------------------------- ! Five prognostic state variables for the f-v dynamics diff --git a/atmos_cubed_sphere/model/fv_dynamics.F90 b/atmos_cubed_sphere/model/fv_dynamics.F90 index f68388eda..5fa21fb2d 100644 --- a/atmos_cubed_sphere/model/fv_dynamics.F90 +++ b/atmos_cubed_sphere/model/fv_dynamics.F90 @@ -241,7 +241,7 @@ subroutine fv_dynamics(npx, npy, npz, nq_tot, ng, bdt, consv_te, fill, type(fv_flags_type), intent(INOUT) :: flagstruct type(fv_nest_type), intent(INOUT) :: neststruct type(domain2d), intent(INOUT) :: domain - type(fv_atmos_type), intent(INOUT) :: parent_grid + type(fv_atmos_type), pointer, intent(IN) :: parent_grid type(fv_diag_type), intent(IN) :: idiag ! Local Arrays diff --git a/atmos_cubed_sphere/model/fv_tracer2d.F90 b/atmos_cubed_sphere/model/fv_tracer2d.F90 index 1b6a42392..f86463717 100644 --- a/atmos_cubed_sphere/model/fv_tracer2d.F90 +++ b/atmos_cubed_sphere/model/fv_tracer2d.F90 @@ -579,7 +579,7 @@ subroutine tracer_2d_nested(q, dp1, mfx, mfy, cx, cy, gridstruct, bd, domain, np real , intent(INOUT) :: cy(bd%isd:bd%ied,bd%js :bd%je +1,npz) !< Courant Number Y-Dir type(fv_grid_type), intent(IN), target :: gridstruct type(fv_nest_type), intent(INOUT) :: neststruct - type(fv_atmos_type), intent(INOUT) :: parent_grid + type(fv_atmos_type), pointer, intent(IN) :: parent_grid type(domain2d), intent(INOUT) :: domain ! Local Arrays diff --git a/atmos_cubed_sphere/tools/fv_diagnostics.F90 b/atmos_cubed_sphere/tools/fv_diagnostics.F90 index 96de7b983..a1982d09a 100644 --- a/atmos_cubed_sphere/tools/fv_diagnostics.F90 +++ b/atmos_cubed_sphere/tools/fv_diagnostics.F90 @@ -2676,7 +2676,6 @@ subroutine fv_diag(Atm, zvir, Time, print_freq) deallocate(a3) endif - if( allocated(wz) ) deallocate (wz) !------------------------------------------------------- diff --git a/atmos_model.F90 b/atmos_model.F90 index af5e8f8fb..a286f8008 100644 --- a/atmos_model.F90 +++ b/atmos_model.F90 @@ -123,6 +123,7 @@ module atmos_model_mod integer, pointer :: pelist(:) =>null() ! pelist where atmosphere is running. integer :: layout(2) ! computer task laytout logical :: regional ! true if domain is regional + logical :: nested ! true if there is a nest integer :: mlon, mlat logical :: pe ! current pe. real(kind=8), pointer, dimension(:) :: ak, bk @@ -411,7 +412,7 @@ subroutine atmos_model_init (Atmos, Time_init, Time, Time_step) call atmosphere_resolution (nlon, nlat, global=.false.) call atmosphere_resolution (mlon, mlat, global=.true.) call alloc_atmos_data_type (nlon, nlat, Atmos) - call atmosphere_domain (Atmos%domain, Atmos%layout, Atmos%regional) + call atmosphere_domain (Atmos%domain, Atmos%layout, Atmos%regional, Atmos%nested, Atmos%pelist) call atmosphere_diag_axes (Atmos%axes) call atmosphere_etalvls (Atmos%ak, Atmos%bk, flip=flip_vc) call atmosphere_grid_bdry (Atmos%lon_bnd, Atmos%lat_bnd, global=.false.) diff --git a/fv3_cap.F90 b/fv3_cap.F90 index abfbdba74..29ca30ae0 100644 --- a/fv3_cap.F90 +++ b/fv3_cap.F90 @@ -696,7 +696,7 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc) call ESMF_StateGet(wrtState(i), & itemName="mirror_"//trim(fcstItemNameList(j)), & fieldbundle=wrtFB(j,i), rc=rc) - if(mype==0) print *,'af get wrtfb=',"mirror_"//trim(fcstItemNameList(j)),'rc=',rc + if(mype==0) print *,'af get wrtfb=',"mirror_"//trim(fcstItemNameList(j)),'rc=',rc if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, file=__FILE__)) return ! bail out @@ -729,9 +729,17 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc) isrctermprocessing = 1 call ESMF_FieldBundleRegridStore(fcstFB(j), wrtFB(j,i), & regridMethod=regridmethod, routehandle=routehandle(j,i), & - srcTermProcessing=isrctermprocessing, rc=rc) - - if(mype==0) print *,'after regrid store, group i=',i,' fb=',j,' time=',mpi_wtime()-timewri + srcTermProcessing=isrctermprocessing, & + rc=rc) + !if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + ! line=__LINE__, & + ! file=__FILE__)) & + ! return ! bail out + if (rc /= ESMF_SUCCESS) then + write(0,*)'fv3_cap.F90:InitializeAdvertise error in ESMF_FieldBundleRegridStore' + call ESMF_LogWrite('fv3_cap.F90:InitializeAdvertise error in ESMF_FieldBundleRegridStore', ESMF_LOGMSG_ERROR, rc=rc) + call ESMF_Finalize(endflag=ESMF_END_ABORT) + end if call ESMF_LogWrite('af FieldBundleRegridStore', ESMF_LOGMSG_INFO, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, file=__FILE__)) return ! bail out diff --git a/gfsphysics/GFS_layer/GFS_typedefs.F90 b/gfsphysics/GFS_layer/GFS_typedefs.F90 index 904e63e60..2ecd55b2f 100644 --- a/gfsphysics/GFS_layer/GFS_typedefs.F90 +++ b/gfsphysics/GFS_layer/GFS_typedefs.F90 @@ -1451,8 +1451,8 @@ subroutine coupling_create (Coupling, IM, Model) Coupling%q2mi_cpl = clear_val Coupling%tsfci_cpl = clear_val Coupling%psurfi_cpl = clear_val -!! Coupling%oro_cpl = clear_val !< pointer to sfcprop%oro -!! Coupling%slmsk_cpl = clear_val !< pointer to sfcprop%slmsk + Coupling%oro_cpl = clear_val !< pointer to sfcprop%oro + Coupling%slmsk_cpl = clear_val !< pointer to sfcprop%slmsk endif !-- cellular automata diff --git a/gfsphysics/physics/gscondp.f b/gfsphysics/physics/gscondp.f old mode 100755 new mode 100644 diff --git a/gfsphysics/physics/h2ointerp.f90 b/gfsphysics/physics/h2ointerp.f90 old mode 100755 new mode 100644 diff --git a/gfsphysics/physics/h2ophys.f b/gfsphysics/physics/h2ophys.f old mode 100755 new mode 100644 diff --git a/gfsphysics/physics/mfpbl.f b/gfsphysics/physics/mfpbl.f old mode 100755 new mode 100644 diff --git a/gfsphysics/physics/mfpblt.f b/gfsphysics/physics/mfpblt.f old mode 100755 new mode 100644 diff --git a/gfsphysics/physics/mfscu.f b/gfsphysics/physics/mfscu.f old mode 100755 new mode 100644 diff --git a/gfsphysics/physics/micro_mg2_0.F90 b/gfsphysics/physics/micro_mg2_0.F90 old mode 100755 new mode 100644 diff --git a/gfsphysics/physics/micro_mg3_0.F90 b/gfsphysics/physics/micro_mg3_0.F90 old mode 100755 new mode 100644 diff --git a/gfsphysics/physics/micro_mg_utils.F90 b/gfsphysics/physics/micro_mg_utils.F90 old mode 100755 new mode 100644 diff --git a/gfsphysics/physics/ozphys_2015.f b/gfsphysics/physics/ozphys_2015.f old mode 100755 new mode 100644 diff --git a/gfsphysics/physics/physparam.f b/gfsphysics/physics/physparam.f old mode 100755 new mode 100644 diff --git a/gfsphysics/physics/precpdp.f b/gfsphysics/physics/precpdp.f old mode 100755 new mode 100644 diff --git a/gfsphysics/physics/rayleigh_damp.f b/gfsphysics/physics/rayleigh_damp.f old mode 100755 new mode 100644 diff --git a/gfsphysics/physics/rayleigh_damp_mesopause.f b/gfsphysics/physics/rayleigh_damp_mesopause.f old mode 100755 new mode 100644 diff --git a/gfsphysics/physics/samfdeepcnv.f b/gfsphysics/physics/samfdeepcnv.f old mode 100755 new mode 100644 diff --git a/gfsphysics/physics/samfshalcnv.f b/gfsphysics/physics/samfshalcnv.f old mode 100755 new mode 100644 diff --git a/io/FV3GFS_io.F90 b/io/FV3GFS_io.F90 index 5897d1cf1..d6632fcbf 100644 --- a/io/FV3GFS_io.F90 +++ b/io/FV3GFS_io.F90 @@ -2133,7 +2133,7 @@ subroutine add_field_to_phybundle(var_name,long_name,units,cell_methods, axes,ph call ESMF_FieldBundleAdd(phys_bundle,(/field/), rc=rc) if( present(rcd)) rcd=rc ! - call ESMF_LogWrite('phys field add to fieldbundle'//trim(var_name), ESMF_LOGMSG_INFO, rc=rc) + call ESMF_LogWrite('phys field add to fieldbundle '//trim(var_name), ESMF_LOGMSG_INFO, rc=rc) end subroutine add_field_to_phybundle ! diff --git a/io/module_write_netcdf.F90 b/io/module_write_netcdf.F90 index dce3f980a..9ef6a3e7a 100644 --- a/io/module_write_netcdf.F90 +++ b/io/module_write_netcdf.F90 @@ -8,10 +8,6 @@ if (status /= nf90_noerr) write(0,*) "line ", __LINE__, trim(nf90_strerror(status)); \ if (status /= nf90_noerr) call ESMF_Finalize(endflag=ESMF_END_ABORT) -#define NF_ERR_STOP(status) \ - if (status /= nf_noerr) write(0,*) "line ", __LINE__, trim(nfmpi_strerror(status)); \ - if (status /= nf_noerr) call ESMF_Finalize(endflag=ESMF_END_ABORT) - module module_write_netcdf use esmf @@ -440,6 +436,17 @@ subroutine add_dim(ncid, dim_name, dimid, grid, rc) ncerr = nf90_put_var(ncid, dim_varid, values=valueListR8 ); NC_ERR_STOP(ncerr) ncerr = nf90_redef(ncid=ncid); NC_ERR_STOP(ncerr) deallocate(valueListR8) +! else if (typekind==ESMF_TYPEKIND_R4) then +! allocate(valueListR4(n)) +! call ESMF_AttributeGet(grid, convention="NetCDF", purpose="FV3", & +! name=trim(dim_name), valueList=valueListR4, rc=rc); ESMF_ERR_RETURN(rc) +! ncerr = nf90_enddef(ncid=ncid); NC_ERR_STOP(ncerr) +! ncerr = nf90_put_var(ncid, dim_varid, values=valueListR4 ); NC_ERR_STOP(ncerr) +! ncerr = nf90_redef(ncid=ncid); NC_ERR_STOP(ncerr) +! deallocate(valueListR4) +! else +! write(0,*)'Error in module_write_netcdf.F90(add_dim) unknown typekind for ',trim(dim_name) +! call ESMF_Finalize(endflag=ESMF_END_ABORT) end if call get_grid_attr(grid, dim_name, ncid, dim_varid, rc) diff --git a/io/module_wrt_grid_comp.F90 b/io/module_wrt_grid_comp.F90 index c18afab19..f316e7c7b 100644 --- a/io/module_wrt_grid_comp.F90 +++ b/io/module_wrt_grid_comp.F90 @@ -509,9 +509,10 @@ subroutine wrt_initialize(wrt_comp, imp_state_write, exp_state_write, clock, rc) name="output_file", value=outfile_name, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & - line=__LINE__, file=__FILE__)) return ! bail out - - CALL ESMF_LogWrite("bf fcstfield, get output_file"//trim(outfile_name)//trim(fieldName),ESMF_LOGMSG_INFO,rc=RC) + line=__LINE__, & + file=__FILE__)) & + return ! bail out + CALL ESMF_LogWrite("bf fcstfield, get output_file "//trim(outfile_name)//" "//trim(fieldName),ESMF_LOGMSG_INFO,rc=RC) if( trim(outfile_name) /= '') then outfilename(j,i) = trim(outfile_name) endif @@ -1184,7 +1185,7 @@ subroutine wrt_run(wrt_comp, imp_state_write, exp_state_write,clock,rc) write(cfhour, cform) nf_hours endif ! - if(mype == lead_write_task) print *,'in wrt run, 2, nf_hours=',nf_hours,nf_minutes,nseconds, & + if(mype == lead_write_task) print *,'in wrt run, nf_hours=',nf_hours,nf_minutes,nseconds, & 'nseconds_num=',nseconds_num,nseconds_den,' FBCount=',FBCount,' cfhour=',trim(cfhour) ! if(mype == lead_write_task) print *,'in wrt run, cfhour=',cfhour, & @@ -1485,6 +1486,10 @@ subroutine recover_fields(file_bundle,rc) ! get filed count call ESMF_FieldBundleGet(file_bundle, fieldCount=fieldCount, & grid=fieldGrid, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out ! CALL ESMF_LogWrite("call recover field on wrt comp",ESMF_LOGMSG_INFO,rc=RC) call ESMF_GridGet(fieldgrid, dimCount=gridDimCount, rc=rc) diff --git a/makefile b/makefile index 18542d9e2..c34e25cec 100644 --- a/makefile +++ b/makefile @@ -18,6 +18,7 @@ else endif FFLAGS += -I$(FMS_DIR) -I$(PHYSP)physics -Iipd -Icpl -Iio -Iatmos_cubed_sphere +CPPDEFS += -DESMF_VERSION_MAJOR=$(ESMF_VERSION_MAJOR) FV3_EXE = fv3.exe FV3CAP_LIB = libfv3cap.a diff --git a/module_fcst_grid_comp.F90 b/module_fcst_grid_comp.F90 index 482a03c9c..f935ba33c 100644 --- a/module_fcst_grid_comp.F90 +++ b/module_fcst_grid_comp.F90 @@ -79,7 +79,7 @@ module module_fcst_grid_comp type atmos_internalstate_type type(atmos_data_type) :: Atm - type (time_type) :: Time_atmos, Time_init, Time_end, & + type(time_type) :: Time_atmos, Time_init, Time_end, & Time_step_atmos, Time_step_ocean, & Time_restart, Time_step_restart integer :: num_atmos_calls, ret, intrm_rst @@ -160,7 +160,7 @@ subroutine fcst_initialize(fcst_comp, importState, exportState, clock, rc) ! integer :: tl, i, j integer,dimension(2,6) :: decomptile !define delayout for the 6 cubed-sphere tiles - integer,dimension(2,1) :: reg_decomptile !define delayout for the regional grid + integer,dimension(2) :: regdecomp !define delayout for the nest grid type(ESMF_FieldBundle) :: fieldbundle ! type(ESMF_Time) :: CurrTime, StartTime, StopTime @@ -196,6 +196,10 @@ subroutine fcst_initialize(fcst_comp, importState, exportState, clock, rc) integer :: n, fcstNpes integer, allocatable, dimension(:) :: isl, iel, jsl, jel integer, allocatable, dimension(:,:,:) :: deBlockList + + integer :: globalTileLayout(2) + integer :: nestRootPet, peListSize(1) + integer, allocatable :: petMap(:) ! !----------------------------------------------------------------------- !*********************************************************************** @@ -355,17 +359,12 @@ subroutine fcst_initialize(fcst_comp, importState, exportState, clock, rc) if (mype == 0) write(0,*)'be create fcst grid' if( quilting ) then - do tl=1,6 - decomptile(1,tl) = atm_int_state%Atm%layout(1) - decomptile(2,tl) = atm_int_state%Atm%layout(2) - enddo - gridfile="grid_spec.nc" ! default if (field_exist("INPUT/grid_spec.nc", "atm_mosaic_file")) then call read_data("INPUT/grid_spec.nc", "atm_mosaic_file", gridfile) endif - if( atm_int_state%Atm%regional) then + if( atm_int_state%Atm%regional ) then call atmosphere_control_data (isc, iec, jsc, jec, nlev) @@ -421,14 +420,96 @@ subroutine fcst_initialize(fcst_comp, importState, exportState, clock, rc) else ! not regional - fcstGrid = ESMF_GridCreateMosaic(filename="INPUT/"//trim(gridfile), & - regDecompPTile=decomptile,tileFilePath="INPUT/", & - staggerlocList=(/ESMF_STAGGERLOC_CENTER, ESMF_STAGGERLOC_CORNER/), & - name='fcst_grid', rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & - line=__LINE__, file=__FILE__)) return ! bail out + if ( .not. atm_int_state%Atm%nested ) then !! global only + + do tl=1,6 + decomptile(1,tl) = atm_int_state%Atm%layout(1) + decomptile(2,tl) = atm_int_state%Atm%layout(2) + enddo + + fcstGrid = ESMF_GridCreateMosaic(filename="INPUT/"//trim(gridfile), & + regDecompPTile=decomptile,tileFilePath="INPUT/", & + staggerlocList=(/ESMF_STAGGERLOC_CENTER, ESMF_STAGGERLOC_CORNER/), & + name='fcst_grid', rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + else !! nesting + +#if ESMF_VERSION_MAJOR >= 8 + if (mype==0) globalTileLayout = atm_int_state%Atm%layout + call ESMF_VMBroadcast(vm, bcstData=globalTileLayout, count=2, & + rootPet=0, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + nestRootPet = globalTileLayout(1) * globalTileLayout(2) * 6 + + if (mype==nestRootPet) then + if (nestRootPet /= atm_int_state%Atm%pelist(1)) then + write(0,*)'error in fcst_initialize: nestRootPet /= atm_int_state%Atm%pelist(1)' + write(0,*)'error in fcst_initialize: nestRootPet = ',nestRootPet + write(0,*)'error in fcst_initialize: atm_int_state%Atm%pelist(1) = ',atm_int_state%Atm%pelist(1) + ESMF_ERR_ABORT(100) + endif + endif + + ! nest rootPet shares peList with others + if (mype==nestRootPet) peListSize(1) = size(atm_int_state%Atm%pelist) + call ESMF_VMBroadcast(vm, bcstData=peListSize, count=1, & + rootPet=nestRootPet, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! nest rootPet shares layout with others + if (mype==nestRootPet) regDecomp = atm_int_state%Atm%layout + call ESMF_VMBroadcast(vm, bcstData=regDecomp, count=2, & + rootPet=nestRootPet, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! prepare petMap variable + allocate(petMap(peListSize(1))) + if (mype==nestRootPet) petMap = atm_int_state%Atm%pelist + ! do the actual broadcast of the petMap + call ESMF_VMBroadcast(vm, bcstData=petMap, count=peListSize(1), & + rootPet=nestRootPet, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! create the DELayout that maps DEs to the PETs in the petMap + delayout = ESMF_DELayoutCreate(petMap=petMap, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out + + ! create the nest Grid by reading it from file but use DELayout + fcstGrid = ESMF_GridCreate(filename='INPUT/grid.nest02.tile7.nc', & + fileformat=ESMF_FILEFORMAT_GRIDSPEC, regDecomp=regDecomp, & + delayout=delayout, isSphere=.false., indexflag=ESMF_INDEX_DELOCAL, & + rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out +#else + write(0,*)'nest quilting is supported only with ESMF 8' + call ESMF_Finalize(endflag=ESMF_END_ABORT) +#endif + endif - end if + endif ! !test to write out vtk file: if( cpl ) then @@ -529,12 +610,18 @@ subroutine fcst_initialize(fcst_comp, importState, exportState, clock, rc) ! for dyn name_FB1 = trim(name_FB)//'_bilinear' fieldbundle = ESMF_FieldBundleCreate(name=trim(name_FB1),rc=rc) - if (mype == 0) write(0,*)'af create fcst fieldbundle, name=',trim(name_FB),'rc=',rc + if(mype==0) print *,'af create fcst fieldbundle, name=',trim(name_FB),' rc=',rc if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & - line=__LINE__, file=__FILE__)) return ! bail out + line=__LINE__, & + file=__FILE__)) & + return ! bail out call fv_dyn_bundle_setup(atm_int_state%Atm%axes, & - fieldbundle, fcstgrid, quilting) + fieldbundle, fcstgrid, quilting, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & + line=__LINE__, & + file=__FILE__)) & + return ! bail out ! Add the field to the importState so parent can connect to it call ESMF_StateAdd(exportState, (/fieldbundle/), rc=rc) @@ -552,7 +639,7 @@ subroutine fcst_initialize(fcst_comp, importState, exportState, clock, rc) name_FB1 = trim(name_FB)//'_bilinear' endif fieldbundlephys(j) = ESMF_FieldBundleCreate(name=trim(name_FB1),rc=rc) - if (mype == 0) write(0,*)'af create fcst fieldbundle, name=',trim(name_FB1),'rc=',rc + if(mype==0) print *,'af create fcst fieldbundle, name=',trim(name_FB1),' rc=',rc if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, & line=__LINE__, file=__FILE__)) return ! bail out enddo @@ -567,6 +654,11 @@ subroutine fcst_initialize(fcst_comp, importState, exportState, clock, rc) line=__LINE__, file=__FILE__)) return ! bail out enddo + else + + write(0,*)' unknown name_FB ', trim(name_FB) + ESMF_ERR_ABORT(101) + endif ! enddo @@ -608,8 +700,6 @@ subroutine fcst_run_phase_1(fcst_comp, importState, exportState,clock,rc) ! !----------------------------------------------------------------------- !*** local variables -! - type(ESMF_FieldBundle) :: file_bundle ! integer :: i,j, mype, na, date(6) character(20) :: compname @@ -660,11 +750,9 @@ subroutine fcst_run_phase_1(fcst_comp, importState, exportState,clock,rc) ! ! IF(RC /= ESMF_SUCCESS) THEN ! if(mype==0) WRITE(0,*)"FAIL: fcst_RUN" -! ELSE - if (mype == 0) WRITE(0,*)"PASS: fcstRUN, na=",na +! ELSE + if(mype==0) WRITE(*,*)"PASS: fcstRUN phase 1, na = ",na, ' time is ', mpi_wtime()-tbeg1 ! ENDIF -! - if (mype == 0) write(0,*)'fcst_run_phase_1 time is ', mpi_wtime()-tbeg1 ! !----------------------------------------------------------------------- ! @@ -687,8 +775,6 @@ subroutine fcst_run_phase_2(fcst_comp, importState, exportState,clock,rc) ! !----------------------------------------------------------------------- !*** local variables -! - type(ESMF_FieldBundle) :: file_bundle ! integer :: i,j, mype, na, date(6) character(20) :: compname @@ -747,12 +833,10 @@ subroutine fcst_run_phase_2(fcst_comp, importState, exportState,clock,rc) !----------------------------------------------------------------------- ! ! IF(RC /= ESMF_SUCCESS) THEN -! if(mype==0) WRITE(0,*)"FAIL: fcst_RUN" -! ELSE - if (mype == 0) WRITE(0,*)"PASS: fcstRUN, na=",na +! if(mype==0) WRITE(0,*)"FAIL: fcst_RUN" +! ELSE + if(mype==0) WRITE(*,*)"PASS: fcstRUN phase 2, na = ",na, ' time is ', mpi_wtime()-tbeg1 ! ENDIF -! - if (mype == 0) write(0,*)'fcst_run_phase_2 time is ', mpi_wtime()-tbeg1 ! !----------------------------------------------------------------------- ! diff --git a/namphysics/NAM_layer/NAM_typedefs.F90 b/namphysics/NAM_layer/NAM_typedefs.F90 index f7bf19d2f..b0efb8c98 100644 --- a/namphysics/NAM_layer/NAM_typedefs.F90 +++ b/namphysics/NAM_layer/NAM_typedefs.F90 @@ -1601,8 +1601,8 @@ subroutine coupling_create (Coupling, IM, Model) Coupling%q2mi_cpl = clear_val Coupling%tsfci_cpl = clear_val Coupling%psurfi_cpl = clear_val -!! Coupling%oro_cpl = clear_val !< pointer to sfcprop%oro -!! Coupling%slmsk_cpl = clear_val !< pointer to sfcprop%slmsk + Coupling%oro_cpl = clear_val !< pointer to sfcprop%oro + Coupling%slmsk_cpl = clear_val !< pointer to sfcprop%slmsk endif !-- cellular automata diff --git a/stochastic_physics/function_indlsev b/stochastic_physics/function_indlsev old mode 100755 new mode 100644 diff --git a/stochastic_physics/function_indlsod b/stochastic_physics/function_indlsod old mode 100755 new mode 100644 diff --git a/stochastic_physics/getcon_lag_stochy.f b/stochastic_physics/getcon_lag_stochy.f old mode 100755 new mode 100644 diff --git a/stochastic_physics/num_parthds_stochy.f b/stochastic_physics/num_parthds_stochy.f old mode 100755 new mode 100644 diff --git a/stochastic_physics/setlats_a_stochy.f b/stochastic_physics/setlats_a_stochy.f old mode 100755 new mode 100644 diff --git a/stochastic_physics/setlats_lag_stochy.f b/stochastic_physics/setlats_lag_stochy.f old mode 100755 new mode 100644 diff --git a/stochastic_physics/stochy_layout_lag.f b/stochastic_physics/stochy_layout_lag.f old mode 100755 new mode 100644 diff --git a/stochastic_physics/sumfln_stochy.f b/stochastic_physics/sumfln_stochy.f old mode 100755 new mode 100644