TRUNK COMMIT, ticket 28: output Gaussian grid nemsio files through wr…

…ite grid component

M       module_fcst_grid_comp.F90
M       atmos_cubed_sphere/tools/fv_nggps_diag.F90
M       atmos_model.F90
M       io/module_wrt_grid_comp.F90
M       io/FV3GFS_io.F90
M       io/module_fv3_io_def.F90
A  +    io/module_write_nemsio.F90
M       io/makefile
M       makefile
M       fv3_cap.F90

atmos_cubed_sphere/tools/fv_nggps_diag.F90

@@ -379,7 +379,7 @@ subroutine fv_dyn_bundle_setup(axes, dyn_bundle, fcst_grid, quilting )
    integer num_axes, id, axis_length, direction, edges
    integer num_attributes, num_field_dyn, axis_typ
    character(255) :: units, long_name, cart_name,axis_direct,edgesS
-   character(128) :: output_name
+   character(128) :: output_name, shydrostatic
    integer currdate(6)
    type(domain1d) :: Domain
    real,dimension(:),allocatable :: axis_data
@@ -407,8 +407,40 @@ subroutine fv_dyn_bundle_setup(axes, dyn_bundle, fcst_grid, quilting )
    num_axes = size(axes)
    allocate(all_axes(num_axes))
    all_axes(1:num_axes) = axes(1:num_axes)
+!   print *,'in fv_dyn bundle,num_axes=',num_axes
 !   if(mpp_pe()==mpp_root_pe())print *,'in fv_dyn bundle,num_axes=',num_axes, 'axes=',axes
-
+!
+!*** add global attributes in the field bundle:
+   call ESMF_AttributeAdd(dyn_bundle, convention="NetCDF", purpose="FV3", &
+     attrList=(/"hydrostatic"/), rc=rc)
+   if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, &
+     line=__LINE__, &
+     file=__FILE__)) &
+     return  ! bail out
+   if (hydrostatico ) then
+     shydrostatic = 'hydrostatic'
+   else
+     shydrostatic = 'non-hydrostatic'
+   endif
+   call ESMF_AttributeSet(dyn_bundle, convention="NetCDF", purpose="FV3", &
+     name="hydrostatic", value=trim(shydrostatic), rc=rc)
+   if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, &
+     line=__LINE__, &
+     file=__FILE__)) &
+     return  ! bail out
+!
+   call ESMF_AttributeAdd(dyn_bundle, convention="NetCDF", purpose="FV3", &
+     attrList=(/"ncnsto"/), rc=rc)
+   if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, &
+     line=__LINE__, &
+     file=__FILE__)) &
+     return  ! bail out
+   call ESMF_AttributeSet(dyn_bundle, convention="NetCDF", purpose="FV3", &
+     name="ncnsto", value=ncnsto, rc=rc)
+   if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, &
+     line=__LINE__, &
+     file=__FILE__)) &
+     return  ! bail out
 !
 !*** get axis names
    allocate(axis_name(num_axes))

atmos_model.F90

@@ -425,7 +425,7 @@ subroutine atmos_model_init (Atmos, Time_init, Time, Time_step)
    !--- update tracers in FV3 with any initialized during the physics/radiation init phase
 !rab   call atmosphere_tracer_postinit (IPD_Data, Atm_block)
 
-   call gfdl_diag_register (Time, IPD_Data(:)%Sfcprop, IPD_Data(:)%IntDiag, Atm_block, Atmos%axes, IPD_Control%nfxr)
+   call gfdl_diag_register (Time, IPD_Data(:)%Sfcprop, IPD_Data(:)%IntDiag, Atm_block, IPD_Control, Atmos%axes, IPD_Control%nfxr)
    call FV3GFS_restart_read (IPD_Data, IPD_Restart, Atm_block, IPD_Control, Atmos%domain)
 
    !--- set the initial diagnostic timestamp

fv3_cap.F90

@@ -28,7 +28,8 @@ module fv3gfs_cap_mod
                                 num_files, filename_base,               &
                                 wrttasks_per_group, n_group,            &
                                 lead_wrttask, last_wrttask,             &
-                                write_netcdfflag, output_grid, imo,jmo
+                                write_nemsiofile, output_grid,          &
+                                imo, jmo
 !
   use module_fcst_grid_comp,  only: fcstSS => SetServices
   use module_wrt_grid_comp,   only: wrtSS => SetServices
@@ -181,11 +182,12 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc)
 
     character(len=*),parameter  :: subname='(mom_cap:InitializeAdvertise)'
     integer nfmout, nfsout , nfmout_hf, nfsout_hf
-    real(kind=8) :: MPI_Wtime, timefs
+    real(kind=8) :: MPI_Wtime, timewri, timeis,timeie,timerhs, timerhe
 !
 !------------------------------------------------------------------------
 !
     rc = ESMF_SUCCESS
+    timeis = mpi_wtime()
 
     call ESMF_GridCompGet(gcomp,name=name,vm=vm,rc=rc)
     if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, &
@@ -199,7 +201,8 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc)
       line=__LINE__, &
       file=__FILE__)) &
       return  ! bail out
-!    print *,'in fv3_cap,initAdvertize,name=',trim(name),'petlist=',petlist,'rc=',rc
+!    print *,'in fv3_cap,initAdvertize,name=',trim(name),'mpi_comm=',mpi_comm_atm, &
+!       'petcount=',petcount,'mype=',mype
 
     clock_fv3=clock
 !
@@ -279,9 +282,13 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc)
       call ESMF_ConfigGetAttribute(config=CF,value=nfhout_hf,label ='nfhout_hf:',rc=rc)
       call ESMF_ConfigGetAttribute(config=CF,value=nsout,    label ='nsout:',rc=rc)
       call ESMF_ConfigGetAttribute(config=CF,value=output_grid, label ='output_grid:',rc=rc)
-      if(trim(output_grid) == 'gaussian grid') then
+      if(mype==0) print *,'af nems config,output_grid=',trim(output_grid)
+      write_nemsiofile=.false.
+      if(trim(output_grid) == 'gaussian_grid') then
         call ESMF_ConfigGetAttribute(config=CF,value=imo, label ='imo:',rc=rc)
         call ESMF_ConfigGetAttribute(config=CF,value=jmo, label ='jmo:',rc=rc)
+        call ESMF_ConfigGetAttribute(config=CF,value=write_nemsiofile, label ='write_nemsiofile:',rc=rc)
+      if(mype==0) print *,'af nems config,imo=',imo,'jmo=',jmo,'write_nemsiofile=', write_nemsiofile
       endif
       if(mype==0) print *,'af nems config,dt_atmos=',dt_atmos,'nfhmax=',nfhmax, &
         'nfhout=',nfhout,nfhmax_hf,nfhout_hf, nsout
@@ -455,6 +462,7 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc)
       if(mype==0) print *,'af allco wrtComp,write_groups=',write_groups
 !
       k = num_pes_fcst
+      timerhs = mpi_wtime()
       do i=1, write_groups
 
 ! prepare petList for wrtComp(i)
@@ -577,9 +585,12 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc)
 
           call ESMF_LogWrite('bf FieldBundleRegridStore', ESMF_LOGMSG_INFO, rc=rc)
 
+          timewri = mpi_wtime()
           call ESMF_FieldBundleRegridStore(fcstFB(j), wrtFB(j,i), &
              regridMethod=regridmethod, routehandle=routehandle(j,i), rc=rc)
 
+!          print *,'after regrid store, group i=',i,' fb=',j,' time=',mpi_wtime()-timewri
+
           call ESMF_LogWrite('af FieldBundleRegridStore', ESMF_LOGMSG_INFO, rc=rc)
           if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, &
             line=__LINE__, &
@@ -590,6 +601,7 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc)
 
 ! end write_groups
       enddo
+!      print *,'in fv3cap init, time wrtcrt/regrdst',mpi_wtime()-timerhs
       deallocate(petList)
 !
 !---------------------------------------------------------------------------------
@@ -658,6 +670,7 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc)
 !end quilting
     endif
 !
+!    print *,'in fv3_cap, init time=',mpi_wtime()-timeis
 !-----------------------------------------------------------------------
 !
   end subroutine InitializeAdvertise
@@ -711,7 +724,6 @@ subroutine ModelAdvance(gcomp, rc)
 
 !-----------------------------------------------------------------------------
 
-    timewri = mpi_wtime()
     rc = ESMF_SUCCESS
     if(profile_memory) call ESMF_VMLogMemInfo("Entering FV3 Model_ADVANCE: ")
 !    
@@ -782,6 +794,7 @@ subroutine ModelAdvance(gcomp, rc)
 !
 !*** for forecast tasks
 
+      timewri = mpi_wtime()
       call ESMF_LogWrite('Model Advance: before fcstcomp run ', ESMF_LOGMSG_INFO, rc=rc)
         if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, &
           line=__LINE__, &
@@ -846,21 +859,23 @@ subroutine ModelAdvance(gcomp, rc)
 
        output: IF(lalarm .or. na==1 ) then
 
+         timerhi = mpi_wtime()
          do i=1, FBCount
 !
 ! get fcst fieldbundle
 !
            call ESMF_FieldBundleRegrid(fcstFB(i), wrtFB(i,n_group),    &
               routehandle=routehandle(i, n_group), rc=rc)
+           timerh = mpi_wtime()
            if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, &
              line=__LINE__, &
              file=__FILE__)) &
              call ESMF_Finalize(endflag=ESMF_END_ABORT)
-!        if (mype == 0 .or. mype == lead_wrttask(n_group)) print *,'aft fieldbundleregrid, rc=',rc, &
-!        'na=',na
 !
 !end FBcount
           enddo
+        if (mype == 0 .or. mype == lead_wrttask(n_group)) print *,'aft fieldbundleregrid,na=',na,  &
+        ' time=', timerh- timerhi
 
 !      if(mype==0 .or. mype==lead_wrttask(1))  print *,'on wrt bf wrt run, na=',na
           call ESMF_LogWrite('Model Advance: before wrtcomp run ', ESMF_LOGMSG_INFO, rc=rc)
@@ -869,7 +884,9 @@ subroutine ModelAdvance(gcomp, rc)
             file=__FILE__)) &
             call ESMF_Finalize(endflag=ESMF_END_ABORT)
 
+          timerhi = mpi_wtime()
           call ESMF_GridCompRun(wrtComp(n_group), importState=wrtState(n_group), clock=clock_fv3,userRc=urc,rc=rc)
+          timerh = mpi_wtime()
           if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, &
             line=__LINE__, &
             file=__FILE__)) &
@@ -878,13 +895,19 @@ subroutine ModelAdvance(gcomp, rc)
             line=__LINE__, &
             file=__FILE__)) &
             call ESMF_Finalize(endflag=ESMF_END_ABORT)
+        if (mype == 0 .or. mype == lead_wrttask(n_group)) print *,'aft wrtgridcomp run,na=',na,  &
+         ' time=', timerh- timerhi
 
           call ESMF_LogWrite('Model Advance: after wrtcomp run ', ESMF_LOGMSG_INFO, rc=rc)
           if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, &
             line=__LINE__, &
             file=__FILE__)) &
             call ESMF_Finalize(endflag=ESMF_END_ABORT)
 
+        if (mype == 0 .or. mype == lead_wrttask(n_group)) print *,'fv3_cap,aft model advance,na=', &
+        na,' time=', mpi_wtime()- timewri
+
+
           if(n_group == write_groups) then
             n_group = 1
           else
@@ -896,6 +919,10 @@ subroutine ModelAdvance(gcomp, rc)
 ! end quilting
       endif
 
+!      if (mype == 0 .or. mype == 1536 .or. mype==2160) then
+!        print *,'fv3_cap,end integrate,na=',na,' time=',mpi_wtime()- timewri
+!      endif
+
 !*** end integreate loop
     enddo integrate
 

io/FV3GFS_io.F90

@@ -92,6 +92,7 @@ module FV3GFS_io_mod
    integer :: tot_diag_idx = 0
    integer :: total_outputlevel = 0
    integer :: isco,ieco,jsco,jeco
+   integer :: fhzero
    integer,dimension(:), allocatable :: nstt
    real(4), dimension(:,:,:), allocatable, target :: buffer_phys
    integer, parameter :: DIAG_SIZE = 250
@@ -1166,13 +1167,14 @@ end subroutine phys_restart_write
 !    13+NFXR - radiation
 !    76+pl_coeff - physics
 !-------------------------------------------------------------------------      
-  subroutine gfdl_diag_register(Time, Sfcprop, Gfs_diag, Atm_block, axes, NFXR)
+  subroutine gfdl_diag_register(Time, Sfcprop, Gfs_diag, Atm_block, Model, axes, NFXR)
     use physcons,  only: con_g
 !--- subroutine interface variable definitions
     type(time_type),           intent(in) :: Time
     type(Gfs_sfcprop_type),    intent(in) :: Sfcprop(:)
     type(GFS_diag_type),       intent(in) :: Gfs_diag(:)
     type (block_control_type), intent(in) :: Atm_block
+    type(IPD_control_type),    intent(in) :: Model
     integer, dimension(4),     intent(in) :: axes
     integer,                   intent(in) :: NFXR
 !--- local variables
@@ -1192,6 +1194,7 @@ subroutine gfdl_diag_register(Time, Sfcprop, Gfs_diag, Atm_block, axes, NFXR)
     ieco = Atm_block%iec
     jsco = Atm_block%jsc
     jeco = Atm_block%jec
+    fhzero = nint(Model%fhzero)
 
     Diag(:)%id = -99
     Diag(:)%axes = -99
@@ -3036,6 +3039,19 @@ subroutine fv_phys_bundle_setup(axes, phys_bundle, fcst_grid, quilting )
 !*** add attributes to the bundle such as subdomain limtis,
 !*** axes, output time, etc
 !------------------------------------------------------------
+!
+   call ESMF_AttributeAdd(phys_bundle, convention="NetCDF", purpose="FV3", &
+     attrList=(/"fhzero"/), rc=rc)
+   if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, &
+     line=__LINE__, &
+     file=__FILE__)) &
+     return  ! bail out
+   call ESMF_AttributeSet(phys_bundle, convention="NetCDF", purpose="FV3", &
+     name="fhzero", value=fhzero, rc=rc)
+   if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, &
+     line=__LINE__, &
+     file=__FILE__)) &
+     return  ! bail out
 !
 !*** add attributes (for phys, set axes to 2)
    num_axes = 2

io/makefile

@@ -28,6 +28,7 @@ SRCS_F   =
 
 SRCS_F90 = \
 		   ./FV3GFS_io.F90                     \
+		   ./module_write_nemsio.F90           \
 		   ./module_fv3_io_def.F90             \
 		   ./module_write_internal_state.F90   \
 		   ./module_wrt_grid_comp.F90
@@ -51,6 +52,8 @@ $(LIBRARY): $(OBJS)
 
 FV3GFS_io.o: FV3GFS_io.F90
 	$(FC) $(CPPDEFS) $(CPPFLAGS) $(FPPFLAGS) $(FFLAGS) $(OTHERFLAGS) $(OTHER_FFLAGS) $(ESMF_INC) -c FV3GFS_io.F90
+module_write_nemsio.o: module_write_nemsio.F90
+	$(FC) $(CPPDEFS) $(CPPFLAGS) $(FPPFLAGS) $(FFLAGS) $(OTHERFLAGS) $(OTHER_FFLAGS) $(ESMF_INC) $(NEMSIOINC) -c module_write_nemsio.F90
 module_write_internal_state.o: module_write_internal_state.F90
 	$(FC) $(CPPDEFS) $(CPPFLAGS) $(FPPFLAGS) $(FFLAGS) $(OTHERFLAGS) $(OTHER_FFLAGS) $(ESMF_INC) -c module_write_internal_state.F90
 module_wrt_grid_comp.o: module_wrt_grid_comp.F90

io/module_fv3_io_def.F90

@@ -12,7 +12,7 @@ module module_fv3_io_def
   integer           :: num_pes_fcst
   integer           :: wrttasks_per_group, write_groups
   integer           :: n_group
-  logical           :: write_netcdfflag
+  logical           :: write_nemsiofile
   integer           :: num_files
   character(255)    :: output_grid
   integer           :: imo,jmo