Reduce the memory usage that is important for ne1024 simulation

cime_config/config_component.xml

@@ -1412,7 +1412,10 @@
     <default_value>idmap</default_value>
     <group>run_domain</group>
     <file>env_run.xml</file>
-    <desc>lnd2rof flux mapping file</desc>
+    <desc>lnd2rof flux (conservative) mapping file. (This mapping file is used for both consd and consf mappings; as discussed in
+    https://github.com/ESCOMP/CMEPS/issues/408, the results of these two mappings are the same within roundoff in CMEPS due to the
+    normalization that is done when applying the mapping weights. For simplicity, and because land mesh files generally don't have
+    a mask, this mapping file should be generated with '--norm_type dstarea'.)</desc>
   </entry>
 
   <entry id="ROF2LND_FMAPNAME">

cime_config/namelist_definition_drv.xml

@@ -2269,7 +2269,10 @@
     <category>mapping</category>
     <input_pathname>abs</input_pathname>
     <group>MED_attributes</group>
-    <desc>lnd to rof mapping, 'unset' or 'idmap' are normal possible values (mapping file given for mizuRoute grids)</desc>
+    <desc>lnd2rof flux (conservative) mapping file; 'unset' or 'idmap' are normal possible values. (This mapping file is used for both
+    consd and consf mappings; as discussed in https://github.com/ESCOMP/CMEPS/issues/408, the results of these two mappings are the
+    same within roundoff in CMEPS due to the normalization that is done when applying the mapping weights. For simplicity, and because
+    land mesh files generally don't have a mask, this mapping file should be generated with '--norm_type dstarea'.)</desc>
     <values>
       <value>$LND2ROF_FMAPNAME</value>
       <value samegrid_lnd_rof='true'>idmap</value>

mediator/esmFldsExchange_cesm_mod.F90

@@ -3320,7 +3320,7 @@ subroutine esmFldsExchange_cesm(gcomp, phase, rc)
        else
           if ( fldchk(is_local%wrap%FBImp(complnd, complnd), 'Flrl_rofsur'//trim(suffix), rc=rc) .and. &
                fldchk(is_local%wrap%FBExp(comprof)         , 'Flrl_rofsur'//trim(suffix), rc=rc)) then
-             call addmap_from(complnd, 'Flrl_rofsur'//trim(suffix), comprof, mapconsf, map_fracname_lnd2rof, 'unset')
+             call addmap_from(complnd, 'Flrl_rofsur'//trim(suffix), comprof, mapconsf, map_fracname_lnd2rof, lnd2rof_map)
              call addmrg_to(comprof, 'Flrl_rofsur'//trim(suffix), &
                   mrg_from=complnd, mrg_fld='Flrl_rofsur'//trim(suffix), &
                   mrg_type='copy_with_weights', mrg_fracname=mrg_fracname_lnd2rof)
@@ -3341,7 +3341,7 @@ subroutine esmFldsExchange_cesm(gcomp, phase, rc)
        else
           if ( fldchk(is_local%wrap%FBImp(complnd, complnd), 'Flrl_rofi'//trim(suffix), rc=rc) .and. &
                fldchk(is_local%wrap%FBExp(comprof)         , 'Flrl_rofi'//trim(suffix), rc=rc)) then
-             call addmap_from(complnd, 'Flrl_rofi'//trim(suffix), comprof, mapconsf, map_fracname_lnd2rof, 'unset')
+             call addmap_from(complnd, 'Flrl_rofi'//trim(suffix), comprof, mapconsf, map_fracname_lnd2rof, lnd2rof_map)
              call addmrg_to(comprof, 'Flrl_rofi'//trim(suffix), &
                   mrg_from=complnd, mrg_fld='Flrl_rofi'//trim(suffix), &
                   mrg_type='copy_with_weights', mrg_fracname=mrg_fracname_lnd2rof)
@@ -3362,7 +3362,7 @@ subroutine esmFldsExchange_cesm(gcomp, phase, rc)
        else
           if ( fldchk(is_local%wrap%FBImp(complnd, complnd), 'Flrl_rofgwl'//trim(suffix), rc=rc) .and. &
                fldchk(is_local%wrap%FBExp(comprof)         , 'Flrl_rofgwl'//trim(suffix), rc=rc)) then
-             call addmap_from(complnd, 'Flrl_rofgwl'//trim(suffix), comprof, mapconsf, map_fracname_lnd2rof, 'unset')
+             call addmap_from(complnd, 'Flrl_rofgwl'//trim(suffix), comprof, mapconsf, map_fracname_lnd2rof, lnd2rof_map)
              call addmrg_to(comprof, 'Flrl_rofgwl'//trim(suffix), &
                   mrg_from=complnd, mrg_fld='Flrl_rofgwl'//trim(suffix), &
                   mrg_type='copy_with_weights', mrg_fracname=mrg_fracname_lnd2rof)
@@ -3383,7 +3383,7 @@ subroutine esmFldsExchange_cesm(gcomp, phase, rc)
        else
           if ( fldchk(is_local%wrap%FBImp(complnd, complnd), 'Flrl_rofsub'//trim(suffix), rc=rc) .and. &
                fldchk(is_local%wrap%FBExp(comprof)         , 'Flrl_rofsub'//trim(suffix), rc=rc)) then
-             call addmap_from(complnd, 'Flrl_rofsub'//trim(suffix), comprof, mapconsf, map_fracname_lnd2rof, 'unset')
+             call addmap_from(complnd, 'Flrl_rofsub'//trim(suffix), comprof, mapconsf, map_fracname_lnd2rof, lnd2rof_map)
              call addmrg_to(comprof, 'Flrl_rofsub'//trim(suffix), &
                   mrg_from=complnd, mrg_fld='Flrl_rofsub'//trim(suffix), &
                   mrg_type='copy_with_weights', mrg_fracname=mrg_fracname_lnd2rof)
@@ -3404,7 +3404,7 @@ subroutine esmFldsExchange_cesm(gcomp, phase, rc)
        else
           if ( fldchk(is_local%wrap%FBImp(complnd, complnd), 'Flrl_irrig'//trim(suffix), rc=rc) .and. &
                fldchk(is_local%wrap%FBExp(comprof)         , 'Flrl_irrig'//trim(suffix), rc=rc)) then
-             call addmap_from(complnd, 'Flrl_irrig'//trim(suffix), comprof, mapconsf, map_fracname_lnd2rof, 'unset')
+             call addmap_from(complnd, 'Flrl_irrig'//trim(suffix), comprof, mapconsf, map_fracname_lnd2rof, lnd2rof_map)
              call addmrg_to(comprof, 'Flrl_irrig'//trim(suffix), &
                   mrg_from=complnd, mrg_fld='Flrl_irrig'//trim(suffix), &
                   mrg_type='copy_with_weights', mrg_fracname=mrg_fracname_lnd2rof)

mediator/med_fraction_mod.F90

@@ -169,7 +169,7 @@ subroutine med_fraction_init(gcomp, rc)
     ! Initialize FBFrac(:) field bundles
 
     use ESMF                  , only : ESMF_LogWrite, ESMF_LOGMSG_INFO
-    use ESMF                  , only : ESMF_SUCCESS
+    use ESMF                  , only : ESMF_SUCCESS, ESMF_FAILURE, ESMF_LogSetError
     use ESMF                  , only : ESMF_GridComp, ESMF_GridCompGet, ESMF_StateIsCreated
     use ESMF                  , only : ESMF_FieldBundle, ESMF_FieldBundleIsCreated, ESMF_FieldBundleDestroy
     use ESMF                  , only : ESMF_FieldBundleGet
@@ -181,6 +181,7 @@ subroutine med_fraction_init(gcomp, rc)
     use med_internalstate_mod , only : InternalState
     use med_map_mod           , only : med_map_routehandles_init, med_map_rh_is_created
     use med_methods_mod       , only : State_getNumFields => med_methods_State_getNumFields
+    use NUOPC                 , only : NUOPC_CompAttributeGet
     use perf_mod              , only : t_startf, t_stopf
 
     ! input/output variables
@@ -208,6 +209,8 @@ subroutine med_fraction_init(gcomp, rc)
     integer             :: n,n1,ns
     integer             :: maptype
     integer             :: fieldCount
+    logical             :: isPresent, isSet, lexist
+    character(len=CX)   :: lnd2rof_map                  ! lnd->rof mapping file (also used for the consd fraction map here)
     logical, save       :: first_call = .true.
     character(len=*),parameter :: subname=' (med_fraction_init)'
     !---------------------------------------
@@ -584,10 +587,37 @@ subroutine med_fraction_init(gcomp, rc)
        if (is_local%wrap%comp_present(complnd)) then
           maptype = mapconsd
           if (.not. med_map_RH_is_created(is_local%wrap%RH(complnd,comprof,:),maptype, rc=rc)) then
-             call med_map_routehandles_init( complnd, comprof, &
-                  FBSrc=is_local%wrap%FBImp(complnd,complnd), &
-                  FBDst=is_local%wrap%FBImp(complnd,comprof), &
-                  mapindex=maptype, RouteHandle=is_local%wrap%RH, rc=rc)
+             ! The lnd->rof fraction map may be the only grid-crossing conservative coupling
+             ! map in a configuration, and computing its weights online (ESMF_FieldRegridStore)
+             ! can be prohibitively memory-heavy at high resolution. If the 'lnd2rof_map'
+             ! attribute names an offline mapping file, read the weights from that file;
+             ! otherwise ('unset' or 'idmap') compute them online. Although this consd map
+             ! shares its mapping file with the consf lnd->rof flux mappings, the two are the
+             ! same within roundoff in CMEPS due to the normalization applied when the weights
+             ! are used (see https://github.com/ESCOMP/CMEPS/issues/408).
+             call NUOPC_CompAttributeGet(gcomp, name='lnd2rof_map', value=lnd2rof_map, &
+                  isPresent=isPresent, isSet=isSet, rc=rc)
+             if (ChkErr(rc,__LINE__,u_FILE_u)) return
+             if (.not. (isPresent .and. isSet)) lnd2rof_map = 'unset'
+             if (lnd2rof_map /= 'unset' .and. lnd2rof_map /= 'idmap') then
+                inquire(file=lnd2rof_map, exist=lexist)
+                if (.not. lexist) then
+                   call ESMF_LogSetError(ESMF_FAILURE, &
+                        msg=trim(subname)//': lnd2rof_map weight file not found: '//trim(lnd2rof_map), &
+                        line=__LINE__, file=u_FILE_u, rcToReturn=rc)
+                   return
+                end if
+                call med_map_routehandles_init( complnd, comprof, &
+                     FBSrc=is_local%wrap%FBImp(complnd,complnd), &
+                     FBDst=is_local%wrap%FBImp(complnd,comprof), &
+                     mapindex=maptype, RouteHandle=is_local%wrap%RH, &
+                     mapfile=trim(lnd2rof_map), rc=rc)
+             else
+                call med_map_routehandles_init( complnd, comprof, &
+                     FBSrc=is_local%wrap%FBImp(complnd,complnd), &
+                     FBDst=is_local%wrap%FBImp(complnd,comprof), &
+                     mapindex=maptype, RouteHandle=is_local%wrap%RH, rc=rc)
+             end if
              if (ChkErr(rc,__LINE__,u_FILE_u)) return
           end if
 

mediator/med_map_mod.F90

@@ -296,7 +296,7 @@ subroutine med_map_RouteHandles_initfrom_esmflds(gcomp, flds_scalar_name, llogun
   end subroutine med_map_RouteHandles_initfrom_esmflds
 
   !================================================================================
-  subroutine med_map_routehandles_initfrom_fieldbundle(n1, n2, FBsrc, FBdst, mapindex, RouteHandle, rc)
+  subroutine med_map_routehandles_initfrom_fieldbundle(n1, n2, FBsrc, FBdst, mapindex, RouteHandle, mapfile, rc)
 
     use ESMF            , only : ESMF_LogWrite, ESMF_LOGMSG_INFO, ESMF_SUCCESS, ESMF_LogFlush
     use ESMF            , only : ESMF_Field, ESMF_FieldBundle, ESMF_RouteHandle
@@ -313,6 +313,7 @@ subroutine med_map_routehandles_initfrom_fieldbundle(n1, n2, FBsrc, FBdst, mapin
     type(ESMF_FieldBundle) , intent(in)    :: fBdst
     integer                , intent(in)    :: mapindex
     type(ESMF_RouteHandle) , intent(inout) :: RouteHandle(:,:,:)
+    character(len=*), optional, intent(in) :: mapfile  ! read offline weights (ESMF_FieldSMMStore) instead of online regrid
     integer                , intent(out)   :: rc
 
     ! local variables
@@ -333,7 +334,12 @@ subroutine med_map_routehandles_initfrom_fieldbundle(n1, n2, FBsrc, FBdst, mapin
     call med_methods_FB_getFieldN(FBDst, 1, flddst, rc)
     if (chkerr(rc,__LINE__,u_FILE_u)) return
 
-    call med_map_routehandles_initfrom_field(n1, n2, fldsrc, flddst, mapindex, routehandle(n1,n2,:), rc=rc)
+    if (present(mapfile)) then
+       call med_map_routehandles_initfrom_field(n1, n2, fldsrc, flddst, mapindex, routehandle(n1,n2,:), &
+            mapfile=mapfile, rc=rc)
+    else
+       call med_map_routehandles_initfrom_field(n1, n2, fldsrc, flddst, mapindex, routehandle(n1,n2,:), rc=rc)
+    end if
     if (chkerr(rc,__LINE__,u_FILE_u)) return
 
     if (dbug_flag > 1) then

mediator/med_phases_aofluxes_mod.F90

@@ -582,7 +582,11 @@ subroutine med_aofluxes_init_ogrid(gcomp, aoflux_in, aoflux_out, rc)
     if (chkerr(rc,__LINE__,u_FILE_u)) return
     call ESMF_FieldGet(lfield, mesh=lmesh, rc=rc)
     if (chkerr(rc,__LINE__,u_FILE_u)) return
-    is_local%wrap%aoflux_mesh = ESMF_MeshCreate(lmesh, rc=rc)
+    ! Reuse the ocean field's mesh directly rather than building a duplicate full
+    ! ESMF mesh per rank (a meaningful memory savings at high resolution). lmesh
+    ! persists in FBArea(compocn) and aoflux_mesh is never destroyed, so sharing
+    ! the handle is safe (ogrid path).
+    is_local%wrap%aoflux_mesh = lmesh
     call ESMF_MeshGet(lmesh, coordSys=coordSys, rc=rc)
     if (chkerr(rc,__LINE__,u_FILE_u)) return
     if (coordSys /= ESMF_COORDSYS_CART) then