diff --git a/Makefile b/Makefile index 07ba5f9556..535b3d085a 100644 --- a/Makefile +++ b/Makefile @@ -684,6 +684,21 @@ intel: # BUILDTARGET Intel oneAPI Fortran, C, and C++ compiler suite CPPINCLUDES = FCINCLUDES = LIBS = +MPAS_PREFIX ?= $(CURDIR) +MPAS_LIBDIR ?= $(MPAS_PREFIX)/lib +MPAS_MODDIR ?= $(MPAS_PREFIX)/mod + +NUOPC ?= false +ifneq ($(filter on ON 1 TRUE,$(NUOPC)),) + override NUOPC := true +endif +ifeq ($(NUOPC), true) + MPAS_ESMF = external + override CPPFLAGS += -DMPAS_NO_ESMF_INIT + NUOPC_MESSAGE="MPAS was built with NUOPC cap libraries." +else + NUOPC_MESSAGE="MPAS was built without NUOPC cap libraries." +endif export MPAS_ESMF ?= embedded ifeq "$(MPAS_ESMF)" "external" @@ -1537,6 +1552,8 @@ SCOTCH_MESSAGE = "MPAS was NOT linked with the Scotch graph partitioning library endif mpas_main: $(MAIN_DEPS) + if [ ! -d $(MPAS_LIBDIR) ]; then mkdir $(MPAS_LIBDIR); fi + if [ ! -d $(MPAS_MODDIR) ]; then mkdir $(MPAS_MODDIR); fi cd src; $(MAKE) FC="$(FC)" \ CC="$(CC)" \ CXX="$(CXX)" \ @@ -1558,7 +1575,11 @@ mpas_main: $(MAIN_DEPS) AUTOCLEAN_DEPS="$(AUTOCLEAN_DEPS)" \ GEN_F90="$(GEN_F90)" \ NAMELIST_SUFFIX="$(NAMELIST_SUFFIX)" \ - EXE_NAME="$(EXE_NAME)" + EXE_NAME="$(EXE_NAME)" \ + MPAS_PREFIX="$(MPAS_PREFIX)" \ + MPAS_LIBDIR="$(MPAS_LIBDIR)" \ + MPAS_MODDIR="$(MPAS_MODDIR)" \ + NUOPC="$(NUOPC)" if [ -e src/$(EXE_NAME) ]; then mv src/$(EXE_NAME) .; fi ( cd src/core_$(CORE); $(MAKE) ROOT_DIR="$(PWD)" post_build ) @@ -1582,14 +1603,17 @@ endif @echo $(TIMER_MESSAGE) @echo $(IO_MESSAGE) @echo $(ESMF_MESSAGE) + @echo $(NUOPC_MESSAGE) @echo "*******************************************************************************" clean: - cd src; $(MAKE) clean RM="$(RM)" CORE="$(CORE)" AUTOCLEAN="$(AUTOCLEAN)" + cd src; $(MAKE) clean RM="$(RM)" CORE="$(CORE)" AUTOCLEAN="$(AUTOCLEAN)" NUOPC="$(NUOPC)" $(RM) $(EXE_NAME) $(RM) namelist.$(NAMELIST_SUFFIX).defaults $(RM) streams.$(NAMELIST_SUFFIX).defaults if [ -f .build_opts.framework ]; then $(RM) .build_opts.framework; fi if [ -f .build_opts.$(CORE) ]; then $(RM) .build_opts.$(CORE); fi + if [ -d $(MPAS_LIBDIR) ]; then $(RM) -r $(MPAS_LIBDIR); fi + if [ -d $(MPAS_MODDIR) ]; then $(RM) -r $(MPAS_MODDIR); fi core_error: @echo "" @@ -1641,6 +1665,7 @@ errmsg: @echo " MPAS_ESMF=opt - Selects the ESMF library to be used for MPAS. Options are:" @echo " MPAS_ESMF=embedded - Use the embedded ESMF timekeeping library (default)" @echo " MPAS_ESMF=external - Use an external ESMF library, determined by ESMFMKFILE" + @echo " NUOPC=true - builds NUOPC library (libmpas_nuopc.a) and installs to lib/ with mod files to mod/. Default is false." @echo "" @echo "Ensure that NETCDF, PNETCDF, PIO, and PAPI (if USE_PAPI=true) are environment variables" @echo "that point to the absolute paths for the libraries." diff --git a/src/Makefile b/src/Makefile index c06a592d95..b308016762 100644 --- a/src/Makefile +++ b/src/Makefile @@ -6,7 +6,12 @@ include Makefile.in.$(ESM) else -all: mpas +BUILD_ALL = mpas +ifeq "$(NUOPC)" "true" +BUILD_ALL = nuopc_lib +endif + +all: $(BUILD_ALL) mpas: $(AUTOCLEAN_DEPS) externals frame ops dycore drver $(LINKER) $(LDFLAGS) -o $(EXE_NAME) driver/*.o -L. -ldycore -lops -lframework $(LIBS) $(MPAS_ESMF_INC) $(MPAS_ESMF_LIB) @@ -35,8 +40,30 @@ dycore: $(AUTOCLEAN_DEPS) build_tools externals frame ops ( cd core_$(CORE); $(MAKE) CPPFLAGS="$(CPPFLAGS)" CPPINCLUDES="$(CPPINCLUDES)" all ) ln -sf core_$(CORE)/libdycore.a libdycore.a +nuopc_lib: $(AUTOCLEAN_DEPS) externals frame ops dycore drver + cp framework/*.mod $(MPAS_MODDIR)/. + ( cd core_$(CORE); $(MAKE) CPPFLAGS="$(CPPFLAGS)" CPPINCLUDES="$(CPPINCLUDES)" FFLAGS="$(FFLAGS)" FCINCLUDES="$(FCINCLUDES)" nuopc ) + ar rcs $(MPAS_LIBDIR)/libmpas_nuopc.a \ + core_$(CORE)/nuopc/*.o \ + core_$(CORE)/libphys/*.o \ + core_$(CORE)/*.o \ + core_$(CORE)/diagnostics/*.o \ + core_$(CORE)/libchem/*.o \ + core_$(CORE)/dynamics/*.o \ + driver/*.o \ + framework/*.o \ + operators/*.o \ + external/ezxml/*.o \ + external/SMIOL/*.o + ranlib $(MPAS_LIBDIR)/libmpas_nuopc.a + cp core_$(CORE)/nuopc/*.mod $(MPAS_MODDIR)/. -clean: clean_shared clean_core +CLEAN_ALL = clean_shared clean_core +ifeq "$(NUOPC)" "true" +CLEAN_ALL += clean_nuopc +endif + +clean: $(CLEAN_ALL) clean_core: ifeq "$(AUTOCLEAN)" "true" @@ -64,3 +91,12 @@ endif ( cd framework; $(MAKE) clean ) ( cd operators; $(MAKE) clean ) ( cd driver; $(MAKE) clean ) + +clean_nuopc: + $(info ) + $(info *********************************************************************************************) + $(info The NUOPC library will be cleaned.) + $(info *********************************************************************************************) + $(info ) + $(RM) $(MPAS_LIBDIR)/libmpas_nuopc.a + $(RM) $(MPAS_MODDIR)/*.mod diff --git a/src/core_atmosphere/Makefile b/src/core_atmosphere/Makefile index bb2bd2c2e7..fc95a4da8e 100644 --- a/src/core_atmosphere/Makefile +++ b/src/core_atmosphere/Makefile @@ -69,6 +69,9 @@ utilities: $(PHYSCORE) atmcore: $(PHYSCORE) dycore diagcore $(OBJS) ar -ru libdycore.a $(OBJS) dynamics/*.o $(PHYS_OBJS) $(CHEM_OBJS) diagnostics/*.o +nuopc: $(PHYSCORE) dycore diagcore $(OBJS) + ( cd nuopc; $(MAKE) all ) + mpas_atm_core_interface.o: mpas_atm_core.o mpas_atm_core.o: dycore diagcore mpas_atm_threading.o mpas_atm_halos.o @@ -81,6 +84,7 @@ clean: ( cd dynamics; $(MAKE) clean ) ( cd diagnostics; $(MAKE) clean ) ( cd utils; $(MAKE) clean ) + ( cd nuopc; $(MAKE) clean ) ( cd ../..; rm -f *TBL ) ( cd ../..; rm -f *DATA* ) $(RM) -r libphys diff --git a/src/core_atmosphere/nuopc/Makefile b/src/core_atmosphere/nuopc/Makefile new file mode 100644 index 0000000000..5bdb0547d5 --- /dev/null +++ b/src/core_atmosphere/nuopc/Makefile @@ -0,0 +1,16 @@ +.SUFFIXES: .F90 .o + +OBJS = mpas_nuopc_atm.o + +all: $(OBJS) + +clean: + $(RM) *.o *.mod + @# Certain systems with intel compilers generate *.i files + @# This removes them during the clean process + $(RM) *.i + +%.o: %.F90 + $(RM) $@ $*.mod + $(FC) $(CPPFLAGS) $(PHYSICS) $(FFLAGS) -c $< -o $@ $(CPPINCLUDES) $(FCINCLUDES) $(MPAS_ESMF_INC) + diff --git a/src/core_atmosphere/nuopc/mpas_nuopc_atm.F90 b/src/core_atmosphere/nuopc/mpas_nuopc_atm.F90 new file mode 100644 index 0000000000..494e079152 --- /dev/null +++ b/src/core_atmosphere/nuopc/mpas_nuopc_atm.F90 @@ -0,0 +1,229 @@ +#define ESMF_ERR_RETURN(rc) if (ESMF_LogFoundError( \ + rcToCheck=rc, \ + msg=ESMF_LOGERR_PASSTHRU, \ + line=__LINE__, \ + file=__FILE__) \ + ) return + +module mpas_nuopc_atm + !> MPAS NUOPC Cap for Atmosphere + + use ESMF + use NUOPC + use NUOPC_Model, & + modelSS => SetServices + + implicit none + + private + + public SetVM, SetServices + + !----------------------------------------------------------------------------- + contains + !----------------------------------------------------------------------------- + + subroutine SetServices(model, rc) + !> Register model entry points: + !> Advertise: advertise import and export fields + !> Realize: realize connected fields + !> SetClock: initialize model clock + !> DataInitialize: initialize data in import and export states + !> Advance: advance model by a single time step + !> Finalize: finalize model and cleanup memory + + ! arguments + type(ESMF_GridComp) :: model + integer, intent(out) :: rc + + rc = ESMF_SUCCESS + + ! derive from NUOPC_Model + call NUOPC_CompDerive(model, modelSS, rc=rc) + ESMF_ERR_RETURN(rc) + + ! specialize model entry points + call NUOPC_CompSpecialize(model, specLabel=label_Advertise, & + specRoutine=Advertise, rc=rc) + ESMF_ERR_RETURN(rc) + call NUOPC_CompSpecialize(model, specLabel=label_RealizeProvided, & + specRoutine=Realize, rc=rc) + ESMF_ERR_RETURN(rc) + call NUOPC_CompSpecialize(model, specLabel=label_SetClock, & + specRoutine=SetClock, rc=rc) + ESMF_ERR_RETURN(rc) + call NUOPC_CompSpecialize(model, specLabel=label_DataInitialize, & + specRoutine=DataInitialize, rc=rc) + ESMF_ERR_RETURN(rc) + call NUOPC_CompSpecialize(model, specLabel=label_Advance, & + specRoutine=Advance, rc=rc) + ESMF_ERR_RETURN(rc) + call NUOPC_CompSpecialize(model, specLabel=label_Finalize, & + specRoutine=Finalize, rc=rc) + ESMF_ERR_RETURN(rc) + + end subroutine SetServices + + !----------------------------------------------------------------------------- + + subroutine Advertise(model, rc) + !> Advertise available export fields and desired import fields + + ! arguments + type(ESMF_GridComp) :: model + integer, intent(out) :: rc + + ! local variables + type(ESMF_State) :: importState, exportState + + rc = ESMF_SUCCESS + + call NUOPC_ModelGet(model, importState=importState, & + exportState=exportState, rc=rc) + ESMF_ERR_RETURN(rc) + + call ESMF_LogWrite(logmsgFlag=ESMF_LOGMSG_ERROR, & + msg="MPAS NUOPC ATM - Advertise has not been implemented", & + line=__LINE__, & + file=__FILE__) + + end subroutine Advertise + + !----------------------------------------------------------------------------- + + subroutine Realize(model, rc) + !> Check field connections and realize connected fields + + ! arguments + type(ESMF_GridComp) :: model + integer, intent(out) :: rc + + ! local variables + type(ESMF_State) :: importState, exportState + + rc = ESMF_SUCCESS + + call NUOPC_ModelGet(model, importState=importState, & + exportState=exportState, rc=rc) + ESMF_ERR_RETURN(rc) + + call ESMF_LogWrite(logmsgFlag=ESMF_LOGMSG_ERROR, & + msg="MPAS NUOPC ATM - Realize has not been implemented", & + line=__LINE__, & + file=__FILE__) + + end subroutine Realize + + !----------------------------------------------------------------------------- + + subroutine SetClock(model, rc) + !> Adjust model clock and time step during initialization + + ! arguments + type(ESMF_GridComp) :: model + integer, intent(out) :: rc + + ! local variables + type(ESMF_Clock) :: clock + type(ESMF_State) :: importState, exportState + + rc = ESMF_SUCCESS + + call NUOPC_ModelGet(model, modelClock=clock, & + importState=importState, & + exportState=exportState, rc=rc) + ESMF_ERR_RETURN(rc) + + call ESMF_LogWrite(logmsgFlag=ESMF_LOGMSG_ERROR, & + msg="MPAS NUOPC ATM - SetClock has not been implemented", & + line=__LINE__, & + file=__FILE__) + + end subroutine SetClock + + !----------------------------------------------------------------------------- + + subroutine DataInitialize(model, rc) + !> Initialize data in import and export states + + ! arguments + type(ESMF_GridComp) :: model + integer, intent(out) :: rc + + ! local variables + type(ESMF_Clock) :: clock + type(ESMF_State) :: importState, exportState + + rc = ESMF_SUCCESS + + call NUOPC_ModelGet(model, modelClock=clock, & + importState=importState, & + exportState=exportState, rc=rc) + ESMF_ERR_RETURN(rc) + + call NUOPC_CompAttributeSet(model, & + name="InitializeDataComplete", value="true", rc=rc) + ESMF_ERR_RETURN(rc) + + call ESMF_LogWrite(logmsgFlag=ESMF_LOGMSG_ERROR, & + msg="MPAS NUOPC ATM - DataInitialize has not been implemented", & + line=__LINE__, & + file=__FILE__) + + end subroutine DataInitialize + + !----------------------------------------------------------------------------- + + subroutine Advance(model, rc) + !> Advance model by a single time step + + ! arguments + type(ESMF_GridComp) :: model + integer, intent(out) :: rc + + ! local variables + type(ESMF_Clock) :: clock + type(ESMF_State) :: importState, exportState + + rc = ESMF_SUCCESS + + call NUOPC_ModelGet(model, modelClock=clock, & + importState=importState, & + exportState=exportState, rc=rc) + ESMF_ERR_RETURN(rc) + + call ESMF_LogWrite(logmsgFlag=ESMF_LOGMSG_ERROR, & + msg="MPAS NUOPC ATM - Advance has not been implemented", & + line=__LINE__, & + file=__FILE__) + + end subroutine Advance + + !----------------------------------------------------------------------------- + + subroutine Finalize(model, rc) + !> Finalize model and cleanup memory allocations + + ! arguments + type(ESMF_GridComp) :: model + integer, intent(out) :: rc + + ! local variables + type(ESMF_State) :: importState, exportState + + rc = ESMF_SUCCESS + + call NUOPC_ModelGet(model, importState=importState, & + exportState=exportState, rc=rc) + ESMF_ERR_RETURN(rc) + + call ESMF_LogWrite(logmsgFlag=ESMF_LOGMSG_ERROR, & + msg="MPAS NUOPC ATM - Finalize has not been implemented", & + line=__LINE__, & + file=__FILE__) + + end subroutine Finalize + + !----------------------------------------------------------------------------- + +end module mpas_nuopc_atm diff --git a/src/core_init_atmosphere/Makefile b/src/core_init_atmosphere/Makefile index 3057a4cc07..28cff7e1aa 100644 --- a/src/core_init_atmosphere/Makefile +++ b/src/core_init_atmosphere/Makefile @@ -42,6 +42,13 @@ core_input_gen: ( cd default_inputs; $(NL_GEN) ../Registry_processed.xml namelist.init_atmosphere in_defaults=true ) ( cd default_inputs; $(ST_GEN) ../Registry_processed.xml streams.init_atmosphere stream_list.init_atmosphere. listed ) +nuopc: + @echo "************ ERROR ************" + @echo "NUOPC build does not exist for the init_atmosphere core. Quitting." + @echo "************ ERROR ************" + @echo "" + exit 1 + gen_includes: $(CPP) $(CPPFLAGS) $(CPPINCLUDES) Registry.xml > Registry_processed.xml (if [ ! -d inc ]; then mkdir -p inc; fi) # To generate *.inc files diff --git a/src/core_landice/Makefile b/src/core_landice/Makefile index 572112f291..daaa465f2b 100644 --- a/src/core_landice/Makefile +++ b/src/core_landice/Makefile @@ -28,6 +28,13 @@ core_input_gen: core_reg: $(CPP) $(CPPFLAGS) $(CPPINCLUDES) Registry.xml > Registry_processed.xml +nuopc: + @echo "************ ERROR ************" + @echo "NUOPC build does not exist for the landice core. Quitting." + @echo "************ ERROR ************" + @echo "" + exit 1 + gen_includes: $(CPP) $(CPPFLAGS) $(CPPINCLUDES) Registry.xml > Registry_processed.xml (if [ ! -d inc ]; then mkdir -p inc; fi) # To generate *.inc files diff --git a/src/core_ocean/Makefile b/src/core_ocean/Makefile index 6108b90395..503880d3ab 100644 --- a/src/core_ocean/Makefile +++ b/src/core_ocean/Makefile @@ -28,6 +28,13 @@ core_input_gen: (cd default_inputs; $(ST_GEN) ../Registry_processed.xml streams.ocean.analysis stream_list.ocean.analysis. mutable mode=analysis ) (cd default_inputs; $(ST_GEN) ../Registry_processed.xml streams.ocean.init stream_list.ocean.init. mutable mode=init ) +nuopc: + @echo "************ ERROR ************" + @echo "NUOPC build does not exist for the ocean core. Quitting." + @echo "************ ERROR ************" + @echo "" + exit 1 + gen_includes: $(CPP) $(CPPFLAGS) $(CPPINCLUDES) Registry.xml > Registry_processed.xml (if [ ! -d inc ]; then mkdir -p inc; fi) # To generate *.inc files diff --git a/src/core_seaice/Makefile b/src/core_seaice/Makefile index 798ad2a573..b210287742 100644 --- a/src/core_seaice/Makefile +++ b/src/core_seaice/Makefile @@ -16,6 +16,13 @@ core_input_gen: (cd default_inputs; $(NL_GEN) ../Registry_processed.xml namelist.seaice ) (cd default_inputs; $(ST_GEN) ../Registry_processed.xml streams.seaice stream_list.seaice. listed ) +nuopc: + @echo "************ ERROR ************" + @echo "NUOPC build does not exist for the seaice core. Quitting." + @echo "************ ERROR ************" + @echo "" + exit 1 + post_build: if [ ! -e $(ROOT_DIR)/default_inputs ]; then mkdir $(ROOT_DIR)/default_inputs; fi cp default_inputs/* $(ROOT_DIR)/default_inputs/. diff --git a/src/core_sw/Makefile b/src/core_sw/Makefile index 038ca62bfe..8be9850f5e 100644 --- a/src/core_sw/Makefile +++ b/src/core_sw/Makefile @@ -18,6 +18,13 @@ core_input_gen: (cd default_inputs; $(NL_GEN) ../Registry_processed.xml namelist.sw ) (cd default_inputs; $(ST_GEN) ../Registry_processed.xml streams.sw stream_list.sw. listed ) +nuopc: + @echo "************ ERROR ************" + @echo "NUOPC build does not exist for the sw core. Quitting." + @echo "************ ERROR ************" + @echo "" + exit 1 + gen_includes: $(CPP) $(CPPFLAGS) $(CPPINCLUDES) Registry.xml > Registry_processed.xml (if [ ! -d inc ]; then mkdir -p inc; fi) # To generate *.inc files diff --git a/src/core_test/Makefile b/src/core_test/Makefile index ab90084721..d89678378e 100644 --- a/src/core_test/Makefile +++ b/src/core_test/Makefile @@ -28,6 +28,13 @@ core_input_gen: (cd default_inputs; $(NL_GEN) ../Registry_processed.xml namelist.test ) (cd default_inputs; $(ST_GEN) ../Registry_processed.xml streams.test stream_list.test. listed ) +nuopc: + @echo "************ ERROR ************" + @echo "NUOPC build does not exist for the test core. Quitting." + @echo "************ ERROR ************" + @echo "" + exit 1 + gen_includes: $(CPP) $(CPPFLAGS) $(CPPINCLUDES) Registry.xml > Registry_processed.xml (if [ ! -d inc ]; then mkdir -p inc; fi) # To generate *.inc files