WIP: Interface allowing ELM to drive ATS

src/executables/CMakeLists.txt

@@ -40,7 +40,7 @@ include_directories(${ATS_SOURCE_DIR}/src/pks)
 include_directories(${ATS_SOURCE_DIR}/src/pks/mpc)
 include_directories(${ATS_SOURCE_DIR}/src/pks/energy)
 include_directories(${ATS_SOURCE_DIR}/src/pks/flow)
-include_directories(${ATS_SOURCE_DIR}/src/pks/deformation)
+include_directories(${ATS_SOURCE_DIR}/src/pks/deform)
 include_directories(${ATS_SOURCE_DIR}/src/pks/transport)
 include_directories(${ATS_SOURCE_DIR}/src/operators/upwinding)
 include_directories(${ATS_SOURCE_DIR}/src/operators/advection)
@@ -177,3 +177,10 @@ add_amanzi_executable(ats
   OUTPUT_NAME ats
   OUTPUT_DIRECTORY ${ATS_BINARY_DIR})
 
+
+# set this here for now
+# add option to bootstrap before completion
+set (BUILD_ELM_ATS_API true)
+if (BUILD_ELM_ATS_API)
+  add_subdirectory(elm_ats_api)
+endif()

src/executables/ats_driver.cc

@@ -49,9 +49,7 @@ namespace ATS {
 
 ATSDriver::ATSDriver(Teuchos::ParameterList& parameter_list,
                      Amanzi::Comm_ptr_type comm)
-    : Coordinator(parameter_list, comm) {}
-
-
+    : Coordinator() { Coordinator::coordinator_init(parameter_list, comm); }
 
 // -----------------------------------------------------------------------------
 // setup and initialize, then run until time >= duration_

src/executables/coordinator.cc

@@ -47,16 +47,18 @@ actual work.
 
 #include "coordinator.hh"
 
-#define DEBUG_MODE 0
-
 namespace ATS {
 
-Coordinator::Coordinator(Teuchos::ParameterList& parameter_list,
-                         Amanzi::Comm_ptr_type comm ) :
-    parameter_list_(Teuchos::rcp(new Teuchos::ParameterList(parameter_list))),
-    comm_(comm),
-    restart_(false)
+// this MUST be be called before using Coordinator
+void
+Coordinator::coordinator_init(Teuchos::ParameterList& parameter_list,
+                              Amanzi::Comm_ptr_type comm)
 {
+  // initialize plist and comm member variables from input args
+  parameter_list_ = Teuchos::rcp(new Teuchos::ParameterList(parameter_list));
+  comm_ = comm;
+  restart_ = false;
+
   // create and start the global timer
   timer_ = Teuchos::rcp(new Teuchos::Time("wallclock_monitor",true));
   setup_timer_ = Teuchos::TimeMonitor::getNewCounter("setup");
@@ -65,13 +67,6 @@ Coordinator::Coordinator(Teuchos::ParameterList& parameter_list,
   // create state.
   S_ = Teuchos::rcp(new Amanzi::State(parameter_list_->sublist("state")));
 
-  coordinator_init();
-
-  vo_ = Teuchos::rcp(new Amanzi::VerboseObject(comm, "Coordinator", *coordinator_list_));
-};
-
-void Coordinator::coordinator_init()
-{
   // create the geometric model and regions
   Teuchos::ParameterList reg_list = parameter_list_->sublist("regions");
   Teuchos::RCP<Amanzi::AmanziGeometry::GeometricModel> gm =
@@ -136,6 +131,9 @@ void Coordinator::coordinator_init()
     }
   }
 
+  // create verbose object
+  vo_ = Teuchos::rcp(new Amanzi::VerboseObject(comm_, "Coordinator", *coordinator_list_));
+
   // create the time step manager
   tsm_ = Teuchos::rcp(new Amanzi::TimeStepManager());
 }

src/executables/coordinator.hh

@@ -39,11 +39,11 @@ namespace ATS {
 class Coordinator {
 
 public:
-  Coordinator(Teuchos::ParameterList& parameter_list,
-              Amanzi::Comm_ptr_type comm);
+
+  Coordinator() {};
 
   // PK methods
-  virtual void setup();
+  void setup();
   virtual void initialize();
   virtual void finalize();
   void report_memory();
@@ -53,7 +53,8 @@ public:
   double get_dt(bool after_fail=false);
 
  protected:
-  void coordinator_init();
+  void coordinator_init(Teuchos::ParameterList& parameter_list,
+              Amanzi::Comm_ptr_type comm);
   void read_parameter_list();
 
   // PK container and factory

src/executables/elm_ats_api/CMakeLists.txt

@@ -0,0 +1,46 @@
+# -*- mode: cmake -*-
+#
+#  ELM_ATS_API
+#    library
+#
+# builds both dynamic and static libs for now
+# need to add build logic to make configurable
+
+# verify the compatibility of the C/Fortran and C++/Fortran compilers
+# Amanzi probably does this somewhere upstream, should check
+include(FortranCInterface)
+FortranCInterface_VERIFY(CXX)
+
+include_directories(${ATS_SOURCE_DIR}/src/executables)
+
+set(elm_ats_src_files
+  ${ATS_SOURCE_DIR}/src/executables/coordinator.cc
+  ${ATS_SOURCE_DIR}/src/executables/ats_mesh_factory.cc
+  elm_ats_driver.cc
+  elm_ats_api.cc
+  )
+
+set(elm_ats_inc_files
+  ${ATS_SOURCE_DIR}/src/executables/coordinator.hh
+  ${ATS_SOURCE_DIR}/src/executables/ats_mesh_factory.hh
+  elm_ats_driver.hh
+  elm_ats_api.h
+  )
+
+## build shared lib
+add_amanzi_library(elm_ats
+  SOURCE ${elm_ats_src_files}
+  HEADERS ${elm_ats_inc_files}
+  LINK_LIBS ${fates_link_libs} ${tpl_link_libs} ${ats_link_libs} ${amanzi_link_libs})
+if (APPLE AND BUILD_SHARED_LIBS)
+  set_target_properties(elm_ats PROPERTIES LINK_FLAGS "-Wl,-undefined,dynamic_lookup")
+endif()
+
+## build static lib
+add_amanzi_library(elm_ats_static
+  STATIC
+  SOURCE ${elm_ats_src_files}
+  HEADERS ${elm_ats_inc_files}
+  LINK_LIBS ${fates_link_libs} ${tpl_link_libs} ${ats_link_libs} ${amanzi_link_libs})
+
+add_subdirectory(test)

src/executables/elm_ats_api/elm_ats_api.cc

@@ -0,0 +1,55 @@
+
+#include "elm_ats_driver.hh"
+#include "elm_ats_api.h"
+
+#ifdef __cplusplus
+extern "C"{
+#endif
+
+// allocate, call constructor and cast ptr to opaque ELM_ATS_DRIVER
+ELM_ATS_DRIVER ats_create() {
+  return reinterpret_cast<ELM_ATS_DRIVER>(new ATS::ELM_ATSDriver());
+}
+// reinterpret as elm_ats_driver and delete (calls destructor)
+void ats_delete(ELM_ATS_DRIVER ats) {
+  auto ats_ptr = reinterpret_cast<ATS::ELM_ATSDriver*>(ats);
+  ats_ptr->finalize();
+  delete ats_ptr;
+}
+// call driver setup()
+void ats_setup(ELM_ATS_DRIVER ats, MPI_Fint *f_comm, const char *input_filename) {
+  return reinterpret_cast<ATS::ELM_ATSDriver*>(ats)->setup(f_comm, input_filename);
+}
+// call driver initialize()
+void ats_initialize(ELM_ATS_DRIVER ats){
+  return reinterpret_cast<ATS::ELM_ATSDriver*>(ats)->initialize();
+}
+// call driver advance()
+void ats_advance(ELM_ATS_DRIVER ats, double *dt) {
+  return reinterpret_cast<ATS::ELM_ATSDriver*>(ats)->advance(dt);
+}
+// call driver advance_test()
+void ats_advance_test(ELM_ATS_DRIVER ats) {
+  return reinterpret_cast<ATS::ELM_ATSDriver*>(ats)->advance_test();
+}
+// call driver set_sources()
+void ats_set_sources(ELM_ATS_DRIVER ats, double *soil_infiltration, double *soil_evaporation,
+  double *root_transpiration, int *ncols, int *ncells) {
+  return reinterpret_cast<ATS::ELM_ATSDriver*>(ats)
+  ->set_sources(soil_infiltration, soil_evaporation, root_transpiration, ncols, ncells);
+}
+// call driver get_waterstate()
+void ats_get_waterstate(ELM_ATS_DRIVER ats, double *surface_pressure, double *soil_pressure,
+  double *saturation, int *ncols, int *ncells) {
+  return reinterpret_cast<ATS::ELM_ATSDriver*>(ats)
+  ->get_waterstate(surface_pressure, soil_pressure, saturation, ncols, ncells);
+}
+// call driver get_mesh_info()
+void ats_get_mesh_info(ELM_ATS_DRIVER ats, int *ncols_local, int *ncols_global, int *ncells_per_col,
+  double *dz, double *depth, double *elev, double *surf_area_m2, double *lat, double *lon) {
+  return reinterpret_cast<ATS::ELM_ATSDriver*>(ats)
+  ->get_mesh_info(ncols_local, ncols_global,ncells_per_col, dz, depth, elev, surf_area_m2, lat, lon);
+}
+#ifdef __cplusplus
+}
+#endif

src/executables/elm_ats_api/elm_ats_api.h

@@ -0,0 +1,51 @@
+
+#ifndef ELM_ATS_API_HH_
+#define ELM_ATS_API_HH_
+
+#ifdef __cplusplus
+extern "C" {
+    // opaque pointer
+    // external caller only sees *ELM_ATS_DRIVER - similar to void*, but better type safety 
+    // ATS resolves ELM_ATS_DRIVER as real ELM_ATSDriver during linking
+    class ELM_ATSDriver;
+    typedef ELM_ATSDriver *ELM_ATS_DRIVER;
+#else
+    // calling code should not dereference the pointer to the ATS object
+    // pointer hidden behind typedef to discourage
+    typedef struct ELM_ATS_DRIVER *ELM_ATS_DRIVER;
+#endif
+
+// allocate, call constructor and cast ptr to opaque ELM_ATS_DRIVER
+ELM_ATS_DRIVER ats_create();
+// reinterpret as elm_ats_driver and delete (calls destructor)
+void ats_delete(ELM_ATS_DRIVER ats);
+// call driver setup()
+void ats_setup(ELM_ATS_DRIVER ats, MPI_Fint *f_comm, const char *input_filename);
+// call driver initialize()
+void ats_initialize(ELM_ATS_DRIVER ats);
+// call driver advance(dt)
+void ats_advance(ELM_ATS_DRIVER ats, double *dt);
+// call driver advance_test()
+void ats_advance_test(ELM_ATS_DRIVER ats);
+// call driver set_sources()
+// soil_infiltration & soil_evaporation are 1D arrays of length ncols
+// root_transpiration is a 1D array array of length (ncells)
+void ats_set_sources(ELM_ATS_DRIVER ats, double *soil_infiltration, double *soil_evaporation,
+  double *root_transpiration, int *ncols, int *ncells);
+// call driver get_waterstate()
+// surface_pressure is a 1D array of length ncols
+// soil_pressure & saturation are 1D arrays array of length (ncells)
+void ats_get_waterstate(ELM_ATS_DRIVER ats, double *surface_pressure, double *soil_pressure,
+  double *saturation, int *ncols, int *ncells);
+// call driver get_mesh_info()
+// ncols_local, ncols_global, and ncells_per_col are scalars
+// dz & depth are 1D arrays array of length (ncells) - these could likely only be ncells_per_col long
+// elev, surf_area_m2, lat, lon are 1D arrays of length ncols - lat lon necessary for every cell?
+void ats_get_mesh_info(ELM_ATS_DRIVER ats, int *ncols_local, int *ncols_global, int *ncells_per_col,
+  double *dz, double *depth, double *elev, double *surf_area_m2, double *lat, double *lon);
+#ifdef __cplusplus
+}
+#endif
+
+// include guard
+#endif