diff --git a/bindings/CXX11/cxx11/Engine.h b/bindings/CXX11/cxx11/Engine.h index 2044ed095d..99af75d22c 100644 --- a/bindings/CXX11/cxx11/Engine.h +++ b/bindings/CXX11/cxx11/Engine.h @@ -246,8 +246,8 @@ class Engine * @param datum user data to be populated, r-value or single data value * @param launch mode policy, optional for API consistency, internally * is always sync - * @exception std::invalid_argument if variable is invalid or nullptr - * &datum + * @exception std::invalid_argument for invalid variableName (variable + * doesn't exist in IO) or nullptr data */ template void Get(const std::string &variableName, T &datum, @@ -277,7 +277,7 @@ class Engine * Get data associated with a Variable from the Engine. * Overloaded version that accepts a std::vector without pre-allocation. * @param variableName find variable by name inside IO that created this - * Engine with Open + * Engine with Open or BeginStep (streaming mode) * @param dataV user data vector to be associated with a variable, it * doesn't need to be pre-allocated. Engine will resize. * @param launch mode policy @@ -289,7 +289,8 @@ class Engine * immediately. * Special case, only use if necessary. * - * @exception std::invalid_argument for invalid variable + * @exception std::invalid_argument for invalid variableName (variable + * doesn't exist in IO) */ template void Get(const std::string &variableName, std::vector &dataV, @@ -322,7 +323,8 @@ class Engine * variable's BlockInfo. Overloaded version * to get variable by name. * @note Preliminary, experimental API, may change soon. - * @param variable contains variable metadata information + * @param variableName find variable by name inside IO that created this + * Engine with Open or BeginStep (streaming mode) * @param info block info struct associated with block selection, * call will link with implementation's block info. * @param launch mode policy @@ -333,7 +335,8 @@ class Engine * immediately. * Special case, only use if necessary. * - * @exception std::invalid_argument for invalid variable or nullptr data + * @exception std::invalid_argument for invalid variableName (variable + * doesn't exist in IO) */ template void Get(const std::string &variableName, typename Variable::Info &info, diff --git a/bindings/CXX11/cxx11/Variable.h b/bindings/CXX11/cxx11/Variable.h index 3f000a2155..09c418ad91 100644 --- a/bindings/CXX11/cxx11/Variable.h +++ b/bindings/CXX11/cxx11/Variable.h @@ -234,17 +234,26 @@ class Variable */ T Max(const size_t step = adios2::DefaultSizeT) const; - /** Contains sub-block information for a particular Variable */ + /** Contains block information for a particular Variable */ struct Info { - adios2::Dims Start; ///< block start - adios2::Dims Count; ///< block count - IOType Min = IOType(); ///< block Min, if IsValue is false - IOType Max = IOType(); ///< block Max, if IsValue is false - IOType Value = IOType(); ///< block Value, if IsValue is true - bool IsValue = false; ///< true: value, false: array - size_t BlockID = 0; ///< block ID for block selections + /** block start */ + adios2::Dims Start; + /** block count */ + adios2::Dims Count; + /** block Min, if IsValue is false */ + IOType Min = IOType(); + /** block Max, if IsValue is false */ + IOType Max = IOType(); + /** block Value, if IsValue is true */ + IOType Value = IOType(); + /** true: value, false: array */ + bool IsValue = false; + /** blockID for Block Selection */ + size_t BlockID = 0; + /** block corresponding step */ size_t Step = 0; + /** reference to internal block data (used by inline Engine) */ const T *Data() const; // allow Engine to set m_Info @@ -268,27 +277,79 @@ class Variable class Span { public: + /** Span can only be created by an Engine */ Span() = delete; + /** Span can't be copied */ Span(const Span &) = delete; + /** Span can be moved */ Span(Span &&) = default; + /** + * Memory is not owned, using RAII for members + */ ~Span() = default; + /** Span can't be copied */ Span &operator=(const Span &) = delete; + /** Span can only be moved */ Span &operator=(Span &&) = default; + /** + * size of the span based on Variable block Count + * @return number of elements + */ size_t size() const noexcept; + + /** + * Pointer to span data, can be modified if new spans are added + * Follows rules of std::vector iterator invalidation. + * Call again to get an updated pointer. + * @return pointer to data + */ T *data() const noexcept; + /** + * Safe access operator that checks bounds and throws an exception + * @param position input offset from 0 = data() + * @return span element at input position + * @throws std::invalid_argument if out of bounds + */ T &at(const size_t position); + + /** + * Safe const access operator that checks bounds and throws an exception + * @param position input offset from 0 = data() + * @return span const (read-only) element at input position + * @throws std::invalid_argument if out of bounds + */ const T &at(const size_t position) const; + /** + * Access operator (unsafe without check overhead) + * @param position input offset from 0 = data() + * @return span element at input position + */ T &operator[](const size_t position); + + /** + * Access const operator (unsafe without check overhead) + * @param position input offset from 0 = data() + * @return span const (read-only) element at input position + */ const T &operator[](const size_t position) const; // engine allowed to set m_Span friend class Engine; + /** + * Custom iterator class from: + * https://gist.github.com/jeetsukumaran/307264#file-custom_iterator-cpp-L26 + */ ADIOS2_CLASS_iterator; + + /** + * Custom iterator class functions from: + * https://gist.github.com/jeetsukumaran/307264#file-custom_iterator-cpp-L26 + */ ADIOS2_iterators_functions(data(), size()); private: diff --git a/bindings/CXX11/cxx11/fstream/ADIOS2fstream.h b/bindings/CXX11/cxx11/fstream/ADIOS2fstream.h index 7c5f77b1ef..84898c6fae 100644 --- a/bindings/CXX11/cxx11/fstream/ADIOS2fstream.h +++ b/bindings/CXX11/cxx11/fstream/ADIOS2fstream.h @@ -234,7 +234,7 @@ class fstream */ template void write_attribute(const std::string &name, const T *data, - const size_t elements, + const size_t size, const std::string &variableName = "", const std::string separator = "/", const bool endStep = false); diff --git a/bindings/CXX11/cxx11/fstream/ADIOS2fstream.tcc b/bindings/CXX11/cxx11/fstream/ADIOS2fstream.tcc index bd84452c8a..5ac9a7c7a4 100644 --- a/bindings/CXX11/cxx11/fstream/ADIOS2fstream.tcc +++ b/bindings/CXX11/cxx11/fstream/ADIOS2fstream.tcc @@ -30,13 +30,13 @@ void fstream::write_attribute(const std::string &name, const T &value, template void fstream::write_attribute(const std::string &name, const T *data, - const size_t elements, + const size_t size, const std::string &variableName, const std::string separator, const bool endStep) { using IOType = typename TypeInfo::IOType; - m_Stream->WriteAttribute(name, reinterpret_cast(data), - elements, variableName, separator, endStep); + m_Stream->WriteAttribute(name, reinterpret_cast(data), size, + variableName, separator, endStep); } template diff --git a/bindings/Python/py11glue.cpp b/bindings/Python/py11glue.cpp index 1476166203..500b664c7a 100644 --- a/bindings/Python/py11glue.cpp +++ b/bindings/Python/py11glue.cpp @@ -743,25 +743,25 @@ PYBIND11_MODULE(adios2, m) pybind11::arg("separator") = "/", pybind11::arg("end_step") = false, R"md( - writes a self-describing single value array (numpy) variable + writes a self-describing single value array (numpy) variable - Parameters - name - attribute name + Parameters + name + attribute name - stringvalue: - attribute single string + stringvalue: + attribute single string - variablename: - if attribute is associated with a variable + variablename: + if attribute is associated with a variable - separator: - concatenation string between variablename and attribute - e.g. variablename + separator + name - var/units. Not used if variablename is empty - endstep - end current step, begin next step and flush - (default = false). + separator: + concatenation string between variablename and attribute + e.g. variablename + separator + name + var/units. Not used if variablename is empty + endstep + end current step, begin next step and flush + (default = false). )md") .def("write_attribute", @@ -774,25 +774,25 @@ PYBIND11_MODULE(adios2, m) pybind11::arg("separator") = "/", pybind11::arg("end_step") = false, R"md( - writes a self-describing single value array (numpy) variable + writes a self-describing single value array (numpy) variable - Parameters - name - attribute name + Parameters + name + attribute name - stringarray: - attribute string array + stringarray: + attribute string array - variablename: - if attribute is associated with a variable + variablename: + if attribute is associated with a variable - separator: - concatenation string between variablename and attribute - e.g. variablename + separator + name - var/units. Not used if variablename is empty - endstep - end current step, begin next step and flush - (default = false). + separator: + concatenation string between variablename and attribute + e.g. variablename + separator + name + var/units. Not used if variablename is empty + endstep + end current step, begin next step and flush + (default = false). )md") .def("read_string", @@ -923,22 +923,22 @@ PYBIND11_MODULE(adios2, m) pybind11::arg("name"), pybind11::arg("variable_name") = "", pybind11::arg("separator") = "/", R"md( - Reads a numpy based attribute + Reads a numpy based attribute - Parameters - name - attribute name - - variablename: - if attribute is associated with a variable - - separator: - concatenation string between variablename and attribute - e.g. variablename + separator + name - var/units. Not used if variablename is empty - Returns - array: numpy - resulting array attribute data + Parameters + name + attribute name + + variablename: + if attribute is associated with a variable + + separator: + concatenation string between variablename and attribute + e.g. variablename + separator + name + var/units. Not used if variablename is empty + Returns + array: numpy + resulting array attribute data )md") .def("read_attribute_string", @@ -949,22 +949,22 @@ PYBIND11_MODULE(adios2, m) pybind11::arg("name"), pybind11::arg("variable_name") = "", pybind11::arg("separator") = "/", R"md( - Read a string attribute + Read a string attribute - Parameters - name - attribute name - - variablename: - if attribute is associated with a variable - - separator: - concatenation string between variablename and attribute - e.g. variablename + separator + name - var/units. Not used if variablename is empty - Returns - list: - resulting string list attribute data + Parameters + name + attribute name + + variablename: + if attribute is associated with a variable + + separator: + concatenation string between variablename and attribute + e.g. variablename + separator + name + var/units. Not used if variablename is empty + Returns + list: + resulting string list attribute data )md") .def("end_step", &adios2::py11::File::EndStep, R"md( diff --git a/docs/user_guide/source/advice/advice.rst b/docs/user_guide/source/advice/advice.rst new file mode 100644 index 0000000000..ef11bc2052 --- /dev/null +++ b/docs/user_guide/source/advice/advice.rst @@ -0,0 +1,84 @@ +****** +Advice +****** + +This list is similar to the Advice sections for each chapter in `The C++ Programming Language, Fourth Edition by Bjarne Stroustrup `_ + +1. Run with debug mode as ON in the ``ADIOS`` constructor or ``adios2_init`` when developing with ADIOS 2 + +2. Use ``MPI_COMM_SELF`` to run MPI compiled versions of ADIOS 2 in "serial" mode + +3. Use a runtime configuration file in the ``ADIOS`` constructor or ``adios2_init`` when targetting multiple engines + +4. Check object validity when developing (similar to ``fstream``): + + - C++: `operator bool` + + .. code-block:: c++ + + if(var) engine.Put(var, data); + + - C: NULL pointer + + .. code-block:: c + + if(var) adios2_put(engine, var, data, adios2_mode_deferred); + + - Python: v2 ``__nonzero__`` v3 ``__bool__``. Note: do not check for None object + .. code-block:: python + + if(var) engine.Put(var, data); + + - Fortran: ``type%valid`` + + .. code-block:: fortran + + if( adios%valid .eqv. .true. ) then + adios2_declare_io(adios, io, "foo") + end if + + +5. C++11 and Python: use ``try-catch`` (``try-except`` in Python) blocks to handle exceptions from ADIOS 2 + +6. C++11: use fixed-width types (``int8_t``, ``uint32_t``) for portability + +7. Define your data structure: set of variables and attributes before developing. Data hierarchies/models can be built on top of ADIOS 2. + +8. Read the documentation for :ref:`Supported Engines` before targetting development for a particular engine + +9. MPI development: treat ``ADIOS`` constructor/destructor (``adios2_init``/``adios2_finalize``) and Engine ``Open`` and ``Close`` always as collective functions. For the most part, ADIOS 2 API functionality is local, but other Engine functions might follow other rules, :ref:`Supported Engines`. + +10. Use `Remove` functions carefully. They create dangling objects/pointers. + +11. Thread-safety: treat ADIOS 2 as NOT thread-safe. Either use a mutex or only handle I/O from a master thread. ADIOS 2 is about performance, adding I/O serial algorithm operations into a parallel execution block may reduce parallel portions from Amdahl's Law. + +12. Prefer the high-level Python and C++ APIs for simple tasks that do not require performance. The more familiar Write/Read overloads for File I/O return native data constructs (``std::vector`` and ``numpy arrays``) immediately for a requested selection. ``open`` only explores the metadata index. + +13. C++: prefer templates to ``void*`` to increase compile-time safety. Use ``IO::InquireVariableType("variableName")`` and ``adios2::GetType()`` to cast upfront to a ``Variable``. C++17 has ``std::any`` as an alternative to ``void*``. ADIOS 2 follows closely the STL model. + +14. Understand ``Put`` and ``Get`` memory contracts from :ref:`Engine` + +15. Prefer Put/Get ``Deferred`` mode, treat ``Sync`` as a special mode + +16. ``Put Span``: create all spans in a step before populating them. Spans follow the same iterator invalidation rules as ``std::vector``, so use ``span.data()`` to always keep the span pointer up-to-date + +17. Prefer populating data before calling ``Put`` in deferred mode, rather than manipulating it between ``Put`` and ``PerformPuts``, ``EndStep``, or ``Close`` + +18. Use ``BeginStep`` and ``EndStep`` to write code that is portable across all ADIOS 2 Engines: file and streaming. + +19. Always use ``Close`` for every call to ``Open``. + +20. C, Fortran: always call ``adios2_finalize`` for every call to ``adios2_init`` to avoid memory leaks. + +21. Reminder: C++, C, Python: Row-Major, while Fortran: Column-Major. ADIOS 2 will handle interoperability between ordering. Remember that :ref:`bpls` is always a Row-Major reader. + +22. Fortran API: use the type members (``var%valid``, ``var%name``, etc.) to get extra type information. + +23. Fortran C interoperability: Fortran bindings support the majority of applications using Fortran 90. We currently don't support the ``ISO_C_BINDING`` interoperability module in Fortran 2003. + +24. Always keep the ``IO`` object self-contained keeping its own set of ``Variables``, ``Attributes`` and ``Engines``. Do not combine Variables with multiple Engines or multiple modes, unless it's 100% guaranteed to be safe in your program avoiding Variable access conflicts. + +25. Developers: explore the testing infrastructure ``ADIOS2/testing`` in ADIOS 2 as a starting point for using ADIOS 2 in your own testing environment. + +26. Become a super-user of :ref:`bpls` to analyze datasets generated by ADIOS 2. + \ No newline at end of file diff --git a/docs/user_guide/source/api_full/api_full.rst b/docs/user_guide/source/api_full/api_full.rst index 863ed9065e..6d3294f819 100644 --- a/docs/user_guide/source/api_full/api_full.rst +++ b/docs/user_guide/source/api_full/api_full.rst @@ -4,7 +4,7 @@ Full Language Bindings APIs .. note:: - Product Application Developers targeting finer-level control for their IO tasks for optimization should use the current full APIs. If you want to use ADIOS2 in simple use cases (*e.g.* reading a file for analysis, interactive Python, or saving some data for a small project) please refer to the :ref:`Simple Language Bindings APIs` for a flat learning curve. + Product Application Developers targeting finer-level control for their IO tasks for optimization should use the current full APIs. If you want to use ADIOS2 in simple use cases (*e.g.* reading a file for analysis, interactive Python, or saving some data for a small project) please refer to the :ref:`High-Level Language Bindings APIs` for a flat learning curve. Currently ADIOS2 support bindings for the following languages and their minimum standards: diff --git a/docs/user_guide/source/api_full/fortran.rst b/docs/user_guide/source/api_full/fortran.rst index d9a4a85bc3..b6659dd3bf 100644 --- a/docs/user_guide/source/api_full/fortran.rst +++ b/docs/user_guide/source/api_full/fortran.rst @@ -593,7 +593,8 @@ ADIOS2 Fortran bindings handlers are mapped 1-to-1 to the ADIOS components descr ! engine handler type(adios2_engine), intent(in) :: engine -* :f90:`subroutine adios2_put` put variable metadata and data into adios2 for IO operations. Default is deferred mode, optional sync mode, see :ref:`Engine API Functions`. Variable and data types must match. +* :f90:`subroutine adios2_put` put variable metadata and data into adios2 for IO operations. Default is deferred mode, optional sync mode, see :ref:`Put: modes and memory contracts`. Variable and data types must match. + .. code-block:: fortran @@ -642,9 +643,8 @@ ADIOS2 Fortran bindings handlers are mapped 1-to-1 to the ADIOS components descr * :f90:`subroutine adios2_get` get variable data into adios2 for IO operations. -Default is deferred mode, optional sync mode, see :ref:`Engine API Functions`. -Variable and data types must match, variable can be obtained from ``adios2_inquire_variable``. -Data must be pre-allocated. +Default is deferred mode, optional sync mode, see :ref:`Get: modes and memory contracts`. Variable and data types must match, variable can be obtained from ``adios2_inquire_variable``. Data must be pre-allocated. + .. code-block:: fortran diff --git a/docs/user_guide/source/api_full/python.rst b/docs/user_guide/source/api_full/python.rst index bdf188eebe..0d81b40fe4 100644 --- a/docs/user_guide/source/api_full/python.rst +++ b/docs/user_guide/source/api_full/python.rst @@ -4,7 +4,7 @@ Python bindings .. note:: - Product Application Developers targeting finer-level control for their IO tasks for optimization should use the current full APIs. If you want to use ADIOS2 in simple use cases (*e.g.* reading a file for analysis, interactive Python, or saving some data for a small project) please refer to the :ref:`Simple Language Bindings APIs` for a flat learning curve. + Product Application Developers targeting finer-level control for their IO tasks for optimization should use the current full APIs. If you want to use ADIOS2 in simple use cases (*e.g.* reading a file for analysis, interactive Python, or saving some data for a small project) please refer to the :ref:`High-Level Language Bindings APIs` for a flat learning curve. The full Python APIs follow very closely the full C++11 API interface. Work in progress. diff --git a/docs/user_guide/source/api_simple/api_simple.rst b/docs/user_guide/source/api_high/api_high.rst similarity index 77% rename from docs/user_guide/source/api_simple/api_simple.rst rename to docs/user_guide/source/api_high/api_high.rst index f2d054454f..ab30fbb425 100644 --- a/docs/user_guide/source/api_simple/api_simple.rst +++ b/docs/user_guide/source/api_high/api_high.rst @@ -1,17 +1,17 @@ -############################# -Simple Language Bindings APIs -############################# +################################# +High-Level Language Bindings APIs +################################# -The current simple APIs are designed for simple and direct tasks in which performance is not a critical aspect. Unlike the :ref:`Full Language Bindings APIs` the simple APIs only require a single object/handler. Thus offering a nearly-flat learning curve for first-time users. +The current simple APIs are designed for simple and direct tasks in which performance is not a critical aspect. Unlike the :ref:`Full Language Bindings APIs` the simple APIs only require a single object/handler offering native resemblance to C++ (fstream) and Python (file I/O) native implementations. Thus, offering a nearly-flat learning curve for first-time users. Typical scenarios for using the simple APIs are: * Reading a file to perform data analysis with libraries (matplotlib, scipy, etc.) * Interactive: few calls make interactive usage easier. * Saving data to files is small or personal projects -* Online frameworks: *e.g.* Jupyter notebooks +* Online frameworks: *e.g.* Jupyter notebooks, see python-mpi examples running on `MyBinder `_ -The designed functionality syntax is closely related to the native language IO bindings for formatted text IO *e.g.* C++ ``fstream``, C ``FILE*``, Fortran and Python file IO. +The designed functionality syntax is closely related to the native language IO bindings for formatted text files *e.g.* C++ ``fstream`` ``getline``, and Python file IO. The main function calls are: ``open`` (or constructor in C++), ``write``, ``read`` and ``close`` (or destructor in C++). In addition, ADIOS2 borrows the corresponding language native syntax for advancing lines to advance the step in write mode, and for a "step-by-step" streaming basis in read mode. See each language section in this chapter for a write/read example. @@ -32,10 +32,6 @@ Currently ADIOS2 support bindings for the following languages and their minimum +----------+----------+-------------------------+-----------------+ | C++ | 11/newer | ``#include adios2.h`` | ``fstream`` | +----------+----------+-------------------------+-----------------+ -| C | 99 | ``#include adios2_c.h`` | stdio ``FILE*`` | -+----------+----------+-------------------------+-----------------+ -| Fortran | 90 | ``use adios2`` | Fortran/stdio | -+----------+----------+-------------------------+-----------------+ | Python | 2.7/3 | ``import adios2`` | Python IO | +----------+----------+-------------------------+-----------------+ | Matlab | | | | @@ -44,8 +40,6 @@ Currently ADIOS2 support bindings for the following languages and their minimum The following sections provide a summary of the API calls on each language and links to Write and Read examples to put it all together. .. include:: cxx11.rst -.. include:: c.rst -.. include:: fortran.rst .. include:: python.rst .. include:: matlab.rst diff --git a/docs/user_guide/source/api_simple/cxx11.rst b/docs/user_guide/source/api_high/cxx11.rst similarity index 93% rename from docs/user_guide/source/api_simple/cxx11.rst rename to docs/user_guide/source/api_high/cxx11.rst index 20178b221d..a8e55d92d7 100644 --- a/docs/user_guide/source/api_simple/cxx11.rst +++ b/docs/user_guide/source/api_high/cxx11.rst @@ -7,7 +7,8 @@ C++ simple bindings DO NOT call ``use namespace adios2`` in your C++ code. Use ``adios2::fstream`` directly to prevent conflicts with ``std::stream``. -Write example: +C++11 Write example +------------------- .. code-block:: c++ @@ -38,14 +39,15 @@ Write example: // T and P are std::vector oStream.write( "temperature", T.data(), shape, start, count ); // adios2::endl will advance the step after writing pressure - oStream.write( "pressure", P.data(), shape, start, count, adios2::endl ); + oStream.write( "pressure", P.data(), shape, start, count, adios2::end_step ); } // optional, but is good practice to indicate oStream shouldn't be used oStream.close(); -Read "stepping/streaming" example: +C++11 Read "step-by-step" example +--------------------------------- .. code-block:: c++ @@ -81,6 +83,7 @@ Read "stepping/streaming" example: } ``adios2::fstream`` API documentation +------------------------------------- .. doxygenclass:: adios2::fstream :project: CXX11 diff --git a/docs/user_guide/source/api_simple/matlab.rst b/docs/user_guide/source/api_high/matlab.rst similarity index 100% rename from docs/user_guide/source/api_simple/matlab.rst rename to docs/user_guide/source/api_high/matlab.rst diff --git a/docs/user_guide/source/api_simple/python.rst b/docs/user_guide/source/api_high/python.rst similarity index 60% rename from docs/user_guide/source/api_simple/python.rst rename to docs/user_guide/source/api_high/python.rst index e75706c3c3..4960e50bca 100644 --- a/docs/user_guide/source/api_simple/python.rst +++ b/docs/user_guide/source/api_high/python.rst @@ -1,11 +1,18 @@ -********************** -Python simple bindings -********************** +********************* +Python high-level API +********************* -Python simple bindings follow closely the :ref:`C++ simple bindings`. Just like the full APIs, the rely on numpy and (optionally) on ``mpi4py``. +Python simple bindings follow closely the :ref:`C++ simple bindings`. Just like the full APIs, they rely on numpy and (optionally) on ``mpi4py`` if ADIOS2 is compiled with MPI. -Write example -------------- +For online examples on MyBinder : + +- `Python-MPI Notebooks `_ + +- `Python-noMPI Notebooks `_ + + +Python Write example +-------------------- .. code-block:: python @@ -34,11 +41,11 @@ Write example # temperature and pressure are numpy arrays fw.write("temperature", temperature, shape, start, count) # advances to next step - fw.write("pressure", pressure, shape, start, count, endl=True) + fw.write("pressure", pressure, shape, start, count, end_step=True) -Read "stepping/streaming" example ---------------------------------- +Python Read "step-by-step" example +---------------------------------- .. code-block:: python @@ -59,21 +66,31 @@ Read "stepping/streaming" example # if only one rank is active pass MPI.COMM_SELF with adios2.open("cfd.bp", "r", MPI.COMM_SELF) as fh: - for fh_step in fh: + for fstep in fh: + # inspect variables in current step + step_vars = fstep.available_variables() + + # print variables information + for name, info in step_vars.items(): + print("variable_name: " + name) + for key, value in info.items(): + print("\t" + key + ": " + value) + print("\n") + + # track current step step = fh_step.currentstep() if( step == 0 ): size_in = fh_step.read("size") + # read variables return a numpy array with corresponding selection physical_time = fh_step.read("physical_time") temperature = fh_step.read("temperature", start, count) pressure = fh_step.read("pressure", start, count) .. caution:: - When reading in stepping mode with the for-in directive, as in the example above, use the step handler (``fh_step``) inside the loop rather than the global handler (``fh``) - - + When reading in stepping mode with the for-in directive, as in the example above, use the step handler (``fstep``) inside the loop rather than the global handler (``fh``) File class API diff --git a/docs/user_guide/source/api_simple/c.rst b/docs/user_guide/source/api_simple/c.rst deleted file mode 100644 index 8f2fc64b4b..0000000000 --- a/docs/user_guide/source/api_simple/c.rst +++ /dev/null @@ -1,114 +0,0 @@ -***************** -C simple bindings -***************** - -Write example: - -.. code-block:: c - - #include - ... - - int rank, size; - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - MPI_Comm_size(MPI_COMM_WORLD, &size); - - // Nx, Ny from application, size_t - size_t shape[2]; - shape[0] = Nx; - shape[1] = Ny * (size_t) size; - - size_t start[2]; - start[0] = 0; - start[1] = Ny * (size_t) rank; - - size_t count[2]; - start[0] = Nx; - start[1] = Ny; - - adios2_FILE* fh = adios2_fopen( "cfd.bp", "w", MPI_COMM_WORLD); - adios2_error errio; //optionally check error return type - // NSteps from aplication - for (size_t step = 0; step < NSteps; ++step) - { - if(rank == 0 && step == 0) // global variable - { - //single value syntax - errio = adios2_fwrite(fh, "varR32", adios2_type_int, &size, 0, - NULL, NULL, NULL, adios2_false); - } - - // physicalTime value - errio = adios2_fwrite(fh, "physicalTime", adios2_type_double, - &physical_time, - 0, NULL, NULL, NULL, adios2_false); - // temperature and pressure arrays T and P - errio = adios2_fwrite(fh, "temperature", adios2_type_float, - T, - 2, shape, start, count, adios2_false); - /// advance to next step by passing adios2_true - errio = adios2_fwrite(fh, "pressure", adios2_type_float, - P, - 2, shape, start, count, adios2_true); - } - - // mandatory, otherwise resource and memory leaks happen - adios2_fclose(fh); - -Read "stepping/streaming" example: - -.. code-block:: c++ - - #include - ... - - int rank, size; - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - MPI_Comm_size(MPI_COMM_WORLD, &size); - - // Selection Window from application, std::size_t - if( rank == 0) - { - size_t start[2]; - start[0] = 0; - start[1] = 0; - - size_t count[2]; - start[0] = selX; - start[1] = selY; - - // if only one rank is active use MPI_COMM_SELF - adios2_FILE* fh = adios2_fopen( "cfd.bp", "r", MPI_COMM_WORLD); - adios2_error errio; //optionally check error return type - - adios2_step* fsh; - - while( adios2_fgets(fsh, fh) != NULL ) - { - // read value signature - errio = adios2_fread(fsh, "physicalTime", adios2_type_double, - &physical_time, 0, NULL, NULL); - - // T and P must be pre-allocated - errio = adios2_fread(fsh, "temperature", adios2_type_float, - T, 2, start, count); - - errio = adios2_fread(fsh, "temperature", adios2_type_float, - P, 2, start, count); - - // use T and P for current step - } - // mandatory, otherwise resource and memory leaks happen - adios2_fclose(fh); - } - - - -``adios2_FILE*`` API documentation - -.. doxygenfile:: adios2_c_FILE.h - :project: C - :path: ../../bindings/C/c/ - - - \ No newline at end of file diff --git a/docs/user_guide/source/api_simple/fortran.rst b/docs/user_guide/source/api_simple/fortran.rst deleted file mode 100644 index ab90a0e4b3..0000000000 --- a/docs/user_guide/source/api_simple/fortran.rst +++ /dev/null @@ -1,112 +0,0 @@ -*********************** -Fortran simple bindings -*********************** - -The Fortran simple API bindings are based on the C ``adios2_FILE*`` interface and the native Fortran IO ``advance`` parameter for stepping at write and read. -They consists only on subroutines and a single ``type(adios2_file)`` handler for adios2 streams. - -On the read side, allocatable arrays must be passed without pre-allocation. ADIOS2 allocates the array memory based on the read selection dimensions and steps. - - -.. note:: - - ADIOS2 supports Fortran 90 and more recent standards only. - - -Write example: - -.. code-block:: fortran - - import adios2 - ... - - type(adios2_file) :: fh - integer(kind=8), dimension(2):: shape_dims, start_dims, count_dims - integer:: rank, size - ... - - call MPI_Comm_rank(MPI_COMM_WORLD, rank, ierr) - call MPI_Comm_size(MPI_COMM_WORLD, size, ierr) - - ! Nx, Ny from application, integer - shape_dims(1) = Nx - shape_dims(2) = Ny * size - - start_dims(1) = 0 - start_dims(2) = Ny * rank - - count_dims(1) = Nx - count_dims(2) = Ny - - call adios2_fopen(fh, 'cfd.bp', 'w', MPI_COMM_WORLD, ierr) - - ! NSteps from aplication - do i = 1, NSteps - ! write global value - if( rank == 0 .and. step == 1 ) then - call adios2_fwrite(fh, "mpi_size", adios2_type_integer4, size, ierr) - end if - - ! physicalTime value - call adios2_fwrite(fh, "physicalTime", adios2_type_dp, & - physical_time, ierr) - ! temperature and pressure arrays - call adios2_fwrite(fh, "temperature", adios2_type_float, & - temperature, shape, start, count, ierr) - ! advance to next step after writing pressure - call adios2_fwrite(fh, "pressure", adios2_type_float, & - pressure, shape, start, count, adios2_advance_yes, ierr) - end do - ! mandatory, otherwise resource and memory leaks happen - call adios2_fclose(fh, ierr); - -Read "stepping/streaming" example: - -.. code-block:: fortran - - import adios2 - ... - - type(adios2_file) :: fh - integer(kind=8), dimension(2):: start_dims, count_dims - integer:: rank, size - integer(kind=4):: size_in - real(kind=8):: physical_time - real, dimension(:,:), allocatable:: temperature, pressure - ... - - call MPI_Comm_rank(MPI_COMM_WORLD, rank, ierr) - call MPI_Comm_size(MPI_COMM_WORLD, size, ierr) - - ! Nx, Ny from application, integer - if( rank == 0 ) then - - start_dims(1) = 0 - start_dims(2) = 0 - - count_dims(1) = selX - count_dims(2) = selY - - ! if only one rank is active use MPI_COMM_SELF - call adios2_fopen(fh, 'cfd.bp', 'w', MPI_COMM_SELF, ierr) - - i = 0 - - do - - if( i == 0) then - call adios2_fread(fh, "mpi_size", size_in, ierr) - end if - - call adios2_fread(fh, "physical_time", physical_time, ierr) - call adios2_fread(fh, "temperature", temperature, & - start_dims, count_dims, ierr) - call adios2_fread(fh, "temperature", temperature, & - start_dims, count_dims, adios2_advance_yes, ierr) - ! exit do loop - if (ierr < 0) exit - i = i + 1 - end do - - ! mandatory, otherwise resource and memory leaks happen - call adios2_fclose(fh, ierr) diff --git a/docs/user_guide/source/components/components.rst b/docs/user_guide/source/components/components.rst index 4d6e59748f..5892a6f0c7 100644 --- a/docs/user_guide/source/components/components.rst +++ b/docs/user_guide/source/components/components.rst @@ -10,3 +10,4 @@ ADIOS2 Interface Components .. include:: engine.rst .. include:: operator.rst .. include:: runtime.rst +.. include:: advice.rst diff --git a/docs/user_guide/source/components/engine.rst b/docs/user_guide/source/components/engine.rst index d6ec12bc28..630751e4b0 100644 --- a/docs/user_guide/source/components/engine.rst +++ b/docs/user_guide/source/components/engine.rst @@ -4,43 +4,385 @@ Engine The Engine abstraction component serves as the base interface to the actual IO Systems executing the heavy-load tasks performed when Producing and Consuming data. -Engine functionality works around two concepts from the application point-of-view: +Engine functionality works around two concepts from the application's point-of-view: -1. Self-describing variables are published and consumed in "steps" in either "File" mode (all steps are available) or "Streaming" mode (steps are available are they are produced). -2. Self-describing variables are published and consumed using a "sync" or "deferred" (lazy evaluation) policy. +1. Self-describing variables are published and consumed in "steps" in either "File" random-access (all steps are available) or "Streaming" (steps are available as they are produced in a step-by-step fashion). +2. Self-describing variables are published (Put) and consumed (Get) using a "sync" or "deferred" (lazy evaluation) policy. .. caution:: - The ADIOS2 "step" is a logical abstraction that means different things depending on the application context. Examples: "time step", "iteration step", "inner loop step", or "interpolation step", "variable section", etc. + The ADIOS 2 "step" is a logical abstraction that means different things depending on the application context. Examples: "time step", "iteration step", "inner loop step", or "interpolation step", "variable section", etc. It only indicates how the variables were passed into ADIOS 2 (e.g. I/O steps) without the user having to index this information on their own. +.. tip:: + + Publishing/Consuming data can be seen as a round-trip in ADIOS 2. Put and Get APIs for write/append and read modes aim to be "symmetric". Hence, reusing similar functions, objects, semantics as much as possible. -Engine API Functions --------------------- - -Recall that Engines are created through the ``IO::Open`` function which must contain a Mode (``Write``, ``Read``, ``Append``). Therefore, the current functionalty of all Engines are provided through the basic API functions: +The rest of the section explains the important concepts -For Publishing data (Write, Append mode) +BeginStep +--------- + + Begin logical step and return the status (via an enum) of the stream to be read/written. In streaming engines BeginStep in where the receiver tries to acquire a new step in the reading process. The full signature allows for a mode and timeout parameters. See :ref:`Supported Engines` for more information on what engine allows. A simplified signature allows each engine to pick reasonable defaults. + +.. code-block:: c++ -* ``Put`` - **Default mode: deferred (lazy evaluation).** Data pointer (or array) to memory must not be reused until first encounter with ``PerformPuts``, ``EndStep`` or ``Close``. Use sync mode to allow the pointer memory to be reusable immediately. This is enabled by passing the flag ``adios2::Mode::Sync`` as the 3rd argument. + // Full signature + StepStatus BeginStep(const StepMode mode, + const float timeoutSeconds = -1.f); + + // Simplified signature + StepStatus BeginStep(); -* ``PerformsPuts`` - Executes all pending Put calls in deferred mode until this line. +EndStep +------- + + Ends logical step, flush to transports depending on IO parameters and engine default behavior. -For Consuming data (Read mode) +.. tip:: + + To write portable code for a step-by-step access across adios2 engines (file and streaming engines) use BeginStep and EndStep. -* ``Get`` - **Default mode: deferred (lazy evaluation).** Data pointer (or array) to memory must not be reused until first encounter with ``PerformPuts``, ``EndStep`` or ``Close``. Use sync mode to populate the data pointer memory immediately. This is enabled by passing the flag ``adios2::Mode::Sync`` as the 3rd argument. +.. danger:: + + Accessing random steps in read mode (e.g. Variable::SetStepSelection in file engines) will create a conflict with BeginStep and EndStep and will throw an exception. In file engines, data is either consumed in a random-access or step-by-step mode, but not both. -* ``PerformsGets`` - Executes all pending deferred Get calls in deferred mode until this line. -Common Functionality (Write, Read, Append modes) +Close +----- - * ``BeginStep`` Begin logical step and return status of stream to be read/written. - * ``EndStep`` End logical step, flush to transports depending on IO parameters and engine default behavior. - * ``Close`` Close current engine and underlying transports. Engine object can't be used after this. + Close current engine and underlying transports. Engine object can't be used after this call. + +.. tip:: + + C++11: despite the fact that we use RAII, always use Close for explicit resource management and guarantee that the Engine data transport operations are concluded. + + +Put: modes and memory contracts +------------------------------- + +``Put`` is the generalized abstract function for publishing data in adios2 when an Engine is created using Write, or Append, mode at ``IO::Open``. Optionally, adios2 Engines can provide direct access to its buffer memory using an overload that returns a span to a variable block (based on a subset of the upcoming `C++20 std::span `_, non-owning contiguous memory piece). Spans act as a 1D memory container meant to be filled out by the application. See :ref:`Supported Engines` for engines that support the span feature (e.g. BP3). + +The following are the current Put signatures: + +1. Deferred (default) or Sync mode, data is contiguous memory + + .. code-block:: c++ + + Put(Variable variable, const T* data, const adios2::Mode = adios2::Mode::Deferred); + +2. Return a span setting a default T() value into a default buffer + + .. code-block:: c++ + + Variable::Span Put(Variable variable); + +3. Return a span setting a constant fill value into a certain buffer + + .. code-block:: c++ + + Variable::Span Put(Variable variable, const size_t bufferID, const T fillValue); + + +The following table summarizes the memory contracts required by adios2 engines between Put signatures and the data memory coming from an application: + ++----------+-------------+----------------------------------------------------+ +| Put | Data Memory | Contract | ++----------+-------------+----------------------------------------------------+ +| | Pointer | do not modify until PerformPuts/EndStep/Close | +| Deferred | | | +| | Contents | consumed at PerformPuts/EndStep/Close | ++----------+-------------+----------------------------------------------------+ +| | Pointer | modify after Put | +| Sync | | | +| | Contents | consumed at Put | ++----------+-------------+----------------------------------------------------+ +| | Pointer | modified by new Spans, updated span iterators/data | +| Span | | | +| | Contents | consumed at PerformPuts/EndStep/Close | ++----------+-------------+----------------------------------------------------+ + + +.. note:: + + In Fortran (array) and Python (numpy array) avoid operations that modify the internal structure of an array (size) to preserve the address. + + +Each Engine will give a concrete meaning to each functions signatures, but all of them must follow the same memory contracts to the "data pointer": the memory address itself, and the "data contents": memory bits (values). + +1. **Put in Deferred or lazy evaluation mode (default)**: this is the preferred mode as it allows Put calls to be "grouped" before potential data transport at the first encounter of ``PerformPuts``, ``EndStep`` or ``Close``. + + .. code-block:: c++ + + Put(variable, data); + Put(variable, data, adios2::Mode::Deferred); + + + Deferred memory contracts: + + - "data pointer" do not modify (e.g. resize) until first call to ``PerformPuts``, ``EndStep`` or ``Close``. + + - "data contents" consumed at first call to ``PerformPuts``, ``EndStep`` or ``Close``. It's recommended practice to set all data contents before Put. + + + Usage: + + .. code-block:: c++ + + // recommended use: + // set "data pointer" and "data contents" + // before Put + data[0] = 10; + + // Puts data pointer into adios2 engine + // associated with current variable metadata + engine.Put(variable, data); + + // valid but not recommended + // risk of changing "data pointer" (e.g. resize) + data[1] = 10; + + // "data contents" must be ready + // "data pointer" must be the same as in Put + engine.EndStep(); + //engine.PerformPuts(); + //engine.Close(); + + // now data pointer can be reused or modified + + .. tip:: + + It's recommended practice to set all data contents before Put in deferred mode to minimize the risk of modifying the data pointer (not just the contents) before PerformPuts/EndStep/Close. + + +2. **Put in Sync mode**: this is the special case, data pointer becomes reusable right after Put. Only use it if absolutely necessary (*e.g.* memory bound application or out of scope data, temporary). + + .. code-block:: c++ + + Put(variable, *data, adios2::Mode::Sync); + + + Sync memory contracts: + + - "data pointer" and "data contents" can be modified after this call. + + + Usage: + + .. code-block:: c++ + + // set "data pointer" and "data contents" + // before Put in Sync mode + data[0] = 10; + + // Puts data pointer into adios2 engine + // associated with current variable metadata + engine.Put(variable, data, adios2::Mode::Sync); + + // data pointer and contents can be reused + // in application + + +3. **Put returning a Span**: signature that allows access to adios2 internal buffer. + + Use cases: + + - population from non-contiguous memory structures + - memory-bound applications + + + Limitations: + + - does not allow operations (compression) + - must keep engine and variables within scope of span usage + + + + Span memory contracts: + + - "data pointer" provided by the engine and returned by span.data(), might change with the generation of a new span. It follows iterator invalidation rules from std::vector. Use `span.data()` or iterators, `span.begin()`, `span.end()` to keep an updated data pointer. + + - span "data contents" are published at the first call to ``PerformPuts``, ``EndStep`` or ``Close`` + + + Usage: + + .. code-block:: c++ + + // return a span into a block of memory + // set memory to default T() + adios2::Variable::Span span1 = Put(var1); + + // just like with std::vector::data() + // iterator invalidation rules + // dataPtr might become invalid + // always use span1.data() directly + T* dataPtr = span1.data(); + + // set memory value to -1 in buffer 0 + adios2::Variable::Span span2 = Put(var2, 0, -1); + + // not returning a span just sets a constant value + Put(var3); + Put(var4, 0, 2); + + // fill span1 + span1[0] = 0; + span1[1] = 1; + span1[2] = 2; + + // fill span2 + span2[1] = 1; + span2[2] = 2; + + // here collect all spans + // they become invalid + engine.EndStep(); + //engine.PerformPuts(); + //engine.Close(); + + // var1 = { 0, 1, 2 }; + // var2 = { -1., 1., 2.}; + // var3 = { 0, 0, 0}; + // var4 = { 2, 2, 2}; + + +PerformsPuts +------------ + + Executes all pending Put calls in deferred mode ad collect spans data + + +Get: modes and memory contracts +------------------------------- + +``Get`` is the generalized abstract function for consuming data in adios2 when an Engine is created using Read mode at ``IO::Open``. ADIOS 2 Put and Get APIs semantics are as symmetric as possible considering that they are opposite operations (*e.g.* Put passes ``const T*``, while Get populates a non-const ``T*``). + +.. + Optionally, adios2 Engines can provide direct access to its buffer memory using an overload that returns a span to a variable block (base on a subset of the upcoming `C++20 std::span `_, non-owning contiguous memory piece). Spans act as a 1D memory container meant to be filled out by the application. See :ref:`Supported Engines` for engines that support the span feature (e.g. BP3). + +The following are the current Get signatures: + +1. Deferred (default) or Sync mode, data is contiguous pre-allocated memory + + .. code-block:: c++ + + Get(Variable variable, const T* data, const adios2::Mode = adios2::Mode::Deferred); + + +2. C++11 only, dataV is automatically resized by adios2 based on Variable selection + + .. code-block:: c++ + + Get(Variable variable, std::vector& dataV, const adios2::Mode = adios2::Mode::Deferred); + + +The following table summarizes the memory contracts required by adios2 engines between Get signatures and the pre-allocated (except when using C++11 ``std::vector``) data memory coming from an application: + ++----------+-------------+-----------------------------------------------+ +| Get | Data Memory | Contract | ++----------+-------------+-----------------------------------------------+ +| | Pointer | do not modify until PerformPuts/EndStep/Close | +| Deferred | | | +| | Contents | populated at PerformPuts/EndStep/Close | ++----------+-------------+-----------------------------------------------+ +| | Pointer | modify after Put | +| Sync | | | +| | Contents | populated at Put | ++----------+-------------+-----------------------------------------------+ + + +1. **Get in Deferred or lazy evaluation mode (default)**: this is the preferred mode as it allows Get calls to be "grouped" before potential data transport at the first encounter of ``PerformPuts``, ``EndStep`` or ``Close``. + + .. code-block:: c++ + + Get(variable, data); + Get(variable, data, adios2::Mode::Deferred); + + + Deferred memory contracts: + + - "data pointer": do not modify (e.g. resize) until first call to ``PerformPuts``, ``EndStep`` or ``Close``. + + - "data contents": populated at first call to ``PerformPuts``, ``EndStep`` or ``Close``. + + Usage: + + .. code-block:: c++ + + std::vector data; + + // resize memory to expected size + data.resize(varBlockSize); + // valid if all memory is populated + // data.reserve(varBlockSize); + + // Gets data pointer to adios2 engine + // associated with current variable metadata + engine.Get(variable, data.data() ); + + // optionally pass data std::vector + // leave resize to adios2 + //engine.Get(variable, data); + + // "data contents" must be ready + // "data pointer" must be the same as in Get + engine.EndStep(); + //engine.PerformPuts(); + //engine.Close(); + + // now data pointer can be reused or modified + + .. caution:: + + Use uninitialized memory at your own risk (e.g. vector reserve, new, malloc). Accessing unitiliazed memory is undefined behavior. + + +2. **Put in Sync mode**: this is the special case, data pointer becomes reusable right after Put. Only use it if absolutely necessary (*e.g.* memory bound application or out of scope data, temporary). + + .. code-block:: c++ + + Get(variable, *data, adios2::Mode::Sync); + + + Sync memory contracts: + + - "data pointer" and "data contents" can be modified after this call. + + + Usage: + + .. code-block:: c++ + + .. code-block:: c++ + + std::vector data; + + // resize memory to expected size + data.resize(varBlockSize); + // valid if all memory is populated + // data.reserve(varBlockSize); + + // Gets data pointer to adios2 engine + // associated with current variable metadata + engine.Get(variable, data.data() ); + + // "data contents" are ready + // "data pointer" can be reused by the application + +.. note:: + + As of v2.4 Get doesn't support returning spans. This is future work required in streaming engines if the application wants a non-owning view into the data buffer for a particular variable block. + + +PerformsGets +------------ + + Executes all pending Get calls in deferred mode + + +Engine usage example +-------------------- The following example illustrates the basic API usage in write mode for data generated at each application step: @@ -54,40 +396,72 @@ The following example illustrates the basic API usage in write mode for data gen engine.BeginStep(); //next "logical" step for this application - engine.Put(variableT, dataT, adios2::Mode::Sync); - // dataT memory already subscribed - // Application can modify its contents - - //deferred functions return immediately (lazy evaluation), - // dataU, dataV and dataW must not be resued - //1st batch - engine.Put(variableU, dataU); - engine.Put(variableV, dataV); + engine.Put(varT, dataT, adios2::Mode::Sync); + // dataT memory already consumed by engine + // Application can modify dataT address and contents + + // deferred functions return immediately (lazy evaluation), + // dataU, dataV and dataW pointers must not be modified + // until PerformPuts, EndStep or Close. + // 1st batch + engine.Put(varU, dataU); + engine.Put(varV, dataV); + // in this case adios2::Mode::Deferred is redundant, // as this is the default option - engine.Put(variableW, dataW, adios2::Mode::Deferred); - // effectively dataU, dataV, dataW memory subscription is "deferred" - // until the first call toPerformPuts, EndStep or Close. - // Application MUST NOT modify their contents. + engine.Put(varW, dataW, adios2::Mode::Deferred); + + // effectively dataU, dataV, dataW are "deferred" + // until the first call to PerformPuts, EndStep or Close. + // Application MUST NOT modify the data pointer (e.g. resize memory). engine.PerformPuts(); - // dataU, dataV, data4W subscribed - // Application can modify their contents - // ... Application modifies dataU, dataV, dataW + // dataU, dataV, dataW pointers/values can now be reused + + // ... Application modifies dataU, dataV, dataW //2nd batch - engine.Put(variableUi, dataU); - engine.Put(variableVi, dataV); - engine.Put(variableWi, dataW); - // Application MUST NOT modify dataU, dataV and dataW - + engine.Put(varU, dataU); + engine.Put(varV, dataV); + engine.Put(varW, dataW); + // Application MUST NOT modify dataU, dataV and dataW pointers (e.g. resize), + // optionally data can be modified, but not recommended + dataU[0] = 10 + dataV[0] = 10 + dataW[0] = 10 + engine.PerformPuts(); + + // dataU, dataV, dataW pointers/values can now be reused + + // Puts a varP block of zeros + adios2::Variable::Span spanP = Put(varP); + + // Not recommended mixing static pointers, + // span follows + // the same pointer/iterator invalidation + // rules as std::vector + T* p = spanP.data(); + + // Puts a varMu block of 1e-6 + adios2::Variable::Span spanMu = Put(varMu, 0, 1e-6); + + // p might be invalidated + // by a new span, use spanP.data() again + foo(spanP.data()); + + // Puts a varRho block with a constant value of 1.225 + Put(varMu, 0, 1.225); + + // it's preferable to start modifying spans + // after all of them are created + foo(spanP.data()); + bar(spanMu.begin(), spanMu.end()); + + engine.EndStep(); + // spanP, spanMu are consumed by the library // end of current logical step, // default behavior: transport data - // if buffering is not fine-tuned with io.SetParameters - - // dataU, dataV, data4W subscribed - // Application can modify their contents } engine.Close(); @@ -96,7 +470,7 @@ The following example illustrates the basic API usage in write mode for data gen .. tip:: - Prefer default Deferred (lazy evaluation) functions as they have the potential to group several variables with the trade-off of not being able to reuse the pointers memory space until ``EndStep``, ``Perform``(``Puts``/``Gets``) or ``Close``. + Prefer default Deferred (lazy evaluation) functions as they have the potential to group several variables with the trade-off of not being able to reuse the pointers memory space until ``EndStep``, ``PerformPuts``, ``PerformGets``, or ``Close``. Only use Sync if you really have to (*e.g.* reuse memory space from pointer). ADIOS2 prefers a step-based IO in which everything is known ahead of time when writing an entire step. @@ -105,8 +479,7 @@ The following example illustrates the basic API usage in write mode for data gen The default behavior of adios2 ``Put`` and ``Get`` calls IS NOT synchronized, but rather deferred. It's actually the opposite of ``MPI_Put`` and more like ``MPI_rPut``. Do not assume the data pointer is usable after a ``Put`` and ``Get``, before ``EndStep``, ``Close`` or the corresponding ``PerformPuts``/``PerformGets``. - Be SAFE and consider using the ``adios2::Mode::Sync`` in the 3rd argument. - Avoid using TEMPORARIES, r-values, and out-of-scope variables in ``Deferred`` mode. + Avoid using TEMPORARIES, r-values, and out-of-scope variables in ``Deferred`` mode, use adios2::Mode::Sync if required. Available Engines diff --git a/docs/user_guide/source/components/overview.rst b/docs/user_guide/source/components/overview.rst index 28ebfab766..76f96ad3b9 100644 --- a/docs/user_guide/source/components/overview.rst +++ b/docs/user_guide/source/components/overview.rst @@ -4,7 +4,7 @@ Components Overview .. note:: - If you are doing simple tasks where performance is a non-critical aspect please go to the :ref:`Simple Language Bindings APIs` section for a quick start. If you are an HPC application developer or you want to use ADIOS2 functionality in full please read this chapter. + If you are doing simple tasks where performance is a non-critical aspect please go to the :ref:`High-Level Language Bindings APIs` section for a quick start. If you are an HPC application developer or you want to use ADIOS2 functionality in full please read this chapter. The simple way to understand the big picture for the ADIOS2 unified user interface components is to map each class to the actual definition of the ADIOS acronym. diff --git a/docs/user_guide/source/conf.py b/docs/user_guide/source/conf.py index 6556011f1c..dd4529e68b 100644 --- a/docs/user_guide/source/conf.py +++ b/docs/user_guide/source/conf.py @@ -94,6 +94,7 @@ # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. exclude_patterns = [ + # 'advice/advice.rst', 'introduction/nutshell.rst', 'introduction/adaptable_io.rst', # 'introduction/introduction.rst', @@ -119,12 +120,10 @@ 'components/io.rst', # 'components/components.rst', 'components/overview.rst', - 'api_simple/fortran.rst', - 'api_simple/matlab.rst', - 'api_simple/python.rst', - 'api_simple/cxx11.rst', - 'api_simple/c.rst', - # 'api_simple/api_simple.rst', + 'api_high/matlab.rst', + 'api_high/python.rst', + 'api_high/cxx11.rst', + # 'api_high/api_high.rst', 'utilities/adios2-config.rst', 'utilities/adios_reorganize.rst', # 'utilities/utilities.rst', diff --git a/docs/user_guide/source/engines/inline.rst b/docs/user_guide/source/engines/inline.rst index 8d8dbfbdc6..a0f6a5e6e8 100644 --- a/docs/user_guide/source/engines/inline.rst +++ b/docs/user_guide/source/engines/inline.rst @@ -1,6 +1,6 @@ -*************** +******************** Inline for zero-copy -*************** +******************** The Inline engine provides in-process communication between the writer and reader, and seeks to avoid copying data buffers. diff --git a/docs/user_guide/source/index.rst b/docs/user_guide/source/index.rst index 0905a8e3ab..173ff0affd 100644 --- a/docs/user_guide/source/index.rst +++ b/docs/user_guide/source/index.rst @@ -17,9 +17,10 @@ Funded by the `Exascale Computing Project (ECP)