Remove adios2.usesteps, set it always to true

This uncovered loads of bugs
franzpoeschel · Oct 19, 2023 · 88cc81f · 88cc81f
1 parent f2db458
commit 88cc81f
Show file tree

Hide file tree

Showing 11 changed files with 175 additions and 162 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -1363,16 +1363,18 @@ if(openPMD_BUILD_TESTING)
                         ${MPI_TEST_EXE} ${Python_EXECUTABLE}                       \
                             ${openPMD_RUNTIME_OUTPUT_DIRECTORY}/openpmd-pipe       \
                             --infile ../samples/git-sample/data00000100.h5         \
-                            --outfile ../samples/git-sample/single_iteration.bp && \
+                            --outfile                                              \
+                                ../samples/git-sample/single_iteration_%T.bp &&    \
                                                                                    \
                         ${MPI_TEST_EXE} ${Python_EXECUTABLE}                       \
                             ${openPMD_RUNTIME_OUTPUT_DIRECTORY}/openpmd-pipe       \
                             --infile ../samples/git-sample/thetaMode/data%T.h5     \
-                            --outfile ../samples/git-sample/thetaMode/data.bp &&   \
+                            --outfile                                              \
+                                ../samples/git-sample/thetaMode/data_%T.bp &&      \
                                                                                    \
                         ${Python_EXECUTABLE}                                       \
                             ${openPMD_RUNTIME_OUTPUT_DIRECTORY}/openpmd-pipe       \
-                            --infile ../samples/git-sample/thetaMode/data.bp       \
+                            --infile ../samples/git-sample/thetaMode/data_%T.bp    \
                             --outfile ../samples/git-sample/thetaMode/data%T.json  \
                         "
                     WORKING_DIRECTORY ${openPMD_RUNTIME_OUTPUT_DIRECTORY}
@@ -1381,17 +1383,17 @@ if(openPMD_BUILD_TESTING)
                 add_test(NAME CLI.pipe.py
                     COMMAND sh -c
                         "${Python_EXECUTABLE}                                      \
-                            ${openPMD_RUNTIME_OUTPUT_DIRECTORY}/openpmd-pipe         \
+                            ${openPMD_RUNTIME_OUTPUT_DIRECTORY}/openpmd-pipe       \
                             --infile ../samples/git-sample/data%T.h5               \
                             --outfile ../samples/git-sample/data%T.bp &&           \
                                                                                    \
                         ${Python_EXECUTABLE}                                       \
-                            ${openPMD_RUNTIME_OUTPUT_DIRECTORY}/openpmd-pipe         \
+                            ${openPMD_RUNTIME_OUTPUT_DIRECTORY}/openpmd-pipe       \
                             --infile ../samples/git-sample/thetaMode/data%T.h5     \
                             --outfile ../samples/git-sample/thetaMode/data%T.bp && \
                                                                                    \
                         ${Python_EXECUTABLE}                                       \
-                            ${openPMD_RUNTIME_OUTPUT_DIRECTORY}/openpmd-pipe         \
+                            ${openPMD_RUNTIME_OUTPUT_DIRECTORY}/openpmd-pipe       \
                             --infile ../samples/git-sample/thetaMode/data%T.bp     \
                             --outfile ../samples/git-sample/thetaMode/data%T.json  \
                         "

diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp
@@ -229,8 +229,12 @@ class ADIOS2IOHandlerImpl
      * If the iteration encoding is variableBased, we default to using a group
      * table, since it is the only reliable way to recover currently active
      * groups.
+     * If group-based encoding is used without group table, then
+     * READ_RANDOM_ACCESS is forbidden as it will be unreliable in reporting
+     * currently available data.
+     * Use AbstractIOHandler::m_encoding for this though
      */
-    IterationEncoding m_iterationEncoding = IterationEncoding::groupBased;
+    // IterationEncoding m_iterationEncoding = IterationEncoding::groupBased;
     /**
      * The ADIOS2 engine type, to be passed to adios2::IO::SetEngine
      */
@@ -434,6 +438,8 @@ namespace ADIOS2Defaults
         "__openPMD_internal/openPMD2_adios2_schema";
     constexpr const_str str_isBoolean = "__is_boolean__";
     constexpr const_str str_activeTablePrefix = "__openPMD_groups";
+    constexpr const_str str_groupBasedWarning =
+        "__openPMD_internal/warning_bugprone_groupbased_encoding";
 } // namespace ADIOS2Defaults
 
 namespace detail
@@ -1064,7 +1070,7 @@ namespace detail
              *    without steps. This is not a workaround since not using steps,
              *    while inefficient in ADIOS2, is something that we support.
              */
-            NoStream,
+            ReadWithoutStream,
             /**
              * Rationale behind this state:
              * When user code opens a Series, series.iterations should contain
@@ -1152,8 +1158,8 @@ namespace detail
         void create_IO();
 
         void configure_IO(ADIOS2IOHandlerImpl &impl);
-        void configure_IO_Read(std::optional<bool> userSpecifiedUsesteps);
-        void configure_IO_Write(std::optional<bool> userSpecifiedUsesteps);
+        void configure_IO_Read();
+        void configure_IO_Write();
     };
 
 } // namespace detail

diff --git a/include/openPMD/IO/AbstractIOHandler.hpp b/include/openPMD/IO/AbstractIOHandler.hpp
@@ -23,6 +23,7 @@
 #include "openPMD/IO/Access.hpp"
 #include "openPMD/IO/Format.hpp"
 #include "openPMD/IO/IOTask.hpp"
+#include "openPMD/IterationEncoding.hpp"
 #include "openPMD/config.hpp"
 
 #if openPMD_HAVE_MPI
@@ -168,6 +169,11 @@ namespace internal
     }
 } // namespace internal
 
+namespace detail
+{
+    class BufferedActions;
+}
+
 /** Interface for communicating between logical and physically persistent data.
  *
  * Input and output operations are channeled through a task queue that is
@@ -179,8 +185,12 @@ namespace internal
 class AbstractIOHandler
 {
     friend class Series;
+    friend class ADIOS2IOHandlerImpl;
+    friend class detail::BufferedActions;
 
 private:
+    IterationEncoding m_encoding = IterationEncoding::groupBased;
+
     void setIterationEncoding(IterationEncoding encoding)
     {
         /*
@@ -193,6 +203,8 @@ class AbstractIOHandler
             // do we really want to have those as const members..?
             *const_cast<Access *>(&m_backendAccess) = Access::CREATE;
         }
+
+        m_encoding = encoding;
     }
 
 public:

diff --git a/include/openPMD/IO/IOTask.hpp b/include/openPMD/IO/IOTask.hpp
@@ -141,7 +141,6 @@ struct OPENPMDAPI_EXPORT Parameter<Operation::CREATE_FILE>
     }
 
     std::string name = "";
-    IterationEncoding encoding = IterationEncoding::groupBased;
 };
 
 template <>
@@ -188,12 +187,6 @@ struct OPENPMDAPI_EXPORT Parameter<Operation::OPEN_FILE>
     }
 
     std::string name = "";
-    /*
-     * The backends might need to ensure availability of certain features
-     * for some iteration encodings, e.g. availability of ADIOS steps for
-     * variableBased encoding.
-     */
-    IterationEncoding encoding = IterationEncoding::groupBased;
     using ParsePreference = internal::ParsePreference;
     std::shared_ptr<ParsePreference> out_parsePreference =
         std::make_shared<ParsePreference>(ParsePreference::UpFront);