patch order across hierarchies not guaranteed by list index

PHAREHUB · Feb 12, 2024 · a8730a3 · a8730a3
1 parent 0c1e705
commit a8730a3
Show file tree

Hide file tree

Showing 3 changed files with 30 additions and 12 deletions.
diff --git a/res/cmake/dep/highfive.cmake b/res/cmake/dep/highfive.cmake
@@ -7,13 +7,13 @@ if(HighFive)
 
   if (NOT EXISTS ${HIGHFIVE_SRC})
     execute_process( # master API breaks us
-      COMMAND ${Git} clone https://github.com/BlueBrain/HighFive ${HIGHFIVE_SRC} -b v2.9.0 --depth 1
+      COMMAND ${Git} clone https://github.com/BlueBrain/HighFive ${HIGHFIVE_SRC} -b master --depth 1
     )
 
   else()
     if(devMode)
-      # message("downloading latest HighFive updates")
-      # execute_process(COMMAND ${Git} pull origin master WORKING_DIRECTORY ${HIGHFIVE_SRC})
+      message("downloading latest HighFive updates")
+      execute_process(COMMAND ${Git} pull origin master WORKING_DIRECTORY ${HIGHFIVE_SRC})
     endif(devMode)
   endif()
 

diff --git a/tests/simulator/__init__.py b/tests/simulator/__init__.py
@@ -265,3 +265,11 @@ def clean_up_diags_dirs(self):
             for diag_dir in self.diag_dirs:
                 if os.path.exists(diag_dir):
                     shutil.rmtree(diag_dir)
+
+    def hierarchy_by_box(self, hier):
+        # comparing hierarchies by index isn't always right
+        box_hier = {i: {} for i in range(len(hier.levels()))}
+        for ilvl, patches in box_hier.items():
+            for patch in hier.level(ilvl).patches:
+                patches[str(patch.box)] = patch
+        return box_hier
diff --git a/tests/simulator/test_restarts.py b/tests/simulator/test_restarts.py
@@ -191,22 +191,23 @@ def count_levels_and_patches(qty):
             )
 
             self.assertEqual(len(datahier0.levels()), len(datahier1.levels()))
+
+            # patch order by list index across hierarchies not guaranteed
+            boxhier1 = self.hierarchy_by_box(datahier1)
+
             for ilvl in range(len(datahier0.levels())):
                 self.assertEqual(
                     len(datahier0.level(ilvl).patches),
                     len(datahier1.level(ilvl).patches),
                 )
-                for patch0, patch1 in zip(
-                    datahier0.level(ilvl).patches, datahier1.level(ilvl).patches
-                ):
-                    self.assertEqual(patch0.box, patch1.box)
+                for patch0 in datahier0.level(ilvl).patches:
+                    self.assertIn(str(patch0.box), boxhier1[ilvl])
 
             self.assertGreater(len(datahier0.levels()), 0)
 
             for ilvl, lvl0 in datahier0.levels().items():
-                patch_level1 = datahier1.levels()[ilvl]
-                for p_idx, patch0 in enumerate(lvl0):
-                    patch1 = patch_level1.patches[p_idx]
+                for patch0 in lvl0:
+                    patch1 = boxhier1[ilvl][str(patch0.box)]
                     for pd_key, pd0 in patch0.patch_datas.items():
                         pd1 = patch1.patch_datas[pd_key]
                         self.assertNotEqual(id(pd0), id(pd1))
@@ -218,7 +219,16 @@ def count_levels_and_patches(qty):
                                     f"FAILED domain particles at time {time} {ilvl} {patch1.box} {patch0.box}"
                                 )
                         else:
-                            np.testing.assert_equal(pd0.dataset[:], pd1.dataset[:])
+                            if cpp.mpi_size() == 1:
+                                np.testing.assert_equal(pd0.dataset[:], pd1.dataset[:])
+                            else:
+                                # rank order for shared nodes not guaranteed
+                                np.testing.assert_allclose(
+                                    pd0.dataset[:],
+                                    pd1.dataset[:],
+                                    atol=1e-14,  # 1e-15 fails empirically
+                                    rtol=0,
+                                )
                         checks += 1
 
             n_levels, n_patches = count_levels_and_patches(run0.GetB(time))
@@ -251,7 +261,7 @@ def test_restarts(self, ndim, interp, simInput, expected_num_levels):
             # three levels has issues with refinementboxes and possibly regridding
             b0 = [[6] * ndim, [15] * ndim]
             simput["refinement_boxes"] = {"L0": {"B0": b0}}
-        else:  # https://github.com/LLNL/SAMRAI/issues/199
+        elif cpp.mpi_size() == 1:  # https://github.com/LLNL/SAMRAI/issues/199
             # tagging can handle more than one timestep as it does not
             #  appear subject to regridding issues, so we make more timesteps
             #  to confirm simulations are still equivalent