fix: reject non-finite model deviations

dpgen2/exploration/render/traj_render_lammps.py

@@ -15,6 +15,9 @@
 
 import dpdata
 import numpy as np
+from dflow.python import (
+    FatalError,
+)
 from dflow.python.opio import (
     HDF5Dataset,
 )
@@ -60,20 +63,47 @@ def get_model_devi(
 
     def _load_one_model_devi(self, fname, model_devi):
         if isinstance(fname, HDF5Dataset):
-            dd = fname.get_data()
+            dd = np.asarray(fname.get_data())
         else:
             dd = np.loadtxt(fname)
-        if (
-            len(np.shape(dd)) == 1  # type: ignore
-        ):  # In case model-devi.out is 1-dimensional
-            dd = dd.reshape((1, len(dd)))  # type: ignore
-
-        model_devi.add(DeviManager.MAX_DEVI_V, dd[:, 1])  # type: ignore
-        model_devi.add(DeviManager.MIN_DEVI_V, dd[:, 2])  # type: ignore
-        model_devi.add(DeviManager.AVG_DEVI_V, dd[:, 3])  # type: ignore
-        model_devi.add(DeviManager.MAX_DEVI_F, dd[:, 4])  # type: ignore
-        model_devi.add(DeviManager.MIN_DEVI_F, dd[:, 5])  # type: ignore
-        model_devi.add(DeviManager.AVG_DEVI_F, dd[:, 6])  # type: ignore
+        if dd.ndim == 1:  # In case model-devi.out is 1-dimensional
+            dd = dd.reshape((1, len(dd)))
+
+        # A NaN or infinity would otherwise reach the report and scheduler, where
+        # comparisons silently produce invalid trust levels. Fail at the artifact
+        # boundary so users can inspect the corresponding LAMMPS task directly.
+        # Validate all six quantities exported by this loader, including virials
+        # that may be used by reports with virial trust levels configured.
+        deviation_names = (
+            DeviManager.MAX_DEVI_V,
+            DeviManager.MIN_DEVI_V,
+            DeviManager.AVG_DEVI_V,
+            DeviManager.MAX_DEVI_F,
+            DeviManager.MIN_DEVI_F,
+            DeviManager.AVG_DEVI_F,
+        )
+        invalid = np.argwhere(~np.isfinite(dd[:, 1:7]))
+        if invalid.size:
+            locations = ", ".join(
+                f"row {int(invalid[index, 0]) + 1} "
+                f"({deviation_names[int(invalid[index, 1])]})"
+                for index in range(min(len(invalid), 10))
+            )
+            if len(invalid) > 10:
+                locations += f", ... ({len(invalid)} non-finite values in total)"
+            source = fname.key if isinstance(fname, HDF5Dataset) else fname
+            raise FatalError(
+                f"Non-finite model-deviation value in {source}: {locations}. "
+                "Inspect the LAMMPS/DeePMD task output before scheduling the "
+                "next exploration iteration."
+            )
+
+        model_devi.add(DeviManager.MAX_DEVI_V, dd[:, 1])
+        model_devi.add(DeviManager.MIN_DEVI_V, dd[:, 2])
+        model_devi.add(DeviManager.AVG_DEVI_V, dd[:, 3])
+        model_devi.add(DeviManager.MAX_DEVI_F, dd[:, 4])
+        model_devi.add(DeviManager.MIN_DEVI_F, dd[:, 5])
+        model_devi.add(DeviManager.AVG_DEVI_F, dd[:, 6])
 
     def get_ele_temp(self, optional_outputs):
         ele_temp = []

tests/exploration/test_traj_render_lammps.py

@@ -1,20 +1,109 @@
 import json
 import os
 import unittest
+from pathlib import (
+    Path,
+)
 
 import dpdata
+import h5py
 import numpy as np
+from dflow.python import (
+    FatalError,
+)
+from dflow.python.opio import (
+    HDF5Dataset,
+)
 
 # isort: off
 from .context import (
     dpgen2,
 )
 from dpgen2.exploration.render import TrajRenderLammps
+from dpgen2.exploration.deviation import DeviManager
 
 # isort: on
 
 
 class TestTrajRenderLammps(unittest.TestCase):
+    deviation_names = (
+        DeviManager.MAX_DEVI_V,
+        DeviManager.MIN_DEVI_V,
+        DeviManager.AVG_DEVI_V,
+        DeviManager.MAX_DEVI_F,
+        DeviManager.MIN_DEVI_F,
+        DeviManager.AVG_DEVI_F,
+    )
+
+    def test_rejects_non_finite_model_deviation(self):
+        """Reject NaN and either infinity in every exported column."""
+        model_devi = Path("model_devi.out")
+        for column, name in enumerate(self.deviation_names, start=1):
+            for value in (np.nan, np.inf, -np.inf):
+                with self.subTest(column=name, value=value):
+                    data = np.zeros((2, 7))
+                    data[1, column] = value
+                    np.savetxt(model_devi, data)
+                    with self.assertRaisesRegex(
+                        FatalError,
+                        rf"Non-finite model-deviation value in model_devi.out: row 2 \({name}\)",
+                    ):
+                        TrajRenderLammps().get_model_devi([model_devi])
+
+    def test_rejects_non_finite_hdf5_model_deviation(self):
+        """An HDF5 error names the dataset, not its Python object address."""
+        key = "task.000000/model_devi"
+        for column, name in enumerate(self.deviation_names, start=1):
+            for value in (np.nan, np.inf, -np.inf):
+                with self.subTest(column=name, value=value):
+                    data = np.zeros((2, 7))
+                    data[1, column] = value
+                    with h5py.File("model_devi.h5", "w") as h5:
+                        h5.create_dataset(key, data=data)
+                        dataset = HDF5Dataset(h5, key)
+                        with self.assertRaises(FatalError) as error:
+                            TrajRenderLammps().get_model_devi([dataset])
+                    message = str(error.exception)
+                    self.assertIn(key, message)
+                    self.assertIn(f"row 2 ({name})", message)
+                    self.assertNotIn("object at", message)
+
+    def test_limits_non_finite_diagnostics(self):
+        """Large failed trajectories give a bounded message and total count."""
+        model_devi = Path("model_devi.out")
+        data = np.full((20000, 7), np.nan)
+        data[:, 0] = np.arange(len(data))
+        np.savetxt(model_devi, data)
+        with self.assertRaises(FatalError) as error:
+            TrajRenderLammps().get_model_devi([model_devi])
+        message = str(error.exception)
+        self.assertEqual(message.count("row "), 10)
+        self.assertIn("120000 non-finite values in total", message)
+        self.assertLess(len(message), 1000)
+
+    def test_loads_finite_model_deviation(self):
+        """Keep single/multiple rows and large finite sentinel values valid."""
+        for nrows in (1, 2):
+            for use_hdf5 in (False, True):
+                with self.subTest(nrows=nrows, use_hdf5=use_hdf5):
+                    data = np.arange(nrows * 7, dtype=float).reshape(nrows, 7)
+                    data[0, 4] = np.finfo(float).max
+                    if use_hdf5:
+                        with h5py.File("model_devi.h5", "w") as h5:
+                            h5.create_dataset(
+                                "model_devi", data=data[0] if nrows == 1 else data
+                            )
+                            source = HDF5Dataset(h5, "model_devi")
+                            deviations = TrajRenderLammps().get_model_devi([source])
+                    else:
+                        source = Path("model_devi.out")
+                        np.savetxt(source, data)
+                        deviations = TrajRenderLammps().get_model_devi([source])
+                    for column, name in enumerate(self.deviation_names, start=1):
+                        np.testing.assert_array_equal(
+                            deviations.get(name)[0], data[:, column]
+                        )
+
     def test_use_ele_temp_1(self):
         with open("job.json", "w") as f:
             json.dump({"ele_temp": 6.6}, f)
@@ -58,5 +147,6 @@ def test_use_ele_temp_2(self):
         np.testing.assert_array_almost_equal(system.data["aparam"], np.array([[[6.6]]]))
 
     def tearDown(self):
-        if os.path.exists("job.json"):
-            os.remove("job.json")
+        for filename in ("job.json", "model_devi.out", "model_devi.h5"):
+            if os.path.exists(filename):
+                os.remove(filename)