implement mechanism to find files to discard after already read measurements

corrlib/meas_io.py

@@ -135,6 +135,15 @@ def write_measurement(path: Path, ensemble: str, measurement: dict[str, dict[str
     return
 
 
+def affected_files(corrs: list[str], ensemble: str, uuid: str) -> list[Path]:
+    file_list = []
+    for corr in corrs:
+        file_in_archive = Path('.') / 'archive' / ensemble / corr / str(uuid + '.json.gz')
+        file_list.append(file_in_archive)
+    file_list = list(set(file_list))
+    return file_list
+
+
 def load_record(path: Path, meas_path: str) -> Union[Corr, Obs]:
     """
     Load a list of records by their paths.

corrlib/toml.py

@@ -15,9 +15,10 @@ import datalad.api as dl
 from .tracker import save
 from .input import sfcf, openQCD
 from .main import import_project, update_aliases
-from .meas_io import write_measurement
+from .meas_io import write_measurement, affected_files
 import os
 from .input.implementations import codes as known_codes
+from tools import step_differences
 from typing import Any
 from pathlib import Path
 
@@ -180,7 +181,29 @@ def import_toml(path: Path, file: str, copy_file: bool=True) -> None:
         uuid = import_project(path, project['url'], aliases=aliases)
     imeas = 1
     nmeas = len(measurements.keys())
-    for mname, md in measurements.items():
+
+    # preparation step
+    affected_file_d = {}
+    mname_list = list(measurements.keys())
+    for mname in mname_list:
+        md = measurements[mname]
+        print(f"Import measurement {imeas}/{nmeas}: {mname}")
+        ensemble = md['ensemble']
+        if project['code'] == 'sfcf':
+            param = sfcf.read_param(path, uuid, md['param_file'])
+            affected_by_meas = affected_files(param['crr'], ensemble, uuid)
+        elif project['code'] == 'openQCD':
+            if md['measurement'] == 'ms1':
+                affected_by_meas = affected_files(param['type'], ensemble, uuid)
+            elif md['measurement'] == 't0':
+                affected_by_meas = affected_files(param['type'], ensemble, uuid)
+            elif md['measurement'] == 't1':
+                affected_by_meas = affected_files(param['type'], ensemble, uuid)
+        affected_file_d[mname] = affected_by_meas
+    discard_after = step_differences(mname_list, affected_file_d)
+
+    for mname in mname_list:
+        md = measurements[mname]
         print(f"Import measurement {imeas}/{nmeas}: {mname}")
         ensemble = md['ensemble']
         if project['code'] == 'sfcf':

corrlib/tools.py

@@ -151,3 +151,22 @@ def cache_enabled(path: Path) -> bool:
         raise ValueError(f"String {cached_str} is not a valid option, only True and False are allowed!")
     cached_bool = cached_str == ('True')
     return cached_bool
+
+
+def step_differences(name_list: list[Any], dict_of_lists: dict[Any, Any]) -> list[set[Any]]:
+    needed_until_step = []
+    for i in range(len(name_list)):
+        nf: set[Any] = set()
+        for k in range(i, len(name_list)):
+            nf = nf.union(dict_of_lists[name_list[k]])
+        needed_until_step.append(nf)
+
+    discard_after = []
+    for i in range(len(needed_until_step)-1):
+        discard_after.append(needed_until_step[i].difference(needed_until_step[i+1]))
+    discard_after.append(needed_until_step[-1])
+
+    print(discard_after)
+    if not set(dict_of_lists[name_list[-1]]) == discard_after[-1]:
+        raise ValueError("Discards and last items diverge.")
+    return discard_after