more intelligent record loading

corrlib/meas_io.py

@@ -51,7 +51,6 @@ def write_measurement(path, ensemble, measurement, uuid, code, parameter_file):
                 meas_path = file + "::" + parHash
                 if known_meas is not None:
                     known_meas[parHash] = measurement[corr][parHash]
-                    # this should be only set if something changed.
                 else:
                     known_meas = measurement[corr]
 
@@ -91,7 +90,7 @@ def load_record(path: str, meas_path: str):
     return load_records(path, [meas_path])[0]
 
 
-def load_records(path: str, meas_paths: list[str]) -> list[Union[Corr, Obs]]:
+def load_records(path: str, meas_paths: list[str], preloaded = {}) -> list[Union[Corr, Obs]]:
     """
     Load a list of records by their paths.
 
@@ -113,15 +112,22 @@ def load_records(path: str, meas_paths: list[str]) -> list[Union[Corr, Obs]]:
             needed_data[file] = []
         key = mpath.split("::")[1]
         needed_data[file].append(key)
-    dl.get([os.path.join(path, file) for file in needed_data.keys()], dataset=path)
+    for filename in needed_data.keys():
+        if not filename in preloaded:
+            preloaded[filename] = preload(path, filename)
     returned_data: list = []
     for filename in needed_data.keys():
-        filedict = pj.load_json_dict(os.path.join(path, filename))
         for key in list(needed_data[filename]):
-            returned_data.append(filedict[key])
+            returned_data.append(preloaded[filename][key])
     return returned_data
 
 
+def preload(path: str, file: str):
+    dl.get(os.path.join(path, file), dataset=path)
+    filedict = pj.load_json_dict(os.path.join(path, file))
+    return filedict
+
+
 def drop_record(path: str, meas_path: str):
     file = meas_path.split("::")[0]
     sub_key = meas_path.split("::")[1]