diff --git a/corrlib/meas_io.py b/corrlib/meas_io.py index 20e58fa..bf6c7ae 100644 --- a/corrlib/meas_io.py +++ b/corrlib/meas_io.py @@ -51,7 +51,6 @@ def write_measurement(path, ensemble, measurement, uuid, code, parameter_file): meas_path = file + "::" + parHash if known_meas is not None: known_meas[parHash] = measurement[corr][parHash] - # this should be only set if something changed. else: known_meas = measurement[corr] @@ -91,7 +90,7 @@ def load_record(path: str, meas_path: str): return load_records(path, [meas_path])[0] -def load_records(path: str, meas_paths: list[str]) -> list[Union[Corr, Obs]]: +def load_records(path: str, meas_paths: list[str], preloaded = {}) -> list[Union[Corr, Obs]]: """ Load a list of records by their paths. @@ -113,15 +112,22 @@ def load_records(path: str, meas_paths: list[str]) -> list[Union[Corr, Obs]]: needed_data[file] = [] key = mpath.split("::")[1] needed_data[file].append(key) - dl.get([os.path.join(path, file) for file in needed_data.keys()], dataset=path) + for filename in needed_data.keys(): + if not filename in preloaded: + preloaded[filename] = preload(path, filename) returned_data: list = [] for filename in needed_data.keys(): - filedict = pj.load_json_dict(os.path.join(path, filename)) for key in list(needed_data[filename]): - returned_data.append(filedict[key]) + returned_data.append(preloaded[filename][key]) return returned_data +def preload(path: str, file: str): + dl.get(os.path.join(path, file), dataset=path) + filedict = pj.load_json_dict(os.path.join(path, file)) + return filedict + + def drop_record(path: str, meas_path: str): file = meas_path.split("::")[0] sub_key = meas_path.split("::")[1]