From f7fc2519a1c85ff310cfb5fd8136fd9c178ff6a5 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Tue, 8 Apr 2025 13:00:30 +0000 Subject: [PATCH 1/2] more intelligent record loading --- corrlib/meas_io.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/corrlib/meas_io.py b/corrlib/meas_io.py index 20e58fa..bf6c7ae 100644 --- a/corrlib/meas_io.py +++ b/corrlib/meas_io.py @@ -51,7 +51,6 @@ def write_measurement(path, ensemble, measurement, uuid, code, parameter_file): meas_path = file + "::" + parHash if known_meas is not None: known_meas[parHash] = measurement[corr][parHash] - # this should be only set if something changed. else: known_meas = measurement[corr] @@ -91,7 +90,7 @@ def load_record(path: str, meas_path: str): return load_records(path, [meas_path])[0] -def load_records(path: str, meas_paths: list[str]) -> list[Union[Corr, Obs]]: +def load_records(path: str, meas_paths: list[str], preloaded = {}) -> list[Union[Corr, Obs]]: """ Load a list of records by their paths. @@ -113,15 +112,22 @@ def load_records(path: str, meas_paths: list[str]) -> list[Union[Corr, Obs]]: needed_data[file] = [] key = mpath.split("::")[1] needed_data[file].append(key) - dl.get([os.path.join(path, file) for file in needed_data.keys()], dataset=path) + for filename in needed_data.keys(): + if not filename in preloaded: + preloaded[filename] = preload(path, filename) returned_data: list = [] for filename in needed_data.keys(): - filedict = pj.load_json_dict(os.path.join(path, filename)) for key in list(needed_data[filename]): - returned_data.append(filedict[key]) + returned_data.append(preloaded[filename][key]) return returned_data +def preload(path: str, file: str): + dl.get(os.path.join(path, file), dataset=path) + filedict = pj.load_json_dict(os.path.join(path, file)) + return filedict + + def drop_record(path: str, meas_path: str): file = meas_path.split("::")[0] sub_key = meas_path.split("::")[1] From 67bbc25321c447663b9990c1f5e0305d6901e7c8 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Tue, 8 Apr 2025 13:03:41 +0000 Subject: [PATCH 2/2] debug result filter, with a first simple fix, better fix needed --- corrlib/find.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/corrlib/find.py b/corrlib/find.py index 4aeb542..0daf9f3 100644 --- a/corrlib/find.py +++ b/corrlib/find.py @@ -73,14 +73,15 @@ def filter_results(results, **kwargs): if (quark_thetas[0] != param['quarks'][0]['thetas'] and quark_thetas[1] != param['quarks'][1]['thetas']) or (quark_thetas[0] != param['quarks'][1]['thetas'] and quark_thetas[1] != param['quarks'][0]['thetas']): drops.append(ind) continue + # careful, this is not save, when multiple contributions are present! if 'wf1' in kwargs: wf1 = kwargs['wf1'] - if (wf1[0] != param['wf1'][0]) or (wf1[1][0] != param['wf1'][1][0]) or (wf1[1][1] != param['wf1'][1][1]): + if (wf1[0][0] != param['wf1'][0][0]) or (wf1[0][1][0] != param['wf1'][0][1][0]) or (wf1[0][1][1] != param['wf1'][0][1][1]): drops.append(ind) continue if 'wf2' in kwargs: wf2 = kwargs['wf2'] - if (wf2[0] != param['wf2'][0]) or (wf2[1][0] != param['wf2'][1][0]) or (wf2[1][1] != param['wf2'][1][1]): + if (wf2[0][0] != param['wf2'][0][0]) or (wf2[0][1][0] != param['wf2'][0][1][0]) or (wf2[0][1][1] != param['wf2'][0][1][1]): drops.append(ind) continue @@ -91,7 +92,7 @@ def find_record(path, ensemble, correlator_name, project=None, code=None, parame db = path + '/backlogger.db' if os.path.exists(db): dl.get(db, dataset=path) - results = _db_lookup(db, ensemble, correlator_name, project, code=None, parameters=None, created_before=created_before, created_after=created_after, updated_before=updated_before, updated_after=updated_after, revision=revision) + results = _db_lookup(db, ensemble, correlator_name, project, code=code, parameters=parameters, created_before=created_before, created_after=created_after, updated_before=updated_before, updated_after=updated_after, revision=revision) results = filter_results(results, **kwargs) print("Found " + str(len(results)) + " results") return results.reset_index()