From c4e1e1aa779492713010323ff399765054c2f82d Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Tue, 8 Apr 2025 09:39:57 +0000 Subject: [PATCH] try sha hash to combat colisions --- corrlib/meas_io.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/corrlib/meas_io.py b/corrlib/meas_io.py index 95cdace..20e58fa 100644 --- a/corrlib/meas_io.py +++ b/corrlib/meas_io.py @@ -6,6 +6,7 @@ from .input import sfcf,openQCD import json from typing import Union from pyerrors import Obs, Corr +from hashlib import sha256 def write_measurement(path, ensemble, measurement, uuid, code, parameter_file): @@ -45,9 +46,11 @@ def write_measurement(path, ensemble, measurement, uuid, code, parameter_file): known_meas = pj.load_json_dict(file) if code == "sfcf": for subkey in measurement[corr].keys(): - meas_path = file + "::" + subkey + pars = sfcf.get_specs(corr + "/" + subkey, parameters) + parHash = sha256(str(pars)) + meas_path = file + "::" + parHash if known_meas is not None: - known_meas[subkey] = measurement[corr][subkey] + known_meas[parHash] = measurement[corr][parHash] # this should be only set if something changed. else: known_meas = measurement[corr] @@ -55,7 +58,8 @@ def write_measurement(path, ensemble, measurement, uuid, code, parameter_file): if c.execute("SELECT * FROM backlogs WHERE path = ?", (meas_path,)).fetchone() is not None: c.execute("UPDATE backlogs SET updated_at = datetime('now') WHERE path = ?", (meas_path, )) else: - c.execute("INSERT INTO backlogs (name, ensemble, code, path, project, parameters, parameter_file, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, datetime('now'), datetime('now'))", (corr, ensemble, code, meas_path, uuid, sfcf.get_specs(corr + "/" + subkey, parameters), parameter_file)) + c.execute("INSERT INTO backlogs (name, ensemble, code, path, project, parameters, parameter_file, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, datetime('now'), datetime('now'))", + (corr, ensemble, code, meas_path, uuid, sfcf.get_specs(corr + "/" + subkey, parameters), parameter_file)) elif code == "openQCD": for i in range(len(parameters["rw_fcts"])): subkey = "/".join(parameters["rw_fcts"][i]) @@ -104,14 +108,15 @@ def load_records(path: str, meas_paths: list[str]) -> list[Union[Corr, Obs]]: """ needed_data: dict[str, list[str]] = {} for mpath in meas_paths: - file = path.split("::")[0] + file = mpath.split("::")[0] if file not in needed_data.keys(): needed_data[file] = [] key = mpath.split("::")[1] needed_data[file].append(key) + dl.get([os.path.join(path, file) for file in needed_data.keys()], dataset=path) returned_data: list = [] for filename in needed_data.keys(): - filedict = pj.load_json_dict(filename) + filedict = pj.load_json_dict(os.path.join(path, filename)) for key in list(needed_data[filename]): returned_data.append(filedict[key]) return returned_data