jkuhl/corrlib
1
0
Fork 0
Code Issues Pull requests Releases Packages Activity

try sha hash to combat colisions

Browse source
This commit is contained in:
Justus Kuhlmann 2025-04-08 09:39:57 +00:00
parent 2998fd2613
commit c4e1e1aa77
1 changed files with 10 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

15
corrlib/meas_io.py
View file

@ -6,6 +6,7 @@ from .input import sfcf,openQCD
import json
from typing import Union
from pyerrors import Obs, Corr
from hashlib import sha256
def write_measurement(path, ensemble, measurement, uuid, code, parameter_file):
@ -45,9 +46,11 @@ def write_measurement(path, ensemble, measurement, uuid, code, parameter_file):
known_meas = pj.load_json_dict(file)
if code == "sfcf":
for subkey in measurement[corr].keys():
meas_path = file + "::" + subkey
pars = sfcf.get_specs(corr + "/" + subkey, parameters)
parHash = sha256(str(pars))
meas_path = file + "::" + parHash
if known_meas is not None:
known_meas[subkey] = measurement[corr][subkey]
known_meas[parHash] = measurement[corr][parHash]
# this should be only set if something changed.
else:
known_meas = measurement[corr]
@ -55,7 +58,8 @@ def write_measurement(path, ensemble, measurement, uuid, code, parameter_file):
if c.execute("SELECT * FROM backlogs WHERE path = ?", (meas_path,)).fetchone() is not None:
c.execute("UPDATE backlogs SET updated_at = datetime('now') WHERE path = ?", (meas_path, ))
else:
c.execute("INSERT INTO backlogs (name, ensemble, code, path, project, parameters, parameter_file, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, datetime('now'), datetime('now'))", (corr, ensemble, code, meas_path, uuid, sfcf.get_specs(corr + "/" + subkey, parameters), parameter_file))
c.execute("INSERT INTO backlogs (name, ensemble, code, path, project, parameters, parameter_file, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, datetime('now'), datetime('now'))",
(corr, ensemble, code, meas_path, uuid, sfcf.get_specs(corr + "/" + subkey, parameters), parameter_file))
elif code == "openQCD":
for i in range(len(parameters["rw_fcts"])):
subkey = "/".join(parameters["rw_fcts"][i])
@ -104,14 +108,15 @@ def load_records(path: str, meas_paths: list[str]) -> list[Union[Corr, Obs]]:
"""
needed_data: dict[str, list[str]] = {}
for mpath in meas_paths:
file = path.split("::")[0]
file = mpath.split("::")[0]
if file not in needed_data.keys():
needed_data[file] = []
key = mpath.split("::")[1]
needed_data[file].append(key)
dl.get([os.path.join(path, file) for file in needed_data.keys()], dataset=path)
returned_data: list = []
for filename in needed_data.keys():
filedict = pj.load_json_dict(filename)
filedict = pj.load_json_dict(os.path.join(path, filename))
for key in list(needed_data[filename]):
returned_data.append(filedict[key])
return returned_data