jkuhl/corrlib
1
0
Fork 0
Code Issues 4 Pull requests Releases Packages Activity

centralize file and key to record concat and back

Browse source
This commit is contained in:
Justus Kuhlmann 2025-11-28 16:42:50 +01:00
commit 5bd94633e8
Signed by: jkuhl
GPG key ID: 00ED992DD79B85A6
2 changed files with 9 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

11
corrlib/meas_io.py
View file

@ -7,7 +7,7 @@ import json
from typing import Union, Optional from typing import Union, Optional
from pyerrors import Obs, Corr, load_object, dump_object from pyerrors import Obs, Corr, load_object, dump_object
from hashlib import sha256, sha1 from hashlib import sha256, sha1
from .tools import cached, record2name_key, make_version_hash from .tools import cached, record2name_key, name_key2record, make_version_hash
from .cache_io import is_in_cache, cache_path, cache_dir, get_version_hash from .cache_io import is_in_cache, cache_path, cache_dir, get_version_hash
@ -82,10 +82,10 @@ def write_measurement(path, ensemble, measurement, uuid, code, parameter_file: O
meas_paths = [] meas_paths = []
for subkey in subkeys: for subkey in subkeys:
par_hash = sha256(str(pars[subkey]).encode('UTF-8')).hexdigest() par_hash = sha256(str(pars[subkey]).encode('UTF-8')).hexdigest()
meas_path = file_in_archive + "::" + par_hash meas_path = name_key2record(file_in_archive, par_hash)
meas_paths.append(meas_path) meas_paths.append(meas_path)
known_meas[par_hash] = measurement[corr][subkey] known_meas[par_hash] = measurement[corr][subkey]
data_hash = sha1(pj.create_json_string(measurement[corr][subkey]).encode('UTF-8')).hexdigest() data_hash = make_version_hash(path, meas_path)
if c.execute("SELECT * FROM backlogs WHERE path = ?", (meas_path,)).fetchone() is None: if c.execute("SELECT * FROM backlogs WHERE path = ?", (meas_path,)).fetchone() is None:
c.execute("INSERT INTO backlogs (name, ensemble, code, path, project, parameters, parameter_file, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, datetime('now'))", c.execute("INSERT INTO backlogs (name, ensemble, code, path, project, parameters, parameter_file, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, datetime('now'))",
(corr, ensemble, code, meas_path, uuid, pars[subkey], parameter_file)) (corr, ensemble, code, meas_path, uuid, pars[subkey], parameter_file))
@ -140,7 +140,7 @@ def load_records(path: str, record_paths: list[str], preloaded = {}) -> list[Uni
returned_data: list = [] returned_data: list = []
for file in needed_data.keys(): for file in needed_data.keys():
for key in list(needed_data[file]): for key in list(needed_data[file]):
record = file + "::" + key record = name_key2record(file, key)
current_version = get_version_hash(path, record) current_version = get_version_hash(path, record)
if is_in_cache(path, record): if is_in_cache(path, record):
returned_data.append(load_object(cache_path(path, file, current_version, key) + ".p")) returned_data.append(load_object(cache_path(path, file, current_version, key) + ".p"))
@ -165,10 +165,9 @@ def preload(path: str, file: str):
def drop_record(path: str, meas_path: str): def drop_record(path: str, meas_path: str):
file_in_archive = meas_path.split("::")[0] file_in_archive, sub_key = record2name_key(meas_path)
file = os.path.join(path, file_in_archive) file = os.path.join(path, file_in_archive)
db = os.path.join(path, 'backlogger.db') db = os.path.join(path, 'backlogger.db')
sub_key = meas_path.split("::")[1]
dl.unlock(db, dataset=path) dl.unlock(db, dataset=path)
conn = sqlite3.connect(db) conn = sqlite3.connect(db)
c = conn.cursor() c = conn.cursor()

4
corrlib/tools.py
View file

@ -24,6 +24,10 @@ def record2name_key(record_path: str):
return file, key return file, key
def name_key2record(name: str, key: str):
return name + "::" + key
def make_version_hash(path, record): def make_version_hash(path, record):
file, key = record2name_key(record) file, key = record2name_key(record)
with open(os.path.join(path, file), 'rb') as fp: with open(os.path.join(path, file), 'rb') as fp: