diff --git a/corrlib/meas_io.py b/corrlib/meas_io.py index b8695ca..a78f6f2 100644 --- a/corrlib/meas_io.py +++ b/corrlib/meas_io.py @@ -7,7 +7,7 @@ import json from typing import Union, Optional from pyerrors import Obs, Corr, load_object, dump_object from hashlib import sha256, sha1 -from .tools import cached, record2name_key, make_version_hash +from .tools import cached, record2name_key, name_key2record, make_version_hash from .cache_io import is_in_cache, cache_path, cache_dir, get_version_hash @@ -82,10 +82,10 @@ def write_measurement(path, ensemble, measurement, uuid, code, parameter_file: O meas_paths = [] for subkey in subkeys: par_hash = sha256(str(pars[subkey]).encode('UTF-8')).hexdigest() - meas_path = file_in_archive + "::" + par_hash + meas_path = name_key2record(file_in_archive, par_hash) meas_paths.append(meas_path) known_meas[par_hash] = measurement[corr][subkey] - data_hash = sha1(pj.create_json_string(measurement[corr][subkey]).encode('UTF-8')).hexdigest() + data_hash = make_version_hash(path, meas_path) if c.execute("SELECT * FROM backlogs WHERE path = ?", (meas_path,)).fetchone() is None: c.execute("INSERT INTO backlogs (name, ensemble, code, path, project, parameters, parameter_file, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, datetime('now'))", (corr, ensemble, code, meas_path, uuid, pars[subkey], parameter_file)) @@ -140,7 +140,7 @@ def load_records(path: str, record_paths: list[str], preloaded = {}) -> list[Uni returned_data: list = [] for file in needed_data.keys(): for key in list(needed_data[file]): - record = file + "::" + key + record = name_key2record(file, key) current_version = get_version_hash(path, record) if is_in_cache(path, record): returned_data.append(load_object(cache_path(path, file, current_version, key) + ".p")) @@ -165,10 +165,9 @@ def preload(path: str, file: str): def drop_record(path: str, meas_path: str): - file_in_archive = meas_path.split("::")[0] + file_in_archive, sub_key = record2name_key(meas_path) file = os.path.join(path, file_in_archive) db = os.path.join(path, 'backlogger.db') - sub_key = meas_path.split("::")[1] dl.unlock(db, dataset=path) conn = sqlite3.connect(db) c = conn.cursor() diff --git a/corrlib/tools.py b/corrlib/tools.py index 70eb518..43ab1ba 100644 --- a/corrlib/tools.py +++ b/corrlib/tools.py @@ -24,6 +24,10 @@ def record2name_key(record_path: str): return file, key +def name_key2record(name: str, key: str): + return name + "::" + key + + def make_version_hash(path, record): file, key = record2name_key(record) with open(os.path.join(path, file), 'rb') as fp: