diff --git a/TODO.md b/TODO.md index 3c568dc..a41f2dd 100644 --- a/TODO.md +++ b/TODO.md @@ -3,13 +3,7 @@ ## Features - [ ] implement import of non-datalad projects - [ ] implement a way to use another backlog repo as a project - -- [ ] find a way to convey the mathematical structure of what EXACTLY is the form of the correlator in a specific project - - this could e.g. be done along the lines of mandatory documentation -- [ ] keep better track of the versions the code, that was used for a specific measurement. - - maybe let this be an input in the project file? - - git repo and commit hash/version tag - [ ] implement local caching with pickle files ## Bugfixes -- [ ] revisit the reimport function for single files +- [ ] revisit the reimport function for single files \ No newline at end of file diff --git a/corrlib/initialization.py b/corrlib/initialization.py index f6ef5aa..9f17e47 100644 --- a/corrlib/initialization.py +++ b/corrlib/initialization.py @@ -46,7 +46,4 @@ def create(path): os.makedirs(path + '/archive') os.makedirs(path + '/toml_imports') os.makedirs(path + '/import_scripts/template.py') - with open(path + "/.gitignore", "w") as fp: - fp.write(".cache") - fp.close() dl.save(path, dataset=path, message="Initialize backlogger directory.") diff --git a/corrlib/meas_io.py b/corrlib/meas_io.py index 5e43b3f..dc8fadc 100644 --- a/corrlib/meas_io.py +++ b/corrlib/meas_io.py @@ -5,9 +5,8 @@ import sqlite3 from .input import sfcf,openQCD import json from typing import Union -from pyerrors import Obs, Corr, dump_obj, load_obj +from pyerrors import Obs, Corr from hashlib import sha256 -from .tools import cached def write_measurement(path, ensemble, measurement, uuid, code, parameter_file=None): @@ -136,35 +135,16 @@ def load_records(path: str, meas_paths: list[str], preloaded = {}) -> list[Union needed_data[file] = [] key = mpath.split("::")[1] needed_data[file].append(key) + for filename in needed_data.keys(): + if not filename in preloaded: + preloaded[filename] = preload(path, filename) returned_data: list = [] - for file in needed_data.keys(): - for key in list(needed_data[file]): - if os.path.exists(cache_path(path, file, key)): - returned_data.append(load_obj(cache_path(path, file, key))) - else: - if file not in preloaded: - preloaded[file] = preload(path, file) - returned_data.append(preloaded[file][key]) - if cached: - if not os.path.exists(cache_dir(path, file)): - os.makedirs(cache_dir(path, file)) - dump_obj(preloaded[file][key], cache_path(path, file, key)) + for filename in needed_data.keys(): + for key in list(needed_data[filename]): + returned_data.append(preloaded[filename][key]) return returned_data -def cache_dir(path, file): - cache_path_list = [path] - cache_path_list.append(".cache") - cache_path_list.extend(file.split("/")[1:]) - cache_path = os.path.join(cache_path_list) - return cache_path - - -def cache_path(path, file, key): - cache_path = os.path.join(cache_dir(path, file), key) - return cache_path + ".p" - - def preload(path: str, file: str): dl.get(os.path.join(path, file), dataset=path) filedict = pj.load_json_dict(os.path.join(path, file)) diff --git a/corrlib/toml.py b/corrlib/toml.py index a65b807..349d304 100644 --- a/corrlib/toml.py +++ b/corrlib/toml.py @@ -104,14 +104,12 @@ def import_toml(path: str, file: str, copy_file: bool=True) -> None: for rwp in ["integrator", "eps", "ntot", "dnms"]: param[rwp] = "Unknown" param['type'] = 't0' - measurement = openQCD.extract_t0(path, uuid, md['path'], param, md["prefix"], md["dtr_read"], md["xmin"], md["spatial_extent"], - fit_range=md.get('fit_range', 5), postfix=md.get('postfix', None), names=md.get('names', None), files=md.get('files', None)) + measurement = openQCD.extract_t0(path, uuid, md['path'], param, md["prefix"], md["dtr_read"], md["xmin"], md["spatial_extent"], fit_range=md.get('fit_range', 5), postfix=md.get('postfix', None), names=md.get('names', None)) elif md['measurement'] == 't1': if 'param_file' in md: param = openQCD.read_ms3_param(path, uuid, md['param_file']) param['type'] = 't1' - measurement = openQCD.extract_t1(path, uuid, md['path'], param, md["prefix"], md["dtr_read"], md["xmin"], md["spatial_extent"], - fit_range=md.get('fit_range', 5), postfix=md.get('postfix', None), names=md.get('names', None), files=md.get('files', None)) + measurement = openQCD.extract_t1(path, uuid, md['path'], param, md["prefix"], md["dtr_read"], md["xmin"], md["spatial_extent"], fit_range=md.get('fit_range', 5), postfix=md.get('postfix', None), names=md.get('names', None)) write_measurement(path, ensemble, measurement, uuid, project['code'], (md['param_file'] if 'param_file' in md else None)) diff --git a/corrlib/tools.py b/corrlib/tools.py index e424b59..a150520 100644 --- a/corrlib/tools.py +++ b/corrlib/tools.py @@ -6,6 +6,4 @@ def str2list(string): def list2str(mylist): s = ",".join(mylist) - return s - -cached = True + return s \ No newline at end of file