From 7fc8e91a3ecc2492052e0d9d928f353143268030 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Thu, 17 Apr 2025 08:36:31 +0000 Subject: [PATCH] implement t0 and t1 import from openQCD, allow no param file for these --- corrlib/input/openQCD.py | 109 +++++++++++++++++++++++++++++++++++---- corrlib/input/sfcf.py | 4 +- corrlib/meas_io.py | 38 ++++++++++---- corrlib/toml.py | 25 +++++++-- 4 files changed, 148 insertions(+), 28 deletions(-) diff --git a/corrlib/input/openQCD.py b/corrlib/input/openQCD.py index d17e943..0342a00 100644 --- a/corrlib/input/openQCD.py +++ b/corrlib/input/openQCD.py @@ -5,13 +5,14 @@ import fnmatch from typing import Any -def read_param(path: str, project: str, file_in_project: str) -> dict[str, Any]: - file = path + "/projects/" + project + '/' + file_in_project - dl.get(file, dataset=path + "/projects/" + project) +def read_ms1_param(path: str, project: str, file_in_project: str) -> dict[str, Any]: + file = os.path.join(path, "projects", project, file_in_project) + ds = os.path.join(path, "projects", project) + dl.get(file, dataset=ds) with open(file, 'r') as fp: lines = fp.readlines() fp.close() - param = {} + param: dict[str, Any] = {} param['rw_fcts'] = [] param['rand'] = {} @@ -48,17 +49,34 @@ def read_param(path: str, project: str, file_in_project: str) -> dict[str, Any]: param["rw_fcts"][nrw]["irp"] = "None" return param - -def read_rwms(path: str, project: str, dir_in_project: str, param: dict[str, Any], prefix: str, postfix: str="ms1", version: str='2.0', names: list[str]=None, files: list[str]=None): - directory = path + "/projects/" + project + '/' + dir_in_project + +def read_ms3_param(path: str, project: str, file_in_project: str) -> dict[str, Any]: + file = os.path.join(path, "projects", project, file_in_project) + ds = os.path.join(path, "projects", project) + dl.get(file, dataset=ds) + with open(file, 'r') as fp: + lines = fp.readlines() + fp.close() + param = {} + for line in lines: + line = line.strip() + for rwp in ["integrator", "eps", "ntot", "dnms"]: + if line.startswith(rwp): + param[rwp] = line.split()[1] + return param + + +def read_rwms(path: str, project: str, dir_in_project: str, param: dict[str, Any], prefix: str, postfix: str="ms1", version: str='2.0', names: list[str]=None, files: list[str]=None) -> dict[str, Any]: + dataset = os.path.join(path, "projects", project) + directory = os.path.join(dataset, dir_in_project) if files is None: files = [] for root, ds, fs in os.walk(directory): for f in fs: if fnmatch.fnmatch(f, prefix + "*" + postfix + ".dat"): files.append(f) - dl.get([directory + "/" + f for f in files], dataset=path + "/projects/" + project) + dl.get([os.path.join(directory, f) for f in files], dataset=dataset) kwargs = {} if names is not None: kwargs['names'] = names @@ -73,4 +91,77 @@ def read_rwms(path: str, project: str, dir_in_project: str, param: dict[str, Any par_list.append(str(param["rw_fcts"][i][k])) pars = "/".join(par_list) rw_dict[param["type"]][pars] = rwms[i] - return rw_dict \ No newline at end of file + return rw_dict + + +def extract_t0(path: str, project: str, dir_in_project: str, param: dict[str, Any], prefix: str, dtr_read: int, xmin: int, spatial_extent: int, fit_range: int = 5, postfix: str=None, names: list[str]=None, files: list[str]=None) -> dict[str, Any]: + dataset = os.path.join(path, "projects", project) + directory = os.path.join(dataset, dir_in_project) + if files is None: + files = [] + for root, ds, fs in os.walk(directory): + for f in fs: + if fnmatch.fnmatch(f, prefix + "*" + postfix + ".dat"): + files.append(f) + dl.get([os.path.join(directory, f) for f in files], dataset=dataset) + kwargs: dict[str, Any] = {} + if names is not None: + kwargs['names'] = names + if files is not None: + kwargs['files'] = files + if postfix is not None: + kwargs['postfix'] = postfix + kwargs['plot_fit'] = False + + t0 = input.extract_t0(directory, + prefix, + dtr_read, + xmin, + spatial_extent, + fit_range=fit_range, + c=0.3, + **kwargs + ) + par_list= [] + for k in ["integrator", "eps", "ntot", "dnms"]: + par_list.append(str(param[k])) + pars = "/".join(par_list) + t0_dict: dict[str, Any] = {} + t0_dict[param["type"]] = {} + t0_dict[param["type"]][pars] = t0 + return t0_dict + + +def extract_t1(path: str, project: str, dir_in_project: str, param: dict[str, Any], prefix: str, dtr_read: int, xmin: int, spatial_extent: int, fit_range: int = 5, postfix: str = None, names: list[str]=None, files: list[str]=None) -> dict[str, Any]: + directory = os.path.join(path, "projects", project, dir_in_project) + if files is None: + files = [] + for root, ds, fs in os.walk(directory): + for f in fs: + if fnmatch.fnmatch(f, prefix + "*" + postfix + ".dat"): + files.append(f) + kwargs: dict[str, Any] = {} + if names is not None: + kwargs['names'] = names + if files is not None: + kwargs['files'] = files + if postfix is not None: + kwargs['postfix'] = postfix + kwargs['plot_fit'] = False + t0 = input.extract_t0(directory, + prefix, + dtr_read, + xmin, + spatial_extent, + fit_range=fit_range, + c=2./3, + **kwargs + ) + par_list= [] + for k in ["integrator", "eps", "ntot", "dnms"]: + par_list.append(str(param[k])) + pars = "/".join(par_list) + t0_dict: dict[str, Any] = {} + t0_dict[param["type"]] = {} + t0_dict[param["type"]][pars] = t0 + return t0_dict diff --git a/corrlib/input/sfcf.py b/corrlib/input/sfcf.py index 403d17a..bd4ae4d 100644 --- a/corrlib/input/sfcf.py +++ b/corrlib/input/sfcf.py @@ -232,7 +232,6 @@ def get_specs(key, parameters, sep='/') -> str: param = _map_params(parameters, key_parts[1:-1]) else: param = _map_params(parameters, key_parts[1:]) - print(param) s = json.dumps(param) return s @@ -305,8 +304,7 @@ def read_data(path, project, dir_in_project, prefix, param, version='1.0c', cfg_ range(len(param['wf_basis'])), range(len(param['wf_basis'])), version, cfg_seperator, keyed_out=True) for key in data_crr.keys(): data[key] = data_crr[key] - # print("Read data:", data_crr) - # print(f"Read data: pe.input.sfcf.read_sfcf_multi({directory}, {prefix}, {param['crr']}, {param['mrr']}, {corr_type_list}, {range(len(param['wf_offsets']))}, {range(len(param['wf_basis']))}, {range(len(param['wf_basis']))}, {version}, {cfg_seperator}, keyed_out=True, names={names})") + if not param['crs'] == []: data_crs = pe.input.sfcf.read_sfcf_multi(directory, param['crs']) for key in data_crs.keys(): diff --git a/corrlib/meas_io.py b/corrlib/meas_io.py index 7c9a32e..dc8fadc 100644 --- a/corrlib/meas_io.py +++ b/corrlib/meas_io.py @@ -9,7 +9,7 @@ from pyerrors import Obs, Corr from hashlib import sha256 -def write_measurement(path, ensemble, measurement, uuid, code, parameter_file): +def write_measurement(path, ensemble, measurement, uuid, code, parameter_file=None): """ Write a measurement to the backlog. If the file for the measurement already exists, update the measurement. @@ -49,17 +49,33 @@ def write_measurement(path, ensemble, measurement, uuid, code, parameter_file): pars[subkey] = sfcf.get_specs(corr + "/" + subkey, parameters) elif code == "openQCD": - parameters = openQCD.read_param(path, uuid, parameter_file) - pars = {} - subkeys = [] - for i in range(len(parameters["rw_fcts"])): - par_list = [] - for k in parameters["rw_fcts"][i].keys(): - par_list.append(str(parameters["rw_fcts"][i][k])) + ms_type = list(measurement.keys())[0] + if ms_type == 'ms1': + parameters = openQCD.read_ms1_param(path, uuid, parameter_file) + pars = {} + subkeys = [] + for i in range(len(parameters["rw_fcts"])): + par_list = [] + for k in parameters["rw_fcts"][i].keys(): + par_list.append(str(parameters["rw_fcts"][i][k])) + subkey = "/".join(par_list) + subkeys.append(subkey) + pars[subkey] = json.dumps(parameters["rw_fcts"][i]) + elif ms_type in ['t0', 't1']: + if parameter_file is not None: + parameters = openQCD.read_ms3_param(path, uuid, parameter_file) + else: + parameters = {} + for rwp in ["integrator", "eps", "ntot", "dnms"]: + parameters[rwp] = "Unknown" + pars = {} + subkeys = [] + par_list= [] + for k in ["integrator", "eps", "ntot", "dnms"]: + par_list.append(str(parameters[k])) subkey = "/".join(par_list) - subkeys.append(subkey) - pars[subkey] = json.dumps(parameters["rw_fcts"][i]) - + subkeys = [subkey] + pars[subkey] = json.dumps(parameters) for subkey in subkeys: parHash = sha256(str(pars[subkey]).encode('UTF-8')).hexdigest() meas_path = file_in_archive + "::" + parHash diff --git a/corrlib/toml.py b/corrlib/toml.py index 9735bad..0004a4e 100644 --- a/corrlib/toml.py +++ b/corrlib/toml.py @@ -36,7 +36,7 @@ def check_measurement_data(measurements: dict, code: str) -> None: if code == "sfcf": var_names = ["path", "ensemble", "param_file", "version", "prefix", "cfg_seperator", "names"] elif code == "openQCD": - var_names = ["path", "ensemble", "measurement", "prefix", "param_file"] + var_names = ["path", "ensemble", "measurement", "prefix"] # , "param_file" for mname, md in measurements.items(): for var_name in var_names: if var_name not in md.keys(): @@ -92,11 +92,26 @@ def import_toml(path: str, file: str, copy_file: bool=True) -> None: version=md['version'], cfg_seperator=md['cfg_seperator'], sep='/') print(mname + " imported.") elif project['code'] == 'openQCD': - param = openQCD.read_param(path, uuid, md['param_file']) - param['type'] = md['measurement'] - measurement = openQCD.read_rwms(path, uuid, md['path'], param, md["prefix"], version=md["version"], names=md['names'], files=md['files']) + if md['measurement'] == 'ms1': + param = openQCD.read_ms1_param(path, uuid, md['param_file']) + param['type'] = 'ms1' + measurement = openQCD.read_rwms(path, uuid, md['path'], param, md["prefix"], version=md["version"], names=md['names'], files=md['files']) + elif md['measurement'] == 't0': + if 'param_file' in md: + param = openQCD.read_ms3_param(path, uuid, md['param_file']) + else: + param = {} + for rwp in ["integrator", "eps", "ntot", "dnms"]: + param[rwp] = "Unknown" + param['type'] = 't0' + measurement = openQCD.extract_t0(path, uuid, md['path'], param, md["prefix"], md["dtr_read"], md["xmin"], md["spatial_extent"], fit_range=md.get('fit_range', 5), postfix=md.get('postfix', None), names=md.get('names', None)) + elif md['measurement'] == 't1': + if 'param_file' in md: + param = openQCD.read_ms3_param(path, uuid, md['param_file']) + param['type'] = 't1' + measurement = openQCD.extract_t1(path, uuid, md['path'], param, md["prefix"], md["dtr_read"], md["xmin"], md["spatial_extent"], fit_range=md.get('fit_range', 5), postfix=md.get('postfix', None), names=md.get('names', None)) - write_measurement(path, ensemble, measurement, uuid, project['code'], md['param_file']) + write_measurement(path, ensemble, measurement, uuid, project['code'], (md['param_file'] if 'param_file' in md else None)) if not os.path.exists(os.path.join(path, "toml_imports", uuid)): os.makedirs(os.path.join(path, "toml_imports", uuid))