""" TOML interface -------------- Remember, that keys with dots have to be quoted. Apart from improting projects yourdelf with python scripts, this package also allows for the import of projects via TOML. """ import tomllib as toml import shutil import datalad.api as dl from .tracker import save from .input import sfcf, openQCD from .main import import_project, update_aliases from .meas_io import write_measurement import os from .input.implementations import codes as known_codes from typing import Any from pathlib import Path def replace_string(string: str, name: str, val: str) -> str: """ Replace a placeholder {name} with a value in a string. Parameters ---------- string: str String in which the placeholders are to be replaced. name: str The name of the placeholder. val: str The value the placeholder is to be replaced with. """ if '{' + name + '}' in string: n = string.replace('{' + name + '}', val) return n else: return string def replace_in_meas(measurements: dict[str, dict[str, Any]], vars: dict[str, str]) -> dict[str, dict[str, Any]]: """ Replace placeholders in the defiitions for a measurement. Parameters ---------- measurements: dict[str, dict[str, Any]] The measurements read from the toml file. vars: dict[str, str] Simple key:value dictionary with the keys to be replaced by the values. """ for name, value in vars.items(): for m in measurements.keys(): for key in measurements[m].keys(): if isinstance(measurements[m][key], str): measurements[m][key] = replace_string(measurements[m][key], name, value) elif isinstance(measurements[m][key], list): for i in range(len(measurements[m][key])): measurements[m][key][i] = replace_string(measurements[m][key][i], name, value) return measurements def fill_cons(measurements: dict[str, dict[str, Any]], constants: dict[str, str]) -> dict[str, dict[str, Any]]: """ Fill in defined constants into the measurements of the toml-file. Parameters ---------- measurements: dict[str, dict[str, Any]] The measurements read from the toml file. constants: dict[str, str] Simple key:value dictionary with the keys to be replaced by the values. """ for m in measurements.keys(): for name, val in constants.items(): if name not in measurements[m].keys(): measurements[m][name] = val return measurements def check_project_data(d: dict[str, dict[str, str]]) -> None: """ Check the data given in the toml import file for the project we want to import. Parameters ---------- d: dict The dictionary holding the data necessary to import the project. """ if 'project' not in d.keys() or 'measurements' not in d.keys() or len(list(d.keys())) > 4: raise ValueError('There should only be maximally be four keys on the top level, "project" and "measurements" are mandatory, "contants" is optional!') project_data = d['project'] if 'url' not in project_data.keys(): raise ValueError('project.url is missing!') if 'code' not in project_data.keys(): raise ValueError('project.code is missing!') if 'measurements' not in d.keys(): raise ValueError('No measurements to import!') return def check_measurement_data(measurements: dict[str, dict[str, str]], code: str) -> None: """ Check syntax of the measurements we want to import. Parameters ---------- measurements: dict[str, dict[str, str]] The dictionary holding the necessary data to import the project. code: str The code used for the project. """ var_names: list[str] = [] if code == "sfcf": var_names = ["path", "ensemble", "param_file", "version", "prefix", "cfg_seperator", "names"] elif code == "openQCD": var_names = ["path", "ensemble", "measurement", "prefix"] # , "param_file" for mname, md in measurements.items(): for var_name in var_names: if var_name not in md.keys(): raise ImportError("Measurment '" + mname + "' does not possess nessecary variable '" + var_name + "'. \ Please add this to the measurements definition.") return def import_tomls(path: Path, files: list[str], copy_files: bool=True) -> None: """ Import multiple toml files. Parameters ---------- path: str Path to the backlog directory. files: list[str] Path to the description files. copy_files: bool, optional Whether the toml-files will be copied into the library. Default is True. """ for file in files: import_toml(path, file, copy_files) return def import_toml(path: Path, file: str, copy_file: bool=True) -> None: """ Import a project decribed by a .toml file. Parameters ---------- path: str Path to the backlog directory. file: str Path to the description file. copy_file: bool, optional Whether the toml-files will be copied into the library. Default is True. """ if not os.path.exists(path): raise FileNotFoundError(f"Corrlib path {path} does not exist.") if not os.path.exists(file): raise FileNotFoundError(f".toml-file {file} does not exist.") print("Import project as decribed in " + file) with open(file, 'rb') as fp: toml_dict = toml.load(fp) check_project_data(toml_dict) project: dict[str, Any] = toml_dict['project'] if project['code'] not in known_codes: raise ValueError('Code' + project['code'] + 'has no import implementation!') measurements: dict[str, dict[str, Any]] = toml_dict['measurements'] measurements = fill_cons(measurements, toml_dict['constants'] if 'constants' in toml_dict else {}) measurements = replace_in_meas(measurements, toml_dict['replace'] if 'replace' in toml_dict else {}) check_measurement_data(measurements, project['code']) aliases = project.get('aliases', []) uuid = project.get('uuid', None) if uuid is not None: if not os.path.exists(path / "projects" / uuid): uuid = import_project(path, project['url'], aliases=aliases) else: update_aliases(path, uuid, aliases) else: uuid = import_project(path, project['url'], aliases=aliases) imeas = 1 nmeas = len(measurements.keys()) for mname, md in measurements.items(): print(f"Import measurement {imeas}/{nmeas}: {mname}") ensemble = md['ensemble'] if project['code'] == 'sfcf': param = sfcf.read_param(path, uuid, md['param_file']) if 'names' in md.keys(): measurement = sfcf.read_data(path, uuid, md['path'], md['prefix'], param, version=md['version'], cfg_seperator=md['cfg_seperator'], sep='/', names=md['names']) else: measurement = sfcf.read_data(path, uuid, md['path'], md['prefix'], param, version=md['version'], cfg_seperator=md['cfg_seperator'], sep='/') elif project['code'] == 'openQCD': if md['measurement'] == 'ms1': if 'param_file' in md.keys(): parameter_file = md['param_file'] if parameter_file.endswith(".ms1.in"): param = openQCD.load_ms1_infile(path, uuid, parameter_file) elif parameter_file.endswith(".ms1.par"): param = openQCD.load_ms1_parfile(path, uuid, parameter_file) else: # Temporary solution parameters: dict[str, Any] = {} parameters["rand"] = {} parameters["rw_fcts"] = [{}] for nrw in range(1): if "nsrc" not in parameters["rw_fcts"][nrw]: parameters["rw_fcts"][nrw]["nsrc"] = 1 if "mu" not in parameters["rw_fcts"][nrw]: parameters["rw_fcts"][nrw]["mu"] = "None" if "np" not in parameters["rw_fcts"][nrw]: parameters["rw_fcts"][nrw]["np"] = "None" if "irp" not in parameters["rw_fcts"][nrw]: parameters["rw_fcts"][nrw]["irp"] = "None" param = parameters param['type'] = 'ms1' measurement = openQCD.read_rwms(path, uuid, md['path'], param, md["prefix"], version=md["version"], names=md['names'], files=md['files']) elif md['measurement'] == 't0': if 'param_file' in md: param = openQCD.load_ms3_infile(path, uuid, md['param_file']) else: param = {} for rwp in ["integrator", "eps", "ntot", "dnms"]: param[rwp] = "Unknown" param['type'] = 't0' measurement = openQCD.extract_t0(path, uuid, md['path'], param, str(md["prefix"]), int(md["dtr_read"]), int(md["xmin"]), int(md["spatial_extent"]), fit_range=int(md.get('fit_range', 5)), postfix=str(md.get('postfix', '')), names=md.get('names', []), files=md.get('files', []), r_start=md.get('r_start', []), r_stop=md.get('r_stop', []), r_step=md.get('r_step', 1)) elif md['measurement'] == 't1': if 'param_file' in md: param = openQCD.load_ms3_infile(path, uuid, md['param_file']) param['type'] = 't1' measurement = openQCD.extract_t1(path, uuid, md['path'], param, str(md["prefix"]), int(md["dtr_read"]), int(md["xmin"]), int(md["spatial_extent"]), fit_range=int(md.get('fit_range', 5)), postfix=str(md.get('postfix', '')), names=md.get('names', []), files=md.get('files', []), r_start=md.get('r_start', []), r_stop=md.get('r_stop', []), r_step=md.get('r_step', 1)) write_measurement(path, ensemble, measurement, uuid, project['code'], (md['param_file'] if 'param_file' in md else None)) imeas += 1 print(mname + " imported.") if not os.path.exists(path / "toml_imports" / uuid): os.makedirs(path / "toml_imports" / uuid) if copy_file: import_file = path / "toml_imports" / uuid / file.split("/")[-1] shutil.copy(file, import_file) save(path, files=[import_file], message=f"Import using {import_file}") print(f"File copied to {import_file}") print("Imported project.") return def reimport_project(path: Path, uuid: str) -> None: """ Reimport an existing project using the files that are already available for this project. Parameters ---------- path: str Path to repository uuid: str uuid of the project that is to be reimported. """ config_path = path / "import_scripts" / uuid for p, filenames, dirnames in os.walk(config_path): for fname in filenames: import_toml(path, os.path.join(config_path, fname), copy_file=False) return def update_project(path: Path, uuid: str) -> None: """ Update all entries associated with a given project. Parameters ---------- path: str The path of the library. uuid: str The unique identifier of the project to be updated. """ dl.update(how='merge', follow='sibling', dataset=os.path.join(path, "projects", uuid)) # reimport_project(path, uuid) return