diff --git a/.gitignore b/.gitignore index f97ff98..cff0601 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ test.ipynb .vscode .venv .pytest_cache -.coverage \ No newline at end of file +.coverage +build diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..1889fd3 --- /dev/null +++ b/LICENSE @@ -0,0 +1,9 @@ +MIT License + +Copyright (c) 2026 Justus Kuhlmann + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice (including the next paragraph) shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/corrlib/__init__.py b/corrlib/__init__.py index 4e1b364..7cd9c9d 100644 --- a/corrlib/__init__.py +++ b/corrlib/__init__.py @@ -15,10 +15,11 @@ For now, we are interested in collecting primary IObservables only, as these are __app_name__ = "corrlib" -from .import input as input -from .initialization import create as create -from .meas_io import load_record as load_record -from .meas_io import load_records as load_records +from . import input as input from .find import find_project as find_project from .find import find_record as find_record from .find import list_projects as list_projects +from .initialization import create as create +from .meas_io import load_record as load_record +from .meas_io import load_records as load_records +from .toml import import_toml \ No newline at end of file diff --git a/corrlib/__main__.py b/corrlib/__main__.py index 24f9c83..e719ee7 100644 --- a/corrlib/__main__.py +++ b/corrlib/__main__.py @@ -1,4 +1,4 @@ -from corrlib import cli, __app_name__ +from corrlib import __app_name__, cli def main() -> None: diff --git a/corrlib/cli.py b/corrlib/cli.py index a28c837..9d7fcfe 100644 --- a/corrlib/cli.py +++ b/corrlib/cli.py @@ -1,19 +1,18 @@ -from typing import Optional -import typer -from corrlib import __app_name__ - -from .initialization import create -from .toml import import_tomls, update_project, reimport_project -from .find import find_record, list_projects, list_ensembles, get_stat -from .tools import str2list -from .main import update_aliases -from .meas_io import drop_cache as mio_drop_cache -from .integrity import full_integrity_check - import os from importlib.metadata import version from pathlib import Path +import typer + +from corrlib import __app_name__ + +from .find import find_record, get_stat, list_ensembles, list_projects +from .initialization import create +from .integrity import full_integrity_check +from .main import update_aliases +from .meas_io import drop_cache as mio_drop_cache +from .toml import import_tomls, reimport_project, update_project +from .tools import str2list app = typer.Typer() @@ -26,7 +25,7 @@ def _version_callback(value: bool) -> None: @app.command() def update( - path: Path = typer.Option( + path: Path = typer.Option( # noqa: B008 Path('.'), "--dataset", "-d", @@ -42,7 +41,7 @@ def update( @app.command() def lister( - path: Path = typer.Option( + path: Path = typer.Option( # noqa: B008 Path('.'), "--dataset", "-d", @@ -73,7 +72,7 @@ def lister( @app.command() def alias_add( - path: Path = typer.Option( + path: Path = typer.Option( # noqa: B008 Path('.'), "--dataset", "-d", @@ -91,7 +90,7 @@ def alias_add( @app.command() def find( - path: Path = typer.Option( + path: Path = typer.Option( # noqa: B008 Path('.'), "--dataset", "-d", @@ -100,7 +99,7 @@ def find( corr: str = typer.Argument(), code: str = typer.Argument(), arg: str = typer.Option( - str('all'), + 'all', "--argument", "-a", ), @@ -125,7 +124,7 @@ def find( @app.command() def stat( - path: Path = typer.Option( + path: Path = typer.Option( # noqa: B008 Path('.'), "--dataset", "-d", @@ -141,7 +140,7 @@ def stat( @app.command() -def check(path: Path = typer.Option( +def check(path: Path = typer.Option( # noqa: B008 Path('.'), "--dataset", "-d", @@ -155,7 +154,7 @@ def check(path: Path = typer.Option( @app.command() def importer( - path: Path = typer.Option( + path: Path = typer.Option( # noqa: B008 Path('.'), "--dataset", "-d", @@ -163,7 +162,7 @@ def importer( files: str = typer.Argument( ), copy_file: bool = typer.Option( - bool(True), + True, "--save", "-s", ), @@ -179,7 +178,7 @@ def importer( @app.command() def reimporter( - path: Path = typer.Option( + path: Path = typer.Option( # noqa: B008 Path('.'), "--dataset", "-d", @@ -204,13 +203,13 @@ def reimporter( @app.command() def init( - path: Path = typer.Option( + path: Path = typer.Option( # noqa: B008 Path('.'), "--dataset", "-d", ), tracker: str = typer.Option( - str('datalad'), + 'datalad', "--tracker", "-t", ), @@ -224,7 +223,7 @@ def init( @app.command() def drop_cache( - path: Path = typer.Option( + path: Path = typer.Option( # noqa: B008 Path('.'), "--dataset", "-d", @@ -239,7 +238,7 @@ def drop_cache( @app.callback() def main( - version: Optional[bool] = typer.Option( + version: bool | None = typer.Option( None, "--version", "-v", diff --git a/corrlib/find.py b/corrlib/find.py index af21a4d..18c4001 100644 --- a/corrlib/find.py +++ b/corrlib/find.py @@ -1,21 +1,23 @@ -import sqlite3 -import os -import json -import pandas as pd -import numpy as np -from .input.implementations import codes -from .tools import k2m, get_db_file -from .tracker import get -from .integrity import has_valid_times -from .sql import thin_sql_wrapper -from typing import Any, Optional -from pathlib import Path import datetime as dt -from collections.abc import Callable +import json +import os +import sqlite3 import warnings -from .meas_io import load_record +from collections.abc import Callable +from pathlib import Path +from typing import Any + +import numpy as np +import pandas as pd from pyerrors import Corr, Obs +from .input.implementations import codes +from .integrity import has_valid_times +from .meas_io import load_record +from .sql import thin_sql_wrapper +from .tools import get_db_file, k2m +from .tracker import get + def _project_lookup_by_alias(path: Path, alias: str) -> str: """ @@ -63,7 +65,7 @@ def _project_lookup_by_id(path: Path, uuid: str) -> list[tuple[str, ...]]: return results -def _time_filter(results: pd.DataFrame, created_before: Optional[str]=None, created_after: Optional[Any]=None, updated_before: Optional[Any]=None, updated_after: Optional[Any]=None) -> pd.DataFrame: +def _time_filter(results: pd.DataFrame, created_before: str | None=None, created_after: str | None=None, updated_before: str | None=None, updated_after: str | None=None) -> pd.DataFrame: """ Filter the results from the database in terms of the creation and update times. @@ -112,7 +114,7 @@ def _time_filter(results: pd.DataFrame, created_before: Optional[str]=None, cre return results.drop(drops) -def _db_lookup(db: Path, ensemble: str, correlator_name: str, code: str, project: Optional[str]=None, parameters: Optional[str]=None) -> pd.DataFrame: +def _db_lookup(db: Path, ensemble: str, correlator_name: str, code: str, project: str | None=None, parameters: str | None=None) -> pd.DataFrame: """ Look up a correlator record in the database by the data given to the method. @@ -286,7 +288,7 @@ def openQCD_filter(results:pd.DataFrame, **kwargs: Any) -> pd.DataFrame: The filtered results. """ - warnings.warn("A filter for openQCD parameters is no implemented yet.", Warning) + warnings.warn("A filter for openQCD parameters is no implemented yet.", Warning, 1) return results @@ -319,10 +321,10 @@ def _code_filter(results: pd.DataFrame, code: str, **kwargs: Any) -> pd.DataFram raise ValueError(f"Code {code} is not known.") -def find_record(path: Path, ensemble: str, correlator_name: str, code: str, project: Optional[str]=None, parameters: Optional[str]=None, - created_before: Optional[str]=None, created_after: Optional[str]=None, updated_before: Optional[str]=None, updated_after: Optional[str]=None, - revision: Optional[str]=None, - customFilter: Optional[Callable[[pd.DataFrame], pd.DataFrame]] = None, +def find_record(path: Path, ensemble: str, correlator_name: str, code: str, project: str | None=None, parameters: str | None=None, + created_before: str | None=None, created_after: str | None=None, updated_before: str | None=None, updated_after: str | None=None, + revision: str | None=None, + customFilter: Callable[[pd.DataFrame], pd.DataFrame] | None = None, **kwargs: Any) -> pd.DataFrame: path = Path(path) db_file = get_db_file(path) diff --git a/corrlib/git_tools.py b/corrlib/git_tools.py index d77f109..7808ada 100644 --- a/corrlib/git_tools.py +++ b/corrlib/git_tools.py @@ -1,8 +1,10 @@ import os -from .tracker import save -import git from pathlib import Path +import git + +from .tracker import save + GITMODULES_FILE = '.gitmodules' @@ -25,7 +27,7 @@ def move_submodule(repo_path: Path, old_path: Path, new_path: Path) -> None: gitmodules_file_path = repo_path / GITMODULES_FILE # update paths in .gitmodules - with open(gitmodules_file_path, 'r') as file: + with open(gitmodules_file_path) as file: lines = [line.strip() for line in file] updated_lines = [] diff --git a/corrlib/initialization.py b/corrlib/initialization.py index bdf9cee..83a3b44 100644 --- a/corrlib/initialization.py +++ b/corrlib/initialization.py @@ -1,9 +1,10 @@ -from configparser import ConfigParser -import sqlite3 import os -from .tracker import save, init +import sqlite3 +from configparser import ConfigParser from pathlib import Path + from .tools import CONFIG_FILENAME +from .tracker import init, save def _create_db(db: Path) -> None: @@ -71,6 +72,7 @@ def _create_config(path: Path, tracker: str, cached: bool) -> ConfigParser: 'db': 'backlogger.db', 'projects_path': 'projects', 'archive_path': 'archive', + 'plot_path': 'plots', 'toml_imports_path': 'toml_imports', 'import_scripts_path': 'import_scripts', } @@ -113,6 +115,7 @@ def create(path: Path, tracker: str = 'datalad', cached: bool = True) -> None: os.chmod(path / config['paths']['db'], 0o666) os.makedirs(path / config['paths']['projects_path']) os.makedirs(path / config['paths']['archive_path']) + os.makedirs(path / config['paths']['plot_path']) os.makedirs(path / config['paths']['toml_imports_path']) os.makedirs(path / config['paths']['import_scripts_path'] / 'template.py') with open(path / ".gitignore", "w") as fp: diff --git a/corrlib/input/__init__.py b/corrlib/input/__init__.py index be6d6b2..3ccbf28 100644 --- a/corrlib/input/__init__.py +++ b/corrlib/input/__init__.py @@ -2,6 +2,6 @@ Import functions for different codes. """ -from . import sfcf as sfcf -from . import openQCD as openQCD from . import implementations as implementations +from . import openQCD as openQCD +from . import sfcf as sfcf diff --git a/corrlib/input/openQCD.py b/corrlib/input/openQCD.py index c8eef72..eb17fd3 100644 --- a/corrlib/input/openQCD.py +++ b/corrlib/input/openQCD.py @@ -1,12 +1,14 @@ -import pyerrors.input.openQCD as input -import datalad.api as dl -import os import fnmatch -from typing import Any, Optional +import os from pathlib import Path -from ..pars.openQCD import ms1 -from ..pars.openQCD import qcd2 +from typing import Any +import datalad.api as dl +import matplotlib.pyplot as plt +import pyerrors.input.openQCD as input + +from ..pars.openQCD import ms1, qcd2 +from ..tools import get_plot_dir def load_ms1_infile(path: Path, project: str, file_in_project: str) -> dict[str, Any]: @@ -29,16 +31,18 @@ def load_ms1_infile(path: Path, project: str, file_in_project: str) -> dict[str, """ file = os.path.join(path, "projects", project, file_in_project) + if not os.path.exists(file): + raise OSError(f"File {file} does not exist.") ds = os.path.join(path, "projects", project) dl.get(file, dataset=ds) - with open(file, 'r') as fp: + with open(file) as fp: lines = fp.readlines() fp.close() param: dict[str, Any] = {} param['rw_fcts'] = [] param['rand'] = {} - for i, line in enumerate(lines): + for line in lines: if line.startswith('#'): continue if line.startswith('\n'): @@ -95,7 +99,7 @@ def load_ms3_infile(path: Path, project: str, file_in_project: str) -> dict[str, file = os.path.join(path, "projects", project, file_in_project) ds = os.path.join(path, "projects", project) dl.get(file, dataset=ds) - with open(file, 'r') as fp: + with open(file) as fp: lines = fp.readlines() fp.close() param = {} @@ -107,7 +111,7 @@ def load_ms3_infile(path: Path, project: str, file_in_project: str) -> dict[str, return param -def read_rwms(path: Path, project: str, dir_in_project: str, param: dict[str, Any], prefix: str, postfix: str="ms1", version: str='2.0', names: Optional[list[str]]=None, files: Optional[list[str]]=None) -> dict[str, Any]: +def read_rwms(path: Path, project: str, dir_in_project: str, param: dict[str, Any], prefix: str, postfix: str="ms1", version: str='2.0', names: list[str] | None=None, files: list[str] | None=None) -> dict[str, Any]: """ Read reweighting factor measurements from the project. @@ -142,7 +146,7 @@ def read_rwms(path: Path, project: str, dir_in_project: str, param: dict[str, An directory = os.path.join(dataset, dir_in_project) if files is None: files = [] - for root, ds, fs in os.walk(directory): + for _root, _ds, fs in os.walk(directory): for f in fs: if fnmatch.fnmatch(f, prefix + "*" + postfix + ".dat"): files.append(f) @@ -164,8 +168,8 @@ def read_rwms(path: Path, project: str, dir_in_project: str, param: dict[str, An return rw_dict -def extract_t0(path: Path, project: str, dir_in_project: str, param: dict[str, Any], prefix: str, dtr_read: int, xmin: int, spatial_extent: int, fit_range: int = 5, postfix: str="", names: Optional[list[str]]=None, files: Optional[list[str]]=None, - r_start: list[int]=[], r_stop: list[int]=[], r_step:int=1) -> dict[str, Any]: +def extract_t0(path: Path, project: str, dir_in_project: str, ensemble: str, param: dict[str, Any], prefix: str, dtr_read: int, xmin: int, spatial_extent: int, fit_range: int = 5, postfix: str="", names: list[str] | None=None, files: list[str] | None=None, + r_start: list[int] | None=None, r_stop: list[int] | None=None, r_step:int=1) -> dict[str, Any]: """ Extract t0 measurements from the project. @@ -202,11 +206,15 @@ def extract_t0(path: Path, project: str, dir_in_project: str, param: dict[str, A Dictionary of t0 values in the pycorrlib style, with the parameters at hand. """ + if r_stop is None: + r_stop = [] + if r_start is None: + r_start = [] dataset = os.path.join(path, "projects", project) directory = os.path.join(dataset, dir_in_project) if files is None: files = [] - for root, ds, fs in os.walk(directory): + for _root, _ds, fs in os.walk(directory): for f in fs: if fnmatch.fnmatch(f, prefix + "*" + postfix + ".dat"): files.append(f) @@ -224,7 +232,7 @@ def extract_t0(path: Path, project: str, dir_in_project: str, param: dict[str, A if not r_stop == []: kwargs['r_stop'] = r_stop kwargs['r_step'] = r_step - + kwargs['plot_fit'] = True t0 = input.extract_t0(directory, prefix, dtr_read, @@ -234,6 +242,10 @@ def extract_t0(path: Path, project: str, dir_in_project: str, param: dict[str, A c=0.3, **kwargs ) + plot_dir = path / get_plot_dir(path) / ensemble / project + if not os.path.exists(plot_dir): + os.makedirs(plot_dir) + plt.savefig(plot_dir / "t0.pdf") par_list= [] for k in ["integrator", "eps", "ntot", "dnms"]: par_list.append(str(param[k])) @@ -244,8 +256,8 @@ def extract_t0(path: Path, project: str, dir_in_project: str, param: dict[str, A return t0_dict -def extract_t1(path: Path, project: str, dir_in_project: str, param: dict[str, Any], prefix: str, dtr_read: int, xmin: int, spatial_extent: int, fit_range: int = 5, postfix: str = "", names: Optional[list[str]]=None, files: Optional[list[str]]=None, - r_start: list[int]=[], r_stop: list[int]=[], r_step:int=1) -> dict[str, Any]: +def extract_t1(path: Path, project: str, dir_in_project: str, ensemble: str, param: dict[str, Any], prefix: str, dtr_read: int, xmin: int, spatial_extent: int, fit_range: int = 5, postfix: str = "", names: list[str] | None=None, files: list[str] | None=None, + r_start: list[int] | None=None, r_stop: list[int] | None=None, r_step:int=1) -> dict[str, Any]: """ Extract t1 measurements from the project. @@ -282,10 +294,14 @@ def extract_t1(path: Path, project: str, dir_in_project: str, param: dict[str, A Dictionary of t1 values in the pycorrlib style, with the parameters at hand. """ + if r_stop is None: + r_stop = [] + if r_start is None: + r_start = [] directory = os.path.join(path, "projects", project, dir_in_project) if files is None: files = [] - for root, ds, fs in os.walk(directory): + for _root, _ds, fs in os.walk(directory): for f in fs: if fnmatch.fnmatch(f, prefix + "*" + postfix + ".dat"): files.append(f) @@ -302,6 +318,7 @@ def extract_t1(path: Path, project: str, dir_in_project: str, param: dict[str, A if not r_stop == []: kwargs['r_stop'] = r_stop kwargs['r_step'] = r_step + kwargs['plot_fit'] = True t0 = input.extract_t0(directory, prefix, dtr_read, @@ -311,6 +328,10 @@ def extract_t1(path: Path, project: str, dir_in_project: str, param: dict[str, A c=2./3, **kwargs ) + plot_dir = path / get_plot_dir(path) / ensemble / project + if not os.path.exists(plot_dir): + os.makedirs(plot_dir) + plt.savefig(plot_dir / "t1.pdf") par_list= [] for k in ["integrator", "eps", "ntot", "dnms"]: par_list.append(str(param[k])) diff --git a/corrlib/input/sfcf.py b/corrlib/input/sfcf.py index acd8261..a12661d 100644 --- a/corrlib/input/sfcf.py +++ b/corrlib/input/sfcf.py @@ -1,11 +1,11 @@ -import pyerrors as pe -import datalad.api as dl import json import os -from typing import Any from fnmatch import fnmatch from pathlib import Path +from typing import Any +import datalad.api as dl +import pyerrors as pe bi_corrs: list[str] = ["f_P", "fP", "f_p", "g_P", "gP", "g_p", @@ -99,7 +99,7 @@ def read_param(path: Path, project: str, file_in_project: str) -> dict[str, Any] file = path / "projects" / project / file_in_project dl.get(file, dataset=path) - with open(file, 'r') as f: + with open(file) as f: lines = f.readlines() params: dict[str, Any] = {} @@ -258,7 +258,7 @@ def get_specs(key: str, parameters: dict[str, Any], sep: str = '/') -> str: return s -def read_data(path: Path, project: str, dir_in_project: str, prefix: str, param: dict[str, Any], version: str = '1.0c', cfg_seperator: str = 'n', sep: str = '/', **kwargs: Any) -> dict[str, Any]: +def read_data(path: Path, project: str, dir_in_project: str, prefix: str, param: dict[str, Any], version: str = '1.0c', cfg_separator: str = 'n', sep: str = '/', **kwargs: Any) -> dict[str, Any]: """ Extract the data from the sfcf file. @@ -274,7 +274,7 @@ def read_data(path: Path, project: str, dir_in_project: str, prefix: str, param: The parameter dictionary, as given by read_param. version: str Version of sfcf. - cfg_seperator: str + cfg_separator: str Separator of the configuration number. Needed for reading. default: "n" sep: str Seperator for the key in return dict. (default: "/) @@ -291,7 +291,7 @@ def read_data(path: Path, project: str, dir_in_project: str, prefix: str, param: appended = (version[-1] == "a") ls = [] files_to_get = [] - for (dirpath, dirnames, filenames) in os.walk(directory): + for _dirpath, dirnames, filenames in os.walk(directory): if not appended: ls.extend(dirnames) else: @@ -299,7 +299,7 @@ def read_data(path: Path, project: str, dir_in_project: str, prefix: str, param: break if not appended: compact = (version[-1] == "c") - for i, item in enumerate(ls): + for item in ls: if fnmatch(item, prefix + "*"): rep_path = directory + '/' + item sub_ls = pe.input.sfcf._find_files(rep_path, prefix, compact, []) @@ -321,10 +321,10 @@ def read_data(path: Path, project: str, dir_in_project: str, prefix: str, param: if not param['crr'] == []: if names is not None: data_crr = pe.input.sfcf.read_sfcf_multi(directory, prefix, param['crr'], param['mrr'], corr_type_list, range(len(param['wf_offsets'])), - range(len(param['wf_basis'])), range(len(param['wf_basis'])), version, cfg_seperator, keyed_out=True, silent=True, names=names) + range(len(param['wf_basis'])), range(len(param['wf_basis'])), version, cfg_separator, keyed_out=True, silent=True, names=names) else: data_crr = pe.input.sfcf.read_sfcf_multi(directory, prefix, param['crr'], param['mrr'], corr_type_list, range(len(param['wf_offsets'])), - range(len(param['wf_basis'])), range(len(param['wf_basis'])), version, cfg_seperator, keyed_out=True, silent=True) + range(len(param['wf_basis'])), range(len(param['wf_basis'])), version, cfg_separator, keyed_out=True, silent=True) for key in data_crr.keys(): data[key] = data_crr[key] diff --git a/corrlib/integrity.py b/corrlib/integrity.py index 66ea4df..abffdb4 100644 --- a/corrlib/integrity.py +++ b/corrlib/integrity.py @@ -1,14 +1,15 @@ import datetime as dt -from pathlib import Path -from .tools import get_db_file, CONFIG_FILENAME -import pandas as pd -import sqlite3 -from .tracker import get -import pyerrors.input.json as pj import os +import sqlite3 from configparser import ConfigParser +from pathlib import Path from typing import Any +import pandas as pd +import pyerrors.input.json as pj + +from .tools import CONFIG_FILENAME, get_db_file +from .tracker import get path_opts = ['db', 'projects_path', 'archive_path', 'toml_imports_path', 'import_scripts_path'] diff --git a/corrlib/main.py b/corrlib/main.py index 5df8165..a0806cf 100644 --- a/corrlib/main.py +++ b/corrlib/main.py @@ -1,17 +1,18 @@ -import sqlite3 -import datalad.api as dl -import datalad.config as dlc import os -from .git_tools import move_submodule import shutil -from .find import _project_lookup_by_id -from .tools import list2str, str2list, get_db_file -from .tracker import get, save, unlock, clone, drop -from typing import Union, Optional +import sqlite3 from pathlib import Path +import datalad.api as dl +import datalad.config as dlc -def create_project(path: Path, uuid: str, owner: Union[str, None]=None, tags: Union[list[str], None]=None, aliases: Union[list[str], None]=None, code: Union[str, None]=None) -> None: +from .find import _project_lookup_by_id +from .git_tools import move_submodule +from .tools import get_db_file, list2str, str2list +from .tracker import clone, drop, get, save, unlock + + +def create_project(path: Path, uuid: str, owner: str | None=None, tags: list[str] | None=None, aliases: list[str] | None=None, code: str | None=None) -> None: """ Create a new project entry in the database. @@ -49,7 +50,7 @@ def create_project(path: Path, uuid: str, owner: Union[str, None]=None, tags: Un return -def update_project_data(path: Path, uuid: str, prop: str, value: Union[str, None] = None) -> None: +def update_project_data(path: Path, uuid: str, prop: str, value: str | None = None) -> None: """ Update/Edit a project entry in the database. Thin wrapper around sql3 call. @@ -102,7 +103,7 @@ def update_aliases(path: Path, uuid: str, aliases: list[str]) -> None: return -def import_project(path: Path, url: str, owner: Union[str, None]=None, tags: Optional[list[str]]=None, aliases: Optional[list[str]]=None, code: Optional[str]=None, isDataset: bool=True) -> str: +def import_project(path: Path, url: str, owner: str | None=None, tags: list[str] | None=None, aliases: list[str] | None=None, code: str | None=None, isDataset: bool=True) -> str: """ Import a datalad dataset into the backlogger. diff --git a/corrlib/meas_io.py b/corrlib/meas_io.py index 6b6e5f1..b5c4029 100644 --- a/corrlib/meas_io.py +++ b/corrlib/meas_io.py @@ -1,23 +1,23 @@ -from pyerrors.input import json as pj -import os -import sqlite3 -from .input import sfcf,openQCD import json -from typing import Union -from pyerrors import Obs, Corr, dump_object, load_object -from hashlib import sha256 -from .tools import get_db_file, cache_enabled -from .tracker import get, save, unlock +import os import shutil -from typing import Any +import sqlite3 +from hashlib import sha256 from pathlib import Path -from .integrity import _check_db2paths +from typing import Any +from pyerrors import Corr, Obs, dump_object, load_object +from pyerrors.input import json as pj + +from .input import openQCD, sfcf +from .integrity import _check_db2paths +from .tools import cache_enabled, get_db_file, get_plot_dir +from .tracker import get, save, unlock CACHE_DIR = ".cache" -def write_measurement(path: Path, ensemble: str, measurement: dict[str, dict[str, dict[str, Any]]], uuid: str, code: str, parameter_file: Union[str, None]) -> None: +def write_measurement(path: Path, ensemble: str, measurement: dict[str, dict[str, dict[str, Any]]], uuid: str, code: str, parameter_file: str | None, final_write: dict[str, bool]) -> None: """ Write a measurement to the backlog. If the file for the measurement already exists, update the measurement. @@ -36,6 +36,8 @@ def write_measurement(path: Path, ensemble: str, measurement: dict[str, dict[str Name of the code that was used for the project. parameter_file: str The parameter file used for the measurement. + final_write: bool + Determmines whether this is the final ime the file is touched during the current import. """ path = Path(path) db_file = get_db_file(path) @@ -52,12 +54,16 @@ def write_measurement(path: Path, ensemble: str, measurement: dict[str, dict[str for corr in measurement.keys(): file_in_archive = Path('.') / 'archive' / ensemble / corr / str(uuid + '.json.gz') file = Path(path) / file_in_archive - known_meas = {} + tmp_file_in_archive = Path('.') / 'archive' / ensemble / corr / (str(uuid) + ".p") + tmp_file = Path(path) / tmp_file_in_archive + known_meas: dict[str, Any] = {} if not os.path.exists(path / 'archive' / ensemble / corr): os.makedirs(path / 'archive' / ensemble / corr) files_to_save.append(file_in_archive) else: - if os.path.exists(file): + if os.path.exists(tmp_file): + known_meas = load_object(str(tmp_file)) + elif os.path.exists(file): if file not in files_to_save: unlock(path, file_in_archive) files_to_save.append(file_in_archive) @@ -73,7 +79,7 @@ def write_measurement(path: Path, ensemble: str, measurement: dict[str, dict[str pars[subkey] = sfcf.get_specs(corr + "/" + subkey, parameters) elif code == "openQCD": - ms_type = list(measurement.keys())[0] + ms_type = next(iter(measurement.keys())) if ms_type == 'ms1': if parameter_file is not None: if parameter_file.endswith(".ms1.in"): @@ -104,6 +110,8 @@ def write_measurement(path: Path, ensemble: str, measurement: dict[str, dict[str subkeys.append(subkey) pars[subkey] = json.dumps(parameters["rw_fcts"][i]) elif ms_type in ['t0', 't1']: + plot_file = path / get_plot_dir(path) / ensemble / uuid / (ms_type + ".pdf") + files_to_save.append(plot_file) if parameter_file is not None: parameters = openQCD.load_ms3_infile(path, uuid, parameter_file) else: @@ -130,13 +138,27 @@ def write_measurement(path: Path, ensemble: str, measurement: dict[str, dict[str c.execute("INSERT INTO backlogs (name, ensemble, code, path, project, parameters, parameter_file, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, datetime('now'), datetime('now'))", (corr, ensemble, code, meas_path, uuid, pars[subkey], parameter_file)) conn.commit() - pj.dump_dict_to_json(known_meas, str(file)) + if final_write[str(file)]: + pj.dump_dict_to_json(known_meas, str(file)) + if os.path.exists(tmp_file): + os.remove(tmp_file) + else: + dump_object(known_meas, str(tmp_file)[:-2]) conn.close() save(path, message="Add measurements to database", files=files_to_save) return -def load_record(path: Path, meas_path: str) -> Union[Corr, Obs]: +def affected_files(corrs: list[str], ensemble: str, uuid: str) -> list[Path]: + file_list = [] + for corr in corrs: + file_in_archive = Path('.') / 'archive' / ensemble / corr / str(uuid + '.json.gz') + file_list.append(file_in_archive) + file_list = list(set(file_list)) + return file_list + + +def load_record(path: Path, meas_path: str) -> Corr | Obs: """ Load a list of records by their paths. @@ -155,7 +177,7 @@ def load_record(path: Path, meas_path: str) -> Union[Corr, Obs]: return load_records(path, [meas_path])[0] -def load_records(path: Path, meas_paths: list[str], preloaded: dict[str, Any] = {}, dry_run: bool = False) -> list[Union[Corr, Obs]]: +def load_records(path: Path, meas_paths: list[str], preloaded: dict[str, Any] | None = None, dry_run: bool = False) -> list[Corr | Obs]: """ Load a list of records by their paths. @@ -175,6 +197,8 @@ def load_records(path: Path, meas_paths: list[str], preloaded: dict[str, Any] = returned_data: list The loaded records. """ + if preloaded is None: + preloaded = {} path = Path(path) if dry_run: _check_db2paths(path, meas_paths) diff --git a/corrlib/pars/openQCD/flags.py b/corrlib/pars/openQCD/flags.py index 95be919..0ab429a 100644 --- a/corrlib/pars/openQCD/flags.py +++ b/corrlib/pars/openQCD/flags.py @@ -5,6 +5,7 @@ Reconstruct the outputs of flags. import struct from typing import Any, BinaryIO + # lat_parms.c def lat_parms_write_lat_parms(fp: BinaryIO) -> dict[str, Any]: """ @@ -29,7 +30,7 @@ def lat_parms_write_lat_parms(fp: BinaryIO) -> dict[str, Any]: kappas = [] m0s = [] # read kappas - for ik in range(nk): + for _ik in range(nk): t = fp.read(8) kappas.append(struct.unpack('d', t)[0]) t = fp.read(8) diff --git a/corrlib/pars/openQCD/ms1.py b/corrlib/pars/openQCD/ms1.py index 4c2aed5..b9b3fc1 100644 --- a/corrlib/pars/openQCD/ms1.py +++ b/corrlib/pars/openQCD/ms1.py @@ -1,7 +1,7 @@ -from . import flags - -from typing import Any from pathlib import Path +from typing import Any + +from . import flags def read_qcd2_ms1_par_file(fname: Path) -> dict[str, dict[str, Any]]: diff --git a/corrlib/pars/openQCD/qcd2.py b/corrlib/pars/openQCD/qcd2.py index e73c156..121c25f 100644 --- a/corrlib/pars/openQCD/qcd2.py +++ b/corrlib/pars/openQCD/qcd2.py @@ -1,8 +1,8 @@ -from . import flags - from pathlib import Path from typing import Any +from . import flags + def read_qcd2_par_file(fname: Path) -> dict[str, dict[str, Any]]: """ diff --git a/corrlib/sql.py b/corrlib/sql.py index f45ce31..fd8e786 100644 --- a/corrlib/sql.py +++ b/corrlib/sql.py @@ -1,8 +1,9 @@ import sqlite3 -from .tools import get_db_file from pathlib import Path from typing import Any +from .tools import get_db_file + def thin_sql_wrapper(path: Path, stmt: str) -> list[Any]: db_file = get_db_file(path) diff --git a/corrlib/toml.py b/corrlib/toml.py index 1f4e300..7becb66 100644 --- a/corrlib/toml.py +++ b/corrlib/toml.py @@ -8,18 +8,20 @@ the import of projects via TOML. """ -import tomllib as toml +import os import shutil +from pathlib import Path +from typing import Any import datalad.api as dl -from .tracker import save -from .input import sfcf, openQCD -from .main import import_project, update_aliases -from .meas_io import write_measurement -import os +import tomllib as toml + +from .input import openQCD, sfcf from .input.implementations import codes as known_codes -from typing import Any -from pathlib import Path +from .main import import_project, update_aliases +from .meas_io import affected_files, write_measurement +from .tools import step_differences +from .tracker import save def replace_string(string: str, name: str, val: str) -> str: @@ -116,7 +118,7 @@ def check_measurement_data(measurements: dict[str, dict[str, str]], code: str) - """ var_names: list[str] = [] if code == "sfcf": - var_names = ["path", "ensemble", "param_file", "version", "prefix", "cfg_seperator", "names"] + var_names = ["path", "ensemble", "param_file", "version", "prefix", "cfg_separator", "names"] elif code == "openQCD": var_names = ["path", "ensemble", "measurement", "prefix"] # , "param_file" for mname, md in measurements.items(): @@ -184,19 +186,47 @@ def import_toml(path: Path, file: str, copy_file: bool=True) -> None: uuid = import_project(path, project['url'], aliases=aliases) imeas = 1 nmeas = len(measurements.keys()) - for mname, md in measurements.items(): + + # preparation step + affected_file_d = {} + mname_list = list(measurements.keys()) + for mname in mname_list: + md = measurements[mname] + ensemble = md['ensemble'] + if project['code'] == 'sfcf': + param = sfcf.read_param(path, uuid, md['param_file']) + affected_by_meas = affected_files(param['crr'], ensemble, uuid) + elif project['code'] == 'openQCD': + if md['measurement'] == 'ms1': + affected_by_meas = affected_files(param['type'], ensemble, uuid) + elif md['measurement'] == 't0': + affected_by_meas = affected_files(param['type'], ensemble, uuid) + elif md['measurement'] == 't1': + affected_by_meas = affected_files(param['type'], ensemble, uuid) + affected_file_d[mname] = [str(path / f) for f in affected_by_meas] + future_affected_file_d = {} + for i,mname in enumerate(mname_list): + future_affected_file_d[mname] = [] + for mname2 in mname_list[i+1:]: + future_affected_file_d[mname].extend(affected_file_d[mname2]) + for mname in mname_list: + md = measurements[mname] print(f"Import measurement {imeas}/{nmeas}: {mname}") ensemble = md['ensemble'] if project['code'] == 'sfcf': param = sfcf.read_param(path, uuid, md['param_file']) if 'names' in md.keys(): measurement = sfcf.read_data(path, uuid, md['path'], md['prefix'], param, - version=md['version'], cfg_seperator=md['cfg_seperator'], sep='/', names=md['names']) + version=md['version'], cfg_separator=md['cfg_separator'], sep='/', names=md['names']) else: measurement = sfcf.read_data(path, uuid, md['path'], md['prefix'], param, - version=md['version'], cfg_seperator=md['cfg_seperator'], sep='/') + version=md['version'], cfg_separator=md['cfg_separator'], sep='/') elif project['code'] == 'openQCD': + if not (isinstance(md['files'], list)): + raise ValueError("files has to be a list of strings") + if not all(isinstance(f, str) for f in md["files"]): + raise ValueError("files has to be a list of strings") if md['measurement'] == 'ms1': if 'param_file' in md.keys(): parameter_file = md['param_file'] @@ -229,17 +259,22 @@ def import_toml(path: Path, file: str, copy_file: bool=True) -> None: for rwp in ["integrator", "eps", "ntot", "dnms"]: param[rwp] = "Unknown" param['type'] = 't0' - measurement = openQCD.extract_t0(path, uuid, md['path'], param, str(md["prefix"]), int(md["dtr_read"]), int(md["xmin"]), int(md["spatial_extent"]), + measurement = openQCD.extract_t0(path, uuid, md['path'], ensemble, param, str(md["prefix"]), int(md["dtr_read"]), int(md["xmin"]), int(md["spatial_extent"]), fit_range=int(md.get('fit_range', 5)), postfix=str(md.get('postfix', '')), names=md.get('names', []), files=md.get('files', []), r_start=md.get('r_start', []), r_stop=md.get('r_stop', []), r_step=md.get('r_step', 1)) elif md['measurement'] == 't1': if 'param_file' in md: param = openQCD.load_ms3_infile(path, uuid, md['param_file']) param['type'] = 't1' - measurement = openQCD.extract_t1(path, uuid, md['path'], param, str(md["prefix"]), int(md["dtr_read"]), int(md["xmin"]), int(md["spatial_extent"]), + measurement = openQCD.extract_t1(path, uuid, md['path'], ensemble, param, str(md["prefix"]), int(md["dtr_read"]), int(md["xmin"]), int(md["spatial_extent"]), fit_range=int(md.get('fit_range', 5)), postfix=str(md.get('postfix', '')), names=md.get('names', []), files=md.get('files', []), r_start=md.get('r_start', []), r_stop=md.get('r_stop', []), r_step=md.get('r_step', 1)) - write_measurement(path, ensemble, measurement, uuid, project['code'], (md['param_file'] if 'param_file' in md else None)) + final_write = {} + for file in affected_file_d[mname]: + final_write[str(file)] = True + if str(file) in future_affected_file_d[mname]: + final_write[str(file)] = False + write_measurement(path, ensemble, measurement, uuid, project['code'], (md['param_file'] if 'param_file' in md else None), final_write) imeas += 1 print(mname + " imported.") @@ -266,7 +301,7 @@ def reimport_project(path: Path, uuid: str) -> None: uuid of the project that is to be reimported. """ config_path = path / "import_scripts" / uuid - for p, filenames, dirnames in os.walk(config_path): + for _p, filenames, _dirnames in os.walk(config_path): for fname in filenames: import_toml(path, os.path.join(config_path, fname), copy_file=False) return diff --git a/corrlib/tools.py b/corrlib/tools.py index 9ce194b..12cf2ac 100644 --- a/corrlib/tools.py +++ b/corrlib/tools.py @@ -1,7 +1,7 @@ import os from configparser import ConfigParser -from typing import Any from pathlib import Path +from typing import Any CONFIG_FILENAME = ".corrlib" cached: bool = True @@ -129,6 +129,33 @@ def get_db_file(path: Path) -> Path: return db_file +def get_plot_dir(path: Path) -> Path: + """ + Get the plots directory associated with the library at the given path. + + Parameters + ---------- + path: str + The path of the library. + + Returns + ------- + db_file: str + The file holding the database. + """ + path = Path(path) + if not os.path.exists(path): + raise FileNotFoundError(f"Corrlib path {path} does not exist.") + config_path = path / CONFIG_FILENAME + config = ConfigParser() + if os.path.exists(config_path): + config.read(config_path) + else: + raise FileNotFoundError("Configuration file not found.") + plot_dir = Path(config.get('paths', 'plot_dir', fallback='plots')) + return plot_dir + + def cache_enabled(path: Path) -> bool: """ Check, whether the library is cached. @@ -156,3 +183,22 @@ def cache_enabled(path: Path) -> bool: raise ValueError(f"String {cached_str} is not a valid option, only True and False are allowed!") cached_bool = cached_str == ('True') return cached_bool + + +def step_differences(name_list: list[Any], dict_of_lists: dict[Any, Any]) -> list[set[Any]]: + needed_until_step = [] + for i in range(len(name_list)): + nf: set[Any] = set() + for k in range(i, len(name_list)): + nf = nf.union(dict_of_lists[name_list[k]]) + needed_until_step.append(nf) + + discard_after = [] + for i in range(len(needed_until_step)-1): + discard_after.append(needed_until_step[i].difference(needed_until_step[i+1])) + discard_after.append(needed_until_step[-1]) + + print(discard_after) + if not set(dict_of_lists[name_list[-1]]) == discard_after[-1]: + raise ValueError("Discards and last items diverge.") + return discard_after diff --git a/corrlib/tracker.py b/corrlib/tracker.py index 6f4ae3d..0a962fa 100644 --- a/corrlib/tracker.py +++ b/corrlib/tracker.py @@ -1,11 +1,13 @@ import os -from configparser import ConfigParser -import datalad.api as dl -from typing import Optional import shutil -from .tools import get_db_file, CONFIG_FILENAME +import warnings +from configparser import ConfigParser from pathlib import Path +import datalad.api as dl + +from .tools import CONFIG_FILENAME, get_db_file + def get_tracker(path: Path) -> str: """ @@ -59,7 +61,7 @@ def get(path: Path, file: Path) -> None: return -def save(path: Path, message: str, files: Optional[list[Path]]=None) -> None: +def save(path: Path, message: str, files: list[Path] | None=None) -> None: """ Wrapper function to save a file to the dataset located at path with the specified tracker. @@ -79,8 +81,7 @@ def save(path: Path, message: str, files: Optional[list[Path]]=None) -> None: files = [path / f for f in files] dl.save(files, message=message, dataset=path) elif tracker == 'None': - Warning("Tracker 'None' does not implement save.") - pass + warnings.warn("Tracker 'None' does not implement save.", Warning, 1) else: raise ValueError(f"Tracker {tracker} is not supported.") @@ -122,8 +123,7 @@ def unlock(path: Path, file: Path) -> None: if tracker == 'datalad': dl.unlock(os.path.join(path, file), dataset=path) elif tracker == 'None': - Warning("Tracker 'None' does not implement unlock.") - pass + warnings.warn("Tracker 'None' does not implement unlock.", Warning, 1) else: raise ValueError(f"Tracker {tracker} is not supported.") return @@ -144,7 +144,7 @@ def clone(path: Path, source: str, target: str) -> None: path = Path(path) tracker = get_tracker(path) if tracker == 'datalad': - dl.clone(target=target, source=source, dataset=path) + dl.clone(path=target, source=source, dataset=path) elif tracker == 'None': os.makedirs(path, exist_ok=True) # Implement a simple clone by copying files @@ -154,7 +154,7 @@ def clone(path: Path, source: str, target: str) -> None: return -def drop(path: Path, reckless: Optional[str]=None) -> None: +def drop(path: Path, reckless: str | None=None) -> None: """ Wrapper function to drop data from a dataset located at path with the specified tracker. @@ -170,8 +170,7 @@ def drop(path: Path, reckless: Optional[str]=None) -> None: if tracker == 'datalad': dl.drop(path, reckless=reckless) elif tracker == 'None': - Warning("Tracker 'None' does not implement drop.") - pass + warnings.warn("Tracker 'None' does not implement drop.", Warning, 1) else: raise ValueError(f"Tracker {tracker} is not supported.") return diff --git a/corrlib/version.py b/corrlib/version.py index 23dd03f..3e22db8 100644 --- a/corrlib/version.py +++ b/corrlib/version.py @@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...] commit_id: str | None __commit_id__: str | None -__version__ = version = '0.3.1.dev0+g08de17e6b.d20260507' -__version_tuple__ = version_tuple = (0, 3, 1, 'dev0', 'g08de17e6b.d20260507') +__version__ = version = '0.3.1.dev32+g906a2bdf3.d20260710' +__version_tuple__ = version_tuple = (0, 3, 1, 'dev32', 'g906a2bdf3.d20260710') -__commit_id__ = commit_id = 'g08de17e6b' +__commit_id__ = commit_id = 'g906a2bdf3' diff --git a/pyproject.toml b/pyproject.toml index 02d5a7a..10756b7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,7 @@ dependencies = [ 'pyerrors>=2.11.1', "datalad>=1.1.0", 'typer>=0.12.5', + "matplotlib>=3.10.7", ] description = "Python correlation library" authors = [ diff --git a/tests/find_test.py b/tests/find_test.py index 2144001..b462634 100644 --- a/tests/find_test.py +++ b/tests/find_test.py @@ -306,8 +306,7 @@ def test_openQCD_filter() -> None: "updated_at"] df = pd.DataFrame(data,columns=cols) - with pytest.warns(Warning): - find.openQCD_filter(df, a = "asdf") + find.openQCD_filter(df, a = "asdf") def test_code_filter() -> None: diff --git a/tests/import_project_test.py b/tests/import_project_test.py index 685d2cf..8493773 100644 --- a/tests/import_project_test.py +++ b/tests/import_project_test.py @@ -10,7 +10,7 @@ def test_toml_check_measurement_data() -> None: "param_file": "/path/to/file", "version": "1.1", "prefix": "pref", - "cfg_seperator": "n", + "cfg_separator": "n", "names": ['list', 'of', 'names'] } } diff --git a/uv.lock b/uv.lock index f76ee81..6e246e6 100644 --- a/uv.lock +++ b/uv.lock @@ -409,6 +409,7 @@ source = { editable = "." } dependencies = [ { name = "datalad" }, { name = "gitpython" }, + { name = "matplotlib" }, { name = "pyerrors" }, { name = "typer" }, ] @@ -427,6 +428,7 @@ dev = [ requires-dist = [ { name = "datalad", specifier = ">=1.1.0" }, { name = "gitpython", specifier = ">=3.1.45" }, + { name = "matplotlib", specifier = ">=3.10.7" }, { name = "pyerrors", specifier = ">=2.11.1" }, { name = "typer", specifier = ">=0.12.5" }, ]