From 614fccfae4a93f4e3f1f24ad4d77804da19e4092 Mon Sep 17 00:00:00 2001 From: jkuhl Date: Thu, 27 Nov 2025 15:54:39 +0100 Subject: [PATCH 01/41] Delete .gitmodules --- .gitmodules | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 .gitmodules diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index d41b9e5..0000000 --- a/.gitmodules +++ /dev/null @@ -1,5 +0,0 @@ -[submodule "projects/tmp"] - path = projects/tmp - url = git@kuhl-mann.de:lattice/charm_SF_data.git - datalad-id = 5f402163-77f2-470e-b6f1-64d7bf9f87d4 - datalad-url = git@kuhl-mann.de:lattice/charm_SF_data.git From 4709e4272714bbb11c722eac56f8b02c4b05c2ec Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Thu, 12 Feb 2026 10:21:45 +0100 Subject: [PATCH 02/41] HOTFIX: ds arg not supported by datalad --- corrlib/main.py | 2 +- corrlib/meas_io.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/corrlib/main.py b/corrlib/main.py index ebc923e..525cbd6 100644 --- a/corrlib/main.py +++ b/corrlib/main.py @@ -122,7 +122,7 @@ def import_project(path: str, url: str, owner: Union[str, None]=None, tags: Unio raise ValueError("The dataset does not have a uuid!") if not os.path.exists(path + "/projects/" + uuid): db = path + "/backlogger.db" - dl.get(db, ds=path) + dl.get(db, dataset=path) dl.unlock(db, dataset=path) create_project(path, uuid, owner, tags, aliases, code) move_submodule(path, 'projects/tmp', 'projects/' + uuid) diff --git a/corrlib/meas_io.py b/corrlib/meas_io.py index b98eb6e..fc373db 100644 --- a/corrlib/meas_io.py +++ b/corrlib/meas_io.py @@ -28,7 +28,7 @@ def write_measurement(path, ensemble, measurement, uuid, code, parameter_file=No The uuid of the project. """ db = os.path.join(path, 'backlogger.db') - dl.get(db, ds=path) + dl.get(db, dataset=path) dl.unlock(db, dataset=path) conn = sqlite3.connect(db) c = conn.cursor() @@ -177,7 +177,7 @@ def drop_record(path: str, meas_path: str): file_in_archive = meas_path.split("::")[0] file = os.path.join(path, file_in_archive) db = os.path.join(path, 'backlogger.db') - dl.get(db, ds=path) + dl.get(db, dataset=path) sub_key = meas_path.split("::")[1] dl.unlock(db, dataset=path) conn = sqlite3.connect(db) From fad703a39feb4cd86b243bca63e1c0c01b68a224 Mon Sep 17 00:00:00 2001 From: jkuhl Date: Thu, 12 Feb 2026 10:27:58 +0100 Subject: [PATCH 03/41] bunmp version to 0.2.4 --- corrlib/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/corrlib/version.py b/corrlib/version.py index d31c31e..788da1f 100644 --- a/corrlib/version.py +++ b/corrlib/version.py @@ -1 +1 @@ -__version__ = "0.2.3" +__version__ = "0.2.4" From d6dde9bb1184c9c814bac5acda9cfd8262b4a60a Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Fri, 13 Feb 2026 10:46:29 +0100 Subject: [PATCH 04/41] add function to only show a single arg with the find cli --- corrlib/cli.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/corrlib/cli.py b/corrlib/cli.py index b808c13..ff74527 100644 --- a/corrlib/cli.py +++ b/corrlib/cli.py @@ -90,6 +90,11 @@ def find( "--dataset", "-d", ), + arg: str = typer.Option( + str('all'), + "--show", + "-s", + ), ensemble: str = typer.Argument(), corr: str = typer.Argument(), code: str = typer.Argument(), @@ -98,8 +103,12 @@ def find( Find a record in the backlog at hand. Through specifying it's ensemble and the measured correlator. """ results = find_record(path, ensemble, corr, code) - print(results) - + if arg == "all": + print(results) + else: + for i in range(len(results)): + print(results[arg].values[i]) + return @app.command() def importer( From 4e3327709e73fab585b17004c19275c76e082f54 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Tue, 28 Apr 2026 11:27:49 +0200 Subject: [PATCH 05/41] HOTFIX: paths in update_aliases --- corrlib/main.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/corrlib/main.py b/corrlib/main.py index 831b69d..5df8165 100644 --- a/corrlib/main.py +++ b/corrlib/main.py @@ -27,7 +27,7 @@ def create_project(path: Path, uuid: str, owner: Union[str, None]=None, tags: Un The code that was used to create the measurements. """ db_file = get_db_file(path) - db = os.path.join(path, db_file) + db = path / db_file get(path, db_file) conn = sqlite3.connect(db) c = conn.cursor() @@ -67,7 +67,7 @@ def update_project_data(path: Path, uuid: str, prop: str, value: Union[str, None """ db_file = get_db_file(path) get(path, db_file) - conn = sqlite3.connect(os.path.join(path, db_file)) + conn = sqlite3.connect(path / db_file) c = conn.cursor() c.execute(f"UPDATE projects SET '{prop}' = '{value}' WHERE id == '{uuid}'") conn.commit() @@ -77,9 +77,8 @@ def update_project_data(path: Path, uuid: str, prop: str, value: Union[str, None def update_aliases(path: Path, uuid: str, aliases: list[str]) -> None: db_file = get_db_file(path) - db = path / db_file get(path, db_file) - known_data = _project_lookup_by_id(db, uuid)[0] + known_data = _project_lookup_by_id(path, uuid)[0] known_aliases = known_data[1] if aliases is None: From b3991ecc6760298d03a99af6e461b40ab043f231 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Thu, 30 Apr 2026 15:09:17 +0200 Subject: [PATCH 06/41] let cli drop cache when importing a new project, add stat flag for find --- corrlib/cli.py | 17 ++++++++++------- corrlib/find.py | 11 +++++++++++ 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/corrlib/cli.py b/corrlib/cli.py index d24d8ef..b4a42e0 100644 --- a/corrlib/cli.py +++ b/corrlib/cli.py @@ -1,14 +1,13 @@ -from typing import Optional +from typing import Optional, Any import typer from corrlib import __app_name__ from .initialization import create from .toml import import_tomls, update_project, reimport_project -from .find import find_record, list_projects +from .find import find_record, list_projects, get_stat from .tools import str2list from .main import update_aliases from .meas_io import drop_cache as mio_drop_cache -from .meas_io import load_record as mio_load_record from .integrity import full_integrity_check import os @@ -116,6 +115,11 @@ def find( if arg == 'all': print(results) else: + if arg == 'stat': + for r in results['path'].values: + stat = get_stat(path, r) + print(stat) + return for r in results[arg].values: print(r) @@ -132,10 +136,7 @@ def stat( """ Show the statistics of a given record. """ - record = mio_load_record(path, record_id) - if isinstance(record, (list, Corr)): - record = record[0] - statistics = record.idl + statistics = get_stat(path, record_id) print(statistics) return @@ -170,6 +171,7 @@ def importer( """ file_list = files.split(",") import_tomls(path, file_list, copy_file) + mio_drop_cache(path) return @@ -194,6 +196,7 @@ def reimporter( raise Exception("This file is not known for this project.") else: reimport_project(path, uuid) + mio_drop_cache(path) return diff --git a/corrlib/find.py b/corrlib/find.py index 1e6b4bf..738c832 100644 --- a/corrlib/find.py +++ b/corrlib/find.py @@ -13,6 +13,8 @@ from pathlib import Path import datetime as dt from collections.abc import Callable import warnings +from .meas_io import load_record +from pyerrors import Corr, Obs def _project_lookup_by_alias(path: Path, alias: str) -> str: @@ -381,3 +383,12 @@ def list_projects(path: Path) -> list[tuple[str, str]]: conn.close() return results + +def get_stat(path: Path, record_id: str) -> Any: + loaded_record: Obs = load_record(path, record_id) + if isinstance(loaded_record, (list, Corr)): + record: Obs = loaded_record[0] + else: + record = loaded_record + return record.idl + From 69348cd151110ccb99fc321470fcc2b8c61d9fd3 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Thu, 30 Apr 2026 15:11:46 +0200 Subject: [PATCH 07/41] fix ruff complaints --- corrlib/cli.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/corrlib/cli.py b/corrlib/cli.py index b4a42e0..3053bb3 100644 --- a/corrlib/cli.py +++ b/corrlib/cli.py @@ -1,4 +1,4 @@ -from typing import Optional, Any +from typing import Optional import typer from corrlib import __app_name__ @@ -11,7 +11,6 @@ from .meas_io import drop_cache as mio_drop_cache from .integrity import full_integrity_check import os -from pyerrors import Corr from importlib.metadata import version from pathlib import Path From 2f83c1f9cba1964360263fbf87778813892c6f70 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Tue, 5 May 2026 16:21:05 +0200 Subject: [PATCH 08/41] throw error if library path is not found in get_db_file --- corrlib/tools.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/corrlib/tools.py b/corrlib/tools.py index 93f0678..f66e35d 100644 --- a/corrlib/tools.py +++ b/corrlib/tools.py @@ -115,6 +115,8 @@ def get_db_file(path: Path) -> Path: db_file: str The file holding the database. """ + if not os.path.exists(path): + raise FileNotFoundError(f"Corrlib path {path} does not exist.") config_path = os.path.join(path, CONFIG_FILENAME) config = ConfigParser() if os.path.exists(config_path): From 6c99653fffcb105f50d107f49a4b3005156b11f3 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Tue, 5 May 2026 16:26:02 +0200 Subject: [PATCH 09/41] check whether paths exist for import --- corrlib/toml.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/corrlib/toml.py b/corrlib/toml.py index 0d4dfc8..29d7de2 100644 --- a/corrlib/toml.py +++ b/corrlib/toml.py @@ -158,6 +158,10 @@ def import_toml(path: Path, file: str, copy_file: bool=True) -> None: copy_file: bool, optional Whether the toml-files will be copied into the library. Default is True. """ + if not os.path.exists(path): + raise FileNotFoundError(f"Corrlib path {path} does not exist.") + if not os.path.exists(file): + raise FileNotFoundError(f".toml-file {file} does not exist.") print("Import project as decribed in " + file) with open(file, 'rb') as fp: toml_dict = toml.load(fp) From 93ca059fc0950d361bf2bd85e468fa8200236e90 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Tue, 5 May 2026 16:26:18 +0200 Subject: [PATCH 10/41] pathlib for concat --- corrlib/tools.py | 6 +++--- corrlib/tracker.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/corrlib/tools.py b/corrlib/tools.py index f66e35d..e75e1cc 100644 --- a/corrlib/tools.py +++ b/corrlib/tools.py @@ -89,7 +89,7 @@ def set_config(path: Path, section: str, option: str, value: Any) -> None: value: Any The value we set the option to. """ - config_path = os.path.join(path, CONFIG_FILENAME) + config_path = path / CONFIG_FILENAME config = ConfigParser() if os.path.exists(config_path): config.read(config_path) @@ -117,7 +117,7 @@ def get_db_file(path: Path) -> Path: """ if not os.path.exists(path): raise FileNotFoundError(f"Corrlib path {path} does not exist.") - config_path = os.path.join(path, CONFIG_FILENAME) + config_path = path / CONFIG_FILENAME config = ConfigParser() if os.path.exists(config_path): config.read(config_path) @@ -142,7 +142,7 @@ def cache_enabled(path: Path) -> bool: cached_bool: bool Whether the given library is cached. """ - config_path = os.path.join(path, CONFIG_FILENAME) + config_path = path / CONFIG_FILENAME config = ConfigParser() if os.path.exists(config_path): config.read(config_path) diff --git a/corrlib/tracker.py b/corrlib/tracker.py index 1bae1b4..98fdec5 100644 --- a/corrlib/tracker.py +++ b/corrlib/tracker.py @@ -21,7 +21,7 @@ def get_tracker(path: Path) -> str: tracker: str The tracker used in the dataset. """ - config_path = os.path.join(path, CONFIG_FILENAME) + config_path = path / CONFIG_FILENAME config = ConfigParser() if os.path.exists(config_path): config.read(config_path) From 656f99a13c2fd0c4fbad292f2a3c145afb567124 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Tue, 5 May 2026 16:47:07 +0200 Subject: [PATCH 11/41] add integrity check for the config-file --- corrlib/integrity.py | 41 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/corrlib/integrity.py b/corrlib/integrity.py index 5f80aa3..74386f4 100644 --- a/corrlib/integrity.py +++ b/corrlib/integrity.py @@ -1,10 +1,12 @@ import datetime as dt from pathlib import Path -from .tools import get_db_file +from .tools import get_db_file, CONFIG_FILENAME import pandas as pd import sqlite3 from .tracker import get import pyerrors.input.json as pj +import os +from configparser import ConfigParser from typing import Any @@ -41,7 +43,6 @@ def check_db_integrity(path: Path) -> None: for _, result in results.iterrows(): if not has_valid_times(result): raise ValueError(f"Result with id {result[id]} has wrong time signatures.") - print("DB:\t✅") return @@ -67,7 +68,6 @@ def _check_db2paths(path: Path, meas_paths: list[str]) -> None: for key in needed_data[file]: if key not in filedict.keys(): raise ValueError(f"Did not find data for key {key} that should be in file {file}.") - print("Links:\t✅") return @@ -79,9 +79,44 @@ def check_db_file_links(path: Path) -> None: _check_db2paths(path, list(results)) +def check_path_and_config(path: Path) -> None: + if not os.path.exists(path): + raise FileNotFoundError(f"Corrlib path {path} does not exist.") + config_path = path / CONFIG_FILENAME + if not os.path.exists(config_path): + raise FileNotFoundError(f"Configuration file {config_path} not found.") + + +def check_config_validity(path: Path) -> None: + config = ConfigParser() + config_path = path / CONFIG_FILENAME + if os.path.exists(config_path): + config.read(config_path) + else: + raise FileNotFoundError("Configuration file not found.") + + if config.has_section('core'): + core_opts = ['version', 'tracker', 'cached'] + has_core_opts = [config.has_option('core', opt) for opt in core_opts] + if not all(has_core_opts): + raise ValueError("One of the options in the 'core' section ('version', 'tracker', 'cached') is missing.") + + if config.has_section('paths'): + path_opts = ['db', 'projects_path', 'archive_path', 'toml_imports_path', 'import_scripts_path'] + has_path_opts = [config.has_option('paths', opt) for opt in path_opts] + if not all(has_path_opts): + raise ValueError("One of the options in the 'path' section ('db', 'projects_path', 'archive_path', 'toml_imports_path', 'import_scripts_path') is missing.") + + def full_integrity_check(path: Path) -> None: + check_path_and_config(path) + print("Path and config-file exist:\t✅") + check_config_validity(path) + print("Configuration is valid:\t✅") check_db_integrity(path) + print("DB:\t✅") check_db_file_links(path) + print("Links:\t✅") print("Full:\t✅") From c3bf36bf52e6f1c9ff4631d75398ad82eff441d9 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Tue, 5 May 2026 17:15:16 +0200 Subject: [PATCH 12/41] add docs, add check for needed paths --- corrlib/integrity.py | 90 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 89 insertions(+), 1 deletion(-) diff --git a/corrlib/integrity.py b/corrlib/integrity.py index 74386f4..f5b2300 100644 --- a/corrlib/integrity.py +++ b/corrlib/integrity.py @@ -12,6 +12,20 @@ from typing import Any def has_valid_times(result: pd.Series) -> bool: + """ + Check, whether the result at hand has time-stamps that are sensible: + A recored is created first, then updated, with both times laying in the past. + + Parameters + ---------- + result: pd.Series + The result to check + + Returns + ------- + b: bool + True, if the timestamps make sense. + """ # we expect created_at <= updated_at <= now created_at = dt.datetime.fromisoformat(result['created_at']) updated_at = dt.datetime.fromisoformat(result['updated_at']) @@ -22,15 +36,41 @@ def has_valid_times(result: pd.Series) -> bool: return True def are_keys_unique(db: Path, table: str, col: str) -> bool: + """ + Check whether the strings listed in a column of a given table are unique. + + Parameters + ---------- + db: Path + The database to check. + table: str + The table to check. + col: str + The column to be checked for uniqueness. + + Returns + ------- + b: bool + True, if the strings are unique. + """ conn = sqlite3.connect(db) c = conn.cursor() - c.execute(f"SELECT COUNT( DISTINCT CAST(path AS nvarchar(4000))), COUNT({col}) FROM {table};") + c.execute(f"SELECT COUNT( DISTINCT CAST({col} AS nvarchar(4000))), COUNT({col}) FROM {table};") results = c.fetchall()[0] conn.close() return bool(results[0] == results[1]) def check_db_integrity(path: Path) -> None: + """ + Check intergrity of the database by checking the uniqueness of the record keys used to load the records + and ensuring that the timestamps of each record is sensible. Throws an error, if issues are detected. + + Parameters + ---------- + path: Path + Path to the backlog-library to check. + """ db = get_db_file(path) if not are_keys_unique(path / db, 'backlogs', 'path'): @@ -47,6 +87,17 @@ def check_db_integrity(path: Path) -> None: def _check_db2paths(path: Path, meas_paths: list[str]) -> None: + """ + Check whether for each record in the given by meas_paths, we can find the data in the file as we expect. + Also check, whether there are unreachable records in the files. If either of the issues arise, throws an error. + + Parameters + ---------- + path: Path + Path to the backlog-library to check. + meas_paths: list[str] + List of measurement paths to check. + """ needed_data: dict[str, list[str]] = {} for mpath in meas_paths: file = mpath.split("::")[0] @@ -72,6 +123,15 @@ def _check_db2paths(path: Path, meas_paths: list[str]) -> None: def check_db_file_links(path: Path) -> None: + """ + Check whether for each record in the given correlator library, we can find the data in the file as we expect. + Also check, whether there are unreachable records in the files. If either of the issues arise, throws an error. + + Parameters + ---------- + path: Path + Path to the backlog-library to check. + """ db = get_db_file(path) search_expr = "SELECT path FROM 'backlogs'" conn = sqlite3.connect(path / db) @@ -80,6 +140,14 @@ def check_db_file_links(path: Path) -> None: def check_path_and_config(path: Path) -> None: + """ + Check whether the given path exists and the cinfigureation file can be found. + + Parameters + ---------- + path: Path + Path to the backlog-library to check. + """ if not os.path.exists(path): raise FileNotFoundError(f"Corrlib path {path} does not exist.") config_path = path / CONFIG_FILENAME @@ -88,6 +156,14 @@ def check_path_and_config(path: Path) -> None: def check_config_validity(path: Path) -> None: + """ + Check whether the configuration file of the given corrlib-dataset path is valid. + + Parameters + ---------- + path: Path + Path to the backlog-library to check. + """ config = ConfigParser() config_path = path / CONFIG_FILENAME if os.path.exists(config_path): @@ -107,8 +183,20 @@ def check_config_validity(path: Path) -> None: if not all(has_path_opts): raise ValueError("One of the options in the 'path' section ('db', 'projects_path', 'archive_path', 'toml_imports_path', 'import_scripts_path') is missing.") + has_paths = [os.path.exists(path / config.get('paths', opt)) for opt in path_opts] + if not all(has_paths): + raise FileNotFoundError("one of the paths needed by the configuration file is not present.") + def full_integrity_check(path: Path) -> None: + """ + Aggregate all checks for easy validation of the backlog-library. + + Parameters + ---------- + path: Path + Path to the backlog-library to check. + """ check_path_and_config(path) print("Path and config-file exist:\t✅") check_config_validity(path) From ba4624d8433d0360aded38d196cbdfac39a9e434 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Tue, 5 May 2026 17:20:20 +0200 Subject: [PATCH 13/41] restruct: needed paths get extra check --- corrlib/integrity.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/corrlib/integrity.py b/corrlib/integrity.py index f5b2300..f2a70bd 100644 --- a/corrlib/integrity.py +++ b/corrlib/integrity.py @@ -11,6 +11,9 @@ from configparser import ConfigParser from typing import Any +path_opts = ['db', 'projects_path', 'archive_path', 'toml_imports_path', 'import_scripts_path'] + + def has_valid_times(result: pd.Series) -> bool: """ Check, whether the result at hand has time-stamps that are sensible: @@ -178,14 +181,29 @@ def check_config_validity(path: Path) -> None: raise ValueError("One of the options in the 'core' section ('version', 'tracker', 'cached') is missing.") if config.has_section('paths'): - path_opts = ['db', 'projects_path', 'archive_path', 'toml_imports_path', 'import_scripts_path'] has_path_opts = [config.has_option('paths', opt) for opt in path_opts] if not all(has_path_opts): raise ValueError("One of the options in the 'path' section ('db', 'projects_path', 'archive_path', 'toml_imports_path', 'import_scripts_path') is missing.") + +def check_paths(path: Path) -> None: + """ + Check whether all paths demanded by the 'paths' section of the configuration-file exist. + + Parameters + ---------- + path: Path + Path to the backlog-library to check. + """ + config = ConfigParser() + config_path = path / CONFIG_FILENAME + if os.path.exists(config_path): + config.read(config_path) + else: + raise FileNotFoundError("Configuration file not found.") has_paths = [os.path.exists(path / config.get('paths', opt)) for opt in path_opts] if not all(has_paths): - raise FileNotFoundError("one of the paths needed by the configuration file is not present.") + raise FileNotFoundError("One of the paths specified in the configuration file is not present.") def full_integrity_check(path: Path) -> None: @@ -201,6 +219,8 @@ def full_integrity_check(path: Path) -> None: print("Path and config-file exist:\t✅") check_config_validity(path) print("Configuration is valid:\t✅") + check_paths(path) + print("Needed paths exist:\t✅") check_db_integrity(path) print("DB:\t✅") check_db_file_links(path) From 3247cdbc40aaccd1c9ff62da2059d407a01d7a51 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Tue, 5 May 2026 17:24:09 +0200 Subject: [PATCH 14/41] neater UX --- corrlib/integrity.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/corrlib/integrity.py b/corrlib/integrity.py index f2a70bd..f660dfe 100644 --- a/corrlib/integrity.py +++ b/corrlib/integrity.py @@ -215,16 +215,17 @@ def full_integrity_check(path: Path) -> None: path: Path Path to the backlog-library to check. """ + print("Run full integrity check...") check_path_and_config(path) - print("Path and config-file exist:\t✅") + print("(1/5) Path and config-file exist: ✅") check_config_validity(path) - print("Configuration is valid:\t✅") + print("(2/5) Configuration is valid: ✅") check_paths(path) - print("Needed paths exist:\t✅") + print("(3/5) Needed paths exist: ✅") check_db_integrity(path) - print("DB:\t✅") + print("(4/5) Database is sane: ✅") check_db_file_links(path) - print("Links:\t✅") - print("Full:\t✅") + print("(5/5) DB2File and File2DB-links are sound: ✅") + print("Full integrity check: ✅") From a2a3346f51ec574e772a1293038558e351e65b69 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Tue, 5 May 2026 22:12:14 +0200 Subject: [PATCH 15/41] provide docstring for repo check --- corrlib/cli.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/corrlib/cli.py b/corrlib/cli.py index d24d8ef..acf4eca 100644 --- a/corrlib/cli.py +++ b/corrlib/cli.py @@ -108,7 +108,7 @@ def find( ), ) -> None: """ - Find a record in the backlog at hand. Through specifying it's ensemble and the measured correlator. + Find a record in the given backlog. """ results = find_record(path, ensemble, corr, code) if results.empty: @@ -147,6 +147,9 @@ def check(path: Path = typer.Option( "-d", ), ) -> None: + """ + Check the integrity of the repository. + """ full_integrity_check(path) From ac400aa9017d72c8f35b4f8664a7a8928e42c8b2 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Tue, 5 May 2026 22:32:13 +0200 Subject: [PATCH 16/41] Breaking change for CLI: change default path to current directory --- corrlib/cli.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/corrlib/cli.py b/corrlib/cli.py index d24d8ef..0e6d96a 100644 --- a/corrlib/cli.py +++ b/corrlib/cli.py @@ -29,7 +29,7 @@ def _version_callback(value: bool) -> None: @app.command() def update( path: Path = typer.Option( - Path('./corrlib'), + Path('.'), "--dataset", "-d", ), @@ -45,7 +45,7 @@ def update( @app.command() def lister( path: Path = typer.Option( - Path('./corrlib'), + Path('.'), "--dataset", "-d", ), @@ -76,7 +76,7 @@ def lister( @app.command() def alias_add( path: Path = typer.Option( - Path('./corrlib'), + Path('.'), "--dataset", "-d", ), @@ -94,7 +94,7 @@ def alias_add( @app.command() def find( path: Path = typer.Option( - Path('./corrlib'), + Path('.'), "--dataset", "-d", ), @@ -123,7 +123,7 @@ def find( @app.command() def stat( path: Path = typer.Option( - Path('./corrlib'), + Path('.'), "--dataset", "-d", ), @@ -142,7 +142,7 @@ def stat( @app.command() def check(path: Path = typer.Option( - Path('./corrlib'), + Path('.'), "--dataset", "-d", ), @@ -153,7 +153,7 @@ def check(path: Path = typer.Option( @app.command() def importer( path: Path = typer.Option( - Path('./corrlib'), + Path('.'), "--dataset", "-d", ), @@ -176,7 +176,7 @@ def importer( @app.command() def reimporter( path: Path = typer.Option( - Path('./corrlib'), + Path('.'), "--dataset", "-d", ), @@ -200,7 +200,7 @@ def reimporter( @app.command() def init( path: Path = typer.Option( - Path('./corrlib'), + Path('.'), "--dataset", "-d", ), @@ -220,7 +220,7 @@ def init( @app.command() def drop_cache( path: Path = typer.Option( - Path('./corrlib'), + Path('.'), "--dataset", "-d", ), From 08d25da188da235638e1c2c6360db130290d4f64 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Wed, 6 May 2026 16:48:35 +0200 Subject: [PATCH 17/41] HOTFIX: nsure path casting in find and meas_io --- corrlib/find.py | 1 + corrlib/meas_io.py | 9 +++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/corrlib/find.py b/corrlib/find.py index 1e6b4bf..ea696b1 100644 --- a/corrlib/find.py +++ b/corrlib/find.py @@ -322,6 +322,7 @@ def find_record(path: Path, ensemble: str, correlator_name: str, code: str, proj revision: Optional[str]=None, customFilter: Optional[Callable[[pd.DataFrame], pd.DataFrame]] = None, **kwargs: Any) -> pd.DataFrame: + path = Path(path) db_file = get_db_file(path) db = path / db_file if code not in codes: diff --git a/corrlib/meas_io.py b/corrlib/meas_io.py index cbd9386..6b6e5f1 100644 --- a/corrlib/meas_io.py +++ b/corrlib/meas_io.py @@ -37,6 +37,7 @@ def write_measurement(path: Path, ensemble: str, measurement: dict[str, dict[str parameter_file: str The parameter file used for the measurement. """ + path = Path(path) db_file = get_db_file(path) db = path / db_file @@ -50,7 +51,7 @@ def write_measurement(path: Path, ensemble: str, measurement: dict[str, dict[str c = conn.cursor() for corr in measurement.keys(): file_in_archive = Path('.') / 'archive' / ensemble / corr / str(uuid + '.json.gz') - file = path / file_in_archive + file = Path(path) / file_in_archive known_meas = {} if not os.path.exists(path / 'archive' / ensemble / corr): os.makedirs(path / 'archive' / ensemble / corr) @@ -174,6 +175,7 @@ def load_records(path: Path, meas_paths: list[str], preloaded: dict[str, Any] = returned_data: list The loaded records. """ + path = Path(path) if dry_run: _check_db2paths(path, meas_paths) return [] @@ -238,6 +240,7 @@ def cache_path(path: Path, file: str, key: str) -> Path: cache_path: str The path at which the measurement of the given file and key is cached. """ + path = Path(path) cache_path = cache_dir(path, file) / key return cache_path @@ -258,6 +261,7 @@ def preload(path: Path, file: Path) -> dict[str, Any]: filedict: dict[str, Any] The data read from the file. """ + path = Path(path) get(path, file) filedict: dict[str, Any] = pj.load_json_dict(str(path / file)) print("> read file") @@ -276,7 +280,7 @@ def drop_record(path: Path, meas_path: str) -> None: The measurement path as noted in the database. """ file_in_archive = meas_path.split("::")[0] - file = path / file_in_archive + file = Path(path) / file_in_archive db_file = get_db_file(path) db = path / db_file get(path, db_file) @@ -310,6 +314,7 @@ def drop_cache(path: Path) -> None: path: str The path of the library. """ + path = Path(path) cache_dir = path / ".cache" for f in os.listdir(cache_dir): shutil.rmtree(cache_dir / f) From fbf802959aff13ac391c17e96825e5c25d29cc51 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Wed, 6 May 2026 16:50:39 +0200 Subject: [PATCH 18/41] HOTFIX: ensure path casting in tools --- corrlib/tools.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/corrlib/tools.py b/corrlib/tools.py index e75e1cc..9ce194b 100644 --- a/corrlib/tools.py +++ b/corrlib/tools.py @@ -89,6 +89,7 @@ def set_config(path: Path, section: str, option: str, value: Any) -> None: value: Any The value we set the option to. """ + path = Path(path) config_path = path / CONFIG_FILENAME config = ConfigParser() if os.path.exists(config_path): @@ -115,6 +116,7 @@ def get_db_file(path: Path) -> Path: db_file: str The file holding the database. """ + path = Path(path) if not os.path.exists(path): raise FileNotFoundError(f"Corrlib path {path} does not exist.") config_path = path / CONFIG_FILENAME @@ -142,6 +144,7 @@ def cache_enabled(path: Path) -> bool: cached_bool: bool Whether the given library is cached. """ + path = Path(path) config_path = path / CONFIG_FILENAME config = ConfigParser() if os.path.exists(config_path): From 075cb2f75633cb0afb30d4320726a781fca8ea73 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Wed, 6 May 2026 16:52:07 +0200 Subject: [PATCH 19/41] HOTFIX: ensure path casting in tracker --- corrlib/tracker.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/corrlib/tracker.py b/corrlib/tracker.py index 98fdec5..6f4ae3d 100644 --- a/corrlib/tracker.py +++ b/corrlib/tracker.py @@ -21,6 +21,7 @@ def get_tracker(path: Path) -> str: tracker: str The tracker used in the dataset. """ + path = Path(path) config_path = path / CONFIG_FILENAME config = ConfigParser() if os.path.exists(config_path): @@ -42,6 +43,7 @@ def get(path: Path, file: Path) -> None: file: str The file to get. """ + path = Path(path) tracker = get_tracker(path) if tracker == 'datalad': if file == get_db_file(path): @@ -70,6 +72,7 @@ def save(path: Path, message: str, files: Optional[list[Path]]=None) -> None: files: list[str], optional The files to save. If None, all changes are saved. """ + path = Path(path) tracker = get_tracker(path) if tracker == 'datalad': if files is not None: @@ -93,6 +96,7 @@ def init(path: Path, tracker: str='datalad') -> None: tracker: str The tracker to use. Currently only 'datalad' and 'None' are supported. """ + path = Path(path) if tracker == 'datalad': dl.create(path) elif tracker == 'None': @@ -113,6 +117,7 @@ def unlock(path: Path, file: Path) -> None: file : str The file to unlock. """ + path = Path(path) tracker = get_tracker(path) if tracker == 'datalad': dl.unlock(os.path.join(path, file), dataset=path) @@ -136,6 +141,7 @@ def clone(path: Path, source: str, target: str) -> None: target: str The target path to clone the dataset to. """ + path = Path(path) tracker = get_tracker(path) if tracker == 'datalad': dl.clone(target=target, source=source, dataset=path) @@ -159,6 +165,7 @@ def drop(path: Path, reckless: Optional[str]=None) -> None: reckless: Optional[str] The datalad's reckless option for dropping data. """ + path = Path(path) tracker = get_tracker(path) if tracker == 'datalad': dl.drop(path, reckless=reckless) From 4c4a5fd670c798d7a87230c4bd383ffb9f21ec1c Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Wed, 6 May 2026 18:02:25 +0200 Subject: [PATCH 20/41] add checks of the format of the paths in the database --- corrlib/cli.py | 8 ++++---- corrlib/find.py | 7 +++++++ corrlib/integrity.py | 30 +++++++++++++++++++++++++++++- 3 files changed, 40 insertions(+), 5 deletions(-) diff --git a/corrlib/cli.py b/corrlib/cli.py index bdee36e..334dd33 100644 --- a/corrlib/cli.py +++ b/corrlib/cli.py @@ -4,7 +4,7 @@ from corrlib import __app_name__ from .initialization import create from .toml import import_tomls, update_project, reimport_project -from .find import find_record, list_projects +from .find import find_record, list_projects, list_ensembles from .tools import str2list from .main import update_aliases from .meas_io import drop_cache as mio_drop_cache @@ -56,9 +56,9 @@ def lister( """ if entities in ['ensembles', 'Ensembles','ENSEMBLES']: print("Ensembles:") - for item in os.listdir(path / "archive"): - if os.path.isdir(path / "archive" / item): - print(item) + results = list_ensembles(path) + for e in results: + print(e) elif entities == 'projects': results = list_projects(path) print("Projects:") diff --git a/corrlib/find.py b/corrlib/find.py index ea696b1..fbdd801 100644 --- a/corrlib/find.py +++ b/corrlib/find.py @@ -382,3 +382,10 @@ def list_projects(path: Path) -> list[tuple[str, str]]: conn.close() return results + +def list_ensembles(path: Path) -> list[str]: + res = [] + for item in os.listdir(path / "archive"): + if os.path.isdir(path / "archive" / item): + res.append(item) + return res diff --git a/corrlib/integrity.py b/corrlib/integrity.py index f660dfe..5a3ae05 100644 --- a/corrlib/integrity.py +++ b/corrlib/integrity.py @@ -7,7 +7,7 @@ from .tracker import get import pyerrors.input.json as pj import os from configparser import ConfigParser - +from .find import list_ensembles, list_projects from typing import Any @@ -64,6 +64,31 @@ def are_keys_unique(db: Path, table: str, col: str) -> bool: return bool(results[0] == results[1]) +def check_path_format(result: pd.Series, ensembles: list[str], projects: list[str]) -> None: + """ + Check whether the path of the given result has the right format. + + Parameters + ---------- + result: pd.Series + The result to be checked. + """ + p = result['path'] + if not p.startswith('archive'): + raise ValueError(f'The path {p} does not start correctly') + + meas_key = p.split('::')[1] + ensemble = p.split('/')[1] + project = p.split('/')[2].split('::')[0] + if not len(meas_key) == 64: + raise ValueError(f'meas_key of {p} is scrambled') + if ensemble not in ensembles: + raise ValueError(f'meas_key of {p} points to an unknown ensemble') + if project not in projects: + raise ValueError(f'meas_key of {p} points to an unknown project id') + + + def check_db_integrity(path: Path) -> None: """ Check intergrity of the database by checking the uniqueness of the record keys used to load the records @@ -82,10 +107,13 @@ def check_db_integrity(path: Path) -> None: search_expr = "SELECT * FROM 'backlogs'" conn = sqlite3.connect(path / db) results = pd.read_sql(search_expr, conn) + ensembles = list_ensembles(path) + projects = [p[0] for p in list_projects(path)] for _, result in results.iterrows(): if not has_valid_times(result): raise ValueError(f"Result with id {result[id]} has wrong time signatures.") + check_path_format(result, ensembles, projects) return From 3c09fb7f8c556bae30b67c42e01fca95b2667ebd Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Wed, 6 May 2026 18:06:14 +0200 Subject: [PATCH 21/41] correct typing issues --- corrlib/cli.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/corrlib/cli.py b/corrlib/cli.py index 334dd33..ad1024d 100644 --- a/corrlib/cli.py +++ b/corrlib/cli.py @@ -56,15 +56,15 @@ def lister( """ if entities in ['ensembles', 'Ensembles','ENSEMBLES']: print("Ensembles:") - results = list_ensembles(path) - for e in results: + ensemble_results = list_ensembles(path) + for e in ensemble_results: print(e) elif entities == 'projects': - results = list_projects(path) + project_results = list_projects(path) print("Projects:") header = "UUID".ljust(37) + "| Aliases" print(header) - for project in results: + for project in project_results: if project[1] is not None: aliases = " | ".join(str2list(project[1])) else: From ac3eb272ad890e4d5b0da546e1d746dd9bb873b5 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Wed, 6 May 2026 18:12:00 +0200 Subject: [PATCH 22/41] get rid of circular import --- corrlib/find.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/corrlib/find.py b/corrlib/find.py index fbdd801..b4d1bfe 100644 --- a/corrlib/find.py +++ b/corrlib/find.py @@ -6,7 +6,6 @@ import numpy as np from .input.implementations import codes from .tools import k2m, get_db_file from .tracker import get -from .integrity import has_valid_times from .sql import thin_sql_wrapper from typing import Any, Optional from pathlib import Path @@ -83,9 +82,6 @@ def _time_filter(results: pd.DataFrame, created_before: Optional[str]=None, cre result = results.iloc[ind] created_at = dt.datetime.fromisoformat(result['created_at']) updated_at = dt.datetime.fromisoformat(result['updated_at']) - db_times_valid = has_valid_times(result) - if not db_times_valid: - raise ValueError('Time stamps not valid for result with path', result["path"]) if created_before is not None: date_created_before = dt.datetime.fromisoformat(created_before) From 46b97acf95d237ea087bc3cd40f278d66df5c392 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Wed, 6 May 2026 18:20:07 +0200 Subject: [PATCH 23/41] get rid of circular imports part 2 --- corrlib/find.py | 4 ++++ corrlib/integrity.py | 37 ++++++++++++++++++++++++++++++++++--- 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/corrlib/find.py b/corrlib/find.py index b4d1bfe..fbdd801 100644 --- a/corrlib/find.py +++ b/corrlib/find.py @@ -6,6 +6,7 @@ import numpy as np from .input.implementations import codes from .tools import k2m, get_db_file from .tracker import get +from .integrity import has_valid_times from .sql import thin_sql_wrapper from typing import Any, Optional from pathlib import Path @@ -82,6 +83,9 @@ def _time_filter(results: pd.DataFrame, created_before: Optional[str]=None, cre result = results.iloc[ind] created_at = dt.datetime.fromisoformat(result['created_at']) updated_at = dt.datetime.fromisoformat(result['updated_at']) + db_times_valid = has_valid_times(result) + if not db_times_valid: + raise ValueError('Time stamps not valid for result with path', result["path"]) if created_before is not None: date_created_before = dt.datetime.fromisoformat(created_before) diff --git a/corrlib/integrity.py b/corrlib/integrity.py index 5a3ae05..4c35c4b 100644 --- a/corrlib/integrity.py +++ b/corrlib/integrity.py @@ -7,7 +7,6 @@ from .tracker import get import pyerrors.input.json as pj import os from configparser import ConfigParser -from .find import list_ensembles, list_projects from typing import Any @@ -64,6 +63,38 @@ def are_keys_unique(db: Path, table: str, col: str) -> bool: return bool(results[0] == results[1]) +def _list_projects(path: Path) -> list[tuple[str, str]]: + """ + List all projects known to the library. + + Parameters + ---------- + path: str + The path of the library. + + Returns + ------- + results: list[Any] + The projects known to the library. + """ + db_file = get_db_file(path) + get(path, db_file) + conn = sqlite3.connect(os.path.join(path, db_file)) + c = conn.cursor() + c.execute("SELECT id,aliases FROM projects") + results = c.fetchall() + conn.close() + return results + + +def _list_ensembles(path: Path) -> list[str]: + res = [] + for item in os.listdir(path / "archive"): + if os.path.isdir(path / "archive" / item): + res.append(item) + return res + + def check_path_format(result: pd.Series, ensembles: list[str], projects: list[str]) -> None: """ Check whether the path of the given result has the right format. @@ -107,8 +138,8 @@ def check_db_integrity(path: Path) -> None: search_expr = "SELECT * FROM 'backlogs'" conn = sqlite3.connect(path / db) results = pd.read_sql(search_expr, conn) - ensembles = list_ensembles(path) - projects = [p[0] for p in list_projects(path)] + ensembles = _list_ensembles(path) + projects = [p[0] for p in _list_projects(path)] for _, result in results.iterrows(): if not has_valid_times(result): From 3640f163fca24fc37cc801645abace8e8b779afd Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Wed, 6 May 2026 19:06:50 +0200 Subject: [PATCH 24/41] Give user a sense of the severity, add basic tests --- corrlib/integrity.py | 5 +++- tests/integrity_test.py | 55 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 tests/integrity_test.py diff --git a/corrlib/integrity.py b/corrlib/integrity.py index 4c35c4b..2fb520d 100644 --- a/corrlib/integrity.py +++ b/corrlib/integrity.py @@ -60,7 +60,10 @@ def are_keys_unique(db: Path, table: str, col: str) -> bool: c.execute(f"SELECT COUNT( DISTINCT CAST({col} AS nvarchar(4000))), COUNT({col}) FROM {table};") results = c.fetchall()[0] conn.close() - return bool(results[0] == results[1]) + res = bool(results[0] == results[1]) + if not res: + print("Unique:", results[0], "All:", results[1]) + return res def _list_projects(path: Path) -> list[tuple[str, str]]: diff --git a/tests/integrity_test.py b/tests/integrity_test.py new file mode 100644 index 0000000..2cf8308 --- /dev/null +++ b/tests/integrity_test.py @@ -0,0 +1,55 @@ +import corrlib.integrity as integ +import corrlib.find as find +import datalad.api as dl +import corrlib.initialization as cinit +import sqlite3 +from pathlib import Path +import os + + +def test_list_ensembles(tmp_path: Path) -> None: + """ + Check against the implementation in find to check if they are the same. + """ + os.mkdir(tmp_path / 'archive') + os.mkdir(tmp_path / 'archive' / 'A') + os.mkdir(tmp_path / 'archive' / 'B') + os.mkdir(tmp_path / 'archive' / 'C') + integ_results = integ._list_ensembles(tmp_path) + assert len(integ_results) == 3 + find_results = find.list_ensembles(tmp_path) + assert len(find_results) == 3 + for f,i in zip(find_results, integ_results): + assert f == i + + +def test_list_projects(tmp_path: Path) -> None: + cinit.create(tmp_path) + db = tmp_path / "backlogger.db" + dl.unlock(str(db), dataset=str(tmp_path)) + conn = sqlite3.connect(db) + c = conn.cursor() + + customTags = "" + owner = "owner" + code = "sfcf" + created_at = "today" + updated_at = "today" + + id = "asdf1" + aliases = "a1,s1,d1,f1" + c.execute("INSERT INTO projects (id, aliases, customTags, owner, code, created_at, updated_at) VALUES (?,?,?,?,?,?,?)", (id, aliases, customTags, owner, code , created_at, updated_at)) + id = "asdf2" + aliases = "a2,s2,d2,f2" + c.execute("INSERT INTO projects (id, aliases, customTags, owner, code, created_at, updated_at) VALUES (?,?,?,?,?,?,?)", (id, aliases, customTags, owner, code , created_at, updated_at)) + id = "asdf3" + aliases = "a3,s3,d3,f3" + c.execute("INSERT INTO projects (id, aliases, customTags, owner, code, created_at, updated_at) VALUES (?,?,?,?,?,?,?)", (id, aliases, customTags, owner, code , created_at, updated_at)) + conn.commit() + conn.close + integ_results = integ._list_projects(tmp_path) + assert len(integ_results) == 3 + find_results = find.list_projects(tmp_path) + assert len(find_results) == 3 + for f,i in zip(find_results, integ_results): + assert f == i From a450601b8076bc3b758d44e1221614de2bb6662b Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Wed, 6 May 2026 19:15:58 +0200 Subject: [PATCH 25/41] add test for has_valid_times --- tests/integrity_test.py | 46 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/tests/integrity_test.py b/tests/integrity_test.py index 2cf8308..8cc074f 100644 --- a/tests/integrity_test.py +++ b/tests/integrity_test.py @@ -5,6 +5,8 @@ import corrlib.initialization as cinit import sqlite3 from pathlib import Path import os +import pandas as pd +import datetime as dt def test_list_ensembles(tmp_path: Path) -> None: @@ -53,3 +55,47 @@ def test_list_projects(tmp_path: Path) -> None: assert len(find_results) == 3 for f,i in zip(find_results, integ_results): assert f == i + + +def test_has_valid_time() -> None: + record_A = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf0", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] # only created + record_B = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf1", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2025-03-26 12:55:18.229966', '2025-04-26 12:55:18.229966'] # created and updated + record_C = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf2", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2026-03-26 12:55:18.229966', '2026-04-14 12:55:18.229966'] # created and updated later + record_D = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf3", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2026-03-26 12:55:18.229966', '2026-03-27 12:55:18.229966'] + record_E = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf4", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2024-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # only created, earlier + record_F = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf5", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2026-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # this is invalid... + record_G = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf2", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2026-03-26 12:55:18.229966', str(dt.datetime.now() + dt.timedelta(days=2, hours=3, minutes=5, seconds=30))] # created and updated later + + data = [record_A, record_B, record_C, record_D, record_E] + cols = ["name", + "ensemble", + "code", + "path", + "project", + "parameters", + "parameter_file", + "created_at", + "updated_at"] + df = pd.DataFrame(data,columns=cols) + for _, result in df.iterrows(): + assert integ.has_valid_times(result) + data = [record_F, record_G] + cols = ["name", + "ensemble", + "code", + "path", + "project", + "parameters", + "parameter_file", + "created_at", + "updated_at"] + df = pd.DataFrame(data,columns=cols) + for _, result in df.iterrows(): + assert not integ.has_valid_times(result) From 9d0b922db922397902d7aa703ec561c08a2f61ef Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Wed, 6 May 2026 19:31:00 +0200 Subject: [PATCH 26/41] add simple test for key uniqueness --- tests/integrity_test.py | 52 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/tests/integrity_test.py b/tests/integrity_test.py index 8cc074f..9946a19 100644 --- a/tests/integrity_test.py +++ b/tests/integrity_test.py @@ -99,3 +99,55 @@ def test_has_valid_time() -> None: df = pd.DataFrame(data,columns=cols) for _, result in df.iterrows(): assert not integ.has_valid_times(result) + + +def test_are_keys_unique(tmp_path: Path) -> None: + db = tmp_path / 'test_success.db' + + record_A = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf0", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] # only created + record_B = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf1", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2025-03-26 12:55:18.229966', '2025-04-26 12:55:18.229966'] # created and updated + record_C = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf2", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2026-03-26 12:55:18.229966', '2026-04-14 12:55:18.229966'] # created and updated later + record_D = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf3", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2026-03-26 12:55:18.229966', '2026-03-27 12:55:18.229966'] + record_E = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf4", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2024-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # only created, earlier + record_F = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf5", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2026-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # this is invalid... + record_G = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf2", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2026-03-26 12:55:18.229966', str(dt.datetime.now() + dt.timedelta(days=2, hours=3, minutes=5, seconds=30))] # created and updated later + + data = [record_A, record_B, record_C, record_D, record_E, record_F] + cols = ["name", + "ensemble", + "code", + "path", + "project", + "parameters", + "parameter_file", + "created_at", + "updated_at"] + df = pd.DataFrame(data,columns=cols) + conn = sqlite3.connect(db) + df.to_sql('backlogs', conn) + conn.close() + assert integ.are_keys_unique(db, 'backlogs', 'path') + + db = tmp_path / 'test_fail.db' + data = [record_A, record_B, record_C, record_D, record_E, record_F, record_G] + cols = ["name", + "ensemble", + "code", + "path", + "project", + "parameters", + "parameter_file", + "created_at", + "updated_at"] + df = pd.DataFrame(data,columns=cols) + conn = sqlite3.connect(db) + df.to_sql('backlogs', conn) + conn.close() + assert not integ.are_keys_unique(db, 'backlogs', 'path') From 32987d5557b4ecf317a9e06acbdcc3ab3b1fce33 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Wed, 6 May 2026 19:47:05 +0200 Subject: [PATCH 27/41] add test whether the ensemble in the database is the one in meas_path --- corrlib/integrity.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/corrlib/integrity.py b/corrlib/integrity.py index 2fb520d..66ea4df 100644 --- a/corrlib/integrity.py +++ b/corrlib/integrity.py @@ -113,13 +113,15 @@ def check_path_format(result: pd.Series, ensembles: list[str], projects: list[st meas_key = p.split('::')[1] ensemble = p.split('/')[1] - project = p.split('/')[2].split('::')[0] + project = p.split('/')[3].split('.')[0] if not len(meas_key) == 64: raise ValueError(f'meas_key of {p} is scrambled') if ensemble not in ensembles: raise ValueError(f'meas_key of {p} points to an unknown ensemble') if project not in projects: - raise ValueError(f'meas_key of {p} points to an unknown project id') + raise ValueError(f'meas_key of {p} points to an unknown project id ({project})') + if not ensemble == result['ensemble']: + raise ValueError(f'Ensemble in database and file does not match for path {p}.') From da62af835cca3c9420e4f00d38996e0b5e843652 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Wed, 6 May 2026 20:10:45 +0200 Subject: [PATCH 28/41] stramline, add tests for path_format check --- tests/integrity_test.py | 68 +++++++++++++++++++++++++++++++---------- 1 file changed, 52 insertions(+), 16 deletions(-) diff --git a/tests/integrity_test.py b/tests/integrity_test.py index 9946a19..8a59d50 100644 --- a/tests/integrity_test.py +++ b/tests/integrity_test.py @@ -7,6 +7,7 @@ from pathlib import Path import os import pandas as pd import datetime as dt +import pytest def test_list_ensembles(tmp_path: Path) -> None: @@ -72,8 +73,6 @@ def test_has_valid_time() -> None: '2026-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # this is invalid... record_G = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf2", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", '2026-03-26 12:55:18.229966', str(dt.datetime.now() + dt.timedelta(days=2, hours=3, minutes=5, seconds=30))] # created and updated later - - data = [record_A, record_B, record_C, record_D, record_E] cols = ["name", "ensemble", "code", @@ -83,19 +82,12 @@ def test_has_valid_time() -> None: "parameter_file", "created_at", "updated_at"] + data = [record_A, record_B, record_C, record_D, record_E] + df = pd.DataFrame(data,columns=cols) for _, result in df.iterrows(): assert integ.has_valid_times(result) data = [record_F, record_G] - cols = ["name", - "ensemble", - "code", - "path", - "project", - "parameters", - "parameter_file", - "created_at", - "updated_at"] df = pd.DataFrame(data,columns=cols) for _, result in df.iterrows(): assert not integ.has_valid_times(result) @@ -119,7 +111,6 @@ def test_are_keys_unique(tmp_path: Path) -> None: record_G = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf2", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", '2026-03-26 12:55:18.229966', str(dt.datetime.now() + dt.timedelta(days=2, hours=3, minutes=5, seconds=30))] # created and updated later - data = [record_A, record_B, record_C, record_D, record_E, record_F] cols = ["name", "ensemble", "code", @@ -129,6 +120,8 @@ def test_are_keys_unique(tmp_path: Path) -> None: "parameter_file", "created_at", "updated_at"] + + data = [record_A, record_B, record_C, record_D, record_E, record_F] df = pd.DataFrame(data,columns=cols) conn = sqlite3.connect(db) df.to_sql('backlogs', conn) @@ -137,6 +130,32 @@ def test_are_keys_unique(tmp_path: Path) -> None: db = tmp_path / 'test_fail.db' data = [record_A, record_B, record_C, record_D, record_E, record_F, record_G] + + df = pd.DataFrame(data,columns=cols) + conn = sqlite3.connect(db) + df.to_sql('backlogs', conn) + conn.close() + assert not integ.are_keys_unique(db, 'backlogs', 'path') + + +def test_check_path_format() -> None: + record_A = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_A.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] # only created + record_B = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_B.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2025-03-26 12:55:18.229966', '2025-04-26 12:55:18.229966'] # created and updated + record_C = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_A.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2026-03-26 12:55:18.229966', '2026-04-14 12:55:18.229966'] # created and updated later + record_D = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_B.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2026-03-26 12:55:18.229966', '2026-03-27 12:55:18.229966'] + record_E = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_A.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2024-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # only created, earlier + record_F = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_B.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2026-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # this is invalid... + record_G = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_A.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfas", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2026-03-26 12:55:18.229966', str(dt.datetime.now() + dt.timedelta(days=2, hours=3, minutes=5, seconds=30))] # created and updated later + + projects = ['Project_A', 'Project_B'] + ensembles = ['ensA'] cols = ["name", "ensemble", "code", @@ -146,8 +165,25 @@ def test_are_keys_unique(tmp_path: Path) -> None: "parameter_file", "created_at", "updated_at"] + + data = [record_A, record_B, record_C, record_D, record_E, record_F] df = pd.DataFrame(data,columns=cols) - conn = sqlite3.connect(db) - df.to_sql('backlogs', conn) - conn.close() - assert not integ.are_keys_unique(db, 'backlogs', 'path') + for _, result in df.iterrows(): + integ.check_path_format(result, ensembles, projects) + + projects = ['Project_A', 'Project_B'] + ensembles = ['ensB'] + for _, result in df.iterrows(): + with pytest.raises(ValueError): + integ.check_path_format(result, ensembles, projects) + + projects = ['Project_A', 'Project_B'] + ensembles = ['ensA', 'ensB'] + for _, result in df.iterrows(): + integ.check_path_format(result, ensembles, projects) + + data = [record_G] + df = pd.DataFrame(data,columns=cols) + for _, result in df.iterrows(): + with pytest.raises(ValueError): + integ.check_path_format(result, ensembles, projects) From b088a282917a1bed18e8620abfcf5ccb0238df48 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Thu, 7 May 2026 15:27:21 +0200 Subject: [PATCH 29/41] bump version --- corrlib/version.py | 28 +++++++++------------------- 1 file changed, 9 insertions(+), 19 deletions(-) diff --git a/corrlib/version.py b/corrlib/version.py index 68d1c45..23dd03f 100644 --- a/corrlib/version.py +++ b/corrlib/version.py @@ -1,5 +1,6 @@ -# file generated by setuptools-scm +# file generated by vcs-versioning # don't change, don't track in version control +from __future__ import annotations __all__ = [ "__version__", @@ -10,25 +11,14 @@ __all__ = [ "commit_id", ] -TYPE_CHECKING = False -if TYPE_CHECKING: - from typing import Tuple - from typing import Union - - VERSION_TUPLE = Tuple[Union[int, str], ...] - COMMIT_ID = Union[str, None] -else: - VERSION_TUPLE = object - COMMIT_ID = object - version: str __version__: str -__version_tuple__: VERSION_TUPLE -version_tuple: VERSION_TUPLE -commit_id: COMMIT_ID -__commit_id__: COMMIT_ID +__version_tuple__: tuple[int | str, ...] +version_tuple: tuple[int | str, ...] +commit_id: str | None +__commit_id__: str | None -__version__ = version = '0.2.4.dev14+g602324f84.d20251202' -__version_tuple__ = version_tuple = (0, 2, 4, 'dev14', 'g602324f84.d20251202') +__version__ = version = '0.3.1.dev0+g08de17e6b.d20260507' +__version_tuple__ = version_tuple = (0, 3, 1, 'dev0', 'g08de17e6b.d20260507') -__commit_id__ = commit_id = 'g602324f84' +__commit_id__ = commit_id = 'g08de17e6b' From 50fb204cb1433ed1df5b16a2042101f3a843d674 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Fri, 8 May 2026 11:32:04 +0200 Subject: [PATCH 30/41] HOTFIX: enable r_start, r_sto, r_step params for t0 and t1 --- corrlib/input/openQCD.py | 16 ++++++++++++++-- corrlib/toml.py | 6 ++++-- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/corrlib/input/openQCD.py b/corrlib/input/openQCD.py index 879b555..c8eef72 100644 --- a/corrlib/input/openQCD.py +++ b/corrlib/input/openQCD.py @@ -164,7 +164,8 @@ def read_rwms(path: Path, project: str, dir_in_project: str, param: dict[str, An return rw_dict -def extract_t0(path: Path, project: str, dir_in_project: str, param: dict[str, Any], prefix: str, dtr_read: int, xmin: int, spatial_extent: int, fit_range: int = 5, postfix: str="", names: Optional[list[str]]=None, files: Optional[list[str]]=None) -> dict[str, Any]: +def extract_t0(path: Path, project: str, dir_in_project: str, param: dict[str, Any], prefix: str, dtr_read: int, xmin: int, spatial_extent: int, fit_range: int = 5, postfix: str="", names: Optional[list[str]]=None, files: Optional[list[str]]=None, + r_start: list[int]=[], r_stop: list[int]=[], r_step:int=1) -> dict[str, Any]: """ Extract t0 measurements from the project. @@ -218,6 +219,11 @@ def extract_t0(path: Path, project: str, dir_in_project: str, param: dict[str, A if postfix is not None: kwargs['postfix'] = postfix kwargs['plot_fit'] = False + if not r_start == []: + kwargs['r_start'] = r_start + if not r_stop == []: + kwargs['r_stop'] = r_stop + kwargs['r_step'] = r_step t0 = input.extract_t0(directory, prefix, @@ -238,7 +244,8 @@ def extract_t0(path: Path, project: str, dir_in_project: str, param: dict[str, A return t0_dict -def extract_t1(path: Path, project: str, dir_in_project: str, param: dict[str, Any], prefix: str, dtr_read: int, xmin: int, spatial_extent: int, fit_range: int = 5, postfix: str = "", names: Optional[list[str]]=None, files: Optional[list[str]]=None) -> dict[str, Any]: +def extract_t1(path: Path, project: str, dir_in_project: str, param: dict[str, Any], prefix: str, dtr_read: int, xmin: int, spatial_extent: int, fit_range: int = 5, postfix: str = "", names: Optional[list[str]]=None, files: Optional[list[str]]=None, + r_start: list[int]=[], r_stop: list[int]=[], r_step:int=1) -> dict[str, Any]: """ Extract t1 measurements from the project. @@ -290,6 +297,11 @@ def extract_t1(path: Path, project: str, dir_in_project: str, param: dict[str, A if postfix is not None: kwargs['postfix'] = postfix kwargs['plot_fit'] = False + if not r_start == []: + kwargs['r_start'] = r_start + if not r_stop == []: + kwargs['r_stop'] = r_stop + kwargs['r_step'] = r_step t0 = input.extract_t0(directory, prefix, dtr_read, diff --git a/corrlib/toml.py b/corrlib/toml.py index 29d7de2..1f4e300 100644 --- a/corrlib/toml.py +++ b/corrlib/toml.py @@ -230,13 +230,15 @@ def import_toml(path: Path, file: str, copy_file: bool=True) -> None: param[rwp] = "Unknown" param['type'] = 't0' measurement = openQCD.extract_t0(path, uuid, md['path'], param, str(md["prefix"]), int(md["dtr_read"]), int(md["xmin"]), int(md["spatial_extent"]), - fit_range=int(md.get('fit_range', 5)), postfix=str(md.get('postfix', '')), names=md.get('names', []), files=md.get('files', [])) + fit_range=int(md.get('fit_range', 5)), postfix=str(md.get('postfix', '')), names=md.get('names', []), files=md.get('files', []), + r_start=md.get('r_start', []), r_stop=md.get('r_stop', []), r_step=md.get('r_step', 1)) elif md['measurement'] == 't1': if 'param_file' in md: param = openQCD.load_ms3_infile(path, uuid, md['param_file']) param['type'] = 't1' measurement = openQCD.extract_t1(path, uuid, md['path'], param, str(md["prefix"]), int(md["dtr_read"]), int(md["xmin"]), int(md["spatial_extent"]), - fit_range=int(md.get('fit_range', 5)), postfix=str(md.get('postfix', '')), names=md.get('names', []), files=md.get('files', [])) + fit_range=int(md.get('fit_range', 5)), postfix=str(md.get('postfix', '')), names=md.get('names', []), files=md.get('files', []), + r_start=md.get('r_start', []), r_stop=md.get('r_stop', []), r_step=md.get('r_step', 1)) write_measurement(path, ensemble, measurement, uuid, project['code'], (md['param_file'] if 'param_file' in md else None)) imeas += 1 print(mname + " imported.") From bbf94e4457c19000b8061fd1c9f630cec74fb045 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Mon, 11 May 2026 22:53:43 +0200 Subject: [PATCH 31/41] update version --- corrlib/version.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/corrlib/version.py b/corrlib/version.py index 68d1c45..60637af 100644 --- a/corrlib/version.py +++ b/corrlib/version.py @@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE commit_id: COMMIT_ID __commit_id__: COMMIT_ID -__version__ = version = '0.2.4.dev14+g602324f84.d20251202' -__version_tuple__ = version_tuple = (0, 2, 4, 'dev14', 'g602324f84.d20251202') +__version__ = version = '0.2.4.dev71+g5e712b64c.d20260213' +__version_tuple__ = version_tuple = (0, 2, 4, 'dev71', 'g5e712b64c.d20260213') -__commit_id__ = commit_id = 'g602324f84' +__commit_id__ = commit_id = 'g5e712b64c' From 481558c5d7fcbec0d43dd0a79a88b17822962c17 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Tue, 12 May 2026 09:16:10 +0200 Subject: [PATCH 32/41] stricter Ruff rules, matching https://github.com/fjosw/pyerrors/pull/282 --- pyproject.toml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index faf7e6c..02d5a7a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,13 +26,17 @@ include = ["corrlib", "corrlib.*"] [tool.setuptools_scm] write_to = "corrlib/version.py" +[tool.ruff] +target-version = "py310" + [tool.ruff.lint] -ignore = ["E501"] -extend-select = [ - "YTT", - "E", - "W", - "F", +extend-select = ["E", "W", "I", "B", "PIE", "PLE", "PLW", "UP", "NPY", "RUF"] +ignore = [ + "F403", # star imports in __init__ files are intentional + "E501", # line too long + "PLC0415", # import outside top level + "PLW2901", # redefined loop name (too noisy) + "RUF002", # ambiguous unicode in docstrings (Greek letters) ] [tool.mypy] From 0798ab9f10f41af1a53e73c907b0732b19272605 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Tue, 19 May 2026 17:16:40 +0200 Subject: [PATCH 33/41] add LICENSE --- LICENSE | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..1889fd3 --- /dev/null +++ b/LICENSE @@ -0,0 +1,9 @@ +MIT License + +Copyright (c) 2026 Justus Kuhlmann + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice (including the next paragraph) shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. From c6ad3f900346a0817621bfe4dca4fbd80f1e35bd Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Tue, 26 May 2026 11:16:18 +0200 Subject: [PATCH 34/41] add automatic saving of plots for t0 and t1 --- corrlib/initialization.py | 2 ++ corrlib/input/openQCD.py | 17 ++++++++++++++--- corrlib/meas_io.py | 4 +++- corrlib/toml.py | 4 ++-- corrlib/tools.py | 27 +++++++++++++++++++++++++++ pyproject.toml | 1 + 6 files changed, 49 insertions(+), 6 deletions(-) diff --git a/corrlib/initialization.py b/corrlib/initialization.py index bdf9cee..99f90d1 100644 --- a/corrlib/initialization.py +++ b/corrlib/initialization.py @@ -71,6 +71,7 @@ def _create_config(path: Path, tracker: str, cached: bool) -> ConfigParser: 'db': 'backlogger.db', 'projects_path': 'projects', 'archive_path': 'archive', + 'plot_path': 'plots', 'toml_imports_path': 'toml_imports', 'import_scripts_path': 'import_scripts', } @@ -113,6 +114,7 @@ def create(path: Path, tracker: str = 'datalad', cached: bool = True) -> None: os.chmod(path / config['paths']['db'], 0o666) os.makedirs(path / config['paths']['projects_path']) os.makedirs(path / config['paths']['archive_path']) + os.makedirs(path / config['paths']['plot_path']) os.makedirs(path / config['paths']['toml_imports_path']) os.makedirs(path / config['paths']['import_scripts_path'] / 'template.py') with open(path / ".gitignore", "w") as fp: diff --git a/corrlib/input/openQCD.py b/corrlib/input/openQCD.py index c8eef72..9dfe85f 100644 --- a/corrlib/input/openQCD.py +++ b/corrlib/input/openQCD.py @@ -4,8 +4,10 @@ import os import fnmatch from typing import Any, Optional from pathlib import Path +import matplotlib.pyplot as plt from ..pars.openQCD import ms1 from ..pars.openQCD import qcd2 +from ..tools import get_plot_dir @@ -164,7 +166,7 @@ def read_rwms(path: Path, project: str, dir_in_project: str, param: dict[str, An return rw_dict -def extract_t0(path: Path, project: str, dir_in_project: str, param: dict[str, Any], prefix: str, dtr_read: int, xmin: int, spatial_extent: int, fit_range: int = 5, postfix: str="", names: Optional[list[str]]=None, files: Optional[list[str]]=None, +def extract_t0(path: Path, project: str, dir_in_project: str, ensemble: str, param: dict[str, Any], prefix: str, dtr_read: int, xmin: int, spatial_extent: int, fit_range: int = 5, postfix: str="", names: Optional[list[str]]=None, files: Optional[list[str]]=None, r_start: list[int]=[], r_stop: list[int]=[], r_step:int=1) -> dict[str, Any]: """ Extract t0 measurements from the project. @@ -224,7 +226,7 @@ def extract_t0(path: Path, project: str, dir_in_project: str, param: dict[str, A if not r_stop == []: kwargs['r_stop'] = r_stop kwargs['r_step'] = r_step - + kwargs['plot_fit'] = True t0 = input.extract_t0(directory, prefix, dtr_read, @@ -234,6 +236,10 @@ def extract_t0(path: Path, project: str, dir_in_project: str, param: dict[str, A c=0.3, **kwargs ) + plot_dir = path / get_plot_dir(path) / ensemble / project + if not os.path.exists(plot_dir): + os.makedirs(plot_dir) + plt.savefig(plot_dir / "t0.pdf") par_list= [] for k in ["integrator", "eps", "ntot", "dnms"]: par_list.append(str(param[k])) @@ -244,7 +250,7 @@ def extract_t0(path: Path, project: str, dir_in_project: str, param: dict[str, A return t0_dict -def extract_t1(path: Path, project: str, dir_in_project: str, param: dict[str, Any], prefix: str, dtr_read: int, xmin: int, spatial_extent: int, fit_range: int = 5, postfix: str = "", names: Optional[list[str]]=None, files: Optional[list[str]]=None, +def extract_t1(path: Path, project: str, dir_in_project: str, ensemble: str, param: dict[str, Any], prefix: str, dtr_read: int, xmin: int, spatial_extent: int, fit_range: int = 5, postfix: str = "", names: Optional[list[str]]=None, files: Optional[list[str]]=None, r_start: list[int]=[], r_stop: list[int]=[], r_step:int=1) -> dict[str, Any]: """ Extract t1 measurements from the project. @@ -302,6 +308,7 @@ def extract_t1(path: Path, project: str, dir_in_project: str, param: dict[str, A if not r_stop == []: kwargs['r_stop'] = r_stop kwargs['r_step'] = r_step + kwargs['plot_fit'] = True t0 = input.extract_t0(directory, prefix, dtr_read, @@ -311,6 +318,10 @@ def extract_t1(path: Path, project: str, dir_in_project: str, param: dict[str, A c=2./3, **kwargs ) + plot_dir = path / get_plot_dir(path) / ensemble / project + if not os.path.exists(plot_dir): + os.makedirs(plot_dir) + plt.savefig(plot_dir / "t1.pdf") par_list= [] for k in ["integrator", "eps", "ntot", "dnms"]: par_list.append(str(param[k])) diff --git a/corrlib/meas_io.py b/corrlib/meas_io.py index 6b6e5f1..52e307e 100644 --- a/corrlib/meas_io.py +++ b/corrlib/meas_io.py @@ -6,7 +6,7 @@ import json from typing import Union from pyerrors import Obs, Corr, dump_object, load_object from hashlib import sha256 -from .tools import get_db_file, cache_enabled +from .tools import get_db_file, cache_enabled, get_plot_dir from .tracker import get, save, unlock import shutil from typing import Any @@ -104,6 +104,8 @@ def write_measurement(path: Path, ensemble: str, measurement: dict[str, dict[str subkeys.append(subkey) pars[subkey] = json.dumps(parameters["rw_fcts"][i]) elif ms_type in ['t0', 't1']: + plot_file = path / get_plot_dir(path) / ensemble / uuid / (ms_type + ".pdf") + files_to_save.append(plot_file) if parameter_file is not None: parameters = openQCD.load_ms3_infile(path, uuid, parameter_file) else: diff --git a/corrlib/toml.py b/corrlib/toml.py index 1f4e300..76533f4 100644 --- a/corrlib/toml.py +++ b/corrlib/toml.py @@ -229,14 +229,14 @@ def import_toml(path: Path, file: str, copy_file: bool=True) -> None: for rwp in ["integrator", "eps", "ntot", "dnms"]: param[rwp] = "Unknown" param['type'] = 't0' - measurement = openQCD.extract_t0(path, uuid, md['path'], param, str(md["prefix"]), int(md["dtr_read"]), int(md["xmin"]), int(md["spatial_extent"]), + measurement = openQCD.extract_t0(path, uuid, md['path'], ensemble, param, str(md["prefix"]), int(md["dtr_read"]), int(md["xmin"]), int(md["spatial_extent"]), fit_range=int(md.get('fit_range', 5)), postfix=str(md.get('postfix', '')), names=md.get('names', []), files=md.get('files', []), r_start=md.get('r_start', []), r_stop=md.get('r_stop', []), r_step=md.get('r_step', 1)) elif md['measurement'] == 't1': if 'param_file' in md: param = openQCD.load_ms3_infile(path, uuid, md['param_file']) param['type'] = 't1' - measurement = openQCD.extract_t1(path, uuid, md['path'], param, str(md["prefix"]), int(md["dtr_read"]), int(md["xmin"]), int(md["spatial_extent"]), + measurement = openQCD.extract_t1(path, uuid, md['path'], ensemble, param, str(md["prefix"]), int(md["dtr_read"]), int(md["xmin"]), int(md["spatial_extent"]), fit_range=int(md.get('fit_range', 5)), postfix=str(md.get('postfix', '')), names=md.get('names', []), files=md.get('files', []), r_start=md.get('r_start', []), r_stop=md.get('r_stop', []), r_step=md.get('r_step', 1)) write_measurement(path, ensemble, measurement, uuid, project['code'], (md['param_file'] if 'param_file' in md else None)) diff --git a/corrlib/tools.py b/corrlib/tools.py index 9ce194b..8dec61a 100644 --- a/corrlib/tools.py +++ b/corrlib/tools.py @@ -129,6 +129,33 @@ def get_db_file(path: Path) -> Path: return db_file +def get_plot_dir(path: Path) -> Path: + """ + Get the plots directory associated with the library at the given path. + + Parameters + ---------- + path: str + The path of the library. + + Returns + ------- + db_file: str + The file holding the database. + """ + path = Path(path) + if not os.path.exists(path): + raise FileNotFoundError(f"Corrlib path {path} does not exist.") + config_path = path / CONFIG_FILENAME + config = ConfigParser() + if os.path.exists(config_path): + config.read(config_path) + else: + raise FileNotFoundError("Configuration file not found.") + plot_dir = Path(config.get('paths', 'plot_dir', fallback='plots')) + return plot_dir + + def cache_enabled(path: Path) -> bool: """ Check, whether the library is cached. diff --git a/pyproject.toml b/pyproject.toml index faf7e6c..68abbe8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,7 @@ dependencies = [ 'pyerrors>=2.11.1', "datalad>=1.1.0", 'typer>=0.12.5', + "matplotlib>=3.10.7", ] description = "Python correlation library" authors = [ From 354cd96407f4d4fa8f2d2e4d2d4cf07b2fde0f9e Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Tue, 26 May 2026 11:40:18 +0200 Subject: [PATCH 35/41] add matplitlib --- uv.lock | 2 ++ 1 file changed, 2 insertions(+) diff --git a/uv.lock b/uv.lock index f76ee81..6e246e6 100644 --- a/uv.lock +++ b/uv.lock @@ -409,6 +409,7 @@ source = { editable = "." } dependencies = [ { name = "datalad" }, { name = "gitpython" }, + { name = "matplotlib" }, { name = "pyerrors" }, { name = "typer" }, ] @@ -427,6 +428,7 @@ dev = [ requires-dist = [ { name = "datalad", specifier = ">=1.1.0" }, { name = "gitpython", specifier = ">=3.1.45" }, + { name = "matplotlib", specifier = ">=3.10.7" }, { name = "pyerrors", specifier = ">=2.11.1" }, { name = "typer", specifier = ">=0.12.5" }, ] From f14fe6f2e3e41d3d00b6a15c94cb3b5015c8557e Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Wed, 1 Jul 2026 10:16:56 +0200 Subject: [PATCH 36/41] add build --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index f97ff98..cff0601 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ test.ipynb .vscode .venv .pytest_cache -.coverage \ No newline at end of file +.coverage +build From 25124025fbf27fcaa322ee34fffde87f9286b50f Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Wed, 1 Jul 2026 10:18:18 +0200 Subject: [PATCH 37/41] Fix: rename clone target to path --- corrlib/tracker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/corrlib/tracker.py b/corrlib/tracker.py index 6f4ae3d..1581fba 100644 --- a/corrlib/tracker.py +++ b/corrlib/tracker.py @@ -144,7 +144,7 @@ def clone(path: Path, source: str, target: str) -> None: path = Path(path) tracker = get_tracker(path) if tracker == 'datalad': - dl.clone(target=target, source=source, dataset=path) + dl.clone(path=target, source=source, dataset=path) elif tracker == 'None': os.makedirs(path, exist_ok=True) # Implement a simple clone by copying files From 4b1c2130906a0455cdc20f40f1dff2eb9b485e1c Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Wed, 1 Jul 2026 14:15:37 +0200 Subject: [PATCH 38/41] Fix: variable typo --- corrlib/input/sfcf.py | 8 ++++---- corrlib/toml.py | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/corrlib/input/sfcf.py b/corrlib/input/sfcf.py index acd8261..1af366f 100644 --- a/corrlib/input/sfcf.py +++ b/corrlib/input/sfcf.py @@ -258,7 +258,7 @@ def get_specs(key: str, parameters: dict[str, Any], sep: str = '/') -> str: return s -def read_data(path: Path, project: str, dir_in_project: str, prefix: str, param: dict[str, Any], version: str = '1.0c', cfg_seperator: str = 'n', sep: str = '/', **kwargs: Any) -> dict[str, Any]: +def read_data(path: Path, project: str, dir_in_project: str, prefix: str, param: dict[str, Any], version: str = '1.0c', cfg_separator: str = 'n', sep: str = '/', **kwargs: Any) -> dict[str, Any]: """ Extract the data from the sfcf file. @@ -274,7 +274,7 @@ def read_data(path: Path, project: str, dir_in_project: str, prefix: str, param: The parameter dictionary, as given by read_param. version: str Version of sfcf. - cfg_seperator: str + cfg_separator: str Separator of the configuration number. Needed for reading. default: "n" sep: str Seperator for the key in return dict. (default: "/) @@ -321,10 +321,10 @@ def read_data(path: Path, project: str, dir_in_project: str, prefix: str, param: if not param['crr'] == []: if names is not None: data_crr = pe.input.sfcf.read_sfcf_multi(directory, prefix, param['crr'], param['mrr'], corr_type_list, range(len(param['wf_offsets'])), - range(len(param['wf_basis'])), range(len(param['wf_basis'])), version, cfg_seperator, keyed_out=True, silent=True, names=names) + range(len(param['wf_basis'])), range(len(param['wf_basis'])), version, cfg_separator, keyed_out=True, silent=True, names=names) else: data_crr = pe.input.sfcf.read_sfcf_multi(directory, prefix, param['crr'], param['mrr'], corr_type_list, range(len(param['wf_offsets'])), - range(len(param['wf_basis'])), range(len(param['wf_basis'])), version, cfg_seperator, keyed_out=True, silent=True) + range(len(param['wf_basis'])), range(len(param['wf_basis'])), version, cfg_separator, keyed_out=True, silent=True) for key in data_crr.keys(): data[key] = data_crr[key] diff --git a/corrlib/toml.py b/corrlib/toml.py index 76533f4..c452dd8 100644 --- a/corrlib/toml.py +++ b/corrlib/toml.py @@ -116,7 +116,7 @@ def check_measurement_data(measurements: dict[str, dict[str, str]], code: str) - """ var_names: list[str] = [] if code == "sfcf": - var_names = ["path", "ensemble", "param_file", "version", "prefix", "cfg_seperator", "names"] + var_names = ["path", "ensemble", "param_file", "version", "prefix", "cfg_separator", "names"] elif code == "openQCD": var_names = ["path", "ensemble", "measurement", "prefix"] # , "param_file" for mname, md in measurements.items(): @@ -191,10 +191,10 @@ def import_toml(path: Path, file: str, copy_file: bool=True) -> None: param = sfcf.read_param(path, uuid, md['param_file']) if 'names' in md.keys(): measurement = sfcf.read_data(path, uuid, md['path'], md['prefix'], param, - version=md['version'], cfg_seperator=md['cfg_seperator'], sep='/', names=md['names']) + version=md['version'], cfg_separator=md['cfg_separator'], sep='/', names=md['names']) else: measurement = sfcf.read_data(path, uuid, md['path'], md['prefix'], param, - version=md['version'], cfg_seperator=md['cfg_seperator'], sep='/') + version=md['version'], cfg_separator=md['cfg_separator'], sep='/') elif project['code'] == 'openQCD': if md['measurement'] == 'ms1': From 07fdc1ba6af9faa1dd597da02b79e4d38b71ec60 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Tue, 7 Jul 2026 08:52:01 +0200 Subject: [PATCH 39/41] check if param file exists --- corrlib/input/openQCD.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/corrlib/input/openQCD.py b/corrlib/input/openQCD.py index 9dfe85f..8f5a8a1 100644 --- a/corrlib/input/openQCD.py +++ b/corrlib/input/openQCD.py @@ -31,6 +31,8 @@ def load_ms1_infile(path: Path, project: str, file_in_project: str) -> dict[str, """ file = os.path.join(path, "projects", project, file_in_project) + if not os.path.exists(file): + raise IOError(f"File {file} does not exist.") ds = os.path.join(path, "projects", project) dl.get(file, dataset=ds) with open(file, 'r') as fp: From 4cf17c3993dfadc8241f8f83965289deefb721d3 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Tue, 7 Jul 2026 13:35:40 +0200 Subject: [PATCH 40/41] use stricter ruff rules --- corrlib/__init__.py | 8 +++--- corrlib/__main__.py | 2 +- corrlib/cli.py | 51 +++++++++++++++++------------------ corrlib/find.py | 43 ++++++++++++++--------------- corrlib/git_tools.py | 8 +++--- corrlib/initialization.py | 7 ++--- corrlib/input/__init__.py | 4 +-- corrlib/input/openQCD.py | 48 +++++++++++++++++++-------------- corrlib/input/sfcf.py | 12 ++++----- corrlib/integrity.py | 13 ++++----- corrlib/main.py | 23 ++++++++-------- corrlib/meas_io.py | 32 +++++++++++----------- corrlib/pars/openQCD/flags.py | 3 ++- corrlib/pars/openQCD/ms1.py | 6 ++--- corrlib/pars/openQCD/qcd2.py | 4 +-- corrlib/sql.py | 3 ++- corrlib/toml.py | 17 ++++++------ corrlib/tools.py | 2 +- corrlib/tracker.py | 22 +++++++-------- corrlib/version.py | 10 +++---- 20 files changed, 167 insertions(+), 151 deletions(-) diff --git a/corrlib/__init__.py b/corrlib/__init__.py index 4e1b364..1fd2bee 100644 --- a/corrlib/__init__.py +++ b/corrlib/__init__.py @@ -15,10 +15,10 @@ For now, we are interested in collecting primary IObservables only, as these are __app_name__ = "corrlib" -from .import input as input -from .initialization import create as create -from .meas_io import load_record as load_record -from .meas_io import load_records as load_records +from . import input as input from .find import find_project as find_project from .find import find_record as find_record from .find import list_projects as list_projects +from .initialization import create as create +from .meas_io import load_record as load_record +from .meas_io import load_records as load_records diff --git a/corrlib/__main__.py b/corrlib/__main__.py index 24f9c83..e719ee7 100644 --- a/corrlib/__main__.py +++ b/corrlib/__main__.py @@ -1,4 +1,4 @@ -from corrlib import cli, __app_name__ +from corrlib import __app_name__, cli def main() -> None: diff --git a/corrlib/cli.py b/corrlib/cli.py index a28c837..9d7fcfe 100644 --- a/corrlib/cli.py +++ b/corrlib/cli.py @@ -1,19 +1,18 @@ -from typing import Optional -import typer -from corrlib import __app_name__ - -from .initialization import create -from .toml import import_tomls, update_project, reimport_project -from .find import find_record, list_projects, list_ensembles, get_stat -from .tools import str2list -from .main import update_aliases -from .meas_io import drop_cache as mio_drop_cache -from .integrity import full_integrity_check - import os from importlib.metadata import version from pathlib import Path +import typer + +from corrlib import __app_name__ + +from .find import find_record, get_stat, list_ensembles, list_projects +from .initialization import create +from .integrity import full_integrity_check +from .main import update_aliases +from .meas_io import drop_cache as mio_drop_cache +from .toml import import_tomls, reimport_project, update_project +from .tools import str2list app = typer.Typer() @@ -26,7 +25,7 @@ def _version_callback(value: bool) -> None: @app.command() def update( - path: Path = typer.Option( + path: Path = typer.Option( # noqa: B008 Path('.'), "--dataset", "-d", @@ -42,7 +41,7 @@ def update( @app.command() def lister( - path: Path = typer.Option( + path: Path = typer.Option( # noqa: B008 Path('.'), "--dataset", "-d", @@ -73,7 +72,7 @@ def lister( @app.command() def alias_add( - path: Path = typer.Option( + path: Path = typer.Option( # noqa: B008 Path('.'), "--dataset", "-d", @@ -91,7 +90,7 @@ def alias_add( @app.command() def find( - path: Path = typer.Option( + path: Path = typer.Option( # noqa: B008 Path('.'), "--dataset", "-d", @@ -100,7 +99,7 @@ def find( corr: str = typer.Argument(), code: str = typer.Argument(), arg: str = typer.Option( - str('all'), + 'all', "--argument", "-a", ), @@ -125,7 +124,7 @@ def find( @app.command() def stat( - path: Path = typer.Option( + path: Path = typer.Option( # noqa: B008 Path('.'), "--dataset", "-d", @@ -141,7 +140,7 @@ def stat( @app.command() -def check(path: Path = typer.Option( +def check(path: Path = typer.Option( # noqa: B008 Path('.'), "--dataset", "-d", @@ -155,7 +154,7 @@ def check(path: Path = typer.Option( @app.command() def importer( - path: Path = typer.Option( + path: Path = typer.Option( # noqa: B008 Path('.'), "--dataset", "-d", @@ -163,7 +162,7 @@ def importer( files: str = typer.Argument( ), copy_file: bool = typer.Option( - bool(True), + True, "--save", "-s", ), @@ -179,7 +178,7 @@ def importer( @app.command() def reimporter( - path: Path = typer.Option( + path: Path = typer.Option( # noqa: B008 Path('.'), "--dataset", "-d", @@ -204,13 +203,13 @@ def reimporter( @app.command() def init( - path: Path = typer.Option( + path: Path = typer.Option( # noqa: B008 Path('.'), "--dataset", "-d", ), tracker: str = typer.Option( - str('datalad'), + 'datalad', "--tracker", "-t", ), @@ -224,7 +223,7 @@ def init( @app.command() def drop_cache( - path: Path = typer.Option( + path: Path = typer.Option( # noqa: B008 Path('.'), "--dataset", "-d", @@ -239,7 +238,7 @@ def drop_cache( @app.callback() def main( - version: Optional[bool] = typer.Option( + version: bool | None = typer.Option( None, "--version", "-v", diff --git a/corrlib/find.py b/corrlib/find.py index af21a4d..dd4be13 100644 --- a/corrlib/find.py +++ b/corrlib/find.py @@ -1,21 +1,22 @@ -import sqlite3 -import os -import json -import pandas as pd -import numpy as np -from .input.implementations import codes -from .tools import k2m, get_db_file -from .tracker import get -from .integrity import has_valid_times -from .sql import thin_sql_wrapper -from typing import Any, Optional -from pathlib import Path import datetime as dt +import json +import os +import sqlite3 from collections.abc import Callable -import warnings -from .meas_io import load_record +from pathlib import Path +from typing import Any + +import numpy as np +import pandas as pd from pyerrors import Corr, Obs +from .input.implementations import codes +from .integrity import has_valid_times +from .meas_io import load_record +from .sql import thin_sql_wrapper +from .tools import get_db_file, k2m +from .tracker import get + def _project_lookup_by_alias(path: Path, alias: str) -> str: """ @@ -63,7 +64,7 @@ def _project_lookup_by_id(path: Path, uuid: str) -> list[tuple[str, ...]]: return results -def _time_filter(results: pd.DataFrame, created_before: Optional[str]=None, created_after: Optional[Any]=None, updated_before: Optional[Any]=None, updated_after: Optional[Any]=None) -> pd.DataFrame: +def _time_filter(results: pd.DataFrame, created_before: str | None=None, created_after: str | None=None, updated_before: str | None=None, updated_after: str | None=None) -> pd.DataFrame: """ Filter the results from the database in terms of the creation and update times. @@ -112,7 +113,7 @@ def _time_filter(results: pd.DataFrame, created_before: Optional[str]=None, cre return results.drop(drops) -def _db_lookup(db: Path, ensemble: str, correlator_name: str, code: str, project: Optional[str]=None, parameters: Optional[str]=None) -> pd.DataFrame: +def _db_lookup(db: Path, ensemble: str, correlator_name: str, code: str, project: str | None=None, parameters: str | None=None) -> pd.DataFrame: """ Look up a correlator record in the database by the data given to the method. @@ -286,7 +287,7 @@ def openQCD_filter(results:pd.DataFrame, **kwargs: Any) -> pd.DataFrame: The filtered results. """ - warnings.warn("A filter for openQCD parameters is no implemented yet.", Warning) + raise Warning("A filter for openQCD parameters is no implemented yet.") return results @@ -319,10 +320,10 @@ def _code_filter(results: pd.DataFrame, code: str, **kwargs: Any) -> pd.DataFram raise ValueError(f"Code {code} is not known.") -def find_record(path: Path, ensemble: str, correlator_name: str, code: str, project: Optional[str]=None, parameters: Optional[str]=None, - created_before: Optional[str]=None, created_after: Optional[str]=None, updated_before: Optional[str]=None, updated_after: Optional[str]=None, - revision: Optional[str]=None, - customFilter: Optional[Callable[[pd.DataFrame], pd.DataFrame]] = None, +def find_record(path: Path, ensemble: str, correlator_name: str, code: str, project: str | None=None, parameters: str | None=None, + created_before: str | None=None, created_after: str | None=None, updated_before: str | None=None, updated_after: str | None=None, + revision: str | None=None, + customFilter: Callable[[pd.DataFrame], pd.DataFrame] | None = None, **kwargs: Any) -> pd.DataFrame: path = Path(path) db_file = get_db_file(path) diff --git a/corrlib/git_tools.py b/corrlib/git_tools.py index d77f109..7808ada 100644 --- a/corrlib/git_tools.py +++ b/corrlib/git_tools.py @@ -1,8 +1,10 @@ import os -from .tracker import save -import git from pathlib import Path +import git + +from .tracker import save + GITMODULES_FILE = '.gitmodules' @@ -25,7 +27,7 @@ def move_submodule(repo_path: Path, old_path: Path, new_path: Path) -> None: gitmodules_file_path = repo_path / GITMODULES_FILE # update paths in .gitmodules - with open(gitmodules_file_path, 'r') as file: + with open(gitmodules_file_path) as file: lines = [line.strip() for line in file] updated_lines = [] diff --git a/corrlib/initialization.py b/corrlib/initialization.py index 99f90d1..83a3b44 100644 --- a/corrlib/initialization.py +++ b/corrlib/initialization.py @@ -1,9 +1,10 @@ -from configparser import ConfigParser -import sqlite3 import os -from .tracker import save, init +import sqlite3 +from configparser import ConfigParser from pathlib import Path + from .tools import CONFIG_FILENAME +from .tracker import init, save def _create_db(db: Path) -> None: diff --git a/corrlib/input/__init__.py b/corrlib/input/__init__.py index be6d6b2..3ccbf28 100644 --- a/corrlib/input/__init__.py +++ b/corrlib/input/__init__.py @@ -2,6 +2,6 @@ Import functions for different codes. """ -from . import sfcf as sfcf -from . import openQCD as openQCD from . import implementations as implementations +from . import openQCD as openQCD +from . import sfcf as sfcf diff --git a/corrlib/input/openQCD.py b/corrlib/input/openQCD.py index 8f5a8a1..eb17fd3 100644 --- a/corrlib/input/openQCD.py +++ b/corrlib/input/openQCD.py @@ -1,14 +1,14 @@ -import pyerrors.input.openQCD as input -import datalad.api as dl -import os import fnmatch -from typing import Any, Optional +import os from pathlib import Path -import matplotlib.pyplot as plt -from ..pars.openQCD import ms1 -from ..pars.openQCD import qcd2 -from ..tools import get_plot_dir +from typing import Any +import datalad.api as dl +import matplotlib.pyplot as plt +import pyerrors.input.openQCD as input + +from ..pars.openQCD import ms1, qcd2 +from ..tools import get_plot_dir def load_ms1_infile(path: Path, project: str, file_in_project: str) -> dict[str, Any]: @@ -32,17 +32,17 @@ def load_ms1_infile(path: Path, project: str, file_in_project: str) -> dict[str, file = os.path.join(path, "projects", project, file_in_project) if not os.path.exists(file): - raise IOError(f"File {file} does not exist.") + raise OSError(f"File {file} does not exist.") ds = os.path.join(path, "projects", project) dl.get(file, dataset=ds) - with open(file, 'r') as fp: + with open(file) as fp: lines = fp.readlines() fp.close() param: dict[str, Any] = {} param['rw_fcts'] = [] param['rand'] = {} - for i, line in enumerate(lines): + for line in lines: if line.startswith('#'): continue if line.startswith('\n'): @@ -99,7 +99,7 @@ def load_ms3_infile(path: Path, project: str, file_in_project: str) -> dict[str, file = os.path.join(path, "projects", project, file_in_project) ds = os.path.join(path, "projects", project) dl.get(file, dataset=ds) - with open(file, 'r') as fp: + with open(file) as fp: lines = fp.readlines() fp.close() param = {} @@ -111,7 +111,7 @@ def load_ms3_infile(path: Path, project: str, file_in_project: str) -> dict[str, return param -def read_rwms(path: Path, project: str, dir_in_project: str, param: dict[str, Any], prefix: str, postfix: str="ms1", version: str='2.0', names: Optional[list[str]]=None, files: Optional[list[str]]=None) -> dict[str, Any]: +def read_rwms(path: Path, project: str, dir_in_project: str, param: dict[str, Any], prefix: str, postfix: str="ms1", version: str='2.0', names: list[str] | None=None, files: list[str] | None=None) -> dict[str, Any]: """ Read reweighting factor measurements from the project. @@ -146,7 +146,7 @@ def read_rwms(path: Path, project: str, dir_in_project: str, param: dict[str, An directory = os.path.join(dataset, dir_in_project) if files is None: files = [] - for root, ds, fs in os.walk(directory): + for _root, _ds, fs in os.walk(directory): for f in fs: if fnmatch.fnmatch(f, prefix + "*" + postfix + ".dat"): files.append(f) @@ -168,8 +168,8 @@ def read_rwms(path: Path, project: str, dir_in_project: str, param: dict[str, An return rw_dict -def extract_t0(path: Path, project: str, dir_in_project: str, ensemble: str, param: dict[str, Any], prefix: str, dtr_read: int, xmin: int, spatial_extent: int, fit_range: int = 5, postfix: str="", names: Optional[list[str]]=None, files: Optional[list[str]]=None, - r_start: list[int]=[], r_stop: list[int]=[], r_step:int=1) -> dict[str, Any]: +def extract_t0(path: Path, project: str, dir_in_project: str, ensemble: str, param: dict[str, Any], prefix: str, dtr_read: int, xmin: int, spatial_extent: int, fit_range: int = 5, postfix: str="", names: list[str] | None=None, files: list[str] | None=None, + r_start: list[int] | None=None, r_stop: list[int] | None=None, r_step:int=1) -> dict[str, Any]: """ Extract t0 measurements from the project. @@ -206,11 +206,15 @@ def extract_t0(path: Path, project: str, dir_in_project: str, ensemble: str, par Dictionary of t0 values in the pycorrlib style, with the parameters at hand. """ + if r_stop is None: + r_stop = [] + if r_start is None: + r_start = [] dataset = os.path.join(path, "projects", project) directory = os.path.join(dataset, dir_in_project) if files is None: files = [] - for root, ds, fs in os.walk(directory): + for _root, _ds, fs in os.walk(directory): for f in fs: if fnmatch.fnmatch(f, prefix + "*" + postfix + ".dat"): files.append(f) @@ -252,8 +256,8 @@ def extract_t0(path: Path, project: str, dir_in_project: str, ensemble: str, par return t0_dict -def extract_t1(path: Path, project: str, dir_in_project: str, ensemble: str, param: dict[str, Any], prefix: str, dtr_read: int, xmin: int, spatial_extent: int, fit_range: int = 5, postfix: str = "", names: Optional[list[str]]=None, files: Optional[list[str]]=None, - r_start: list[int]=[], r_stop: list[int]=[], r_step:int=1) -> dict[str, Any]: +def extract_t1(path: Path, project: str, dir_in_project: str, ensemble: str, param: dict[str, Any], prefix: str, dtr_read: int, xmin: int, spatial_extent: int, fit_range: int = 5, postfix: str = "", names: list[str] | None=None, files: list[str] | None=None, + r_start: list[int] | None=None, r_stop: list[int] | None=None, r_step:int=1) -> dict[str, Any]: """ Extract t1 measurements from the project. @@ -290,10 +294,14 @@ def extract_t1(path: Path, project: str, dir_in_project: str, ensemble: str, par Dictionary of t1 values in the pycorrlib style, with the parameters at hand. """ + if r_stop is None: + r_stop = [] + if r_start is None: + r_start = [] directory = os.path.join(path, "projects", project, dir_in_project) if files is None: files = [] - for root, ds, fs in os.walk(directory): + for _root, _ds, fs in os.walk(directory): for f in fs: if fnmatch.fnmatch(f, prefix + "*" + postfix + ".dat"): files.append(f) diff --git a/corrlib/input/sfcf.py b/corrlib/input/sfcf.py index 1af366f..a12661d 100644 --- a/corrlib/input/sfcf.py +++ b/corrlib/input/sfcf.py @@ -1,11 +1,11 @@ -import pyerrors as pe -import datalad.api as dl import json import os -from typing import Any from fnmatch import fnmatch from pathlib import Path +from typing import Any +import datalad.api as dl +import pyerrors as pe bi_corrs: list[str] = ["f_P", "fP", "f_p", "g_P", "gP", "g_p", @@ -99,7 +99,7 @@ def read_param(path: Path, project: str, file_in_project: str) -> dict[str, Any] file = path / "projects" / project / file_in_project dl.get(file, dataset=path) - with open(file, 'r') as f: + with open(file) as f: lines = f.readlines() params: dict[str, Any] = {} @@ -291,7 +291,7 @@ def read_data(path: Path, project: str, dir_in_project: str, prefix: str, param: appended = (version[-1] == "a") ls = [] files_to_get = [] - for (dirpath, dirnames, filenames) in os.walk(directory): + for _dirpath, dirnames, filenames in os.walk(directory): if not appended: ls.extend(dirnames) else: @@ -299,7 +299,7 @@ def read_data(path: Path, project: str, dir_in_project: str, prefix: str, param: break if not appended: compact = (version[-1] == "c") - for i, item in enumerate(ls): + for item in ls: if fnmatch(item, prefix + "*"): rep_path = directory + '/' + item sub_ls = pe.input.sfcf._find_files(rep_path, prefix, compact, []) diff --git a/corrlib/integrity.py b/corrlib/integrity.py index 66ea4df..abffdb4 100644 --- a/corrlib/integrity.py +++ b/corrlib/integrity.py @@ -1,14 +1,15 @@ import datetime as dt -from pathlib import Path -from .tools import get_db_file, CONFIG_FILENAME -import pandas as pd -import sqlite3 -from .tracker import get -import pyerrors.input.json as pj import os +import sqlite3 from configparser import ConfigParser +from pathlib import Path from typing import Any +import pandas as pd +import pyerrors.input.json as pj + +from .tools import CONFIG_FILENAME, get_db_file +from .tracker import get path_opts = ['db', 'projects_path', 'archive_path', 'toml_imports_path', 'import_scripts_path'] diff --git a/corrlib/main.py b/corrlib/main.py index 5df8165..a0806cf 100644 --- a/corrlib/main.py +++ b/corrlib/main.py @@ -1,17 +1,18 @@ -import sqlite3 -import datalad.api as dl -import datalad.config as dlc import os -from .git_tools import move_submodule import shutil -from .find import _project_lookup_by_id -from .tools import list2str, str2list, get_db_file -from .tracker import get, save, unlock, clone, drop -from typing import Union, Optional +import sqlite3 from pathlib import Path +import datalad.api as dl +import datalad.config as dlc -def create_project(path: Path, uuid: str, owner: Union[str, None]=None, tags: Union[list[str], None]=None, aliases: Union[list[str], None]=None, code: Union[str, None]=None) -> None: +from .find import _project_lookup_by_id +from .git_tools import move_submodule +from .tools import get_db_file, list2str, str2list +from .tracker import clone, drop, get, save, unlock + + +def create_project(path: Path, uuid: str, owner: str | None=None, tags: list[str] | None=None, aliases: list[str] | None=None, code: str | None=None) -> None: """ Create a new project entry in the database. @@ -49,7 +50,7 @@ def create_project(path: Path, uuid: str, owner: Union[str, None]=None, tags: Un return -def update_project_data(path: Path, uuid: str, prop: str, value: Union[str, None] = None) -> None: +def update_project_data(path: Path, uuid: str, prop: str, value: str | None = None) -> None: """ Update/Edit a project entry in the database. Thin wrapper around sql3 call. @@ -102,7 +103,7 @@ def update_aliases(path: Path, uuid: str, aliases: list[str]) -> None: return -def import_project(path: Path, url: str, owner: Union[str, None]=None, tags: Optional[list[str]]=None, aliases: Optional[list[str]]=None, code: Optional[str]=None, isDataset: bool=True) -> str: +def import_project(path: Path, url: str, owner: str | None=None, tags: list[str] | None=None, aliases: list[str] | None=None, code: str | None=None, isDataset: bool=True) -> str: """ Import a datalad dataset into the backlogger. diff --git a/corrlib/meas_io.py b/corrlib/meas_io.py index 52e307e..cd5db2e 100644 --- a/corrlib/meas_io.py +++ b/corrlib/meas_io.py @@ -1,23 +1,23 @@ -from pyerrors.input import json as pj -import os -import sqlite3 -from .input import sfcf,openQCD import json -from typing import Union -from pyerrors import Obs, Corr, dump_object, load_object -from hashlib import sha256 -from .tools import get_db_file, cache_enabled, get_plot_dir -from .tracker import get, save, unlock +import os import shutil -from typing import Any +import sqlite3 +from hashlib import sha256 from pathlib import Path -from .integrity import _check_db2paths +from typing import Any +from pyerrors import Corr, Obs, dump_object, load_object +from pyerrors.input import json as pj + +from .input import openQCD, sfcf +from .integrity import _check_db2paths +from .tools import cache_enabled, get_db_file, get_plot_dir +from .tracker import get, save, unlock CACHE_DIR = ".cache" -def write_measurement(path: Path, ensemble: str, measurement: dict[str, dict[str, dict[str, Any]]], uuid: str, code: str, parameter_file: Union[str, None]) -> None: +def write_measurement(path: Path, ensemble: str, measurement: dict[str, dict[str, dict[str, Any]]], uuid: str, code: str, parameter_file: str | None) -> None: """ Write a measurement to the backlog. If the file for the measurement already exists, update the measurement. @@ -73,7 +73,7 @@ def write_measurement(path: Path, ensemble: str, measurement: dict[str, dict[str pars[subkey] = sfcf.get_specs(corr + "/" + subkey, parameters) elif code == "openQCD": - ms_type = list(measurement.keys())[0] + ms_type = next(iter(measurement.keys())) if ms_type == 'ms1': if parameter_file is not None: if parameter_file.endswith(".ms1.in"): @@ -138,7 +138,7 @@ def write_measurement(path: Path, ensemble: str, measurement: dict[str, dict[str return -def load_record(path: Path, meas_path: str) -> Union[Corr, Obs]: +def load_record(path: Path, meas_path: str) -> Corr | Obs: """ Load a list of records by their paths. @@ -157,7 +157,7 @@ def load_record(path: Path, meas_path: str) -> Union[Corr, Obs]: return load_records(path, [meas_path])[0] -def load_records(path: Path, meas_paths: list[str], preloaded: dict[str, Any] = {}, dry_run: bool = False) -> list[Union[Corr, Obs]]: +def load_records(path: Path, meas_paths: list[str], preloaded: dict[str, Any] | None = None, dry_run: bool = False) -> list[Corr | Obs]: """ Load a list of records by their paths. @@ -177,6 +177,8 @@ def load_records(path: Path, meas_paths: list[str], preloaded: dict[str, Any] = returned_data: list The loaded records. """ + if preloaded is None: + preloaded = {} path = Path(path) if dry_run: _check_db2paths(path, meas_paths) diff --git a/corrlib/pars/openQCD/flags.py b/corrlib/pars/openQCD/flags.py index 95be919..0ab429a 100644 --- a/corrlib/pars/openQCD/flags.py +++ b/corrlib/pars/openQCD/flags.py @@ -5,6 +5,7 @@ Reconstruct the outputs of flags. import struct from typing import Any, BinaryIO + # lat_parms.c def lat_parms_write_lat_parms(fp: BinaryIO) -> dict[str, Any]: """ @@ -29,7 +30,7 @@ def lat_parms_write_lat_parms(fp: BinaryIO) -> dict[str, Any]: kappas = [] m0s = [] # read kappas - for ik in range(nk): + for _ik in range(nk): t = fp.read(8) kappas.append(struct.unpack('d', t)[0]) t = fp.read(8) diff --git a/corrlib/pars/openQCD/ms1.py b/corrlib/pars/openQCD/ms1.py index 4c2aed5..b9b3fc1 100644 --- a/corrlib/pars/openQCD/ms1.py +++ b/corrlib/pars/openQCD/ms1.py @@ -1,7 +1,7 @@ -from . import flags - -from typing import Any from pathlib import Path +from typing import Any + +from . import flags def read_qcd2_ms1_par_file(fname: Path) -> dict[str, dict[str, Any]]: diff --git a/corrlib/pars/openQCD/qcd2.py b/corrlib/pars/openQCD/qcd2.py index e73c156..121c25f 100644 --- a/corrlib/pars/openQCD/qcd2.py +++ b/corrlib/pars/openQCD/qcd2.py @@ -1,8 +1,8 @@ -from . import flags - from pathlib import Path from typing import Any +from . import flags + def read_qcd2_par_file(fname: Path) -> dict[str, dict[str, Any]]: """ diff --git a/corrlib/sql.py b/corrlib/sql.py index f45ce31..fd8e786 100644 --- a/corrlib/sql.py +++ b/corrlib/sql.py @@ -1,8 +1,9 @@ import sqlite3 -from .tools import get_db_file from pathlib import Path from typing import Any +from .tools import get_db_file + def thin_sql_wrapper(path: Path, stmt: str) -> list[Any]: db_file = get_db_file(path) diff --git a/corrlib/toml.py b/corrlib/toml.py index c452dd8..a89e24b 100644 --- a/corrlib/toml.py +++ b/corrlib/toml.py @@ -8,18 +8,19 @@ the import of projects via TOML. """ -import tomllib as toml +import os import shutil +from pathlib import Path +from typing import Any import datalad.api as dl -from .tracker import save -from .input import sfcf, openQCD +import tomllib as toml + +from .input import openQCD, sfcf +from .input.implementations import codes as known_codes from .main import import_project, update_aliases from .meas_io import write_measurement -import os -from .input.implementations import codes as known_codes -from typing import Any -from pathlib import Path +from .tracker import save def replace_string(string: str, name: str, val: str) -> str: @@ -266,7 +267,7 @@ def reimport_project(path: Path, uuid: str) -> None: uuid of the project that is to be reimported. """ config_path = path / "import_scripts" / uuid - for p, filenames, dirnames in os.walk(config_path): + for _p, filenames, _dirnames in os.walk(config_path): for fname in filenames: import_toml(path, os.path.join(config_path, fname), copy_file=False) return diff --git a/corrlib/tools.py b/corrlib/tools.py index 8dec61a..f39a140 100644 --- a/corrlib/tools.py +++ b/corrlib/tools.py @@ -1,7 +1,7 @@ import os from configparser import ConfigParser -from typing import Any from pathlib import Path +from typing import Any CONFIG_FILENAME = ".corrlib" cached: bool = True diff --git a/corrlib/tracker.py b/corrlib/tracker.py index 1581fba..bd84bff 100644 --- a/corrlib/tracker.py +++ b/corrlib/tracker.py @@ -1,11 +1,12 @@ import os -from configparser import ConfigParser -import datalad.api as dl -from typing import Optional import shutil -from .tools import get_db_file, CONFIG_FILENAME +from configparser import ConfigParser from pathlib import Path +import datalad.api as dl + +from .tools import CONFIG_FILENAME, get_db_file + def get_tracker(path: Path) -> str: """ @@ -59,7 +60,7 @@ def get(path: Path, file: Path) -> None: return -def save(path: Path, message: str, files: Optional[list[Path]]=None) -> None: +def save(path: Path, message: str, files: list[Path] | None=None) -> None: """ Wrapper function to save a file to the dataset located at path with the specified tracker. @@ -79,8 +80,7 @@ def save(path: Path, message: str, files: Optional[list[Path]]=None) -> None: files = [path / f for f in files] dl.save(files, message=message, dataset=path) elif tracker == 'None': - Warning("Tracker 'None' does not implement save.") - pass + raise Warning("Tracker 'None' does not implement save.") else: raise ValueError(f"Tracker {tracker} is not supported.") @@ -122,8 +122,7 @@ def unlock(path: Path, file: Path) -> None: if tracker == 'datalad': dl.unlock(os.path.join(path, file), dataset=path) elif tracker == 'None': - Warning("Tracker 'None' does not implement unlock.") - pass + raise Warning("Tracker 'None' does not implement unlock.") else: raise ValueError(f"Tracker {tracker} is not supported.") return @@ -154,7 +153,7 @@ def clone(path: Path, source: str, target: str) -> None: return -def drop(path: Path, reckless: Optional[str]=None) -> None: +def drop(path: Path, reckless: str | None=None) -> None: """ Wrapper function to drop data from a dataset located at path with the specified tracker. @@ -170,8 +169,7 @@ def drop(path: Path, reckless: Optional[str]=None) -> None: if tracker == 'datalad': dl.drop(path, reckless=reckless) elif tracker == 'None': - Warning("Tracker 'None' does not implement drop.") - pass + raise Warning("Tracker 'None' does not implement drop.") else: raise ValueError(f"Tracker {tracker} is not supported.") return diff --git a/corrlib/version.py b/corrlib/version.py index 23dd03f..23ea4de 100644 --- a/corrlib/version.py +++ b/corrlib/version.py @@ -3,12 +3,12 @@ from __future__ import annotations __all__ = [ + "__commit_id__", "__version__", "__version_tuple__", + "commit_id", "version", "version_tuple", - "__commit_id__", - "commit_id", ] version: str @@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...] commit_id: str | None __commit_id__: str | None -__version__ = version = '0.3.1.dev0+g08de17e6b.d20260507' -__version_tuple__ = version_tuple = (0, 3, 1, 'dev0', 'g08de17e6b.d20260507') +__version__ = version = '0.3.1.dev22+g4b1c21309.d20260701' +__version_tuple__ = version_tuple = (0, 3, 1, 'dev22', 'g4b1c21309.d20260701') -__commit_id__ = commit_id = 'g08de17e6b' +__commit_id__ = commit_id = 'g4b1c21309' From 48b95f27003cc319e1b2ca043a5f938ffeec3359 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Tue, 7 Jul 2026 13:51:58 +0200 Subject: [PATCH 41/41] use warnings better, small typo correction in test --- corrlib/find.py | 3 ++- corrlib/tracker.py | 7 ++++--- tests/find_test.py | 3 +-- tests/import_project_test.py | 2 +- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/corrlib/find.py b/corrlib/find.py index dd4be13..18c4001 100644 --- a/corrlib/find.py +++ b/corrlib/find.py @@ -2,6 +2,7 @@ import datetime as dt import json import os import sqlite3 +import warnings from collections.abc import Callable from pathlib import Path from typing import Any @@ -287,7 +288,7 @@ def openQCD_filter(results:pd.DataFrame, **kwargs: Any) -> pd.DataFrame: The filtered results. """ - raise Warning("A filter for openQCD parameters is no implemented yet.") + warnings.warn("A filter for openQCD parameters is no implemented yet.", Warning, 1) return results diff --git a/corrlib/tracker.py b/corrlib/tracker.py index bd84bff..0a962fa 100644 --- a/corrlib/tracker.py +++ b/corrlib/tracker.py @@ -1,5 +1,6 @@ import os import shutil +import warnings from configparser import ConfigParser from pathlib import Path @@ -80,7 +81,7 @@ def save(path: Path, message: str, files: list[Path] | None=None) -> None: files = [path / f for f in files] dl.save(files, message=message, dataset=path) elif tracker == 'None': - raise Warning("Tracker 'None' does not implement save.") + warnings.warn("Tracker 'None' does not implement save.", Warning, 1) else: raise ValueError(f"Tracker {tracker} is not supported.") @@ -122,7 +123,7 @@ def unlock(path: Path, file: Path) -> None: if tracker == 'datalad': dl.unlock(os.path.join(path, file), dataset=path) elif tracker == 'None': - raise Warning("Tracker 'None' does not implement unlock.") + warnings.warn("Tracker 'None' does not implement unlock.", Warning, 1) else: raise ValueError(f"Tracker {tracker} is not supported.") return @@ -169,7 +170,7 @@ def drop(path: Path, reckless: str | None=None) -> None: if tracker == 'datalad': dl.drop(path, reckless=reckless) elif tracker == 'None': - raise Warning("Tracker 'None' does not implement drop.") + warnings.warn("Tracker 'None' does not implement drop.", Warning, 1) else: raise ValueError(f"Tracker {tracker} is not supported.") return diff --git a/tests/find_test.py b/tests/find_test.py index 2144001..b462634 100644 --- a/tests/find_test.py +++ b/tests/find_test.py @@ -306,8 +306,7 @@ def test_openQCD_filter() -> None: "updated_at"] df = pd.DataFrame(data,columns=cols) - with pytest.warns(Warning): - find.openQCD_filter(df, a = "asdf") + find.openQCD_filter(df, a = "asdf") def test_code_filter() -> None: diff --git a/tests/import_project_test.py b/tests/import_project_test.py index 685d2cf..8493773 100644 --- a/tests/import_project_test.py +++ b/tests/import_project_test.py @@ -10,7 +10,7 @@ def test_toml_check_measurement_data() -> None: "param_file": "/path/to/file", "version": "1.1", "prefix": "pref", - "cfg_seperator": "n", + "cfg_separator": "n", "names": ['list', 'of', 'names'] } }