From 614fccfae4a93f4e3f1f24ad4d77804da19e4092 Mon Sep 17 00:00:00 2001 From: jkuhl Date: Thu, 27 Nov 2025 15:54:39 +0100 Subject: [PATCH 01/85] Delete .gitmodules --- .gitmodules | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 .gitmodules diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index d41b9e5..0000000 --- a/.gitmodules +++ /dev/null @@ -1,5 +0,0 @@ -[submodule "projects/tmp"] - path = projects/tmp - url = git@kuhl-mann.de:lattice/charm_SF_data.git - datalad-id = 5f402163-77f2-470e-b6f1-64d7bf9f87d4 - datalad-url = git@kuhl-mann.de:lattice/charm_SF_data.git From 4709e4272714bbb11c722eac56f8b02c4b05c2ec Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Thu, 12 Feb 2026 10:21:45 +0100 Subject: [PATCH 02/85] HOTFIX: ds arg not supported by datalad --- corrlib/main.py | 2 +- corrlib/meas_io.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/corrlib/main.py b/corrlib/main.py index ebc923e..525cbd6 100644 --- a/corrlib/main.py +++ b/corrlib/main.py @@ -122,7 +122,7 @@ def import_project(path: str, url: str, owner: Union[str, None]=None, tags: Unio raise ValueError("The dataset does not have a uuid!") if not os.path.exists(path + "/projects/" + uuid): db = path + "/backlogger.db" - dl.get(db, ds=path) + dl.get(db, dataset=path) dl.unlock(db, dataset=path) create_project(path, uuid, owner, tags, aliases, code) move_submodule(path, 'projects/tmp', 'projects/' + uuid) diff --git a/corrlib/meas_io.py b/corrlib/meas_io.py index b98eb6e..fc373db 100644 --- a/corrlib/meas_io.py +++ b/corrlib/meas_io.py @@ -28,7 +28,7 @@ def write_measurement(path, ensemble, measurement, uuid, code, parameter_file=No The uuid of the project. """ db = os.path.join(path, 'backlogger.db') - dl.get(db, ds=path) + dl.get(db, dataset=path) dl.unlock(db, dataset=path) conn = sqlite3.connect(db) c = conn.cursor() @@ -177,7 +177,7 @@ def drop_record(path: str, meas_path: str): file_in_archive = meas_path.split("::")[0] file = os.path.join(path, file_in_archive) db = os.path.join(path, 'backlogger.db') - dl.get(db, ds=path) + dl.get(db, dataset=path) sub_key = meas_path.split("::")[1] dl.unlock(db, dataset=path) conn = sqlite3.connect(db) From fad703a39feb4cd86b243bca63e1c0c01b68a224 Mon Sep 17 00:00:00 2001 From: jkuhl Date: Thu, 12 Feb 2026 10:27:58 +0100 Subject: [PATCH 03/85] bunmp version to 0.2.4 --- corrlib/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/corrlib/version.py b/corrlib/version.py index d31c31e..788da1f 100644 --- a/corrlib/version.py +++ b/corrlib/version.py @@ -1 +1 @@ -__version__ = "0.2.3" +__version__ = "0.2.4" From d6dde9bb1184c9c814bac5acda9cfd8262b4a60a Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Fri, 13 Feb 2026 10:46:29 +0100 Subject: [PATCH 04/85] add function to only show a single arg with the find cli --- corrlib/cli.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/corrlib/cli.py b/corrlib/cli.py index b808c13..ff74527 100644 --- a/corrlib/cli.py +++ b/corrlib/cli.py @@ -90,6 +90,11 @@ def find( "--dataset", "-d", ), + arg: str = typer.Option( + str('all'), + "--show", + "-s", + ), ensemble: str = typer.Argument(), corr: str = typer.Argument(), code: str = typer.Argument(), @@ -98,8 +103,12 @@ def find( Find a record in the backlog at hand. Through specifying it's ensemble and the measured correlator. """ results = find_record(path, ensemble, corr, code) - print(results) - + if arg == "all": + print(results) + else: + for i in range(len(results)): + print(results[arg].values[i]) + return @app.command() def importer( From bd34b7c3785ddb509f2fbfffbf01ce145ee0463d Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Mon, 23 Mar 2026 22:33:01 +0100 Subject: [PATCH 05/85] write first trivial find test --- tests/find_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/find_test.py b/tests/find_test.py index b63b246..8cc7923 100644 --- a/tests/find_test.py +++ b/tests/find_test.py @@ -10,6 +10,7 @@ def make_sql(path: Path) -> Path: cinit._create_db(db) return db + def test_find_lookup_by_one_alias(tmp_path: Path) -> None: db = make_sql(tmp_path) conn = sqlite3.connect(db) From f8566207e36978af1cefded2ae6c8fc521e732cd Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Mon, 23 Mar 2026 22:43:39 +0100 Subject: [PATCH 06/85] add id lookup test --- tests/find_test.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/find_test.py b/tests/find_test.py index 8cc7923..e0730e9 100644 --- a/tests/find_test.py +++ b/tests/find_test.py @@ -32,3 +32,25 @@ def test_find_lookup_by_one_alias(tmp_path: Path) -> None: with pytest.raises(Exception): assert uuid == find._project_lookup_by_alias(db, "fun_project") conn.close() + + +def test_find_lookup_by_id(tmp_path: Path) -> None: + db = make_sql(tmp_path) + conn = sqlite3.connect(db) + c = conn.cursor() + uuid = "test_uuid" + alias_str = "fun_project" + tag_str = "tt" + owner = "tester" + code = "test_code" + c.execute("INSERT INTO projects (id, aliases, customTags, owner, code, created_at, updated_at) VALUES (?, ?, ?, ?, ?, datetime('now'), datetime('now'))", + (uuid, alias_str, tag_str, owner, code)) + conn.commit() + conn.close() + result = find._project_lookup_by_id(db, uuid)[0] + assert uuid == result[0] + assert alias_str == result[1] + assert tag_str == result[2] + assert owner == result[3] + assert code == result[4] + From d0d5f9aa8775c4ce78f547d7cbe3dea5199743cb Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Mon, 23 Mar 2026 23:37:22 +0100 Subject: [PATCH 07/85] rewrite time filter --- corrlib/find.py | 46 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 35 insertions(+), 11 deletions(-) diff --git a/corrlib/find.py b/corrlib/find.py index 4c51e05..e4ee735 100644 --- a/corrlib/find.py +++ b/corrlib/find.py @@ -8,6 +8,7 @@ from .tools import k2m, get_db_file from .tracker import get from typing import Any, Optional from pathlib import Path +import datetime as dt def _project_lookup_by_alias(db: Path, alias: str) -> str: @@ -62,8 +63,37 @@ def _project_lookup_by_id(db: Path, uuid: str) -> list[tuple[str, str]]: return results -def _db_lookup(db: Path, ensemble: str, correlator_name: str, code: str, project: Optional[str]=None, parameters: Optional[str]=None, - created_before: Optional[str]=None, created_after: Optional[Any]=None, updated_before: Optional[Any]=None, updated_after: Optional[Any]=None) -> pd.DataFrame: +def _time_filter(results: pd.DataFrame, created_before: Optional[str]=None, created_after: Optional[Any]=None, updated_before: Optional[Any]=None, updated_after: Optional[Any]=None) -> pd.DataFrame: + drops = [] + for ind in len(results): + result = results.iloc[ind] + created_at = dt.datetime.fromisoformat(result['created_at']) + updated_at = dt.datetime.fromisoformat(result['updated_at']) + + if created_before is not None: + created_before = dt.datetime.fromisoformat(created_before) + if created_before < created_at: + drops.append(ind) + continue + if created_after is not None: + created_after = dt.datetime.fromisoformat(created_after) + if created_before > created_at: + drops.append(ind) + continue + if updated_before is not None: + updated_before = dt.datetime.fromisoformat(updated_before) + if updated_before < updated_at: + drops.append(ind) + continue + if updated_after is not None: + updated_after = dt.datetime.fromisoformat(updated_after) + if updated_after > updated_at: + drops.append(ind) + continue + return results.drop(drops) + + +def _db_lookup(db: Path, ensemble: str, correlator_name: str, code: str, project: Optional[str]=None, parameters: Optional[str]=None) -> pd.DataFrame: """ Look up a correlator record in the database by the data given to the method. @@ -105,14 +135,6 @@ def _db_lookup(db: Path, ensemble: str, correlator_name: str, code: str, project search_expr += f" AND code = '{code}'" if parameters: search_expr += f" AND parameters = '{parameters}'" - if created_before: - search_expr += f" AND created_at < '{created_before}'" - if created_after: - search_expr += f" AND created_at > '{created_after}'" - if updated_before: - search_expr += f" AND updated_at < '{updated_before}'" - if updated_after: - search_expr += f" AND updated_at > '{updated_after}'" conn = sqlite3.connect(db) results = pd.read_sql(search_expr, conn) conn.close() @@ -236,7 +258,9 @@ def find_record(path: Path, ensemble: str, correlator_name: str, code: str, proj if code not in codes: raise ValueError("Code " + code + "unknown, take one of the following:" + ", ".join(codes)) get(path, db_file) - results = _db_lookup(db, ensemble, correlator_name,code, project, parameters=parameters, created_before=created_before, created_after=created_after, updated_before=updated_before, updated_after=updated_after) + results = _db_lookup(db, ensemble, correlator_name,code, project, parameters=parameters) + if Any([created_before, created_after, updated_before, updated_after]): + results = _time_filter(results, created_before, created_after, updated_before, updated_after) if code == "sfcf": results = sfcf_filter(results, **kwargs) elif code == "openQCD": From 29558a734b1522c94979858fb1ed0a12f8ed20d3 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Mon, 23 Mar 2026 23:38:40 +0100 Subject: [PATCH 08/85] add test for db lookup --- tests/find_test.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/tests/find_test.py b/tests/find_test.py index e0730e9..da1bfc1 100644 --- a/tests/find_test.py +++ b/tests/find_test.py @@ -54,3 +54,42 @@ def test_find_lookup_by_id(tmp_path: Path) -> None: assert owner == result[3] assert code == result[4] + +def test_db_lookup(tmp_path: Path) -> None: + db = make_sql(tmp_path) + conn = sqlite3.connect(db) + c = conn.cursor() + + corr = "f_A" + ensemble = "SF_A" + code = "openQCD" + meas_path = "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf" + uuid = "Project_A" + pars = "{par_A: 3.0, par_B: 5.0}" + parameter_file = "projects/Project_A/myinput.in" + c.execute("INSERT INTO backlogs (name, ensemble, code, path, project, parameters, parameter_file, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, datetime('now'), datetime('now'))", + (corr, ensemble, code, meas_path, uuid, pars, parameter_file)) + conn.commit() + conn.close() + + results = find._db_lookup(db, ensemble, corr, code) + assert len(results) == 1 + results = find._db_lookup(db, "SF_B", corr, code) + assert results.empty + results = find._db_lookup(db, ensemble, "g_A", code) + assert results.empty + results = find._db_lookup(db, ensemble, corr, "sfcf") + assert results.empty + results = find._db_lookup(db, ensemble, corr, code, project = "Project_A") + assert len(results) == 1 + results = find._db_lookup(db, ensemble, corr, code, project = "Project_B") + assert results.empty + results = find._db_lookup(db, ensemble, corr, code, parameters = pars) + assert len(results) == 1 + results = find._db_lookup(db, ensemble, corr, code, parameters = "{par_A: 3.0, par_B: 4.0}") + assert results.empty + #results = find._db_lookup(db, ensemble, corr, code, project, parameters, created_before, created_after, updated_before, updated_after) + + #results = find._db_lookup(db, ensemble, corr, code, project, parameters, created_before, created_after, updated_before, updated_after) + + From 402ca07edbecda8bb5828596e98527c9ed2de8a4 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Mon, 23 Mar 2026 23:42:42 +0100 Subject: [PATCH 09/85] linting and hotfix --- corrlib/find.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/corrlib/find.py b/corrlib/find.py index e4ee735..3e62344 100644 --- a/corrlib/find.py +++ b/corrlib/find.py @@ -65,32 +65,32 @@ def _project_lookup_by_id(db: Path, uuid: str) -> list[tuple[str, str]]: def _time_filter(results: pd.DataFrame, created_before: Optional[str]=None, created_after: Optional[Any]=None, updated_before: Optional[Any]=None, updated_after: Optional[Any]=None) -> pd.DataFrame: drops = [] - for ind in len(results): + for ind in range(len(results)): result = results.iloc[ind] created_at = dt.datetime.fromisoformat(result['created_at']) updated_at = dt.datetime.fromisoformat(result['updated_at']) if created_before is not None: - created_before = dt.datetime.fromisoformat(created_before) - if created_before < created_at: + date_created_before = dt.datetime.fromisoformat(created_before) + if date_created_before < created_at: drops.append(ind) continue if created_after is not None: - created_after = dt.datetime.fromisoformat(created_after) - if created_before > created_at: + date_created_after = dt.datetime.fromisoformat(created_after) + if date_created_after > created_at: drops.append(ind) continue if updated_before is not None: - updated_before = dt.datetime.fromisoformat(updated_before) - if updated_before < updated_at: + date_updated_before = dt.datetime.fromisoformat(updated_before) + if date_updated_before < updated_at: drops.append(ind) continue if updated_after is not None: - updated_after = dt.datetime.fromisoformat(updated_after) - if updated_after > updated_at: + date_updated_after = dt.datetime.fromisoformat(updated_after) + if date_updated_after > updated_at: drops.append(ind) continue - return results.drop(drops) + return results.drop(drops) def _db_lookup(db: Path, ensemble: str, correlator_name: str, code: str, project: Optional[str]=None, parameters: Optional[str]=None) -> pd.DataFrame: From b50ffc4c6b898de970e8ded6c2287c96a6c6389b Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Mon, 23 Mar 2026 23:45:22 +0100 Subject: [PATCH 10/85] any hotfix --- corrlib/find.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/corrlib/find.py b/corrlib/find.py index 3e62344..14b1772 100644 --- a/corrlib/find.py +++ b/corrlib/find.py @@ -259,7 +259,7 @@ def find_record(path: Path, ensemble: str, correlator_name: str, code: str, proj raise ValueError("Code " + code + "unknown, take one of the following:" + ", ".join(codes)) get(path, db_file) results = _db_lookup(db, ensemble, correlator_name,code, project, parameters=parameters) - if Any([created_before, created_after, updated_before, updated_after]): + if any(arg is not None for arg in [created_before, created_after, updated_before, updated_after]): results = _time_filter(results, created_before, created_after, updated_before, updated_after) if code == "sfcf": results = sfcf_filter(results, **kwargs) From c431145a23764015d52ed6a1fd3da007d554cc3f Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Tue, 24 Mar 2026 09:23:30 +0100 Subject: [PATCH 11/85] some more db lookup --- tests/find_test.py | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/tests/find_test.py b/tests/find_test.py index da1bfc1..e895b85 100644 --- a/tests/find_test.py +++ b/tests/find_test.py @@ -70,7 +70,6 @@ def test_db_lookup(tmp_path: Path) -> None: c.execute("INSERT INTO backlogs (name, ensemble, code, path, project, parameters, parameter_file, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, datetime('now'), datetime('now'))", (corr, ensemble, code, meas_path, uuid, pars, parameter_file)) conn.commit() - conn.close() results = find._db_lookup(db, ensemble, corr, code) assert len(results) == 1 @@ -88,8 +87,38 @@ def test_db_lookup(tmp_path: Path) -> None: assert len(results) == 1 results = find._db_lookup(db, ensemble, corr, code, parameters = "{par_A: 3.0, par_B: 4.0}") assert results.empty - #results = find._db_lookup(db, ensemble, corr, code, project, parameters, created_before, created_after, updated_before, updated_after) - #results = find._db_lookup(db, ensemble, corr, code, project, parameters, created_before, created_after, updated_before, updated_after) + corr = "g_A" + ensemble = "SF_A" + code = "openQCD" + meas_path = "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf" + uuid = "Project_A" + pars = "{par_A: 3.0, par_B: 4.0}" + parameter_file = "projects/Project_A/myinput.in" + c.execute("INSERT INTO backlogs (name, ensemble, code, path, project, parameters, parameter_file, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, datetime('now'), datetime('now'))", + (corr, ensemble, code, meas_path, uuid, pars, parameter_file)) + conn.commit() + corr = "f_A" + results = find._db_lookup(db, ensemble, corr, code) + assert len(results) == 1 + results = find._db_lookup(db, "SF_B", corr, code) + assert results.empty + results = find._db_lookup(db, ensemble, "g_A", code) + assert len(results) == 1 + results = find._db_lookup(db, ensemble, corr, "sfcf") + assert results.empty + results = find._db_lookup(db, ensemble, corr, code, project = "Project_A") + assert len(results) == 1 + results = find._db_lookup(db, ensemble, "g_A", code, project = "Project_A") + assert len(results) == 1 + results = find._db_lookup(db, ensemble, corr, code, project = "Project_B") + assert results.empty + results = find._db_lookup(db, ensemble, "g_A", code, project = "Project_B") + assert results.empty + results = find._db_lookup(db, ensemble, corr, code, parameters = pars) + assert results.empty + results = find._db_lookup(db, ensemble, "g_A", code, parameters = "{par_A: 3.0, par_B: 4.0}") + assert len(results) == 1 + conn.close() From 3fd557f3eebd2a57b9340b727a23f72586f6e68e Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Tue, 24 Mar 2026 09:24:12 +0100 Subject: [PATCH 12/85] add customtFilter --- corrlib/find.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/corrlib/find.py b/corrlib/find.py index 14b1772..8934854 100644 --- a/corrlib/find.py +++ b/corrlib/find.py @@ -6,9 +6,10 @@ import numpy as np from .input.implementations import codes from .tools import k2m, get_db_file from .tracker import get -from typing import Any, Optional +from typing import Any, Optional, Union from pathlib import Path import datetime as dt +from collections.abc import Callable def _project_lookup_by_alias(db: Path, alias: str) -> str: @@ -251,22 +252,31 @@ def sfcf_filter(results: pd.DataFrame, **kwargs: Any) -> pd.DataFrame: return results.drop(drops) +def openQCD_filter(results:pd.DataFrame, **kwargs: Any) -> pd.DataFrame: + return results + + def find_record(path: Path, ensemble: str, correlator_name: str, code: str, project: Optional[str]=None, parameters: Optional[str]=None, - created_before: Optional[str]=None, created_after: Optional[str]=None, updated_before: Optional[str]=None, updated_after: Optional[str]=None, revision: Optional[str]=None, **kwargs: Any) -> pd.DataFrame: + created_before: Optional[str]=None, created_after: Optional[str]=None, updated_before: Optional[str]=None, updated_after: Optional[str]=None, + revision: Optional[str]=None, + customFilter: Optional[Callable[[pd.DataFrame], pd.DataFrame]] = None, + **kwargs: Any) -> pd.DataFrame: db_file = get_db_file(path) db = path / db_file if code not in codes: raise ValueError("Code " + code + "unknown, take one of the following:" + ", ".join(codes)) get(path, db_file) results = _db_lookup(db, ensemble, correlator_name,code, project, parameters=parameters) - if any(arg is not None for arg in [created_before, created_after, updated_before, updated_after]): + if any([arg is not None for arg in [created_before, created_after, updated_before, updated_after]]): results = _time_filter(results, created_before, created_after, updated_before, updated_after) + if customFilter is not None: + results = customFilter(results) if code == "sfcf": results = sfcf_filter(results, **kwargs) elif code == "openQCD": - pass + results = openQCD_filter(results, **kwargs) else: - raise Exception + raise ValueError(f"Code {code} is not known.") print("Found " + str(len(results)) + " result" + ("s" if len(results)>1 else "")) return results.reset_index() From 3fe8e28a68a58a4cf8bce7a29d43f60286000c81 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Tue, 24 Mar 2026 09:25:21 +0100 Subject: [PATCH 13/85] customtFilter after general filters --- corrlib/find.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/corrlib/find.py b/corrlib/find.py index 8934854..e099aea 100644 --- a/corrlib/find.py +++ b/corrlib/find.py @@ -269,14 +269,14 @@ def find_record(path: Path, ensemble: str, correlator_name: str, code: str, proj results = _db_lookup(db, ensemble, correlator_name,code, project, parameters=parameters) if any([arg is not None for arg in [created_before, created_after, updated_before, updated_after]]): results = _time_filter(results, created_before, created_after, updated_before, updated_after) - if customFilter is not None: - results = customFilter(results) if code == "sfcf": results = sfcf_filter(results, **kwargs) elif code == "openQCD": results = openQCD_filter(results, **kwargs) else: raise ValueError(f"Code {code} is not known.") + if customFilter is not None: + results = customFilter(results) print("Found " + str(len(results)) + " result" + ("s" if len(results)>1 else "")) return results.reset_index() From 4516ca3149cac8b2f0420903c41576b471b7ed8f Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Tue, 24 Mar 2026 18:39:00 +0100 Subject: [PATCH 14/85] better type annotation fir id lookup --- corrlib/find.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/corrlib/find.py b/corrlib/find.py index e099aea..dd3a9a6 100644 --- a/corrlib/find.py +++ b/corrlib/find.py @@ -40,7 +40,7 @@ def _project_lookup_by_alias(db: Path, alias: str) -> str: return str(results[0][0]) -def _project_lookup_by_id(db: Path, uuid: str) -> list[tuple[str, str]]: +def _project_lookup_by_id(db: Path, uuid: str) -> list[tuple[str, ...]]: """ Return the project information available in the database by UUID. From 2fd46d452b84cf326d19d0a814a465631b5c8241 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Tue, 24 Mar 2026 18:40:46 +0100 Subject: [PATCH 15/85] hotfix ensure that path is a Path --- corrlib/meas_io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/corrlib/meas_io.py b/corrlib/meas_io.py index be80b6f..de19727 100644 --- a/corrlib/meas_io.py +++ b/corrlib/meas_io.py @@ -195,7 +195,7 @@ def cache_dir(path: Path, file: str) -> Path: The path holding the cached data for the given file. """ cache_path_list = file.split("/")[1:] - cache_path = path / CACHE_DIR + cache_path = Path(path) / CACHE_DIR for directory in cache_path_list: cache_path /= directory return cache_path From b8121811f967530f174202d5c67e72a0132295de Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Tue, 24 Mar 2026 18:49:04 +0100 Subject: [PATCH 16/85] HOTFIX: hand over path as str --- corrlib/meas_io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/corrlib/meas_io.py b/corrlib/meas_io.py index de19727..a87e227 100644 --- a/corrlib/meas_io.py +++ b/corrlib/meas_io.py @@ -238,7 +238,7 @@ def preload(path: Path, file: Path) -> dict[str, Any]: The data read from the file. """ get(path, file) - filedict: dict[str, Any] = pj.load_json_dict(path / file) + filedict: dict[str, Any] = pj.load_json_dict(str(path / file)) print("> read file") return filedict From 38b4983fed0af22231b27b57dc32a4efb121a63a Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Tue, 24 Mar 2026 18:50:30 +0100 Subject: [PATCH 17/85] HOTFIX: hand over path as str 2 --- corrlib/meas_io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/corrlib/meas_io.py b/corrlib/meas_io.py index a87e227..48017a1 100644 --- a/corrlib/meas_io.py +++ b/corrlib/meas_io.py @@ -175,7 +175,7 @@ def load_records(path: Path, meas_paths: list[str], preloaded: dict[str, Any] = if cache_enabled(path): if not os.path.exists(cache_dir(path, file)): os.makedirs(cache_dir(path, file)) - dump_object(preloaded[file][key], cache_path(path, file, key)) + dump_object(preloaded[file][key], str(cache_path(path, file, key))) return returned_data From cc14e68b4429a122ee0c9b299555f4e7ca8fef45 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Thu, 26 Mar 2026 17:19:58 +0100 Subject: [PATCH 18/85] add tests for time filter and find project, add a first check for integrity of the database --- corrlib/find.py | 4 ++ corrlib/integrity.py | 5 ++ tests/find_test.py | 116 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 125 insertions(+) create mode 100644 corrlib/integrity.py diff --git a/corrlib/find.py b/corrlib/find.py index dd3a9a6..3cbe09b 100644 --- a/corrlib/find.py +++ b/corrlib/find.py @@ -6,6 +6,7 @@ import numpy as np from .input.implementations import codes from .tools import k2m, get_db_file from .tracker import get +from .integrity import check_time_validity from typing import Any, Optional, Union from pathlib import Path import datetime as dt @@ -70,6 +71,9 @@ def _time_filter(results: pd.DataFrame, created_before: Optional[str]=None, cre result = results.iloc[ind] created_at = dt.datetime.fromisoformat(result['created_at']) updated_at = dt.datetime.fromisoformat(result['updated_at']) + db_times_valid = check_time_validity(created_at=created_at, updated_at=updated_at) + if not db_times_valid: + raise ValueError('Time stamps not valid for result with path', result["path"]) if created_before is not None: date_created_before = dt.datetime.fromisoformat(created_before) diff --git a/corrlib/integrity.py b/corrlib/integrity.py new file mode 100644 index 0000000..bf890db --- /dev/null +++ b/corrlib/integrity.py @@ -0,0 +1,5 @@ +import datetime as dt + + +def check_time_validity(created_at: dt.datetime, updated_at: dt.datetime) -> bool: + return not (created_at > updated_at) diff --git a/tests/find_test.py b/tests/find_test.py index e895b85..573f87e 100644 --- a/tests/find_test.py +++ b/tests/find_test.py @@ -3,6 +3,8 @@ import sqlite3 from pathlib import Path import corrlib.initialization as cinit import pytest +import pandas as pd +import datalad.api as dl def make_sql(path: Path) -> Path: @@ -34,6 +36,34 @@ def test_find_lookup_by_one_alias(tmp_path: Path) -> None: conn.close() +def test_find_project(tmp_path: Path) -> None: + cinit.create(tmp_path) + db = tmp_path / "backlogger.db" + dl.unlock(str(db), dataset=str(tmp_path)) + conn = sqlite3.connect(db) + c = conn.cursor() + uuid = "test_uuid" + alias_str = "fun_project" + tag_str = "tt" + owner = "tester" + code = "test_code" + c.execute("INSERT INTO projects (id, aliases, customTags, owner, code, created_at, updated_at) VALUES (?, ?, ?, ?, ?, datetime('now'), datetime('now'))", + (uuid, alias_str, tag_str, owner, code)) + conn.commit() + + assert uuid == find.find_project(tmp_path, "fun_project") + + uuid = "test_uuid2" + alias_str = "fun_project" + c.execute("INSERT INTO projects (id, aliases, customTags, owner, code, created_at, updated_at) VALUES (?, ?, ?, ?, ?, datetime('now'), datetime('now'))", + (uuid, alias_str, tag_str, owner, code)) + conn.commit() + + with pytest.raises(Exception): + assert uuid == find._project_lookup_by_alias(tmp_path, "fun_project") + conn.close() + + def test_find_lookup_by_id(tmp_path: Path) -> None: db = make_sql(tmp_path) conn = sqlite3.connect(db) @@ -122,3 +152,89 @@ def test_db_lookup(tmp_path: Path) -> None: assert len(results) == 1 conn.close() + + +def test_time_filter() -> None: + record_A = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{par_A: 5.0, par_B: 5.0}', "projects/SF_A/input.in", + '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] # only created + record_B = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{par_A: 5.0, par_B: 5.0}', "projects/SF_A/input.in", + '2025-03-26 12:55:18.229966', '2025-04-26 12:55:18.229966'] # created and updated + record_C = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{par_A: 5.0, par_B: 5.0}', "projects/SF_A/input.in", + '2026-03-26 12:55:18.229966', '2026-05-26 12:55:18.229966'] # created and updated later + record_D = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{par_A: 5.0, par_B: 5.0}', "projects/SF_A/input.in", + '2026-03-26 12:55:18.229966', '2026-03-27 12:55:18.229966'] + record_E = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{par_A: 5.0, par_B: 5.0}', "projects/SF_A/input.in", + '2024-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # only created, earlier + record_F = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{par_A: 5.0, par_B: 5.0}', "projects/SF_A/input.in", + '2026-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # this is invalid... + + data = [record_A, record_B, record_C, record_D, record_E] + cols = ["name", + "ensemble", + "code", + "path", + "project", + "parameters", + "parameter_file", + "created_at", + "updated_at"] + df = pd.DataFrame(data,columns=cols) + + results = find._time_filter(df, created_before='2023-03-26 12:55:18.229966') + assert results.empty + results = find._time_filter(df, created_before='2027-03-26 12:55:18.229966') + assert len(results) == 5 + results = find._time_filter(df, created_before='2026-03-25 12:55:18.229966') + assert len(results) == 3 + results = find._time_filter(df, created_before='2026-03-26 12:55:18.229965') + assert len(results) == 3 + results = find._time_filter(df, created_before='2025-03-04 12:55:18.229965') + assert len(results) == 1 + + results = find._time_filter(df, created_after='2023-03-26 12:55:18.229966') + assert len(results) == 5 + results = find._time_filter(df, created_after='2027-03-26 12:55:18.229966') + assert results.empty + results = find._time_filter(df, created_after='2026-03-25 12:55:18.229966') + assert len(results) == 2 + results = find._time_filter(df, created_after='2026-03-26 12:55:18.229965') + assert len(results) == 2 + results = find._time_filter(df, created_after='2025-03-04 12:55:18.229965') + assert len(results) == 4 + + results = find._time_filter(df, updated_before='2023-03-26 12:55:18.229966') + assert results.empty + results = find._time_filter(df, updated_before='2027-03-26 12:55:18.229966') + assert len(results) == 5 + results = find._time_filter(df, updated_before='2026-03-25 12:55:18.229966') + assert len(results) == 3 + results = find._time_filter(df, updated_before='2026-03-26 12:55:18.229965') + assert len(results) == 3 + results = find._time_filter(df, updated_before='2025-03-04 12:55:18.229965') + assert len(results) == 1 + + results = find._time_filter(df, updated_after='2023-03-26 12:55:18.229966') + assert len(results) == 5 + results = find._time_filter(df, updated_after='2027-03-26 12:55:18.229966') + assert results.empty + results = find._time_filter(df, updated_after='2026-03-25 12:55:18.229966') + assert len(results) == 2 + results = find._time_filter(df, updated_after='2026-03-26 12:55:18.229965') + assert len(results) == 2 + results = find._time_filter(df, updated_after='2025-03-04 12:55:18.229965') + assert len(results) == 4 + + data = [record_A, record_B, record_C, record_D, record_F] + cols = ["name", + "ensemble", + "code", + "path", + "project", + "parameters", + "parameter_file", + "created_at", + "updated_at"] + df = pd.DataFrame(data,columns=cols) + + with pytest.raises(ValueError): + results = find._time_filter(df, created_before='2023-03-26 12:55:18.229966') From 81af9579dcad49d0b5c3095b0d467cf49d2282e6 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Thu, 26 Mar 2026 17:25:57 +0100 Subject: [PATCH 19/85] add a docstring for time filter --- corrlib/find.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/corrlib/find.py b/corrlib/find.py index 3cbe09b..cb85130 100644 --- a/corrlib/find.py +++ b/corrlib/find.py @@ -66,6 +66,22 @@ def _project_lookup_by_id(db: Path, uuid: str) -> list[tuple[str, ...]]: def _time_filter(results: pd.DataFrame, created_before: Optional[str]=None, created_after: Optional[Any]=None, updated_before: Optional[Any]=None, updated_after: Optional[Any]=None) -> pd.DataFrame: + """ + Filter the results from the database in terms of the creation and update times. + + Parameters + ---------- + results: pd.DataFrame + The dataframe holding the unfilteres results from the database. + created_before: str + Contraint on the creation date in datetime.datetime.isoformat. Note that this is exclusive. The creation date has to be truly before the date and time given. + created_after: str + Contraint on the creation date in datetime.datetime.isoformat. Note that this is exclusive. The creation date has to be truly after the date and time given. + updated_before: str + Contraint on the creation date in datetime.datetime.isoformat. Note that this is exclusive. The date of the last update has to be truly before the date and time given. + updated_after: str + Contraint on the creation date in datetime.datetime.isoformat. Note that this is exclusive. The date of the last update has to be truly after the date and time given. + """ drops = [] for ind in range(len(results)): result = results.iloc[ind] From e8360c88b938cbd3636b8b8cfa30ce0b3375e7ed Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Fri, 27 Mar 2026 11:53:07 +0100 Subject: [PATCH 20/85] add more templates --- tests/find_test.py | 278 +++++++++++++++++++++++++++++---------------- 1 file changed, 180 insertions(+), 98 deletions(-) diff --git a/tests/find_test.py b/tests/find_test.py index 573f87e..944ae5f 100644 --- a/tests/find_test.py +++ b/tests/find_test.py @@ -35,35 +35,6 @@ def test_find_lookup_by_one_alias(tmp_path: Path) -> None: assert uuid == find._project_lookup_by_alias(db, "fun_project") conn.close() - -def test_find_project(tmp_path: Path) -> None: - cinit.create(tmp_path) - db = tmp_path / "backlogger.db" - dl.unlock(str(db), dataset=str(tmp_path)) - conn = sqlite3.connect(db) - c = conn.cursor() - uuid = "test_uuid" - alias_str = "fun_project" - tag_str = "tt" - owner = "tester" - code = "test_code" - c.execute("INSERT INTO projects (id, aliases, customTags, owner, code, created_at, updated_at) VALUES (?, ?, ?, ?, ?, datetime('now'), datetime('now'))", - (uuid, alias_str, tag_str, owner, code)) - conn.commit() - - assert uuid == find.find_project(tmp_path, "fun_project") - - uuid = "test_uuid2" - alias_str = "fun_project" - c.execute("INSERT INTO projects (id, aliases, customTags, owner, code, created_at, updated_at) VALUES (?, ?, ?, ?, ?, datetime('now'), datetime('now'))", - (uuid, alias_str, tag_str, owner, code)) - conn.commit() - - with pytest.raises(Exception): - assert uuid == find._project_lookup_by_alias(tmp_path, "fun_project") - conn.close() - - def test_find_lookup_by_id(tmp_path: Path) -> None: db = make_sql(tmp_path) conn = sqlite3.connect(db) @@ -85,75 +56,6 @@ def test_find_lookup_by_id(tmp_path: Path) -> None: assert code == result[4] -def test_db_lookup(tmp_path: Path) -> None: - db = make_sql(tmp_path) - conn = sqlite3.connect(db) - c = conn.cursor() - - corr = "f_A" - ensemble = "SF_A" - code = "openQCD" - meas_path = "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf" - uuid = "Project_A" - pars = "{par_A: 3.0, par_B: 5.0}" - parameter_file = "projects/Project_A/myinput.in" - c.execute("INSERT INTO backlogs (name, ensemble, code, path, project, parameters, parameter_file, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, datetime('now'), datetime('now'))", - (corr, ensemble, code, meas_path, uuid, pars, parameter_file)) - conn.commit() - - results = find._db_lookup(db, ensemble, corr, code) - assert len(results) == 1 - results = find._db_lookup(db, "SF_B", corr, code) - assert results.empty - results = find._db_lookup(db, ensemble, "g_A", code) - assert results.empty - results = find._db_lookup(db, ensemble, corr, "sfcf") - assert results.empty - results = find._db_lookup(db, ensemble, corr, code, project = "Project_A") - assert len(results) == 1 - results = find._db_lookup(db, ensemble, corr, code, project = "Project_B") - assert results.empty - results = find._db_lookup(db, ensemble, corr, code, parameters = pars) - assert len(results) == 1 - results = find._db_lookup(db, ensemble, corr, code, parameters = "{par_A: 3.0, par_B: 4.0}") - assert results.empty - - corr = "g_A" - ensemble = "SF_A" - code = "openQCD" - meas_path = "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf" - uuid = "Project_A" - pars = "{par_A: 3.0, par_B: 4.0}" - parameter_file = "projects/Project_A/myinput.in" - c.execute("INSERT INTO backlogs (name, ensemble, code, path, project, parameters, parameter_file, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, datetime('now'), datetime('now'))", - (corr, ensemble, code, meas_path, uuid, pars, parameter_file)) - conn.commit() - - corr = "f_A" - results = find._db_lookup(db, ensemble, corr, code) - assert len(results) == 1 - results = find._db_lookup(db, "SF_B", corr, code) - assert results.empty - results = find._db_lookup(db, ensemble, "g_A", code) - assert len(results) == 1 - results = find._db_lookup(db, ensemble, corr, "sfcf") - assert results.empty - results = find._db_lookup(db, ensemble, corr, code, project = "Project_A") - assert len(results) == 1 - results = find._db_lookup(db, ensemble, "g_A", code, project = "Project_A") - assert len(results) == 1 - results = find._db_lookup(db, ensemble, corr, code, project = "Project_B") - assert results.empty - results = find._db_lookup(db, ensemble, "g_A", code, project = "Project_B") - assert results.empty - results = find._db_lookup(db, ensemble, corr, code, parameters = pars) - assert results.empty - results = find._db_lookup(db, ensemble, "g_A", code, parameters = "{par_A: 3.0, par_B: 4.0}") - assert len(results) == 1 - - conn.close() - - def test_time_filter() -> None: record_A = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{par_A: 5.0, par_B: 5.0}', "projects/SF_A/input.in", '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] # only created @@ -238,3 +140,183 @@ def test_time_filter() -> None: with pytest.raises(ValueError): results = find._time_filter(df, created_before='2023-03-26 12:55:18.229966') + + +def test_db_lookup(tmp_path: Path) -> None: + db = make_sql(tmp_path) + conn = sqlite3.connect(db) + c = conn.cursor() + + corr = "f_A" + ensemble = "SF_A" + code = "openQCD" + meas_path = "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf" + uuid = "Project_A" + pars = "{par_A: 3.0, par_B: 5.0}" + parameter_file = "projects/Project_A/myinput.in" + c.execute("INSERT INTO backlogs (name, ensemble, code, path, project, parameters, parameter_file, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, datetime('now'), datetime('now'))", + (corr, ensemble, code, meas_path, uuid, pars, parameter_file)) + conn.commit() + + results = find._db_lookup(db, ensemble, corr, code) + assert len(results) == 1 + results = find._db_lookup(db, "SF_B", corr, code) + assert results.empty + results = find._db_lookup(db, ensemble, "g_A", code) + assert results.empty + results = find._db_lookup(db, ensemble, corr, "sfcf") + assert results.empty + results = find._db_lookup(db, ensemble, corr, code, project = "Project_A") + assert len(results) == 1 + results = find._db_lookup(db, ensemble, corr, code, project = "Project_B") + assert results.empty + results = find._db_lookup(db, ensemble, corr, code, parameters = pars) + assert len(results) == 1 + results = find._db_lookup(db, ensemble, corr, code, parameters = "{par_A: 3.0, par_B: 4.0}") + assert results.empty + + corr = "g_A" + ensemble = "SF_A" + code = "openQCD" + meas_path = "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf" + uuid = "Project_A" + pars = "{par_A: 3.0, par_B: 4.0}" + parameter_file = "projects/Project_A/myinput.in" + c.execute("INSERT INTO backlogs (name, ensemble, code, path, project, parameters, parameter_file, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, datetime('now'), datetime('now'))", + (corr, ensemble, code, meas_path, uuid, pars, parameter_file)) + conn.commit() + + corr = "f_A" + results = find._db_lookup(db, ensemble, corr, code) + assert len(results) == 1 + results = find._db_lookup(db, "SF_B", corr, code) + assert results.empty + results = find._db_lookup(db, ensemble, "g_A", code) + assert len(results) == 1 + results = find._db_lookup(db, ensemble, corr, "sfcf") + assert results.empty + results = find._db_lookup(db, ensemble, corr, code, project = "Project_A") + assert len(results) == 1 + results = find._db_lookup(db, ensemble, "g_A", code, project = "Project_A") + assert len(results) == 1 + results = find._db_lookup(db, ensemble, corr, code, project = "Project_B") + assert results.empty + results = find._db_lookup(db, ensemble, "g_A", code, project = "Project_B") + assert results.empty + results = find._db_lookup(db, ensemble, corr, code, parameters = pars) + assert results.empty + results = find._db_lookup(db, ensemble, "g_A", code, parameters = "{par_A: 3.0, par_B: 4.0}") + assert len(results) == 1 + + conn.close() + + +def test_sfcf_filter() -> None: + record_0 = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{par_A: 5.0, par_B: 5.0}', "projects/SF_A/input.in", + '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] + record_1 = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{par_A: 5.0, par_B: 5.0}', "projects/SF_A/input.in", + '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] + record_2 = ["f_P", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{par_A: 5.0, par_B: 5.0}', "projects/SF_A/input.in", + '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] + record_3 = ["f_P", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{par_A: 5.0, par_B: 5.0}', "projects/SF_A/input.in", + '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] + record_4 = [] + record_5 = [] + record_6 = [] + record_7 = [] + record_8 = [] + record_9 = [] + data = [ + record_0, + record_1, + record_2, + record_3, + record_4, + record_5, + record_6, + record_7, + record_8, + record_9, + ] + cols = ["name", + "ensemble", + "code", + "path", + "project", + "parameters", + "parameter_file", + "created_at", + "updated_at"] + df = pd.DataFrame(data,columns=cols) + + assert True + + +def test_openQCD_filter() -> None: + assert True + + +def test_find_record() -> None: + assert True + + +def test_find_project(tmp_path: Path) -> None: + cinit.create(tmp_path) + db = tmp_path / "backlogger.db" + dl.unlock(str(db), dataset=str(tmp_path)) + conn = sqlite3.connect(db) + c = conn.cursor() + uuid = "test_uuid" + alias_str = "fun_project" + tag_str = "tt" + owner = "tester" + code = "test_code" + c.execute("INSERT INTO projects (id, aliases, customTags, owner, code, created_at, updated_at) VALUES (?, ?, ?, ?, ?, datetime('now'), datetime('now'))", + (uuid, alias_str, tag_str, owner, code)) + conn.commit() + + assert uuid == find.find_project(tmp_path, "fun_project") + + uuid = "test_uuid2" + alias_str = "fun_project" + c.execute("INSERT INTO projects (id, aliases, customTags, owner, code, created_at, updated_at) VALUES (?, ?, ?, ?, ?, datetime('now'), datetime('now'))", + (uuid, alias_str, tag_str, owner, code)) + conn.commit() + + with pytest.raises(Exception): + assert uuid == find._project_lookup_by_alias(tmp_path, "fun_project") + conn.close() + + +def test_list_projects(tmp_path: Path) -> None: + cinit.create(tmp_path) + db = tmp_path / "backlogger.db" + dl.unlock(str(db), dataset=str(tmp_path)) + conn = sqlite3.connect(db) + c = conn.cursor() + uuid = "test_uuid" + alias_str = "fun_project" + tag_str = "tt" + owner = "tester" + code = "test_code" + + c.execute("INSERT INTO projects (id, aliases, customTags, owner, code, created_at, updated_at) VALUES (?, ?, ?, ?, ?, datetime('now'), datetime('now'))", + (uuid, alias_str, tag_str, owner, code)) + uuid = "test_uuid2" + alias_str = "fun_project2" + c.execute("INSERT INTO projects (id, aliases, customTags, owner, code, created_at, updated_at) VALUES (?, ?, ?, ?, ?, datetime('now'), datetime('now'))", + (uuid, alias_str, tag_str, owner, code)) + uuid = "test_uuid3" + alias_str = "fun_project3" + c.execute("INSERT INTO projects (id, aliases, customTags, owner, code, created_at, updated_at) VALUES (?, ?, ?, ?, ?, datetime('now'), datetime('now'))", + (uuid, alias_str, tag_str, owner, code)) + uuid = "test_uuid4" + alias_str = "fun_project4" + c.execute("INSERT INTO projects (id, aliases, customTags, owner, code, created_at, updated_at) VALUES (?, ?, ?, ?, ?, datetime('now'), datetime('now'))", + (uuid, alias_str, tag_str, owner, code)) + conn.commit() + conn.close() + results = find.list_projects(tmp_path) + assert len(results) == 4 + for i in range(4): + assert len(results[i]) == 2 From 1a1ac5121dbd623513bfaca70de0aa829352029c Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Fri, 27 Mar 2026 11:53:39 +0100 Subject: [PATCH 21/85] restructure: make code filter --- corrlib/find.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/corrlib/find.py b/corrlib/find.py index cb85130..9b2c201 100644 --- a/corrlib/find.py +++ b/corrlib/find.py @@ -276,6 +276,15 @@ def openQCD_filter(results:pd.DataFrame, **kwargs: Any) -> pd.DataFrame: return results +def _code_filter(results: pd.DataFrame, code: str, **kwargs: Any) -> pd.DataFrame: + if code == "sfcf": + return sfcf_filter(results, **kwargs) + elif code == "openQCD": + return openQCD_filter(results, **kwargs) + else: + raise ValueError(f"Code {code} is not known.") + + def find_record(path: Path, ensemble: str, correlator_name: str, code: str, project: Optional[str]=None, parameters: Optional[str]=None, created_before: Optional[str]=None, created_after: Optional[str]=None, updated_before: Optional[str]=None, updated_after: Optional[str]=None, revision: Optional[str]=None, @@ -289,12 +298,7 @@ def find_record(path: Path, ensemble: str, correlator_name: str, code: str, proj results = _db_lookup(db, ensemble, correlator_name,code, project, parameters=parameters) if any([arg is not None for arg in [created_before, created_after, updated_before, updated_after]]): results = _time_filter(results, created_before, created_after, updated_before, updated_after) - if code == "sfcf": - results = sfcf_filter(results, **kwargs) - elif code == "openQCD": - results = openQCD_filter(results, **kwargs) - else: - raise ValueError(f"Code {code} is not known.") + results = _code_filter(results, code, **kwargs) if customFilter is not None: results = customFilter(results) print("Found " + str(len(results)) + " result" + ("s" if len(results)>1 else "")) From f98521b5a1f6839d1624579186677ee8ce5cbf5d Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Fri, 27 Mar 2026 11:56:40 +0100 Subject: [PATCH 22/85] HOTFIX: strings for pyerrors 3 --- corrlib/meas_io.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/corrlib/meas_io.py b/corrlib/meas_io.py index 48017a1..0f9ac02 100644 --- a/corrlib/meas_io.py +++ b/corrlib/meas_io.py @@ -59,7 +59,7 @@ def write_measurement(path: Path, ensemble: str, measurement: dict[str, dict[str if file not in files_to_save: unlock(path, file_in_archive) files_to_save.append(file_in_archive) - known_meas = pj.load_json_dict(file, verbose=False) + known_meas = pj.load_json_dict(str(file), verbose=False) if code == "sfcf": if parameter_file is not None: parameters = sfcf.read_param(path, uuid, parameter_file) @@ -113,7 +113,7 @@ def write_measurement(path: Path, ensemble: str, measurement: dict[str, dict[str c.execute("INSERT INTO backlogs (name, ensemble, code, path, project, parameters, parameter_file, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, datetime('now'), datetime('now'))", (corr, ensemble, code, meas_path, uuid, pars[subkey], parameter_file)) conn.commit() - pj.dump_dict_to_json(known_meas, file) + pj.dump_dict_to_json(known_meas, str(file)) conn.close() save(path, message="Add measurements to database", files=files_to_save) return @@ -269,11 +269,11 @@ def drop_record(path: Path, meas_path: str) -> None: raise ValueError("This measurement does not exist as an entry!") conn.commit() - known_meas = pj.load_json_dict(file) + known_meas = pj.load_json_dict(str(file)) if sub_key in known_meas: del known_meas[sub_key] unlock(path, Path(file_in_archive)) - pj.dump_dict_to_json(known_meas, file) + pj.dump_dict_to_json(known_meas, str(file)) save(path, message="Drop measurements to database", files=[db, file]) return else: From 4673751dc3fd56dcb9776fb8d79e2d5b60f9e4b2 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Tue, 7 Apr 2026 11:29:10 +0200 Subject: [PATCH 23/85] add docstrings for openQCD filter --- corrlib/find.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/corrlib/find.py b/corrlib/find.py index 9b2c201..d368973 100644 --- a/corrlib/find.py +++ b/corrlib/find.py @@ -273,10 +273,43 @@ def sfcf_filter(results: pd.DataFrame, **kwargs: Any) -> pd.DataFrame: def openQCD_filter(results:pd.DataFrame, **kwargs: Any) -> pd.DataFrame: + """ + Filter for parameters of openQCD. + + Parameters + ---------- + results: pd.DataFrame + The unfiltered list of results from the database. + + Returns + ------- + results: pd.DataFrame + The filtered results. + + """ return results def _code_filter(results: pd.DataFrame, code: str, **kwargs: Any) -> pd.DataFrame: + """ + Abstraction of the filters for the different codes that are available. + At the moment, only openQCD and SFCF are known. + The possible key words for the parameters can be seen in the descriptionso f the code-specific filters. + + Parameters + ---------- + results: pd.DataFrame + The unfiltered list of results from the database. + code: str + The name of the code that produced the record at hand. + kwargs: + The keyworkd args that are handed over to the code-specific filters. + + Returns + ------- + results: pd.DataFrame + The filtered results. + """ if code == "sfcf": return sfcf_filter(results, **kwargs) elif code == "openQCD": From 8db8d46a06c76bed244bcd9df374c6060d1886ff Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Tue, 7 Apr 2026 11:40:48 +0200 Subject: [PATCH 24/85] add very simple tests or code filter and openQCD filter, fix json par strings --- tests/find_test.py | 127 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 102 insertions(+), 25 deletions(-) diff --git a/tests/find_test.py b/tests/find_test.py index 944ae5f..156e5fe 100644 --- a/tests/find_test.py +++ b/tests/find_test.py @@ -57,17 +57,17 @@ def test_find_lookup_by_id(tmp_path: Path) -> None: def test_time_filter() -> None: - record_A = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{par_A: 5.0, par_B: 5.0}', "projects/SF_A/input.in", + record_A = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] # only created - record_B = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{par_A: 5.0, par_B: 5.0}', "projects/SF_A/input.in", + record_B = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", '2025-03-26 12:55:18.229966', '2025-04-26 12:55:18.229966'] # created and updated - record_C = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{par_A: 5.0, par_B: 5.0}', "projects/SF_A/input.in", + record_C = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", '2026-03-26 12:55:18.229966', '2026-05-26 12:55:18.229966'] # created and updated later - record_D = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{par_A: 5.0, par_B: 5.0}', "projects/SF_A/input.in", + record_D = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", '2026-03-26 12:55:18.229966', '2026-03-27 12:55:18.229966'] - record_E = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{par_A: 5.0, par_B: 5.0}', "projects/SF_A/input.in", + record_E = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", '2024-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # only created, earlier - record_F = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{par_A: 5.0, par_B: 5.0}', "projects/SF_A/input.in", + record_F = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", '2026-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # this is invalid... data = [record_A, record_B, record_C, record_D, record_E] @@ -172,7 +172,7 @@ def test_db_lookup(tmp_path: Path) -> None: assert results.empty results = find._db_lookup(db, ensemble, corr, code, parameters = pars) assert len(results) == 1 - results = find._db_lookup(db, ensemble, corr, code, parameters = "{par_A: 3.0, par_B: 4.0}") + results = find._db_lookup(db, ensemble, corr, code, parameters = '{"par_A": 3.0, "par_B": 4.0}') assert results.empty corr = "g_A" @@ -180,7 +180,7 @@ def test_db_lookup(tmp_path: Path) -> None: code = "openQCD" meas_path = "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf" uuid = "Project_A" - pars = "{par_A: 3.0, par_B: 4.0}" + pars = '{"par_A": 3.0, "par_B": 4.0}' parameter_file = "projects/Project_A/myinput.in" c.execute("INSERT INTO backlogs (name, ensemble, code, path, project, parameters, parameter_file, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, datetime('now'), datetime('now'))", (corr, ensemble, code, meas_path, uuid, pars, parameter_file)) @@ -205,38 +205,26 @@ def test_db_lookup(tmp_path: Path) -> None: assert results.empty results = find._db_lookup(db, ensemble, corr, code, parameters = pars) assert results.empty - results = find._db_lookup(db, ensemble, "g_A", code, parameters = "{par_A: 3.0, par_B: 4.0}") + results = find._db_lookup(db, ensemble, "g_A", code, parameters = '{"par_A": 3.0, "par_B": 4.0}') assert len(results) == 1 conn.close() def test_sfcf_filter() -> None: - record_0 = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{par_A: 5.0, par_B: 5.0}', "projects/SF_A/input.in", + record_0 = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] - record_1 = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{par_A: 5.0, par_B: 5.0}', "projects/SF_A/input.in", + record_1 = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] - record_2 = ["f_P", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{par_A: 5.0, par_B: 5.0}', "projects/SF_A/input.in", + record_2 = ["f_P", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] - record_3 = ["f_P", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{par_A: 5.0, par_B: 5.0}', "projects/SF_A/input.in", + record_3 = ["f_P", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] - record_4 = [] - record_5 = [] - record_6 = [] - record_7 = [] - record_8 = [] - record_9 = [] data = [ record_0, record_1, record_2, record_3, - record_4, - record_5, - record_6, - record_7, - record_8, - record_9, ] cols = ["name", "ensemble", @@ -253,9 +241,98 @@ def test_sfcf_filter() -> None: def test_openQCD_filter() -> None: + record_0 = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] + record_1 = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] + record_2 = ["f_P", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] + record_3 = ["f_P", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] + data = [ + record_0, + record_1, + record_2, + record_3, + ] + cols = ["name", + "ensemble", + "code", + "path", + "project", + "parameters", + "parameter_file", + "created_at", + "updated_at"] + df = pd.DataFrame(data,columns=cols) + + find.openQCD_filter(df) assert True +def test_code_filter() -> None: + record_0 = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] + record_1 = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] + record_2 = ["f_P", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] + record_3 = ["f_P", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] + record_4 = ["f_A", "ensA", "openQCD", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] + record_5 = ["f_A", "ensA", "openQCD", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] + record_6 = ["f_P", "ensA", "openQCD", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] + record_7 = ["f_P", "ensA", "openQCD", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] + record_8 = ["f_P", "ensA", "openQCD", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] + data = [ + record_0, + record_1, + record_2, + record_3, + ] + cols = ["name", + "ensemble", + "code", + "path", + "project", + "parameters", + "parameter_file", + "created_at", + "updated_at"] + df = pd.DataFrame(data,columns=cols) + + res = find._code_filter(df, "sfcf") + assert len(res) == 4 + + data = [ + record_4, + record_5, + record_6, + record_7, + record_8, + ] + cols = ["name", + "ensemble", + "code", + "path", + "project", + "parameters", + "parameter_file", + "created_at", + "updated_at"] + df = pd.DataFrame(data,columns=cols) + + res = find._code_filter(df, "openQCD") + assert len(res) == 5 + with pytest.raises(ValueError): + res = find._code_filter(df, "asdf") + + def test_find_record() -> None: assert True From 3a1e41808b00763ec270ef32700a9ba45fcf74ee Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Wed, 8 Apr 2026 17:26:38 +0200 Subject: [PATCH 25/85] correct minor typos in doc --- corrlib/find.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/corrlib/find.py b/corrlib/find.py index d368973..660e4bf 100644 --- a/corrlib/find.py +++ b/corrlib/find.py @@ -179,9 +179,9 @@ def sfcf_filter(results: pd.DataFrame, **kwargs: Any) -> pd.DataFrame: qk2: float, optional Mass parameter $\kappa_2$ of the first quark. qm1: float, optional - Bare quak mass $m_1$ of the first quark. + Bare quark mass $m_1$ of the first quark. qm2: float, optional - Bare quak mass $m_1$ of the first quark. + Bare quark mass $m_2$ of the first quark. quarks_thetas: list[list[float]], optional wf1: optional wf2: optional From 7275fdd4f33c67d0927ca65990e59865ce77cac2 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Wed, 8 Apr 2026 17:28:54 +0200 Subject: [PATCH 26/85] remove unnecessary output when results are empty --- corrlib/cli.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/corrlib/cli.py b/corrlib/cli.py index b28692a..6c1c3c5 100644 --- a/corrlib/cli.py +++ b/corrlib/cli.py @@ -108,6 +108,8 @@ def find( Find a record in the backlog at hand. Through specifying it's ensemble and the measured correlator. """ results = find_record(path, ensemble, corr, code) + if results.empty: + return if arg == 'all': print(results) else: From 5c37c06b1356ab7c53dfdc1524470d5cd4df330b Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Thu, 9 Apr 2026 09:54:39 +0200 Subject: [PATCH 27/85] add an implementation to read the first ~200 bytes of the par file of openQCD's qcd2 --- corrlib/input/openQCD.py | 56 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/corrlib/input/openQCD.py b/corrlib/input/openQCD.py index a3bce6f..9c5fbbc 100644 --- a/corrlib/input/openQCD.py +++ b/corrlib/input/openQCD.py @@ -4,6 +4,7 @@ import os import fnmatch from typing import Any, Optional from pathlib import Path +import struct def read_ms1_param(path: Path, project: str, file_in_project: str) -> dict[str, Any]: @@ -304,3 +305,58 @@ def extract_t1(path: Path, project: str, dir_in_project: str, param: dict[str, A t1_dict[param["type"]] = {} t1_dict[param["type"]][pars] = t0 return t1_dict + + +def read_par_file(fname: str) -> dict[str, dict[str, Any]]: + + def _qcd2_write_lat_parms() -> dict[str, Any]: + lat_pars = {} + + t = fp.read(16) + lat_pars["N"] = list(struct.unpack('iiii', t)) # lattice extends + t = fp.read(8) + nk, isw = struct.unpack('ii', t) + lat_pars["nk"] = nk + lat_pars["isw"] = isw + t = fp.read(8) + lat_pars["beta"] = struct.unpack('d', t)[0] + t = fp.read(8) + lat_pars["c0"] = struct.unpack('d', t)[0] + t = fp.read(8) + lat_pars["c1"] = struct.unpack('d', t)[0] + t = fp.read(8) + lat_pars["csw"] = struct.unpack('d', t)[0] + kappas = [] + m0s = [] + for ik in range(nk): + t = fp.read(8) + kappas.append(struct.unpack('d', t)[0]) + t = fp.read(8) + m0s.append(struct.unpack('d', t)[0]) + lat_pars["kappas"] = kappas + lat_pars["m0s"] = m0s + return lat_pars + + def _qcd2_write_bc_parms() -> dict[str, Any]: + bc_pars = {} + t = fp.read(4) + bc_pars["type"] = struct.unpack('i', t)[0] + t = fp.read(104) + bc_parms = struct.unpack('d'*13, t) + bc_pars["cG"] = list(bc_parms[:2]) + bc_pars["cF"] = list(bc_parms[2:4]) + phi = [[], []] + phi[0] = list(bc_parms[4:7]) + phi[1] = list(bc_parms[7:10]) + bc_pars["phi"] = phi + bc_pars["theta"] = list(bc_parms[10:]) + return bc_pars + + with open(fname, "rb") as fp: + lat_par_dict = _qcd2_write_lat_parms() + bc_par_dict = _qcd2_write_bc_parms() + fp.close() + par_dict = {} + par_dict["lat"] = lat_par_dict + par_dict["bc"] = bc_par_dict + return par_dict From 5ea832675702ae6715b07215de120315657ca03d Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Thu, 9 Apr 2026 10:26:47 +0200 Subject: [PATCH 28/85] add thin wrapper to accomodate for input conventions, add comments --- corrlib/input/openQCD.py | 62 +++++++++++++++++++++++++++++++++------- 1 file changed, 52 insertions(+), 10 deletions(-) diff --git a/corrlib/input/openQCD.py b/corrlib/input/openQCD.py index 9c5fbbc..1d36e7f 100644 --- a/corrlib/input/openQCD.py +++ b/corrlib/input/openQCD.py @@ -307,27 +307,42 @@ def extract_t1(path: Path, project: str, dir_in_project: str, param: dict[str, A return t1_dict -def read_par_file(fname: str) -> dict[str, dict[str, Any]]: +def read_qcd2_par_file(fname: Path) -> dict[str, dict[str, Any]]: + """ + The subroutines written here have names according to the openQCD programs and functions that write out the data. + Parameters + ---------- + fname: Path + Location of the parameter file. + + Returns + ------- + par_dict: dict + Dictionary holding the parameters specified in the given file. + """ def _qcd2_write_lat_parms() -> dict[str, Any]: + """ + Unpack the lattice parameters written by write_lat_parms. + """ lat_pars = {} - t = fp.read(16) lat_pars["N"] = list(struct.unpack('iiii', t)) # lattice extends t = fp.read(8) - nk, isw = struct.unpack('ii', t) + nk, isw = struct.unpack('ii', t) # number of kappas and isw parameter lat_pars["nk"] = nk lat_pars["isw"] = isw t = fp.read(8) - lat_pars["beta"] = struct.unpack('d', t)[0] + lat_pars["beta"] = struct.unpack('d', t)[0] # beta t = fp.read(8) lat_pars["c0"] = struct.unpack('d', t)[0] t = fp.read(8) lat_pars["c1"] = struct.unpack('d', t)[0] t = fp.read(8) - lat_pars["csw"] = struct.unpack('d', t)[0] + lat_pars["csw"] = struct.unpack('d', t)[0] # csw factor kappas = [] m0s = [] + # read kappas for ik in range(nk): t = fp.read(8) kappas.append(struct.unpack('d', t)[0]) @@ -338,14 +353,17 @@ def read_par_file(fname: str) -> dict[str, dict[str, Any]]: return lat_pars def _qcd2_write_bc_parms() -> dict[str, Any]: - bc_pars = {} + """ + Unpack the boundary parameters written by write_bc_parms. + """ + bc_pars: dict[str, Any] = {} t = fp.read(4) - bc_pars["type"] = struct.unpack('i', t)[0] + bc_pars["type"] = struct.unpack('i', t)[0] # type of hte boundaries t = fp.read(104) bc_parms = struct.unpack('d'*13, t) - bc_pars["cG"] = list(bc_parms[:2]) - bc_pars["cF"] = list(bc_parms[2:4]) - phi = [[], []] + bc_pars["cG"] = list(bc_parms[:2]) # boundary gauge field improvement + bc_pars["cF"] = list(bc_parms[2:4]) # boundary fermion field improvement + phi: list[list[float]] = [[], []] phi[0] = list(bc_parms[4:7]) phi[1] = list(bc_parms[7:10]) bc_pars["phi"] = phi @@ -360,3 +378,27 @@ def read_par_file(fname: str) -> dict[str, dict[str, Any]]: par_dict["lat"] = lat_par_dict par_dict["bc"] = bc_par_dict return par_dict + + +def load_qcd2_pars(path: Path, project: str, file_in_project: str) -> dict[str, Any]: + """ + Thin wrapper around read_qcd2_par_file, getting the file before reading. + + Parameters + ---------- + path: Path + Path of the corrlib repository. + project: str + UUID of the project of the parameter-file. + file_in_project: str + The loaction of the file in the project directory. + + Returns + ------- + par_dict: dict + The dict with the parameters read from the .par-file. + """ + fname = path / "projects" / project / file_in_project + ds = os.path.join(path, "projects", project) + dl.get(fname, dataset=ds) + return read_qcd2_par_file(fname) From 71332264cf36ad27c9a9d840c7f14c86ba7835ce Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Thu, 9 Apr 2026 10:47:19 +0200 Subject: [PATCH 29/85] restruct: give each openQCD prog it's own file --- corrlib/input/openQCD.py | 103 +++++++++---------------------- corrlib/pars/openQCD/__init__.py | 3 + corrlib/pars/openQCD/ms1.py | 81 ++++++++++++++++++++++++ corrlib/pars/openQCD/qcd2.py | 77 +++++++++++++++++++++++ 4 files changed, 189 insertions(+), 75 deletions(-) create mode 100644 corrlib/pars/openQCD/__init__.py create mode 100644 corrlib/pars/openQCD/ms1.py create mode 100644 corrlib/pars/openQCD/qcd2.py diff --git a/corrlib/input/openQCD.py b/corrlib/input/openQCD.py index 1d36e7f..8a2b41e 100644 --- a/corrlib/input/openQCD.py +++ b/corrlib/input/openQCD.py @@ -4,7 +4,9 @@ import os import fnmatch from typing import Any, Optional from pathlib import Path -import struct +from ..pars.openQCD import ms1 +from ..pars.openQCD import qcd2 + def read_ms1_param(path: Path, project: str, file_in_project: str) -> dict[str, Any]: @@ -307,79 +309,6 @@ def extract_t1(path: Path, project: str, dir_in_project: str, param: dict[str, A return t1_dict -def read_qcd2_par_file(fname: Path) -> dict[str, dict[str, Any]]: - """ - The subroutines written here have names according to the openQCD programs and functions that write out the data. - - Parameters - ---------- - fname: Path - Location of the parameter file. - - Returns - ------- - par_dict: dict - Dictionary holding the parameters specified in the given file. - """ - def _qcd2_write_lat_parms() -> dict[str, Any]: - """ - Unpack the lattice parameters written by write_lat_parms. - """ - lat_pars = {} - t = fp.read(16) - lat_pars["N"] = list(struct.unpack('iiii', t)) # lattice extends - t = fp.read(8) - nk, isw = struct.unpack('ii', t) # number of kappas and isw parameter - lat_pars["nk"] = nk - lat_pars["isw"] = isw - t = fp.read(8) - lat_pars["beta"] = struct.unpack('d', t)[0] # beta - t = fp.read(8) - lat_pars["c0"] = struct.unpack('d', t)[0] - t = fp.read(8) - lat_pars["c1"] = struct.unpack('d', t)[0] - t = fp.read(8) - lat_pars["csw"] = struct.unpack('d', t)[0] # csw factor - kappas = [] - m0s = [] - # read kappas - for ik in range(nk): - t = fp.read(8) - kappas.append(struct.unpack('d', t)[0]) - t = fp.read(8) - m0s.append(struct.unpack('d', t)[0]) - lat_pars["kappas"] = kappas - lat_pars["m0s"] = m0s - return lat_pars - - def _qcd2_write_bc_parms() -> dict[str, Any]: - """ - Unpack the boundary parameters written by write_bc_parms. - """ - bc_pars: dict[str, Any] = {} - t = fp.read(4) - bc_pars["type"] = struct.unpack('i', t)[0] # type of hte boundaries - t = fp.read(104) - bc_parms = struct.unpack('d'*13, t) - bc_pars["cG"] = list(bc_parms[:2]) # boundary gauge field improvement - bc_pars["cF"] = list(bc_parms[2:4]) # boundary fermion field improvement - phi: list[list[float]] = [[], []] - phi[0] = list(bc_parms[4:7]) - phi[1] = list(bc_parms[7:10]) - bc_pars["phi"] = phi - bc_pars["theta"] = list(bc_parms[10:]) - return bc_pars - - with open(fname, "rb") as fp: - lat_par_dict = _qcd2_write_lat_parms() - bc_par_dict = _qcd2_write_bc_parms() - fp.close() - par_dict = {} - par_dict["lat"] = lat_par_dict - par_dict["bc"] = bc_par_dict - return par_dict - - def load_qcd2_pars(path: Path, project: str, file_in_project: str) -> dict[str, Any]: """ Thin wrapper around read_qcd2_par_file, getting the file before reading. @@ -401,4 +330,28 @@ def load_qcd2_pars(path: Path, project: str, file_in_project: str) -> dict[str, fname = path / "projects" / project / file_in_project ds = os.path.join(path, "projects", project) dl.get(fname, dataset=ds) - return read_qcd2_par_file(fname) + return qcd2.read_qcd2_par_file(fname) + + +def load_ms1_pars(path: Path, project: str, file_in_project: str) -> dict[str, Any]: + """ + Thin wrapper around read_qcd2_ms1_par_file, getting the file before reading. + + Parameters + ---------- + path: Path + Path of the corrlib repository. + project: str + UUID of the project of the parameter-file. + file_in_project: str + The loaction of the file in the project directory. + + Returns + ------- + par_dict: dict + The dict with the parameters read from the .par-file. + """ + fname = path / "projects" / project / file_in_project + ds = os.path.join(path, "projects", project) + dl.get(fname, dataset=ds) + return ms1.read_qcd2_ms1_par_file(fname) diff --git a/corrlib/pars/openQCD/__init__.py b/corrlib/pars/openQCD/__init__.py new file mode 100644 index 0000000..edbac71 --- /dev/null +++ b/corrlib/pars/openQCD/__init__.py @@ -0,0 +1,3 @@ + +from . import ms1 as ms1 +from . import qcd2 as qcd2 diff --git a/corrlib/pars/openQCD/ms1.py b/corrlib/pars/openQCD/ms1.py new file mode 100644 index 0000000..9aabc54 --- /dev/null +++ b/corrlib/pars/openQCD/ms1.py @@ -0,0 +1,81 @@ +import struct + +from typing import Any +from pathlib import Path + + +def read_qcd2_ms1_par_file(fname: Path) -> dict[str, dict[str, Any]]: + """ + The subroutines written here have names according to the openQCD programs and functions that write out the data. + Parameters + ---------- + fname: Path + Location of the parameter file. + + Returns + ------- + par_dict: dict + Dictionary holding the parameters specified in the given file. + """ + + def _qcd2_write_lat_parms() -> dict[str, Any]: + """ + NOTE: This is a duplcation from qcd2. + Unpack the lattice parameters written by write_lat_parms. + """ + lat_pars = {} + t = fp.read(16) + lat_pars["N"] = list(struct.unpack('iiii', t)) # lattice extends + t = fp.read(8) + nk, isw = struct.unpack('ii', t) # number of kappas and isw parameter + lat_pars["nk"] = nk + lat_pars["isw"] = isw + t = fp.read(8) + lat_pars["beta"] = struct.unpack('d', t)[0] # beta + t = fp.read(8) + lat_pars["c0"] = struct.unpack('d', t)[0] + t = fp.read(8) + lat_pars["c1"] = struct.unpack('d', t)[0] + t = fp.read(8) + lat_pars["csw"] = struct.unpack('d', t)[0] # csw factor + kappas = [] + m0s = [] + # read kappas + for ik in range(nk): + t = fp.read(8) + kappas.append(struct.unpack('d', t)[0]) + t = fp.read(8) + m0s.append(struct.unpack('d', t)[0]) + lat_pars["kappas"] = kappas + lat_pars["m0s"] = m0s + return lat_pars + + def _qcd2_write_bc_parms() -> dict[str, Any]: + """ + NOTE: This is a duplcation from qcd2. + Unpack the boundary parameters written by write_bc_parms. + """ + bc_pars: dict[str, Any] = {} + t = fp.read(4) + bc_pars["type"] = struct.unpack('i', t)[0] # type of hte boundaries + t = fp.read(104) + bc_parms = struct.unpack('d'*13, t) + bc_pars["cG"] = list(bc_parms[:2]) # boundary gauge field improvement + bc_pars["cF"] = list(bc_parms[2:4]) # boundary fermion field improvement + phi: list[list[float]] = [[], []] + phi[0] = list(bc_parms[4:7]) + phi[1] = list(bc_parms[7:10]) + bc_pars["phi"] = phi + bc_pars["theta"] = list(bc_parms[10:]) + return bc_pars + + with open(fname, "rb") as fp: + lat_par_dict = _qcd2_write_lat_parms() + bc_par_dict = _qcd2_write_bc_parms() + fp.close() + par_dict = {} + par_dict["lat"] = lat_par_dict + par_dict["bc"] = bc_par_dict + return par_dict + + diff --git a/corrlib/pars/openQCD/qcd2.py b/corrlib/pars/openQCD/qcd2.py new file mode 100644 index 0000000..9d63689 --- /dev/null +++ b/corrlib/pars/openQCD/qcd2.py @@ -0,0 +1,77 @@ +import struct + +from pathlib import Path +from typing import Any + + +def read_qcd2_par_file(fname: Path) -> dict[str, dict[str, Any]]: + """ + The subroutines written here have names according to the openQCD programs and functions that write out the data. + + Parameters + ---------- + fname: Path + Location of the parameter file. + + Returns + ------- + par_dict: dict + Dictionary holding the parameters specified in the given file. + """ + def _qcd2_write_lat_parms() -> dict[str, Any]: + """ + Unpack the lattice parameters written by write_lat_parms. + """ + lat_pars = {} + t = fp.read(16) + lat_pars["N"] = list(struct.unpack('iiii', t)) # lattice extends + t = fp.read(8) + nk, isw = struct.unpack('ii', t) # number of kappas and isw parameter + lat_pars["nk"] = nk + lat_pars["isw"] = isw + t = fp.read(8) + lat_pars["beta"] = struct.unpack('d', t)[0] # beta + t = fp.read(8) + lat_pars["c0"] = struct.unpack('d', t)[0] + t = fp.read(8) + lat_pars["c1"] = struct.unpack('d', t)[0] + t = fp.read(8) + lat_pars["csw"] = struct.unpack('d', t)[0] # csw factor + kappas = [] + m0s = [] + # read kappas + for ik in range(nk): + t = fp.read(8) + kappas.append(struct.unpack('d', t)[0]) + t = fp.read(8) + m0s.append(struct.unpack('d', t)[0]) + lat_pars["kappas"] = kappas + lat_pars["m0s"] = m0s + return lat_pars + + def _qcd2_write_bc_parms() -> dict[str, Any]: + """ + Unpack the boundary parameters written by write_bc_parms. + """ + bc_pars: dict[str, Any] = {} + t = fp.read(4) + bc_pars["type"] = struct.unpack('i', t)[0] # type of hte boundaries + t = fp.read(104) + bc_parms = struct.unpack('d'*13, t) + bc_pars["cG"] = list(bc_parms[:2]) # boundary gauge field improvement + bc_pars["cF"] = list(bc_parms[2:4]) # boundary fermion field improvement + phi: list[list[float]] = [[], []] + phi[0] = list(bc_parms[4:7]) + phi[1] = list(bc_parms[7:10]) + bc_pars["phi"] = phi + bc_pars["theta"] = list(bc_parms[10:]) + return bc_pars + + with open(fname, "rb") as fp: + lat_par_dict = _qcd2_write_lat_parms() + bc_par_dict = _qcd2_write_bc_parms() + fp.close() + par_dict = {} + par_dict["lat"] = lat_par_dict + par_dict["bc"] = bc_par_dict + return par_dict From e654d7c1bb6b4948eb43037977142358c37dcd7b Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Thu, 9 Apr 2026 11:10:54 +0200 Subject: [PATCH 30/85] restruct: introduce a file for flags --- corrlib/pars/openQCD/flags.py | 59 +++++++++++++++++++++++++++++++++++ corrlib/pars/openQCD/ms1.py | 57 ++------------------------------- corrlib/pars/openQCD/qcd2.py | 55 ++------------------------------ 3 files changed, 65 insertions(+), 106 deletions(-) create mode 100644 corrlib/pars/openQCD/flags.py diff --git a/corrlib/pars/openQCD/flags.py b/corrlib/pars/openQCD/flags.py new file mode 100644 index 0000000..2e4ab17 --- /dev/null +++ b/corrlib/pars/openQCD/flags.py @@ -0,0 +1,59 @@ +""" +Reconstruct the outputs of flags. +""" + +import struct +from typing import Any, BinaryIO + +# lat_parms.c +def lat_parms_write_lat_parms(fp: BinaryIO) -> dict[str, Any]: + """ + NOTE: This is a duplcation from qcd2. + Unpack the lattice parameters written by write_lat_parms. + """ + lat_pars = {} + t = fp.read(16) + lat_pars["N"] = list(struct.unpack('iiii', t)) # lattice extends + t = fp.read(8) + nk, isw = struct.unpack('ii', t) # number of kappas and isw parameter + lat_pars["nk"] = nk + lat_pars["isw"] = isw + t = fp.read(8) + lat_pars["beta"] = struct.unpack('d', t)[0] # beta + t = fp.read(8) + lat_pars["c0"] = struct.unpack('d', t)[0] + t = fp.read(8) + lat_pars["c1"] = struct.unpack('d', t)[0] + t = fp.read(8) + lat_pars["csw"] = struct.unpack('d', t)[0] # csw factor + kappas = [] + m0s = [] + # read kappas + for ik in range(nk): + t = fp.read(8) + kappas.append(struct.unpack('d', t)[0]) + t = fp.read(8) + m0s.append(struct.unpack('d', t)[0]) + lat_pars["kappas"] = kappas + lat_pars["m0s"] = m0s + return lat_pars + + +def lat_parms_write_bc_parms(fp: BinaryIO) -> dict[str, Any]: + """ + NOTE: This is a duplcation from qcd2. + Unpack the boundary parameters written by write_bc_parms. + """ + bc_pars: dict[str, Any] = {} + t = fp.read(4) + bc_pars["type"] = struct.unpack('i', t)[0] # type of hte boundaries + t = fp.read(104) + bc_parms = struct.unpack('d'*13, t) + bc_pars["cG"] = list(bc_parms[:2]) # boundary gauge field improvement + bc_pars["cF"] = list(bc_parms[2:4]) # boundary fermion field improvement + phi: list[list[float]] = [[], []] + phi[0] = list(bc_parms[4:7]) + phi[1] = list(bc_parms[7:10]) + bc_pars["phi"] = phi + bc_pars["theta"] = list(bc_parms[10:]) + return bc_pars \ No newline at end of file diff --git a/corrlib/pars/openQCD/ms1.py b/corrlib/pars/openQCD/ms1.py index 9aabc54..4c2aed5 100644 --- a/corrlib/pars/openQCD/ms1.py +++ b/corrlib/pars/openQCD/ms1.py @@ -1,4 +1,4 @@ -import struct +from . import flags from typing import Any from pathlib import Path @@ -18,60 +18,9 @@ def read_qcd2_ms1_par_file(fname: Path) -> dict[str, dict[str, Any]]: Dictionary holding the parameters specified in the given file. """ - def _qcd2_write_lat_parms() -> dict[str, Any]: - """ - NOTE: This is a duplcation from qcd2. - Unpack the lattice parameters written by write_lat_parms. - """ - lat_pars = {} - t = fp.read(16) - lat_pars["N"] = list(struct.unpack('iiii', t)) # lattice extends - t = fp.read(8) - nk, isw = struct.unpack('ii', t) # number of kappas and isw parameter - lat_pars["nk"] = nk - lat_pars["isw"] = isw - t = fp.read(8) - lat_pars["beta"] = struct.unpack('d', t)[0] # beta - t = fp.read(8) - lat_pars["c0"] = struct.unpack('d', t)[0] - t = fp.read(8) - lat_pars["c1"] = struct.unpack('d', t)[0] - t = fp.read(8) - lat_pars["csw"] = struct.unpack('d', t)[0] # csw factor - kappas = [] - m0s = [] - # read kappas - for ik in range(nk): - t = fp.read(8) - kappas.append(struct.unpack('d', t)[0]) - t = fp.read(8) - m0s.append(struct.unpack('d', t)[0]) - lat_pars["kappas"] = kappas - lat_pars["m0s"] = m0s - return lat_pars - - def _qcd2_write_bc_parms() -> dict[str, Any]: - """ - NOTE: This is a duplcation from qcd2. - Unpack the boundary parameters written by write_bc_parms. - """ - bc_pars: dict[str, Any] = {} - t = fp.read(4) - bc_pars["type"] = struct.unpack('i', t)[0] # type of hte boundaries - t = fp.read(104) - bc_parms = struct.unpack('d'*13, t) - bc_pars["cG"] = list(bc_parms[:2]) # boundary gauge field improvement - bc_pars["cF"] = list(bc_parms[2:4]) # boundary fermion field improvement - phi: list[list[float]] = [[], []] - phi[0] = list(bc_parms[4:7]) - phi[1] = list(bc_parms[7:10]) - bc_pars["phi"] = phi - bc_pars["theta"] = list(bc_parms[10:]) - return bc_pars - with open(fname, "rb") as fp: - lat_par_dict = _qcd2_write_lat_parms() - bc_par_dict = _qcd2_write_bc_parms() + lat_par_dict = flags.lat_parms_write_lat_parms(fp) + bc_par_dict = flags.lat_parms_write_bc_parms(fp) fp.close() par_dict = {} par_dict["lat"] = lat_par_dict diff --git a/corrlib/pars/openQCD/qcd2.py b/corrlib/pars/openQCD/qcd2.py index 9d63689..3b6e277 100644 --- a/corrlib/pars/openQCD/qcd2.py +++ b/corrlib/pars/openQCD/qcd2.py @@ -1,9 +1,8 @@ -import struct +from . import flags from pathlib import Path from typing import Any - def read_qcd2_par_file(fname: Path) -> dict[str, dict[str, Any]]: """ The subroutines written here have names according to the openQCD programs and functions that write out the data. @@ -18,58 +17,10 @@ def read_qcd2_par_file(fname: Path) -> dict[str, dict[str, Any]]: par_dict: dict Dictionary holding the parameters specified in the given file. """ - def _qcd2_write_lat_parms() -> dict[str, Any]: - """ - Unpack the lattice parameters written by write_lat_parms. - """ - lat_pars = {} - t = fp.read(16) - lat_pars["N"] = list(struct.unpack('iiii', t)) # lattice extends - t = fp.read(8) - nk, isw = struct.unpack('ii', t) # number of kappas and isw parameter - lat_pars["nk"] = nk - lat_pars["isw"] = isw - t = fp.read(8) - lat_pars["beta"] = struct.unpack('d', t)[0] # beta - t = fp.read(8) - lat_pars["c0"] = struct.unpack('d', t)[0] - t = fp.read(8) - lat_pars["c1"] = struct.unpack('d', t)[0] - t = fp.read(8) - lat_pars["csw"] = struct.unpack('d', t)[0] # csw factor - kappas = [] - m0s = [] - # read kappas - for ik in range(nk): - t = fp.read(8) - kappas.append(struct.unpack('d', t)[0]) - t = fp.read(8) - m0s.append(struct.unpack('d', t)[0]) - lat_pars["kappas"] = kappas - lat_pars["m0s"] = m0s - return lat_pars - - def _qcd2_write_bc_parms() -> dict[str, Any]: - """ - Unpack the boundary parameters written by write_bc_parms. - """ - bc_pars: dict[str, Any] = {} - t = fp.read(4) - bc_pars["type"] = struct.unpack('i', t)[0] # type of hte boundaries - t = fp.read(104) - bc_parms = struct.unpack('d'*13, t) - bc_pars["cG"] = list(bc_parms[:2]) # boundary gauge field improvement - bc_pars["cF"] = list(bc_parms[2:4]) # boundary fermion field improvement - phi: list[list[float]] = [[], []] - phi[0] = list(bc_parms[4:7]) - phi[1] = list(bc_parms[7:10]) - bc_pars["phi"] = phi - bc_pars["theta"] = list(bc_parms[10:]) - return bc_pars with open(fname, "rb") as fp: - lat_par_dict = _qcd2_write_lat_parms() - bc_par_dict = _qcd2_write_bc_parms() + lat_par_dict = flags.lat_parms_qcd2_write_lat_parms(fp) + bc_par_dict = flags.lat_parms_qcd2_write_bc_parms(fp) fp.close() par_dict = {} par_dict["lat"] = lat_par_dict From 9498c1dd735ea498207eedafe4cb37f458c008c2 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Thu, 9 Apr 2026 11:11:48 +0200 Subject: [PATCH 31/85] correct function names --- corrlib/pars/openQCD/qcd2.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/corrlib/pars/openQCD/qcd2.py b/corrlib/pars/openQCD/qcd2.py index 3b6e277..e73c156 100644 --- a/corrlib/pars/openQCD/qcd2.py +++ b/corrlib/pars/openQCD/qcd2.py @@ -3,6 +3,7 @@ from . import flags from pathlib import Path from typing import Any + def read_qcd2_par_file(fname: Path) -> dict[str, dict[str, Any]]: """ The subroutines written here have names according to the openQCD programs and functions that write out the data. @@ -19,8 +20,8 @@ def read_qcd2_par_file(fname: Path) -> dict[str, dict[str, Any]]: """ with open(fname, "rb") as fp: - lat_par_dict = flags.lat_parms_qcd2_write_lat_parms(fp) - bc_par_dict = flags.lat_parms_qcd2_write_bc_parms(fp) + lat_par_dict = flags.lat_parms_write_lat_parms(fp) + bc_par_dict = flags.lat_parms_write_bc_parms(fp) fp.close() par_dict = {} par_dict["lat"] = lat_par_dict From 8394b1fdbdaeaf0e647b7e568c9338f0a4a3a166 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Thu, 9 Apr 2026 11:23:28 +0200 Subject: [PATCH 32/85] rename functions, let write_measurement decide which file type is given --- corrlib/input/openQCD.py | 6 +++--- corrlib/meas_io.py | 7 +++++-- corrlib/toml.py | 10 +++++++--- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/corrlib/input/openQCD.py b/corrlib/input/openQCD.py index 8a2b41e..879b555 100644 --- a/corrlib/input/openQCD.py +++ b/corrlib/input/openQCD.py @@ -9,7 +9,7 @@ from ..pars.openQCD import qcd2 -def read_ms1_param(path: Path, project: str, file_in_project: str) -> dict[str, Any]: +def load_ms1_infile(path: Path, project: str, file_in_project: str) -> dict[str, Any]: """ Read the parameters for ms1 measurements from a parameter file in the project. @@ -73,7 +73,7 @@ def read_ms1_param(path: Path, project: str, file_in_project: str) -> dict[str, return param -def read_ms3_param(path: Path, project: str, file_in_project: str) -> dict[str, Any]: +def load_ms3_infile(path: Path, project: str, file_in_project: str) -> dict[str, Any]: """ Read the parameters for ms3 measurements from a parameter file in the project. @@ -333,7 +333,7 @@ def load_qcd2_pars(path: Path, project: str, file_in_project: str) -> dict[str, return qcd2.read_qcd2_par_file(fname) -def load_ms1_pars(path: Path, project: str, file_in_project: str) -> dict[str, Any]: +def load_ms1_parfile(path: Path, project: str, file_in_project: str) -> dict[str, Any]: """ Thin wrapper around read_qcd2_ms1_par_file, getting the file before reading. diff --git a/corrlib/meas_io.py b/corrlib/meas_io.py index 0f9ac02..a48f546 100644 --- a/corrlib/meas_io.py +++ b/corrlib/meas_io.py @@ -74,7 +74,10 @@ def write_measurement(path: Path, ensemble: str, measurement: dict[str, dict[str ms_type = list(measurement.keys())[0] if ms_type == 'ms1': if parameter_file is not None: - parameters = openQCD.read_ms1_param(path, uuid, parameter_file) + if parameter_file.endswith(".ms1.in"): + parameters = openQCD.load_ms1_infile(path, uuid, parameter_file) + elif parameter_file.endswith(".ms1.par"): + parameters = openQCD.load_ms1_parfile(path, uuid, parameter_file) else: raise Exception("Need parameter file for this code!") pars = {} @@ -88,7 +91,7 @@ def write_measurement(path: Path, ensemble: str, measurement: dict[str, dict[str pars[subkey] = json.dumps(parameters["rw_fcts"][i]) elif ms_type in ['t0', 't1']: if parameter_file is not None: - parameters = openQCD.read_ms3_param(path, uuid, parameter_file) + parameters = openQCD.load_ms3_infile(path, uuid, parameter_file) else: parameters = {} for rwp in ["integrator", "eps", "ntot", "dnms"]: diff --git a/corrlib/toml.py b/corrlib/toml.py index add3739..93ba0f3 100644 --- a/corrlib/toml.py +++ b/corrlib/toml.py @@ -192,12 +192,16 @@ def import_toml(path: Path, file: str, copy_file: bool=True) -> None: elif project['code'] == 'openQCD': if md['measurement'] == 'ms1': - param = openQCD.read_ms1_param(path, uuid, md['param_file']) + parameter_file = md['param_file'] + if parameter_file.endswith(".ms1.in"): + param = openQCD.load_ms1_infile(path, uuid, parameter_file) + elif parameter_file.endswith(".ms1.par"): + param = openQCD.load_ms1_parfile(path, uuid, parameter_file) param['type'] = 'ms1' measurement = openQCD.read_rwms(path, uuid, md['path'], param, md["prefix"], version=md["version"], names=md['names'], files=md['files']) elif md['measurement'] == 't0': if 'param_file' in md: - param = openQCD.read_ms3_param(path, uuid, md['param_file']) + param = openQCD.load_ms3_infile(path, uuid, md['param_file']) else: param = {} for rwp in ["integrator", "eps", "ntot", "dnms"]: @@ -207,7 +211,7 @@ def import_toml(path: Path, file: str, copy_file: bool=True) -> None: fit_range=int(md.get('fit_range', 5)), postfix=str(md.get('postfix', '')), names=md.get('names', []), files=md.get('files', [])) elif md['measurement'] == 't1': if 'param_file' in md: - param = openQCD.read_ms3_param(path, uuid, md['param_file']) + param = openQCD.load_ms3_infile(path, uuid, md['param_file']) param['type'] = 't1' measurement = openQCD.extract_t1(path, uuid, md['path'], param, str(md["prefix"]), int(md["dtr_read"]), int(md["xmin"]), int(md["spatial_extent"]), fit_range=int(md.get('fit_range', 5)), postfix=str(md.get('postfix', '')), names=md.get('names', []), files=md.get('files', [])) From 26607632328e4615fce343f7d700baddaad9fdb2 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Thu, 9 Apr 2026 11:24:25 +0200 Subject: [PATCH 33/85] lint --- corrlib/pars/openQCD/flags.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/corrlib/pars/openQCD/flags.py b/corrlib/pars/openQCD/flags.py index 2e4ab17..95be919 100644 --- a/corrlib/pars/openQCD/flags.py +++ b/corrlib/pars/openQCD/flags.py @@ -56,4 +56,4 @@ def lat_parms_write_bc_parms(fp: BinaryIO) -> dict[str, Any]: phi[1] = list(bc_parms[7:10]) bc_pars["phi"] = phi bc_pars["theta"] = list(bc_parms[10:]) - return bc_pars \ No newline at end of file + return bc_pars From 3b6a8be0cc1d1ed1e64b014d7b88d96c2dcc446f Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Thu, 9 Apr 2026 12:12:38 +0200 Subject: [PATCH 34/85] TEMPFIX: allow ms1 to not have an in or par file --- corrlib/meas_io.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/corrlib/meas_io.py b/corrlib/meas_io.py index a48f546..82f2e7d 100644 --- a/corrlib/meas_io.py +++ b/corrlib/meas_io.py @@ -79,7 +79,19 @@ def write_measurement(path: Path, ensemble: str, measurement: dict[str, dict[str elif parameter_file.endswith(".ms1.par"): parameters = openQCD.load_ms1_parfile(path, uuid, parameter_file) else: - raise Exception("Need parameter file for this code!") + # Temporary solution + parameters = {} + parameters["rand"] = {} + for nrw in range(1): + if "nsrc" not in parameters["rw_fcts"][nrw]: + parameters["rw_fcts"][nrw]["nsrc"] = 1 + if "mu" not in parameters["rw_fcts"][nrw]: + parameters["rw_fcts"][nrw]["mu"] = "None" + if "np" not in parameters["rw_fcts"][nrw]: + parameters["rw_fcts"][nrw]["np"] = "None" + if "irp" not in parameters["rw_fcts"][nrw]: + parameters["rw_fcts"][nrw]["irp"] = "None" + # raise Exception("Need parameter file for this code!") pars = {} subkeys = [] for i in range(len(parameters["rw_fcts"])): From 9b628abc69f0ac44f5c2fd392be1da1c48a85d6c Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Thu, 9 Apr 2026 12:16:11 +0200 Subject: [PATCH 35/85] TEMPFIX: allow ms1 to not have an in or par file part 2 --- corrlib/toml.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/corrlib/toml.py b/corrlib/toml.py index 93ba0f3..14ec058 100644 --- a/corrlib/toml.py +++ b/corrlib/toml.py @@ -192,11 +192,26 @@ def import_toml(path: Path, file: str, copy_file: bool=True) -> None: elif project['code'] == 'openQCD': if md['measurement'] == 'ms1': - parameter_file = md['param_file'] - if parameter_file.endswith(".ms1.in"): - param = openQCD.load_ms1_infile(path, uuid, parameter_file) - elif parameter_file.endswith(".ms1.par"): - param = openQCD.load_ms1_parfile(path, uuid, parameter_file) + if 'param_file' in md.keys(): + parameter_file = md['param_file'] + if parameter_file.endswith(".ms1.in"): + param = openQCD.load_ms1_infile(path, uuid, parameter_file) + elif parameter_file.endswith(".ms1.par"): + param = openQCD.load_ms1_parfile(path, uuid, parameter_file) + else: + # Temporary solution + parameters = {} + parameters["rand"] = {} + for nrw in range(1): + if "nsrc" not in parameters["rw_fcts"][nrw]: + parameters["rw_fcts"][nrw]["nsrc"] = 1 + if "mu" not in parameters["rw_fcts"][nrw]: + parameters["rw_fcts"][nrw]["mu"] = "None" + if "np" not in parameters["rw_fcts"][nrw]: + parameters["rw_fcts"][nrw]["np"] = "None" + if "irp" not in parameters["rw_fcts"][nrw]: + parameters["rw_fcts"][nrw]["irp"] = "None" + param = parameters param['type'] = 'ms1' measurement = openQCD.read_rwms(path, uuid, md['path'], param, md["prefix"], version=md["version"], names=md['names'], files=md['files']) elif md['measurement'] == 't0': From 16dcca3f3dea23022e56e20e03b7ff9c14ed04bc Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Thu, 9 Apr 2026 12:17:29 +0200 Subject: [PATCH 36/85] TEMPFIX: allow ms1 to not have an in or par file part 3 --- corrlib/toml.py | 1 + 1 file changed, 1 insertion(+) diff --git a/corrlib/toml.py b/corrlib/toml.py index 14ec058..f0ba525 100644 --- a/corrlib/toml.py +++ b/corrlib/toml.py @@ -202,6 +202,7 @@ def import_toml(path: Path, file: str, copy_file: bool=True) -> None: # Temporary solution parameters = {} parameters["rand"] = {} + parameters["rw_fcts"] = [] for nrw in range(1): if "nsrc" not in parameters["rw_fcts"][nrw]: parameters["rw_fcts"][nrw]["nsrc"] = 1 From e3be65beec6c7bbb4fef67098c51ebe502bde125 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Thu, 9 Apr 2026 12:18:59 +0200 Subject: [PATCH 37/85] TEMPFIX: allow ms1 to not have an in or par file part 4 --- corrlib/meas_io.py | 2 +- corrlib/toml.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/corrlib/meas_io.py b/corrlib/meas_io.py index 82f2e7d..f4e8a83 100644 --- a/corrlib/meas_io.py +++ b/corrlib/meas_io.py @@ -82,6 +82,7 @@ def write_measurement(path: Path, ensemble: str, measurement: dict[str, dict[str # Temporary solution parameters = {} parameters["rand"] = {} + parameters["rw_fcts"] = [{}] for nrw in range(1): if "nsrc" not in parameters["rw_fcts"][nrw]: parameters["rw_fcts"][nrw]["nsrc"] = 1 @@ -91,7 +92,6 @@ def write_measurement(path: Path, ensemble: str, measurement: dict[str, dict[str parameters["rw_fcts"][nrw]["np"] = "None" if "irp" not in parameters["rw_fcts"][nrw]: parameters["rw_fcts"][nrw]["irp"] = "None" - # raise Exception("Need parameter file for this code!") pars = {} subkeys = [] for i in range(len(parameters["rw_fcts"])): diff --git a/corrlib/toml.py b/corrlib/toml.py index f0ba525..eb40d5a 100644 --- a/corrlib/toml.py +++ b/corrlib/toml.py @@ -202,7 +202,7 @@ def import_toml(path: Path, file: str, copy_file: bool=True) -> None: # Temporary solution parameters = {} parameters["rand"] = {} - parameters["rw_fcts"] = [] + parameters["rw_fcts"] = [{}] for nrw in range(1): if "nsrc" not in parameters["rw_fcts"][nrw]: parameters["rw_fcts"][nrw]["nsrc"] = 1 From e95edcb0932815e352011da5a69f297a92bfedd1 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Thu, 9 Apr 2026 23:27:31 +0200 Subject: [PATCH 38/85] restruct for easier tests, test drop of sfcf params --- corrlib/find.py | 149 +++++++++++++++++++++++---------------------- tests/find_test.py | 93 +++++++++++++--------------- 2 files changed, 116 insertions(+), 126 deletions(-) diff --git a/corrlib/find.py b/corrlib/find.py index 660e4bf..9d07a1c 100644 --- a/corrlib/find.py +++ b/corrlib/find.py @@ -162,6 +162,78 @@ def _db_lookup(db: Path, ensemble: str, correlator_name: str, code: str, project return results +def _sfcf_drop(param, **kwargs): + if 'offset' in kwargs: + if kwargs.get('offset') != param['offset']: + return True + if 'quark_kappas' in kwargs: + kappas = kwargs['quark_kappas'] + if (not np.isclose(kappas[0], param['quarks'][0]['mass']) or not np.isclose(kappas[1], param['quarks'][1]['mass'])): + return True + if 'quark_masses' in kwargs: + masses = kwargs['quark_masses'] + if (not np.isclose(masses[0], k2m(param['quarks'][0]['mass'])) or not np.isclose(masses[1], k2m(param['quarks'][1]['mass']))): + return True + if 'qk1' in kwargs: + quark_kappa1 = kwargs['qk1'] + if not isinstance(quark_kappa1, list): + if (not np.isclose(quark_kappa1, param['quarks'][0]['mass'])): + return True + else: + if len(quark_kappa1) == 2: + if (quark_kappa1[0] > param['quarks'][0]['mass']) or (quark_kappa1[1] < param['quarks'][0]['mass']): + return True + else: + raise ValueError("quark_kappa1 has to have length 2") + if 'qk2' in kwargs: + quark_kappa2 = kwargs['qk2'] + if not isinstance(quark_kappa2, list): + if (not np.isclose(quark_kappa2, param['quarks'][1]['mass'])): + return True + else: + if len(quark_kappa2) == 2: + if (quark_kappa2[0] > param['quarks'][1]['mass']) or (quark_kappa2[1] < param['quarks'][1]['mass']): + return True + else: + raise ValueError("quark_kappa2 has to have length 2") + if 'qm1' in kwargs: + quark_mass1 = kwargs['qm1'] + if not isinstance(quark_mass1, list): + if (not np.isclose(quark_mass1, k2m(param['quarks'][0]['mass']))): + return True + else: + if len(quark_mass1) == 2: + if (quark_mass1[0] > k2m(param['quarks'][0]['mass'])) or (quark_mass1[1] < k2m(param['quarks'][0]['mass'])): + return True + else: + raise ValueError("quark_mass1 has to have length 2") + if 'qm2' in kwargs: + quark_mass2 = kwargs['qm2'] + if not isinstance(quark_mass2, list): + if (not np.isclose(quark_mass2, k2m(param['quarks'][1]['mass']))): + return True + else: + if len(quark_mass2) == 2: + if (quark_mass2[0] > k2m(param['quarks'][1]['mass'])) or (quark_mass2[1] < k2m(param['quarks'][1]['mass'])): + return True + else: + raise ValueError("quark_mass2 has to have length 2") + if 'quark_thetas' in kwargs: + quark_thetas = kwargs['quark_thetas'] + if (quark_thetas[0] != param['quarks'][0]['thetas'] and quark_thetas[1] != param['quarks'][1]['thetas']) or (quark_thetas[0] != param['quarks'][1]['thetas'] and quark_thetas[1] != param['quarks'][0]['thetas']): + return True + # careful, this is not save, when multiple contributions are present! + if 'wf1' in kwargs: + wf1 = kwargs['wf1'] + if not (np.isclose(wf1[0][0], param['wf1'][0][0], 1e-8) and np.isclose(wf1[0][1][0], param['wf1'][0][1][0], 1e-8) and np.isclose(wf1[0][1][1], param['wf1'][0][1][1], 1e-8)): + return True + if 'wf2' in kwargs: + wf2 = kwargs['wf2'] + if not (np.isclose(wf2[0][0], param['wf2'][0][0], 1e-8) and np.isclose(wf2[0][1][0], param['wf2'][0][1][0], 1e-8) and np.isclose(wf2[0][1][1], param['wf2'][0][1][1], 1e-8)): + return True + return False + + def sfcf_filter(results: pd.DataFrame, **kwargs: Any) -> pd.DataFrame: r""" Filter method for the Database entries holding SFCF calculations. @@ -191,84 +263,13 @@ def sfcf_filter(results: pd.DataFrame, **kwargs: Any) -> pd.DataFrame: results: pd.DataFrame The filtered DataFrame, only holding the records that fit to the parameters given. """ + drops = [] for ind in range(len(results)): result = results.iloc[ind] param = json.loads(result['parameters']) - if 'offset' in kwargs: - if kwargs.get('offset') != param['offset']: - drops.append(ind) - continue - if 'quark_kappas' in kwargs: - kappas = kwargs['quark_kappas'] - if (not np.isclose(kappas[0], param['quarks'][0]['mass']) or not np.isclose(kappas[1], param['quarks'][1]['mass'])): - drops.append(ind) - continue - if 'quark_masses' in kwargs: - masses = kwargs['quark_masses'] - if (not np.isclose(masses[0], k2m(param['quarks'][0]['mass'])) or not np.isclose(masses[1], k2m(param['quarks'][1]['mass']))): - drops.append(ind) - continue - if 'qk1' in kwargs: - quark_kappa1 = kwargs['qk1'] - if not isinstance(quark_kappa1, list): - if (not np.isclose(quark_kappa1, param['quarks'][0]['mass'])): - drops.append(ind) - continue - else: - if len(quark_kappa1) == 2: - if (quark_kappa1[0] > param['quarks'][0]['mass']) or (quark_kappa1[1] < param['quarks'][0]['mass']): - drops.append(ind) - continue - if 'qk2' in kwargs: - quark_kappa2 = kwargs['qk2'] - if not isinstance(quark_kappa2, list): - if (not np.isclose(quark_kappa2, param['quarks'][1]['mass'])): - drops.append(ind) - continue - else: - if len(quark_kappa2) == 2: - if (quark_kappa2[0] > param['quarks'][1]['mass']) or (quark_kappa2[1] < param['quarks'][1]['mass']): - drops.append(ind) - continue - if 'qm1' in kwargs: - quark_mass1 = kwargs['qm1'] - if not isinstance(quark_mass1, list): - if (not np.isclose(quark_mass1, k2m(param['quarks'][0]['mass']))): - drops.append(ind) - continue - else: - if len(quark_mass1) == 2: - if (quark_mass1[0] > k2m(param['quarks'][0]['mass'])) or (quark_mass1[1] < k2m(param['quarks'][0]['mass'])): - drops.append(ind) - continue - if 'qm2' in kwargs: - quark_mass2 = kwargs['qm2'] - if not isinstance(quark_mass2, list): - if (not np.isclose(quark_mass2, k2m(param['quarks'][1]['mass']))): - drops.append(ind) - continue - else: - if len(quark_mass2) == 2: - if (quark_mass2[0] > k2m(param['quarks'][1]['mass'])) or (quark_mass2[1] < k2m(param['quarks'][1]['mass'])): - drops.append(ind) - continue - if 'quark_thetas' in kwargs: - quark_thetas = kwargs['quark_thetas'] - if (quark_thetas[0] != param['quarks'][0]['thetas'] and quark_thetas[1] != param['quarks'][1]['thetas']) or (quark_thetas[0] != param['quarks'][1]['thetas'] and quark_thetas[1] != param['quarks'][0]['thetas']): - drops.append(ind) - continue - # careful, this is not save, when multiple contributions are present! - if 'wf1' in kwargs: - wf1 = kwargs['wf1'] - if not (np.isclose(wf1[0][0], param['wf1'][0][0], 1e-8) and np.isclose(wf1[0][1][0], param['wf1'][0][1][0], 1e-8) and np.isclose(wf1[0][1][1], param['wf1'][0][1][1], 1e-8)): - drops.append(ind) - continue - if 'wf2' in kwargs: - wf2 = kwargs['wf2'] - if not (np.isclose(wf2[0][0], param['wf2'][0][0], 1e-8) and np.isclose(wf2[0][1][0], param['wf2'][0][1][0], 1e-8) and np.isclose(wf2[0][1][1], param['wf2'][0][1][1], 1e-8)): - drops.append(ind) - continue + if _sfcf_drop(param, **kwargs): + drops.append(ind) return results.drop(drops) diff --git a/tests/find_test.py b/tests/find_test.py index 156e5fe..36d687e 100644 --- a/tests/find_test.py +++ b/tests/find_test.py @@ -211,62 +211,51 @@ def test_db_lookup(tmp_path: Path) -> None: conn.close() -def test_sfcf_filter() -> None: - record_0 = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", - '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] - record_1 = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", - '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] - record_2 = ["f_P", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", - '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] - record_3 = ["f_P", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", - '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] - data = [ - record_0, - record_1, - record_2, - record_3, - ] - cols = ["name", - "ensemble", - "code", - "path", - "project", - "parameters", - "parameter_file", - "created_at", - "updated_at"] - df = pd.DataFrame(data,columns=cols) +def test_sfcf_drop() -> None: + parameters0 = { + 'offset': [0,0,0], + 'quarks': [{'mass': 1, 'thetas': [0,0,0]}, {'mass': 2, 'thetas': [0,0,1]}], # m0s = -3.5, -3.75 + 'wf1': [[1, [0, 0]], [0.5, [1, 0]], [.75, [.5, .5]]], + 'wf2': [[1, [2, 1]], [2, [0.5, -0.5]], [.5, [.75, .72]]], + } - assert True + assert not find._sfcf_drop(parameters0, offset=[0,0,0]) + assert find._sfcf_drop(parameters0, offset=[1,0,0]) + + assert not find._sfcf_drop(parameters0, quark_kappas = [1, 2]) + assert find._sfcf_drop(parameters0, quark_kappas = [-3.1, -3.72]) + + assert not find._sfcf_drop(parameters0, quark_masses = [-3.5, -3.75]) + assert find._sfcf_drop(parameters0, quark_masses = [-3.1, -3.72]) + + assert not find._sfcf_drop(parameters0, qk1 = 1) + assert not find._sfcf_drop(parameters0, qk2 = 2) + assert find._sfcf_drop(parameters0, qk1 = 2) + assert find._sfcf_drop(parameters0, qk2 = 1) + + assert not find._sfcf_drop(parameters0, qk1 = [0.5,1.5]) + assert not find._sfcf_drop(parameters0, qk2 = [1.5,2.5]) + assert find._sfcf_drop(parameters0, qk1 = 2) + assert find._sfcf_drop(parameters0, qk2 = 1) + with pytest.raises(ValueError): + assert not find._sfcf_drop(parameters0, qk1 = [0.5,1,5]) + with pytest.raises(ValueError): + assert not find._sfcf_drop(parameters0, qk2 = [1,5,2.5]) + + assert find._sfcf_drop(parameters0, qm1 = 1.2) + assert find._sfcf_drop(parameters0, qm2 = 2.2) + assert not find._sfcf_drop(parameters0, qm1 = -3.5) + assert not find._sfcf_drop(parameters0, qm2 = -3.75) + + assert find._sfcf_drop(parameters0, qm2 = 1.2) + assert find._sfcf_drop(parameters0, qm1 = 2.2) + with pytest.raises(ValueError): + assert not find._sfcf_drop(parameters0, qm1 = [0.5,1,5]) + with pytest.raises(ValueError): + assert not find._sfcf_drop(parameters0, qm2 = [1,5,2.5]) def test_openQCD_filter() -> None: - record_0 = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", - '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] - record_1 = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", - '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] - record_2 = ["f_P", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", - '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] - record_3 = ["f_P", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", - '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] - data = [ - record_0, - record_1, - record_2, - record_3, - ] - cols = ["name", - "ensemble", - "code", - "path", - "project", - "parameters", - "parameter_file", - "created_at", - "updated_at"] - df = pd.DataFrame(data,columns=cols) - - find.openQCD_filter(df) assert True From 6e886aa06d638a9af4b32f6351d0534e65f70c99 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Fri, 10 Apr 2026 09:16:46 +0200 Subject: [PATCH 39/85] add counter for measurements in tomls --- corrlib/toml.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/corrlib/toml.py b/corrlib/toml.py index eb40d5a..e2e257d 100644 --- a/corrlib/toml.py +++ b/corrlib/toml.py @@ -178,8 +178,10 @@ def import_toml(path: Path, file: str, copy_file: bool=True) -> None: update_aliases(path, uuid, aliases) else: uuid = import_project(path, project['url'], aliases=aliases) + imeas = 1 + nmeas = len(measurements.keys()) for mname, md in measurements.items(): - print("Import measurement: " + mname) + print(f"Import measurement {imeas}/{nmeas}: {mname}") ensemble = md['ensemble'] if project['code'] == 'sfcf': param = sfcf.read_param(path, uuid, md['param_file']) @@ -232,6 +234,7 @@ def import_toml(path: Path, file: str, copy_file: bool=True) -> None: measurement = openQCD.extract_t1(path, uuid, md['path'], param, str(md["prefix"]), int(md["dtr_read"]), int(md["xmin"]), int(md["spatial_extent"]), fit_range=int(md.get('fit_range', 5)), postfix=str(md.get('postfix', '')), names=md.get('names', []), files=md.get('files', [])) write_measurement(path, ensemble, measurement, uuid, project['code'], (md['param_file'] if 'param_file' in md else None)) + imeas += 1 print(mname + " imported.") if not os.path.exists(path / "toml_imports" / uuid): From 74d99f8d5f4a9f24339bf04cf9bdac1e37a3d6d8 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Fri, 10 Apr 2026 09:38:31 +0200 Subject: [PATCH 40/85] fix mypy issue --- corrlib/toml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/corrlib/toml.py b/corrlib/toml.py index e2e257d..0d4dfc8 100644 --- a/corrlib/toml.py +++ b/corrlib/toml.py @@ -202,7 +202,7 @@ def import_toml(path: Path, file: str, copy_file: bool=True) -> None: param = openQCD.load_ms1_parfile(path, uuid, parameter_file) else: # Temporary solution - parameters = {} + parameters: dict[str, Any] = {} parameters["rand"] = {} parameters["rw_fcts"] = [{}] for nrw in range(1): From 6d1f8f7f1baa7efc26b0964af041af97347cb491 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Fri, 10 Apr 2026 10:28:28 +0200 Subject: [PATCH 41/85] add NotImplemented warning for openQCD filter --- corrlib/find.py | 5 ++++- tests/find_test.py | 28 +++++++++++++++++++++++++++- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/corrlib/find.py b/corrlib/find.py index 9d07a1c..1c985e2 100644 --- a/corrlib/find.py +++ b/corrlib/find.py @@ -11,6 +11,7 @@ from typing import Any, Optional, Union from pathlib import Path import datetime as dt from collections.abc import Callable +import warnings def _project_lookup_by_alias(db: Path, alias: str) -> str: @@ -162,7 +163,7 @@ def _db_lookup(db: Path, ensemble: str, correlator_name: str, code: str, project return results -def _sfcf_drop(param, **kwargs): +def _sfcf_drop(param: dict[str, Any], **kwargs: Any) -> bool: if 'offset' in kwargs: if kwargs.get('offset') != param['offset']: return True @@ -288,6 +289,8 @@ def openQCD_filter(results:pd.DataFrame, **kwargs: Any) -> pd.DataFrame: The filtered results. """ + warnings.warn("A filter for openQCD parameters is no implemented yet.", Warning) + return results diff --git a/tests/find_test.py b/tests/find_test.py index 36d687e..f512f15 100644 --- a/tests/find_test.py +++ b/tests/find_test.py @@ -256,7 +256,33 @@ def test_sfcf_drop() -> None: def test_openQCD_filter() -> None: - assert True + record_0 = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] + record_1 = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] + record_2 = ["f_P", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] + record_3 = ["f_P", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] + data = [ + record_0, + record_1, + record_2, + record_3, + ] + cols = ["name", + "ensemble", + "code", + "path", + "project", + "parameters", + "parameter_file", + "created_at", + "updated_at"] + df = pd.DataFrame(data,columns=cols) + + with pytest.warns(Warning): + find.openQCD_filter(df, a = "asdf") def test_code_filter() -> None: From 91938c3c5a3f590ad48d471e0a19a8702ba94349 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Tue, 14 Apr 2026 14:17:41 +0200 Subject: [PATCH 42/85] add second time integrity check --- corrlib/integrity.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/corrlib/integrity.py b/corrlib/integrity.py index bf890db..f1459d0 100644 --- a/corrlib/integrity.py +++ b/corrlib/integrity.py @@ -2,4 +2,9 @@ import datetime as dt def check_time_validity(created_at: dt.datetime, updated_at: dt.datetime) -> bool: - return not (created_at > updated_at) + # we expect created_at <= updated_at <= now + if created_at > updated_at: + return False + if updated_at > dt.datetime.now(): + return False + return True From 0b8c041ee559af903d6aa1526ed1a59753ab775d Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Tue, 14 Apr 2026 15:34:05 +0200 Subject: [PATCH 43/85] add wrapper functions to check for the validity of the database --- corrlib/integrity.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/corrlib/integrity.py b/corrlib/integrity.py index f1459d0..db242f6 100644 --- a/corrlib/integrity.py +++ b/corrlib/integrity.py @@ -1,10 +1,32 @@ import datetime as dt +from pathlib import Path +from .tools import get_db_file +import pandas as pd +import sqlite3 -def check_time_validity(created_at: dt.datetime, updated_at: dt.datetime) -> bool: +def has_valid_times(result: pd.DataFrame) -> bool: # we expect created_at <= updated_at <= now + created_at = dt.datetime.fromisoformat(result['created_at']) + updated_at = dt.datetime.fromisoformat(result['updated_at']) if created_at > updated_at: return False if updated_at > dt.datetime.now(): return False return True + + +def check_db_integrity(path: Path) -> None: + db = get_db_file(path) + search_expr = "SELECT * FROM 'backlogs'" + conn = sqlite3.connect(db) + results = pd.read_sql(search_expr, conn) + + for result in results: + if not has_valid_times(result): + raise ValueError(f"Result with id {result[id]} has wrong time signatures.") + + +def full_integrity_check(path: Path) -> None: + check_db_integrity(path) + From 65cd55ec0a8d2afbe5a54159cff393d80da466bd Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Tue, 14 Apr 2026 16:36:31 +0200 Subject: [PATCH 44/85] add test on whether paths are indeed unique --- corrlib/integrity.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/corrlib/integrity.py b/corrlib/integrity.py index db242f6..70e4694 100644 --- a/corrlib/integrity.py +++ b/corrlib/integrity.py @@ -15,6 +15,14 @@ def has_valid_times(result: pd.DataFrame) -> bool: return False return True +def are_keys_unique(db: Path, table: str, col: str) -> bool: + conn = sqlite3.connect(db) + c = conn.cursor() + c.execute(f"SELECT COUNT( DISTINCT CAST(path AS nvarchar(4000))), COUNT({col}) FROM {table};") + results = c.fetchall()[0] + conn.close() + return bool(results[0] == results[1]) + def check_db_integrity(path: Path) -> None: db = get_db_file(path) @@ -27,6 +35,7 @@ def check_db_integrity(path: Path) -> None: raise ValueError(f"Result with id {result[id]} has wrong time signatures.") + def full_integrity_check(path: Path) -> None: check_db_integrity(path) From 85698c377bca7405d69c63d13d3ef918d35aaf1a Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Tue, 14 Apr 2026 16:42:39 +0200 Subject: [PATCH 45/85] use uniqueness for complete db check --- corrlib/integrity.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/corrlib/integrity.py b/corrlib/integrity.py index 70e4694..8a414bf 100644 --- a/corrlib/integrity.py +++ b/corrlib/integrity.py @@ -26,6 +26,10 @@ def are_keys_unique(db: Path, table: str, col: str) -> bool: def check_db_integrity(path: Path) -> None: db = get_db_file(path) + + if not are_keys_unique(db, 'backlogs', 'path'): + raise Exception("The paths the backlog table of the database links are not unique.") + search_expr = "SELECT * FROM 'backlogs'" conn = sqlite3.connect(db) results = pd.read_sql(search_expr, conn) From d8bb9e4080017070bc928d19fadd7e175abeebf0 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Wed, 15 Apr 2026 10:49:03 +0200 Subject: [PATCH 46/85] fix import --- corrlib/find.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/corrlib/find.py b/corrlib/find.py index 1c985e2..7b07321 100644 --- a/corrlib/find.py +++ b/corrlib/find.py @@ -6,8 +6,8 @@ import numpy as np from .input.implementations import codes from .tools import k2m, get_db_file from .tracker import get -from .integrity import check_time_validity -from typing import Any, Optional, Union +from .integrity import has_valid_times +from typing import Any, Optional from pathlib import Path import datetime as dt from collections.abc import Callable @@ -88,7 +88,7 @@ def _time_filter(results: pd.DataFrame, created_before: Optional[str]=None, cre result = results.iloc[ind] created_at = dt.datetime.fromisoformat(result['created_at']) updated_at = dt.datetime.fromisoformat(result['updated_at']) - db_times_valid = check_time_validity(created_at=created_at, updated_at=updated_at) + db_times_valid = has_valid_times(result) if not db_times_valid: raise ValueError('Time stamps not valid for result with path', result["path"]) From dc424c3e18ecdeeda834865b05dc9bfac6e41e5a Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Wed, 15 Apr 2026 11:24:25 +0200 Subject: [PATCH 47/85] fix time tests --- tests/find_test.py | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/tests/find_test.py b/tests/find_test.py index f512f15..cc455f9 100644 --- a/tests/find_test.py +++ b/tests/find_test.py @@ -5,6 +5,7 @@ import corrlib.initialization as cinit import pytest import pandas as pd import datalad.api as dl +import datetime as dt def make_sql(path: Path) -> Path: @@ -57,18 +58,20 @@ def test_find_lookup_by_id(tmp_path: Path) -> None: def test_time_filter() -> None: - record_A = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + record_A = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf0", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] # only created - record_B = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + record_B = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf1", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", '2025-03-26 12:55:18.229966', '2025-04-26 12:55:18.229966'] # created and updated - record_C = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", - '2026-03-26 12:55:18.229966', '2026-05-26 12:55:18.229966'] # created and updated later - record_D = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + record_C = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf2", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2026-03-26 12:55:18.229966', '2026-04-14 12:55:18.229966'] # created and updated later + record_D = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf3", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", '2026-03-26 12:55:18.229966', '2026-03-27 12:55:18.229966'] - record_E = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + record_E = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf4", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", '2024-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # only created, earlier - record_F = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + record_F = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf5", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", '2026-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # this is invalid... + record_G = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf2", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2026-03-26 12:55:18.229966', str(dt.datetime.now() + dt.timedelta(days=2, hours=3, minutes=5, seconds=30))] # created and updated later data = [record_A, record_B, record_C, record_D, record_E] cols = ["name", @@ -141,6 +144,21 @@ def test_time_filter() -> None: with pytest.raises(ValueError): results = find._time_filter(df, created_before='2023-03-26 12:55:18.229966') + data = [record_A, record_B, record_C, record_D, record_G] + cols = ["name", + "ensemble", + "code", + "path", + "project", + "parameters", + "parameter_file", + "created_at", + "updated_at"] + df = pd.DataFrame(data,columns=cols) + + with pytest.raises(ValueError): + results = find._time_filter(df, created_before='2023-03-26 12:55:18.229966') + def test_db_lookup(tmp_path: Path) -> None: db = make_sql(tmp_path) From b625bf92438ba3fcae0729bddd57554f68275fdd Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Wed, 15 Apr 2026 12:02:03 +0200 Subject: [PATCH 48/85] proper row interation --- corrlib/integrity.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/corrlib/integrity.py b/corrlib/integrity.py index 8a414bf..d865944 100644 --- a/corrlib/integrity.py +++ b/corrlib/integrity.py @@ -5,7 +5,7 @@ import pandas as pd import sqlite3 -def has_valid_times(result: pd.DataFrame) -> bool: +def has_valid_times(result: pd.Series) -> bool: # we expect created_at <= updated_at <= now created_at = dt.datetime.fromisoformat(result['created_at']) updated_at = dt.datetime.fromisoformat(result['updated_at']) @@ -34,7 +34,7 @@ def check_db_integrity(path: Path) -> None: conn = sqlite3.connect(db) results = pd.read_sql(search_expr, conn) - for result in results: + for _, result in results.iterrows(): if not has_valid_times(result): raise ValueError(f"Result with id {result[id]} has wrong time signatures.") From 0b1ff3cbad724456e6c77a5a52dd521bb6ac5ffe Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Fri, 17 Apr 2026 16:24:31 +0200 Subject: [PATCH 49/85] prepare implementation --- corrlib/cli.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/corrlib/cli.py b/corrlib/cli.py index 6c1c3c5..2d1a9ee 100644 --- a/corrlib/cli.py +++ b/corrlib/cli.py @@ -1,6 +1,7 @@ from typing import Optional import typer from corrlib import __app_name__ + from .initialization import create from .toml import import_tomls, update_project, reimport_project from .find import find_record, list_projects @@ -8,6 +9,8 @@ from .tools import str2list from .main import update_aliases from .meas_io import drop_cache as mio_drop_cache from .meas_io import load_record as mio_load_record +# from .integrity import + import os from pyerrors import Corr from importlib.metadata import version @@ -137,6 +140,23 @@ def stat( return +@app.command() +def check(path: Path = typer.Option( + Path('./corrlib'), + "--dataset", + "-d", + ), + files: str = typer.Argument( + ), + copy_file: bool = typer.Option( + bool(True), + "--save", + "-s", + ),) -> None: + + "✅" : "❌" + + @app.command() def importer( path: Path = typer.Option( From 23b5d066f7b4e4733629b07786fd1ffbb117efcd Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Fri, 17 Apr 2026 16:34:30 +0200 Subject: [PATCH 50/85] make integrity checks accassible from cli --- corrlib/cli.py | 13 +++---------- corrlib/integrity.py | 8 +++++--- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/corrlib/cli.py b/corrlib/cli.py index 2d1a9ee..d24d8ef 100644 --- a/corrlib/cli.py +++ b/corrlib/cli.py @@ -9,7 +9,7 @@ from .tools import str2list from .main import update_aliases from .meas_io import drop_cache as mio_drop_cache from .meas_io import load_record as mio_load_record -# from .integrity import +from .integrity import full_integrity_check import os from pyerrors import Corr @@ -146,15 +146,8 @@ def check(path: Path = typer.Option( "--dataset", "-d", ), - files: str = typer.Argument( - ), - copy_file: bool = typer.Option( - bool(True), - "--save", - "-s", - ),) -> None: - - "✅" : "❌" + ) -> None: + full_integrity_check(path) @app.command() diff --git a/corrlib/integrity.py b/corrlib/integrity.py index d865944..dc1216c 100644 --- a/corrlib/integrity.py +++ b/corrlib/integrity.py @@ -27,19 +27,21 @@ def are_keys_unique(db: Path, table: str, col: str) -> bool: def check_db_integrity(path: Path) -> None: db = get_db_file(path) - if not are_keys_unique(db, 'backlogs', 'path'): + if not are_keys_unique(path / db, 'backlogs', 'path'): raise Exception("The paths the backlog table of the database links are not unique.") search_expr = "SELECT * FROM 'backlogs'" - conn = sqlite3.connect(db) + conn = sqlite3.connect(path / db) results = pd.read_sql(search_expr, conn) for _, result in results.iterrows(): if not has_valid_times(result): raise ValueError(f"Result with id {result[id]} has wrong time signatures.") - + print("DB:\t✅") def full_integrity_check(path: Path) -> None: check_db_integrity(path) + print("Full:\t✅") + From b13136a248f294eb2275da54747cc105473655f3 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Fri, 17 Apr 2026 17:32:22 +0200 Subject: [PATCH 51/85] add check for links to files --- corrlib/integrity.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/corrlib/integrity.py b/corrlib/integrity.py index dc1216c..63572a9 100644 --- a/corrlib/integrity.py +++ b/corrlib/integrity.py @@ -3,6 +3,8 @@ from pathlib import Path from .tools import get_db_file import pandas as pd import sqlite3 +from .tracker import get +import pyerrors.input.json as pj def has_valid_times(result: pd.Series) -> bool: @@ -38,10 +40,45 @@ def check_db_integrity(path: Path) -> None: if not has_valid_times(result): raise ValueError(f"Result with id {result[id]} has wrong time signatures.") print("DB:\t✅") + return + + +def _check_db2paths(path: Path, meas_paths: list[str]) -> None: + needed_data: dict[str, list[str]] = {} + for mpath in meas_paths: + file = mpath.split("::")[0] + if file not in needed_data.keys(): + needed_data[file] = [] + key = mpath.split("::")[1] + needed_data[file].append(key) + + for file in needed_data.keys(): + get(path, Path(file)) + filedict: dict[str, Any] = pj.load_json_dict(str(path / file)) + if not set(filedict.keys()).issubset(needed_data[file]): + for key in filedict.keys(): + if key not in needed_data[file]: + raise ValueError(f"Found unintended key {key} in file {file}.") + elif not set(needed_data[file]).issubset(filedict.keys()): + for key in needed_data[file]: + if key not in filedict.keys(): + raise ValueError(f"Did not find data for key {key} that should be in file {file}.") + print("Links:\t✅") + return + + +def check_db_file_links(path: Path) -> None: + db = get_db_file(path) + search_expr = "SELECT path FROM 'backlogs'" + conn = sqlite3.connect(path / db) + results = pd.read_sql(search_expr, conn)['path'].values + print(results) + _check_db2paths(path, results) def full_integrity_check(path: Path) -> None: check_db_integrity(path) + check_db_file_links(path) print("Full:\t✅") From 29ebafc1c44e5e0fe30dcb38fdf22d408f746cec Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Fri, 17 Apr 2026 17:34:53 +0200 Subject: [PATCH 52/85] show progress a little --- corrlib/integrity.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/corrlib/integrity.py b/corrlib/integrity.py index 63572a9..8722840 100644 --- a/corrlib/integrity.py +++ b/corrlib/integrity.py @@ -52,7 +52,9 @@ def _check_db2paths(path: Path, meas_paths: list[str]) -> None: key = mpath.split("::")[1] needed_data[file].append(key) - for file in needed_data.keys(): + totf = len(needed_data.keys()) + for i, file in enumerate(needed_data.keys()): + print(f"Check against file {i}/{totf}: {file}") get(path, Path(file)) filedict: dict[str, Any] = pj.load_json_dict(str(path / file)) if not set(filedict.keys()).issubset(needed_data[file]): From 37ae8185897b62017f5822ec0b727685a905a389 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Fri, 17 Apr 2026 17:37:46 +0200 Subject: [PATCH 53/85] small logic issue --- corrlib/integrity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/corrlib/integrity.py b/corrlib/integrity.py index 8722840..23fbe0e 100644 --- a/corrlib/integrity.py +++ b/corrlib/integrity.py @@ -61,7 +61,7 @@ def _check_db2paths(path: Path, meas_paths: list[str]) -> None: for key in filedict.keys(): if key not in needed_data[file]: raise ValueError(f"Found unintended key {key} in file {file}.") - elif not set(needed_data[file]).issubset(filedict.keys()): + if not set(needed_data[file]).issubset(filedict.keys()): for key in needed_data[file]: if key not in filedict.keys(): raise ValueError(f"Did not find data for key {key} that should be in file {file}.") From 0535e19bf08ebced63834ff5796591a815ae20f0 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Fri, 17 Apr 2026 17:42:47 +0200 Subject: [PATCH 54/85] fix typing --- corrlib/integrity.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/corrlib/integrity.py b/corrlib/integrity.py index 23fbe0e..5f80aa3 100644 --- a/corrlib/integrity.py +++ b/corrlib/integrity.py @@ -6,6 +6,8 @@ import sqlite3 from .tracker import get import pyerrors.input.json as pj +from typing import Any + def has_valid_times(result: pd.Series) -> bool: # we expect created_at <= updated_at <= now @@ -74,8 +76,7 @@ def check_db_file_links(path: Path) -> None: search_expr = "SELECT path FROM 'backlogs'" conn = sqlite3.connect(path / db) results = pd.read_sql(search_expr, conn)['path'].values - print(results) - _check_db2paths(path, results) + _check_db2paths(path, list(results)) def full_integrity_check(path: Path) -> None: From 083d7ee3ce6ec2868a2da1b9c4fd73ef22362f50 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Fri, 17 Apr 2026 17:52:18 +0200 Subject: [PATCH 55/85] add dry run for loading data using the integrity functions --- corrlib/meas_io.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/corrlib/meas_io.py b/corrlib/meas_io.py index f4e8a83..731da66 100644 --- a/corrlib/meas_io.py +++ b/corrlib/meas_io.py @@ -11,6 +11,7 @@ from .tracker import get, save, unlock import shutil from typing import Any from pathlib import Path +from .integrity import _check_db2paths CACHE_DIR = ".cache" @@ -153,7 +154,7 @@ def load_record(path: Path, meas_path: str) -> Union[Corr, Obs]: return load_records(path, [meas_path])[0] -def load_records(path: Path, meas_paths: list[str], preloaded: dict[str, Any] = {}) -> list[Union[Corr, Obs]]: +def load_records(path: Path, meas_paths: list[str], preloaded: dict[str, Any] = {}, dry_run: bool = False) -> list[Union[Corr, Obs]]: """ Load a list of records by their paths. @@ -163,14 +164,19 @@ def load_records(path: Path, meas_paths: list[str], preloaded: dict[str, Any] = Path of the correlator library. meas_paths: list[str] A list of the paths to the correlator in the backlog system. - perloaded: dict[str, Any] - The data that is already prelaoded. Of interest if data has alread been loaded in the same script. + preloaded: dict[str, Any] + The data that is already preloaded. Of interest if data has alread been loaded in the same script. + dry_run: bool + Do not load datda, just check whether we can reach the data we are interested in. Returns ------- - retruned_data: list + returned_data: list The loaded records. """ + if dry_run: + _check_db2paths(path, meas_paths) + return needed_data: dict[str, list[str]] = {} for mpath in meas_paths: file = mpath.split("::")[0] From 51ae53aa024365f47436c11bf69bc376184ac6b4 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Fri, 17 Apr 2026 17:53:13 +0200 Subject: [PATCH 56/85] add empty return --- corrlib/meas_io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/corrlib/meas_io.py b/corrlib/meas_io.py index 731da66..cbd9386 100644 --- a/corrlib/meas_io.py +++ b/corrlib/meas_io.py @@ -176,7 +176,7 @@ def load_records(path: Path, meas_paths: list[str], preloaded: dict[str, Any] = """ if dry_run: _check_db2paths(path, meas_paths) - return + return [] needed_data: dict[str, list[str]] = {} for mpath in meas_paths: file = mpath.split("::")[0] From d6de8e6387f9bd6aea96082b122c5f1ad2a848d5 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Tue, 21 Apr 2026 10:22:46 +0200 Subject: [PATCH 57/85] Introduce thin wrapper for SQL calls --- corrlib/find.py | 21 ++++++++------------- corrlib/sql.py | 17 +++++++++++++++++ tests/find_test.py | 13 ++++++++++--- 3 files changed, 35 insertions(+), 16 deletions(-) create mode 100644 corrlib/sql.py diff --git a/corrlib/find.py b/corrlib/find.py index 7b07321..1e6b4bf 100644 --- a/corrlib/find.py +++ b/corrlib/find.py @@ -7,6 +7,7 @@ from .input.implementations import codes from .tools import k2m, get_db_file from .tracker import get from .integrity import has_valid_times +from .sql import thin_sql_wrapper from typing import Any, Optional from pathlib import Path import datetime as dt @@ -14,7 +15,7 @@ from collections.abc import Callable import warnings -def _project_lookup_by_alias(db: Path, alias: str) -> str: +def _project_lookup_by_alias(path: Path, alias: str) -> str: """ Lookup a projects UUID by its (human-readable) alias. @@ -30,11 +31,8 @@ def _project_lookup_by_alias(db: Path, alias: str) -> str: uuid: str The UUID of the project with the given alias. """ - conn = sqlite3.connect(db) - c = conn.cursor() - c.execute(f"SELECT * FROM 'projects' WHERE aliases = '{alias}'") - results = c.fetchall() - conn.close() + stmt = f"SELECT * FROM 'projects' WHERE aliases = '{alias}'" + results = thin_sql_wrapper(path, stmt) if len(results)>1: print("Error: multiple projects found with alias " + alias) elif len(results) == 0: @@ -42,7 +40,7 @@ def _project_lookup_by_alias(db: Path, alias: str) -> str: return str(results[0][0]) -def _project_lookup_by_id(db: Path, uuid: str) -> list[tuple[str, ...]]: +def _project_lookup_by_id(path: Path, uuid: str) -> list[tuple[str, ...]]: """ Return the project information available in the database by UUID. @@ -58,11 +56,8 @@ def _project_lookup_by_id(db: Path, uuid: str) -> list[tuple[str, ...]]: results: list The row of the project in the database. """ - conn = sqlite3.connect(db) - c = conn.cursor() - c.execute(f"SELECT * FROM 'projects' WHERE id = '{uuid}'") - results = c.fetchall() - conn.close() + stmt = f"SELECT * FROM 'projects' WHERE id = '{uuid}'" + results = thin_sql_wrapper(path, stmt) return results @@ -360,7 +355,7 @@ def find_project(path: Path, name: str) -> str: """ db_file = get_db_file(path) get(path, db_file) - return _project_lookup_by_alias(path / db_file, name) + return _project_lookup_by_alias(path, name) def list_projects(path: Path) -> list[tuple[str, str]]: diff --git a/corrlib/sql.py b/corrlib/sql.py new file mode 100644 index 0000000..f45ce31 --- /dev/null +++ b/corrlib/sql.py @@ -0,0 +1,17 @@ +import sqlite3 +from .tools import get_db_file +from pathlib import Path +from typing import Any + + +def thin_sql_wrapper(path: Path, stmt: str) -> list[Any]: + db_file = get_db_file(path) + db = path / db_file + conn = sqlite3.connect(db) + c = conn.cursor() + + c.execute(stmt) + results = c.fetchall() + conn.commit() + conn.close() + return results diff --git a/tests/find_test.py b/tests/find_test.py index cc455f9..2144001 100644 --- a/tests/find_test.py +++ b/tests/find_test.py @@ -9,12 +9,17 @@ import datetime as dt def make_sql(path: Path) -> Path: - db = path / "test.db" + db = path / "backlogger.db" cinit._create_db(db) return db +def make_config(path: Path) -> None: + cinit._write_config(path, cinit._create_config(path, "datalad", False)) + + def test_find_lookup_by_one_alias(tmp_path: Path) -> None: + make_config(tmp_path) db = make_sql(tmp_path) conn = sqlite3.connect(db) c = conn.cursor() @@ -26,7 +31,7 @@ def test_find_lookup_by_one_alias(tmp_path: Path) -> None: c.execute("INSERT INTO projects (id, aliases, customTags, owner, code, created_at, updated_at) VALUES (?, ?, ?, ?, ?, datetime('now'), datetime('now'))", (uuid, alias_str, tag_str, owner, code)) conn.commit() - assert uuid == find._project_lookup_by_alias(db, "fun_project") + assert uuid == find._project_lookup_by_alias(tmp_path, "fun_project") uuid = "test_uuid2" alias_str = "fun_project" c.execute("INSERT INTO projects (id, aliases, customTags, owner, code, created_at, updated_at) VALUES (?, ?, ?, ?, ?, datetime('now'), datetime('now'))", @@ -36,7 +41,9 @@ def test_find_lookup_by_one_alias(tmp_path: Path) -> None: assert uuid == find._project_lookup_by_alias(db, "fun_project") conn.close() + def test_find_lookup_by_id(tmp_path: Path) -> None: + make_config(tmp_path) db = make_sql(tmp_path) conn = sqlite3.connect(db) c = conn.cursor() @@ -49,7 +56,7 @@ def test_find_lookup_by_id(tmp_path: Path) -> None: (uuid, alias_str, tag_str, owner, code)) conn.commit() conn.close() - result = find._project_lookup_by_id(db, uuid)[0] + result = find._project_lookup_by_id(tmp_path, uuid)[0] assert uuid == result[0] assert alias_str == result[1] assert tag_str == result[2] From 1b338b3f6ca78b9c8c2bbdf03bbc906e9bc32c32 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Tue, 21 Apr 2026 16:15:41 +0200 Subject: [PATCH 58/85] add FileNotFound to db tests --- tests/tools_test.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/tools_test.py b/tests/tools_test.py index 541674f..917e1a1 100644 --- a/tests/tools_test.py +++ b/tests/tools_test.py @@ -69,6 +69,8 @@ def test_get_db_file(tmp_path: Path) -> None: # config is not yet available tl.set_config(tmp_path, section, option, value) assert tl.get_db_file(tmp_path) == Path("test_value") + with pytest.raises(FileNotFoundError): + tl.get_db_file(tmp_path / "doesnotexist") def test_cache_enabled(tmp_path: Path) -> None: @@ -82,3 +84,5 @@ def test_cache_enabled(tmp_path: Path) -> None: tl.set_config(tmp_path, section, option, "lalala") with pytest.raises(ValueError): tl.cache_enabled(tmp_path) + with pytest.raises(FileNotFoundError): + tl.cache_enabled(tmp_path / "doesnotexist") From 5649a0a38ca8be076e103ebc265e00f6c93d0d31 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Tue, 21 Apr 2026 16:21:20 +0200 Subject: [PATCH 59/85] use CONFIG_FILENAMR --- corrlib/initialization.py | 3 ++- corrlib/tracker.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/corrlib/initialization.py b/corrlib/initialization.py index c06a201..bdf9cee 100644 --- a/corrlib/initialization.py +++ b/corrlib/initialization.py @@ -3,6 +3,7 @@ import sqlite3 import os from .tracker import save, init from pathlib import Path +from .tools import CONFIG_FILENAME def _create_db(db: Path) -> None: @@ -87,7 +88,7 @@ def _write_config(path: Path, config: ConfigParser) -> None: config: ConfigParser The configuration to be used as a ConfigParser, e.g. generated by _create_config. """ - with open(os.path.join(path, '.corrlib'), 'w') as configfile: + with open(os.path.join(path, CONFIG_FILENAME), 'w') as configfile: config.write(configfile) return diff --git a/corrlib/tracker.py b/corrlib/tracker.py index a6e9bf4..1bae1b4 100644 --- a/corrlib/tracker.py +++ b/corrlib/tracker.py @@ -3,7 +3,7 @@ from configparser import ConfigParser import datalad.api as dl from typing import Optional import shutil -from .tools import get_db_file +from .tools import get_db_file, CONFIG_FILENAME from pathlib import Path @@ -21,7 +21,7 @@ def get_tracker(path: Path) -> str: tracker: str The tracker used in the dataset. """ - config_path = os.path.join(path, '.corrlib') + config_path = os.path.join(path, CONFIG_FILENAME) config = ConfigParser() if os.path.exists(config_path): config.read(config_path) From 4e3327709e73fab585b17004c19275c76e082f54 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Tue, 28 Apr 2026 11:27:49 +0200 Subject: [PATCH 60/85] HOTFIX: paths in update_aliases --- corrlib/main.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/corrlib/main.py b/corrlib/main.py index 831b69d..5df8165 100644 --- a/corrlib/main.py +++ b/corrlib/main.py @@ -27,7 +27,7 @@ def create_project(path: Path, uuid: str, owner: Union[str, None]=None, tags: Un The code that was used to create the measurements. """ db_file = get_db_file(path) - db = os.path.join(path, db_file) + db = path / db_file get(path, db_file) conn = sqlite3.connect(db) c = conn.cursor() @@ -67,7 +67,7 @@ def update_project_data(path: Path, uuid: str, prop: str, value: Union[str, None """ db_file = get_db_file(path) get(path, db_file) - conn = sqlite3.connect(os.path.join(path, db_file)) + conn = sqlite3.connect(path / db_file) c = conn.cursor() c.execute(f"UPDATE projects SET '{prop}' = '{value}' WHERE id == '{uuid}'") conn.commit() @@ -77,9 +77,8 @@ def update_project_data(path: Path, uuid: str, prop: str, value: Union[str, None def update_aliases(path: Path, uuid: str, aliases: list[str]) -> None: db_file = get_db_file(path) - db = path / db_file get(path, db_file) - known_data = _project_lookup_by_id(db, uuid)[0] + known_data = _project_lookup_by_id(path, uuid)[0] known_aliases = known_data[1] if aliases is None: From b3991ecc6760298d03a99af6e461b40ab043f231 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Thu, 30 Apr 2026 15:09:17 +0200 Subject: [PATCH 61/85] let cli drop cache when importing a new project, add stat flag for find --- corrlib/cli.py | 17 ++++++++++------- corrlib/find.py | 11 +++++++++++ 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/corrlib/cli.py b/corrlib/cli.py index d24d8ef..b4a42e0 100644 --- a/corrlib/cli.py +++ b/corrlib/cli.py @@ -1,14 +1,13 @@ -from typing import Optional +from typing import Optional, Any import typer from corrlib import __app_name__ from .initialization import create from .toml import import_tomls, update_project, reimport_project -from .find import find_record, list_projects +from .find import find_record, list_projects, get_stat from .tools import str2list from .main import update_aliases from .meas_io import drop_cache as mio_drop_cache -from .meas_io import load_record as mio_load_record from .integrity import full_integrity_check import os @@ -116,6 +115,11 @@ def find( if arg == 'all': print(results) else: + if arg == 'stat': + for r in results['path'].values: + stat = get_stat(path, r) + print(stat) + return for r in results[arg].values: print(r) @@ -132,10 +136,7 @@ def stat( """ Show the statistics of a given record. """ - record = mio_load_record(path, record_id) - if isinstance(record, (list, Corr)): - record = record[0] - statistics = record.idl + statistics = get_stat(path, record_id) print(statistics) return @@ -170,6 +171,7 @@ def importer( """ file_list = files.split(",") import_tomls(path, file_list, copy_file) + mio_drop_cache(path) return @@ -194,6 +196,7 @@ def reimporter( raise Exception("This file is not known for this project.") else: reimport_project(path, uuid) + mio_drop_cache(path) return diff --git a/corrlib/find.py b/corrlib/find.py index 1e6b4bf..738c832 100644 --- a/corrlib/find.py +++ b/corrlib/find.py @@ -13,6 +13,8 @@ from pathlib import Path import datetime as dt from collections.abc import Callable import warnings +from .meas_io import load_record +from pyerrors import Corr, Obs def _project_lookup_by_alias(path: Path, alias: str) -> str: @@ -381,3 +383,12 @@ def list_projects(path: Path) -> list[tuple[str, str]]: conn.close() return results + +def get_stat(path: Path, record_id: str) -> Any: + loaded_record: Obs = load_record(path, record_id) + if isinstance(loaded_record, (list, Corr)): + record: Obs = loaded_record[0] + else: + record = loaded_record + return record.idl + From 69348cd151110ccb99fc321470fcc2b8c61d9fd3 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Thu, 30 Apr 2026 15:11:46 +0200 Subject: [PATCH 62/85] fix ruff complaints --- corrlib/cli.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/corrlib/cli.py b/corrlib/cli.py index b4a42e0..3053bb3 100644 --- a/corrlib/cli.py +++ b/corrlib/cli.py @@ -1,4 +1,4 @@ -from typing import Optional, Any +from typing import Optional import typer from corrlib import __app_name__ @@ -11,7 +11,6 @@ from .meas_io import drop_cache as mio_drop_cache from .integrity import full_integrity_check import os -from pyerrors import Corr from importlib.metadata import version from pathlib import Path From 2f83c1f9cba1964360263fbf87778813892c6f70 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Tue, 5 May 2026 16:21:05 +0200 Subject: [PATCH 63/85] throw error if library path is not found in get_db_file --- corrlib/tools.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/corrlib/tools.py b/corrlib/tools.py index 93f0678..f66e35d 100644 --- a/corrlib/tools.py +++ b/corrlib/tools.py @@ -115,6 +115,8 @@ def get_db_file(path: Path) -> Path: db_file: str The file holding the database. """ + if not os.path.exists(path): + raise FileNotFoundError(f"Corrlib path {path} does not exist.") config_path = os.path.join(path, CONFIG_FILENAME) config = ConfigParser() if os.path.exists(config_path): From 6c99653fffcb105f50d107f49a4b3005156b11f3 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Tue, 5 May 2026 16:26:02 +0200 Subject: [PATCH 64/85] check whether paths exist for import --- corrlib/toml.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/corrlib/toml.py b/corrlib/toml.py index 0d4dfc8..29d7de2 100644 --- a/corrlib/toml.py +++ b/corrlib/toml.py @@ -158,6 +158,10 @@ def import_toml(path: Path, file: str, copy_file: bool=True) -> None: copy_file: bool, optional Whether the toml-files will be copied into the library. Default is True. """ + if not os.path.exists(path): + raise FileNotFoundError(f"Corrlib path {path} does not exist.") + if not os.path.exists(file): + raise FileNotFoundError(f".toml-file {file} does not exist.") print("Import project as decribed in " + file) with open(file, 'rb') as fp: toml_dict = toml.load(fp) From 93ca059fc0950d361bf2bd85e468fa8200236e90 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Tue, 5 May 2026 16:26:18 +0200 Subject: [PATCH 65/85] pathlib for concat --- corrlib/tools.py | 6 +++--- corrlib/tracker.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/corrlib/tools.py b/corrlib/tools.py index f66e35d..e75e1cc 100644 --- a/corrlib/tools.py +++ b/corrlib/tools.py @@ -89,7 +89,7 @@ def set_config(path: Path, section: str, option: str, value: Any) -> None: value: Any The value we set the option to. """ - config_path = os.path.join(path, CONFIG_FILENAME) + config_path = path / CONFIG_FILENAME config = ConfigParser() if os.path.exists(config_path): config.read(config_path) @@ -117,7 +117,7 @@ def get_db_file(path: Path) -> Path: """ if not os.path.exists(path): raise FileNotFoundError(f"Corrlib path {path} does not exist.") - config_path = os.path.join(path, CONFIG_FILENAME) + config_path = path / CONFIG_FILENAME config = ConfigParser() if os.path.exists(config_path): config.read(config_path) @@ -142,7 +142,7 @@ def cache_enabled(path: Path) -> bool: cached_bool: bool Whether the given library is cached. """ - config_path = os.path.join(path, CONFIG_FILENAME) + config_path = path / CONFIG_FILENAME config = ConfigParser() if os.path.exists(config_path): config.read(config_path) diff --git a/corrlib/tracker.py b/corrlib/tracker.py index 1bae1b4..98fdec5 100644 --- a/corrlib/tracker.py +++ b/corrlib/tracker.py @@ -21,7 +21,7 @@ def get_tracker(path: Path) -> str: tracker: str The tracker used in the dataset. """ - config_path = os.path.join(path, CONFIG_FILENAME) + config_path = path / CONFIG_FILENAME config = ConfigParser() if os.path.exists(config_path): config.read(config_path) From 656f99a13c2fd0c4fbad292f2a3c145afb567124 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Tue, 5 May 2026 16:47:07 +0200 Subject: [PATCH 66/85] add integrity check for the config-file --- corrlib/integrity.py | 41 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/corrlib/integrity.py b/corrlib/integrity.py index 5f80aa3..74386f4 100644 --- a/corrlib/integrity.py +++ b/corrlib/integrity.py @@ -1,10 +1,12 @@ import datetime as dt from pathlib import Path -from .tools import get_db_file +from .tools import get_db_file, CONFIG_FILENAME import pandas as pd import sqlite3 from .tracker import get import pyerrors.input.json as pj +import os +from configparser import ConfigParser from typing import Any @@ -41,7 +43,6 @@ def check_db_integrity(path: Path) -> None: for _, result in results.iterrows(): if not has_valid_times(result): raise ValueError(f"Result with id {result[id]} has wrong time signatures.") - print("DB:\t✅") return @@ -67,7 +68,6 @@ def _check_db2paths(path: Path, meas_paths: list[str]) -> None: for key in needed_data[file]: if key not in filedict.keys(): raise ValueError(f"Did not find data for key {key} that should be in file {file}.") - print("Links:\t✅") return @@ -79,9 +79,44 @@ def check_db_file_links(path: Path) -> None: _check_db2paths(path, list(results)) +def check_path_and_config(path: Path) -> None: + if not os.path.exists(path): + raise FileNotFoundError(f"Corrlib path {path} does not exist.") + config_path = path / CONFIG_FILENAME + if not os.path.exists(config_path): + raise FileNotFoundError(f"Configuration file {config_path} not found.") + + +def check_config_validity(path: Path) -> None: + config = ConfigParser() + config_path = path / CONFIG_FILENAME + if os.path.exists(config_path): + config.read(config_path) + else: + raise FileNotFoundError("Configuration file not found.") + + if config.has_section('core'): + core_opts = ['version', 'tracker', 'cached'] + has_core_opts = [config.has_option('core', opt) for opt in core_opts] + if not all(has_core_opts): + raise ValueError("One of the options in the 'core' section ('version', 'tracker', 'cached') is missing.") + + if config.has_section('paths'): + path_opts = ['db', 'projects_path', 'archive_path', 'toml_imports_path', 'import_scripts_path'] + has_path_opts = [config.has_option('paths', opt) for opt in path_opts] + if not all(has_path_opts): + raise ValueError("One of the options in the 'path' section ('db', 'projects_path', 'archive_path', 'toml_imports_path', 'import_scripts_path') is missing.") + + def full_integrity_check(path: Path) -> None: + check_path_and_config(path) + print("Path and config-file exist:\t✅") + check_config_validity(path) + print("Configuration is valid:\t✅") check_db_integrity(path) + print("DB:\t✅") check_db_file_links(path) + print("Links:\t✅") print("Full:\t✅") From c3bf36bf52e6f1c9ff4631d75398ad82eff441d9 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Tue, 5 May 2026 17:15:16 +0200 Subject: [PATCH 67/85] add docs, add check for needed paths --- corrlib/integrity.py | 90 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 89 insertions(+), 1 deletion(-) diff --git a/corrlib/integrity.py b/corrlib/integrity.py index 74386f4..f5b2300 100644 --- a/corrlib/integrity.py +++ b/corrlib/integrity.py @@ -12,6 +12,20 @@ from typing import Any def has_valid_times(result: pd.Series) -> bool: + """ + Check, whether the result at hand has time-stamps that are sensible: + A recored is created first, then updated, with both times laying in the past. + + Parameters + ---------- + result: pd.Series + The result to check + + Returns + ------- + b: bool + True, if the timestamps make sense. + """ # we expect created_at <= updated_at <= now created_at = dt.datetime.fromisoformat(result['created_at']) updated_at = dt.datetime.fromisoformat(result['updated_at']) @@ -22,15 +36,41 @@ def has_valid_times(result: pd.Series) -> bool: return True def are_keys_unique(db: Path, table: str, col: str) -> bool: + """ + Check whether the strings listed in a column of a given table are unique. + + Parameters + ---------- + db: Path + The database to check. + table: str + The table to check. + col: str + The column to be checked for uniqueness. + + Returns + ------- + b: bool + True, if the strings are unique. + """ conn = sqlite3.connect(db) c = conn.cursor() - c.execute(f"SELECT COUNT( DISTINCT CAST(path AS nvarchar(4000))), COUNT({col}) FROM {table};") + c.execute(f"SELECT COUNT( DISTINCT CAST({col} AS nvarchar(4000))), COUNT({col}) FROM {table};") results = c.fetchall()[0] conn.close() return bool(results[0] == results[1]) def check_db_integrity(path: Path) -> None: + """ + Check intergrity of the database by checking the uniqueness of the record keys used to load the records + and ensuring that the timestamps of each record is sensible. Throws an error, if issues are detected. + + Parameters + ---------- + path: Path + Path to the backlog-library to check. + """ db = get_db_file(path) if not are_keys_unique(path / db, 'backlogs', 'path'): @@ -47,6 +87,17 @@ def check_db_integrity(path: Path) -> None: def _check_db2paths(path: Path, meas_paths: list[str]) -> None: + """ + Check whether for each record in the given by meas_paths, we can find the data in the file as we expect. + Also check, whether there are unreachable records in the files. If either of the issues arise, throws an error. + + Parameters + ---------- + path: Path + Path to the backlog-library to check. + meas_paths: list[str] + List of measurement paths to check. + """ needed_data: dict[str, list[str]] = {} for mpath in meas_paths: file = mpath.split("::")[0] @@ -72,6 +123,15 @@ def _check_db2paths(path: Path, meas_paths: list[str]) -> None: def check_db_file_links(path: Path) -> None: + """ + Check whether for each record in the given correlator library, we can find the data in the file as we expect. + Also check, whether there are unreachable records in the files. If either of the issues arise, throws an error. + + Parameters + ---------- + path: Path + Path to the backlog-library to check. + """ db = get_db_file(path) search_expr = "SELECT path FROM 'backlogs'" conn = sqlite3.connect(path / db) @@ -80,6 +140,14 @@ def check_db_file_links(path: Path) -> None: def check_path_and_config(path: Path) -> None: + """ + Check whether the given path exists and the cinfigureation file can be found. + + Parameters + ---------- + path: Path + Path to the backlog-library to check. + """ if not os.path.exists(path): raise FileNotFoundError(f"Corrlib path {path} does not exist.") config_path = path / CONFIG_FILENAME @@ -88,6 +156,14 @@ def check_path_and_config(path: Path) -> None: def check_config_validity(path: Path) -> None: + """ + Check whether the configuration file of the given corrlib-dataset path is valid. + + Parameters + ---------- + path: Path + Path to the backlog-library to check. + """ config = ConfigParser() config_path = path / CONFIG_FILENAME if os.path.exists(config_path): @@ -107,8 +183,20 @@ def check_config_validity(path: Path) -> None: if not all(has_path_opts): raise ValueError("One of the options in the 'path' section ('db', 'projects_path', 'archive_path', 'toml_imports_path', 'import_scripts_path') is missing.") + has_paths = [os.path.exists(path / config.get('paths', opt)) for opt in path_opts] + if not all(has_paths): + raise FileNotFoundError("one of the paths needed by the configuration file is not present.") + def full_integrity_check(path: Path) -> None: + """ + Aggregate all checks for easy validation of the backlog-library. + + Parameters + ---------- + path: Path + Path to the backlog-library to check. + """ check_path_and_config(path) print("Path and config-file exist:\t✅") check_config_validity(path) From ba4624d8433d0360aded38d196cbdfac39a9e434 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Tue, 5 May 2026 17:20:20 +0200 Subject: [PATCH 68/85] restruct: needed paths get extra check --- corrlib/integrity.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/corrlib/integrity.py b/corrlib/integrity.py index f5b2300..f2a70bd 100644 --- a/corrlib/integrity.py +++ b/corrlib/integrity.py @@ -11,6 +11,9 @@ from configparser import ConfigParser from typing import Any +path_opts = ['db', 'projects_path', 'archive_path', 'toml_imports_path', 'import_scripts_path'] + + def has_valid_times(result: pd.Series) -> bool: """ Check, whether the result at hand has time-stamps that are sensible: @@ -178,14 +181,29 @@ def check_config_validity(path: Path) -> None: raise ValueError("One of the options in the 'core' section ('version', 'tracker', 'cached') is missing.") if config.has_section('paths'): - path_opts = ['db', 'projects_path', 'archive_path', 'toml_imports_path', 'import_scripts_path'] has_path_opts = [config.has_option('paths', opt) for opt in path_opts] if not all(has_path_opts): raise ValueError("One of the options in the 'path' section ('db', 'projects_path', 'archive_path', 'toml_imports_path', 'import_scripts_path') is missing.") + +def check_paths(path: Path) -> None: + """ + Check whether all paths demanded by the 'paths' section of the configuration-file exist. + + Parameters + ---------- + path: Path + Path to the backlog-library to check. + """ + config = ConfigParser() + config_path = path / CONFIG_FILENAME + if os.path.exists(config_path): + config.read(config_path) + else: + raise FileNotFoundError("Configuration file not found.") has_paths = [os.path.exists(path / config.get('paths', opt)) for opt in path_opts] if not all(has_paths): - raise FileNotFoundError("one of the paths needed by the configuration file is not present.") + raise FileNotFoundError("One of the paths specified in the configuration file is not present.") def full_integrity_check(path: Path) -> None: @@ -201,6 +219,8 @@ def full_integrity_check(path: Path) -> None: print("Path and config-file exist:\t✅") check_config_validity(path) print("Configuration is valid:\t✅") + check_paths(path) + print("Needed paths exist:\t✅") check_db_integrity(path) print("DB:\t✅") check_db_file_links(path) From 3247cdbc40aaccd1c9ff62da2059d407a01d7a51 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Tue, 5 May 2026 17:24:09 +0200 Subject: [PATCH 69/85] neater UX --- corrlib/integrity.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/corrlib/integrity.py b/corrlib/integrity.py index f2a70bd..f660dfe 100644 --- a/corrlib/integrity.py +++ b/corrlib/integrity.py @@ -215,16 +215,17 @@ def full_integrity_check(path: Path) -> None: path: Path Path to the backlog-library to check. """ + print("Run full integrity check...") check_path_and_config(path) - print("Path and config-file exist:\t✅") + print("(1/5) Path and config-file exist: ✅") check_config_validity(path) - print("Configuration is valid:\t✅") + print("(2/5) Configuration is valid: ✅") check_paths(path) - print("Needed paths exist:\t✅") + print("(3/5) Needed paths exist: ✅") check_db_integrity(path) - print("DB:\t✅") + print("(4/5) Database is sane: ✅") check_db_file_links(path) - print("Links:\t✅") - print("Full:\t✅") + print("(5/5) DB2File and File2DB-links are sound: ✅") + print("Full integrity check: ✅") From a2a3346f51ec574e772a1293038558e351e65b69 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Tue, 5 May 2026 22:12:14 +0200 Subject: [PATCH 70/85] provide docstring for repo check --- corrlib/cli.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/corrlib/cli.py b/corrlib/cli.py index d24d8ef..acf4eca 100644 --- a/corrlib/cli.py +++ b/corrlib/cli.py @@ -108,7 +108,7 @@ def find( ), ) -> None: """ - Find a record in the backlog at hand. Through specifying it's ensemble and the measured correlator. + Find a record in the given backlog. """ results = find_record(path, ensemble, corr, code) if results.empty: @@ -147,6 +147,9 @@ def check(path: Path = typer.Option( "-d", ), ) -> None: + """ + Check the integrity of the repository. + """ full_integrity_check(path) From ac400aa9017d72c8f35b4f8664a7a8928e42c8b2 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Tue, 5 May 2026 22:32:13 +0200 Subject: [PATCH 71/85] Breaking change for CLI: change default path to current directory --- corrlib/cli.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/corrlib/cli.py b/corrlib/cli.py index d24d8ef..0e6d96a 100644 --- a/corrlib/cli.py +++ b/corrlib/cli.py @@ -29,7 +29,7 @@ def _version_callback(value: bool) -> None: @app.command() def update( path: Path = typer.Option( - Path('./corrlib'), + Path('.'), "--dataset", "-d", ), @@ -45,7 +45,7 @@ def update( @app.command() def lister( path: Path = typer.Option( - Path('./corrlib'), + Path('.'), "--dataset", "-d", ), @@ -76,7 +76,7 @@ def lister( @app.command() def alias_add( path: Path = typer.Option( - Path('./corrlib'), + Path('.'), "--dataset", "-d", ), @@ -94,7 +94,7 @@ def alias_add( @app.command() def find( path: Path = typer.Option( - Path('./corrlib'), + Path('.'), "--dataset", "-d", ), @@ -123,7 +123,7 @@ def find( @app.command() def stat( path: Path = typer.Option( - Path('./corrlib'), + Path('.'), "--dataset", "-d", ), @@ -142,7 +142,7 @@ def stat( @app.command() def check(path: Path = typer.Option( - Path('./corrlib'), + Path('.'), "--dataset", "-d", ), @@ -153,7 +153,7 @@ def check(path: Path = typer.Option( @app.command() def importer( path: Path = typer.Option( - Path('./corrlib'), + Path('.'), "--dataset", "-d", ), @@ -176,7 +176,7 @@ def importer( @app.command() def reimporter( path: Path = typer.Option( - Path('./corrlib'), + Path('.'), "--dataset", "-d", ), @@ -200,7 +200,7 @@ def reimporter( @app.command() def init( path: Path = typer.Option( - Path('./corrlib'), + Path('.'), "--dataset", "-d", ), @@ -220,7 +220,7 @@ def init( @app.command() def drop_cache( path: Path = typer.Option( - Path('./corrlib'), + Path('.'), "--dataset", "-d", ), From 08d25da188da235638e1c2c6360db130290d4f64 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Wed, 6 May 2026 16:48:35 +0200 Subject: [PATCH 72/85] HOTFIX: nsure path casting in find and meas_io --- corrlib/find.py | 1 + corrlib/meas_io.py | 9 +++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/corrlib/find.py b/corrlib/find.py index 1e6b4bf..ea696b1 100644 --- a/corrlib/find.py +++ b/corrlib/find.py @@ -322,6 +322,7 @@ def find_record(path: Path, ensemble: str, correlator_name: str, code: str, proj revision: Optional[str]=None, customFilter: Optional[Callable[[pd.DataFrame], pd.DataFrame]] = None, **kwargs: Any) -> pd.DataFrame: + path = Path(path) db_file = get_db_file(path) db = path / db_file if code not in codes: diff --git a/corrlib/meas_io.py b/corrlib/meas_io.py index cbd9386..6b6e5f1 100644 --- a/corrlib/meas_io.py +++ b/corrlib/meas_io.py @@ -37,6 +37,7 @@ def write_measurement(path: Path, ensemble: str, measurement: dict[str, dict[str parameter_file: str The parameter file used for the measurement. """ + path = Path(path) db_file = get_db_file(path) db = path / db_file @@ -50,7 +51,7 @@ def write_measurement(path: Path, ensemble: str, measurement: dict[str, dict[str c = conn.cursor() for corr in measurement.keys(): file_in_archive = Path('.') / 'archive' / ensemble / corr / str(uuid + '.json.gz') - file = path / file_in_archive + file = Path(path) / file_in_archive known_meas = {} if not os.path.exists(path / 'archive' / ensemble / corr): os.makedirs(path / 'archive' / ensemble / corr) @@ -174,6 +175,7 @@ def load_records(path: Path, meas_paths: list[str], preloaded: dict[str, Any] = returned_data: list The loaded records. """ + path = Path(path) if dry_run: _check_db2paths(path, meas_paths) return [] @@ -238,6 +240,7 @@ def cache_path(path: Path, file: str, key: str) -> Path: cache_path: str The path at which the measurement of the given file and key is cached. """ + path = Path(path) cache_path = cache_dir(path, file) / key return cache_path @@ -258,6 +261,7 @@ def preload(path: Path, file: Path) -> dict[str, Any]: filedict: dict[str, Any] The data read from the file. """ + path = Path(path) get(path, file) filedict: dict[str, Any] = pj.load_json_dict(str(path / file)) print("> read file") @@ -276,7 +280,7 @@ def drop_record(path: Path, meas_path: str) -> None: The measurement path as noted in the database. """ file_in_archive = meas_path.split("::")[0] - file = path / file_in_archive + file = Path(path) / file_in_archive db_file = get_db_file(path) db = path / db_file get(path, db_file) @@ -310,6 +314,7 @@ def drop_cache(path: Path) -> None: path: str The path of the library. """ + path = Path(path) cache_dir = path / ".cache" for f in os.listdir(cache_dir): shutil.rmtree(cache_dir / f) From fbf802959aff13ac391c17e96825e5c25d29cc51 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Wed, 6 May 2026 16:50:39 +0200 Subject: [PATCH 73/85] HOTFIX: ensure path casting in tools --- corrlib/tools.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/corrlib/tools.py b/corrlib/tools.py index e75e1cc..9ce194b 100644 --- a/corrlib/tools.py +++ b/corrlib/tools.py @@ -89,6 +89,7 @@ def set_config(path: Path, section: str, option: str, value: Any) -> None: value: Any The value we set the option to. """ + path = Path(path) config_path = path / CONFIG_FILENAME config = ConfigParser() if os.path.exists(config_path): @@ -115,6 +116,7 @@ def get_db_file(path: Path) -> Path: db_file: str The file holding the database. """ + path = Path(path) if not os.path.exists(path): raise FileNotFoundError(f"Corrlib path {path} does not exist.") config_path = path / CONFIG_FILENAME @@ -142,6 +144,7 @@ def cache_enabled(path: Path) -> bool: cached_bool: bool Whether the given library is cached. """ + path = Path(path) config_path = path / CONFIG_FILENAME config = ConfigParser() if os.path.exists(config_path): From 075cb2f75633cb0afb30d4320726a781fca8ea73 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Wed, 6 May 2026 16:52:07 +0200 Subject: [PATCH 74/85] HOTFIX: ensure path casting in tracker --- corrlib/tracker.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/corrlib/tracker.py b/corrlib/tracker.py index 98fdec5..6f4ae3d 100644 --- a/corrlib/tracker.py +++ b/corrlib/tracker.py @@ -21,6 +21,7 @@ def get_tracker(path: Path) -> str: tracker: str The tracker used in the dataset. """ + path = Path(path) config_path = path / CONFIG_FILENAME config = ConfigParser() if os.path.exists(config_path): @@ -42,6 +43,7 @@ def get(path: Path, file: Path) -> None: file: str The file to get. """ + path = Path(path) tracker = get_tracker(path) if tracker == 'datalad': if file == get_db_file(path): @@ -70,6 +72,7 @@ def save(path: Path, message: str, files: Optional[list[Path]]=None) -> None: files: list[str], optional The files to save. If None, all changes are saved. """ + path = Path(path) tracker = get_tracker(path) if tracker == 'datalad': if files is not None: @@ -93,6 +96,7 @@ def init(path: Path, tracker: str='datalad') -> None: tracker: str The tracker to use. Currently only 'datalad' and 'None' are supported. """ + path = Path(path) if tracker == 'datalad': dl.create(path) elif tracker == 'None': @@ -113,6 +117,7 @@ def unlock(path: Path, file: Path) -> None: file : str The file to unlock. """ + path = Path(path) tracker = get_tracker(path) if tracker == 'datalad': dl.unlock(os.path.join(path, file), dataset=path) @@ -136,6 +141,7 @@ def clone(path: Path, source: str, target: str) -> None: target: str The target path to clone the dataset to. """ + path = Path(path) tracker = get_tracker(path) if tracker == 'datalad': dl.clone(target=target, source=source, dataset=path) @@ -159,6 +165,7 @@ def drop(path: Path, reckless: Optional[str]=None) -> None: reckless: Optional[str] The datalad's reckless option for dropping data. """ + path = Path(path) tracker = get_tracker(path) if tracker == 'datalad': dl.drop(path, reckless=reckless) From 4c4a5fd670c798d7a87230c4bd383ffb9f21ec1c Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Wed, 6 May 2026 18:02:25 +0200 Subject: [PATCH 75/85] add checks of the format of the paths in the database --- corrlib/cli.py | 8 ++++---- corrlib/find.py | 7 +++++++ corrlib/integrity.py | 30 +++++++++++++++++++++++++++++- 3 files changed, 40 insertions(+), 5 deletions(-) diff --git a/corrlib/cli.py b/corrlib/cli.py index bdee36e..334dd33 100644 --- a/corrlib/cli.py +++ b/corrlib/cli.py @@ -4,7 +4,7 @@ from corrlib import __app_name__ from .initialization import create from .toml import import_tomls, update_project, reimport_project -from .find import find_record, list_projects +from .find import find_record, list_projects, list_ensembles from .tools import str2list from .main import update_aliases from .meas_io import drop_cache as mio_drop_cache @@ -56,9 +56,9 @@ def lister( """ if entities in ['ensembles', 'Ensembles','ENSEMBLES']: print("Ensembles:") - for item in os.listdir(path / "archive"): - if os.path.isdir(path / "archive" / item): - print(item) + results = list_ensembles(path) + for e in results: + print(e) elif entities == 'projects': results = list_projects(path) print("Projects:") diff --git a/corrlib/find.py b/corrlib/find.py index ea696b1..fbdd801 100644 --- a/corrlib/find.py +++ b/corrlib/find.py @@ -382,3 +382,10 @@ def list_projects(path: Path) -> list[tuple[str, str]]: conn.close() return results + +def list_ensembles(path: Path) -> list[str]: + res = [] + for item in os.listdir(path / "archive"): + if os.path.isdir(path / "archive" / item): + res.append(item) + return res diff --git a/corrlib/integrity.py b/corrlib/integrity.py index f660dfe..5a3ae05 100644 --- a/corrlib/integrity.py +++ b/corrlib/integrity.py @@ -7,7 +7,7 @@ from .tracker import get import pyerrors.input.json as pj import os from configparser import ConfigParser - +from .find import list_ensembles, list_projects from typing import Any @@ -64,6 +64,31 @@ def are_keys_unique(db: Path, table: str, col: str) -> bool: return bool(results[0] == results[1]) +def check_path_format(result: pd.Series, ensembles: list[str], projects: list[str]) -> None: + """ + Check whether the path of the given result has the right format. + + Parameters + ---------- + result: pd.Series + The result to be checked. + """ + p = result['path'] + if not p.startswith('archive'): + raise ValueError(f'The path {p} does not start correctly') + + meas_key = p.split('::')[1] + ensemble = p.split('/')[1] + project = p.split('/')[2].split('::')[0] + if not len(meas_key) == 64: + raise ValueError(f'meas_key of {p} is scrambled') + if ensemble not in ensembles: + raise ValueError(f'meas_key of {p} points to an unknown ensemble') + if project not in projects: + raise ValueError(f'meas_key of {p} points to an unknown project id') + + + def check_db_integrity(path: Path) -> None: """ Check intergrity of the database by checking the uniqueness of the record keys used to load the records @@ -82,10 +107,13 @@ def check_db_integrity(path: Path) -> None: search_expr = "SELECT * FROM 'backlogs'" conn = sqlite3.connect(path / db) results = pd.read_sql(search_expr, conn) + ensembles = list_ensembles(path) + projects = [p[0] for p in list_projects(path)] for _, result in results.iterrows(): if not has_valid_times(result): raise ValueError(f"Result with id {result[id]} has wrong time signatures.") + check_path_format(result, ensembles, projects) return From 3c09fb7f8c556bae30b67c42e01fca95b2667ebd Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Wed, 6 May 2026 18:06:14 +0200 Subject: [PATCH 76/85] correct typing issues --- corrlib/cli.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/corrlib/cli.py b/corrlib/cli.py index 334dd33..ad1024d 100644 --- a/corrlib/cli.py +++ b/corrlib/cli.py @@ -56,15 +56,15 @@ def lister( """ if entities in ['ensembles', 'Ensembles','ENSEMBLES']: print("Ensembles:") - results = list_ensembles(path) - for e in results: + ensemble_results = list_ensembles(path) + for e in ensemble_results: print(e) elif entities == 'projects': - results = list_projects(path) + project_results = list_projects(path) print("Projects:") header = "UUID".ljust(37) + "| Aliases" print(header) - for project in results: + for project in project_results: if project[1] is not None: aliases = " | ".join(str2list(project[1])) else: From ac3eb272ad890e4d5b0da546e1d746dd9bb873b5 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Wed, 6 May 2026 18:12:00 +0200 Subject: [PATCH 77/85] get rid of circular import --- corrlib/find.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/corrlib/find.py b/corrlib/find.py index fbdd801..b4d1bfe 100644 --- a/corrlib/find.py +++ b/corrlib/find.py @@ -6,7 +6,6 @@ import numpy as np from .input.implementations import codes from .tools import k2m, get_db_file from .tracker import get -from .integrity import has_valid_times from .sql import thin_sql_wrapper from typing import Any, Optional from pathlib import Path @@ -83,9 +82,6 @@ def _time_filter(results: pd.DataFrame, created_before: Optional[str]=None, cre result = results.iloc[ind] created_at = dt.datetime.fromisoformat(result['created_at']) updated_at = dt.datetime.fromisoformat(result['updated_at']) - db_times_valid = has_valid_times(result) - if not db_times_valid: - raise ValueError('Time stamps not valid for result with path', result["path"]) if created_before is not None: date_created_before = dt.datetime.fromisoformat(created_before) From 46b97acf95d237ea087bc3cd40f278d66df5c392 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Wed, 6 May 2026 18:20:07 +0200 Subject: [PATCH 78/85] get rid of circular imports part 2 --- corrlib/find.py | 4 ++++ corrlib/integrity.py | 37 ++++++++++++++++++++++++++++++++++--- 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/corrlib/find.py b/corrlib/find.py index b4d1bfe..fbdd801 100644 --- a/corrlib/find.py +++ b/corrlib/find.py @@ -6,6 +6,7 @@ import numpy as np from .input.implementations import codes from .tools import k2m, get_db_file from .tracker import get +from .integrity import has_valid_times from .sql import thin_sql_wrapper from typing import Any, Optional from pathlib import Path @@ -82,6 +83,9 @@ def _time_filter(results: pd.DataFrame, created_before: Optional[str]=None, cre result = results.iloc[ind] created_at = dt.datetime.fromisoformat(result['created_at']) updated_at = dt.datetime.fromisoformat(result['updated_at']) + db_times_valid = has_valid_times(result) + if not db_times_valid: + raise ValueError('Time stamps not valid for result with path', result["path"]) if created_before is not None: date_created_before = dt.datetime.fromisoformat(created_before) diff --git a/corrlib/integrity.py b/corrlib/integrity.py index 5a3ae05..4c35c4b 100644 --- a/corrlib/integrity.py +++ b/corrlib/integrity.py @@ -7,7 +7,6 @@ from .tracker import get import pyerrors.input.json as pj import os from configparser import ConfigParser -from .find import list_ensembles, list_projects from typing import Any @@ -64,6 +63,38 @@ def are_keys_unique(db: Path, table: str, col: str) -> bool: return bool(results[0] == results[1]) +def _list_projects(path: Path) -> list[tuple[str, str]]: + """ + List all projects known to the library. + + Parameters + ---------- + path: str + The path of the library. + + Returns + ------- + results: list[Any] + The projects known to the library. + """ + db_file = get_db_file(path) + get(path, db_file) + conn = sqlite3.connect(os.path.join(path, db_file)) + c = conn.cursor() + c.execute("SELECT id,aliases FROM projects") + results = c.fetchall() + conn.close() + return results + + +def _list_ensembles(path: Path) -> list[str]: + res = [] + for item in os.listdir(path / "archive"): + if os.path.isdir(path / "archive" / item): + res.append(item) + return res + + def check_path_format(result: pd.Series, ensembles: list[str], projects: list[str]) -> None: """ Check whether the path of the given result has the right format. @@ -107,8 +138,8 @@ def check_db_integrity(path: Path) -> None: search_expr = "SELECT * FROM 'backlogs'" conn = sqlite3.connect(path / db) results = pd.read_sql(search_expr, conn) - ensembles = list_ensembles(path) - projects = [p[0] for p in list_projects(path)] + ensembles = _list_ensembles(path) + projects = [p[0] for p in _list_projects(path)] for _, result in results.iterrows(): if not has_valid_times(result): From 3640f163fca24fc37cc801645abace8e8b779afd Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Wed, 6 May 2026 19:06:50 +0200 Subject: [PATCH 79/85] Give user a sense of the severity, add basic tests --- corrlib/integrity.py | 5 +++- tests/integrity_test.py | 55 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 tests/integrity_test.py diff --git a/corrlib/integrity.py b/corrlib/integrity.py index 4c35c4b..2fb520d 100644 --- a/corrlib/integrity.py +++ b/corrlib/integrity.py @@ -60,7 +60,10 @@ def are_keys_unique(db: Path, table: str, col: str) -> bool: c.execute(f"SELECT COUNT( DISTINCT CAST({col} AS nvarchar(4000))), COUNT({col}) FROM {table};") results = c.fetchall()[0] conn.close() - return bool(results[0] == results[1]) + res = bool(results[0] == results[1]) + if not res: + print("Unique:", results[0], "All:", results[1]) + return res def _list_projects(path: Path) -> list[tuple[str, str]]: diff --git a/tests/integrity_test.py b/tests/integrity_test.py new file mode 100644 index 0000000..2cf8308 --- /dev/null +++ b/tests/integrity_test.py @@ -0,0 +1,55 @@ +import corrlib.integrity as integ +import corrlib.find as find +import datalad.api as dl +import corrlib.initialization as cinit +import sqlite3 +from pathlib import Path +import os + + +def test_list_ensembles(tmp_path: Path) -> None: + """ + Check against the implementation in find to check if they are the same. + """ + os.mkdir(tmp_path / 'archive') + os.mkdir(tmp_path / 'archive' / 'A') + os.mkdir(tmp_path / 'archive' / 'B') + os.mkdir(tmp_path / 'archive' / 'C') + integ_results = integ._list_ensembles(tmp_path) + assert len(integ_results) == 3 + find_results = find.list_ensembles(tmp_path) + assert len(find_results) == 3 + for f,i in zip(find_results, integ_results): + assert f == i + + +def test_list_projects(tmp_path: Path) -> None: + cinit.create(tmp_path) + db = tmp_path / "backlogger.db" + dl.unlock(str(db), dataset=str(tmp_path)) + conn = sqlite3.connect(db) + c = conn.cursor() + + customTags = "" + owner = "owner" + code = "sfcf" + created_at = "today" + updated_at = "today" + + id = "asdf1" + aliases = "a1,s1,d1,f1" + c.execute("INSERT INTO projects (id, aliases, customTags, owner, code, created_at, updated_at) VALUES (?,?,?,?,?,?,?)", (id, aliases, customTags, owner, code , created_at, updated_at)) + id = "asdf2" + aliases = "a2,s2,d2,f2" + c.execute("INSERT INTO projects (id, aliases, customTags, owner, code, created_at, updated_at) VALUES (?,?,?,?,?,?,?)", (id, aliases, customTags, owner, code , created_at, updated_at)) + id = "asdf3" + aliases = "a3,s3,d3,f3" + c.execute("INSERT INTO projects (id, aliases, customTags, owner, code, created_at, updated_at) VALUES (?,?,?,?,?,?,?)", (id, aliases, customTags, owner, code , created_at, updated_at)) + conn.commit() + conn.close + integ_results = integ._list_projects(tmp_path) + assert len(integ_results) == 3 + find_results = find.list_projects(tmp_path) + assert len(find_results) == 3 + for f,i in zip(find_results, integ_results): + assert f == i From a450601b8076bc3b758d44e1221614de2bb6662b Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Wed, 6 May 2026 19:15:58 +0200 Subject: [PATCH 80/85] add test for has_valid_times --- tests/integrity_test.py | 46 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/tests/integrity_test.py b/tests/integrity_test.py index 2cf8308..8cc074f 100644 --- a/tests/integrity_test.py +++ b/tests/integrity_test.py @@ -5,6 +5,8 @@ import corrlib.initialization as cinit import sqlite3 from pathlib import Path import os +import pandas as pd +import datetime as dt def test_list_ensembles(tmp_path: Path) -> None: @@ -53,3 +55,47 @@ def test_list_projects(tmp_path: Path) -> None: assert len(find_results) == 3 for f,i in zip(find_results, integ_results): assert f == i + + +def test_has_valid_time() -> None: + record_A = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf0", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] # only created + record_B = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf1", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2025-03-26 12:55:18.229966', '2025-04-26 12:55:18.229966'] # created and updated + record_C = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf2", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2026-03-26 12:55:18.229966', '2026-04-14 12:55:18.229966'] # created and updated later + record_D = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf3", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2026-03-26 12:55:18.229966', '2026-03-27 12:55:18.229966'] + record_E = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf4", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2024-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # only created, earlier + record_F = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf5", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2026-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # this is invalid... + record_G = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf2", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2026-03-26 12:55:18.229966', str(dt.datetime.now() + dt.timedelta(days=2, hours=3, minutes=5, seconds=30))] # created and updated later + + data = [record_A, record_B, record_C, record_D, record_E] + cols = ["name", + "ensemble", + "code", + "path", + "project", + "parameters", + "parameter_file", + "created_at", + "updated_at"] + df = pd.DataFrame(data,columns=cols) + for _, result in df.iterrows(): + assert integ.has_valid_times(result) + data = [record_F, record_G] + cols = ["name", + "ensemble", + "code", + "path", + "project", + "parameters", + "parameter_file", + "created_at", + "updated_at"] + df = pd.DataFrame(data,columns=cols) + for _, result in df.iterrows(): + assert not integ.has_valid_times(result) From 9d0b922db922397902d7aa703ec561c08a2f61ef Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Wed, 6 May 2026 19:31:00 +0200 Subject: [PATCH 81/85] add simple test for key uniqueness --- tests/integrity_test.py | 52 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/tests/integrity_test.py b/tests/integrity_test.py index 8cc074f..9946a19 100644 --- a/tests/integrity_test.py +++ b/tests/integrity_test.py @@ -99,3 +99,55 @@ def test_has_valid_time() -> None: df = pd.DataFrame(data,columns=cols) for _, result in df.iterrows(): assert not integ.has_valid_times(result) + + +def test_are_keys_unique(tmp_path: Path) -> None: + db = tmp_path / 'test_success.db' + + record_A = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf0", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] # only created + record_B = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf1", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2025-03-26 12:55:18.229966', '2025-04-26 12:55:18.229966'] # created and updated + record_C = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf2", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2026-03-26 12:55:18.229966', '2026-04-14 12:55:18.229966'] # created and updated later + record_D = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf3", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2026-03-26 12:55:18.229966', '2026-03-27 12:55:18.229966'] + record_E = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf4", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2024-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # only created, earlier + record_F = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf5", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2026-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # this is invalid... + record_G = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf2", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2026-03-26 12:55:18.229966', str(dt.datetime.now() + dt.timedelta(days=2, hours=3, minutes=5, seconds=30))] # created and updated later + + data = [record_A, record_B, record_C, record_D, record_E, record_F] + cols = ["name", + "ensemble", + "code", + "path", + "project", + "parameters", + "parameter_file", + "created_at", + "updated_at"] + df = pd.DataFrame(data,columns=cols) + conn = sqlite3.connect(db) + df.to_sql('backlogs', conn) + conn.close() + assert integ.are_keys_unique(db, 'backlogs', 'path') + + db = tmp_path / 'test_fail.db' + data = [record_A, record_B, record_C, record_D, record_E, record_F, record_G] + cols = ["name", + "ensemble", + "code", + "path", + "project", + "parameters", + "parameter_file", + "created_at", + "updated_at"] + df = pd.DataFrame(data,columns=cols) + conn = sqlite3.connect(db) + df.to_sql('backlogs', conn) + conn.close() + assert not integ.are_keys_unique(db, 'backlogs', 'path') From 32987d5557b4ecf317a9e06acbdcc3ab3b1fce33 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Wed, 6 May 2026 19:47:05 +0200 Subject: [PATCH 82/85] add test whether the ensemble in the database is the one in meas_path --- corrlib/integrity.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/corrlib/integrity.py b/corrlib/integrity.py index 2fb520d..66ea4df 100644 --- a/corrlib/integrity.py +++ b/corrlib/integrity.py @@ -113,13 +113,15 @@ def check_path_format(result: pd.Series, ensembles: list[str], projects: list[st meas_key = p.split('::')[1] ensemble = p.split('/')[1] - project = p.split('/')[2].split('::')[0] + project = p.split('/')[3].split('.')[0] if not len(meas_key) == 64: raise ValueError(f'meas_key of {p} is scrambled') if ensemble not in ensembles: raise ValueError(f'meas_key of {p} points to an unknown ensemble') if project not in projects: - raise ValueError(f'meas_key of {p} points to an unknown project id') + raise ValueError(f'meas_key of {p} points to an unknown project id ({project})') + if not ensemble == result['ensemble']: + raise ValueError(f'Ensemble in database and file does not match for path {p}.') From da62af835cca3c9420e4f00d38996e0b5e843652 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Wed, 6 May 2026 20:10:45 +0200 Subject: [PATCH 83/85] stramline, add tests for path_format check --- tests/integrity_test.py | 68 +++++++++++++++++++++++++++++++---------- 1 file changed, 52 insertions(+), 16 deletions(-) diff --git a/tests/integrity_test.py b/tests/integrity_test.py index 9946a19..8a59d50 100644 --- a/tests/integrity_test.py +++ b/tests/integrity_test.py @@ -7,6 +7,7 @@ from pathlib import Path import os import pandas as pd import datetime as dt +import pytest def test_list_ensembles(tmp_path: Path) -> None: @@ -72,8 +73,6 @@ def test_has_valid_time() -> None: '2026-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # this is invalid... record_G = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf2", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", '2026-03-26 12:55:18.229966', str(dt.datetime.now() + dt.timedelta(days=2, hours=3, minutes=5, seconds=30))] # created and updated later - - data = [record_A, record_B, record_C, record_D, record_E] cols = ["name", "ensemble", "code", @@ -83,19 +82,12 @@ def test_has_valid_time() -> None: "parameter_file", "created_at", "updated_at"] + data = [record_A, record_B, record_C, record_D, record_E] + df = pd.DataFrame(data,columns=cols) for _, result in df.iterrows(): assert integ.has_valid_times(result) data = [record_F, record_G] - cols = ["name", - "ensemble", - "code", - "path", - "project", - "parameters", - "parameter_file", - "created_at", - "updated_at"] df = pd.DataFrame(data,columns=cols) for _, result in df.iterrows(): assert not integ.has_valid_times(result) @@ -119,7 +111,6 @@ def test_are_keys_unique(tmp_path: Path) -> None: record_G = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf2", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", '2026-03-26 12:55:18.229966', str(dt.datetime.now() + dt.timedelta(days=2, hours=3, minutes=5, seconds=30))] # created and updated later - data = [record_A, record_B, record_C, record_D, record_E, record_F] cols = ["name", "ensemble", "code", @@ -129,6 +120,8 @@ def test_are_keys_unique(tmp_path: Path) -> None: "parameter_file", "created_at", "updated_at"] + + data = [record_A, record_B, record_C, record_D, record_E, record_F] df = pd.DataFrame(data,columns=cols) conn = sqlite3.connect(db) df.to_sql('backlogs', conn) @@ -137,6 +130,32 @@ def test_are_keys_unique(tmp_path: Path) -> None: db = tmp_path / 'test_fail.db' data = [record_A, record_B, record_C, record_D, record_E, record_F, record_G] + + df = pd.DataFrame(data,columns=cols) + conn = sqlite3.connect(db) + df.to_sql('backlogs', conn) + conn.close() + assert not integ.are_keys_unique(db, 'backlogs', 'path') + + +def test_check_path_format() -> None: + record_A = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_A.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] # only created + record_B = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_B.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2025-03-26 12:55:18.229966', '2025-04-26 12:55:18.229966'] # created and updated + record_C = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_A.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2026-03-26 12:55:18.229966', '2026-04-14 12:55:18.229966'] # created and updated later + record_D = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_B.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2026-03-26 12:55:18.229966', '2026-03-27 12:55:18.229966'] + record_E = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_A.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2024-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # only created, earlier + record_F = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_B.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2026-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # this is invalid... + record_G = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_A.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfas", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2026-03-26 12:55:18.229966', str(dt.datetime.now() + dt.timedelta(days=2, hours=3, minutes=5, seconds=30))] # created and updated later + + projects = ['Project_A', 'Project_B'] + ensembles = ['ensA'] cols = ["name", "ensemble", "code", @@ -146,8 +165,25 @@ def test_are_keys_unique(tmp_path: Path) -> None: "parameter_file", "created_at", "updated_at"] + + data = [record_A, record_B, record_C, record_D, record_E, record_F] df = pd.DataFrame(data,columns=cols) - conn = sqlite3.connect(db) - df.to_sql('backlogs', conn) - conn.close() - assert not integ.are_keys_unique(db, 'backlogs', 'path') + for _, result in df.iterrows(): + integ.check_path_format(result, ensembles, projects) + + projects = ['Project_A', 'Project_B'] + ensembles = ['ensB'] + for _, result in df.iterrows(): + with pytest.raises(ValueError): + integ.check_path_format(result, ensembles, projects) + + projects = ['Project_A', 'Project_B'] + ensembles = ['ensA', 'ensB'] + for _, result in df.iterrows(): + integ.check_path_format(result, ensembles, projects) + + data = [record_G] + df = pd.DataFrame(data,columns=cols) + for _, result in df.iterrows(): + with pytest.raises(ValueError): + integ.check_path_format(result, ensembles, projects) From b088a282917a1bed18e8620abfcf5ccb0238df48 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Thu, 7 May 2026 15:27:21 +0200 Subject: [PATCH 84/85] bump version --- corrlib/version.py | 28 +++++++++------------------- 1 file changed, 9 insertions(+), 19 deletions(-) diff --git a/corrlib/version.py b/corrlib/version.py index 68d1c45..23dd03f 100644 --- a/corrlib/version.py +++ b/corrlib/version.py @@ -1,5 +1,6 @@ -# file generated by setuptools-scm +# file generated by vcs-versioning # don't change, don't track in version control +from __future__ import annotations __all__ = [ "__version__", @@ -10,25 +11,14 @@ __all__ = [ "commit_id", ] -TYPE_CHECKING = False -if TYPE_CHECKING: - from typing import Tuple - from typing import Union - - VERSION_TUPLE = Tuple[Union[int, str], ...] - COMMIT_ID = Union[str, None] -else: - VERSION_TUPLE = object - COMMIT_ID = object - version: str __version__: str -__version_tuple__: VERSION_TUPLE -version_tuple: VERSION_TUPLE -commit_id: COMMIT_ID -__commit_id__: COMMIT_ID +__version_tuple__: tuple[int | str, ...] +version_tuple: tuple[int | str, ...] +commit_id: str | None +__commit_id__: str | None -__version__ = version = '0.2.4.dev14+g602324f84.d20251202' -__version_tuple__ = version_tuple = (0, 2, 4, 'dev14', 'g602324f84.d20251202') +__version__ = version = '0.3.1.dev0+g08de17e6b.d20260507' +__version_tuple__ = version_tuple = (0, 3, 1, 'dev0', 'g08de17e6b.d20260507') -__commit_id__ = commit_id = 'g602324f84' +__commit_id__ = commit_id = 'g08de17e6b' From 50fb204cb1433ed1df5b16a2042101f3a843d674 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Fri, 8 May 2026 11:32:04 +0200 Subject: [PATCH 85/85] HOTFIX: enable r_start, r_sto, r_step params for t0 and t1 --- corrlib/input/openQCD.py | 16 ++++++++++++++-- corrlib/toml.py | 6 ++++-- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/corrlib/input/openQCD.py b/corrlib/input/openQCD.py index 879b555..c8eef72 100644 --- a/corrlib/input/openQCD.py +++ b/corrlib/input/openQCD.py @@ -164,7 +164,8 @@ def read_rwms(path: Path, project: str, dir_in_project: str, param: dict[str, An return rw_dict -def extract_t0(path: Path, project: str, dir_in_project: str, param: dict[str, Any], prefix: str, dtr_read: int, xmin: int, spatial_extent: int, fit_range: int = 5, postfix: str="", names: Optional[list[str]]=None, files: Optional[list[str]]=None) -> dict[str, Any]: +def extract_t0(path: Path, project: str, dir_in_project: str, param: dict[str, Any], prefix: str, dtr_read: int, xmin: int, spatial_extent: int, fit_range: int = 5, postfix: str="", names: Optional[list[str]]=None, files: Optional[list[str]]=None, + r_start: list[int]=[], r_stop: list[int]=[], r_step:int=1) -> dict[str, Any]: """ Extract t0 measurements from the project. @@ -218,6 +219,11 @@ def extract_t0(path: Path, project: str, dir_in_project: str, param: dict[str, A if postfix is not None: kwargs['postfix'] = postfix kwargs['plot_fit'] = False + if not r_start == []: + kwargs['r_start'] = r_start + if not r_stop == []: + kwargs['r_stop'] = r_stop + kwargs['r_step'] = r_step t0 = input.extract_t0(directory, prefix, @@ -238,7 +244,8 @@ def extract_t0(path: Path, project: str, dir_in_project: str, param: dict[str, A return t0_dict -def extract_t1(path: Path, project: str, dir_in_project: str, param: dict[str, Any], prefix: str, dtr_read: int, xmin: int, spatial_extent: int, fit_range: int = 5, postfix: str = "", names: Optional[list[str]]=None, files: Optional[list[str]]=None) -> dict[str, Any]: +def extract_t1(path: Path, project: str, dir_in_project: str, param: dict[str, Any], prefix: str, dtr_read: int, xmin: int, spatial_extent: int, fit_range: int = 5, postfix: str = "", names: Optional[list[str]]=None, files: Optional[list[str]]=None, + r_start: list[int]=[], r_stop: list[int]=[], r_step:int=1) -> dict[str, Any]: """ Extract t1 measurements from the project. @@ -290,6 +297,11 @@ def extract_t1(path: Path, project: str, dir_in_project: str, param: dict[str, A if postfix is not None: kwargs['postfix'] = postfix kwargs['plot_fit'] = False + if not r_start == []: + kwargs['r_start'] = r_start + if not r_stop == []: + kwargs['r_stop'] = r_stop + kwargs['r_step'] = r_step t0 = input.extract_t0(directory, prefix, dtr_read, diff --git a/corrlib/toml.py b/corrlib/toml.py index 29d7de2..1f4e300 100644 --- a/corrlib/toml.py +++ b/corrlib/toml.py @@ -230,13 +230,15 @@ def import_toml(path: Path, file: str, copy_file: bool=True) -> None: param[rwp] = "Unknown" param['type'] = 't0' measurement = openQCD.extract_t0(path, uuid, md['path'], param, str(md["prefix"]), int(md["dtr_read"]), int(md["xmin"]), int(md["spatial_extent"]), - fit_range=int(md.get('fit_range', 5)), postfix=str(md.get('postfix', '')), names=md.get('names', []), files=md.get('files', [])) + fit_range=int(md.get('fit_range', 5)), postfix=str(md.get('postfix', '')), names=md.get('names', []), files=md.get('files', []), + r_start=md.get('r_start', []), r_stop=md.get('r_stop', []), r_step=md.get('r_step', 1)) elif md['measurement'] == 't1': if 'param_file' in md: param = openQCD.load_ms3_infile(path, uuid, md['param_file']) param['type'] = 't1' measurement = openQCD.extract_t1(path, uuid, md['path'], param, str(md["prefix"]), int(md["dtr_read"]), int(md["xmin"]), int(md["spatial_extent"]), - fit_range=int(md.get('fit_range', 5)), postfix=str(md.get('postfix', '')), names=md.get('names', []), files=md.get('files', [])) + fit_range=int(md.get('fit_range', 5)), postfix=str(md.get('postfix', '')), names=md.get('names', []), files=md.get('files', []), + r_start=md.get('r_start', []), r_stop=md.get('r_stop', []), r_step=md.get('r_step', 1)) write_measurement(path, ensemble, measurement, uuid, project['code'], (md['param_file'] if 'param_file' in md else None)) imeas += 1 print(mname + " imported.")