From 4c4a5fd670c798d7a87230c4bd383ffb9f21ec1c Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Wed, 6 May 2026 18:02:25 +0200 Subject: [PATCH 1/9] add checks of the format of the paths in the database --- corrlib/cli.py | 8 ++++---- corrlib/find.py | 7 +++++++ corrlib/integrity.py | 30 +++++++++++++++++++++++++++++- 3 files changed, 40 insertions(+), 5 deletions(-) diff --git a/corrlib/cli.py b/corrlib/cli.py index bdee36e..334dd33 100644 --- a/corrlib/cli.py +++ b/corrlib/cli.py @@ -4,7 +4,7 @@ from corrlib import __app_name__ from .initialization import create from .toml import import_tomls, update_project, reimport_project -from .find import find_record, list_projects +from .find import find_record, list_projects, list_ensembles from .tools import str2list from .main import update_aliases from .meas_io import drop_cache as mio_drop_cache @@ -56,9 +56,9 @@ def lister( """ if entities in ['ensembles', 'Ensembles','ENSEMBLES']: print("Ensembles:") - for item in os.listdir(path / "archive"): - if os.path.isdir(path / "archive" / item): - print(item) + results = list_ensembles(path) + for e in results: + print(e) elif entities == 'projects': results = list_projects(path) print("Projects:") diff --git a/corrlib/find.py b/corrlib/find.py index ea696b1..fbdd801 100644 --- a/corrlib/find.py +++ b/corrlib/find.py @@ -382,3 +382,10 @@ def list_projects(path: Path) -> list[tuple[str, str]]: conn.close() return results + +def list_ensembles(path: Path) -> list[str]: + res = [] + for item in os.listdir(path / "archive"): + if os.path.isdir(path / "archive" / item): + res.append(item) + return res diff --git a/corrlib/integrity.py b/corrlib/integrity.py index f660dfe..5a3ae05 100644 --- a/corrlib/integrity.py +++ b/corrlib/integrity.py @@ -7,7 +7,7 @@ from .tracker import get import pyerrors.input.json as pj import os from configparser import ConfigParser - +from .find import list_ensembles, list_projects from typing import Any @@ -64,6 +64,31 @@ def are_keys_unique(db: Path, table: str, col: str) -> bool: return bool(results[0] == results[1]) +def check_path_format(result: pd.Series, ensembles: list[str], projects: list[str]) -> None: + """ + Check whether the path of the given result has the right format. + + Parameters + ---------- + result: pd.Series + The result to be checked. + """ + p = result['path'] + if not p.startswith('archive'): + raise ValueError(f'The path {p} does not start correctly') + + meas_key = p.split('::')[1] + ensemble = p.split('/')[1] + project = p.split('/')[2].split('::')[0] + if not len(meas_key) == 64: + raise ValueError(f'meas_key of {p} is scrambled') + if ensemble not in ensembles: + raise ValueError(f'meas_key of {p} points to an unknown ensemble') + if project not in projects: + raise ValueError(f'meas_key of {p} points to an unknown project id') + + + def check_db_integrity(path: Path) -> None: """ Check intergrity of the database by checking the uniqueness of the record keys used to load the records @@ -82,10 +107,13 @@ def check_db_integrity(path: Path) -> None: search_expr = "SELECT * FROM 'backlogs'" conn = sqlite3.connect(path / db) results = pd.read_sql(search_expr, conn) + ensembles = list_ensembles(path) + projects = [p[0] for p in list_projects(path)] for _, result in results.iterrows(): if not has_valid_times(result): raise ValueError(f"Result with id {result[id]} has wrong time signatures.") + check_path_format(result, ensembles, projects) return From 3c09fb7f8c556bae30b67c42e01fca95b2667ebd Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Wed, 6 May 2026 18:06:14 +0200 Subject: [PATCH 2/9] correct typing issues --- corrlib/cli.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/corrlib/cli.py b/corrlib/cli.py index 334dd33..ad1024d 100644 --- a/corrlib/cli.py +++ b/corrlib/cli.py @@ -56,15 +56,15 @@ def lister( """ if entities in ['ensembles', 'Ensembles','ENSEMBLES']: print("Ensembles:") - results = list_ensembles(path) - for e in results: + ensemble_results = list_ensembles(path) + for e in ensemble_results: print(e) elif entities == 'projects': - results = list_projects(path) + project_results = list_projects(path) print("Projects:") header = "UUID".ljust(37) + "| Aliases" print(header) - for project in results: + for project in project_results: if project[1] is not None: aliases = " | ".join(str2list(project[1])) else: From ac3eb272ad890e4d5b0da546e1d746dd9bb873b5 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Wed, 6 May 2026 18:12:00 +0200 Subject: [PATCH 3/9] get rid of circular import --- corrlib/find.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/corrlib/find.py b/corrlib/find.py index fbdd801..b4d1bfe 100644 --- a/corrlib/find.py +++ b/corrlib/find.py @@ -6,7 +6,6 @@ import numpy as np from .input.implementations import codes from .tools import k2m, get_db_file from .tracker import get -from .integrity import has_valid_times from .sql import thin_sql_wrapper from typing import Any, Optional from pathlib import Path @@ -83,9 +82,6 @@ def _time_filter(results: pd.DataFrame, created_before: Optional[str]=None, cre result = results.iloc[ind] created_at = dt.datetime.fromisoformat(result['created_at']) updated_at = dt.datetime.fromisoformat(result['updated_at']) - db_times_valid = has_valid_times(result) - if not db_times_valid: - raise ValueError('Time stamps not valid for result with path', result["path"]) if created_before is not None: date_created_before = dt.datetime.fromisoformat(created_before) From 46b97acf95d237ea087bc3cd40f278d66df5c392 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Wed, 6 May 2026 18:20:07 +0200 Subject: [PATCH 4/9] get rid of circular imports part 2 --- corrlib/find.py | 4 ++++ corrlib/integrity.py | 37 ++++++++++++++++++++++++++++++++++--- 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/corrlib/find.py b/corrlib/find.py index b4d1bfe..fbdd801 100644 --- a/corrlib/find.py +++ b/corrlib/find.py @@ -6,6 +6,7 @@ import numpy as np from .input.implementations import codes from .tools import k2m, get_db_file from .tracker import get +from .integrity import has_valid_times from .sql import thin_sql_wrapper from typing import Any, Optional from pathlib import Path @@ -82,6 +83,9 @@ def _time_filter(results: pd.DataFrame, created_before: Optional[str]=None, cre result = results.iloc[ind] created_at = dt.datetime.fromisoformat(result['created_at']) updated_at = dt.datetime.fromisoformat(result['updated_at']) + db_times_valid = has_valid_times(result) + if not db_times_valid: + raise ValueError('Time stamps not valid for result with path', result["path"]) if created_before is not None: date_created_before = dt.datetime.fromisoformat(created_before) diff --git a/corrlib/integrity.py b/corrlib/integrity.py index 5a3ae05..4c35c4b 100644 --- a/corrlib/integrity.py +++ b/corrlib/integrity.py @@ -7,7 +7,6 @@ from .tracker import get import pyerrors.input.json as pj import os from configparser import ConfigParser -from .find import list_ensembles, list_projects from typing import Any @@ -64,6 +63,38 @@ def are_keys_unique(db: Path, table: str, col: str) -> bool: return bool(results[0] == results[1]) +def _list_projects(path: Path) -> list[tuple[str, str]]: + """ + List all projects known to the library. + + Parameters + ---------- + path: str + The path of the library. + + Returns + ------- + results: list[Any] + The projects known to the library. + """ + db_file = get_db_file(path) + get(path, db_file) + conn = sqlite3.connect(os.path.join(path, db_file)) + c = conn.cursor() + c.execute("SELECT id,aliases FROM projects") + results = c.fetchall() + conn.close() + return results + + +def _list_ensembles(path: Path) -> list[str]: + res = [] + for item in os.listdir(path / "archive"): + if os.path.isdir(path / "archive" / item): + res.append(item) + return res + + def check_path_format(result: pd.Series, ensembles: list[str], projects: list[str]) -> None: """ Check whether the path of the given result has the right format. @@ -107,8 +138,8 @@ def check_db_integrity(path: Path) -> None: search_expr = "SELECT * FROM 'backlogs'" conn = sqlite3.connect(path / db) results = pd.read_sql(search_expr, conn) - ensembles = list_ensembles(path) - projects = [p[0] for p in list_projects(path)] + ensembles = _list_ensembles(path) + projects = [p[0] for p in _list_projects(path)] for _, result in results.iterrows(): if not has_valid_times(result): From 3640f163fca24fc37cc801645abace8e8b779afd Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Wed, 6 May 2026 19:06:50 +0200 Subject: [PATCH 5/9] Give user a sense of the severity, add basic tests --- corrlib/integrity.py | 5 +++- tests/integrity_test.py | 55 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 tests/integrity_test.py diff --git a/corrlib/integrity.py b/corrlib/integrity.py index 4c35c4b..2fb520d 100644 --- a/corrlib/integrity.py +++ b/corrlib/integrity.py @@ -60,7 +60,10 @@ def are_keys_unique(db: Path, table: str, col: str) -> bool: c.execute(f"SELECT COUNT( DISTINCT CAST({col} AS nvarchar(4000))), COUNT({col}) FROM {table};") results = c.fetchall()[0] conn.close() - return bool(results[0] == results[1]) + res = bool(results[0] == results[1]) + if not res: + print("Unique:", results[0], "All:", results[1]) + return res def _list_projects(path: Path) -> list[tuple[str, str]]: diff --git a/tests/integrity_test.py b/tests/integrity_test.py new file mode 100644 index 0000000..2cf8308 --- /dev/null +++ b/tests/integrity_test.py @@ -0,0 +1,55 @@ +import corrlib.integrity as integ +import corrlib.find as find +import datalad.api as dl +import corrlib.initialization as cinit +import sqlite3 +from pathlib import Path +import os + + +def test_list_ensembles(tmp_path: Path) -> None: + """ + Check against the implementation in find to check if they are the same. + """ + os.mkdir(tmp_path / 'archive') + os.mkdir(tmp_path / 'archive' / 'A') + os.mkdir(tmp_path / 'archive' / 'B') + os.mkdir(tmp_path / 'archive' / 'C') + integ_results = integ._list_ensembles(tmp_path) + assert len(integ_results) == 3 + find_results = find.list_ensembles(tmp_path) + assert len(find_results) == 3 + for f,i in zip(find_results, integ_results): + assert f == i + + +def test_list_projects(tmp_path: Path) -> None: + cinit.create(tmp_path) + db = tmp_path / "backlogger.db" + dl.unlock(str(db), dataset=str(tmp_path)) + conn = sqlite3.connect(db) + c = conn.cursor() + + customTags = "" + owner = "owner" + code = "sfcf" + created_at = "today" + updated_at = "today" + + id = "asdf1" + aliases = "a1,s1,d1,f1" + c.execute("INSERT INTO projects (id, aliases, customTags, owner, code, created_at, updated_at) VALUES (?,?,?,?,?,?,?)", (id, aliases, customTags, owner, code , created_at, updated_at)) + id = "asdf2" + aliases = "a2,s2,d2,f2" + c.execute("INSERT INTO projects (id, aliases, customTags, owner, code, created_at, updated_at) VALUES (?,?,?,?,?,?,?)", (id, aliases, customTags, owner, code , created_at, updated_at)) + id = "asdf3" + aliases = "a3,s3,d3,f3" + c.execute("INSERT INTO projects (id, aliases, customTags, owner, code, created_at, updated_at) VALUES (?,?,?,?,?,?,?)", (id, aliases, customTags, owner, code , created_at, updated_at)) + conn.commit() + conn.close + integ_results = integ._list_projects(tmp_path) + assert len(integ_results) == 3 + find_results = find.list_projects(tmp_path) + assert len(find_results) == 3 + for f,i in zip(find_results, integ_results): + assert f == i From a450601b8076bc3b758d44e1221614de2bb6662b Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Wed, 6 May 2026 19:15:58 +0200 Subject: [PATCH 6/9] add test for has_valid_times --- tests/integrity_test.py | 46 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/tests/integrity_test.py b/tests/integrity_test.py index 2cf8308..8cc074f 100644 --- a/tests/integrity_test.py +++ b/tests/integrity_test.py @@ -5,6 +5,8 @@ import corrlib.initialization as cinit import sqlite3 from pathlib import Path import os +import pandas as pd +import datetime as dt def test_list_ensembles(tmp_path: Path) -> None: @@ -53,3 +55,47 @@ def test_list_projects(tmp_path: Path) -> None: assert len(find_results) == 3 for f,i in zip(find_results, integ_results): assert f == i + + +def test_has_valid_time() -> None: + record_A = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf0", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] # only created + record_B = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf1", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2025-03-26 12:55:18.229966', '2025-04-26 12:55:18.229966'] # created and updated + record_C = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf2", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2026-03-26 12:55:18.229966', '2026-04-14 12:55:18.229966'] # created and updated later + record_D = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf3", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2026-03-26 12:55:18.229966', '2026-03-27 12:55:18.229966'] + record_E = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf4", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2024-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # only created, earlier + record_F = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf5", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2026-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # this is invalid... + record_G = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf2", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2026-03-26 12:55:18.229966', str(dt.datetime.now() + dt.timedelta(days=2, hours=3, minutes=5, seconds=30))] # created and updated later + + data = [record_A, record_B, record_C, record_D, record_E] + cols = ["name", + "ensemble", + "code", + "path", + "project", + "parameters", + "parameter_file", + "created_at", + "updated_at"] + df = pd.DataFrame(data,columns=cols) + for _, result in df.iterrows(): + assert integ.has_valid_times(result) + data = [record_F, record_G] + cols = ["name", + "ensemble", + "code", + "path", + "project", + "parameters", + "parameter_file", + "created_at", + "updated_at"] + df = pd.DataFrame(data,columns=cols) + for _, result in df.iterrows(): + assert not integ.has_valid_times(result) From 9d0b922db922397902d7aa703ec561c08a2f61ef Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Wed, 6 May 2026 19:31:00 +0200 Subject: [PATCH 7/9] add simple test for key uniqueness --- tests/integrity_test.py | 52 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/tests/integrity_test.py b/tests/integrity_test.py index 8cc074f..9946a19 100644 --- a/tests/integrity_test.py +++ b/tests/integrity_test.py @@ -99,3 +99,55 @@ def test_has_valid_time() -> None: df = pd.DataFrame(data,columns=cols) for _, result in df.iterrows(): assert not integ.has_valid_times(result) + + +def test_are_keys_unique(tmp_path: Path) -> None: + db = tmp_path / 'test_success.db' + + record_A = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf0", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] # only created + record_B = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf1", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2025-03-26 12:55:18.229966', '2025-04-26 12:55:18.229966'] # created and updated + record_C = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf2", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2026-03-26 12:55:18.229966', '2026-04-14 12:55:18.229966'] # created and updated later + record_D = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf3", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2026-03-26 12:55:18.229966', '2026-03-27 12:55:18.229966'] + record_E = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf4", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2024-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # only created, earlier + record_F = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf5", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2026-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # this is invalid... + record_G = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf2", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2026-03-26 12:55:18.229966', str(dt.datetime.now() + dt.timedelta(days=2, hours=3, minutes=5, seconds=30))] # created and updated later + + data = [record_A, record_B, record_C, record_D, record_E, record_F] + cols = ["name", + "ensemble", + "code", + "path", + "project", + "parameters", + "parameter_file", + "created_at", + "updated_at"] + df = pd.DataFrame(data,columns=cols) + conn = sqlite3.connect(db) + df.to_sql('backlogs', conn) + conn.close() + assert integ.are_keys_unique(db, 'backlogs', 'path') + + db = tmp_path / 'test_fail.db' + data = [record_A, record_B, record_C, record_D, record_E, record_F, record_G] + cols = ["name", + "ensemble", + "code", + "path", + "project", + "parameters", + "parameter_file", + "created_at", + "updated_at"] + df = pd.DataFrame(data,columns=cols) + conn = sqlite3.connect(db) + df.to_sql('backlogs', conn) + conn.close() + assert not integ.are_keys_unique(db, 'backlogs', 'path') From 32987d5557b4ecf317a9e06acbdcc3ab3b1fce33 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Wed, 6 May 2026 19:47:05 +0200 Subject: [PATCH 8/9] add test whether the ensemble in the database is the one in meas_path --- corrlib/integrity.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/corrlib/integrity.py b/corrlib/integrity.py index 2fb520d..66ea4df 100644 --- a/corrlib/integrity.py +++ b/corrlib/integrity.py @@ -113,13 +113,15 @@ def check_path_format(result: pd.Series, ensembles: list[str], projects: list[st meas_key = p.split('::')[1] ensemble = p.split('/')[1] - project = p.split('/')[2].split('::')[0] + project = p.split('/')[3].split('.')[0] if not len(meas_key) == 64: raise ValueError(f'meas_key of {p} is scrambled') if ensemble not in ensembles: raise ValueError(f'meas_key of {p} points to an unknown ensemble') if project not in projects: - raise ValueError(f'meas_key of {p} points to an unknown project id') + raise ValueError(f'meas_key of {p} points to an unknown project id ({project})') + if not ensemble == result['ensemble']: + raise ValueError(f'Ensemble in database and file does not match for path {p}.') From da62af835cca3c9420e4f00d38996e0b5e843652 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Wed, 6 May 2026 20:10:45 +0200 Subject: [PATCH 9/9] stramline, add tests for path_format check --- tests/integrity_test.py | 68 +++++++++++++++++++++++++++++++---------- 1 file changed, 52 insertions(+), 16 deletions(-) diff --git a/tests/integrity_test.py b/tests/integrity_test.py index 9946a19..8a59d50 100644 --- a/tests/integrity_test.py +++ b/tests/integrity_test.py @@ -7,6 +7,7 @@ from pathlib import Path import os import pandas as pd import datetime as dt +import pytest def test_list_ensembles(tmp_path: Path) -> None: @@ -72,8 +73,6 @@ def test_has_valid_time() -> None: '2026-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # this is invalid... record_G = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf2", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", '2026-03-26 12:55:18.229966', str(dt.datetime.now() + dt.timedelta(days=2, hours=3, minutes=5, seconds=30))] # created and updated later - - data = [record_A, record_B, record_C, record_D, record_E] cols = ["name", "ensemble", "code", @@ -83,19 +82,12 @@ def test_has_valid_time() -> None: "parameter_file", "created_at", "updated_at"] + data = [record_A, record_B, record_C, record_D, record_E] + df = pd.DataFrame(data,columns=cols) for _, result in df.iterrows(): assert integ.has_valid_times(result) data = [record_F, record_G] - cols = ["name", - "ensemble", - "code", - "path", - "project", - "parameters", - "parameter_file", - "created_at", - "updated_at"] df = pd.DataFrame(data,columns=cols) for _, result in df.iterrows(): assert not integ.has_valid_times(result) @@ -119,7 +111,6 @@ def test_are_keys_unique(tmp_path: Path) -> None: record_G = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf2", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", '2026-03-26 12:55:18.229966', str(dt.datetime.now() + dt.timedelta(days=2, hours=3, minutes=5, seconds=30))] # created and updated later - data = [record_A, record_B, record_C, record_D, record_E, record_F] cols = ["name", "ensemble", "code", @@ -129,6 +120,8 @@ def test_are_keys_unique(tmp_path: Path) -> None: "parameter_file", "created_at", "updated_at"] + + data = [record_A, record_B, record_C, record_D, record_E, record_F] df = pd.DataFrame(data,columns=cols) conn = sqlite3.connect(db) df.to_sql('backlogs', conn) @@ -137,6 +130,32 @@ def test_are_keys_unique(tmp_path: Path) -> None: db = tmp_path / 'test_fail.db' data = [record_A, record_B, record_C, record_D, record_E, record_F, record_G] + + df = pd.DataFrame(data,columns=cols) + conn = sqlite3.connect(db) + df.to_sql('backlogs', conn) + conn.close() + assert not integ.are_keys_unique(db, 'backlogs', 'path') + + +def test_check_path_format() -> None: + record_A = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_A.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] # only created + record_B = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_B.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2025-03-26 12:55:18.229966', '2025-04-26 12:55:18.229966'] # created and updated + record_C = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_A.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2026-03-26 12:55:18.229966', '2026-04-14 12:55:18.229966'] # created and updated later + record_D = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_B.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2026-03-26 12:55:18.229966', '2026-03-27 12:55:18.229966'] + record_E = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_A.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2024-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # only created, earlier + record_F = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_B.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2026-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # this is invalid... + record_G = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_A.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfas", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2026-03-26 12:55:18.229966', str(dt.datetime.now() + dt.timedelta(days=2, hours=3, minutes=5, seconds=30))] # created and updated later + + projects = ['Project_A', 'Project_B'] + ensembles = ['ensA'] cols = ["name", "ensemble", "code", @@ -146,8 +165,25 @@ def test_are_keys_unique(tmp_path: Path) -> None: "parameter_file", "created_at", "updated_at"] + + data = [record_A, record_B, record_C, record_D, record_E, record_F] df = pd.DataFrame(data,columns=cols) - conn = sqlite3.connect(db) - df.to_sql('backlogs', conn) - conn.close() - assert not integ.are_keys_unique(db, 'backlogs', 'path') + for _, result in df.iterrows(): + integ.check_path_format(result, ensembles, projects) + + projects = ['Project_A', 'Project_B'] + ensembles = ['ensB'] + for _, result in df.iterrows(): + with pytest.raises(ValueError): + integ.check_path_format(result, ensembles, projects) + + projects = ['Project_A', 'Project_B'] + ensembles = ['ensA', 'ensB'] + for _, result in df.iterrows(): + integ.check_path_format(result, ensembles, projects) + + data = [record_G] + df = pd.DataFrame(data,columns=cols) + for _, result in df.iterrows(): + with pytest.raises(ValueError): + integ.check_path_format(result, ensembles, projects)