From a450601b8076bc3b758d44e1221614de2bb6662b Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Wed, 6 May 2026 19:15:58 +0200 Subject: [PATCH 1/4] add test for has_valid_times --- tests/integrity_test.py | 46 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/tests/integrity_test.py b/tests/integrity_test.py index 2cf8308..8cc074f 100644 --- a/tests/integrity_test.py +++ b/tests/integrity_test.py @@ -5,6 +5,8 @@ import corrlib.initialization as cinit import sqlite3 from pathlib import Path import os +import pandas as pd +import datetime as dt def test_list_ensembles(tmp_path: Path) -> None: @@ -53,3 +55,47 @@ def test_list_projects(tmp_path: Path) -> None: assert len(find_results) == 3 for f,i in zip(find_results, integ_results): assert f == i + + +def test_has_valid_time() -> None: + record_A = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf0", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] # only created + record_B = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf1", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2025-03-26 12:55:18.229966', '2025-04-26 12:55:18.229966'] # created and updated + record_C = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf2", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2026-03-26 12:55:18.229966', '2026-04-14 12:55:18.229966'] # created and updated later + record_D = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf3", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2026-03-26 12:55:18.229966', '2026-03-27 12:55:18.229966'] + record_E = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf4", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2024-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # only created, earlier + record_F = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf5", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2026-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # this is invalid... + record_G = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf2", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2026-03-26 12:55:18.229966', str(dt.datetime.now() + dt.timedelta(days=2, hours=3, minutes=5, seconds=30))] # created and updated later + + data = [record_A, record_B, record_C, record_D, record_E] + cols = ["name", + "ensemble", + "code", + "path", + "project", + "parameters", + "parameter_file", + "created_at", + "updated_at"] + df = pd.DataFrame(data,columns=cols) + for _, result in df.iterrows(): + assert integ.has_valid_times(result) + data = [record_F, record_G] + cols = ["name", + "ensemble", + "code", + "path", + "project", + "parameters", + "parameter_file", + "created_at", + "updated_at"] + df = pd.DataFrame(data,columns=cols) + for _, result in df.iterrows(): + assert not integ.has_valid_times(result) From 9d0b922db922397902d7aa703ec561c08a2f61ef Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Wed, 6 May 2026 19:31:00 +0200 Subject: [PATCH 2/4] add simple test for key uniqueness --- tests/integrity_test.py | 52 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/tests/integrity_test.py b/tests/integrity_test.py index 8cc074f..9946a19 100644 --- a/tests/integrity_test.py +++ b/tests/integrity_test.py @@ -99,3 +99,55 @@ def test_has_valid_time() -> None: df = pd.DataFrame(data,columns=cols) for _, result in df.iterrows(): assert not integ.has_valid_times(result) + + +def test_are_keys_unique(tmp_path: Path) -> None: + db = tmp_path / 'test_success.db' + + record_A = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf0", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] # only created + record_B = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf1", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2025-03-26 12:55:18.229966', '2025-04-26 12:55:18.229966'] # created and updated + record_C = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf2", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2026-03-26 12:55:18.229966', '2026-04-14 12:55:18.229966'] # created and updated later + record_D = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf3", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2026-03-26 12:55:18.229966', '2026-03-27 12:55:18.229966'] + record_E = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf4", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2024-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # only created, earlier + record_F = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf5", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2026-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # this is invalid... + record_G = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf2", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2026-03-26 12:55:18.229966', str(dt.datetime.now() + dt.timedelta(days=2, hours=3, minutes=5, seconds=30))] # created and updated later + + data = [record_A, record_B, record_C, record_D, record_E, record_F] + cols = ["name", + "ensemble", + "code", + "path", + "project", + "parameters", + "parameter_file", + "created_at", + "updated_at"] + df = pd.DataFrame(data,columns=cols) + conn = sqlite3.connect(db) + df.to_sql('backlogs', conn) + conn.close() + assert integ.are_keys_unique(db, 'backlogs', 'path') + + db = tmp_path / 'test_fail.db' + data = [record_A, record_B, record_C, record_D, record_E, record_F, record_G] + cols = ["name", + "ensemble", + "code", + "path", + "project", + "parameters", + "parameter_file", + "created_at", + "updated_at"] + df = pd.DataFrame(data,columns=cols) + conn = sqlite3.connect(db) + df.to_sql('backlogs', conn) + conn.close() + assert not integ.are_keys_unique(db, 'backlogs', 'path') From 32987d5557b4ecf317a9e06acbdcc3ab3b1fce33 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Wed, 6 May 2026 19:47:05 +0200 Subject: [PATCH 3/4] add test whether the ensemble in the database is the one in meas_path --- corrlib/integrity.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/corrlib/integrity.py b/corrlib/integrity.py index 2fb520d..66ea4df 100644 --- a/corrlib/integrity.py +++ b/corrlib/integrity.py @@ -113,13 +113,15 @@ def check_path_format(result: pd.Series, ensembles: list[str], projects: list[st meas_key = p.split('::')[1] ensemble = p.split('/')[1] - project = p.split('/')[2].split('::')[0] + project = p.split('/')[3].split('.')[0] if not len(meas_key) == 64: raise ValueError(f'meas_key of {p} is scrambled') if ensemble not in ensembles: raise ValueError(f'meas_key of {p} points to an unknown ensemble') if project not in projects: - raise ValueError(f'meas_key of {p} points to an unknown project id') + raise ValueError(f'meas_key of {p} points to an unknown project id ({project})') + if not ensemble == result['ensemble']: + raise ValueError(f'Ensemble in database and file does not match for path {p}.') From da62af835cca3c9420e4f00d38996e0b5e843652 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Wed, 6 May 2026 20:10:45 +0200 Subject: [PATCH 4/4] stramline, add tests for path_format check --- tests/integrity_test.py | 68 +++++++++++++++++++++++++++++++---------- 1 file changed, 52 insertions(+), 16 deletions(-) diff --git a/tests/integrity_test.py b/tests/integrity_test.py index 9946a19..8a59d50 100644 --- a/tests/integrity_test.py +++ b/tests/integrity_test.py @@ -7,6 +7,7 @@ from pathlib import Path import os import pandas as pd import datetime as dt +import pytest def test_list_ensembles(tmp_path: Path) -> None: @@ -72,8 +73,6 @@ def test_has_valid_time() -> None: '2026-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # this is invalid... record_G = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf2", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", '2026-03-26 12:55:18.229966', str(dt.datetime.now() + dt.timedelta(days=2, hours=3, minutes=5, seconds=30))] # created and updated later - - data = [record_A, record_B, record_C, record_D, record_E] cols = ["name", "ensemble", "code", @@ -83,19 +82,12 @@ def test_has_valid_time() -> None: "parameter_file", "created_at", "updated_at"] + data = [record_A, record_B, record_C, record_D, record_E] + df = pd.DataFrame(data,columns=cols) for _, result in df.iterrows(): assert integ.has_valid_times(result) data = [record_F, record_G] - cols = ["name", - "ensemble", - "code", - "path", - "project", - "parameters", - "parameter_file", - "created_at", - "updated_at"] df = pd.DataFrame(data,columns=cols) for _, result in df.iterrows(): assert not integ.has_valid_times(result) @@ -119,7 +111,6 @@ def test_are_keys_unique(tmp_path: Path) -> None: record_G = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf2", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", '2026-03-26 12:55:18.229966', str(dt.datetime.now() + dt.timedelta(days=2, hours=3, minutes=5, seconds=30))] # created and updated later - data = [record_A, record_B, record_C, record_D, record_E, record_F] cols = ["name", "ensemble", "code", @@ -129,6 +120,8 @@ def test_are_keys_unique(tmp_path: Path) -> None: "parameter_file", "created_at", "updated_at"] + + data = [record_A, record_B, record_C, record_D, record_E, record_F] df = pd.DataFrame(data,columns=cols) conn = sqlite3.connect(db) df.to_sql('backlogs', conn) @@ -137,6 +130,32 @@ def test_are_keys_unique(tmp_path: Path) -> None: db = tmp_path / 'test_fail.db' data = [record_A, record_B, record_C, record_D, record_E, record_F, record_G] + + df = pd.DataFrame(data,columns=cols) + conn = sqlite3.connect(db) + df.to_sql('backlogs', conn) + conn.close() + assert not integ.are_keys_unique(db, 'backlogs', 'path') + + +def test_check_path_format() -> None: + record_A = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_A.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] # only created + record_B = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_B.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2025-03-26 12:55:18.229966', '2025-04-26 12:55:18.229966'] # created and updated + record_C = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_A.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2026-03-26 12:55:18.229966', '2026-04-14 12:55:18.229966'] # created and updated later + record_D = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_B.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2026-03-26 12:55:18.229966', '2026-03-27 12:55:18.229966'] + record_E = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_A.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2024-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # only created, earlier + record_F = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_B.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2026-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # this is invalid... + record_G = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_A.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfas", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", + '2026-03-26 12:55:18.229966', str(dt.datetime.now() + dt.timedelta(days=2, hours=3, minutes=5, seconds=30))] # created and updated later + + projects = ['Project_A', 'Project_B'] + ensembles = ['ensA'] cols = ["name", "ensemble", "code", @@ -146,8 +165,25 @@ def test_are_keys_unique(tmp_path: Path) -> None: "parameter_file", "created_at", "updated_at"] + + data = [record_A, record_B, record_C, record_D, record_E, record_F] df = pd.DataFrame(data,columns=cols) - conn = sqlite3.connect(db) - df.to_sql('backlogs', conn) - conn.close() - assert not integ.are_keys_unique(db, 'backlogs', 'path') + for _, result in df.iterrows(): + integ.check_path_format(result, ensembles, projects) + + projects = ['Project_A', 'Project_B'] + ensembles = ['ensB'] + for _, result in df.iterrows(): + with pytest.raises(ValueError): + integ.check_path_format(result, ensembles, projects) + + projects = ['Project_A', 'Project_B'] + ensembles = ['ensA', 'ensB'] + for _, result in df.iterrows(): + integ.check_path_format(result, ensembles, projects) + + data = [record_G] + df = pd.DataFrame(data,columns=cols) + for _, result in df.iterrows(): + with pytest.raises(ValueError): + integ.check_path_format(result, ensembles, projects)