corrlib/integrity.py

@@ -113,15 +113,13 @@ def check_path_format(result: pd.Series, ensembles: list[str], projects: list[st
 
     meas_key = p.split('::')[1]
     ensemble = p.split('/')[1]
-    project = p.split('/')[3].split('.')[0]
+    project = p.split('/')[2].split('::')[0]
     if not len(meas_key) == 64:
         raise ValueError(f'meas_key of {p} is scrambled')
     if ensemble not in ensembles:
         raise ValueError(f'meas_key of {p} points to an unknown ensemble')
     if project not in projects:
-        raise ValueError(f'meas_key of {p} points to an unknown project id ({project})')
+        raise ValueError(f'meas_key of {p} points to an unknown project id')
-    if not ensemble == result['ensemble']:
-        raise ValueError(f'Ensemble in database and file does not match for path {p}.')
 
 
 

tests/integrity_test.py

@@ -5,9 +5,6 @@ import corrlib.initialization as cinit
 import sqlite3
 from pathlib import Path
 import os
-import pandas as pd
-import datetime as dt
-import pytest
 
 
 def test_list_ensembles(tmp_path: Path) -> None:
@@ -56,134 +53,3 @@ def test_list_projects(tmp_path: Path) -> None:
     assert len(find_results) == 3
     for f,i in zip(find_results, integ_results):
         assert f == i
-
-
-def test_has_valid_time() -> None:
-    record_A = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf0", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
-            '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] # only created
-    record_B = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf1", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
-                '2025-03-26 12:55:18.229966', '2025-04-26 12:55:18.229966'] # created and updated
-    record_C = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf2", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
-                '2026-03-26 12:55:18.229966', '2026-04-14 12:55:18.229966'] # created and updated later
-    record_D = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf3", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
-                '2026-03-26 12:55:18.229966', '2026-03-27 12:55:18.229966']
-    record_E = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf4", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
-                '2024-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # only created, earlier
-    record_F = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf5", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
-                '2026-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # this is invalid...
-    record_G = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf2", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
-                '2026-03-26 12:55:18.229966', str(dt.datetime.now() + dt.timedelta(days=2, hours=3, minutes=5, seconds=30))] # created and updated later
-    cols = ["name",
-            "ensemble",
-            "code",
-            "path",
-            "project",
-            "parameters",
-            "parameter_file",
-            "created_at",
-            "updated_at"]
-    data = [record_A, record_B, record_C, record_D, record_E]
-
-    df = pd.DataFrame(data,columns=cols)
-    for _, result in df.iterrows():
-        assert integ.has_valid_times(result)
-    data = [record_F, record_G]
-    df = pd.DataFrame(data,columns=cols)
-    for _, result in df.iterrows():
-        assert not integ.has_valid_times(result)
-
-
-def test_are_keys_unique(tmp_path: Path) -> None:
-    db = tmp_path / 'test_success.db'
-
-    record_A = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf0", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
-            '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] # only created
-    record_B = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf1", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
-                '2025-03-26 12:55:18.229966', '2025-04-26 12:55:18.229966'] # created and updated
-    record_C = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf2", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
-                '2026-03-26 12:55:18.229966', '2026-04-14 12:55:18.229966'] # created and updated later
-    record_D = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf3", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
-                '2026-03-26 12:55:18.229966', '2026-03-27 12:55:18.229966']
-    record_E = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf4", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
-                '2024-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # only created, earlier
-    record_F = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf5", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
-                '2026-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # this is invalid...
-    record_G = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf2", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
-                '2026-03-26 12:55:18.229966', str(dt.datetime.now() + dt.timedelta(days=2, hours=3, minutes=5, seconds=30))] # created and updated later
-
-    cols = ["name",
-            "ensemble",
-            "code",
-            "path",
-            "project",
-            "parameters",
-            "parameter_file",
-            "created_at",
-            "updated_at"]
-
-    data = [record_A, record_B, record_C, record_D, record_E, record_F]
-    df = pd.DataFrame(data,columns=cols)
-    conn = sqlite3.connect(db)
-    df.to_sql('backlogs', conn)
-    conn.close()
-    assert integ.are_keys_unique(db, 'backlogs', 'path')
-
-    db = tmp_path / 'test_fail.db'
-    data = [record_A, record_B, record_C, record_D, record_E, record_F, record_G]
-
-    df = pd.DataFrame(data,columns=cols)
-    conn = sqlite3.connect(db)
-    df.to_sql('backlogs', conn)
-    conn.close()
-    assert not integ.are_keys_unique(db, 'backlogs', 'path')
-
-
-def test_check_path_format() -> None:
-    record_A = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_A.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
-            '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] # only created
-    record_B = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_B.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
-                '2025-03-26 12:55:18.229966', '2025-04-26 12:55:18.229966'] # created and updated
-    record_C = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_A.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
-                '2026-03-26 12:55:18.229966', '2026-04-14 12:55:18.229966'] # created and updated later
-    record_D = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_B.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
-                '2026-03-26 12:55:18.229966', '2026-03-27 12:55:18.229966']
-    record_E = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_A.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
-                '2024-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # only created, earlier
-    record_F = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_B.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
-                '2026-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # this is invalid...
-    record_G = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_A.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfas", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
-                '2026-03-26 12:55:18.229966', str(dt.datetime.now() + dt.timedelta(days=2, hours=3, minutes=5, seconds=30))] # created and updated later
-
-    projects = ['Project_A', 'Project_B']
-    ensembles = ['ensA']
-    cols = ["name",
-            "ensemble",
-            "code",
-            "path",
-            "project",
-            "parameters",
-            "parameter_file",
-            "created_at",
-            "updated_at"]
-
-    data = [record_A, record_B, record_C, record_D, record_E, record_F]
-    df = pd.DataFrame(data,columns=cols)
-    for _, result in df.iterrows():
-        integ.check_path_format(result, ensembles, projects)
-
-    projects = ['Project_A', 'Project_B']
-    ensembles = ['ensB']
-    for _, result in df.iterrows():
-        with pytest.raises(ValueError):
-            integ.check_path_format(result, ensembles, projects)
-
-    projects = ['Project_A', 'Project_B']
-    ensembles = ['ensA', 'ensB']
-    for _, result in df.iterrows():
-        integ.check_path_format(result, ensembles, projects)
-
-    data = [record_G]
-    df = pd.DataFrame(data,columns=cols)
-    for _, result in df.iterrows():
-        with pytest.raises(ValueError):
-            integ.check_path_format(result, ensembles, projects)