jkuhl/corrlib
1
0
Fork 0
Code Issues 11 Pull requests 1 Releases Packages Activity Actions

Compare commits

base: jkuhl:da62af835cca3c9420e4f00d38996e0b5e843652
Branches Tags
jkuhl:develop
jkuhl:feat/plots
jkuhl:lint/strict_ruff
jkuhl:master
jkuhl:fix/cache
jkuhl:0.3.0
jkuhl:0.2.4
jkuhl:0.2.3
jkuhl:0.2.2
jkuhl:0.2.1
jkuhl:0.2.0
jkuhl:0.1.4
jkuhl:0.1.3
jkuhl:0.1.2
jkuhl:0.1.1
jkuhl:0.1.0
jkuhl:v0.0.2
jkuhl:v0.0.1
..
compare: jkuhl:3640f163fca24fc37cc801645abace8e8b779afd
Branches Tags
jkuhl:develop
jkuhl:feat/plots
jkuhl:lint/strict_ruff
jkuhl:master
jkuhl:fix/cache
jkuhl:0.3.0
jkuhl:0.2.4
jkuhl:0.2.3
jkuhl:0.2.2
jkuhl:0.2.1
jkuhl:0.2.0
jkuhl:0.1.4
jkuhl:0.1.3
jkuhl:0.1.2
jkuhl:0.1.1
jkuhl:0.1.0
jkuhl:v0.0.2
jkuhl:v0.0.1

No commits in common. "da62af835cca3c9420e4f00d38996e0b5e843652" and "3640f163fca24fc37cc801645abace8e8b779afd" have entirely different histories.
da62af835c ... 3640f163fc

2 changed files with 2 additions and 138 deletions
Download patch file Download diff file Expand all files Collapse all files

6
corrlib/integrity.py
View file

@ -113,15 +113,13 @@ def check_path_format(result: pd.Series, ensembles: list[str], projects: list[st
meas_key = p.split('::')[1]
ensemble = p.split('/')[1]
project = p.split('/')[3].split('.')[0]
project = p.split('/')[2].split('::')[0]
if not len(meas_key) == 64:
raise ValueError(f'meas_key of {p} is scrambled')
if ensemble not in ensembles:
raise ValueError(f'meas_key of {p} points to an unknown ensemble')
if project not in projects:
raise ValueError(f'meas_key of {p} points to an unknown project id ({project})')
if not ensemble == result['ensemble']:
raise ValueError(f'Ensemble in database and file does not match for path {p}.')
raise ValueError(f'meas_key of {p} points to an unknown project id')

134
tests/integrity_test.py
View file

@ -5,9 +5,6 @@ import corrlib.initialization as cinit
import sqlite3
from pathlib import Path
import os
import pandas as pd
import datetime as dt
import pytest
def test_list_ensembles(tmp_path: Path) -> None:
@ -56,134 +53,3 @@ def test_list_projects(tmp_path: Path) -> None:
assert len(find_results) == 3
for f,i in zip(find_results, integ_results):
assert f == i
def test_has_valid_time() -> None:
record_A = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf0", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
'2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] # only created
record_B = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf1", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
'2025-03-26 12:55:18.229966', '2025-04-26 12:55:18.229966'] # created and updated
record_C = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf2", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
'2026-03-26 12:55:18.229966', '2026-04-14 12:55:18.229966'] # created and updated later
record_D = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf3", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
'2026-03-26 12:55:18.229966', '2026-03-27 12:55:18.229966']
record_E = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf4", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
'2024-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # only created, earlier
record_F = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf5", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
'2026-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # this is invalid...
record_G = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf2", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
'2026-03-26 12:55:18.229966', str(dt.datetime.now() + dt.timedelta(days=2, hours=3, minutes=5, seconds=30))] # created and updated later
cols = ["name",
"ensemble",
"code",
"path",
"project",
"parameters",
"parameter_file",
"created_at",
"updated_at"]
data = [record_A, record_B, record_C, record_D, record_E]
df = pd.DataFrame(data,columns=cols)
for _, result in df.iterrows():
assert integ.has_valid_times(result)
data = [record_F, record_G]
df = pd.DataFrame(data,columns=cols)
for _, result in df.iterrows():
assert not integ.has_valid_times(result)
def test_are_keys_unique(tmp_path: Path) -> None:
db = tmp_path / 'test_success.db'
record_A = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf0", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
'2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] # only created
record_B = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf1", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
'2025-03-26 12:55:18.229966', '2025-04-26 12:55:18.229966'] # created and updated
record_C = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf2", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
'2026-03-26 12:55:18.229966', '2026-04-14 12:55:18.229966'] # created and updated later
record_D = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf3", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
'2026-03-26 12:55:18.229966', '2026-03-27 12:55:18.229966']
record_E = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf4", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
'2024-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # only created, earlier
record_F = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf5", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
'2026-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # this is invalid...
record_G = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf2", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
'2026-03-26 12:55:18.229966', str(dt.datetime.now() + dt.timedelta(days=2, hours=3, minutes=5, seconds=30))] # created and updated later
cols = ["name",
"ensemble",
"code",
"path",
"project",
"parameters",
"parameter_file",
"created_at",
"updated_at"]
data = [record_A, record_B, record_C, record_D, record_E, record_F]
df = pd.DataFrame(data,columns=cols)
conn = sqlite3.connect(db)
df.to_sql('backlogs', conn)
conn.close()
assert integ.are_keys_unique(db, 'backlogs', 'path')
db = tmp_path / 'test_fail.db'
data = [record_A, record_B, record_C, record_D, record_E, record_F, record_G]
df = pd.DataFrame(data,columns=cols)
conn = sqlite3.connect(db)
df.to_sql('backlogs', conn)
conn.close()
assert not integ.are_keys_unique(db, 'backlogs', 'path')
def test_check_path_format() -> None:
record_A = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_A.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
'2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] # only created
record_B = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_B.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
'2025-03-26 12:55:18.229966', '2025-04-26 12:55:18.229966'] # created and updated
record_C = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_A.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
'2026-03-26 12:55:18.229966', '2026-04-14 12:55:18.229966'] # created and updated later
record_D = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_B.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
'2026-03-26 12:55:18.229966', '2026-03-27 12:55:18.229966']
record_E = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_A.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
'2024-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # only created, earlier
record_F = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_B.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
'2026-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # this is invalid...
record_G = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_A.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfas", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
'2026-03-26 12:55:18.229966', str(dt.datetime.now() + dt.timedelta(days=2, hours=3, minutes=5, seconds=30))] # created and updated later
projects = ['Project_A', 'Project_B']
ensembles = ['ensA']
cols = ["name",
"ensemble",
"code",
"path",
"project",
"parameters",
"parameter_file",
"created_at",
"updated_at"]
data = [record_A, record_B, record_C, record_D, record_E, record_F]
df = pd.DataFrame(data,columns=cols)
for _, result in df.iterrows():
integ.check_path_format(result, ensembles, projects)
projects = ['Project_A', 'Project_B']
ensembles = ['ensB']
for _, result in df.iterrows():
with pytest.raises(ValueError):
integ.check_path_format(result, ensembles, projects)
projects = ['Project_A', 'Project_B']
ensembles = ['ensA', 'ensB']
for _, result in df.iterrows():
integ.check_path_format(result, ensembles, projects)
data = [record_G]
df = pd.DataFrame(data,columns=cols)
for _, result in df.iterrows():
with pytest.raises(ValueError):
integ.check_path_format(result, ensembles, projects)