diff --git a/corrlib/cli.py b/corrlib/cli.py index ad1024d..bdee36e 100644 --- a/corrlib/cli.py +++ b/corrlib/cli.py @@ -4,7 +4,7 @@ from corrlib import __app_name__ from .initialization import create from .toml import import_tomls, update_project, reimport_project -from .find import find_record, list_projects, list_ensembles +from .find import find_record, list_projects from .tools import str2list from .main import update_aliases from .meas_io import drop_cache as mio_drop_cache @@ -56,15 +56,15 @@ def lister( """ if entities in ['ensembles', 'Ensembles','ENSEMBLES']: print("Ensembles:") - ensemble_results = list_ensembles(path) - for e in ensemble_results: - print(e) + for item in os.listdir(path / "archive"): + if os.path.isdir(path / "archive" / item): + print(item) elif entities == 'projects': - project_results = list_projects(path) + results = list_projects(path) print("Projects:") header = "UUID".ljust(37) + "| Aliases" print(header) - for project in project_results: + for project in results: if project[1] is not None: aliases = " | ".join(str2list(project[1])) else: diff --git a/corrlib/find.py b/corrlib/find.py index fbdd801..ea696b1 100644 --- a/corrlib/find.py +++ b/corrlib/find.py @@ -382,10 +382,3 @@ def list_projects(path: Path) -> list[tuple[str, str]]: conn.close() return results - -def list_ensembles(path: Path) -> list[str]: - res = [] - for item in os.listdir(path / "archive"): - if os.path.isdir(path / "archive" / item): - res.append(item) - return res diff --git a/corrlib/integrity.py b/corrlib/integrity.py index 66ea4df..f660dfe 100644 --- a/corrlib/integrity.py +++ b/corrlib/integrity.py @@ -7,6 +7,7 @@ from .tracker import get import pyerrors.input.json as pj import os from configparser import ConfigParser + from typing import Any @@ -60,69 +61,7 @@ def are_keys_unique(db: Path, table: str, col: str) -> bool: c.execute(f"SELECT COUNT( DISTINCT CAST({col} AS nvarchar(4000))), COUNT({col}) FROM {table};") results = c.fetchall()[0] conn.close() - res = bool(results[0] == results[1]) - if not res: - print("Unique:", results[0], "All:", results[1]) - return res - - -def _list_projects(path: Path) -> list[tuple[str, str]]: - """ - List all projects known to the library. - - Parameters - ---------- - path: str - The path of the library. - - Returns - ------- - results: list[Any] - The projects known to the library. - """ - db_file = get_db_file(path) - get(path, db_file) - conn = sqlite3.connect(os.path.join(path, db_file)) - c = conn.cursor() - c.execute("SELECT id,aliases FROM projects") - results = c.fetchall() - conn.close() - return results - - -def _list_ensembles(path: Path) -> list[str]: - res = [] - for item in os.listdir(path / "archive"): - if os.path.isdir(path / "archive" / item): - res.append(item) - return res - - -def check_path_format(result: pd.Series, ensembles: list[str], projects: list[str]) -> None: - """ - Check whether the path of the given result has the right format. - - Parameters - ---------- - result: pd.Series - The result to be checked. - """ - p = result['path'] - if not p.startswith('archive'): - raise ValueError(f'The path {p} does not start correctly') - - meas_key = p.split('::')[1] - ensemble = p.split('/')[1] - project = p.split('/')[3].split('.')[0] - if not len(meas_key) == 64: - raise ValueError(f'meas_key of {p} is scrambled') - if ensemble not in ensembles: - raise ValueError(f'meas_key of {p} points to an unknown ensemble') - if project not in projects: - raise ValueError(f'meas_key of {p} points to an unknown project id ({project})') - if not ensemble == result['ensemble']: - raise ValueError(f'Ensemble in database and file does not match for path {p}.') - + return bool(results[0] == results[1]) def check_db_integrity(path: Path) -> None: @@ -143,13 +82,10 @@ def check_db_integrity(path: Path) -> None: search_expr = "SELECT * FROM 'backlogs'" conn = sqlite3.connect(path / db) results = pd.read_sql(search_expr, conn) - ensembles = _list_ensembles(path) - projects = [p[0] for p in _list_projects(path)] for _, result in results.iterrows(): if not has_valid_times(result): raise ValueError(f"Result with id {result[id]} has wrong time signatures.") - check_path_format(result, ensembles, projects) return diff --git a/tests/integrity_test.py b/tests/integrity_test.py deleted file mode 100644 index 8a59d50..0000000 --- a/tests/integrity_test.py +++ /dev/null @@ -1,189 +0,0 @@ -import corrlib.integrity as integ -import corrlib.find as find -import datalad.api as dl -import corrlib.initialization as cinit -import sqlite3 -from pathlib import Path -import os -import pandas as pd -import datetime as dt -import pytest - - -def test_list_ensembles(tmp_path: Path) -> None: - """ - Check against the implementation in find to check if they are the same. - """ - os.mkdir(tmp_path / 'archive') - os.mkdir(tmp_path / 'archive' / 'A') - os.mkdir(tmp_path / 'archive' / 'B') - os.mkdir(tmp_path / 'archive' / 'C') - integ_results = integ._list_ensembles(tmp_path) - assert len(integ_results) == 3 - find_results = find.list_ensembles(tmp_path) - assert len(find_results) == 3 - for f,i in zip(find_results, integ_results): - assert f == i - - -def test_list_projects(tmp_path: Path) -> None: - cinit.create(tmp_path) - db = tmp_path / "backlogger.db" - dl.unlock(str(db), dataset=str(tmp_path)) - conn = sqlite3.connect(db) - c = conn.cursor() - - customTags = "" - owner = "owner" - code = "sfcf" - created_at = "today" - updated_at = "today" - - id = "asdf1" - aliases = "a1,s1,d1,f1" - c.execute("INSERT INTO projects (id, aliases, customTags, owner, code, created_at, updated_at) VALUES (?,?,?,?,?,?,?)", (id, aliases, customTags, owner, code , created_at, updated_at)) - id = "asdf2" - aliases = "a2,s2,d2,f2" - c.execute("INSERT INTO projects (id, aliases, customTags, owner, code, created_at, updated_at) VALUES (?,?,?,?,?,?,?)", (id, aliases, customTags, owner, code , created_at, updated_at)) - id = "asdf3" - aliases = "a3,s3,d3,f3" - c.execute("INSERT INTO projects (id, aliases, customTags, owner, code, created_at, updated_at) VALUES (?,?,?,?,?,?,?)", (id, aliases, customTags, owner, code , created_at, updated_at)) - conn.commit() - conn.close - integ_results = integ._list_projects(tmp_path) - assert len(integ_results) == 3 - find_results = find.list_projects(tmp_path) - assert len(find_results) == 3 - for f,i in zip(find_results, integ_results): - assert f == i - - -def test_has_valid_time() -> None: - record_A = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf0", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", - '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] # only created - record_B = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf1", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", - '2025-03-26 12:55:18.229966', '2025-04-26 12:55:18.229966'] # created and updated - record_C = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf2", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", - '2026-03-26 12:55:18.229966', '2026-04-14 12:55:18.229966'] # created and updated later - record_D = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf3", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", - '2026-03-26 12:55:18.229966', '2026-03-27 12:55:18.229966'] - record_E = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf4", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", - '2024-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # only created, earlier - record_F = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf5", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", - '2026-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # this is invalid... - record_G = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf2", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", - '2026-03-26 12:55:18.229966', str(dt.datetime.now() + dt.timedelta(days=2, hours=3, minutes=5, seconds=30))] # created and updated later - cols = ["name", - "ensemble", - "code", - "path", - "project", - "parameters", - "parameter_file", - "created_at", - "updated_at"] - data = [record_A, record_B, record_C, record_D, record_E] - - df = pd.DataFrame(data,columns=cols) - for _, result in df.iterrows(): - assert integ.has_valid_times(result) - data = [record_F, record_G] - df = pd.DataFrame(data,columns=cols) - for _, result in df.iterrows(): - assert not integ.has_valid_times(result) - - -def test_are_keys_unique(tmp_path: Path) -> None: - db = tmp_path / 'test_success.db' - - record_A = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf0", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", - '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] # only created - record_B = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf1", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", - '2025-03-26 12:55:18.229966', '2025-04-26 12:55:18.229966'] # created and updated - record_C = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf2", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", - '2026-03-26 12:55:18.229966', '2026-04-14 12:55:18.229966'] # created and updated later - record_D = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf3", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", - '2026-03-26 12:55:18.229966', '2026-03-27 12:55:18.229966'] - record_E = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf4", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", - '2024-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # only created, earlier - record_F = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf5", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", - '2026-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # this is invalid... - record_G = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf2", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", - '2026-03-26 12:55:18.229966', str(dt.datetime.now() + dt.timedelta(days=2, hours=3, minutes=5, seconds=30))] # created and updated later - - cols = ["name", - "ensemble", - "code", - "path", - "project", - "parameters", - "parameter_file", - "created_at", - "updated_at"] - - data = [record_A, record_B, record_C, record_D, record_E, record_F] - df = pd.DataFrame(data,columns=cols) - conn = sqlite3.connect(db) - df.to_sql('backlogs', conn) - conn.close() - assert integ.are_keys_unique(db, 'backlogs', 'path') - - db = tmp_path / 'test_fail.db' - data = [record_A, record_B, record_C, record_D, record_E, record_F, record_G] - - df = pd.DataFrame(data,columns=cols) - conn = sqlite3.connect(db) - df.to_sql('backlogs', conn) - conn.close() - assert not integ.are_keys_unique(db, 'backlogs', 'path') - - -def test_check_path_format() -> None: - record_A = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_A.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", - '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] # only created - record_B = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_B.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", - '2025-03-26 12:55:18.229966', '2025-04-26 12:55:18.229966'] # created and updated - record_C = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_A.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", - '2026-03-26 12:55:18.229966', '2026-04-14 12:55:18.229966'] # created and updated later - record_D = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_B.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", - '2026-03-26 12:55:18.229966', '2026-03-27 12:55:18.229966'] - record_E = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_A.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", - '2024-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # only created, earlier - record_F = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_B.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", - '2026-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # this is invalid... - record_G = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_A.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfas", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in", - '2026-03-26 12:55:18.229966', str(dt.datetime.now() + dt.timedelta(days=2, hours=3, minutes=5, seconds=30))] # created and updated later - - projects = ['Project_A', 'Project_B'] - ensembles = ['ensA'] - cols = ["name", - "ensemble", - "code", - "path", - "project", - "parameters", - "parameter_file", - "created_at", - "updated_at"] - - data = [record_A, record_B, record_C, record_D, record_E, record_F] - df = pd.DataFrame(data,columns=cols) - for _, result in df.iterrows(): - integ.check_path_format(result, ensembles, projects) - - projects = ['Project_A', 'Project_B'] - ensembles = ['ensB'] - for _, result in df.iterrows(): - with pytest.raises(ValueError): - integ.check_path_format(result, ensembles, projects) - - projects = ['Project_A', 'Project_B'] - ensembles = ['ensA', 'ensB'] - for _, result in df.iterrows(): - integ.check_path_format(result, ensembles, projects) - - data = [record_G] - df = pd.DataFrame(data,columns=cols) - for _, result in df.iterrows(): - with pytest.raises(ValueError): - integ.check_path_format(result, ensembles, projects)