296 lines
9.3 KiB
Python
296 lines
9.3 KiB
Python
import datetime as dt
|
|
import os
|
|
import sqlite3
|
|
from configparser import ConfigParser
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import pandas as pd
|
|
import pyerrors.input.json as pj
|
|
|
|
from .tools import CONFIG_FILENAME, get_db_file
|
|
from .tracker import get
|
|
|
|
path_opts = ['db', 'projects_path', 'archive_path', 'toml_imports_path', 'import_scripts_path']
|
|
|
|
|
|
def has_valid_times(result: pd.Series) -> bool:
|
|
"""
|
|
Check, whether the result at hand has time-stamps that are sensible:
|
|
A recored is created first, then updated, with both times laying in the past.
|
|
|
|
Parameters
|
|
----------
|
|
result: pd.Series
|
|
The result to check
|
|
|
|
Returns
|
|
-------
|
|
b: bool
|
|
True, if the timestamps make sense.
|
|
"""
|
|
# we expect created_at <= updated_at <= now
|
|
created_at = dt.datetime.fromisoformat(result['created_at'])
|
|
updated_at = dt.datetime.fromisoformat(result['updated_at'])
|
|
if created_at > updated_at:
|
|
return False
|
|
if updated_at > dt.datetime.now():
|
|
return False
|
|
return True
|
|
|
|
def are_keys_unique(db: Path, table: str, col: str) -> bool:
|
|
"""
|
|
Check whether the strings listed in a column of a given table are unique.
|
|
|
|
Parameters
|
|
----------
|
|
db: Path
|
|
The database to check.
|
|
table: str
|
|
The table to check.
|
|
col: str
|
|
The column to be checked for uniqueness.
|
|
|
|
Returns
|
|
-------
|
|
b: bool
|
|
True, if the strings are unique.
|
|
"""
|
|
conn = sqlite3.connect(db)
|
|
c = conn.cursor()
|
|
c.execute(f"SELECT COUNT( DISTINCT CAST({col} AS nvarchar(4000))), COUNT({col}) FROM {table};")
|
|
results = c.fetchall()[0]
|
|
conn.close()
|
|
res = bool(results[0] == results[1])
|
|
if not res:
|
|
print("Unique:", results[0], "All:", results[1])
|
|
return res
|
|
|
|
|
|
def _list_projects(path: Path) -> list[tuple[str, str]]:
|
|
"""
|
|
List all projects known to the library.
|
|
|
|
Parameters
|
|
----------
|
|
path: str
|
|
The path of the library.
|
|
|
|
Returns
|
|
-------
|
|
results: list[Any]
|
|
The projects known to the library.
|
|
"""
|
|
db_file = get_db_file(path)
|
|
get(path, db_file)
|
|
conn = sqlite3.connect(os.path.join(path, db_file))
|
|
c = conn.cursor()
|
|
c.execute("SELECT id,aliases FROM projects")
|
|
results = c.fetchall()
|
|
conn.close()
|
|
return results
|
|
|
|
|
|
def _list_ensembles(path: Path) -> list[str]:
|
|
res = []
|
|
for item in os.listdir(path / "archive"):
|
|
if os.path.isdir(path / "archive" / item):
|
|
res.append(item)
|
|
return res
|
|
|
|
|
|
def check_path_format(result: pd.Series, ensembles: list[str], projects: list[str]) -> None:
|
|
"""
|
|
Check whether the path of the given result has the right format.
|
|
|
|
Parameters
|
|
----------
|
|
result: pd.Series
|
|
The result to be checked.
|
|
"""
|
|
p = result['path']
|
|
if not p.startswith('archive'):
|
|
raise ValueError(f'The path {p} does not start correctly')
|
|
|
|
meas_key = p.split('::')[1]
|
|
ensemble = p.split('/')[1]
|
|
project = p.split('/')[3].split('.')[0]
|
|
if not len(meas_key) == 64:
|
|
raise ValueError(f'meas_key of {p} is scrambled')
|
|
if ensemble not in ensembles:
|
|
raise ValueError(f'meas_key of {p} points to an unknown ensemble')
|
|
if project not in projects:
|
|
raise ValueError(f'meas_key of {p} points to an unknown project id ({project})')
|
|
if not ensemble == result['ensemble']:
|
|
raise ValueError(f'Ensemble in database and file does not match for path {p}.')
|
|
|
|
|
|
|
|
def check_db_integrity(path: Path) -> None:
|
|
"""
|
|
Check intergrity of the database by checking the uniqueness of the record keys used to load the records
|
|
and ensuring that the timestamps of each record is sensible. Throws an error, if issues are detected.
|
|
|
|
Parameters
|
|
----------
|
|
path: Path
|
|
Path to the backlog-library to check.
|
|
"""
|
|
db = get_db_file(path)
|
|
|
|
if not are_keys_unique(path / db, 'backlogs', 'path'):
|
|
raise Exception("The paths the backlog table of the database links are not unique.")
|
|
|
|
search_expr = "SELECT * FROM 'backlogs'"
|
|
conn = sqlite3.connect(path / db)
|
|
results = pd.read_sql(search_expr, conn)
|
|
ensembles = _list_ensembles(path)
|
|
projects = [p[0] for p in _list_projects(path)]
|
|
|
|
for _, result in results.iterrows():
|
|
if not has_valid_times(result):
|
|
raise ValueError(f"Result with id {result[id]} has wrong time signatures.")
|
|
check_path_format(result, ensembles, projects)
|
|
return
|
|
|
|
|
|
def _check_db2paths(path: Path, meas_paths: list[str]) -> None:
|
|
"""
|
|
Check whether for each record in the given by meas_paths, we can find the data in the file as we expect.
|
|
Also check, whether there are unreachable records in the files. If either of the issues arise, throws an error.
|
|
|
|
Parameters
|
|
----------
|
|
path: Path
|
|
Path to the backlog-library to check.
|
|
meas_paths: list[str]
|
|
List of measurement paths to check.
|
|
"""
|
|
needed_data: dict[str, list[str]] = {}
|
|
for mpath in meas_paths:
|
|
file = mpath.split("::")[0]
|
|
if file not in needed_data.keys():
|
|
needed_data[file] = []
|
|
key = mpath.split("::")[1]
|
|
needed_data[file].append(key)
|
|
|
|
totf = len(needed_data.keys())
|
|
for i, file in enumerate(needed_data.keys()):
|
|
print(f"Check against file {i}/{totf}: {file}")
|
|
get(path, Path(file))
|
|
filedict: dict[str, Any] = pj.load_json_dict(str(path / file))
|
|
if not set(filedict.keys()).issubset(needed_data[file]):
|
|
for key in filedict.keys():
|
|
if key not in needed_data[file]:
|
|
raise ValueError(f"Found unintended key {key} in file {file}.")
|
|
if not set(needed_data[file]).issubset(filedict.keys()):
|
|
for key in needed_data[file]:
|
|
if key not in filedict.keys():
|
|
raise ValueError(f"Did not find data for key {key} that should be in file {file}.")
|
|
return
|
|
|
|
|
|
def check_db_file_links(path: Path) -> None:
|
|
"""
|
|
Check whether for each record in the given correlator library, we can find the data in the file as we expect.
|
|
Also check, whether there are unreachable records in the files. If either of the issues arise, throws an error.
|
|
|
|
Parameters
|
|
----------
|
|
path: Path
|
|
Path to the backlog-library to check.
|
|
"""
|
|
db = get_db_file(path)
|
|
search_expr = "SELECT path FROM 'backlogs'"
|
|
conn = sqlite3.connect(path / db)
|
|
results = pd.read_sql(search_expr, conn)['path'].values
|
|
_check_db2paths(path, list(results))
|
|
|
|
|
|
def check_path_and_config(path: Path) -> None:
|
|
"""
|
|
Check whether the given path exists and the cinfigureation file can be found.
|
|
|
|
Parameters
|
|
----------
|
|
path: Path
|
|
Path to the backlog-library to check.
|
|
"""
|
|
if not os.path.exists(path):
|
|
raise FileNotFoundError(f"Corrlib path {path} does not exist.")
|
|
config_path = path / CONFIG_FILENAME
|
|
if not os.path.exists(config_path):
|
|
raise FileNotFoundError(f"Configuration file {config_path} not found.")
|
|
|
|
|
|
def check_config_validity(path: Path) -> None:
|
|
"""
|
|
Check whether the configuration file of the given corrlib-dataset path is valid.
|
|
|
|
Parameters
|
|
----------
|
|
path: Path
|
|
Path to the backlog-library to check.
|
|
"""
|
|
config = ConfigParser()
|
|
config_path = path / CONFIG_FILENAME
|
|
if os.path.exists(config_path):
|
|
config.read(config_path)
|
|
else:
|
|
raise FileNotFoundError("Configuration file not found.")
|
|
|
|
if config.has_section('core'):
|
|
core_opts = ['version', 'tracker', 'cached']
|
|
has_core_opts = [config.has_option('core', opt) for opt in core_opts]
|
|
if not all(has_core_opts):
|
|
raise ValueError("One of the options in the 'core' section ('version', 'tracker', 'cached') is missing.")
|
|
|
|
if config.has_section('paths'):
|
|
has_path_opts = [config.has_option('paths', opt) for opt in path_opts]
|
|
if not all(has_path_opts):
|
|
raise ValueError("One of the options in the 'path' section ('db', 'projects_path', 'archive_path', 'toml_imports_path', 'import_scripts_path') is missing.")
|
|
|
|
|
|
def check_paths(path: Path) -> None:
|
|
"""
|
|
Check whether all paths demanded by the 'paths' section of the configuration-file exist.
|
|
|
|
Parameters
|
|
----------
|
|
path: Path
|
|
Path to the backlog-library to check.
|
|
"""
|
|
config = ConfigParser()
|
|
config_path = path / CONFIG_FILENAME
|
|
if os.path.exists(config_path):
|
|
config.read(config_path)
|
|
else:
|
|
raise FileNotFoundError("Configuration file not found.")
|
|
has_paths = [os.path.exists(path / config.get('paths', opt)) for opt in path_opts]
|
|
if not all(has_paths):
|
|
raise FileNotFoundError("One of the paths specified in the configuration file is not present.")
|
|
|
|
|
|
def full_integrity_check(path: Path) -> None:
|
|
"""
|
|
Aggregate all checks for easy validation of the backlog-library.
|
|
|
|
Parameters
|
|
----------
|
|
path: Path
|
|
Path to the backlog-library to check.
|
|
"""
|
|
print("Run full integrity check...")
|
|
check_path_and_config(path)
|
|
print("(1/5) Path and config-file exist: ✅")
|
|
check_config_validity(path)
|
|
print("(2/5) Configuration is valid: ✅")
|
|
check_paths(path)
|
|
print("(3/5) Needed paths exist: ✅")
|
|
check_db_integrity(path)
|
|
print("(4/5) Database is sane: ✅")
|
|
check_db_file_links(path)
|
|
print("(5/5) DB2File and File2DB-links are sound: ✅")
|
|
print("Full integrity check: ✅")
|
|
|
|
|