jkuhl/corrlib
1
0
Fork 0
Code Issues 11 Pull requests 1 Releases Packages Activity Actions

Merge branch 'develop' into feat/fast_import

Browse source
This commit is contained in:
Justus Kuhlmann 2026-07-07 14:39:25 +02:00
commit 21750ec362
Signed by: jkuhl
GPG key ID: 00ED992DD79B85A6
27 changed files with 736 additions and 210 deletions
Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -6,3 +6,4 @@ test.ipynb
.venv
.pytest_cache
.coverage
build

9
LICENSE Normal file
View file

@ -0,0 +1,9 @@
MIT License
Copyright (c) 2026 Justus Kuhlmann
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice (including the next paragraph) shall be included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

6
corrlib/__init__.py
View file

@ -16,9 +16,9 @@ For now, we are interested in collecting primary IObservables only, as these are
__app_name__ = "corrlib"
from . import input as input
from .initialization import create as create
from .meas_io import load_record as load_record
from .meas_io import load_records as load_records
from .find import find_project as find_project
from .find import find_record as find_record
from .find import list_projects as list_projects
from .initialization import create as create
from .meas_io import load_record as load_record
from .meas_io import load_records as load_records

2
corrlib/__main__.py
View file

@ -1,4 +1,4 @@
from corrlib import cli, __app_name__
from corrlib import __app_name__, cli
def main() -> None:

100
corrlib/cli.py
View file

@ -1,21 +1,18 @@
from typing import Optional
import typer
from corrlib import __app_name__
from .initialization import create
from .toml import import_tomls, update_project, reimport_project
from .find import find_record, list_projects
from .tools import str2list
from .main import update_aliases
from .meas_io import drop_cache as mio_drop_cache
from .meas_io import load_record as mio_load_record
from .integrity import full_integrity_check
import os
from pyerrors import Corr
from importlib.metadata import version
from pathlib import Path
import typer
from corrlib import __app_name__
from .find import find_record, get_stat, list_ensembles, list_projects
from .initialization import create
from .integrity import full_integrity_check
from .main import update_aliases
from .meas_io import drop_cache as mio_drop_cache
from .toml import import_tomls, reimport_project, update_project
from .tools import str2list
app = typer.Typer()
@ -28,8 +25,8 @@ def _version_callback(value: bool) -> None:
@app.command()
def update(
path: Path = typer.Option(
Path('./corrlib'),
path: Path = typer.Option( # noqa: B008
Path('.'),
"--dataset",
"-d",
),
@ -44,8 +41,8 @@ def update(
@app.command()
def lister(
path: Path = typer.Option(
Path('./corrlib'),
path: Path = typer.Option( # noqa: B008
Path('.'),
"--dataset",
"-d",
),
@ -56,15 +53,15 @@ def lister(
"""
if entities in ['ensembles', 'Ensembles','ENSEMBLES']:
print("Ensembles:")
for item in os.listdir(path / "archive"):
if os.path.isdir(path / "archive" / item):
print(item)
ensemble_results = list_ensembles(path)
for e in ensemble_results:
print(e)
elif entities == 'projects':
results = list_projects(path)
project_results = list_projects(path)
print("Projects:")
header = "UUID".ljust(37) + "| Aliases"
print(header)
for project in results:
for project in project_results:
if project[1] is not None:
aliases = " | ".join(str2list(project[1]))
else:
@ -75,8 +72,8 @@ def lister(
@app.command()
def alias_add(
path: Path = typer.Option(
Path('./corrlib'),
path: Path = typer.Option( # noqa: B008
Path('.'),
"--dataset",
"-d",
),
@ -93,8 +90,8 @@ def alias_add(
@app.command()
def find(
path: Path = typer.Option(
Path('./corrlib'),
path: Path = typer.Option( # noqa: B008
Path('.'),
"--dataset",
"-d",
),
@ -102,13 +99,13 @@ def find(
corr: str = typer.Argument(),
code: str = typer.Argument(),
arg: str = typer.Option(
str('all'),
'all',
"--argument",
"-a",
),
) -> None:
"""
Find a record in the backlog at hand. Through specifying it's ensemble and the measured correlator.
Find a record in the given backlog.
"""
results = find_record(path, ensemble, corr, code)
if results.empty:
@ -116,14 +113,19 @@ def find(
if arg == 'all':
print(results)
else:
if arg == 'stat':
for r in results['path'].values:
stat = get_stat(path, r)
print(stat)
return
for r in results[arg].values:
print(r)
@app.command()
def stat(
path: Path = typer.Option(
Path('./corrlib'),
path: Path = typer.Option( # noqa: B008
Path('.'),
"--dataset",
"-d",
),
@ -132,35 +134,35 @@ def stat(
"""
Show the statistics of a given record.
"""
record = mio_load_record(path, record_id)
if isinstance(record, (list, Corr)):
record = record[0]
statistics = record.idl
statistics = get_stat(path, record_id)
print(statistics)
return
@app.command()
def check(path: Path = typer.Option(
Path('./corrlib'),
def check(path: Path = typer.Option( # noqa: B008
Path('.'),
"--dataset",
"-d",
),
) -> None:
"""
Check the integrity of the repository.
"""
full_integrity_check(path)
@app.command()
def importer(
path: Path = typer.Option(
Path('./corrlib'),
path: Path = typer.Option( # noqa: B008
Path('.'),
"--dataset",
"-d",
),
files: str = typer.Argument(
),
copy_file: bool = typer.Option(
bool(True),
True,
"--save",
"-s",
),
@ -170,13 +172,14 @@ def importer(
"""
file_list = files.split(",")
import_tomls(path, file_list, copy_file)
mio_drop_cache(path)
return
@app.command()
def reimporter(
path: Path = typer.Option(
Path('./corrlib'),
path: Path = typer.Option( # noqa: B008
Path('.'),
"--dataset",
"-d",
),
@ -194,18 +197,19 @@ def reimporter(
raise Exception("This file is not known for this project.")
else:
reimport_project(path, uuid)
mio_drop_cache(path)
return
@app.command()
def init(
path: Path = typer.Option(
Path('./corrlib'),
path: Path = typer.Option( # noqa: B008
Path('.'),
"--dataset",
"-d",
),
tracker: str = typer.Option(
str('datalad'),
'datalad',
"--tracker",
"-t",
),
@ -219,8 +223,8 @@ def init(
@app.command()
def drop_cache(
path: Path = typer.Option(
Path('./corrlib'),
path: Path = typer.Option( # noqa: B008
Path('.'),
"--dataset",
"-d",
),
@ -234,7 +238,7 @@ def drop_cache(
@app.callback()
def main(
version: Optional[bool] = typer.Option(
version: bool | None = typer.Option(
None,
"--version",
"-v",

61
corrlib/find.py
View file

@ -1,18 +1,22 @@
import sqlite3
import os
import json
import pandas as pd
import numpy as np
from .input.implementations import codes
from .tools import k2m, get_db_file
from .tracker import get
from .integrity import has_valid_times
from .sql import thin_sql_wrapper
from typing import Any, Optional
from pathlib import Path
import datetime as dt
from collections.abc import Callable
import json
import os
import sqlite3
import warnings
from collections.abc import Callable
from pathlib import Path
from typing import Any
import numpy as np
import pandas as pd
from pyerrors import Corr, Obs
from .input.implementations import codes
from .integrity import has_valid_times
from .meas_io import load_record
from .sql import thin_sql_wrapper
from .tools import get_db_file, k2m
from .tracker import get
def _project_lookup_by_alias(path: Path, alias: str) -> str:
@ -61,7 +65,7 @@ def _project_lookup_by_id(path: Path, uuid: str) -> list[tuple[str, ...]]:
return results
def _time_filter(results: pd.DataFrame, created_before: Optional[str]=None, created_after: Optional[Any]=None, updated_before: Optional[Any]=None, updated_after: Optional[Any]=None) -> pd.DataFrame:
def _time_filter(results: pd.DataFrame, created_before: str | None=None, created_after: str | None=None, updated_before: str | None=None, updated_after: str | None=None) -> pd.DataFrame:
"""
Filter the results from the database in terms of the creation and update times.
@ -110,7 +114,7 @@ def _time_filter(results: pd.DataFrame, created_before: Optional[str]=None, cre
return results.drop(drops)
def _db_lookup(db: Path, ensemble: str, correlator_name: str, code: str, project: Optional[str]=None, parameters: Optional[str]=None) -> pd.DataFrame:
def _db_lookup(db: Path, ensemble: str, correlator_name: str, code: str, project: str | None=None, parameters: str | None=None) -> pd.DataFrame:
"""
Look up a correlator record in the database by the data given to the method.
@ -284,7 +288,7 @@ def openQCD_filter(results:pd.DataFrame, **kwargs: Any) -> pd.DataFrame:
The filtered results.
"""
warnings.warn("A filter for openQCD parameters is no implemented yet.", Warning)
warnings.warn("A filter for openQCD parameters is no implemented yet.", Warning, 1)
return results
@ -317,11 +321,12 @@ def _code_filter(results: pd.DataFrame, code: str, **kwargs: Any) -> pd.DataFram
raise ValueError(f"Code {code} is not known.")
def find_record(path: Path, ensemble: str, correlator_name: str, code: str, project: Optional[str]=None, parameters: Optional[str]=None,
created_before: Optional[str]=None, created_after: Optional[str]=None, updated_before: Optional[str]=None, updated_after: Optional[str]=None,
revision: Optional[str]=None,
customFilter: Optional[Callable[[pd.DataFrame], pd.DataFrame]] = None,
def find_record(path: Path, ensemble: str, correlator_name: str, code: str, project: str | None=None, parameters: str | None=None,
created_before: str | None=None, created_after: str | None=None, updated_before: str | None=None, updated_after: str | None=None,
revision: str | None=None,
customFilter: Callable[[pd.DataFrame], pd.DataFrame] | None = None,
**kwargs: Any) -> pd.DataFrame:
path = Path(path)
db_file = get_db_file(path)
db = path / db_file
if code not in codes:
@ -381,3 +386,19 @@ def list_projects(path: Path) -> list[tuple[str, str]]:
conn.close()
return results
def list_ensembles(path: Path) -> list[str]:
res = []
for item in os.listdir(path / "archive"):
if os.path.isdir(path / "archive" / item):
res.append(item)
return res
def get_stat(path: Path, record_id: str) -> Any:
loaded_record: Obs = load_record(path, record_id)
if isinstance(loaded_record, (list, Corr)):
record: Obs = loaded_record[0]
else:
record = loaded_record
return record.idl

8
corrlib/git_tools.py
View file

@ -1,8 +1,10 @@
import os
from .tracker import save
import git
from pathlib import Path
import git
from .tracker import save
GITMODULES_FILE = '.gitmodules'
@ -25,7 +27,7 @@ def move_submodule(repo_path: Path, old_path: Path, new_path: Path) -> None:
gitmodules_file_path = repo_path / GITMODULES_FILE
# update paths in .gitmodules
with open(gitmodules_file_path, 'r') as file:
with open(gitmodules_file_path) as file:
lines = [line.strip() for line in file]
updated_lines = []

9
corrlib/initialization.py
View file

@ -1,9 +1,10 @@
from configparser import ConfigParser
import sqlite3
import os
from .tracker import save, init
import sqlite3
from configparser import ConfigParser
from pathlib import Path
from .tools import CONFIG_FILENAME
from .tracker import init, save
def _create_db(db: Path) -> None:
@ -71,6 +72,7 @@ def _create_config(path: Path, tracker: str, cached: bool) -> ConfigParser:
'db': 'backlogger.db',
'projects_path': 'projects',
'archive_path': 'archive',
'plot_path': 'plots',
'toml_imports_path': 'toml_imports',
'import_scripts_path': 'import_scripts',
}
@ -113,6 +115,7 @@ def create(path: Path, tracker: str = 'datalad', cached: bool = True) -> None:
os.chmod(path / config['paths']['db'], 0o666)
os.makedirs(path / config['paths']['projects_path'])
os.makedirs(path / config['paths']['archive_path'])
os.makedirs(path / config['paths']['plot_path'])
os.makedirs(path / config['paths']['toml_imports_path'])
os.makedirs(path / config['paths']['import_scripts_path'] / 'template.py')
with open(path / ".gitignore", "w") as fp:

4
corrlib/input/__init__.py
View file

@ -2,6 +2,6 @@
Import functions for different codes.
"""
from . import sfcf as sfcf
from . import openQCD as openQCD
from . import implementations as implementations
from . import openQCD as openQCD
from . import sfcf as sfcf

65
corrlib/input/openQCD.py
View file

@ -1,12 +1,14 @@
import pyerrors.input.openQCD as input
import datalad.api as dl
import os
import fnmatch
from typing import Any, Optional
import os
from pathlib import Path
from ..pars.openQCD import ms1
from ..pars.openQCD import qcd2
from typing import Any
import datalad.api as dl
import matplotlib.pyplot as plt
import pyerrors.input.openQCD as input
from ..pars.openQCD import ms1, qcd2
from ..tools import get_plot_dir
def load_ms1_infile(path: Path, project: str, file_in_project: str) -> dict[str, Any]:
@ -29,16 +31,18 @@ def load_ms1_infile(path: Path, project: str, file_in_project: str) -> dict[str,
"""
file = os.path.join(path, "projects", project, file_in_project)
if not os.path.exists(file):
raise OSError(f"File {file} does not exist.")
ds = os.path.join(path, "projects", project)
dl.get(file, dataset=ds)
with open(file, 'r') as fp:
with open(file) as fp:
lines = fp.readlines()
fp.close()
param: dict[str, Any] = {}
param['rw_fcts'] = []
param['rand'] = {}
for i, line in enumerate(lines):
for line in lines:
if line.startswith('#'):
continue
if line.startswith('\n'):
@ -95,7 +99,7 @@ def load_ms3_infile(path: Path, project: str, file_in_project: str) -> dict[str,
file = os.path.join(path, "projects", project, file_in_project)
ds = os.path.join(path, "projects", project)
dl.get(file, dataset=ds)
with open(file, 'r') as fp:
with open(file) as fp:
lines = fp.readlines()
fp.close()
param = {}
@ -107,7 +111,7 @@ def load_ms3_infile(path: Path, project: str, file_in_project: str) -> dict[str,
return param
def read_rwms(path: Path, project: str, dir_in_project: str, param: dict[str, Any], prefix: str, postfix: str="ms1", version: str='2.0', names: Optional[list[str]]=None, files: Optional[list[str]]=None) -> dict[str, Any]:
def read_rwms(path: Path, project: str, dir_in_project: str, param: dict[str, Any], prefix: str, postfix: str="ms1", version: str='2.0', names: list[str] | None=None, files: list[str] | None=None) -> dict[str, Any]:
"""
Read reweighting factor measurements from the project.
@ -142,7 +146,7 @@ def read_rwms(path: Path, project: str, dir_in_project: str, param: dict[str, An
directory = os.path.join(dataset, dir_in_project)
if files is None:
files = []
for root, ds, fs in os.walk(directory):
for _root, _ds, fs in os.walk(directory):
for f in fs:
if fnmatch.fnmatch(f, prefix + "*" + postfix + ".dat"):
files.append(f)
@ -164,7 +168,8 @@ def read_rwms(path: Path, project: str, dir_in_project: str, param: dict[str, An
return rw_dict
def extract_t0(path: Path, project: str, dir_in_project: str, param: dict[str, Any], prefix: str, dtr_read: int, xmin: int, spatial_extent: int, fit_range: int = 5, postfix: str="", names: Optional[list[str]]=None, files: Optional[list[str]]=None) -> dict[str, Any]:
def extract_t0(path: Path, project: str, dir_in_project: str, ensemble: str, param: dict[str, Any], prefix: str, dtr_read: int, xmin: int, spatial_extent: int, fit_range: int = 5, postfix: str="", names: list[str] | None=None, files: list[str] | None=None,
r_start: list[int] | None=None, r_stop: list[int] | None=None, r_step:int=1) -> dict[str, Any]:
"""
Extract t0 measurements from the project.
@ -201,11 +206,15 @@ def extract_t0(path: Path, project: str, dir_in_project: str, param: dict[str, A
Dictionary of t0 values in the pycorrlib style, with the parameters at hand.
"""
if r_stop is None:
r_stop = []
if r_start is None:
r_start = []
dataset = os.path.join(path, "projects", project)
directory = os.path.join(dataset, dir_in_project)
if files is None:
files = []
for root, ds, fs in os.walk(directory):
for _root, _ds, fs in os.walk(directory):
for f in fs:
if fnmatch.fnmatch(f, prefix + "*" + postfix + ".dat"):
files.append(f)
@ -218,7 +227,12 @@ def extract_t0(path: Path, project: str, dir_in_project: str, param: dict[str, A
if postfix is not None:
kwargs['postfix'] = postfix
kwargs['plot_fit'] = False
if not r_start == []:
kwargs['r_start'] = r_start
if not r_stop == []:
kwargs['r_stop'] = r_stop
kwargs['r_step'] = r_step
kwargs['plot_fit'] = True
t0 = input.extract_t0(directory,
prefix,
dtr_read,
@ -228,6 +242,10 @@ def extract_t0(path: Path, project: str, dir_in_project: str, param: dict[str, A
c=0.3,
**kwargs
)
plot_dir = path / get_plot_dir(path) / ensemble / project
if not os.path.exists(plot_dir):
os.makedirs(plot_dir)
plt.savefig(plot_dir / "t0.pdf")
par_list= []
for k in ["integrator", "eps", "ntot", "dnms"]:
par_list.append(str(param[k]))
@ -238,7 +256,8 @@ def extract_t0(path: Path, project: str, dir_in_project: str, param: dict[str, A
return t0_dict
def extract_t1(path: Path, project: str, dir_in_project: str, param: dict[str, Any], prefix: str, dtr_read: int, xmin: int, spatial_extent: int, fit_range: int = 5, postfix: str = "", names: Optional[list[str]]=None, files: Optional[list[str]]=None) -> dict[str, Any]:
def extract_t1(path: Path, project: str, dir_in_project: str, ensemble: str, param: dict[str, Any], prefix: str, dtr_read: int, xmin: int, spatial_extent: int, fit_range: int = 5, postfix: str = "", names: list[str] | None=None, files: list[str] | None=None,
r_start: list[int] | None=None, r_stop: list[int] | None=None, r_step:int=1) -> dict[str, Any]:
"""
Extract t1 measurements from the project.
@ -275,10 +294,14 @@ def extract_t1(path: Path, project: str, dir_in_project: str, param: dict[str, A
Dictionary of t1 values in the pycorrlib style, with the parameters at hand.
"""
if r_stop is None:
r_stop = []
if r_start is None:
r_start = []
directory = os.path.join(path, "projects", project, dir_in_project)
if files is None:
files = []
for root, ds, fs in os.walk(directory):
for _root, _ds, fs in os.walk(directory):
for f in fs:
if fnmatch.fnmatch(f, prefix + "*" + postfix + ".dat"):
files.append(f)
@ -290,6 +313,12 @@ def extract_t1(path: Path, project: str, dir_in_project: str, param: dict[str, A
if postfix is not None:
kwargs['postfix'] = postfix
kwargs['plot_fit'] = False
if not r_start == []:
kwargs['r_start'] = r_start
if not r_stop == []:
kwargs['r_stop'] = r_stop
kwargs['r_step'] = r_step
kwargs['plot_fit'] = True
t0 = input.extract_t0(directory,
prefix,
dtr_read,
@ -299,6 +328,10 @@ def extract_t1(path: Path, project: str, dir_in_project: str, param: dict[str, A
c=2./3,
**kwargs
)
plot_dir = path / get_plot_dir(path) / ensemble / project
if not os.path.exists(plot_dir):
os.makedirs(plot_dir)
plt.savefig(plot_dir / "t1.pdf")
par_list= []
for k in ["integrator", "eps", "ntot", "dnms"]:
par_list.append(str(param[k]))

20
corrlib/input/sfcf.py
View file

@ -1,11 +1,11 @@
import pyerrors as pe
import datalad.api as dl
import json
import os
from typing import Any
from fnmatch import fnmatch
from pathlib import Path
from typing import Any
import datalad.api as dl
import pyerrors as pe
bi_corrs: list[str] = ["f_P", "fP", "f_p",
"g_P", "gP", "g_p",
@ -99,7 +99,7 @@ def read_param(path: Path, project: str, file_in_project: str) -> dict[str, Any]
file = path / "projects" / project / file_in_project
dl.get(file, dataset=path)
with open(file, 'r') as f:
with open(file) as f:
lines = f.readlines()
params: dict[str, Any] = {}
@ -258,7 +258,7 @@ def get_specs(key: str, parameters: dict[str, Any], sep: str = '/') -> str:
return s
def read_data(path: Path, project: str, dir_in_project: str, prefix: str, param: dict[str, Any], version: str = '1.0c', cfg_seperator: str = 'n', sep: str = '/', **kwargs: Any) -> dict[str, Any]:
def read_data(path: Path, project: str, dir_in_project: str, prefix: str, param: dict[str, Any], version: str = '1.0c', cfg_separator: str = 'n', sep: str = '/', **kwargs: Any) -> dict[str, Any]:
"""
Extract the data from the sfcf file.
@ -274,7 +274,7 @@ def read_data(path: Path, project: str, dir_in_project: str, prefix: str, param:
The parameter dictionary, as given by read_param.
version: str
Version of sfcf.
cfg_seperator: str
cfg_separator: str
Separator of the configuration number. Needed for reading. default: "n"
sep: str
Seperator for the key in return dict. (default: "/)
@ -291,7 +291,7 @@ def read_data(path: Path, project: str, dir_in_project: str, prefix: str, param:
appended = (version[-1] == "a")
ls = []
files_to_get = []
for (dirpath, dirnames, filenames) in os.walk(directory):
for _dirpath, dirnames, filenames in os.walk(directory):
if not appended:
ls.extend(dirnames)
else:
@ -299,7 +299,7 @@ def read_data(path: Path, project: str, dir_in_project: str, prefix: str, param:
break
if not appended:
compact = (version[-1] == "c")
for i, item in enumerate(ls):
for item in ls:
if fnmatch(item, prefix + "*"):
rep_path = directory + '/' + item
sub_ls = pe.input.sfcf._find_files(rep_path, prefix, compact, [])
@ -321,10 +321,10 @@ def read_data(path: Path, project: str, dir_in_project: str, prefix: str, param:
if not param['crr'] == []:
if names is not None:
data_crr = pe.input.sfcf.read_sfcf_multi(directory, prefix, param['crr'], param['mrr'], corr_type_list, range(len(param['wf_offsets'])),
range(len(param['wf_basis'])), range(len(param['wf_basis'])), version, cfg_seperator, keyed_out=True, silent=True, names=names)
range(len(param['wf_basis'])), range(len(param['wf_basis'])), version, cfg_separator, keyed_out=True, silent=True, names=names)
else:
data_crr = pe.input.sfcf.read_sfcf_multi(directory, prefix, param['crr'], param['mrr'], corr_type_list, range(len(param['wf_offsets'])),
range(len(param['wf_basis'])), range(len(param['wf_basis'])), version, cfg_seperator, keyed_out=True, silent=True)
range(len(param['wf_basis'])), range(len(param['wf_basis'])), version, cfg_separator, keyed_out=True, silent=True)
for key in data_crr.keys():
data[key] = data_crr[key]

229
corrlib/integrity.py
View file

@ -1,15 +1,34 @@
import datetime as dt
from pathlib import Path
from .tools import get_db_file
import pandas as pd
import os
import sqlite3
from .tracker import get
from configparser import ConfigParser
from pathlib import Path
from typing import Any
import pandas as pd
import pyerrors.input.json as pj
from typing import Any
from .tools import CONFIG_FILENAME, get_db_file
from .tracker import get
path_opts = ['db', 'projects_path', 'archive_path', 'toml_imports_path', 'import_scripts_path']
def has_valid_times(result: pd.Series) -> bool:
"""
Check, whether the result at hand has time-stamps that are sensible:
A recored is created first, then updated, with both times laying in the past.
Parameters
----------
result: pd.Series
The result to check
Returns
-------
b: bool
True, if the timestamps make sense.
"""
# we expect created_at <= updated_at <= now
created_at = dt.datetime.fromisoformat(result['created_at'])
updated_at = dt.datetime.fromisoformat(result['updated_at'])
@ -20,15 +39,103 @@ def has_valid_times(result: pd.Series) -> bool:
return True
def are_keys_unique(db: Path, table: str, col: str) -> bool:
"""
Check whether the strings listed in a column of a given table are unique.
Parameters
----------
db: Path
The database to check.
table: str
The table to check.
col: str
The column to be checked for uniqueness.
Returns
-------
b: bool
True, if the strings are unique.
"""
conn = sqlite3.connect(db)
c = conn.cursor()
c.execute(f"SELECT COUNT( DISTINCT CAST(path AS nvarchar(4000))), COUNT({col}) FROM {table};")
c.execute(f"SELECT COUNT( DISTINCT CAST({col} AS nvarchar(4000))), COUNT({col}) FROM {table};")
results = c.fetchall()[0]
conn.close()
return bool(results[0] == results[1])
res = bool(results[0] == results[1])
if not res:
print("Unique:", results[0], "All:", results[1])
return res
def _list_projects(path: Path) -> list[tuple[str, str]]:
"""
List all projects known to the library.
Parameters
----------
path: str
The path of the library.
Returns
-------
results: list[Any]
The projects known to the library.
"""
db_file = get_db_file(path)
get(path, db_file)
conn = sqlite3.connect(os.path.join(path, db_file))
c = conn.cursor()
c.execute("SELECT id,aliases FROM projects")
results = c.fetchall()
conn.close()
return results
def _list_ensembles(path: Path) -> list[str]:
res = []
for item in os.listdir(path / "archive"):
if os.path.isdir(path / "archive" / item):
res.append(item)
return res
def check_path_format(result: pd.Series, ensembles: list[str], projects: list[str]) -> None:
"""
Check whether the path of the given result has the right format.
Parameters
----------
result: pd.Series
The result to be checked.
"""
p = result['path']
if not p.startswith('archive'):
raise ValueError(f'The path {p} does not start correctly')
meas_key = p.split('::')[1]
ensemble = p.split('/')[1]
project = p.split('/')[3].split('.')[0]
if not len(meas_key) == 64:
raise ValueError(f'meas_key of {p} is scrambled')
if ensemble not in ensembles:
raise ValueError(f'meas_key of {p} points to an unknown ensemble')
if project not in projects:
raise ValueError(f'meas_key of {p} points to an unknown project id ({project})')
if not ensemble == result['ensemble']:
raise ValueError(f'Ensemble in database and file does not match for path {p}.')
def check_db_integrity(path: Path) -> None:
"""
Check intergrity of the database by checking the uniqueness of the record keys used to load the records
and ensuring that the timestamps of each record is sensible. Throws an error, if issues are detected.
Parameters
----------
path: Path
Path to the backlog-library to check.
"""
db = get_db_file(path)
if not are_keys_unique(path / db, 'backlogs', 'path'):
@ -37,15 +144,28 @@ def check_db_integrity(path: Path) -> None:
search_expr = "SELECT * FROM 'backlogs'"
conn = sqlite3.connect(path / db)
results = pd.read_sql(search_expr, conn)
ensembles = _list_ensembles(path)
projects = [p[0] for p in _list_projects(path)]
for _, result in results.iterrows():
if not has_valid_times(result):
raise ValueError(f"Result with id {result[id]} has wrong time signatures.")
print("DB:\t✅")
check_path_format(result, ensembles, projects)
return
def _check_db2paths(path: Path, meas_paths: list[str]) -> None:
"""
Check whether for each record in the given by meas_paths, we can find the data in the file as we expect.
Also check, whether there are unreachable records in the files. If either of the issues arise, throws an error.
Parameters
----------
path: Path
Path to the backlog-library to check.
meas_paths: list[str]
List of measurement paths to check.
"""
needed_data: dict[str, list[str]] = {}
for mpath in meas_paths:
file = mpath.split("::")[0]
@ -67,11 +187,19 @@ def _check_db2paths(path: Path, meas_paths: list[str]) -> None:
for key in needed_data[file]:
if key not in filedict.keys():
raise ValueError(f"Did not find data for key {key} that should be in file {file}.")
print("Links:\t✅")
return
def check_db_file_links(path: Path) -> None:
"""
Check whether for each record in the given correlator library, we can find the data in the file as we expect.
Also check, whether there are unreachable records in the files. If either of the issues arise, throws an error.
Parameters
----------
path: Path
Path to the backlog-library to check.
"""
db = get_db_file(path)
search_expr = "SELECT path FROM 'backlogs'"
conn = sqlite3.connect(path / db)
@ -79,9 +207,90 @@ def check_db_file_links(path: Path) -> None:
_check_db2paths(path, list(results))
def check_path_and_config(path: Path) -> None:
"""
Check whether the given path exists and the cinfigureation file can be found.
Parameters
----------
path: Path
Path to the backlog-library to check.
"""
if not os.path.exists(path):
raise FileNotFoundError(f"Corrlib path {path} does not exist.")
config_path = path / CONFIG_FILENAME
if not os.path.exists(config_path):
raise FileNotFoundError(f"Configuration file {config_path} not found.")
def check_config_validity(path: Path) -> None:
"""
Check whether the configuration file of the given corrlib-dataset path is valid.
Parameters
----------
path: Path
Path to the backlog-library to check.
"""
config = ConfigParser()
config_path = path / CONFIG_FILENAME
if os.path.exists(config_path):
config.read(config_path)
else:
raise FileNotFoundError("Configuration file not found.")
if config.has_section('core'):
core_opts = ['version', 'tracker', 'cached']
has_core_opts = [config.has_option('core', opt) for opt in core_opts]
if not all(has_core_opts):
raise ValueError("One of the options in the 'core' section ('version', 'tracker', 'cached') is missing.")
if config.has_section('paths'):
has_path_opts = [config.has_option('paths', opt) for opt in path_opts]
if not all(has_path_opts):
raise ValueError("One of the options in the 'path' section ('db', 'projects_path', 'archive_path', 'toml_imports_path', 'import_scripts_path') is missing.")
def check_paths(path: Path) -> None:
"""
Check whether all paths demanded by the 'paths' section of the configuration-file exist.
Parameters
----------
path: Path
Path to the backlog-library to check.
"""
config = ConfigParser()
config_path = path / CONFIG_FILENAME
if os.path.exists(config_path):
config.read(config_path)
else:
raise FileNotFoundError("Configuration file not found.")
has_paths = [os.path.exists(path / config.get('paths', opt)) for opt in path_opts]
if not all(has_paths):
raise FileNotFoundError("One of the paths specified in the configuration file is not present.")
def full_integrity_check(path: Path) -> None:
"""
Aggregate all checks for easy validation of the backlog-library.
Parameters
----------
path: Path
Path to the backlog-library to check.
"""
print("Run full integrity check...")
check_path_and_config(path)
print("(1/5) Path and config-file exist: ✅")
check_config_validity(path)
print("(2/5) Configuration is valid: ✅")
check_paths(path)
print("(3/5) Needed paths exist: ✅")
check_db_integrity(path)
print("(4/5) Database is sane: ✅")
check_db_file_links(path)
print("Full:\t✅")
print("(5/5) DB2File and File2DB-links are sound: ✅")
print("Full integrity check: ✅")

30
corrlib/main.py
View file

@ -1,17 +1,18 @@
import sqlite3
import datalad.api as dl
import datalad.config as dlc
import os
from .git_tools import move_submodule
import shutil
from .find import _project_lookup_by_id
from .tools import list2str, str2list, get_db_file
from .tracker import get, save, unlock, clone, drop
from typing import Union, Optional
import sqlite3
from pathlib import Path
import datalad.api as dl
import datalad.config as dlc
def create_project(path: Path, uuid: str, owner: Union[str, None]=None, tags: Union[list[str], None]=None, aliases: Union[list[str], None]=None, code: Union[str, None]=None) -> None:
from .find import _project_lookup_by_id
from .git_tools import move_submodule
from .tools import get_db_file, list2str, str2list
from .tracker import clone, drop, get, save, unlock
def create_project(path: Path, uuid: str, owner: str | None=None, tags: list[str] | None=None, aliases: list[str] | None=None, code: str | None=None) -> None:
"""
Create a new project entry in the database.
@ -27,7 +28,7 @@ def create_project(path: Path, uuid: str, owner: Union[str, None]=None, tags: Un
The code that was used to create the measurements.
"""
db_file = get_db_file(path)
db = os.path.join(path, db_file)
db = path / db_file
get(path, db_file)
conn = sqlite3.connect(db)
c = conn.cursor()
@ -49,7 +50,7 @@ def create_project(path: Path, uuid: str, owner: Union[str, None]=None, tags: Un
return
def update_project_data(path: Path, uuid: str, prop: str, value: Union[str, None] = None) -> None:
def update_project_data(path: Path, uuid: str, prop: str, value: str | None = None) -> None:
"""
Update/Edit a project entry in the database.
Thin wrapper around sql3 call.
@ -67,7 +68,7 @@ def update_project_data(path: Path, uuid: str, prop: str, value: Union[str, None
"""
db_file = get_db_file(path)
get(path, db_file)
conn = sqlite3.connect(os.path.join(path, db_file))
conn = sqlite3.connect(path / db_file)
c = conn.cursor()
c.execute(f"UPDATE projects SET '{prop}' = '{value}' WHERE id == '{uuid}'")
conn.commit()
@ -77,9 +78,8 @@ def update_project_data(path: Path, uuid: str, prop: str, value: Union[str, None
def update_aliases(path: Path, uuid: str, aliases: list[str]) -> None:
db_file = get_db_file(path)
db = path / db_file
get(path, db_file)
known_data = _project_lookup_by_id(db, uuid)[0]
known_data = _project_lookup_by_id(path, uuid)[0]
known_aliases = known_data[1]
if aliases is None:
@ -103,7 +103,7 @@ def update_aliases(path: Path, uuid: str, aliases: list[str]) -> None:
return
def import_project(path: Path, url: str, owner: Union[str, None]=None, tags: Optional[list[str]]=None, aliases: Optional[list[str]]=None, code: Optional[str]=None, isDataset: bool=True) -> str:
def import_project(path: Path, url: str, owner: str | None=None, tags: list[str] | None=None, aliases: list[str] | None=None, code: str | None=None, isDataset: bool=True) -> str:
"""
Import a datalad dataset into the backlogger.

43
corrlib/meas_io.py
View file

@ -1,23 +1,23 @@
from pyerrors.input import json as pj
import os
import sqlite3
from .input import sfcf,openQCD
import json
from typing import Union
from pyerrors import Obs, Corr, dump_object, load_object
from hashlib import sha256
from .tools import get_db_file, cache_enabled
from .tracker import get, save, unlock
import os
import shutil
from typing import Any
import sqlite3
from hashlib import sha256
from pathlib import Path
from .integrity import _check_db2paths
from typing import Any
from pyerrors import Corr, Obs, dump_object, load_object
from pyerrors.input import json as pj
from .input import openQCD, sfcf
from .integrity import _check_db2paths
from .tools import cache_enabled, get_db_file, get_plot_dir
from .tracker import get, save, unlock
CACHE_DIR = ".cache"
def write_measurement(path: Path, ensemble: str, measurement: dict[str, dict[str, dict[str, Any]]], uuid: str, code: str, parameter_file: Union[str, None]) -> None:
def write_measurement(path: Path, ensemble: str, measurement: dict[str, dict[str, dict[str, Any]]], uuid: str, code: str, parameter_file: str | None) -> None:
"""
Write a measurement to the backlog.
If the file for the measurement already exists, update the measurement.
@ -37,6 +37,7 @@ def write_measurement(path: Path, ensemble: str, measurement: dict[str, dict[str
parameter_file: str
The parameter file used for the measurement.
"""
path = Path(path)
db_file = get_db_file(path)
db = path / db_file
@ -50,7 +51,7 @@ def write_measurement(path: Path, ensemble: str, measurement: dict[str, dict[str
c = conn.cursor()
for corr in measurement.keys():
file_in_archive = Path('.') / 'archive' / ensemble / corr / str(uuid + '.json.gz')
file = path / file_in_archive
file = Path(path) / file_in_archive
known_meas = {}
if not os.path.exists(path / 'archive' / ensemble / corr):
os.makedirs(path / 'archive' / ensemble / corr)
@ -72,7 +73,7 @@ def write_measurement(path: Path, ensemble: str, measurement: dict[str, dict[str
pars[subkey] = sfcf.get_specs(corr + "/" + subkey, parameters)
elif code == "openQCD":
ms_type = list(measurement.keys())[0]
ms_type = next(iter(measurement.keys()))
if ms_type == 'ms1':
if parameter_file is not None:
if parameter_file.endswith(".ms1.in"):
@ -103,6 +104,8 @@ def write_measurement(path: Path, ensemble: str, measurement: dict[str, dict[str
subkeys.append(subkey)
pars[subkey] = json.dumps(parameters["rw_fcts"][i])
elif ms_type in ['t0', 't1']:
plot_file = path / get_plot_dir(path) / ensemble / uuid / (ms_type + ".pdf")
files_to_save.append(plot_file)
if parameter_file is not None:
parameters = openQCD.load_ms3_infile(path, uuid, parameter_file)
else:
@ -144,7 +147,7 @@ def affected_files(corrs: list[str], ensemble: str, uuid: str) -> list[Path]:
return file_list
def load_record(path: Path, meas_path: str) -> Union[Corr, Obs]:
def load_record(path: Path, meas_path: str) -> Corr | Obs:
"""
Load a list of records by their paths.
@ -163,7 +166,7 @@ def load_record(path: Path, meas_path: str) -> Union[Corr, Obs]:
return load_records(path, [meas_path])[0]
def load_records(path: Path, meas_paths: list[str], preloaded: dict[str, Any] = {}, dry_run: bool = False) -> list[Union[Corr, Obs]]:
def load_records(path: Path, meas_paths: list[str], preloaded: dict[str, Any] | None = None, dry_run: bool = False) -> list[Corr | Obs]:
"""
Load a list of records by their paths.
@ -183,6 +186,9 @@ def load_records(path: Path, meas_paths: list[str], preloaded: dict[str, Any] =
returned_data: list
The loaded records.
"""
if preloaded is None:
preloaded = {}
path = Path(path)
if dry_run:
_check_db2paths(path, meas_paths)
return []
@ -247,6 +253,7 @@ def cache_path(path: Path, file: str, key: str) -> Path:
cache_path: str
The path at which the measurement of the given file and key is cached.
"""
path = Path(path)
cache_path = cache_dir(path, file) / key
return cache_path
@ -267,6 +274,7 @@ def preload(path: Path, file: Path) -> dict[str, Any]:
filedict: dict[str, Any]
The data read from the file.
"""
path = Path(path)
get(path, file)
filedict: dict[str, Any] = pj.load_json_dict(str(path / file))
print("> read file")
@ -285,7 +293,7 @@ def drop_record(path: Path, meas_path: str) -> None:
The measurement path as noted in the database.
"""
file_in_archive = meas_path.split("::")[0]
file = path / file_in_archive
file = Path(path) / file_in_archive
db_file = get_db_file(path)
db = path / db_file
get(path, db_file)
@ -319,6 +327,7 @@ def drop_cache(path: Path) -> None:
path: str
The path of the library.
"""
path = Path(path)
cache_dir = path / ".cache"
for f in os.listdir(cache_dir):
shutil.rmtree(cache_dir / f)

3
corrlib/pars/openQCD/flags.py
View file

@ -5,6 +5,7 @@ Reconstruct the outputs of flags.
import struct
from typing import Any, BinaryIO
# lat_parms.c
def lat_parms_write_lat_parms(fp: BinaryIO) -> dict[str, Any]:
"""
@ -29,7 +30,7 @@ def lat_parms_write_lat_parms(fp: BinaryIO) -> dict[str, Any]:
kappas = []
m0s = []
# read kappas
for ik in range(nk):
for _ik in range(nk):
t = fp.read(8)
kappas.append(struct.unpack('d', t)[0])
t = fp.read(8)

6
corrlib/pars/openQCD/ms1.py
View file

@ -1,7 +1,7 @@
from . import flags
from typing import Any
from pathlib import Path
from typing import Any
from . import flags
def read_qcd2_ms1_par_file(fname: Path) -> dict[str, dict[str, Any]]:

4
corrlib/pars/openQCD/qcd2.py
View file

@ -1,8 +1,8 @@
from . import flags
from pathlib import Path
from typing import Any
from . import flags
def read_qcd2_par_file(fname: Path) -> dict[str, dict[str, Any]]:
"""

3
corrlib/sql.py
View file

@ -1,8 +1,9 @@
import sqlite3
from .tools import get_db_file
from pathlib import Path
from typing import Any
from .tools import get_db_file
def thin_sql_wrapper(path: Path, stmt: str) -> list[Any]:
db_file = get_db_file(path)

36
corrlib/toml.py
View file

@ -8,20 +8,24 @@ the import of projects via TOML.
"""
import tomllib as toml
import os
import shutil
from pathlib import Path
from typing import Any
import datalad.api as dl
from .tracker import save
from .input import sfcf, openQCD
import tomllib as toml
from .input import openQCD, sfcf
from .input.implementations import codes as known_codes
from .main import import_project, update_aliases
from .meas_io import write_measurement, affected_files
import os
from .input.implementations import codes as known_codes
from tools import step_differences
from typing import Any
from pathlib import Path
from .tools import step_differences
from .tracker import save
def replace_string(string: str, name: str, val: str) -> str:
"""
@ -117,7 +121,7 @@ def check_measurement_data(measurements: dict[str, dict[str, str]], code: str) -
"""
var_names: list[str] = []
if code == "sfcf":
var_names = ["path", "ensemble", "param_file", "version", "prefix", "cfg_seperator", "names"]
var_names = ["path", "ensemble", "param_file", "version", "prefix", "cfg_separator", "names"]
elif code == "openQCD":
var_names = ["path", "ensemble", "measurement", "prefix"] # , "param_file"
for mname, md in measurements.items():
@ -159,6 +163,10 @@ def import_toml(path: Path, file: str, copy_file: bool=True) -> None:
copy_file: bool, optional
Whether the toml-files will be copied into the library. Default is True.
"""
if not os.path.exists(path):
raise FileNotFoundError(f"Corrlib path {path} does not exist.")
if not os.path.exists(file):
raise FileNotFoundError(f".toml-file {file} does not exist.")
print("Import project as decribed in " + file)
with open(file, 'rb') as fp:
toml_dict = toml.load(fp)
@ -210,10 +218,10 @@ def import_toml(path: Path, file: str, copy_file: bool=True) -> None:
param = sfcf.read_param(path, uuid, md['param_file'])
if 'names' in md.keys():
measurement = sfcf.read_data(path, uuid, md['path'], md['prefix'], param,
version=md['version'], cfg_seperator=md['cfg_seperator'], sep='/', names=md['names'])
version=md['version'], cfg_separator=md['cfg_separator'], sep='/', names=md['names'])
else:
measurement = sfcf.read_data(path, uuid, md['path'], md['prefix'], param,
version=md['version'], cfg_seperator=md['cfg_seperator'], sep='/')
version=md['version'], cfg_separator=md['cfg_separator'], sep='/')
elif project['code'] == 'openQCD':
if md['measurement'] == 'ms1':
@ -248,14 +256,16 @@ def import_toml(path: Path, file: str, copy_file: bool=True) -> None:
for rwp in ["integrator", "eps", "ntot", "dnms"]:
param[rwp] = "Unknown"
param['type'] = 't0'
measurement = openQCD.extract_t0(path, uuid, md['path'], param, str(md["prefix"]), int(md["dtr_read"]), int(md["xmin"]), int(md["spatial_extent"]),
fit_range=int(md.get('fit_range', 5)), postfix=str(md.get('postfix', '')), names=md.get('names', []), files=md.get('files', []))
measurement = openQCD.extract_t0(path, uuid, md['path'], ensemble, param, str(md["prefix"]), int(md["dtr_read"]), int(md["xmin"]), int(md["spatial_extent"]),
fit_range=int(md.get('fit_range', 5)), postfix=str(md.get('postfix', '')), names=md.get('names', []), files=md.get('files', []),
r_start=md.get('r_start', []), r_stop=md.get('r_stop', []), r_step=md.get('r_step', 1))
elif md['measurement'] == 't1':
if 'param_file' in md:
param = openQCD.load_ms3_infile(path, uuid, md['param_file'])
param['type'] = 't1'
measurement = openQCD.extract_t1(path, uuid, md['path'], param, str(md["prefix"]), int(md["dtr_read"]), int(md["xmin"]), int(md["spatial_extent"]),
fit_range=int(md.get('fit_range', 5)), postfix=str(md.get('postfix', '')), names=md.get('names', []), files=md.get('files', []))
measurement = openQCD.extract_t1(path, uuid, md['path'], ensemble, param, str(md["prefix"]), int(md["dtr_read"]), int(md["xmin"]), int(md["spatial_extent"]),
fit_range=int(md.get('fit_range', 5)), postfix=str(md.get('postfix', '')), names=md.get('names', []), files=md.get('files', []),
r_start=md.get('r_start', []), r_stop=md.get('r_stop', []), r_step=md.get('r_step', 1))
write_measurement(path, ensemble, measurement, uuid, project['code'], (md['param_file'] if 'param_file' in md else None))
imeas += 1
print(mname + " imported.")
@ -283,7 +293,7 @@ def reimport_project(path: Path, uuid: str) -> None:
uuid of the project that is to be reimported.
"""
config_path = path / "import_scripts" / uuid
for p, filenames, dirnames in os.walk(config_path):
for _p, filenames, _dirnames in os.walk(config_path):
for fname in filenames:
import_toml(path, os.path.join(config_path, fname), copy_file=False)
return

40
corrlib/tools.py
View file

@ -1,7 +1,7 @@
import os
from configparser import ConfigParser
from typing import Any
from pathlib import Path
from typing import Any
CONFIG_FILENAME = ".corrlib"
cached: bool = True
@ -89,7 +89,8 @@ def set_config(path: Path, section: str, option: str, value: Any) -> None:
value: Any
The value we set the option to.
"""
config_path = os.path.join(path, CONFIG_FILENAME)
path = Path(path)
config_path = path / CONFIG_FILENAME
config = ConfigParser()
if os.path.exists(config_path):
config.read(config_path)
@ -115,7 +116,10 @@ def get_db_file(path: Path) -> Path:
db_file: str
The file holding the database.
"""
config_path = os.path.join(path, CONFIG_FILENAME)
path = Path(path)
if not os.path.exists(path):
raise FileNotFoundError(f"Corrlib path {path} does not exist.")
config_path = path / CONFIG_FILENAME
config = ConfigParser()
if os.path.exists(config_path):
config.read(config_path)
@ -125,6 +129,33 @@ def get_db_file(path: Path) -> Path:
return db_file
def get_plot_dir(path: Path) -> Path:
"""
Get the plots directory associated with the library at the given path.
Parameters
----------
path: str
The path of the library.
Returns
-------
db_file: str
The file holding the database.
"""
path = Path(path)
if not os.path.exists(path):
raise FileNotFoundError(f"Corrlib path {path} does not exist.")
config_path = path / CONFIG_FILENAME
config = ConfigParser()
if os.path.exists(config_path):
config.read(config_path)
else:
raise FileNotFoundError("Configuration file not found.")
plot_dir = Path(config.get('paths', 'plot_dir', fallback='plots'))
return plot_dir
def cache_enabled(path: Path) -> bool:
"""
Check, whether the library is cached.
@ -140,7 +171,8 @@ def cache_enabled(path: Path) -> bool:
cached_bool: bool
Whether the given library is cached.
"""
config_path = os.path.join(path, CONFIG_FILENAME)
path = Path(path)
config_path = path / CONFIG_FILENAME
config = ConfigParser()
if os.path.exists(config_path):
config.read(config_path)

34
corrlib/tracker.py
View file

@ -1,11 +1,13 @@
import os
from configparser import ConfigParser
import datalad.api as dl
from typing import Optional
import shutil
from .tools import get_db_file, CONFIG_FILENAME
import warnings
from configparser import ConfigParser
from pathlib import Path
import datalad.api as dl
from .tools import CONFIG_FILENAME, get_db_file
def get_tracker(path: Path) -> str:
"""
@ -21,7 +23,8 @@ def get_tracker(path: Path) -> str:
tracker: str
The tracker used in the dataset.
"""
config_path = os.path.join(path, CONFIG_FILENAME)
path = Path(path)
config_path = path / CONFIG_FILENAME
config = ConfigParser()
if os.path.exists(config_path):
config.read(config_path)
@ -42,6 +45,7 @@ def get(path: Path, file: Path) -> None:
file: str
The file to get.
"""
path = Path(path)
tracker = get_tracker(path)
if tracker == 'datalad':
if file == get_db_file(path):
@ -57,7 +61,7 @@ def get(path: Path, file: Path) -> None:
return
def save(path: Path, message: str, files: Optional[list[Path]]=None) -> None:
def save(path: Path, message: str, files: list[Path] | None=None) -> None:
"""
Wrapper function to save a file to the dataset located at path with the specified tracker.
@ -70,14 +74,14 @@ def save(path: Path, message: str, files: Optional[list[Path]]=None) -> None:
files: list[str], optional
The files to save. If None, all changes are saved.
"""
path = Path(path)
tracker = get_tracker(path)
if tracker == 'datalad':
if files is not None:
files = [path / f for f in files]
dl.save(files, message=message, dataset=path)
elif tracker == 'None':
Warning("Tracker 'None' does not implement save.")
pass
warnings.warn("Tracker 'None' does not implement save.", Warning, 1)
else:
raise ValueError(f"Tracker {tracker} is not supported.")
@ -93,6 +97,7 @@ def init(path: Path, tracker: str='datalad') -> None:
tracker: str
The tracker to use. Currently only 'datalad' and 'None' are supported.
"""
path = Path(path)
if tracker == 'datalad':
dl.create(path)
elif tracker == 'None':
@ -113,12 +118,12 @@ def unlock(path: Path, file: Path) -> None:
file : str
The file to unlock.
"""
path = Path(path)
tracker = get_tracker(path)
if tracker == 'datalad':
dl.unlock(os.path.join(path, file), dataset=path)
elif tracker == 'None':
Warning("Tracker 'None' does not implement unlock.")
pass
warnings.warn("Tracker 'None' does not implement unlock.", Warning, 1)
else:
raise ValueError(f"Tracker {tracker} is not supported.")
return
@ -136,9 +141,10 @@ def clone(path: Path, source: str, target: str) -> None:
target: str
The target path to clone the dataset to.
"""
path = Path(path)
tracker = get_tracker(path)
if tracker == 'datalad':
dl.clone(target=target, source=source, dataset=path)
dl.clone(path=target, source=source, dataset=path)
elif tracker == 'None':
os.makedirs(path, exist_ok=True)
# Implement a simple clone by copying files
@ -148,7 +154,7 @@ def clone(path: Path, source: str, target: str) -> None:
return
def drop(path: Path, reckless: Optional[str]=None) -> None:
def drop(path: Path, reckless: str | None=None) -> None:
"""
Wrapper function to drop data from a dataset located at path with the specified tracker.
@ -159,12 +165,12 @@ def drop(path: Path, reckless: Optional[str]=None) -> None:
reckless: Optional[str]
The datalad's reckless option for dropping data.
"""
path = Path(path)
tracker = get_tracker(path)
if tracker == 'datalad':
dl.drop(path, reckless=reckless)
elif tracker == 'None':
Warning("Tracker 'None' does not implement drop.")
pass
warnings.warn("Tracker 'None' does not implement drop.", Warning, 1)
else:
raise ValueError(f"Tracker {tracker} is not supported.")
return

32
corrlib/version.py
View file

@ -1,34 +1,24 @@
# file generated by setuptools-scm
# file generated by vcs-versioning
# don't change, don't track in version control
from __future__ import annotations
__all__ = [
"__commit_id__",
"__version__",
"__version_tuple__",
"commit_id",
"version",
"version_tuple",
"__commit_id__",
"commit_id",
]
TYPE_CHECKING = False
if TYPE_CHECKING:
from typing import Tuple
from typing import Union
VERSION_TUPLE = Tuple[Union[int, str], ...]
COMMIT_ID = Union[str, None]
else:
VERSION_TUPLE = object
COMMIT_ID = object
version: str
__version__: str
__version_tuple__: VERSION_TUPLE
version_tuple: VERSION_TUPLE
commit_id: COMMIT_ID
__commit_id__: COMMIT_ID
__version_tuple__: tuple[int | str, ...]
version_tuple: tuple[int | str, ...]
commit_id: str | None
__commit_id__: str | None
__version__ = version = '0.2.4.dev14+g602324f84.d20251202'
__version_tuple__ = version_tuple = (0, 2, 4, 'dev14', 'g602324f84.d20251202')
__version__ = version = '0.3.1.dev22+g4b1c21309.d20260701'
__version_tuple__ = version_tuple = (0, 3, 1, 'dev22', 'g4b1c21309.d20260701')
__commit_id__ = commit_id = 'g602324f84'
__commit_id__ = commit_id = 'g4b1c21309'

17
pyproject.toml
View file

@ -11,6 +11,7 @@ dependencies = [
'pyerrors>=2.11.1',
"datalad>=1.1.0",
'typer>=0.12.5',
"matplotlib>=3.10.7",
]
description = "Python correlation library"
authors = [
@ -26,13 +27,17 @@ include = ["corrlib", "corrlib.*"]
[tool.setuptools_scm]
write_to = "corrlib/version.py"
[tool.ruff]
target-version = "py310"
[tool.ruff.lint]
ignore = ["E501"]
extend-select = [
"YTT",
"E",
"W",
"F",
extend-select = ["E", "W", "I", "B", "PIE", "PLE", "PLW", "UP", "NPY", "RUF"]
ignore = [
"F403", # star imports in __init__ files are intentional
"E501", # line too long
"PLC0415", # import outside top level
"PLW2901", # redefined loop name (too noisy)
"RUF002", # ambiguous unicode in docstrings (Greek letters)
]
[tool.mypy]

1
tests/find_test.py
View file

@ -306,7 +306,6 @@ def test_openQCD_filter() -> None:
"updated_at"]
df = pd.DataFrame(data,columns=cols)
with pytest.warns(Warning):
find.openQCD_filter(df, a = "asdf")

2
tests/import_project_test.py
View file

@ -10,7 +10,7 @@ def test_toml_check_measurement_data() -> None:
"param_file": "/path/to/file",
"version": "1.1",
"prefix": "pref",
"cfg_seperator": "n",
"cfg_separator": "n",
"names": ['list', 'of', 'names']
}
}

189
tests/integrity_test.py Normal file
View file

@ -0,0 +1,189 @@
import corrlib.integrity as integ
import corrlib.find as find
import datalad.api as dl
import corrlib.initialization as cinit
import sqlite3
from pathlib import Path
import os
import pandas as pd
import datetime as dt
import pytest
def test_list_ensembles(tmp_path: Path) -> None:
"""
Check against the implementation in find to check if they are the same.
"""
os.mkdir(tmp_path / 'archive')
os.mkdir(tmp_path / 'archive' / 'A')
os.mkdir(tmp_path / 'archive' / 'B')
os.mkdir(tmp_path / 'archive' / 'C')
integ_results = integ._list_ensembles(tmp_path)
assert len(integ_results) == 3
find_results = find.list_ensembles(tmp_path)
assert len(find_results) == 3
for f,i in zip(find_results, integ_results):
assert f == i
def test_list_projects(tmp_path: Path) -> None:
cinit.create(tmp_path)
db = tmp_path / "backlogger.db"
dl.unlock(str(db), dataset=str(tmp_path))
conn = sqlite3.connect(db)
c = conn.cursor()
customTags = ""
owner = "owner"
code = "sfcf"
created_at = "today"
updated_at = "today"
id = "asdf1"
aliases = "a1,s1,d1,f1"
c.execute("INSERT INTO projects (id, aliases, customTags, owner, code, created_at, updated_at) VALUES (?,?,?,?,?,?,?)", (id, aliases, customTags, owner, code , created_at, updated_at))
id = "asdf2"
aliases = "a2,s2,d2,f2"
c.execute("INSERT INTO projects (id, aliases, customTags, owner, code, created_at, updated_at) VALUES (?,?,?,?,?,?,?)", (id, aliases, customTags, owner, code , created_at, updated_at))
id = "asdf3"
aliases = "a3,s3,d3,f3"
c.execute("INSERT INTO projects (id, aliases, customTags, owner, code, created_at, updated_at) VALUES (?,?,?,?,?,?,?)", (id, aliases, customTags, owner, code , created_at, updated_at))
conn.commit()
conn.close
integ_results = integ._list_projects(tmp_path)
assert len(integ_results) == 3
find_results = find.list_projects(tmp_path)
assert len(find_results) == 3
for f,i in zip(find_results, integ_results):
assert f == i
def test_has_valid_time() -> None:
record_A = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf0", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
'2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] # only created
record_B = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf1", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
'2025-03-26 12:55:18.229966', '2025-04-26 12:55:18.229966'] # created and updated
record_C = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf2", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
'2026-03-26 12:55:18.229966', '2026-04-14 12:55:18.229966'] # created and updated later
record_D = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf3", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
'2026-03-26 12:55:18.229966', '2026-03-27 12:55:18.229966']
record_E = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf4", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
'2024-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # only created, earlier
record_F = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf5", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
'2026-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # this is invalid...
record_G = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf2", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
'2026-03-26 12:55:18.229966', str(dt.datetime.now() + dt.timedelta(days=2, hours=3, minutes=5, seconds=30))] # created and updated later
cols = ["name",
"ensemble",
"code",
"path",
"project",
"parameters",
"parameter_file",
"created_at",
"updated_at"]
data = [record_A, record_B, record_C, record_D, record_E]
df = pd.DataFrame(data,columns=cols)
for _, result in df.iterrows():
assert integ.has_valid_times(result)
data = [record_F, record_G]
df = pd.DataFrame(data,columns=cols)
for _, result in df.iterrows():
assert not integ.has_valid_times(result)
def test_are_keys_unique(tmp_path: Path) -> None:
db = tmp_path / 'test_success.db'
record_A = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf0", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
'2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] # only created
record_B = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf1", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
'2025-03-26 12:55:18.229966', '2025-04-26 12:55:18.229966'] # created and updated
record_C = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf2", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
'2026-03-26 12:55:18.229966', '2026-04-14 12:55:18.229966'] # created and updated later
record_D = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf3", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
'2026-03-26 12:55:18.229966', '2026-03-27 12:55:18.229966']
record_E = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf4", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
'2024-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # only created, earlier
record_F = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf5", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
'2026-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # this is invalid...
record_G = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf2", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
'2026-03-26 12:55:18.229966', str(dt.datetime.now() + dt.timedelta(days=2, hours=3, minutes=5, seconds=30))] # created and updated later
cols = ["name",
"ensemble",
"code",
"path",
"project",
"parameters",
"parameter_file",
"created_at",
"updated_at"]
data = [record_A, record_B, record_C, record_D, record_E, record_F]
df = pd.DataFrame(data,columns=cols)
conn = sqlite3.connect(db)
df.to_sql('backlogs', conn)
conn.close()
assert integ.are_keys_unique(db, 'backlogs', 'path')
db = tmp_path / 'test_fail.db'
data = [record_A, record_B, record_C, record_D, record_E, record_F, record_G]
df = pd.DataFrame(data,columns=cols)
conn = sqlite3.connect(db)
df.to_sql('backlogs', conn)
conn.close()
assert not integ.are_keys_unique(db, 'backlogs', 'path')
def test_check_path_format() -> None:
record_A = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_A.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
'2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] # only created
record_B = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_B.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
'2025-03-26 12:55:18.229966', '2025-04-26 12:55:18.229966'] # created and updated
record_C = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_A.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
'2026-03-26 12:55:18.229966', '2026-04-14 12:55:18.229966'] # created and updated later
record_D = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_B.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
'2026-03-26 12:55:18.229966', '2026-03-27 12:55:18.229966']
record_E = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_A.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
'2024-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # only created, earlier
record_F = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_B.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
'2026-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # this is invalid...
record_G = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_A.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfas", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
'2026-03-26 12:55:18.229966', str(dt.datetime.now() + dt.timedelta(days=2, hours=3, minutes=5, seconds=30))] # created and updated later
projects = ['Project_A', 'Project_B']
ensembles = ['ensA']
cols = ["name",
"ensemble",
"code",
"path",
"project",
"parameters",
"parameter_file",
"created_at",
"updated_at"]
data = [record_A, record_B, record_C, record_D, record_E, record_F]
df = pd.DataFrame(data,columns=cols)
for _, result in df.iterrows():
integ.check_path_format(result, ensembles, projects)
projects = ['Project_A', 'Project_B']
ensembles = ['ensB']
for _, result in df.iterrows():
with pytest.raises(ValueError):
integ.check_path_format(result, ensembles, projects)
projects = ['Project_A', 'Project_B']
ensembles = ['ensA', 'ensB']
for _, result in df.iterrows():
integ.check_path_format(result, ensembles, projects)
data = [record_G]
df = pd.DataFrame(data,columns=cols)
for _, result in df.iterrows():
with pytest.raises(ValueError):
integ.check_path_format(result, ensembles, projects)

2
uv.lock generated
View file

@ -409,6 +409,7 @@ source = { editable = "." }
dependencies = [
{ name = "datalad" },
{ name = "gitpython" },
{ name = "matplotlib" },
{ name = "pyerrors" },
{ name = "typer" },
]
@ -427,6 +428,7 @@ dev = [
requires-dist = [
{ name = "datalad", specifier = ">=1.1.0" },
{ name = "gitpython", specifier = ">=3.1.45" },
{ name = "matplotlib", specifier = ">=3.10.7" },
{ name = "pyerrors", specifier = ">=2.11.1" },
{ name = "typer", specifier = ">=0.12.5" },
]