Merge pull request 'develop' (#41) from develop into master

Reviewed-on: #41

.github/workflows/mypy.yaml

@@ -0,0 +1,29 @@
+name: Mypy
+
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  mypy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install git-annex
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y git-annex 
+      - name: Check out the repository
+        uses: https://github.com/RouxAntoine/checkout@v4.1.8
+        with:
+          show-progress: true
+      - name: Setup python
+        uses: https://github.com/actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Install uv
+        uses: https://github.com/astral-sh/setup-uv@v5
+      - name: Install corrlib
+        run: uv sync --locked --all-extras --dev --python "3.12"
+      - name: Run tests
+        run: uv run mypy corrlib

.github/workflows/pytest.yaml

@@ -0,0 +1,42 @@
+name: Pytest
+
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 4 1 * *'
+
+jobs:
+  pytest:
+    strategy:
+      matrix:
+        python-version:
+          - "3.12"
+          - "3.13"
+          - "3.14"
+
+    runs-on: ubuntu-latest
+    steps:
+      - name: Setup git
+        run: |
+          git config --global user.email "tester@example.com"
+          git config --global user.name "Tester"
+      - name: Install git-annex
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y git-annex
+      - name: Check out the repository
+        uses: https://github.com/RouxAntoine/checkout@v4.1.8
+        with:
+          show-progress: true
+      - name: Setup python
+        uses: https://github.com/actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install uv
+        uses: https://github.com/astral-sh/setup-uv@v5
+      - name: Install corrlib
+        run: uv sync --locked --all-extras --dev --python ${{ matrix.python-version }}
+      - name: Run tests
+        run: uv run pytest --cov=corrlib tests

.github/workflows/ruff.yaml

@@ -0,0 +1,30 @@
+name: Ruff
+
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  ruff:
+
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install git-annex
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y git-annex
+      - name: Check out the repository
+        uses: https://github.com/RouxAntoine/checkout@v4.1.8
+        with:
+          show-progress: true
+      - name: Setup python
+        uses: https://github.com/actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Install uv
+        uses: https://github.com/astral-sh/setup-uv@v5
+      - name: Install corrlib
+        run: uv sync --locked --all-extras --dev --python "3.12"
+      - name: Run tests
+        run: uv run ruff check corrlib

.gitignore

@@ -2,3 +2,7 @@ pyerrors_corrlib.egg-info
 __pycache__
 *.egg-info
 test.ipynb
+.vscode
+.venv
+.pytest_cache
+.coverage

corrlib/__init__.py

@@ -15,9 +15,10 @@ For now, we are interested in collecting primary IObservables only, as these are
 
 __app_name__ = "corrlib"
 
-from .main import *
 from .import input as input
-from .initialization import *
-from .meas_io import *
-from .find import *
-from .version import __version__
+from .initialization import create as create
+from .meas_io import load_record as load_record
+from .meas_io import load_records as load_records
+from .find import find_project as find_project
+from .find import find_record as find_record
+from .find import list_projects as list_projects

corrlib/__main__.py

@@ -1,8 +1,9 @@
 from corrlib import cli, __app_name__
 
 
-def main():
+def main() -> None:
     cli.app(prog_name=__app_name__)
+    return
 
 
 if __name__ == "__main__":

corrlib/cli.py

@@ -1,13 +1,20 @@
 from typing import Optional
 import typer
-from corrlib import __app_name__, __version__
+from corrlib import __app_name__
+
 from .initialization import create
 from .toml import import_tomls, update_project, reimport_project
-from .find import find_record, list_projects
+from .find import find_record, list_projects, list_ensembles
 from .tools import str2list
 from .main import update_aliases
 from .meas_io import drop_cache as mio_drop_cache
+from .meas_io import load_record as mio_load_record
+from .integrity import full_integrity_check
+
 import os
+from pyerrors import Corr
+from importlib.metadata import version
+from pathlib import Path
 
 
 app = typer.Typer()
@@ -15,48 +22,49 @@ app = typer.Typer()
 
 def _version_callback(value: bool) -> None:
     if value:
-        typer.echo(f"{__app_name__} v{__version__}")
+        print(__app_name__, version(__app_name__))
         raise typer.Exit()
 
 
 @app.command()
 def update(
-    path: str = typer.Option(
-        str('./corrlib'),
+    path: Path = typer.Option(
+        Path('.'),
         "--dataset",
         "-d",
         ),
     uuid: str = typer.Argument(),
-) -> None:
+    ) -> None:
     """
     Update a project by it's UUID.
     """
     update_project(path, uuid)
     return
 
+
 @app.command()
-def list(
-    path: str = typer.Option(
-        str('./corrlib'),
+def lister(
+    path: Path = typer.Option(
+        Path('.'),
         "--dataset",
         "-d",
     ),
     entities: str = typer.Argument('ensembles'),
-) -> None:
+    ) -> None:
     """
     List entities. (ensembles, projects)
     """
     if entities in ['ensembles', 'Ensembles','ENSEMBLES']:
         print("Ensembles:")
-        for item in os.listdir(path + "/archive"):
-            if os.path.isdir(os.path.join(path + "/archive", item)):
-                print(item)
+        ensemble_results = list_ensembles(path)
+        for e in ensemble_results:
+            print(e)
     elif entities == 'projects':
-        results = list_projects(path)
+        project_results = list_projects(path)
         print("Projects:")
         header = "UUID".ljust(37) + "| Aliases"
         print(header)
-        for project in results:
+        for project in project_results:
             if project[1] is not None:
                 aliases = " | ".join(str2list(project[1]))
             else:
@@ -67,14 +75,14 @@ def list(
 
 @app.command()
 def alias_add(
-    path: str = typer.Option(
-        str('./corrlib'),
+    path: Path = typer.Option(
+        Path('.'),
         "--dataset",
         "-d",
         ),
     uuid: str = typer.Argument(),
     alias: str = typer.Argument(),
-) -> None:
+    ) -> None:
     """
     Add an alias to a project UUID.
     """
@@ -85,35 +93,70 @@ def alias_add(
 
 @app.command()
 def find(
-    path: str = typer.Option(
-        str('./corrlib'),
+    path: Path = typer.Option(
+        Path('.'),
         "--dataset",
         "-d",
         ),
-    arg: str = typer.Option(
-        str('all'),
-        "--show",
-        "-s",
-    ),
     ensemble: str = typer.Argument(),
     corr: str = typer.Argument(),
     code: str = typer.Argument(),
-) -> None:
+    arg: str = typer.Option(
+        str('all'),
+        "--argument",
+        "-a",
+    ),
+    ) -> None:
     """
-    Find a record in the backlog at hand. Through specifying it's ensemble and the measured correlator.
+    Find a record in the given backlog.
     """
     results = find_record(path, ensemble, corr, code)
-    if arg == "all":
+    if results.empty:
+        return
+    if arg == 'all':
         print(results)
     else:
-        for i in range(len(results)):
-            print(results[arg].values[i])
+        for r in results[arg].values:
+            print(r)
+
+
+@app.command()
+def stat(
+    path: Path = typer.Option(
+        Path('.'),
+        "--dataset",
+        "-d",
+        ),
+    record_id: str = typer.Argument(),
+    ) -> None:
+    """
+    Show the statistics of a given record.
+    """
+    record = mio_load_record(path, record_id)
+    if isinstance(record, (list, Corr)):
+        record = record[0]
+    statistics = record.idl
+    print(statistics)
     return
 
+
+@app.command()
+def check(path: Path = typer.Option(
+        Path('.'),
+        "--dataset",
+        "-d",
+        ),
+    ) -> None:
+    """
+    Check the integrity of the repository.
+    """
+    full_integrity_check(path)
+
+
 @app.command()
 def importer(
-    path: str = typer.Option(
-        str('./corrlib'),
+    path: Path = typer.Option(
+        Path('.'),
         "--dataset",
         "-d",
         ),
@@ -124,7 +167,7 @@ def importer(
         "--save",
         "-s",
         ),
-) -> None:
+    ) -> None:
     """
     Import a project from a .toml-file via CLI.
     """
@@ -135,13 +178,16 @@ def importer(
 
 @app.command()
 def reimporter(
-    path: str = typer.Option(
-        str('./corrlib'),
+    path: Path = typer.Option(
+        Path('.'),
         "--dataset",
         "-d",
         ),
     ident: str = typer.Argument()
     ) -> None:
+    """
+    Reimport the toml file identfied by the ident string.
+    """
     uuid = ident.split("::")[0]
     if len(ident.split("::")) > 1:
         toml_file = os.path.join(path, "toml_imports", ident.split("::")[1])
@@ -156,27 +202,32 @@ def reimporter(
 
 @app.command()
 def init(
-    path: str = typer.Option(
-        str('./corrlib'),
+    path: Path = typer.Option(
+        Path('.'),
         "--dataset",
         "-d",
         ),
-) -> None:
+    tracker: str = typer.Option(
+        str('datalad'),
+        "--tracker",
+        "-t",
+        ),
+    ) -> None:
     """
     Initialize a new backlog-database.
     """
-    create(path)
+    create(path, tracker)
     return
 
 
 @app.command()
 def drop_cache(
-    path: str = typer.Option(
-        str('./corrlib'),
+    path: Path = typer.Option(
+        Path('.'),
         "--dataset",
         "-d",
         ),
-) -> None:
+    ) -> None:
     """
     Drop the currect cache directory of the dataset.
     """
@@ -194,5 +245,5 @@ def main(
         callback=_version_callback,
         is_eager=True,
         )
-) -> None:
+    ) -> None:
     return

corrlib/find.py

@@ -1,38 +1,148 @@
 import sqlite3
-import datalad.api as dl
 import os
 import json
 import pandas as pd
 import numpy as np
 from .input.implementations import codes
-from .tools import k2m
-# this will implement the search functionality
+from .tools import k2m, get_db_file
+from .tracker import get
+from .integrity import has_valid_times
+from .sql import thin_sql_wrapper
+from typing import Any, Optional
+from pathlib import Path
+import datetime as dt
+from collections.abc import Callable
+import warnings
 
 
-def _project_lookup_by_alias(db, alias):
-    # this will lookup the project name based on the alias
-    conn = sqlite3.connect(db)
-    c = conn.cursor()
-    c.execute(f"SELECT * FROM 'projects' WHERE alias = '{alias}'")
-    results = c.fetchall()
-    conn.close()
-    if len(results) > 1:
+def _project_lookup_by_alias(path: Path, alias: str) -> str:
+    """
+    Lookup a projects UUID by its (human-readable) alias.
+
+    Parameters
+    ----------
+    db: str
+        The database to look up the project.
+    alias: str
+        The alias to look up.
+
+    Returns
+    -------
+    uuid: str
+        The UUID of the project with the given alias.
+    """
+    stmt = f"SELECT * FROM 'projects' WHERE aliases = '{alias}'"
+    results = thin_sql_wrapper(path, stmt)
+    if len(results)>1:
         print("Error: multiple projects found with alias " + alias)
     elif len(results) == 0:
         raise Exception("Error: no project found with alias " + alias)
-    return results[0][0]
+    return str(results[0][0])
 
 
-def _project_lookup_by_id(db, uuid):
-    conn = sqlite3.connect(db)
-    c = conn.cursor()
-    c.execute(f"SELECT * FROM 'projects' WHERE id = '{uuid}'")
-    results = c.fetchall()
-    conn.close()
+def _project_lookup_by_id(path: Path, uuid: str) -> list[tuple[str, ...]]:
+    """
+    Return the project information available in the database by UUID.
+
+    Parameters
+    ----------
+    db: str
+        The database to look up the project.
+    uuid: str
+        The uuid of the project in question.
+
+    Returns
+    -------
+    results: list
+        The row of the project in the database.
+    """
+    stmt = f"SELECT * FROM 'projects' WHERE id = '{uuid}'"
+    results = thin_sql_wrapper(path, stmt)
     return results
 
 
-def _db_lookup(db, ensemble, correlator_name,code, project=None,  parameters=None, created_before=None, created_after=None, updated_before=None, updated_after=None, revision=None):
+def _time_filter(results: pd.DataFrame,  created_before: Optional[str]=None, created_after: Optional[Any]=None, updated_before: Optional[Any]=None, updated_after: Optional[Any]=None) -> pd.DataFrame:
+    """
+    Filter the results from the database in terms of the creation and update times.
+
+    Parameters
+    ----------
+    results: pd.DataFrame
+        The dataframe holding the unfilteres results from the database.
+    created_before: str
+        Contraint on the creation date in datetime.datetime.isoformat. Note that this is exclusive. The creation date has to be truly before the date and time given.
+    created_after: str
+        Contraint on the creation date in datetime.datetime.isoformat. Note that this is exclusive. The creation date has to be truly after the date and time given.
+    updated_before: str
+        Contraint on the creation date in datetime.datetime.isoformat. Note that this is exclusive. The date of the last update has to be truly before the date and time given.
+    updated_after: str
+        Contraint on the creation date in datetime.datetime.isoformat. Note that this is exclusive. The date of the last update has to be truly after the date and time given.
+    """
+    drops = []
+    for ind in range(len(results)):
+        result = results.iloc[ind]
+        created_at = dt.datetime.fromisoformat(result['created_at'])
+        updated_at = dt.datetime.fromisoformat(result['updated_at'])
+        db_times_valid = has_valid_times(result)
+        if not db_times_valid:
+            raise ValueError('Time stamps not valid for result with path', result["path"])
+
+        if created_before is not None:
+            date_created_before = dt.datetime.fromisoformat(created_before)
+            if date_created_before < created_at:
+                drops.append(ind)
+                continue
+        if created_after is not None:
+            date_created_after = dt.datetime.fromisoformat(created_after)
+            if date_created_after > created_at:
+                drops.append(ind)
+                continue
+        if updated_before is not None:
+            date_updated_before = dt.datetime.fromisoformat(updated_before)
+            if date_updated_before < updated_at:
+                drops.append(ind)
+                continue
+        if updated_after is not None:
+            date_updated_after = dt.datetime.fromisoformat(updated_after)
+            if date_updated_after > updated_at:
+                drops.append(ind)
+                continue
+    return results.drop(drops)
+
+
+def _db_lookup(db: Path, ensemble: str, correlator_name: str, code: str, project: Optional[str]=None,  parameters: Optional[str]=None) -> pd.DataFrame:
+    """
+    Look up a correlator record in the database by the data given to the method.
+
+    Parameters
+    ----------
+    db: str
+        The database to look up the record.
+    ensemble: str
+        The ensemble the record is associated with.
+    correlator_name: str
+        The name of the correlator in question.
+    code: str
+        The name of the code which was used to calculate the correlator.
+    project: str, optional
+        The UUID of the project the correlator was calculated in.
+    parameters: str, optional
+        A dictionary holding the exact parameters for the measurement that are held in the database.
+    created_before: str, optional
+        Timestamp string before which the meaurement has been created.
+    created_after: str, optional
+        Timestamp string after which the meaurement has been created.
+    updated_before: str, optional
+        Timestamp string before which the meaurement has been updated.
+    updated_after: str, optional
+        Timestamp string after which the meaurement has been updated.
+
+    Returns
+    -------
+    results: pd.DataFrame
+        A pandas DataFrame holding the information received form the DB query.
+    """
+
     project_str = project
 
     search_expr = f"SELECT * FROM 'backlogs' WHERE name = '{correlator_name}' AND ensemble = '{ensemble}'"
@@ -42,127 +152,240 @@ def _db_lookup(db, ensemble, correlator_name,code, project=None,  parameters=Non
         search_expr += f" AND code = '{code}'"
     if parameters:
         search_expr += f" AND parameters = '{parameters}'"
-    if created_before:
-        search_expr += f" AND created_at < '{created_before}'"
-    if created_after:
-        search_expr += f" AND created_at > '{created_after}'"
-    if updated_before:
-        search_expr += f" AND updated_at < '{updated_before}'"
-    if updated_after:
-        search_expr += f" AND updated_at > '{updated_after}'"
     conn = sqlite3.connect(db)
     results = pd.read_sql(search_expr, conn)
     conn.close()
     return results
 
 
-def sfcf_filter(results, **kwargs):
-    drops = []
-    for ind in range(len(results)):
-        result = results.iloc[ind]
-        if result['code'] == 'sfcf':
-            param = json.loads(result['parameters'])
+def _sfcf_drop(param: dict[str, Any], **kwargs: Any) -> bool:
         if 'offset' in kwargs:
             if kwargs.get('offset') != param['offset']:
-                    drops.append(ind)
-                    continue
+                return True
         if 'quark_kappas' in kwargs:
             kappas = kwargs['quark_kappas']
             if (not np.isclose(kappas[0], param['quarks'][0]['mass']) or not np.isclose(kappas[1], param['quarks'][1]['mass'])):
-                    drops.append(ind)
-                    continue
+                return True
         if 'quark_masses' in kwargs:
             masses = kwargs['quark_masses']
             if (not np.isclose(masses[0], k2m(param['quarks'][0]['mass'])) or not np.isclose(masses[1], k2m(param['quarks'][1]['mass']))):
-                    drops.append(ind)
-                    continue
+                return True
         if 'qk1' in kwargs:
             quark_kappa1 = kwargs['qk1']
             if not isinstance(quark_kappa1, list):
                 if (not np.isclose(quark_kappa1, param['quarks'][0]['mass'])):
-                        drops.append(ind)
-                        continue
+                    return True
             else:
                 if len(quark_kappa1) == 2:
                     if (quark_kappa1[0] > param['quarks'][0]['mass']) or (quark_kappa1[1] < param['quarks'][0]['mass']):
-                            drops.append(ind)
-                            continue
+                        return True
+                else:
+                    raise ValueError("quark_kappa1 has to have length 2")
         if 'qk2' in kwargs:
             quark_kappa2 = kwargs['qk2']
             if not isinstance(quark_kappa2, list):
                 if (not np.isclose(quark_kappa2, param['quarks'][1]['mass'])):
-                        drops.append(ind)
-                        continue
+                    return True
             else:
                 if len(quark_kappa2) == 2:
                     if (quark_kappa2[0] > param['quarks'][1]['mass']) or (quark_kappa2[1] < param['quarks'][1]['mass']):
-                            drops.append(ind)
-                            continue
+                        return True
+                else:
+                    raise ValueError("quark_kappa2 has to have length 2")
         if 'qm1' in kwargs:
             quark_mass1 = kwargs['qm1']
             if not isinstance(quark_mass1, list):
                 if (not np.isclose(quark_mass1, k2m(param['quarks'][0]['mass']))):
-                        drops.append(ind)
-                        continue
+                    return True
             else:
                 if len(quark_mass1) == 2:
                     if (quark_mass1[0] > k2m(param['quarks'][0]['mass'])) or (quark_mass1[1] < k2m(param['quarks'][0]['mass'])):
-                            drops.append(ind)
-                            continue
+                        return True
+                else:
+                    raise ValueError("quark_mass1 has to have length 2")
         if 'qm2' in kwargs:
             quark_mass2 = kwargs['qm2']
             if not isinstance(quark_mass2, list):
                 if (not np.isclose(quark_mass2, k2m(param['quarks'][1]['mass']))):
-                        drops.append(ind)
-                        continue
+                    return True
             else:
                 if len(quark_mass2) == 2:
                     if (quark_mass2[0] > k2m(param['quarks'][1]['mass'])) or (quark_mass2[1] < k2m(param['quarks'][1]['mass'])):
-                            drops.append(ind)
-                            continue
+                        return True
+                else:
+                    raise ValueError("quark_mass2 has to have length 2")
         if 'quark_thetas' in kwargs:
             quark_thetas = kwargs['quark_thetas']
             if (quark_thetas[0] != param['quarks'][0]['thetas'] and quark_thetas[1] != param['quarks'][1]['thetas']) or (quark_thetas[0] != param['quarks'][1]['thetas'] and quark_thetas[1] != param['quarks'][0]['thetas']):
-                    drops.append(ind)
-                    continue
+                return True
         # careful, this is not save, when multiple contributions are present!
         if 'wf1' in kwargs:
             wf1 = kwargs['wf1']
             if not (np.isclose(wf1[0][0], param['wf1'][0][0], 1e-8) and np.isclose(wf1[0][1][0], param['wf1'][0][1][0], 1e-8) and np.isclose(wf1[0][1][1], param['wf1'][0][1][1], 1e-8)):
-                    drops.append(ind)
-                    continue
+                return True
         if 'wf2' in kwargs:
             wf2 = kwargs['wf2']
             if not (np.isclose(wf2[0][0], param['wf2'][0][0], 1e-8) and np.isclose(wf2[0][1][0], param['wf2'][0][1][0], 1e-8) and np.isclose(wf2[0][1][1], param['wf2'][0][1][1], 1e-8)):
+                return True
+        return False
+
+
+def sfcf_filter(results: pd.DataFrame, **kwargs: Any) -> pd.DataFrame:
+    r"""
+    Filter method for the Database entries holding SFCF calculations.
+
+    Parameters
+    ----------
+    results: pd.DataFrame
+        The unfiltered pandas DataFrame holding the entries from the database.
+
+    offset: list[float], optional
+    quark_kappas: list[float]
+    quarks_masses: list[float]
+    qk1: float, optional
+        Mass parameter $\kappa_1$ of the first quark.
+    qk2: float, optional
+        Mass parameter $\kappa_2$ of the first quark.
+    qm1: float, optional
+        Bare quark mass $m_1$ of the first quark.
+    qm2: float, optional
+        Bare quark mass $m_2$ of the first quark.
+    quarks_thetas: list[list[float]], optional
+    wf1: optional
+    wf2: optional
+
+    Results
+    -------
+    results: pd.DataFrame
+        The filtered DataFrame, only holding the records that fit to the parameters given.
+    """
+
+    drops = []
+    for ind in range(len(results)):
+        result = results.iloc[ind]
+        param = json.loads(result['parameters'])
+        if _sfcf_drop(param, **kwargs):
             drops.append(ind)
-                    continue
     return results.drop(drops)
 
 
-def find_record(path, ensemble, correlator_name, code, project=None, parameters=None, created_before=None, created_after=None, updated_before=None, updated_after=None, revision=None, **kwargs):
-    db = path + '/backlogger.db'
+def openQCD_filter(results:pd.DataFrame, **kwargs: Any) -> pd.DataFrame:
+    """
+    Filter for parameters of openQCD.
+
+    Parameters
+    ----------
+    results: pd.DataFrame
+        The unfiltered list of results from the database.
+
+    Returns
+    -------
+    results: pd.DataFrame
+        The filtered results.
+
+    """
+    warnings.warn("A filter for openQCD parameters is no implemented yet.", Warning)
+
+    return results
+
+
+def _code_filter(results: pd.DataFrame, code: str, **kwargs: Any) -> pd.DataFrame:
+    """
+    Abstraction of the filters for the different codes that are available.
+    At the moment, only openQCD and SFCF are known.
+    The possible key words for the parameters can be seen in the descriptionso f the code-specific filters.
+
+    Parameters
+    ----------
+    results: pd.DataFrame
+        The unfiltered list of results from the database.
+    code: str
+        The name of the code that produced the record at hand.
+    kwargs:
+        The keyworkd args that are handed over to the code-specific filters.
+
+    Returns
+    -------
+    results: pd.DataFrame
+        The filtered results.
+    """
+    if code == "sfcf":
+        return sfcf_filter(results, **kwargs)
+    elif code == "openQCD":
+        return openQCD_filter(results, **kwargs)
+    else:
+        raise ValueError(f"Code {code} is not known.")
+
+
+def find_record(path: Path, ensemble: str, correlator_name: str, code: str, project: Optional[str]=None, parameters: Optional[str]=None,
+                created_before: Optional[str]=None, created_after: Optional[str]=None, updated_before: Optional[str]=None, updated_after: Optional[str]=None,
+                revision: Optional[str]=None,
+                customFilter: Optional[Callable[[pd.DataFrame], pd.DataFrame]] = None,
+                **kwargs: Any) -> pd.DataFrame:
+    path = Path(path)
+    db_file = get_db_file(path)
+    db = path / db_file
     if code not in codes:
         raise ValueError("Code " + code + "unknown, take one of the following:" + ", ".join(codes))
-    dl.get(db, dataset=path)
-    results = _db_lookup(db, ensemble, correlator_name,code, project, parameters=parameters, created_before=created_before, created_after=created_after, updated_before=updated_before, updated_after=updated_after, revision=revision)
-    if code == "sfcf":
-        results = sfcf_filter(results, **kwargs)
+    get(path, db_file)
+    results = _db_lookup(db, ensemble, correlator_name,code, project, parameters=parameters)
+    if any([arg is not None for arg in [created_before, created_after, updated_before, updated_after]]):
+        results = _time_filter(results, created_before, created_after, updated_before, updated_after)
+    results = _code_filter(results, code, **kwargs)
+    if customFilter is not None:
+        results =  customFilter(results)
     print("Found " + str(len(results)) + " result" + ("s" if len(results)>1 else ""))
     return results.reset_index()
 
 
-def find_project(path, db, name):
-    dl.get(db, dataset=path)
-    return _project_lookup_by_alias(db, name)
+def find_project(path: Path, name: str) -> str:
+    """
+    Find a project by it's human readable name.
+
+    Parameters
+    ----------
+    path: str
+        The path of the library.
+    name: str
+        The name of the project to look for in the library.
+
+    Returns
+    -------
+    uuid: str
+        The uuid of the project in question.
+    """
+    db_file = get_db_file(path)
+    get(path, db_file)
+    return _project_lookup_by_alias(path, name)
 
 
-def list_projects(path):
-    db = path + '/backlogger.db'
-    dl.get(db, dataset=path)
-    conn = sqlite3.connect(db)
+def list_projects(path: Path) -> list[tuple[str, str]]:
+    """
+    List all projects known to the library.
+
+    Parameters
+    ----------
+    path: str
+        The path of the library.
+
+    Returns
+    -------
+    results: list[Any]
+        The projects known to the library.
+    """
+    db_file = get_db_file(path)
+    get(path, db_file)
+    conn = sqlite3.connect(os.path.join(path, db_file))
     c = conn.cursor()
     c.execute("SELECT id,aliases FROM projects")
     results = c.fetchall()
     conn.close()
     return results
 
+
+def list_ensembles(path: Path) -> list[str]:
+    res = []
+    for item in os.listdir(path / "archive"):
+        if os.path.isdir(path / "archive" / item):
+            res.append(item)
+    return res

corrlib/git_tools.py

@@ -1,27 +1,28 @@
 import os
-import datalad.api as dl
+from .tracker import save
 import git
+from pathlib import Path
 
 GITMODULES_FILE = '.gitmodules'
 
 
-def move_submodule(repo_path, old_path, new_path):
+def move_submodule(repo_path: Path, old_path: Path, new_path: Path) -> None:
     """
     Move a submodule to a new location.
 
     Parameters
     ----------
-    repo_path: str
+    repo_path: Path
     Path to the repository.
-    old_path: str
+    old_path: Path
         The old path of the module.
-    new_path: str
+    new_path: Path
         The new path of the module.
     """
 
-    os.rename(os.path.join(repo_path, old_path), os.path.join(repo_path, new_path))
+    os.rename(repo_path / old_path, repo_path / new_path)
 
-    gitmodules_file_path = os.path.join(repo_path, GITMODULES_FILE)
+    gitmodules_file_path = repo_path / GITMODULES_FILE
 
     # update paths in .gitmodules
     with open(gitmodules_file_path, 'r') as file:
@@ -29,8 +30,8 @@ def move_submodule(repo_path, old_path, new_path):
 
     updated_lines = []
     for line in lines:
-        if old_path in line:
-            line = line.replace(old_path, new_path)
+        if str(old_path) in line:
+            line = line.replace(str(old_path), str(new_path))
         updated_lines.append(line)
 
     with open(gitmodules_file_path, 'w') as file:
@@ -40,4 +41,6 @@ def move_submodule(repo_path, old_path, new_path):
     repo = git.Repo(repo_path)
     repo.git.add('.gitmodules')
     # save new state of the dataset
-    dl.save(repo_path, message=f"Move module from {old_path} to {new_path}", dataset=repo_path)
+    save(repo_path, message=f"Move module from {old_path} to {new_path}", files=[Path('.gitmodules'), repo_path])
+
+    return

corrlib/initialization.py

@@ -1,12 +1,19 @@
+from configparser import ConfigParser
 import sqlite3
-import datalad.api as dl
 import os
+from .tracker import save, init
+from pathlib import Path
+from .tools import CONFIG_FILENAME
 
 
-def _create_db(db):
+def _create_db(db: Path) -> None:
     """
     Create the database file and the table.
 
+    Parameters
+    ----------
+    db: str
+        Path of the database file.
     """
     conn = sqlite3.connect(db)
     c = conn.cursor()
@@ -32,21 +39,84 @@ def _create_db(db):
                   updated_at TEXT)''')
     conn.commit()
     conn.close()
+    return
 
 
-def create(path):
+def _create_config(path: Path, tracker: str, cached: bool) -> ConfigParser:
+    """
+    Create the config file construction for backlogger.
+
+    Parameters
+    ----------
+    path: str
+        The path of the libaray to create.
+    tracker: str
+        Type of the tracker to use for the library (only DataLad is supported at the moment).
+    cached: bool
+        Whether or not the library will create a cache folder for multiple reads when downloaded.
+
+    Returns
+    -------
+    config: ConfigParser
+        Cpnfig parser with the default configuration printed.
+
+    """
+    config = ConfigParser()
+    config['core'] = {
+        'version': '1.0',
+        'tracker': tracker,
+        'cached': str(cached),
+    }
+    config['paths'] = {
+        'db': 'backlogger.db',
+        'projects_path': 'projects',
+        'archive_path': 'archive',
+        'toml_imports_path': 'toml_imports',
+        'import_scripts_path': 'import_scripts',
+    }
+    return config
+
+
+def _write_config(path: Path, config: ConfigParser) -> None:
+    """
+    Write the config file to disk.
+
+    Parameters
+    ----------
+    path: str
+        The path of the libaray to create.
+    config: ConfigParser
+        The configuration to be used as a ConfigParser, e.g. generated by _create_config.
+    """
+    with open(os.path.join(path, CONFIG_FILENAME), 'w') as configfile:
+        config.write(configfile)
+    return
+
+
+def create(path: Path, tracker: str = 'datalad', cached: bool = True) -> None:
     """
     Create folder of backlogs.
 
+    Parameters
+    ----------
+    path: str
+        The path at which the library will be created.
+    tracker: str, optional
+        The tracker to use for the library. The delauft is DataLad, which is also the only one that is supported at the moment.
+    cached: bool, optional
+        Whether or not hte librarby will be cached. By default, it does cache already read entries.
     """
-    dl.create(path)
-    _create_db(path + '/backlogger.db')
-    os.chmod(path + '/backlogger.db', 0o666)  # why does this not work?
-    os.makedirs(path + '/projects')
-    os.makedirs(path + '/archive')
-    os.makedirs(path + '/toml_imports')
-    os.makedirs(path + '/import_scripts/template.py')
-    with open(path + "/.gitignore", "w") as fp:
+    config = _create_config(path, tracker, cached)
+    init(path, tracker)
+    _write_config(path, config)
+    _create_db(path / config['paths']['db'])
+    os.chmod(path / config['paths']['db'], 0o666)
+    os.makedirs(path / config['paths']['projects_path'])
+    os.makedirs(path / config['paths']['archive_path'])
+    os.makedirs(path / config['paths']['toml_imports_path'])
+    os.makedirs(path / config['paths']['import_scripts_path'] / 'template.py')
+    with open(path / ".gitignore", "w") as fp:
         fp.write(".cache")
         fp.close()
-    dl.save(path, dataset=path, message="Initialize backlogger directory.")
+    save(path, message="Initialized correlator library")
+    return

corrlib/input/__init__.py

@@ -2,6 +2,6 @@
 Import functions for different codes.
 """
 
-from . import sfcf
-from . import openQCD
-from . import implementations
+from . import sfcf as sfcf
+from . import openQCD as openQCD
+from . import implementations as implementations

corrlib/input/implementations.py

@@ -1,2 +1,2 @@
-
+# List of supported input implementations
 codes = ['sfcf', 'openQCD']

corrlib/input/openQCD.py

@@ -2,10 +2,32 @@ import pyerrors.input.openQCD as input
 import datalad.api as dl
 import os
 import fnmatch
-from typing import Any
+from typing import Any, Optional
+from pathlib import Path
+from ..pars.openQCD import ms1
+from ..pars.openQCD import qcd2
 
 
-def read_ms1_param(path: str, project: str, file_in_project: str) -> dict[str, Any]:
+
+def load_ms1_infile(path: Path, project: str, file_in_project: str) -> dict[str, Any]:
+    """
+    Read the parameters for ms1 measurements from a parameter file in the project.
+
+    Parameters
+    ----------
+    path: str
+        The path to the backlogger folder.
+    project: str
+        The project from which to read the parameter file.
+    file_in_project: str
+        The path to the parameter file within the project.
+
+    Returns
+    -------
+    param: dict[str, Any]
+        The parameters read from the file.
+    """
+
     file = os.path.join(path, "projects", project, file_in_project)
     ds = os.path.join(path, "projects", project)
     dl.get(file, dataset=ds)
@@ -51,7 +73,25 @@ def read_ms1_param(path: str, project: str, file_in_project: str) -> dict[str, A
     return param
 
 
-def read_ms3_param(path: str, project: str, file_in_project: str) -> dict[str, Any]:
+def load_ms3_infile(path: Path, project: str, file_in_project: str) -> dict[str, Any]:
+    """
+    Read the parameters for ms3 measurements from a parameter file in the project.
+
+    Parameters
+    ----------
+    path: str
+        The path to the backlogger folder.
+    project: str
+        The project from which to read the parameter file.
+    file_in_project: str
+        The path to the parameter file within the project.
+
+    Returns
+    -------
+    param: dict[str, Any]
+        The parameters read from the file.
+    """
+
     file = os.path.join(path, "projects", project, file_in_project)
     ds = os.path.join(path, "projects", project)
     dl.get(file, dataset=ds)
@@ -67,7 +107,37 @@ def read_ms3_param(path: str, project: str, file_in_project: str) -> dict[str, A
     return param
 
 
-def read_rwms(path: str, project: str, dir_in_project: str, param: dict[str, Any], prefix: str, postfix: str="ms1", version: str='2.0', names: list[str]=None, files: list[str]=None) -> dict[str, Any]:
+def read_rwms(path: Path, project: str, dir_in_project: str, param: dict[str, Any], prefix: str, postfix: str="ms1", version: str='2.0', names: Optional[list[str]]=None, files: Optional[list[str]]=None) -> dict[str, Any]:
+    """
+    Read reweighting factor measurements from the project.
+
+    Parameters
+    ----------
+    path: str
+        The path to the backlogger folder.
+    project: str
+        The project from which to read the measurements.
+    dir_in_project: str
+        The directory within the project where the measurements are located.
+    param: dict[str, Any]
+        The parameters for the measurements.
+    prefix: str
+        The prefix of the measurement files.
+    postfix: str
+        The postfix of the measurement files.
+    version: str
+        The version of the openQCD used.
+    names: list[str]
+        Specific names for the replica of the ensemble the measurement file belongs to.
+    files: list[str]
+        Specific files to read.
+
+    Returns
+    -------
+    rw_dict: dict[str, dict[str, Any]]
+        The reweighting factor measurements read from the files.
+    """
+
     dataset = os.path.join(path, "projects", project)
     directory = os.path.join(dataset, dir_in_project)
     if files is None:
@@ -94,7 +164,43 @@ def read_rwms(path: str, project: str, dir_in_project: str, param: dict[str, Any
     return rw_dict
 
 
-def extract_t0(path: str, project: str, dir_in_project: str, param: dict[str, Any], prefix: str, dtr_read: int, xmin: int, spatial_extent: int, fit_range: int = 5, postfix: str=None, names: list[str]=None, files: list[str]=None) -> dict[str, Any]:
+def extract_t0(path: Path, project: str, dir_in_project: str, param: dict[str, Any], prefix: str, dtr_read: int, xmin: int, spatial_extent: int, fit_range: int = 5, postfix: str="", names: Optional[list[str]]=None, files: Optional[list[str]]=None) -> dict[str, Any]:
+    """
+    Extract t0 measurements from the project.
+
+    Parameters
+    ----------
+    path: str
+        The path to the backlogger folder.
+    project: str
+        The project from which to read the measurements.
+    dir_in_project: str
+        The directory within the project where the measurements are located.
+    param: dict[str, Any]
+        The parameters for the measurements.
+    prefix: str
+        The prefix of the measurement files.
+    dtr_read: int
+        The dtr_read parameter for the extraction.
+    xmin: int
+        The xmin parameter for the extraction.
+    spatial_extent: int
+        The spatial_extent parameter for the extraction.
+    fit_range: int
+        The fit_range parameter for the extraction.
+    postfix: str
+        The postfix of the measurement files.
+    names: list[str]
+        Specific names for the replica of the ensemble the measurement file belongs to.
+    files: list[str]
+        Specific files to read.
+
+    Returns
+    -------
+    t0_dict: dict
+        Dictionary of t0 values in the pycorrlib style, with the parameters at hand.
+    """
+
     dataset = os.path.join(path, "projects", project)
     directory = os.path.join(dataset, dir_in_project)
     if files is None:
@@ -132,7 +238,43 @@ def extract_t0(path: str, project: str, dir_in_project: str, param: dict[str, An
     return t0_dict
 
 
-def extract_t1(path: str, project: str, dir_in_project: str, param: dict[str, Any], prefix: str, dtr_read: int, xmin: int, spatial_extent: int, fit_range: int = 5, postfix: str = None, names: list[str]=None, files: list[str]=None) -> dict[str, Any]:
+def extract_t1(path: Path, project: str, dir_in_project: str, param: dict[str, Any], prefix: str, dtr_read: int, xmin: int, spatial_extent: int, fit_range: int = 5, postfix: str = "", names: Optional[list[str]]=None, files: Optional[list[str]]=None) -> dict[str, Any]:
+    """
+    Extract t1 measurements from the project.
+
+    Parameters
+    ----------
+    path: str
+        The path to the backlogger folder.
+    project: str
+        The project from which to read the measurements.
+    dir_in_project: str
+        The directory within the project where the measurements are located.
+    param: dict[str, Any]
+        The parameters for the measurements.
+    prefix: str
+        The prefix of the measurement files.
+    dtr_read: int
+        The dtr_read parameter for the extraction.
+    xmin: int
+        The xmin parameter for the extraction.
+    spatial_extent: int
+        The spatial_extent parameter for the extraction.
+    fit_range: int
+        The fit_range parameter for the extraction.
+    postfix: str
+        The postfix of the measurement files.
+    names: list[str]
+        Specific names for the replica of the ensemble the measurement file belongs to.
+    files: list[str]
+        Specific files to read.
+
+    Returns
+    -------
+    t1_dict: dict
+        Dictionary of t1 values in the pycorrlib style, with the parameters at hand.
+    """
+
     directory = os.path.join(path, "projects", project, dir_in_project)
     if files is None:
         files = []
@@ -161,7 +303,55 @@ def extract_t1(path: str, project: str, dir_in_project: str, param: dict[str, An
     for k in ["integrator", "eps", "ntot", "dnms"]:
         par_list.append(str(param[k]))
     pars = "/".join(par_list)
-    t0_dict: dict[str, Any] = {}
-    t0_dict[param["type"]] = {}
-    t0_dict[param["type"]][pars] = t0
-    return t0_dict
+    t1_dict: dict[str, Any] = {}
+    t1_dict[param["type"]] = {}
+    t1_dict[param["type"]][pars] = t0
+    return t1_dict
+
+
+def load_qcd2_pars(path: Path, project: str, file_in_project: str)  -> dict[str, Any]:
+    """
+    Thin wrapper around read_qcd2_par_file, getting the file before reading.
+
+    Parameters
+    ----------
+    path: Path
+        Path of the corrlib repository.
+    project: str
+        UUID of the project of the parameter-file.
+    file_in_project: str
+        The loaction of the file in the project directory.
+
+    Returns
+    -------
+    par_dict: dict
+        The dict with the parameters read from the .par-file.
+    """
+    fname = path / "projects" / project / file_in_project
+    ds = os.path.join(path, "projects", project)
+    dl.get(fname, dataset=ds)
+    return qcd2.read_qcd2_par_file(fname)
+
+
+def load_ms1_parfile(path: Path, project: str, file_in_project: str)  -> dict[str, Any]:
+    """
+    Thin wrapper around read_qcd2_ms1_par_file, getting the file before reading.
+
+    Parameters
+    ----------
+    path: Path
+        Path of the corrlib repository.
+    project: str
+        UUID of the project of the parameter-file.
+    file_in_project: str
+        The loaction of the file in the project directory.
+
+    Returns
+    -------
+    par_dict: dict
+        The dict with the parameters read from the .par-file.
+    """
+    fname = path / "projects" / project / file_in_project
+    ds = os.path.join(path, "projects", project)
+    dl.get(fname, dataset=ds)
+    return ms1.read_qcd2_ms1_par_file(fname)

corrlib/input/sfcf.py

@@ -3,9 +3,11 @@ import datalad.api as dl
 import json
 import os
 from typing import Any
+from fnmatch import fnmatch
+from pathlib import Path
 
 
-bi_corrs: list = ["f_P", "fP", "f_p",
+bi_corrs: list[str] = ["f_P", "fP", "f_p",
             "g_P", "gP", "g_p",
             "fA0", "f_A", "f_a",
             "gA0", "g_A", "g_a",
@@ -43,7 +45,7 @@ bi_corrs: list = ["f_P", "fP", "f_p",
             "l3A2", "l3_A2", "g_av23",
             ]
 
-bb_corrs: list = [
+bb_corrs: list[str] = [
             'F1',
             'F_1',
             'f_1',
@@ -64,7 +66,7 @@ bb_corrs: list = [
             'F_sPdP_d',
             ]
 
-bib_corrs: list = [
+bib_corrs: list[str] = [
             'F_V0',
             'K_V0',
             ]
@@ -79,7 +81,7 @@ for c in bib_corrs:
     corr_types[c] = 'bib'
 
 
-def read_param(path: str, project: str, file_in_project: str) -> dict[str, Any]:
+def read_param(path: Path, project: str, file_in_project: str) -> dict[str, Any]:
     """
     Read the parameters from the sfcf file.
 
@@ -95,7 +97,7 @@ def read_param(path: str, project: str, file_in_project: str) -> dict[str, Any]:
 
     """
 
-    file = path + "/projects/" + project + '/' + file_in_project
+    file = path / "projects" / project / file_in_project
     dl.get(file, dataset=path)
     with open(file, 'r') as f:
         lines = f.readlines()
@@ -184,7 +186,7 @@ def read_param(path: str, project: str, file_in_project: str) -> dict[str, Any]:
     return params
 
 
-def _map_params(params: dict, spec_list: list) -> dict[str, Any]:
+def _map_params(params: dict[str, Any], spec_list: list[str]) -> dict[str, Any]:
     """
     Map the extracted parameters to the extracted data.
 
@@ -228,7 +230,25 @@ def _map_params(params: dict, spec_list: list) -> dict[str, Any]:
     return new_specs
 
 
-def get_specs(key, parameters, sep='/') -> str:
+def get_specs(key: str, parameters: dict[str, Any], sep: str = '/') -> str:
+    """
+    Get sepcification from the parameter file for a specific key in the read measurements
+
+    Parameters
+    ----------
+    key: str
+        The key for whioch the parameters are to be looked up.
+    parameters: dict[str, Any]
+        The dictionary with the parameters from the parameter file.
+    sep: str
+        Separator string for the key. (default="/")
+
+    Return
+    ------
+    s: str
+        json string holding the parameters.
+    """
+
     key_parts = key.split(sep)
     if corr_types[key_parts[0]] == 'bi':
         param = _map_params(parameters, key_parts[1:-1])
@@ -238,7 +258,7 @@ def get_specs(key, parameters, sep='/') -> str:
     return s
 
 
-def read_data(path, project, dir_in_project, prefix, param, version='1.0c', cfg_seperator='n', sep='/', **kwargs) -> dict:
+def read_data(path: Path, project: str, dir_in_project: str, prefix: str, param: dict[str, Any], version: str = '1.0c', cfg_seperator: str = 'n', sep: str = '/', **kwargs: Any) -> dict[str, Any]:
     """
     Extract the data from the sfcf file.
 
@@ -280,6 +300,7 @@ def read_data(path, project, dir_in_project, prefix, param, version='1.0c', cfg_
     if not appended:
         compact = (version[-1] == "c")
         for i, item in enumerate(ls):
+            if fnmatch(item, prefix + "*"):
                 rep_path = directory + '/' + item
                 sub_ls = pe.input.sfcf._find_files(rep_path, prefix, compact, [])
                 files_to_get.extend([rep_path + "/" + filename for filename in sub_ls])
@@ -300,10 +321,10 @@ def read_data(path, project, dir_in_project, prefix, param, version='1.0c', cfg_
     if not param['crr'] == []:
         if names is not None:
             data_crr = pe.input.sfcf.read_sfcf_multi(directory, prefix, param['crr'], param['mrr'], corr_type_list, range(len(param['wf_offsets'])),
-                                                     range(len(param['wf_basis'])), range(len(param['wf_basis'])), version, cfg_seperator, keyed_out=True, names=names)
+                                                     range(len(param['wf_basis'])), range(len(param['wf_basis'])), version, cfg_seperator, keyed_out=True, silent=True, names=names)
         else:
             data_crr = pe.input.sfcf.read_sfcf_multi(directory, prefix, param['crr'], param['mrr'], corr_type_list, range(len(param['wf_offsets'])),
-                                                     range(len(param['wf_basis'])), range(len(param['wf_basis'])), version, cfg_seperator, keyed_out=True)
+                                                     range(len(param['wf_basis'])), range(len(param['wf_basis'])), version, cfg_seperator, keyed_out=True, silent=True)
         for key in data_crr.keys():
             data[key] = data_crr[key]
 

corrlib/integrity.py

@@ -0,0 +1,295 @@
+import datetime as dt
+from pathlib import Path
+from .tools import get_db_file, CONFIG_FILENAME
+import pandas as pd
+import sqlite3
+from .tracker import get
+import pyerrors.input.json as pj
+import os
+from configparser import ConfigParser
+from typing import Any
+
+
+path_opts = ['db', 'projects_path', 'archive_path', 'toml_imports_path', 'import_scripts_path']
+
+
+def has_valid_times(result: pd.Series) -> bool:
+    """
+    Check, whether the result at hand has time-stamps that are sensible:
+    A recored is created first, then updated, with both times laying in the past.
+
+    Parameters
+    ----------
+    result: pd.Series
+        The result to check
+
+    Returns
+    -------
+    b: bool
+        True, if the timestamps make sense.
+    """
+    # we expect created_at <= updated_at <= now
+    created_at = dt.datetime.fromisoformat(result['created_at'])
+    updated_at = dt.datetime.fromisoformat(result['updated_at'])
+    if created_at > updated_at:
+        return False
+    if updated_at > dt.datetime.now():
+        return False
+    return True
+
+def are_keys_unique(db: Path, table: str, col: str) -> bool:
+    """
+    Check whether the strings listed in a column of a given table are unique.
+
+    Parameters
+    ----------
+    db: Path
+        The database to check.
+    table: str
+        The table to check.
+    col: str
+        The column to be checked for uniqueness.
+
+    Returns
+    -------
+    b: bool
+        True, if the strings are unique.
+    """
+    conn = sqlite3.connect(db)
+    c = conn.cursor()
+    c.execute(f"SELECT COUNT( DISTINCT CAST({col} AS nvarchar(4000))), COUNT({col}) FROM {table};")
+    results = c.fetchall()[0]
+    conn.close()
+    res = bool(results[0] == results[1])
+    if not res:
+        print("Unique:", results[0], "All:", results[1])
+    return res
+
+
+def _list_projects(path: Path) -> list[tuple[str, str]]:
+    """
+    List all projects known to the library.
+
+    Parameters
+    ----------
+    path: str
+        The path of the library.
+
+    Returns
+    -------
+    results: list[Any]
+        The projects known to the library.
+    """
+    db_file = get_db_file(path)
+    get(path, db_file)
+    conn = sqlite3.connect(os.path.join(path, db_file))
+    c = conn.cursor()
+    c.execute("SELECT id,aliases FROM projects")
+    results = c.fetchall()
+    conn.close()
+    return results
+
+
+def _list_ensembles(path: Path) -> list[str]:
+    res = []
+    for item in os.listdir(path / "archive"):
+        if os.path.isdir(path / "archive" / item):
+            res.append(item)
+    return res
+
+
+def check_path_format(result: pd.Series, ensembles: list[str], projects: list[str]) -> None:
+    """
+    Check whether the path of the given result has the right format.
+
+    Parameters
+    ----------
+    result: pd.Series
+        The result to be checked.
+    """
+    p = result['path']
+    if not p.startswith('archive'):
+        raise ValueError(f'The path {p} does not start correctly')
+
+    meas_key = p.split('::')[1]
+    ensemble = p.split('/')[1]
+    project = p.split('/')[3].split('.')[0]
+    if not len(meas_key) == 64:
+        raise ValueError(f'meas_key of {p} is scrambled')
+    if ensemble not in ensembles:
+        raise ValueError(f'meas_key of {p} points to an unknown ensemble')
+    if project not in projects:
+        raise ValueError(f'meas_key of {p} points to an unknown project id ({project})')
+    if not ensemble == result['ensemble']:
+        raise ValueError(f'Ensemble in database and file does not match for path {p}.')
+
+
+
+def check_db_integrity(path: Path) -> None:
+    """
+    Check intergrity of the database by checking the uniqueness of the record keys used to load the records
+    and ensuring that the timestamps of each record is sensible. Throws an error, if issues are detected.
+
+    Parameters
+    ----------
+    path: Path
+        Path to the backlog-library to check.
+    """
+    db = get_db_file(path)
+
+    if not are_keys_unique(path / db, 'backlogs', 'path'):
+        raise Exception("The paths the backlog table of the database links are not unique.")
+
+    search_expr = "SELECT * FROM 'backlogs'"
+    conn = sqlite3.connect(path / db)
+    results = pd.read_sql(search_expr, conn)
+    ensembles = _list_ensembles(path)
+    projects = [p[0] for p in _list_projects(path)]
+
+    for _, result in results.iterrows():
+        if not has_valid_times(result):
+            raise ValueError(f"Result with id {result[id]} has wrong time signatures.")
+        check_path_format(result, ensembles, projects)
+    return
+
+
+def _check_db2paths(path: Path, meas_paths: list[str]) -> None:
+    """
+    Check whether for each record in the given by meas_paths, we can find the data in the file as we expect.
+    Also check, whether there are unreachable records in the files. If either of the issues arise, throws an error.
+
+    Parameters
+    ----------
+    path: Path
+        Path to the backlog-library to check.
+    meas_paths: list[str]
+        List of measurement paths to check.
+    """
+    needed_data: dict[str, list[str]] = {}
+    for mpath in meas_paths:
+        file = mpath.split("::")[0]
+        if file not in needed_data.keys():
+            needed_data[file] = []
+        key = mpath.split("::")[1]
+        needed_data[file].append(key)
+
+    totf = len(needed_data.keys())
+    for i, file in enumerate(needed_data.keys()):
+        print(f"Check against file {i}/{totf}: {file}")
+        get(path, Path(file))
+        filedict: dict[str, Any] = pj.load_json_dict(str(path / file))
+        if not set(filedict.keys()).issubset(needed_data[file]):
+            for key in filedict.keys():
+                if key not in needed_data[file]:
+                    raise ValueError(f"Found unintended key {key} in file {file}.")
+        if not set(needed_data[file]).issubset(filedict.keys()):
+            for key in needed_data[file]:
+                if key not in filedict.keys():
+                    raise ValueError(f"Did not find data for key {key} that should be in file {file}.")
+    return
+
+
+def check_db_file_links(path: Path) -> None:
+    """
+    Check whether for each record in the given correlator library, we can find the data in the file as we expect.
+    Also check, whether there are unreachable records in the files. If either of the issues arise, throws an error.
+
+    Parameters
+    ----------
+    path: Path
+        Path to the backlog-library to check.
+    """
+    db = get_db_file(path)
+    search_expr = "SELECT path FROM 'backlogs'"
+    conn = sqlite3.connect(path / db)
+    results = pd.read_sql(search_expr, conn)['path'].values
+    _check_db2paths(path, list(results))
+
+
+def check_path_and_config(path: Path) -> None:
+    """
+    Check whether the given path exists and the cinfigureation file can be found.
+
+    Parameters
+    ----------
+    path: Path
+        Path to the backlog-library to check.
+    """
+    if not os.path.exists(path):
+        raise FileNotFoundError(f"Corrlib path {path} does not exist.")
+    config_path = path / CONFIG_FILENAME
+    if not os.path.exists(config_path):
+        raise FileNotFoundError(f"Configuration file {config_path} not found.")
+
+
+def check_config_validity(path: Path) -> None:
+    """
+    Check whether the configuration file of the given corrlib-dataset path is valid.
+
+    Parameters
+    ----------
+    path: Path
+        Path to the backlog-library to check.
+    """
+    config = ConfigParser()
+    config_path = path / CONFIG_FILENAME
+    if os.path.exists(config_path):
+        config.read(config_path)
+    else:
+        raise FileNotFoundError("Configuration file not found.")
+
+    if config.has_section('core'):
+        core_opts = ['version', 'tracker', 'cached']
+        has_core_opts = [config.has_option('core', opt) for opt in core_opts]
+        if not all(has_core_opts):
+            raise ValueError("One of the options in the 'core' section  ('version', 'tracker', 'cached') is missing.")
+
+    if config.has_section('paths'):
+        has_path_opts = [config.has_option('paths', opt) for opt in path_opts]
+        if not all(has_path_opts):
+            raise ValueError("One of the options in the 'path' section  ('db', 'projects_path', 'archive_path', 'toml_imports_path', 'import_scripts_path') is missing.")
+
+
+def check_paths(path: Path) -> None:
+    """
+    Check whether all paths demanded by the 'paths' section of the configuration-file exist.
+
+    Parameters
+    ----------
+    path: Path
+        Path to the backlog-library to check.
+    """
+    config = ConfigParser()
+    config_path = path / CONFIG_FILENAME
+    if os.path.exists(config_path):
+        config.read(config_path)
+    else:
+        raise FileNotFoundError("Configuration file not found.")
+    has_paths = [os.path.exists(path / config.get('paths', opt)) for opt in path_opts]
+    if not all(has_paths):
+        raise FileNotFoundError("One of the paths specified in the configuration file is not present.")
+
+
+def full_integrity_check(path: Path) -> None:
+    """
+    Aggregate all checks for easy validation of the backlog-library.
+
+    Parameters
+    ----------
+    path: Path
+        Path to the backlog-library to check.
+    """
+    print("Run full integrity check...")
+    check_path_and_config(path)
+    print("(1/5) Path and config-file exist: ✅")
+    check_config_validity(path)
+    print("(2/5) Configuration is valid: ✅")
+    check_paths(path)
+    print("(3/5) Needed paths exist: ✅")
+    check_db_integrity(path)
+    print("(4/5) Database is sane: ✅")
+    check_db_file_links(path)
+    print("(5/5) DB2File and File2DB-links are sound: ✅")
+    print("Full integrity check: ✅")
+
+

corrlib/main.py

@@ -5,11 +5,13 @@ import os
 from .git_tools import move_submodule
 import shutil
 from .find import _project_lookup_by_id
-from .tools import list2str, str2list
-from typing import Union
+from .tools import list2str, str2list, get_db_file
+from .tracker import get, save, unlock, clone, drop
+from typing import Union, Optional
+from pathlib import Path
 
 
-def create_project(path: str, uuid: str, owner: Union[str, None]=None, tags: Union[str, None]=None, aliases: Union[str, None]=None, code: Union[str, None]=None):
+def create_project(path: Path, uuid: str, owner: Union[str, None]=None, tags: Union[list[str], None]=None, aliases: Union[list[str], None]=None, code: Union[str, None]=None) -> None:
     """
     Create a new project entry in the database.
 
@@ -24,30 +26,48 @@ def create_project(path: str, uuid: str, owner: Union[str, None]=None, tags: Uni
     code: str (optional)
         The code that was used to create the measurements.
     """
-    db = path + "/backlogger.db"
-    dl.get(db, dataset=path)
+    db_file = get_db_file(path)
+    db = path / db_file
+    get(path, db_file)
     conn = sqlite3.connect(db)
     c = conn.cursor()
     known_projects = c.execute("SELECT * FROM projects WHERE id=?", (uuid,))
     if known_projects.fetchone():
         raise ValueError("Project already imported, use update_project() instead.")
 
-    dl.unlock(db, dataset=path)
-    alias_str = None
+    unlock(path, db_file)
+    alias_str = ""
     if aliases is not None:
         alias_str = list2str(aliases)
-    tag_str = None
+    tag_str = ""
     if tags is not None:
         tag_str = list2str(tags)
     c.execute("INSERT INTO projects (id, aliases, customTags, owner, code, created_at, updated_at) VALUES (?, ?, ?, ?, ?, datetime('now'), datetime('now'))", (uuid, alias_str, tag_str, owner, code))
     conn.commit()
     conn.close()
-    dl.save(db, message="Added entry for project " + uuid + " to database", dataset=path)
+    save(path, message="Added entry for project " + uuid + " to database", files=[db_file])
+    return
 
 
-def update_project_data(path, db, uuid, prop, value = None):
-    dl.get(db, dataset=path)
-    conn = sqlite3.connect(db)
+def update_project_data(path: Path, uuid: str, prop: str, value: Union[str, None] = None) -> None:
+    """
+    Update/Edit a project entry in the database.
+    Thin wrapper around sql3 call.
+
+    Parameters
+    ----------
+    path: str
+        The path to the backlogger folder.
+    uuid: str
+        The uuid of the project.
+    prop: str
+        Property of the entry to edit
+    value: str or None
+        Value to se `prop` to.
+    """
+    db_file = get_db_file(path)
+    get(path, db_file)
+    conn = sqlite3.connect(path / db_file)
     c = conn.cursor()
     c.execute(f"UPDATE projects SET '{prop}' = '{value}' WHERE id == '{uuid}'")
     conn.commit()
@@ -55,10 +75,10 @@ def update_project_data(path, db, uuid, prop, value = None):
     return
 
 
-def update_aliases(path: str, uuid: str, aliases: list[str]):
-    db = os.path.join(path, "backlogger.db")
-    dl.get(db, dataset=path)
-    known_data = _project_lookup_by_id(db, uuid)[0]
+def update_aliases(path: Path, uuid: str, aliases: list[str]) -> None:
+    db_file = get_db_file(path)
+    get(path, db_file)
+    known_data = _project_lookup_by_id(path, uuid)[0]
     known_aliases = known_data[1]
 
     if aliases is None:
@@ -76,14 +96,16 @@ def update_aliases(path: str, uuid: str, aliases: list[str]):
 
     if not len(new_alias_list) == len(known_alias_list):
         alias_str = list2str(new_alias_list)
-        dl.unlock(db, dataset=path)
-        update_project_data(db, uuid, "aliases", alias_str)
-        dl.save(db, dataset=path)
+        unlock(path, db_file)
+        update_project_data(path, uuid, "aliases", alias_str)
+        save(path, message="Updated aliases for project " + uuid, files=[db_file])
     return
 
 
-def import_project(path: str, url: str, owner: Union[str, None]=None, tags: Union[str, None]=None, aliases: Union[str, None]=None, code: Union[str, None]=None, isDataset: bool=True):
+def import_project(path: Path, url: str, owner: Union[str, None]=None, tags: Optional[list[str]]=None, aliases: Optional[list[str]]=None, code: Optional[str]=None, isDataset: bool=True) -> str:
     """
+    Import a datalad dataset into the backlogger.
+
     Parameters
     ----------
 
@@ -91,43 +113,35 @@ def import_project(path: str, url: str, owner: Union[str, None]=None, tags: Unio
         The url of the project to import. This can be any url that datalad can handle.
     path: str
         The path to the backlogger folder.
-    aliases: list[str]
-        Custom name of the project, alias of the project.
-    code: str
+    owner: str, optional
+        Person responsible for the maintainance of the project to be impoerted.
+    tags: list[str], optional
+        Custom tags of the imported project.
+    aliases: list[str], optional
+        Custom names of the project, alias of the project.
+    code: str, optional
         Code that was used to create the measurements.
 
-    Import a datalad dataset into the backlogger.
-
-    Parameters
-    ----------
-    path: str
-        The path to the backlogger directory.
-    url: str
-        The url of the project to import. This can be any url that datalad can handle.
-        Also supported are non-datalad datasets, which will be converted to datalad datasets,
-        in order to receive a uuid and have a consistent interface.
-
+    Returns
+    -------
+    uuid: str
+        The unique identifier of the imported project.
     """
-    tmp_path = path + '/projects/tmp'
-    if not isDataset:
-        dl.create(tmp_path, dataset=path)
-        shutil.copytree(url + "/*", path + '/projects/tmp/')
-        dl.save(tmp_path, dataset=path)
-    else:
-        dl.install(path=tmp_path, source=url, dataset=path)
+    tmp_path = os.path.join(path, 'projects/tmp')
+    clone(path, source=url, target=tmp_path)
     tmp_ds = dl.Dataset(tmp_path)
     conf = dlc.ConfigManager(tmp_ds)
-    uuid = conf.get("datalad.dataset.id")
+    uuid = str(conf.get("datalad.dataset.id"))
     if not uuid:
         raise ValueError("The dataset does not have a uuid!")
-    if not os.path.exists(path + "/projects/" + uuid):
-        db = path + "/backlogger.db"
-        dl.get(db, dataset=path)
-        dl.unlock(db, dataset=path)
+    if not os.path.exists(path / "projects" / uuid):
+        db_file = get_db_file(path)
+        get(path, db_file)
+        unlock(path, db_file)
         create_project(path, uuid, owner, tags, aliases, code)
-        move_submodule(path, 'projects/tmp', 'projects/' + uuid)
-        os.mkdir(path + '/import_scripts/' + uuid)
-        dl.save([db, path + '/projects/' + uuid], message="Import project from " + url, dataset=path)
+        move_submodule(path, Path('projects/tmp'), Path('projects') / uuid)
+        os.mkdir(path / 'import_scripts' / uuid)
+        save(path, message="Import project from " + url, files=[Path(f'projects/{uuid}'), db_file])
     else:
         dl.drop(tmp_path, reckless='kill')
         shutil.rmtree(tmp_path)
@@ -142,9 +156,18 @@ def import_project(path: str, url: str, owner: Union[str, None]=None, tags: Unio
     return uuid
 
 
-def drop_project_data(path: str, uuid: str, path_in_project: str = ""):
+def drop_project_data(path: Path, uuid: str, path_in_project: str = "") -> None:
     """
-    Drop (parts of) a prject to free up diskspace
-    """
-    dl.drop(path + "/projects/" + uuid + "/" + path_in_project)
+    Drop (parts of) a project to free up diskspace
 
+    Parameters
+    ----------
+    path: str
+        Path of the library.
+    uuid: str
+        The UUID ofthe project rom which data is to be dropped.
+    path_pn_project: str, optional
+        If set, only the given path within the project is dropped.
+    """
+    drop(path / "projects" / uuid / path_in_project)
+    return

corrlib/meas_io.py

@@ -1,17 +1,23 @@
 from pyerrors.input import json as pj
 import os
-import datalad.api as dl
 import sqlite3
 from .input import sfcf,openQCD
 import json
 from typing import Union
 from pyerrors import Obs, Corr, dump_object, load_object
 from hashlib import sha256
-from .tools import cached
+from .tools import get_db_file, cache_enabled
+from .tracker import get, save, unlock
 import shutil
+from typing import Any
+from pathlib import Path
+from .integrity import _check_db2paths
 
 
-def write_measurement(path, ensemble, measurement, uuid, code, parameter_file=None):
+CACHE_DIR = ".cache"
+
+
+def write_measurement(path: Path, ensemble: str, measurement: dict[str, dict[str, dict[str, Any]]], uuid: str, code: str, parameter_file: Union[str, None]) -> None:
     """
     Write a measurement to the backlog.
     If the file for the measurement already exists, update the measurement.
@@ -26,26 +32,41 @@ def write_measurement(path, ensemble, measurement, uuid, code, parameter_file=No
         Measurements to be captured in the backlogging system.
     uuid: str
         The uuid of the project.
+    code: str
+        Name of the code that was used for the project.
+    parameter_file: str
+        The parameter file used for the measurement.
     """
-    db = os.path.join(path, 'backlogger.db')
-    dl.get(db, dataset=path)
-    dl.unlock(db, dataset=path)
+    path = Path(path)
+    db_file = get_db_file(path)
+    db = path / db_file
+
+    files_to_save = []
+
+    get(path, db_file)
+    unlock(path, db_file)
+    files_to_save.append(db_file)
+
     conn = sqlite3.connect(db)
     c = conn.cursor()
-    files = []
     for corr in measurement.keys():
-        file_in_archive = os.path.join('.', 'archive', ensemble, corr,  uuid + '.json.gz')
-        file = os.path.join(path, file_in_archive)
-        files.append(file)
+        file_in_archive = Path('.') / 'archive' / ensemble / corr / str(uuid + '.json.gz')
+        file = Path(path) / file_in_archive
         known_meas = {}
-        if not os.path.exists(os.path.join(path, '.', 'archive', ensemble, corr)):
-            os.makedirs(os.path.join(path, '.', 'archive', ensemble, corr))
+        if not os.path.exists(path / 'archive' / ensemble / corr):
+            os.makedirs(path / 'archive' / ensemble / corr)
+            files_to_save.append(file_in_archive)
         else:
             if os.path.exists(file):
-                dl.unlock(file, dataset=path)
-                known_meas = pj.load_json_dict(file)
+                if file not in files_to_save:
+                    unlock(path, file_in_archive)
+                    files_to_save.append(file_in_archive)
+                known_meas = pj.load_json_dict(str(file), verbose=False)
         if code == "sfcf":
+            if parameter_file is not None:
                 parameters = sfcf.read_param(path, uuid, parameter_file)
+            else:
+                raise Exception("Need parameter file for this code!")
             pars = {}
             subkeys = list(measurement[corr].keys())
             for subkey in subkeys:
@@ -54,7 +75,25 @@ def write_measurement(path, ensemble, measurement, uuid, code, parameter_file=No
         elif code == "openQCD":
             ms_type = list(measurement.keys())[0]
             if ms_type == 'ms1':
-                parameters = openQCD.read_ms1_param(path, uuid, parameter_file)
+                if parameter_file is not None:
+                    if parameter_file.endswith(".ms1.in"):
+                        parameters = openQCD.load_ms1_infile(path, uuid, parameter_file)
+                    elif parameter_file.endswith(".ms1.par"):
+                        parameters = openQCD.load_ms1_parfile(path, uuid, parameter_file)
+                else:
+                    # Temporary solution
+                    parameters = {}
+                    parameters["rand"] = {}
+                    parameters["rw_fcts"] = [{}]
+                    for nrw in range(1):
+                        if "nsrc" not in parameters["rw_fcts"][nrw]:
+                            parameters["rw_fcts"][nrw]["nsrc"] = 1
+                        if "mu" not in parameters["rw_fcts"][nrw]:
+                            parameters["rw_fcts"][nrw]["mu"] = "None"
+                        if "np" not in parameters["rw_fcts"][nrw]:
+                            parameters["rw_fcts"][nrw]["np"] = "None"
+                        if "irp" not in parameters["rw_fcts"][nrw]:
+                            parameters["rw_fcts"][nrw]["irp"] = "None"
                 pars = {}
                 subkeys = []
                 for i in range(len(parameters["rw_fcts"])):
@@ -66,7 +105,7 @@ def write_measurement(path, ensemble, measurement, uuid, code, parameter_file=No
                     pars[subkey] = json.dumps(parameters["rw_fcts"][i])
             elif ms_type in ['t0', 't1']:
                 if parameter_file is not None:
-                    parameters = openQCD.read_ms3_param(path, uuid, parameter_file)
+                    parameters = openQCD.load_ms3_infile(path, uuid, parameter_file)
                 else:
                     parameters = {}
                     for rwp in ["integrator", "eps", "ntot", "dnms"]:
@@ -81,7 +120,7 @@ def write_measurement(path, ensemble, measurement, uuid, code, parameter_file=No
                 pars[subkey] = json.dumps(parameters)
         for subkey in subkeys:
             parHash = sha256(str(pars[subkey]).encode('UTF-8')).hexdigest()
-            meas_path = file_in_archive + "::" + parHash
+            meas_path = str(file_in_archive) + "::" + parHash
 
             known_meas[parHash] = measurement[corr][subkey]
 
@@ -91,13 +130,13 @@ def write_measurement(path, ensemble, measurement, uuid, code, parameter_file=No
                 c.execute("INSERT INTO backlogs (name, ensemble, code, path, project, parameters, parameter_file, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, datetime('now'), datetime('now'))",
                                                 (corr, ensemble, code, meas_path, uuid, pars[subkey], parameter_file))
         conn.commit()
-        pj.dump_dict_to_json(known_meas, file)
-    files.append(path + '/backlogger.db')
+        pj.dump_dict_to_json(known_meas, str(file))
     conn.close()
-    dl.save(files, message="Add measurements to database", dataset=path)
+    save(path, message="Add measurements to database", files=files_to_save)
+    return
 
 
-def load_record(path: str, meas_path: str):
+def load_record(path: Path, meas_path: str) -> Union[Corr, Obs]:
     """
     Load a list of records by their paths.
 
@@ -116,7 +155,7 @@ def load_record(path: str, meas_path: str):
     return load_records(path, [meas_path])[0]
 
 
-def load_records(path: str, meas_paths: list[str], preloaded = {}) -> list[Union[Corr, Obs]]:
+def load_records(path: Path, meas_paths: list[str], preloaded: dict[str, Any] = {}, dry_run: bool = False) -> list[Union[Corr, Obs]]:
     """
     Load a list of records by their paths.
 
@@ -126,11 +165,20 @@ def load_records(path: str, meas_paths: list[str], preloaded = {}) -> list[Union
         Path of the correlator library.
     meas_paths: list[str]
         A list of the paths to the correlator in the backlog system.
+    preloaded: dict[str, Any]
+        The data that is already preloaded. Of interest if data has alread been loaded in the same script.
+    dry_run: bool
+        Do not load datda, just check whether we can reach the data we are interested in.
 
     Returns
     -------
-    List
+    returned_data: list
+        The loaded records.
     """
+    path = Path(path)
+    if dry_run:
+        _check_db2paths(path, meas_paths)
+        return []
     needed_data: dict[str, list[str]] = {}
     for mpath in meas_paths:
         file = mpath.split("::")[0]
@@ -138,48 +186,106 @@ def load_records(path: str, meas_paths: list[str], preloaded = {}) -> list[Union
             needed_data[file] = []
         key = mpath.split("::")[1]
         needed_data[file].append(key)
-    returned_data: list = []
+    returned_data: list[Any] = []
     for file in needed_data.keys():
         for key in list(needed_data[file]):
-            if os.path.exists(cache_path(path, file, key) + ".p"):
-                returned_data.append(load_object(cache_path(path, file, key) + ".p"))
+            if os.path.exists(str(cache_path(path, file, key)) + ".p"):
+                returned_data.append(load_object(str(cache_path(path, file, key)) + ".p"))
             else:
                 if file not in preloaded:
-                    preloaded[file] = preload(path, file)
+                    preloaded[file] = preload(path, Path(file))
                 returned_data.append(preloaded[file][key])
-                if cached:
+                if cache_enabled(path):
                     if not os.path.exists(cache_dir(path, file)):
                         os.makedirs(cache_dir(path, file))
-                    dump_object(preloaded[file][key], cache_path(path, file, key))
+                    dump_object(preloaded[file][key], str(cache_path(path, file, key)))
     return returned_data
 
 
-def cache_dir(path, file):
-    cache_path_list = [path]
-    cache_path_list.append(".cache")
-    cache_path_list.extend(file.split("/")[1:])
-    cache_path = "/".join(cache_path_list)
+def cache_dir(path: Path, file: str) -> Path:
+    """
+    Returns the directory corresponding to the cache for the given file.
+
+    Parameters
+    ----------
+    path: str
+        The path of the library.
+    file: str
+        The file in the library that we want to access the cached data of.
+    Returns
+    -------
+    cache_path: str
+        The path holding the cached data for the given file.
+    """
+    cache_path_list = file.split("/")[1:]
+    cache_path = Path(path) / CACHE_DIR
+    for directory in cache_path_list:
+        cache_path /= directory
     return cache_path
 
 
-def cache_path(path, file, key):
-    cache_path = os.path.join(cache_dir(path, file), key)
+def cache_path(path: Path, file: str, key: str) -> Path:
+    """
+    Parameters
+    ----------
+    path: str
+        The path of the library.
+    file: str
+        The file in the library that we want to access the cached data of.
+    key: str
+        The key within the archive file.
+
+    Returns
+    -------
+    cache_path: str
+        The path at which the measurement of the given file and key is cached.
+    """
+    path = Path(path)
+    cache_path = cache_dir(path, file) / key
     return cache_path
 
 
-def preload(path: str, file: str):
-    dl.get(os.path.join(path, file), dataset=path)
-    filedict = pj.load_json_dict(os.path.join(path, file))
+def preload(path: Path, file: Path) -> dict[str, Any]:
+    """
+    Read the contents of a file into a json dictionary with the pyerrors.json.load_json_dict method.
+
+    Parameters
+    ----------
+    path: str
+        The path of the library.
+    file: str
+        The file within the library to be laoded.
+
+    Returns
+    -------
+    filedict: dict[str, Any]
+        The data read from the file.
+    """
+    path = Path(path)
+    get(path, file)
+    filedict: dict[str, Any] = pj.load_json_dict(str(path / file))
+    print("> read file")
     return filedict
 
 
-def drop_record(path: str, meas_path: str):
+def drop_record(path: Path, meas_path: str) -> None:
+    """
+    Drop a record by it's path.
+
+    Parameters
+    ----------
+    path: str
+        The path of the library.
+    meas_path: str
+        The measurement path as noted in the database.
+    """
     file_in_archive = meas_path.split("::")[0]
-    file = os.path.join(path, file_in_archive)
-    db = os.path.join(path, 'backlogger.db')
-    dl.get(db, dataset=path)
+    file = Path(path) / file_in_archive
+    db_file = get_db_file(path)
+    db = path / db_file
+    get(path, db_file)
     sub_key = meas_path.split("::")[1]
-    dl.unlock(db, dataset=path)
+    unlock(path, db_file)
     conn = sqlite3.connect(db)
     c = conn.cursor()
     if c.execute("SELECT * FROM backlogs WHERE path = ?", (meas_path, )).fetchone() is not None:
@@ -188,17 +294,28 @@ def drop_record(path: str, meas_path: str):
         raise ValueError("This measurement does not exist as an entry!")
 
     conn.commit()
-    known_meas = pj.load_json_dict(file)
+    known_meas = pj.load_json_dict(str(file))
     if sub_key in known_meas:
         del known_meas[sub_key]
-        dl.unlock(file, dataset=path)
-        pj.dump_dict_to_json(known_meas, file)
-        dl.save([db, file], message="Drop measurements to database", dataset=path)
+        unlock(path, Path(file_in_archive))
+        pj.dump_dict_to_json(known_meas, str(file))
+        save(path, message="Drop measurements to database", files=[db, file])
         return
     else:
         raise ValueError("This measurement does not exist as a file!")
 
-def drop_cache(path: str):
-    cache_dir = os.path.join(path, ".cache")
+
+def drop_cache(path: Path) -> None:
+    """
+    Drop the cache directory of the library.
+
+    Parameters
+    ----------
+    path: str
+        The path of the library.
+    """
+    path = Path(path)
+    cache_dir = path / ".cache"
     for f in os.listdir(cache_dir):
-        shutil.rmtree(os.path.join(cache_dir, f))
+        shutil.rmtree(cache_dir / f)
+    return

corrlib/pars/openQCD/__init__.py

@@ -0,0 +1,3 @@
+
+from . import ms1 as ms1
+from . import qcd2 as qcd2

corrlib/pars/openQCD/flags.py

@@ -0,0 +1,59 @@
+"""
+Reconstruct the outputs of flags.
+"""
+
+import struct
+from typing import Any, BinaryIO
+
+# lat_parms.c
+def lat_parms_write_lat_parms(fp: BinaryIO) -> dict[str, Any]:
+        """
+        NOTE: This is a duplcation from qcd2.
+        Unpack the lattice parameters written by write_lat_parms.
+        """
+        lat_pars = {}
+        t = fp.read(16)
+        lat_pars["N"] = list(struct.unpack('iiii', t)) # lattice extends
+        t = fp.read(8)
+        nk, isw = struct.unpack('ii', t) # number of kappas and isw parameter
+        lat_pars["nk"] = nk
+        lat_pars["isw"] = isw
+        t = fp.read(8)
+        lat_pars["beta"] = struct.unpack('d', t)[0] # beta
+        t = fp.read(8)
+        lat_pars["c0"] = struct.unpack('d', t)[0]
+        t = fp.read(8)
+        lat_pars["c1"] = struct.unpack('d', t)[0]
+        t = fp.read(8)
+        lat_pars["csw"] = struct.unpack('d', t)[0] # csw factor
+        kappas = []
+        m0s = []
+        # read kappas
+        for ik in range(nk):
+            t = fp.read(8)
+            kappas.append(struct.unpack('d', t)[0])
+            t = fp.read(8)
+            m0s.append(struct.unpack('d', t)[0])
+        lat_pars["kappas"] = kappas
+        lat_pars["m0s"] = m0s
+        return lat_pars
+
+
+def lat_parms_write_bc_parms(fp: BinaryIO) -> dict[str, Any]:
+        """
+        NOTE: This is a duplcation from qcd2.
+        Unpack the boundary parameters written by write_bc_parms.
+        """
+        bc_pars: dict[str, Any] = {}
+        t = fp.read(4)
+        bc_pars["type"] = struct.unpack('i', t)[0] # type of hte boundaries
+        t = fp.read(104)
+        bc_parms = struct.unpack('d'*13, t)
+        bc_pars["cG"] = list(bc_parms[:2]) # boundary gauge field improvement
+        bc_pars["cF"] = list(bc_parms[2:4]) # boundary fermion field improvement
+        phi: list[list[float]] = [[], []]
+        phi[0] = list(bc_parms[4:7])
+        phi[1] = list(bc_parms[7:10])
+        bc_pars["phi"] = phi
+        bc_pars["theta"] = list(bc_parms[10:])
+        return bc_pars

corrlib/pars/openQCD/ms1.py

@@ -0,0 +1,30 @@
+from . import flags
+
+from typing import Any
+from pathlib import Path
+
+
+def read_qcd2_ms1_par_file(fname: Path) -> dict[str, dict[str, Any]]:
+    """
+    The subroutines written here have names according to the openQCD programs and functions that write out the data.
+    Parameters
+    ----------
+    fname: Path
+        Location of the parameter file.
+
+    Returns
+    -------
+    par_dict: dict
+        Dictionary holding the parameters specified in the given file.
+    """
+
+    with open(fname, "rb") as fp:
+        lat_par_dict = flags.lat_parms_write_lat_parms(fp)
+        bc_par_dict = flags.lat_parms_write_bc_parms(fp)
+        fp.close()
+    par_dict = {}
+    par_dict["lat"] = lat_par_dict
+    par_dict["bc"] = bc_par_dict
+    return par_dict
+
+

corrlib/pars/openQCD/qcd2.py

@@ -0,0 +1,29 @@
+from . import flags
+
+from pathlib import Path
+from typing import Any
+
+
+def read_qcd2_par_file(fname: Path) -> dict[str, dict[str, Any]]:
+    """
+    The subroutines written here have names according to the openQCD programs and functions that write out the data.
+
+    Parameters
+    ----------
+    fname: Path
+        Location of the parameter file.
+
+    Returns
+    -------
+    par_dict: dict
+        Dictionary holding the parameters specified in the given file.
+    """
+
+    with open(fname, "rb") as fp:
+        lat_par_dict = flags.lat_parms_write_lat_parms(fp)
+        bc_par_dict = flags.lat_parms_write_bc_parms(fp)
+        fp.close()
+    par_dict = {}
+    par_dict["lat"] = lat_par_dict
+    par_dict["bc"] = bc_par_dict
+    return par_dict

corrlib/sql.py

@@ -0,0 +1,17 @@
+import sqlite3
+from .tools import get_db_file
+from pathlib import Path
+from typing import Any
+
+
+def thin_sql_wrapper(path: Path, stmt: str) -> list[Any]:
+    db_file = get_db_file(path)
+    db = path / db_file
+    conn = sqlite3.connect(db)
+    c = conn.cursor()
+
+    c.execute(stmt)
+    results = c.fetchall()
+    conn.commit()
+    conn.close()
+    return results

corrlib/toml.py

@@ -10,22 +10,49 @@ the import of projects via TOML.
 
 import tomllib as toml
 import shutil
+
+import datalad.api as dl
+from .tracker import save
 from .input import sfcf, openQCD
 from .main import import_project, update_aliases
 from .meas_io import write_measurement
-import datalad.api as dl
 import os
 from .input.implementations import codes as known_codes
+from typing import Any
+from pathlib import Path
 
-def replace_string(string: str, name: str, val: str):
+
+def replace_string(string: str, name: str, val: str) -> str:
+    """
+    Replace a placeholder {name} with a value in a string.
+
+    Parameters
+    ----------
+    string: str
+        String in which the placeholders are to be replaced.
+    name: str
+        The name of the placeholder.
+    val: str
+        The value the placeholder is to be replaced with.
+    """
     if '{' + name + '}' in string:
         n = string.replace('{' + name + '}', val)
         return n
     else:
         return string
 
-def replace_in_meas(measurements: dict, vars: dict[str, str]):
-    # replace global variables
+
+def replace_in_meas(measurements: dict[str, dict[str, Any]], vars: dict[str, str]) -> dict[str, dict[str, Any]]:
+    """
+    Replace placeholders in the defiitions for a measurement.
+
+    Parameters
+    ----------
+    measurements: dict[str, dict[str, Any]]
+        The measurements read from the toml file.
+    vars: dict[str, str]
+        Simple key:value dictionary with the keys to be replaced by the values.
+    """
     for name, value in vars.items():
         for m in measurements.keys():
             for key in measurements[m].keys():
@@ -36,7 +63,18 @@ def replace_in_meas(measurements: dict, vars: dict[str, str]):
                         measurements[m][key][i] = replace_string(measurements[m][key][i], name, value)
     return measurements
 
-def fill_cons(measurements, constants):
+
+def fill_cons(measurements: dict[str, dict[str, Any]], constants: dict[str, str]) -> dict[str, dict[str, Any]]:
+    """
+    Fill in defined constants into the measurements of the toml-file.
+
+    Parameters
+    ----------
+    measurements: dict[str, dict[str, Any]]
+        The measurements read from the toml file.
+    constants: dict[str, str]
+        Simple key:value dictionary with the keys to be replaced by the values.
+    """
     for m in measurements.keys():
         for name, val in constants.items():
             if name not in measurements[m].keys():
@@ -44,7 +82,15 @@ def fill_cons(measurements, constants):
     return measurements
 
 
-def check_project_data(d: dict) -> None:
+def check_project_data(d: dict[str, dict[str, str]]) -> None:
+    """
+    Check the data given in the toml import file for the project we want to import.
+
+    Parameters
+    ----------
+    d: dict
+        The dictionary holding the data necessary to import the project.
+    """
     if 'project' not in d.keys() or 'measurements' not in d.keys() or len(list(d.keys())) > 4:
         raise ValueError('There should only be maximally be four keys on the top level, "project" and "measurements" are mandatory, "contants" is optional!')
     project_data = d['project']
@@ -57,7 +103,17 @@ def check_project_data(d: dict) -> None:
     return
 
 
-def check_measurement_data(measurements: dict, code: str) -> None:
+def check_measurement_data(measurements: dict[str, dict[str, str]], code: str) -> None:
+    """
+    Check syntax of the measurements we want to import.
+
+    Parameters
+    ----------
+    measurements: dict[str, dict[str, str]]
+        The dictionary holding the necessary data to import the project.
+    code: str
+        The code used for the project.
+    """
     var_names: list[str] = []
     if code == "sfcf":
         var_names = ["path", "ensemble", "param_file", "version", "prefix", "cfg_seperator", "names"]
@@ -71,12 +127,25 @@ def check_measurement_data(measurements: dict, code: str) -> None:
     return
 
 
-def import_tomls(path: str, files: list[str], copy_files: bool=True) -> None:
+def import_tomls(path: Path, files: list[str], copy_files: bool=True) -> None:
+    """
+    Import multiple toml files.
+
+    Parameters
+    ----------
+    path: str
+        Path to the backlog directory.
+    files: list[str]
+        Path to the description files.
+    copy_files: bool, optional
+        Whether the toml-files will be copied into the library. Default is True.
+    """
     for file in files:
         import_toml(path, file, copy_files)
+    return
 
 
-def import_toml(path: str, file: str, copy_file: bool=True) -> None:
+def import_toml(path: Path, file: str, copy_file: bool=True) -> None:
     """
     Import a project decribed by a .toml file.
 
@@ -86,29 +155,37 @@ def import_toml(path: str, file: str, copy_file: bool=True) -> None:
         Path to the backlog directory.
     file: str
         Path to the description file.
+    copy_file: bool, optional
+        Whether the toml-files will be copied into the library. Default is True.
     """
+    if not os.path.exists(path):
+        raise FileNotFoundError(f"Corrlib path {path} does not exist.")
+    if not os.path.exists(file):
+        raise FileNotFoundError(f".toml-file {file} does not exist.")
     print("Import project as decribed in " + file)
     with open(file, 'rb') as fp:
         toml_dict = toml.load(fp)
     check_project_data(toml_dict)
-    project: dict = toml_dict['project']
+    project: dict[str, Any] = toml_dict['project']
     if project['code'] not in known_codes:
         raise ValueError('Code' + project['code'] + 'has no import implementation!')
-    measurements: dict = toml_dict['measurements']
+    measurements: dict[str, dict[str, Any]] = toml_dict['measurements']
     measurements = fill_cons(measurements, toml_dict['constants'] if 'constants' in toml_dict else {})
     measurements = replace_in_meas(measurements, toml_dict['replace'] if 'replace' in toml_dict else {})
     check_measurement_data(measurements, project['code'])
-    aliases = project.get('aliases', None)
+    aliases = project.get('aliases', [])
     uuid = project.get('uuid', None)
     if uuid is not None:
-        if not os.path.exists(path + "/projects/" + uuid):
+        if not os.path.exists(path / "projects" / uuid):
             uuid = import_project(path, project['url'], aliases=aliases)
         else:
             update_aliases(path, uuid, aliases)
     else:
         uuid = import_project(path, project['url'], aliases=aliases)
+    imeas = 1
+    nmeas = len(measurements.keys())
     for mname, md in measurements.items():
-        print("Import measurement: " + mname)
+        print(f"Import measurement {imeas}/{nmeas}: {mname}")
         ensemble = md['ensemble']
         if project['code'] == 'sfcf':
             param = sfcf.read_param(path, uuid, md['param_file'])
@@ -119,43 +196,63 @@ def import_toml(path: str, file: str, copy_file: bool=True) -> None:
                 measurement = sfcf.read_data(path, uuid, md['path'], md['prefix'], param,
                                             version=md['version'], cfg_seperator=md['cfg_seperator'], sep='/')
 
-            print(mname + " imported.")
         elif project['code'] == 'openQCD':
             if md['measurement'] == 'ms1':
-                param = openQCD.read_ms1_param(path, uuid, md['param_file'])
+                if 'param_file' in md.keys():
+                    parameter_file = md['param_file']
+                    if parameter_file.endswith(".ms1.in"):
+                        param = openQCD.load_ms1_infile(path, uuid, parameter_file)
+                    elif parameter_file.endswith(".ms1.par"):
+                        param = openQCD.load_ms1_parfile(path, uuid, parameter_file)
+                else:
+                    # Temporary solution
+                    parameters: dict[str, Any] = {}
+                    parameters["rand"] = {}
+                    parameters["rw_fcts"] = [{}]
+                    for nrw in range(1):
+                        if "nsrc" not in parameters["rw_fcts"][nrw]:
+                            parameters["rw_fcts"][nrw]["nsrc"] = 1
+                        if "mu" not in parameters["rw_fcts"][nrw]:
+                            parameters["rw_fcts"][nrw]["mu"] = "None"
+                        if "np" not in parameters["rw_fcts"][nrw]:
+                            parameters["rw_fcts"][nrw]["np"] = "None"
+                        if "irp" not in parameters["rw_fcts"][nrw]:
+                            parameters["rw_fcts"][nrw]["irp"] = "None"
+                    param = parameters
                 param['type'] = 'ms1'
                 measurement = openQCD.read_rwms(path, uuid, md['path'], param, md["prefix"], version=md["version"], names=md['names'], files=md['files'])
             elif md['measurement'] == 't0':
                 if 'param_file' in md:
-                    param = openQCD.read_ms3_param(path, uuid, md['param_file'])
+                    param = openQCD.load_ms3_infile(path, uuid, md['param_file'])
                 else:
                     param = {}
                     for rwp in ["integrator", "eps", "ntot", "dnms"]:
                         param[rwp] = "Unknown"
                 param['type'] = 't0'
-                measurement = openQCD.extract_t0(path, uuid, md['path'], param, md["prefix"], md["dtr_read"], md["xmin"], md["spatial_extent"],
-                                                 fit_range=md.get('fit_range', 5), postfix=md.get('postfix', None), names=md.get('names', None), files=md.get('files', None))
+                measurement = openQCD.extract_t0(path, uuid, md['path'], param, str(md["prefix"]), int(md["dtr_read"]), int(md["xmin"]), int(md["spatial_extent"]),
+                                                 fit_range=int(md.get('fit_range', 5)), postfix=str(md.get('postfix', '')), names=md.get('names', []), files=md.get('files', []))
             elif md['measurement'] == 't1':
                 if 'param_file' in md:
-                    param = openQCD.read_ms3_param(path, uuid, md['param_file'])
+                    param = openQCD.load_ms3_infile(path, uuid, md['param_file'])
                 param['type'] = 't1'
-                measurement = openQCD.extract_t1(path, uuid, md['path'], param, md["prefix"], md["dtr_read"], md["xmin"], md["spatial_extent"],
-                                                 fit_range=md.get('fit_range', 5), postfix=md.get('postfix', None), names=md.get('names', None), files=md.get('files', None))
-
+                measurement = openQCD.extract_t1(path, uuid, md['path'], param, str(md["prefix"]), int(md["dtr_read"]), int(md["xmin"]), int(md["spatial_extent"]),
+                                                 fit_range=int(md.get('fit_range', 5)), postfix=str(md.get('postfix', '')), names=md.get('names', []), files=md.get('files', []))
         write_measurement(path, ensemble, measurement, uuid, project['code'], (md['param_file'] if 'param_file' in md else None))
+        imeas += 1
+        print(mname + " imported.")
 
-    if not os.path.exists(os.path.join(path, "toml_imports", uuid)):
-        os.makedirs(os.path.join(path, "toml_imports", uuid))
+    if not os.path.exists(path / "toml_imports" / uuid):
+        os.makedirs(path / "toml_imports" / uuid)
     if copy_file:
-        import_file = os.path.join(path, "toml_imports", uuid, file.split("/")[-1])
+        import_file = path / "toml_imports" / uuid / file.split("/")[-1]
         shutil.copy(file, import_file)
-        dl.save(import_file, message="Import using " + import_file, dataset=path)
-        print("File copied to " + import_file)
+        save(path, files=[import_file], message=f"Import using {import_file}")
+        print(f"File copied to {import_file}")
     print("Imported project.")
     return
 
 
-def reimport_project(path, uuid):
+def reimport_project(path: Path, uuid: str) -> None:
     """
     Reimport an existing project using the files that are already available for this project.
 
@@ -166,13 +263,24 @@ def reimport_project(path, uuid):
     uuid: str
     uuid of the project that is to be reimported.
     """
-    config_path = "/".join([path, "import_scripts", uuid])
+    config_path = path / "import_scripts" / uuid
     for p, filenames, dirnames in os.walk(config_path):
         for fname in filenames:
             import_toml(path, os.path.join(config_path, fname), copy_file=False)
     return
 
 
-def update_project(path, uuid):
+def update_project(path: Path, uuid: str) -> None:
+    """
+    Update all entries associated with a given project.
+
+    Parameters
+    ----------
+    path: str
+        The path of the library.
+    uuid: str
+        The unique identifier of the project to be updated.
+    """
     dl.update(how='merge', follow='sibling', dataset=os.path.join(path, "projects", uuid))
     # reimport_project(path, uuid)
+    return

corrlib/tools.py

@@ -1,17 +1,158 @@
+import os
+from configparser import ConfigParser
+from typing import Any
+from pathlib import Path
+
+CONFIG_FILENAME = ".corrlib"
+cached: bool = True
 
 
-def str2list(string):
+def str2list(string: str) -> list[str]:
+    """
+    Convert a comma-separated string to a list.
+
+    Parameters
+    ----------
+    string: str
+        The sting holding a comma-sparated list.
+
+    Returns
+    -------
+    s: list[str]
+        The list of strings that was held bythe comma separated string.
+    """
     return string.split(",")
 
-def list2str(mylist):
+def list2str(mylist: list[str]) -> str:
+    """
+    Convert a list to a comma-separated string.
+
+    Parameters
+    ----------
+    mylist: list[str]
+        A list of strings to be concatinated.
+
+    Returns
+    -------
+    s: list[str]
+        The sting holding a comma-sparated list.
+    """
     s = ",".join(mylist)
     return s
 
-cached = True
+def m2k(m: float) -> float:
+    """
+    Convert to bare quark mas $m$ to inverse mass parameter $kappa$.
 
-def m2k(m):
+    Parameters
+    ----------
+    m: float
+        Bare quark mass.
+
+    Returns
+    -------
+    k: float
+        The corresponing $kappa$.
+    """
     return 1/(2*m+8)
 
 
-def k2m(k):
+def k2m(k: float) -> float:
+    """
+    Convert from the inverse bare quark parameter $kappa$ to the bare quark mass $m$.
+
+    Parameters
+    ----------
+    k: float
+        Inverse bare quark mass parameter $kappa$.
+
+    Returns
+    -------
+    m: float
+        The corresponing bare quark mass.
+    """
     return (1/(2*k))-4
+
+
+def set_config(path: Path, section: str, option: str, value: Any) -> None:
+    """
+    Set configuration parameters for the library.
+
+    Parameters
+    ----------
+    path: str
+        The path of the library.
+    section: str
+        The section within the configuration file.
+    option: str
+        The option to be set to value.
+    value: Any
+        The value we set the option to.
+    """
+    path = Path(path)
+    config_path = path / CONFIG_FILENAME
+    config = ConfigParser()
+    if os.path.exists(config_path):
+        config.read(config_path)
+    if not config.has_section(section):
+        config.add_section(section)
+    config.set(section, option, value)
+    with open(config_path, 'w') as configfile:
+        config.write(configfile)
+    return
+
+
+def get_db_file(path: Path) -> Path:
+    """
+    Get the database file associated with the library at the given path.
+
+    Parameters
+    ----------
+    path: str
+        The path of the library.
+
+    Returns
+    -------
+    db_file: str
+        The file holding the database.
+    """
+    path = Path(path)
+    if not os.path.exists(path):
+        raise FileNotFoundError(f"Corrlib path {path} does not exist.")
+    config_path = path / CONFIG_FILENAME
+    config = ConfigParser()
+    if os.path.exists(config_path):
+        config.read(config_path)
+    else:
+        raise FileNotFoundError("Configuration file not found.")
+    db_file = Path(config.get('paths', 'db', fallback='backlogger.db'))
+    return db_file
+
+
+def cache_enabled(path: Path) -> bool:
+    """
+    Check, whether the library is cached.
+    Fallback is true.
+
+    Parameters
+    ----------
+    path: str
+        The path of the library.
+
+    Returns
+    -------
+    cached_bool: bool
+        Whether the given library is cached.
+    """
+    path = Path(path)
+    config_path = path / CONFIG_FILENAME
+    config = ConfigParser()
+    if os.path.exists(config_path):
+        config.read(config_path)
+    else:
+        raise FileNotFoundError("Configuration file not found.")
+    cached_str = config.get('core', 'cached', fallback='True')
+    if cached_str not in ['True', 'False']:
+        raise ValueError(f"String {cached_str} is not a valid option, only True and False are allowed!")
+    cached_bool = cached_str == ('True')
+    return cached_bool

corrlib/tracker.py

@@ -0,0 +1,177 @@
+import os
+from configparser import ConfigParser
+import datalad.api as dl
+from typing import Optional
+import shutil
+from .tools import get_db_file, CONFIG_FILENAME
+from pathlib import Path
+
+
+def get_tracker(path: Path) -> str:
+    """
+    Get the tracker used in the dataset located at path.
+
+    Parameters
+    ----------
+    path: str
+        The path to the backlogger folder.
+
+    Returns
+    -------
+    tracker: str
+        The tracker used in the dataset.
+    """
+    path = Path(path)
+    config_path = path / CONFIG_FILENAME
+    config = ConfigParser()
+    if os.path.exists(config_path):
+        config.read(config_path)
+    else:
+        raise FileNotFoundError(f"No config file found in {path}.")
+    tracker = config.get('core', 'tracker', fallback='datalad')
+    return tracker
+
+
+def get(path: Path, file: Path) -> None:
+    """
+    Wrapper function to get a file from the dataset located at path with the specified tracker.
+
+    Parameters
+    ----------
+    path: str
+        The path to the backlogger folder.
+    file: str
+        The file to get.
+    """
+    path = Path(path)
+    tracker = get_tracker(path)
+    if tracker == 'datalad':
+        if file == get_db_file(path):
+            print("Downloading database...")
+        else:
+            print("Downloading data...")
+        dl.get(os.path.join(path, file), dataset=path)
+        print("> downloaded file")
+    elif tracker == 'None':
+        pass
+    else:
+        raise ValueError(f"Tracker {tracker} is not supported.")
+    return
+
+
+def save(path: Path, message: str, files: Optional[list[Path]]=None) -> None:
+    """
+    Wrapper function to save a file to the dataset located at path with the specified tracker.
+
+    Parameters
+    ----------
+    path: str
+        The path to the backlogger folder.
+    message: str
+        The commit message.
+    files: list[str], optional
+        The files to save. If None, all changes are saved.
+    """
+    path = Path(path)
+    tracker = get_tracker(path)
+    if tracker == 'datalad':
+        if files is not None:
+            files = [path / f for f in files]
+        dl.save(files, message=message, dataset=path)
+    elif tracker == 'None':
+        Warning("Tracker 'None' does not implement save.")
+        pass
+    else:
+        raise ValueError(f"Tracker {tracker} is not supported.")
+
+
+def init(path: Path, tracker: str='datalad') -> None:
+    """
+    Initialize a dataset at the specified path with the specified tracker.
+
+    Parameters
+    ----------
+    path: str
+        The path to initialize the dataset.
+    tracker: str
+        The tracker to use. Currently only 'datalad' and 'None' are supported.
+    """
+    path = Path(path)
+    if tracker == 'datalad':
+        dl.create(path)
+    elif tracker == 'None':
+        os.makedirs(path, exist_ok=True)
+    else:
+        raise ValueError(f"Tracker {tracker} is not supported.")
+    return
+
+
+def unlock(path: Path, file: Path) -> None:
+    """
+    Wrapper function to unlock a file in the dataset located at path with the specified tracker.
+
+    Parameters
+    ----------
+    path    : str
+        The path to the backlogger folder.
+    file    : str
+        The file to unlock.
+    """
+    path = Path(path)
+    tracker = get_tracker(path)
+    if tracker == 'datalad':
+        dl.unlock(os.path.join(path, file), dataset=path)
+    elif tracker == 'None':
+        Warning("Tracker 'None' does not implement unlock.")
+        pass
+    else:
+        raise ValueError(f"Tracker {tracker} is not supported.")
+    return
+
+
+def clone(path: Path, source: str, target: str) -> None:
+    """
+    Wrapper function to clone a dataset from source to target with the specified tracker.
+    Parameters
+    ----------
+    path: str
+        The path to the backlogger folder.
+    source: str
+        The source dataset to clone.
+    target: str
+        The target path to clone the dataset to.
+    """
+    path = Path(path)
+    tracker = get_tracker(path)
+    if tracker == 'datalad':
+        dl.clone(target=target, source=source, dataset=path)
+    elif tracker == 'None':
+        os.makedirs(path, exist_ok=True)
+        # Implement a simple clone by copying files
+        shutil.copytree(source, target, dirs_exist_ok=False)
+    else:
+        raise ValueError(f"Tracker {tracker} is not supported.")
+    return
+
+
+def drop(path: Path, reckless: Optional[str]=None) -> None:
+    """
+    Wrapper function to drop data from a dataset located at path with the specified tracker.
+
+    Parameters
+    ----------
+    path: str
+        The path to the backlogger folder.
+    reckless: Optional[str]
+        The datalad's reckless option for dropping data.
+    """
+    path = Path(path)
+    tracker = get_tracker(path)
+    if tracker == 'datalad':
+        dl.drop(path, reckless=reckless)
+    elif tracker == 'None':
+        Warning("Tracker 'None' does not implement drop.")
+        pass
+    else:
+        raise ValueError(f"Tracker {tracker} is not supported.")
+    return

corrlib/version.py

@@ -1 +1,24 @@
-__version__ = "0.2.4"
+# file generated by vcs-versioning
+# don't change, don't track in version control
+from __future__ import annotations
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+version: str
+__version__: str
+__version_tuple__: tuple[int | str, ...]
+version_tuple: tuple[int | str, ...]
+commit_id: str | None
+__commit_id__: str | None
+
+__version__ = version = '0.3.1.dev0+g08de17e6b.d20260507'
+__version_tuple__ = version_tuple = (0, 3, 1, 'dev0', 'g08de17e6b.d20260507')
+
+__commit_id__ = commit_id = 'g08de17e6b'

pyproject.toml

@@ -1,6 +1,52 @@
 [build-system]
-requires = ["setuptools >= 63.0.0", "wheel"]
+requires = ["setuptools >= 63.0.0", "wheel", "setuptools-scm"]
 build-backend = "setuptools.build_meta"
 
+[project]
+requires-python = ">=3.10"
+name = "corrlib"
+dynamic = ["version"]
+dependencies = [
+    "gitpython>=3.1.45",
+    'pyerrors>=2.11.1',
+    "datalad>=1.1.0",
+    'typer>=0.12.5',
+]
+description = "Python correlation library"
+authors = [
+  { name = 'Justus Kuhlmann', email = 'j_kuhl19@uni-muenster.de'}
+]
+
+[project.scripts]
+pcl = "corrlib.cli:app"
+
+[tool.setuptools.packages.find]
+include = ["corrlib", "corrlib.*"]
+
+[tool.setuptools_scm]
+write_to = "corrlib/version.py"
+
 [tool.ruff.lint]
-ignore = ["F403"]
+ignore = ["E501"]
+extend-select = [
+  "YTT",
+  "E",
+  "W",
+  "F",
+]
+
+[tool.mypy]
+strict = true
+implicit_reexport = false
+follow_untyped_imports = false
+ignore_missing_imports = true
+
+[dependency-groups]
+dev = [
+    "mypy>=1.19.0",
+    "pandas-stubs>=2.3.3.251201",
+    "pytest>=9.0.1",
+    "pytest-cov>=7.0.0",
+    "pytest-pretty>=1.3.0",
+    "ruff>=0.14.7",
+]

setup.py

@@ -1,18 +0,0 @@
-from setuptools import setup
-from distutils.util import convert_path
-
-
-version = {}
-with open(convert_path('corrlib/version.py')) as ver_file:
-    exec(ver_file.read(), version)
-
-setup(name='pycorrlib',
-      version=version['__version__'],
-      author='Justus Kuhlmann',
-      author_email='j_kuhl19@uni-muenster.de',
-      install_requires=['pyerrors>=2.11.1', 'datalad>=1.1.0', 'typer>=0.12.5'],
-      entry_points = {
-        'console_scripts': ['pcl=corrlib.cli:app'],
-        },
-      packages=['corrlib', 'corrlib.input']
-      )

tests/cli_test.py

@@ -0,0 +1,92 @@
+from typer.testing import CliRunner
+from corrlib.cli import app
+import os
+import sqlite3 as sql
+from pathlib import Path
+
+
+runner = CliRunner()
+
+
+def test_version() -> None:
+    result = runner.invoke(app, ["--version"])
+    assert result.exit_code == 0
+    assert "corrlib" in result.output
+
+
+def test_init_folders(tmp_path: Path) -> None:
+    dataset_path = tmp_path / "test_dataset"
+    result = runner.invoke(app, ["init", "--dataset", str(dataset_path)])
+    assert result.exit_code == 0
+    assert os.path.exists(str(dataset_path))
+    assert os.path.exists(str(dataset_path / "backlogger.db"))
+
+
+def test_init_db(tmp_path: Path) -> None:
+    dataset_path = tmp_path / "test_dataset"
+    result = runner.invoke(app, ["init", "--dataset", str(dataset_path)])
+    assert result.exit_code == 0
+    assert os.path.exists(str(dataset_path / "backlogger.db"))
+    conn = sql.connect(str(dataset_path / "backlogger.db"))
+    cursor = conn.cursor()
+    cursor.execute("SELECT name FROM sqlite_master WHERE type='table';")
+    tables = cursor.fetchall()
+    expected_tables = [
+        'projects',
+        'backlogs',
+        ]
+    table_names = [table[0] for table in tables]
+    for expected_table in expected_tables:
+        assert expected_table in table_names
+
+    cursor.execute("SELECT * FROM projects;")
+    projects = cursor.fetchall()
+    assert len(projects) == 0
+
+    cursor.execute("SELECT * FROM backlogs;")
+    backlogs = cursor.fetchall()
+    assert len(backlogs) == 0
+
+    cursor.execute("PRAGMA table_info('projects');")
+    project_columns = cursor.fetchall()
+    expected_project_columns = [
+        "id",
+        "aliases",
+        "customTags",
+        "owner",
+        "code",
+        "created_at",
+        "updated_at"
+        ]
+    project_column_names = [col[1] for col in project_columns]
+    for expected_col in expected_project_columns:
+        assert expected_col in project_column_names
+
+    cursor.execute("PRAGMA table_info('backlogs');")
+    backlog_columns = cursor.fetchall()
+    expected_backlog_columns = [
+        "id",
+        "name",
+        "ensemble",
+        "code",
+        "path",
+        "project",
+        "customTags",
+        "parameters",
+        "parameter_file",
+        "created_at",
+        "updated_at"
+        ]
+    backlog_column_names = [col[1] for col in backlog_columns]
+    for expected_col in expected_backlog_columns:
+        assert expected_col in backlog_column_names
+
+
+def test_list(tmp_path: Path) -> None:
+    dataset_path = tmp_path / "test_dataset"
+    result = runner.invoke(app, ["init", "--dataset", str(dataset_path)])
+    assert result.exit_code == 0
+    result = runner.invoke(app, ["lister", "--dataset", str(dataset_path), "ensembles"])
+    assert result.exit_code == 0
+    result = runner.invoke(app, ["lister", "--dataset", str(dataset_path), "projects"])
+    assert result.exit_code == 0

tests/find_test.py

@@ -0,0 +1,439 @@
+import corrlib.find as find
+import sqlite3
+from pathlib import Path
+import corrlib.initialization as cinit
+import pytest
+import pandas as pd
+import datalad.api as dl
+import datetime as dt
+
+
+def make_sql(path: Path) -> Path:
+    db = path / "backlogger.db"
+    cinit._create_db(db)
+    return db
+
+
+def make_config(path: Path) -> None:
+    cinit._write_config(path,  cinit._create_config(path, "datalad", False))
+
+
+def test_find_lookup_by_one_alias(tmp_path: Path) -> None:
+    make_config(tmp_path)
+    db = make_sql(tmp_path)
+    conn = sqlite3.connect(db)
+    c = conn.cursor()
+    uuid = "test_uuid"
+    alias_str = "fun_project"
+    tag_str = "tt"
+    owner = "tester"
+    code = "test_code"
+    c.execute("INSERT INTO projects (id, aliases, customTags, owner, code, created_at, updated_at) VALUES (?, ?, ?, ?, ?, datetime('now'), datetime('now'))",
+              (uuid, alias_str, tag_str, owner, code))
+    conn.commit()
+    assert uuid == find._project_lookup_by_alias(tmp_path, "fun_project")
+    uuid = "test_uuid2"
+    alias_str = "fun_project"
+    c.execute("INSERT INTO projects (id, aliases, customTags, owner, code, created_at, updated_at) VALUES (?, ?, ?, ?, ?, datetime('now'), datetime('now'))",
+              (uuid, alias_str, tag_str, owner, code))
+    conn.commit()
+    with pytest.raises(Exception):
+        assert uuid == find._project_lookup_by_alias(db, "fun_project")
+    conn.close()
+
+
+def test_find_lookup_by_id(tmp_path: Path) -> None:
+    make_config(tmp_path)
+    db = make_sql(tmp_path)
+    conn = sqlite3.connect(db)
+    c = conn.cursor()
+    uuid = "test_uuid"
+    alias_str = "fun_project"
+    tag_str = "tt"
+    owner = "tester"
+    code = "test_code"
+    c.execute("INSERT INTO projects (id, aliases, customTags, owner, code, created_at, updated_at) VALUES (?, ?, ?, ?, ?, datetime('now'), datetime('now'))",
+              (uuid, alias_str, tag_str, owner, code))
+    conn.commit()
+    conn.close()
+    result = find._project_lookup_by_id(tmp_path, uuid)[0]
+    assert uuid == result[0]
+    assert alias_str == result[1]
+    assert tag_str == result[2]
+    assert owner == result[3]
+    assert code == result[4]
+
+
+def test_time_filter() -> None:
+    record_A = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf0", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+            '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] # only created
+    record_B = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf1", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2025-03-26 12:55:18.229966', '2025-04-26 12:55:18.229966'] # created and updated
+    record_C = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf2", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2026-03-26 12:55:18.229966', '2026-04-14 12:55:18.229966'] # created and updated later
+    record_D = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf3", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2026-03-26 12:55:18.229966', '2026-03-27 12:55:18.229966']
+    record_E = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf4", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2024-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # only created, earlier
+    record_F = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf5", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2026-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # this is invalid...
+    record_G = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf2", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2026-03-26 12:55:18.229966', str(dt.datetime.now() + dt.timedelta(days=2, hours=3, minutes=5, seconds=30))] # created and updated later
+
+    data = [record_A, record_B, record_C, record_D, record_E]
+    cols = ["name",
+            "ensemble",
+            "code",
+            "path",
+            "project",
+            "parameters",
+            "parameter_file",
+            "created_at",
+            "updated_at"]
+    df = pd.DataFrame(data,columns=cols)
+
+    results = find._time_filter(df, created_before='2023-03-26 12:55:18.229966')
+    assert results.empty
+    results = find._time_filter(df, created_before='2027-03-26 12:55:18.229966')
+    assert len(results) == 5
+    results = find._time_filter(df, created_before='2026-03-25 12:55:18.229966')
+    assert len(results) == 3
+    results = find._time_filter(df, created_before='2026-03-26 12:55:18.229965')
+    assert len(results) == 3
+    results = find._time_filter(df, created_before='2025-03-04 12:55:18.229965')
+    assert len(results) == 1
+
+    results = find._time_filter(df, created_after='2023-03-26 12:55:18.229966')
+    assert len(results) == 5
+    results = find._time_filter(df, created_after='2027-03-26 12:55:18.229966')
+    assert results.empty
+    results = find._time_filter(df, created_after='2026-03-25 12:55:18.229966')
+    assert len(results) == 2
+    results = find._time_filter(df, created_after='2026-03-26 12:55:18.229965')
+    assert len(results) == 2
+    results = find._time_filter(df, created_after='2025-03-04 12:55:18.229965')
+    assert len(results) == 4
+
+    results = find._time_filter(df, updated_before='2023-03-26 12:55:18.229966')
+    assert results.empty
+    results = find._time_filter(df, updated_before='2027-03-26 12:55:18.229966')
+    assert len(results) == 5
+    results = find._time_filter(df, updated_before='2026-03-25 12:55:18.229966')
+    assert len(results) == 3
+    results = find._time_filter(df, updated_before='2026-03-26 12:55:18.229965')
+    assert len(results) == 3
+    results = find._time_filter(df, updated_before='2025-03-04 12:55:18.229965')
+    assert len(results) == 1
+
+    results = find._time_filter(df, updated_after='2023-03-26 12:55:18.229966')
+    assert len(results) == 5
+    results = find._time_filter(df, updated_after='2027-03-26 12:55:18.229966')
+    assert results.empty
+    results = find._time_filter(df, updated_after='2026-03-25 12:55:18.229966')
+    assert len(results) == 2
+    results = find._time_filter(df, updated_after='2026-03-26 12:55:18.229965')
+    assert len(results) == 2
+    results = find._time_filter(df, updated_after='2025-03-04 12:55:18.229965')
+    assert len(results) == 4
+
+    data = [record_A, record_B, record_C, record_D, record_F]
+    cols = ["name",
+            "ensemble",
+            "code",
+            "path",
+            "project",
+            "parameters",
+            "parameter_file",
+            "created_at",
+            "updated_at"]
+    df = pd.DataFrame(data,columns=cols)
+
+    with pytest.raises(ValueError):
+        results = find._time_filter(df, created_before='2023-03-26 12:55:18.229966')
+
+    data = [record_A, record_B, record_C, record_D, record_G]
+    cols = ["name",
+            "ensemble",
+            "code",
+            "path",
+            "project",
+            "parameters",
+            "parameter_file",
+            "created_at",
+            "updated_at"]
+    df = pd.DataFrame(data,columns=cols)
+
+    with pytest.raises(ValueError):
+        results = find._time_filter(df, created_before='2023-03-26 12:55:18.229966')
+
+
+def test_db_lookup(tmp_path: Path) -> None:
+    db = make_sql(tmp_path)
+    conn = sqlite3.connect(db)
+    c = conn.cursor()
+
+    corr = "f_A"
+    ensemble = "SF_A"
+    code = "openQCD"
+    meas_path = "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf"
+    uuid = "Project_A"
+    pars = "{par_A: 3.0, par_B: 5.0}"
+    parameter_file = "projects/Project_A/myinput.in"
+    c.execute("INSERT INTO backlogs (name, ensemble, code, path, project, parameters, parameter_file, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, datetime('now'), datetime('now'))",
+                                                (corr, ensemble, code, meas_path, uuid, pars, parameter_file))
+    conn.commit()
+
+    results = find._db_lookup(db, ensemble, corr, code)
+    assert len(results) == 1
+    results = find._db_lookup(db, "SF_B", corr, code)
+    assert results.empty
+    results = find._db_lookup(db, ensemble, "g_A", code)
+    assert results.empty
+    results = find._db_lookup(db, ensemble, corr, "sfcf")
+    assert results.empty
+    results = find._db_lookup(db, ensemble, corr, code, project = "Project_A")
+    assert len(results) == 1
+    results = find._db_lookup(db, ensemble, corr, code, project = "Project_B")
+    assert results.empty
+    results = find._db_lookup(db, ensemble, corr, code, parameters = pars)
+    assert len(results) == 1
+    results = find._db_lookup(db, ensemble, corr, code, parameters = '{"par_A": 3.0, "par_B": 4.0}')
+    assert results.empty
+
+    corr = "g_A"
+    ensemble = "SF_A"
+    code = "openQCD"
+    meas_path = "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf"
+    uuid = "Project_A"
+    pars = '{"par_A": 3.0, "par_B": 4.0}'
+    parameter_file = "projects/Project_A/myinput.in"
+    c.execute("INSERT INTO backlogs (name, ensemble, code, path, project, parameters, parameter_file, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, datetime('now'), datetime('now'))",
+                                                (corr, ensemble, code, meas_path, uuid, pars, parameter_file))
+    conn.commit()
+
+    corr = "f_A"
+    results = find._db_lookup(db, ensemble, corr, code)
+    assert len(results) == 1
+    results = find._db_lookup(db, "SF_B", corr, code)
+    assert results.empty
+    results = find._db_lookup(db, ensemble, "g_A", code)
+    assert len(results) == 1
+    results = find._db_lookup(db, ensemble, corr, "sfcf")
+    assert results.empty
+    results = find._db_lookup(db, ensemble, corr, code, project = "Project_A")
+    assert len(results) == 1
+    results = find._db_lookup(db, ensemble, "g_A", code, project = "Project_A")
+    assert len(results) == 1
+    results = find._db_lookup(db, ensemble, corr, code, project = "Project_B")
+    assert results.empty
+    results = find._db_lookup(db, ensemble, "g_A", code, project = "Project_B")
+    assert results.empty
+    results = find._db_lookup(db, ensemble, corr, code, parameters = pars)
+    assert results.empty
+    results = find._db_lookup(db, ensemble, "g_A", code, parameters = '{"par_A": 3.0, "par_B": 4.0}')
+    assert len(results) == 1
+
+    conn.close()
+
+
+def test_sfcf_drop() -> None:
+    parameters0 = {
+                    'offset': [0,0,0],
+                    'quarks': [{'mass': 1, 'thetas': [0,0,0]}, {'mass': 2, 'thetas': [0,0,1]}], # m0s = -3.5, -3.75
+                    'wf1': [[1, [0, 0]], [0.5, [1, 0]], [.75, [.5, .5]]],
+                    'wf2': [[1, [2, 1]], [2, [0.5, -0.5]], [.5, [.75, .72]]],
+                   }
+
+    assert not find._sfcf_drop(parameters0, offset=[0,0,0])
+    assert find._sfcf_drop(parameters0, offset=[1,0,0])
+
+    assert not find._sfcf_drop(parameters0, quark_kappas = [1, 2])
+    assert find._sfcf_drop(parameters0, quark_kappas = [-3.1, -3.72])
+
+    assert not find._sfcf_drop(parameters0, quark_masses = [-3.5, -3.75])
+    assert find._sfcf_drop(parameters0, quark_masses = [-3.1, -3.72])
+
+    assert not find._sfcf_drop(parameters0, qk1 = 1)
+    assert not find._sfcf_drop(parameters0, qk2 = 2)
+    assert find._sfcf_drop(parameters0, qk1 = 2)
+    assert find._sfcf_drop(parameters0, qk2 = 1)
+
+    assert not find._sfcf_drop(parameters0, qk1 = [0.5,1.5])
+    assert not find._sfcf_drop(parameters0, qk2 = [1.5,2.5])
+    assert find._sfcf_drop(parameters0, qk1 = 2)
+    assert find._sfcf_drop(parameters0, qk2 = 1)
+    with pytest.raises(ValueError):
+        assert not find._sfcf_drop(parameters0, qk1 = [0.5,1,5])
+    with pytest.raises(ValueError):
+        assert not find._sfcf_drop(parameters0, qk2 = [1,5,2.5])
+
+    assert find._sfcf_drop(parameters0, qm1 = 1.2)
+    assert find._sfcf_drop(parameters0, qm2 = 2.2)
+    assert not find._sfcf_drop(parameters0, qm1 = -3.5)
+    assert not find._sfcf_drop(parameters0, qm2 = -3.75)
+
+    assert find._sfcf_drop(parameters0, qm2 = 1.2)
+    assert find._sfcf_drop(parameters0, qm1 = 2.2)
+    with pytest.raises(ValueError):
+        assert not find._sfcf_drop(parameters0, qm1 = [0.5,1,5])
+    with pytest.raises(ValueError):
+        assert not find._sfcf_drop(parameters0, qm2 = [1,5,2.5])
+
+
+def test_openQCD_filter() -> None:
+    record_0 = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+            '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966']
+    record_1 = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+            '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966']
+    record_2 = ["f_P", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+            '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966']
+    record_3 = ["f_P", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+            '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966']
+    data = [
+        record_0,
+        record_1,
+        record_2,
+        record_3,
+    ]
+    cols = ["name",
+            "ensemble",
+            "code",
+            "path",
+            "project",
+            "parameters",
+            "parameter_file",
+            "created_at",
+            "updated_at"]
+    df = pd.DataFrame(data,columns=cols)
+
+    with pytest.warns(Warning):
+        find.openQCD_filter(df, a = "asdf")
+
+
+def test_code_filter() -> None:
+    record_0 = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+            '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966']
+    record_1 = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+            '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966']
+    record_2 = ["f_P", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+            '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966']
+    record_3 = ["f_P", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+            '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966']
+    record_4 = ["f_A", "ensA", "openQCD", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+            '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966']
+    record_5 = ["f_A", "ensA", "openQCD", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+            '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966']
+    record_6 = ["f_P", "ensA", "openQCD", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+            '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966']
+    record_7 = ["f_P", "ensA", "openQCD", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+            '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966']
+    record_8 = ["f_P", "ensA", "openQCD", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+            '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966']
+    data = [
+        record_0,
+        record_1,
+        record_2,
+        record_3,
+    ]
+    cols = ["name",
+            "ensemble",
+            "code",
+            "path",
+            "project",
+            "parameters",
+            "parameter_file",
+            "created_at",
+            "updated_at"]
+    df = pd.DataFrame(data,columns=cols)
+
+    res = find._code_filter(df, "sfcf")
+    assert len(res) == 4
+
+    data = [
+        record_4,
+        record_5,
+        record_6,
+        record_7,
+        record_8,
+    ]
+    cols = ["name",
+            "ensemble",
+            "code",
+            "path",
+            "project",
+            "parameters",
+            "parameter_file",
+            "created_at",
+            "updated_at"]
+    df = pd.DataFrame(data,columns=cols)
+
+    res = find._code_filter(df, "openQCD")
+    assert len(res) == 5
+    with pytest.raises(ValueError):
+        res = find._code_filter(df, "asdf")
+
+
+def test_find_record() -> None:
+    assert True
+
+
+def test_find_project(tmp_path: Path) -> None:
+    cinit.create(tmp_path)
+    db = tmp_path / "backlogger.db"
+    dl.unlock(str(db), dataset=str(tmp_path))
+    conn = sqlite3.connect(db)
+    c = conn.cursor()
+    uuid = "test_uuid"
+    alias_str = "fun_project"
+    tag_str = "tt"
+    owner = "tester"
+    code = "test_code"
+    c.execute("INSERT INTO projects (id, aliases, customTags, owner, code, created_at, updated_at) VALUES (?, ?, ?, ?, ?, datetime('now'), datetime('now'))",
+              (uuid, alias_str, tag_str, owner, code))
+    conn.commit()
+
+    assert uuid == find.find_project(tmp_path, "fun_project")
+
+    uuid = "test_uuid2"
+    alias_str = "fun_project"
+    c.execute("INSERT INTO projects (id, aliases, customTags, owner, code, created_at, updated_at) VALUES (?, ?, ?, ?, ?, datetime('now'), datetime('now'))",
+              (uuid, alias_str, tag_str, owner, code))
+    conn.commit()
+
+    with pytest.raises(Exception):
+        assert uuid == find._project_lookup_by_alias(tmp_path, "fun_project")
+    conn.close()
+
+
+def test_list_projects(tmp_path: Path) -> None:
+    cinit.create(tmp_path)
+    db = tmp_path / "backlogger.db"
+    dl.unlock(str(db), dataset=str(tmp_path))
+    conn = sqlite3.connect(db)
+    c = conn.cursor()
+    uuid = "test_uuid"
+    alias_str = "fun_project"
+    tag_str = "tt"
+    owner = "tester"
+    code = "test_code"
+
+    c.execute("INSERT INTO projects (id, aliases, customTags, owner, code, created_at, updated_at) VALUES (?, ?, ?, ?, ?, datetime('now'), datetime('now'))",
+              (uuid, alias_str, tag_str, owner, code))
+    uuid = "test_uuid2"
+    alias_str = "fun_project2"
+    c.execute("INSERT INTO projects (id, aliases, customTags, owner, code, created_at, updated_at) VALUES (?, ?, ?, ?, ?, datetime('now'), datetime('now'))",
+              (uuid, alias_str, tag_str, owner, code))
+    uuid = "test_uuid3"
+    alias_str = "fun_project3"
+    c.execute("INSERT INTO projects (id, aliases, customTags, owner, code, created_at, updated_at) VALUES (?, ?, ?, ?, ?, datetime('now'), datetime('now'))",
+              (uuid, alias_str, tag_str, owner, code))
+    uuid = "test_uuid4"
+    alias_str = "fun_project4"
+    c.execute("INSERT INTO projects (id, aliases, customTags, owner, code, created_at, updated_at) VALUES (?, ?, ?, ?, ?, datetime('now'), datetime('now'))",
+              (uuid, alias_str, tag_str, owner, code))
+    conn.commit()
+    conn.close()
+    results = find.list_projects(tmp_path)
+    assert len(results) == 4
+    for i in range(4):
+        assert len(results[i]) == 2

tests/import_project_test.py

@@ -1,7 +1,7 @@
 import corrlib.toml as t
 
 
-def test_toml_check_measurement_data():
+def test_toml_check_measurement_data() -> None:
     measurements = {
                     "a":
                         {
@@ -14,4 +14,4 @@ def test_toml_check_measurement_data():
                             "names": ['list', 'of', 'names']
                         }
                     }
-    t.check_measurement_data(measurements)
+    t.check_measurement_data(measurements, "sfcf")

tests/initialization_test.py

@@ -0,0 +1,94 @@
+import corrlib.initialization as init
+import os
+import sqlite3 as sql
+from pathlib import Path
+
+def test_init_folders(tmp_path: Path) -> None:
+    dataset_path = tmp_path / "test_dataset"
+    init.create(dataset_path)
+    assert os.path.exists(str(dataset_path))
+    assert os.path.exists(str(dataset_path / "backlogger.db"))
+
+
+def test_init_folders_no_tracker(tmp_path: Path) -> None:
+    dataset_path = tmp_path / "test_dataset"
+    init.create(dataset_path, tracker="None")
+    assert os.path.exists(str(dataset_path))
+    assert os.path.exists(str(dataset_path / "backlogger.db"))
+
+
+def test_init_config(tmp_path: Path) -> None:
+    dataset_path = tmp_path / "test_dataset"
+    init.create(dataset_path, tracker="None")
+    config_path = dataset_path / ".corrlib"
+    assert os.path.exists(str(config_path))
+    from configparser import ConfigParser
+    config = ConfigParser()
+    config.read(str(config_path))
+    assert config.get("core", "tracker") == "None"
+    assert config.get("core", "version") == "1.0"
+    assert config.get("core", "cached") == "True"
+    assert config.get("paths", "db") == "backlogger.db"
+    assert config.get("paths", "projects_path") == "projects"
+    assert config.get("paths", "archive_path") == "archive"
+    assert config.get("paths", "toml_imports_path") == "toml_imports"
+    assert config.get("paths", "import_scripts_path") == "import_scripts"
+
+
+def test_init_db(tmp_path: Path) -> None:
+    dataset_path = tmp_path / "test_dataset"
+    init.create(dataset_path)
+    assert os.path.exists(str(dataset_path / "backlogger.db"))
+    conn = sql.connect(str(dataset_path / "backlogger.db"))
+    cursor = conn.cursor()
+    cursor.execute("SELECT name FROM sqlite_master WHERE type='table';")
+    tables = cursor.fetchall()
+    expected_tables = [
+        'projects',
+        'backlogs',
+        ]
+    table_names = [table[0] for table in tables]
+    for expected_table in expected_tables:
+        assert expected_table in table_names
+
+    cursor.execute("SELECT * FROM projects;")
+    projects = cursor.fetchall()
+    assert len(projects) == 0
+
+    cursor.execute("SELECT * FROM backlogs;")
+    backlogs = cursor.fetchall()
+    assert len(backlogs) == 0
+
+    cursor.execute("PRAGMA table_info('projects');")
+    project_columns = cursor.fetchall()
+    expected_project_columns = [
+        "id",
+        "aliases",
+        "customTags",
+        "owner",
+        "code",
+        "created_at",
+        "updated_at"
+        ]
+    project_column_names = [col[1] for col in project_columns]
+    for expected_col in expected_project_columns:
+        assert expected_col in project_column_names
+
+    cursor.execute("PRAGMA table_info('backlogs');")
+    backlog_columns = cursor.fetchall()
+    expected_backlog_columns = [
+        "id",
+        "name",
+        "ensemble",
+        "code",
+        "path",
+        "project",
+        "customTags",
+        "parameters",
+        "parameter_file",
+        "created_at",
+        "updated_at"
+        ]
+    backlog_column_names = [col[1] for col in backlog_columns]
+    for expected_col in expected_backlog_columns:
+        assert expected_col in backlog_column_names

tests/integrity_test.py

@@ -0,0 +1,189 @@
+import corrlib.integrity as integ
+import corrlib.find as find
+import datalad.api as dl
+import corrlib.initialization as cinit
+import sqlite3
+from pathlib import Path
+import os
+import pandas as pd
+import datetime as dt
+import pytest
+
+
+def test_list_ensembles(tmp_path: Path) -> None:
+    """
+    Check against the implementation in find to check if they are the same.
+    """
+    os.mkdir(tmp_path / 'archive')
+    os.mkdir(tmp_path / 'archive' / 'A')
+    os.mkdir(tmp_path / 'archive' / 'B')
+    os.mkdir(tmp_path / 'archive' / 'C')
+    integ_results = integ._list_ensembles(tmp_path)
+    assert len(integ_results) == 3
+    find_results = find.list_ensembles(tmp_path)
+    assert len(find_results) == 3
+    for f,i in zip(find_results, integ_results):
+        assert f == i
+
+
+def test_list_projects(tmp_path: Path) -> None:
+    cinit.create(tmp_path)
+    db = tmp_path / "backlogger.db"
+    dl.unlock(str(db), dataset=str(tmp_path))
+    conn = sqlite3.connect(db)
+    c = conn.cursor()
+
+    customTags = ""
+    owner = "owner"
+    code = "sfcf"
+    created_at = "today"
+    updated_at = "today"
+
+    id = "asdf1"
+    aliases = "a1,s1,d1,f1"
+    c.execute("INSERT INTO projects (id, aliases, customTags, owner, code, created_at, updated_at) VALUES (?,?,?,?,?,?,?)", (id, aliases, customTags, owner, code , created_at, updated_at))
+    id = "asdf2"
+    aliases = "a2,s2,d2,f2"
+    c.execute("INSERT INTO projects (id, aliases, customTags, owner, code, created_at, updated_at) VALUES (?,?,?,?,?,?,?)", (id, aliases, customTags, owner, code , created_at, updated_at))
+    id = "asdf3"
+    aliases = "a3,s3,d3,f3"
+    c.execute("INSERT INTO projects (id, aliases, customTags, owner, code, created_at, updated_at) VALUES (?,?,?,?,?,?,?)", (id, aliases, customTags, owner, code , created_at, updated_at))
+    conn.commit()
+    conn.close
+    integ_results = integ._list_projects(tmp_path)
+    assert len(integ_results) == 3
+    find_results = find.list_projects(tmp_path)
+    assert len(find_results) == 3
+    for f,i in zip(find_results, integ_results):
+        assert f == i
+
+
+def test_has_valid_time() -> None:
+    record_A = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf0", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+            '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] # only created
+    record_B = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf1", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2025-03-26 12:55:18.229966', '2025-04-26 12:55:18.229966'] # created and updated
+    record_C = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf2", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2026-03-26 12:55:18.229966', '2026-04-14 12:55:18.229966'] # created and updated later
+    record_D = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf3", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2026-03-26 12:55:18.229966', '2026-03-27 12:55:18.229966']
+    record_E = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf4", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2024-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # only created, earlier
+    record_F = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf5", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2026-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # this is invalid...
+    record_G = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf2", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2026-03-26 12:55:18.229966', str(dt.datetime.now() + dt.timedelta(days=2, hours=3, minutes=5, seconds=30))] # created and updated later
+    cols = ["name",
+            "ensemble",
+            "code",
+            "path",
+            "project",
+            "parameters",
+            "parameter_file",
+            "created_at",
+            "updated_at"]
+    data = [record_A, record_B, record_C, record_D, record_E]
+
+    df = pd.DataFrame(data,columns=cols)
+    for _, result in df.iterrows():
+        assert integ.has_valid_times(result)
+    data = [record_F, record_G]
+    df = pd.DataFrame(data,columns=cols)
+    for _, result in df.iterrows():
+        assert not integ.has_valid_times(result)
+
+
+def test_are_keys_unique(tmp_path: Path) -> None:
+    db = tmp_path / 'test_success.db'
+
+    record_A = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf0", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+            '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] # only created
+    record_B = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf1", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2025-03-26 12:55:18.229966', '2025-04-26 12:55:18.229966'] # created and updated
+    record_C = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf2", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2026-03-26 12:55:18.229966', '2026-04-14 12:55:18.229966'] # created and updated later
+    record_D = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf3", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2026-03-26 12:55:18.229966', '2026-03-27 12:55:18.229966']
+    record_E = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf4", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2024-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # only created, earlier
+    record_F = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf5", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2026-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # this is invalid...
+    record_G = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf2", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2026-03-26 12:55:18.229966', str(dt.datetime.now() + dt.timedelta(days=2, hours=3, minutes=5, seconds=30))] # created and updated later
+
+    cols = ["name",
+            "ensemble",
+            "code",
+            "path",
+            "project",
+            "parameters",
+            "parameter_file",
+            "created_at",
+            "updated_at"]
+
+    data = [record_A, record_B, record_C, record_D, record_E, record_F]
+    df = pd.DataFrame(data,columns=cols)
+    conn = sqlite3.connect(db)
+    df.to_sql('backlogs', conn)
+    conn.close()
+    assert integ.are_keys_unique(db, 'backlogs', 'path')
+
+    db = tmp_path / 'test_fail.db'
+    data = [record_A, record_B, record_C, record_D, record_E, record_F, record_G]
+
+    df = pd.DataFrame(data,columns=cols)
+    conn = sqlite3.connect(db)
+    df.to_sql('backlogs', conn)
+    conn.close()
+    assert not integ.are_keys_unique(db, 'backlogs', 'path')
+
+
+def test_check_path_format() -> None:
+    record_A = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_A.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+            '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] # only created
+    record_B = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_B.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2025-03-26 12:55:18.229966', '2025-04-26 12:55:18.229966'] # created and updated
+    record_C = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_A.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2026-03-26 12:55:18.229966', '2026-04-14 12:55:18.229966'] # created and updated later
+    record_D = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_B.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2026-03-26 12:55:18.229966', '2026-03-27 12:55:18.229966']
+    record_E = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_A.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2024-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # only created, earlier
+    record_F = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_B.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2026-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # this is invalid...
+    record_G = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_A.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfas", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2026-03-26 12:55:18.229966', str(dt.datetime.now() + dt.timedelta(days=2, hours=3, minutes=5, seconds=30))] # created and updated later
+
+    projects = ['Project_A', 'Project_B']
+    ensembles = ['ensA']
+    cols = ["name",
+            "ensemble",
+            "code",
+            "path",
+            "project",
+            "parameters",
+            "parameter_file",
+            "created_at",
+            "updated_at"]
+
+    data = [record_A, record_B, record_C, record_D, record_E, record_F]
+    df = pd.DataFrame(data,columns=cols)
+    for _, result in df.iterrows():
+        integ.check_path_format(result, ensembles, projects)
+
+    projects = ['Project_A', 'Project_B']
+    ensembles = ['ensB']
+    for _, result in df.iterrows():
+        with pytest.raises(ValueError):
+            integ.check_path_format(result, ensembles, projects)
+
+    projects = ['Project_A', 'Project_B']
+    ensembles = ['ensA', 'ensB']
+    for _, result in df.iterrows():
+        integ.check_path_format(result, ensembles, projects)
+
+    data = [record_G]
+    df = pd.DataFrame(data,columns=cols)
+    for _, result in df.iterrows():
+        with pytest.raises(ValueError):
+            integ.check_path_format(result, ensembles, projects)

tests/sfcf_in_test.py

@@ -1,7 +1,7 @@
 import corrlib.input.sfcf as input
 import json
 
-def test_get_specs():
+def test_get_specs() -> None:
     parameters = {
         'crr': [
             'f_P', 'f_A'

tests/tools_test.py

@@ -0,0 +1,88 @@
+from corrlib import tools as tl
+from configparser import ConfigParser
+from pathlib import Path
+import pytest
+
+
+def test_m2k() -> None:
+    for m in [0.1, 0.5, 1.0]:
+        expected_k = 1 / (2 * m + 8)
+        assert tl.m2k(m) == expected_k
+
+
+def test_k2m() -> None:
+    for m in [0.1, 0.5, 1.0]:
+        assert tl.k2m(m) == (1/(2*m))-4
+
+
+def test_k2m_m2k() -> None:
+    for m in [0.1, 0.5, 1.0]:
+        k = tl.m2k(m)
+        m_converted = tl.k2m(k)
+        assert abs(m - m_converted) < 1e-9
+
+
+def test_str2list() -> None:
+    assert tl.str2list("a,b,c") == ["a", "b", "c"]
+    assert tl.str2list("1,2,3") == ["1", "2", "3"]
+
+
+def test_list2str() -> None:
+    assert tl.list2str(["a", "b", "c"]) == "a,b,c"
+    assert tl.list2str(["1", "2", "3"]) == "1,2,3"
+
+
+def test_set_config(tmp_path: Path) -> None:
+    section = "core"
+    option = "test_option"
+    value = "test_value"
+    # config is not yet available
+    tl.set_config(tmp_path, section, option, value)
+    config_path = tmp_path / '.corrlib'
+    config = ConfigParser()
+    config.read(config_path)
+    assert config.get('core', 'test_option', fallback="not the value") == "test_value"
+    # now, a config file is already present
+    section = "core"
+    option = "test_option2"
+    value = "test_value2"
+    tl.set_config(tmp_path, section, option, value)
+    config_path = tmp_path / '.corrlib'
+    config = ConfigParser()
+    config.read(config_path)
+    assert config.get('core', 'test_option2', fallback="not the value") == "test_value2"
+    # update option 2
+    section = "core"
+    option = "test_option2"
+    value = "test_value3"
+    tl.set_config(tmp_path, section, option, value)
+    config_path = tmp_path / '.corrlib'
+    config = ConfigParser()
+    config.read(config_path)
+    assert config.get('core', 'test_option2', fallback="not the value") == "test_value3"
+
+
+def test_get_db_file(tmp_path: Path) -> None:
+    section = "paths"
+    option = "db"
+    value = "test_value"
+    # config is not yet available
+    tl.set_config(tmp_path, section, option, value)
+    assert tl.get_db_file(tmp_path) == Path("test_value")
+    with pytest.raises(FileNotFoundError):
+        tl.get_db_file(tmp_path / "doesnotexist")
+
+
+def test_cache_enabled(tmp_path: Path) -> None:
+    section = "core"
+    option = "cached"
+    # config is not yet available
+    tl.set_config(tmp_path, section, option, "True")
+    assert tl.cache_enabled(tmp_path)
+    tl.set_config(tmp_path, section, option, "False")
+    assert not tl.cache_enabled(tmp_path)
+    tl.set_config(tmp_path, section, option, "lalala")
+    with pytest.raises(ValueError):
+        tl.cache_enabled(tmp_path)
+    with pytest.raises(FileNotFoundError):
+        tl.cache_enabled(tmp_path / "doesnotexist")