Merge pull request 'develop' (#41) from develop into master

Reviewed-on: #41

.github/workflows/mypy.yaml

@@ -0,0 +1,29 @@
+name: Mypy
+
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  mypy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install git-annex
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y git-annex 
+      - name: Check out the repository
+        uses: https://github.com/RouxAntoine/checkout@v4.1.8
+        with:
+          show-progress: true
+      - name: Setup python
+        uses: https://github.com/actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Install uv
+        uses: https://github.com/astral-sh/setup-uv@v5
+      - name: Install corrlib
+        run: uv sync --locked --all-extras --dev --python "3.12"
+      - name: Run tests
+        run: uv run mypy corrlib

.github/workflows/pytest.yaml

@@ -0,0 +1,42 @@
+name: Pytest
+
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 4 1 * *'
+
+jobs:
+  pytest:
+    strategy:
+      matrix:
+        python-version:
+          - "3.12"
+          - "3.13"
+          - "3.14"
+
+    runs-on: ubuntu-latest
+    steps:
+      - name: Setup git
+        run: |
+          git config --global user.email "tester@example.com"
+          git config --global user.name "Tester"
+      - name: Install git-annex
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y git-annex
+      - name: Check out the repository
+        uses: https://github.com/RouxAntoine/checkout@v4.1.8
+        with:
+          show-progress: true
+      - name: Setup python
+        uses: https://github.com/actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install uv
+        uses: https://github.com/astral-sh/setup-uv@v5
+      - name: Install corrlib
+        run: uv sync --locked --all-extras --dev --python ${{ matrix.python-version }}
+      - name: Run tests
+        run: uv run pytest --cov=corrlib tests

.github/workflows/ruff.yaml

@@ -0,0 +1,30 @@
+name: Ruff
+
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  ruff:
+
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install git-annex
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y git-annex
+      - name: Check out the repository
+        uses: https://github.com/RouxAntoine/checkout@v4.1.8
+        with:
+          show-progress: true
+      - name: Setup python
+        uses: https://github.com/actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Install uv
+        uses: https://github.com/astral-sh/setup-uv@v5
+      - name: Install corrlib
+        run: uv sync --locked --all-extras --dev --python "3.12"
+      - name: Run tests
+        run: uv run ruff check corrlib

.gitignore

@@ -2,3 +2,7 @@ pyerrors_corrlib.egg-info
 __pycache__
 *.egg-info
 test.ipynb
+.vscode
+.venv
+.pytest_cache
+.coverage

corrlib/__init__.py

@@ -1,5 +1,5 @@
 """
-The aim of this project is to extend pyerrors to be able to collect measurements from different projects and make them easily accessable to 
+The aim of this project is to extend pyerrors to be able to collect measurements from different projects and make them easily accessable to
 the research group. The idea is to build a database, in which the researcher can easily search for measurements on a correlator basis,
 which may be reusable.
 As a standard to store the measurements, we will use the .json.gz format from pyerrors.
@@ -15,9 +15,10 @@ For now, we are interested in collecting primary IObservables only, as these are
 
 __app_name__ = "corrlib"
 
-from .main import *
 from .import input as input
-from .initialization import *
+from .initialization import create as create
-from .meas_io import *
+from .meas_io import load_record as load_record
-from .find import *
+from .meas_io import load_records as load_records
-from .version import __version__
+from .find import find_project as find_project
+from .find import find_record as find_record
+from .find import list_projects as list_projects

corrlib/__main__.py

@@ -1,8 +1,9 @@
 from corrlib import cli, __app_name__
 
 
-def main():
+def main() -> None:
     cli.app(prog_name=__app_name__)
+    return
 
 
 if __name__ == "__main__":

corrlib/cli.py

@@ -1,13 +1,20 @@
 from typing import Optional
 import typer
-from corrlib import __app_name__, __version__
+from corrlib import __app_name__
+
 from .initialization import create
 from .toml import import_tomls, update_project, reimport_project
-from .find import find_record, list_projects
+from .find import find_record, list_projects, list_ensembles
 from .tools import str2list
 from .main import update_aliases
 from .meas_io import drop_cache as mio_drop_cache
+from .meas_io import load_record as mio_load_record
+from .integrity import full_integrity_check
+
 import os
+from pyerrors import Corr
+from importlib.metadata import version
+from pathlib import Path
 
 
 app = typer.Typer()
@@ -15,48 +22,49 @@ app = typer.Typer()
 
 def _version_callback(value: bool) -> None:
     if value:
-        typer.echo(f"{__app_name__} v{__version__}")
+        print(__app_name__, version(__app_name__))
         raise typer.Exit()
 
 
 @app.command()
 def update(
-    path: str = typer.Option(
+    path: Path = typer.Option(
-        str('./corrlib'),
+        Path('.'),
         "--dataset",
         "-d",
-    ),
+        ),
     uuid: str = typer.Argument(),
-) -> None:
+    ) -> None:
     """
     Update a project by it's UUID.
     """
     update_project(path, uuid)
     return
 
+
 @app.command()
-def list(
+def lister(
-    path: str = typer.Option(
+    path: Path = typer.Option(
-        str('./corrlib'),
+        Path('.'),
         "--dataset",
         "-d",
     ),
     entities: str = typer.Argument('ensembles'),
-) -> None:
+    ) -> None:
     """
     List entities. (ensembles, projects)
     """
     if entities in ['ensembles', 'Ensembles','ENSEMBLES']:
         print("Ensembles:")
-        for item in os.listdir(path + "/archive"):
+        ensemble_results = list_ensembles(path)
-            if os.path.isdir(os.path.join(path + "/archive", item)):
+        for e in ensemble_results:
-                print(item)
+            print(e)
     elif entities == 'projects':
-        results = list_projects(path)
+        project_results = list_projects(path)
         print("Projects:")
         header = "UUID".ljust(37) + "| Aliases"
         print(header)
-        for project in results:
+        for project in project_results:
             if project[1] is not None:
                 aliases = " | ".join(str2list(project[1]))
             else:
@@ -67,14 +75,14 @@ def list(
 
 @app.command()
 def alias_add(
-    path: str = typer.Option(
+    path: Path = typer.Option(
-        str('./corrlib'),
+        Path('.'),
         "--dataset",
         "-d",
-    ),
+        ),
     uuid: str = typer.Argument(),
     alias: str = typer.Argument(),
-) -> None:
+    ) -> None:
     """
     Add an alias to a project UUID.
     """
@@ -85,46 +93,81 @@ def alias_add(
 
 @app.command()
 def find(
-    path: str = typer.Option(
+    path: Path = typer.Option(
-        str('./corrlib'),
+        Path('.'),
         "--dataset",
         "-d",
-    ),
+        ),
-    arg: str = typer.Option(
-        str('all'),
-        "--show",
-        "-s",
-    ),
     ensemble: str = typer.Argument(),
     corr: str = typer.Argument(),
     code: str = typer.Argument(),
-) -> None:
+    arg: str = typer.Option(
+        str('all'),
+        "--argument",
+        "-a",
+    ),
+    ) -> None:
     """
-    Find a record in the backlog at hand. Through specifying it's ensemble and the measured correlator.
+    Find a record in the given backlog.
     """
     results = find_record(path, ensemble, corr, code)
-    if arg == "all":
+    if results.empty:
+        return
+    if arg == 'all':
         print(results)
     else:
-        for i in range(len(results)):
+        for r in results[arg].values:
-            print(results[arg].values[i])
+            print(r)
+
+
+@app.command()
+def stat(
+    path: Path = typer.Option(
+        Path('.'),
+        "--dataset",
+        "-d",
+        ),
+    record_id: str = typer.Argument(),
+    ) -> None:
+    """
+    Show the statistics of a given record.
+    """
+    record = mio_load_record(path, record_id)
+    if isinstance(record, (list, Corr)):
+        record = record[0]
+    statistics = record.idl
+    print(statistics)
     return
 
+
+@app.command()
+def check(path: Path = typer.Option(
+        Path('.'),
+        "--dataset",
+        "-d",
+        ),
+    ) -> None:
+    """
+    Check the integrity of the repository.
+    """
+    full_integrity_check(path)
+
+
 @app.command()
 def importer(
-    path: str = typer.Option(
+    path: Path = typer.Option(
-        str('./corrlib'),
+        Path('.'),
         "--dataset",
         "-d",
-    ),
+        ),
     files: str = typer.Argument(
-    ),
+        ),
     copy_file: bool = typer.Option(
         bool(True),
         "--save",
         "-s",
-    ),
+        ),
-) -> None:
+    ) -> None:
     """
     Import a project from a .toml-file via CLI.
     """
@@ -135,13 +178,16 @@ def importer(
 
 @app.command()
 def reimporter(
-    path: str = typer.Option(
+    path: Path = typer.Option(
-        str('./corrlib'),
+        Path('.'),
         "--dataset",
         "-d",
         ),
     ident: str = typer.Argument()
     ) -> None:
+    """
+    Reimport the toml file identfied by the ident string.
+    """
     uuid = ident.split("::")[0]
     if len(ident.split("::")) > 1:
         toml_file = os.path.join(path, "toml_imports", ident.split("::")[1])
@@ -156,27 +202,32 @@ def reimporter(
 
 @app.command()
 def init(
-    path: str = typer.Option(
+    path: Path = typer.Option(
-        str('./corrlib'),
+        Path('.'),
         "--dataset",
         "-d",
-    ),
+        ),
-) -> None:
+    tracker: str = typer.Option(
+        str('datalad'),
+        "--tracker",
+        "-t",
+        ),
+    ) -> None:
     """
     Initialize a new backlog-database.
     """
-    create(path)
+    create(path, tracker)
     return
 
 
 @app.command()
 def drop_cache(
-    path: str = typer.Option(
+    path: Path = typer.Option(
-        str('./corrlib'),
+        Path('.'),
         "--dataset",
         "-d",
-    ),
+        ),
-) -> None:
+    ) -> None:
     """
     Drop the currect cache directory of the dataset.
     """
@@ -193,6 +244,6 @@ def main(
         help="Show the application's version and exit.",
         callback=_version_callback,
         is_eager=True,
-    )
+        )
-) -> None:
+    ) -> None:
     return

corrlib/find.py

@@ -1,38 +1,148 @@
 import sqlite3
-import datalad.api as dl
 import os
 import json
 import pandas as pd
 import numpy as np
 from .input.implementations import codes
-from .tools import k2m
+from .tools import k2m, get_db_file
-# this will implement the search functionality
+from .tracker import get
+from .integrity import has_valid_times
+from .sql import thin_sql_wrapper
+from typing import Any, Optional
+from pathlib import Path
+import datetime as dt
+from collections.abc import Callable
+import warnings
 
 
-def _project_lookup_by_alias(db, alias):
+def _project_lookup_by_alias(path: Path, alias: str) -> str:
-    # this will lookup the project name based on the alias
+    """
-    conn = sqlite3.connect(db)
+    Lookup a projects UUID by its (human-readable) alias.
-    c = conn.cursor()
+
-    c.execute(f"SELECT * FROM 'projects' WHERE alias = '{alias}'")
+    Parameters
-    results = c.fetchall()
+    ----------
-    conn.close()
+    db: str
-    if len(results) > 1:
+        The database to look up the project.
+    alias: str
+        The alias to look up.
+
+    Returns
+    -------
+    uuid: str
+        The UUID of the project with the given alias.
+    """
+    stmt = f"SELECT * FROM 'projects' WHERE aliases = '{alias}'"
+    results = thin_sql_wrapper(path, stmt)
+    if len(results)>1:
         print("Error: multiple projects found with alias " + alias)
     elif len(results) == 0:
         raise Exception("Error: no project found with alias " + alias)
-    return results[0][0]
+    return str(results[0][0])
 
 
-def _project_lookup_by_id(db, uuid):
+def _project_lookup_by_id(path: Path, uuid: str) -> list[tuple[str, ...]]:
-    conn = sqlite3.connect(db)
+    """
-    c = conn.cursor()
+    Return the project information available in the database by UUID.
-    c.execute(f"SELECT * FROM 'projects' WHERE id = '{uuid}'")
+
-    results = c.fetchall()
+    Parameters
-    conn.close()
+    ----------
+    db: str
+        The database to look up the project.
+    uuid: str
+        The uuid of the project in question.
+
+    Returns
+    -------
+    results: list
+        The row of the project in the database.
+    """
+    stmt = f"SELECT * FROM 'projects' WHERE id = '{uuid}'"
+    results = thin_sql_wrapper(path, stmt)
     return results
 
 
-def _db_lookup(db, ensemble, correlator_name,code, project=None,  parameters=None, created_before=None, created_after=None, updated_before=None, updated_after=None, revision=None):
+def _time_filter(results: pd.DataFrame,  created_before: Optional[str]=None, created_after: Optional[Any]=None, updated_before: Optional[Any]=None, updated_after: Optional[Any]=None) -> pd.DataFrame:
+    """
+    Filter the results from the database in terms of the creation and update times.
+
+    Parameters
+    ----------
+    results: pd.DataFrame
+        The dataframe holding the unfilteres results from the database.
+    created_before: str
+        Contraint on the creation date in datetime.datetime.isoformat. Note that this is exclusive. The creation date has to be truly before the date and time given.
+    created_after: str
+        Contraint on the creation date in datetime.datetime.isoformat. Note that this is exclusive. The creation date has to be truly after the date and time given.
+    updated_before: str
+        Contraint on the creation date in datetime.datetime.isoformat. Note that this is exclusive. The date of the last update has to be truly before the date and time given.
+    updated_after: str
+        Contraint on the creation date in datetime.datetime.isoformat. Note that this is exclusive. The date of the last update has to be truly after the date and time given.
+    """
+    drops = []
+    for ind in range(len(results)):
+        result = results.iloc[ind]
+        created_at = dt.datetime.fromisoformat(result['created_at'])
+        updated_at = dt.datetime.fromisoformat(result['updated_at'])
+        db_times_valid = has_valid_times(result)
+        if not db_times_valid:
+            raise ValueError('Time stamps not valid for result with path', result["path"])
+
+        if created_before is not None:
+            date_created_before = dt.datetime.fromisoformat(created_before)
+            if date_created_before < created_at:
+                drops.append(ind)
+                continue
+        if created_after is not None:
+            date_created_after = dt.datetime.fromisoformat(created_after)
+            if date_created_after > created_at:
+                drops.append(ind)
+                continue
+        if updated_before is not None:
+            date_updated_before = dt.datetime.fromisoformat(updated_before)
+            if date_updated_before < updated_at:
+                drops.append(ind)
+                continue
+        if updated_after is not None:
+            date_updated_after = dt.datetime.fromisoformat(updated_after)
+            if date_updated_after > updated_at:
+                drops.append(ind)
+                continue
+    return results.drop(drops)
+
+
+def _db_lookup(db: Path, ensemble: str, correlator_name: str, code: str, project: Optional[str]=None,  parameters: Optional[str]=None) -> pd.DataFrame:
+    """
+    Look up a correlator record in the database by the data given to the method.
+
+    Parameters
+    ----------
+    db: str
+        The database to look up the record.
+    ensemble: str
+        The ensemble the record is associated with.
+    correlator_name: str
+        The name of the correlator in question.
+    code: str
+        The name of the code which was used to calculate the correlator.
+    project: str, optional
+        The UUID of the project the correlator was calculated in.
+    parameters: str, optional
+        A dictionary holding the exact parameters for the measurement that are held in the database.
+    created_before: str, optional
+        Timestamp string before which the meaurement has been created.
+    created_after: str, optional
+        Timestamp string after which the meaurement has been created.
+    updated_before: str, optional
+        Timestamp string before which the meaurement has been updated.
+    updated_after: str, optional
+        Timestamp string after which the meaurement has been updated.
+
+    Returns
+    -------
+    results: pd.DataFrame
+        A pandas DataFrame holding the information received form the DB query.
+    """
+
     project_str = project
 
     search_expr = f"SELECT * FROM 'backlogs' WHERE name = '{correlator_name}' AND ensemble = '{ensemble}'"
@@ -42,127 +152,240 @@ def _db_lookup(db, ensemble, correlator_name,code, project=None,  parameters=Non
         search_expr += f" AND code = '{code}'"
     if parameters:
         search_expr += f" AND parameters = '{parameters}'"
-    if created_before:
-        search_expr += f" AND created_at < '{created_before}'"
-    if created_after:
-        search_expr += f" AND created_at > '{created_after}'"
-    if updated_before:
-        search_expr += f" AND updated_at < '{updated_before}'"
-    if updated_after:
-        search_expr += f" AND updated_at > '{updated_after}'"
     conn = sqlite3.connect(db)
     results = pd.read_sql(search_expr, conn)
     conn.close()
     return results
 
 
-def sfcf_filter(results, **kwargs):
+def _sfcf_drop(param: dict[str, Any], **kwargs: Any) -> bool:
+        if 'offset' in kwargs:
+            if kwargs.get('offset') != param['offset']:
+                return True
+        if 'quark_kappas' in kwargs:
+            kappas = kwargs['quark_kappas']
+            if (not np.isclose(kappas[0], param['quarks'][0]['mass']) or not np.isclose(kappas[1], param['quarks'][1]['mass'])):
+                return True
+        if 'quark_masses' in kwargs:
+            masses = kwargs['quark_masses']
+            if (not np.isclose(masses[0], k2m(param['quarks'][0]['mass'])) or not np.isclose(masses[1], k2m(param['quarks'][1]['mass']))):
+                return True
+        if 'qk1' in kwargs:
+            quark_kappa1 = kwargs['qk1']
+            if not isinstance(quark_kappa1, list):
+                if (not np.isclose(quark_kappa1, param['quarks'][0]['mass'])):
+                    return True
+            else:
+                if len(quark_kappa1) == 2:
+                    if (quark_kappa1[0] > param['quarks'][0]['mass']) or (quark_kappa1[1] < param['quarks'][0]['mass']):
+                        return True
+                else:
+                    raise ValueError("quark_kappa1 has to have length 2")
+        if 'qk2' in kwargs:
+            quark_kappa2 = kwargs['qk2']
+            if not isinstance(quark_kappa2, list):
+                if (not np.isclose(quark_kappa2, param['quarks'][1]['mass'])):
+                    return True
+            else:
+                if len(quark_kappa2) == 2:
+                    if (quark_kappa2[0] > param['quarks'][1]['mass']) or (quark_kappa2[1] < param['quarks'][1]['mass']):
+                        return True
+                else:
+                    raise ValueError("quark_kappa2 has to have length 2")
+        if 'qm1' in kwargs:
+            quark_mass1 = kwargs['qm1']
+            if not isinstance(quark_mass1, list):
+                if (not np.isclose(quark_mass1, k2m(param['quarks'][0]['mass']))):
+                    return True
+            else:
+                if len(quark_mass1) == 2:
+                    if (quark_mass1[0] > k2m(param['quarks'][0]['mass'])) or (quark_mass1[1] < k2m(param['quarks'][0]['mass'])):
+                        return True
+                else:
+                    raise ValueError("quark_mass1 has to have length 2")
+        if 'qm2' in kwargs:
+            quark_mass2 = kwargs['qm2']
+            if not isinstance(quark_mass2, list):
+                if (not np.isclose(quark_mass2, k2m(param['quarks'][1]['mass']))):
+                    return True
+            else:
+                if len(quark_mass2) == 2:
+                    if (quark_mass2[0] > k2m(param['quarks'][1]['mass'])) or (quark_mass2[1] < k2m(param['quarks'][1]['mass'])):
+                        return True
+                else:
+                    raise ValueError("quark_mass2 has to have length 2")
+        if 'quark_thetas' in kwargs:
+            quark_thetas = kwargs['quark_thetas']
+            if (quark_thetas[0] != param['quarks'][0]['thetas'] and quark_thetas[1] != param['quarks'][1]['thetas']) or (quark_thetas[0] != param['quarks'][1]['thetas'] and quark_thetas[1] != param['quarks'][0]['thetas']):
+                return True
+        # careful, this is not save, when multiple contributions are present!
+        if 'wf1' in kwargs:
+            wf1 = kwargs['wf1']
+            if not (np.isclose(wf1[0][0], param['wf1'][0][0], 1e-8) and np.isclose(wf1[0][1][0], param['wf1'][0][1][0], 1e-8) and np.isclose(wf1[0][1][1], param['wf1'][0][1][1], 1e-8)):
+                return True
+        if 'wf2' in kwargs:
+            wf2 = kwargs['wf2']
+            if not (np.isclose(wf2[0][0], param['wf2'][0][0], 1e-8) and np.isclose(wf2[0][1][0], param['wf2'][0][1][0], 1e-8) and np.isclose(wf2[0][1][1], param['wf2'][0][1][1], 1e-8)):
+                return True
+        return False
+
+
+def sfcf_filter(results: pd.DataFrame, **kwargs: Any) -> pd.DataFrame:
+    r"""
+    Filter method for the Database entries holding SFCF calculations.
+
+    Parameters
+    ----------
+    results: pd.DataFrame
+        The unfiltered pandas DataFrame holding the entries from the database.
+
+    offset: list[float], optional
+    quark_kappas: list[float]
+    quarks_masses: list[float]
+    qk1: float, optional
+        Mass parameter $\kappa_1$ of the first quark.
+    qk2: float, optional
+        Mass parameter $\kappa_2$ of the first quark.
+    qm1: float, optional
+        Bare quark mass $m_1$ of the first quark.
+    qm2: float, optional
+        Bare quark mass $m_2$ of the first quark.
+    quarks_thetas: list[list[float]], optional
+    wf1: optional
+    wf2: optional
+
+    Results
+    -------
+    results: pd.DataFrame
+        The filtered DataFrame, only holding the records that fit to the parameters given.
+    """
+
     drops = []
     for ind in range(len(results)):
         result = results.iloc[ind]
-        if result['code'] == 'sfcf':
+        param = json.loads(result['parameters'])
-            param = json.loads(result['parameters'])
+        if _sfcf_drop(param, **kwargs):
-            if 'offset' in kwargs:
+            drops.append(ind)
-                if kwargs.get('offset') != param['offset']:
-                    drops.append(ind)
-                    continue
-            if 'quark_kappas' in kwargs:
-                kappas = kwargs['quark_kappas']
-                if (not np.isclose(kappas[0], param['quarks'][0]['mass']) or not np.isclose(kappas[1], param['quarks'][1]['mass'])):
-                    drops.append(ind)
-                    continue
-            if 'quark_masses' in kwargs:
-                masses = kwargs['quark_masses']
-                if (not np.isclose(masses[0], k2m(param['quarks'][0]['mass'])) or not np.isclose(masses[1], k2m(param['quarks'][1]['mass']))):
-                    drops.append(ind)
-                    continue
-            if 'qk1' in kwargs:
-                quark_kappa1 = kwargs['qk1']
-                if not isinstance(quark_kappa1, list):
-                    if (not np.isclose(quark_kappa1, param['quarks'][0]['mass'])):
-                        drops.append(ind)
-                        continue
-                else:
-                    if len(quark_kappa1) == 2:
-                        if (quark_kappa1[0] > param['quarks'][0]['mass']) or (quark_kappa1[1] < param['quarks'][0]['mass']):
-                            drops.append(ind)
-                            continue
-            if 'qk2' in kwargs:
-                quark_kappa2 = kwargs['qk2']
-                if not isinstance(quark_kappa2, list):
-                    if (not np.isclose(quark_kappa2, param['quarks'][1]['mass'])):
-                        drops.append(ind)
-                        continue
-                else:
-                    if len(quark_kappa2) == 2:
-                        if (quark_kappa2[0] > param['quarks'][1]['mass']) or (quark_kappa2[1] < param['quarks'][1]['mass']):
-                            drops.append(ind)
-                            continue
-            if 'qm1' in kwargs:
-                quark_mass1 = kwargs['qm1']
-                if not isinstance(quark_mass1, list):
-                    if (not np.isclose(quark_mass1, k2m(param['quarks'][0]['mass']))):
-                        drops.append(ind)
-                        continue
-                else:
-                    if len(quark_mass1) == 2:
-                        if (quark_mass1[0] > k2m(param['quarks'][0]['mass'])) or (quark_mass1[1] < k2m(param['quarks'][0]['mass'])):
-                            drops.append(ind)
-                            continue
-            if 'qm2' in kwargs:
-                quark_mass2 = kwargs['qm2']
-                if not isinstance(quark_mass2, list):
-                    if (not np.isclose(quark_mass2, k2m(param['quarks'][1]['mass']))):
-                        drops.append(ind)
-                        continue
-                else:
-                    if len(quark_mass2) == 2:
-                        if (quark_mass2[0] > k2m(param['quarks'][1]['mass'])) or (quark_mass2[1] < k2m(param['quarks'][1]['mass'])):
-                            drops.append(ind)
-                            continue
-            if 'quark_thetas' in kwargs:
-                quark_thetas = kwargs['quark_thetas']
-                if (quark_thetas[0] != param['quarks'][0]['thetas'] and quark_thetas[1] != param['quarks'][1]['thetas']) or (quark_thetas[0] != param['quarks'][1]['thetas'] and quark_thetas[1] != param['quarks'][0]['thetas']):
-                    drops.append(ind)
-                    continue
-            # careful, this is not save, when multiple contributions are present!
-            if 'wf1' in kwargs:
-                wf1 = kwargs['wf1']
-                if not (np.isclose(wf1[0][0], param['wf1'][0][0], 1e-8) and np.isclose(wf1[0][1][0], param['wf1'][0][1][0], 1e-8) and np.isclose(wf1[0][1][1], param['wf1'][0][1][1], 1e-8)):
-                    drops.append(ind)
-                    continue
-            if 'wf2' in kwargs:
-                wf2 = kwargs['wf2']
-                if not (np.isclose(wf2[0][0], param['wf2'][0][0], 1e-8) and np.isclose(wf2[0][1][0], param['wf2'][0][1][0], 1e-8) and np.isclose(wf2[0][1][1], param['wf2'][0][1][1], 1e-8)):
-                    drops.append(ind)
-                    continue
     return results.drop(drops)
 
 
-def find_record(path, ensemble, correlator_name, code, project=None, parameters=None, created_before=None, created_after=None, updated_before=None, updated_after=None, revision=None, **kwargs):
+def openQCD_filter(results:pd.DataFrame, **kwargs: Any) -> pd.DataFrame:
-    db = path + '/backlogger.db'
+    """
+    Filter for parameters of openQCD.
+
+    Parameters
+    ----------
+    results: pd.DataFrame
+        The unfiltered list of results from the database.
+
+    Returns
+    -------
+    results: pd.DataFrame
+        The filtered results.
+
+    """
+    warnings.warn("A filter for openQCD parameters is no implemented yet.", Warning)
+
+    return results
+
+
+def _code_filter(results: pd.DataFrame, code: str, **kwargs: Any) -> pd.DataFrame:
+    """
+    Abstraction of the filters for the different codes that are available.
+    At the moment, only openQCD and SFCF are known.
+    The possible key words for the parameters can be seen in the descriptionso f the code-specific filters.
+
+    Parameters
+    ----------
+    results: pd.DataFrame
+        The unfiltered list of results from the database.
+    code: str
+        The name of the code that produced the record at hand.
+    kwargs:
+        The keyworkd args that are handed over to the code-specific filters.
+
+    Returns
+    -------
+    results: pd.DataFrame
+        The filtered results.
+    """
+    if code == "sfcf":
+        return sfcf_filter(results, **kwargs)
+    elif code == "openQCD":
+        return openQCD_filter(results, **kwargs)
+    else:
+        raise ValueError(f"Code {code} is not known.")
+
+
+def find_record(path: Path, ensemble: str, correlator_name: str, code: str, project: Optional[str]=None, parameters: Optional[str]=None,
+                created_before: Optional[str]=None, created_after: Optional[str]=None, updated_before: Optional[str]=None, updated_after: Optional[str]=None,
+                revision: Optional[str]=None,
+                customFilter: Optional[Callable[[pd.DataFrame], pd.DataFrame]] = None,
+                **kwargs: Any) -> pd.DataFrame:
+    path = Path(path)
+    db_file = get_db_file(path)
+    db = path / db_file
     if code not in codes:
         raise ValueError("Code " + code + "unknown, take one of the following:" + ", ".join(codes))
-    dl.get(db, dataset=path)
+    get(path, db_file)
-    results = _db_lookup(db, ensemble, correlator_name,code, project, parameters=parameters, created_before=created_before, created_after=created_after, updated_before=updated_before, updated_after=updated_after, revision=revision)
+    results = _db_lookup(db, ensemble, correlator_name,code, project, parameters=parameters)
-    if code == "sfcf":
+    if any([arg is not None for arg in [created_before, created_after, updated_before, updated_after]]):
-        results = sfcf_filter(results, **kwargs)
+        results = _time_filter(results, created_before, created_after, updated_before, updated_after)
+    results = _code_filter(results, code, **kwargs)
+    if customFilter is not None:
+        results =  customFilter(results)
     print("Found " + str(len(results)) + " result" + ("s" if len(results)>1 else ""))
     return results.reset_index()
 
 
-def find_project(path, db, name):
+def find_project(path: Path, name: str) -> str:
-    dl.get(db, dataset=path)
+    """
-    return _project_lookup_by_alias(db, name)
+    Find a project by it's human readable name.
+
+    Parameters
+    ----------
+    path: str
+        The path of the library.
+    name: str
+        The name of the project to look for in the library.
+
+    Returns
+    -------
+    uuid: str
+        The uuid of the project in question.
+    """
+    db_file = get_db_file(path)
+    get(path, db_file)
+    return _project_lookup_by_alias(path, name)
 
 
-def list_projects(path):
+def list_projects(path: Path) -> list[tuple[str, str]]:
-    db = path + '/backlogger.db'
+    """
-    dl.get(db, dataset=path)
+    List all projects known to the library.
-    conn = sqlite3.connect(db)
+
+    Parameters
+    ----------
+    path: str
+        The path of the library.
+
+    Returns
+    -------
+    results: list[Any]
+        The projects known to the library.
+    """
+    db_file = get_db_file(path)
+    get(path, db_file)
+    conn = sqlite3.connect(os.path.join(path, db_file))
     c = conn.cursor()
     c.execute("SELECT id,aliases FROM projects")
     results = c.fetchall()
     conn.close()
     return results
 
+
+def list_ensembles(path: Path) -> list[str]:
+    res = []
+    for item in os.listdir(path / "archive"):
+        if os.path.isdir(path / "archive" / item):
+            res.append(item)
+    return res

corrlib/git_tools.py

@@ -1,27 +1,28 @@
 import os
-import datalad.api as dl
+from .tracker import save
 import git
+from pathlib import Path
 
 GITMODULES_FILE = '.gitmodules'
 
 
-def move_submodule(repo_path, old_path, new_path):
+def move_submodule(repo_path: Path, old_path: Path, new_path: Path) -> None:
     """
     Move a submodule to a new location.
 
     Parameters
     ----------
-    repo_path: str
+    repo_path: Path
     Path to the repository.
-    old_path: str
+    old_path: Path
         The old path of the module.
-    new_path: str
+    new_path: Path
         The new path of the module.
     """
 
-    os.rename(os.path.join(repo_path, old_path), os.path.join(repo_path, new_path))
+    os.rename(repo_path / old_path, repo_path / new_path)
 
-    gitmodules_file_path = os.path.join(repo_path, GITMODULES_FILE)
+    gitmodules_file_path = repo_path / GITMODULES_FILE
 
     # update paths in .gitmodules
     with open(gitmodules_file_path, 'r') as file:
@@ -29,8 +30,8 @@ def move_submodule(repo_path, old_path, new_path):
 
     updated_lines = []
     for line in lines:
-        if old_path in line:
+        if str(old_path) in line:
-            line = line.replace(old_path, new_path)
+            line = line.replace(str(old_path), str(new_path))
         updated_lines.append(line)
 
     with open(gitmodules_file_path, 'w') as file:
@@ -40,4 +41,6 @@ def move_submodule(repo_path, old_path, new_path):
     repo = git.Repo(repo_path)
     repo.git.add('.gitmodules')
     # save new state of the dataset
-    dl.save(repo_path, message=f"Move module from {old_path} to {new_path}", dataset=repo_path)
+    save(repo_path, message=f"Move module from {old_path} to {new_path}", files=[Path('.gitmodules'), repo_path])
+
+    return

corrlib/initialization.py

@@ -1,12 +1,19 @@
+from configparser import ConfigParser
 import sqlite3
-import datalad.api as dl
 import os
+from .tracker import save, init
+from pathlib import Path
+from .tools import CONFIG_FILENAME
 
 
-def _create_db(db):
+def _create_db(db: Path) -> None:
     """
     Create the database file and the table.
 
+    Parameters
+    ----------
+    db: str
+        Path of the database file.
     """
     conn = sqlite3.connect(db)
     c = conn.cursor()
@@ -32,21 +39,84 @@ def _create_db(db):
                   updated_at TEXT)''')
     conn.commit()
     conn.close()
+    return
 
 
-def create(path):
+def _create_config(path: Path, tracker: str, cached: bool) -> ConfigParser:
+    """
+    Create the config file construction for backlogger.
+
+    Parameters
+    ----------
+    path: str
+        The path of the libaray to create.
+    tracker: str
+        Type of the tracker to use for the library (only DataLad is supported at the moment).
+    cached: bool
+        Whether or not the library will create a cache folder for multiple reads when downloaded.
+
+    Returns
+    -------
+    config: ConfigParser
+        Cpnfig parser with the default configuration printed.
+
+    """
+    config = ConfigParser()
+    config['core'] = {
+        'version': '1.0',
+        'tracker': tracker,
+        'cached': str(cached),
+    }
+    config['paths'] = {
+        'db': 'backlogger.db',
+        'projects_path': 'projects',
+        'archive_path': 'archive',
+        'toml_imports_path': 'toml_imports',
+        'import_scripts_path': 'import_scripts',
+    }
+    return config
+
+
+def _write_config(path: Path, config: ConfigParser) -> None:
+    """
+    Write the config file to disk.
+
+    Parameters
+    ----------
+    path: str
+        The path of the libaray to create.
+    config: ConfigParser
+        The configuration to be used as a ConfigParser, e.g. generated by _create_config.
+    """
+    with open(os.path.join(path, CONFIG_FILENAME), 'w') as configfile:
+        config.write(configfile)
+    return
+
+
+def create(path: Path, tracker: str = 'datalad', cached: bool = True) -> None:
     """
     Create folder of backlogs.
 
+    Parameters
+    ----------
+    path: str
+        The path at which the library will be created.
+    tracker: str, optional
+        The tracker to use for the library. The delauft is DataLad, which is also the only one that is supported at the moment.
+    cached: bool, optional
+        Whether or not hte librarby will be cached. By default, it does cache already read entries.
     """
-    dl.create(path)
+    config = _create_config(path, tracker, cached)
-    _create_db(path + '/backlogger.db')
+    init(path, tracker)
-    os.chmod(path + '/backlogger.db', 0o666)  # why does this not work?
+    _write_config(path, config)
-    os.makedirs(path + '/projects')
+    _create_db(path / config['paths']['db'])
-    os.makedirs(path + '/archive')
+    os.chmod(path / config['paths']['db'], 0o666)
-    os.makedirs(path + '/toml_imports')
+    os.makedirs(path / config['paths']['projects_path'])
-    os.makedirs(path + '/import_scripts/template.py')
+    os.makedirs(path / config['paths']['archive_path'])
-    with open(path + "/.gitignore", "w") as fp:
+    os.makedirs(path / config['paths']['toml_imports_path'])
+    os.makedirs(path / config['paths']['import_scripts_path'] / 'template.py')
+    with open(path / ".gitignore", "w") as fp:
         fp.write(".cache")
         fp.close()
-    dl.save(path, dataset=path, message="Initialize backlogger directory.")
+    save(path, message="Initialized correlator library")
+    return

corrlib/input/__init__.py

@@ -2,6 +2,6 @@
 Import functions for different codes.
 """
 
-from . import sfcf
+from . import sfcf as sfcf
-from . import openQCD
+from . import openQCD as openQCD
-from . import implementations
+from . import implementations as implementations

corrlib/input/implementations.py

@@ -1,2 +1,2 @@
-
+# List of supported input implementations
 codes = ['sfcf', 'openQCD']

corrlib/input/openQCD.py

@@ -2,10 +2,32 @@ import pyerrors.input.openQCD as input
 import datalad.api as dl
 import os
 import fnmatch
-from typing import Any
+from typing import Any, Optional
+from pathlib import Path
+from ..pars.openQCD import ms1
+from ..pars.openQCD import qcd2
 
 
-def read_ms1_param(path: str, project: str, file_in_project: str) -> dict[str, Any]:
+
+def load_ms1_infile(path: Path, project: str, file_in_project: str) -> dict[str, Any]:
+    """
+    Read the parameters for ms1 measurements from a parameter file in the project.
+
+    Parameters
+    ----------
+    path: str
+        The path to the backlogger folder.
+    project: str
+        The project from which to read the parameter file.
+    file_in_project: str
+        The path to the parameter file within the project.
+
+    Returns
+    -------
+    param: dict[str, Any]
+        The parameters read from the file.
+    """
+
     file = os.path.join(path, "projects", project, file_in_project)
     ds = os.path.join(path, "projects", project)
     dl.get(file, dataset=ds)
@@ -51,7 +73,25 @@ def read_ms1_param(path: str, project: str, file_in_project: str) -> dict[str, A
     return param
 
 
-def read_ms3_param(path: str, project: str, file_in_project: str) -> dict[str, Any]:
+def load_ms3_infile(path: Path, project: str, file_in_project: str) -> dict[str, Any]:
+    """
+    Read the parameters for ms3 measurements from a parameter file in the project.
+
+    Parameters
+    ----------
+    path: str
+        The path to the backlogger folder.
+    project: str
+        The project from which to read the parameter file.
+    file_in_project: str
+        The path to the parameter file within the project.
+
+    Returns
+    -------
+    param: dict[str, Any]
+        The parameters read from the file.
+    """
+
     file = os.path.join(path, "projects", project, file_in_project)
     ds = os.path.join(path, "projects", project)
     dl.get(file, dataset=ds)
@@ -67,7 +107,37 @@ def read_ms3_param(path: str, project: str, file_in_project: str) -> dict[str, A
     return param
 
 
-def read_rwms(path: str, project: str, dir_in_project: str, param: dict[str, Any], prefix: str, postfix: str="ms1", version: str='2.0', names: list[str]=None, files: list[str]=None) -> dict[str, Any]:
+def read_rwms(path: Path, project: str, dir_in_project: str, param: dict[str, Any], prefix: str, postfix: str="ms1", version: str='2.0', names: Optional[list[str]]=None, files: Optional[list[str]]=None) -> dict[str, Any]:
+    """
+    Read reweighting factor measurements from the project.
+
+    Parameters
+    ----------
+    path: str
+        The path to the backlogger folder.
+    project: str
+        The project from which to read the measurements.
+    dir_in_project: str
+        The directory within the project where the measurements are located.
+    param: dict[str, Any]
+        The parameters for the measurements.
+    prefix: str
+        The prefix of the measurement files.
+    postfix: str
+        The postfix of the measurement files.
+    version: str
+        The version of the openQCD used.
+    names: list[str]
+        Specific names for the replica of the ensemble the measurement file belongs to.
+    files: list[str]
+        Specific files to read.
+
+    Returns
+    -------
+    rw_dict: dict[str, dict[str, Any]]
+        The reweighting factor measurements read from the files.
+    """
+
     dataset = os.path.join(path, "projects", project)
     directory = os.path.join(dataset, dir_in_project)
     if files is None:
@@ -94,7 +164,43 @@ def read_rwms(path: str, project: str, dir_in_project: str, param: dict[str, Any
     return rw_dict
 
 
-def extract_t0(path: str, project: str, dir_in_project: str, param: dict[str, Any], prefix: str, dtr_read: int, xmin: int, spatial_extent: int, fit_range: int = 5, postfix: str=None, names: list[str]=None, files: list[str]=None) -> dict[str, Any]:
+def extract_t0(path: Path, project: str, dir_in_project: str, param: dict[str, Any], prefix: str, dtr_read: int, xmin: int, spatial_extent: int, fit_range: int = 5, postfix: str="", names: Optional[list[str]]=None, files: Optional[list[str]]=None) -> dict[str, Any]:
+    """
+    Extract t0 measurements from the project.
+
+    Parameters
+    ----------
+    path: str
+        The path to the backlogger folder.
+    project: str
+        The project from which to read the measurements.
+    dir_in_project: str
+        The directory within the project where the measurements are located.
+    param: dict[str, Any]
+        The parameters for the measurements.
+    prefix: str
+        The prefix of the measurement files.
+    dtr_read: int
+        The dtr_read parameter for the extraction.
+    xmin: int
+        The xmin parameter for the extraction.
+    spatial_extent: int
+        The spatial_extent parameter for the extraction.
+    fit_range: int
+        The fit_range parameter for the extraction.
+    postfix: str
+        The postfix of the measurement files.
+    names: list[str]
+        Specific names for the replica of the ensemble the measurement file belongs to.
+    files: list[str]
+        Specific files to read.
+
+    Returns
+    -------
+    t0_dict: dict
+        Dictionary of t0 values in the pycorrlib style, with the parameters at hand.
+    """
+
     dataset = os.path.join(path, "projects", project)
     directory = os.path.join(dataset, dir_in_project)
     if files is None:
@@ -132,7 +238,43 @@ def extract_t0(path: str, project: str, dir_in_project: str, param: dict[str, An
     return t0_dict
 
 
-def extract_t1(path: str, project: str, dir_in_project: str, param: dict[str, Any], prefix: str, dtr_read: int, xmin: int, spatial_extent: int, fit_range: int = 5, postfix: str = None, names: list[str]=None, files: list[str]=None) -> dict[str, Any]:
+def extract_t1(path: Path, project: str, dir_in_project: str, param: dict[str, Any], prefix: str, dtr_read: int, xmin: int, spatial_extent: int, fit_range: int = 5, postfix: str = "", names: Optional[list[str]]=None, files: Optional[list[str]]=None) -> dict[str, Any]:
+    """
+    Extract t1 measurements from the project.
+
+    Parameters
+    ----------
+    path: str
+        The path to the backlogger folder.
+    project: str
+        The project from which to read the measurements.
+    dir_in_project: str
+        The directory within the project where the measurements are located.
+    param: dict[str, Any]
+        The parameters for the measurements.
+    prefix: str
+        The prefix of the measurement files.
+    dtr_read: int
+        The dtr_read parameter for the extraction.
+    xmin: int
+        The xmin parameter for the extraction.
+    spatial_extent: int
+        The spatial_extent parameter for the extraction.
+    fit_range: int
+        The fit_range parameter for the extraction.
+    postfix: str
+        The postfix of the measurement files.
+    names: list[str]
+        Specific names for the replica of the ensemble the measurement file belongs to.
+    files: list[str]
+        Specific files to read.
+
+    Returns
+    -------
+    t1_dict: dict
+        Dictionary of t1 values in the pycorrlib style, with the parameters at hand.
+    """
+
     directory = os.path.join(path, "projects", project, dir_in_project)
     if files is None:
         files = []
@@ -161,7 +303,55 @@ def extract_t1(path: str, project: str, dir_in_project: str, param: dict[str, An
     for k in ["integrator", "eps", "ntot", "dnms"]:
         par_list.append(str(param[k]))
     pars = "/".join(par_list)
-    t0_dict: dict[str, Any] = {}
+    t1_dict: dict[str, Any] = {}
-    t0_dict[param["type"]] = {}
+    t1_dict[param["type"]] = {}
-    t0_dict[param["type"]][pars] = t0
+    t1_dict[param["type"]][pars] = t0
-    return t0_dict
+    return t1_dict
+
+
+def load_qcd2_pars(path: Path, project: str, file_in_project: str)  -> dict[str, Any]:
+    """
+    Thin wrapper around read_qcd2_par_file, getting the file before reading.
+
+    Parameters
+    ----------
+    path: Path
+        Path of the corrlib repository.
+    project: str
+        UUID of the project of the parameter-file.
+    file_in_project: str
+        The loaction of the file in the project directory.
+
+    Returns
+    -------
+    par_dict: dict
+        The dict with the parameters read from the .par-file.
+    """
+    fname = path / "projects" / project / file_in_project
+    ds = os.path.join(path, "projects", project)
+    dl.get(fname, dataset=ds)
+    return qcd2.read_qcd2_par_file(fname)
+
+
+def load_ms1_parfile(path: Path, project: str, file_in_project: str)  -> dict[str, Any]:
+    """
+    Thin wrapper around read_qcd2_ms1_par_file, getting the file before reading.
+
+    Parameters
+    ----------
+    path: Path
+        Path of the corrlib repository.
+    project: str
+        UUID of the project of the parameter-file.
+    file_in_project: str
+        The loaction of the file in the project directory.
+
+    Returns
+    -------
+    par_dict: dict
+        The dict with the parameters read from the .par-file.
+    """
+    fname = path / "projects" / project / file_in_project
+    ds = os.path.join(path, "projects", project)
+    dl.get(fname, dataset=ds)
+    return ms1.read_qcd2_ms1_par_file(fname)

corrlib/input/sfcf.py

@@ -3,9 +3,11 @@ import datalad.api as dl
 import json
 import os
 from typing import Any
+from fnmatch import fnmatch
+from pathlib import Path
 
 
-bi_corrs: list = ["f_P", "fP", "f_p",
+bi_corrs: list[str] = ["f_P", "fP", "f_p",
             "g_P", "gP", "g_p",
             "fA0", "f_A", "f_a",
             "gA0", "g_A", "g_a",
@@ -43,7 +45,7 @@ bi_corrs: list = ["f_P", "fP", "f_p",
             "l3A2", "l3_A2", "g_av23",
             ]
 
-bb_corrs: list = [
+bb_corrs: list[str] = [
             'F1',
             'F_1',
             'f_1',
@@ -64,7 +66,7 @@ bb_corrs: list = [
             'F_sPdP_d',
             ]
 
-bib_corrs: list = [
+bib_corrs: list[str] = [
             'F_V0',
             'K_V0',
             ]
@@ -79,7 +81,7 @@ for c in bib_corrs:
     corr_types[c] = 'bib'
 
 
-def read_param(path: str, project: str, file_in_project: str) -> dict[str, Any]:
+def read_param(path: Path, project: str, file_in_project: str) -> dict[str, Any]:
     """
     Read the parameters from the sfcf file.
 
@@ -95,7 +97,7 @@ def read_param(path: str, project: str, file_in_project: str) -> dict[str, Any]:
 
     """
 
-    file = path + "/projects/" + project + '/' + file_in_project
+    file = path / "projects" / project / file_in_project
     dl.get(file, dataset=path)
     with open(file, 'r') as f:
         lines = f.readlines()
@@ -184,7 +186,7 @@ def read_param(path: str, project: str, file_in_project: str) -> dict[str, Any]:
     return params
 
 
-def _map_params(params: dict, spec_list: list) -> dict[str, Any]:
+def _map_params(params: dict[str, Any], spec_list: list[str]) -> dict[str, Any]:
     """
     Map the extracted parameters to the extracted data.
 
@@ -194,7 +196,7 @@ def _map_params(params: dict, spec_list: list) -> dict[str, Any]:
         The parameters extracted from the parameter (input) file. in the dict form given by read_param.
     spec_list: list
         The list of specifications that belongs to the dorrelator in question.
-    
+
     Return
     ------
     new_specs: dict
@@ -228,7 +230,25 @@ def _map_params(params: dict, spec_list: list) -> dict[str, Any]:
     return new_specs
 
 
-def get_specs(key, parameters, sep='/') -> str:
+def get_specs(key: str, parameters: dict[str, Any], sep: str = '/') -> str:
+    """
+    Get sepcification from the parameter file for a specific key in the read measurements
+
+    Parameters
+    ----------
+    key: str
+        The key for whioch the parameters are to be looked up.
+    parameters: dict[str, Any]
+        The dictionary with the parameters from the parameter file.
+    sep: str
+        Separator string for the key. (default="/")
+
+    Return
+    ------
+    s: str
+        json string holding the parameters.
+    """
+
     key_parts = key.split(sep)
     if corr_types[key_parts[0]] == 'bi':
         param = _map_params(parameters, key_parts[1:-1])
@@ -238,7 +258,7 @@ def get_specs(key, parameters, sep='/') -> str:
     return s
 
 
-def read_data(path, project, dir_in_project, prefix, param, version='1.0c', cfg_seperator='n', sep='/', **kwargs) -> dict:
+def read_data(path: Path, project: str, dir_in_project: str, prefix: str, param: dict[str, Any], version: str = '1.0c', cfg_seperator: str = 'n', sep: str = '/', **kwargs: Any) -> dict[str, Any]:
     """
     Extract the data from the sfcf file.
 
@@ -280,9 +300,10 @@ def read_data(path, project, dir_in_project, prefix, param, version='1.0c', cfg_
     if not appended:
         compact = (version[-1] == "c")
         for i, item in enumerate(ls):
-            rep_path = directory + '/' + item
+            if fnmatch(item, prefix + "*"):
-            sub_ls = pe.input.sfcf._find_files(rep_path, prefix, compact, [])
+                rep_path = directory + '/' + item
-            files_to_get.extend([rep_path + "/" + filename for filename in sub_ls])
+                sub_ls = pe.input.sfcf._find_files(rep_path, prefix, compact, [])
+                files_to_get.extend([rep_path + "/" + filename for filename in sub_ls])
 
 
     print("Getting data, this might take a while...")
@@ -300,10 +321,10 @@ def read_data(path, project, dir_in_project, prefix, param, version='1.0c', cfg_
     if not param['crr'] == []:
         if names is not None:
             data_crr = pe.input.sfcf.read_sfcf_multi(directory, prefix, param['crr'], param['mrr'], corr_type_list, range(len(param['wf_offsets'])),
-                                                     range(len(param['wf_basis'])), range(len(param['wf_basis'])), version, cfg_seperator, keyed_out=True, names=names)
+                                                     range(len(param['wf_basis'])), range(len(param['wf_basis'])), version, cfg_seperator, keyed_out=True, silent=True, names=names)
         else:
             data_crr = pe.input.sfcf.read_sfcf_multi(directory, prefix, param['crr'], param['mrr'], corr_type_list, range(len(param['wf_offsets'])),
-                                                     range(len(param['wf_basis'])), range(len(param['wf_basis'])), version, cfg_seperator, keyed_out=True)
+                                                     range(len(param['wf_basis'])), range(len(param['wf_basis'])), version, cfg_seperator, keyed_out=True, silent=True)
         for key in data_crr.keys():
             data[key] = data_crr[key]
 

corrlib/integrity.py

@@ -0,0 +1,295 @@
+import datetime as dt
+from pathlib import Path
+from .tools import get_db_file, CONFIG_FILENAME
+import pandas as pd
+import sqlite3
+from .tracker import get
+import pyerrors.input.json as pj
+import os
+from configparser import ConfigParser
+from typing import Any
+
+
+path_opts = ['db', 'projects_path', 'archive_path', 'toml_imports_path', 'import_scripts_path']
+
+
+def has_valid_times(result: pd.Series) -> bool:
+    """
+    Check, whether the result at hand has time-stamps that are sensible:
+    A recored is created first, then updated, with both times laying in the past.
+
+    Parameters
+    ----------
+    result: pd.Series
+        The result to check
+
+    Returns
+    -------
+    b: bool
+        True, if the timestamps make sense.
+    """
+    # we expect created_at <= updated_at <= now
+    created_at = dt.datetime.fromisoformat(result['created_at'])
+    updated_at = dt.datetime.fromisoformat(result['updated_at'])
+    if created_at > updated_at:
+        return False
+    if updated_at > dt.datetime.now():
+        return False
+    return True
+
+def are_keys_unique(db: Path, table: str, col: str) -> bool:
+    """
+    Check whether the strings listed in a column of a given table are unique.
+
+    Parameters
+    ----------
+    db: Path
+        The database to check.
+    table: str
+        The table to check.
+    col: str
+        The column to be checked for uniqueness.
+
+    Returns
+    -------
+    b: bool
+        True, if the strings are unique.
+    """
+    conn = sqlite3.connect(db)
+    c = conn.cursor()
+    c.execute(f"SELECT COUNT( DISTINCT CAST({col} AS nvarchar(4000))), COUNT({col}) FROM {table};")
+    results = c.fetchall()[0]
+    conn.close()
+    res = bool(results[0] == results[1])
+    if not res:
+        print("Unique:", results[0], "All:", results[1])
+    return res
+
+
+def _list_projects(path: Path) -> list[tuple[str, str]]:
+    """
+    List all projects known to the library.
+
+    Parameters
+    ----------
+    path: str
+        The path of the library.
+
+    Returns
+    -------
+    results: list[Any]
+        The projects known to the library.
+    """
+    db_file = get_db_file(path)
+    get(path, db_file)
+    conn = sqlite3.connect(os.path.join(path, db_file))
+    c = conn.cursor()
+    c.execute("SELECT id,aliases FROM projects")
+    results = c.fetchall()
+    conn.close()
+    return results
+
+
+def _list_ensembles(path: Path) -> list[str]:
+    res = []
+    for item in os.listdir(path / "archive"):
+        if os.path.isdir(path / "archive" / item):
+            res.append(item)
+    return res
+
+
+def check_path_format(result: pd.Series, ensembles: list[str], projects: list[str]) -> None:
+    """
+    Check whether the path of the given result has the right format.
+
+    Parameters
+    ----------
+    result: pd.Series
+        The result to be checked.
+    """
+    p = result['path']
+    if not p.startswith('archive'):
+        raise ValueError(f'The path {p} does not start correctly')
+
+    meas_key = p.split('::')[1]
+    ensemble = p.split('/')[1]
+    project = p.split('/')[3].split('.')[0]
+    if not len(meas_key) == 64:
+        raise ValueError(f'meas_key of {p} is scrambled')
+    if ensemble not in ensembles:
+        raise ValueError(f'meas_key of {p} points to an unknown ensemble')
+    if project not in projects:
+        raise ValueError(f'meas_key of {p} points to an unknown project id ({project})')
+    if not ensemble == result['ensemble']:
+        raise ValueError(f'Ensemble in database and file does not match for path {p}.')
+
+
+
+def check_db_integrity(path: Path) -> None:
+    """
+    Check intergrity of the database by checking the uniqueness of the record keys used to load the records
+    and ensuring that the timestamps of each record is sensible. Throws an error, if issues are detected.
+
+    Parameters
+    ----------
+    path: Path
+        Path to the backlog-library to check.
+    """
+    db = get_db_file(path)
+
+    if not are_keys_unique(path / db, 'backlogs', 'path'):
+        raise Exception("The paths the backlog table of the database links are not unique.")
+
+    search_expr = "SELECT * FROM 'backlogs'"
+    conn = sqlite3.connect(path / db)
+    results = pd.read_sql(search_expr, conn)
+    ensembles = _list_ensembles(path)
+    projects = [p[0] for p in _list_projects(path)]
+
+    for _, result in results.iterrows():
+        if not has_valid_times(result):
+            raise ValueError(f"Result with id {result[id]} has wrong time signatures.")
+        check_path_format(result, ensembles, projects)
+    return
+
+
+def _check_db2paths(path: Path, meas_paths: list[str]) -> None:
+    """
+    Check whether for each record in the given by meas_paths, we can find the data in the file as we expect.
+    Also check, whether there are unreachable records in the files. If either of the issues arise, throws an error.
+
+    Parameters
+    ----------
+    path: Path
+        Path to the backlog-library to check.
+    meas_paths: list[str]
+        List of measurement paths to check.
+    """
+    needed_data: dict[str, list[str]] = {}
+    for mpath in meas_paths:
+        file = mpath.split("::")[0]
+        if file not in needed_data.keys():
+            needed_data[file] = []
+        key = mpath.split("::")[1]
+        needed_data[file].append(key)
+
+    totf = len(needed_data.keys())
+    for i, file in enumerate(needed_data.keys()):
+        print(f"Check against file {i}/{totf}: {file}")
+        get(path, Path(file))
+        filedict: dict[str, Any] = pj.load_json_dict(str(path / file))
+        if not set(filedict.keys()).issubset(needed_data[file]):
+            for key in filedict.keys():
+                if key not in needed_data[file]:
+                    raise ValueError(f"Found unintended key {key} in file {file}.")
+        if not set(needed_data[file]).issubset(filedict.keys()):
+            for key in needed_data[file]:
+                if key not in filedict.keys():
+                    raise ValueError(f"Did not find data for key {key} that should be in file {file}.")
+    return
+
+
+def check_db_file_links(path: Path) -> None:
+    """
+    Check whether for each record in the given correlator library, we can find the data in the file as we expect.
+    Also check, whether there are unreachable records in the files. If either of the issues arise, throws an error.
+
+    Parameters
+    ----------
+    path: Path
+        Path to the backlog-library to check.
+    """
+    db = get_db_file(path)
+    search_expr = "SELECT path FROM 'backlogs'"
+    conn = sqlite3.connect(path / db)
+    results = pd.read_sql(search_expr, conn)['path'].values
+    _check_db2paths(path, list(results))
+
+
+def check_path_and_config(path: Path) -> None:
+    """
+    Check whether the given path exists and the cinfigureation file can be found.
+
+    Parameters
+    ----------
+    path: Path
+        Path to the backlog-library to check.
+    """
+    if not os.path.exists(path):
+        raise FileNotFoundError(f"Corrlib path {path} does not exist.")
+    config_path = path / CONFIG_FILENAME
+    if not os.path.exists(config_path):
+        raise FileNotFoundError(f"Configuration file {config_path} not found.")
+
+
+def check_config_validity(path: Path) -> None:
+    """
+    Check whether the configuration file of the given corrlib-dataset path is valid.
+
+    Parameters
+    ----------
+    path: Path
+        Path to the backlog-library to check.
+    """
+    config = ConfigParser()
+    config_path = path / CONFIG_FILENAME
+    if os.path.exists(config_path):
+        config.read(config_path)
+    else:
+        raise FileNotFoundError("Configuration file not found.")
+
+    if config.has_section('core'):
+        core_opts = ['version', 'tracker', 'cached']
+        has_core_opts = [config.has_option('core', opt) for opt in core_opts]
+        if not all(has_core_opts):
+            raise ValueError("One of the options in the 'core' section  ('version', 'tracker', 'cached') is missing.")
+
+    if config.has_section('paths'):
+        has_path_opts = [config.has_option('paths', opt) for opt in path_opts]
+        if not all(has_path_opts):
+            raise ValueError("One of the options in the 'path' section  ('db', 'projects_path', 'archive_path', 'toml_imports_path', 'import_scripts_path') is missing.")
+
+
+def check_paths(path: Path) -> None:
+    """
+    Check whether all paths demanded by the 'paths' section of the configuration-file exist.
+
+    Parameters
+    ----------
+    path: Path
+        Path to the backlog-library to check.
+    """
+    config = ConfigParser()
+    config_path = path / CONFIG_FILENAME
+    if os.path.exists(config_path):
+        config.read(config_path)
+    else:
+        raise FileNotFoundError("Configuration file not found.")
+    has_paths = [os.path.exists(path / config.get('paths', opt)) for opt in path_opts]
+    if not all(has_paths):
+        raise FileNotFoundError("One of the paths specified in the configuration file is not present.")
+
+
+def full_integrity_check(path: Path) -> None:
+    """
+    Aggregate all checks for easy validation of the backlog-library.
+
+    Parameters
+    ----------
+    path: Path
+        Path to the backlog-library to check.
+    """
+    print("Run full integrity check...")
+    check_path_and_config(path)
+    print("(1/5) Path and config-file exist: ✅")
+    check_config_validity(path)
+    print("(2/5) Configuration is valid: ✅")
+    check_paths(path)
+    print("(3/5) Needed paths exist: ✅")
+    check_db_integrity(path)
+    print("(4/5) Database is sane: ✅")
+    check_db_file_links(path)
+    print("(5/5) DB2File and File2DB-links are sound: ✅")
+    print("Full integrity check: ✅")
+
+

corrlib/main.py

@@ -5,11 +5,13 @@ import os
 from .git_tools import move_submodule
 import shutil
 from .find import _project_lookup_by_id
-from .tools import list2str, str2list
+from .tools import list2str, str2list, get_db_file
-from typing import Union
+from .tracker import get, save, unlock, clone, drop
+from typing import Union, Optional
+from pathlib import Path
 
 
-def create_project(path: str, uuid: str, owner: Union[str, None]=None, tags: Union[str, None]=None, aliases: Union[str, None]=None, code: Union[str, None]=None):
+def create_project(path: Path, uuid: str, owner: Union[str, None]=None, tags: Union[list[str], None]=None, aliases: Union[list[str], None]=None, code: Union[str, None]=None) -> None:
     """
     Create a new project entry in the database.
 
@@ -24,30 +26,48 @@ def create_project(path: str, uuid: str, owner: Union[str, None]=None, tags: Uni
     code: str (optional)
         The code that was used to create the measurements.
     """
-    db = path + "/backlogger.db"
+    db_file = get_db_file(path)
-    dl.get(db, dataset=path)
+    db = path / db_file
+    get(path, db_file)
     conn = sqlite3.connect(db)
     c = conn.cursor()
     known_projects = c.execute("SELECT * FROM projects WHERE id=?", (uuid,))
     if known_projects.fetchone():
         raise ValueError("Project already imported, use update_project() instead.")
 
-    dl.unlock(db, dataset=path)
+    unlock(path, db_file)
-    alias_str = None
+    alias_str = ""
     if aliases is not None:
         alias_str = list2str(aliases)
-    tag_str = None
+    tag_str = ""
     if tags is not None:
         tag_str = list2str(tags)
     c.execute("INSERT INTO projects (id, aliases, customTags, owner, code, created_at, updated_at) VALUES (?, ?, ?, ?, ?, datetime('now'), datetime('now'))", (uuid, alias_str, tag_str, owner, code))
     conn.commit()
     conn.close()
-    dl.save(db, message="Added entry for project " + uuid + " to database", dataset=path)
+    save(path, message="Added entry for project " + uuid + " to database", files=[db_file])
+    return
 
 
-def update_project_data(path, db, uuid, prop, value = None):
+def update_project_data(path: Path, uuid: str, prop: str, value: Union[str, None] = None) -> None:
-    dl.get(db, dataset=path)
+    """
-    conn = sqlite3.connect(db)
+    Update/Edit a project entry in the database.
+    Thin wrapper around sql3 call.
+
+    Parameters
+    ----------
+    path: str
+        The path to the backlogger folder.
+    uuid: str
+        The uuid of the project.
+    prop: str
+        Property of the entry to edit
+    value: str or None
+        Value to se `prop` to.
+    """
+    db_file = get_db_file(path)
+    get(path, db_file)
+    conn = sqlite3.connect(path / db_file)
     c = conn.cursor()
     c.execute(f"UPDATE projects SET '{prop}' = '{value}' WHERE id == '{uuid}'")
     conn.commit()
@@ -55,10 +75,10 @@ def update_project_data(path, db, uuid, prop, value = None):
     return
 
 
-def update_aliases(path: str, uuid: str, aliases: list[str]):
+def update_aliases(path: Path, uuid: str, aliases: list[str]) -> None:
-    db = os.path.join(path, "backlogger.db")
+    db_file = get_db_file(path)
-    dl.get(db, dataset=path)
+    get(path, db_file)
-    known_data = _project_lookup_by_id(db, uuid)[0]
+    known_data = _project_lookup_by_id(path, uuid)[0]
     known_aliases = known_data[1]
 
     if aliases is None:
@@ -76,14 +96,16 @@ def update_aliases(path: str, uuid: str, aliases: list[str]):
 
     if not len(new_alias_list) == len(known_alias_list):
         alias_str = list2str(new_alias_list)
-        dl.unlock(db, dataset=path)
+        unlock(path, db_file)
-        update_project_data(db, uuid, "aliases", alias_str)
+        update_project_data(path, uuid, "aliases", alias_str)
-        dl.save(db, dataset=path)
+        save(path, message="Updated aliases for project " + uuid, files=[db_file])
     return
 
 
-def import_project(path: str, url: str, owner: Union[str, None]=None, tags: Union[str, None]=None, aliases: Union[str, None]=None, code: Union[str, None]=None, isDataset: bool=True):
+def import_project(path: Path, url: str, owner: Union[str, None]=None, tags: Optional[list[str]]=None, aliases: Optional[list[str]]=None, code: Optional[str]=None, isDataset: bool=True) -> str:
     """
+    Import a datalad dataset into the backlogger.
+
     Parameters
     ----------
 
@@ -91,60 +113,61 @@ def import_project(path: str, url: str, owner: Union[str, None]=None, tags: Unio
         The url of the project to import. This can be any url that datalad can handle.
     path: str
         The path to the backlogger folder.
-    aliases: list[str]
+    owner: str, optional
-        Custom name of the project, alias of the project.
+        Person responsible for the maintainance of the project to be impoerted.
-    code: str
+    tags: list[str], optional
+        Custom tags of the imported project.
+    aliases: list[str], optional
+        Custom names of the project, alias of the project.
+    code: str, optional
         Code that was used to create the measurements.
 
-    Import a datalad dataset into the backlogger.
+    Returns
-
+    -------
-    Parameters
+    uuid: str
-    ----------
+        The unique identifier of the imported project.
-    path: str
-        The path to the backlogger directory.
-    url: str
-        The url of the project to import. This can be any url that datalad can handle.
-        Also supported are non-datalad datasets, which will be converted to datalad datasets,
-        in order to receive a uuid and have a consistent interface.
-
     """
-    tmp_path = path + '/projects/tmp'
+    tmp_path = os.path.join(path, 'projects/tmp')
-    if not isDataset:
+    clone(path, source=url, target=tmp_path)
-        dl.create(tmp_path, dataset=path)
-        shutil.copytree(url + "/*", path + '/projects/tmp/')
-        dl.save(tmp_path, dataset=path)
-    else:
-        dl.install(path=tmp_path, source=url, dataset=path)
     tmp_ds = dl.Dataset(tmp_path)
     conf = dlc.ConfigManager(tmp_ds)
-    uuid = conf.get("datalad.dataset.id")
+    uuid = str(conf.get("datalad.dataset.id"))
     if not uuid:
         raise ValueError("The dataset does not have a uuid!")
-    if not os.path.exists(path + "/projects/" + uuid):
+    if not os.path.exists(path / "projects" / uuid):
-        db = path + "/backlogger.db"
+        db_file = get_db_file(path)
-        dl.get(db, dataset=path)
+        get(path, db_file)
-        dl.unlock(db, dataset=path)
+        unlock(path, db_file)
         create_project(path, uuid, owner, tags, aliases, code)
-        move_submodule(path, 'projects/tmp', 'projects/' + uuid)
+        move_submodule(path, Path('projects/tmp'), Path('projects') / uuid)
-        os.mkdir(path + '/import_scripts/' + uuid)
+        os.mkdir(path / 'import_scripts' / uuid)
-        dl.save([db, path + '/projects/' + uuid], message="Import project from " + url, dataset=path)
+        save(path, message="Import project from " + url, files=[Path(f'projects/{uuid}'), db_file])
     else:
         dl.drop(tmp_path, reckless='kill')
-        shutil.rmtree(tmp_path)    
+        shutil.rmtree(tmp_path)
         if aliases is not None:
             if isinstance(aliases, str):
                 alias_list = [aliases]
             else:
                 alias_list = aliases
             update_aliases(path, uuid, alias_list)
-            
+
     # make this more concrete
     return uuid
 
 
-def drop_project_data(path: str, uuid: str, path_in_project: str = ""):
+def drop_project_data(path: Path, uuid: str, path_in_project: str = "") -> None:
     """
-    Drop (parts of) a prject to free up diskspace
+    Drop (parts of) a project to free up diskspace
-    """
-    dl.drop(path + "/projects/" + uuid + "/" + path_in_project)
 
+    Parameters
+    ----------
+    path: str
+        Path of the library.
+    uuid: str
+        The UUID ofthe project rom which data is to be dropped.
+    path_pn_project: str, optional
+        If set, only the given path within the project is dropped.
+    """
+    drop(path / "projects" / uuid / path_in_project)
+    return

corrlib/meas_io.py

@@ -1,17 +1,23 @@
 from pyerrors.input import json as pj
 import os
-import datalad.api as dl
 import sqlite3
 from .input import sfcf,openQCD
 import json
 from typing import Union
 from pyerrors import Obs, Corr, dump_object, load_object
 from hashlib import sha256
-from .tools import cached
+from .tools import get_db_file, cache_enabled
+from .tracker import get, save, unlock
 import shutil
+from typing import Any
+from pathlib import Path
+from .integrity import _check_db2paths
 
 
-def write_measurement(path, ensemble, measurement, uuid, code, parameter_file=None):
+CACHE_DIR = ".cache"
+
+
+def write_measurement(path: Path, ensemble: str, measurement: dict[str, dict[str, dict[str, Any]]], uuid: str, code: str, parameter_file: Union[str, None]) -> None:
     """
     Write a measurement to the backlog.
     If the file for the measurement already exists, update the measurement.
@@ -26,26 +32,41 @@ def write_measurement(path, ensemble, measurement, uuid, code, parameter_file=No
         Measurements to be captured in the backlogging system.
     uuid: str
         The uuid of the project.
+    code: str
+        Name of the code that was used for the project.
+    parameter_file: str
+        The parameter file used for the measurement.
     """
-    db = os.path.join(path, 'backlogger.db')
+    path = Path(path)
-    dl.get(db, dataset=path)
+    db_file = get_db_file(path)
-    dl.unlock(db, dataset=path)
+    db = path / db_file
+
+    files_to_save = []
+
+    get(path, db_file)
+    unlock(path, db_file)
+    files_to_save.append(db_file)
+
     conn = sqlite3.connect(db)
     c = conn.cursor()
-    files = []
     for corr in measurement.keys():
-        file_in_archive = os.path.join('.', 'archive', ensemble, corr,  uuid + '.json.gz')
+        file_in_archive = Path('.') / 'archive' / ensemble / corr / str(uuid + '.json.gz')
-        file = os.path.join(path, file_in_archive)
+        file = Path(path) / file_in_archive
-        files.append(file)
         known_meas = {}
-        if not os.path.exists(os.path.join(path, '.', 'archive', ensemble, corr)):
+        if not os.path.exists(path / 'archive' / ensemble / corr):
-            os.makedirs(os.path.join(path, '.', 'archive', ensemble, corr))
+            os.makedirs(path / 'archive' / ensemble / corr)
+            files_to_save.append(file_in_archive)
         else:
             if os.path.exists(file):
-                dl.unlock(file, dataset=path)
+                if file not in files_to_save:
-                known_meas = pj.load_json_dict(file)
+                    unlock(path, file_in_archive)
+                    files_to_save.append(file_in_archive)
+                known_meas = pj.load_json_dict(str(file), verbose=False)
         if code == "sfcf":
-            parameters = sfcf.read_param(path, uuid, parameter_file)
+            if parameter_file is not None:
+                parameters = sfcf.read_param(path, uuid, parameter_file)
+            else:
+                raise Exception("Need parameter file for this code!")
             pars = {}
             subkeys = list(measurement[corr].keys())
             for subkey in subkeys:
@@ -54,11 +75,29 @@ def write_measurement(path, ensemble, measurement, uuid, code, parameter_file=No
         elif code == "openQCD":
             ms_type = list(measurement.keys())[0]
             if ms_type == 'ms1':
-                parameters = openQCD.read_ms1_param(path, uuid, parameter_file)
+                if parameter_file is not None:
+                    if parameter_file.endswith(".ms1.in"):
+                        parameters = openQCD.load_ms1_infile(path, uuid, parameter_file)
+                    elif parameter_file.endswith(".ms1.par"):
+                        parameters = openQCD.load_ms1_parfile(path, uuid, parameter_file)
+                else:
+                    # Temporary solution
+                    parameters = {}
+                    parameters["rand"] = {}
+                    parameters["rw_fcts"] = [{}]
+                    for nrw in range(1):
+                        if "nsrc" not in parameters["rw_fcts"][nrw]:
+                            parameters["rw_fcts"][nrw]["nsrc"] = 1
+                        if "mu" not in parameters["rw_fcts"][nrw]:
+                            parameters["rw_fcts"][nrw]["mu"] = "None"
+                        if "np" not in parameters["rw_fcts"][nrw]:
+                            parameters["rw_fcts"][nrw]["np"] = "None"
+                        if "irp" not in parameters["rw_fcts"][nrw]:
+                            parameters["rw_fcts"][nrw]["irp"] = "None"
                 pars = {}
                 subkeys = []
                 for i in range(len(parameters["rw_fcts"])):
-                    par_list = []                
+                    par_list = []
                     for k in parameters["rw_fcts"][i].keys():
                         par_list.append(str(parameters["rw_fcts"][i][k]))
                     subkey = "/".join(par_list)
@@ -66,7 +105,7 @@ def write_measurement(path, ensemble, measurement, uuid, code, parameter_file=No
                     pars[subkey] = json.dumps(parameters["rw_fcts"][i])
             elif ms_type in ['t0', 't1']:
                 if parameter_file is not None:
-                    parameters = openQCD.read_ms3_param(path, uuid, parameter_file)
+                    parameters = openQCD.load_ms3_infile(path, uuid, parameter_file)
                 else:
                     parameters = {}
                     for rwp in ["integrator", "eps", "ntot", "dnms"]:
@@ -79,25 +118,25 @@ def write_measurement(path, ensemble, measurement, uuid, code, parameter_file=No
                 subkey = "/".join(par_list)
                 subkeys = [subkey]
                 pars[subkey] = json.dumps(parameters)
-        for subkey in subkeys:                
+        for subkey in subkeys:
             parHash = sha256(str(pars[subkey]).encode('UTF-8')).hexdigest()
-            meas_path = file_in_archive + "::" + parHash
+            meas_path = str(file_in_archive) + "::" + parHash
 
             known_meas[parHash] = measurement[corr][subkey]
-            
+
             if c.execute("SELECT * FROM backlogs WHERE path = ?", (meas_path,)).fetchone() is not None:
                 c.execute("UPDATE backlogs SET updated_at = datetime('now') WHERE path = ?", (meas_path, ))
             else:
                 c.execute("INSERT INTO backlogs (name, ensemble, code, path, project, parameters, parameter_file, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, datetime('now'), datetime('now'))",
                                                 (corr, ensemble, code, meas_path, uuid, pars[subkey], parameter_file))
         conn.commit()
-        pj.dump_dict_to_json(known_meas, file)
+        pj.dump_dict_to_json(known_meas, str(file))
-    files.append(path + '/backlogger.db')
     conn.close()
-    dl.save(files, message="Add measurements to database", dataset=path)
+    save(path, message="Add measurements to database", files=files_to_save)
+    return
 
 
-def load_record(path: str, meas_path: str):
+def load_record(path: Path, meas_path: str) -> Union[Corr, Obs]:
     """
     Load a list of records by their paths.
 
@@ -107,7 +146,7 @@ def load_record(path: str, meas_path: str):
         Path of the correlator library.
     meas_path: str
         The path to the correlator in the backlog system.
-    
+
     Returns
     -------
     co : Corr or Obs
@@ -116,7 +155,7 @@ def load_record(path: str, meas_path: str):
     return load_records(path, [meas_path])[0]
 
 
-def load_records(path: str, meas_paths: list[str], preloaded = {}) -> list[Union[Corr, Obs]]:
+def load_records(path: Path, meas_paths: list[str], preloaded: dict[str, Any] = {}, dry_run: bool = False) -> list[Union[Corr, Obs]]:
     """
     Load a list of records by their paths.
 
@@ -126,11 +165,20 @@ def load_records(path: str, meas_paths: list[str], preloaded = {}) -> list[Union
         Path of the correlator library.
     meas_paths: list[str]
         A list of the paths to the correlator in the backlog system.
-    
+    preloaded: dict[str, Any]
+        The data that is already preloaded. Of interest if data has alread been loaded in the same script.
+    dry_run: bool
+        Do not load datda, just check whether we can reach the data we are interested in.
+
     Returns
     -------
-    List
+    returned_data: list
+        The loaded records.
     """
+    path = Path(path)
+    if dry_run:
+        _check_db2paths(path, meas_paths)
+        return []
     needed_data: dict[str, list[str]] = {}
     for mpath in meas_paths:
         file = mpath.split("::")[0]
@@ -138,48 +186,106 @@ def load_records(path: str, meas_paths: list[str], preloaded = {}) -> list[Union
             needed_data[file] = []
         key = mpath.split("::")[1]
         needed_data[file].append(key)
-    returned_data: list = []
+    returned_data: list[Any] = []
     for file in needed_data.keys():
         for key in list(needed_data[file]):
-            if os.path.exists(cache_path(path, file, key) + ".p"):
+            if os.path.exists(str(cache_path(path, file, key)) + ".p"):
-                returned_data.append(load_object(cache_path(path, file, key) + ".p"))
+                returned_data.append(load_object(str(cache_path(path, file, key)) + ".p"))
             else:
                 if file not in preloaded:
-                    preloaded[file] = preload(path, file)
+                    preloaded[file] = preload(path, Path(file))
                 returned_data.append(preloaded[file][key])
-                if cached:
+                if cache_enabled(path):
                     if not os.path.exists(cache_dir(path, file)):
                         os.makedirs(cache_dir(path, file))
-                    dump_object(preloaded[file][key], cache_path(path, file, key))
+                    dump_object(preloaded[file][key], str(cache_path(path, file, key)))
     return returned_data
 
 
-def cache_dir(path, file):
+def cache_dir(path: Path, file: str) -> Path:
-    cache_path_list = [path]
+    """
-    cache_path_list.append(".cache")
+    Returns the directory corresponding to the cache for the given file.
-    cache_path_list.extend(file.split("/")[1:])
+
-    cache_path = "/".join(cache_path_list)
+    Parameters
+    ----------
+    path: str
+        The path of the library.
+    file: str
+        The file in the library that we want to access the cached data of.
+    Returns
+    -------
+    cache_path: str
+        The path holding the cached data for the given file.
+    """
+    cache_path_list = file.split("/")[1:]
+    cache_path = Path(path) / CACHE_DIR
+    for directory in cache_path_list:
+        cache_path /= directory
     return cache_path
 
 
-def cache_path(path, file, key):
+def cache_path(path: Path, file: str, key: str) -> Path:
-    cache_path = os.path.join(cache_dir(path, file), key)
+    """
+    Parameters
+    ----------
+    path: str
+        The path of the library.
+    file: str
+        The file in the library that we want to access the cached data of.
+    key: str
+        The key within the archive file.
+
+    Returns
+    -------
+    cache_path: str
+        The path at which the measurement of the given file and key is cached.
+    """
+    path = Path(path)
+    cache_path = cache_dir(path, file) / key
     return cache_path
 
 
-def preload(path: str, file: str):
+def preload(path: Path, file: Path) -> dict[str, Any]:
-    dl.get(os.path.join(path, file), dataset=path)
+    """
-    filedict = pj.load_json_dict(os.path.join(path, file))
+    Read the contents of a file into a json dictionary with the pyerrors.json.load_json_dict method.
+
+    Parameters
+    ----------
+    path: str
+        The path of the library.
+    file: str
+        The file within the library to be laoded.
+
+    Returns
+    -------
+    filedict: dict[str, Any]
+        The data read from the file.
+    """
+    path = Path(path)
+    get(path, file)
+    filedict: dict[str, Any] = pj.load_json_dict(str(path / file))
+    print("> read file")
     return filedict
 
 
-def drop_record(path: str, meas_path: str):
+def drop_record(path: Path, meas_path: str) -> None:
+    """
+    Drop a record by it's path.
+
+    Parameters
+    ----------
+    path: str
+        The path of the library.
+    meas_path: str
+        The measurement path as noted in the database.
+    """
     file_in_archive = meas_path.split("::")[0]
-    file = os.path.join(path, file_in_archive)
+    file = Path(path) / file_in_archive
-    db = os.path.join(path, 'backlogger.db')
+    db_file = get_db_file(path)
-    dl.get(db, dataset=path)
+    db = path / db_file
+    get(path, db_file)
     sub_key = meas_path.split("::")[1]
-    dl.unlock(db, dataset=path)
+    unlock(path, db_file)
     conn = sqlite3.connect(db)
     c = conn.cursor()
     if c.execute("SELECT * FROM backlogs WHERE path = ?", (meas_path, )).fetchone() is not None:
@@ -188,17 +294,28 @@ def drop_record(path: str, meas_path: str):
         raise ValueError("This measurement does not exist as an entry!")
 
     conn.commit()
-    known_meas = pj.load_json_dict(file)
+    known_meas = pj.load_json_dict(str(file))
     if sub_key in known_meas:
         del known_meas[sub_key]
-        dl.unlock(file, dataset=path)
+        unlock(path, Path(file_in_archive))
-        pj.dump_dict_to_json(known_meas, file)
+        pj.dump_dict_to_json(known_meas, str(file))
-        dl.save([db, file], message="Drop measurements to database", dataset=path)
+        save(path, message="Drop measurements to database", files=[db, file])
         return
     else:
         raise ValueError("This measurement does not exist as a file!")
 
-def drop_cache(path: str):
+
-    cache_dir = os.path.join(path, ".cache")
+def drop_cache(path: Path) -> None:
+    """
+    Drop the cache directory of the library.
+
+    Parameters
+    ----------
+    path: str
+        The path of the library.
+    """
+    path = Path(path)
+    cache_dir = path / ".cache"
     for f in os.listdir(cache_dir):
-        shutil.rmtree(os.path.join(cache_dir, f))
+        shutil.rmtree(cache_dir / f)
+    return

corrlib/pars/openQCD/__init__.py

@@ -0,0 +1,3 @@
+
+from . import ms1 as ms1
+from . import qcd2 as qcd2

corrlib/pars/openQCD/flags.py

@@ -0,0 +1,59 @@
+"""
+Reconstruct the outputs of flags.
+"""
+
+import struct
+from typing import Any, BinaryIO
+
+# lat_parms.c
+def lat_parms_write_lat_parms(fp: BinaryIO) -> dict[str, Any]:
+        """
+        NOTE: This is a duplcation from qcd2.
+        Unpack the lattice parameters written by write_lat_parms.
+        """
+        lat_pars = {}
+        t = fp.read(16)
+        lat_pars["N"] = list(struct.unpack('iiii', t)) # lattice extends
+        t = fp.read(8)
+        nk, isw = struct.unpack('ii', t) # number of kappas and isw parameter
+        lat_pars["nk"] = nk
+        lat_pars["isw"] = isw
+        t = fp.read(8)
+        lat_pars["beta"] = struct.unpack('d', t)[0] # beta
+        t = fp.read(8)
+        lat_pars["c0"] = struct.unpack('d', t)[0]
+        t = fp.read(8)
+        lat_pars["c1"] = struct.unpack('d', t)[0]
+        t = fp.read(8)
+        lat_pars["csw"] = struct.unpack('d', t)[0] # csw factor
+        kappas = []
+        m0s = []
+        # read kappas
+        for ik in range(nk):
+            t = fp.read(8)
+            kappas.append(struct.unpack('d', t)[0])
+            t = fp.read(8)
+            m0s.append(struct.unpack('d', t)[0])
+        lat_pars["kappas"] = kappas
+        lat_pars["m0s"] = m0s
+        return lat_pars
+
+
+def lat_parms_write_bc_parms(fp: BinaryIO) -> dict[str, Any]:
+        """
+        NOTE: This is a duplcation from qcd2.
+        Unpack the boundary parameters written by write_bc_parms.
+        """
+        bc_pars: dict[str, Any] = {}
+        t = fp.read(4)
+        bc_pars["type"] = struct.unpack('i', t)[0] # type of hte boundaries
+        t = fp.read(104)
+        bc_parms = struct.unpack('d'*13, t)
+        bc_pars["cG"] = list(bc_parms[:2]) # boundary gauge field improvement
+        bc_pars["cF"] = list(bc_parms[2:4]) # boundary fermion field improvement
+        phi: list[list[float]] = [[], []]
+        phi[0] = list(bc_parms[4:7])
+        phi[1] = list(bc_parms[7:10])
+        bc_pars["phi"] = phi
+        bc_pars["theta"] = list(bc_parms[10:])
+        return bc_pars

corrlib/pars/openQCD/ms1.py

@@ -0,0 +1,30 @@
+from . import flags
+
+from typing import Any
+from pathlib import Path
+
+
+def read_qcd2_ms1_par_file(fname: Path) -> dict[str, dict[str, Any]]:
+    """
+    The subroutines written here have names according to the openQCD programs and functions that write out the data.
+    Parameters
+    ----------
+    fname: Path
+        Location of the parameter file.
+
+    Returns
+    -------
+    par_dict: dict
+        Dictionary holding the parameters specified in the given file.
+    """
+
+    with open(fname, "rb") as fp:
+        lat_par_dict = flags.lat_parms_write_lat_parms(fp)
+        bc_par_dict = flags.lat_parms_write_bc_parms(fp)
+        fp.close()
+    par_dict = {}
+    par_dict["lat"] = lat_par_dict
+    par_dict["bc"] = bc_par_dict
+    return par_dict
+
+

corrlib/pars/openQCD/qcd2.py

@@ -0,0 +1,29 @@
+from . import flags
+
+from pathlib import Path
+from typing import Any
+
+
+def read_qcd2_par_file(fname: Path) -> dict[str, dict[str, Any]]:
+    """
+    The subroutines written here have names according to the openQCD programs and functions that write out the data.
+
+    Parameters
+    ----------
+    fname: Path
+        Location of the parameter file.
+
+    Returns
+    -------
+    par_dict: dict
+        Dictionary holding the parameters specified in the given file.
+    """
+
+    with open(fname, "rb") as fp:
+        lat_par_dict = flags.lat_parms_write_lat_parms(fp)
+        bc_par_dict = flags.lat_parms_write_bc_parms(fp)
+        fp.close()
+    par_dict = {}
+    par_dict["lat"] = lat_par_dict
+    par_dict["bc"] = bc_par_dict
+    return par_dict

corrlib/sql.py

@@ -0,0 +1,17 @@
+import sqlite3
+from .tools import get_db_file
+from pathlib import Path
+from typing import Any
+
+
+def thin_sql_wrapper(path: Path, stmt: str) -> list[Any]:
+    db_file = get_db_file(path)
+    db = path / db_file
+    conn = sqlite3.connect(db)
+    c = conn.cursor()
+
+    c.execute(stmt)
+    results = c.fetchall()
+    conn.commit()
+    conn.close()
+    return results

corrlib/toml.py

@@ -10,22 +10,49 @@ the import of projects via TOML.
 
 import tomllib as toml
 import shutil
+
+import datalad.api as dl
+from .tracker import save
 from .input import sfcf, openQCD
 from .main import import_project, update_aliases
 from .meas_io import write_measurement
-import datalad.api as dl
 import os
 from .input.implementations import codes as known_codes
+from typing import Any
+from pathlib import Path
 
-def replace_string(string: str, name: str, val: str):
+
+def replace_string(string: str, name: str, val: str) -> str:
+    """
+    Replace a placeholder {name} with a value in a string.
+
+    Parameters
+    ----------
+    string: str
+        String in which the placeholders are to be replaced.
+    name: str
+        The name of the placeholder.
+    val: str
+        The value the placeholder is to be replaced with.
+    """
     if '{' + name + '}' in string:
         n = string.replace('{' + name + '}', val)
         return n
     else:
         return string
 
-def replace_in_meas(measurements: dict, vars: dict[str, str]):
+
-    # replace global variables
+def replace_in_meas(measurements: dict[str, dict[str, Any]], vars: dict[str, str]) -> dict[str, dict[str, Any]]:
+    """
+    Replace placeholders in the defiitions for a measurement.
+
+    Parameters
+    ----------
+    measurements: dict[str, dict[str, Any]]
+        The measurements read from the toml file.
+    vars: dict[str, str]
+        Simple key:value dictionary with the keys to be replaced by the values.
+    """
     for name, value in vars.items():
         for m in measurements.keys():
             for key in measurements[m].keys():
@@ -36,7 +63,18 @@ def replace_in_meas(measurements: dict, vars: dict[str, str]):
                         measurements[m][key][i] = replace_string(measurements[m][key][i], name, value)
     return measurements
 
-def fill_cons(measurements, constants):
+
+def fill_cons(measurements: dict[str, dict[str, Any]], constants: dict[str, str]) -> dict[str, dict[str, Any]]:
+    """
+    Fill in defined constants into the measurements of the toml-file.
+
+    Parameters
+    ----------
+    measurements: dict[str, dict[str, Any]]
+        The measurements read from the toml file.
+    constants: dict[str, str]
+        Simple key:value dictionary with the keys to be replaced by the values.
+    """
     for m in measurements.keys():
         for name, val in constants.items():
             if name not in measurements[m].keys():
@@ -44,7 +82,15 @@ def fill_cons(measurements, constants):
     return measurements
 
 
-def check_project_data(d: dict) -> None:
+def check_project_data(d: dict[str, dict[str, str]]) -> None:
+    """
+    Check the data given in the toml import file for the project we want to import.
+
+    Parameters
+    ----------
+    d: dict
+        The dictionary holding the data necessary to import the project.
+    """
     if 'project' not in d.keys() or 'measurements' not in d.keys() or len(list(d.keys())) > 4:
         raise ValueError('There should only be maximally be four keys on the top level, "project" and "measurements" are mandatory, "contants" is optional!')
     project_data = d['project']
@@ -57,7 +103,17 @@ def check_project_data(d: dict) -> None:
     return
 
 
-def check_measurement_data(measurements: dict, code: str) -> None:
+def check_measurement_data(measurements: dict[str, dict[str, str]], code: str) -> None:
+    """
+    Check syntax of the measurements we want to import.
+
+    Parameters
+    ----------
+    measurements: dict[str, dict[str, str]]
+        The dictionary holding the necessary data to import the project.
+    code: str
+        The code used for the project.
+    """
     var_names: list[str] = []
     if code == "sfcf":
         var_names = ["path", "ensemble", "param_file", "version", "prefix", "cfg_seperator", "names"]
@@ -71,12 +127,25 @@ def check_measurement_data(measurements: dict, code: str) -> None:
     return
 
 
-def import_tomls(path: str, files: list[str], copy_files: bool=True) -> None:
+def import_tomls(path: Path, files: list[str], copy_files: bool=True) -> None:
+    """
+    Import multiple toml files.
+
+    Parameters
+    ----------
+    path: str
+        Path to the backlog directory.
+    files: list[str]
+        Path to the description files.
+    copy_files: bool, optional
+        Whether the toml-files will be copied into the library. Default is True.
+    """
     for file in files:
         import_toml(path, file, copy_files)
+    return
 
 
-def import_toml(path: str, file: str, copy_file: bool=True) -> None:
+def import_toml(path: Path, file: str, copy_file: bool=True) -> None:
     """
     Import a project decribed by a .toml file.
 
@@ -86,29 +155,37 @@ def import_toml(path: str, file: str, copy_file: bool=True) -> None:
         Path to the backlog directory.
     file: str
         Path to the description file.
+    copy_file: bool, optional
+        Whether the toml-files will be copied into the library. Default is True.
     """
+    if not os.path.exists(path):
+        raise FileNotFoundError(f"Corrlib path {path} does not exist.")
+    if not os.path.exists(file):
+        raise FileNotFoundError(f".toml-file {file} does not exist.")
     print("Import project as decribed in " + file)
     with open(file, 'rb') as fp:
         toml_dict = toml.load(fp)
     check_project_data(toml_dict)
-    project: dict = toml_dict['project']
+    project: dict[str, Any] = toml_dict['project']
     if project['code'] not in known_codes:
         raise ValueError('Code' + project['code'] + 'has no import implementation!')
-    measurements: dict = toml_dict['measurements']
+    measurements: dict[str, dict[str, Any]] = toml_dict['measurements']
     measurements = fill_cons(measurements, toml_dict['constants'] if 'constants' in toml_dict else {})
     measurements = replace_in_meas(measurements, toml_dict['replace'] if 'replace' in toml_dict else {})
     check_measurement_data(measurements, project['code'])
-    aliases = project.get('aliases', None)
+    aliases = project.get('aliases', [])
     uuid = project.get('uuid', None)
     if uuid is not None:
-        if not os.path.exists(path + "/projects/" + uuid):
+        if not os.path.exists(path / "projects" / uuid):
             uuid = import_project(path, project['url'], aliases=aliases)
         else:
             update_aliases(path, uuid, aliases)
     else:
         uuid = import_project(path, project['url'], aliases=aliases)
+    imeas = 1
+    nmeas = len(measurements.keys())
     for mname, md in measurements.items():
-        print("Import measurement: " + mname)
+        print(f"Import measurement {imeas}/{nmeas}: {mname}")
         ensemble = md['ensemble']
         if project['code'] == 'sfcf':
             param = sfcf.read_param(path, uuid, md['param_file'])
@@ -119,43 +196,63 @@ def import_toml(path: str, file: str, copy_file: bool=True) -> None:
                 measurement = sfcf.read_data(path, uuid, md['path'], md['prefix'], param,
                                             version=md['version'], cfg_seperator=md['cfg_seperator'], sep='/')
 
-            print(mname + " imported.")
         elif project['code'] == 'openQCD':
             if md['measurement'] == 'ms1':
-                param = openQCD.read_ms1_param(path, uuid, md['param_file'])
+                if 'param_file' in md.keys():
+                    parameter_file = md['param_file']
+                    if parameter_file.endswith(".ms1.in"):
+                        param = openQCD.load_ms1_infile(path, uuid, parameter_file)
+                    elif parameter_file.endswith(".ms1.par"):
+                        param = openQCD.load_ms1_parfile(path, uuid, parameter_file)
+                else:
+                    # Temporary solution
+                    parameters: dict[str, Any] = {}
+                    parameters["rand"] = {}
+                    parameters["rw_fcts"] = [{}]
+                    for nrw in range(1):
+                        if "nsrc" not in parameters["rw_fcts"][nrw]:
+                            parameters["rw_fcts"][nrw]["nsrc"] = 1
+                        if "mu" not in parameters["rw_fcts"][nrw]:
+                            parameters["rw_fcts"][nrw]["mu"] = "None"
+                        if "np" not in parameters["rw_fcts"][nrw]:
+                            parameters["rw_fcts"][nrw]["np"] = "None"
+                        if "irp" not in parameters["rw_fcts"][nrw]:
+                            parameters["rw_fcts"][nrw]["irp"] = "None"
+                    param = parameters
                 param['type'] = 'ms1'
                 measurement = openQCD.read_rwms(path, uuid, md['path'], param, md["prefix"], version=md["version"], names=md['names'], files=md['files'])
             elif md['measurement'] == 't0':
                 if 'param_file' in md:
-                    param = openQCD.read_ms3_param(path, uuid, md['param_file'])
+                    param = openQCD.load_ms3_infile(path, uuid, md['param_file'])
                 else:
                     param = {}
                     for rwp in ["integrator", "eps", "ntot", "dnms"]:
                         param[rwp] = "Unknown"
                 param['type'] = 't0'
-                measurement = openQCD.extract_t0(path, uuid, md['path'], param, md["prefix"], md["dtr_read"], md["xmin"], md["spatial_extent"],
+                measurement = openQCD.extract_t0(path, uuid, md['path'], param, str(md["prefix"]), int(md["dtr_read"]), int(md["xmin"]), int(md["spatial_extent"]),
-                                                 fit_range=md.get('fit_range', 5), postfix=md.get('postfix', None), names=md.get('names', None), files=md.get('files', None))
+                                                 fit_range=int(md.get('fit_range', 5)), postfix=str(md.get('postfix', '')), names=md.get('names', []), files=md.get('files', []))
             elif md['measurement'] == 't1':
                 if 'param_file' in md:
-                    param = openQCD.read_ms3_param(path, uuid, md['param_file'])
+                    param = openQCD.load_ms3_infile(path, uuid, md['param_file'])
                 param['type'] = 't1'
-                measurement = openQCD.extract_t1(path, uuid, md['path'], param, md["prefix"], md["dtr_read"], md["xmin"], md["spatial_extent"],
+                measurement = openQCD.extract_t1(path, uuid, md['path'], param, str(md["prefix"]), int(md["dtr_read"]), int(md["xmin"]), int(md["spatial_extent"]),
-                                                 fit_range=md.get('fit_range', 5), postfix=md.get('postfix', None), names=md.get('names', None), files=md.get('files', None))
+                                                 fit_range=int(md.get('fit_range', 5)), postfix=str(md.get('postfix', '')), names=md.get('names', []), files=md.get('files', []))
-
         write_measurement(path, ensemble, measurement, uuid, project['code'], (md['param_file'] if 'param_file' in md else None))
+        imeas += 1
+        print(mname + " imported.")
 
-    if not os.path.exists(os.path.join(path, "toml_imports", uuid)):
+    if not os.path.exists(path / "toml_imports" / uuid):
-        os.makedirs(os.path.join(path, "toml_imports", uuid))
+        os.makedirs(path / "toml_imports" / uuid)
     if copy_file:
-        import_file = os.path.join(path, "toml_imports", uuid, file.split("/")[-1])
+        import_file = path / "toml_imports" / uuid / file.split("/")[-1]
         shutil.copy(file, import_file)
-        dl.save(import_file, message="Import using " + import_file, dataset=path)
+        save(path, files=[import_file], message=f"Import using {import_file}")
-        print("File copied to " + import_file)
+        print(f"File copied to {import_file}")
     print("Imported project.")
     return
 
 
-def reimport_project(path, uuid):
+def reimport_project(path: Path, uuid: str) -> None:
     """
     Reimport an existing project using the files that are already available for this project.
 
@@ -166,13 +263,24 @@ def reimport_project(path, uuid):
     uuid: str
     uuid of the project that is to be reimported.
     """
-    config_path = "/".join([path, "import_scripts", uuid])
+    config_path = path / "import_scripts" / uuid
     for p, filenames, dirnames in os.walk(config_path):
         for fname in filenames:
             import_toml(path, os.path.join(config_path, fname), copy_file=False)
     return
 
 
-def update_project(path, uuid):
+def update_project(path: Path, uuid: str) -> None:
+    """
+    Update all entries associated with a given project.
+
+    Parameters
+    ----------
+    path: str
+        The path of the library.
+    uuid: str
+        The unique identifier of the project to be updated.
+    """
     dl.update(how='merge', follow='sibling', dataset=os.path.join(path, "projects", uuid))
     # reimport_project(path, uuid)
+    return

corrlib/tools.py

@@ -1,17 +1,158 @@
+import os
+from configparser import ConfigParser
+from typing import Any
+from pathlib import Path
+
+CONFIG_FILENAME = ".corrlib"
+cached: bool = True
 
 
-def str2list(string):
+def str2list(string: str) -> list[str]:
+    """
+    Convert a comma-separated string to a list.
+
+    Parameters
+    ----------
+    string: str
+        The sting holding a comma-sparated list.
+
+    Returns
+    -------
+    s: list[str]
+        The list of strings that was held bythe comma separated string.
+    """
     return string.split(",")
 
-def list2str(mylist):
+def list2str(mylist: list[str]) -> str:
+    """
+    Convert a list to a comma-separated string.
+
+    Parameters
+    ----------
+    mylist: list[str]
+        A list of strings to be concatinated.
+
+    Returns
+    -------
+    s: list[str]
+        The sting holding a comma-sparated list.
+    """
     s = ",".join(mylist)
     return s
 
-cached = True
+def m2k(m: float) -> float:
+    """
+    Convert to bare quark mas $m$ to inverse mass parameter $kappa$.
 
-def m2k(m):
+    Parameters
+    ----------
+    m: float
+        Bare quark mass.
+
+    Returns
+    -------
+    k: float
+        The corresponing $kappa$.
+    """
     return 1/(2*m+8)
 
 
-def k2m(k):
+def k2m(k: float) -> float:
+    """
+    Convert from the inverse bare quark parameter $kappa$ to the bare quark mass $m$.
+
+    Parameters
+    ----------
+    k: float
+        Inverse bare quark mass parameter $kappa$.
+
+    Returns
+    -------
+    m: float
+        The corresponing bare quark mass.
+    """
     return (1/(2*k))-4
+
+
+def set_config(path: Path, section: str, option: str, value: Any) -> None:
+    """
+    Set configuration parameters for the library.
+
+    Parameters
+    ----------
+    path: str
+        The path of the library.
+    section: str
+        The section within the configuration file.
+    option: str
+        The option to be set to value.
+    value: Any
+        The value we set the option to.
+    """
+    path = Path(path)
+    config_path = path / CONFIG_FILENAME
+    config = ConfigParser()
+    if os.path.exists(config_path):
+        config.read(config_path)
+    if not config.has_section(section):
+        config.add_section(section)
+    config.set(section, option, value)
+    with open(config_path, 'w') as configfile:
+        config.write(configfile)
+    return
+
+
+def get_db_file(path: Path) -> Path:
+    """
+    Get the database file associated with the library at the given path.
+
+    Parameters
+    ----------
+    path: str
+        The path of the library.
+
+    Returns
+    -------
+    db_file: str
+        The file holding the database.
+    """
+    path = Path(path)
+    if not os.path.exists(path):
+        raise FileNotFoundError(f"Corrlib path {path} does not exist.")
+    config_path = path / CONFIG_FILENAME
+    config = ConfigParser()
+    if os.path.exists(config_path):
+        config.read(config_path)
+    else:
+        raise FileNotFoundError("Configuration file not found.")
+    db_file = Path(config.get('paths', 'db', fallback='backlogger.db'))
+    return db_file
+
+
+def cache_enabled(path: Path) -> bool:
+    """
+    Check, whether the library is cached.
+    Fallback is true.
+
+    Parameters
+    ----------
+    path: str
+        The path of the library.
+
+    Returns
+    -------
+    cached_bool: bool
+        Whether the given library is cached.
+    """
+    path = Path(path)
+    config_path = path / CONFIG_FILENAME
+    config = ConfigParser()
+    if os.path.exists(config_path):
+        config.read(config_path)
+    else:
+        raise FileNotFoundError("Configuration file not found.")
+    cached_str = config.get('core', 'cached', fallback='True')
+    if cached_str not in ['True', 'False']:
+        raise ValueError(f"String {cached_str} is not a valid option, only True and False are allowed!")
+    cached_bool = cached_str == ('True')
+    return cached_bool

corrlib/tracker.py

@@ -0,0 +1,177 @@
+import os
+from configparser import ConfigParser
+import datalad.api as dl
+from typing import Optional
+import shutil
+from .tools import get_db_file, CONFIG_FILENAME
+from pathlib import Path
+
+
+def get_tracker(path: Path) -> str:
+    """
+    Get the tracker used in the dataset located at path.
+
+    Parameters
+    ----------
+    path: str
+        The path to the backlogger folder.
+
+    Returns
+    -------
+    tracker: str
+        The tracker used in the dataset.
+    """
+    path = Path(path)
+    config_path = path / CONFIG_FILENAME
+    config = ConfigParser()
+    if os.path.exists(config_path):
+        config.read(config_path)
+    else:
+        raise FileNotFoundError(f"No config file found in {path}.")
+    tracker = config.get('core', 'tracker', fallback='datalad')
+    return tracker
+
+
+def get(path: Path, file: Path) -> None:
+    """
+    Wrapper function to get a file from the dataset located at path with the specified tracker.
+
+    Parameters
+    ----------
+    path: str
+        The path to the backlogger folder.
+    file: str
+        The file to get.
+    """
+    path = Path(path)
+    tracker = get_tracker(path)
+    if tracker == 'datalad':
+        if file == get_db_file(path):
+            print("Downloading database...")
+        else:
+            print("Downloading data...")
+        dl.get(os.path.join(path, file), dataset=path)
+        print("> downloaded file")
+    elif tracker == 'None':
+        pass
+    else:
+        raise ValueError(f"Tracker {tracker} is not supported.")
+    return
+
+
+def save(path: Path, message: str, files: Optional[list[Path]]=None) -> None:
+    """
+    Wrapper function to save a file to the dataset located at path with the specified tracker.
+
+    Parameters
+    ----------
+    path: str
+        The path to the backlogger folder.
+    message: str
+        The commit message.
+    files: list[str], optional
+        The files to save. If None, all changes are saved.
+    """
+    path = Path(path)
+    tracker = get_tracker(path)
+    if tracker == 'datalad':
+        if files is not None:
+            files = [path / f for f in files]
+        dl.save(files, message=message, dataset=path)
+    elif tracker == 'None':
+        Warning("Tracker 'None' does not implement save.")
+        pass
+    else:
+        raise ValueError(f"Tracker {tracker} is not supported.")
+
+
+def init(path: Path, tracker: str='datalad') -> None:
+    """
+    Initialize a dataset at the specified path with the specified tracker.
+
+    Parameters
+    ----------
+    path: str
+        The path to initialize the dataset.
+    tracker: str
+        The tracker to use. Currently only 'datalad' and 'None' are supported.
+    """
+    path = Path(path)
+    if tracker == 'datalad':
+        dl.create(path)
+    elif tracker == 'None':
+        os.makedirs(path, exist_ok=True)
+    else:
+        raise ValueError(f"Tracker {tracker} is not supported.")
+    return
+
+
+def unlock(path: Path, file: Path) -> None:
+    """
+    Wrapper function to unlock a file in the dataset located at path with the specified tracker.
+
+    Parameters
+    ----------
+    path    : str
+        The path to the backlogger folder.
+    file    : str
+        The file to unlock.
+    """
+    path = Path(path)
+    tracker = get_tracker(path)
+    if tracker == 'datalad':
+        dl.unlock(os.path.join(path, file), dataset=path)
+    elif tracker == 'None':
+        Warning("Tracker 'None' does not implement unlock.")
+        pass
+    else:
+        raise ValueError(f"Tracker {tracker} is not supported.")
+    return
+
+
+def clone(path: Path, source: str, target: str) -> None:
+    """
+    Wrapper function to clone a dataset from source to target with the specified tracker.
+    Parameters
+    ----------
+    path: str
+        The path to the backlogger folder.
+    source: str
+        The source dataset to clone.
+    target: str
+        The target path to clone the dataset to.
+    """
+    path = Path(path)
+    tracker = get_tracker(path)
+    if tracker == 'datalad':
+        dl.clone(target=target, source=source, dataset=path)
+    elif tracker == 'None':
+        os.makedirs(path, exist_ok=True)
+        # Implement a simple clone by copying files
+        shutil.copytree(source, target, dirs_exist_ok=False)
+    else:
+        raise ValueError(f"Tracker {tracker} is not supported.")
+    return
+
+
+def drop(path: Path, reckless: Optional[str]=None) -> None:
+    """
+    Wrapper function to drop data from a dataset located at path with the specified tracker.
+
+    Parameters
+    ----------
+    path: str
+        The path to the backlogger folder.
+    reckless: Optional[str]
+        The datalad's reckless option for dropping data.
+    """
+    path = Path(path)
+    tracker = get_tracker(path)
+    if tracker == 'datalad':
+        dl.drop(path, reckless=reckless)
+    elif tracker == 'None':
+        Warning("Tracker 'None' does not implement drop.")
+        pass
+    else:
+        raise ValueError(f"Tracker {tracker} is not supported.")
+    return

corrlib/version.py

@@ -1 +1,24 @@
-__version__ = "0.2.4"
+# file generated by vcs-versioning
+# don't change, don't track in version control
+from __future__ import annotations
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+version: str
+__version__: str
+__version_tuple__: tuple[int | str, ...]
+version_tuple: tuple[int | str, ...]
+commit_id: str | None
+__commit_id__: str | None
+
+__version__ = version = '0.3.1.dev0+g08de17e6b.d20260507'
+__version_tuple__ = version_tuple = (0, 3, 1, 'dev0', 'g08de17e6b.d20260507')
+
+__commit_id__ = commit_id = 'g08de17e6b'

pyproject.toml

@@ -1,6 +1,52 @@
 [build-system]
-requires = ["setuptools >= 63.0.0", "wheel"]
+requires = ["setuptools >= 63.0.0", "wheel", "setuptools-scm"]
 build-backend = "setuptools.build_meta"
 
+[project]
+requires-python = ">=3.10"
+name = "corrlib"
+dynamic = ["version"]
+dependencies = [
+    "gitpython>=3.1.45",
+    'pyerrors>=2.11.1',
+    "datalad>=1.1.0",
+    'typer>=0.12.5',
+]
+description = "Python correlation library"
+authors = [
+  { name = 'Justus Kuhlmann', email = 'j_kuhl19@uni-muenster.de'}
+]
+
+[project.scripts]
+pcl = "corrlib.cli:app"
+
+[tool.setuptools.packages.find]
+include = ["corrlib", "corrlib.*"]
+
+[tool.setuptools_scm]
+write_to = "corrlib/version.py"
+
 [tool.ruff.lint]
-ignore = ["F403"]
+ignore = ["E501"]
+extend-select = [
+  "YTT",
+  "E",
+  "W",
+  "F",
+]
+
+[tool.mypy]
+strict = true
+implicit_reexport = false
+follow_untyped_imports = false
+ignore_missing_imports = true
+
+[dependency-groups]
+dev = [
+    "mypy>=1.19.0",
+    "pandas-stubs>=2.3.3.251201",
+    "pytest>=9.0.1",
+    "pytest-cov>=7.0.0",
+    "pytest-pretty>=1.3.0",
+    "ruff>=0.14.7",
+]

setup.py

@@ -1,18 +0,0 @@
-from setuptools import setup
-from distutils.util import convert_path
-
-
-version = {}
-with open(convert_path('corrlib/version.py')) as ver_file:
-    exec(ver_file.read(), version)
-
-setup(name='pycorrlib',
-      version=version['__version__'],
-      author='Justus Kuhlmann',
-      author_email='j_kuhl19@uni-muenster.de',
-      install_requires=['pyerrors>=2.11.1', 'datalad>=1.1.0', 'typer>=0.12.5'],
-      entry_points = {
-        'console_scripts': ['pcl=corrlib.cli:app'],
-        },
-      packages=['corrlib', 'corrlib.input']
-      )

tests/cli_test.py

@@ -0,0 +1,92 @@
+from typer.testing import CliRunner
+from corrlib.cli import app
+import os
+import sqlite3 as sql
+from pathlib import Path
+
+
+runner = CliRunner()
+
+
+def test_version() -> None:
+    result = runner.invoke(app, ["--version"])
+    assert result.exit_code == 0
+    assert "corrlib" in result.output
+
+
+def test_init_folders(tmp_path: Path) -> None:
+    dataset_path = tmp_path / "test_dataset"
+    result = runner.invoke(app, ["init", "--dataset", str(dataset_path)])
+    assert result.exit_code == 0
+    assert os.path.exists(str(dataset_path))
+    assert os.path.exists(str(dataset_path / "backlogger.db"))
+
+
+def test_init_db(tmp_path: Path) -> None:
+    dataset_path = tmp_path / "test_dataset"
+    result = runner.invoke(app, ["init", "--dataset", str(dataset_path)])
+    assert result.exit_code == 0
+    assert os.path.exists(str(dataset_path / "backlogger.db"))
+    conn = sql.connect(str(dataset_path / "backlogger.db"))
+    cursor = conn.cursor()
+    cursor.execute("SELECT name FROM sqlite_master WHERE type='table';")
+    tables = cursor.fetchall()
+    expected_tables = [
+        'projects',
+        'backlogs',
+        ]
+    table_names = [table[0] for table in tables]
+    for expected_table in expected_tables:
+        assert expected_table in table_names
+
+    cursor.execute("SELECT * FROM projects;")
+    projects = cursor.fetchall()
+    assert len(projects) == 0
+
+    cursor.execute("SELECT * FROM backlogs;")
+    backlogs = cursor.fetchall()
+    assert len(backlogs) == 0
+
+    cursor.execute("PRAGMA table_info('projects');")
+    project_columns = cursor.fetchall()
+    expected_project_columns = [
+        "id",
+        "aliases",
+        "customTags",
+        "owner",
+        "code",
+        "created_at",
+        "updated_at"
+        ]
+    project_column_names = [col[1] for col in project_columns]
+    for expected_col in expected_project_columns:
+        assert expected_col in project_column_names
+
+    cursor.execute("PRAGMA table_info('backlogs');")
+    backlog_columns = cursor.fetchall()
+    expected_backlog_columns = [
+        "id",
+        "name",
+        "ensemble",
+        "code",
+        "path",
+        "project",
+        "customTags",
+        "parameters",
+        "parameter_file",
+        "created_at",
+        "updated_at"
+        ]
+    backlog_column_names = [col[1] for col in backlog_columns]
+    for expected_col in expected_backlog_columns:
+        assert expected_col in backlog_column_names
+
+
+def test_list(tmp_path: Path) -> None:
+    dataset_path = tmp_path / "test_dataset"
+    result = runner.invoke(app, ["init", "--dataset", str(dataset_path)])
+    assert result.exit_code == 0
+    result = runner.invoke(app, ["lister", "--dataset", str(dataset_path), "ensembles"])
+    assert result.exit_code == 0
+    result = runner.invoke(app, ["lister", "--dataset", str(dataset_path), "projects"])
+    assert result.exit_code == 0

tests/find_test.py

@@ -0,0 +1,439 @@
+import corrlib.find as find
+import sqlite3
+from pathlib import Path
+import corrlib.initialization as cinit
+import pytest
+import pandas as pd
+import datalad.api as dl
+import datetime as dt
+
+
+def make_sql(path: Path) -> Path:
+    db = path / "backlogger.db"
+    cinit._create_db(db)
+    return db
+
+
+def make_config(path: Path) -> None:
+    cinit._write_config(path,  cinit._create_config(path, "datalad", False))
+
+
+def test_find_lookup_by_one_alias(tmp_path: Path) -> None:
+    make_config(tmp_path)
+    db = make_sql(tmp_path)
+    conn = sqlite3.connect(db)
+    c = conn.cursor()
+    uuid = "test_uuid"
+    alias_str = "fun_project"
+    tag_str = "tt"
+    owner = "tester"
+    code = "test_code"
+    c.execute("INSERT INTO projects (id, aliases, customTags, owner, code, created_at, updated_at) VALUES (?, ?, ?, ?, ?, datetime('now'), datetime('now'))",
+              (uuid, alias_str, tag_str, owner, code))
+    conn.commit()
+    assert uuid == find._project_lookup_by_alias(tmp_path, "fun_project")
+    uuid = "test_uuid2"
+    alias_str = "fun_project"
+    c.execute("INSERT INTO projects (id, aliases, customTags, owner, code, created_at, updated_at) VALUES (?, ?, ?, ?, ?, datetime('now'), datetime('now'))",
+              (uuid, alias_str, tag_str, owner, code))
+    conn.commit()
+    with pytest.raises(Exception):
+        assert uuid == find._project_lookup_by_alias(db, "fun_project")
+    conn.close()
+
+
+def test_find_lookup_by_id(tmp_path: Path) -> None:
+    make_config(tmp_path)
+    db = make_sql(tmp_path)
+    conn = sqlite3.connect(db)
+    c = conn.cursor()
+    uuid = "test_uuid"
+    alias_str = "fun_project"
+    tag_str = "tt"
+    owner = "tester"
+    code = "test_code"
+    c.execute("INSERT INTO projects (id, aliases, customTags, owner, code, created_at, updated_at) VALUES (?, ?, ?, ?, ?, datetime('now'), datetime('now'))",
+              (uuid, alias_str, tag_str, owner, code))
+    conn.commit()
+    conn.close()
+    result = find._project_lookup_by_id(tmp_path, uuid)[0]
+    assert uuid == result[0]
+    assert alias_str == result[1]
+    assert tag_str == result[2]
+    assert owner == result[3]
+    assert code == result[4]
+
+
+def test_time_filter() -> None:
+    record_A = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf0", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+            '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] # only created
+    record_B = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf1", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2025-03-26 12:55:18.229966', '2025-04-26 12:55:18.229966'] # created and updated
+    record_C = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf2", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2026-03-26 12:55:18.229966', '2026-04-14 12:55:18.229966'] # created and updated later
+    record_D = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf3", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2026-03-26 12:55:18.229966', '2026-03-27 12:55:18.229966']
+    record_E = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf4", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2024-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # only created, earlier
+    record_F = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf5", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2026-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # this is invalid...
+    record_G = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf2", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2026-03-26 12:55:18.229966', str(dt.datetime.now() + dt.timedelta(days=2, hours=3, minutes=5, seconds=30))] # created and updated later
+
+    data = [record_A, record_B, record_C, record_D, record_E]
+    cols = ["name",
+            "ensemble",
+            "code",
+            "path",
+            "project",
+            "parameters",
+            "parameter_file",
+            "created_at",
+            "updated_at"]
+    df = pd.DataFrame(data,columns=cols)
+
+    results = find._time_filter(df, created_before='2023-03-26 12:55:18.229966')
+    assert results.empty
+    results = find._time_filter(df, created_before='2027-03-26 12:55:18.229966')
+    assert len(results) == 5
+    results = find._time_filter(df, created_before='2026-03-25 12:55:18.229966')
+    assert len(results) == 3
+    results = find._time_filter(df, created_before='2026-03-26 12:55:18.229965')
+    assert len(results) == 3
+    results = find._time_filter(df, created_before='2025-03-04 12:55:18.229965')
+    assert len(results) == 1
+
+    results = find._time_filter(df, created_after='2023-03-26 12:55:18.229966')
+    assert len(results) == 5
+    results = find._time_filter(df, created_after='2027-03-26 12:55:18.229966')
+    assert results.empty
+    results = find._time_filter(df, created_after='2026-03-25 12:55:18.229966')
+    assert len(results) == 2
+    results = find._time_filter(df, created_after='2026-03-26 12:55:18.229965')
+    assert len(results) == 2
+    results = find._time_filter(df, created_after='2025-03-04 12:55:18.229965')
+    assert len(results) == 4
+
+    results = find._time_filter(df, updated_before='2023-03-26 12:55:18.229966')
+    assert results.empty
+    results = find._time_filter(df, updated_before='2027-03-26 12:55:18.229966')
+    assert len(results) == 5
+    results = find._time_filter(df, updated_before='2026-03-25 12:55:18.229966')
+    assert len(results) == 3
+    results = find._time_filter(df, updated_before='2026-03-26 12:55:18.229965')
+    assert len(results) == 3
+    results = find._time_filter(df, updated_before='2025-03-04 12:55:18.229965')
+    assert len(results) == 1
+
+    results = find._time_filter(df, updated_after='2023-03-26 12:55:18.229966')
+    assert len(results) == 5
+    results = find._time_filter(df, updated_after='2027-03-26 12:55:18.229966')
+    assert results.empty
+    results = find._time_filter(df, updated_after='2026-03-25 12:55:18.229966')
+    assert len(results) == 2
+    results = find._time_filter(df, updated_after='2026-03-26 12:55:18.229965')
+    assert len(results) == 2
+    results = find._time_filter(df, updated_after='2025-03-04 12:55:18.229965')
+    assert len(results) == 4
+
+    data = [record_A, record_B, record_C, record_D, record_F]
+    cols = ["name",
+            "ensemble",
+            "code",
+            "path",
+            "project",
+            "parameters",
+            "parameter_file",
+            "created_at",
+            "updated_at"]
+    df = pd.DataFrame(data,columns=cols)
+
+    with pytest.raises(ValueError):
+        results = find._time_filter(df, created_before='2023-03-26 12:55:18.229966')
+
+    data = [record_A, record_B, record_C, record_D, record_G]
+    cols = ["name",
+            "ensemble",
+            "code",
+            "path",
+            "project",
+            "parameters",
+            "parameter_file",
+            "created_at",
+            "updated_at"]
+    df = pd.DataFrame(data,columns=cols)
+
+    with pytest.raises(ValueError):
+        results = find._time_filter(df, created_before='2023-03-26 12:55:18.229966')
+
+
+def test_db_lookup(tmp_path: Path) -> None:
+    db = make_sql(tmp_path)
+    conn = sqlite3.connect(db)
+    c = conn.cursor()
+
+    corr = "f_A"
+    ensemble = "SF_A"
+    code = "openQCD"
+    meas_path = "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf"
+    uuid = "Project_A"
+    pars = "{par_A: 3.0, par_B: 5.0}"
+    parameter_file = "projects/Project_A/myinput.in"
+    c.execute("INSERT INTO backlogs (name, ensemble, code, path, project, parameters, parameter_file, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, datetime('now'), datetime('now'))",
+                                                (corr, ensemble, code, meas_path, uuid, pars, parameter_file))
+    conn.commit()
+
+    results = find._db_lookup(db, ensemble, corr, code)
+    assert len(results) == 1
+    results = find._db_lookup(db, "SF_B", corr, code)
+    assert results.empty
+    results = find._db_lookup(db, ensemble, "g_A", code)
+    assert results.empty
+    results = find._db_lookup(db, ensemble, corr, "sfcf")
+    assert results.empty
+    results = find._db_lookup(db, ensemble, corr, code, project = "Project_A")
+    assert len(results) == 1
+    results = find._db_lookup(db, ensemble, corr, code, project = "Project_B")
+    assert results.empty
+    results = find._db_lookup(db, ensemble, corr, code, parameters = pars)
+    assert len(results) == 1
+    results = find._db_lookup(db, ensemble, corr, code, parameters = '{"par_A": 3.0, "par_B": 4.0}')
+    assert results.empty
+
+    corr = "g_A"
+    ensemble = "SF_A"
+    code = "openQCD"
+    meas_path = "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf"
+    uuid = "Project_A"
+    pars = '{"par_A": 3.0, "par_B": 4.0}'
+    parameter_file = "projects/Project_A/myinput.in"
+    c.execute("INSERT INTO backlogs (name, ensemble, code, path, project, parameters, parameter_file, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, datetime('now'), datetime('now'))",
+                                                (corr, ensemble, code, meas_path, uuid, pars, parameter_file))
+    conn.commit()
+
+    corr = "f_A"
+    results = find._db_lookup(db, ensemble, corr, code)
+    assert len(results) == 1
+    results = find._db_lookup(db, "SF_B", corr, code)
+    assert results.empty
+    results = find._db_lookup(db, ensemble, "g_A", code)
+    assert len(results) == 1
+    results = find._db_lookup(db, ensemble, corr, "sfcf")
+    assert results.empty
+    results = find._db_lookup(db, ensemble, corr, code, project = "Project_A")
+    assert len(results) == 1
+    results = find._db_lookup(db, ensemble, "g_A", code, project = "Project_A")
+    assert len(results) == 1
+    results = find._db_lookup(db, ensemble, corr, code, project = "Project_B")
+    assert results.empty
+    results = find._db_lookup(db, ensemble, "g_A", code, project = "Project_B")
+    assert results.empty
+    results = find._db_lookup(db, ensemble, corr, code, parameters = pars)
+    assert results.empty
+    results = find._db_lookup(db, ensemble, "g_A", code, parameters = '{"par_A": 3.0, "par_B": 4.0}')
+    assert len(results) == 1
+
+    conn.close()
+
+
+def test_sfcf_drop() -> None:
+    parameters0 = {
+                    'offset': [0,0,0],
+                    'quarks': [{'mass': 1, 'thetas': [0,0,0]}, {'mass': 2, 'thetas': [0,0,1]}], # m0s = -3.5, -3.75
+                    'wf1': [[1, [0, 0]], [0.5, [1, 0]], [.75, [.5, .5]]],
+                    'wf2': [[1, [2, 1]], [2, [0.5, -0.5]], [.5, [.75, .72]]],
+                   }
+
+    assert not find._sfcf_drop(parameters0, offset=[0,0,0])
+    assert find._sfcf_drop(parameters0, offset=[1,0,0])
+
+    assert not find._sfcf_drop(parameters0, quark_kappas = [1, 2])
+    assert find._sfcf_drop(parameters0, quark_kappas = [-3.1, -3.72])
+
+    assert not find._sfcf_drop(parameters0, quark_masses = [-3.5, -3.75])
+    assert find._sfcf_drop(parameters0, quark_masses = [-3.1, -3.72])
+
+    assert not find._sfcf_drop(parameters0, qk1 = 1)
+    assert not find._sfcf_drop(parameters0, qk2 = 2)
+    assert find._sfcf_drop(parameters0, qk1 = 2)
+    assert find._sfcf_drop(parameters0, qk2 = 1)
+
+    assert not find._sfcf_drop(parameters0, qk1 = [0.5,1.5])
+    assert not find._sfcf_drop(parameters0, qk2 = [1.5,2.5])
+    assert find._sfcf_drop(parameters0, qk1 = 2)
+    assert find._sfcf_drop(parameters0, qk2 = 1)
+    with pytest.raises(ValueError):
+        assert not find._sfcf_drop(parameters0, qk1 = [0.5,1,5])
+    with pytest.raises(ValueError):
+        assert not find._sfcf_drop(parameters0, qk2 = [1,5,2.5])
+
+    assert find._sfcf_drop(parameters0, qm1 = 1.2)
+    assert find._sfcf_drop(parameters0, qm2 = 2.2)
+    assert not find._sfcf_drop(parameters0, qm1 = -3.5)
+    assert not find._sfcf_drop(parameters0, qm2 = -3.75)
+
+    assert find._sfcf_drop(parameters0, qm2 = 1.2)
+    assert find._sfcf_drop(parameters0, qm1 = 2.2)
+    with pytest.raises(ValueError):
+        assert not find._sfcf_drop(parameters0, qm1 = [0.5,1,5])
+    with pytest.raises(ValueError):
+        assert not find._sfcf_drop(parameters0, qm2 = [1,5,2.5])
+
+
+def test_openQCD_filter() -> None:
+    record_0 = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+            '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966']
+    record_1 = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+            '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966']
+    record_2 = ["f_P", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+            '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966']
+    record_3 = ["f_P", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+            '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966']
+    data = [
+        record_0,
+        record_1,
+        record_2,
+        record_3,
+    ]
+    cols = ["name",
+            "ensemble",
+            "code",
+            "path",
+            "project",
+            "parameters",
+            "parameter_file",
+            "created_at",
+            "updated_at"]
+    df = pd.DataFrame(data,columns=cols)
+
+    with pytest.warns(Warning):
+        find.openQCD_filter(df, a = "asdf")
+
+
+def test_code_filter() -> None:
+    record_0 = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+            '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966']
+    record_1 = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+            '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966']
+    record_2 = ["f_P", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+            '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966']
+    record_3 = ["f_P", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+            '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966']
+    record_4 = ["f_A", "ensA", "openQCD", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+            '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966']
+    record_5 = ["f_A", "ensA", "openQCD", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+            '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966']
+    record_6 = ["f_P", "ensA", "openQCD", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+            '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966']
+    record_7 = ["f_P", "ensA", "openQCD", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+            '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966']
+    record_8 = ["f_P", "ensA", "openQCD", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+            '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966']
+    data = [
+        record_0,
+        record_1,
+        record_2,
+        record_3,
+    ]
+    cols = ["name",
+            "ensemble",
+            "code",
+            "path",
+            "project",
+            "parameters",
+            "parameter_file",
+            "created_at",
+            "updated_at"]
+    df = pd.DataFrame(data,columns=cols)
+
+    res = find._code_filter(df, "sfcf")
+    assert len(res) == 4
+
+    data = [
+        record_4,
+        record_5,
+        record_6,
+        record_7,
+        record_8,
+    ]
+    cols = ["name",
+            "ensemble",
+            "code",
+            "path",
+            "project",
+            "parameters",
+            "parameter_file",
+            "created_at",
+            "updated_at"]
+    df = pd.DataFrame(data,columns=cols)
+
+    res = find._code_filter(df, "openQCD")
+    assert len(res) == 5
+    with pytest.raises(ValueError):
+        res = find._code_filter(df, "asdf")
+
+
+def test_find_record() -> None:
+    assert True
+
+
+def test_find_project(tmp_path: Path) -> None:
+    cinit.create(tmp_path)
+    db = tmp_path / "backlogger.db"
+    dl.unlock(str(db), dataset=str(tmp_path))
+    conn = sqlite3.connect(db)
+    c = conn.cursor()
+    uuid = "test_uuid"
+    alias_str = "fun_project"
+    tag_str = "tt"
+    owner = "tester"
+    code = "test_code"
+    c.execute("INSERT INTO projects (id, aliases, customTags, owner, code, created_at, updated_at) VALUES (?, ?, ?, ?, ?, datetime('now'), datetime('now'))",
+              (uuid, alias_str, tag_str, owner, code))
+    conn.commit()
+
+    assert uuid == find.find_project(tmp_path, "fun_project")
+
+    uuid = "test_uuid2"
+    alias_str = "fun_project"
+    c.execute("INSERT INTO projects (id, aliases, customTags, owner, code, created_at, updated_at) VALUES (?, ?, ?, ?, ?, datetime('now'), datetime('now'))",
+              (uuid, alias_str, tag_str, owner, code))
+    conn.commit()
+
+    with pytest.raises(Exception):
+        assert uuid == find._project_lookup_by_alias(tmp_path, "fun_project")
+    conn.close()
+
+
+def test_list_projects(tmp_path: Path) -> None:
+    cinit.create(tmp_path)
+    db = tmp_path / "backlogger.db"
+    dl.unlock(str(db), dataset=str(tmp_path))
+    conn = sqlite3.connect(db)
+    c = conn.cursor()
+    uuid = "test_uuid"
+    alias_str = "fun_project"
+    tag_str = "tt"
+    owner = "tester"
+    code = "test_code"
+
+    c.execute("INSERT INTO projects (id, aliases, customTags, owner, code, created_at, updated_at) VALUES (?, ?, ?, ?, ?, datetime('now'), datetime('now'))",
+              (uuid, alias_str, tag_str, owner, code))
+    uuid = "test_uuid2"
+    alias_str = "fun_project2"
+    c.execute("INSERT INTO projects (id, aliases, customTags, owner, code, created_at, updated_at) VALUES (?, ?, ?, ?, ?, datetime('now'), datetime('now'))",
+              (uuid, alias_str, tag_str, owner, code))
+    uuid = "test_uuid3"
+    alias_str = "fun_project3"
+    c.execute("INSERT INTO projects (id, aliases, customTags, owner, code, created_at, updated_at) VALUES (?, ?, ?, ?, ?, datetime('now'), datetime('now'))",
+              (uuid, alias_str, tag_str, owner, code))
+    uuid = "test_uuid4"
+    alias_str = "fun_project4"
+    c.execute("INSERT INTO projects (id, aliases, customTags, owner, code, created_at, updated_at) VALUES (?, ?, ?, ?, ?, datetime('now'), datetime('now'))",
+              (uuid, alias_str, tag_str, owner, code))
+    conn.commit()
+    conn.close()
+    results = find.list_projects(tmp_path)
+    assert len(results) == 4
+    for i in range(4):
+        assert len(results[i]) == 2

tests/import_project_test.py

@@ -1,7 +1,7 @@
 import corrlib.toml as t
 
 
-def test_toml_check_measurement_data():
+def test_toml_check_measurement_data() -> None:
     measurements = {
                     "a":
                         {
@@ -14,4 +14,4 @@ def test_toml_check_measurement_data():
                             "names": ['list', 'of', 'names']
                         }
                     }
-    t.check_measurement_data(measurements)
+    t.check_measurement_data(measurements, "sfcf")

tests/initialization_test.py

@@ -0,0 +1,94 @@
+import corrlib.initialization as init
+import os
+import sqlite3 as sql
+from pathlib import Path
+
+def test_init_folders(tmp_path: Path) -> None:
+    dataset_path = tmp_path / "test_dataset"
+    init.create(dataset_path)
+    assert os.path.exists(str(dataset_path))
+    assert os.path.exists(str(dataset_path / "backlogger.db"))
+
+
+def test_init_folders_no_tracker(tmp_path: Path) -> None:
+    dataset_path = tmp_path / "test_dataset"
+    init.create(dataset_path, tracker="None")
+    assert os.path.exists(str(dataset_path))
+    assert os.path.exists(str(dataset_path / "backlogger.db"))
+
+
+def test_init_config(tmp_path: Path) -> None:
+    dataset_path = tmp_path / "test_dataset"
+    init.create(dataset_path, tracker="None")
+    config_path = dataset_path / ".corrlib"
+    assert os.path.exists(str(config_path))
+    from configparser import ConfigParser
+    config = ConfigParser()
+    config.read(str(config_path))
+    assert config.get("core", "tracker") == "None"
+    assert config.get("core", "version") == "1.0"
+    assert config.get("core", "cached") == "True"
+    assert config.get("paths", "db") == "backlogger.db"
+    assert config.get("paths", "projects_path") == "projects"
+    assert config.get("paths", "archive_path") == "archive"
+    assert config.get("paths", "toml_imports_path") == "toml_imports"
+    assert config.get("paths", "import_scripts_path") == "import_scripts"
+
+
+def test_init_db(tmp_path: Path) -> None:
+    dataset_path = tmp_path / "test_dataset"
+    init.create(dataset_path)
+    assert os.path.exists(str(dataset_path / "backlogger.db"))
+    conn = sql.connect(str(dataset_path / "backlogger.db"))
+    cursor = conn.cursor()
+    cursor.execute("SELECT name FROM sqlite_master WHERE type='table';")
+    tables = cursor.fetchall()
+    expected_tables = [
+        'projects',
+        'backlogs',
+        ]
+    table_names = [table[0] for table in tables]
+    for expected_table in expected_tables:
+        assert expected_table in table_names
+
+    cursor.execute("SELECT * FROM projects;")
+    projects = cursor.fetchall()
+    assert len(projects) == 0
+
+    cursor.execute("SELECT * FROM backlogs;")
+    backlogs = cursor.fetchall()
+    assert len(backlogs) == 0
+
+    cursor.execute("PRAGMA table_info('projects');")
+    project_columns = cursor.fetchall()
+    expected_project_columns = [
+        "id",
+        "aliases",
+        "customTags",
+        "owner",
+        "code",
+        "created_at",
+        "updated_at"
+        ]
+    project_column_names = [col[1] for col in project_columns]
+    for expected_col in expected_project_columns:
+        assert expected_col in project_column_names
+
+    cursor.execute("PRAGMA table_info('backlogs');")
+    backlog_columns = cursor.fetchall()
+    expected_backlog_columns = [
+        "id",
+        "name",
+        "ensemble",
+        "code",
+        "path",
+        "project",
+        "customTags",
+        "parameters",
+        "parameter_file",
+        "created_at",
+        "updated_at"
+        ]
+    backlog_column_names = [col[1] for col in backlog_columns]
+    for expected_col in expected_backlog_columns:
+        assert expected_col in backlog_column_names

tests/integrity_test.py

@@ -0,0 +1,189 @@
+import corrlib.integrity as integ
+import corrlib.find as find
+import datalad.api as dl
+import corrlib.initialization as cinit
+import sqlite3
+from pathlib import Path
+import os
+import pandas as pd
+import datetime as dt
+import pytest
+
+
+def test_list_ensembles(tmp_path: Path) -> None:
+    """
+    Check against the implementation in find to check if they are the same.
+    """
+    os.mkdir(tmp_path / 'archive')
+    os.mkdir(tmp_path / 'archive' / 'A')
+    os.mkdir(tmp_path / 'archive' / 'B')
+    os.mkdir(tmp_path / 'archive' / 'C')
+    integ_results = integ._list_ensembles(tmp_path)
+    assert len(integ_results) == 3
+    find_results = find.list_ensembles(tmp_path)
+    assert len(find_results) == 3
+    for f,i in zip(find_results, integ_results):
+        assert f == i
+
+
+def test_list_projects(tmp_path: Path) -> None:
+    cinit.create(tmp_path)
+    db = tmp_path / "backlogger.db"
+    dl.unlock(str(db), dataset=str(tmp_path))
+    conn = sqlite3.connect(db)
+    c = conn.cursor()
+
+    customTags = ""
+    owner = "owner"
+    code = "sfcf"
+    created_at = "today"
+    updated_at = "today"
+
+    id = "asdf1"
+    aliases = "a1,s1,d1,f1"
+    c.execute("INSERT INTO projects (id, aliases, customTags, owner, code, created_at, updated_at) VALUES (?,?,?,?,?,?,?)", (id, aliases, customTags, owner, code , created_at, updated_at))
+    id = "asdf2"
+    aliases = "a2,s2,d2,f2"
+    c.execute("INSERT INTO projects (id, aliases, customTags, owner, code, created_at, updated_at) VALUES (?,?,?,?,?,?,?)", (id, aliases, customTags, owner, code , created_at, updated_at))
+    id = "asdf3"
+    aliases = "a3,s3,d3,f3"
+    c.execute("INSERT INTO projects (id, aliases, customTags, owner, code, created_at, updated_at) VALUES (?,?,?,?,?,?,?)", (id, aliases, customTags, owner, code , created_at, updated_at))
+    conn.commit()
+    conn.close
+    integ_results = integ._list_projects(tmp_path)
+    assert len(integ_results) == 3
+    find_results = find.list_projects(tmp_path)
+    assert len(find_results) == 3
+    for f,i in zip(find_results, integ_results):
+        assert f == i
+
+
+def test_has_valid_time() -> None:
+    record_A = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf0", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+            '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] # only created
+    record_B = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf1", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2025-03-26 12:55:18.229966', '2025-04-26 12:55:18.229966'] # created and updated
+    record_C = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf2", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2026-03-26 12:55:18.229966', '2026-04-14 12:55:18.229966'] # created and updated later
+    record_D = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf3", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2026-03-26 12:55:18.229966', '2026-03-27 12:55:18.229966']
+    record_E = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf4", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2024-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # only created, earlier
+    record_F = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf5", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2026-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # this is invalid...
+    record_G = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf2", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2026-03-26 12:55:18.229966', str(dt.datetime.now() + dt.timedelta(days=2, hours=3, minutes=5, seconds=30))] # created and updated later
+    cols = ["name",
+            "ensemble",
+            "code",
+            "path",
+            "project",
+            "parameters",
+            "parameter_file",
+            "created_at",
+            "updated_at"]
+    data = [record_A, record_B, record_C, record_D, record_E]
+
+    df = pd.DataFrame(data,columns=cols)
+    for _, result in df.iterrows():
+        assert integ.has_valid_times(result)
+    data = [record_F, record_G]
+    df = pd.DataFrame(data,columns=cols)
+    for _, result in df.iterrows():
+        assert not integ.has_valid_times(result)
+
+
+def test_are_keys_unique(tmp_path: Path) -> None:
+    db = tmp_path / 'test_success.db'
+
+    record_A = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf0", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+            '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] # only created
+    record_B = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf1", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2025-03-26 12:55:18.229966', '2025-04-26 12:55:18.229966'] # created and updated
+    record_C = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf2", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2026-03-26 12:55:18.229966', '2026-04-14 12:55:18.229966'] # created and updated later
+    record_D = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf3", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2026-03-26 12:55:18.229966', '2026-03-27 12:55:18.229966']
+    record_E = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf4", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2024-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # only created, earlier
+    record_F = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf5", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2026-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # this is invalid...
+    record_G = ["f_A", "ensA", "sfcf", "archive/SF_A/f_A/Project_A.json.gz::asdfasdfasdf2", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2026-03-26 12:55:18.229966', str(dt.datetime.now() + dt.timedelta(days=2, hours=3, minutes=5, seconds=30))] # created and updated later
+
+    cols = ["name",
+            "ensemble",
+            "code",
+            "path",
+            "project",
+            "parameters",
+            "parameter_file",
+            "created_at",
+            "updated_at"]
+
+    data = [record_A, record_B, record_C, record_D, record_E, record_F]
+    df = pd.DataFrame(data,columns=cols)
+    conn = sqlite3.connect(db)
+    df.to_sql('backlogs', conn)
+    conn.close()
+    assert integ.are_keys_unique(db, 'backlogs', 'path')
+
+    db = tmp_path / 'test_fail.db'
+    data = [record_A, record_B, record_C, record_D, record_E, record_F, record_G]
+
+    df = pd.DataFrame(data,columns=cols)
+    conn = sqlite3.connect(db)
+    df.to_sql('backlogs', conn)
+    conn.close()
+    assert not integ.are_keys_unique(db, 'backlogs', 'path')
+
+
+def test_check_path_format() -> None:
+    record_A = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_A.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+            '2025-03-26 12:55:18.229966', '2025-03-26 12:55:18.229966'] # only created
+    record_B = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_B.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2025-03-26 12:55:18.229966', '2025-04-26 12:55:18.229966'] # created and updated
+    record_C = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_A.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2026-03-26 12:55:18.229966', '2026-04-14 12:55:18.229966'] # created and updated later
+    record_D = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_B.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2026-03-26 12:55:18.229966', '2026-03-27 12:55:18.229966']
+    record_E = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_A.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2024-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # only created, earlier
+    record_F = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_B.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdf", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2026-03-26 12:55:18.229966', '2024-03-26 12:55:18.229966'] # this is invalid...
+    record_G = ["f_A", "ensA", "sfcf", "archive/ensA/f_A/Project_A.json.gz::asdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfasdfas", "SF_A", '{"par_A": 5.0, "par_B": 5.0}', "projects/SF_A/input.in",
+                '2026-03-26 12:55:18.229966', str(dt.datetime.now() + dt.timedelta(days=2, hours=3, minutes=5, seconds=30))] # created and updated later
+
+    projects = ['Project_A', 'Project_B']
+    ensembles = ['ensA']
+    cols = ["name",
+            "ensemble",
+            "code",
+            "path",
+            "project",
+            "parameters",
+            "parameter_file",
+            "created_at",
+            "updated_at"]
+
+    data = [record_A, record_B, record_C, record_D, record_E, record_F]
+    df = pd.DataFrame(data,columns=cols)
+    for _, result in df.iterrows():
+        integ.check_path_format(result, ensembles, projects)
+
+    projects = ['Project_A', 'Project_B']
+    ensembles = ['ensB']
+    for _, result in df.iterrows():
+        with pytest.raises(ValueError):
+            integ.check_path_format(result, ensembles, projects)
+
+    projects = ['Project_A', 'Project_B']
+    ensembles = ['ensA', 'ensB']
+    for _, result in df.iterrows():
+        integ.check_path_format(result, ensembles, projects)
+
+    data = [record_G]
+    df = pd.DataFrame(data,columns=cols)
+    for _, result in df.iterrows():
+        with pytest.raises(ValueError):
+            integ.check_path_format(result, ensembles, projects)

tests/sfcf_in_test.py

@@ -1,7 +1,7 @@
 import corrlib.input.sfcf as input
 import json
 
-def test_get_specs():
+def test_get_specs() -> None:
     parameters = {
         'crr': [
             'f_P', 'f_A'
@@ -26,4 +26,4 @@ def test_get_specs():
     key = "f_P/q1 q2/1/0/0"
     specs = json.loads(input.get_specs(key, parameters))
     assert specs['quarks'] == ['a', 'b']
-    assert specs['wf1'][0] == [1, [0, 0]]
+    assert specs['wf1'][0] == [1, [0, 0]]

tests/tools_test.py

@@ -0,0 +1,88 @@
+from corrlib import tools as tl
+from configparser import ConfigParser
+from pathlib import Path
+import pytest
+
+
+def test_m2k() -> None:
+    for m in [0.1, 0.5, 1.0]:
+        expected_k = 1 / (2 * m + 8)
+        assert tl.m2k(m) == expected_k
+
+
+def test_k2m() -> None:
+    for m in [0.1, 0.5, 1.0]:
+        assert tl.k2m(m) == (1/(2*m))-4
+
+
+def test_k2m_m2k() -> None:
+    for m in [0.1, 0.5, 1.0]:
+        k = tl.m2k(m)
+        m_converted = tl.k2m(k)
+        assert abs(m - m_converted) < 1e-9
+
+
+def test_str2list() -> None:
+    assert tl.str2list("a,b,c") == ["a", "b", "c"]
+    assert tl.str2list("1,2,3") == ["1", "2", "3"]
+
+
+def test_list2str() -> None:
+    assert tl.list2str(["a", "b", "c"]) == "a,b,c"
+    assert tl.list2str(["1", "2", "3"]) == "1,2,3"
+
+
+def test_set_config(tmp_path: Path) -> None:
+    section = "core"
+    option = "test_option"
+    value = "test_value"
+    # config is not yet available
+    tl.set_config(tmp_path, section, option, value)
+    config_path = tmp_path / '.corrlib'
+    config = ConfigParser()
+    config.read(config_path)
+    assert config.get('core', 'test_option', fallback="not the value") == "test_value"
+    # now, a config file is already present
+    section = "core"
+    option = "test_option2"
+    value = "test_value2"
+    tl.set_config(tmp_path, section, option, value)
+    config_path = tmp_path / '.corrlib'
+    config = ConfigParser()
+    config.read(config_path)
+    assert config.get('core', 'test_option2', fallback="not the value") == "test_value2"
+    # update option 2
+    section = "core"
+    option = "test_option2"
+    value = "test_value3"
+    tl.set_config(tmp_path, section, option, value)
+    config_path = tmp_path / '.corrlib'
+    config = ConfigParser()
+    config.read(config_path)
+    assert config.get('core', 'test_option2', fallback="not the value") == "test_value3"
+
+
+def test_get_db_file(tmp_path: Path) -> None:
+    section = "paths"
+    option = "db"
+    value = "test_value"
+    # config is not yet available
+    tl.set_config(tmp_path, section, option, value)
+    assert tl.get_db_file(tmp_path) == Path("test_value")
+    with pytest.raises(FileNotFoundError):
+        tl.get_db_file(tmp_path / "doesnotexist")
+
+
+def test_cache_enabled(tmp_path: Path) -> None:
+    section = "core"
+    option = "cached"
+    # config is not yet available
+    tl.set_config(tmp_path, section, option, "True")
+    assert tl.cache_enabled(tmp_path)
+    tl.set_config(tmp_path, section, option, "False")
+    assert not tl.cache_enabled(tmp_path)
+    tl.set_config(tmp_path, section, option, "lalala")
+    with pytest.raises(ValueError):
+        tl.cache_enabled(tmp_path)
+    with pytest.raises(FileNotFoundError):
+        tl.cache_enabled(tmp_path / "doesnotexist")