.github/workflows/mypy.yaml

@@ -1,30 +0,0 @@
-name: Mypy
-
-on:
-  push:
-  pull_request:
-  workflow_dispatch:
-
-jobs:
-  mypy:
-    runs-on: ubuntu-latest
-    env:
-      UV_CACHE_DIR: /tmp/.uv-cache
-    steps:
-      - name: Install git-annex
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y git-annex
-      - name: Check out the repository
-        uses: https://github.com/RouxAntoine/checkout@v4.1.8
-        with:
-          show-progress: true
-      - name: Install uv
-        uses: astral-sh/setup-uv@v7
-        with:
-          python-version: ${{ matrix.python-version }}
-          enable-cache: true
-      - name: Install corrlib
-        run: uv sync --locked --all-extras --dev --python "3.12"
-      - name: Run tests
-        run: uv run mypy corrlib

.github/workflows/pytest.yaml

@@ -1,39 +0,0 @@
-name: Pytest
-
-on:
-  push:
-  pull_request:
-  workflow_dispatch:
-  schedule:
-    - cron: '0 4 1 * *'
-
-jobs:
-  pytest:
-    strategy:
-      matrix:
-        python-version:
-          - "3.12"
-          - "3.13"
-          - "3.14"
-
-    runs-on: ubuntu-latest
-    env:
-      UV_CACHE_DIR: /tmp/.uv-cache
-    steps:
-      - name: Install git-annex
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y git-annex
-      - name: Check out the repository
-        uses: https://github.com/RouxAntoine/checkout@v4.1.8
-        with:
-          show-progress: true
-      - name: Install uv
-        uses: astral-sh/setup-uv@v7
-        with:
-          python-version: ${{ matrix.python-version }}
-          enable-cache: true
-      - name: Install corrlib
-        run: uv sync --locked --all-extras --dev --python ${{ matrix.python-version }}
-      - name: Run tests
-        run: uv run pytest --cov=corrlib tests

.github/workflows/ruff.yaml

@@ -1,30 +0,0 @@
-name: Ruff
-
-on:
-  push:
-  pull_request:
-  workflow_dispatch:
-
-jobs:
-  ruff:
-
-    runs-on: ubuntu-latest
-    env:
-      UV_CACHE_DIR: /tmp/.uv-cache
-    steps:
-      - name: Install git-annex
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y git-annex
-      - name: Check out the repository
-        uses: https://github.com/RouxAntoine/checkout@v4.1.8
-        with:
-          show-progress: true
-      - name: Install uv
-        uses: astral-sh/setup-uv@v7
-        with:
-          enable-cache: true
-      - name: Install corrlib
-        run: uv sync --locked --all-extras --dev --python "3.12"
-      - name: Run tests
-        run: uv run ruff check corrlib

.gitignore

@@ -2,7 +2,3 @@ pyerrors_corrlib.egg-info
 __pycache__
 *.egg-info
 test.ipynb
-.vscode
-.venv
-.pytest_cache
-.coverage

.gitmodules

@@ -0,0 +1,5 @@
+[submodule "projects/tmp"]
+	path = projects/tmp
+	url = git@kuhl-mann.de:lattice/charm_SF_data.git
+	datalad-id = 5f402163-77f2-470e-b6f1-64d7bf9f87d4
+	datalad-url = git@kuhl-mann.de:lattice/charm_SF_data.git

corrlib/__init__.py

@@ -15,11 +15,9 @@ For now, we are interested in collecting primary IObservables only, as these are
 
 __app_name__ = "corrlib"
 
+from .main import *
 from .import input as input
-from .initialization import create as create
+from .initialization import *
-from .meas_io import load_record as load_record
+from .meas_io import *
-from .meas_io import load_records as load_records
+from .find import *
-from .find import find_project as find_project
+from .version import __version__
-from .find import find_record as find_record
-from .find import list_projects as list_projects
-from .config import *

corrlib/__main__.py

@@ -1,9 +1,8 @@
 from corrlib import cli, __app_name__
 
 
-def main() -> None:
+def main():
     cli.app(prog_name=__app_name__)
-    return
 
 
 if __name__ == "__main__":

corrlib/cli.py

@@ -1,6 +1,6 @@
 from typing import Optional
 import typer
-from corrlib import __app_name__
+from corrlib import __app_name__, __version__
 from .initialization import create
 from .toml import import_tomls, update_project, reimport_project
 from .find import find_record, list_projects
@@ -8,7 +8,6 @@ from .tools import str2list
 from .main import update_aliases
 from .meas_io import drop_cache as mio_drop_cache
 import os
-from importlib.metadata import version
 
 
 app = typer.Typer()
@@ -16,7 +15,7 @@ app = typer.Typer()
 
 def _version_callback(value: bool) -> None:
     if value:
-        print(__app_name__, version(__app_name__))
+        typer.echo(f"{__app_name__} v{__version__}")
         raise typer.Exit()
 
 

corrlib/find.py

@@ -1,30 +1,29 @@
 import sqlite3
+import datalad.api as dl
 import os
 import json
 import pandas as pd
 import numpy as np
 from .input.implementations import codes
-from .tools import k2m
+from .tools import k2m, get_file
-from .tracker import get
-from typing import Any, Optional
 # this will implement the search functionality
 
 
-def _project_lookup_by_alias(db: str, alias: str) -> str:
+def _project_lookup_by_alias(db, alias):
     # this will lookup the project name based on the alias
     conn = sqlite3.connect(db)
     c = conn.cursor()
     c.execute(f"SELECT * FROM 'projects' WHERE alias = '{alias}'")
     results = c.fetchall()
     conn.close()
-    if len(results)>1:
+    if len(results) > 1:
         print("Error: multiple projects found with alias " + alias)
     elif len(results) == 0:
         raise Exception("Error: no project found with alias " + alias)
-    return str(results[0][0])
+    return results[0][0]
 
 
-def _project_lookup_by_id(db: str, uuid: str) -> list[tuple[str, str]]:
+def _project_lookup_by_id(db, uuid):
     conn = sqlite3.connect(db)
     c = conn.cursor()
     c.execute(f"SELECT * FROM 'projects' WHERE id = '{uuid}'")
@@ -33,8 +32,7 @@ def _project_lookup_by_id(db: str, uuid: str) -> list[tuple[str, str]]:
     return results
 
 
-def _db_lookup(db: str, ensemble: str, correlator_name: str, code: str, project: Optional[str]=None,  parameters: Optional[str]=None,
+def _db_lookup(db, ensemble, correlator_name,code, project=None,  parameters=None, created_before=None, created_after=None, updated_before=None, updated_after=None, revision=None):
-               created_before: Optional[str]=None, created_after: Optional[Any]=None, updated_before: Optional[Any]=None, updated_after: Optional[Any]=None) -> pd.DataFrame:
     project_str = project
 
     search_expr = f"SELECT * FROM 'backlogs' WHERE name = '{correlator_name}' AND ensemble = '{ensemble}'"
@@ -58,7 +56,7 @@ def _db_lookup(db: str, ensemble: str, correlator_name: str, code: str, project:
     return results
 
 
-def sfcf_filter(results: pd.DataFrame, **kwargs: Any) -> pd.DataFrame:
+def sfcf_filter(results, **kwargs):
     drops = []
     for ind in range(len(results)):
         result = results.iloc[ind]
@@ -141,27 +139,26 @@ def sfcf_filter(results: pd.DataFrame, **kwargs: Any) -> pd.DataFrame:
     return results.drop(drops)
 
 
-def find_record(path: str, ensemble: str, correlator_name: str, code: str, project: Optional[str]=None, parameters: Optional[str]=None,
+def find_record(path, ensemble, correlator_name, code, project=None, parameters=None, created_before=None, created_after=None, updated_before=None, updated_after=None, revision=None, **kwargs):
-                created_before: Optional[str]=None, created_after: Optional[str]=None, updated_before: Optional[str]=None, updated_after: Optional[str]=None, revision: Optional[str]=None, **kwargs: Any) -> pd.DataFrame:
     db = path + '/backlogger.db'
     if code not in codes:
         raise ValueError("Code " + code + "unknown, take one of the following:" + ", ".join(codes))
-    get(path, "backlogger.db")
+    get_file(path, "backlogger.db")
-    results = _db_lookup(db, ensemble, correlator_name,code, project, parameters=parameters, created_before=created_before, created_after=created_after, updated_before=updated_before, updated_after=updated_after)
+    results = _db_lookup(db, ensemble, correlator_name,code, project, parameters=parameters, created_before=created_before, created_after=created_after, updated_before=updated_before, updated_after=updated_after, revision=revision)
     if code == "sfcf":
         results = sfcf_filter(results, **kwargs)
     print("Found " + str(len(results)) + " result" + ("s" if len(results)>1 else ""))
     return results.reset_index()
 
 
-def find_project(path: str, name: str) -> str:
+def find_project(path, name):
-    get(path, "backlogger.db")
+    get_file(path, "backlogger.db")
     return _project_lookup_by_alias(os.path.join(path, "backlogger.db"), name)
 
 
-def list_projects(path: str) -> list[tuple[str, str]]:
+def list_projects(path):
     db = path + '/backlogger.db'
-    get(path, "backlogger.db")
+    get_file(path, "backlogger.db")
     conn = sqlite3.connect(db)
     c = conn.cursor()
     c.execute("SELECT id,aliases FROM projects")

corrlib/git_tools.py

@@ -5,7 +5,7 @@ import git
 GITMODULES_FILE = '.gitmodules'
 
 
-def move_submodule(repo_path: str, old_path: str, new_path: str) -> None:
+def move_submodule(repo_path, old_path, new_path):
     """
     Move a submodule to a new location.
 
@@ -41,4 +41,3 @@ def move_submodule(repo_path: str, old_path: str, new_path: str) -> None:
     repo.git.add('.gitmodules')
     # save new state of the dataset
     dl.save(repo_path, message=f"Move module from {old_path} to {new_path}", dataset=repo_path)
-    return

corrlib/initialization.py

@@ -1,11 +1,9 @@
-from configparser import ConfigParser
 import sqlite3
 import datalad.api as dl
 import os
-import tracker as tr
 
 
-def _create_db(db: str) -> None:
+def _create_db(db):
     """
     Create the database file and the table.
 
@@ -34,65 +32,21 @@ def _create_db(db: str) -> None:
                   updated_at TEXT)''')
     conn.commit()
     conn.close()
-    return
 
 
-def _create_config(path):
+def create(path):
-    """
-    Create the config file for backlogger.
-
-    """
-    config = ConfigParser()
-    config['core'] = {
-        'version': '1.0',
-        'db_path': os.path.join(path, 'backlogger.db'),
-        'projects_path': os.path.join(path, 'projects'),
-        'archive_path': os.path.join(path, 'archive'),
-        'toml_imports_path': os.path.join(path, 'toml_imports'),
-        'import_scripts_path': os.path.join(path, 'import_scripts'),
-        'tracker': 'datalad',
-        'cached': True,
-    }
-    with open(os.path.join(path, '.corrlib'), 'w') as configfile:
-        config.write(configfile)
-
-
-def _create_config(path: str) -> None:
-    """
-    Create the config file for backlogger.
-
-    """
-    config = ConfigParser()
-    config['core'] = {
-        'version': '1.0',
-        'db_path': os.path.join(path, 'backlogger.db'),
-        'projects_path': os.path.join(path, 'projects'),
-        'archive_path': os.path.join(path, 'archive'),
-        'toml_imports_path': os.path.join(path, 'toml_imports'),
-        'import_scripts_path': os.path.join(path, 'import_scripts'),
-        'tracker': 'datalad',
-        'cached': True,
-    }
-    with open(os.path.join(path, '.corrlib'), 'w') as configfile:
-        config.write(configfile)
-    return
-
-
-def create(path: str) -> None:
     """
     Create folder of backlogs.
 
     """
     dl.create(path)
-    _create_db(os.path.join(path, 'backlogger.db'))
+    _create_db(path + '/backlogger.db')
-    os.chmod(os.path.join(path, 'backlogger.db'), 0o666)  # why does this not work?
+    os.chmod(path + '/backlogger.db', 0o666)  # why does this not work?
-    _create_config(path)
+    os.makedirs(path + '/projects')
-    os.makedirs(os.path.join(path, 'projects'))
+    os.makedirs(path + '/archive')
-    os.makedirs(os.path.join(path, 'archive'))
+    os.makedirs(path + '/toml_imports')
-    os.makedirs(os.path.join(path, 'toml_imports'))
+    os.makedirs(path + '/import_scripts/template.py')
-    os.makedirs(os.path.join(path, 'import_scripts/template.py'))
+    with open(path + "/.gitignore", "w") as fp:
-    with open(os.path.join(path, ".gitignore"), "w") as fp:
         fp.write(".cache")
         fp.close()
-    tr.save(path, message="Initialized correlator library", dataset=path)
+    dl.save(path, dataset=path, message="Initialize backlogger directory.")
-    return

corrlib/input/__init__.py

@@ -2,6 +2,6 @@
 Import functions for different codes.
 """
 
-from . import sfcf as sfcf
+from . import sfcf
-from . import openQCD as openQCD
+from . import openQCD
-from . import implementations as implementations
+from . import implementations

corrlib/input/openQCD.py

@@ -2,7 +2,7 @@ import pyerrors.input.openQCD as input
 import datalad.api as dl
 import os
 import fnmatch
-from typing import Any, Optional
+from typing import Any
 
 
 def read_ms1_param(path: str, project: str, file_in_project: str) -> dict[str, Any]:
@@ -67,7 +67,7 @@ def read_ms3_param(path: str, project: str, file_in_project: str) -> dict[str, A
     return param
 
 
-def read_rwms(path: str, project: str, dir_in_project: str, param: dict[str, Any], prefix: str, postfix: str="ms1", version: str='2.0', names: Optional[list[str]]=None, files: Optional[list[str]]=None) -> dict[str, Any]:
+def read_rwms(path: str, project: str, dir_in_project: str, param: dict[str, Any], prefix: str, postfix: str="ms1", version: str='2.0', names: list[str]=None, files: list[str]=None) -> dict[str, Any]:
     dataset = os.path.join(path, "projects", project)
     directory = os.path.join(dataset, dir_in_project)
     if files is None:
@@ -94,7 +94,7 @@ def read_rwms(path: str, project: str, dir_in_project: str, param: dict[str, Any
     return rw_dict
 
 
-def extract_t0(path: str, project: str, dir_in_project: str, param: dict[str, Any], prefix: str, dtr_read: int, xmin: int, spatial_extent: int, fit_range: int = 5, postfix: str="", names: Optional[list[str]]=None, files: Optional[list[str]]=None) -> dict[str, Any]:
+def extract_t0(path: str, project: str, dir_in_project: str, param: dict[str, Any], prefix: str, dtr_read: int, xmin: int, spatial_extent: int, fit_range: int = 5, postfix: str=None, names: list[str]=None, files: list[str]=None) -> dict[str, Any]:
     dataset = os.path.join(path, "projects", project)
     directory = os.path.join(dataset, dir_in_project)
     if files is None:
@@ -132,7 +132,7 @@ def extract_t0(path: str, project: str, dir_in_project: str, param: dict[str, An
     return t0_dict
 
 
-def extract_t1(path: str, project: str, dir_in_project: str, param: dict[str, Any], prefix: str, dtr_read: int, xmin: int, spatial_extent: int, fit_range: int = 5, postfix: str = "", names: Optional[list[str]]=None, files: Optional[list[str]]=None) -> dict[str, Any]:
+def extract_t1(path: str, project: str, dir_in_project: str, param: dict[str, Any], prefix: str, dtr_read: int, xmin: int, spatial_extent: int, fit_range: int = 5, postfix: str = None, names: list[str]=None, files: list[str]=None) -> dict[str, Any]:
     directory = os.path.join(path, "projects", project, dir_in_project)
     if files is None:
         files = []

corrlib/input/sfcf.py

@@ -5,7 +5,7 @@ import os
 from typing import Any
 
 
-bi_corrs: list[str] = ["f_P", "fP", "f_p",
+bi_corrs: list = ["f_P", "fP", "f_p",
             "g_P", "gP", "g_p",
             "fA0", "f_A", "f_a",
             "gA0", "g_A", "g_a",
@@ -43,7 +43,7 @@ bi_corrs: list[str] = ["f_P", "fP", "f_p",
             "l3A2", "l3_A2", "g_av23",
             ]
 
-bb_corrs: list[str] = [
+bb_corrs: list = [
             'F1',
             'F_1',
             'f_1',
@@ -64,7 +64,7 @@ bb_corrs: list[str] = [
             'F_sPdP_d',
             ]
 
-bib_corrs: list[str] = [
+bib_corrs: list = [
             'F_V0',
             'K_V0',
             ]
@@ -184,7 +184,7 @@ def read_param(path: str, project: str, file_in_project: str) -> dict[str, Any]:
     return params
 
 
-def _map_params(params: dict[str, Any], spec_list: list[str]) -> dict[str, Any]:
+def _map_params(params: dict, spec_list: list) -> dict[str, Any]:
     """
     Map the extracted parameters to the extracted data.
 
@@ -228,7 +228,7 @@ def _map_params(params: dict[str, Any], spec_list: list[str]) -> dict[str, Any]:
     return new_specs
 
 
-def get_specs(key: str, parameters: dict[str, Any], sep: str = '/') -> str:
+def get_specs(key, parameters, sep='/') -> str:
     key_parts = key.split(sep)
     if corr_types[key_parts[0]] == 'bi':
         param = _map_params(parameters, key_parts[1:-1])
@@ -238,7 +238,7 @@ def get_specs(key: str, parameters: dict[str, Any], sep: str = '/') -> str:
     return s
 
 
-def read_data(path: str, project: str, dir_in_project: str, prefix: str, param: dict[str, Any], version: str = '1.0c', cfg_seperator: str = 'n', sep: str = '/', **kwargs: Any) -> dict[str, Any]:
+def read_data(path, project, dir_in_project, prefix, param, version='1.0c', cfg_seperator='n', sep='/', **kwargs) -> dict:
     """
     Extract the data from the sfcf file.
 

corrlib/main.py

@@ -5,12 +5,11 @@ import os
 from .git_tools import move_submodule
 import shutil
 from .find import _project_lookup_by_id
-from .tools import list2str, str2list
+from .tools import list2str, str2list, get_file
-from .tracker import get_file
+from typing import Union
-from typing import Union, Optional
 
 
-def create_project(path: str, uuid: str, owner: Union[str, None]=None, tags: Union[list[str], None]=None, aliases: Union[list[str], None]=None, code: Union[str, None]=None) -> None:
+def create_project(path: str, uuid: str, owner: Union[str, None]=None, tags: Union[str, None]=None, aliases: Union[str, None]=None, code: Union[str, None]=None):
     """
     Create a new project entry in the database.
 
@@ -34,10 +33,10 @@ def create_project(path: str, uuid: str, owner: Union[str, None]=None, tags: Uni
         raise ValueError("Project already imported, use update_project() instead.")
 
     dl.unlock(db, dataset=path)
-    alias_str = ""
+    alias_str = None
     if aliases is not None:
         alias_str = list2str(aliases)
-    tag_str = ""
+    tag_str = None
     if tags is not None:
         tag_str = list2str(tags)
     c.execute("INSERT INTO projects (id, aliases, customTags, owner, code, created_at, updated_at) VALUES (?, ?, ?, ?, ?, datetime('now'), datetime('now'))", (uuid, alias_str, tag_str, owner, code))
@@ -46,7 +45,7 @@ def create_project(path: str, uuid: str, owner: Union[str, None]=None, tags: Uni
     dl.save(db, message="Added entry for project " + uuid + " to database", dataset=path)
 
 
-def update_project_data(path: str, uuid: str, prop: str, value: Union[str, None] = None) -> None:
+def update_project_data(path, uuid, prop, value = None):
     get_file(path, "backlogger.db")
     conn = sqlite3.connect(os.path.join(path, "backlogger.db"))
     c = conn.cursor()
@@ -56,7 +55,7 @@ def update_project_data(path: str, uuid: str, prop: str, value: Union[str, None]
     return
 
 
-def update_aliases(path: str, uuid: str, aliases: list[str]) -> None:
+def update_aliases(path: str, uuid: str, aliases: list[str]):
     db = os.path.join(path, "backlogger.db")
     get_file(path, "backlogger.db")
     known_data = _project_lookup_by_id(db, uuid)[0]
@@ -83,7 +82,7 @@ def update_aliases(path: str, uuid: str, aliases: list[str]) -> None:
     return
 
 
-def import_project(path: str, url: str, owner: Union[str, None]=None, tags: Optional[list[str]]=None, aliases: Optional[list[str]]=None, code: Optional[str]=None, isDataset: bool=True) -> str:
+def import_project(path: str, url: str, owner: Union[str, None]=None, tags: Union[str, None]=None, aliases: Union[str, None]=None, code: Union[str, None]=None, isDataset: bool=True):
     """
     Parameters
     ----------
@@ -118,7 +117,7 @@ def import_project(path: str, url: str, owner: Union[str, None]=None, tags: Opti
         dl.install(path=tmp_path, source=url, dataset=path)
     tmp_ds = dl.Dataset(tmp_path)
     conf = dlc.ConfigManager(tmp_ds)
-    uuid = str(conf.get("datalad.dataset.id"))
+    uuid = conf.get("datalad.dataset.id")
     if not uuid:
         raise ValueError("The dataset does not have a uuid!")
     if not os.path.exists(path + "/projects/" + uuid):
@@ -143,10 +142,9 @@ def import_project(path: str, url: str, owner: Union[str, None]=None, tags: Opti
     return uuid
 
 
-def drop_project_data(path: str, uuid: str, path_in_project: str = "") -> None:
+def drop_project_data(path: str, uuid: str, path_in_project: str = ""):
     """
     Drop (parts of) a prject to free up diskspace
     """
     dl.drop(path + "/projects/" + uuid + "/" + path_in_project)
-    return
 

corrlib/meas_io.py

@@ -7,13 +7,11 @@ import json
 from typing import Union
 from pyerrors import Obs, Corr, dump_object, load_object
 from hashlib import sha256
-from .tools import cached
+from .tools import cached, get_file
-from .tracker import get
 import shutil
-from typing import Any
 
 
-def write_measurement(path: str, ensemble: str, measurement: dict[str, dict[str, dict[str, Any]]], uuid: str, code: str, parameter_file: str) -> None:
+def write_measurement(path, ensemble, measurement, uuid, code, parameter_file=None):
     """
     Write a measurement to the backlog.
     If the file for the measurement already exists, update the measurement.
@@ -30,7 +28,7 @@ def write_measurement(path: str, ensemble: str, measurement: dict[str, dict[str,
         The uuid of the project.
     """
     db = os.path.join(path, 'backlogger.db')
-    get(path, "backlogger.db")
+    get_file(path, "backlogger.db")
     dl.unlock(db, dataset=path)
     conn = sqlite3.connect(db)
     c = conn.cursor()
@@ -99,7 +97,7 @@ def write_measurement(path: str, ensemble: str, measurement: dict[str, dict[str,
     dl.save(files, message="Add measurements to database", dataset=path)
 
 
-def load_record(path: str, meas_path: str) -> Union[Corr, Obs]:
+def load_record(path: str, meas_path: str):
     """
     Load a list of records by their paths.
 
@@ -118,7 +116,7 @@ def load_record(path: str, meas_path: str) -> Union[Corr, Obs]:
     return load_records(path, [meas_path])[0]
 
 
-def load_records(path: str, meas_paths: list[str], preloaded: dict[str, Any] = {}) -> list[Union[Corr, Obs]]:
+def load_records(path: str, meas_paths: list[str], preloaded = {}) -> list[Union[Corr, Obs]]:
     """
     Load a list of records by their paths.
 
@@ -140,7 +138,7 @@ def load_records(path: str, meas_paths: list[str], preloaded: dict[str, Any] = {
             needed_data[file] = []
         key = mpath.split("::")[1]
         needed_data[file].append(key)
-    returned_data: list[Any] = []
+    returned_data: list = []
     for file in needed_data.keys():
         for key in list(needed_data[file]):
             if os.path.exists(cache_path(path, file, key) + ".p"):
@@ -156,7 +154,7 @@ def load_records(path: str, meas_paths: list[str], preloaded: dict[str, Any] = {
     return returned_data
 
 
-def cache_dir(path: str, file: str) -> str:
+def cache_dir(path, file):
     cache_path_list = [path]
     cache_path_list.append(".cache")
     cache_path_list.extend(file.split("/")[1:])
@@ -164,23 +162,23 @@ def cache_dir(path: str, file: str) -> str:
     return cache_path
 
 
-def cache_path(path: str, file: str, key: str) -> str:
+def cache_path(path, file, key):
     cache_path = os.path.join(cache_dir(path, file), key)
     return cache_path
 
 
-def preload(path: str, file: str) -> dict[str, Any]:
+def preload(path: str, file: str):
-    get(path, file)
+    get_file(path, file)
-    filedict: dict[str, Any] = pj.load_json_dict(os.path.join(path, file))
+    filedict = pj.load_json_dict(os.path.join(path, file))
     print("> read file")
     return filedict
 
 
-def drop_record(path: str, meas_path: str) -> None:
+def drop_record(path: str, meas_path: str):
     file_in_archive = meas_path.split("::")[0]
     file = os.path.join(path, file_in_archive)
     db = os.path.join(path, 'backlogger.db')
-    get(path, 'backlogger.db')
+    get_file(path, 'backlogger.db')
     sub_key = meas_path.split("::")[1]
     dl.unlock(db, dataset=path)
     conn = sqlite3.connect(db)
@@ -201,9 +199,7 @@ def drop_record(path: str, meas_path: str) -> None:
     else:
         raise ValueError("This measurement does not exist as a file!")
 
-
+def drop_cache(path: str):
-def drop_cache(path: str) -> None:
     cache_dir = os.path.join(path, ".cache")
     for f in os.listdir(cache_dir):
         shutil.rmtree(os.path.join(cache_dir, f))
-    return

corrlib/toml.py

@@ -16,16 +16,15 @@ from .meas_io import write_measurement
 import datalad.api as dl
 import os
 from .input.implementations import codes as known_codes
-from typing import Any
 
-def replace_string(string: str, name: str, val: str) -> str:
+def replace_string(string: str, name: str, val: str):
     if '{' + name + '}' in string:
         n = string.replace('{' + name + '}', val)
         return n
     else:
         return string
 
-def replace_in_meas(measurements: dict[str, dict[str, Any]], vars: dict[str, str]) -> dict[str, dict[str, Any]]:
+def replace_in_meas(measurements: dict, vars: dict[str, str]):
     # replace global variables
     for name, value in vars.items():
         for m in measurements.keys():
@@ -37,7 +36,7 @@ def replace_in_meas(measurements: dict[str, dict[str, Any]], vars: dict[str, str
                         measurements[m][key][i] = replace_string(measurements[m][key][i], name, value)
     return measurements
 
-def fill_cons(measurements: dict[str, dict[str, Any]], constants: dict[str, str]) -> dict[str, dict[str, Any]]:
+def fill_cons(measurements, constants):
     for m in measurements.keys():
         for name, val in constants.items():
             if name not in measurements[m].keys():
@@ -45,7 +44,7 @@ def fill_cons(measurements: dict[str, dict[str, Any]], constants: dict[str, str]
     return measurements
 
 
-def check_project_data(d: dict[str, dict[str, str]]) -> None:
+def check_project_data(d: dict) -> None:
     if 'project' not in d.keys() or 'measurements' not in d.keys() or len(list(d.keys())) > 4:
         raise ValueError('There should only be maximally be four keys on the top level, "project" and "measurements" are mandatory, "contants" is optional!')
     project_data = d['project']
@@ -58,7 +57,7 @@ def check_project_data(d: dict[str, dict[str, str]]) -> None:
     return
 
 
-def check_measurement_data(measurements: dict[str, dict[str, str]], code: str) -> None:
+def check_measurement_data(measurements: dict, code: str) -> None:
     var_names: list[str] = []
     if code == "sfcf":
         var_names = ["path", "ensemble", "param_file", "version", "prefix", "cfg_seperator", "names"]
@@ -92,14 +91,14 @@ def import_toml(path: str, file: str, copy_file: bool=True) -> None:
     with open(file, 'rb') as fp:
         toml_dict = toml.load(fp)
     check_project_data(toml_dict)
-    project: dict[str, Any] = toml_dict['project']
+    project: dict = toml_dict['project']
     if project['code'] not in known_codes:
         raise ValueError('Code' + project['code'] + 'has no import implementation!')
-    measurements: dict[str, dict[str, Any]] = toml_dict['measurements']
+    measurements: dict = toml_dict['measurements']
     measurements = fill_cons(measurements, toml_dict['constants'] if 'constants' in toml_dict else {})
     measurements = replace_in_meas(measurements, toml_dict['replace'] if 'replace' in toml_dict else {})
     check_measurement_data(measurements, project['code'])
-    aliases = project.get('aliases', [])
+    aliases = project.get('aliases', None)
     uuid = project.get('uuid', None)
     if uuid is not None:
         if not os.path.exists(path + "/projects/" + uuid):
@@ -134,16 +133,16 @@ def import_toml(path: str, file: str, copy_file: bool=True) -> None:
                     for rwp in ["integrator", "eps", "ntot", "dnms"]:
                         param[rwp] = "Unknown"
                 param['type'] = 't0'
-                measurement = openQCD.extract_t0(path, uuid, md['path'], param, str(md["prefix"]), int(md["dtr_read"]), int(md["xmin"]), int(md["spatial_extent"]),
+                measurement = openQCD.extract_t0(path, uuid, md['path'], param, md["prefix"], md["dtr_read"], md["xmin"], md["spatial_extent"],
-                                                 fit_range=int(md.get('fit_range', 5)), postfix=str(md.get('postfix', '')), names=md.get('names', []), files=md.get('files', []))
+                                                 fit_range=md.get('fit_range', 5), postfix=md.get('postfix', None), names=md.get('names', None), files=md.get('files', None))
             elif md['measurement'] == 't1':
                 if 'param_file' in md:
                     param = openQCD.read_ms3_param(path, uuid, md['param_file'])
                 param['type'] = 't1'
-                measurement = openQCD.extract_t1(path, uuid, md['path'], param, str(md["prefix"]), int(md["dtr_read"]), int(md["xmin"]), int(md["spatial_extent"]),
+                measurement = openQCD.extract_t1(path, uuid, md['path'], param, md["prefix"], md["dtr_read"], md["xmin"], md["spatial_extent"],
-                                                 fit_range=int(md.get('fit_range', 5)), postfix=str(md.get('postfix', '')), names=md.get('names', []), files=md.get('files', []))
+                                                 fit_range=md.get('fit_range', 5), postfix=md.get('postfix', None), names=md.get('names', None), files=md.get('files', None))
 
-        write_measurement(path, ensemble, measurement, uuid, project['code'], (md['param_file'] if 'param_file' in md else ''))
+        write_measurement(path, ensemble, measurement, uuid, project['code'], (md['param_file'] if 'param_file' in md else None))
 
     if not os.path.exists(os.path.join(path, "toml_imports", uuid)):
         os.makedirs(os.path.join(path, "toml_imports", uuid))
@@ -156,7 +155,7 @@ def import_toml(path: str, file: str, copy_file: bool=True) -> None:
     return
 
 
-def reimport_project(path: str, uuid: str) -> None:
+def reimport_project(path, uuid):
     """
     Reimport an existing project using the files that are already available for this project.
 
@@ -174,7 +173,6 @@ def reimport_project(path: str, uuid: str) -> None:
     return
 
 
-def update_project(path: str, uuid: str) -> None:
+def update_project(path, uuid):
     dl.update(how='merge', follow='sibling', dataset=os.path.join(path, "projects", uuid))
     # reimport_project(path, uuid)
-    return

corrlib/tools.py

@@ -1,32 +1,20 @@
 import os
-from configparser import ConfigParser
+import datalad.api as dl
 
 
-def str2list(string: str) -> list[str]:
+def str2list(string):
     return string.split(",")
 
-def list2str(mylist: list[str]) -> str:
+def list2str(mylist):
     s = ",".join(mylist)
     return s
 
-cached: bool = True
+cached = True
 
-def m2k(m: float) -> float:
+def m2k(m):
     return 1/(2*m+8)
 
 
-def k2m(k: float) -> float:
+def k2m(k):
     return (1/(2*k))-4
     
-
-def set_config(path, section, option, value):
-    config_path = os.path.join(path, '.corrlib')
-    config = ConfigParser()
-    if os.path.exists(config_path):
-        config.read(config_path)
-    if not config.has_section(section):
-        config.add_section(section)
-    config.set(section, option, value)
-    with open(config_path, 'w') as configfile:
-        config.write(configfile)
-    return

corrlib/tracker.py

@@ -1,29 +0,0 @@
-import os
-from configparser import ConfigParser
-from .trackers import datalad as dl
-
-
-def get_tracker(path):
-    config_path = os.path.join(path, '.corrlib')
-    config = ConfigParser()
-    if os.path.exists(config_path):
-        config.read(config_path)
-    tracker = config.get('core', 'tracker', fallback='datalad')
-    return tracker
-
-
-def get(path, file):
-    tracker = get_tracker(path)
-    if tracker == 'datalad':
-        dl.get_file(path, file)
-    else:
-        raise ValueError(f"Tracker {tracker} is not supported.")
-    return
-
-
-def save(path, message, files):
-    tracker = get_tracker(path)
-    if tracker == 'datalad':
-        dl.save(files, message=message, dataset=path)
-    else:
-        raise ValueError(f"Tracker {tracker} is not supported.")

corrlib/trackers/datalad.py

@@ -1,20 +0,0 @@
-import datalad.api as dl
-import os
-
-
-def get(path, file):
-    if file == "backlogger.db":
-        print("Downloading database...")
-    else:
-        print("Downloading data...")
-    dl.get(os.path.join(path, file), dataset=path)
-    print("> downloaded file")
-
-
-def save(path, message, files= None):
-    if files is None:
-        files = path
-    else:
-        files = [os.path.join(path, f) for f in files]
-    dl.save(files, message=message, dataset=path)
-    return

corrlib/version.py

@@ -1,34 +1 @@
-# file generated by setuptools-scm
+__version__ = "0.2.3"
-# don't change, don't track in version control
-
-__all__ = [
-    "__version__",
-    "__version_tuple__",
-    "version",
-    "version_tuple",
-    "__commit_id__",
-    "commit_id",
-]
-
-TYPE_CHECKING = False
-if TYPE_CHECKING:
-    from typing import Tuple
-    from typing import Union
-
-    VERSION_TUPLE = Tuple[Union[int, str], ...]
-    COMMIT_ID = Union[str, None]
-else:
-    VERSION_TUPLE = object
-    COMMIT_ID = object
-
-version: str
-__version__: str
-__version_tuple__: VERSION_TUPLE
-version_tuple: VERSION_TUPLE
-commit_id: COMMIT_ID
-__commit_id__: COMMIT_ID
-
-__version__ = version = '0.2.4.dev14+g602324f84.d20251202'
-__version_tuple__ = version_tuple = (0, 2, 4, 'dev14', 'g602324f84.d20251202')
-
-__commit_id__ = commit_id = 'g602324f84'

pyproject.toml

@@ -1,52 +1,6 @@
 [build-system]
-requires = ["setuptools >= 63.0.0", "wheel", "setuptools-scm"]
+requires = ["setuptools >= 63.0.0", "wheel"]
 build-backend = "setuptools.build_meta"
 
-[project]
-requires-python = ">=3.10"
-name = "corrlib"
-dynamic = ["version"]
-dependencies = [
-    "gitpython>=3.1.45",
-    'pyerrors>=2.11.1',
-    "datalad>=1.1.0",
-    'typer>=0.12.5',
-]
-description = "Python correlation library"
-authors = [
-  { name = 'Justus Kuhlmann', email = 'j_kuhl19@uni-muenster.de'}
-]
-
-[project.scripts]
-pcl = "corrlib.cli:app"
-
-[tool.setuptools.packages.find]
-include = ["corrlib", "corrlib.*"]
-
-[tool.setuptools_scm]
-write_to = "corrlib/version.py"
-
 [tool.ruff.lint]
-ignore = ["E501"]
+ignore = ["F403"]
-extend-select = [
-  "YTT",
-  "E",
-  "W",
-  "F",
-]
-
-[tool.mypy]
-strict = true
-implicit_reexport = false
-follow_untyped_imports = false
-ignore_missing_imports = true
-
-[dependency-groups]
-dev = [
-    "mypy>=1.19.0",
-    "pandas-stubs>=2.3.3.251201",
-    "pytest>=9.0.1",
-    "pytest-cov>=7.0.0",
-    "pytest-pretty>=1.3.0",
-    "ruff>=0.14.7",
-]

setup.py

@@ -0,0 +1,18 @@
+from setuptools import setup
+from distutils.util import convert_path
+
+
+version = {}
+with open(convert_path('corrlib/version.py')) as ver_file:
+    exec(ver_file.read(), version)
+
+setup(name='pycorrlib',
+      version=version['__version__'],
+      author='Justus Kuhlmann',
+      author_email='j_kuhl19@uni-muenster.de',
+      install_requires=['pyerrors>=2.11.1', 'datalad>=1.1.0', 'typer>=0.12.5'],
+      entry_points = {
+        'console_scripts': ['pcl=corrlib.cli:app'],
+        },
+      packages=['corrlib', 'corrlib.input']
+      )

tests/cli_test.py

@@ -1,91 +0,0 @@
-from typer.testing import CliRunner
-from corrlib.cli import app
-import os
-import sqlite3 as sql
-
-
-runner = CliRunner()
-
-
-def test_version():
-    result = runner.invoke(app, ["--version"])
-    assert result.exit_code == 0
-    assert "corrlib" in result.output
-
-
-def test_init_folders(tmp_path):
-    dataset_path = tmp_path / "test_dataset"
-    result = runner.invoke(app, ["init", "--dataset", str(dataset_path)])
-    assert result.exit_code == 0
-    assert os.path.exists(str(dataset_path))
-    assert os.path.exists(str(dataset_path / "backlogger.db"))
-
-
-def test_init_db(tmp_path):
-    dataset_path = tmp_path / "test_dataset"
-    result = runner.invoke(app, ["init", "--dataset", str(dataset_path)])
-    assert result.exit_code == 0
-    assert os.path.exists(str(dataset_path / "backlogger.db"))
-    conn = sql.connect(str(dataset_path / "backlogger.db"))
-    cursor = conn.cursor()
-    cursor.execute("SELECT name FROM sqlite_master WHERE type='table';")
-    tables = cursor.fetchall()
-    expected_tables = [
-        'projects',
-        'backlogs',
-        ]
-    table_names = [table[0] for table in tables]
-    for expected_table in expected_tables:
-        assert expected_table in table_names
-    
-    cursor.execute("SELECT * FROM projects;")
-    projects = cursor.fetchall()
-    assert len(projects) == 0
-
-    cursor.execute("SELECT * FROM backlogs;")
-    backlogs = cursor.fetchall()
-    assert len(backlogs) == 0
-
-    cursor.execute("PRAGMA table_info('projects');")
-    project_columns = cursor.fetchall()
-    expected_project_columns = [
-        "id",
-        "aliases",
-        "customTags",
-        "owner",
-        "code",
-        "created_at",
-        "updated_at"
-        ]
-    project_column_names = [col[1] for col in project_columns]
-    for expected_col in expected_project_columns:
-        assert expected_col in project_column_names
-    
-    cursor.execute("PRAGMA table_info('backlogs');")
-    backlog_columns = cursor.fetchall()
-    expected_backlog_columns = [
-        "id",
-        "name",
-        "ensemble",
-        "code",
-        "path",
-        "project",
-        "customTags",
-        "parameters",
-        "parameter_file",
-        "created_at",
-        "updated_at"
-        ]
-    backlog_column_names = [col[1] for col in backlog_columns]
-    for expected_col in expected_backlog_columns:
-        assert expected_col in backlog_column_names
-
-
-def test_list(tmp_path):
-    dataset_path = tmp_path / "test_dataset"
-    result = runner.invoke(app, ["init", "--dataset", str(dataset_path)])
-    assert result.exit_code == 0
-    result = runner.invoke(app, ["list", "--dataset", str(dataset_path), "ensembles"])
-    assert result.exit_code == 0
-    result = runner.invoke(app, ["list", "--dataset", str(dataset_path), "projects"])
-    assert result.exit_code == 0

tests/import_project_test.py

@@ -14,4 +14,4 @@ def test_toml_check_measurement_data():
                             "names": ['list', 'of', 'names']
                         }
                     }
-    t.check_measurement_data(measurements, "sfcf")
+    t.check_measurement_data(measurements)

tests/test_initialization.py

@@ -1,68 +0,0 @@
-import corrlib.initialization as init
-import os
-import sqlite3 as sql
-
-def test_init_folders(tmp_path):
-    dataset_path = tmp_path / "test_dataset"
-    init.create(str(dataset_path))
-    assert os.path.exists(str(dataset_path))
-    assert os.path.exists(str(dataset_path / "backlogger.db"))
-
-
-def test_init_db(tmp_path):
-    dataset_path = tmp_path / "test_dataset"
-    init.create(str(dataset_path))
-    assert os.path.exists(str(dataset_path / "backlogger.db"))
-    conn = sql.connect(str(dataset_path / "backlogger.db"))
-    cursor = conn.cursor()
-    cursor.execute("SELECT name FROM sqlite_master WHERE type='table';")
-    tables = cursor.fetchall()
-    expected_tables = [
-        'projects',
-        'backlogs',
-        ]
-    table_names = [table[0] for table in tables]
-    for expected_table in expected_tables:
-        assert expected_table in table_names
-    
-    cursor.execute("SELECT * FROM projects;")
-    projects = cursor.fetchall()
-    assert len(projects) == 0
-
-    cursor.execute("SELECT * FROM backlogs;")
-    backlogs = cursor.fetchall()
-    assert len(backlogs) == 0
-
-    cursor.execute("PRAGMA table_info('projects');")
-    project_columns = cursor.fetchall()
-    expected_project_columns = [
-        "id",
-        "aliases",
-        "customTags",
-        "owner",
-        "code",
-        "created_at",
-        "updated_at"
-        ]
-    project_column_names = [col[1] for col in project_columns]
-    for expected_col in expected_project_columns:
-        assert expected_col in project_column_names
-    
-    cursor.execute("PRAGMA table_info('backlogs');")
-    backlog_columns = cursor.fetchall()
-    expected_backlog_columns = [
-        "id",
-        "name",
-        "ensemble",
-        "code",
-        "path",
-        "project",
-        "customTags",
-        "parameters",
-        "parameter_file",
-        "created_at",
-        "updated_at"
-        ]
-    backlog_column_names = [col[1] for col in backlog_columns]
-    for expected_col in expected_backlog_columns:
-        assert expected_col in backlog_column_names

tests/tools_test.py

@@ -1,31 +0,0 @@
-
-
-from corrlib import tools as tl
-
-
-def test_m2k():
-    for m in [0.1, 0.5, 1.0]:
-        expected_k = 1 / (2 * m + 8)
-        assert tl.m2k(m) == expected_k
-
-
-def test_k2m():
-    for m in [0.1, 0.5, 1.0]:
-        assert tl.k2m(m) == (1/(2*m))-4
-
-
-def test_k2m_m2k():
-    for m in [0.1, 0.5, 1.0]:
-        k = tl.m2k(m)
-        m_converted = tl.k2m(k)
-        assert abs(m - m_converted) < 1e-9
-
-
-def test_str2list():
-    assert tl.str2list("a,b,c") == ["a", "b", "c"]
-    assert tl.str2list("1,2,3") == ["1", "2", "3"]
-
-
-def test_list2str():
-    assert tl.list2str(["a", "b", "c"]) == "a,b,c"
-    assert tl.list2str(["1", "2", "3"]) == "1,2,3"