From 44ab402c6c5b62dcda975e9f5f4789e65bae5a9d Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Fri, 28 Nov 2025 16:13:53 +0100 Subject: [PATCH 01/35] start refactor for data backend by only improting datalad into one submodule --- corrlib/data_backend/databackend.py | 7 +++++++ corrlib/data_backend/datalad.py | 13 +++++++++++++ corrlib/tools.py | 9 --------- 3 files changed, 20 insertions(+), 9 deletions(-) create mode 100644 corrlib/data_backend/databackend.py create mode 100644 corrlib/data_backend/datalad.py diff --git a/corrlib/data_backend/databackend.py b/corrlib/data_backend/databackend.py new file mode 100644 index 0000000..dad9fee --- /dev/null +++ b/corrlib/data_backend/databackend.py @@ -0,0 +1,7 @@ +import .datalad as dl + + +class DataBackend: + @abstractmethod + def get_file(self, data, file): + pass \ No newline at end of file diff --git a/corrlib/data_backend/datalad.py b/corrlib/data_backend/datalad.py new file mode 100644 index 0000000..9ccf1a6 --- /dev/null +++ b/corrlib/data_backend/datalad.py @@ -0,0 +1,13 @@ +import datalad.api as dl +import os +from corrlib.data_backend.databackend import + +def DLBackend() + +def get_file(path, file): + if file == "backlogger.db": + print("Downloading database...") + else: + print("Downloading data...") + dl.get(os.path.join(path, file), dataset=path) + print("> downloaded file") \ No newline at end of file diff --git a/corrlib/tools.py b/corrlib/tools.py index 14bfc05..6bc8b27 100644 --- a/corrlib/tools.py +++ b/corrlib/tools.py @@ -17,13 +17,4 @@ def m2k(m): def k2m(k): return (1/(2*k))-4 - - -def get_file(path, file): - if file == "backlogger.db": - print("Downloading database...") - else: - print("Downloading data...") - dl.get(os.path.join(path, file), dataset=path) - print("> downloaded file") \ No newline at end of file From 3963b07c5f5eacda164e876453fce640ca406340 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Thu, 4 Dec 2025 10:47:53 +0100 Subject: [PATCH 02/35] add tracker module, moularize tracking system --- corrlib/__init__.py | 1 + corrlib/find.py | 3 ++- corrlib/main.py | 3 ++- corrlib/meas_io.py | 3 ++- corrlib/tracker.py | 22 ++++++++++++++++++++++ corrlib/trackers/datalad.py | 11 +++++++++++ 6 files changed, 40 insertions(+), 3 deletions(-) create mode 100644 corrlib/tracker.py create mode 100644 corrlib/trackers/datalad.py diff --git a/corrlib/__init__.py b/corrlib/__init__.py index 91b07f4..d9aff5f 100644 --- a/corrlib/__init__.py +++ b/corrlib/__init__.py @@ -21,3 +21,4 @@ from .initialization import * from .meas_io import * from .find import * from .version import __version__ +from .config import * diff --git a/corrlib/find.py b/corrlib/find.py index 3e73240..6aa795c 100644 --- a/corrlib/find.py +++ b/corrlib/find.py @@ -5,7 +5,8 @@ import json import pandas as pd import numpy as np from .input.implementations import codes -from .tools import k2m, get_file +from .tools import k2m +from .tracker import get_file # this will implement the search functionality diff --git a/corrlib/main.py b/corrlib/main.py index fc7663f..90e786f 100644 --- a/corrlib/main.py +++ b/corrlib/main.py @@ -5,7 +5,8 @@ import os from .git_tools import move_submodule import shutil from .find import _project_lookup_by_id -from .tools import list2str, str2list, get_file +from .tools import list2str, str2list +from .tracker import get_file from typing import Union diff --git a/corrlib/meas_io.py b/corrlib/meas_io.py index 7122ca0..594e668 100644 --- a/corrlib/meas_io.py +++ b/corrlib/meas_io.py @@ -7,7 +7,8 @@ import json from typing import Union from pyerrors import Obs, Corr, dump_object, load_object from hashlib import sha256 -from .tools import cached, get_file +from .tools import cached +from .tracker import get_file import shutil diff --git a/corrlib/tracker.py b/corrlib/tracker.py new file mode 100644 index 0000000..7230386 --- /dev/null +++ b/corrlib/tracker.py @@ -0,0 +1,22 @@ +import os +from configparser import ConfigParser +from .trackers import datalad as dl + + +def get_tracker(path): + config_path = os.path.join(path, '.corrlib') + config = ConfigParser() + if os.path.exists(config_path): + config.read(config_path) + tracker = config.get('core', 'tracker', fallback='datalad') + return tracker + + +def get_file(path, file): + tracker = get_tracker(path) + if tracker == 'datalad': + dl.get_file(path, file) + else: + raise ValueError(f"Tracker {tracker} is not supported.") + return + diff --git a/corrlib/trackers/datalad.py b/corrlib/trackers/datalad.py new file mode 100644 index 0000000..5d3deaa --- /dev/null +++ b/corrlib/trackers/datalad.py @@ -0,0 +1,11 @@ +import datalad.api as dl +import os + + +def get_file(path, file): + if file == "backlogger.db": + print("Downloading database...") + else: + print("Downloading data...") + dl.get(os.path.join(path, file), dataset=path) + print("> downloaded file") \ No newline at end of file From d5a48b91f07b7ec31750dee81e39af656d7bb614 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Thu, 4 Dec 2025 11:07:33 +0100 Subject: [PATCH 03/35] implement save method --- corrlib/find.py | 8 ++++---- corrlib/trackers/datalad.py | 13 +++++++++++-- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/corrlib/find.py b/corrlib/find.py index 6aa795c..57a4d1a 100644 --- a/corrlib/find.py +++ b/corrlib/find.py @@ -6,7 +6,7 @@ import pandas as pd import numpy as np from .input.implementations import codes from .tools import k2m -from .tracker import get_file +from .tracker import get # this will implement the search functionality @@ -144,7 +144,7 @@ def find_record(path, ensemble, correlator_name, code, project=None, parameters= db = path + '/backlogger.db' if code not in codes: raise ValueError("Code " + code + "unknown, take one of the following:" + ", ".join(codes)) - get_file(path, "backlogger.db") + get(path, "backlogger.db") results = _db_lookup(db, ensemble, correlator_name,code, project, parameters=parameters, created_before=created_before, created_after=created_after, updated_before=updated_before, updated_after=updated_after, revision=revision) if code == "sfcf": results = sfcf_filter(results, **kwargs) @@ -153,13 +153,13 @@ def find_record(path, ensemble, correlator_name, code, project=None, parameters= def find_project(path, name): - get_file(path, "backlogger.db") + get(path, "backlogger.db") return _project_lookup_by_alias(os.path.join(path, "backlogger.db"), name) def list_projects(path): db = path + '/backlogger.db' - get_file(path, "backlogger.db") + get(path, "backlogger.db") conn = sqlite3.connect(db) c = conn.cursor() c.execute("SELECT id,aliases FROM projects") diff --git a/corrlib/trackers/datalad.py b/corrlib/trackers/datalad.py index 5d3deaa..e9a6e9f 100644 --- a/corrlib/trackers/datalad.py +++ b/corrlib/trackers/datalad.py @@ -2,10 +2,19 @@ import datalad.api as dl import os -def get_file(path, file): +def get(path, file): if file == "backlogger.db": print("Downloading database...") else: print("Downloading data...") dl.get(os.path.join(path, file), dataset=path) - print("> downloaded file") \ No newline at end of file + print("> downloaded file") + + +def save(path, message, files= None): + if files is None: + files = path + else: + files = [os.path.join(path, f) for f in files] + dl.save(files, message=message, dataset=path) + return From 057f214e33cb9046d622326423745cc3570944e7 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Thu, 4 Dec 2025 11:08:05 +0100 Subject: [PATCH 04/35] create config on init --- corrlib/initialization.py | 39 +++++++++++++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/corrlib/initialization.py b/corrlib/initialization.py index f6ef5aa..28cf4c7 100644 --- a/corrlib/initialization.py +++ b/corrlib/initialization.py @@ -1,6 +1,8 @@ +from configparser import ConfigParser import sqlite3 import datalad.api as dl import os +import tracker as tr def _create_db(db): @@ -34,19 +36,40 @@ def _create_db(db): conn.close() +def _create_config(path): + """ + Create the config file for backlogger. + + """ + config = ConfigParser() + config['core'] = { + 'version': '1.0', + 'db_path': os.path.join(path, 'backlogger.db'), + 'projects_path': os.path.join(path, 'projects'), + 'archive_path': os.path.join(path, 'archive'), + 'toml_imports_path': os.path.join(path, 'toml_imports'), + 'import_scripts_path': os.path.join(path, 'import_scripts'), + 'tracker': 'datalad', + 'cached': True, + } + with open(os.path.join(path, '.corrlib'), 'w') as configfile: + config.write(configfile) + + def create(path): """ Create folder of backlogs. """ dl.create(path) - _create_db(path + '/backlogger.db') - os.chmod(path + '/backlogger.db', 0o666) # why does this not work? - os.makedirs(path + '/projects') - os.makedirs(path + '/archive') - os.makedirs(path + '/toml_imports') - os.makedirs(path + '/import_scripts/template.py') - with open(path + "/.gitignore", "w") as fp: + _create_db(os.path.join(path, 'backlogger.db')) + os.chmod(os.path.join(path, 'backlogger.db'), 0o666) # why does this not work? + _create_config(path) + os.makedirs(os.path.join(path, 'projects')) + os.makedirs(os.path.join(path, 'archive')) + os.makedirs(os.path.join(path, 'toml_imports')) + os.makedirs(os.path.join(path, 'import_scripts/template.py')) + with open(os.path.join(path, ".gitignore"), "w") as fp: fp.write(".cache") fp.close() - dl.save(path, dataset=path, message="Initialize backlogger directory.") + tr.save(path, message="Initialized correlator library", dataset=path) From 15fd97af8e88b51cf462eea13ff937efa3de5d7f Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Thu, 4 Dec 2025 11:09:03 +0100 Subject: [PATCH 05/35] rename getter method --- corrlib/meas_io.py | 8 ++++---- corrlib/tools.py | 16 ++++++++++++++-- corrlib/tracker.py | 9 ++++++++- 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/corrlib/meas_io.py b/corrlib/meas_io.py index 594e668..6a50f7e 100644 --- a/corrlib/meas_io.py +++ b/corrlib/meas_io.py @@ -8,7 +8,7 @@ from typing import Union from pyerrors import Obs, Corr, dump_object, load_object from hashlib import sha256 from .tools import cached -from .tracker import get_file +from .tracker import get import shutil @@ -29,7 +29,7 @@ def write_measurement(path, ensemble, measurement, uuid, code, parameter_file=No The uuid of the project. """ db = os.path.join(path, 'backlogger.db') - get_file(path, "backlogger.db") + get(path, "backlogger.db") dl.unlock(db, dataset=path) conn = sqlite3.connect(db) c = conn.cursor() @@ -169,7 +169,7 @@ def cache_path(path, file, key): def preload(path: str, file: str): - get_file(path, file) + get(path, file) filedict = pj.load_json_dict(os.path.join(path, file)) print("> read file") return filedict @@ -179,7 +179,7 @@ def drop_record(path: str, meas_path: str): file_in_archive = meas_path.split("::")[0] file = os.path.join(path, file_in_archive) db = os.path.join(path, 'backlogger.db') - get_file(path, 'backlogger.db') + get(path, 'backlogger.db') sub_key = meas_path.split("::")[1] dl.unlock(db, dataset=path) conn = sqlite3.connect(db) diff --git a/corrlib/tools.py b/corrlib/tools.py index 6bc8b27..1aaa4d3 100644 --- a/corrlib/tools.py +++ b/corrlib/tools.py @@ -1,5 +1,5 @@ import os -import datalad.api as dl +from configparser import ConfigParser def str2list(string): @@ -17,4 +17,16 @@ def m2k(m): def k2m(k): return (1/(2*k))-4 - \ No newline at end of file + + +def set_config(path, section, option, value): + config_path = os.path.join(path, '.corrlib') + config = ConfigParser() + if os.path.exists(config_path): + config.read(config_path) + if not config.has_section(section): + config.add_section(section) + config.set(section, option, value) + with open(config_path, 'w') as configfile: + config.write(configfile) + return diff --git a/corrlib/tracker.py b/corrlib/tracker.py index 7230386..fcf3994 100644 --- a/corrlib/tracker.py +++ b/corrlib/tracker.py @@ -12,7 +12,7 @@ def get_tracker(path): return tracker -def get_file(path, file): +def get(path, file): tracker = get_tracker(path) if tracker == 'datalad': dl.get_file(path, file) @@ -20,3 +20,10 @@ def get_file(path, file): raise ValueError(f"Tracker {tracker} is not supported.") return + +def save(path, message, files): + tracker = get_tracker(path) + if tracker == 'datalad': + dl.save(files, message=message, dataset=path) + else: + raise ValueError(f"Tracker {tracker} is not supported.") From 2100d90511ba28ed849a35e9414bd2f9e4c75b2c Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Thu, 4 Dec 2025 11:21:13 +0100 Subject: [PATCH 06/35] chore: cleanup --- corrlib/data_backend/databackend.py | 7 ------- corrlib/data_backend/datalad.py | 13 ------------- tests/test_import_project.py | 0 3 files changed, 20 deletions(-) delete mode 100644 corrlib/data_backend/databackend.py delete mode 100644 corrlib/data_backend/datalad.py delete mode 100644 tests/test_import_project.py diff --git a/corrlib/data_backend/databackend.py b/corrlib/data_backend/databackend.py deleted file mode 100644 index dad9fee..0000000 --- a/corrlib/data_backend/databackend.py +++ /dev/null @@ -1,7 +0,0 @@ -import .datalad as dl - - -class DataBackend: - @abstractmethod - def get_file(self, data, file): - pass \ No newline at end of file diff --git a/corrlib/data_backend/datalad.py b/corrlib/data_backend/datalad.py deleted file mode 100644 index 9ccf1a6..0000000 --- a/corrlib/data_backend/datalad.py +++ /dev/null @@ -1,13 +0,0 @@ -import datalad.api as dl -import os -from corrlib.data_backend.databackend import - -def DLBackend() - -def get_file(path, file): - if file == "backlogger.db": - print("Downloading database...") - else: - print("Downloading data...") - dl.get(os.path.join(path, file), dataset=path) - print("> downloaded file") \ No newline at end of file diff --git a/tests/test_import_project.py b/tests/test_import_project.py deleted file mode 100644 index e69de29..0000000 From 3b64d8457b7f92daf3ba788e0c99b56fe8d6b08c Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Thu, 4 Dec 2025 11:23:07 +0100 Subject: [PATCH 07/35] import correct file as tracker --- corrlib/initialization.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/corrlib/initialization.py b/corrlib/initialization.py index 03e7f3f..24a45ca 100644 --- a/corrlib/initialization.py +++ b/corrlib/initialization.py @@ -2,7 +2,7 @@ from configparser import ConfigParser import sqlite3 import datalad.api as dl import os -import tracker as tr +import .tracker as tr def _create_db(db: str) -> None: From 18843f4d9f0485e83667fb5f035e8376f1589fc0 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Thu, 4 Dec 2025 11:32:30 +0100 Subject: [PATCH 08/35] correct syntax --- corrlib/initialization.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/corrlib/initialization.py b/corrlib/initialization.py index 24a45ca..242ab45 100644 --- a/corrlib/initialization.py +++ b/corrlib/initialization.py @@ -2,7 +2,7 @@ from configparser import ConfigParser import sqlite3 import datalad.api as dl import os -import .tracker as tr +from .tracker import save def _create_db(db: str) -> None: @@ -94,5 +94,5 @@ def create(path: str) -> None: with open(os.path.join(path, ".gitignore"), "w") as fp: fp.write(".cache") fp.close() - tr.save(path, message="Initialized correlator library", dataset=path) + save(path, message="Initialized correlator library", dataset=path) return From 38d00316e1cb9957d763bf4d452262a7e7974340 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Thu, 4 Dec 2025 11:33:29 +0100 Subject: [PATCH 09/35] remove duplicate method --- corrlib/initialization.py | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/corrlib/initialization.py b/corrlib/initialization.py index 242ab45..130cea8 100644 --- a/corrlib/initialization.py +++ b/corrlib/initialization.py @@ -37,26 +37,6 @@ def _create_db(db: str) -> None: return -def _create_config(path): - """ - Create the config file for backlogger. - - """ - config = ConfigParser() - config['core'] = { - 'version': '1.0', - 'db_path': os.path.join(path, 'backlogger.db'), - 'projects_path': os.path.join(path, 'projects'), - 'archive_path': os.path.join(path, 'archive'), - 'toml_imports_path': os.path.join(path, 'toml_imports'), - 'import_scripts_path': os.path.join(path, 'import_scripts'), - 'tracker': 'datalad', - 'cached': True, - } - with open(os.path.join(path, '.corrlib'), 'w') as configfile: - config.write(configfile) - - def _create_config(path: str) -> None: """ Create the config file for backlogger. From 28ea48ad66e2effa3e0f6b0c26189e995ab0c3c6 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Thu, 4 Dec 2025 11:34:43 +0100 Subject: [PATCH 10/35] typing --- corrlib/tracker.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/corrlib/tracker.py b/corrlib/tracker.py index fcf3994..e6d4ad5 100644 --- a/corrlib/tracker.py +++ b/corrlib/tracker.py @@ -3,7 +3,7 @@ from configparser import ConfigParser from .trackers import datalad as dl -def get_tracker(path): +def get_tracker(path: str) -> str: config_path = os.path.join(path, '.corrlib') config = ConfigParser() if os.path.exists(config_path): @@ -12,7 +12,7 @@ def get_tracker(path): return tracker -def get(path, file): +def get(path: str, file: str) -> None: tracker = get_tracker(path) if tracker == 'datalad': dl.get_file(path, file) @@ -21,7 +21,7 @@ def get(path, file): return -def save(path, message, files): +def save(path: str, message: str, files: list[str]) -> None: tracker = get_tracker(path) if tracker == 'datalad': dl.save(files, message=message, dataset=path) From 7240d29b46e07f45efba2fbc22bb793ec462bafc Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Thu, 4 Dec 2025 11:40:50 +0100 Subject: [PATCH 11/35] add thin create wrapper --- corrlib/initialization.py | 2 +- corrlib/tracker.py | 9 +++++++++ corrlib/trackers/datalad.py | 5 +++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/corrlib/initialization.py b/corrlib/initialization.py index 130cea8..14bcaf0 100644 --- a/corrlib/initialization.py +++ b/corrlib/initialization.py @@ -63,7 +63,7 @@ def create(path: str) -> None: Create folder of backlogs. """ - dl.create(path) + create(path) _create_db(os.path.join(path, 'backlogger.db')) os.chmod(os.path.join(path, 'backlogger.db'), 0o666) # why does this not work? _create_config(path) diff --git a/corrlib/tracker.py b/corrlib/tracker.py index e6d4ad5..7f63e9b 100644 --- a/corrlib/tracker.py +++ b/corrlib/tracker.py @@ -27,3 +27,12 @@ def save(path: str, message: str, files: list[str]) -> None: dl.save(files, message=message, dataset=path) else: raise ValueError(f"Tracker {tracker} is not supported.") + + +def create(path: str) -> None: + tracker = get_tracker(path) + if tracker == 'datalad': + dl.create(path) + else: + raise ValueError(f"Tracker {tracker} is not supported.") + return diff --git a/corrlib/trackers/datalad.py b/corrlib/trackers/datalad.py index e9a6e9f..4eccf0f 100644 --- a/corrlib/trackers/datalad.py +++ b/corrlib/trackers/datalad.py @@ -18,3 +18,8 @@ def save(path, message, files= None): files = [os.path.join(path, f) for f in files] dl.save(files, message=message, dataset=path) return + + +def create(path): + dl.create(path) + return From 6c9b88a22f41f800effbe409eec6575176233a7c Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Thu, 4 Dec 2025 11:41:32 +0100 Subject: [PATCH 12/35] import save from tracker --- corrlib/toml.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/corrlib/toml.py b/corrlib/toml.py index 11065fe..7b02e33 100644 --- a/corrlib/toml.py +++ b/corrlib/toml.py @@ -10,10 +10,12 @@ the import of projects via TOML. import tomllib as toml import shutil + +import datalad.api as dl +from .tracker import save from .input import sfcf, openQCD from .main import import_project, update_aliases from .meas_io import write_measurement -import datalad.api as dl import os from .input.implementations import codes as known_codes from typing import Any @@ -150,7 +152,7 @@ def import_toml(path: str, file: str, copy_file: bool=True) -> None: if copy_file: import_file = os.path.join(path, "toml_imports", uuid, file.split("/")[-1]) shutil.copy(file, import_file) - dl.save(import_file, message="Import using " + import_file, dataset=path) + save(path, files=[import_file], message="Import using " + import_file) print("File copied to " + import_file) print("Imported project.") return From 8993fbe4c126317a0974544f444b3b7e670efeae Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Thu, 4 Dec 2025 11:41:58 +0100 Subject: [PATCH 13/35] remove config import --- corrlib/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/corrlib/__init__.py b/corrlib/__init__.py index afc6776..4e1b364 100644 --- a/corrlib/__init__.py +++ b/corrlib/__init__.py @@ -22,4 +22,3 @@ from .meas_io import load_records as load_records from .find import find_project as find_project from .find import find_record as find_record from .find import list_projects as list_projects -from .config import * From d104d994f8b35a7704bd27f963e0e4d57a8e36a1 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Thu, 4 Dec 2025 11:49:52 +0100 Subject: [PATCH 14/35] correct typing errors --- corrlib/main.py | 10 +++++----- corrlib/tools.py | 3 ++- corrlib/tracker.py | 4 ++-- corrlib/trackers/datalad.py | 8 +++++--- 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/corrlib/main.py b/corrlib/main.py index df303cd..2065206 100644 --- a/corrlib/main.py +++ b/corrlib/main.py @@ -6,7 +6,7 @@ from .git_tools import move_submodule import shutil from .find import _project_lookup_by_id from .tools import list2str, str2list -from .tracker import get_file +from .tracker import get from typing import Union, Optional @@ -26,7 +26,7 @@ def create_project(path: str, uuid: str, owner: Union[str, None]=None, tags: Uni The code that was used to create the measurements. """ db = path + "/backlogger.db" - get_file(path, "backlogger.db") + get(path, "backlogger.db") conn = sqlite3.connect(db) c = conn.cursor() known_projects = c.execute("SELECT * FROM projects WHERE id=?", (uuid,)) @@ -47,7 +47,7 @@ def create_project(path: str, uuid: str, owner: Union[str, None]=None, tags: Uni def update_project_data(path: str, uuid: str, prop: str, value: Union[str, None] = None) -> None: - get_file(path, "backlogger.db") + get(path, "backlogger.db") conn = sqlite3.connect(os.path.join(path, "backlogger.db")) c = conn.cursor() c.execute(f"UPDATE projects SET '{prop}' = '{value}' WHERE id == '{uuid}'") @@ -58,7 +58,7 @@ def update_project_data(path: str, uuid: str, prop: str, value: Union[str, None] def update_aliases(path: str, uuid: str, aliases: list[str]) -> None: db = os.path.join(path, "backlogger.db") - get_file(path, "backlogger.db") + get(path, "backlogger.db") known_data = _project_lookup_by_id(db, uuid)[0] known_aliases = known_data[1] @@ -123,7 +123,7 @@ def import_project(path: str, url: str, owner: Union[str, None]=None, tags: Opti raise ValueError("The dataset does not have a uuid!") if not os.path.exists(path + "/projects/" + uuid): db = path + "/backlogger.db" - get_file(path, "backlogger.db") + get(path, "backlogger.db") dl.unlock(db, dataset=path) create_project(path, uuid, owner, tags, aliases, code) move_submodule(path, 'projects/tmp', 'projects/' + uuid) diff --git a/corrlib/tools.py b/corrlib/tools.py index 4d444cc..77bfd2e 100644 --- a/corrlib/tools.py +++ b/corrlib/tools.py @@ -1,5 +1,6 @@ import os from configparser import ConfigParser +from typing import Any def str2list(string: str) -> list[str]: @@ -19,7 +20,7 @@ def k2m(k: float) -> float: return (1/(2*k))-4 -def set_config(path, section, option, value): +def set_config(path: str, section: str, option: str, value: Any) -> None: config_path = os.path.join(path, '.corrlib') config = ConfigParser() if os.path.exists(config_path): diff --git a/corrlib/tracker.py b/corrlib/tracker.py index 7f63e9b..900268c 100644 --- a/corrlib/tracker.py +++ b/corrlib/tracker.py @@ -15,7 +15,7 @@ def get_tracker(path: str) -> str: def get(path: str, file: str) -> None: tracker = get_tracker(path) if tracker == 'datalad': - dl.get_file(path, file) + dl.get(path, file) else: raise ValueError(f"Tracker {tracker} is not supported.") return @@ -24,7 +24,7 @@ def get(path: str, file: str) -> None: def save(path: str, message: str, files: list[str]) -> None: tracker = get_tracker(path) if tracker == 'datalad': - dl.save(files, message=message, dataset=path) + dl.save(path, message, files) else: raise ValueError(f"Tracker {tracker} is not supported.") diff --git a/corrlib/trackers/datalad.py b/corrlib/trackers/datalad.py index 4eccf0f..e3be365 100644 --- a/corrlib/trackers/datalad.py +++ b/corrlib/trackers/datalad.py @@ -1,17 +1,19 @@ import datalad.api as dl import os +from typing import Optional -def get(path, file): +def get(path: str, file: str) -> None: if file == "backlogger.db": print("Downloading database...") else: print("Downloading data...") dl.get(os.path.join(path, file), dataset=path) print("> downloaded file") + return -def save(path, message, files= None): +def save(path: str, message: str, files: Optional[list[str]]=None) -> None: if files is None: files = path else: @@ -20,6 +22,6 @@ def save(path, message, files= None): return -def create(path): +def create(path: str) -> None: dl.create(path) return From 59519eae3be2d1a3dcbb6304d32bf474d6479be1 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Thu, 4 Dec 2025 11:59:43 +0100 Subject: [PATCH 15/35] rename tracker create --- corrlib/initialization.py | 4 ++-- corrlib/tracker.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/corrlib/initialization.py b/corrlib/initialization.py index 14bcaf0..76b062f 100644 --- a/corrlib/initialization.py +++ b/corrlib/initialization.py @@ -2,7 +2,7 @@ from configparser import ConfigParser import sqlite3 import datalad.api as dl import os -from .tracker import save +from .tracker import save, init def _create_db(db: str) -> None: @@ -74,5 +74,5 @@ def create(path: str) -> None: with open(os.path.join(path, ".gitignore"), "w") as fp: fp.write(".cache") fp.close() - save(path, message="Initialized correlator library", dataset=path) + save(path, message="Initialized correlator library") return diff --git a/corrlib/tracker.py b/corrlib/tracker.py index 900268c..f513b56 100644 --- a/corrlib/tracker.py +++ b/corrlib/tracker.py @@ -29,7 +29,7 @@ def save(path: str, message: str, files: list[str]) -> None: raise ValueError(f"Tracker {tracker} is not supported.") -def create(path: str) -> None: +def init(path: str) -> None: tracker = get_tracker(path) if tracker == 'datalad': dl.create(path) From 07173e499917764728227046587ccec49062d06c Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Thu, 4 Dec 2025 12:04:46 +0100 Subject: [PATCH 16/35] make file arg optional --- corrlib/tracker.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/corrlib/tracker.py b/corrlib/tracker.py index f513b56..e6e9c04 100644 --- a/corrlib/tracker.py +++ b/corrlib/tracker.py @@ -1,6 +1,7 @@ import os from configparser import ConfigParser from .trackers import datalad as dl +from typing import Optional def get_tracker(path: str) -> str: @@ -21,7 +22,7 @@ def get(path: str, file: str) -> None: return -def save(path: str, message: str, files: list[str]) -> None: +def save(path: str, message: str, files: Optional[list[str]]=None) -> None: tracker = get_tracker(path) if tracker == 'datalad': dl.save(path, message, files) From b0ef8c3fc0b84670b0ab3d5ed8e80fe063b231ad Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Thu, 4 Dec 2025 12:05:17 +0100 Subject: [PATCH 17/35] use init, not create --- corrlib/initialization.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/corrlib/initialization.py b/corrlib/initialization.py index 76b062f..e3a078e 100644 --- a/corrlib/initialization.py +++ b/corrlib/initialization.py @@ -1,6 +1,5 @@ from configparser import ConfigParser import sqlite3 -import datalad.api as dl import os from .tracker import save, init @@ -63,7 +62,7 @@ def create(path: str) -> None: Create folder of backlogs. """ - create(path) + init(path) _create_db(os.path.join(path, 'backlogger.db')) os.chmod(os.path.join(path, 'backlogger.db'), 0o666) # why does this not work? _create_config(path) From 294df0a6c9bd0c644b9ff26ec16fbcdead49edca Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Thu, 4 Dec 2025 12:07:42 +0100 Subject: [PATCH 18/35] cleaner implementation --- corrlib/trackers/datalad.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/corrlib/trackers/datalad.py b/corrlib/trackers/datalad.py index e3be365..c4e3e70 100644 --- a/corrlib/trackers/datalad.py +++ b/corrlib/trackers/datalad.py @@ -14,9 +14,7 @@ def get(path: str, file: str) -> None: def save(path: str, message: str, files: Optional[list[str]]=None) -> None: - if files is None: - files = path - else: + if files is not None: files = [os.path.join(path, f) for f in files] dl.save(files, message=message, dataset=path) return From 1264a09ed0a3847d8f89acbf4dcc8b4a8d6cf589 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Thu, 4 Dec 2025 12:09:26 +0100 Subject: [PATCH 19/35] change cached to be string --- corrlib/initialization.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/corrlib/initialization.py b/corrlib/initialization.py index e3a078e..13b6e66 100644 --- a/corrlib/initialization.py +++ b/corrlib/initialization.py @@ -50,7 +50,7 @@ def _create_config(path: str) -> None: 'toml_imports_path': os.path.join(path, 'toml_imports'), 'import_scripts_path': os.path.join(path, 'import_scripts'), 'tracker': 'datalad', - 'cached': True, + 'cached': 'True', } with open(os.path.join(path, '.corrlib'), 'w') as configfile: config.write(configfile) From b3256e0b7c1390a8c407ea242425a2489131a75e Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Thu, 4 Dec 2025 12:16:15 +0100 Subject: [PATCH 20/35] add option to not track --- corrlib/tracker.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/corrlib/tracker.py b/corrlib/tracker.py index e6e9c04..e645470 100644 --- a/corrlib/tracker.py +++ b/corrlib/tracker.py @@ -17,6 +17,8 @@ def get(path: str, file: str) -> None: tracker = get_tracker(path) if tracker == 'datalad': dl.get(path, file) + elif tracker == 'None': + pass else: raise ValueError(f"Tracker {tracker} is not supported.") return @@ -26,6 +28,8 @@ def save(path: str, message: str, files: Optional[list[str]]=None) -> None: tracker = get_tracker(path) if tracker == 'datalad': dl.save(path, message, files) + elif tracker == 'None': + pass else: raise ValueError(f"Tracker {tracker} is not supported.") @@ -34,6 +38,8 @@ def init(path: str) -> None: tracker = get_tracker(path) if tracker == 'datalad': dl.create(path) + elif tracker == 'None': + os.path.makedirs(path, exist_ok=True) else: raise ValueError(f"Tracker {tracker} is not supported.") return From 2537fea06c73afc51fdbc148da73dd4a322ef501 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Thu, 4 Dec 2025 12:29:31 +0100 Subject: [PATCH 21/35] roll out save replacement --- corrlib/git_tools.py | 5 +++-- corrlib/main.py | 12 ++++++------ corrlib/meas_io.py | 6 +++--- corrlib/tracker.py | 2 +- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/corrlib/git_tools.py b/corrlib/git_tools.py index bde9871..c6e7522 100644 --- a/corrlib/git_tools.py +++ b/corrlib/git_tools.py @@ -1,5 +1,5 @@ import os -import datalad.api as dl +from .tracker import save import git GITMODULES_FILE = '.gitmodules' @@ -40,5 +40,6 @@ def move_submodule(repo_path: str, old_path: str, new_path: str) -> None: repo = git.Repo(repo_path) repo.git.add('.gitmodules') # save new state of the dataset - dl.save(repo_path, message=f"Move module from {old_path} to {new_path}", dataset=repo_path) + save(repo_path, message=f"Move module from {old_path} to {new_path}", files=['.gitmodules', repo_path]) + return diff --git a/corrlib/main.py b/corrlib/main.py index 2065206..dfed6ea 100644 --- a/corrlib/main.py +++ b/corrlib/main.py @@ -6,7 +6,7 @@ from .git_tools import move_submodule import shutil from .find import _project_lookup_by_id from .tools import list2str, str2list -from .tracker import get +from .tracker import get, save from typing import Union, Optional @@ -43,7 +43,7 @@ def create_project(path: str, uuid: str, owner: Union[str, None]=None, tags: Uni c.execute("INSERT INTO projects (id, aliases, customTags, owner, code, created_at, updated_at) VALUES (?, ?, ?, ?, ?, datetime('now'), datetime('now'))", (uuid, alias_str, tag_str, owner, code)) conn.commit() conn.close() - dl.save(db, message="Added entry for project " + uuid + " to database", dataset=path) + save(path, message="Added entry for project " + uuid + " to database", files=["backlogger.db"]) def update_project_data(path: str, uuid: str, prop: str, value: Union[str, None] = None) -> None: @@ -79,7 +79,7 @@ def update_aliases(path: str, uuid: str, aliases: list[str]) -> None: alias_str = list2str(new_alias_list) dl.unlock(db, dataset=path) update_project_data(path, uuid, "aliases", alias_str) - dl.save(db, dataset=path) + save(path, message="Updated aliases for project " + uuid, files=["backlogger.db"]) return @@ -109,11 +109,11 @@ def import_project(path: str, url: str, owner: Union[str, None]=None, tags: Opti in order to receive a uuid and have a consistent interface. """ - tmp_path = path + '/projects/tmp' + tmp_path = os.path.join(path, 'projects/tmp') if not isDataset: dl.create(tmp_path, dataset=path) shutil.copytree(url + "/*", path + '/projects/tmp/') - dl.save(tmp_path, dataset=path) + save(path, message="Created temporary project dataset", files=['projects/tmp']) else: dl.install(path=tmp_path, source=url, dataset=path) tmp_ds = dl.Dataset(tmp_path) @@ -128,7 +128,7 @@ def import_project(path: str, url: str, owner: Union[str, None]=None, tags: Opti create_project(path, uuid, owner, tags, aliases, code) move_submodule(path, 'projects/tmp', 'projects/' + uuid) os.mkdir(path + '/import_scripts/' + uuid) - dl.save([db, path + '/projects/' + uuid], message="Import project from " + url, dataset=path) + save(path, message="Import project from " + url, files=['projects/' + uuid, 'backlogger.db']) else: dl.drop(tmp_path, reckless='kill') shutil.rmtree(tmp_path) diff --git a/corrlib/meas_io.py b/corrlib/meas_io.py index 4bf404b..645746d 100644 --- a/corrlib/meas_io.py +++ b/corrlib/meas_io.py @@ -8,7 +8,7 @@ from typing import Union from pyerrors import Obs, Corr, dump_object, load_object from hashlib import sha256 from .tools import cached -from .tracker import get +from .tracker import get, save import shutil from typing import Any @@ -96,7 +96,7 @@ def write_measurement(path: str, ensemble: str, measurement: dict[str, dict[str, pj.dump_dict_to_json(known_meas, file) files.append(path + '/backlogger.db') conn.close() - dl.save(files, message="Add measurements to database", dataset=path) + save(path, message="Add measurements to database", files=files) def load_record(path: str, meas_path: str) -> Union[Corr, Obs]: @@ -196,7 +196,7 @@ def drop_record(path: str, meas_path: str) -> None: del known_meas[sub_key] dl.unlock(file, dataset=path) pj.dump_dict_to_json(known_meas, file) - dl.save([db, file], message="Drop measurements to database", dataset=path) + save(path, message="Drop measurements to database", files=[db, file]) return else: raise ValueError("This measurement does not exist as a file!") diff --git a/corrlib/tracker.py b/corrlib/tracker.py index e645470..2603c78 100644 --- a/corrlib/tracker.py +++ b/corrlib/tracker.py @@ -39,7 +39,7 @@ def init(path: str) -> None: if tracker == 'datalad': dl.create(path) elif tracker == 'None': - os.path.makedirs(path, exist_ok=True) + os.makedirs(path, exist_ok=True) else: raise ValueError(f"Tracker {tracker} is not supported.") return From aa51d5d786787d017fedac85d707069cefb25318 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Thu, 4 Dec 2025 12:30:50 +0100 Subject: [PATCH 22/35] throw error if not config file found --- corrlib/tracker.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/corrlib/tracker.py b/corrlib/tracker.py index 2603c78..f85ac39 100644 --- a/corrlib/tracker.py +++ b/corrlib/tracker.py @@ -9,6 +9,8 @@ def get_tracker(path: str) -> str: config = ConfigParser() if os.path.exists(config_path): config.read(config_path) + else: + raise FileNotFoundError(f"No config file found in {path}.") tracker = config.get('core', 'tracker', fallback='datalad') return tracker From 821bc14f4b59deb7d725b5df57156546b4aa380a Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Thu, 4 Dec 2025 12:36:41 +0100 Subject: [PATCH 23/35] avoid looking for a tracker before config exists --- corrlib/initialization.py | 6 +++--- corrlib/tracker.py | 3 +-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/corrlib/initialization.py b/corrlib/initialization.py index 13b6e66..fd1ad07 100644 --- a/corrlib/initialization.py +++ b/corrlib/initialization.py @@ -57,14 +57,14 @@ def _create_config(path: str) -> None: return -def create(path: str) -> None: +def create(path: str, tracker = 'datalad') -> None: """ Create folder of backlogs. """ - init(path) + init(path, tracker) _create_db(os.path.join(path, 'backlogger.db')) - os.chmod(os.path.join(path, 'backlogger.db'), 0o666) # why does this not work? + os.chmod(os.path.join(path, 'backlogger.db'), 0o666) _create_config(path) os.makedirs(os.path.join(path, 'projects')) os.makedirs(os.path.join(path, 'archive')) diff --git a/corrlib/tracker.py b/corrlib/tracker.py index f85ac39..d41d2d8 100644 --- a/corrlib/tracker.py +++ b/corrlib/tracker.py @@ -36,8 +36,7 @@ def save(path: str, message: str, files: Optional[list[str]]=None) -> None: raise ValueError(f"Tracker {tracker} is not supported.") -def init(path: str) -> None: - tracker = get_tracker(path) +def init(path: str, tracker: str='datalad') -> None: if tracker == 'datalad': dl.create(path) elif tracker == 'None': From c1aef6cdf262d968b632d1b20e28ea0e7269dce5 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Thu, 4 Dec 2025 12:39:30 +0100 Subject: [PATCH 24/35] expose tracker option in cli --- corrlib/cli.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/corrlib/cli.py b/corrlib/cli.py index 6f0d8cf..e9e8ebc 100644 --- a/corrlib/cli.py +++ b/corrlib/cli.py @@ -152,12 +152,17 @@ def init( str('./corrlib'), "--dataset", "-d", - ), -) -> None: + ), + tracker: str = typer.Option( + str('datalad'), + "--tracker", + "-t", + ), + ) -> None: """ Initialize a new backlog-database. """ - create(path) + create(path, tracker) return @@ -167,8 +172,8 @@ def drop_cache( str('./corrlib'), "--dataset", "-d", - ), -) -> None: + ), + ) -> None: """ Drop the currect cache directory of the dataset. """ From 1c06383f76810e0eaac251352302155e29accfb5 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Thu, 4 Dec 2025 12:41:14 +0100 Subject: [PATCH 25/35] nicer indents --- corrlib/cli.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/corrlib/cli.py b/corrlib/cli.py index e9e8ebc..3f4eb8f 100644 --- a/corrlib/cli.py +++ b/corrlib/cli.py @@ -26,9 +26,9 @@ def update( str('./corrlib'), "--dataset", "-d", - ), + ), uuid: str = typer.Argument(), -) -> None: + ) -> None: """ Update a project by it's UUID. """ @@ -43,7 +43,7 @@ def list( "-d", ), entities: str = typer.Argument('ensembles'), -) -> None: + ) -> None: """ List entities. (ensembles, projects) """ @@ -72,10 +72,10 @@ def alias_add( str('./corrlib'), "--dataset", "-d", - ), + ), uuid: str = typer.Argument(), alias: str = typer.Argument(), -) -> None: + ) -> None: """ Add an alias to a project UUID. """ @@ -90,11 +90,11 @@ def find( str('./corrlib'), "--dataset", "-d", - ), + ), ensemble: str = typer.Argument(), corr: str = typer.Argument(), code: str = typer.Argument(), -) -> None: + ) -> None: """ Find a record in the backlog at hand. Through specifying it's ensemble and the measured correlator. """ @@ -108,15 +108,15 @@ def importer( str('./corrlib'), "--dataset", "-d", - ), + ), files: str = typer.Argument( - ), + ), copy_file: bool = typer.Option( bool(True), "--save", "-s", - ), -) -> None: + ), + ) -> None: """ Import a project from a .toml-file via CLI. """ @@ -190,6 +190,6 @@ def main( help="Show the application's version and exit.", callback=_version_callback, is_eager=True, - ) -) -> None: + ) + ) -> None: return From f83eab785c0bf40b63b64a4cca6d8c9a8b5f4e2e Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Thu, 4 Dec 2025 12:42:17 +0100 Subject: [PATCH 26/35] typing --- corrlib/initialization.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/corrlib/initialization.py b/corrlib/initialization.py index fd1ad07..b46010a 100644 --- a/corrlib/initialization.py +++ b/corrlib/initialization.py @@ -57,7 +57,7 @@ def _create_config(path: str) -> None: return -def create(path: str, tracker = 'datalad') -> None: +def create(path: str, tracker: str = 'datalad') -> None: """ Create folder of backlogs. From 4b552276422aa7c90bd71ae1d545b70575ef5e4f Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Thu, 4 Dec 2025 12:56:02 +0100 Subject: [PATCH 27/35] use config in initialization --- corrlib/initialization.py | 42 ++++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/corrlib/initialization.py b/corrlib/initialization.py index b46010a..8aa8287 100644 --- a/corrlib/initialization.py +++ b/corrlib/initialization.py @@ -36,7 +36,7 @@ def _create_db(db: str) -> None: return -def _create_config(path: str) -> None: +def _create_config(path: str, tracker: str, cached: bool) -> ConfigParser: """ Create the config file for backlogger. @@ -44,32 +44,42 @@ def _create_config(path: str) -> None: config = ConfigParser() config['core'] = { 'version': '1.0', - 'db_path': os.path.join(path, 'backlogger.db'), - 'projects_path': os.path.join(path, 'projects'), - 'archive_path': os.path.join(path, 'archive'), - 'toml_imports_path': os.path.join(path, 'toml_imports'), - 'import_scripts_path': os.path.join(path, 'import_scripts'), - 'tracker': 'datalad', - 'cached': 'True', + 'tracker': tracker, + 'cached': str(cached), } + config['paths'] = { + 'db': 'backlogger.db', + 'projects_path': 'projects', + 'archive_path': 'archive', + 'toml_imports_path': 'toml_imports', + 'import_scripts_path': 'import_scripts', + } + return config + + +def _write_config(path: str, config: ConfigParser) -> None: + """ + Write the config file to disk. + """ with open(os.path.join(path, '.corrlib'), 'w') as configfile: config.write(configfile) return -def create(path: str, tracker: str = 'datalad') -> None: +def create(path: str, tracker: str = 'datalad', cached: bool = True) -> None: """ Create folder of backlogs. """ + config = _create_config(path, tracker, cached) init(path, tracker) - _create_db(os.path.join(path, 'backlogger.db')) - os.chmod(os.path.join(path, 'backlogger.db'), 0o666) - _create_config(path) - os.makedirs(os.path.join(path, 'projects')) - os.makedirs(os.path.join(path, 'archive')) - os.makedirs(os.path.join(path, 'toml_imports')) - os.makedirs(os.path.join(path, 'import_scripts/template.py')) + _write_config(path, config) + _create_db(os.path.join(path, config['paths']['db'])) + os.chmod(os.path.join(path, config['paths']['db']), 0o666) + os.makedirs(os.path.join(path, config['paths']['projects_path'])) + os.makedirs(os.path.join(path, config['paths']['archive_path'])) + os.makedirs(os.path.join(path, config['paths']['toml_imports_path'])) + os.makedirs(os.path.join(path, config['paths']['import_scripts_path'], 'template.py')) with open(os.path.join(path, ".gitignore"), "w") as fp: fp.write(".cache") fp.close() From 0626b34337686ca25f904ffeb47b8d0a767d952d Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Thu, 4 Dec 2025 14:31:53 +0100 Subject: [PATCH 28/35] implement dynamic db name from config --- corrlib/find.py | 19 +++++++++++-------- corrlib/main.py | 28 ++++++++++++++++------------ corrlib/meas_io.py | 16 +++++++++------- corrlib/toml.py | 3 +++ corrlib/tools.py | 21 +++++++++++++++++++++ 5 files changed, 60 insertions(+), 27 deletions(-) diff --git a/corrlib/find.py b/corrlib/find.py index ac38044..901c09c 100644 --- a/corrlib/find.py +++ b/corrlib/find.py @@ -4,9 +4,10 @@ import json import pandas as pd import numpy as np from .input.implementations import codes -from .tools import k2m +from .tools import k2m, get_db_file from .tracker import get from typing import Any, Optional + # this will implement the search functionality @@ -143,10 +144,11 @@ def sfcf_filter(results: pd.DataFrame, **kwargs: Any) -> pd.DataFrame: def find_record(path: str, ensemble: str, correlator_name: str, code: str, project: Optional[str]=None, parameters: Optional[str]=None, created_before: Optional[str]=None, created_after: Optional[str]=None, updated_before: Optional[str]=None, updated_after: Optional[str]=None, revision: Optional[str]=None, **kwargs: Any) -> pd.DataFrame: - db = path + '/backlogger.db' + db_file = get_db_file(path) + db = os.path.join(path, db_file) if code not in codes: raise ValueError("Code " + code + "unknown, take one of the following:" + ", ".join(codes)) - get(path, "backlogger.db") + get(path, db_file) results = _db_lookup(db, ensemble, correlator_name,code, project, parameters=parameters, created_before=created_before, created_after=created_after, updated_before=updated_before, updated_after=updated_after) if code == "sfcf": results = sfcf_filter(results, **kwargs) @@ -155,14 +157,15 @@ def find_record(path: str, ensemble: str, correlator_name: str, code: str, proje def find_project(path: str, name: str) -> str: - get(path, "backlogger.db") - return _project_lookup_by_alias(os.path.join(path, "backlogger.db"), name) + db_file = get_db_file(path) + get(path, db_file) + return _project_lookup_by_alias(os.path.join(path, db_file), name) def list_projects(path: str) -> list[tuple[str, str]]: - db = path + '/backlogger.db' - get(path, "backlogger.db") - conn = sqlite3.connect(db) + db_file = get_db_file(path) + get(path, db_file) + conn = sqlite3.connect(os.path.join(path, db_file)) c = conn.cursor() c.execute("SELECT id,aliases FROM projects") results = c.fetchall() diff --git a/corrlib/main.py b/corrlib/main.py index dfed6ea..c4f2832 100644 --- a/corrlib/main.py +++ b/corrlib/main.py @@ -5,7 +5,7 @@ import os from .git_tools import move_submodule import shutil from .find import _project_lookup_by_id -from .tools import list2str, str2list +from .tools import list2str, str2list, get_db_file from .tracker import get, save from typing import Union, Optional @@ -25,8 +25,9 @@ def create_project(path: str, uuid: str, owner: Union[str, None]=None, tags: Uni code: str (optional) The code that was used to create the measurements. """ - db = path + "/backlogger.db" - get(path, "backlogger.db") + db_file = get_db_file(path) + db = os.path.join(path, db_file) + get(path, db_file) conn = sqlite3.connect(db) c = conn.cursor() known_projects = c.execute("SELECT * FROM projects WHERE id=?", (uuid,)) @@ -43,12 +44,13 @@ def create_project(path: str, uuid: str, owner: Union[str, None]=None, tags: Uni c.execute("INSERT INTO projects (id, aliases, customTags, owner, code, created_at, updated_at) VALUES (?, ?, ?, ?, ?, datetime('now'), datetime('now'))", (uuid, alias_str, tag_str, owner, code)) conn.commit() conn.close() - save(path, message="Added entry for project " + uuid + " to database", files=["backlogger.db"]) + save(path, message="Added entry for project " + uuid + " to database", files=[db_file]) def update_project_data(path: str, uuid: str, prop: str, value: Union[str, None] = None) -> None: - get(path, "backlogger.db") - conn = sqlite3.connect(os.path.join(path, "backlogger.db")) + db_file = get_db_file(path) + get(path, db_file) + conn = sqlite3.connect(os.path.join(path, db_file)) c = conn.cursor() c.execute(f"UPDATE projects SET '{prop}' = '{value}' WHERE id == '{uuid}'") conn.commit() @@ -57,8 +59,8 @@ def update_project_data(path: str, uuid: str, prop: str, value: Union[str, None] def update_aliases(path: str, uuid: str, aliases: list[str]) -> None: - db = os.path.join(path, "backlogger.db") - get(path, "backlogger.db") + db_file = get_db_file(path) + get(path, db_file) known_data = _project_lookup_by_id(db, uuid)[0] known_aliases = known_data[1] @@ -77,9 +79,10 @@ def update_aliases(path: str, uuid: str, aliases: list[str]) -> None: if not len(new_alias_list) == len(known_alias_list): alias_str = list2str(new_alias_list) + db = os.path.join(path, db_file) dl.unlock(db, dataset=path) update_project_data(path, uuid, "aliases", alias_str) - save(path, message="Updated aliases for project " + uuid, files=["backlogger.db"]) + save(path, message="Updated aliases for project " + uuid, files=[db_file]) return @@ -122,13 +125,14 @@ def import_project(path: str, url: str, owner: Union[str, None]=None, tags: Opti if not uuid: raise ValueError("The dataset does not have a uuid!") if not os.path.exists(path + "/projects/" + uuid): - db = path + "/backlogger.db" - get(path, "backlogger.db") + db_file = get_db_file(path) + db = os.path.join(path, db_file) + get(path, db_file) dl.unlock(db, dataset=path) create_project(path, uuid, owner, tags, aliases, code) move_submodule(path, 'projects/tmp', 'projects/' + uuid) os.mkdir(path + '/import_scripts/' + uuid) - save(path, message="Import project from " + url, files=['projects/' + uuid, 'backlogger.db']) + save(path, message="Import project from " + url, files=['projects/' + uuid, db_file]) else: dl.drop(tmp_path, reckless='kill') shutil.rmtree(tmp_path) diff --git a/corrlib/meas_io.py b/corrlib/meas_io.py index 645746d..d17750e 100644 --- a/corrlib/meas_io.py +++ b/corrlib/meas_io.py @@ -7,7 +7,7 @@ import json from typing import Union from pyerrors import Obs, Corr, dump_object, load_object from hashlib import sha256 -from .tools import cached +from .tools import get_db_file, cache_enabled from .tracker import get, save import shutil from typing import Any @@ -29,8 +29,9 @@ def write_measurement(path: str, ensemble: str, measurement: dict[str, dict[str, uuid: str The uuid of the project. """ - db = os.path.join(path, 'backlogger.db') - get(path, "backlogger.db") + db_file = get_db_file(path) + db = os.path.join(path, db_file) + get(path, db_file) dl.unlock(db, dataset=path) conn = sqlite3.connect(db) c = conn.cursor() @@ -94,7 +95,7 @@ def write_measurement(path: str, ensemble: str, measurement: dict[str, dict[str, (corr, ensemble, code, meas_path, uuid, pars[subkey], parameter_file)) conn.commit() pj.dump_dict_to_json(known_meas, file) - files.append(path + '/backlogger.db') + files.append(os.path.join(path, db_file)) conn.close() save(path, message="Add measurements to database", files=files) @@ -149,7 +150,7 @@ def load_records(path: str, meas_paths: list[str], preloaded: dict[str, Any] = { if file not in preloaded: preloaded[file] = preload(path, file) returned_data.append(preloaded[file][key]) - if cached: + if cache_enabled(path): if not os.path.exists(cache_dir(path, file)): os.makedirs(cache_dir(path, file)) dump_object(preloaded[file][key], cache_path(path, file, key)) @@ -179,8 +180,9 @@ def preload(path: str, file: str) -> dict[str, Any]: def drop_record(path: str, meas_path: str) -> None: file_in_archive = meas_path.split("::")[0] file = os.path.join(path, file_in_archive) - db = os.path.join(path, 'backlogger.db') - get(path, 'backlogger.db') + db_file = get_db_file(path) + db = os.path.join(path, db_file) + get(path, db_file) sub_key = meas_path.split("::")[1] dl.unlock(db, dataset=path) conn = sqlite3.connect(db) diff --git a/corrlib/toml.py b/corrlib/toml.py index 7b02e33..c1c4d5b 100644 --- a/corrlib/toml.py +++ b/corrlib/toml.py @@ -20,6 +20,7 @@ import os from .input.implementations import codes as known_codes from typing import Any + def replace_string(string: str, name: str, val: str) -> str: if '{' + name + '}' in string: n = string.replace('{' + name + '}', val) @@ -27,6 +28,7 @@ def replace_string(string: str, name: str, val: str) -> str: else: return string + def replace_in_meas(measurements: dict[str, dict[str, Any]], vars: dict[str, str]) -> dict[str, dict[str, Any]]: # replace global variables for name, value in vars.items(): @@ -39,6 +41,7 @@ def replace_in_meas(measurements: dict[str, dict[str, Any]], vars: dict[str, str measurements[m][key][i] = replace_string(measurements[m][key][i], name, value) return measurements + def fill_cons(measurements: dict[str, dict[str, Any]], constants: dict[str, str]) -> dict[str, dict[str, Any]]: for m in measurements.keys(): for name, val in constants.items(): diff --git a/corrlib/tools.py b/corrlib/tools.py index 77bfd2e..9c39d7c 100644 --- a/corrlib/tools.py +++ b/corrlib/tools.py @@ -2,6 +2,8 @@ import os from configparser import ConfigParser from typing import Any +CONFIG_FILENAME = ".corrlib" + def str2list(string: str) -> list[str]: return string.split(",") @@ -31,3 +33,22 @@ def set_config(path: str, section: str, option: str, value: Any) -> None: with open(config_path, 'w') as configfile: config.write(configfile) return + + +def get_db_file(path: str) -> str: + config_path = os.path.join(path, CONFIG_FILENAME) + config = ConfigParser() + if os.path.exists(config_path): + config.read(config_path) + db_file = config.get('paths', 'db', fallback='backlogger.db') + return db_file + + +def cache_enabled(path: str) -> bool: + config_path = os.path.join(path, CONFIG_FILENAME) + config = ConfigParser() + if os.path.exists(config_path): + config.read(config_path) + cached_str = config.get('core', 'cached', fallback='True') + cached_bool = cached_str == ('True') + return cached_bool From 303dbdd2dc52c461fc9a0c6be1df13fbc45fe349 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Thu, 4 Dec 2025 14:34:06 +0100 Subject: [PATCH 29/35] be more careful with definitions --- corrlib/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/corrlib/main.py b/corrlib/main.py index c4f2832..f6e3e0e 100644 --- a/corrlib/main.py +++ b/corrlib/main.py @@ -60,6 +60,7 @@ def update_project_data(path: str, uuid: str, prop: str, value: Union[str, None] def update_aliases(path: str, uuid: str, aliases: list[str]) -> None: db_file = get_db_file(path) + db = os.path.join(path, db_file) get(path, db_file) known_data = _project_lookup_by_id(db, uuid)[0] known_aliases = known_data[1] @@ -79,7 +80,6 @@ def update_aliases(path: str, uuid: str, aliases: list[str]) -> None: if not len(new_alias_list) == len(known_alias_list): alias_str = list2str(new_alias_list) - db = os.path.join(path, db_file) dl.unlock(db, dataset=path) update_project_data(path, uuid, "aliases", alias_str) save(path, message="Updated aliases for project " + uuid, files=[db_file]) @@ -149,7 +149,7 @@ def import_project(path: str, url: str, owner: Union[str, None]=None, tags: Opti def drop_project_data(path: str, uuid: str, path_in_project: str = "") -> None: """ - Drop (parts of) a prject to free up diskspace + Drop (parts of) a project to free up diskspace """ dl.drop(path + "/projects/" + uuid + "/" + path_in_project) return From 0be5cb18e239c085c7f1af42619a0150a8e94ff7 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Thu, 4 Dec 2025 14:44:05 +0100 Subject: [PATCH 30/35] add more simple init tests --- tests/test_initialization.py | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/tests/test_initialization.py b/tests/test_initialization.py index 38357bf..966bd07 100644 --- a/tests/test_initialization.py +++ b/tests/test_initialization.py @@ -9,6 +9,29 @@ def test_init_folders(tmp_path): assert os.path.exists(str(dataset_path / "backlogger.db")) +def test_init_folders_no_tracker(tmp_path): + dataset_path = tmp_path / "test_dataset" + init.create(str(dataset_path), tracker="None") + assert os.path.exists(str(dataset_path)) + assert os.path.exists(str(dataset_path / "backlogger.db")) + + +def test_init_config(tmp_path): + dataset_path = tmp_path / "test_dataset" + init.create(str(dataset_path), tracker="None") + config_path = dataset_path / ".corrlib" + assert os.path.exists(str(config_path)) + from configparser import ConfigParser + config = ConfigParser() + config.read(str(config_path)) + assert config.get("core", "tracker") == "None" + assert config.get("paths", "db") == "backlogger.db" + assert config.get("paths", "projects_path") == "projects" + assert config.get("paths", "archive_path") == "archive" + assert config.get("paths", "toml_imports_path") == "toml_imports" + assert config.get("paths", "import_scripts_path") == "import_scripts" + + def test_init_db(tmp_path): dataset_path = tmp_path / "test_dataset" init.create(str(dataset_path)) @@ -24,7 +47,7 @@ def test_init_db(tmp_path): table_names = [table[0] for table in tables] for expected_table in expected_tables: assert expected_table in table_names - + cursor.execute("SELECT * FROM projects;") projects = cursor.fetchall() assert len(projects) == 0 @@ -47,7 +70,7 @@ def test_init_db(tmp_path): project_column_names = [col[1] for col in project_columns] for expected_col in expected_project_columns: assert expected_col in project_column_names - + cursor.execute("PRAGMA table_info('backlogs');") backlog_columns = cursor.fetchall() expected_backlog_columns = [ From 3e6c7a4fdb2a6e1caf3f8d448378a9182802fae8 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Thu, 4 Dec 2025 14:46:16 +0100 Subject: [PATCH 31/35] assert more vars in config --- tests/test_initialization.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_initialization.py b/tests/test_initialization.py index 966bd07..1ea0ece 100644 --- a/tests/test_initialization.py +++ b/tests/test_initialization.py @@ -25,6 +25,8 @@ def test_init_config(tmp_path): config = ConfigParser() config.read(str(config_path)) assert config.get("core", "tracker") == "None" + assert config.get("core", "version") == "1.0" + assert config.get("core", "cached") == "True" assert config.get("paths", "db") == "backlogger.db" assert config.get("paths", "projects_path") == "projects" assert config.get("paths", "archive_path") == "archive" From bc57087a5aa9d9a22ce888a48216abd43fa4b64f Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Thu, 4 Dec 2025 15:14:28 +0100 Subject: [PATCH 32/35] remove temporary non-datalad implementation --- corrlib/main.py | 18 +++++-------- corrlib/tracker.py | 50 ++++++++++++++++++++++++++++++++++--- corrlib/trackers/datalad.py | 25 ------------------- 3 files changed, 53 insertions(+), 40 deletions(-) delete mode 100644 corrlib/trackers/datalad.py diff --git a/corrlib/main.py b/corrlib/main.py index f6e3e0e..d2cbc6a 100644 --- a/corrlib/main.py +++ b/corrlib/main.py @@ -6,7 +6,7 @@ from .git_tools import move_submodule import shutil from .find import _project_lookup_by_id from .tools import list2str, str2list, get_db_file -from .tracker import get, save +from .tracker import get, save, unlock, init, clone, drop from typing import Union, Optional @@ -34,7 +34,7 @@ def create_project(path: str, uuid: str, owner: Union[str, None]=None, tags: Uni if known_projects.fetchone(): raise ValueError("Project already imported, use update_project() instead.") - dl.unlock(db, dataset=path) + unlock(path, db_file) alias_str = "" if aliases is not None: alias_str = list2str(aliases) @@ -80,7 +80,7 @@ def update_aliases(path: str, uuid: str, aliases: list[str]) -> None: if not len(new_alias_list) == len(known_alias_list): alias_str = list2str(new_alias_list) - dl.unlock(db, dataset=path) + unlock(path, db_file) update_project_data(path, uuid, "aliases", alias_str) save(path, message="Updated aliases for project " + uuid, files=[db_file]) return @@ -113,12 +113,7 @@ def import_project(path: str, url: str, owner: Union[str, None]=None, tags: Opti """ tmp_path = os.path.join(path, 'projects/tmp') - if not isDataset: - dl.create(tmp_path, dataset=path) - shutil.copytree(url + "/*", path + '/projects/tmp/') - save(path, message="Created temporary project dataset", files=['projects/tmp']) - else: - dl.install(path=tmp_path, source=url, dataset=path) + clone(path, source=url, target=tmp_path) tmp_ds = dl.Dataset(tmp_path) conf = dlc.ConfigManager(tmp_ds) uuid = str(conf.get("datalad.dataset.id")) @@ -126,9 +121,8 @@ def import_project(path: str, url: str, owner: Union[str, None]=None, tags: Opti raise ValueError("The dataset does not have a uuid!") if not os.path.exists(path + "/projects/" + uuid): db_file = get_db_file(path) - db = os.path.join(path, db_file) get(path, db_file) - dl.unlock(db, dataset=path) + unlock(path, db_file) create_project(path, uuid, owner, tags, aliases, code) move_submodule(path, 'projects/tmp', 'projects/' + uuid) os.mkdir(path + '/import_scripts/' + uuid) @@ -151,6 +145,6 @@ def drop_project_data(path: str, uuid: str, path_in_project: str = "") -> None: """ Drop (parts of) a project to free up diskspace """ - dl.drop(path + "/projects/" + uuid + "/" + path_in_project) + drop(path + "/projects/" + uuid + "/" + path_in_project) return diff --git a/corrlib/tracker.py b/corrlib/tracker.py index d41d2d8..565263a 100644 --- a/corrlib/tracker.py +++ b/corrlib/tracker.py @@ -1,7 +1,9 @@ import os from configparser import ConfigParser -from .trackers import datalad as dl +import datalad.api as dl from typing import Optional +import shutil +from .tools import get_db_file def get_tracker(path: str) -> str: @@ -18,7 +20,12 @@ def get_tracker(path: str) -> str: def get(path: str, file: str) -> None: tracker = get_tracker(path) if tracker == 'datalad': - dl.get(path, file) + if file == get_db_file(path): + print("Downloading database...") + else: + print("Downloading data...") + dl.get(os.path.join(path, file), dataset=path) + print("> downloaded file") elif tracker == 'None': pass else: @@ -29,7 +36,9 @@ def get(path: str, file: str) -> None: def save(path: str, message: str, files: Optional[list[str]]=None) -> None: tracker = get_tracker(path) if tracker == 'datalad': - dl.save(path, message, files) + if files is not None: + files = [os.path.join(path, f) for f in files] + dl.save(files, message=message, dataset=path) elif tracker == 'None': pass else: @@ -44,3 +53,38 @@ def init(path: str, tracker: str='datalad') -> None: else: raise ValueError(f"Tracker {tracker} is not supported.") return + + +def unlock(path: str, file: str) -> None: + tracker = get_tracker(path) + if tracker == 'datalad': + dl.unlock(file, dataset=path) + elif tracker == 'None': + pass + else: + raise ValueError(f"Tracker {tracker} is not supported.") + return + + +def clone(path: str, source: str, target: str) -> None: + tracker = get_tracker(path) + if tracker == 'datalad': + dl.clone(target=target, source=source, dataset=path) + elif tracker == 'None': + os.makedirs(path, exist_ok=True) + # Implement a simple clone by copying files + shutil.copytree(source, target, dirs_exist_ok=False) + else: + raise ValueError(f"Tracker {tracker} is not supported.") + return + + +def drop(path: str, reckless: Optional[str]=None) -> None: + tracker = get_tracker(path) + if tracker == 'datalad': + dl.drop(path, reckless=reckless) + elif tracker == 'None': + shutil.rmtree(path) + else: + raise ValueError(f"Tracker {tracker} is not supported.") + return \ No newline at end of file diff --git a/corrlib/trackers/datalad.py b/corrlib/trackers/datalad.py deleted file mode 100644 index c4e3e70..0000000 --- a/corrlib/trackers/datalad.py +++ /dev/null @@ -1,25 +0,0 @@ -import datalad.api as dl -import os -from typing import Optional - - -def get(path: str, file: str) -> None: - if file == "backlogger.db": - print("Downloading database...") - else: - print("Downloading data...") - dl.get(os.path.join(path, file), dataset=path) - print("> downloaded file") - return - - -def save(path: str, message: str, files: Optional[list[str]]=None) -> None: - if files is not None: - files = [os.path.join(path, f) for f in files] - dl.save(files, message=message, dataset=path) - return - - -def create(path: str) -> None: - dl.create(path) - return From 00ec9f7f8a7b19747e4dbdf29cca58200e95319f Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Thu, 4 Dec 2025 15:15:24 +0100 Subject: [PATCH 33/35] roll out tracker unlock implementation --- corrlib/meas_io.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/corrlib/meas_io.py b/corrlib/meas_io.py index d17750e..aec7891 100644 --- a/corrlib/meas_io.py +++ b/corrlib/meas_io.py @@ -8,7 +8,7 @@ from typing import Union from pyerrors import Obs, Corr, dump_object, load_object from hashlib import sha256 from .tools import get_db_file, cache_enabled -from .tracker import get, save +from .tracker import get, save, unlock import shutil from typing import Any @@ -32,7 +32,7 @@ def write_measurement(path: str, ensemble: str, measurement: dict[str, dict[str, db_file = get_db_file(path) db = os.path.join(path, db_file) get(path, db_file) - dl.unlock(db, dataset=path) + unlock(path, db_file) conn = sqlite3.connect(db) c = conn.cursor() files = [] @@ -45,7 +45,7 @@ def write_measurement(path: str, ensemble: str, measurement: dict[str, dict[str, os.makedirs(os.path.join(path, '.', 'archive', ensemble, corr)) else: if os.path.exists(file): - dl.unlock(file, dataset=path) + unlock(path, file_in_archive) known_meas = pj.load_json_dict(file) if code == "sfcf": parameters = sfcf.read_param(path, uuid, parameter_file) @@ -184,7 +184,7 @@ def drop_record(path: str, meas_path: str) -> None: db = os.path.join(path, db_file) get(path, db_file) sub_key = meas_path.split("::")[1] - dl.unlock(db, dataset=path) + unlock(path, db_file) conn = sqlite3.connect(db) c = conn.cursor() if c.execute("SELECT * FROM backlogs WHERE path = ?", (meas_path, )).fetchone() is not None: @@ -196,7 +196,7 @@ def drop_record(path: str, meas_path: str) -> None: known_meas = pj.load_json_dict(file) if sub_key in known_meas: del known_meas[sub_key] - dl.unlock(file, dataset=path) + unlock(path, file_in_archive) pj.dump_dict_to_json(known_meas, file) save(path, message="Drop measurements to database", files=[db, file]) return From df25acfe0a375386431f320e6355342fc4a03e7d Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Thu, 4 Dec 2025 15:25:28 +0100 Subject: [PATCH 34/35] lint --- corrlib/main.py | 2 +- corrlib/meas_io.py | 1 - corrlib/tracker.py | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/corrlib/main.py b/corrlib/main.py index d2cbc6a..24d5103 100644 --- a/corrlib/main.py +++ b/corrlib/main.py @@ -6,7 +6,7 @@ from .git_tools import move_submodule import shutil from .find import _project_lookup_by_id from .tools import list2str, str2list, get_db_file -from .tracker import get, save, unlock, init, clone, drop +from .tracker import get, save, unlock, clone, drop from typing import Union, Optional diff --git a/corrlib/meas_io.py b/corrlib/meas_io.py index aec7891..a00079e 100644 --- a/corrlib/meas_io.py +++ b/corrlib/meas_io.py @@ -1,6 +1,5 @@ from pyerrors.input import json as pj import os -import datalad.api as dl import sqlite3 from .input import sfcf,openQCD import json diff --git a/corrlib/tracker.py b/corrlib/tracker.py index 565263a..5cc07de 100644 --- a/corrlib/tracker.py +++ b/corrlib/tracker.py @@ -87,4 +87,4 @@ def drop(path: str, reckless: Optional[str]=None) -> None: shutil.rmtree(path) else: raise ValueError(f"Tracker {tracker} is not supported.") - return \ No newline at end of file + return From 8f8f9b472a76bdbb50d595febe834cb34e3ed570 Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Thu, 4 Dec 2025 15:38:31 +0100 Subject: [PATCH 35/35] docstrings --- corrlib/tracker.py | 81 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 80 insertions(+), 1 deletion(-) diff --git a/corrlib/tracker.py b/corrlib/tracker.py index 5cc07de..5cc281c 100644 --- a/corrlib/tracker.py +++ b/corrlib/tracker.py @@ -7,6 +7,19 @@ from .tools import get_db_file def get_tracker(path: str) -> str: + """ + Get the tracker used in the dataset located at path. + + Parameters + ---------- + path: str + The path to the backlogger folder. + + Returns + ------- + tracker: str + The tracker used in the dataset. + """ config_path = os.path.join(path, '.corrlib') config = ConfigParser() if os.path.exists(config_path): @@ -18,6 +31,16 @@ def get_tracker(path: str) -> str: def get(path: str, file: str) -> None: + """ + Wrapper function to get a file from the dataset located at path with the specified tracker. + + Parameters + ---------- + path: str + The path to the backlogger folder. + file: str + The file to get. + """ tracker = get_tracker(path) if tracker == 'datalad': if file == get_db_file(path): @@ -34,18 +57,41 @@ def get(path: str, file: str) -> None: def save(path: str, message: str, files: Optional[list[str]]=None) -> None: + """ + Wrapper function to save a file to the dataset located at path with the specified tracker. + + Parameters + ---------- + path: str + The path to the backlogger folder. + message: str + The commit message. + files: list[str], optional + The files to save. If None, all changes are saved. + """ tracker = get_tracker(path) if tracker == 'datalad': if files is not None: files = [os.path.join(path, f) for f in files] dl.save(files, message=message, dataset=path) elif tracker == 'None': + Warning("Tracker 'None' does not implement save.") pass else: raise ValueError(f"Tracker {tracker} is not supported.") def init(path: str, tracker: str='datalad') -> None: + """ + Initialize a dataset at the specified path with the specified tracker. + + Parameters + ---------- + path: str + The path to initialize the dataset. + tracker: str + The tracker to use. Currently only 'datalad' and 'None' are supported. + """ if tracker == 'datalad': dl.create(path) elif tracker == 'None': @@ -56,10 +102,21 @@ def init(path: str, tracker: str='datalad') -> None: def unlock(path: str, file: str) -> None: + """ + Wrapper function to unlock a file in the dataset located at path with the specified tracker. + + Parameters + ---------- + path : str + The path to the backlogger folder. + file : str + The file to unlock. + """ tracker = get_tracker(path) if tracker == 'datalad': dl.unlock(file, dataset=path) elif tracker == 'None': + Warning("Tracker 'None' does not implement unlock.") pass else: raise ValueError(f"Tracker {tracker} is not supported.") @@ -67,6 +124,17 @@ def unlock(path: str, file: str) -> None: def clone(path: str, source: str, target: str) -> None: + """ + Wrapper function to clone a dataset from source to target with the specified tracker. + Parameters + ---------- + path: str + The path to the backlogger folder. + source: str + The source dataset to clone. + target: str + The target path to clone the dataset to. + """ tracker = get_tracker(path) if tracker == 'datalad': dl.clone(target=target, source=source, dataset=path) @@ -80,11 +148,22 @@ def clone(path: str, source: str, target: str) -> None: def drop(path: str, reckless: Optional[str]=None) -> None: + """ + Wrapper function to drop data from a dataset located at path with the specified tracker. + + Parameters + ---------- + path: str + The path to the backlogger folder. + reckless: Optional[str] + The datalad's reckless option for dropping data. + """ tracker = get_tracker(path) if tracker == 'datalad': dl.drop(path, reckless=reckless) elif tracker == 'None': - shutil.rmtree(path) + Warning("Tracker 'None' does not implement drop.") + pass else: raise ValueError(f"Tracker {tracker} is not supported.") return