From bc57087a5aa9d9a22ce888a48216abd43fa4b64f Mon Sep 17 00:00:00 2001 From: Justus Kuhlmann Date: Thu, 4 Dec 2025 15:14:28 +0100 Subject: [PATCH] remove temporary non-datalad implementation --- corrlib/main.py | 18 +++++-------- corrlib/tracker.py | 50 ++++++++++++++++++++++++++++++++++--- corrlib/trackers/datalad.py | 25 ------------------- 3 files changed, 53 insertions(+), 40 deletions(-) delete mode 100644 corrlib/trackers/datalad.py diff --git a/corrlib/main.py b/corrlib/main.py index f6e3e0e..d2cbc6a 100644 --- a/corrlib/main.py +++ b/corrlib/main.py @@ -6,7 +6,7 @@ from .git_tools import move_submodule import shutil from .find import _project_lookup_by_id from .tools import list2str, str2list, get_db_file -from .tracker import get, save +from .tracker import get, save, unlock, init, clone, drop from typing import Union, Optional @@ -34,7 +34,7 @@ def create_project(path: str, uuid: str, owner: Union[str, None]=None, tags: Uni if known_projects.fetchone(): raise ValueError("Project already imported, use update_project() instead.") - dl.unlock(db, dataset=path) + unlock(path, db_file) alias_str = "" if aliases is not None: alias_str = list2str(aliases) @@ -80,7 +80,7 @@ def update_aliases(path: str, uuid: str, aliases: list[str]) -> None: if not len(new_alias_list) == len(known_alias_list): alias_str = list2str(new_alias_list) - dl.unlock(db, dataset=path) + unlock(path, db_file) update_project_data(path, uuid, "aliases", alias_str) save(path, message="Updated aliases for project " + uuid, files=[db_file]) return @@ -113,12 +113,7 @@ def import_project(path: str, url: str, owner: Union[str, None]=None, tags: Opti """ tmp_path = os.path.join(path, 'projects/tmp') - if not isDataset: - dl.create(tmp_path, dataset=path) - shutil.copytree(url + "/*", path + '/projects/tmp/') - save(path, message="Created temporary project dataset", files=['projects/tmp']) - else: - dl.install(path=tmp_path, source=url, dataset=path) + clone(path, source=url, target=tmp_path) tmp_ds = dl.Dataset(tmp_path) conf = dlc.ConfigManager(tmp_ds) uuid = str(conf.get("datalad.dataset.id")) @@ -126,9 +121,8 @@ def import_project(path: str, url: str, owner: Union[str, None]=None, tags: Opti raise ValueError("The dataset does not have a uuid!") if not os.path.exists(path + "/projects/" + uuid): db_file = get_db_file(path) - db = os.path.join(path, db_file) get(path, db_file) - dl.unlock(db, dataset=path) + unlock(path, db_file) create_project(path, uuid, owner, tags, aliases, code) move_submodule(path, 'projects/tmp', 'projects/' + uuid) os.mkdir(path + '/import_scripts/' + uuid) @@ -151,6 +145,6 @@ def drop_project_data(path: str, uuid: str, path_in_project: str = "") -> None: """ Drop (parts of) a project to free up diskspace """ - dl.drop(path + "/projects/" + uuid + "/" + path_in_project) + drop(path + "/projects/" + uuid + "/" + path_in_project) return diff --git a/corrlib/tracker.py b/corrlib/tracker.py index d41d2d8..565263a 100644 --- a/corrlib/tracker.py +++ b/corrlib/tracker.py @@ -1,7 +1,9 @@ import os from configparser import ConfigParser -from .trackers import datalad as dl +import datalad.api as dl from typing import Optional +import shutil +from .tools import get_db_file def get_tracker(path: str) -> str: @@ -18,7 +20,12 @@ def get_tracker(path: str) -> str: def get(path: str, file: str) -> None: tracker = get_tracker(path) if tracker == 'datalad': - dl.get(path, file) + if file == get_db_file(path): + print("Downloading database...") + else: + print("Downloading data...") + dl.get(os.path.join(path, file), dataset=path) + print("> downloaded file") elif tracker == 'None': pass else: @@ -29,7 +36,9 @@ def get(path: str, file: str) -> None: def save(path: str, message: str, files: Optional[list[str]]=None) -> None: tracker = get_tracker(path) if tracker == 'datalad': - dl.save(path, message, files) + if files is not None: + files = [os.path.join(path, f) for f in files] + dl.save(files, message=message, dataset=path) elif tracker == 'None': pass else: @@ -44,3 +53,38 @@ def init(path: str, tracker: str='datalad') -> None: else: raise ValueError(f"Tracker {tracker} is not supported.") return + + +def unlock(path: str, file: str) -> None: + tracker = get_tracker(path) + if tracker == 'datalad': + dl.unlock(file, dataset=path) + elif tracker == 'None': + pass + else: + raise ValueError(f"Tracker {tracker} is not supported.") + return + + +def clone(path: str, source: str, target: str) -> None: + tracker = get_tracker(path) + if tracker == 'datalad': + dl.clone(target=target, source=source, dataset=path) + elif tracker == 'None': + os.makedirs(path, exist_ok=True) + # Implement a simple clone by copying files + shutil.copytree(source, target, dirs_exist_ok=False) + else: + raise ValueError(f"Tracker {tracker} is not supported.") + return + + +def drop(path: str, reckless: Optional[str]=None) -> None: + tracker = get_tracker(path) + if tracker == 'datalad': + dl.drop(path, reckless=reckless) + elif tracker == 'None': + shutil.rmtree(path) + else: + raise ValueError(f"Tracker {tracker} is not supported.") + return \ No newline at end of file diff --git a/corrlib/trackers/datalad.py b/corrlib/trackers/datalad.py deleted file mode 100644 index c4e3e70..0000000 --- a/corrlib/trackers/datalad.py +++ /dev/null @@ -1,25 +0,0 @@ -import datalad.api as dl -import os -from typing import Optional - - -def get(path: str, file: str) -> None: - if file == "backlogger.db": - print("Downloading database...") - else: - print("Downloading data...") - dl.get(os.path.join(path, file), dataset=path) - print("> downloaded file") - return - - -def save(path: str, message: str, files: Optional[list[str]]=None) -> None: - if files is not None: - files = [os.path.join(path, f) for f in files] - dl.save(files, message=message, dataset=path) - return - - -def create(path: str) -> None: - dl.create(path) - return