integ/links #34

Merged
jkuhl merged 6 commits from integ/links into develop 2026-04-17 18:09:18 +02:00
Showing only changes of commit 083d7ee3ce - Show all commits

add dry run for loading data using the integrity functions
Some checks failed
Pytest / pytest (3.12) (push) Waiting to run
Pytest / pytest (3.13) (push) Waiting to run
Pytest / pytest (3.14) (push) Waiting to run
Ruff / ruff (push) Waiting to run
Mypy / mypy (pull_request) Waiting to run
Pytest / pytest (3.12) (pull_request) Waiting to run
Pytest / pytest (3.13) (pull_request) Waiting to run
Pytest / pytest (3.14) (pull_request) Waiting to run
Ruff / ruff (pull_request) Waiting to run
Mypy / mypy (push) Has been cancelled

Justus Kuhlmann 2026-04-17 17:52:18 +02:00
Signed by: jkuhl
GPG key ID: 00ED992DD79B85A6

View file

@ -11,6 +11,7 @@ from .tracker import get, save, unlock
import shutil
from typing import Any
from pathlib import Path
from .integrity import _check_db2paths
CACHE_DIR = ".cache"
@ -153,7 +154,7 @@ def load_record(path: Path, meas_path: str) -> Union[Corr, Obs]:
return load_records(path, [meas_path])[0]
def load_records(path: Path, meas_paths: list[str], preloaded: dict[str, Any] = {}) -> list[Union[Corr, Obs]]:
def load_records(path: Path, meas_paths: list[str], preloaded: dict[str, Any] = {}, dry_run: bool = False) -> list[Union[Corr, Obs]]:
"""
Load a list of records by their paths.
@ -163,14 +164,19 @@ def load_records(path: Path, meas_paths: list[str], preloaded: dict[str, Any] =
Path of the correlator library.
meas_paths: list[str]
A list of the paths to the correlator in the backlog system.
perloaded: dict[str, Any]
The data that is already prelaoded. Of interest if data has alread been loaded in the same script.
preloaded: dict[str, Any]
The data that is already preloaded. Of interest if data has alread been loaded in the same script.
dry_run: bool
Do not load datda, just check whether we can reach the data we are interested in.
Returns
-------
retruned_data: list
returned_data: list
The loaded records.
"""
if dry_run:
_check_db2paths(path, meas_paths)
return
needed_data: dict[str, list[str]] = {}
for mpath in meas_paths:
file = mpath.split("::")[0]