diff --git a/.github/workflows/mypy.yaml b/.github/workflows/mypy.yaml index 791243f..fbd51ec 100644 --- a/.github/workflows/mypy.yaml +++ b/.github/workflows/mypy.yaml @@ -8,22 +8,21 @@ on: jobs: mypy: runs-on: ubuntu-latest - env: - UV_CACHE_DIR: /tmp/.uv-cache steps: - name: Install git-annex run: | sudo apt-get update - sudo apt-get install -y git-annex + sudo apt-get install -y git-annex - name: Check out the repository uses: https://github.com/RouxAntoine/checkout@v4.1.8 with: show-progress: true - - name: Install uv - uses: astral-sh/setup-uv@v7 + - name: Setup python + uses: https://github.com/actions/setup-python@v5 with: - python-version: ${{ matrix.python-version }} - enable-cache: true + python-version: "3.12" + - name: Install uv + uses: https://github.com/astral-sh/setup-uv@v5 - name: Install corrlib run: uv sync --locked --all-extras --dev --python "3.12" - name: Run tests diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index 1fcb8fe..b1a4d94 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -17,9 +17,11 @@ jobs: - "3.14" runs-on: ubuntu-latest - env: - UV_CACHE_DIR: /tmp/.uv-cache steps: + - name: Setup git + run: | + git config --global user.email "tester@example.com" + git config --global user.name "Tester" - name: Install git-annex run: | sudo apt-get update @@ -28,11 +30,12 @@ jobs: uses: https://github.com/RouxAntoine/checkout@v4.1.8 with: show-progress: true - - name: Install uv - uses: astral-sh/setup-uv@v7 + - name: Setup python + uses: https://github.com/actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - enable-cache: true + - name: Install uv + uses: https://github.com/astral-sh/setup-uv@v5 - name: Install corrlib run: uv sync --locked --all-extras --dev --python ${{ matrix.python-version }} - name: Run tests diff --git a/.github/workflows/ruff.yaml b/.github/workflows/ruff.yaml index 4de4b0b..1da1225 100644 --- a/.github/workflows/ruff.yaml +++ b/.github/workflows/ruff.yaml @@ -9,8 +9,6 @@ jobs: ruff: runs-on: ubuntu-latest - env: - UV_CACHE_DIR: /tmp/.uv-cache steps: - name: Install git-annex run: | @@ -20,10 +18,12 @@ jobs: uses: https://github.com/RouxAntoine/checkout@v4.1.8 with: show-progress: true - - name: Install uv - uses: astral-sh/setup-uv@v7 + - name: Setup python + uses: https://github.com/actions/setup-python@v5 with: - enable-cache: true + python-version: "3.12" + - name: Install uv + uses: https://github.com/astral-sh/setup-uv@v5 - name: Install corrlib run: uv sync --locked --all-extras --dev --python "3.12" - name: Run tests diff --git a/corrlib/cli.py b/corrlib/cli.py index f205026..4e1b65e 100644 --- a/corrlib/cli.py +++ b/corrlib/cli.py @@ -9,6 +9,7 @@ from .main import update_aliases from .meas_io import drop_cache as mio_drop_cache from .meas_io import load_record as mio_load_record import os +from pyerrors import Corr from importlib.metadata import version @@ -38,7 +39,7 @@ def update( @app.command() -def list( +def lister( path: str = typer.Option( str('./corrlib'), "--dataset", @@ -120,12 +121,14 @@ def stat( "--dataset", "-d", ), - record: str = typer.Argument(), + record_id: str = typer.Argument(), ) -> None: """ Show the statistics of a given record. """ - record = mio_load_record(path, record)[0] + record = mio_load_record(path, record_id) + if isinstance(record, (list, Corr)): + record = record[0] statistics = record.idl print(statistics) return diff --git a/corrlib/find.py b/corrlib/find.py index 21063ec..022a3f5 100644 --- a/corrlib/find.py +++ b/corrlib/find.py @@ -119,7 +119,7 @@ def _db_lookup(db: str, ensemble: str, correlator_name: str, code: str, project: def sfcf_filter(results: pd.DataFrame, **kwargs: Any) -> pd.DataFrame: - """ + r""" Filter method for the Database entries holding SFCF calculations. Parameters diff --git a/corrlib/input/sfcf.py b/corrlib/input/sfcf.py index 621f736..8b6e1a3 100644 --- a/corrlib/input/sfcf.py +++ b/corrlib/input/sfcf.py @@ -320,10 +320,10 @@ def read_data(path: str, project: str, dir_in_project: str, prefix: str, param: if not param['crr'] == []: if names is not None: data_crr = pe.input.sfcf.read_sfcf_multi(directory, prefix, param['crr'], param['mrr'], corr_type_list, range(len(param['wf_offsets'])), - range(len(param['wf_basis'])), range(len(param['wf_basis'])), version, cfg_seperator, keyed_out=True, names=names) + range(len(param['wf_basis'])), range(len(param['wf_basis'])), version, cfg_seperator, keyed_out=True, silent=True, names=names) else: data_crr = pe.input.sfcf.read_sfcf_multi(directory, prefix, param['crr'], param['mrr'], corr_type_list, range(len(param['wf_offsets'])), - range(len(param['wf_basis'])), range(len(param['wf_basis'])), version, cfg_seperator, keyed_out=True) + range(len(param['wf_basis'])), range(len(param['wf_basis'])), version, cfg_seperator, keyed_out=True, silent=True) for key in data_crr.keys(): data[key] = data_crr[key] diff --git a/corrlib/meas_io.py b/corrlib/meas_io.py index 65a0569..8e5855d 100644 --- a/corrlib/meas_io.py +++ b/corrlib/meas_io.py @@ -12,7 +12,7 @@ import shutil from typing import Any -def write_measurement(path: str, ensemble: str, measurement: dict[str, dict[str, dict[str, Any]]], uuid: str, code: str, parameter_file: str) -> None: +def write_measurement(path: str, ensemble: str, measurement: dict[str, dict[str, dict[str, Any]]], uuid: str, code: str, parameter_file: Union[str, None]) -> None: """ Write a measurement to the backlog. If the file for the measurement already exists, update the measurement. @@ -34,24 +34,33 @@ def write_measurement(path: str, ensemble: str, measurement: dict[str, dict[str, """ db_file = get_db_file(path) db = os.path.join(path, db_file) + + files_to_save = [] + get(path, db_file) unlock(path, db_file) + files_to_save.append(db_file) + conn = sqlite3.connect(db) c = conn.cursor() - files = [] for corr in measurement.keys(): file_in_archive = os.path.join('.', 'archive', ensemble, corr, uuid + '.json.gz') file = os.path.join(path, file_in_archive) - files.append(file) known_meas = {} if not os.path.exists(os.path.join(path, '.', 'archive', ensemble, corr)): os.makedirs(os.path.join(path, '.', 'archive', ensemble, corr)) + files_to_save.append(file_in_archive) else: if os.path.exists(file): - unlock(path, file_in_archive) - known_meas = pj.load_json_dict(file) + if file not in files_to_save: + unlock(path, file_in_archive) + files_to_save.append(file_in_archive) + known_meas = pj.load_json_dict(file, verbose=False) if code == "sfcf": - parameters = sfcf.read_param(path, uuid, parameter_file) + if parameter_file is not None: + parameters = sfcf.read_param(path, uuid, parameter_file) + else: + raise Exception("Need parameter file for this code!") pars = {} subkeys = list(measurement[corr].keys()) for subkey in subkeys: @@ -60,7 +69,10 @@ def write_measurement(path: str, ensemble: str, measurement: dict[str, dict[str, elif code == "openQCD": ms_type = list(measurement.keys())[0] if ms_type == 'ms1': - parameters = openQCD.read_ms1_param(path, uuid, parameter_file) + if parameter_file is not None: + parameters = openQCD.read_ms1_param(path, uuid, parameter_file) + else: + raise Exception("Need parameter file for this code!") pars = {} subkeys = [] for i in range(len(parameters["rw_fcts"])): @@ -98,9 +110,8 @@ def write_measurement(path: str, ensemble: str, measurement: dict[str, dict[str, (corr, ensemble, code, meas_path, uuid, pars[subkey], parameter_file)) conn.commit() pj.dump_dict_to_json(known_meas, file) - files.append(os.path.join(path, db_file)) conn.close() - save(path, message="Add measurements to database", files=files) + save(path, message="Add measurements to database", files=files_to_save) return diff --git a/corrlib/toml.py b/corrlib/toml.py index 629a499..feafaf6 100644 --- a/corrlib/toml.py +++ b/corrlib/toml.py @@ -189,7 +189,6 @@ def import_toml(path: str, file: str, copy_file: bool=True) -> None: measurement = sfcf.read_data(path, uuid, md['path'], md['prefix'], param, version=md['version'], cfg_seperator=md['cfg_seperator'], sep='/') - print(mname + " imported.") elif project['code'] == 'openQCD': if md['measurement'] == 'ms1': param = openQCD.read_ms1_param(path, uuid, md['param_file']) @@ -211,8 +210,8 @@ def import_toml(path: str, file: str, copy_file: bool=True) -> None: param['type'] = 't1' measurement = openQCD.extract_t1(path, uuid, md['path'], param, str(md["prefix"]), int(md["dtr_read"]), int(md["xmin"]), int(md["spatial_extent"]), fit_range=int(md.get('fit_range', 5)), postfix=str(md.get('postfix', '')), names=md.get('names', []), files=md.get('files', [])) - - write_measurement(path, ensemble, measurement, uuid, project['code'], (md['param_file'] if 'param_file' in md else '')) + write_measurement(path, ensemble, measurement, uuid, project['code'], (md['param_file'] if 'param_file' in md else None)) + print(mname + " imported.") if not os.path.exists(os.path.join(path, "toml_imports", uuid)): os.makedirs(os.path.join(path, "toml_imports", uuid)) diff --git a/corrlib/tracker.py b/corrlib/tracker.py index 5cc281c..e535b03 100644 --- a/corrlib/tracker.py +++ b/corrlib/tracker.py @@ -114,7 +114,7 @@ def unlock(path: str, file: str) -> None: """ tracker = get_tracker(path) if tracker == 'datalad': - dl.unlock(file, dataset=path) + dl.unlock(os.path.join(path, file), dataset=path) elif tracker == 'None': Warning("Tracker 'None' does not implement unlock.") pass diff --git a/tests/cli_test.py b/tests/cli_test.py index a6b0bd7..d4a4045 100644 --- a/tests/cli_test.py +++ b/tests/cli_test.py @@ -37,7 +37,7 @@ def test_init_db(tmp_path): table_names = [table[0] for table in tables] for expected_table in expected_tables: assert expected_table in table_names - + cursor.execute("SELECT * FROM projects;") projects = cursor.fetchall() assert len(projects) == 0 @@ -60,7 +60,7 @@ def test_init_db(tmp_path): project_column_names = [col[1] for col in project_columns] for expected_col in expected_project_columns: assert expected_col in project_column_names - + cursor.execute("PRAGMA table_info('backlogs');") backlog_columns = cursor.fetchall() expected_backlog_columns = [ @@ -85,7 +85,7 @@ def test_list(tmp_path): dataset_path = tmp_path / "test_dataset" result = runner.invoke(app, ["init", "--dataset", str(dataset_path)]) assert result.exit_code == 0 - result = runner.invoke(app, ["list", "--dataset", str(dataset_path), "ensembles"]) + result = runner.invoke(app, ["lister", "--dataset", str(dataset_path), "ensembles"]) assert result.exit_code == 0 - result = runner.invoke(app, ["list", "--dataset", str(dataset_path), "projects"]) + result = runner.invoke(app, ["lister", "--dataset", str(dataset_path), "projects"]) assert result.exit_code == 0