Merge pull request 'fix/uv-setup' (#21) from fix/uv-setup into develop

Reviewed-on: https://www.kuhl-mann.de/git/git/jkuhl/corrlib/pulls/21

.github/workflows/mypy.yaml

@@ -8,22 +8,21 @@ on:
 jobs:
   mypy:
     runs-on: ubuntu-latest
-    env:
-      UV_CACHE_DIR: /tmp/.uv-cache
     steps:
       - name: Install git-annex
         run: |
           sudo apt-get update
-          sudo apt-get install -y git-annex
+          sudo apt-get install -y git-annex 
       - name: Check out the repository
         uses: https://github.com/RouxAntoine/checkout@v4.1.8
         with:
           show-progress: true
-      - name: Install uv
+      - name: Setup python
-        uses: astral-sh/setup-uv@v7
+        uses: https://github.com/actions/setup-python@v5
         with:
-          python-version: ${{ matrix.python-version }}
+          python-version: "3.12"
-          enable-cache: true
+      - name: Install uv
+        uses: https://github.com/astral-sh/setup-uv@v5
       - name: Install corrlib
         run: uv sync --locked --all-extras --dev --python "3.12"
       - name: Run tests

.github/workflows/pytest.yaml

@@ -17,9 +17,11 @@ jobs:
           - "3.14"
 
     runs-on: ubuntu-latest
-    env:
-      UV_CACHE_DIR: /tmp/.uv-cache
     steps:
+      - name: Setup git
+        run: |
+          git config --global user.email "tester@example.com"
+          git config --global user.name "Tester"
       - name: Install git-annex
         run: |
           sudo apt-get update
@@ -28,11 +30,12 @@ jobs:
         uses: https://github.com/RouxAntoine/checkout@v4.1.8
         with:
           show-progress: true
-      - name: Install uv
+      - name: Setup python
-        uses: astral-sh/setup-uv@v7
+        uses: https://github.com/actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-          enable-cache: true
+      - name: Install uv
+        uses: https://github.com/astral-sh/setup-uv@v5
       - name: Install corrlib
         run: uv sync --locked --all-extras --dev --python ${{ matrix.python-version }}
       - name: Run tests

.github/workflows/ruff.yaml

@@ -9,8 +9,6 @@ jobs:
   ruff:
 
     runs-on: ubuntu-latest
-    env:
-      UV_CACHE_DIR: /tmp/.uv-cache
     steps:
       - name: Install git-annex
         run: |
@@ -20,10 +18,12 @@ jobs:
         uses: https://github.com/RouxAntoine/checkout@v4.1.8
         with:
           show-progress: true
-      - name: Install uv
+      - name: Setup python
-        uses: astral-sh/setup-uv@v7
+        uses: https://github.com/actions/setup-python@v5
         with:
-          enable-cache: true
+          python-version: "3.12"
+      - name: Install uv
+        uses: https://github.com/astral-sh/setup-uv@v5
       - name: Install corrlib
         run: uv sync --locked --all-extras --dev --python "3.12"
       - name: Run tests

corrlib/cli.py

@@ -9,6 +9,7 @@ from .main import update_aliases
 from .meas_io import drop_cache as mio_drop_cache
 from .meas_io import load_record as mio_load_record
 import os
+from pyerrors import Corr
 from importlib.metadata import version
 
 
@@ -38,7 +39,7 @@ def update(
 
 
 @app.command()
-def list(
+def lister(
     path: str = typer.Option(
         str('./corrlib'),
         "--dataset",
@@ -120,12 +121,14 @@ def stat(
         "--dataset",
         "-d",
         ),
-    record: str = typer.Argument(),
+    record_id: str = typer.Argument(),
     ) -> None:
     """
     Show the statistics of a given record.
     """
-    record = mio_load_record(path, record)[0]
+    record = mio_load_record(path, record_id)
+    if isinstance(record, (list, Corr)):
+        record = record[0]
     statistics = record.idl
     print(statistics)
     return

corrlib/find.py

@@ -119,7 +119,7 @@ def _db_lookup(db: str, ensemble: str, correlator_name: str, code: str, project:
 
 
 def sfcf_filter(results: pd.DataFrame, **kwargs: Any) -> pd.DataFrame:
-    """
+    r"""
     Filter method for the Database entries holding SFCF calculations.
 
     Parameters

corrlib/input/sfcf.py

@@ -320,10 +320,10 @@ def read_data(path: str, project: str, dir_in_project: str, prefix: str, param:
     if not param['crr'] == []:
         if names is not None:
             data_crr = pe.input.sfcf.read_sfcf_multi(directory, prefix, param['crr'], param['mrr'], corr_type_list, range(len(param['wf_offsets'])),
-                                                     range(len(param['wf_basis'])), range(len(param['wf_basis'])), version, cfg_seperator, keyed_out=True, names=names)
+                                                     range(len(param['wf_basis'])), range(len(param['wf_basis'])), version, cfg_seperator, keyed_out=True, silent=True, names=names)
         else:
             data_crr = pe.input.sfcf.read_sfcf_multi(directory, prefix, param['crr'], param['mrr'], corr_type_list, range(len(param['wf_offsets'])),
-                                                     range(len(param['wf_basis'])), range(len(param['wf_basis'])), version, cfg_seperator, keyed_out=True)
+                                                     range(len(param['wf_basis'])), range(len(param['wf_basis'])), version, cfg_seperator, keyed_out=True, silent=True)
         for key in data_crr.keys():
             data[key] = data_crr[key]
 

corrlib/meas_io.py

@@ -12,7 +12,7 @@ import shutil
 from typing import Any
 
 
-def write_measurement(path: str, ensemble: str, measurement: dict[str, dict[str, dict[str, Any]]], uuid: str, code: str, parameter_file: str) -> None:
+def write_measurement(path: str, ensemble: str, measurement: dict[str, dict[str, dict[str, Any]]], uuid: str, code: str, parameter_file: Union[str, None]) -> None:
     """
     Write a measurement to the backlog.
     If the file for the measurement already exists, update the measurement.
@@ -34,24 +34,33 @@ def write_measurement(path: str, ensemble: str, measurement: dict[str, dict[str,
     """
     db_file = get_db_file(path)
     db = os.path.join(path, db_file)
+
+    files_to_save = []
+
     get(path, db_file)
     unlock(path, db_file)
+    files_to_save.append(db_file)
+
     conn = sqlite3.connect(db)
     c = conn.cursor()
-    files = []
     for corr in measurement.keys():
         file_in_archive = os.path.join('.', 'archive', ensemble, corr,  uuid + '.json.gz')
         file = os.path.join(path, file_in_archive)
-        files.append(file)
         known_meas = {}
         if not os.path.exists(os.path.join(path, '.', 'archive', ensemble, corr)):
             os.makedirs(os.path.join(path, '.', 'archive', ensemble, corr))
+            files_to_save.append(file_in_archive)
         else:
             if os.path.exists(file):
-                unlock(path, file_in_archive)
+                if file not in files_to_save:
-                known_meas = pj.load_json_dict(file)
+                    unlock(path, file_in_archive)
+                    files_to_save.append(file_in_archive)
+                known_meas = pj.load_json_dict(file, verbose=False)
         if code == "sfcf":
-            parameters = sfcf.read_param(path, uuid, parameter_file)
+            if parameter_file is not None:
+                parameters = sfcf.read_param(path, uuid, parameter_file)
+            else:
+                raise Exception("Need parameter file for this code!")
             pars = {}
             subkeys = list(measurement[corr].keys())
             for subkey in subkeys:
@@ -60,7 +69,10 @@ def write_measurement(path: str, ensemble: str, measurement: dict[str, dict[str,
         elif code == "openQCD":
             ms_type = list(measurement.keys())[0]
             if ms_type == 'ms1':
-                parameters = openQCD.read_ms1_param(path, uuid, parameter_file)
+                if parameter_file is not None:
+                    parameters = openQCD.read_ms1_param(path, uuid, parameter_file)
+                else:
+                    raise Exception("Need parameter file for this code!")
                 pars = {}
                 subkeys = []
                 for i in range(len(parameters["rw_fcts"])):
@@ -98,9 +110,8 @@ def write_measurement(path: str, ensemble: str, measurement: dict[str, dict[str,
                                                 (corr, ensemble, code, meas_path, uuid, pars[subkey], parameter_file))
         conn.commit()
         pj.dump_dict_to_json(known_meas, file)
-    files.append(os.path.join(path, db_file))
     conn.close()
-    save(path, message="Add measurements to database", files=files)
+    save(path, message="Add measurements to database", files=files_to_save)
     return
 
 

corrlib/toml.py

@@ -189,7 +189,6 @@ def import_toml(path: str, file: str, copy_file: bool=True) -> None:
                 measurement = sfcf.read_data(path, uuid, md['path'], md['prefix'], param,
                                             version=md['version'], cfg_seperator=md['cfg_seperator'], sep='/')
 
-            print(mname + " imported.")
         elif project['code'] == 'openQCD':
             if md['measurement'] == 'ms1':
                 param = openQCD.read_ms1_param(path, uuid, md['param_file'])
@@ -211,8 +210,8 @@ def import_toml(path: str, file: str, copy_file: bool=True) -> None:
                 param['type'] = 't1'
                 measurement = openQCD.extract_t1(path, uuid, md['path'], param, str(md["prefix"]), int(md["dtr_read"]), int(md["xmin"]), int(md["spatial_extent"]),
                                                  fit_range=int(md.get('fit_range', 5)), postfix=str(md.get('postfix', '')), names=md.get('names', []), files=md.get('files', []))
-
+        write_measurement(path, ensemble, measurement, uuid, project['code'], (md['param_file'] if 'param_file' in md else None))
-        write_measurement(path, ensemble, measurement, uuid, project['code'], (md['param_file'] if 'param_file' in md else ''))
+        print(mname + " imported.")
 
     if not os.path.exists(os.path.join(path, "toml_imports", uuid)):
         os.makedirs(os.path.join(path, "toml_imports", uuid))

corrlib/tracker.py

@@ -114,7 +114,7 @@ def unlock(path: str, file: str) -> None:
     """
     tracker = get_tracker(path)
     if tracker == 'datalad':
-        dl.unlock(file, dataset=path)
+        dl.unlock(os.path.join(path, file), dataset=path)
     elif tracker == 'None':
         Warning("Tracker 'None' does not implement unlock.")
         pass

tests/cli_test.py

@@ -37,7 +37,7 @@ def test_init_db(tmp_path):
     table_names = [table[0] for table in tables]
     for expected_table in expected_tables:
         assert expected_table in table_names
-    
+
     cursor.execute("SELECT * FROM projects;")
     projects = cursor.fetchall()
     assert len(projects) == 0
@@ -60,7 +60,7 @@ def test_init_db(tmp_path):
     project_column_names = [col[1] for col in project_columns]
     for expected_col in expected_project_columns:
         assert expected_col in project_column_names
-    
+
     cursor.execute("PRAGMA table_info('backlogs');")
     backlog_columns = cursor.fetchall()
     expected_backlog_columns = [
@@ -85,7 +85,7 @@ def test_list(tmp_path):
     dataset_path = tmp_path / "test_dataset"
     result = runner.invoke(app, ["init", "--dataset", str(dataset_path)])
     assert result.exit_code == 0
-    result = runner.invoke(app, ["list", "--dataset", str(dataset_path), "ensembles"])
+    result = runner.invoke(app, ["lister", "--dataset", str(dataset_path), "ensembles"])
     assert result.exit_code == 0
-    result = runner.invoke(app, ["list", "--dataset", str(dataset_path), "projects"])
+    result = runner.invoke(app, ["lister", "--dataset", str(dataset_path), "projects"])
     assert result.exit_code == 0