diff --git a/corrlib/meas_io.py b/corrlib/meas_io.py index 393a843..b5c4029 100644 --- a/corrlib/meas_io.py +++ b/corrlib/meas_io.py @@ -17,7 +17,7 @@ from .tracker import get, save, unlock CACHE_DIR = ".cache" -def write_measurement(path: Path, ensemble: str, measurement: dict[str, dict[str, dict[str, Any]]], uuid: str, code: str, parameter_file: str | None) -> None: +def write_measurement(path: Path, ensemble: str, measurement: dict[str, dict[str, dict[str, Any]]], uuid: str, code: str, parameter_file: str | None, final_write: dict[str, bool]) -> None: """ Write a measurement to the backlog. If the file for the measurement already exists, update the measurement. @@ -36,6 +36,8 @@ def write_measurement(path: Path, ensemble: str, measurement: dict[str, dict[str Name of the code that was used for the project. parameter_file: str The parameter file used for the measurement. + final_write: bool + Determmines whether this is the final ime the file is touched during the current import. """ path = Path(path) db_file = get_db_file(path) @@ -52,12 +54,16 @@ def write_measurement(path: Path, ensemble: str, measurement: dict[str, dict[str for corr in measurement.keys(): file_in_archive = Path('.') / 'archive' / ensemble / corr / str(uuid + '.json.gz') file = Path(path) / file_in_archive - known_meas = {} + tmp_file_in_archive = Path('.') / 'archive' / ensemble / corr / (str(uuid) + ".p") + tmp_file = Path(path) / tmp_file_in_archive + known_meas: dict[str, Any] = {} if not os.path.exists(path / 'archive' / ensemble / corr): os.makedirs(path / 'archive' / ensemble / corr) files_to_save.append(file_in_archive) else: - if os.path.exists(file): + if os.path.exists(tmp_file): + known_meas = load_object(str(tmp_file)) + elif os.path.exists(file): if file not in files_to_save: unlock(path, file_in_archive) files_to_save.append(file_in_archive) @@ -132,7 +138,12 @@ def write_measurement(path: Path, ensemble: str, measurement: dict[str, dict[str c.execute("INSERT INTO backlogs (name, ensemble, code, path, project, parameters, parameter_file, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, datetime('now'), datetime('now'))", (corr, ensemble, code, meas_path, uuid, pars[subkey], parameter_file)) conn.commit() - pj.dump_dict_to_json(known_meas, str(file)) + if final_write[str(file)]: + pj.dump_dict_to_json(known_meas, str(file)) + if os.path.exists(tmp_file): + os.remove(tmp_file) + else: + dump_object(known_meas, str(tmp_file)[:-2]) conn.close() save(path, message="Add measurements to database", files=files_to_save) return diff --git a/corrlib/toml.py b/corrlib/toml.py index 058da2d..7becb66 100644 --- a/corrlib/toml.py +++ b/corrlib/toml.py @@ -192,7 +192,6 @@ def import_toml(path: Path, file: str, copy_file: bool=True) -> None: mname_list = list(measurements.keys()) for mname in mname_list: md = measurements[mname] - print(f"Import measurement {imeas}/{nmeas}: {mname}") ensemble = md['ensemble'] if project['code'] == 'sfcf': param = sfcf.read_param(path, uuid, md['param_file']) @@ -204,9 +203,12 @@ def import_toml(path: Path, file: str, copy_file: bool=True) -> None: affected_by_meas = affected_files(param['type'], ensemble, uuid) elif md['measurement'] == 't1': affected_by_meas = affected_files(param['type'], ensemble, uuid) - affected_file_d[mname] = affected_by_meas - discard_after = step_differences(mname_list, affected_file_d) - + affected_file_d[mname] = [str(path / f) for f in affected_by_meas] + future_affected_file_d = {} + for i,mname in enumerate(mname_list): + future_affected_file_d[mname] = [] + for mname2 in mname_list[i+1:]: + future_affected_file_d[mname].extend(affected_file_d[mname2]) for mname in mname_list: md = measurements[mname] print(f"Import measurement {imeas}/{nmeas}: {mname}") @@ -267,7 +269,12 @@ def import_toml(path: Path, file: str, copy_file: bool=True) -> None: measurement = openQCD.extract_t1(path, uuid, md['path'], ensemble, param, str(md["prefix"]), int(md["dtr_read"]), int(md["xmin"]), int(md["spatial_extent"]), fit_range=int(md.get('fit_range', 5)), postfix=str(md.get('postfix', '')), names=md.get('names', []), files=md.get('files', []), r_start=md.get('r_start', []), r_stop=md.get('r_stop', []), r_step=md.get('r_step', 1)) - write_measurement(path, ensemble, measurement, uuid, project['code'], (md['param_file'] if 'param_file' in md else None)) + final_write = {} + for file in affected_file_d[mname]: + final_write[str(file)] = True + if str(file) in future_affected_file_d[mname]: + final_write[str(file)] = False + write_measurement(path, ensemble, measurement, uuid, project['code'], (md['param_file'] if 'param_file' in md else None), final_write) imeas += 1 print(mname + " imported.") diff --git a/corrlib/version.py b/corrlib/version.py index 23ea4de..3e22db8 100644 --- a/corrlib/version.py +++ b/corrlib/version.py @@ -3,12 +3,12 @@ from __future__ import annotations __all__ = [ - "__commit_id__", "__version__", "__version_tuple__", - "commit_id", "version", "version_tuple", + "__commit_id__", + "commit_id", ] version: str @@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...] commit_id: str | None __commit_id__: str | None -__version__ = version = '0.3.1.dev22+g4b1c21309.d20260701' -__version_tuple__ = version_tuple = (0, 3, 1, 'dev22', 'g4b1c21309.d20260701') +__version__ = version = '0.3.1.dev32+g906a2bdf3.d20260710' +__version_tuple__ = version_tuple = (0, 3, 1, 'dev32', 'g906a2bdf3.d20260710') -__commit_id__ = commit_id = 'g4b1c21309' +__commit_id__ = commit_id = 'g906a2bdf3'