jkuhl/corrlib
1
0
Fork 0
Code Issues Pull requests Releases Packages Activity

implement write to db prperly

Browse source
This commit is contained in:
Justus Kuhlmann 2024-08-13 15:14:34 +00:00
parent 4c81410798
commit 391fa556ef
2 changed files with 53 additions and 32 deletions
Download patch file Download diff file Expand all files Collapse all files

47
backlogger/io.py
View file

@ -2,9 +2,11 @@ from pyerrors.input import json as pj
import os
import datalad.api as dl
import sqlite3
from .input.sfcf import get_specs
from .input.sfcf import read_param
def write_measurement(path, ensemble, measurement, uuid, code, parameters, parameter_file):
def write_measurement(path, ensemble, measurement, uuid, code, parameter_file):
"""
Write a measurement to the backlog.
If the file for the measurement already exists, update the measurement.
@ -20,27 +22,36 @@ def write_measurement(path, ensemble, measurement, uuid, code, parameters, param
uuid: str
The uuid of the project.
"""
parameters = read_param(path, uuid, parameter_file)
print(parameters)
dl.unlock(path + '/backlogger.db', dataset=path)
conn = sqlite3.connect(path + '/backlogger.db')
c = conn.cursor()
files = []
for corr in measurement.keys():
file = path + "/archive/" + ensemble + "/" + corr + '/' + uuid + '.json.gz'
files.append(file)
if not os.path.exists(path + "/archive/" + ensemble + "/" + corr):
os.makedirs(path + "/archive/" + ensemble + "/" + corr)
conn = sqlite3.connect(path + '/backlogger.db')
c = conn.cursor()
if os.path.exists(file):
dl.unlock(file, dataset=path)
known_meas = pj.load_json_dict(file)
for key in measurement[corr].keys():
known_meas[key] = measurement[corr][key]
else:
known_meas = measurement[corr]
pj.dump_dict_to_json(measurement[corr], file)
for subkey in measurement[corr].keys():
meas_path = file + "::" + subkey
if not os.path.exists(file):
c.execute("INSERT INTO backlogs (name, ensemble, code, path, project, parameters, parameter_file, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, datetime('now'), datetime('now'))", (corr, ensemble, code, meas_path, uuid, parameters, parameter_file))
else:
c.execute("UPDATE backlogs SET updated_at=datetime('now') WHERE path=?", (file,))
if os.path.exists(file):
dl.unlock(file, dataset=path)
known_meas = pj.load_json_dict(file)
for key in measurement[corr].keys():
known_meas[key] = measurement[corr][key]
# this should be only set if something changed.
else:
known_meas = measurement[corr]
if c.execute("SELECT * FROM backlogs WHERE path = ?", (meas_path,)).fetchone() is not None:
c.execute("UPDATE backlogs SET name = ?, ensemble = ?, code = ?, project = ?, parameters = ?, parameter_file = ?, updated_at = datetime('now') WHERE path = ?", (meas_path, ))
else:
c.execute("INSERT INTO backlogs (name, ensemble, code, path, project, parameters, parameter_file, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, datetime('now'), datetime('now'))", (corr, ensemble, code, meas_path, uuid, get_specs(corr + "/" + subkey, parameters), parameter_file))
conn.commit()
conn.close()
dl.save([path + '/backlogger.db', file], message="Add measurement to database", dataset=path)
pj.dump_dict_to_json(measurement[corr], file)
files.append(path + '/backlogger.db')
conn.close()
dl.save(files, message="Add measurement to database", dataset=path)