jkuhl/corrlib
1
0
Fork 0
Code Issues Pull requests Releases Packages Activity

better io for measurement management

Browse source
This commit is contained in:
Justus Kuhlmann 2024-08-15 16:05:12 +00:00
parent fe8f63465a
commit 36496a4fb8
1 changed files with 39 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

47
backlogger/io.py → backlogger/meas_io.py
View file

@ -31,27 +31,58 @@ def write_measurement(path, ensemble, measurement, uuid, code, parameter_file):
for corr in measurement.keys():
file = path + "/archive/" + ensemble + "/" + corr + '/' + uuid + '.json.gz'
files.append(file)
known_meas = None
if not os.path.exists(path + "/archive/" + ensemble + "/" + corr):
os.makedirs(path + "/archive/" + ensemble + "/" + corr)
for subkey in measurement[corr].keys():
meas_path = file + "::" + subkey
else:
if os.path.exists(file):
dl.unlock(file, dataset=path)
known_meas = pj.load_json_dict(file)
for key in measurement[corr].keys():
known_meas[key] = measurement[corr][key]
for subkey in measurement[corr].keys():
meas_path = file + "::" + subkey
if known_meas is not None:
known_meas[subkey] = measurement[corr][subkey]
# this should be only set if something changed.
else:
known_meas = measurement[corr]
if c.execute("SELECT * FROM backlogs WHERE path = ?", (meas_path,)).fetchone() is not None:
c.execute("UPDATE backlogs SET name = ?, ensemble = ?, code = ?, project = ?, parameters = ?, parameter_file = ?, updated_at = datetime('now') WHERE path = ?", (meas_path, ))
c.execute("UPDATE backlogs SET updated_at = datetime('now') WHERE path = ?", (meas_path, ))
else:
c.execute("INSERT INTO backlogs (name, ensemble, code, path, project, parameters, parameter_file, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, datetime('now'), datetime('now'))", (corr, ensemble, code, meas_path, uuid, get_specs(corr + "/" + subkey, parameters), parameter_file))
conn.commit()
pj.dump_dict_to_json(measurement[corr], file)
files.append(path + '/backlogger.db')
conn.close()
dl.save(files, message="Add measurement to database", dataset=path)
dl.save(files, message="Add measurements to database", dataset=path)
def get_record(path, meas_path):
file = meas_path.split("::")[0]
sub_key = meas_path.split("::")[1]
dl.get(file, dataset=path)
return pj.load_json_dict(file)[sub_key]
def drop_record(path, meas_path):
file = meas_path.split("::")[0]
sub_key = meas_path.split("::")[1]
dl.unlock(path + '/backlogger.db', dataset=path)
conn = sqlite3.connect(path + '/backlogger.db')
c = conn.cursor()
if c.execute("SELECT * FROM backlogs WHERE path = ?", (meas_path, )).fetchone() is not None:
c.execute("DELETE FROM backlogs WHERE path = ?", (meas_path, ))
else:
raise ValueError("This measurement does not exist as an entry!")
conn.commit()
known_meas = pj.load_json_dict(file)
if sub_key in known_meas:
del known_meas[sub_key]
dl.unlock(file, dataset=path)
pj.dump_dict_to_json(known_meas, file)
dl.save([path + '/backlogger.db', file], message="Drop measurements to database", dataset=path)
return
else:
raise ValueError("This measurement does not exist as a file!")