From da279b3575fd8696b53f8e38389cbf7084061627 Mon Sep 17 00:00:00 2001
From: Justus Kuhlmann <j_kuhl19@uni-muenster.de>
Date: Mon, 12 May 2025 20:30:18 +0000
Subject: [PATCH] first version of caching

---
 corrlib/initialization.py |  3 +++
 corrlib/meas_io.py        | 34 +++++++++++++++++++++++++++-------
 corrlib/tools.py          |  4 +++-
 3 files changed, 33 insertions(+), 8 deletions(-)

diff --git a/corrlib/initialization.py b/corrlib/initialization.py
index 9f17e47..f6ef5aa 100644
--- a/corrlib/initialization.py
+++ b/corrlib/initialization.py
@@ -46,4 +46,7 @@ def create(path):
     os.makedirs(path + '/archive')
     os.makedirs(path + '/toml_imports')
     os.makedirs(path + '/import_scripts/template.py')
+    with open(path + "/.gitignore", "w") as fp:
+        fp.write(".cache")
+        fp.close()
     dl.save(path, dataset=path, message="Initialize backlogger directory.")
diff --git a/corrlib/meas_io.py b/corrlib/meas_io.py
index dc8fadc..5e43b3f 100644
--- a/corrlib/meas_io.py
+++ b/corrlib/meas_io.py
@@ -5,8 +5,9 @@ import sqlite3
 from .input import sfcf,openQCD
 import json
 from typing import Union
-from pyerrors import Obs, Corr
+from pyerrors import Obs, Corr, dump_obj, load_obj
 from hashlib import sha256
+from .tools import cached
 
 
 def write_measurement(path, ensemble, measurement, uuid, code, parameter_file=None):
@@ -135,16 +136,35 @@ def load_records(path: str, meas_paths: list[str], preloaded = {}) -> list[Union
             needed_data[file] = []
         key = mpath.split("::")[1]
         needed_data[file].append(key)
-    for filename in needed_data.keys():
-        if not filename in preloaded:
-            preloaded[filename] = preload(path, filename)
     returned_data: list = []
-    for filename in needed_data.keys():
-        for key in list(needed_data[filename]):
-            returned_data.append(preloaded[filename][key])
+    for file in needed_data.keys():
+        for key in list(needed_data[file]):
+            if os.path.exists(cache_path(path, file, key)):
+                returned_data.append(load_obj(cache_path(path, file, key)))
+            else:
+                if file not in preloaded:
+                    preloaded[file] = preload(path, file)
+                returned_data.append(preloaded[file][key])
+                if cached:
+                    if not os.path.exists(cache_dir(path, file)):
+                        os.makedirs(cache_dir(path, file))
+                    dump_obj(preloaded[file][key], cache_path(path, file, key))
     return returned_data
 
 
+def cache_dir(path, file):
+    cache_path_list = [path]
+    cache_path_list.append(".cache")
+    cache_path_list.extend(file.split("/")[1:])
+    cache_path = os.path.join(cache_path_list)
+    return cache_path
+
+
+def cache_path(path, file, key):
+    cache_path = os.path.join(cache_dir(path, file), key)
+    return cache_path + ".p"
+
+
 def preload(path: str, file: str):
     dl.get(os.path.join(path, file), dataset=path)
     filedict = pj.load_json_dict(os.path.join(path, file))
diff --git a/corrlib/tools.py b/corrlib/tools.py
index a150520..e424b59 100644
--- a/corrlib/tools.py
+++ b/corrlib/tools.py
@@ -6,4 +6,6 @@ def str2list(string):
 
 def list2str(mylist):
     s = ",".join(mylist)
-    return s
\ No newline at end of file
+    return s
+
+cached = True