Feat/read sfcf multi (#210)

* make template

* read_sfcf_multi running with compact format

* fix append mode, norrmal tests work

* improve readability

* add simple test for multi_read

* simple multi_test works

* add first method to check sfcf param hashes

* add docstring

* simple test for o format working

* use benedict to make loops easier

* introduce python-benedict as dep

* no nice_out, less error prone, found bug in tests

* Revert "introduce python-benedict as dep"

This reverts commit 9696d68b7a.

* Revert "use benedict to make loops easier"

This reverts commit fa3987479b.

* no nice output after reverts

* [build] Added jkuhl-uni as CODEOWNER for sfcf.

* refactor: flatten internal dicts

* very small test extension

* ...flake8

* docu

* Delete second sep init

---------

Co-authored-by: Fabian Joswig <fabian.joswig@uni-muenster.de>

.github/CODEOWNERS

@@ -2,3 +2,5 @@
 /pyerrors/covobs.py @s-kuberski
 /pyerrors/input/json.py @s-kuberski
 /pyerrors/input/dobs.py @s-kuberski
+/pyerrors/input/sfcf.py @jkuhl-uni
+/tests/sfcf_in_test.py @jkuhl-uni

pyerrors/input/sfcf.py

@@ -4,9 +4,13 @@ import re
 import numpy as np  # Thinly-wrapped numpy
 from ..obs import Obs
 from .utils import sort_names, check_idl
+import itertools
 
 
-def read_sfcf(path, prefix, name, quarks='.*', corr_type='bi', noffset=0, wf=0, wf2=0, version="1.0c", cfg_separator="n", silent=False, **kwargs):
+sep = "/"
+
+
+def read_sfcf(path, prefix, name, quarks='.*', corr_type="bi", noffset=0, wf=0, wf2=0, version="1.0c", cfg_separator="n", silent=False, **kwargs):
     """Read sfcf files from given folder structure.
 
     Parameters
@@ -65,6 +69,76 @@ def read_sfcf(path, prefix, name, quarks='.*', corr_type='bi', noffset=0, wf=0,
         list of Observables with length T, observable per timeslice.
         bb-type correlators have length 1.
     """
+    ret = read_sfcf_multi(path, prefix, [name], quarks_list=[quarks], corr_type_list=[corr_type],
+                          noffset_list=[noffset], wf_list=[wf], wf2_list=[wf2], version=version,
+                          cfg_separator=cfg_separator, silent=silent, **kwargs)
+    return ret[name][quarks][str(noffset)][str(wf)][str(wf2)]
+
+
+def read_sfcf_multi(path, prefix, name_list, quarks_list=['.*'], corr_type_list=['bi'], noffset_list=[0], wf_list=[0], wf2_list=[0], version="1.0c", cfg_separator="n", silent=False, keyed_out=False, **kwargs):
+    """Read sfcf files from given folder structure.
+
+    Parameters
+    ----------
+    path : str
+        Path to the sfcf files.
+    prefix : str
+        Prefix of the sfcf files.
+    name : str
+        Name of the correlation function to read.
+    quarks_list : list[str]
+        Label of the quarks used in the sfcf input file. e.g. "quark quark"
+        for version 0.0 this does NOT need to be given with the typical " - "
+        that is present in the output file,
+        this is done automatically for this version
+    corr_type_list : list[str]
+        Type of correlation function to read. Can be
+        - 'bi' for boundary-inner
+        - 'bb' for boundary-boundary
+        - 'bib' for boundary-inner-boundary
+    noffset_list : list[int]
+        Offset of the source (only relevant when wavefunctions are used)
+    wf_list : int
+        ID of wave function
+    wf2_list : list[int]
+        ID of the second wavefunction
+        (only relevant for boundary-to-boundary correlation functions)
+    im : bool
+        if True, read imaginary instead of real part
+        of the correlation function.
+    names : list
+        Alternative labeling for replicas/ensembles.
+        Has to have the appropriate length
+    ens_name : str
+        replaces the name of the ensemble
+    version: str
+        version of SFCF, with which the measurement was done.
+        if the compact output option (-c) was specified,
+        append a "c" to the version (e.g. "1.0c")
+        if the append output option (-a) was specified,
+        append an "a" to the version
+    cfg_separator : str
+        String that separates the ensemble identifier from the configuration number (default 'n').
+    replica: list
+        list of replica to be read, default is all
+    files: list
+        list of files to be read per replica, default is all.
+        for non-compact output format, hand the folders to be read here.
+    check_configs: list[list[int]]
+        list of list of supposed configs, eg. [range(1,1000)]
+        for one replicum with 1000 configs
+
+    Returns
+    -------
+    result: dict[list[Obs]]
+        dict with one of the following properties:
+        if keyed_out:
+            dict[key] = list[Obs]
+            where key has the form name/quarks/offset/wf/wf2
+        if not keyed_out:
+            dict[name][quarks][offset][wf][wf2] = list[Obs]
+    """
+
     if kwargs.get('im'):
         im = 1
         part = 'imaginary'
@@ -72,16 +146,6 @@ def read_sfcf(path, prefix, name, quarks='.*', corr_type='bi', noffset=0, wf=0,
         im = 0
         part = 'real'
 
-    if corr_type == 'bb':
-        b2b = True
-        single = True
-    elif corr_type == 'bib':
-        b2b = True
-        single = False
-    else:
-        b2b = False
-        single = False
-
     known_versions = ["0.0", "1.0", "2.0", "1.0c", "2.0c", "1.0a", "2.0a"]
 
     if version not in known_versions:
@@ -121,7 +185,7 @@ def read_sfcf(path, prefix, name, quarks='.*', corr_type='bi', noffset=0, wf=0,
     else:
         replica = len([file.split(".")[-1] for file in ls]) // len(set([file.split(".")[-1] for file in ls]))
     if not silent:
-        print('Read', part, 'part of', name, 'from', prefix[:-1], ',', replica, 'replica')
+        print('Read', part, 'part of', name_list, 'from', prefix[:-1], ',', replica, 'replica')
 
     if 'names' in kwargs:
         new_names = kwargs.get('names')
@@ -135,10 +199,38 @@ def read_sfcf(path, prefix, name, quarks='.*', corr_type='bi', noffset=0, wf=0,
         if not appended:
             new_names = _get_rep_names(ls, ens_name)
         else:
-            new_names = _get_appended_rep_names(ls, prefix, name, ens_name)
+            new_names = _get_appended_rep_names(ls, prefix, name_list[0], ens_name)
         new_names = sort_names(new_names)
 
     idl = []
+
+    noffset_list = [str(x) for x in noffset_list]
+    wf_list = [str(x) for x in wf_list]
+    wf2_list = [str(x) for x in wf2_list]
+
+    # setup dict structures
+    intern = {}
+    for name, corr_type in zip(name_list, corr_type_list):
+        intern[name] = {}
+        b2b, single = _extract_corr_type(corr_type)
+        intern[name]["b2b"] = b2b
+        intern[name]["single"] = single
+        intern[name]["spec"] = {}
+        for quarks in quarks_list:
+            intern[name]["spec"][quarks] = {}
+            for off in noffset_list:
+                intern[name]["spec"][quarks][off] = {}
+                for w in wf_list:
+                    intern[name]["spec"][quarks][off][w] = {}
+                    for w2 in wf2_list:
+                        intern[name]["spec"][quarks][off][w][w2] = {}
+                        intern[name]["spec"][quarks][off][w][w2]["pattern"] = _make_pattern(version, name, off, w, w2, intern[name]['b2b'], quarks)
+
+    internal_ret_dict = {}
+    needed_keys = _lists2key(name_list, quarks_list, noffset_list, wf_list, wf2_list)
+    for key in needed_keys:
+        internal_ret_dict[key] = []
+
     if not appended:
         for i, item in enumerate(ls):
             rep_path = path + '/' + item
@@ -164,67 +256,83 @@ def read_sfcf(path, prefix, name, quarks='.*', corr_type='bi', noffset=0, wf=0,
             idl.append(rep_idl)
             # here we have found all the files we need to look into.
             if i == 0:
+                if version != "0.0" and compact:
+                    file = path + '/' + item + '/' + sub_ls[0]
+                for name in name_list:
+                    if version == "0.0" or not compact:
+                        file = path + '/' + item + '/' + sub_ls[0] + '/' + name
+                    for key in _lists2key(quarks_list, noffset_list, wf_list, wf2_list):
+                        specs = _key2specs(key)
+                        quarks = specs[0]
+                        off = specs[1]
+                        w = specs[2]
+                        w2 = specs[3]
                         # here, we want to find the place within the file,
                         # where the correlator we need is stored.
                         # to do so, the pattern needed is put together
                         # from the input values
-                if version == "0.0":
+                        start_read, T = _find_correlator(file, version, intern[name]["spec"][quarks][str(off)][str(w)][str(w2)]["pattern"], intern[name]['b2b'], silent=silent)
-                    file = path + '/' + item + '/' + sub_ls[0] + '/' + name
+                        intern[name]["spec"][quarks][str(off)][str(w)][str(w2)]["start"] = start_read
-                else:
+                        intern[name]["T"] = T
-                    if compact:
-                        file = path + '/' + item + '/' + sub_ls[0]
-                    else:
-                        file = path + '/' + item + '/' + sub_ls[0] + '/' + name
-
-                pattern = _make_pattern(version, name, noffset, wf, wf2, b2b, quarks)
-                start_read, T = _find_correlator(file, version, pattern, b2b, silent=silent)
-
                         # preparing the datastructure
                         # the correlators get parsed into...
                         deltas = []
-                for j in range(T):
+                        for j in range(intern[name]["T"]):
                             deltas.append([])
+                        internal_ret_dict[sep.join([name, key])] = deltas
 
             if compact:
-                rep_deltas = _read_compact_rep(path, item, sub_ls, start_read, T, b2b, name, im)
+                rep_deltas = _read_compact_rep(path, item, sub_ls, intern, needed_keys, im)
-
+                for key in needed_keys:
-                for t in range(T):
+                    name = _key2specs(key)[0]
-                    deltas[t].append(rep_deltas[t])
+                    for t in range(intern[name]["T"]):
+                        internal_ret_dict[key][t].append(rep_deltas[key][t])
             else:
-                for t in range(T):
+                for key in needed_keys:
-                    deltas[t].append(np.zeros(no_cfg))
+                    rep_data = []
-                for cnfg, subitem in enumerate(sub_ls):
+                    name = _key2specs(key)[0]
-                    with open(path + '/' + item + '/' + subitem + '/' + name) as fp:
+                    for subitem in sub_ls:
-                        for k, line in enumerate(fp):
+                        cfg_path = path + '/' + item + '/' + subitem
-                            if (k >= start_read and k < start_read + T):
+                        file_data = _read_o_file(cfg_path, name, needed_keys, intern, version, im)
-                                floats = list(map(float, line.split()))
+                        rep_data.append(file_data)
-                                if version == "0.0":
+                    print(rep_data)
-                                    deltas[k - start_read][i][cnfg] = floats[im - single]
+                    for t in range(intern[name]["T"]):
-                                else:
+                        internal_ret_dict[key][t].append([])
-                                    deltas[k - start_read][i][cnfg] = floats[1 + im - single]
+                        for cfg in range(no_cfg):
-
+                            internal_ret_dict[key][t][i].append(rep_data[cfg][key][t])
     else:
+        for key in needed_keys:
+            specs = _key2specs(key)
+            name = specs[0]
+            quarks = specs[1]
+            off = specs[2]
+            w = specs[3]
+            w2 = specs[4]
             if "files" in kwargs:
                 ls = kwargs.get("files")
             else:
-            for exc in ls:
+                name_ls = ls
+                for exc in name_ls:
                     if not fnmatch.fnmatch(exc, prefix + '*.' + name):
-                    ls = list(set(ls) - set([exc]))
+                        name_ls = list(set(name_ls) - set([exc]))
-            ls = sort_names(ls)
+            name_ls = sort_names(name_ls)
-        pattern = _make_pattern(version, name, noffset, wf, wf2, b2b, quarks)
+            pattern = intern[name]['spec'][quarks][off][w][w2]['pattern']
             deltas = []
-        for rep, file in enumerate(ls):
+            for rep, file in enumerate(name_ls):
                 rep_idl = []
                 filename = path + '/' + file
-            T, rep_idl, rep_data = _read_append_rep(filename, pattern, b2b, cfg_separator, im, single)
+                T, rep_idl, rep_data = _read_append_rep(filename, pattern, intern[name]['b2b'], cfg_separator, im, intern[name]['single'])
                 if rep == 0:
-                for t in range(T):
+                    intern[name]['T'] = T
+                    for t in range(intern[name]['T']):
                         deltas.append([])
-            for t in range(T):
+                for t in range(intern[name]['T']):
                     deltas[t].append(rep_data[t])
+                internal_ret_dict[key] = deltas
+                if name == name_list[0]:
                     idl.append(rep_idl)
 
-    if "check_configs" in kwargs:
+    if kwargs.get("check_configs") is True:
         if not silent:
             print("Checking for missing configs...")
         che = kwargs.get("check_configs")
@@ -236,10 +344,83 @@ def read_sfcf(path, prefix, name, quarks='.*', corr_type='bi', noffset=0, wf=0,
             check_idl(idl[r], che[r])
         if not silent:
             print("Done")
+
+    result_dict = {}
+    if keyed_out:
+        for key in needed_keys:
             result = []
-    for t in range(T):
+            for t in range(intern[name]["T"]):
-        result.append(Obs(deltas[t], new_names, idl=idl))
+                result.append(Obs(internal_ret_dict[key][t], new_names, idl=idl))
-    return result
+            result_dict[key] = result
+    else:
+        for name in name_list:
+            result_dict[name] = {}
+            for quarks in quarks_list:
+                result_dict[name][quarks] = {}
+                for off in noffset_list:
+                    result_dict[name][quarks][off] = {}
+                    for w in wf_list:
+                        result_dict[name][quarks][off][w] = {}
+                        for w2 in wf2_list:
+                            key = _specs2key(name, quarks, off, w, w2)
+                            result = []
+                            for t in range(intern[name]["T"]):
+                                result.append(Obs(internal_ret_dict[key][t], new_names, idl=idl))
+                            result_dict[name][quarks][str(off)][str(w)][str(w2)] = result
+    return result_dict
+
+
+def _lists2key(*lists):
+    keys = []
+    for tup in itertools.product(*lists):
+        keys.append(sep.join(tup))
+    return keys
+
+
+def _key2specs(key):
+    return key.split(sep)
+
+
+def _specs2key(*specs):
+    return sep.join(specs)
+
+
+def _read_o_file(cfg_path, name, needed_keys, intern, version, im):
+    return_vals = {}
+    for key in needed_keys:
+        file = cfg_path + '/' + name
+        specs = _key2specs(key)
+        if specs[0] == name:
+            with open(file) as fp:
+                lines = fp.readlines()
+                quarks = specs[1]
+                off = specs[2]
+                w = specs[3]
+                w2 = specs[4]
+                T = intern[name]["T"]
+                start_read = intern[name]["spec"][quarks][off][w][w2]["start"]
+                deltas = []
+                for line in lines[start_read:start_read + T]:
+                    floats = list(map(float, line.split()))
+                    if version == "0.0":
+                        deltas.append(floats[im - intern[name]["single"]])
+                    else:
+                        deltas.append(floats[1 + im - intern[name]["single"]])
+                return_vals[key] = deltas
+    return return_vals
+
+
+def _extract_corr_type(corr_type):
+    if corr_type == 'bb':
+        b2b = True
+        single = True
+    elif corr_type == 'bib':
+        b2b = True
+        single = False
+    else:
+        b2b = False
+        single = False
+    return b2b, single
 
 
 def _find_files(rep_path, prefix, compact, files=[]):
@@ -309,38 +490,57 @@ def _find_correlator(file_name, version, pattern, b2b, silent=False):
     return start_read, T
 
 
-def _read_compact_file(rep_path, config_file, start_read, T, b2b, name, im):
+def _read_compact_file(rep_path, cfg_file, intern, needed_keys, im):
-    with open(rep_path + config_file) as fp:
+    return_vals = {}
+    with open(rep_path + cfg_file) as fp:
         lines = fp.readlines()
+        for key in needed_keys:
+            keys = _key2specs(key)
+            name = keys[0]
+            quarks = keys[1]
+            off = keys[2]
+            w = keys[3]
+            w2 = keys[4]
+
+            T = intern[name]["T"]
+            start_read = intern[name]["spec"][quarks][off][w][w2]["start"]
             # check, if the correlator is in fact
             # printed completely
             if (start_read + T + 1 > len(lines)):
-            raise Exception("EOF before end of correlator data! Maybe " + rep_path + config_file + " is corrupted?")
+                raise Exception("EOF before end of correlator data! Maybe " + rep_path + cfg_file + " is corrupted?")
             corr_lines = lines[start_read - 6: start_read + T]
-        del lines
             t_vals = []
 
-        if corr_lines[1 - b2b].strip() != 'name      ' + name:
+            if corr_lines[1 - intern[name]["b2b"]].strip() != 'name      ' + name:
-            raise Exception('Wrong format in file', config_file)
+                raise Exception('Wrong format in file', cfg_file)
 
             for k in range(6, T + 6):
                 floats = list(map(float, corr_lines[k].split()))
                 t_vals.append(floats[-2:][im])
-    return t_vals
+            return_vals[key] = t_vals
+    return return_vals
 
 
-def _read_compact_rep(path, rep, sub_ls, start_read, T, b2b, name, im):
+def _read_compact_rep(path, rep, sub_ls, intern, needed_keys, im):
     rep_path = path + '/' + rep + '/'
     no_cfg = len(sub_ls)
+
+    return_vals = {}
+    for key in needed_keys:
+        name = _key2specs(key)[0]
         deltas = []
-    for t in range(T):
+        for t in range(intern[name]["T"]):
             deltas.append(np.zeros(no_cfg))
+        return_vals[key] = deltas
+
     for cfg in range(no_cfg):
         cfg_file = sub_ls[cfg]
-        cfg_data = _read_compact_file(rep_path, cfg_file, start_read, T, b2b, name, im)
+        cfg_data = _read_compact_file(rep_path, cfg_file, intern, needed_keys, im)
-        for t in range(T):
+        for key in needed_keys:
-            deltas[t][cfg] = cfg_data[t]
+            name = _key2specs(key)[0]
-    return deltas
+            for t in range(intern[name]["T"]):
+                return_vals[key][t][cfg] = cfg_data[key][t]
+    return return_vals
 
 
 def _read_chunk(chunk, gauge_line, cfg_sep, start_read, T, corr_line, b2b, pattern, im, single):

pyerrors/input/utils.py

@@ -1,6 +1,9 @@
-import re
 """Utilities for the input"""
 
+import re
+import fnmatch
+import os
+
 
 def sort_names(ll):
     """Sorts a list of names of replika with searches for `r` and `id` in the replikum string.
@@ -17,6 +20,7 @@ def sort_names(ll):
     ll: list
         sorted list
     """
+
     if len(ll) > 1:
         sorted = False
         r_pattern = r'r(\d+)'
@@ -63,6 +67,7 @@ def check_idl(idl, che):
     miss_str : str
         string with integers of which idls are missing
     """
+
     missing = []
     for c in che:
         if c not in idl:
@@ -75,3 +80,65 @@ def check_idl(idl, che):
             miss_str += "," + str(i)
         print(miss_str)
     return miss_str
+
+
+def check_params(path, param_hash, prefix, param_prefix="parameters_"):
+    """
+    Check if, for sfcf, the parameter hashes at the end of the parameter files are in fact the expected one.
+
+    Parameters
+    ----------
+    path: str
+        measurement path, same as for sfcf read method
+    param_hash: str
+        expected parameter hash
+    prefix: str
+        data prefix to find the appropriate replicum folders in path
+    param_prefix: str
+        prefix of the parameter file. Defaults to 'parameters_'
+
+    Returns
+    -------
+    nums: dict
+        dictionary of faulty parameter files sorted by the replica paths
+    """
+
+    ls = []
+    for (dirpath, dirnames, filenames) in os.walk(path):
+        ls.extend(dirnames)
+        break
+    if not ls:
+        raise Exception('Error, directory not found')
+    # Exclude folders with different names
+    for exc in ls:
+        if not fnmatch.fnmatch(exc, prefix + '*'):
+            ls = list(set(ls) - set([exc]))
+
+    ls = sort_names(ls)
+    nums = {}
+    for rep in ls:
+        rep_path = path + '/' + rep
+        # files of replicum
+        sub_ls = []
+        for (dirpath, dirnames, filenames) in os.walk(rep_path):
+            sub_ls.extend(filenames)
+
+        # filter
+        param_files = []
+        for file in sub_ls:
+            if fnmatch.fnmatch(file, param_prefix + '*'):
+                param_files.append(file)
+
+        rep_nums = ''
+        for file in param_files:
+            with open(rep_path + '/' + file) as fp:
+                for line in fp:
+                    pass
+                last_line = line
+                if last_line.split()[2] != param_hash:
+                    rep_nums += file.split("_")[1] + ','
+        nums[rep_path] = rep_nums
+
+        if not len(rep_nums) == 0:
+            raise Warning("found differing parameter hash in the param files in " + rep_path)
+    return nums

tests/sfcf_in_test.py

@@ -1,7 +1,6 @@
 import os
 import sys
 import inspect
-import pyerrors as pe
 import pyerrors.input.sfcf as sfin
 import shutil
 import pytest
@@ -14,107 +13,299 @@ sys.path.insert(0, parent_dir)
 def build_test_environment(path, env_type, cfgs, reps):
     shutil.copytree("tests/data/sfcf_test/data_"+env_type, (path + "/data_" + env_type))
     if env_type == "o":
-        for i in range(2,cfgs+1):
+        for i in range(2, cfgs+1):
             shutil.copytree(path + "/data_o/test_r0/cfg1", path + "/data_o/test_r0/cfg"+str(i))
-        for i in range(1,reps):
+        for i in range(1, reps):
             shutil.copytree(path + "/data_o/test_r0", path + "/data_o/test_r"+str(i))
     elif env_type == "c":
-        for i in range(2,cfgs+1):
+        for i in range(2, cfgs+1):
             shutil.copy(path + "/data_c/data_c_r0/data_c_r0_n1", path + "/data_c/data_c_r0/data_c_r0_n"+str(i))
-        for i in range(1,reps):
+        for i in range(1, reps):
             os.mkdir(path + "/data_c/data_c_r"+str(i))
-            for j in range(1,cfgs+1):
+            for j in range(1, cfgs+1):
-                shutil.copy(path + "/data_c/data_c_r0/data_c_r0_n1",path + "/data_c/data_c_r"+str(i)+"/data_c_r"+str(i)+"_n"+str(j))
+                shutil.copy(path + "/data_c/data_c_r0/data_c_r0_n1", path + "/data_c/data_c_r"+str(i)+"/data_c_r"+str(i)+"_n"+str(j))
     elif env_type == "a":
-        for i in range(1,reps):
+        for i in range(1, reps):
             for corr in ["f_1", "f_A", "F_V0"]:
                 shutil.copy(path + "/data_a/data_a_r0." + corr, path + "/data_a/data_a_r" + str(i) + "." + corr)
 
 
 def test_o_bb(tmp_path):
-    build_test_environment(str(tmp_path), "o",5,3)
+    build_test_environment(str(tmp_path), "o", 5, 3)
-    f_1 = sfin.read_sfcf(str(tmp_path) + "/data_o", "test", "f_1",quarks="lquark lquark", wf = 0, wf2=0, version = "2.0", corr_type="bb")
+    f_1 = sfin.read_sfcf(str(tmp_path) + "/data_o", "test", "f_1", quarks="lquark lquark", wf=0, wf2=0, version="2.0", corr_type="bb")
     print(f_1)
     assert len(f_1) == 1
-    assert list(f_1[0].shape.keys()) == ["test_|r0","test_|r1","test_|r2"]
+    assert list(f_1[0].shape.keys()) == ["test_|r0", "test_|r1", "test_|r2"]
     assert f_1[0].value == 351.1941525454502
 
+
 def test_o_bi(tmp_path):
-    build_test_environment(str(tmp_path), "o",5,3)
+    build_test_environment(str(tmp_path), "o", 5, 3)
-    f_A = sfin.read_sfcf(str(tmp_path) + "/data_o", "test", "f_A",quarks="lquark lquark", wf = 0, version = "2.0")
+    f_A = sfin.read_sfcf(str(tmp_path) + "/data_o", "test", "f_A", quarks="lquark lquark", wf=0, version="2.0")
     print(f_A)
     assert len(f_A) == 3
-    assert list(f_A[0].shape.keys()) == ["test_|r0","test_|r1","test_|r2"]
+    assert list(f_A[0].shape.keys()) == ["test_|r0", "test_|r1", "test_|r2"]
     assert f_A[0].value == 65.4711887279723
     assert f_A[1].value == 1.0447210336915187
     assert f_A[2].value == -41.025094911185185
 
+
 def test_o_bib(tmp_path):
-    build_test_environment(str(tmp_path), "o",5,3)
+    build_test_environment(str(tmp_path), "o", 5, 3)
-    f_V0 = sfin.read_sfcf(str(tmp_path) + "/data_o", "test", "F_V0",quarks="lquark lquark", wf = 0, wf2 = 0, version = "2.0", corr_type="bib")
+    f_V0 = sfin.read_sfcf(str(tmp_path) + "/data_o", "test", "F_V0", quarks="lquark lquark", wf=0, wf2=0, version="2.0", corr_type="bib")
     print(f_V0)
     assert len(f_V0) == 3
-    assert list(f_V0[0].shape.keys()) == ["test_|r0","test_|r1","test_|r2"]
+    assert list(f_V0[0].shape.keys()) == ["test_|r0", "test_|r1", "test_|r2"]
     assert f_V0[0] == 683.6776090085115
     assert f_V0[1] == 661.3188585582334
     assert f_V0[2] == 683.6776090081005
 
+
+def test_simple_multi_o(tmp_path):
+    build_test_environment(str(tmp_path), "o", 5, 3)
+    corrs = sfin.read_sfcf_multi(str(tmp_path) + "/data_o", "test", ["F_V0"], quarks_list=["lquark lquark"], wf1_list=[0], wf2_list=[0], version="2.0", corr_type_list=["bib"])
+    f_V0 = corrs["F_V0"]['lquark lquark']['0']['0']['0']
+    assert len(f_V0) == 3
+    assert list(f_V0[0].shape.keys()) == ["test_|r0", "test_|r1", "test_|r2"]
+    assert f_V0[0] == 683.6776090085115
+    assert f_V0[1] == 661.3188585582334
+    assert f_V0[2] == 683.6776090081005
+
+
+def test_dict_multi_o(tmp_path):
+    build_test_environment(str(tmp_path), "o", 5, 3)
+    corrs = sfin.read_sfcf_multi(str(tmp_path) + "/data_o", "test",
+                                 ["F_V0", "f_A", "f_1"], quarks_list=["lquark lquark"],
+                                 wf_list=[0], wf2_list=[0], version="2.0",
+                                 corr_type_list=["bib", "bi", "bb"], nice_output=False)
+    print(corrs)
+    f_1 = corrs["f_1"]['lquark lquark']['0']['0']['0']
+    f_A = corrs["f_A"]['lquark lquark']['0']['0']['0']
+    f_V0 = corrs["F_V0"]['lquark lquark']['0']['0']['0']
+
+    assert len(f_1) == 1
+    assert list(f_1[0].shape.keys()) == ["test_|r0", "test_|r1", "test_|r2"]
+    assert f_1[0].value == 351.1941525454502
+
+    assert len(f_A) == 3
+    assert list(f_A[0].shape.keys()) == ["test_|r0", "test_|r1", "test_|r2"]
+    assert f_A[0].value == 65.4711887279723
+    assert f_A[1].value == 1.0447210336915187
+    assert f_A[2].value == -41.025094911185185
+
+    assert len(f_V0) == 3
+    assert list(f_V0[0].shape.keys()) == ["test_|r0", "test_|r1", "test_|r2"]
+    assert f_V0[0] == 683.6776090085115
+    assert f_V0[1] == 661.3188585582334
+    assert f_V0[2] == 683.6776090081005
+
+
 def test_c_bb(tmp_path):
-    build_test_environment(str(tmp_path), "c",5,3)
+    build_test_environment(str(tmp_path), "c", 5, 3)
-    f_1 = sfin.read_sfcf(str(tmp_path) + "/data_c", "data_c", "f_1", quarks="lquark lquark", wf = 0, wf2=0, version = "2.0c", corr_type="bb")
+    f_1 = sfin.read_sfcf(str(tmp_path) + "/data_c", "data_c", "f_1", quarks="lquark lquark", wf=0, wf2=0, version="2.0c", corr_type="bb")
     print(f_1)
     assert len(f_1) == 1
-    assert list(f_1[0].shape.keys()) == ["data_c_|r0","data_c_|r1","data_c_|r2"]
+    assert list(f_1[0].shape.keys()) == ["data_c_|r0", "data_c_|r1", "data_c_|r2"]
     assert f_1[0].value == 351.1941525454502
 
+
 def test_c_bi(tmp_path):
-    build_test_environment(str(tmp_path), "c",5,3)
+    build_test_environment(str(tmp_path), "c", 5, 3)
-    f_A = sfin.read_sfcf(str(tmp_path) + "/data_c", "data_c", "f_A", quarks="lquark lquark", wf = 0, version = "2.0c")
+    f_A = sfin.read_sfcf(str(tmp_path) + "/data_c", "data_c", "f_A", quarks="lquark lquark", wf=0, version="2.0c")
     print(f_A)
     assert len(f_A) == 3
-    assert list(f_A[0].shape.keys()) == ["data_c_|r0","data_c_|r1","data_c_|r2"]
+    assert list(f_A[0].shape.keys()) == ["data_c_|r0", "data_c_|r1", "data_c_|r2"]
     assert f_A[0].value == 65.4711887279723
     assert f_A[1].value == 1.0447210336915187
     assert f_A[2].value == -41.025094911185185
 
+
 def test_c_bib(tmp_path):
-    build_test_environment(str(tmp_path), "c",5,3)
+    build_test_environment(str(tmp_path), "c", 5, 3)
-    f_V0 = sfin.read_sfcf(str(tmp_path) + "/data_c", "data_c", "F_V0",quarks="lquark lquark", wf = 0, wf2 = 0, version = "2.0c", corr_type="bib")
+    f_V0 = sfin.read_sfcf(str(tmp_path) + "/data_c", "data_c", "F_V0", quarks="lquark lquark", wf=0, wf2=0, version="2.0c", corr_type="bib")
     print(f_V0)
     assert len(f_V0) == 3
-    assert list(f_V0[0].shape.keys()) == ["data_c_|r0","data_c_|r1","data_c_|r2"]
+    assert list(f_V0[0].shape.keys()) == ["data_c_|r0", "data_c_|r1", "data_c_|r2"]
     assert f_V0[0] == 683.6776090085115
     assert f_V0[1] == 661.3188585582334
     assert f_V0[2] == 683.6776090081005
 
-def test_a_bb(tmp_path):
+
-    build_test_environment(str(tmp_path), "a",5,3)
+def test_simple_multi_c(tmp_path):
-    f_1 = sfin.read_sfcf(str(tmp_path) + "/data_a", "data_a", "f_1", quarks="lquark lquark", wf = 0, wf2=0, version = "2.0a", corr_type="bb")
+    build_test_environment(str(tmp_path), "c", 5, 3)
-    print(f_1)
+    corrs = sfin.read_sfcf_multi(str(tmp_path) + "/data_c", "data_c", ["F_V0"], quarks_list=["lquark lquark"], wf1_list=[0], wf2_list=[0], version="2.0c", corr_type_list=["bib"])
+    f_V0 = corrs["F_V0"]['lquark lquark']['0']['0']['0']
+    assert len(f_V0) == 3
+    assert list(f_V0[0].shape.keys()) == ["data_c_|r0", "data_c_|r1", "data_c_|r2"]
+    assert f_V0[0] == 683.6776090085115
+    assert f_V0[1] == 661.3188585582334
+    assert f_V0[2] == 683.6776090081005
+
+
+def test_dict_multi_c(tmp_path):
+    build_test_environment(str(tmp_path), "c", 5, 3)
+    corrs = sfin.read_sfcf_multi(str(tmp_path) + "/data_c", "data_c",
+                                 ["F_V0", "f_A", "f_1"], quarks_list=["lquark lquark"],
+                                 wf_list=[0], wf2_list=[0], version="2.0c",
+                                 corr_type_list=["bib", "bi", "bb"], nice_output=False)
+    print(corrs)
+    f_1 = corrs["f_1"]['lquark lquark']['0']['0']['0']
+    f_A = corrs["f_A"]['lquark lquark']['0']['0']['0']
+    f_V0 = corrs["F_V0"]['lquark lquark']['0']['0']['0']
+
     assert len(f_1) == 1
-    assert list(f_1[0].shape.keys()) == ["data_a_|r0","data_a_|r1","data_a_|r2"]
+    assert list(f_1[0].shape.keys()) == ["data_c_|r0", "data_c_|r1", "data_c_|r2"]
     assert f_1[0].value == 351.1941525454502
 
-def test_a_bi(tmp_path):
-    build_test_environment(str(tmp_path), "a",5,3)
-    f_A = sfin.read_sfcf(str(tmp_path) + "/data_a", "data_a", "f_A", quarks="lquark lquark", wf = 0, version = "2.0a")
-    print(f_A)
     assert len(f_A) == 3
-    assert list(f_A[0].shape.keys()) == ["data_a_|r0","data_a_|r1","data_a_|r2"]
+    assert list(f_A[0].shape.keys()) == ["data_c_|r0", "data_c_|r1", "data_c_|r2"]
     assert f_A[0].value == 65.4711887279723
     assert f_A[1].value == 1.0447210336915187
     assert f_A[2].value == -41.025094911185185
 
-def test_a_bib(tmp_path):
-    build_test_environment(str(tmp_path), "a",5,3)
-    f_V0 = sfin.read_sfcf(str(tmp_path) + "/data_a", "data_a", "F_V0",quarks="lquark lquark", wf = 0, wf2 = 0, version = "2.0a", corr_type="bib")
-    print(f_V0)
     assert len(f_V0) == 3
-    assert list(f_V0[0].shape.keys()) == ["data_a_|r0","data_a_|r1","data_a_|r2"]
+    assert list(f_V0[0].shape.keys()) == ["data_c_|r0", "data_c_|r1", "data_c_|r2"]
     assert f_V0[0] == 683.6776090085115
     assert f_V0[1] == 661.3188585582334
     assert f_V0[2] == 683.6776090081005
 
+
+def test_dict_multi_wf_c(tmp_path):
+    build_test_environment(str(tmp_path), "c", 5, 3)
+    corrs = sfin.read_sfcf_multi(str(tmp_path) + "/data_c", "data_c",
+                                 ["F_V0", "f_A", "f_1"], quarks_list=["lquark lquark"],
+                                 wf_list=[0, 1], wf2_list=[0, 1], version="2.0c",
+                                 corr_type_list=["bib", "bi", "bb"], nice_output=False)
+    rep_names = ["data_c_|r0", "data_c_|r1", "data_c_|r2"]
+    f_1_00 = corrs["f_1"]['lquark lquark']['0']['0']['0']
+    f_1_01 = corrs["f_1"]['lquark lquark']['0']['0']['1']
+    f_1_10 = corrs["f_1"]['lquark lquark']['0']['1']['0']
+    f_1_11 = corrs["f_1"]['lquark lquark']['0']['1']['1']
+
+    assert len(f_1_00) == 1
+    assert list(f_1_00[0].shape.keys()) == rep_names
+    assert f_1_00[0].value == 351.1941525454502
+
+    assert len(f_1_01) == 1
+    assert list(f_1_01[0].shape.keys()) == rep_names
+    assert f_1_01[0].value == 351.20703575855345
+
+    assert len(f_1_10) == 1
+    assert list(f_1_10[0].shape.keys()) == rep_names
+    assert f_1_10[0].value == 351.20703575855515
+
+    assert len(f_1_11) == 1
+    assert list(f_1_11[0].shape.keys()) == rep_names
+    assert f_1_11[0].value == 351.22001235609065
+
+    f_A = corrs["f_A"]['lquark lquark']['0']['0']['0']
+
+    assert len(f_A) == 3
+    assert list(f_A[0].shape.keys()) == rep_names
+    assert f_A[0].value == 65.4711887279723
+    assert f_A[1].value == 1.0447210336915187
+    assert f_A[2].value == -41.025094911185185
+
+    f_V0_00 = corrs["F_V0"]['lquark lquark']['0']['0']['0']
+    f_V0_01 = corrs["F_V0"]['lquark lquark']['0']['0']['1']
+    f_V0_10 = corrs["F_V0"]['lquark lquark']['0']['1']['0']
+    f_V0_11 = corrs["F_V0"]['lquark lquark']['0']['1']['1']
+
+    assert len(f_V0_00) == 3
+    assert list(f_V0_00[0].shape.keys()) == rep_names
+    assert f_V0_00[0].value == 683.6776090085115
+    assert f_V0_00[1].value == 661.3188585582334
+    assert f_V0_00[2].value == 683.6776090081005
+
+    assert len(f_V0_01) == 3
+    assert list(f_V0_01[0].shape.keys()) == rep_names
+    assert f_V0_01[0].value == 683.7028316879306
+    assert f_V0_01[1].value == 661.3432563640756
+    assert f_V0_01[2].value == 683.7028316875197
+
+    assert len(f_V0_10) == 3
+    assert list(f_V0_10[0].shape.keys()) == rep_names
+    assert f_V0_10[0].value == 683.7028316879289
+    assert f_V0_10[1].value == 661.343256364074
+    assert f_V0_10[2].value == 683.702831687518
+
+    assert len(f_V0_11) == 3
+    assert list(f_V0_11[0].shape.keys()) == rep_names
+    assert f_V0_11[0].value == 683.7280552978792
+    assert f_V0_11[1].value == 661.3676550700158
+    assert f_V0_11[2].value == 683.7280552974681
+
+
+def test_a_bb(tmp_path):
+    build_test_environment(str(tmp_path), "a", 5, 3)
+    f_1 = sfin.read_sfcf(str(tmp_path) + "/data_a", "data_a", "f_1", quarks="lquark lquark", wf=0, wf2=0, version="2.0a", corr_type="bb")
+    print(f_1)
+    assert len(f_1) == 1
+    assert list(f_1[0].shape.keys()) == ["data_a_|r0", "data_a_|r1", "data_a_|r2"]
+    assert f_1[0].value == 351.1941525454502
+
+
+def test_a_bi(tmp_path):
+    build_test_environment(str(tmp_path), "a", 5, 3)
+    f_A = sfin.read_sfcf(str(tmp_path) + "/data_a", "data_a", "f_A", quarks="lquark lquark", wf=0, version="2.0a")
+    print(f_A)
+    assert len(f_A) == 3
+    assert list(f_A[0].shape.keys()) == ["data_a_|r0", "data_a_|r1", "data_a_|r2"]
+    assert f_A[0].value == 65.4711887279723
+    assert f_A[1].value == 1.0447210336915187
+    assert f_A[2].value == -41.025094911185185
+
+
+def test_a_bib(tmp_path):
+    build_test_environment(str(tmp_path), "a", 5, 3)
+    f_V0 = sfin.read_sfcf(str(tmp_path) + "/data_a", "data_a", "F_V0", quarks="lquark lquark", wf=0, wf2=0, version="2.0a", corr_type="bib")
+    print(f_V0)
+    assert len(f_V0) == 3
+    assert list(f_V0[0].shape.keys()) == ["data_a_|r0", "data_a_|r1", "data_a_|r2"]
+    assert f_V0[0] == 683.6776090085115
+    assert f_V0[1] == 661.3188585582334
+    assert f_V0[2] == 683.6776090081005
+
+
+def test_simple_multi_a(tmp_path):
+    build_test_environment(str(tmp_path), "a", 5, 3)
+    corrs = sfin.read_sfcf_multi(str(tmp_path) + "/data_a", "data_a", ["F_V0"], quarks_list=["lquark lquark"], wf1_list=[0], wf2_list=[0], version="2.0a", corr_type_list=["bib"])
+    f_V0 = corrs["F_V0"]['lquark lquark']['0']['0']['0']
+    assert len(f_V0) == 3
+    assert list(f_V0[0].shape.keys()) == ["data_a_|r0", "data_a_|r1", "data_a_|r2"]
+    assert f_V0[0] == 683.6776090085115
+    assert f_V0[1] == 661.3188585582334
+    assert f_V0[2] == 683.6776090081005
+
+
+def test_dict_multi_a(tmp_path):
+    build_test_environment(str(tmp_path), "a", 5, 3)
+    corrs = sfin.read_sfcf_multi(str(tmp_path) + "/data_a", "data_a",
+                                 ["F_V0", "f_A", "f_1"], quarks_list=["lquark lquark"],
+                                 wf_list=[0], wf2_list=[0], version="2.0a",
+                                 corr_type_list=["bib", "bi", "bb"], nice_output=False)
+    print(corrs)
+    f_1 = corrs["f_1"]['lquark lquark']['0']['0']['0']
+    f_A = corrs["f_A"]['lquark lquark']['0']['0']['0']
+    f_V0 = corrs["F_V0"]['lquark lquark']['0']['0']['0']
+
+    assert len(f_1) == 1
+    assert list(f_1[0].shape.keys()) == ["data_a_|r0", "data_a_|r1", "data_a_|r2"]
+    assert f_1[0].value == 351.1941525454502
+
+    assert len(f_A) == 3
+    assert list(f_A[0].shape.keys()) == ["data_a_|r0", "data_a_|r1", "data_a_|r2"]
+    assert f_A[0].value == 65.4711887279723
+    assert f_A[1].value == 1.0447210336915187
+    assert f_A[2].value == -41.025094911185185
+
+    assert len(f_V0) == 3
+    assert list(f_V0[0].shape.keys()) == ["data_a_|r0", "data_a_|r1", "data_a_|r2"]
+    assert f_V0[0] == 683.6776090085115
+    assert f_V0[1] == 661.3188585582334
+    assert f_V0[2] == 683.6776090081005
+
+
 def test_find_corr():
     pattern = 'name      ' + "f_A" + '\nquarks    ' + "lquark lquark" + '\noffset    ' + str(0) + '\nwf        ' + str(0)
     start_read, T = sfin._find_correlator("tests/data/sfcf_test/data_c/data_c_r0/data_c_r0_n1", "2.0c", pattern, False)
@@ -129,7 +320,8 @@ def test_find_corr():
     with pytest.raises(ValueError):
         sfin._find_correlator("tests/data/sfcf_test/broken_data_c/data_c_r0/data_c_r0_n1", "2.0c", pattern, False)
 
-def test_read_compact_file(tmp_path):
+
+def test_read_compact_file():
     rep_path = "tests/data/sfcf_test/broken_data_c/data_c_r0/"
     config_file = "data_c_r0_n1"
     start_read = 469
@@ -139,3 +331,10 @@ def test_read_compact_file(tmp_path):
     im = False
     with pytest.raises(Exception):
         sfin._read_compact_file(rep_path, config_file, start_read, T, b2b, name, im)
+
+
+def test_find_correlator():
+    file = "tests/data/sfcf_test/data_c/data_c_r0/data_c_r0_n1"
+    found_start, found_T = sfin._find_correlator(file, "2.0", "name      f_A\nquarks    lquark lquark\noffset    0\nwf        0", False, False)
+    assert found_start == 21
+    assert found_T == 3